env cleanup

rewards adjustment
pybullet logic contained in SimManager

ml/train.py

@@ -178,13 +178,18 @@ def train(
 
     device = resolve_device(device)
 
+    policy_kwargs = dict(
+    log_std_init=-1.5,  # Sets initial std ~ 0.22 instead of 1.0
+    net_arch=dict(pi=[256, 256], vf=[256, 256])
+)
+
     model = PPO(
         "MlpPolicy",
         env,
         verbose=1,
         seed=seed,
-        learning_rate=3.5e-4,      # Cut LR in half (from 3e-4) to smooth out updates
-        n_steps=2048,              # Larger rollout buffer per env for stable gradients
+        learning_rate=1.5e-4,      # Cut LR in half (from 3e-4) to smooth out updates
+        n_steps=1024,              # Larger rollout buffer per env for stable gradients
         batch_size=128,            # Larger minibatches reduce noise
         n_epochs=10,               # Number of epoch updates per rollout
         gamma=0.99,                # Discount factor
@@ -195,6 +200,7 @@ def train(
         vf_coef=0.5,
         max_grad_norm=0.5,
         device=device,
+        policy_kwargs=policy_kwargs,
         tensorboard_log=str(Path(__file__).resolve().parent / "tensorboard"),
     )