direct and residual movement inconsistency fixed

2026-08-27 21:40:57 +02:00
parent cd870a4afc
commit 4cc2d37d94
8 changed files with 109 additions and 73 deletions
@@ -86,7 +86,12 @@ def main():
        model = PPO.load(
            args.pretrained_model,
            env=vec_env,
-            learning_rate=1e-4,  # Lower learning rate so RL fine-tunes without destroying base gait
+            learning_rate=5e-5,  # Lower learning rate so RL fine-tunes without destroying base gait
+            ent_coef=0.001,
+            target_kl=0.05,
+            vf_coef=0.5,
+            max_grad_norm=0.5,
+            verbose=2,
            tensorboard_log=args.log_dir,
            device="cpu",
        )
@@ -95,18 +100,18 @@ def main():
        model = PPO(
            policy="MlpPolicy",
            env=vec_env,
-            learning_rate=1e-4,
+            learning_rate=5e-5,
            n_steps=256,
            batch_size=256,
            n_epochs=10,
            gamma=0.99,
            gae_lambda=0.95,
            clip_range=0.2,
-            ent_coef=0.01,
+            ent_coef=0.001,
            target_kl=0.05,
            vf_coef=0.5,
            max_grad_norm=0.5,
-            verbose=1,
+            verbose=2,
            tensorboard_log=args.log_dir,
            device="cpu",
        )
@@ -127,7 +132,7 @@ def main():
        eval_env,
        best_model_save_path=best_model_path,
        log_path="ml/logs/results",
-        eval_freq=max(1, 20_000 // args.num_workers),
+        eval_freq=max(1, 50_000 // args.num_workers),
        deterministic=True,
        render=False,
    )