JackM323/JackBot
1
0
Fork 0
Code Issues 3 Pull Requests Actions Packages Projects 2 Releases Wiki Activity

learning tweaks

Browse Source
This commit is contained in:
JackM323
2026-08-07 10:42:28 +02:00
parent 405e3ad5f2
commit f9a3e8ddba
5 changed files with 80 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
ml/env.py
+50 -16
View File
@@ -81,9 +81,11 @@ class JackBotEnv(gym.Env):
self.action_space = spaces.Box(-1.0, 1.0, shape=(action_dim,), dtype=np.float32)
self.observation_space = spaces.Box(-np.inf, np.inf, shape=(obs_dim,), dtype=np.float32)
self.min_joint_limits, self.max_joint_limits = self.sim._get_urdf_joint_limits()
self.command = np.zeros(3, dtype=np.float32) # [vx, vy, omega]
self.last_action = np.zeros(action_dim, dtype=np.float32)
self.last_last_action = np.zeros(action_dim, dtype=np.float32)
self.target_height = 0.122
self.collapse_height_fraction = 0.55
self.tilt_failure_rad = 0.9
@@ -143,7 +145,7 @@ class JackBotEnv(gym.Env):
self.last_reward_components = {}
self.episode_reward_components_sum = defaultdict(float)
spawn_pos = [0.0, 0.0, 0.20]
spawn_pos = [0.0, 0.0, 0.15]
spawn_orn = [0.0, 0.0, 0.0, 1.0]
# 1. Reset base pose and velocities
@@ -155,7 +157,9 @@ class JackBotEnv(gym.Env):
if self.use_gui:
self.sim.set_robot_color([1.0, 1.0, 1.0, 1.0])
self.last_action = np.zeros(self.action_space.shape[0], dtype=np.float32)
action_dim = self.action_space.shape[0]
self.last_action = np.zeros(action_dim, dtype=np.float32)
self.last_last_action = np.zeros(action_dim, dtype=np.float32)
self.max_distance_from_start = 0.0
self.max_survival_steps = 0
@@ -170,8 +174,13 @@ class JackBotEnv(gym.Env):
self.start_position = list(pos)
# Drop settlement
self.target_height = self.sim.settle_and_measure_height(steps=200, fallback_height=0.122)
self.target_height = self.sim.settle_and_measure_height(
target_angles=self.robot.current_rad,
steps=300,
fallback_height=0.122
)
self.default_joint_angles = self.sim.get_robot_joint_angles()
self.default_action = np.clip(self.default_joint_angles / (np.pi / 2.0), -1.0, 1.0)
if self.use_gui:
self.hud.reset()
@@ -183,9 +192,11 @@ class JackBotEnv(gym.Env):
return self.robot.get_observation(command=self.command)
def step(self, action: np.ndarray) -> Tuple[np.ndarray, float, bool, bool, Dict[str, Any]]:
previous_action = self.last_action.copy()
self.step_count += 1
self.total_steps += 1
previous_action = self.last_action.copy()
self.last_last_action = self.last_action.copy()
self.last_action = action.copy()
# Command resampling
@@ -200,9 +211,14 @@ class JackBotEnv(gym.Env):
# Mirror main.py input resolution logic: update robot_state and vector_dirmov directly
self.robot.robot_state = "walking" if (abs(cmd_vx) > 0.01 or abs(cmd_vy) > 0.01 or abs(cmd_omega) > 0.01) else "idle"
self.robot.vector_dirmov = [float(cmd_vx), float(cmd_vy), float(cmd_omega)]
# Delegate execution tick to Robot instance
self.robot.tick(action=action)
target_angles = np.where(
action < 0.0,
self.default_joint_angles + action * (self.default_joint_angles - self.min_joint_limits),
self.default_joint_angles + action * (self.max_joint_limits - self.default_joint_angles)
)
self.robot.tick(action=target_angles)
if self.robot_mode != "kinematics" and self.step_count % 60 == 0:
random_force = np.random.uniform(-2.0, 2.0, size=2)
@@ -265,9 +281,8 @@ class JackBotEnv(gym.Env):
def _update_curriculum(self):
self._curriculum_advanced = False
forced_forward = (self.curriculum_phase < CurriculumPhase.FORWARD) and (self.step_count >= 2500)
if self.curriculum_phase < CurriculumPhase.FORWARD and (self._phase_progress_ready(CurriculumPhase.FORWARD) or forced_forward):
if self.curriculum_phase < CurriculumPhase.FORWARD and (self._phase_progress_ready(CurriculumPhase.FORWARD)):
self.curriculum_phase = CurriculumPhase.FORWARD
self._curriculum_advanced = True
elif self.curriculum_phase < CurriculumPhase.TURN_AND_DIRECTION and self._phase_progress_ready(CurriculumPhase.TURN_AND_DIRECTION):
@@ -289,12 +304,19 @@ class JackBotEnv(gym.Env):
cmd_norm = math.hypot(cmd_vx, cmd_vy)
raw_speed = math.hypot(linear_vel[0], linear_vel[1])
# Forgiving zones: ignore noise below 0.04 m/s and 0.05 rad/s
filtered_vx, filtered_vy = (linear_vel[0], linear_vel[1]) if raw_speed >= 0.04 else (0.0, 0.0)
filtered_speed = raw_speed if raw_speed >= 0.04 else 0.0
raw_yaw_rate = abs(angular_vel[2])
filtered_yaw_rate = angular_vel[2] if raw_yaw_rate >= 0.05 else 0.0
filtered_yaw_rate = raw_yaw_rate if raw_yaw_rate >= 0.05 else 0.0
# --- 1. FIXED JITTER PENALTY ---
# Scaled way down (0.005) and capped so it can never dominate the reward
action_accel = action - 2.0 * self.last_action + self.last_last_action
raw_jitter = float(np.mean(np.square(action_accel)))
jitter_penalty = -0.005 * min(raw_jitter, 10.0)
# --- Base Components ---
height_error = pos[2] - self.target_height
r_height = math.exp(-150.0 * (height_error ** 2))
r_stability = math.exp(-25.0 * (roll**2 + pitch**2))
@@ -302,12 +324,20 @@ class JackBotEnv(gym.Env):
r_smoothness = math.exp(-0.1 * np.mean(np.square(action - previous_action)))
r_lin_vel, r_ang_vel, stillness_penalty = 0.0, 0.0, 0.0
stand_penalty = 0.0
# --- 2. COMMAND IS ZERO: STANDING MODE ---
if cmd_norm < 0.05 and abs(cmd_yaw) < 0.05:
w_height, w_stability, w_pose, w_smoothness = 0.35, 0.35, 0.20, 0.10
total_reward = (w_height * r_height) + (w_stability * r_stability) + (w_pose * r_pose) + (w_smoothness * r_smoothness)
base_reward = (w_height * r_height) + (w_stability * r_stability) + (w_pose * r_pose) + (w_smoothness * r_smoothness)
# Soft quadratic penalty on filtered speed (gives a forgiving gap near 0)
stand_penalty = -0.5 * filtered_speed - 0.1 * filtered_yaw_rate
total_reward = base_reward + stand_penalty
# --- 3. COMMAND IS NON-ZERO: WALKING MODE ---
else:
is_moving = (filtered_speed > 0.0) or (abs(filtered_yaw_rate) > 0.0)
is_moving = (filtered_speed > 0.0) or (filtered_yaw_rate > 0.0)
target_vx, target_vy = cmd_vx * self.max_robot_speed, cmd_vy * self.max_robot_speed
target_speed = math.hypot(target_vx, target_vy)
@@ -320,7 +350,7 @@ class JackBotEnv(gym.Env):
if target_speed > 0.08 and raw_speed < 0.03:
r_lin_vel = 0.0
stillness_penalty = -0.25
stillness_penalty = -0.1 # Softened from -0.25
w_lin_vel, w_ang_vel, w_height, w_stability, w_smoothness = 0.55, 0.15, 0.10, 0.12, 0.08
total_reward = (
@@ -328,7 +358,9 @@ class JackBotEnv(gym.Env):
+ (w_stability * r_stability) + (w_smoothness * r_smoothness) + stillness_penalty
)
final_reward = float(total_reward / 10.0)
step_reward = float(total_reward / 10.0)
alive_bonus = 0.01
final_reward = max(0.0, step_reward + jitter_penalty + alive_bonus)
self.last_reward_components = {
"height": float(r_height),
@@ -337,13 +369,15 @@ class JackBotEnv(gym.Env):
"smoothness": float(r_smoothness),
"lin_vel": float(r_lin_vel),
"ang_vel": float(r_ang_vel),
"jitter_penalty": float(jitter_penalty),
"stand_penalty": float(stand_penalty),
"total": final_reward,
}
for k, v in self.last_reward_components.items():
self.episode_reward_components_sum[k] += v
return final_reward
def get_reward_component_averages(self) -> Dict[str, float]:
steps = max(1, self.step_count)
return {k: v / steps for k, v in self.episode_reward_components_sum.items()}
@@ -377,7 +411,7 @@ class JackBotEnv(gym.Env):
)
def _update_robot_failure(self):
if self.is_failed or self.step_count < 15:
if self.is_failed:
return
position, (roll, pitch, _) = self.sim.get_robot_pose_and_rpy()
ml/run_eval_training.py
+3
View File
@@ -4,6 +4,9 @@ ml/run_eval_training.py - Benchmark reward system across all curriculum phases.
import time
import numpy as np
from ml.env import JackBotEnv, CurriculumPhase
import sys
from pathlib import Path
sys.path.append(str(Path(__file__).resolve().parent.parent))
def evaluate_kinematics(episode_length: int = 1000):
env = JackBotEnv(
ml/run_train.py
+1 -1
View File
@@ -110,7 +110,7 @@ def main():
eval_env,
best_model_save_path=best_model_path,
log_path="ml/logs/results",
eval_freq=max(1, 10_000 // args.num_workers),
eval_freq=max(1, 20_000 // args.num_workers),
deterministic=True,
render=False,
)