JackBot/ml/env.py

import math
import os
import random
import numpy as np
import pybullet as p
import pybullet_data
import gymnasium as gym
from gymnasium import spaces
import config as cfg
import robot_init as ri


class JackBotEnv(gym.Env):
    """Gymnasium environment for joint-command learning with the JackBot URDF."""

    metadata = {"render_modes": ["human", "rgb_array"]}

    def __init__(
        self,
        urdf_path: str | None = None,
        use_gui: bool = True,
        frame_skip: int = 4,
        random_command: bool = True,
        max_episode_steps: int = 2000,
        num_robots: int = 1,
        robot_spacing: float = 0.5,
        start_pose: str = "init_deg",
    ):
        """
        Initialize the JackBot environment.

        Args:
            urdf_path: Path to the robot's URDF file. Defaults to config value.
            use_gui: If True, uses p.GUI (shows window), else p.DIRECT (headless).
            frame_skip: Number of physics simulation steps per agent step. 
                Higher values mean the agent takes actions less frequently (e.g., 4 steps at 240Hz = 60Hz control).
            random_command: If True, assigns a random velocity command (vx, vy, omega) to each robot every reset.
            max_episode_steps: Maximum number of steps (interactions) before the episode is truncated.
            num_robots: Number of robots to spawn for parallel training in the same scene.
            robot_spacing: Distance in meters between robots when spawning multiple.
            start_pose: The initial joint configuration name (e.g., 'init_deg' or 'init90_deg').
        """
        self.urdf_path = urdf_path or cfg.urdf_path
        self.use_gui = use_gui
        self.frame_skip = frame_skip
        self.random_command = random_command
        self.max_episode_steps = max_episode_steps
        self.num_robots = max(1, num_robots)
        self.robot_spacing = robot_spacing
        self.start_pose = start_pose
        self.action_scale = math.radians(8.0)
        self.physics_client = None
        self.robots = []
        self.plane = None
        self.robot_joint_indices = []
        self.joint_lower = None
        self.joint_upper = None
        self.initial_angles = None
        self.commands = None
        self.step_count = 0
        self.last_action = None
        self._first_reset = True
        self._min_steps_before_done = 150

        self._connect_sim()

        total_joints = self.num_robots * len(self.robot_joint_indices[0])
        observation_dim = total_joints + self.num_robots * 4
        action_dim = total_joints

        self.observation_space = spaces.Box(
            low=-np.inf,
            high=np.inf,
            shape=(observation_dim,),
            dtype=np.float32,
        )
        self.action_space = spaces.Box(
            low=-1.0,
            high=1.0,
            shape=(action_dim,),
            dtype=np.float32,
        )

    def _connect_sim(self):
        if self.physics_client is not None:
            p.disconnect(self.physics_client)

        flags = p.GUI if self.use_gui else p.DIRECT
        self.physics_client = p.connect(flags)
        p.setAdditionalSearchPath(pybullet_data.getDataPath())
        p.setGravity(0, 0, -9.81)

        self.plane = p.loadURDF("plane.urdf")
        self.robots = []
        self.robot_joint_indices = []

        for robot_id in range(self.num_robots):
            base_pos = self._robot_base_position(robot_id)
            robot = p.loadURDF(self.urdf_path, basePosition=base_pos, useFixedBase=False)
            self.robots.append(robot)
            joint_indices = [
                i
                for i in range(p.getNumJoints(robot))
                if p.getJointInfo(robot, i)[2] == p.JOINT_REVOLUTE
            ]
            self.robot_joint_indices.append(joint_indices)

        joint_count = len(self.robot_joint_indices[0])
        if any(len(indices) != joint_count for indices in self.robot_joint_indices):
            raise ValueError("All robots must have the same number of revolute joints.")

        lower_limits = []
        upper_limits = []
        for joint_index in self.robot_joint_indices[0]:
            info = p.getJointInfo(self.robots[0], joint_index)
            lower = info[8]
            upper = info[9]
            if lower >= upper:
                lower = -math.pi
                upper = math.pi
            lower_limits.append(lower)
            upper_limits.append(upper)

        limits = np.array([lower_limits, upper_limits], dtype=np.float32)
        self.joint_lower = np.tile(limits[0], (self.num_robots, 1))
        self.joint_upper = np.tile(limits[1], (self.num_robots, 1))

        pose = ri.init_deg if self.start_pose == "init_deg" else ri.init90_deg
        initial_pose = pose.to_rad().data.flatten()
        self.initial_angles = np.tile(initial_pose, (self.num_robots, 1)).astype(np.float32)
        self.initial_angles = np.clip(
            self.initial_angles,
            self.joint_lower,
            self.joint_upper,
        )

        self.last_action = np.zeros(self.num_robots * joint_count, dtype=np.float32)
        self.commands = np.zeros((self.num_robots, 4), dtype=np.float32)

    def reset(self, command: np.ndarray | None = None, seed: int | None = None, options: dict | None = None):
        if self._first_reset:
            self.step_count = 0
            self.last_action = np.zeros(self.num_robots * len(self.robot_joint_indices[0]), dtype=np.float32)

            if seed is not None:
                self.seed(seed)

            if command is not None:
                self.commands = np.array(command, dtype=np.float32).reshape(self.num_robots, 4)
            elif self.random_command:
                self.commands = np.stack([self.sample_command() for _ in range(self.num_robots)])
            else:
                self.commands = np.zeros((self.num_robots, 4), dtype=np.float32)

            self._reset_pose()
            self._first_reset = False
            return self._get_obs(), {}

        p.resetSimulation()
        p.setGravity(0, 0, -9.81)
        p.setAdditionalSearchPath(pybullet_data.getDataPath())
        self.plane = p.loadURDF("plane.urdf")
        self.robots = []
        self.robot_joint_indices = []

        for robot_id in range(self.num_robots):
            base_pos = self._robot_base_position(robot_id)
            robot = p.loadURDF(self.urdf_path, basePosition=base_pos, useFixedBase=False)
            self.robots.append(robot)
            joint_indices = [
                i
                for i in range(p.getNumJoints(robot))
                if p.getJointInfo(robot, i)[2] == p.JOINT_REVOLUTE
            ]
            self.robot_joint_indices.append(joint_indices)

        self.step_count = 0
        self.last_action = np.zeros(self.num_robots * len(self.robot_joint_indices[0]), dtype=np.float32)

        if seed is not None:
            self.seed(seed)

        if command is not None:
            self.commands = np.array(command, dtype=np.float32).reshape(self.num_robots, 4)
        elif self.random_command:
            self.commands = np.stack([self.sample_command() for _ in range(self.num_robots)])
        else:
            self.commands = np.zeros((self.num_robots, 4), dtype=np.float32)

        self._reset_pose()
        return self._get_obs(), {}

    def seed(self, seed: int | None = None):
        """Set RNG seed for reproducibility (Gym compatibility)."""
        if seed is None:
            seed = int(np.random.randint(0, 2**31 - 1))
        random.seed(seed)
        np.random.seed(seed)
        self._seed = seed
        return [seed]

    def _reset_pose(self):
        for robot, joint_indices, init_angles in zip(self.robots, self.robot_joint_indices, self.initial_angles):
            for joint_index, target_angle in zip(joint_indices, init_angles):
                p.resetJointState(robot, joint_index, target_angle)
                p.setJointMotorControl2(
                    bodyIndex=robot,
                    jointIndex=joint_index,
                    controlMode=p.POSITION_CONTROL,
                    targetPosition=target_angle,
                    force=250,
                )

        for _ in range(400):
            p.stepSimulation()

        self.joint_angles = self.initial_angles.copy()

    def sample_command(self) -> np.ndarray:
        vx = np.random.uniform(-1.0, 1.0)
        vy = np.random.uniform(-0.5, 0.5)
        vz = 0.0
        omega = np.random.uniform(-1.0, 1.0)
        return np.array([vx, vy, vz, omega], dtype=np.float32)

    def _get_obs(self) -> np.ndarray:
        return np.concatenate([self.joint_angles.flatten(), self.commands.flatten()]).astype(np.float32)

    def step(self, action: np.ndarray):
        action = np.clip(action, self.action_space.low, self.action_space.high).astype(np.float32)
        self.last_action = action
        action_matrix = action.reshape(self.num_robots, -1)
        self.joint_angles = np.clip(
            self.joint_angles + action_matrix * self.action_scale,
            self.joint_lower,
            self.joint_upper,
        )

        for robot, joint_indices, angles in zip(self.robots, self.robot_joint_indices, self.joint_angles):
            for joint_index, target_angle in zip(joint_indices, angles):
                p.setJointMotorControl2(
                    bodyIndex=robot,
                    jointIndex=joint_index,
                    controlMode=p.POSITION_CONTROL,
                    targetPosition=target_angle,
                    force=250,
                )

        for _ in range(self.frame_skip):
            p.stepSimulation()

        self.step_count += 1
        observation = self._get_obs()
        reward = self._compute_reward()
        done = self._is_done()
        if self.step_count <= self._min_steps_before_done:
            done = False
        info = {"step": self.step_count}

        # Gymnasium-style return: (obs, reward, terminated, truncated, info)
        if done:
            terminated = True
            truncated = False
        else:
            terminated = False
            truncated = False

        return observation, float(reward), terminated, truncated, info

    def _compute_reward(self) -> float:
        rewards = []
        for robot_id, robot in enumerate(self.robots):
            linear_vel, angular_vel = p.getBaseVelocity(robot)
            _, orientation = p.getBasePositionAndOrientation(robot)
            roll, pitch, _ = p.getEulerFromQuaternion(orientation)
            command = self.commands[robot_id]

            forward_reward = command[0] * linear_vel[0] + command[1] * linear_vel[1]
            rotation_reward = command[3] * angular_vel[2]
            stability_penalty = abs(roll) + abs(pitch)
            action_penalty = float(np.sum(np.square(self.last_action.reshape(self.num_robots, -1)[robot_id]))) * 0.01

            rewards.append(0.1 + forward_reward + rotation_reward - 0.2 * stability_penalty - action_penalty)

        return float(np.sum(rewards))

    def _is_done(self) -> bool:
        for robot in self.robots:
            _, orientation = p.getBasePositionAndOrientation(robot)
            roll, pitch, _ = p.getEulerFromQuaternion(orientation)
            if abs(roll) > 0.7 or abs(pitch) > 0.7:
                return True
            position, _ = p.getBasePositionAndOrientation(robot)
            if position[2] < 0.02:
                return True

        if self.step_count >= self.max_episode_steps:
            return True
        return False

    def render(self, mode="human"):
        if self.use_gui:
            return None
        raise NotImplementedError("Render is only supported with use_gui=True")

    def _robot_base_position(self, robot_id: int) -> list[float]:
        cols = int(math.sqrt(self.num_robots - 1)) + 1
        row = robot_id // cols
        col = robot_id % cols
        x = (col - (cols - 1) / 2.0) * self.robot_spacing
        y = (row - (cols - 1) / 2.0) * self.robot_spacing
        return [x, y, 0.1]

    def close(self):
        if self.physics_client is not None:
            p.disconnect(self.physics_client)
            self.physics_client = None