diff --git a/dimos/robot/all_blueprints.py b/dimos/robot/all_blueprints.py index c97dd8df1a..06990afe00 100644 --- a/dimos/robot/all_blueprints.py +++ b/dimos/robot/all_blueprints.py @@ -68,6 +68,7 @@ "drone-agentic": "dimos.robot.drone.blueprints.agentic.drone_agentic:drone_agentic", "drone-basic": "dimos.robot.drone.blueprints.basic.drone_basic:drone_basic", "dual-openyam-planner-coordinator": "dimos.robot.manipulators.dual_openyam.blueprints.basic:dual_openyam_planner_coordinator", + "dual-openyam-quest-collection": "dimos.robot.manipulators.dual_openyam.blueprints.learning_collection:dual_openyam_quest_collection", "dual-xarm6-planner-coordinator": "dimos.robot.manipulators.xarm.blueprints.basic:dual_xarm6_planner_coordinator", "go2-zenoh-basic": "dimos.robot.unitree.go2.zenoh.blueprints:go2_zenoh_basic", "go2-zenoh-htc": "dimos.robot.unitree.go2.zenoh.blueprints:go2_zenoh_htc", diff --git a/dimos/robot/manipulators/dual_openyam/blueprints/learning_collection.py b/dimos/robot/manipulators/dual_openyam/blueprints/learning_collection.py new file mode 100644 index 0000000000..6f26123f5c --- /dev/null +++ b/dimos/robot/manipulators/dual_openyam/blueprints/learning_collection.py @@ -0,0 +1,53 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Dual OpenYAM collection using ordinary blueprint configuration.""" + +from dimos.core.coordination.blueprints import autoconnect +from dimos.hardware.sensors.camera.module import CameraModule +from dimos.hardware.sensors.camera.webcam import WebcamConfig +from dimos.imitation.collection.episode_monitor import EpisodeMonitorModule +from dimos.imitation.collection.recorder import collection_recorder +from dimos.manipulation.visualization.viser.config import ViserVisualizationConfig +from dimos.robot.manipulators.dual_openyam.blueprints.teleop import build_dual_openyam_webxr +from dimos.robot.manipulators.dual_openyam.learning import DUAL_OPENYAM_COLLECTION + +dual_openyam_quest_collection = autoconnect( + build_dual_openyam_webxr(visualization=ViserVisualizationConfig(host="0.0.0.0")), + CameraModule.blueprint( + instance_name="left_wrist", + hardware=WebcamConfig( + camera_index=0, width=640, height=480, fps=30, frame_id_prefix="left_wrist_image" + ), + frame_id="left_wrist_camera_link", + ), + CameraModule.blueprint( + instance_name="right_wrist", + hardware=WebcamConfig( + camera_index=1, width=640, height=480, fps=30, frame_id_prefix="right_wrist_image" + ), + frame_id="right_wrist_camera_link", + ), + collection_recorder(profile=DUAL_OPENYAM_COLLECTION), + EpisodeMonitorModule.blueprint(instance_name="episodes"), +).remappings( + [ + ("left_wrist", "color_image", "left_wrist_image"), + ("left_wrist", "camera_info", "left_wrist_camera_info"), + ("left_wrist", "tf", "left_wrist_tf"), + ("right_wrist", "color_image", "right_wrist_image"), + ("right_wrist", "camera_info", "right_wrist_camera_info"), + ("right_wrist", "tf", "right_wrist_tf"), + ] +) diff --git a/dimos/robot/manipulators/dual_openyam/blueprints/test_learning.py b/dimos/robot/manipulators/dual_openyam/blueprints/test_learning.py new file mode 100644 index 0000000000..da62bf7bdd --- /dev/null +++ b/dimos/robot/manipulators/dual_openyam/blueprints/test_learning.py @@ -0,0 +1,129 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +import numpy as np + +from dimos.core.coordination.blueprint_config.parser import BlueprintConfigParser +from dimos.hardware.sensors.camera.module import CameraModule +from dimos.imitation.dataprep.core import resolve_field +from dimos.manipulation.manipulation_module import ManipulationModule +from dimos.msgs.sensor_msgs.JointState import JointState +from dimos.robot.manipulators.dual_openyam.blueprints.learning_collection import ( + dual_openyam_quest_collection, +) +from dimos.robot.manipulators.dual_openyam.blueprints.teleop import teleop_webxr_dual_openyam +from dimos.robot.manipulators.dual_openyam.joints import DUAL_OPENYAM_JOINTS +from dimos.robot.manipulators.dual_openyam.learning import DUAL_OPENYAM_COLLECTION + + +def test_dual_profile_projects_canonical_joint_order_and_records_both_cameras(): + profile = DUAL_OPENYAM_COLLECTION + schema = profile.to_schema() + message = JointState( + name=list(reversed(DUAL_OPENYAM_JOINTS)), position=list(reversed(range(14))) + ) + np.testing.assert_array_equal(resolve_field(message, schema.action["action"]), np.arange(14)) + np.testing.assert_array_equal( + resolve_field(message, schema.observation["observation.state"]), np.arange(14) + ) + assert schema.action["action"].source_kind == "joint_position_updates" + assert schema.observation["observation.state"].source_kind == "snapshot" + assert { + key: feature.stream + for key, feature in schema.observation.items() + if feature.dtype == "video" + } == { + "observation.images.left_wrist": "left_wrist_image", + "observation.images.right_wrist": "right_wrist_image", + } + [recorder] = [ + atom for atom in dual_openyam_quest_collection.active_blueprints if atom.name == "recorder" + ] + # Every Module inherits tf; the profile recorder does not record that input. + assert { + port.name for port in recorder.streams if port.direction == "in" and port.name != "tf" + } == { + "left_wrist_image", + "right_wrist_image", + "coordinator_joint_state", + "applied_joint_position_command", + "status", + } + + +def test_dual_collection_configures_both_cameras_and_buses_through_run(tmp_path): + blueprint = dual_openyam_quest_collection + parsed = BlueprintConfigParser(blueprint).parse( + [ + "--recorder.recording", + str(tmp_path / "dual"), + "--episodes.task", + "fold towel", + "--controlcoordinator.left-can-port", + "follower_l", + "--controlcoordinator.right-can-port", + "follower_r", + "--left-wrist.hardware.camera-index", + "/dev/video2", + "--right-wrist.hardware.camera-index", + "/dev/video4", + ], + environ={}, + ) + cameras = [atom for atom in blueprint.active_blueprints if atom.module is CameraModule] + assert [camera.name for camera in cameras] == ["left_wrist", "right_wrist"] + assert blueprint.remapping_map[("left_wrist", "color_image")] == "left_wrist_image" + assert blueprint.remapping_map[("right_wrist", "color_image")] == "right_wrist_image" + assert parsed.module_configs["ControlCoordinator"]["left_can_port"] == "follower_l" + assert parsed.module_configs["ControlCoordinator"]["right_can_port"] == "follower_r" + assert parsed.module_configs["left_wrist"]["hardware"]["camera_index"] == "/dev/video2" + assert parsed.module_configs["right_wrist"]["hardware"]["camera_index"] == "/dev/video4" + + +def test_collection_exposes_inherited_viser_without_changing_teleop(tmp_path, mocker): + download = mocker.patch( + "dimos.utils.data.get_data", side_effect=AssertionError("Config must not download models") + ) + args = ["--recorder.recording", str(tmp_path / "dual"), "--episodes.task", "fold towel"] + parser = BlueprintConfigParser(dual_openyam_quest_collection) + collection = parser.parse(args, environ={}).module_kwargs("manipulationmodule") + teleop = ( + BlueprintConfigParser(teleop_webxr_dual_openyam) + .parse(environ={}) + .module_kwargs("manipulationmodule") + ) + + assert collection["visualization"]["host"] == "0.0.0.0" + assert teleop["visualization"]["host"] == "127.0.0.1" + assert collection["visualization"] == {**teleop["visualization"], "host": "0.0.0.0"} + assert collection["kinematics"] == teleop["kinematics"] + [collection_module] = [ + atom + for atom in dual_openyam_quest_collection.active_blueprints + if atom.module is ManipulationModule + ] + [teleop_module] = [ + atom + for atom in teleop_webxr_dual_openyam.active_blueprints + if atom.module is ManipulationModule + ] + # Check inheritance without resolving the copied lazy model paths. + assert collection_module.kwargs["model"] is teleop_module.kwargs["model"] + + restricted = parser.parse( + [*args, "--manipulationmodule.visualization.host", "127.0.0.1"], environ={} + ).module_kwargs("manipulationmodule") + assert restricted["visualization"] == teleop["visualization"] + download.assert_not_called() diff --git a/dimos/robot/manipulators/dual_openyam/learning.py b/dimos/robot/manipulators/dual_openyam/learning.py new file mode 100644 index 0000000000..c927a68e53 --- /dev/null +++ b/dimos/robot/manipulators/dual_openyam/learning.py @@ -0,0 +1,60 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Dual-arm collection preset, independent of policy backends and hardware.""" + +from dimos.imitation.collection.profile import CollectionFeature, CollectionProfile +from dimos.imitation.dataprep.core import QualityConfig, SyncConfig +from dimos.msgs.sensor_msgs.Image import Image +from dimos.msgs.sensor_msgs.JointState import JointState +from dimos.robot.manipulators.dual_openyam.joints import DUAL_OPENYAM_JOINTS + +DUAL_OPENYAM_COLLECTION = CollectionProfile( + name="dual-openyam-quest", + robot_type="dual_openyam", + observations={ + **{ + f"observation.images.{side}_wrist": CollectionFeature( + stream=f"{side}_wrist_image", + message_type=Image, + field="data", + dtype="video", + shape=(480, 640, 3), + names=["height", "width", "channels"], + ) + for side in ("left", "right") + }, + "observation.state": CollectionFeature( + stream="coordinator_joint_state", + message_type=JointState, + field="position", + dtype="float32", + shape=(len(DUAL_OPENYAM_JOINTS),), + names=list(DUAL_OPENYAM_JOINTS), + ), + }, + actions={ + "action": CollectionFeature( + stream="applied_joint_position_command", + source_kind="joint_position_updates", + message_type=JointState, + field="position", + dtype="float32", + shape=(len(DUAL_OPENYAM_JOINTS),), + names=list(DUAL_OPENYAM_JOINTS), + ), + }, + sync=SyncConfig(anchor="observation.images.left_wrist", rate_hz=30, tolerance_ms=20), + quality=QualityConfig(mode="strict"), +) diff --git a/docs/capabilities/manipulation/imitation-learning.md b/docs/capabilities/manipulation/imitation-learning.md index 6ece47a3e9..da40c6fed6 100644 --- a/docs/capabilities/manipulation/imitation-learning.md +++ b/docs/capabilities/manipulation/imitation-learning.md @@ -36,6 +36,7 @@ before switching from collection to rollout. | --- | --- | --- | | `openyam-teach-collection` | Wrist RGB | Measured 7-D joints for both | | `openyam-quest-collection` | Wrist RGB | Measured state, accepted commands | +| `dual-openyam-quest-collection` | Two wrist RGB cameras | Measured state, accepted commands; 14-D | ```bash dimos --can-port follower_l run openyam-teach-collection --daemon \ @@ -74,6 +75,35 @@ the arm before stopping the stack, since shutdown may de-torque it: dimos stop ``` +### Dual-arm collection with Quest + +For two arms with Quest, launch the dual collection blueprint and attach the +same episode controls: + +```bash skip +dimos run dual-openyam-quest-collection --daemon \ + --recorder.recording recordings/fold-001 \ + --recorder.format mcap \ + --episodes.task "fold the towel" \ + --controlcoordinator.left-can-port follower_l \ + --controlcoordinator.right-can-port follower_r \ + --left-wrist.hardware.camera-index /dev/video0 \ + --right-wrist.hardware.camera-index /dev/video2 +dimos imitation collect +``` + +The dual profile uses 640x480 RGB images at 30 Hz with a 20 ms alignment +tolerance anchored on the left wrist. Joint order is left arm joints 1-6, +right arm joints 1-6, left gripper, right gripper. Images and measured state +are snapshots; accepted actions declare `source_kind="joint_position_updates"`. +Prepare and train using the same commands below with the dual recording and +dataset paths. Dual-arm collection does not imply support for dual-arm rollout. + +The collection Blueprint constructs dual teleop with +`ViserVisualizationConfig(host="0.0.0.0")`; ordinary dual teleop retains its +local-only default. Open `http://ROBOT_HOST:8095` on a trusted network, or restrict +access with `--manipulationmodule.visualization.host 127.0.0.1`. + ## Recording directories ```text