From 6fd0cc1dd9c5ddc8c9d6d44f9b6189b4475295a7 Mon Sep 17 00:00:00 2001
From: Dan Wahl <hi@danwahl.net>
Date: Wed, 16 Sep 2026 17:50:09 -0500
Subject: [PATCH] feat(lab_sim): select the camera renderer from one setting

MOVEIT_PRO_CAMERA_RENDERER picks mujoco (the default), nyx, or both. The
robot description resolves the MuJoCo camera and point-cloud rates from it
and runtime.launch.xml reads the same variable to decide whether to start
picknik_nyx_renderer, so the two halves cannot disagree.

The point-cloud rate has to follow the render rate rather than stand alone:
the cloud timer publishes what the render timer produced, so running it
against a stopped renderer reports every camera as never having rendered.
Nyx has no depth output, so selecting it gives up point clouds.

An unrecognized value fails the description rather than leaving the config
with no cameras and no explanation.

Needs the Nyx renderer package, which is not in a released image yet, so
anything other than the default requires a moveit_pro build that carries it.

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_019YCk2yNsEQkVUSvhAmsJ4m
---
 src/lab_sim/description/picknik_ur.xacro | 17 +++++++++++++----
 src/lab_sim/launch/runtime.launch.xml    | 24 ++++++++++++++++++++++++
 2 files changed, 37 insertions(+), 4 deletions(-)

diff --git a/src/lab_sim/description/picknik_ur.xacro b/src/lab_sim/description/picknik_ur.xacro
index 470c2020f..536e7a741 100644
--- a/src/lab_sim/description/picknik_ur.xacro
+++ b/src/lab_sim/description/picknik_ur.xacro
@@ -2,6 +2,15 @@
 <robot name="ur5e" xmlns:xacro="http://ros.org/wiki/xacro">
   <xacro:arg name="mujoco_model" default="description/scene.xml" />
   <xacro:arg name="mujoco_model_package" default="lab_sim" />
+  <!-- Which renderer produces this config's camera topics: mujoco (the default), nyx, or both on separate topics. runtime.launch.xml reads the same variable to decide whether to start the Nyx node, so the two halves cannot disagree. An unrecognized value fails the description rather than leaving the config with no cameras. -->
+  <xacro:arg
+    name="camera_renderer"
+    default="$(optenv MOVEIT_PRO_CAMERA_RENDERER mujoco)"
+  />
+  <xacro:property
+    name="mujoco_cameras"
+    value="${ dict(mujoco=True, nyx=False, both=True)['$(arg camera_renderer)'] }"
+  />
   <xacro:arg
     name="point_cloud_publish_rate"
     default="$(optenv MOVEIT_PRO_POINT_CLOUD_PUBLISH_RATE 10)"
@@ -1203,10 +1212,10 @@
       <plugin>picknik_mujoco_ros/MujocoSystem</plugin>
       <param name="mujoco_model">$(arg mujoco_model)</param>
       <param name="mujoco_model_package">$(arg mujoco_model_package)</param>
-      <!-- The rate <camera> data is published at -->
-      <param name="render_publish_rate">10</param>
-      <!-- The rate point clouds generated from depth images are published at -->
-      <param name="point_cloud_publish_rate">$(arg point_cloud_publish_rate)</param>
+      <!-- The rate <camera> data is published at. Zero switches MuJoCo's cameras off: the image timer is only created for a non-zero rate. -->
+      <param name="render_publish_rate">${10 if mujoco_cameras else 0}</param>
+      <!-- The rate point clouds generated from depth images are published at. Zero for the same reason, and it follows the render rate rather than standing alone: the cloud timer publishes what the render timer produced, so against a stopped renderer it would report every camera as never having rendered. Nyx has no depth output, so selecting it gives up point clouds. -->
+      <param name="point_cloud_publish_rate">${$(arg point_cloud_publish_rate) if mujoco_cameras else 0}</param>
       <!-- The rate transforms to <site> poses are published at -->
       <param name="tf_publish_rate">60</param>
       <!-- The rate <rangefinder> data is published at -->
diff --git a/src/lab_sim/launch/runtime.launch.xml b/src/lab_sim/launch/runtime.launch.xml
index b2aa75434..cb3591eca 100644
--- a/src/lab_sim/launch/runtime.launch.xml
+++ b/src/lab_sim/launch/runtime.launch.xml
@@ -3,4 +3,28 @@
   <include
     file="$(find-pkg-share moveit_studio_agent)/launch/studio_agent_bridge.launch.xml"
   />
+
+  <!-- Nyx path-traced cameras, started only when this config asks for them. The same
+       MOVEIT_PRO_CAMERA_RENDERER the robot description reads to set the MuJoCo camera rate:
+         mujoco  MuJoCo cameras only. The default, and the only option on an image carrying no Nyx
+                 runtime, which is every arm64 image and the amd64 CPU image.
+         nyx     Nyx only, on <camera>/nyx_color. MuJoCo's camera and point-cloud rates go to 0.
+         both    Both, on their own topics, at the cost of running two renderers.
+       Nyx needs a scene exported ahead of time with nyx_export_scene; see
+       src/nyx_renderer/README.md in moveit_pro. The node reports a missing scene and exits. -->
+  <let
+    name="camera_renderer"
+    value="$(env MOVEIT_PRO_CAMERA_RENDERER mujoco)"
+  />
+  <include
+    file="$(find-pkg-share picknik_nyx_renderer)/launch/nyx_camera.launch.py"
+    if="$(eval '&quot;$(var camera_renderer)&quot; in (&quot;nyx&quot;, &quot;both&quot;)')"
+  >
+    <arg name="mjcf" value="$(find-pkg-share lab_sim)/description/scene.xml" />
+    <arg
+      name="scene_dir"
+      value="$(env USER_WS $(env HOME)/user_ws)/.nyx_scenes/lab_sim"
+    />
+    <arg name="camera" value="scene_camera" />
+  </include>
 </launch>