diff --git a/dimos/e2e_tests/test_map_click_browser.py b/dimos/e2e_tests/test_map_click_browser.py new file mode 100644 index 0000000000..2152eb80e9 --- /dev/null +++ b/dimos/e2e_tests/test_map_click_browser.py @@ -0,0 +1,140 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Browser-to-bridge coverage for map clicks and path-driven cancellation.""" + +from collections.abc import Iterator +import threading +from typing import NamedTuple + +import pytest +from reactivex.disposable import Disposable + +from dimos.core.transport import pLCMTransport +from dimos.msgs.geometry_msgs.PointStamped import PointStamped +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.msgs.nav_msgs.Path import Path +from dimos.web.cockpit import Map2D, cockpit +from dimos.web.relay_bridge.e2e_support import stop_module +from dimos.web.relay_bridge.gen_costmap_fixtures import grid_msg +from dimos.web.relay_bridge.module_test_support import wait_until +from dimos.web.relay_bridge.relay_bridge_module import RelayBridgeModule + +pytest.importorskip("playwright") + +from playwright.sync_api import Page, expect, sync_playwright + +pytestmark = pytest.mark.web_browser + +CANVAS = '[data-testid="map2d-global_costmap-canvas"]' +# 2x2 cells at 0.5 m with the origin corner at (0.25, -0.5): world centre (0.75, 0). +GRID = grid_msg([[0, 50], [100, -1]], 0.5, 0.25, -0.5, 0.0) + + +class MapBridge(NamedTuple): + url: str + module: RelayBridgeModule + path: pLCMTransport + + +def _path(*xy: tuple[float, float]) -> Path: + poses = [ + PoseStamped(ts=1.0, position=[x, y, 0.0], orientation=[0.0, 0.0, 0.0, 1.0]) for x, y in xy + ] + return Path(ts=1.0, frame_id="world", poses=poses) + + +@pytest.fixture(scope="module") +def map_bridge() -> Iterator[MapBridge]: + layout = Map2D(pose=None, path="path", click="clicked_point", stop="stop_movement") + (atom,) = cockpit(layout=layout).blueprints + module = atom.module(local_port=0, open_browser=False, robot_id="map-click-e2e", **atom.kwargs) + transports: list[pLCMTransport] = [] + publishers: dict[str, pLCMTransport] = {} + for ch in ("global_costmap", "path"): + topic = f"/map_click_e2e/{ch}" + bridge_side = pLCMTransport(topic) + bridge_side.start() + getattr(module, ch).transport = bridge_side + publishers[ch] = pLCMTransport(topic) + publishers[ch].start() + transports += [bridge_side, publishers[ch]] + stop = threading.Event() + + def mapper() -> None: + # Frames flow only once the bridge lazily subscribes, so keep them coming. + while not stop.is_set(): + publishers["global_costmap"].publish(GRID) + stop.wait(0.5) + + thread = threading.Thread(target=mapper, daemon=True) + thread.start() + try: + module.start() + relay = module._relay + assert relay is not None and relay.info is not None + yield MapBridge(relay.info.open_url, module, publishers["path"]) + finally: + stop.set() + thread.join(timeout=2) + stop_module(module) + for transport in transports: + transport.stop() + + +@pytest.fixture +def chromium_page() -> Iterator[Page]: + with sync_playwright() as p: + browser = p.chromium.launch() + try: + yield browser.new_page() + finally: + browser.close() + + +def test_click_to_goal_and_cancel(map_bridge: MapBridge, chromium_page: Page) -> None: + clicks: list[PointStamped] = [] + stops: list[bool] = [] + map_bridge.module.register_disposable( + Disposable(map_bridge.module.clicked_point.subscribe(clicks.append)) + ) + map_bridge.module.register_disposable( + Disposable(map_bridge.module.stop_movement.subscribe(lambda msg: stops.append(msg.data))) + ) + + chromium_page.goto(map_bridge.url) + # The first grid sizes the backing store from the layout (300 is the + # unsized default): the map is drawn once that happens. + chromium_page.wait_for_function( + f"""() => {{ + const canvas = document.querySelector('{CANVAS}'); + return canvas !== null && canvas.width !== 300; + }}""", + timeout=120_000, + ) + chromium_page.click(CANVAS) + assert wait_until(lambda: len(clicks) == 1, timeout=15.0) + assert clicks[0].x == pytest.approx(0.75, abs=0.01) + assert clicks[0].y == pytest.approx(0.0, abs=0.01) + assert clicks[0].frame_id == "world" + + cancel = chromium_page.get_by_test_id("map2d-global_costmap-cancel") + expect(cancel).to_have_count(0) + map_bridge.path.publish(_path((0.3, -0.4), (0.75, 0.0), (1.2, 0.4))) + expect(cancel).to_be_visible(timeout=30_000) + cancel.click() + assert wait_until(lambda: stops == [True], timeout=15.0) + + map_bridge.path.publish(Path()) + expect(cancel).to_have_count(0, timeout=30_000) diff --git a/dimos/navigation/basic_path_follower/module.py b/dimos/navigation/basic_path_follower/module.py index b83024de20..f27068fbb2 100644 --- a/dimos/navigation/basic_path_follower/module.py +++ b/dimos/navigation/basic_path_follower/module.py @@ -19,7 +19,7 @@ import time from typing import Any -from dimos_lcm.std_msgs import Bool # type: ignore[import-untyped] +from dimos_lcm.std_msgs import Bool import numpy as np from numpy.typing import NDArray from reactivex.disposable import Disposable diff --git a/dimos/navigation/dannav/holonomic_tc/module.py b/dimos/navigation/dannav/holonomic_tc/module.py index 060fa81ae0..73fc8e7752 100644 --- a/dimos/navigation/dannav/holonomic_tc/module.py +++ b/dimos/navigation/dannav/holonomic_tc/module.py @@ -31,7 +31,7 @@ import traceback from typing import Any, Literal, TypeAlias -from dimos_lcm.std_msgs import Bool # type: ignore[import-untyped] +from dimos_lcm.std_msgs import Bool import numpy as np from reactivex import Subject from reactivex.disposable import Disposable diff --git a/dimos/navigation/dannav/holonomic_tc/test_dan_holonomic_tc.py b/dimos/navigation/dannav/holonomic_tc/test_dan_holonomic_tc.py index 9a7a81fa87..50a0096c8e 100644 --- a/dimos/navigation/dannav/holonomic_tc/test_dan_holonomic_tc.py +++ b/dimos/navigation/dannav/holonomic_tc/test_dan_holonomic_tc.py @@ -23,7 +23,7 @@ import time from typing import Any, Literal -from dimos_lcm.std_msgs import Bool # type: ignore[import-untyped] +from dimos_lcm.std_msgs import Bool import pytest from dimos.core.stream import Stream, Transport diff --git a/dimos/navigation/movement_manager/movement_manager.py b/dimos/navigation/movement_manager/movement_manager.py index ed12dc93ac..4f83428f9a 100644 --- a/dimos/navigation/movement_manager/movement_manager.py +++ b/dimos/navigation/movement_manager/movement_manager.py @@ -25,7 +25,7 @@ import time from typing import Any -from dimos_lcm.std_msgs import Bool # type: ignore[import-untyped] +from dimos_lcm.std_msgs import Bool from reactivex.disposable import Disposable from dimos.core.core import rpc diff --git a/dimos/robot/galaxea/r1pro/connection.py b/dimos/robot/galaxea/r1pro/connection.py index 127dba59c8..3275427ce8 100644 --- a/dimos/robot/galaxea/r1pro/connection.py +++ b/dimos/robot/galaxea/r1pro/connection.py @@ -111,7 +111,7 @@ def _make_qos() -> Any: """BEST_EFFORT + VOLATILE QoS — the profile the R1 Pro topics expect.""" from rclpy.qos import DurabilityPolicy, QoSProfile, ReliabilityPolicy - return QoSProfile( + return QoSProfile( # type: ignore[no-untyped-call] depth=10, reliability=ReliabilityPolicy.BEST_EFFORT, durability=DurabilityPolicy.VOLATILE, diff --git a/dimos/robot/unitree/go2/blueprints/agentic/unitree_go2_agentic_cockpit.py b/dimos/robot/unitree/go2/blueprints/agentic/unitree_go2_agentic_cockpit.py index ca9c5f0efc..9d41b04a46 100644 --- a/dimos/robot/unitree/go2/blueprints/agentic/unitree_go2_agentic_cockpit.py +++ b/dimos/robot/unitree/go2/blueprints/agentic/unitree_go2_agentic_cockpit.py @@ -38,7 +38,7 @@ layout=Row( Video("color_image", title="Front camera"), Col( - Map2D(costmap="global_costmap", pose="odom", title="Map"), + Map2D(path="path", click="clicked_point", stop="stop_movement", title="Map"), Teleop(title="Keyboard teleop"), shares=[3, 1], ), diff --git a/dimos/robot/unitree/go2/blueprints/smart/unitree_go2_cockpit.py b/dimos/robot/unitree/go2/blueprints/smart/unitree_go2_cockpit.py index 0b53383665..371c900c06 100644 --- a/dimos/robot/unitree/go2/blueprints/smart/unitree_go2_cockpit.py +++ b/dimos/robot/unitree/go2/blueprints/smart/unitree_go2_cockpit.py @@ -30,7 +30,11 @@ cockpit( layout=Row( Video("color_image"), - Col(Map2D(costmap="global_costmap", pose="odom"), Teleop(), shares=[3, 1]), + Col( + Map2D(path="path", click="clicked_point", stop="stop_movement"), + Teleop(), + shares=[3, 1], + ), shares=[2, 1], ), ), diff --git a/dimos/teleop/hosted/arm_command.py b/dimos/teleop/hosted/arm_command.py index aaafb2e230..47b8aec626 100644 --- a/dimos/teleop/hosted/arm_command.py +++ b/dimos/teleop/hosted/arm_command.py @@ -38,6 +38,7 @@ from dimos.teleop.webxr.controller_types import Hand from dimos.teleop.webxr.extensions import ArmTeleopModule from dimos.teleop.webxr.module import WebXRTeleopConfig +from dimos.utils.generic import finite_number from dimos.utils.logging_config import setup_logger logger = setup_logger() @@ -219,8 +220,8 @@ def _handle_teleop_scale(self, msg: dict[str, Any]) -> None: self._send_ack(nonce, False) return try: - self._set_translation_scale(float(msg["scale"])) - except (KeyError, TypeError, ValueError): + self._set_translation_scale(finite_number(msg.get("scale"), "scale")) + except ValueError: self._send_ack(nonce, False) return self._send_ack(nonce, True) diff --git a/dimos/teleop/hosted/go2_command.py b/dimos/teleop/hosted/go2_command.py index 73044aad52..bdac4661e7 100644 --- a/dimos/teleop/hosted/go2_command.py +++ b/dimos/teleop/hosted/go2_command.py @@ -39,6 +39,7 @@ from dimos.msgs.geometry_msgs.TwistStamped import TwistStamped from dimos.robot.unitree.go2.connection import GO2Connection from dimos.teleop.hosted.command_executor import SerializedCommandExecutor +from dimos.utils.generic import finite_number from dimos.utils.logging_config import setup_logger logger = setup_logger() @@ -319,14 +320,11 @@ def _handle_light(self, msg: dict[str, Any]) -> None: if raw is None: raw = 1.0 if msg.get("enabled") else 0.0 # legacy on/off toggle try: - brightness = float(raw) - except (TypeError, ValueError): + brightness = finite_number(raw, "brightness") + except ValueError: logger.warning("light: malformed brightness %r", raw) self._send_ack(nonce, False) return - if math.isnan(brightness): - self._send_ack(nonce, False) - return brightness = max(0.0, min(1.0, brightness)) level = round(brightness * 10) @@ -349,13 +347,13 @@ def _handle_nav_goal(self, msg: dict[str, Any]) -> None: self._send_ack(nonce, False) return try: - x, y = float(msg["x"]), float(msg["y"]) - except (KeyError, TypeError, ValueError): + x, y = finite_number(msg.get("x"), "x"), finite_number(msg.get("y"), "y") + except ValueError: logger.warning("nav_goal: malformed %r", msg) self._send_ack(nonce, False) return limit = self.config.max_nav_goal_m - if not (math.isfinite(x) and math.isfinite(y)) or abs(x) > limit or abs(y) > limit: + if abs(x) > limit or abs(y) > limit: logger.warning("nav_goal: out-of-range (%r, %r)", x, y) self._send_ack(nonce, False) return diff --git a/dimos/teleop/hosted/test_go2_command.py b/dimos/teleop/hosted/test_go2_command.py index a7046f4636..17430fdb16 100644 --- a/dimos/teleop/hosted/test_go2_command.py +++ b/dimos/teleop/hosted/test_go2_command.py @@ -381,6 +381,18 @@ def test_nav_goal_publishes_and_acks( assert acks == [(11, True)] +def test_nav_goal_rejects_non_numbers( + module: Go2CommandModule, monkeypatch: pytest.MonkeyPatch +) -> None: + acks: list[tuple[Any, bool]] = [] + monkeypatch.setattr(module, "_send_ack", lambda nonce, ok: acks.append((nonce, ok))) + + module._handle_nav_goal({"x": "2.5", "y": 1.0, "nonce": 17}) + + module.goal_request.publish.assert_not_called() + assert acks == [(17, False)] + + def test_nav_goal_rejected_when_estopped( module: Go2CommandModule, monkeypatch: pytest.MonkeyPatch ) -> None: diff --git a/dimos/utils/generic.py b/dimos/utils/generic.py index 200c7c6d86..3b92c9ffe8 100644 --- a/dimos/utils/generic.py +++ b/dimos/utils/generic.py @@ -15,6 +15,7 @@ from collections.abc import Callable import hashlib import json +import math import os import socket import string @@ -73,6 +74,13 @@ def extract_json_from_llm_response(response: str) -> Any: return None +def finite_number(value: Any, name: str) -> float: + """Validate an untrusted JSON number: not a bool, not a string, finite.""" + if isinstance(value, bool) or not isinstance(value, (int, float)) or not math.isfinite(value): + raise ValueError(f"{name} must be a finite number, got {value!r}") + return float(value) + + def short_id(from_string: str | None = None) -> str: alphabet = string.digits + string.ascii_letters base = len(alphabet) diff --git a/dimos/utils/test_generic.py b/dimos/utils/test_generic.py index 0f691bc23c..a0293026d3 100644 --- a/dimos/utils/test_generic.py +++ b/dimos/utils/test_generic.py @@ -12,9 +12,12 @@ # See the License for the specific language governing permissions and # limitations under the License. +import math from uuid import UUID -from dimos.utils.generic import short_id +import pytest + +from dimos.utils.generic import finite_number, short_id def test_short_id_hello_world() -> None: @@ -29,3 +32,14 @@ def test_short_id_uuid_one(mocker) -> None: def test_short_id_uuid_zero(mocker) -> None: mocker.patch("uuid.uuid4", return_value=UUID("00000000-0000-0000-0000-000000000000")) assert short_id() == "000000000000000000" + + +def test_finite_number_coerces_to_float() -> None: + assert finite_number(2, "x") == 2.0 + assert isinstance(finite_number(2, "x"), float) + + +@pytest.mark.parametrize("value", [None, "1.5", True, math.nan, math.inf, [1.0]]) +def test_finite_number_rejects_non_numbers(value: object) -> None: + with pytest.raises(ValueError, match="x must be a finite number"): + finite_number(value, "x") diff --git a/dimos/web/cockpit.py b/dimos/web/cockpit.py index c06c7f66f5..471724de06 100644 --- a/dimos/web/cockpit.py +++ b/dimos/web/cockpit.py @@ -46,6 +46,10 @@ else: from typing_extensions import Self +from dimos_lcm.std_msgs import Bool + +from dimos.msgs.geometry_msgs.PointStamped import PointStamped +from dimos.msgs.nav_msgs.Path import Path from dimos.web.relay_bridge.manifest import ( MANIFEST_VERSION, MAX_MANIFEST_ID_LEN, @@ -145,10 +149,6 @@ class ChannelRequest: delivery: Delivery = field(default="reliable", kw_only=True) publish: Publish = field(default="none", kw_only=True) required_scope: str | None = field(default=None, kw_only=True) - # Event streams (chat): the bridge meets max_hz by spacing sends, never by - # dropping, so a burst of messages crosses complete and in order. Panels - # only; a plain Channel is sampled at max_hz. - paced: bool = field(default=False, kw_only=True) @dataclass(frozen=True) @@ -172,6 +172,10 @@ class Channel: (the teleop panel); publish="exclusive" arrives with the lease ticket (W8). `required_scope` names the operator scope a remote relay demands (the local relay never checks scopes). + + rx channels are sampled at max_hz. `paced` spaces sends instead (event + streams: a burst crosses complete and in order); `resend_on_subscribe` + replays the last message when the first viewer subscribes (state streams). """ stream: str @@ -183,6 +187,8 @@ class Channel: params: Mapping[str, Any] | None = field(default=None, kw_only=True) publish: Literal["none", "shared", "exclusive"] = field(default="none", kw_only=True) required_scope: str | None = field(default=None, kw_only=True) + paced: bool = field(default=False, kw_only=True) + resend_on_subscribe: bool = field(default=False, kw_only=True) def __post_init__(self) -> None: _check_stream("stream", self.stream) @@ -237,6 +243,12 @@ def __post_init__(self) -> None: f"required_scope must be 1..{MAX_MANIFEST_ID_LEN} chars, " f"got {self.required_scope!r}" ) + for name in ("paced", "resend_on_subscribe"): + flag = getattr(self, name) + if not isinstance(flag, bool): + raise ValueError(f"{name} must be a bool, got {flag!r}") + if flag and self.dir != "rx": + raise ValueError(f"{name} applies to rx channels only") if self.params is not None and not isinstance(self.params, Mapping): raise ValueError(f"params must be a mapping or None, got {self.params!r}") params = {} if self.params is None else dict(self.params) @@ -324,22 +336,70 @@ def _channel_requests(self) -> tuple[ChannelRequest, ...]: @dataclass(frozen=True) class Map2D(Panel): - """2D costmap with an optional pose overlay (pose=None drops it).""" + """2D costmap with optional pose and path overlays. + + `click` publishes PointStamped goals; `stop` publishes Bool cancellation + requests while a path is active, regardless of who set the goal. + """ kind: ClassVar[str] = "map2d" costmap: str = "global_costmap" pose: str | None = "odom" + path: str | None = field(default=None, kw_only=True) + click: str | None = field(default=None, kw_only=True) + stop: str | None = field(default=None, kw_only=True) costmap_hz: float = field(default=5.0, kw_only=True) pose_hz: float = field(default=20.0, kw_only=True) title: str = field(default="", kw_only=True) def __post_init__(self) -> None: _check_stream("costmap", self.costmap) - if self.pose is not None: - _check_stream("pose", self.pose) + for name in ("pose", "path", "click", "stop"): + stream = getattr(self, name) + if stream is not None: + _check_stream(name, stream) _check_rate("costmap_hz", self.costmap_hz) _check_rate("pose_hz", self.pose_hz) + def _channels(self) -> tuple[Channel, ...]: + channels = [] + if self.path is not None: + # Sampling can drop the final path or clear in a planner burst. + channels.append( + Channel( + self.path, + Path, + encoding="path.json.v1", + delivery="latest", + max_hz=10.0, + paced=True, + resend_on_subscribe=True, + ) + ) + if self.click is not None: + channels.append( + Channel( + self.click, + PointStamped, + dir="tx", + encoding="point.json.v1", + publish="shared", + max_hz=5.0, + ) + ) + if self.stop is not None: + channels.append( + Channel( + self.stop, Bool, dir="tx", encoding="bool.json.v1", publish="shared", max_hz=5.0 + ) + ) + return tuple(channels) + + def _panel_params(self) -> dict[str, Any]: + # Keep the existing costmap/pose slots compatible with older viewers. + bound = {"path": self.path, "click": self.click, "stop": self.stop} + return {key: stream for key, stream in bound.items() if stream is not None} + def _channel_requests(self) -> tuple[ChannelRequest, ...]: requests = [ ChannelRequest( @@ -435,8 +495,10 @@ def _channels(self) -> tuple[Channel, ...]: Channel( self.input, str, dir="tx", encoding="text.json.v1", publish="shared", max_hz=5.0 ), - Channel(self.messages, BaseMessage, encoding="chat.json.v1", max_hz=20.0), - Channel(self.idle, bool, delivery="latest", max_hz=20.0), + # Every agent message and idle flip must reach the viewer: paced, + # not sampled. + Channel(self.messages, BaseMessage, encoding="chat.json.v1", max_hz=20.0, paced=True), + Channel(self.idle, bool, delivery="latest", max_hz=20.0, paced=True), Channel( self.audio, AudioChunk, @@ -448,9 +510,7 @@ def _channels(self) -> tuple[Channel, ...]: ) def _channel_requests(self) -> tuple[ChannelRequest, ...]: - # Every agent message and idle flip must reach the viewer: paced, - # not sampled. - return tuple(replace(_request_of(channel), paced=True) for channel in self._channels()) + return tuple(_request_of(channel) for channel in self._channels()) @dataclass(frozen=True) @@ -604,6 +664,8 @@ def merge(request: ChannelRequest) -> None: def add_panel(panel: Panel) -> str: panel_id = f"p{len(panels_out)}" + for channel in panel._channels(): + merge(_request_of(channel)) requests = panel._channel_requests() for request in requests: merge(request) @@ -775,19 +837,22 @@ def cockpit( from dimos.web.codecs import resolve_decoder, resolve_encoder layout = _default_preset() if layout is None and not declared else layout - # Panels binding non-built-in streams (Chat) declare them like explicit - # channels. An explicit declaration for the same stream stands, provided - # it agrees (the rest of the agreement check is build_manifest_data's). - paced: set[str] = set() - for panel in (*_panels(layout), *_panels(tuple(pages))): - paced.update(r.stream for r in panel._channel_requests() if r.paced) - for channel in panel._channels(): - previous = declared.setdefault(channel.stream, channel) - if previous.message_type is not channel.message_type: - raise ValueError( - f"conflicting declarations for stream {channel.stream!r}: " - f"{previous!r} vs {channel!r}" - ) + # Explicit declarations must agree with the panel; build_manifest_data + # checks the wire requirements. Bridge flags are combined below. + panel_channels = [ + channel + for panel in (*_panels(layout), *_panels(tuple(pages))) + for channel in panel._channels() + ] + for channel in panel_channels: + previous = declared.setdefault(channel.stream, channel) + if previous.message_type is not channel.message_type: + raise ValueError( + f"conflicting declarations for stream {channel.stream!r}: " + f"{previous!r} vs {channel!r}" + ) + paced = {c.stream for c in (*channels, *panel_channels) if c.paced} + resend = {c.stream for c in (*channels, *panel_channels) if c.resend_on_subscribe} atom = RelayBridgeModule.blueprint().blueprints[0] port_types = {s.name: s.type for s in atom.streams} @@ -863,7 +928,9 @@ def cockpit( params=dict(wire["params"]), encoder=codec.encode, encoder_takes_params=codec.takes_params, - resend_on_subscribe=builtin.resend_on_subscribe if builtin is not None else False, + resend_on_subscribe=( + ch in resend or (builtin is not None and builtin.resend_on_subscribe) + ), paced=ch in paced, ) ) diff --git a/dimos/web/relay_bridge/builtin_codecs.py b/dimos/web/relay_bridge/builtin_codecs.py index a5928f445d..6efbbbd4c0 100644 --- a/dimos/web/relay_bridge/builtin_codecs.py +++ b/dimos/web/relay_bridge/builtin_codecs.py @@ -13,7 +13,7 @@ # limitations under the License. """Built-in web codecs (jpeg.v1, pose.json.v1, costmap.zlib.v1, text.json.v1, -stats.json.v1). +stats.json.v1, path.json.v1, point.json.v1, bool.json.v1). Registered into dimos.web.codecs at import time; relay_bridge_module imports this module so every bridge process (parent and worker) has the built-ins. @@ -26,11 +26,15 @@ from typing import Any import zlib +from dimos_lcm.std_msgs import Bool import numpy as np +from dimos.msgs.geometry_msgs.PointStamped import PointStamped from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped from dimos.msgs.nav_msgs.OccupancyGrid import OccupancyGrid, block_max_reduce +from dimos.msgs.nav_msgs.Path import Path from dimos.msgs.sensor_msgs.Image import Image +from dimos.utils.generic import finite_number from dimos.web.codecs import EncodedPayload, web_decoder, web_encoder # Custom jpeg channels authored without a quality param; the built-in @@ -77,6 +81,29 @@ def encode_pose(msg: PoseStamped) -> bytes: return json.dumps(pose, separators=(",", ":")).encode() +@web_encoder("path.json.v1") +def encode_path(msg: Path) -> bytes: + # Empty paths must reach the viewer to clear the overlay. + points = [[round(p.x, 3), round(p.y, 3)] for p in msg.poses] + return json.dumps(points, separators=(",", ":"), allow_nan=False).encode() + + +@web_decoder("point.json.v1") +def decode_point(msg: dict[str, Any]) -> PointStamped: + if not isinstance(msg, dict): + raise ValueError(f"point.json.v1 wants an object, got {type(msg).__name__}") + return PointStamped( + finite_number(msg.get("x"), "x"), finite_number(msg.get("y"), "y"), frame_id="world" + ) + + +@web_decoder("bool.json.v1") +def decode_bool(msg: bool) -> Bool: + if not isinstance(msg, bool): + raise ValueError(f"bool.json.v1 wants a boolean, got {type(msg).__name__}") + return Bool(data=msg) + + # The historical costmap encoder's choice (websocket_vis/optimized_costmap.py); # full grids compress to ~10-30 KB at <= 5 Hz, so speed over ratio is fine. _COSTMAP_ZLIB_LEVEL = 6 diff --git a/dimos/web/relay_bridge/relay_bridge_module.py b/dimos/web/relay_bridge/relay_bridge_module.py index e374366056..7d2100ae7c 100644 --- a/dimos/web/relay_bridge/relay_bridge_module.py +++ b/dimos/web/relay_bridge/relay_bridge_module.py @@ -60,6 +60,7 @@ from dimos.msgs.geometry_msgs.Vector3 import Vector3 from dimos.msgs.nav_msgs.OccupancyGrid import OccupancyGrid from dimos.msgs.sensor_msgs.Image import Image +from dimos.utils.generic import finite_number from dimos.utils.logging_config import setup_logger # No import cycle: cockpit.py only imports this module lazily inside @@ -1173,17 +1174,13 @@ def _resolve_teleop_params(self, spec: ChannelSpec) -> _TeleopParams: values: dict[str, float] = {} for key, default in _TELEOP_PARAM_DEFAULTS.items(): candidate = spec.params.get(key, default) - if ( - isinstance(candidate, bool) - or not isinstance(candidate, (int, float)) - or not math.isfinite(candidate) - or candidate <= 0 - ): + value = finite_number(candidate, f"manifest channel {spec.ch!r} {key}") + if value <= 0: raise RuntimeError( f"manifest channel {spec.ch!r} {key} must be a positive number, " f"got {candidate!r}" ) - values[key] = float(candidate) + values[key] = value return _TeleopParams( max_linear=values["maxLinear"], max_angular=values["maxAngular"], diff --git a/dimos/web/relay_bridge/test_map_codecs.py b/dimos/web/relay_bridge/test_map_codecs.py new file mode 100644 index 0000000000..e26fa14345 --- /dev/null +++ b/dimos/web/relay_bridge/test_map_codecs.py @@ -0,0 +1,73 @@ +# Copyright 2026 Dimensional Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""The map panel's nav codecs: path.json.v1 out, point.json.v1 and bool.json.v1 in.""" + +import math + +from dimos_lcm.std_msgs import Bool +import pytest + +from dimos.msgs.geometry_msgs.PointStamped import PointStamped +from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped +from dimos.msgs.nav_msgs.Path import Path +from dimos.web.codecs import resolve_decoder, resolve_encoder +from dimos.web.relay_bridge.builtin_codecs import decode_bool, decode_point, encode_path + + +def _pose(x: float, y: float) -> PoseStamped: + return PoseStamped(ts=1.0, position=[x, y, 0.0], orientation=[0.0, 0.0, 0.0, 1.0]) + + +def test_path_encodes_a_rounded_xy_polyline() -> None: + path = Path(ts=1.0, frame_id="world", poses=[_pose(1.23456, -2.0), _pose(0.0004, 3.5)]) + assert encode_path(path) == b"[[1.235,-2.0],[0.0,3.5]]" + assert resolve_encoder("path.json.v1", Path).encode is encode_path + + +def test_empty_path_encodes_as_a_clear() -> None: + # The planner publishes Path() on cancel and arrival: the overlay must go. + assert encode_path(Path()) == b"[]" + + +def test_point_decodes_a_click() -> None: + point = decode_point({"x": 1.5, "y": -2}) + assert isinstance(point, PointStamped) + assert (point.x, point.y, point.z, point.frame_id) == (1.5, -2.0, 0.0, "world") + assert resolve_decoder("point.json.v1", PointStamped).decode is decode_point + + +@pytest.mark.parametrize( + "value", + [ + [1.5, 2.0], + {"x": 1.5}, + {"x": "1.5", "y": 2.0}, + {"x": True, "y": 2.0}, + {"x": math.inf, "y": 0.0}, + ], + ids=["list", "missing_y", "string", "bool", "inf"], +) +def test_point_rejects_a_malformed_click(value: object) -> None: + with pytest.raises(ValueError, match="point.json.v1|finite number"): + decode_point(value) + + +def test_bool_decodes_to_std_msgs_bool() -> None: + assert isinstance(decode_bool(True), Bool) + assert decode_bool(True).data is True + assert decode_bool(False).data is False + assert resolve_decoder("bool.json.v1", Bool).decode is decode_bool + with pytest.raises(ValueError, match="bool.json.v1"): + decode_bool(1) diff --git a/dimos/web/relay_bridge/test_relay_bridge_authoring.py b/dimos/web/relay_bridge/test_relay_bridge_authoring.py index ce45c2064d..1758c61370 100644 --- a/dimos/web/relay_bridge/test_relay_bridge_authoring.py +++ b/dimos/web/relay_bridge/test_relay_bridge_authoring.py @@ -30,17 +30,19 @@ from langchain_core.messages import AIMessage import numpy as np import pytest +from reactivex.disposable import Disposable from dimos.core.coordination.blueprints import autoconnect from dimos.core.module import Module, ModuleConfig from dimos.core.resource_monitor.stats import ProcessStats, WorkerStats from dimos.core.stream import In, Out +from dimos.msgs.geometry_msgs.PointStamped import PointStamped from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped from dimos.msgs.geometry_msgs.Twist import Twist from dimos.msgs.nav_msgs.OccupancyGrid import OccupancyGrid from dimos.msgs.nav_msgs.Path import Path as NavPath from dimos.msgs.sensor_msgs.Image import Image -from dimos.web.cockpit import Channel, Chat, Stats, Video, cockpit +from dimos.web.cockpit import Channel, Chat, Map2D, Stats, Video, cockpit from dimos.web.codecs import EncodedPayload, PublishContext, web_decoder, web_encoder from dimos.web.relay_bridge import builtin_codecs, relay_bridge_module from dimos.web.relay_bridge.audio_codec import AudioChunk @@ -716,3 +718,91 @@ def test_publish_frame_with_unusable_meta_is_dropped(monkeypatch) -> None: assert module._pub_invalid == 8 finally: stop_module(module) + + +def _nav_path(*xy: tuple[float, float]) -> NavPath: + poses = [ + PoseStamped(ts=1.0, position=[x, y, 0.0], orientation=[0.0, 0.0, 0.0, 1.0]) for x, y in xy + ] + return NavPath(ts=1.0, frame_id="world", poses=poses) + + +@pytest.fixture +def map_bridge(monkeypatch): + blueprint = cockpit(layout=Map2D(path="path", click="clicked_point", stop="stop_movement")) + module, clients = start_authored( + monkeypatch, blueprint, wire=("global_costmap", "odom", "path") + ) + try: + yield module, clients + finally: + stop_module(module) + + +def test_map2d_rejects_an_incompatible_path_encoder() -> None: + with pytest.raises(ValueError, match="conflicting requirements for stream 'path'"): + cockpit( + layout=Map2D(path="path"), + channels=[Channel("path", NavPath, encoding="path.rbm.v1", delivery="latest")], + ) + + +def test_builtin_channel_replays_when_explicitly_requested(monkeypatch) -> None: + blueprint = cockpit( + channels=[Channel("odom", PoseStamped, encoding="pose.json.v1", resend_on_subscribe=True)] + ) + module, clients = start_authored(monkeypatch, blueprint, wire=("odom",)) + try: + transport_of(module, "odom").publish(_NAV_PATH.poses[0]) + push(module, clients[0], Subs(chs=["odom"], n=1)) + assert wait_until(lambda: clients[0].frames) + ch, payload, delivery, meta = clients[0].frames[0] + assert (ch, delivery, meta) == ("odom", "reliable", None) + assert json.loads(payload) == {"x": 1.5, "y": -2.5, "z": 0.0, "yaw": 0.0, "ts": 1.0} + finally: + stop_module(module) + + +def test_map2d_path_replays_and_paces(map_bridge) -> None: + module, clients = map_bridge + path = transport_of(module, "path") + offers = clients[0].writers["path"].offers + path.publish(_NAV_PATH) # nobody watching; the cache keeps it + push(module, clients[0], Subs(chs=["path"], n=1)) + assert wait_until(lambda: len(offers) == 1) + assert json.loads(offers[0][0]) == [[1.5, -2.5]] + + # A planner burst must not leave the viewer stuck on the empty path. + path.publish(NavPath()) + path.publish(_nav_path((0.25, 0.5), (1.0, 1.0), (1.75, 0.5))) + assert wait_until(lambda: len(offers) == 3) + assert [json.loads(payload) for payload, _ in offers[1:]] == [ + [], + [[0.25, 0.5], [1.0, 1.0], [1.75, 0.5]], + ] + + +def test_map2d_click_and_stop_publish(map_bridge) -> None: + module, clients = map_bridge + points: list[PointStamped] = [] + stops: list[bool] = [] + module.register_disposable(Disposable(module.clicked_point.subscribe(points.append))) + module.register_disposable( + Disposable(module.stop_movement.subscribe(lambda msg: stops.append(msg.data))) + ) + + click = json.dumps({"x": 1.5, "y": -2.25}).encode() + push(module, clients[0], _pub_frame(click, ch="clicked_point")) + assert wait_until(lambda: clients[0].control_frames) + assert isinstance(clients[0].control_frames[0], PubAck) + (point,) = points + assert (point.x, point.y, point.z, point.frame_id) == (1.5, -2.25, 0.0, "world") + + push(module, clients[0], _pub_frame(b"true", ch="stop_movement", seq=2)) + assert wait_until(lambda: stops == [True]) + + push(module, clients[0], _pub_frame(b'{"x": "1", "y": 2}', ch="clicked_point", seq=3)) + assert wait_until(lambda: len(clients[0].control_frames) == 3) + nack = clients[0].control_frames[2] + assert isinstance(nack, PubNack) and nack.code == "decode_failed" + assert len(points) == 1 diff --git a/dimos/web/test_cockpit.py b/dimos/web/test_cockpit.py index 583ff01a69..34c132520a 100644 --- a/dimos/web/test_cockpit.py +++ b/dimos/web/test_cockpit.py @@ -20,11 +20,13 @@ import subprocess import sys +from dimos_lcm.std_msgs import Bool from langchain_core.messages import BaseMessage import pytest from dimos.core.coordination.blueprint_config.parser import BlueprintConfigParser from dimos.core.coordination.blueprints import autoconnect +from dimos.msgs.geometry_msgs.PointStamped import PointStamped from dimos.msgs.geometry_msgs.PoseStamped import PoseStamped from dimos.msgs.geometry_msgs.Twist import Twist from dimos.msgs.nav_msgs.Path import Path @@ -44,7 +46,13 @@ ) from dimos.web.codecs import EncodedPayload, decode_json_v1, encode_json_v1, web_encoder from dimos.web.relay_bridge.audio_codec import AudioChunk, decode_audio_chunk -from dimos.web.relay_bridge.builtin_codecs import decode_text, encode_stats +from dimos.web.relay_bridge.builtin_codecs import ( + decode_bool, + decode_point, + decode_text, + encode_path, + encode_stats, +) from dimos.web.relay_bridge.chat_codec import encode_chat from dimos.web.relay_bridge.manifest import ManifestError, parse_manifest from dimos.web.relay_bridge.protocol import ( @@ -220,6 +228,7 @@ def test_pages_get_ids_after_the_grid() -> None: lambda: Video("color_image", quality=True), lambda: Map2D(costmap=""), lambda: Map2D(costmap_hz=-5.0), + lambda: Map2D(click=""), lambda: Teleop(stream=""), lambda: Teleop(max_linear=0), lambda: Teleop(boost=-2.0), @@ -240,6 +249,7 @@ def test_pages_get_ids_after_the_grid() -> None: "video_quality_bool", "map2d_empty_costmap", "map2d_negative_rate", + "map2d_empty_click", "teleop_empty_stream", "teleop_zero_linear", "teleop_negative_boost", @@ -318,6 +328,11 @@ def test_channel_publish_policy_rules() -> None: # Generic publish is reliable-only. with pytest.raises(ValueError, match="delivery='reliable'"): Channel("goal", dict, dir="tx", publish="shared", delivery="latest") + # Pacing and replay are bridge-side rx behaviours. + with pytest.raises(ValueError, match="paced applies to rx"): + Channel("goal", dict, dir="tx", publish="shared", paced=True) + with pytest.raises(ValueError, match="resend_on_subscribe applies to rx"): + Channel("goal", dict, dir="tx", publish="shared", resend_on_subscribe=True) # Scope uses the manifest id bound. with pytest.raises(ValueError, match="required_scope must be 1..64"): Channel("goal", dict, dir="tx", publish="shared", required_scope="") @@ -525,6 +540,10 @@ def test_chat_panel_blueprint() -> None: # by hand are sampled like any channel. (atom,) = cockpit(channels=[Channel("agent", BaseMessage, encoding="chat.json.v1")]).blueprints assert not atom.kwargs["channels"][0].paced + (atom,) = cockpit( + channels=[Channel("agent", BaseMessage, encoding="chat.json.v1", paced=True)] + ).blueprints + assert atom.kwargs["channels"][0].paced def test_chat_panel_declarations_merge_or_conflict() -> None: @@ -539,6 +558,50 @@ def test_chat_panel_declarations_merge_or_conflict() -> None: assert next(s for s in atom.kwargs["channels"] if s.ch == "agent").paced +def test_map2d_nav_channels_blueprint() -> None: + blueprint = cockpit(layout=Map2D(path="path", click="clicked_point", stop="stop_movement")) + (atom,) = blueprint.blueprints + manifest = atom.kwargs["manifest"] + assert [ + (c["ch"], c["dir"], c["encoding"], c["delivery"], c["maxHz"], c["publish"]) + for c in manifest["channels"] + ] == [ + ("odom", "rx", "pose.json.v1", "reliable", 20.0, "none"), + ("global_costmap", "rx", "costmap.zlib.v1", "latest", 5.0, "none"), + ("path", "rx", "path.json.v1", "latest", 10.0, "none"), + ("clicked_point", "tx", "point.json.v1", "reliable", 5.0, "shared"), + ("stop_movement", "tx", "bool.json.v1", "reliable", 5.0, "shared"), + ] + (panel,) = manifest["panels"] + # The map2d slots stay costmap + pose; the nav streams ride the params. + assert panel["channels"] == ["global_costmap", "odom"] + assert panel["params"] == {"path": "path", "click": "clicked_point", "stop": "stop_movement"} + assert parse_manifest(manifest).model_dump() == manifest + # Generated ports autoconnect to the planner's by name + type. + ports = {(s.name, s.direction): s.type for s in atom.streams} + assert ports[("path", "in")] is Path + assert ports[("clicked_point", "out")] is PointStamped + assert ports[("stop_movement", "out")] is Bool + specs = {s.ch: s for s in atom.kwargs["channels"]} + assert specs["path"].encoder is encode_path + assert specs["path"].paced and specs["path"].resend_on_subscribe + assert specs["clicked_point"].decoder is decode_point + assert specs["stop_movement"].decoder is decode_bool + restored = pickle.loads(pickle.dumps(blueprint)) + (ratom,) = restored.blueprints + assert {s.ch: s.decoder for s in ratom.kwargs["channels"]}["clicked_point"] is decode_point + + +def test_map2d_path_flags_survive_an_explicit_declaration() -> None: + (atom,) = cockpit( + layout=Map2D(path="path"), + channels=[Channel("path", Path, encoding="path.json.v1", delivery="latest", max_hz=20.0)], + ).blueprints + spec = next(s for s in atom.kwargs["channels"] if s.ch == "path") + assert spec.max_hz == 20.0 + assert spec.paced and spec.resend_on_subscribe + + def test_stats_panel_blueprint() -> None: blueprint = cockpit(layout=Video("color_image"), pages=[Stats()]) (atom,) = blueprint.blueprints diff --git a/web/README.md b/web/README.md index 80104e312d..b5fccc5fce 100644 --- a/web/README.md +++ b/web/README.md @@ -132,7 +132,11 @@ Panels are authored in Python (`dimos.web.cockpit`: `Video`, `Map2D`, `Teleop`, compiled into the manifest; `cockpit(pages=[...])` panels render as full-page tabs in the header, next to Overview and the panels/channels toggle. `Stats()` is dtop as a tab: the bridge re-encodes the resource monitor's `/resource_stats` dict as `stats.json.v1`, and the blueprint switches the -monitor on (`GlobalConfig.dtop`) by itself. +monitor on (`GlobalConfig.dtop`) by itself. `Map2D(path=, click=, stop=)` adds the planner's path +overlay, click-to-goal (a click publishes a `PointStamped` on `click`) and a cancel button (a `Bool` +on `stop`, shown while a path is active). `Channel(paced=True)` spaces sends instead of sampling and +`Channel(resend_on_subscribe=True)` replays the last message when the first viewer subscribes. +Additional viewers wait for the next publish on channels already being watched. Dev workflow: run the relay (`deno task dev` in `web/`, or just `dimos run --local-relay`) and the vite server side by side. `localhost:5173` is a secure context; vite proxies `/api` to the relay diff --git a/web/cockpit/src/panels/ChatPanel.test.tsx b/web/cockpit/src/panels/ChatPanel.test.tsx index e577038fd8..e1ab0f79ea 100644 --- a/web/cockpit/src/panels/ChatPanel.test.tsx +++ b/web/cockpit/src/panels/ChatPanel.test.tsx @@ -4,8 +4,9 @@ import { act } from "react"; import { createRoot, type Root } from "react-dom/client"; import type { FrameHeader, Msg, PanelSpec } from "@dimos/shared"; import type { Manifest } from "@dimos/shared/manifest"; -import { ChannelStore, PublishError, type Session, StatusStore } from "@dimos/sdk"; +import { PublishError, StatusStore } from "@dimos/sdk"; import type { TeleopHooks } from "@dimos/sdk/internal/teleop"; +import { FakeSession } from "../testing/fakeSession.ts"; import { ChatPanel } from "./ChatPanel.tsx"; import { TeleopPanel } from "./TeleopPanel.tsx"; @@ -70,22 +71,6 @@ function header(ch: string, seq: number): FrameHeader { return { ch, seq, ts: 1_700_000_000 + seq, delivery: ch === "agent" ? "reliable" : "latest" }; } -class FakeSession implements Session { - status = new StatusStore(); - store = new ChannelStore(); - published: [string, unknown][] = []; - reject: PublishError | null = null; - watch = () => new Promise(() => {}); - subscribe = () => () => {}; - publish = (ch: string, value: unknown) => { - this.published.push([ch, value]); - return this.reject === null - ? Promise.resolve({ ch, relayTs: 1, bridgeTs: 2 }) - : Promise.reject(this.reject); - }; - close = () => {}; -} - class FakeHooks implements TeleopHooks { controls: Msg[] = []; datagrams: Msg[] = []; diff --git a/web/cockpit/src/panels/MapPanel.module.css b/web/cockpit/src/panels/MapPanel.module.css index 86adcb6a01..62494dfb49 100644 --- a/web/cockpit/src/panels/MapPanel.module.css +++ b/web/cockpit/src/panels/MapPanel.module.css @@ -5,9 +5,41 @@ width: 100%; } +.clickable { + cursor: crosshair; +} + .waiting { color: var(--fg-muted); font-family: var(--font-mono); font-size: 12px; position: absolute; } + +.error { + color: var(--danger); + font-family: var(--font-mono); + font-size: 12px; + left: 8px; + position: absolute; + top: 8px; +} + +.cancel { + background: var(--bg-2); + border: 1px solid var(--danger); + border-radius: var(--radius); + bottom: 8px; + color: var(--danger); + cursor: pointer; + font-family: var(--font-ui); + font-size: 12px; + left: 8px; + padding: 4px 10px; + position: absolute; +} + +.cancel:hover { + background: var(--danger); + color: var(--bg-0); +} diff --git a/web/cockpit/src/panels/MapPanel.tsx b/web/cockpit/src/panels/MapPanel.tsx index 47cc6a1afe..61ff694d68 100644 --- a/web/cockpit/src/panels/MapPanel.tsx +++ b/web/cockpit/src/panels/MapPanel.tsx @@ -1,19 +1,20 @@ -// Live 2D costmap: canvas drawing driven by the store's direct-subscribe path -// (React is not involved at grid or pose rate; the badge rides the 500 ms UI -// tick). channels[0] is the costmap, channels[1] (optional) the pose overlay -// - both bindings come from the manifest, never hardcoded stream names. +// Canvas updates bypass React; badges and cancellation use the slower UI tick. -import { useEffect, useRef } from "react"; +import { useEffect, useRef, useState } from "react"; +import type { JsonValue, PanelSpec } from "@dimos/shared"; import { Badge, type DrawHealth, PanelFrame } from "../layout/PanelFrame.tsx"; -import { type ChannelStore, type CostmapValue, inflateCostmap } from "@dimos/sdk"; +import { type ChannelStore, type CostmapValue, inflateCostmap, type Session } from "@dimos/sdk"; import { useStoreChannel } from "@dimos/sdk/react"; import styles from "./MapPanel.module.css"; import { + canvasToWorld, + drawPath, drawPose, fitTransform, gridBlit, type GridPlacement, gridToImageData, + type PathPoint, type Pose2d, } from "./mapRenderer.ts"; import type { PanelProps } from "./registry.tsx"; @@ -31,6 +32,13 @@ export interface MapSinkDeps { observeResize?: (el: Element, cb: () => void) => () => void; } +export interface MapSinkOptions { + costmap: string; + pose?: string; + path?: string; + onClick?: (x: number, y: number) => void; +} + function isCostmapValue(v: unknown): v is CostmapValue { return typeof v === "object" && v !== null && (v as CostmapValue).bytes instanceof Uint8Array && @@ -44,21 +52,22 @@ function readPose(v: unknown): Pose2d | null { return { x, y, yaw }; } +function readPath(v: unknown): PathPoint[] | null { + if (!Array.isArray(v)) return null; + const ok = v.every((p) => + Array.isArray(p) && p.length === 2 && Number.isFinite(p[0]) && Number.isFinite(p[1]) + ); + return ok ? v as PathPoint[] : null; +} + /** - * Drive `canvas` from the costmap channel's slot: at most one inflate in - * flight, and on completion the pump re-checks the slot, so a burst of grids - * costs one inflate of the newest (latest-wins, same shedding rule as - * everywhere else in the pipeline). The inflated grid lands on an offscreen - * bitmap at cell resolution; the display canvas redraws (scaled blit + pose - * triangle) on new grids, pose ingests, resizes, and visibility changes, - * which is how two consumers share the odom channel without extra - * subscriptions upstream. While the document is hidden nothing inflates or - * draws. Returns the cleanup function. + * Inflate one grid at a time, skipping to the newest after each completion. + * Cache the bitmap so overlays and resizes don't repeat decompression. + * Hidden documents pause both inflation and drawing. */ export function startMapSink( store: ChannelStore, - costmapCh: string, - poseCh: string | undefined, + { costmap: costmapCh, pose: poseCh, path: pathCh, onClick }: MapSinkOptions, canvas: HTMLCanvasElement, health: DrawHealth, deps: MapSinkDeps = {}, @@ -105,6 +114,8 @@ export function startMapSink( ctx.rotate(rot); ctx.drawImage(grid, 0, -dh, dw, dh); ctx.restore(); + const path = pathCh === undefined ? null : readPath(store.get(pathCh)?.value); + if (path !== null && path.length > 1) drawPath(ctx, t, path, dpr); const pose = poseCh === undefined ? null : readPose(store.get(poseCh)?.value); if (pose !== null) drawPose(ctx, t, pose, dpr); }; @@ -145,9 +156,20 @@ export function startMapSink( }); }; + // Mouse coordinates are CSS pixels; the fitted map uses backing-store pixels. + const onCanvasClick = (e: MouseEvent): void => { + const rect = canvas.getBoundingClientRect(); + if (place === null || rect.width === 0 || rect.height === 0) return; + const t = fitTransform(place, canvas.width, canvas.height); + const px = (e.clientX - rect.left) * canvas.width / rect.width; + const py = (e.clientY - rect.top) * canvas.height / rect.height; + onClick?.(...canvasToWorld(t, px, py)); + }; + if (onClick !== undefined) canvas.addEventListener("click", onCanvasClick); + const unsubscribeGrid = store.subscribe(costmapCh, pump); - // Pose redraws reuse the cached grid bitmap: no inflate at odom rate. const unsubscribePose = poseCh === undefined ? null : store.subscribe(poseCh, draw); + const unsubscribePath = pathCh === undefined ? null : store.subscribe(pathCh, draw); const disposeResize = observeResize(canvas, draw); const onVisibility = (): void => { pump(); @@ -157,14 +179,33 @@ export function startMapSink( pump(); // a slot may predate the mount return () => { stopped = true; + canvas.removeEventListener("click", onCanvasClick); unsubscribeGrid(); unsubscribePose?.(); + unsubscribePath?.(); disposeResize(); document.removeEventListener("visibilitychange", onVisibility); }; } -export function MapPanel({ spec, store }: PanelProps) { +function param(spec: PanelSpec, key: string): string | undefined { + const value = spec.params[key]; + return typeof value === "string" ? value : undefined; +} + +function send( + session: Session, + ch: string, + value: JsonValue, + onError: (message: string | null) => void, +): void { + session.publish(ch, value).then( + () => onError(null), + (err: unknown) => onError(`send failed: ${err instanceof Error ? err.message : String(err)}`), + ); +} + +export function MapPanel({ spec, store, session }: PanelProps) { const costmapCh = spec.channels[0] as string | undefined; if (costmapCh === undefined) { // A map panel without a costmap channel is a bridge authoring mistake; @@ -179,27 +220,40 @@ export function MapPanel({ spec, store }: PanelProps) { ); } function MapCanvas( - { spec, store, costmapCh, poseCh }: PanelProps & { + { spec, store, session, costmapCh, poseCh, pathCh, clickCh, stopCh }: PanelProps & { costmapCh: string; poseCh: string | undefined; + pathCh: string | undefined; + clickCh: string | undefined; + stopCh: string | undefined; }, ) { const canvasRef = useRef(null); const health = useRef({ lastDrawOkAtMs: Date.now(), failures: 0 }).current; const { slot } = useStoreChannel(store, costmapCh); + const [error, setError] = useState(null); + const clickable = session !== undefined && clickCh !== undefined; useEffect(() => { const canvas = canvasRef.current; if (canvas === null) return; - return startMapSink(store, costmapCh, poseCh, canvas, health); - }, [store, costmapCh, poseCh, health]); + const onClick = session !== undefined && clickCh !== undefined + ? (x: number, y: number) => send(session, clickCh, { x, y }, setError) + : undefined; + const opts = { costmap: costmapCh, pose: poseCh, path: pathCh, onClick }; + return startMapSink(store, opts, canvas, health); + }, [store, session, costmapCh, poseCh, pathCh, clickCh, health]); return ( {slot === null && waiting for data...} + {error !== null && ( + + {error} + + )} + {session !== undefined && pathCh !== undefined && stopCh !== undefined && ( + + )} ); } + +function CancelButton({ store, session, pathCh, stopCh, testId, onError }: { + store: ChannelStore; + session: Session; + pathCh: string; + stopCh: string; + testId: string; + onError: (message: string | null) => void; +}) { + const path = readPath(useStoreChannel(store, pathCh).slot?.value); + if (path === null || path.length === 0) return null; + return ( + + ); +} diff --git a/web/cockpit/src/panels/mapRenderer.test.ts b/web/cockpit/src/panels/mapRenderer.test.ts index 923f5a3866..92dd68ef3e 100644 --- a/web/cockpit/src/panels/mapRenderer.test.ts +++ b/web/cockpit/src/panels/mapRenderer.test.ts @@ -1,10 +1,12 @@ -import { describe, expect, it } from "vitest"; +import { describe, expect, it, vi } from "vitest"; import { canvasToWorld, + drawPath, fitTransform, gridBlit, gridToImageData, OCCUPANCY_PALETTE, + PATH_COLOR, posePath, worldToCanvas, } from "./mapRenderer.ts"; @@ -192,3 +194,24 @@ describe("posePath", () => { expect(right[1] - cy).toBeCloseTo(-8.4, 9); }); }); + +describe("drawPath", () => { + it("strokes one polyline through the world points, dpr-scaled", () => { + const t = fitTransform({ w: 100, h: 100, res: 1.0, origin: [0.0, 0.0, 0.0] }, 100, 100); + const ctx = { + beginPath: vi.fn(), + moveTo: vi.fn(), + lineTo: vi.fn(), + stroke: vi.fn(), + strokeStyle: "", + lineWidth: 0, + }; + const points: [number, number][] = [[10, 20], [30, 40], [50, 50]]; + drawPath(ctx as unknown as CanvasRenderingContext2D, t, points, 2); + expect(ctx.moveTo.mock.calls).toEqual([[10, 80]]); + expect(ctx.lineTo.mock.calls).toEqual([[30, 60], [50, 50]]); + expect(ctx.strokeStyle).toBe(PATH_COLOR); + expect(ctx.lineWidth).toBe(4); + expect(ctx.stroke).toHaveBeenCalledTimes(1); + }); +}); diff --git a/web/cockpit/src/panels/mapRenderer.ts b/web/cockpit/src/panels/mapRenderer.ts index 6afde21d6b..21ccb66289 100644 --- a/web/cockpit/src/panels/mapRenderer.ts +++ b/web/cockpit/src/panels/mapRenderer.ts @@ -132,6 +132,27 @@ export function gridBlit( }; } +export type PathPoint = [number, number]; +export const PATH_COLOR = "#3fb950"; +const PATH_PX = 2; + +export function drawPath( + ctx: CanvasRenderingContext2D, + t: MapTransform, + points: PathPoint[], + dpr = 1, +): void { + ctx.beginPath(); + points.forEach(([wx, wy], i) => { + const [cx, cy] = worldToCanvas(t, wx, wy); + if (i === 0) ctx.moveTo(cx, cy); + else ctx.lineTo(cx, cy); + }); + ctx.strokeStyle = PATH_COLOR; + ctx.lineWidth = PATH_PX * dpr; + ctx.stroke(); +} + export const POSE_COLOR = "#ff5c5c"; // Triangle length in CSS pixels: screen-constant so the marker stays legible // however far the fit zooms out. The canvas backing store is DPR-scaled, so diff --git a/web/cockpit/src/panels/panels.test.tsx b/web/cockpit/src/panels/panels.test.tsx index e77f65a451..e426e203a3 100644 --- a/web/cockpit/src/panels/panels.test.tsx +++ b/web/cockpit/src/panels/panels.test.tsx @@ -4,8 +4,9 @@ import { act } from "react"; import { createRoot, type Root } from "react-dom/client"; import type { FrameHeader, PanelSpec } from "@dimos/shared"; import type { CostmapValue } from "@dimos/sdk"; -import { ChannelStore } from "@dimos/sdk"; +import { ChannelStore, PublishError } from "@dimos/sdk"; import type { DrawHealth } from "../layout/PanelFrame.tsx"; +import { FakeSession } from "../testing/fakeSession.ts"; import { MapPanel, startMapSink } from "./MapPanel.tsx"; import { fitTransform, posePath } from "./mapRenderer.ts"; import { ChatPanel } from "./ChatPanel.tsx"; @@ -371,6 +372,8 @@ describe("registry", () => { const MAP_CH = "global_costmap"; const POSE_CH = "odom"; +const PATH_CH = "path"; +const CHS = { costmap: MAP_CH, pose: POSE_CH }; function costmapValue(seq: number, w = 2, h = 2): CostmapValue { return { bytes: new Uint8Array([seq]), w, h, res: 0.5, origin: [0.25, -0.5, 0.0] }; @@ -387,6 +390,22 @@ function poseFrame(store: ChannelStore, seq: number): void { store.ingest(POSE_CH, { ch: POSE_CH, seq, ts: seq, delivery: "reliable" }, value, true); } +function pathFrame(store: ChannelStore, seq: number, points: [number, number][]): void { + store.ingest(PATH_CH, { ch: PATH_CH, seq, ts: seq, delivery: "latest" }, points, true); +} + +/** happy-dom has no layout; pin the on-screen rect the click handler reads. */ +function defineRect(canvas: HTMLCanvasElement, left: number, top: number, w: number, h: number) { + Object.defineProperty(canvas, "getBoundingClientRect", { + configurable: true, + value: () => ({ left, top, width: w, height: h }), + }); +} + +function click(el: Element, clientX: number, clientY: number): void { + el.dispatchEvent(new MouseEvent("click", { clientX, clientY, bubbles: true })); +} + /** Inflate stub whose promises settle only when the test says so. */ function deferredInflate() { const calls: CostmapValue[] = []; @@ -418,6 +437,7 @@ describe("startMapSink", () => { lineTo: ReturnType; closePath: ReturnType; fill: ReturnType; + stroke: ReturnType; } let store: ChannelStore; let canvas: HTMLCanvasElement; @@ -447,6 +467,7 @@ describe("startMapSink", () => { lineTo: vi.fn(), closePath: vi.fn(), fill: vi.fn(), + stroke: vi.fn(), }; contexts.push(fake); return fake as unknown as CanvasRenderingContext2D; @@ -463,7 +484,7 @@ describe("startMapSink", () => { it("inflates one grid at a time and skips straight to the newest", async () => { const { inflate, calls, settlers } = deferredInflate(); - stop = startMapSink(store, MAP_CH, POSE_CH, canvas, health, { inflate, hidden: () => false }); + stop = startMapSink(store, CHS, canvas, health, { inflate, hidden: () => false }); const first = gridFrame(store, 1); expect(calls).toEqual([first]); @@ -488,7 +509,7 @@ describe("startMapSink", () => { it("redraws the pose from the cached bitmap without a new inflate", async () => { const { inflate, calls, settlers } = deferredInflate(); - stop = startMapSink(store, MAP_CH, POSE_CH, canvas, health, { inflate, hidden: () => false }); + stop = startMapSink(store, CHS, canvas, health, { inflate, hidden: () => false }); gridFrame(store, 1); settlers[0].resolve(new Uint8Array(4)); await flush(); @@ -503,14 +524,14 @@ describe("startMapSink", () => { it("ignores pose frames until a grid has drawn", () => { const { inflate } = deferredInflate(); - stop = startMapSink(store, MAP_CH, POSE_CH, canvas, health, { inflate, hidden: () => false }); + stop = startMapSink(store, CHS, canvas, health, { inflate, hidden: () => false }); poseFrame(store, 1); expect(display().drawImage).not.toHaveBeenCalled(); }); it("counts inflate rejections and recovers on the next grid", async () => { const { inflate, settlers } = deferredInflate(); - stop = startMapSink(store, MAP_CH, POSE_CH, canvas, health, { inflate, hidden: () => false }); + stop = startMapSink(store, CHS, canvas, health, { inflate, hidden: () => false }); const stamp = health.lastDrawOkAtMs; gridFrame(store, 1); settlers[0].reject(new Error("corrupt zlib")); @@ -526,7 +547,7 @@ describe("startMapSink", () => { it("skips a slot that is not a costmap value without spinning", () => { const { inflate, calls } = deferredInflate(); - stop = startMapSink(store, MAP_CH, POSE_CH, canvas, health, { inflate, hidden: () => false }); + stop = startMapSink(store, CHS, canvas, health, { inflate, hidden: () => false }); store.ingest( MAP_CH, { ch: MAP_CH, seq: 1, ts: 1, delivery: "latest" }, @@ -539,7 +560,7 @@ describe("startMapSink", () => { it("does not inflate while hidden and catches up on visibilitychange", async () => { const { inflate, calls, settlers } = deferredInflate(); let hidden = true; - stop = startMapSink(store, MAP_CH, POSE_CH, canvas, health, { inflate, hidden: () => hidden }); + stop = startMapSink(store, CHS, canvas, health, { inflate, hidden: () => hidden }); gridFrame(store, 1); const newest = gridFrame(store, 2); expect(calls.length).toBe(0); // a backgrounded panel costs no inflate @@ -556,7 +577,7 @@ describe("startMapSink", () => { const { inflate, calls, settlers } = deferredInflate(); let resize: (() => void) | null = null; const dispose = vi.fn(); - stop = startMapSink(store, MAP_CH, POSE_CH, canvas, health, { + stop = startMapSink(store, CHS, canvas, health, { inflate, hidden: () => false, observeResize: (_el, cb) => { @@ -582,7 +603,7 @@ describe("startMapSink", () => { it("stops inflating and drawing after cleanup", async () => { const { inflate, calls, settlers } = deferredInflate(); - stop = startMapSink(store, MAP_CH, POSE_CH, canvas, health, { inflate, hidden: () => false }); + stop = startMapSink(store, CHS, canvas, health, { inflate, hidden: () => false }); gridFrame(store, 1); stop(); stop = null; @@ -599,7 +620,7 @@ describe("startMapSink", () => { it("rotates the grid blit by -yaw and restores before the pose", async () => { const { inflate, settlers } = deferredInflate(); - stop = startMapSink(store, MAP_CH, POSE_CH, canvas, health, { inflate, hidden: () => false }); + stop = startMapSink(store, CHS, canvas, health, { inflate, hidden: () => false }); const value = { ...costmapValue(1), origin: [0.25, -0.5, 0.25] as [number, number, number] }; store.ingest(MAP_CH, { ch: MAP_CH, seq: 1, ts: 1, delivery: "latest" }, value, true); settlers[0].resolve(new Uint8Array(4)); @@ -621,7 +642,7 @@ describe("startMapSink", () => { it("reuses the ImageData buffer across same-size grids", async () => { const { inflate, settlers } = deferredInflate(); - stop = startMapSink(store, MAP_CH, POSE_CH, canvas, health, { inflate, hidden: () => false }); + stop = startMapSink(store, CHS, canvas, health, { inflate, hidden: () => false }); gridFrame(store, 1); settlers[0].resolve(new Uint8Array(4)); await flush(); @@ -641,7 +662,7 @@ describe("startMapSink", () => { it("sizes the backing store and pose marker by devicePixelRatio", async () => { vi.stubGlobal("devicePixelRatio", 2); const { inflate, settlers } = deferredInflate(); - stop = startMapSink(store, MAP_CH, POSE_CH, canvas, health, { inflate, hidden: () => false }); + stop = startMapSink(store, CHS, canvas, health, { inflate, hidden: () => false }); gridFrame(store, 1); settlers[0].resolve(new Uint8Array(4)); await flush(); @@ -654,6 +675,63 @@ describe("startMapSink", () => { expect(nx).toBeCloseTo(ex, 9); // the sink passed its dpr to the marker expect(ny).toBeCloseTo(ey, 9); }); + + it("draws the path under the pose and clears it on an empty path", async () => { + const { inflate, calls, settlers } = deferredInflate(); + const opts = { ...CHS, path: PATH_CH }; + stop = startMapSink(store, opts, canvas, health, { inflate, hidden: () => false }); + gridFrame(store, 1); + settlers[0].resolve(new Uint8Array(4)); + await flush(); + poseFrame(store, 1); + expect(display().stroke).not.toHaveBeenCalled(); + + // The 1x1 m grid fits the 100x80 canvas at 80 px/m, letterboxed to + // x 10..90 with world y -0.5 on canvas row 80. + pathFrame(store, 1, [[0.5, -0.25], [0.75, 0], [1.0, 0.25]]); + expect(display().stroke).toHaveBeenCalledTimes(1); + expect(display().moveTo.mock.calls).toContainEqual([30, 60]); + expect(display().lineTo.mock.calls).toContainEqual([50, 40]); + expect(display().lineTo.mock.calls).toContainEqual([70, 20]); + // Under the pose: the stroke precedes this draw's triangle fill. + const fills = display().fill.mock.invocationCallOrder; + expect(display().stroke.mock.invocationCallOrder[0]).toBeLessThan(fills[fills.length - 1]); + expect(calls.length).toBe(1); // no re-inflate for an overlay + + pathFrame(store, 2, []); // cancel/arrival: the overlay goes, the pose stays + expect(display().stroke).toHaveBeenCalledTimes(1); + expect(display().fill).toHaveBeenCalledTimes(3); + }); + + it("maps a click through the fitted transform at the device pixel ratio", async () => { + vi.stubGlobal("devicePixelRatio", 2); + const { inflate, settlers } = deferredInflate(); + const clicks: [number, number][] = []; + const opts = { ...CHS, onClick: (x: number, y: number) => clicks.push([x, y]) }; + stop = startMapSink(store, opts, canvas, health, { inflate, hidden: () => false }); + defineRect(canvas, 10, 20, 100, 80); + click(canvas, 60, 60); + expect(clicks).toEqual([]); // no grid yet: no world frame + + gridFrame(store, 1); + settlers[0].resolve(new Uint8Array(4)); + await flush(); + // The canvas centre is the centre of the fitted grid: world (0.75, 0). + click(canvas, 60, 60); + // CSS (10, 60) is backing-store (20, 120): the AABB's left edge, a + // quarter of the way up. + click(canvas, 20, 80); + expect(clicks.length).toBe(2); + expect(clicks[0][0]).toBeCloseTo(0.75, 9); + expect(clicks[0][1]).toBeCloseTo(0, 9); + expect(clicks[1][0]).toBeCloseTo(0.25, 9); + expect(clicks[1][1]).toBeCloseTo(-0.25, 9); + + stop!(); + stop = null; + click(canvas, 60, 60); + expect(clicks.length).toBe(2); // the listener left with the sink + }); }); describe("MapPanel", () => { @@ -791,4 +869,61 @@ describe("MapPanel", () => { expect(container.textContent).toContain("no channel bound"); expect(container.querySelector("canvas")).toBeNull(); }); + + const NAV_SPEC: PanelSpec = { + ...SPEC, + params: { path: PATH_CH, click: "clicked_point", stop: "stop_movement" }, + }; + const cancel = () => container.querySelector(`[data-testid="map2d-${MAP_CH}-cancel"]`); + + it("publishes a click as {x, y} on the click channel and reports a rejection", async () => { + const session = new FakeSession(); + act(() => root.render()); + const canvas = container.querySelector("canvas")!; + defineRect(canvas, 0, 0, 100, 80); + await act(async () => { + realGridFrame(1, now / 1000); + await flush(); + }); + act(() => click(canvas, 50, 40)); // the centre of the fitted grid + expect(session.published).toEqual([["clicked_point", { x: 0.75, y: 0 }]]); + await act(async () => {}); + expect(container.querySelector('[role="alert"]')).toBeNull(); + + session.reject = new PublishError("rejected", "not_connected", "no session"); + act(() => click(canvas, 50, 40)); + await act(async () => {}); + expect(container.querySelector('[role="alert"]')!.textContent).toBe( + "send failed: not_connected: no session", + ); + }); + + it("shows the cancel button only while a path is active and publishes the stop", () => { + const session = new FakeSession(); + act(() => root.render()); + expect(cancel()).toBeNull(); + act(() => { + pathFrame(store, 1, [[0.5, 0], [1.0, 0]]); + store.publishUi(); + }); + expect(cancel()).not.toBeNull(); + act(() => (cancel() as HTMLButtonElement).click()); + expect(session.published).toEqual([["stop_movement", true]]); + + act(() => { + pathFrame(store, 2, []); // the planner cleared it: nothing left to cancel + store.publishUi(); + }); + expect(cancel()).toBeNull(); + }); + + it("renders neither the cancel button nor the crosshair without a session", () => { + act(() => root.render()); + act(() => { + pathFrame(store, 1, [[0.5, 0], [1.0, 0]]); + store.publishUi(); + }); + expect(cancel()).toBeNull(); + expect(container.querySelector("canvas")!.className).not.toContain("clickable"); + }); }); diff --git a/web/cockpit/src/testing/fakeSession.ts b/web/cockpit/src/testing/fakeSession.ts new file mode 100644 index 0000000000..87292ba35a --- /dev/null +++ b/web/cockpit/src/testing/fakeSession.ts @@ -0,0 +1,17 @@ +import { ChannelStore, type PublishError, type Session, StatusStore } from "@dimos/sdk"; + +export class FakeSession implements Session { + status = new StatusStore(); + store = new ChannelStore(); + published: [string, unknown][] = []; + reject: PublishError | null = null; + watch = () => new Promise(() => {}); + subscribe = () => () => {}; + publish = (ch: string, value: unknown) => { + this.published.push([ch, value]); + return this.reject === null + ? Promise.resolve({ ch, relayTs: 1, bridgeTs: 2 }) + : Promise.reject(this.reject); + }; + close = () => {}; +}