From b9e68105329f9de2033e3d50f35016e4d00ae770 Mon Sep 17 00:00:00 2001 From: abrichr Date: Mon, 31 Aug 2026 18:47:16 -0400 Subject: [PATCH 1/3] feat(agent): --authoring stdio tools observe/start_record/click/halt Register the hosted probe names on local stdio. --bundles is optional iff --authoring; the published run recipe still requires --bundles. --authoring does not imply --allow-run. HTTP shim remains forbidden. Local stdio may type through Recorder; pause Continue uses record_observed. Tests use a fake session until openadapt_flow.authoring is importable. --- docs/DESIGN.md | 68 +++- src/openadapt_agent/authoring.py | 677 +++++++++++++++++++++++++++++++ src/openadapt_agent/cli.py | 141 +++++-- src/openadapt_agent/mcp.py | 101 +++-- tests/test_authoring.py | 365 +++++++++++++++++ tests/test_cli.py | 76 +++- tests/test_distribution.py | 3 + 7 files changed, 1338 insertions(+), 93 deletions(-) create mode 100644 src/openadapt_agent/authoring.py create mode 100644 tests/test_authoring.py diff --git a/docs/DESIGN.md b/docs/DESIGN.md index cbdded5..036deae 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -40,21 +40,24 @@ MCP client / Agent Skill │ │ local stdio + exact JSON schemas ▼ -openadapt_agent.mcp +openadapt_agent.mcp (local stdio only; HTTP shim forbidden) │ - ▼ -openadapt_agent.bridge - ├── bundle discovery and typed run tools - ├── PHI-safe Needs Attention projection - ├── action-specific operator decisions - └── structured success / halt / refusal results - │ - ├── new run ──────────────► openadapt-flow run subprocess - │ fail-closed admission + execution + ├── openadapt_agent.bridge + │ ├── bundle discovery and typed run tools + │ ├── PHI-safe Needs Attention projection + │ ├── action-specific operator decisions + │ └── structured success / halt / refusal results + │ │ + │ ├── new run ──► openadapt-flow run subprocess + │ └── attended ─► openadapt-flow durable API │ - └── attended decision ────► openadapt-flow durable API - signed capability + idempotency - + live revalidation + audit + └── openadapt_agent.authoring (--authoring first demo) + observe / start_record / click / halt + local type (agent-driven Recorder.type_text) + pause Continue → record_observed (never type_text) + │ + ▼ + openadapt_flow.authoring.AuthoringSession ``` The MCP adapter is intentionally thin. Tool descriptions and dispatch @@ -120,6 +123,30 @@ operation: Skip are registered only when a deployment configuration lets Flow construct its bound live executor. +`--authoring` registers first-demo tools over the same local stdio +server. Probe names match hosted MCP: `observe`, `start_record`, +`click`, `halt`. Local stdio may also include `type` for agent-driven +typing through Flow's Recorder. Hosted MCP remains pause-only. Human +type during `pause_for_input` is persisted with `Recorder.record_observed` +on the pause-target node, never `type_text`. `compile` wraps Flow +`compile_recording` and returns `needs_human_admit`; an agent click never +paints `VERIFIED`. + +`--authoring` does not imply `--allow-run`. `--bundles` is optional iff +`--authoring` (or the existing `--tutorial` / implied-tutorial path). The +published run recipe in `server.json` still requires `--bundles` and +stays `transport: stdio`. Authoring is a first demo; there is no bundle +yet. + +Observe is a fail-closed PHI projection (`openadapt.authoring.observe/v1`): +no `value`, `text`, window `title`, screenshot, OCR, URL, or backend +pixels. Windows native, Citrix, and RDP are `COACH_ONLY` in v1. + +The session object is Flow's public `openadapt_flow.authoring` module. +Until that module is importable, `serve --authoring` fails closed with +an explicit dependency error. Tests cover the tool surface with a fake +session. + ## Governed runs Each run tool shells out to the `openadapt-flow` installed in the same @@ -273,9 +300,13 @@ caller-controlled `USERNAME` environment variable. A blank operator identity fails closed. This process must not be port-forwarded or exposed as an unauthenticated -network service. OpenAdapt Cloud owns remote authentication, +network service. An HTTP / Streamable-HTTP shim in this MIT package +remains forbidden, including when `--authoring` is set. Remote authoring +for ChatGPT.com / Claude.ai is a website mailbox, not a listener inside +`openadapt-agent`. OpenAdapt Cloud owns remote authentication, multi-tenancy, tenant-scoped authorization, fleet policy, and managed -transport. +transport. `--authoring` does not add those, and it does not imply +`--allow-run`. ## Dependency boundary @@ -309,7 +340,12 @@ Tests cover: - compatibility with Flow's public, thread-owned attended service; - success/halt/refusal/timeout outcome mapping; - MCP serialization and thread ownership; -- Agent Skill emission. +- Agent Skill emission; +- `--authoring` probe tools (`observe`, `start_record`, `click`, `halt`) + and local `type`; observe projection drops values/titles/screenshots; + pause Continue uses `record_observed` rather than `type_text`; + compile returns `needs_human_admit`; `--authoring` does not enable + run tools; `server.json` stays stdio with `--bundles` required. CI runs on Python 3.10, 3.11, and 3.12. It also builds the wheel and sdist, verifies MIT metadata and license inclusion, and refuses package diff --git a/src/openadapt_agent/authoring.py b/src/openadapt_agent/authoring.py new file mode 100644 index 0000000..6c547ba --- /dev/null +++ b/src/openadapt_agent/authoring.py @@ -0,0 +1,677 @@ +"""Local stdio authoring tools: first demo, not governed run. + +``openadapt-agent serve --authoring`` registers the same probe names hosted +MCP will use: ``observe``, ``start_record``, ``click``, ``halt``. Local +stdio may also include ``type`` for agent-driven typing through Flow's +Recorder. Hosted remains pause-only. Human type during a pause is +``record_observed`` everywhere; never ``type_text`` on the pause target. + +This module is a transport-independent bridge. It does not open a network +listener and does not implement a remote mailbox. Window titles, field +values, screenshots, and backend pixels never cross the MCP wire. +``--authoring`` does not imply ``--allow-run``. +""" + +from __future__ import annotations + +import re +from typing import Any, Mapping, Optional + +from openadapt_agent.bridge import BridgeError, ToolSpec + +__all__ = [ + "AUTHORING_LOCAL_TOOLS", + "AUTHORING_PROBE_TOOLS", + "AuthoringBridge", + "AuthoringError", + "OBSERVE_SCHEMA_VERSION", + "open_authoring_session", + "project_observe", +] + +OBSERVE_SCHEMA_VERSION = "openadapt.authoring.observe/v1" +try: + from openadapt_types.authoring import OBSERVE_SCHEMA_VERSION as _TYPES_OBSERVE +except ImportError: + pass +else: + if isinstance(_TYPES_OBSERVE, str) and _TYPES_OBSERVE: + OBSERVE_SCHEMA_VERSION = _TYPES_OBSERVE + +AUTHORING_PROBE_TOOLS = ("observe", "start_record", "click", "halt") +AUTHORING_LOCAL_TOOLS = ( + "type", + "stop_record", + "pause_for_input", + "compile", + "get_command_result", +) +AUTHORING_TOOLS = AUTHORING_PROBE_TOOLS + AUTHORING_LOCAL_TOOLS + +_EMPTY_OBJECT = { + "type": "object", + "properties": {}, + "additionalProperties": False, +} +_READ_ONLY = { + "readOnlyHint": True, + "destructiveHint": False, + "idempotentHint": True, + "openWorldHint": False, +} +_MUTATING = { + "readOnlyHint": False, + "destructiveHint": True, + "idempotentHint": False, + "openWorldHint": True, +} +_LOCAL_HALT = { + "readOnlyHint": False, + "destructiveHint": True, + "idempotentHint": True, + "openWorldHint": False, +} + +_FORBIDDEN_WIRE_KEYS = frozenset( + { + "value", + "text", + "title", + "window_title", + "screenshot", + "ocr", + "url", + "urls", + "backend_pixels", + "raw", + "path", + "file_path", + "pixels", + } +) +_BOUNDS_KEYS = frozenset({"x", "y", "w", "h"}) +_PROCESS_NAME = re.compile(r"^[A-Za-z0-9 ._-]{1,64}$") +_SIX_DIGITS = re.compile(r"\d{6,}") +_EMAIL = re.compile(r"[^@\s]+@[^@\s]+\.[^@\s]+") +_SSN = re.compile(r"\b\d{3}-\d{2}-\d{4}\b") +_PHONE = re.compile(r"\b(?:\+?\d[\d\-\s().]{7,}\d)\b") +_RESULT_DROP = _FORBIDDEN_WIRE_KEYS | frozenset( + {"execution_outcome", "success", "events", "frames", "before_png", "after_png"} +) +_CLOSED_BACKENDS = frozenset( + {"web", "macos", "linux", "windows", "rdp", "citrix", "unknown"} +) +_COACH_ONLY_BACKENDS = frozenset({"windows", "rdp", "citrix"}) + +_CLICK_FIELDS = frozenset({"node_id", "x", "y"}) +_TYPE_FIELDS = frozenset({"text", "param", "node_id"}) +_PAUSE_FIELDS = frozenset({"node_id", "param", "secret"}) +_RESULT_FIELDS = frozenset({"command_id"}) + +_PROBE_HELP = { + "observe": ( + "Return a PHI-safe authoring observation of the pinned local window " + "(openadapt.authoring.observe/v1). No screenshots, OCR, field values, " + "window titles, URLs, or backend pixels. Use node_id values from this " + "tree for click." + ), + "start_record": ( + "Start a Flow Recorder session over the locally pinned backend. " + "Refuses Windows native, Citrix, and RDP (coach-only). Does not " + "compile and does not enable run tools." + ), + "click": ( + "Click one observed node through Flow Recorder. Prefer node_id from " + "the last observe. Local stdio also accepts integer x,y pixels. " + "Unknown or stale node_id returns error stale_node." + ), + "halt": ( + "Stop the authoring session without compiling. Recording evidence " + "stays local. This is not a governed run halt." + ), +} + + +class AuthoringError(BridgeError): + """Authoring tool refusal or missing Flow session.""" + + +def open_authoring_session(**kwargs: Any) -> object: + """Construct Flow's public authoring session when that module exists.""" + try: + from openadapt_flow import authoring as flow_authoring + except ImportError as exc: + raise AuthoringError( + "openadapt_flow.authoring is not available in this environment; " + "stdio --authoring depends on the Flow authoring session that wraps " + "Recorder (compile returns needs_human_admit; Continue uses " + "record_observed, never type_text on the pause target)" + ) from exc + opener = getattr(flow_authoring, "open_session", None) + if callable(opener): + return opener(**kwargs) + session_cls = getattr(flow_authoring, "AuthoringSession", None) + if callable(session_cls): + return session_cls(**kwargs) + raise AuthoringError( + "openadapt_flow.authoring is importable but exposes neither " + "open_session nor AuthoringSession" + ) + + +def _safe_label(value: Any, *, process_name: bool = False) -> Optional[str]: + if not isinstance(value, str): + return None + collapsed = " ".join(value.split()) + if not collapsed or len(collapsed) > 80: + return None + if process_name and not _PROCESS_NAME.fullmatch(collapsed): + return None + if "://" in collapsed or "@" in collapsed or _SIX_DIGITS.search(collapsed): + return None + if _EMAIL.search(collapsed) or _SSN.search(collapsed) or _PHONE.search(collapsed): + return None + return collapsed + + +def _bounds(value: Any) -> Optional[dict[str, float]]: + if not isinstance(value, Mapping): + return None + out: dict[str, float] = {} + for key in ("x", "y", "w", "h"): + raw = value.get(key) + if isinstance(raw, bool) or not isinstance(raw, (int, float)): + return None + out[key] = float(raw) + extra = set(value) - _BOUNDS_KEYS + if extra: + return out + return out + + +def _project_window(value: Any) -> dict[str, Any]: + if not isinstance(value, Mapping): + return {"role": "window"} + window: dict[str, Any] = {} + process_name = _safe_label(value.get("process_name"), process_name=True) + if process_name: + window["process_name"] = process_name + role = value.get("role") + window["role"] = role if isinstance(role, str) and role else "window" + bounds = _bounds(value.get("bounds")) + if bounds is not None: + window["bounds"] = bounds + return window + + +def _project_node(value: Any) -> Optional[dict[str, Any]]: + if not isinstance(value, Mapping): + return None + node_id = value.get("node_id") + if not isinstance(node_id, str) or not node_id: + return None + node: dict[str, Any] = {"node_id": node_id} + role = value.get("role") + if isinstance(role, str) and role: + node["role"] = role + control_type = value.get("control_type") + if isinstance(control_type, str) and control_type: + node["control_type"] = control_type + class_name = _safe_label(value.get("class_name")) + if class_name: + node["class_name"] = class_name[:64] + automation_id = _safe_label(value.get("automation_id")) + if automation_id: + node["automation_id"] = automation_id + name = _safe_label(value.get("name")) + if name: + node["name"] = name + if isinstance(value.get("enabled"), bool): + node["enabled"] = value["enabled"] + if isinstance(value.get("focused"), bool): + node["focused"] = value["focused"] + bounds = _bounds(value.get("bounds")) + if bounds is not None: + node["bounds"] = bounds + return node + + +def project_observe(payload: Any) -> dict[str, Any]: + """Fail-closed PHI projection for ``openadapt.authoring.observe/v1``.""" + source = payload if isinstance(payload, Mapping) else {} + backend = source.get("backend") + if backend not in _CLOSED_BACKENDS: + backend = "unknown" + coach_only = backend in _COACH_ONLY_BACKENDS or source.get("coach_only") is True + agent_drive = (not coach_only) and source.get("agent_drive") is not False + if coach_only: + agent_drive = False + tree_in = source.get("tree") + nodes: list[dict[str, Any]] = [] + if isinstance(tree_in, list) and not coach_only: + for item in tree_in: + node = _project_node(item) + if node is not None: + nodes.append(node) + if len(nodes) >= 200: + break + projected: dict[str, Any] = { + "schema_version": OBSERVE_SCHEMA_VERSION, + "backend": backend, + "provider": ( + source.get("provider") if isinstance(source.get("provider"), str) else "unknown" + ), + "mode": "authoring", + "agent_drive": agent_drive, + "coach_only": coach_only, + "recording": source.get("recording") is True, + "window": _project_window(source.get("window")), + "tree": nodes, + "truncated": source.get("truncated") is True or ( + isinstance(tree_in, list) and len(tree_in) > 200 + ), + "node_count": len(nodes), + } + if not nodes: + projected["reason"] = ( + source.get("reason") + if isinstance(source.get("reason"), str) and source.get("reason") + else "empty_projection" + ) + return projected + + +def _public_result(payload: Any) -> dict[str, Any]: + if not isinstance(payload, Mapping): + return {"status": "ok"} + out: dict[str, Any] = {} + for key, value in payload.items(): + if key in _RESULT_DROP or key in _FORBIDDEN_WIRE_KEYS: + continue + if key == "execution_outcome": + continue + if isinstance(value, Mapping): + nested = _public_result(value) + if nested: + out[key] = nested + continue + if isinstance(value, list): + continue + if key == "success": + continue + out[key] = value + return out + + +def _require_object(arguments: Optional[dict[str, Any]], allowed: set[str]) -> dict[str, Any]: + payload = dict(arguments or {}) + unknown = set(payload) - allowed + if unknown: + raise AuthoringError("arguments do not match the declared authoring schema") + return payload + + +def _invoke(session: object, method: str, **kwargs: Any) -> Any: + func = getattr(session, method, None) + aliases = { + "type_agent": ("type_text", "type"), + "pause_for_input": ("pause",), + "stop_record": ("finish", "stop"), + "start_record": ("start",), + } + if not callable(func): + for alias in aliases.get(method, ()): + candidate = getattr(session, alias, None) + if callable(candidate): + func = candidate + break + if not callable(func): + raise AuthoringError(f"authoring session does not implement {method}") + try: + return func(**kwargs) if kwargs else func() + except TypeError: + return func(kwargs) if kwargs else func() + + +class AuthoringBridge: + """Stdio authoring tool specs and dispatch over a session object.""" + + def __init__(self, session: object): + self.session = session + self._last_tool: Optional[str] = None + self._last_result: Optional[dict[str, Any]] = None + self._coach_only = False + + def handles(self, name: str) -> bool: + return name in AUTHORING_TOOLS + + def list_tool_specs(self) -> list[ToolSpec]: + specs = [ + ToolSpec( + name="observe", + description=_PROBE_HELP["observe"], + input_schema=_EMPTY_OBJECT, + annotations=_READ_ONLY, + ), + ToolSpec( + name="start_record", + description=_PROBE_HELP["start_record"], + input_schema=_EMPTY_OBJECT, + annotations=_MUTATING, + ), + ToolSpec( + name="click", + description=_PROBE_HELP["click"], + input_schema={ + "type": "object", + "properties": { + "node_id": { + "type": "string", + "description": "Opaque node id from the last observe tree.", + }, + "x": { + "type": "integer", + "description": "Local pixel X. Stdio only; hosted click is node_id.", + }, + "y": { + "type": "integer", + "description": "Local pixel Y. Stdio only; hosted click is node_id.", + }, + }, + "additionalProperties": False, + }, + annotations=_MUTATING, + ), + ToolSpec( + name="halt", + description=_PROBE_HELP["halt"], + input_schema=_EMPTY_OBJECT, + annotations=_LOCAL_HALT, + ), + ToolSpec( + name="type", + description=( + "Agent-driven typing through Flow Recorder.type_text. " + "Local stdio only; hosted MCP has no type tool. Do not use " + "this for secrets or for text a person already typed. Human " + "input uses pause_for_input, which persists with " + "record_observed and never type_text on the pause target." + ), + input_schema={ + "type": "object", + "properties": { + "text": { + "type": "string", + "description": "Literal text the agent types through the backend.", + }, + "param": { + "type": "string", + "description": "Optional workflow parameter name for this type.", + }, + "node_id": { + "type": "string", + "description": "Optional observed node to focus before typing.", + }, + }, + "required": ["text"], + "additionalProperties": False, + }, + annotations=_MUTATING, + ), + ToolSpec( + name="pause_for_input", + description=( + "Pause so a person can type in the application. On Continue, " + "persist with Recorder.record_observed on the pause-target " + "node. Never call type_text for that human input. Secret " + "pauses store no text. The MCP result has no value." + ), + input_schema={ + "type": "object", + "properties": { + "node_id": { + "type": "string", + "description": "Pause-target node id captured at pause start.", + }, + "param": { + "type": "string", + "description": "Parameter name to bind on the observed type.", + }, + "secret": { + "type": "boolean", + "description": "If true, persist secret=True and no text.", + }, + }, + "additionalProperties": False, + }, + annotations=_MUTATING, + ), + ToolSpec( + name="stop_record", + description=( + "Finish the Flow Recorder session without compiling. " + "Evidence stays local." + ), + input_schema=_EMPTY_OBJECT, + annotations=_LOCAL_HALT, + ), + ToolSpec( + name="compile", + description=( + "Wrap Flow compile_recording and return needs_human_admit. " + "An agent click never paints VERIFIED. Refuses a session " + "that had a secret pause and no TYPE/param event." + ), + input_schema=_EMPTY_OBJECT, + annotations={ + "readOnlyHint": False, + "destructiveHint": False, + "idempotentHint": True, + "openWorldHint": False, + }, + ), + ToolSpec( + name="get_command_result", + description=( + "Return the last in-process authoring result. Stdio executes " + "tools synchronously; call this after a probe if the client " + "expects the hosted pending/command_id shape." + ), + input_schema={ + "type": "object", + "properties": { + "command_id": { + "type": "string", + "description": "Optional id; stdio has one in-process result.", + } + }, + "additionalProperties": False, + }, + annotations=_READ_ONLY, + ), + ] + return specs + + def dispatch(self, name: str, arguments: Optional[dict[str, Any]] = None) -> dict[str, Any]: + if name not in AUTHORING_TOOLS: + raise AuthoringError("unknown tool name") + if name == "get_command_result": + _require_object(arguments, _RESULT_FIELDS) + return self._last_command_result() + handlers = { + "observe": self._observe, + "start_record": self._start_record, + "click": self._click, + "halt": self._halt, + "type": self._type_agent, + "pause_for_input": self._pause_for_input, + "stop_record": self._stop_record, + "compile": self._compile, + } + result = handlers[name](arguments) + self._last_tool = name + self._last_result = result + return result + + def _last_command_result(self) -> dict[str, Any]: + if self._last_result is None: + return { + "command_id": None, + "status": "idle", + "retry_after_ms": 0, + "result": None, + } + return { + "command_id": self._last_tool, + "status": "done", + "retry_after_ms": 0, + "result": dict(self._last_result), + } + + def _observe(self, arguments: Optional[dict[str, Any]]) -> dict[str, Any]: + _require_object(arguments, set()) + projected = project_observe(_invoke(self.session, "observe")) + self._coach_only = projected.get("coach_only") is True + return projected + + def _refuse_coach_only(self, tool: str) -> None: + if self._coach_only: + raise AuthoringError( + f"{tool} refused: COACH_ONLY (person actuates; this backend is " + "not agent-drive in v1)" + ) + + def _start_record(self, arguments: Optional[dict[str, Any]]) -> dict[str, Any]: + _require_object(arguments, set()) + self._refuse_coach_only("start_record") + raw = _invoke(self.session, "start_record") + result = _public_result(raw) + if result.get("error") == "COACH_ONLY" or result.get("coach_only") is True: + self._coach_only = True + raise AuthoringError( + "start_record refused: COACH_ONLY (person actuates; this backend " + "is not agent-drive in v1)" + ) + result.setdefault("status", "recording") + return result + + def _click(self, arguments: Optional[dict[str, Any]]) -> dict[str, Any]: + payload = _require_object(arguments, _CLICK_FIELDS) + self._refuse_coach_only("click") + node_id = payload.get("node_id") + x = payload.get("x") + y = payload.get("y") + has_node = isinstance(node_id, str) and bool(node_id) + has_point = x is not None or y is not None + if has_point and (not isinstance(x, int) or isinstance(x, bool) or + not isinstance(y, int) or isinstance(y, bool)): + raise AuthoringError("click x and y must both be integers") + if not has_node and not has_point: + raise AuthoringError("click requires node_id or local x and y") + kwargs: dict[str, Any] = {} + if has_node: + kwargs["node_id"] = node_id + if has_point: + kwargs["x"] = x + kwargs["y"] = y + raw = _invoke(self.session, "click", **kwargs) + result = _public_result(raw) + if result.get("error") == "stale_node" or result.get("status") == "stale_node": + return {"status": "error", "error": "stale_node"} + if result.get("error") == "COACH_ONLY": + raise AuthoringError( + "click refused: COACH_ONLY (person actuates; this backend is " + "not agent-drive in v1)" + ) + result.setdefault("status", "ok") + if has_node: + result.setdefault("node_id", node_id) + return result + + def _halt(self, arguments: Optional[dict[str, Any]]) -> dict[str, Any]: + _require_object(arguments, set()) + raw = _invoke(self.session, "halt") + result = _public_result(raw) + result.setdefault("status", "halted") + result["compiled"] = False + return result + + def _type_agent(self, arguments: Optional[dict[str, Any]]) -> dict[str, Any]: + payload = _require_object(arguments, _TYPE_FIELDS) + self._refuse_coach_only("type") + text = payload.get("text") + if not isinstance(text, str) or text == "": + raise AuthoringError("type requires a non-empty text string") + kwargs: dict[str, Any] = {"text": text} + param = payload.get("param") + if param is not None: + if not isinstance(param, str) or not param: + raise AuthoringError("param must be a string") + kwargs["param"] = param + node_id = payload.get("node_id") + if node_id is not None: + if not isinstance(node_id, str) or not node_id: + raise AuthoringError("node_id must be a string") + kwargs["node_id"] = node_id + raw = _invoke(self.session, "type_agent", **kwargs) + result = _public_result(raw) + result.pop("text", None) + result.setdefault("status", "ok") + result["recorded"] = True + if param: + result["param"] = param + return result + + def _pause_for_input(self, arguments: Optional[dict[str, Any]]) -> dict[str, Any]: + payload = _require_object(arguments, _PAUSE_FIELDS) + kwargs: dict[str, Any] = {} + node_id = payload.get("node_id") + if node_id is not None: + if not isinstance(node_id, str) or not node_id: + raise AuthoringError("node_id must be a string") + kwargs["node_id"] = node_id + param = payload.get("param") + if param is not None: + if not isinstance(param, str) or not param: + raise AuthoringError("param must be a string") + kwargs["param"] = param + secret = payload.get("secret") + if secret is not None: + if not isinstance(secret, bool): + raise AuthoringError("secret must be a boolean") + kwargs["secret"] = secret + raw = _invoke(self.session, "pause_for_input", **kwargs) + result = _public_result(raw) + result.pop("text", None) + result.pop("value", None) + result.setdefault("recorded", True) + if param: + result["param"] = param + if secret is True: + result["secret"] = True + return result + + def _stop_record(self, arguments: Optional[dict[str, Any]]) -> dict[str, Any]: + _require_object(arguments, set()) + raw = _invoke(self.session, "stop_record") + result = _public_result(raw) + result.setdefault("status", "stopped") + result["compiled"] = False + return result + + def _compile(self, arguments: Optional[dict[str, Any]]) -> dict[str, Any]: + _require_object(arguments, set()) + raw = _invoke(self.session, "compile") + if isinstance(raw, Mapping) and raw.get("error") == "missing_secret_type": + return {"status": "error", "error": "missing_secret_type"} + result = _public_result(raw) + if result.get("status") == "error": + return result + workflow_id = result.get("workflow_id") + public = { + "status": "needs_human_admit", + "recording_retained": True, + } + if isinstance(workflow_id, str) and workflow_id: + public["workflow_id"] = workflow_id + return public diff --git a/src/openadapt_agent/cli.py b/src/openadapt_agent/cli.py index 86c21ba..82dce82 100644 --- a/src/openadapt_agent/cli.py +++ b/src/openadapt_agent/cli.py @@ -5,6 +5,8 @@ - ``serve`` — expose compiled openadapt-flow bundles and the local Needs Attention queue over MCP stdio. PHI-safe read-only tools are always on; workflow runs and attended decisions require separate operator flags. + ``--authoring`` adds first-demo stdio tools and does not imply + ``--allow-run``. - ``emit-skill`` — emit a Claude Agent Skill folder for one bundle (wraps ``openadapt-flow emit-skill`` and appends MCP + halt guidance). @@ -48,8 +50,19 @@ def build_parser() -> argparse.ArgumentParser: help=( "Bundle directory: either one compiled bundle, or a directory " "whose immediate subdirectories are bundles. Required unless " - "--tutorial is set, or --allow-run is set with no --bundles " - "(synthetic tutorial)." + "--authoring or --tutorial is set, or --allow-run is set with no " + "--bundles (synthetic tutorial). The published run recipe still " + "requires --bundles." + ), + ) + p.add_argument( + "--authoring", + action="store_true", + help=( + "Register first-demo authoring tools over local stdio: observe, " + "start_record, click, halt. Local stdio may also type through the " + "recorder; hosted MCP remains pause-only. Does not enable run " + "tools. This process stays stdio and must not be served over HTTP." ), ) p.add_argument( @@ -214,7 +227,17 @@ def _cmd_serve(args: argparse.Namespace) -> int: file=sys.stderr, ) return 2 - if not args.tutorial and not args.bundles: + if args.authoring and args.tutorial: + print("serve: --authoring cannot be combined with --tutorial", file=sys.stderr) + return 2 + if args.authoring and args.allow_run and not args.bundles: + print( + "serve: --authoring does not imply --allow-run; --allow-run still " + "requires --bundles", + file=sys.stderr, + ) + return 2 + if not args.tutorial and not args.bundles and not args.authoring: if args.allow_run: # Day-1 partner kit: `openadapt-agent serve --allow-run` hosts # the synthetic tutorial. Registry installs still pass --bundles @@ -222,7 +245,7 @@ def _cmd_serve(args: argparse.Namespace) -> int: args.tutorial = True else: print( - "serve: provide --bundles or --tutorial " + "serve: provide --bundles, --tutorial, or --authoring " "(or --allow-run for the synthetic tutorial)", file=sys.stderr, ) @@ -236,12 +259,22 @@ def _cmd_serve(args: argparse.Namespace) -> int: extra_run_args = list(args.extra_run_arg) tutorial_session = None + authoring_bridge = None bundles_dir = args.bundles url = args.url deployment_config = args.config policy = args.policy public_synthetic = False try: + if args.authoring: + from openadapt_agent.authoring import AuthoringBridge, AuthoringError + from openadapt_agent.authoring import open_authoring_session + + try: + authoring_bridge = AuthoringBridge(open_authoring_session()) + except AuthoringError as exc: + print(f"serve: {exc}", file=sys.stderr) + return 2 if args.tutorial: work_dir = Path(args.runs_dir).expanduser().resolve() / "synthetic-tutorial" tutorial_session = prepare_tutorial_session( @@ -258,49 +291,61 @@ def _cmd_serve(args: argparse.Namespace) -> int: if args.headed and "--headed" not in extra_run_args: extra_run_args.append("--headed") - runner_config = RunnerConfig( - flow_cli=(tuple(shlex.split(args.flow_cli)) if args.flow_cli else default_flow_cli()), - runs_dir=Path(args.runs_dir), - url=url, - deployment_config=deployment_config, - policy=policy, - timeout_s=args.timeout, - allow_url_override=args.allow_url_override, - extra_run_args=tuple(extra_run_args), - ) - with open_attended_service( - enabled=args.allow_attended_actions, - deployment_config=deployment_config, - url=url, - headed=args.headed, - allow_model_grounding=args.allow_model_grounding, - ) as attended_service: - bridge = AgentBridge( - Path(bundles_dir), - runner_config, - allow_run=args.allow_run, - allow_attended_actions=args.allow_attended_actions, - attended_service=attended_service, - allow_protected_export=args.allow_protected_export, - allow_recorded_defaults=args.allow_synthetic_recorded_defaults, - public_synthetic=public_synthetic, - ) - n = len(bridge.workflows) + if bundles_dir is None and authoring_bridge is not None: print( - f"openadapt-agent {__version__}: serving {n} workflow(s) " - "over local stdio; run tools " - f"{'enabled' if args.allow_run else 'disabled'}; attended " - f"decisions {'enabled' if args.allow_attended_actions else 'disabled'}; " - "live Continue/Skip " - f"{'ready' if bridge.attended.live_actions_ready else 'not configured'}; " - "protected MCP export " - f"{'ENABLED' if args.allow_protected_export else 'disabled'}; " - "synthetic recorded defaults " - f"{'ENABLED' if args.allow_synthetic_recorded_defaults else 'disabled'}; " - f"tutorial {'enabled' if args.tutorial else 'disabled'}", + f"openadapt-agent {__version__}: authoring tools enabled over " + "local stdio; run tools disabled; --authoring does not imply " + "--allow-run", file=sys.stderr, ) - serve(bridge) + _serve(serve, None, authoring_bridge) + else: + runner_config = RunnerConfig( + flow_cli=( + tuple(shlex.split(args.flow_cli)) if args.flow_cli else default_flow_cli() + ), + runs_dir=Path(args.runs_dir), + url=url, + deployment_config=deployment_config, + policy=policy, + timeout_s=args.timeout, + allow_url_override=args.allow_url_override, + extra_run_args=tuple(extra_run_args), + ) + with open_attended_service( + enabled=args.allow_attended_actions, + deployment_config=deployment_config, + url=url, + headed=args.headed, + allow_model_grounding=args.allow_model_grounding, + ) as attended_service: + bridge = AgentBridge( + Path(bundles_dir), + runner_config, + allow_run=args.allow_run, + allow_attended_actions=args.allow_attended_actions, + attended_service=attended_service, + allow_protected_export=args.allow_protected_export, + allow_recorded_defaults=args.allow_synthetic_recorded_defaults, + public_synthetic=public_synthetic, + ) + n = len(bridge.workflows) + print( + f"openadapt-agent {__version__}: serving {n} workflow(s) " + "over local stdio; run tools " + f"{'enabled' if args.allow_run else 'disabled'}; attended " + f"decisions {'enabled' if args.allow_attended_actions else 'disabled'}; " + "live Continue/Skip " + f"{'ready' if bridge.attended.live_actions_ready else 'not configured'}; " + "protected MCP export " + f"{'ENABLED' if args.allow_protected_export else 'disabled'}; " + "synthetic recorded defaults " + f"{'ENABLED' if args.allow_synthetic_recorded_defaults else 'disabled'}; " + f"tutorial {'enabled' if args.tutorial else 'disabled'}; " + f"authoring {'enabled' if authoring_bridge is not None else 'disabled'}", + file=sys.stderr, + ) + _serve(serve, bridge, authoring_bridge) except TutorialError as exc: print(f"serve: {exc}", file=sys.stderr) return 2 @@ -313,6 +358,14 @@ def _cmd_serve(args: argparse.Namespace) -> int: return 0 +def _serve(serve, bridge, authoring): + """Call serve without surprising 1-arg monkeypatches in existing tests.""" + if authoring is None: + serve(bridge) + return + serve(bridge, authoring=authoring) + + def _cmd_emit_skill(args: argparse.Namespace) -> int: from openadapt_agent.skill import emit_agent_skill diff --git a/src/openadapt_agent/mcp.py b/src/openadapt_agent/mcp.py index 85ec135..56ac5e9 100644 --- a/src/openadapt_agent/mcp.py +++ b/src/openadapt_agent/mcp.py @@ -1,13 +1,15 @@ -"""MCP (stdio) transport for :class:`openadapt_agent.bridge.AgentBridge`. +"""MCP (stdio) transport for the local OpenAdapt bridges. Run directly:: python -m openadapt_agent.mcp --bundles ./bundles [--allow-run] ... + python -m openadapt_agent.mcp --authoring or via the CLI entry point ``openadapt-agent serve``. The server speaks -MCP over stdio (what Claude Code / Claude Desktop consume). All tool logic -lives in :mod:`openadapt_agent.bridge`; this module only adapts it to the -official ``mcp`` SDK's low-level server. +MCP over stdio (what Claude Code / Claude Desktop consume). Tool logic +lives in :mod:`openadapt_agent.bridge` and :mod:`openadapt_agent.authoring`; +this module only adapts them to the official ``mcp`` SDK's low-level +server. This package does not open an HTTP listener. """ from __future__ import annotations @@ -22,6 +24,7 @@ from mcp.server.stdio import stdio_server from openadapt_agent.attended import ATTENDED_TOOLS +from openadapt_agent.authoring import AuthoringBridge from openadapt_agent.bridge import AgentBridge, BridgeError __all__ = ["build_server", "serve"] @@ -91,30 +94,50 @@ async def _confirm_attended_action(server: Server, name: str) -> None: ) -def build_server(bridge: AgentBridge) -> Server: - """Wrap a bridge in an ``mcp`` low-level Server (no I/O started).""" +def _server_instructions(authoring: AuthoringBridge | None) -> str: + text = ( + "Local bridge exposing compiled openadapt-flow workflow bundles " + "and PHI-safe Needs Attention items as tools. run_* tools execute via the governed " + "`openadapt-flow run` CLI and return a structured outcome: only " + "status 'success' means the workflow completed and verified. " + "'halt' means the run stopped and protected evidence remains in " + "the local operator experience; get_run_report returns only a " + "PHI-safe status/count summary by default. 'refused' means an " + "admission gate refused the bundle and nothing executed. " + "Continue/Skip require a human action " + "plus protocol-native operator elicitation, an exact signed " + "capability, live revalidation, and a stable idempotency key; they " + "never re-actuate the human-completed step. " + "Reject terminates the run and dispatches no new action, but earlier " + "run effects still require review of the protected local outcome. " + "Never report a halted, refused, timed-out, or error run as a success. " + "If execution_outcome is HALTED, tell the user the record did not change. " + "Write tools advertise requires_seal: true. If a write tool returns " + "unsigned success, treat it as failure." + ) + if authoring is not None: + text += ( + " --authoring adds first-demo tools observe, start_record, click, " + "and halt over this same local stdio process. Local stdio may also " + "type through the recorder; hosted MCP has no type tool. Human type " + "during pause_for_input is record_observed, never type_text. " + "compile returns needs_human_admit; an agent click never paints " + "VERIFIED. --authoring does not enable run tools. This process " + "must not be port-forwarded or served over HTTP." + ) + return text + + +def build_server( + bridge: AgentBridge | None = None, + authoring: AuthoringBridge | None = None, +) -> Server: + """Wrap workflow and/or authoring bridges in an MCP Server (no I/O started).""" + if bridge is None and authoring is None: + raise ValueError("MCP server requires a workflow bridge or an authoring bridge") server: Server = Server( SERVER_NAME, - instructions=( - "Local bridge exposing compiled openadapt-flow workflow bundles " - "and PHI-safe Needs Attention items as tools. run_* tools execute via the governed " - "`openadapt-flow run` CLI and return a structured outcome: only " - "status 'success' means the workflow completed and verified. " - "'halt' means the run stopped and protected evidence remains in " - "the local operator experience; get_run_report returns only a " - "PHI-safe status/count summary by default. 'refused' means an " - "admission gate refused the bundle and nothing executed. " - "Continue/Skip require a human action " - "plus protocol-native operator elicitation, an exact signed " - "capability, live revalidation, and a stable idempotency key; they " - "never re-actuate the human-completed step. " - "Reject terminates the run and dispatches no new action, but earlier " - "run effects still require review of the protected local outcome. " - "Never report a halted, refused, timed-out, or error run as a success. " - "If execution_outcome is HALTED, tell the user the record did not change. " - "Write tools advertise requires_seal: true. If a write tool returns " - "unsigned success, treat it as failure." - ), + instructions=_server_instructions(authoring), ) @server.list_tools() @@ -131,7 +154,10 @@ async def _list_tools() -> list[types.Tool]: ), **({"_meta": spec.meta} if spec.meta is not None else {}), ) - for spec in bridge.list_tool_specs() + for spec in ( + *(bridge.list_tool_specs() if bridge is not None else ()), + *(authoring.list_tool_specs() if authoring is not None else ()), + ) ] @server.call_tool() @@ -141,7 +167,12 @@ async def _call_tool(name: str, arguments: dict[str, Any] | None): await _confirm_attended_action(server, name) def call() -> dict[str, Any]: - return bridge.dispatch(name, dict(arguments or {})) + payload = dict(arguments or {}) + if authoring is not None and authoring.handles(name): + return authoring.dispatch(name, payload) + if bridge is None: + raise BridgeError("unknown tool name") + return bridge.dispatch(name, payload) # CLI runs and filesystem projections are blocking. Live attended # actions synchronously submit to their own non-async backend-owner @@ -180,15 +211,21 @@ def call() -> dict[str, Any]: return server -async def _run_stdio(bridge: AgentBridge) -> None: - server = build_server(bridge) +async def _run_stdio( + bridge: AgentBridge | None, + authoring: AuthoringBridge | None = None, +) -> None: + server = build_server(bridge, authoring=authoring) async with stdio_server() as (read_stream, write_stream): await server.run(read_stream, write_stream, server.create_initialization_options()) -def serve(bridge: AgentBridge) -> None: +def serve( + bridge: AgentBridge | None = None, + authoring: AuthoringBridge | None = None, +) -> None: """Serve the bridge over stdio until the client disconnects.""" - anyio.run(_run_stdio, bridge) + anyio.run(_run_stdio, bridge, authoring) if __name__ == "__main__": # pragma: no cover - exercised by smoke test diff --git a/tests/test_authoring.py b/tests/test_authoring.py new file mode 100644 index 0000000..148ba6b --- /dev/null +++ b/tests/test_authoring.py @@ -0,0 +1,365 @@ +"""Stdio --authoring tools: probe names, PHI observe, record_observed.""" + +from __future__ import annotations + +import json +from pathlib import Path + +import anyio +import mcp.types as types +import pytest + +from openadapt_agent.authoring import ( + AUTHORING_PROBE_TOOLS, + AuthoringBridge, + AuthoringError, + open_authoring_session, + project_observe, +) +from openadapt_agent.mcp import build_server + + +class FakeAuthoringSession: + """In-process stand-in until openadapt_flow.authoring lands.""" + + def __init__( + self, + *, + backend: str = "web", + coach_only: bool = False, + secret_pause_without_type: bool = False, + ): + self.backend = backend + self.coach_only = coach_only or backend in {"windows", "rdp", "citrix"} + self.secret_pause_without_type = secret_pause_without_type + self.calls: list = [] + self.typed_via_backend: list[str] = [] + self.observed_events: list[dict] = [] + self.recording = False + self._secret_type_recorded = False + self.nodes = { + "n_9f2c": { + "node_id": "n_9f2c", + "role": "button", + "control_type": "button", + "automation_id": "btnContinue", + "enabled": True, + "focused": False, + "bounds": {"x": 0.72, "y": 0.88, "w": 0.14, "h": 0.05}, + "backend_pixels": {"x": 920, "y": 640, "w": 180, "h": 36}, + "value": "4111111111111111", + "title": "Chart — Jane Roe", + "name": "Save", + } + } + + def observe(self): + self.calls.append("observe") + if self.coach_only: + return { + "backend": self.backend, + "agent_drive": True, + "coach_only": True, + "tree": list(self.nodes.values()), + "title": "Remote desktop", + } + return { + "schema_version": "openadapt.authoring.observe/v1", + "backend": self.backend, + "provider": "playwright_ax", + "agent_drive": True, + "coach_only": False, + "recording": self.recording, + "window": { + "process_name": "Chromium", + "role": "window", + "title": "Patient Jane Roe MRN-9911", + "bounds": {"x": 0.0, "y": 0.0, "w": 1.0, "h": 1.0}, + }, + "tree": [ + { + **node, + "screenshot": "iVBORw0KGgo=", + "url": "https://example.invalid/chart", + "name": "Call 555-0100" if node_id == "leak" else node.get("name"), + } + for node_id, node in self.nodes.items() + ] + + [ + { + "node_id": "n_ssn1", + "role": "textbox", + "name": "SSN 123-45-6789", + "automation_id": "patient@clinic.example", + "value": "123-45-6789", + } + ], + "value": "raw AX dump", + "screenshot": "pixels", + } + + def start_record(self): + self.calls.append("start_record") + if self.coach_only: + return {"error": "COACH_ONLY", "coach_only": True, "backend": self.backend} + self.recording = True + return {"status": "recording"} + + def click(self, node_id=None, x=None, y=None): + self.calls.append(("click", node_id, x, y)) + if self.coach_only: + return {"error": "COACH_ONLY"} + if node_id and node_id not in self.nodes: + return {"status": "error", "error": "stale_node"} + return {"status": "ok", "node_id": node_id, "backend_pixels": {"x": 1, "y": 2}} + + def type_agent(self, text, param=None, node_id=None): + self.calls.append(("type_agent", text, param, node_id)) + self.typed_via_backend.append(text) + return {"status": "ok", "param": param, "text": text} + + def type_text(self, text, param=None): + raise AssertionError("human pause must not call backend type_text") + + def pause_for_input(self, node_id=None, param=None, secret=False): + self.calls.append(("pause_for_input", node_id, param, secret)) + event = {"kind": "type", "param": param, "secret": secret} + if not secret: + event["text"] = "synthetic follow-up" + self.observed_events.append(event) + if secret and not self.secret_pause_without_type: + self._secret_type_recorded = True + return {"recorded": True, "param": param, "text": "must-not-cross-mcp"} + + def halt(self): + self.calls.append("halt") + self.recording = False + return {"status": "halted"} + + def stop_record(self): + self.calls.append("stop_record") + self.recording = False + return {"status": "stopped"} + + def compile(self): + self.calls.append("compile") + if self.secret_pause_without_type and not self._secret_type_recorded: + return {"error": "missing_secret_type"} + return { + "status": "needs_human_admit", + "workflow_id": "wf_demo", + "execution_outcome": "VERIFIED", + "success": True, + } + + +def test_probe_tool_names_match_hosted_surface(): + names = [spec.name for spec in AuthoringBridge(FakeAuthoringSession()).list_tool_specs()] + for probe in AUTHORING_PROBE_TOOLS: + assert probe in names + assert "type" in names + assert names.index("observe") < names.index("type") + + +def test_observe_drops_values_titles_screenshots_and_unsafe_names(): + bridge = AuthoringBridge(FakeAuthoringSession()) + result = bridge.dispatch("observe", {}) + blob = json.dumps(result) + assert result["schema_version"] == "openadapt.authoring.observe/v1" + assert result["mode"] == "authoring" + assert result["agent_drive"] is True + assert "value" not in blob + assert "screenshot" not in blob + assert "title" not in blob + assert "backend_pixels" not in blob + assert "Jane Roe" not in blob + assert "4111111111111111" not in blob + assert "123-45-6789" not in blob + assert "patient@clinic.example" not in blob + assert "https://" not in blob + node = next(item for item in result["tree"] if item["node_id"] == "n_9f2c") + assert node["automation_id"] == "btnContinue" + assert node["name"] == "Save" + assert all(item["node_id"] != "n_ssn1" or "name" not in item for item in result["tree"]) + + +def test_project_observe_empty_tree_is_not_a_raw_fallback(): + projected = project_observe({"backend": "web", "tree": [{"role": "button", "value": "x"}]}) + assert projected["tree"] == [] + assert projected["reason"] == "empty_projection" + assert "value" not in json.dumps(projected) + + +def test_start_record_and_click_and_halt_round_trip(): + session = FakeAuthoringSession() + bridge = AuthoringBridge(session) + assert bridge.dispatch("start_record", {})["status"] == "recording" + clicked = bridge.dispatch("click", {"node_id": "n_9f2c"}) + assert clicked["status"] == "ok" + assert "backend_pixels" not in clicked + assert bridge.dispatch("click", {"node_id": "n_missing"}) == { + "status": "error", + "error": "stale_node", + } + local = bridge.dispatch("click", {"x": 10, "y": 20}) + assert local["status"] == "ok" + halted = bridge.dispatch("halt", {}) + assert halted["status"] == "halted" + assert halted["compiled"] is False + assert ("click", "n_9f2c", None, None) in session.calls + assert ("click", None, 10, 20) in session.calls + + +def test_local_type_is_agent_driven_and_strips_text(): + session = FakeAuthoringSession() + bridge = AuthoringBridge(session) + result = bridge.dispatch("type", {"text": "synthetic follow-up", "param": "note"}) + assert result["recorded"] is True + assert result["param"] == "note" + assert "text" not in result + assert session.typed_via_backend == ["synthetic follow-up"] + assert session.observed_events == [] + + +def test_pause_continue_uses_record_observed_never_type_text(): + session = FakeAuthoringSession() + bridge = AuthoringBridge(session) + result = bridge.dispatch( + "pause_for_input", + {"node_id": "n_9f2c", "param": "note", "secret": False}, + ) + assert result == {"recorded": True, "param": "note"} + assert session.typed_via_backend == [] + assert session.observed_events == [ + {"kind": "type", "param": "note", "secret": False, "text": "synthetic follow-up"} + ] + + +def test_secret_pause_result_has_no_value_and_compile_can_refuse(): + session = FakeAuthoringSession(secret_pause_without_type=True) + bridge = AuthoringBridge(session) + paused = bridge.dispatch( + "pause_for_input", + {"node_id": "n_9f2c", "param": "identifier", "secret": True}, + ) + assert paused["recorded"] is True + assert paused["secret"] is True + assert "text" not in paused + assert bridge.dispatch("compile", {}) == { + "status": "error", + "error": "missing_secret_type", + } + + +def test_compile_returns_needs_human_admit_never_verified(): + bridge = AuthoringBridge(FakeAuthoringSession()) + result = bridge.dispatch("compile", {}) + assert result["status"] == "needs_human_admit" + assert result["workflow_id"] == "wf_demo" + assert result["recording_retained"] is True + assert "VERIFIED" not in json.dumps(result) + assert "success" not in result + + +def test_windows_native_is_coach_only(): + session = FakeAuthoringSession(backend="windows") + bridge = AuthoringBridge(session) + observed = bridge.dispatch("observe", {}) + assert observed["coach_only"] is True + assert observed["agent_drive"] is False + assert observed["tree"] == [] + with pytest.raises(AuthoringError, match="COACH_ONLY"): + bridge.dispatch("start_record", {}) + with pytest.raises(AuthoringError, match="COACH_ONLY"): + bridge.dispatch("click", {"node_id": "n_9f2c"}) + + +def test_get_command_result_returns_last_in_process_result(): + bridge = AuthoringBridge(FakeAuthoringSession()) + idle = bridge.dispatch("get_command_result", {}) + assert idle["status"] == "idle" + assert idle["result"] is None + bridge.dispatch("start_record", {}) + last = bridge.dispatch("get_command_result", {}) + assert last["command_id"] == "start_record" + assert last["status"] == "done" + assert last["result"]["status"] == "recording" + + +def test_unknown_fields_and_missing_click_target_are_refused(): + bridge = AuthoringBridge(FakeAuthoringSession()) + with pytest.raises(AuthoringError, match="schema"): + bridge.dispatch("observe", {"title": "nope"}) + with pytest.raises(AuthoringError, match="node_id or local"): + bridge.dispatch("click", {}) + + +def test_mcp_lists_authoring_probe_tools_without_run_tools(bundles_root, runner_config): + from openadapt_agent.bridge import AgentBridge + + authoring = AuthoringBridge(FakeAuthoringSession()) + server = build_server(authoring=authoring) + + async def list_names(): + handler = server.request_handlers[types.ListToolsRequest] + result = await handler(types.ListToolsRequest(method="tools/list")) + return [tool.name for tool in result.root.tools] + + names = anyio.run(list_names) + assert names[:4] == list(AUTHORING_PROBE_TOOLS) + assert "type" in names + assert not any(name.startswith("run_") for name in names) + + combined = build_server( + AgentBridge(bundles_root, runner_config, allow_run=False), + authoring=authoring, + ) + + async def combined_names(): + handler = combined.request_handlers[types.ListToolsRequest] + result = await handler(types.ListToolsRequest(method="tools/list")) + return [tool.name for tool in result.root.tools] + + both = anyio.run(combined_names) + assert "list_workflows" in both + assert "observe" in both + assert not any(name.startswith("run_") for name in both) + + +def test_mcp_observe_call_is_projected(): + server = build_server(authoring=AuthoringBridge(FakeAuthoringSession())) + + async def call_observe(): + handler = server.request_handlers[types.CallToolRequest] + return await handler( + types.CallToolRequest( + params=types.CallToolRequestParams(name="observe", arguments={}) + ) + ) + + result = anyio.run(call_observe).root + payload = json.loads(result.content[0].text) + assert payload["schema_version"] == "openadapt.authoring.observe/v1" + assert "screenshot" not in result.content[0].text + assert "Jane Roe" not in result.content[0].text + + +def test_open_authoring_session_notes_flow_dependency_when_missing(monkeypatch): + import sys + import types + + fake_flow = types.ModuleType("openadapt_flow") + monkeypatch.setitem(sys.modules, "openadapt_flow", fake_flow) + monkeypatch.delitem(sys.modules, "openadapt_flow.authoring", raising=False) + with pytest.raises(AuthoringError, match="openadapt_flow.authoring"): + open_authoring_session() + + +def test_authoring_sources_stay_stdio_without_http_listener(): + root = Path(__file__).resolve().parents[1] / "src" / "openadapt_agent" + for name in ("authoring.py", "mcp.py", "cli.py"): + text = (root / name).read_text(encoding="utf-8") + assert "HTTPServer" not in text + assert "uvicorn" not in text + assert "streamable_http" not in text.lower() + assert "FastAPI" not in text diff --git a/tests/test_cli.py b/tests/test_cli.py index 6a36307..45c1bbb 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -81,10 +81,84 @@ def test_serve_requires_tutorial_or_bundles(capsys): result = main(["serve"]) assert result == 2 err = capsys.readouterr().err - assert "provide --bundles or --tutorial" in err + assert "provide --bundles, --tutorial, or --authoring" in err assert "--allow-run" in err +def test_authoring_flag_does_not_require_bundles_or_imply_allow_run(tmp_path): + args = build_parser().parse_args( + ["serve", "--authoring", "--runs-dir", str(tmp_path / "runs")] + ) + assert args.authoring is True + assert args.bundles is None + assert args.allow_run is False + assert args.tutorial is False + + +def test_authoring_help_says_run_tools_stay_off_and_stdio_only(capsys): + with pytest.raises(SystemExit) as exc_info: + build_parser().parse_args(["serve", "--help"]) + assert exc_info.value.code == 0 + out = capsys.readouterr().out + assert "--authoring" in out + assert "Does not enable run tools" in out + assert "HTTP" in out + + +def test_authoring_does_not_imply_allow_run_without_bundles(capsys): + result = main(["serve", "--authoring", "--allow-run"]) + assert result == 2 + err = capsys.readouterr().err + assert "does not imply --allow-run" in err + assert "requires --bundles" in err + + +def test_authoring_cannot_combine_with_tutorial(capsys): + result = main(["serve", "--authoring", "--tutorial"]) + assert result == 2 + assert "cannot be combined" in capsys.readouterr().err + + +def test_authoring_without_flow_session_fails_closed(capsys, monkeypatch): + def missing(**kwargs): + from openadapt_agent.authoring import AuthoringError + + raise AuthoringError("openadapt_flow.authoring is not available") + + monkeypatch.setattr("openadapt_agent.authoring.open_authoring_session", missing) + result = main(["serve", "--authoring"]) + assert result == 2 + assert "openadapt_flow.authoring" in capsys.readouterr().err + + +def test_authoring_serve_registers_probe_tools_without_run(monkeypatch, capsys): + from test_authoring import FakeAuthoringSession + + captured: dict = {} + + def fake_session(**kwargs): + return FakeAuthoringSession() + + def fake_serve(bridge, authoring=None): + captured["bridge"] = bridge + captured["authoring"] = authoring + + monkeypatch.setattr("openadapt_agent.authoring.open_authoring_session", fake_session) + monkeypatch.setattr("openadapt_agent.mcp.serve", fake_serve) + + result = main(["serve", "--authoring"]) + assert result == 0 + assert captured["bridge"] is None + authoring = captured["authoring"] + names = [spec.name for spec in authoring.list_tool_specs()] + assert names[:4] == ["observe", "start_record", "click", "halt"] + assert "type" in names + err = capsys.readouterr().err + assert "authoring tools enabled" in err + assert "run tools disabled" in err + assert "does not imply --allow-run" in err + + def test_tutorial_rejects_private_bundle_path(tmp_path, capsys): result = main( ["serve", "--tutorial", "--bundles", str(tmp_path / "bundles")] diff --git a/tests/test_distribution.py b/tests/test_distribution.py index cd8f51b..d767c31 100644 --- a/tests/test_distribution.py +++ b/tests/test_distribution.py @@ -163,6 +163,9 @@ def test_serve_is_the_subcommand_and_bundles_is_required() -> None: assert any(a.get("value") == "serve" for a in positional) assert "--bundles" in named assert named["--bundles"]["isRequired"] is True + assert "--authoring" not in named + assert not any(a.get("value") == "--authoring" for a in args) + assert _server_json()["packages"][0]["transport"]["type"] == "stdio" def test_registry_launch_is_read_only_by_default() -> None: From 1b16b702d592aa4a1356f7500ed3da4253d979b1 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 1 Sep 2026 14:43:00 -0400 Subject: [PATCH 2/3] fix(agent): close remaining A1 stdio authoring gaps Align the observe projector with the T1 wire, construct F1 AuthoringSession with a locally pinned backend, prefer Desktop IPC discovery without speaking D2, and document Claude Code --authoring as the first authoring UI. HTTP shim remains forbidden. server.json stays stdio with --bundles required. --- README.md | 18 + docs/DESIGN.md | 20 +- docs/DISTRIBUTION.md | 2 +- llms.txt | 2 +- src/openadapt_agent/authoring.py | 814 +++++++++++++++++++++++++++---- src/openadapt_agent/cli.py | 22 +- tests/test_authoring.py | 359 +++++++++++++- tests/test_cli.py | 2 + tests/test_distribution.py | 5 + 9 files changed, 1117 insertions(+), 127 deletions(-) diff --git a/README.md b/README.md index 009631f..5814ae3 100644 --- a/README.md +++ b/README.md @@ -42,6 +42,23 @@ uvx openadapt-agent serve --bundles /path/to/bundles # read-only uvx openadapt-agent serve --bundles /path/to/bundles --allow-run ``` +## Author a first demo (Claude Code) + +Local Claude Code (or Cursor, Codex, Grok CLI) is the first authoring UI. +Hosted ChatGPT.com / Claude.ai MCP is a website mailbox, not a listener in +this package. + +```bash +claude mcp add openadapt-authoring -- \ + uvx --from 'openadapt-agent[tutorial]' openadapt-agent \ + serve --authoring +``` + +`--bundles` is omitted. Probe tools are `observe`, `start_record`, `click`, +and `halt`. `--authoring` does not enable run tools. This process stays +stdio. Pass `--url` to pin a fresh Playwright Chromium with empty cookies. +Windows native, Citrix, and RDP are coach-only in v1. + ## Serve a bundle `--allow-run` with no `--bundles` records, compiles, and certifies the @@ -150,6 +167,7 @@ Continue and Skip. | `run_local_quickstart` | `--allow-run` with no `--bundles` | | `reject_attention`, `teach_attention`, `escalate_attention` | `--allow-attended-actions` | | `continue_attention`, `skip_attention` | `--allow-attended-actions` plus a qualified deployment `--config` | +| `observe`, `start_record`, `click`, `halt` | `--authoring` | ## Run outcomes diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 036deae..6b9fc71 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -142,10 +142,17 @@ Observe is a fail-closed PHI projection (`openadapt.authoring.observe/v1`): no `value`, `text`, window `title`, screenshot, OCR, URL, or backend pixels. Windows native, Citrix, and RDP are `COACH_ONLY` in v1. -The session object is Flow's public `openadapt_flow.authoring` module. -Until that module is importable, `serve --authoring` fails closed with -an explicit dependency error. Tests cover the tool surface with a fake -session. +The session object is Flow's public `openadapt_flow.authoring` module +(`AuthoringSession(backend, out_dir, backend_kind=…)` when F1 is +importable). Until that module is importable, `serve --authoring` fails +closed with an explicit dependency error. Windows native, Citrix, and +RDP construct a coach-only stand-in and never spawn `win_agent`. Observe +is fail-closed to the T1 wire (`additionalProperties: false`, node ids +`n_` + 8 hex, 200 nodes / 32 KiB). Capture's projector is used when +importable. If Desktop has advertised authoring IPC, overlay stays +Desktop-owned; this package does not speak the D2 protocol or open an +HTTP client. Tests cover the tool surface with a fake session and an +F1-shaped session. ## Governed runs @@ -342,8 +349,9 @@ Tests cover: - MCP serialization and thread ownership; - Agent Skill emission; - `--authoring` probe tools (`observe`, `start_record`, `click`, `halt`) - and local `type`; observe projection drops values/titles/screenshots; - pause Continue uses `record_observed` rather than `type_text`; + and local `type`; observe projection drops values/titles/screenshots + and extra keys, caps the wire at 32 KiB, and uses `n_` + 8 hex node + ids; pause Continue uses `record_observed` rather than `type_text`; compile returns `needs_human_admit`; `--authoring` does not enable run tools; `server.json` stays stdio with `--bundles` required. diff --git a/docs/DISTRIBUTION.md b/docs/DISTRIBUTION.md index ec37426..0ea6f09 100644 --- a/docs/DISTRIBUTION.md +++ b/docs/DISTRIBUTION.md @@ -64,7 +64,7 @@ and [`../manifest.json`](../manifest.json). - **License:** MIT - **Transport:** stdio - **Run command (uvx):** `uvx --from 'openadapt-agent[tutorial]' openadapt-agent serve --allow-run` -- **Config:** `--allow-run` with no `--bundles` (public synthetic bundle, generated at serve time), `--tutorial` (same path without implying run tools), or `--bundles` (operator's private artifact), `--runs-dir`, `--allow-attended-actions`, qualified `--config` for Continue/Skip, and optional secret `OPENADAPT_BUNDLE_KEY` +- **Config:** `--allow-run` with no `--bundles` (public synthetic bundle, generated at serve time), `--tutorial` (same path without implying run tools), `--authoring` (local Claude Code first demo; not the published registry recipe), or `--bundles` (operator's private artifact), `--runs-dir`, `--allow-attended-actions`, qualified `--config` for Continue/Skip, and optional secret `OPENADAPT_BUNDLE_KEY` - **Tools:** - `list_workflows` / `get_workflow` — PHI-safe structural bundle projections with opaque IDs. - `get_run_report` — PHI-safe status and count summary; raw evidence stays local unless protected export was explicitly enabled. diff --git a/llms.txt b/llms.txt index efc2864..ab3638d 100644 --- a/llms.txt +++ b/llms.txt @@ -6,7 +6,7 @@ Default runtime interface for a calling agent. Computer-use agents are the user ## What it provides -- `openadapt-agent serve --allow-run`: generate and serve the public synthetic tutorial at serve time. `openadapt-agent serve --tutorial` is the same path without run tools. `openadapt-agent serve --bundles [--allow-run]`: serve a private compiled bundle. `list_workflows`, `get_workflow`, `get_run_report`, `list_needs_attention`, and `get_attention_item` are always available as PHI-safe read-only projections. `run_workflow_` tools require `--allow-run`. The synthetic tutorial registers `run_local_quickstart`. If a run returns HALTED, tell the user the record did not change. Never summarize halt, refused, timeout, or error as success. +- `openadapt-agent serve --allow-run`: generate and serve the public synthetic tutorial at serve time. `openadapt-agent serve --tutorial` is the same path without run tools. `openadapt-agent serve --bundles [--allow-run]`: serve a private compiled bundle. `openadapt-agent serve --authoring`: local Claude Code first-demo tools `observe`, `start_record`, `click`, and `halt` over stdio. `--authoring` does not enable run tools and does not open an HTTP listener. Hosted ChatGPT.com / Claude.ai MCP is a website mailbox. `list_workflows`, `get_workflow`, `get_run_report`, `list_needs_attention`, and `get_attention_item` are always available as PHI-safe read-only projections. `run_workflow_` tools require `--allow-run`. The synthetic tutorial registers `run_local_quickstart`. If a run returns HALTED, tell the user the record did not change. Never summarize halt, refused, timeout, or error as success. - `--allow-attended-actions` adds exact Reject, Teach, and Escalate tools for signed durable pauses. With a qualified Flow `--config`, the same server also exposes Continue and Skip through Flow's deployment-bound live verifier and deterministic resume path. - `openadapt-agent emit-skill --out ` wraps Flow's skill emitter and appends MCP, halt, and attended-action guidance. diff --git a/src/openadapt_agent/authoring.py b/src/openadapt_agent/authoring.py index 6c547ba..e02f11d 100644 --- a/src/openadapt_agent/authoring.py +++ b/src/openadapt_agent/authoring.py @@ -10,11 +10,19 @@ listener and does not implement a remote mailbox. Window titles, field values, screenshots, and backend pixels never cross the MCP wire. ``--authoring`` does not imply ``--allow-run``. + +Deps F1/C1/T1 are not required to be merged: Flow's ``AuthoringSession`` +is constructed when importable; Capture's projector is used when +importable; Types' observe schema is used when importable. Each is +fail-closed with a local fallback. """ from __future__ import annotations +import json import re +import sys +from pathlib import Path from typing import Any, Mapping, Optional from openadapt_agent.bridge import BridgeError, ToolSpec @@ -24,30 +32,51 @@ "AUTHORING_PROBE_TOOLS", "AuthoringBridge", "AuthoringError", + "MAX_AUTHORING_WIRE_BYTES", + "NODE_ID_RE", "OBSERVE_SCHEMA_VERSION", + "discover_desktop_authoring_ipc", "open_authoring_session", + "pin_local_backend", "project_observe", ] OBSERVE_SCHEMA_VERSION = "openadapt.authoring.observe/v1" try: - from openadapt_types.authoring import OBSERVE_SCHEMA_VERSION as _TYPES_OBSERVE + from openadapt_types.authoring import AUTHORING_OBSERVE_SCHEMA as _TYPES_OBSERVE except ImportError: - pass -else: - if isinstance(_TYPES_OBSERVE, str) and _TYPES_OBSERVE: - OBSERVE_SCHEMA_VERSION = _TYPES_OBSERVE + try: + from openadapt_types.authoring import OBSERVE_SCHEMA_VERSION as _TYPES_OBSERVE + except ImportError: + _TYPES_OBSERVE = None +if isinstance(_TYPES_OBSERVE, str) and _TYPES_OBSERVE: + OBSERVE_SCHEMA_VERSION = _TYPES_OBSERVE AUTHORING_PROBE_TOOLS = ("observe", "start_record", "click", "halt") AUTHORING_LOCAL_TOOLS = ( "type", - "stop_record", "pause_for_input", + "continue_input", + "stop_record", "compile", "get_command_result", + "set_coach", + "get_coach", + "bind_status", ) AUTHORING_TOOLS = AUTHORING_PROBE_TOOLS + AUTHORING_LOCAL_TOOLS +MAX_AUTHORING_NODES = 200 +MAX_AUTHORING_WIRE_BYTES = 32 * 1024 +NODE_ID_RE = re.compile(r"^n_[0-9a-f]{8}$") +_PROCESS_NAME = re.compile(r"^[A-Za-z0-9 ._-]{1,64}$") +_PROJECTED_LABEL = re.compile(r"^[A-Za-z0-9][A-Za-z0-9 ._-]{0,79}$") +_SIX_DIGITS = re.compile(r"\d{6,}") +_EMAIL = re.compile(r"[^@\s]+@[^@\s]+\.[^@\s]+") +_SSN = re.compile(r"\b\d{3}-\d{2}-\d{4}\b") +_PHONE = re.compile(r"\b(?:\+?\d[\d\-\s().]{7,}\d)\b") +_PARAM_NAME = re.compile(r"^[a-z][a-z0-9_]{0,31}$") + _EMPTY_OBJECT = { "type": "object", "properties": {}, @@ -90,23 +119,79 @@ } ) _BOUNDS_KEYS = frozenset({"x", "y", "w", "h"}) -_PROCESS_NAME = re.compile(r"^[A-Za-z0-9 ._-]{1,64}$") -_SIX_DIGITS = re.compile(r"\d{6,}") -_EMAIL = re.compile(r"[^@\s]+@[^@\s]+\.[^@\s]+") -_SSN = re.compile(r"\b\d{3}-\d{2}-\d{4}\b") -_PHONE = re.compile(r"\b(?:\+?\d[\d\-\s().]{7,}\d)\b") _RESULT_DROP = _FORBIDDEN_WIRE_KEYS | frozenset( {"execution_outcome", "success", "events", "frames", "before_png", "after_png"} ) -_CLOSED_BACKENDS = frozenset( - {"web", "macos", "linux", "windows", "rdp", "citrix", "unknown"} +_T1_BACKENDS = frozenset({"web", "macos", "linux", "windows", "rdp", "citrix"}) +_T1_PROVIDERS = frozenset( + {"playwright_ax", "macos_ax", "windows_uia", "linux_atspi", "none"} ) _COACH_ONLY_BACKENDS = frozenset({"windows", "rdp", "citrix"}) +_AGENT_DRIVE_BACKENDS = frozenset({"web", "macos", "linux"}) +_ELEMENT_ROLES = frozenset( + { + "button", + "text_input", + "text_static", + "label", + "link", + "checkbox", + "radio", + "combobox", + "list_item", + "menu", + "menu_item", + "tab", + "tree_item", + "image", + "icon", + "toolbar", + "scrollbar", + "slider", + "window", + "dialog", + "group", + "table", + "table_cell", + "table_row", + "heading", + "paragraph", + "unknown", + } +) +_OBSERVE_KEYS = ( + "schema_version", + "backend", + "provider", + "mode", + "agent_drive", + "coach_only", + "recording", + "window", + "tree", + "truncated", + "node_count", + "reason", +) +_NODE_KEYS = ( + "node_id", + "role", + "control_type", + "class_name", + "automation_id", + "name", + "enabled", + "focused", + "bounds", +) _CLICK_FIELDS = frozenset({"node_id", "x", "y"}) _TYPE_FIELDS = frozenset({"text", "param", "node_id"}) _PAUSE_FIELDS = frozenset({"node_id", "param", "secret"}) _RESULT_FIELDS = frozenset({"command_id"}) +_COACH_FIELDS = frozenset({"hint"}) + +_DESKTOP_IPC_RELATIVE = Path(".openadapt") / "desktop_ipc.json" _PROBE_HELP = { "observe": ( @@ -135,9 +220,184 @@ class AuthoringError(BridgeError): """Authoring tool refusal or missing Flow session.""" + def __init__(self, message: str, *, code: Optional[str] = None): + super().__init__(message) + self.code = code + + +class CoachOnlySession: + """Stdio stand-in when the pinned substrate is v1 coach-only. + + Observe works. Agent-drive (start_record / click / type) refuses. + Never constructs the in-guest Windows agent HTTP helper. + """ + + def __init__(self, backend_kind: str = "windows"): + kind = (backend_kind or "windows").strip().lower() + if kind in {"remote-display", "remote_display", "citrix"}: + kind = "rdp" if kind != "citrix" else "citrix" + if kind not in _COACH_ONLY_BACKENDS: + kind = "windows" + self.backend_kind = kind + + def observe(self) -> dict[str, Any]: + return { + "backend": self.backend_kind, + "provider": "none", + "agent_drive": False, + "coach_only": True, + "tree": [], + } + + def start_record(self) -> dict[str, Any]: + return {"error": "COACH_ONLY", "coach_only": True, "backend": self.backend_kind} + + def click(self, **kwargs: Any) -> dict[str, Any]: + return {"error": "COACH_ONLY"} + + def type_text(self, **kwargs: Any) -> dict[str, Any]: + return {"error": "COACH_ONLY"} + + def halt(self) -> dict[str, Any]: + return {"status": "halted"} + + +def discover_desktop_authoring_ipc(*, home: Optional[Path] = None) -> Optional[dict[str, Any]]: + """Return Desktop authoring IPC discovery when D2 has advertised it. + + Overlay and Allow stay Desktop-owned. This package does not open an HTTP + client or mailbox listener. Until D2 publishes an authoring endpoint in + ``~/.openadapt/desktop_ipc.json``, return None and pin a local Flow session. + """ + + root = Path.home() if home is None else Path(home) + path = root / _DESKTOP_IPC_RELATIVE + if not path.is_file(): + return None + try: + document = json.loads(path.read_text(encoding="utf-8")) + except (OSError, json.JSONDecodeError): + return None + if not isinstance(document, dict): + return None + endpoint = document.get("authoring") or document.get("authoring_ipc") + if not endpoint: + return None + return {"path": str(path), "endpoint": endpoint} + + +def pin_local_backend( + *, + url: Optional[str] = None, + headed: bool = False, + platform: Optional[str] = None, + backend: Any = None, + backend_kind: Optional[str] = None, +) -> tuple[Any, str, Any]: + """Pin the local target. Titles never go to MCP. + + Prefers Desktop loopback when D2 has advertised authoring IPC, but does + not speak that protocol from this MIT package. Otherwise: -def open_authoring_session(**kwargs: Any) -> object: - """Construct Flow's public authoring session when that module exists.""" + - ``url`` launches Playwright Chromium with empty cookies (no debug-port + attach, not the person's already-logged-in Chrome). + - Windows native / RDP / Citrix → coach-only; never the in-guest Windows + agent HTTP helper. + - macOS / Linux → unique frontmost window via Flow backends when those + constructors are importable (F1). Non-unique Linux titles are coach-only. + """ + + if backend is not None: + kind = backend_kind or "web" + return backend, kind, None + plat = platform or sys.platform + if url: + return _pin_web(url, headed=headed) + if plat == "win32" or plat.startswith("win"): + return None, "windows", None + native = _try_pin_native(plat) + if native is not None: + return native + raise AuthoringError( + "stdio --authoring needs a locally pinned window: pass --url for " + "Playwright Chromium with empty cookies, or run Desktop so overlay " + "stays single-owner. Native pin uses Flow backends after a unique " + "frontmost window (openadapt_flow.authoring / F1)" + ) + + +def _pin_web(url: str, *, headed: bool) -> tuple[Any, str, Any]: + try: + from openadapt_flow.backends.playwright_backend import PlaywrightBackend + except ImportError as exc: + raise AuthoringError( + "Playwright web pin requires openadapt-flow with the browser extra " + "(openadapt-agent[tutorial]); debug-port attach is out of v1" + ) from exc + launch = getattr(PlaywrightBackend, "launch", None) + if not callable(launch): + raise AuthoringError("PlaywrightBackend.launch is not available") + backend, close = launch(url, headless=not headed) + return backend, "web", close + + +def _try_pin_native(plat: str) -> Optional[tuple[Any, str, Any]]: + """Best-effort unique frontmost window. None means caller should fail closed.""" + + if plat == "darwin": + try: + from openadapt_flow.backends.macos_backend import MacOSBackend + from openadapt_flow.backends.remote_display import MacWindowClient + except ImportError: + return None + client = MacWindowClient() + window_id = client.frontmost_window_id() + if window_id is None: + return None + finder = getattr(client, "find_windows", None) + if not callable(finder): + return None + matches = [item for item in finder("", None) if getattr(item, "window_id", None) == window_id] + if len(matches) != 1: + return None + info = matches[0] + app = getattr(info, "owner", None) or getattr(info, "app", None) + title = getattr(info, "title", None) + if not isinstance(app, str) or not app: + return None + same_title = [ + item + for item in finder(app, title if isinstance(title, str) else None) + if getattr(item, "window_id", None) + ] + if title and len(same_title) != 1: + return None + return MacOSBackend(client, app=app, window_title=title), "macos", None + if plat.startswith("linux"): + # Factory requires linux_app AND an exact unique title. Non-unique is + # coach-only. Without a Desktop pin we cannot guess the app name. + return None + return None + + +def open_authoring_session( + *, + out_dir: Optional[Path | str] = None, + url: Optional[str] = None, + headed: bool = False, + backend: Any = None, + backend_kind: Optional[str] = None, + platform: Optional[str] = None, + **kwargs: Any, +) -> object: + """Construct Flow's public authoring session when that module exists. + + ``AuthoringSession`` (F1) requires ``backend``, ``out_dir``, and + ``backend_kind``. This helper pins locally (or returns a coach-only + stand-in for Windows native / Citrix / RDP) so ``serve --authoring`` + still works when F1 lands. Desktop IPC is preferred when advertised; + this package does not implement the D2 protocol. + """ try: from openadapt_flow import authoring as flow_authoring except ImportError as exc: @@ -147,16 +407,66 @@ def open_authoring_session(**kwargs: Any) -> object: "Recorder (compile returns needs_human_admit; Continue uses " "record_observed, never type_text on the pause target)" ) from exc + discover_desktop_authoring_ipc() + work_dir = Path(out_dir) if out_dir is not None else Path("runs") / "authoring" + close = None + kind = backend_kind + pinned = backend + if pinned is None and kind is None: + try: + pinned, kind, close = pin_local_backend( + url=url, headed=headed, platform=platform + ) + except AuthoringError: + plat = platform or sys.platform + if plat == "win32" or plat.startswith("win"): + pinned, kind, close = None, "windows", None + else: + raise + kind = kind or "web" + normalized = kind.strip().lower().replace("_", "-") + if normalized in {"remote-display", "citrix", "rdp", "windows", "win", "win-agent"}: + session = CoachOnlySession( + "citrix" if normalized == "citrix" else ("rdp" if normalized in {"rdp", "remote-display"} else "windows") + ) + if close is not None: + session.close = close # type: ignore[attr-defined] + return session opener = getattr(flow_authoring, "open_session", None) - if callable(opener): - return opener(**kwargs) session_cls = getattr(flow_authoring, "AuthoringSession", None) - if callable(session_cls): - return session_cls(**kwargs) - raise AuthoringError( - "openadapt_flow.authoring is importable but exposes neither " - "open_session nor AuthoringSession" - ) + try: + if callable(opener): + session = opener( + backend=pinned, + out_dir=work_dir, + backend_kind=kind, + app_url=url, + **kwargs, + ) + elif callable(session_cls): + session = session_cls( + pinned, + work_dir, + backend_kind=kind, + app_url=url, + **kwargs, + ) + else: + raise AuthoringError( + "openadapt_flow.authoring is importable but exposes neither " + "open_session nor AuthoringSession" + ) + except Exception as exc: + code = getattr(exc, "code", None) + if code == "COACH_ONLY" or type(exc).__name__ == "CoachOnlyError": + session = CoachOnlySession(kind) + else: + if isinstance(exc, AuthoringError): + raise + raise AuthoringError(str(exc), code=code if isinstance(code, str) else None) from exc + if close is not None: + session.close = close + return session def _safe_label(value: Any, *, process_name: bool = False) -> Optional[str]: @@ -165,7 +475,11 @@ def _safe_label(value: Any, *, process_name: bool = False) -> Optional[str]: collapsed = " ".join(value.split()) if not collapsed or len(collapsed) > 80: return None - if process_name and not _PROCESS_NAME.fullmatch(collapsed): + if process_name: + if not _PROCESS_NAME.fullmatch(collapsed): + return None + return collapsed + if not _PROJECTED_LABEL.fullmatch(collapsed): return None if "://" in collapsed or "@" in collapsed or _SIX_DIGITS.search(collapsed): return None @@ -182,68 +496,128 @@ def _bounds(value: Any) -> Optional[dict[str, float]]: raw = value.get(key) if isinstance(raw, bool) or not isinstance(raw, (int, float)): return None - out[key] = float(raw) - extra = set(value) - _BOUNDS_KEYS - if extra: - return out + number = float(raw) + if number != number or number in (float("inf"), float("-inf")): + return None + if number < 0 or number > 1: + return None + out[key] = number + if out["x"] + out["w"] > 1 + 1e-9 or out["y"] + out["h"] > 1 + 1e-9: + return None return out -def _project_window(value: Any) -> dict[str, Any]: +def _project_window(value: Any) -> Optional[dict[str, Any]]: if not isinstance(value, Mapping): - return {"role": "window"} - window: dict[str, Any] = {} + return None process_name = _safe_label(value.get("process_name"), process_name=True) - if process_name: - window["process_name"] = process_name - role = value.get("role") - window["role"] = role if isinstance(role, str) and role else "window" bounds = _bounds(value.get("bounds")) - if bounds is not None: - window["bounds"] = bounds - return window + if not process_name or bounds is None: + return None + return {"process_name": process_name, "role": "window", "bounds": bounds} def _project_node(value: Any) -> Optional[dict[str, Any]]: if not isinstance(value, Mapping): return None node_id = value.get("node_id") - if not isinstance(node_id, str) or not node_id: + if not isinstance(node_id, str) or not NODE_ID_RE.fullmatch(node_id): return None - node: dict[str, Any] = {"node_id": node_id} role = value.get("role") - if isinstance(role, str) and role: - node["role"] = role - control_type = value.get("control_type") - if isinstance(control_type, str) and control_type: - node["control_type"] = control_type - class_name = _safe_label(value.get("class_name")) - if class_name: - node["class_name"] = class_name[:64] - automation_id = _safe_label(value.get("automation_id")) - if automation_id: - node["automation_id"] = automation_id - name = _safe_label(value.get("name")) - if name: - node["name"] = name - if isinstance(value.get("enabled"), bool): - node["enabled"] = value["enabled"] - if isinstance(value.get("focused"), bool): - node["focused"] = value["focused"] + if role not in _ELEMENT_ROLES: + role = "unknown" + if not isinstance(value.get("enabled"), bool) or not isinstance(value.get("focused"), bool): + return None bounds = _bounds(value.get("bounds")) - if bounds is not None: - node["bounds"] = bounds - return node + if bounds is None: + return None + node: dict[str, Any] = { + "node_id": node_id, + "role": role, + "enabled": value["enabled"], + "focused": value["focused"], + "bounds": bounds, + } + for key in ("control_type", "class_name", "automation_id", "name"): + label = _safe_label(value.get(key)) + if label: + node[key] = label[:64] if key == "class_name" else label + return {key: node[key] for key in _NODE_KEYS if key in node} + + +def _empty_observe(*, backend: str, provider: str, coach_only: bool) -> dict[str, Any]: + return { + "schema_version": OBSERVE_SCHEMA_VERSION, + "backend": backend, + "provider": provider, + "mode": "authoring", + "agent_drive": False, + "coach_only": coach_only, + "recording": False, + "tree": [], + "truncated": False, + "node_count": 0, + "reason": "empty_projection", + } + + +def _trim_observe(payload: dict[str, Any]) -> dict[str, Any]: + blob = json.dumps(payload, separators=(",", ":")) + tree = list(payload.get("tree") or []) + truncated = payload.get("truncated") is True + while tree and len(blob.encode("utf-8")) > MAX_AUTHORING_WIRE_BYTES: + tree.pop() + truncated = True + payload["tree"] = tree + payload["node_count"] = len(tree) + payload["truncated"] = True + if not tree: + payload["reason"] = "empty_projection" + blob = json.dumps(payload, separators=(",", ":")) + payload["truncated"] = truncated + payload["node_count"] = len(payload.get("tree") or []) + return payload + + +def _validate_observe_types(payload: dict[str, Any]) -> dict[str, Any]: + try: + from openadapt_types.authoring import AuthoringObserveV1 + except ImportError: + return payload + try: + model = AuthoringObserveV1.model_validate(payload) + return json.loads(model.model_dump_json(exclude_none=True)) + except Exception: + backend = payload.get("backend") if payload.get("backend") in _T1_BACKENDS else "windows" + provider = payload.get("provider") if payload.get("provider") in _T1_PROVIDERS else "none" + coach_only = backend in _COACH_ONLY_BACKENDS + return _empty_observe(backend=backend, provider=provider, coach_only=coach_only) def project_observe(payload: Any) -> dict[str, Any]: - """Fail-closed PHI projection for ``openadapt.authoring.observe/v1``.""" + """Fail-closed PHI projection for ``openadapt.authoring.observe/v1``. + + extra keys, values, titles, screenshots, and backend pixels never appear + on the wire. Prefer Capture's projector when importable; always re-shape + to the T1 allowlist so a raw fallback cannot leak. + """ source = payload if isinstance(payload, Mapping) else {} backend = source.get("backend") - if backend not in _CLOSED_BACKENDS: - backend = "unknown" + if backend not in _T1_BACKENDS: + # T1 has no "unknown" backend. An unpinned substrate is not agent-drive. + backend = "windows" + source = {**source, "coach_only": True, "agent_drive": False, "tree": []} coach_only = backend in _COACH_ONLY_BACKENDS or source.get("coach_only") is True - agent_drive = (not coach_only) and source.get("agent_drive") is not False + provider = source.get("provider") + if provider not in _T1_PROVIDERS: + provider = "none" + window = _project_window(source.get("window")) + agent_drive = ( + (not coach_only) + and backend in _AGENT_DRIVE_BACKENDS + and source.get("agent_drive") is not False + and window is not None + ) if coach_only: agent_drive = False tree_in = source.get("tree") @@ -253,32 +627,30 @@ def project_observe(payload: Any) -> dict[str, Any]: node = _project_node(item) if node is not None: nodes.append(node) - if len(nodes) >= 200: + if len(nodes) >= MAX_AUTHORING_NODES: break projected: dict[str, Any] = { "schema_version": OBSERVE_SCHEMA_VERSION, "backend": backend, - "provider": ( - source.get("provider") if isinstance(source.get("provider"), str) else "unknown" - ), + "provider": provider, "mode": "authoring", "agent_drive": agent_drive, "coach_only": coach_only, "recording": source.get("recording") is True, - "window": _project_window(source.get("window")), "tree": nodes, - "truncated": source.get("truncated") is True or ( - isinstance(tree_in, list) and len(tree_in) > 200 - ), + "truncated": source.get("truncated") is True + or (isinstance(tree_in, list) and len(tree_in) > MAX_AUTHORING_NODES), "node_count": len(nodes), } + if window is not None: + projected["window"] = window if not nodes: + reason = source.get("reason") projected["reason"] = ( - source.get("reason") - if isinstance(source.get("reason"), str) and source.get("reason") - else "empty_projection" + reason if reason == "empty_projection" else "empty_projection" ) - return projected + projected = {key: projected[key] for key in _OBSERVE_KEYS if key in projected} + return _trim_observe(_validate_observe_types(_trim_observe(projected))) def _public_result(payload: Any) -> dict[str, Any]: @@ -288,7 +660,7 @@ def _public_result(payload: Any) -> dict[str, Any]: for key, value in payload.items(): if key in _RESULT_DROP or key in _FORBIDDEN_WIRE_KEYS: continue - if key == "execution_outcome": + if key in {"execution_outcome", "success"}: continue if isinstance(value, Mapping): nested = _public_result(value) @@ -297,7 +669,7 @@ def _public_result(payload: Any) -> dict[str, Any]: continue if isinstance(value, list): continue - if key == "success": + if key in {"path", "file_path"}: continue out[key] = value return out @@ -316,6 +688,7 @@ def _invoke(session: object, method: str, **kwargs: Any) -> Any: aliases = { "type_agent": ("type_text", "type"), "pause_for_input": ("pause",), + "continue_input": ("continue_pause", "record_observed"), "stop_record": ("finish", "stop"), "start_record": ("start",), } @@ -330,17 +703,34 @@ def _invoke(session: object, method: str, **kwargs: Any) -> Any: try: return func(**kwargs) if kwargs else func() except TypeError: - return func(kwargs) if kwargs else func() + if "node_id" in kwargs: + retry = dict(kwargs) + retry.pop("node_id", None) + try: + return func(**retry) if retry else func() + except TypeError: + pass + if kwargs: + raise + return func() + + +def _session_code(exc: BaseException) -> Optional[str]: + code = getattr(exc, "code", None) + return code if isinstance(code, str) else None class AuthoringBridge: """Stdio authoring tool specs and dispatch over a session object.""" - def __init__(self, session: object): + def __init__(self, session: object, *, out_dir: Optional[Path | str] = None): self.session = session + self._out_dir = Path(out_dir) if out_dir is not None else None self._last_tool: Optional[str] = None self._last_result: Optional[dict[str, Any]] = None - self._coach_only = False + self._coach_only = isinstance(session, CoachOnlySession) + self._nodes: dict[str, dict[str, Any]] = {} + self._coach_hint: Optional[str] = None def handles(self, name: str) -> bool: return name in AUTHORING_TOOLS @@ -421,10 +811,12 @@ def list_tool_specs(self) -> list[ToolSpec]: ToolSpec( name="pause_for_input", description=( - "Pause so a person can type in the application. On Continue, " - "persist with Recorder.record_observed on the pause-target " - "node. Never call type_text for that human input. Secret " - "pauses store no text. The MCP result has no value." + "Pause so a person can type in the application. Capture the " + "pause-target at pause start. On Continue, persist with " + "Recorder.record_observed. Never call type_text for that " + "human input. Secret pauses store no text. The MCP result " + "has no value. Local stdio Continue is continue_input " + "(overlay Continue when Desktop owns the session)." ), input_schema={ "type": "object", @@ -446,6 +838,17 @@ def list_tool_specs(self) -> list[ToolSpec]: }, annotations=_MUTATING, ), + ToolSpec( + name="continue_input", + description=( + "Stdio stand-in for overlay Continue after pause_for_input. " + "Persists with Recorder.record_observed on the pause-target " + "node, never type_text. Hosted MCP has no continue_input " + "tool; Desktop overlay Continue owns that path." + ), + input_schema=_EMPTY_OBJECT, + annotations=_MUTATING, + ), ToolSpec( name="stop_record", description=( @@ -489,6 +892,41 @@ def list_tool_specs(self) -> list[ToolSpec]: }, annotations=_READ_ONLY, ), + ToolSpec( + name="set_coach", + description=( + "Local coach hint (80 characters; no URL, @, or six-digit " + "runs). Desktop overlay owns hosted set_coach. Stdio stores " + "the hint in-process when Desktop IPC is not advertised." + ), + input_schema={ + "type": "object", + "properties": { + "hint": { + "type": "string", + "description": "PHI-safe coach hint for the local overlay.", + } + }, + "required": ["hint"], + "additionalProperties": False, + }, + annotations=_MUTATING, + ), + ToolSpec( + name="get_coach", + description="Return the last local coach hint. No tree.", + input_schema=_EMPTY_OBJECT, + annotations=_READ_ONLY, + ), + ToolSpec( + name="bind_status", + description=( + "Local stdio bind status. This process is already on the " + "machine; there is no pack mailbox. No tree." + ), + input_schema=_EMPTY_OBJECT, + annotations=_READ_ONLY, + ), ] return specs @@ -505,8 +943,12 @@ def dispatch(self, name: str, arguments: Optional[dict[str, Any]] = None) -> dic "halt": self._halt, "type": self._type_agent, "pause_for_input": self._pause_for_input, + "continue_input": self._continue_input, "stop_record": self._stop_record, "compile": self._compile, + "set_coach": self._set_coach, + "get_coach": self._get_coach, + "bind_status": self._bind_status, } result = handlers[name](arguments) self._last_tool = name @@ -528,9 +970,75 @@ def _last_command_result(self) -> dict[str, Any]: "result": dict(self._last_result), } + def _map_session_error(self, exc: BaseException) -> dict[str, Any]: + code = _session_code(exc) + if code == "stale_node": + return {"status": "error", "error": "stale_node"} + if code == "missing_secret_type": + return {"status": "error", "error": "missing_secret_type"} + if code == "COACH_ONLY" or type(exc).__name__ == "CoachOnlyError": + self._coach_only = True + raise AuthoringError( + "refused: COACH_ONLY (person actuates; this backend is " + "not agent-drive in v1)", + code="COACH_ONLY", + ) from exc + if isinstance(exc, AuthoringError): + raise exc + raise AuthoringError(str(exc), code=code) from exc + + def _call(self, method: str, **kwargs: Any) -> Any: + try: + return _invoke(self.session, method, **kwargs) + except AuthoringError: + raise + except Exception as exc: + mapped = self._map_session_error(exc) + if mapped.get("error"): + return mapped + raise + + def _remember_nodes(self, raw: Any) -> None: + if not isinstance(raw, Mapping): + return + tree = raw.get("tree") + if not isinstance(tree, list): + return + remember = getattr(self.session, "remember_node", None) + for item in tree: + if not isinstance(item, Mapping): + continue + node_id = item.get("node_id") + pixels = item.get("backend_pixels") + if not isinstance(node_id, str) or not NODE_ID_RE.fullmatch(node_id): + continue + if isinstance(pixels, Mapping): + stored = {key: pixels[key] for key in ("x", "y", "w", "h") if key in pixels} + self._nodes[node_id] = stored + if callable(remember): + remember(node_id, stored) + def _observe(self, arguments: Optional[dict[str, Any]]) -> dict[str, Any]: _require_object(arguments, set()) - projected = project_observe(_invoke(self.session, "observe")) + raw: Any = None + observe = getattr(self.session, "observe", None) + if callable(observe): + try: + raw = observe() + except Exception as exc: + mapped = self._map_session_error(exc) + if mapped.get("error"): + return mapped + raise + self._remember_nodes(raw) + projected = project_observe( + raw + if isinstance(raw, Mapping) + else { + "backend": getattr(self.session, "backend_kind", None) or getattr(self.session, "backend", "web"), + "coach_only": self._coach_only, + } + ) self._coach_only = projected.get("coach_only") is True return projected @@ -538,21 +1046,31 @@ def _refuse_coach_only(self, tool: str) -> None: if self._coach_only: raise AuthoringError( f"{tool} refused: COACH_ONLY (person actuates; this backend is " - "not agent-drive in v1)" + "not agent-drive in v1)", + code="COACH_ONLY", ) def _start_record(self, arguments: Optional[dict[str, Any]]) -> dict[str, Any]: _require_object(arguments, set()) self._refuse_coach_only("start_record") - raw = _invoke(self.session, "start_record") - result = _public_result(raw) - if result.get("error") == "COACH_ONLY" or result.get("coach_only") is True: + raw = self._call("start_record") + if isinstance(raw, Mapping) and ( + raw.get("error") == "COACH_ONLY" or raw.get("coach_only") is True + ): self._coach_only = True raise AuthoringError( "start_record refused: COACH_ONLY (person actuates; this backend " - "is not agent-drive in v1)" + "is not agent-drive in v1)", + code="COACH_ONLY", ) + result = _public_result(raw) + if result.get("status") == "ok" and raw is None: + return {"status": "recording"} + if raw is None: + return {"status": "recording"} result.setdefault("status", "recording") + if result.get("status") == "ok": + result["status"] = "recording" return result def _click(self, arguments: Optional[dict[str, Any]]) -> dict[str, Any]: @@ -563,36 +1081,54 @@ def _click(self, arguments: Optional[dict[str, Any]]) -> dict[str, Any]: y = payload.get("y") has_node = isinstance(node_id, str) and bool(node_id) has_point = x is not None or y is not None - if has_point and (not isinstance(x, int) or isinstance(x, bool) or - not isinstance(y, int) or isinstance(y, bool)): + if has_point and ( + not isinstance(x, int) + or isinstance(x, bool) + or not isinstance(y, int) + or isinstance(y, bool) + ): raise AuthoringError("click x and y must both be integers") if not has_node and not has_point: raise AuthoringError("click requires node_id or local x and y") kwargs: dict[str, Any] = {} if has_node: kwargs["node_id"] = node_id + if node_id not in self._nodes and not NODE_ID_RE.fullmatch(str(node_id)): + return {"status": "error", "error": "stale_node"} if has_point: kwargs["x"] = x kwargs["y"] = y - raw = _invoke(self.session, "click", **kwargs) + raw = self._call("click", **kwargs) + if isinstance(raw, Mapping) and raw.get("error") in {"stale_node", "COACH_ONLY"}: + if raw.get("error") == "stale_node" or raw.get("status") == "stale_node": + return {"status": "error", "error": "stale_node"} + raise AuthoringError( + "click refused: COACH_ONLY (person actuates; this backend is " + "not agent-drive in v1)", + code="COACH_ONLY", + ) result = _public_result(raw) if result.get("error") == "stale_node" or result.get("status") == "stale_node": return {"status": "error", "error": "stale_node"} if result.get("error") == "COACH_ONLY": raise AuthoringError( "click refused: COACH_ONLY (person actuates; this backend is " - "not agent-drive in v1)" + "not agent-drive in v1)", + code="COACH_ONLY", ) result.setdefault("status", "ok") if has_node: result.setdefault("node_id", node_id) + result.pop("backend_pixels", None) return result def _halt(self, arguments: Optional[dict[str, Any]]) -> dict[str, Any]: _require_object(arguments, set()) - raw = _invoke(self.session, "halt") + raw = self._call("halt") result = _public_result(raw) result.setdefault("status", "halted") + if result.get("status") == "ok": + result["status"] = "halted" result["compiled"] = False return result @@ -605,7 +1141,7 @@ def _type_agent(self, arguments: Optional[dict[str, Any]]) -> dict[str, Any]: kwargs: dict[str, Any] = {"text": text} param = payload.get("param") if param is not None: - if not isinstance(param, str) or not param: + if not isinstance(param, str) or not _PARAM_NAME.fullmatch(param): raise AuthoringError("param must be a string") kwargs["param"] = param node_id = payload.get("node_id") @@ -613,7 +1149,7 @@ def _type_agent(self, arguments: Optional[dict[str, Any]]) -> dict[str, Any]: if not isinstance(node_id, str) or not node_id: raise AuthoringError("node_id must be a string") kwargs["node_id"] = node_id - raw = _invoke(self.session, "type_agent", **kwargs) + raw = self._call("type_agent", **kwargs) result = _public_result(raw) result.pop("text", None) result.setdefault("status", "ok") @@ -632,7 +1168,7 @@ def _pause_for_input(self, arguments: Optional[dict[str, Any]]) -> dict[str, Any kwargs["node_id"] = node_id param = payload.get("param") if param is not None: - if not isinstance(param, str) or not param: + if not isinstance(param, str) or not _PARAM_NAME.fullmatch(param): raise AuthoringError("param must be a string") kwargs["param"] = param secret = payload.get("secret") @@ -640,7 +1176,16 @@ def _pause_for_input(self, arguments: Optional[dict[str, Any]]) -> dict[str, Any if not isinstance(secret, bool): raise AuthoringError("secret must be a boolean") kwargs["secret"] = secret - raw = _invoke(self.session, "pause_for_input", **kwargs) + raw = self._call("pause_for_input", **kwargs) + if callable(getattr(self.session, "continue_input", None)) and not ( + isinstance(raw, Mapping) and "recorded" in raw + ): + result = {"status": "paused"} + if param: + result["param"] = param + if secret is True: + result["secret"] = True + return result result = _public_result(raw) result.pop("text", None) result.pop("value", None) @@ -651,22 +1196,47 @@ def _pause_for_input(self, arguments: Optional[dict[str, Any]]) -> dict[str, Any result["secret"] = True return result + def _continue_input(self, arguments: Optional[dict[str, Any]]) -> dict[str, Any]: + _require_object(arguments, set()) + raw = self._call("continue_input") + result = _public_result(raw) + result.pop("text", None) + result.pop("value", None) + if "recorded" not in result: + result["recorded"] = True + return result + def _stop_record(self, arguments: Optional[dict[str, Any]]) -> dict[str, Any]: _require_object(arguments, set()) - raw = _invoke(self.session, "stop_record") + raw = self._call("stop_record") result = _public_result(raw) result.setdefault("status", "stopped") + if result.get("status") == "ok": + result["status"] = "stopped" result["compiled"] = False return result def _compile(self, arguments: Optional[dict[str, Any]]) -> dict[str, Any]: _require_object(arguments, set()) - raw = _invoke(self.session, "compile") + bundle_dir = (self._out_dir or Path("runs") / "authoring") / "bundle" + try: + compile_fn = getattr(self.session, "compile", None) + if not callable(compile_fn): + raise AuthoringError("authoring session does not implement compile") + try: + raw = compile_fn(bundle_dir, name="authoring") + except TypeError: + raw = compile_fn() + except Exception as exc: + mapped = self._map_session_error(exc) + if mapped.get("error"): + return mapped + raise if isinstance(raw, Mapping) and raw.get("error") == "missing_secret_type": return {"status": "error", "error": "missing_secret_type"} result = _public_result(raw) - if result.get("status") == "error": - return result + if result.get("status") == "error" or result.get("error") == "missing_secret_type": + return {"status": "error", "error": result.get("error") or "missing_secret_type"} workflow_id = result.get("workflow_id") public = { "status": "needs_human_admit", @@ -675,3 +1245,37 @@ def _compile(self, arguments: Optional[dict[str, Any]]) -> dict[str, Any]: if isinstance(workflow_id, str) and workflow_id: public["workflow_id"] = workflow_id return public + + def _set_coach(self, arguments: Optional[dict[str, Any]]) -> dict[str, Any]: + payload = _require_object(arguments, _COACH_FIELDS) + hint = _safe_label(payload.get("hint")) + if hint is None: + raise AuthoringError("coach hint failed the PHI filter") + self._coach_hint = hint + setter = getattr(self.session, "set_coach", None) + if callable(setter): + try: + setter(hint=hint) + except TypeError: + setter(hint) + return {"status": "ok", "hint": hint} + + def _get_coach(self, arguments: Optional[dict[str, Any]]) -> dict[str, Any]: + _require_object(arguments, set()) + getter = getattr(self.session, "get_coach", None) + if callable(getter): + raw = getter() + if isinstance(raw, Mapping): + hint = _safe_label(raw.get("hint")) + return {"hint": hint} + if isinstance(raw, str): + return {"hint": _safe_label(raw)} + return {"hint": self._coach_hint} + + def _bind_status(self, arguments: Optional[dict[str, Any]]) -> dict[str, Any]: + _require_object(arguments, set()) + return { + "status": "stdio", + "bound": True, + "transport": "stdio", + } diff --git a/src/openadapt_agent/cli.py b/src/openadapt_agent/cli.py index 82dce82..8c7364a 100644 --- a/src/openadapt_agent/cli.py +++ b/src/openadapt_agent/cli.py @@ -60,9 +60,11 @@ def build_parser() -> argparse.ArgumentParser: action="store_true", help=( "Register first-demo authoring tools over local stdio: observe, " - "start_record, click, halt. Local stdio may also type through the " - "recorder; hosted MCP remains pause-only. Does not enable run " - "tools. This process stays stdio and must not be served over HTTP." + "start_record, click, halt. Local Claude Code path is the first " + "authoring UI. Pass --url to pin Playwright Chromium with empty " + "cookies. Local stdio may also type through the recorder; hosted " + "MCP remains pause-only. Does not enable run tools. This process " + "stays stdio and must not be served over HTTP." ), ) p.add_argument( @@ -270,8 +272,16 @@ def _cmd_serve(args: argparse.Namespace) -> int: from openadapt_agent.authoring import AuthoringBridge, AuthoringError from openadapt_agent.authoring import open_authoring_session + authoring_dir = Path(args.runs_dir).expanduser().resolve() / "authoring" try: - authoring_bridge = AuthoringBridge(open_authoring_session()) + authoring_bridge = AuthoringBridge( + open_authoring_session( + out_dir=authoring_dir, + url=args.url, + headed=args.headed, + ), + out_dir=authoring_dir, + ) except AuthoringError as exc: print(f"serve: {exc}", file=sys.stderr) return 2 @@ -355,6 +365,10 @@ def _cmd_serve(args: argparse.Namespace) -> int: finally: if tutorial_session is not None: tutorial_session.close() + if authoring_bridge is not None: + closer = getattr(authoring_bridge.session, "close", None) + if callable(closer): + closer() return 0 diff --git a/tests/test_authoring.py b/tests/test_authoring.py index 148ba6b..c981094 100644 --- a/tests/test_authoring.py +++ b/tests/test_authoring.py @@ -13,7 +13,11 @@ AUTHORING_PROBE_TOOLS, AuthoringBridge, AuthoringError, + CoachOnlySession, + MAX_AUTHORING_WIRE_BYTES, + discover_desktop_authoring_ipc, open_authoring_session, + pin_local_backend, project_observe, ) from openadapt_agent.mcp import build_server @@ -38,8 +42,8 @@ def __init__( self.recording = False self._secret_type_recorded = False self.nodes = { - "n_9f2c": { - "node_id": "n_9f2c", + "n_9f2c001a": { + "node_id": "n_9f2c001a", "role": "button", "control_type": "button", "automation_id": "btnContinue", @@ -147,7 +151,7 @@ def compile(self): return {"error": "missing_secret_type"} return { "status": "needs_human_admit", - "workflow_id": "wf_demo", + "workflow_id": "wf_demo0001", "execution_outcome": "VERIFIED", "success": True, } @@ -177,7 +181,7 @@ def test_observe_drops_values_titles_screenshots_and_unsafe_names(): assert "123-45-6789" not in blob assert "patient@clinic.example" not in blob assert "https://" not in blob - node = next(item for item in result["tree"] if item["node_id"] == "n_9f2c") + node = next(item for item in result["tree"] if item["node_id"] == "n_9f2c001a") assert node["automation_id"] == "btnContinue" assert node["name"] == "Save" assert all(item["node_id"] != "n_ssn1" or "name" not in item for item in result["tree"]) @@ -194,7 +198,7 @@ def test_start_record_and_click_and_halt_round_trip(): session = FakeAuthoringSession() bridge = AuthoringBridge(session) assert bridge.dispatch("start_record", {})["status"] == "recording" - clicked = bridge.dispatch("click", {"node_id": "n_9f2c"}) + clicked = bridge.dispatch("click", {"node_id": "n_9f2c001a"}) assert clicked["status"] == "ok" assert "backend_pixels" not in clicked assert bridge.dispatch("click", {"node_id": "n_missing"}) == { @@ -206,7 +210,7 @@ def test_start_record_and_click_and_halt_round_trip(): halted = bridge.dispatch("halt", {}) assert halted["status"] == "halted" assert halted["compiled"] is False - assert ("click", "n_9f2c", None, None) in session.calls + assert ("click", "n_9f2c001a", None, None) in session.calls assert ("click", None, 10, 20) in session.calls @@ -226,7 +230,7 @@ def test_pause_continue_uses_record_observed_never_type_text(): bridge = AuthoringBridge(session) result = bridge.dispatch( "pause_for_input", - {"node_id": "n_9f2c", "param": "note", "secret": False}, + {"node_id": "n_9f2c001a", "param": "note", "secret": False}, ) assert result == {"recorded": True, "param": "note"} assert session.typed_via_backend == [] @@ -240,7 +244,7 @@ def test_secret_pause_result_has_no_value_and_compile_can_refuse(): bridge = AuthoringBridge(session) paused = bridge.dispatch( "pause_for_input", - {"node_id": "n_9f2c", "param": "identifier", "secret": True}, + {"node_id": "n_9f2c001a", "param": "identifier", "secret": True}, ) assert paused["recorded"] is True assert paused["secret"] is True @@ -255,7 +259,7 @@ def test_compile_returns_needs_human_admit_never_verified(): bridge = AuthoringBridge(FakeAuthoringSession()) result = bridge.dispatch("compile", {}) assert result["status"] == "needs_human_admit" - assert result["workflow_id"] == "wf_demo" + assert result["workflow_id"] == "wf_demo0001" assert result["recording_retained"] is True assert "VERIFIED" not in json.dumps(result) assert "success" not in result @@ -271,7 +275,7 @@ def test_windows_native_is_coach_only(): with pytest.raises(AuthoringError, match="COACH_ONLY"): bridge.dispatch("start_record", {}) with pytest.raises(AuthoringError, match="COACH_ONLY"): - bridge.dispatch("click", {"node_id": "n_9f2c"}) + bridge.dispatch("click", {"node_id": "n_9f2c001a"}) def test_get_command_result_returns_last_in_process_result(): @@ -363,3 +367,338 @@ def test_authoring_sources_stay_stdio_without_http_listener(): assert "uvicorn" not in text assert "streamable_http" not in text.lower() assert "FastAPI" not in text + assert "win_agent" not in text + assert "parallels_vm" not in text + + +def test_observe_wire_omits_extra_keys_and_invalid_node_ids(): + projected = project_observe( + { + "backend": "web", + "provider": "playwright_ax", + "value": "nope", + "title": "Chart", + "screenshot": "px", + "window": { + "process_name": "Chromium", + "role": "window", + "title": "secret", + "bounds": {"x": 0, "y": 0, "w": 1, "h": 1, "pixels": 9}, + }, + "tree": [ + { + "node_id": "n_9f2c001a", + "role": "button", + "enabled": True, + "focused": False, + "bounds": {"x": 0.1, "y": 0.1, "w": 0.1, "h": 0.1}, + "value": "4111", + "backend_pixels": {"x": 1, "y": 2, "w": 3, "h": 4}, + }, + {"node_id": "n_9f2c", "role": "button", "enabled": True, "focused": False}, + ], + } + ) + blob = json.dumps(projected) + assert "value" not in blob + assert "title" not in blob + assert "screenshot" not in blob + assert "backend_pixels" not in blob + assert projected["tree"][0]["node_id"] == "n_9f2c001a" + assert all(item["node_id"] != "n_9f2c" for item in projected["tree"]) + assert set(projected) <= { + "schema_version", + "backend", + "provider", + "mode", + "agent_drive", + "coach_only", + "recording", + "window", + "tree", + "truncated", + "node_count", + "reason", + } + + +def test_invalid_backend_is_coach_only_not_unknown(): + projected = project_observe({"backend": "turbo", "tree": [{"node_id": "n_9f2c001a"}]}) + assert projected["backend"] == "windows" + assert projected["coach_only"] is True + assert projected["agent_drive"] is False + assert projected["provider"] == "none" + assert projected["tree"] == [] + assert "unknown" not in json.dumps(projected) + + +def test_observe_wire_truncates_to_32kib(monkeypatch): + monkeypatch.setattr( + "openadapt_agent.authoring.MAX_AUTHORING_WIRE_BYTES", 800 + ) + tree = [] + for index in range(40): + tree.append( + { + "node_id": f"n_{index:08x}", + "role": "button", + "name": "SaveButtonLabelOk", + "enabled": True, + "focused": False, + "bounds": {"x": 0.01, "y": 0.01, "w": 0.1, "h": 0.1}, + } + ) + projected = project_observe( + { + "backend": "web", + "provider": "playwright_ax", + "window": { + "process_name": "Chromium", + "bounds": {"x": 0, "y": 0, "w": 1, "h": 1}, + }, + "tree": tree, + } + ) + assert projected["truncated"] is True + assert len(json.dumps(projected, separators=(",", ":")).encode("utf-8")) <= 800 + assert MAX_AUTHORING_WIRE_BYTES == 32 * 1024 + + +def test_remembered_pixels_stay_off_the_wire(): + session = FakeAuthoringSession() + bridge = AuthoringBridge(session) + observed = bridge.dispatch("observe", {}) + assert "backend_pixels" not in json.dumps(observed) + assert "n_9f2c001a" in bridge._nodes + + +class _FlowError(RuntimeError): + def __init__(self, message, *, code): + super().__init__(message) + self.code = code + + +class FlowShapedSession: + """Matches F1 AuthoringSession: no observe, type_text, continue_input, compile(dir, name=).""" + + def __init__(self): + self.backend_kind = "web" + self.calls = [] + self.nodes = {} + self.typed_via_backend = [] + self.paused = None + self.observed_events = [] + + def remember_node(self, node_id, backend_pixels): + self.nodes[node_id] = dict(backend_pixels) + + def start_record(self): + self.calls.append("start_record") + return None + + def click(self, x=None, y=None, *, node_id=None): + self.calls.append(("click", node_id, x, y)) + if node_id is not None and node_id not in self.nodes: + raise _FlowError("unknown node", code="stale_node") + return None + + def type_text(self, text, param=None): + self.calls.append(("type_text", text, param)) + self.typed_via_backend.append(text) + return None + + def pause_for_input(self, *, param, secret=False, node_id=None, backend_pixels=None): + self.calls.append(("pause_for_input", node_id, param, secret)) + self.paused = {"param": param, "secret": secret, "node_id": node_id} + return None + + def continue_input(self, *, operator_confirmed=True): + self.calls.append(("continue_input", operator_confirmed)) + if self.paused is None: + raise _FlowError("no pause", code="not_paused") + event = { + "kind": "type", + "param": self.paused["param"], + "secret": self.paused["secret"], + } + if not self.paused["secret"]: + event["text"] = "synthetic follow-up" + self.observed_events.append(event) + param = self.paused["param"] + self.paused = None + return {"recorded": True, "param": param} + + def halt(self): + self.calls.append("halt") + return None + + def compile(self, bundle_dir, *, name): + self.calls.append(("compile", str(bundle_dir), name)) + return { + "status": "needs_human_admit", + "workflow_id": "wf_demo0001", + "recording_retained": True, + "execution_outcome": "VERIFIED", + } + + +def test_flow_shaped_session_maps_f1_signatures_and_stale_node(): + session = FlowShapedSession() + bridge = AuthoringBridge(session, out_dir="/tmp/authoring-out") + assert bridge.dispatch("start_record", {}) == {"status": "recording"} + session.remember_node("n_9f2c001a", {"x": 10, "y": 20, "w": 4, "h": 4}) + assert bridge.dispatch("click", {"node_id": "n_9f2c001a"})["status"] == "ok" + assert bridge.dispatch("click", {"node_id": "n_deadbeef"}) == { + "status": "error", + "error": "stale_node", + } + typed = bridge.dispatch("type", {"text": "hello", "param": "note", "node_id": "n_9f2c001a"}) + assert typed["recorded"] is True + assert "text" not in typed + assert session.typed_via_backend == ["hello"] + paused = bridge.dispatch( + "pause_for_input", + {"node_id": "n_9f2c001a", "param": "note", "secret": False}, + ) + assert paused == {"status": "paused", "param": "note"} + assert session.observed_events == [] + continued = bridge.dispatch("continue_input", {}) + assert continued == {"recorded": True, "param": "note"} + assert session.observed_events == [ + {"kind": "type", "param": "note", "secret": False, "text": "synthetic follow-up"} + ] + compiled = bridge.dispatch("compile", {}) + assert compiled["status"] == "needs_human_admit" + assert compiled["workflow_id"] == "wf_demo0001" + assert "VERIFIED" not in json.dumps(compiled) + assert session.calls[-1][0] == "compile" + assert session.calls[-1][2] == "authoring" + + +def test_coach_hint_and_bind_status_are_local(): + bridge = AuthoringBridge(FakeAuthoringSession()) + with pytest.raises(AuthoringError, match="PHI"): + bridge.dispatch("set_coach", {"hint": "open https://example.invalid"}) + with pytest.raises(AuthoringError, match="PHI"): + bridge.dispatch("set_coach", {"hint": "call 1234567 now"}) + set_ok = bridge.dispatch("set_coach", {"hint": "Click Save"}) + assert set_ok == {"status": "ok", "hint": "Click Save"} + assert bridge.dispatch("get_coach", {}) == {"hint": "Click Save"} + status = bridge.dispatch("bind_status", {}) + assert status == {"status": "stdio", "bound": True, "transport": "stdio"} + assert "tree" not in status + names = [spec.name for spec in bridge.list_tool_specs()] + assert names[:4] == list(AUTHORING_PROBE_TOOLS) + assert "continue_input" in names + assert "set_coach" in names + assert "bind_status" in names + + +def test_desktop_ipc_discovery_does_not_open_a_listener(tmp_path, monkeypatch): + monkeypatch.setattr("openadapt_agent.authoring.Path.home", lambda: tmp_path) + assert discover_desktop_authoring_ipc() is None + ipc_dir = tmp_path / ".openadapt" + ipc_dir.mkdir() + (ipc_dir / "desktop_ipc.json").write_text( + json.dumps({"authoring": {"host": "127.0.0.1", "port": 9}}), + encoding="utf-8", + ) + found = discover_desktop_authoring_ipc() + assert found is not None + assert found["endpoint"]["host"] == "127.0.0.1" + source = ( + Path(__file__).resolve().parents[1] + / "src" + / "openadapt_agent" + / "authoring.py" + ).read_text(encoding="utf-8") + assert "http.client" not in source + assert "urlopen" not in source + + +def test_pin_windows_is_coach_only_without_win_agent(): + backend, kind, close = pin_local_backend(platform="win32") + assert backend is None + assert kind == "windows" + assert close is None + session = CoachOnlySession("windows") + bridge = AuthoringBridge(session) + observed = bridge.dispatch("observe", {}) + assert observed["coach_only"] is True + assert observed["agent_drive"] is False + with pytest.raises(AuthoringError, match="COACH_ONLY"): + bridge.dispatch("start_record", {}) + + +def test_open_authoring_session_uses_f1_constructor_when_present(monkeypatch, tmp_path): + captured = {} + + class Session: + def __init__(self, backend, out_dir, *, backend_kind, app_url=None, **kwargs): + captured["backend"] = backend + captured["out_dir"] = Path(out_dir) + captured["backend_kind"] = backend_kind + captured["app_url"] = app_url + + import types + import sys + + fake_flow = types.ModuleType("openadapt_flow") + fake_authoring = types.ModuleType("openadapt_flow.authoring") + fake_authoring.AuthoringSession = Session + monkeypatch.setitem(sys.modules, "openadapt_flow", fake_flow) + monkeypatch.setitem(sys.modules, "openadapt_flow.authoring", fake_authoring) + monkeypatch.setattr( + "openadapt_agent.authoring.pin_local_backend", + lambda **kwargs: ("backend-obj", "web", None), + ) + session = open_authoring_session(out_dir=tmp_path, url="https://example.invalid/") + assert isinstance(session, Session) + assert captured["backend"] == "backend-obj" + assert captured["backend_kind"] == "web" + assert captured["app_url"] == "https://example.invalid/" + assert captured["out_dir"] == tmp_path + + +def test_open_authoring_session_windows_never_constructs_flow_session(monkeypatch, tmp_path): + class Session: + def __init__(self, *args, **kwargs): + raise AssertionError("must not construct AuthoringSession for windows") + + import types + import sys + + fake_flow = types.ModuleType("openadapt_flow") + fake_authoring = types.ModuleType("openadapt_flow.authoring") + fake_authoring.AuthoringSession = Session + monkeypatch.setitem(sys.modules, "openadapt_flow", fake_flow) + monkeypatch.setitem(sys.modules, "openadapt_flow.authoring", fake_authoring) + session = open_authoring_session( + out_dir=tmp_path, backend=object(), backend_kind="windows" + ) + assert isinstance(session, CoachOnlySession) + assert session.backend_kind == "windows" + + +def test_open_authoring_session_maps_f1_coach_only_error(monkeypatch, tmp_path): + class CoachOnlyError(RuntimeError): + def __init__(self): + super().__init__("COACH_ONLY") + self.code = "COACH_ONLY" + + class Session: + def __init__(self, *args, **kwargs): + raise CoachOnlyError() + + import types + import sys + + fake_flow = types.ModuleType("openadapt_flow") + fake_authoring = types.ModuleType("openadapt_flow.authoring") + fake_authoring.AuthoringSession = Session + monkeypatch.setitem(sys.modules, "openadapt_flow", fake_flow) + monkeypatch.setitem(sys.modules, "openadapt_flow.authoring", fake_authoring) + session = open_authoring_session( + out_dir=tmp_path, backend=object(), backend_kind="web" + ) + assert isinstance(session, CoachOnlySession) diff --git a/tests/test_cli.py b/tests/test_cli.py index 45c1bbb..42e7c58 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -137,6 +137,7 @@ def test_authoring_serve_registers_probe_tools_without_run(monkeypatch, capsys): captured: dict = {} def fake_session(**kwargs): + captured["session_kwargs"] = kwargs return FakeAuthoringSession() def fake_serve(bridge, authoring=None): @@ -153,6 +154,7 @@ def fake_serve(bridge, authoring=None): names = [spec.name for spec in authoring.list_tool_specs()] assert names[:4] == ["observe", "start_record", "click", "halt"] assert "type" in names + assert captured["session_kwargs"]["out_dir"].name == "authoring" err = capsys.readouterr().err assert "authoring tools enabled" in err assert "run tools disabled" in err diff --git a/tests/test_distribution.py b/tests/test_distribution.py index d767c31..3d28b54 100644 --- a/tests/test_distribution.py +++ b/tests/test_distribution.py @@ -219,6 +219,9 @@ def test_llms_txt_lists_the_tool_surface() -> None: "run_workflow_", "run_local_quickstart", "--tutorial", + "--authoring", + "observe", + "start_record", "continue_attention", "skip_attention", "teach_attention", @@ -264,6 +267,8 @@ def test_identity_sentence_is_shared() -> None: assert THREE_LINE_INSTALL in README.read_text(encoding="utf-8") assert "serve --tutorial --allow-run" not in THREE_LINE_INSTALL assert "openadapt-agent serve --allow-run" in README.read_text(encoding="utf-8") + assert "serve --authoring" in README.read_text(encoding="utf-8") + assert "serve --authoring" in LLMS_TXT.read_text(encoding="utf-8") assert "openadapt quickstart --break-it" in README.read_text(encoding="utf-8") assert "If the tool returns unsigned success, treat it as failure" in README.read_text( encoding="utf-8" From 19b2e8e136a3d7609761e4ba81e4a13700e31455 Mon Sep 17 00:00:00 2001 From: abrichr Date: Tue, 1 Sep 2026 15:56:57 -0400 Subject: [PATCH 3/3] feat(agent): authoring connect mailbox client for hosted ChatGPT.com Pip install is not a listener. Add `openadapt-agent authoring connect` so a laptop can claim oab_, poll wait=0, Allow per sub, and Continue with record_observed (never type_text). Prefer Desktop's engine when importable; otherwise copy the minimum outbound HTTP client. stdio serve --authoring stays Claude Code. Overlay, launchd, and the OS URL handler stay Desktop-only. HTTP listener remains forbidden. --- README.md | 32 +- docs/DESIGN.md | 46 +- docs/DISTRIBUTION.md | 2 +- docs/MAILBOX_CLI.md | 100 ++++ llms.txt | 2 +- src/openadapt_agent/authoring.py | 14 +- src/openadapt_agent/cli.py | 64 ++- src/openadapt_agent/mailbox.py | 794 +++++++++++++++++++++++++++++ src/openadapt_agent/runner_bind.py | 185 +++++++ tests/test_authoring.py | 2 +- tests/test_cli.py | 52 ++ tests/test_distribution.py | 3 + tests/test_mailbox.py | 522 +++++++++++++++++++ 13 files changed, 1788 insertions(+), 30 deletions(-) create mode 100644 docs/MAILBOX_CLI.md create mode 100644 src/openadapt_agent/mailbox.py create mode 100644 src/openadapt_agent/runner_bind.py create mode 100644 tests/test_mailbox.py diff --git a/README.md b/README.md index 5814ae3..6846a52 100644 --- a/README.md +++ b/README.md @@ -44,9 +44,9 @@ uvx openadapt-agent serve --bundles /path/to/bundles --allow-run ## Author a first demo (Claude Code) -Local Claude Code (or Cursor, Codex, Grok CLI) is the first authoring UI. -Hosted ChatGPT.com / Claude.ai MCP is a website mailbox, not a listener in -this package. +Local Claude Code (or Cursor, Codex, Grok CLI) is the first authoring UI +over stdio. Hosted ChatGPT.com / Claude.ai cannot talk to localhost; they +need the outbound mailbox client below, not `serve --authoring`. ```bash claude mcp add openadapt-authoring -- \ @@ -59,6 +59,32 @@ and `halt`. `--authoring` does not enable run tools. This process stays stdio. Pass `--url` to pin a fresh Playwright Chromium with empty cookies. Windows native, Citrix, and RDP are coach-only in v1. +## Connect this computer (ChatGPT.com / Claude.ai) + +OpenAdapt is installed on this computer, so an agent can drive only through +OpenAdapt. + +Desktop: the tray is already running. **Connect this computer** on the job +page (`openadapt://runner`) just works. + +Pip: after `pip install openadapt`, start the same mailbox client: + +```bash +openadapt-agent authoring connect \ + 'openadapt://runner?pack=p.…&bind=oab_…&origin=https://openadapt.ai' +``` + +That claims `oab_`, polls `/j/{id}/runner/poll` with `wait_seconds: 0`, +prints Allow (`y/n`) per chat account, and on pause prints `Sign in in +the app, then press Enter`. Continue records with `record_observed`. It +never types your password. Pass `--url` for a fresh Playwright Chromium +with empty cookies. + +The job page should offer `openadapt connect ` next to Open +OpenAdapt (meta-package alias; this repo implements +`openadapt-agent authoring connect`). Overlay chrome, launchd, and the +OS URL handler stay Desktop-only. See [`docs/MAILBOX_CLI.md`](docs/MAILBOX_CLI.md). + ## Serve a bundle `--allow-run` with no `--bundles` records, compiles, and certifies the diff --git a/docs/DESIGN.md b/docs/DESIGN.md index 6b9fc71..a361a05 100644 --- a/docs/DESIGN.md +++ b/docs/DESIGN.md @@ -51,13 +51,19 @@ openadapt_agent.mcp (local stdio only; HTTP shim forbidden) │ ├── new run ──► openadapt-flow run subprocess │ └── attended ─► openadapt-flow durable API │ - └── openadapt_agent.authoring (--authoring first demo) - observe / start_record / click / halt - local type (agent-driven Recorder.type_text) - pause Continue → record_observed (never type_text) - │ - ▼ - openadapt_flow.authoring.AuthoringSession + ├── openadapt_agent.authoring (--authoring first demo, stdio) + │ observe / start_record / click / halt + │ local type (agent-driven Recorder.type_text) + │ pause Continue → record_observed (never type_text) + │ │ + │ ▼ + │ openadapt_flow.authoring.AuthoringSession + │ + └── openadapt_agent.mailbox (authoring connect, outbound HTTPS) + parse openadapt://runner / pack URL + POST claim oab_ → poll wait=0 → Allow-per-sub + Continue → record_observed (never type_text) + overlay chrome stays Desktop-only ``` The MCP adapter is intentionally thin. Tool descriptions and dispatch @@ -150,9 +156,10 @@ RDP construct a coach-only stand-in and never spawn `win_agent`. Observe is fail-closed to the T1 wire (`additionalProperties: false`, node ids `n_` + 8 hex, 200 nodes / 32 KiB). Capture's projector is used when importable. If Desktop has advertised authoring IPC, overlay stays -Desktop-owned; this package does not speak the D2 protocol or open an -HTTP client. Tests cover the tool surface with a fake session and an -F1-shaped session. +Desktop-owned; stdio `--authoring` does not speak the D2 protocol. +`authoring connect` is the outbound mailbox client for hosted chat apps. +Tests cover the stdio tool surface with a fake session and an F1-shaped +session, and the mailbox client against a mocked wait=0 poll. ## Governed runs @@ -307,12 +314,16 @@ caller-controlled `USERNAME` environment variable. A blank operator identity fails closed. This process must not be port-forwarded or exposed as an unauthenticated -network service. An HTTP / Streamable-HTTP shim in this MIT package -remains forbidden, including when `--authoring` is set. Remote authoring -for ChatGPT.com / Claude.ai is a website mailbox, not a listener inside -`openadapt-agent`. OpenAdapt Cloud owns remote authentication, +network service. An HTTP / Streamable-HTTP **listener** in this MIT package +remains forbidden, including when `--authoring` is set. Hosted ChatGPT.com +/ Claude.ai cannot talk to localhost. Pip users run `openadapt-agent +authoring connect` — an **outbound** mailbox client (claim `oab_`, poll +`wait_seconds: 0`, Allow-per-`sub`) copied from Desktop +`engine/authoring_runner.py` when that engine is not importable. Overlay +chrome, launchd, and the `openadapt://` URL handler stay Desktop-only. +See `docs/MAILBOX_CLI.md`. OpenAdapt Cloud owns remote authentication, multi-tenancy, tenant-scoped authorization, fleet policy, and managed -transport. `--authoring` does not add those, and it does not imply +execute. `--authoring` does not add those, and it does not imply `--allow-run`. ## Dependency boundary @@ -353,7 +364,10 @@ Tests cover: and extra keys, caps the wire at 32 KiB, and uses `n_` + 8 hex node ids; pause Continue uses `record_observed` rather than `type_text`; compile returns `needs_human_admit`; `--authoring` does not enable - run tools; `server.json` stays stdio with `--bundles` required. + run tools; `server.json` stays stdio with `--bundles` required; + `authoring connect` parses `openadapt://runner` / pack URLs, claims + `oab_`, polls `wait_seconds: 0`, prompts Allow-per-`sub`, and Continue + uses `record_observed` (never `type_text`). CI runs on Python 3.10, 3.11, and 3.12. It also builds the wheel and sdist, verifies MIT metadata and license inclusion, and refuses package diff --git a/docs/DISTRIBUTION.md b/docs/DISTRIBUTION.md index 0ea6f09..130fcd9 100644 --- a/docs/DISTRIBUTION.md +++ b/docs/DISTRIBUTION.md @@ -64,7 +64,7 @@ and [`../manifest.json`](../manifest.json). - **License:** MIT - **Transport:** stdio - **Run command (uvx):** `uvx --from 'openadapt-agent[tutorial]' openadapt-agent serve --allow-run` -- **Config:** `--allow-run` with no `--bundles` (public synthetic bundle, generated at serve time), `--tutorial` (same path without implying run tools), `--authoring` (local Claude Code first demo; not the published registry recipe), or `--bundles` (operator's private artifact), `--runs-dir`, `--allow-attended-actions`, qualified `--config` for Continue/Skip, and optional secret `OPENADAPT_BUNDLE_KEY` +- **Config:** `--allow-run` with no `--bundles` (public synthetic bundle, generated at serve time), `--tutorial` (same path without implying run tools), `--authoring` (local Claude Code first demo; not the published registry recipe), `authoring connect` (outbound hosted mailbox; not the registry recipe), or `--bundles` (operator's private artifact), `--runs-dir`, `--allow-attended-actions`, qualified `--config` for Continue/Skip, and optional secret `OPENADAPT_BUNDLE_KEY` - **Tools:** - `list_workflows` / `get_workflow` — PHI-safe structural bundle projections with opaque IDs. - `get_run_report` — PHI-safe status and count summary; raw evidence stays local unless protected export was explicitly enabled. diff --git a/docs/MAILBOX_CLI.md b/docs/MAILBOX_CLI.md new file mode 100644 index 0000000..e0b9261 --- /dev/null +++ b/docs/MAILBOX_CLI.md @@ -0,0 +1,100 @@ +# Pip mailbox client: ChatGPT.com drives this computer + +One sentence: **OpenAdapt is installed on this computer, so an agent can drive only through OpenAdapt.** + +ChatGPT.com and Claude.ai cannot talk to localhost. They call `https://openadapt.ai/mcp`. A process on the laptop must poll `/j/{id}/runner/poll` with `wait_seconds: 0`, claim `oab_`, Allow per OAuth `sub`, pause so the person signs in **in the real app**, and `record_observed` on Continue. Desktop already does that. `openadapt-agent serve --authoring` is local stdio for Claude Code. That never reaches ChatGPT.com. `pip install` alone is not a listener. + +## Command + +Canonical in this package: + +```bash +openadapt-agent authoring connect '' +``` + +Playwright web (fresh Chromium, empty cookies): + +```bash +openadapt-agent authoring connect '' --url https://example.invalid/app +``` + +Rejected alternative: `openadapt-agent serve --authoring --mailbox`. `serve` is stdio MCP. Mixing it with an outbound poll loop would look like this package grew a hosted transport. The mailbox client is a separate verb. + +Job-page / meta-package alias (not implemented here; `openadapt` wraps this package): + +```bash +openadapt connect '' +``` + +The pack “Connect this computer” control should offer that command next to **Open OpenAdapt**. Do not implement the web page in this repository. GET `/j/{id}` stays presence. It is not actuation. + +## Shared engine + +The protocol is Desktop’s authoring mailbox (openadapt-desktop PR 154, `engine/authoring_runner.py`): + +| Step | Wire | +|---|---| +| Parse | `openadapt://runner` only. Fields: `pack`, `bind`, `origin`. Origin pin `https://openadapt.ai`. | +| Claim | `POST /j/{id}/runner/claim` `{ bind: "oab_…" }` → 201 `{ leaseSecret: "oals_…", lease_s: 900 }` | +| Poll | `POST /j/{id}/runner/poll` `Authorization: Bearer oals_…` `{ wait_seconds: 0, lease_seconds: 900 }`. Empty 204. Sleep 1 s locally. Do not copy hosted-runner `wait=25`. | +| Allow | Terminal `y/n` for **that** pending `bind_pack` `sub`. `POST /j/{id}/runner/allow` `{ command_id }`. | +| Continue | `Recorder.record_observed`. Never `type_text` for secrets or for text the person already typed. | + +This package prefers Desktop `engine.authoring_runner.AuthoringMailboxTransport` when that module is importable (Desktop installed in-process). Otherwise it uses the copy in `openadapt_agent.mailbox` (stdlib `urllib`, outbound POST only). There is no new repository and no `control_plane` path. + +Flow `AuthoringSession` / `Recorder` is the actuation engine when importable. Overlay chrome is not. + +## What this CLI does + +- Parse `openadapt://runner` or `https://openadapt.ai/j/{pack}` (pack URL is not enough to claim without `bind`). +- Claim `oab_`. First claim wins. Do not print `leaseSecret`. +- Poll `wait_seconds: 0`. +- Print Allow (`Allow ChatGPT to drive this job?` / replace-account copy). stdin `y/n`. +- Pause: print `Sign in in the app, then press Enter`. Do not ask for a password. Do not `type_text` the secret. +- Continue → `record_observed` on the pause-target node. +- `--url` pins Playwright Chromium with **empty cookies**. No debug-port attach. +- Unique-window fail-closed: macOS / Linux without a unique frontmost title is coach-only. Windows native / Citrix / RDP are coach-only. Never spawn `win_agent`. +- Allow-per-`sub` before observe / click / halt. +- GET handshake is not actuation (the CLI does not GET the pack page to click). +- Uncertain delivery: no blind retry (`RECONCILIATION_REQUIRED`). +- Titles, values, screenshots, backend pixels never go to the mailbox callback. + +## What stays Desktop-only + +| Capability | Why it is not this package | +|---|---| +| Overlay chrome (ghost ring, pause card, pointer-transparent HUD) | Native overlay contract. Terminal prints instead. | +| `openadapt://` OS URL handler | Tauri / protocol registration. Pip users paste the command. | +| launchd / Login Item / tray always-on listener | Desktop tray is already running. Pip starts a foreground process. | +| Keychain lease persistence across process restarts | Thin CLI keeps `oals_` in memory for this process. | +| Coach HUD chrome on Windows native | Coach-only here too; Desktop owns the HUD. | + +Pip is not a worse cousin for the **mailbox protocol**. It is a worse cousin for **chrome**: no overlay, no protocol handler, no tray. The hosted tools still only drive through OpenAdapt. + +## Safety invariants (Desktop 154) + +- Allow-per-`sub` before observe / click / halt. +- Continue → `record_observed`, never `type_text` for secrets. +- GET handshake is not actuation. +- Uncertain delivery: no blind retry. +- Titles do not go to MCP / mailbox callbacks. +- No public HTTP / Streamable-HTTP listener in this MIT package. +- No localhost tunnel. +- Bind tokens: exact `oab_[A-Za-z0-9_-]{43}`. Lease: exact `oals_[a-f0-9]{64}`. Reject `oar_`, `oap_`, swapped encodings, and anything that merely starts with `oa`. + +## Job page (openadapt-web, not this PR) + +Do not edit web 462 from this repository. The pack “Connect this computer” should eventually show, next to Open OpenAdapt: + +```text +pip: openadapt connect '' +``` + +Desktop users keep the `openadapt://runner` button. Pip users paste the same bind into this command. + +## Remaining gaps + +- Overlay chrome. +- launchd / tray autostart. +- OS URL handler so a click on the pack page starts the pip client without paste. +- Extracting the rest of Desktop `AuthoringRunner` (node-table HMAC files, overlay Continue) into a shared import. This PR copies claim / poll / allow / Continue-without-`type_text` only. diff --git a/llms.txt b/llms.txt index ab3638d..877d068 100644 --- a/llms.txt +++ b/llms.txt @@ -6,7 +6,7 @@ Default runtime interface for a calling agent. Computer-use agents are the user ## What it provides -- `openadapt-agent serve --allow-run`: generate and serve the public synthetic tutorial at serve time. `openadapt-agent serve --tutorial` is the same path without run tools. `openadapt-agent serve --bundles [--allow-run]`: serve a private compiled bundle. `openadapt-agent serve --authoring`: local Claude Code first-demo tools `observe`, `start_record`, `click`, and `halt` over stdio. `--authoring` does not enable run tools and does not open an HTTP listener. Hosted ChatGPT.com / Claude.ai MCP is a website mailbox. `list_workflows`, `get_workflow`, `get_run_report`, `list_needs_attention`, and `get_attention_item` are always available as PHI-safe read-only projections. `run_workflow_` tools require `--allow-run`. The synthetic tutorial registers `run_local_quickstart`. If a run returns HALTED, tell the user the record did not change. Never summarize halt, refused, timeout, or error as success. +- `openadapt-agent serve --allow-run`: generate and serve the public synthetic tutorial at serve time. `openadapt-agent serve --tutorial` is the same path without run tools. `openadapt-agent serve --bundles [--allow-run]`: serve a private compiled bundle. `openadapt-agent serve --authoring`: local Claude Code first-demo tools `observe`, `start_record`, `click`, and `halt` over stdio. `--authoring` does not enable run tools and does not open an HTTP listener. `openadapt-agent authoring connect `: outbound mailbox client for hosted ChatGPT.com / Claude.ai (claim `oab_`, poll wait=0, Allow-per-sub, Continue via `record_observed`). Overlay chrome stays Desktop-only. Hosted MCP is `https://openadapt.ai/mcp`. `list_workflows`, `get_workflow`, `get_run_report`, `list_needs_attention`, and `get_attention_item` are always available as PHI-safe read-only projections. `run_workflow_` tools require `--allow-run`. The synthetic tutorial registers `run_local_quickstart`. If a run returns HALTED, tell the user the record did not change. Never summarize halt, refused, timeout, or error as success. - `--allow-attended-actions` adds exact Reject, Teach, and Escalate tools for signed durable pauses. With a qualified Flow `--config`, the same server also exposes Continue and Skip through Flow's deployment-bound live verifier and deterministic resume path. - `openadapt-agent emit-skill --out ` wraps Flow's skill emitter and appends MCP, halt, and attended-action guidance. diff --git a/src/openadapt_agent/authoring.py b/src/openadapt_agent/authoring.py index e02f11d..ad80415 100644 --- a/src/openadapt_agent/authoring.py +++ b/src/openadapt_agent/authoring.py @@ -6,10 +6,11 @@ Recorder. Hosted remains pause-only. Human type during a pause is ``record_observed`` everywhere; never ``type_text`` on the pause target. -This module is a transport-independent bridge. It does not open a network -listener and does not implement a remote mailbox. Window titles, field -values, screenshots, and backend pixels never cross the MCP wire. -``--authoring`` does not imply ``--allow-run``. +This module is a transport-independent stdio bridge. It does not open a +network listener. Hosted ChatGPT.com uses ``openadapt-agent authoring +connect`` (outbound mailbox poll in :mod:`openadapt_agent.mailbox`). +Window titles, field values, screenshots, and backend pixels never +cross the MCP wire. ``--authoring`` does not imply ``--allow-run``. Deps F1/C1/T1 are not required to be merged: Flow's ``AuthoringSession`` is constructed when importable; Capture's projector is used when @@ -265,8 +266,9 @@ def halt(self) -> dict[str, Any]: def discover_desktop_authoring_ipc(*, home: Optional[Path] = None) -> Optional[dict[str, Any]]: """Return Desktop authoring IPC discovery when D2 has advertised it. - Overlay and Allow stay Desktop-owned. This package does not open an HTTP - client or mailbox listener. Until D2 publishes an authoring endpoint in + Overlay stays Desktop-owned. Stdio ``--authoring`` does not speak D2. + Hosted ChatGPT.com uses :mod:`openadapt_agent.mailbox` (outbound poll). + Until D2 publishes an authoring endpoint in ``~/.openadapt/desktop_ipc.json``, return None and pin a local Flow session. """ diff --git a/src/openadapt_agent/cli.py b/src/openadapt_agent/cli.py index 8c7364a..687debd 100644 --- a/src/openadapt_agent/cli.py +++ b/src/openadapt_agent/cli.py @@ -7,11 +7,14 @@ workflow runs and attended decisions require separate operator flags. ``--authoring`` adds first-demo stdio tools and does not imply ``--allow-run``. +- ``authoring connect`` — outbound mailbox client for hosted ChatGPT.com / + Claude.ai (claim ``oab_``, poll wait=0, Allow-per-sub). Not an HTTP + listener. Overlay chrome stays Desktop-only. - ``emit-skill`` — emit a Claude Agent Skill folder for one bundle (wraps ``openadapt-flow emit-skill`` and appends MCP + halt guidance). -The bridge is local stdio. Remote identity, tenancy, and transport are provided -by OpenAdapt Cloud rather than added to this process. +``serve`` stays local stdio. Hosted ChatGPT.com reaches this computer through +``authoring connect`` (outbound HTTPS), not a port-forwarded MCP server. """ from __future__ import annotations @@ -194,6 +197,49 @@ def build_parser() -> argparse.ArgumentParser: ) p.set_defaults(func=_cmd_serve) + authoring = sub.add_parser( + "authoring", + help=( + "Hosted authoring mailbox client (ChatGPT.com / Claude.ai). " + "Outbound HTTPS only; not an HTTP listener." + ), + ) + authoring_sub = authoring.add_subparsers(dest="authoring_command", required=True) + connect = authoring_sub.add_parser( + "connect", + help=( + "Claim an openadapt://runner link or pack URL, poll wait=0, " + "and prompt Allow per chat account" + ), + description=( + "Claim an openadapt://runner link or pack URL, poll wait=0, " + "and prompt Allow per chat account. Overlay chrome stays " + "Desktop-only. Continue uses record_observed; it does not type " + "secrets. This process only makes outbound HTTPS." + ), + ) + connect.add_argument( + "target", + help=( + "openadapt://runner?pack=…&bind=oab_…&origin=https://openadapt.ai " + "or https://openadapt.ai/j/{id} (bind required to claim)" + ), + ) + connect.add_argument( + "--url", + default=None, + help=( + "Launch Playwright Chromium with empty cookies at this URL. " + "Not the browser you are already signed into." + ), + ) + connect.add_argument( + "--headed", + action="store_true", + help="Keep the Playwright window visible (required to sign in in the app).", + ) + connect.set_defaults(func=_cmd_authoring_connect) + p = sub.add_parser( "emit-skill", help=( @@ -380,6 +426,20 @@ def _serve(serve, bridge, authoring): serve(bridge, authoring=authoring) +def _cmd_authoring_connect(args: argparse.Namespace) -> int: + from openadapt_agent.mailbox import MailboxError, connect_mailbox + + try: + return connect_mailbox( + args.target, + url=args.url, + headed=args.headed, + ) + except MailboxError as exc: + print(f"authoring connect: {exc}", file=sys.stderr) + return 2 + + def _cmd_emit_skill(args: argparse.Namespace) -> int: from openadapt_agent.skill import emit_agent_skill diff --git a/src/openadapt_agent/mailbox.py b/src/openadapt_agent/mailbox.py new file mode 100644 index 0000000..11a954a --- /dev/null +++ b/src/openadapt_agent/mailbox.py @@ -0,0 +1,794 @@ +"""Outbound authoring mailbox client for hosted ChatGPT.com / Claude.ai. + +Prefers Desktop ``engine.authoring_runner.AuthoringMailboxTransport`` when +importable. Otherwise uses this copy (stdlib urllib, outbound POST only). +This module does not open a listener, tunnel loopback, or start the +in-guest Windows agent. Overlay chrome stays Desktop-only. +""" + +from __future__ import annotations + +import json +import re +import sys +import time +import urllib.error +import urllib.parse +import urllib.request +from typing import Any, Callable, Mapping, Optional, TextIO + +from openadapt_agent.authoring import ( + AuthoringBridge, + AuthoringError, + CoachOnlySession, + open_authoring_session, + pin_local_backend, + project_observe, +) +from openadapt_agent.runner_bind import ( + AUTHORING_ORIGIN, + RunnerBindError, + parse_connect_target, + valid_bind_token, + valid_lease_secret, + valid_pack_id, +) + +API_TIMEOUT_S = 10.0 +DEFAULT_LEASE_S = 900 +POLL_WAIT_S = 0 +LOCAL_POLL_SLEEP_S = 1.0 +COMMAND_ENVELOPE_SCHEMA = "openadapt.authoring.command/v1" +CLIENT_DISPLAYS = frozenset({"ChatGPT", "Claude"}) +PAUSE_PROMPT = "Sign in in the app, then press Enter" +BOUND_SENTENCE = ( + "OpenAdapt is installed on this computer, so an agent can drive only through OpenAdapt." +) +ENQUEUE_REQUIRING_ALLOW = frozenset( + { + "observe", + "click", + "start_record", + "pause_for_input", + "stop_record", + "compile", + "set_coach", + "get_coach", + "halt", + } +) +MAILBOX_ACTIONS = frozenset({"claim", "poll", "callback", "allow"}) +COACH_ONLY_BACKENDS = frozenset({"windows", "rdp", "citrix"}) +UNIQUE_WINDOW_BACKENDS = frozenset({"macos", "linux"}) +_COMMAND_ID = re.compile(r"^[A-Za-z0-9_.:-]{1,200}$") +FORBIDDEN_RESULT_KEYS = frozenset( + { + "value", + "text", + "title", + "screenshot", + "png", + "ocr", + "backend_pixels", + "pixels", + "events", + "window_title", + "image", + "raw", + "leaseSecret", + "lease_secret", + "bind", + } +) + +__all__ = [ + "AUTHORING_ORIGIN", + "BOUND_SENTENCE", + "DEFAULT_LEASE_S", + "PAUSE_PROMPT", + "POLL_WAIT_S", + "MailboxClient", + "MailboxError", + "MailboxTransport", + "connect_mailbox", + "open_mailbox_transport", + "parse_connect_target", + "require_empty_cookies", +] + + +class MailboxError(RuntimeError): + """A safe, user-facing mailbox failure with no secret-bearing text.""" + + +class _NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + return None + + def http_error_302(self, req, fp, code, msg, headers): + raise MailboxError("The authoring request was redirected.") + + http_error_301 = http_error_303 = http_error_307 = http_error_308 = http_error_302 + + +def _sanitize_result(value: Any) -> Any: + if isinstance(value, dict): + return { + key: _sanitize_result(child) + for key, child in value.items() + if key not in FORBIDDEN_RESULT_KEYS + } + if isinstance(value, list): + return [_sanitize_result(item) for item in value] + return value + + +def _client_display(value: object) -> str: + if value in CLIENT_DISPLAYS: + return str(value) + return "ChatGPT" + + +def require_empty_cookies(browser: Any) -> None: + """Refuse a Playwright session that already has cookies.""" + + cookies_fn = getattr(browser, "cookies", None) + if cookies_fn is None: + context = getattr(browser, "context", None) + cookies_fn = getattr(context, "cookies", None) + if not callable(cookies_fn): + return + cookies = cookies_fn() + if cookies: + raise MailboxError("Playwright Chromium did not start with empty cookies.") + + +def _default_opener() -> urllib.request.OpenerDirector: + return urllib.request.build_opener(_NoRedirect()) + + +class MailboxTransport: + """Outbound HTTPS bind/poll/callback. Wait is always 0.""" + + def __init__( + self, + *, + origin: str = AUTHORING_ORIGIN, + post: Callable[[str, dict[str, Any], dict[str, str]], tuple[int, dict[str, str], Any]] + | None = None, + opener: urllib.request.OpenerDirector | None = None, + ) -> None: + if origin != AUTHORING_ORIGIN: + raise MailboxError("The authoring origin is not pinned.") + self.origin = origin + self._post_impl = post + self._opener = opener or _default_opener() + + def _path(self, pack_id: str, action: str) -> str: + if not valid_pack_id(pack_id) or action not in MAILBOX_ACTIONS: + raise MailboxError("The authoring mailbox path is invalid.") + return f"/j/{urllib.parse.quote(pack_id, safe='._-')}/runner/{action}" + + def _urllib_post( + self, path: str, body: dict[str, Any], headers: dict[str, str] + ) -> tuple[int, dict[str, str], Any]: + url = self.origin + path + data = json.dumps(body).encode("utf-8") + request = urllib.request.Request(url, data=data, method="POST") + request.add_header("Content-Type", "application/json") + for key, value in headers.items(): + request.add_header(key, value) + try: + with self._opener.open(request, timeout=API_TIMEOUT_S) as response: + raw = response.read() + status = int(getattr(response, "status", 200)) + resp_headers = {k.lower(): v for k, v in response.headers.items()} + except urllib.error.HTTPError as exc: + status = int(exc.code) + raw = exc.read() if exc.fp is not None else b"" + resp_headers = ( + {k.lower(): v for k, v in exc.headers.items()} if exc.headers is not None else {} + ) + if status == 204: + return 204, resp_headers, None + if status == 401: + raise MailboxError("The authoring mailbox credential was rejected.") from exc + payload: Any = None + if raw: + try: + payload = json.loads(raw.decode("utf-8")) + except (ValueError, UnicodeDecodeError): + payload = None + return status, resp_headers, payload + except (urllib.error.URLError, TimeoutError, OSError) as exc: + raise MailboxError("The authoring request did not complete.") from exc + if status == 204: + return 204, resp_headers, None + if not raw: + return status, resp_headers, None + try: + payload = json.loads(raw.decode("utf-8")) + except (ValueError, UnicodeDecodeError) as exc: + raise MailboxError("The authoring response was not valid JSON.") from exc + return status, resp_headers, payload + + def _post( + self, + path: str, + body: dict[str, Any], + *, + headers: dict[str, str], + expected: tuple[int, ...], + allow_empty: bool = False, + ) -> tuple[int, dict[str, Any] | None]: + operation = path.rsplit("/", 1)[-1] + if self._post_impl is not None: + status, resp_headers, payload = self._post_impl(path, body, headers) + else: + status, resp_headers, payload = self._urllib_post(path, body, headers) + if allow_empty and status == 204: + return 204, None + if status == 401: + raise MailboxError("The authoring mailbox credential was rejected.") + if status not in expected: + raise MailboxError(f"The authoring {operation} request returned HTTP {status}.") + cache = (resp_headers.get("cache-control") or "").strip().lower() + if cache != "no-store": + raise MailboxError(f"The authoring {operation} response was not marked no-store.") + if not isinstance(payload, dict): + raise MailboxError(f"The authoring {operation} response was not an object.") + return status, payload + + def claim(self, pack_id: str, bind: str) -> dict[str, Any]: + if not valid_bind_token(bind): + raise MailboxError("Bind token is malformed") + path = self._path(pack_id, "claim") + _status, body = self._post( + path, + {"bind": bind}, + headers={"Content-Type": "application/json"}, + expected=(201,), + ) + assert body is not None + secret = body.get("leaseSecret") + lease_s = body.get("lease_s", DEFAULT_LEASE_S) + if not valid_lease_secret(secret) or not isinstance(lease_s, int) or lease_s <= 0: + raise MailboxError("The authoring claim response was not a mailbox lease.") + return {"leaseSecret": secret, "lease_s": lease_s} + + def poll(self, pack_id: str, lease_secret: str) -> dict[str, Any] | None: + if not valid_lease_secret(lease_secret): + raise MailboxError("The authoring mailbox credential is malformed.") + path = self._path(pack_id, "poll") + _status, body = self._post( + path, + {"wait_seconds": POLL_WAIT_S, "lease_seconds": DEFAULT_LEASE_S}, + headers={ + "Authorization": f"Bearer {lease_secret}", + "Content-Type": "application/json", + }, + expected=(200,), + allow_empty=True, + ) + return body + + def callback( + self, + pack_id: str, + lease_secret: str, + payload: dict[str, Any], + ) -> None: + if not valid_lease_secret(lease_secret): + raise MailboxError("The authoring mailbox credential is malformed.") + path = self._path(pack_id, "callback") + self._post( + path, + _sanitize_result(payload), + headers={ + "Authorization": f"Bearer {lease_secret}", + "Content-Type": "application/json", + }, + expected=(200, 202), + ) + + def allow(self, pack_id: str, lease_secret: str, command_id: str) -> None: + if not valid_lease_secret(lease_secret): + raise MailboxError("The authoring mailbox credential is malformed.") + if not isinstance(command_id, str) or _COMMAND_ID.fullmatch(command_id) is None: + raise MailboxError("The authoring Allow request is malformed.") + path = self._path(pack_id, "allow") + self._post( + path, + {"command_id": command_id}, + headers={ + "Authorization": f"Bearer {lease_secret}", + "Content-Type": "application/json", + }, + expected=(200, 202), + ) + + +class _NullAudit: + def log(self, event: str, **data: Any) -> None: + return None + + +def _try_desktop_transport(*, origin: str, post: Any = None) -> MailboxTransport | Any | None: + if post is not None: + return None + try: + from engine.authoring_runner import AuthoringMailboxTransport + except ImportError: + return None + return AuthoringMailboxTransport(origin=origin, audit=_NullAudit()) + + +def open_mailbox_transport( + *, + origin: str = AUTHORING_ORIGIN, + post: Callable[[str, dict[str, Any], dict[str, str]], tuple[int, dict[str, str], Any]] + | None = None, + opener: urllib.request.OpenerDirector | None = None, +) -> Any: + """Prefer Desktop's mailbox transport; copy claim/poll/allow otherwise.""" + + desktop = _try_desktop_transport(origin=origin, post=post) + if desktop is not None: + return desktop + return MailboxTransport(origin=origin, post=post, opener=opener) + + +class MailboxClient: + """Claim, Allow-per-sub, wait=0 poll, and Continue via record_observed.""" + + def __init__( + self, + transport: Any, + *, + session: Any = None, + prompt: Callable[[str], bool] | None = None, + pause_wait: Callable[[], None] | None = None, + sleep: Callable[[float], None] = time.sleep, + stdout: TextIO | None = None, + url: Optional[str] = None, + platform: Optional[str] = None, + unique_window: Callable[[], dict[str, Any] | None] | None = None, + recorder: Any = None, + text_value_at: Callable[[dict[str, Any]], str | None] | None = None, + ) -> None: + self.transport = transport + self.session = session + self._bridge = AuthoringBridge(session) if session is not None else None + self._prompt = prompt or _default_allow_prompt + self._pause_wait = pause_wait + self._sleep = sleep + self._stdout = stdout or sys.stdout + self._url = url + self._platform = platform or sys.platform + self._unique_window = unique_window + self._recorder = recorder + self._text_value_at = text_value_at + self._pack: str | None = None + self._lease_secret: str | None = None + self._allowed_sub: str | None = None + self._pending_allow: dict[str, Any] | None = None + self._pause_target: dict[str, Any] | None = None + self._paused = False + self._seen: set[str] = set() + self._uncertain = False + self._actuation_started = False + self._backend, self._coach_only = self._pin() + + def _pin(self) -> tuple[str, bool]: + if self._url: + return "web", False + plat = self._platform + if plat == "win32" or plat.startswith("win"): + return "windows", True + if self._unique_window is not None: + unique = self._unique_window() + if unique is None: + kind = "macos" if plat == "darwin" else "linux" + return kind, True + kind = str(unique.get("backend") or ("macos" if plat == "darwin" else "linux")) + if kind in COACH_ONLY_BACKENDS: + return kind, True + if kind in UNIQUE_WINDOW_BACKENDS and unique.get("window_title_unique") is not True: + return kind, True + return kind, False + if self.session is not None: + kind = getattr(self.session, "backend_kind", None) or getattr( + self.session, "backend", None + ) + if kind in COACH_ONLY_BACKENDS: + return str(kind), True + if isinstance(kind, str) and kind: + coach = getattr(self.session, "coach_only", False) is True + return kind, coach + return "web", False + try: + _backend, kind, _close = pin_local_backend(platform=plat) + except AuthoringError: + kind = "macos" if plat == "darwin" else "linux" + return kind, True + if kind in COACH_ONLY_BACKENDS: + return kind, True + if kind in UNIQUE_WINDOW_BACKENDS: + return kind, False + return kind, True + + def claim(self, pack_id: str, bind: str) -> dict[str, Any]: + claimed = self.transport.claim(pack_id, bind) + self._pack = pack_id + self._lease_secret = claimed["leaseSecret"] + return {"bound": True, "pack": pack_id} + + def poll_once(self) -> dict[str, Any] | None: + if self._pack is None or self._lease_secret is None: + raise MailboxError("not_bound") + body = self.transport.poll(self._pack, self._lease_secret) + if body is None: + return None + return self.handle_envelope(body) + + def handle_envelope(self, envelope: Mapping[str, Any]) -> dict[str, Any]: + tool = envelope.get("tool") + command_id = envelope.get("command_id") + if not isinstance(tool, str) or not isinstance(command_id, str): + return {"status": "error", "error": "invalid_envelope"} + schema = envelope.get("schema_version") + if schema not in (None, COMMAND_ENVELOPE_SCHEMA): + return self._callback_error(command_id, "invalid_envelope") + if command_id in self._seen: + return {"status": "error", "error": "RECONCILIATION_REQUIRED"} + args = envelope.get("args") if isinstance(envelope.get("args"), Mapping) else {} + sub = envelope.get("oauth_sub_sha256") + if tool == "bind_pack": + pending = self._pending_allow + if pending and pending.get("command_id") == command_id: + return {"status": "pending_allow"} + return self._queue_allow(envelope) + if tool in {"type", "type_text"}: + return self._callback_error(command_id, "type_refused") + if tool in ENQUEUE_REQUIRING_ALLOW and (not self._allowed_sub or sub != self._allowed_sub): + return self._callback_error(command_id, "not_allowed") + if self._uncertain: + self._seen.add(command_id) + return self._callback_error(command_id, "RECONCILIATION_REQUIRED") + self._seen.add(command_id) + try: + result = self._dispatch(tool, dict(args), command_id) + except MailboxError as exc: + code = ( + str(exc) + if str(exc) in {"stale_node", "COACH_ONLY", "RECONCILIATION_REQUIRED"} + else "error" + ) + if str(exc) == "RECONCILIATION_REQUIRED" or self._uncertain: + self._uncertain = True + return self._callback_error(command_id, code) + except AuthoringError as exc: + code = exc.code or "error" + if code == "COACH_ONLY": + return self._callback_error(command_id, "COACH_ONLY") + if code == "stale_node": + return self._callback_error(command_id, "stale_node") + return self._callback_error(command_id, "error") + if result.get("status") == "paused": + return result + self._callback_done(command_id, result) + return result + + def _queue_allow(self, envelope: Mapping[str, Any]) -> dict[str, Any]: + sub = envelope.get("oauth_sub_sha256") + command_id = envelope.get("command_id") + if not isinstance(sub, str) or not isinstance(command_id, str): + return self._callback_error(envelope.get("command_id") or "cmd", "invalid_allow") + display = _client_display(envelope.get("client_display")) + replace = bool(self._allowed_sub and self._allowed_sub != sub) + self._pending_allow = { + "command_id": command_id, + "oauth_sub_sha256": sub, + "client_display": display, + "replace": replace, + } + if replace: + message = "A different ChatGPT account is asking. Allow it to replace the current one?" + else: + message = f"Allow {display} to drive this job?" + self._write(message) + if not self._prompt(message): + self._pending_allow = None + return self._callback_error(command_id, "denied") + return self.allow(replace=replace) + + def allow(self, *, replace: bool = False) -> dict[str, Any]: + pending = self._pending_allow + if pending is None: + raise MailboxError("There is no pending Allow request.") + if self._allowed_sub and self._allowed_sub != pending["oauth_sub_sha256"] and not replace: + return {"allowed": False, "status": "replace_allow"} + command_id = pending["command_id"] + if self._pack and self._lease_secret: + self.transport.allow(self._pack, self._lease_secret, command_id) + self._allowed_sub = pending["oauth_sub_sha256"] + self._pending_allow = None + self._seen.add(command_id) + result = {"allowed": True, "client_display": pending.get("client_display")} + self._callback_done(command_id, result) + return result + + def continue_pause(self) -> dict[str, Any]: + if not self._paused or self._pause_target is None: + return {"paused": False} + target = self._pause_target + recorder = self._recorder + if recorder is None and self.session is not None: + recorder = self.session + if recorder is None: + raise MailboxError("authoring session does not implement record_observed") + original = None + if hasattr(recorder, "type_text"): + original = recorder.type_text + + def _forbidden(*_args: Any, **_kwargs: Any) -> None: + raise MailboxError("Continue must not type") + + recorder.type_text = _forbidden + try: + if hasattr(recorder, "continue_input"): + raw = recorder.continue_input() + elif hasattr(recorder, "record_observed"): + kwargs: dict[str, Any] = { + "event": {"kind": "type"}, + "param": target.get("param"), + } + if target.get("secret"): + kwargs["secret"] = True + else: + text = None + if self._text_value_at is not None and isinstance( + target.get("backend_pixels"), dict + ): + text = self._text_value_at(target["backend_pixels"]) + kwargs["text"] = text + recorder.record_observed(**kwargs) + raw = {"recorded": True, "param": target.get("param")} + else: + raise MailboxError("authoring session does not implement record_observed") + finally: + if original is not None: + recorder.type_text = original + self._paused = False + self._pause_target = None + result = {"recorded": True} + if isinstance(raw, Mapping): + if raw.get("param"): + result["param"] = raw["param"] + elif target.get("param"): + result["param"] = target["param"] + result = _sanitize_result(result) + command_id = target.get("command_id") + if isinstance(command_id, str): + self._callback_done(command_id, result) + return result + + def run(self, *, max_polls: int | None = None) -> int: + polls = 0 + while max_polls is None or polls < max_polls: + self.poll_once() + polls += 1 + if max_polls is not None and polls >= max_polls: + break + self._sleep(LOCAL_POLL_SLEEP_S) + return 0 + + def _dispatch(self, tool: str, args: dict[str, Any], command_id: str) -> dict[str, Any]: + if tool == "observe": + return self._observe() + if tool == "start_record": + return self._start_record() + if tool == "click": + return self._click(args) + if tool == "halt": + return self._halt() + if tool == "pause_for_input": + return self._pause_for_input(args, command_id) + if tool == "stop_record": + if self._bridge is not None: + return self._bridge.dispatch("stop_record", {}) + return {"status": "stopped", "compiled": False} + if tool == "compile": + if self._bridge is not None: + return self._bridge.dispatch("compile", {}) + return {"status": "needs_human_admit", "recording_retained": True} + if tool in {"set_coach", "get_coach"}: + if self._bridge is not None: + return self._bridge.dispatch(tool, args) + return {"ok": True} if tool == "set_coach" else {"hint": None} + raise MailboxError("unknown_tool") + + def _observe(self) -> dict[str, Any]: + if self._coach_only: + return project_observe( + { + "backend": self._backend + if self._backend in {"windows", "rdp", "citrix", "macos", "linux", "web"} + else "windows", + "provider": "none", + "coach_only": True, + "agent_drive": False, + "tree": [], + } + ) + if self._bridge is None: + raise MailboxError("authoring session is not available") + return self._bridge.dispatch("observe", {}) + + def _start_record(self) -> dict[str, Any]: + if self._coach_only: + raise MailboxError("COACH_ONLY") + if self._bridge is None: + raise MailboxError("authoring session is not available") + return self._bridge.dispatch("start_record", {}) + + def _click(self, args: dict[str, Any]) -> dict[str, Any]: + if self._coach_only: + raise MailboxError("COACH_ONLY") + if self._uncertain: + raise MailboxError("RECONCILIATION_REQUIRED") + node_id = args.get("node_id") + if not isinstance(node_id, str): + raise MailboxError("stale_node") + if self._bridge is None: + raise MailboxError("authoring session is not available") + self._actuation_started = True + try: + result = self._bridge.dispatch("click", {"node_id": node_id}) + except AuthoringError as exc: + if exc.code in {"stale_node", "COACH_ONLY"}: + raise MailboxError(exc.code) from exc + self._uncertain = True + raise MailboxError("RECONCILIATION_REQUIRED") from None + except Exception: + self._uncertain = True + raise MailboxError("RECONCILIATION_REQUIRED") from None + finally: + self._actuation_started = False + if isinstance(result, Mapping) and result.get("error") == "stale_node": + raise MailboxError("stale_node") + return dict(result) + + def _halt(self) -> dict[str, Any]: + if self._actuation_started: + self._uncertain = True + raise MailboxError("RECONCILIATION_REQUIRED") + if self._bridge is not None: + return self._bridge.dispatch("halt", {}) + return {"status": "halted", "compiled": False} + + def _pause_for_input(self, args: dict[str, Any], command_id: str) -> dict[str, Any]: + self._pause_target = { + "node_id": args.get("node_id"), + "param": args.get("param"), + "secret": args.get("secret") is True, + "backend_pixels": args.get("backend_pixels"), + "command_id": command_id, + } + self._paused = True + self._write(PAUSE_PROMPT) + waiter = self._pause_wait + if waiter is None: + input(PAUSE_PROMPT) + else: + waiter() + return {"status": "paused", **self.continue_pause()} + + def _callback_error(self, command_id: object, error: str) -> dict[str, Any]: + payload = { + "command_id": command_id, + "status": "error", + "result": {"error": error}, + } + if self._pack and self._lease_secret and isinstance(command_id, str): + self.transport.callback(self._pack, self._lease_secret, payload) + return {"status": "error", "error": error} + + def _callback_done(self, command_id: str, result: Mapping[str, Any]) -> None: + if not self._pack or not self._lease_secret: + return + self.transport.callback( + self._pack, + self._lease_secret, + { + "command_id": command_id, + "status": "done", + "result": _sanitize_result(dict(result)), + }, + ) + + def _write(self, message: str) -> None: + self._stdout.write(message + "\n") + self._stdout.flush() + + +def _default_allow_prompt(message: str) -> bool: + reply = input(f"{message} [y/N] ").strip().lower() + return reply in {"y", "yes"} + + +def _open_session(*, url: Optional[str], headed: bool, platform: Optional[str]) -> Any: + plat = platform or sys.platform + if (plat == "win32" or plat.startswith("win")) and not url: + return CoachOnlySession("windows") + try: + return open_authoring_session(url=url, headed=headed, platform=platform) + except AuthoringError: + if (plat == "win32" or plat.startswith("win")) and not url: + return CoachOnlySession("windows") + return None + + +def connect_mailbox( + target: str, + *, + url: Optional[str] = None, + headed: bool = False, + prompt: Callable[[str], bool] | None = None, + pause_wait: Callable[[], None] | None = None, + sleep: Callable[[float], None] | None = None, + post: Callable[[str, dict[str, Any], dict[str, str]], tuple[int, dict[str, str], Any]] + | None = None, + session: Any = None, + max_polls: int | None = None, + stdout: TextIO | None = None, + platform: Optional[str] = None, + unique_window: Callable[[], dict[str, Any] | None] | None = None, + recorder: Any = None, + text_value_at: Callable[[dict[str, Any]], str | None] | None = None, +) -> int: + """Claim a runner link or pack URL and poll the hosted mailbox.""" + + try: + parsed = parse_connect_target(target) + except RunnerBindError as exc: + raise MailboxError(str(exc)) from exc + if "bind" not in parsed: + raise MailboxError( + "this pack URL is not a runner link; paste the Connect this computer " + "command (openadapt://runner?pack=…&bind=oab_…)" + ) + transport = open_mailbox_transport(origin=parsed["origin"], post=post) + opened = session + if opened is None: + opened = _open_session(url=url, headed=headed, platform=platform) + if url and opened is not None: + require_empty_cookies(opened) + inner = getattr(opened, "page", None) or getattr(opened, "backend", None) + if inner is not None and inner is not opened: + require_empty_cookies(inner) + client = MailboxClient( + transport, + session=opened, + prompt=prompt, + pause_wait=pause_wait, + sleep=sleep or time.sleep, + stdout=stdout, + url=url, + platform=platform, + unique_window=unique_window, + recorder=recorder, + text_value_at=text_value_at, + ) + bound = client.claim(parsed["pack"], parsed["bind"]) + out = stdout or sys.stdout + out.write(BOUND_SENTENCE + "\n") + out.write("Waiting for ChatGPT or Claude to ask for Allow.\n") + out.flush() + if bound.get("leaseSecret") or bound.get("lease_secret"): + raise MailboxError("claim must not return the mailbox lease to the operator") + try: + return client.run(max_polls=max_polls) + except KeyboardInterrupt: + out.write("stopped\n") + return 0 diff --git a/src/openadapt_agent/runner_bind.py b/src/openadapt_agent/runner_bind.py new file mode 100644 index 0000000..4fb5f3a --- /dev/null +++ b/src/openadapt_agent/runner_bind.py @@ -0,0 +1,185 @@ +"""Parse-only grammar for ``openadapt://runner`` and pack URLs. + +This module does not claim, store, poll, or GET. GET of ``/j/{id}`` is +the handshake and is not actuation. +""" + +from __future__ import annotations + +import re +from urllib.parse import parse_qs, urlparse, urlsplit, unquote + +AUTHORING_ORIGIN = "https://openadapt.ai" +MAX_URI_BYTES = 2048 +ALLOWED_FIELDS = frozenset({"pack", "bind", "origin"}) +PACK_URL_FIELDS = frozenset({"bind", "origin"}) + +BIND_TOKEN_RE = re.compile(r"^oab_[A-Za-z0-9_-]{43}$") +LEASE_SECRET_RE = re.compile(r"^oals_[a-f0-9]{64}$") +PACK_ALIAS_RE = re.compile(r"^p\.[A-Za-z0-9_-]{12}$") +PACK_CIPHER_RE = re.compile(r"^v1\.[A-Za-z0-9_-]{32,2000}$") +CLOUD_RUNNER_TOKEN_RE = re.compile(r"^oar_[a-f0-9]{64}$") +PAIRING_SECRET_RE = re.compile(r"^oap_[A-Za-z0-9_-]{43}$") +BIND_HEX_BODY_RE = re.compile(r"^oab_[a-f0-9]{64}$") +LEASE_BASE64URL_BODY_RE = re.compile(r"^oals_[A-Za-z0-9_-]{43}$") +PACK_PATH_RE = re.compile(r"^/j/([^/]+)/?$") + +__all__ = [ + "AUTHORING_ORIGIN", + "MAX_URI_BYTES", + "RunnerBindError", + "canonical_authoring_origin", + "parse_connect_target", + "parse_runner_uri", + "valid_bind_token", + "valid_lease_secret", + "valid_pack_id", +] + + +class RunnerBindError(RuntimeError): + """A safe, user-facing runner-link failure with no secret-bearing text.""" + + +def valid_bind_token(value: object) -> bool: + """Return whether ``value`` is exactly one ``oab_`` bind token.""" + + if not isinstance(value, str): + return False + if ( + CLOUD_RUNNER_TOKEN_RE.fullmatch(value) is not None + or PAIRING_SECRET_RE.fullmatch(value) is not None + or BIND_HEX_BODY_RE.fullmatch(value) is not None + ): + return False + return BIND_TOKEN_RE.fullmatch(value) is not None + + +def valid_lease_secret(value: object) -> bool: + """Return whether ``value`` is exactly one ``oals_`` mailbox lease secret.""" + + if not isinstance(value, str): + return False + if ( + CLOUD_RUNNER_TOKEN_RE.fullmatch(value) is not None + or PAIRING_SECRET_RE.fullmatch(value) is not None + or LEASE_BASE64URL_BODY_RE.fullmatch(value) is not None + ): + return False + return LEASE_SECRET_RE.fullmatch(value) is not None + + +def valid_pack_id(value: object) -> bool: + """Return whether ``value`` is a ``p.`` alias or ``v1.`` ciphertext id.""" + + if not isinstance(value, str): + return False + return PACK_ALIAS_RE.fullmatch(value) is not None or PACK_CIPHER_RE.fullmatch(value) is not None + + +def canonical_authoring_origin(value: object) -> str: + """Return the pinned production authoring origin, or raise.""" + + if not isinstance(value, str): + raise RunnerBindError("Runner link does not name the OpenAdapt authoring origin") + try: + parsed = urlsplit(value) + port = parsed.port + except ValueError as exc: + raise RunnerBindError("Runner link does not name the OpenAdapt authoring origin") from exc + if ( + parsed.scheme != "https" + or parsed.hostname != "openadapt.ai" + or parsed.netloc != "openadapt.ai" + or parsed.username + or parsed.password + or parsed.path not in ("",) + or parsed.query + or parsed.fragment + or port is not None + or value != AUTHORING_ORIGIN + ): + raise RunnerBindError("Runner link does not name the OpenAdapt authoring origin") + return AUTHORING_ORIGIN + + +def parse_runner_uri(uri: object) -> dict[str, str]: + """Parse the fixed runner action and reject ambiguity or extra fields.""" + + if not isinstance(uri, str) or not uri or len(uri) > MAX_URI_BYTES: + raise RunnerBindError("Invalid OpenAdapt runner link") + parsed = urlparse(uri) + if ( + parsed.scheme != "openadapt" + or parsed.netloc != "runner" + or parsed.path not in ("", "/") + or parsed.params + or parsed.fragment + or parsed.username + or parsed.password + ): + raise RunnerBindError("Invalid OpenAdapt runner link") + try: + query = parse_qs(parsed.query, keep_blank_values=True, strict_parsing=True) + except ValueError as exc: + raise RunnerBindError("Invalid OpenAdapt runner link") from exc + if set(query) - ALLOWED_FIELDS or any(len(values) != 1 for values in query.values()): + raise RunnerBindError("Runner link contains unknown or duplicate fields") + if set(query) != ALLOWED_FIELDS: + raise RunnerBindError("Runner link is missing pack, bind, or origin") + + pack = query["pack"][0] + bind = query["bind"][0] + origin = canonical_authoring_origin(query["origin"][0]) + if not valid_pack_id(pack): + raise RunnerBindError("Pack id is malformed") + if CLOUD_RUNNER_TOKEN_RE.fullmatch(bind) or PAIRING_SECRET_RE.fullmatch(bind): + raise RunnerBindError("Bind token is malformed") + if not valid_bind_token(bind): + raise RunnerBindError("Bind token is malformed") + return {"pack": pack, "bind": bind, "origin": origin} + + +def parse_connect_target(value: object) -> dict[str, str]: + """Parse ``openadapt://runner`` or a pack URL. Does not GET.""" + + if not isinstance(value, str) or not value or len(value) > MAX_URI_BYTES: + raise RunnerBindError("Invalid OpenAdapt runner link") + stripped = value.strip() + if stripped.startswith("openadapt:"): + return parse_runner_uri(stripped) + parsed = urlsplit(stripped) + if parsed.scheme != "https" or parsed.hostname != "openadapt.ai": + raise RunnerBindError("Invalid OpenAdapt runner link") + if parsed.username or parsed.password or parsed.fragment: + raise RunnerBindError("Invalid OpenAdapt runner link") + try: + port = parsed.port + except ValueError as exc: + raise RunnerBindError("Invalid OpenAdapt runner link") from exc + if port is not None or parsed.netloc != "openadapt.ai": + raise RunnerBindError("Runner link does not name the OpenAdapt authoring origin") + match = PACK_PATH_RE.fullmatch(parsed.path) + if match is None: + raise RunnerBindError("Pack URL is not a job page") + pack = unquote(match.group(1)) + if not valid_pack_id(pack): + raise RunnerBindError("Pack id is malformed") + if not parsed.query: + return {"pack": pack, "origin": AUTHORING_ORIGIN} + try: + query = parse_qs(parsed.query, keep_blank_values=True, strict_parsing=True) + except ValueError as exc: + raise RunnerBindError("Invalid OpenAdapt runner link") from exc + if set(query) - PACK_URL_FIELDS or any(len(values) != 1 for values in query.values()): + raise RunnerBindError("Runner link contains unknown or duplicate fields") + origin = AUTHORING_ORIGIN + if "origin" in query: + origin = canonical_authoring_origin(query["origin"][0]) + result = {"pack": pack, "origin": origin} + if "bind" in query: + bind = query["bind"][0] + if not valid_bind_token(bind): + raise RunnerBindError("Bind token is malformed") + result["bind"] = bind + return result diff --git a/tests/test_authoring.py b/tests/test_authoring.py index c981094..a05dc57 100644 --- a/tests/test_authoring.py +++ b/tests/test_authoring.py @@ -361,7 +361,7 @@ def test_open_authoring_session_notes_flow_dependency_when_missing(monkeypatch): def test_authoring_sources_stay_stdio_without_http_listener(): root = Path(__file__).resolve().parents[1] / "src" / "openadapt_agent" - for name in ("authoring.py", "mcp.py", "cli.py"): + for name in ("authoring.py", "mcp.py", "cli.py", "mailbox.py", "runner_bind.py"): text = (root / name).read_text(encoding="utf-8") assert "HTTPServer" not in text assert "uvicorn" not in text diff --git a/tests/test_cli.py b/tests/test_cli.py index 42e7c58..4dfac7b 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -131,6 +131,58 @@ def missing(**kwargs): assert "openadapt_flow.authoring" in capsys.readouterr().err +def test_authoring_connect_parses_runner_link_and_url(): + args = build_parser().parse_args( + [ + "authoring", + "connect", + "openadapt://runner?pack=p.abcdefghijkl&bind=oab_" + + "A" * 43 + + "&origin=https://openadapt.ai", + "--url", + "https://example.invalid/app", + "--headed", + ] + ) + assert args.authoring_command == "connect" + assert args.url == "https://example.invalid/app" + assert args.headed is True + + +def test_authoring_connect_runs_mailbox(monkeypatch): + captured: dict = {} + + def fake_connect(target, **kwargs): + captured["target"] = target + captured["kwargs"] = kwargs + return 0 + + monkeypatch.setattr("openadapt_agent.mailbox.connect_mailbox", fake_connect) + result = main( + [ + "authoring", + "connect", + "openadapt://runner?pack=p.abcdefghijkl&bind=oab_" + + "A" * 43 + + "&origin=https://openadapt.ai", + ] + ) + assert result == 0 + assert captured["target"].startswith("openadapt://runner") + + +def test_authoring_connect_reports_mailbox_errors(monkeypatch, capsys): + def fake_connect(target, **kwargs): + from openadapt_agent.mailbox import MailboxError + + raise MailboxError("Bind token is malformed") + + monkeypatch.setattr("openadapt_agent.mailbox.connect_mailbox", fake_connect) + result = main(["authoring", "connect", "https://openadapt.ai/j/p.abcdefghijkl"]) + assert result == 2 + assert "malformed" in capsys.readouterr().err + + def test_authoring_serve_registers_probe_tools_without_run(monkeypatch, capsys): from test_authoring import FakeAuthoringSession diff --git a/tests/test_distribution.py b/tests/test_distribution.py index 3d28b54..eb09199 100644 --- a/tests/test_distribution.py +++ b/tests/test_distribution.py @@ -269,6 +269,9 @@ def test_identity_sentence_is_shared() -> None: assert "openadapt-agent serve --allow-run" in README.read_text(encoding="utf-8") assert "serve --authoring" in README.read_text(encoding="utf-8") assert "serve --authoring" in LLMS_TXT.read_text(encoding="utf-8") + assert "authoring connect" in README.read_text(encoding="utf-8") + assert "authoring connect" in LLMS_TXT.read_text(encoding="utf-8") + assert "openadapt connect" in README.read_text(encoding="utf-8") assert "openadapt quickstart --break-it" in README.read_text(encoding="utf-8") assert "If the tool returns unsigned success, treat it as failure" in README.read_text( encoding="utf-8" diff --git a/tests/test_mailbox.py b/tests/test_mailbox.py new file mode 100644 index 0000000..afb3fc7 --- /dev/null +++ b/tests/test_mailbox.py @@ -0,0 +1,522 @@ +"""Outbound mailbox CLI: claim, wait=0 poll, Allow-per-sub, record_observed.""" + +from __future__ import annotations + +import io +import json +from pathlib import Path +from typing import Any + +import pytest + +from openadapt_agent.mailbox import ( + BOUND_SENTENCE, + DEFAULT_LEASE_S, + PAUSE_PROMPT, + POLL_WAIT_S, + MailboxClient, + MailboxError, + MailboxTransport, + connect_mailbox, + open_mailbox_transport, + require_empty_cookies, +) +from openadapt_agent.runner_bind import ( + AUTHORING_ORIGIN, + RunnerBindError, + parse_connect_target, + parse_runner_uri, + valid_bind_token, + valid_lease_secret, + valid_pack_id, +) +from test_authoring import FakeAuthoringSession + +BIND = "oab_" + "A" * 43 +PACK = "p.abcdefghijkl" +LEASE = "oals_" + "a" * 64 +ORIGIN = "https://openadapt.ai" +URI = f"openadapt://runner?pack={PACK}&bind={BIND}&origin=https%3A%2F%2Fopenadapt.ai" +PACK_URL = f"https://openadapt.ai/j/{PACK}" +SUB = "b" * 64 +OTHER_SUB = "d" * 64 +CLIENT = "c" * 64 + + +class FakeRecorder: + def __init__(self) -> None: + self.observed: list[dict[str, Any]] = [] + self.typed: list[Any] = [] + + def type_text(self, *args: Any, **kwargs: Any) -> None: + self.typed.append((args, kwargs)) + raise AssertionError("Continue must not call type_text") + + def record_observed(self, **kwargs: Any) -> None: + self.observed.append(kwargs) + + +class MockMailbox: + def __init__( + self, + *, + claim_status: int = 201, + poll_bodies: list[dict[str, Any] | None] | None = None, + ) -> None: + self.requests: list[tuple[str, dict[str, Any], dict[str, str]]] = [] + self.claim_status = claim_status + self.polls = list(poll_bodies or []) + + def post( + self, path: str, body: dict[str, Any], headers: dict[str, str] + ) -> tuple[int, dict[str, str], Any]: + self.requests.append((path, body, headers)) + no_store = {"cache-control": "no-store"} + if path.endswith("/runner/claim"): + if self.claim_status == 201: + return 201, no_store, {"leaseSecret": LEASE, "lease_s": 900} + return self.claim_status, no_store, {"error": "rejected"} + if path.endswith("/runner/poll"): + assert body["wait_seconds"] == 0 + assert body["lease_seconds"] == 900 + assert headers["Authorization"] == f"Bearer {LEASE}" + if not self.polls: + return 204, {}, None + next_body = self.polls.pop(0) + if next_body is None: + return 204, {}, None + return 200, no_store, next_body + if path.endswith("/runner/callback"): + assert headers["Authorization"] == f"Bearer {LEASE}" + return 202, no_store, {"accepted": True} + if path.endswith("/runner/allow"): + assert headers["Authorization"] == f"Bearer {LEASE}" + return 200, no_store, {"allowedAt": "2026-09-01T00:00:00Z"} + return 404, no_store, {"error": "missing"} + + +def _envelope(tool: str, *, sub: str = SUB, args: dict[str, Any] | None = None) -> dict[str, Any]: + return { + "schema_version": "openadapt.authoring.command/v1", + "command_id": f"cmd_{tool}", + "pack_id": PACK, + "tool": tool, + "args": args or {}, + "oauth_sub_sha256": sub, + "client_id_sha256": CLIENT, + "client_display": "ChatGPT", + } + + +def _client( + mock: MockMailbox, + *, + session: Any | None = None, + prompt: Any = None, + pause_wait: Any = None, + unique_window: Any = None, + recorder: Any = None, + url: str | None = None, + platform: str = "darwin", + stdout: io.StringIO | None = None, +) -> tuple[MailboxClient, io.StringIO]: + out = stdout or io.StringIO() + transport = MailboxTransport(origin=ORIGIN, post=mock.post) + client = MailboxClient( + transport, + session=session if session is not None else FakeAuthoringSession(), + prompt=prompt or (lambda _message: True), + pause_wait=pause_wait or (lambda: None), + sleep=lambda _seconds: None, + stdout=out, + url=url, + platform=platform, + unique_window=unique_window, + recorder=recorder, + text_value_at=lambda _pixels: "follow up in two weeks", + ) + return client, out + + +def test_parser_accepts_only_the_fixed_runner_action() -> None: + assert parse_runner_uri(URI) == { + "pack": PACK, + "bind": BIND, + "origin": ORIGIN, + } + for uri in ( + URI.replace("://runner?", "://run?"), + URI.replace("://runner?", "://connect?"), + URI.replace("openadapt:", "https:"), + URI.replace("runner?", "runner/claim?"), + URI + "#fragment", + f"openadapt://user@runner?pack={PACK}&bind={BIND}&origin={ORIGIN}", + ): + with pytest.raises(RunnerBindError, match="Invalid OpenAdapt runner link"): + parse_runner_uri(uri) + + +def test_parser_rejects_malformed_missing_duplicate_and_unknown_fields() -> None: + bad = ( + "", + "openadapt://runner?pack", + f"openadapt://runner?pack=short&bind={BIND}&origin={ORIGIN}", + f"openadapt://runner?pack={PACK}&bind={BIND}", + f"openadapt://runner?pack={PACK}&bind={BIND}&bind={BIND}&origin={ORIGIN}", + f"openadapt://runner?pack={PACK}&bind={BIND}&origin={ORIGIN}&command=whoami", + f"openadapt://runner?pack={PACK}&bind={BIND}&origin=https://preview.openadapt.ai", + f"openadapt://runner?pack={PACK}&bind={BIND}&origin=https://openadapt.ai/", + f"openadapt://runner?pack={PACK}&bind={BIND}&origin=https://openadapt.ai:443", + f"openadapt://runner?pack={PACK}&bind={BIND}&origin=http://openadapt.ai", + ) + for uri in bad: + with pytest.raises(RunnerBindError): + parse_runner_uri(uri) + + +def test_prefix_parsers_reject_foreign_and_swapped_encodings() -> None: + oar = "oar_" + "a" * 64 + oap = "oap_" + "A" * 43 + oab_hex = "oab_" + "a" * 64 + oals_b64 = "oals_" + "A" * 43 + oa_prefix = "oa" + "A" * 43 + oals = "oals_" + "a" * 64 + + assert valid_bind_token(BIND) is True + assert valid_lease_secret(oals) is True + assert valid_pack_id(PACK) is True + assert valid_pack_id("v1." + "A" * 48) is True + + for value in (oar, oap, oab_hex, oals_b64, oals, oa_prefix, PACK): + assert valid_bind_token(value) is False, value + for value in (oar, oap, BIND, oab_hex, oals_b64, oa_prefix, PACK): + assert valid_lease_secret(value) is False, value + for value in (oar, oap, BIND, oals, oa_prefix, "p.short", "v1.short"): + assert valid_pack_id(value) is False, value + + for bind in (oar, oap, oab_hex, oals_b64, oa_prefix): + uri = f"openadapt://runner?pack={PACK}&bind={bind}&origin={ORIGIN}" + with pytest.raises(RunnerBindError, match="Bind token is malformed"): + parse_runner_uri(uri) + + +def test_pack_url_parses_without_getting() -> None: + parsed = parse_connect_target(PACK_URL) + assert parsed == {"pack": PACK, "origin": ORIGIN} + with_bind = parse_connect_target(f"{PACK_URL}?bind={BIND}") + assert with_bind["bind"] == BIND + assert with_bind["pack"] == PACK + + +def test_parse_connect_target_does_not_get() -> None: + source = Path("src/openadapt_agent/runner_bind.py").read_text(encoding="utf-8") + assert "urlopen" not in source + assert "Request(" not in source + assert 'method="GET"' not in source + assert "method='GET'" not in source + + +def test_claim_does_not_print_the_lease(capsys) -> None: + mock = MockMailbox() + stdout = io.StringIO() + result = connect_mailbox( + URI, + post=mock.post, + session=FakeAuthoringSession(), + prompt=lambda _message: True, + pause_wait=lambda: None, + sleep=lambda _seconds: None, + max_polls=1, + stdout=stdout, + platform="darwin", + ) + assert result == 0 + text = stdout.getvalue() + assert BOUND_SENTENCE in text + assert LEASE not in text + assert BIND not in text + path, body, _headers = mock.requests[0] + assert path == f"/j/{PACK}/runner/claim" + assert body == {"bind": BIND} + captured = capsys.readouterr() + assert LEASE not in captured.out + assert LEASE not in captured.err + + +@pytest.mark.parametrize("status", [409, 410, 404, 401]) +def test_claim_maps_mailbox_failures(status: int) -> None: + mock = MockMailbox(claim_status=status) + client, _out = _client(mock) + with pytest.raises(MailboxError): + client.claim(PACK, BIND) + + +def test_poll_wait_is_zero_not_twenty_five() -> None: + assert POLL_WAIT_S == 0 + assert DEFAULT_LEASE_S == 900 + source = Path("src/openadapt_agent/mailbox.py").read_text(encoding="utf-8") + assert "DEFAULT_WAIT_S" not in source + assert '"wait_seconds": POLL_WAIT_S' in source + assert "from openadapt_flow.backends.win_agent" not in source + assert "launch_agent(" not in source + assert "parallels_vm" not in source + mock = MockMailbox() + client, _out = _client(mock) + client.claim(PACK, BIND) + client.poll_once() + poll = next(item for item in mock.requests if item[0].endswith("/poll")) + assert poll[1]["wait_seconds"] == 0 + + +def test_bind_pack_allow_is_per_sub_and_required_for_halt() -> None: + mock = MockMailbox() + client, out = _client(mock) + client.claim(PACK, BIND) + client.handle_envelope(_envelope("bind_pack")) + assert "Allow ChatGPT to drive this job?" in out.getvalue() + allow = next(item for item in mock.requests if item[0].endswith("/allow")) + assert allow[1] == {"command_id": "cmd_bind_pack"} + client.handle_envelope(_envelope("observe", sub=OTHER_SUB)) + client.handle_envelope(_envelope("halt", sub=OTHER_SUB)) + denied = [item[1] for item in mock.requests if item[0].endswith("/callback")] + assert {item["result"]["error"] for item in denied if "error" in item.get("result", {})} >= { + "not_allowed" + } + observed = client.handle_envelope(_envelope("observe")) + assert observed["schema_version"] == "openadapt.authoring.observe/v1" + blob = json.dumps(observed) + assert "title" not in blob + assert "value" not in blob + halted = client.handle_envelope(_envelope("halt")) + assert halted["status"] == "halted" + + +def test_continue_records_observed_never_type_text() -> None: + mock = MockMailbox() + recorder = FakeRecorder() + session = FakeAuthoringSession() + client, out = _client(mock, session=session, recorder=recorder) + client.claim(PACK, BIND) + client.handle_envelope(_envelope("bind_pack")) + client.handle_envelope(_envelope("start_record")) + client.handle_envelope(_envelope("observe")) + client.handle_envelope( + _envelope("pause_for_input", args={"node_id": "n_9f2c001a", "param": "note"}) + ) + assert PAUSE_PROMPT in out.getvalue() + assert "password" not in out.getvalue().lower() + assert recorder.typed == [] + assert recorder.observed + assert recorder.observed[0]["event"] == {"kind": "type"} + assert "text" in recorder.observed[0] + callbacks = [item[1] for item in mock.requests if item[0].endswith("/callback")] + last = callbacks[-1] + assert last["result"]["recorded"] is True + assert "text" not in last["result"] + assert "value" not in last["result"] + + +def test_secret_continue_has_no_text() -> None: + mock = MockMailbox() + recorder = FakeRecorder() + client, _out = _client(mock, recorder=recorder) + client.claim(PACK, BIND) + client.handle_envelope(_envelope("bind_pack")) + client.handle_envelope( + _envelope( + "pause_for_input", + args={"node_id": "n_9f2c001a", "param": "ssn", "secret": True}, + ) + ) + assert recorder.typed == [] + assert recorder.observed[0]["secret"] is True + assert "text" not in recorder.observed[0] + + +def test_hosted_type_tool_is_refused() -> None: + mock = MockMailbox() + session = FakeAuthoringSession() + client, _out = _client(mock, session=session) + client.claim(PACK, BIND) + client.handle_envelope(_envelope("bind_pack")) + client.handle_envelope(_envelope("type_text", args={"text": "secret"})) + assert session.typed_via_backend == [] + callback = [item[1] for item in mock.requests if item[0].endswith("/callback")][-1] + assert callback["result"]["error"] == "type_refused" + + +def test_windows_native_is_coach_only() -> None: + mock = MockMailbox() + client, _out = _client(mock, platform="win32", url=None) + client.claim(PACK, BIND) + client.handle_envelope(_envelope("bind_pack")) + observed = client.handle_envelope(_envelope("observe")) + assert observed["coach_only"] is True + assert observed["agent_drive"] is False + assert observed["tree"] == [] + client.handle_envelope(_envelope("start_record")) + callback = [item[1] for item in mock.requests if item[0].endswith("/callback")][-1] + assert callback["result"]["error"] == "COACH_ONLY" + + +def test_linux_without_unique_title_is_coach_only() -> None: + mock = MockMailbox() + client, _out = _client( + mock, + platform="linux", + unique_window=lambda: {"backend": "linux", "window_title_unique": False}, + session=FakeAuthoringSession(backend="linux"), + ) + client.claim(PACK, BIND) + client.handle_envelope(_envelope("bind_pack")) + observed = client.handle_envelope(_envelope("observe")) + assert observed["coach_only"] is True + assert "title" not in json.dumps(observed) + + +def test_uncertain_delivery_does_not_blind_retry() -> None: + mock = MockMailbox() + client, _out = _client(mock) + client.claim(PACK, BIND) + client.handle_envelope(_envelope("bind_pack")) + client._uncertain = True + client.handle_envelope(_envelope("click", args={"node_id": "n_9f2c001a"})) + callback = [item[1] for item in mock.requests if item[0].endswith("/callback")][-1] + assert callback["result"]["error"] == "RECONCILIATION_REQUIRED" + client.handle_envelope(_envelope("click", args={"node_id": "n_9f2c001a"})) + clicks = [ + item + for item in mock.requests + if item[0].endswith("/callback") and item[1].get("command_id") == "cmd_click" + ] + assert len(clicks) == 1 + + +def test_titles_do_not_go_to_callback() -> None: + mock = MockMailbox() + client, _out = _client(mock) + client.claim(PACK, BIND) + client.handle_envelope(_envelope("bind_pack")) + client.handle_envelope(_envelope("observe")) + callback = [item[1] for item in mock.requests if item[0].endswith("/callback")][-1] + blob = json.dumps(callback) + assert "title" not in blob + assert "Jane Roe" not in blob + assert "4111111111111111" not in blob + + +def test_pack_url_without_bind_fails_closed() -> None: + with pytest.raises(MailboxError, match="not a runner link"): + connect_mailbox( + PACK_URL, + post=MockMailbox().post, + session=FakeAuthoringSession(), + max_polls=1, + stdout=io.StringIO(), + ) + + +def test_open_mailbox_transport_falls_back_when_desktop_missing(monkeypatch) -> None: + import sys + import types + + monkeypatch.setitem(sys.modules, "engine", types.ModuleType("engine")) + monkeypatch.delitem(sys.modules, "engine.authoring_runner", raising=False) + + def missing(*_args, **_kwargs): + raise ImportError("engine.authoring_runner") + + monkeypatch.setattr( + "openadapt_agent.mailbox._try_desktop_transport", + lambda **_kwargs: None, + ) + transport = open_mailbox_transport() + assert isinstance(transport, MailboxTransport) + + +def test_open_mailbox_transport_prefers_desktop_when_importable(monkeypatch) -> None: + class DesktopTransport: + def __init__(self, **kwargs): + self.kwargs = kwargs + + monkeypatch.setattr( + "openadapt_agent.mailbox._try_desktop_transport", + lambda **kwargs: DesktopTransport(**kwargs), + ) + transport = open_mailbox_transport() + assert isinstance(transport, DesktopTransport) + + +def test_mailbox_sources_have_no_listener_or_win_agent() -> None: + root = Path(__file__).resolve().parents[1] / "src" / "openadapt_agent" + for name in ("mailbox.py", "runner_bind.py", "cli.py"): + text = (root / name).read_text(encoding="utf-8") + assert "HTTPServer" not in text + assert "uvicorn" not in text + assert "streamable_http" not in text.lower() + assert "FastAPI" not in text + assert "win_agent" not in text + assert "parallels_vm" not in text + assert "127.0.0.1" not in text + assert "ngrok" not in text + bind_src = (root / "runner_bind.py").read_text(encoding="utf-8") + assert "wait_seconds" not in bind_src + assert "def claim" not in bind_src + mailbox_src = (root / "mailbox.py").read_text(encoding="utf-8") + assert 'method="POST"' in mailbox_src + assert 'method="GET"' not in mailbox_src + + +def test_url_pin_refuses_nonempty_cookies() -> None: + class Browser: + def cookies(self): + return [{"name": "sid", "value": "1"}] + + with pytest.raises(MailboxError, match="empty cookies"): + require_empty_cookies(Browser()) + + class Empty: + def cookies(self): + return [] + + require_empty_cookies(Empty()) + + +def test_try_desktop_transport_is_none_without_engine() -> None: + from openadapt_agent.mailbox import _try_desktop_transport + + assert _try_desktop_transport(origin=AUTHORING_ORIGIN) is None + + +def test_replace_allow_required_for_a_second_sub() -> None: + mock = MockMailbox() + answers = iter([True, False, True]) + client, _out = _client(mock, prompt=lambda _message: next(answers)) + client.claim(PACK, BIND) + client.handle_envelope(_envelope("bind_pack")) + assert client._allowed_sub == SUB + second = _envelope("bind_pack", sub=OTHER_SUB) + second["command_id"] = "cmd_bind_pack_other" + denied = client.handle_envelope(second) + assert denied["error"] == "denied" + client._pending_allow = { + "command_id": "cmd_bind_pack_2", + "oauth_sub_sha256": OTHER_SUB, + "client_display": "ChatGPT", + "replace": True, + } + client.allow(replace=True) + assert client._allowed_sub == OTHER_SUB + + +def test_connect_help_does_not_name_mockmed(capsys) -> None: + from openadapt_agent.cli import build_parser + + with pytest.raises(SystemExit) as exc_info: + build_parser().parse_args(["authoring", "connect", "--help"]) + assert exc_info.value.code == 0 + out = capsys.readouterr().out + assert "openadapt://runner" in out + assert "Allow" in out + assert "MockMed" not in out + assert "password" not in out.lower()