diff --git a/CHANGELOG.md b/CHANGELOG.md index 29f0f252..0172fa5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,11 +13,14 @@ All notable changes to SkillOpt are documented here. This project adheres to - **OpenCode transcript source** (`--source opencode`) for SkillOpt-Sleep. It reads visible user/assistant text and tool names from OpenCode's local SQLite history without requiring its CLI, login, or a provider connection. -- **OpenCode CLI backend** (`--backend opencode`) for SkillOpt-Sleep model calls, - including plain task replay, using an installed OpenCode CLI with the user's - existing login and file-based global configuration. Calls parse OpenCode's - JSONL output and disable project configuration, tool use, external plugins, - and configured MCP servers. Tool-aware replay remains follow-up work. +- **OpenCode CLI backend** (`--backend opencode`) for SkillOpt-Sleep model calls. + It uses an installed OpenCode CLI with the user's existing login and + file-based global configuration, and supports plain task replay plus opt-in + tool-aware replay. Plain calls disable project configuration, + model-initiated tool invocation, external plugins, and configured MCP servers. + Tool-aware replay exposes only temporary synthetic tools with randomized names + and fixed results, verifies which tools OpenCode actually invokes, and never + retains or replays historical tool arguments or results. - **GitHub Copilot CLI backend**, in two forms: `copilot_chat` (usable as both optimizer and target) and `copilot_exec` (target-only execution harness). Because the Copilot CLI carries its own sign-in, `--backend copilot` selects @@ -106,8 +109,8 @@ All notable changes to SkillOpt are documented here. This project adheres to @Alphaxalchemy's #129). ### Tests -- Add focused OpenCode backend coverage and opt-in real-CLI smoke tests for a - plain call and a seeded cycle-level run. +- Add focused OpenCode backend coverage and opt-in real-CLI smoke tests for + tool-aware replay and a seeded plain cycle. - Strengthen SkillOpt-Sleep verifier-discipline assertions, including recorded scores and gate actions (thanks @Tanmay9223, #96). - Add focused coverage for the validation-gate decision core and edit-budget diff --git a/docs/reference/cli.md b/docs/reference/cli.md index 9fdcb91c..3f7a8db7 100644 --- a/docs/reference/cli.md +++ b/docs/reference/cli.md @@ -139,6 +139,7 @@ Actions are `run`, `dry-run`, `status`, `adopt`, `harvest`, `schedule`, and | `--pi-path PATH` | Path to the installed Pi coding-agent CLI | | `--opencode-path PATH` | Path to the installed OpenCode CLI | | `--opencode-db PATH` | Path to the OpenCode SQLite history database | +| `--opencode-tool-replay` | Enable OpenCode tool-aware replay for `tool_called` checks in rule judges | | `--preferences TEXT` | House rules supplied to reflection | | `--lookback-hours N` | Initial transcript lookback; `0` scans all history | | `--max-sessions N` / `--max-tasks N` | Bound the harvested workload | @@ -256,6 +257,9 @@ Install and configure OpenCode using its [official documentation](https://opencode.ai/docs/), then confirm the CLI is available with `opencode --version`. +SkillOpt's OpenCode backend, including tool-aware replay, has been tested with +OpenCode CLI `1.18.15`. Other versions may work but have not been validated. + If OpenCode is on `PATH`, no path option is needed. Otherwise use `--opencode-path`, the `opencode_path` config key, or `SKILLOPT_SLEEP_OPENCODE_PATH`. Use `--model`, the `model` config key, or @@ -268,28 +272,41 @@ skillopt-sleep run --project "$(pwd)" \ --model provider/model --max-sessions 5 --max-tasks 3 --progress ``` -Plain calls run from a temporary directory with project configuration, tool -use, and external plugins disabled. Before contacting the model, SkillOpt -discovers the resolved MCP configuration, disables every -configured MCP server for the call, and verifies that none remains enabled. If -that check fails, the model call is not made. Tool-aware replay is not yet -supported. - -The child process keeps normal OpenCode file-based global configuration and -data directories. SkillOpt sets `OPENCODE_CONFIG_CONTENT` for the child process -to define the temporary agent and disable configured MCP servers. This replaces -the user's existing value in that child process, so settings supplied only -through that value are unavailable. Because `--pure` skips external plugins, +Plain calls run from a temporary directory with project configuration and +model-initiated tool invocation disabled. Before contacting the model, SkillOpt +discovers the resolved MCP configuration, disables every configured MCP server +for the call, and verifies that none remains enabled. If that check fails, the +model call is not made. + +OpenCode tool-aware replay is disabled by default. Enable it explicitly with +`--opencode-tool-replay` or `"opencode_tool_replay": true` in +`~/.skillopt-sleep/config.json`. It applies only to tasks whose rule judge +contains a non-empty `tool_called` check. In a fresh temporary workspace, +SkillOpt creates synthetic tools with randomized names and fixed results, then +verifies which tools OpenCode actually invoked. Historical tool arguments and +results are neither retained nor replayed. Configured MCP servers remain +disabled, and the invocation allowlist includes only these temporary tools. + +Both modes continue to use OpenCode's normal data directory and file-based +global configuration. OpenCode may discover or initialize custom JS/TS tools +from its global configuration directories, although SkillOpt does not allow the +model to invoke them. SkillOpt replaces `OPENCODE_CONFIG_CONTENT` in the child +process to define the temporary agent and disable configured MCP servers, so +settings present only in the user's original value are unavailable. Because +`--pure` skips external plugins, authentication or provider setup that depends on one of those plugins is also -unavailable. Calls may appear in the user's normal OpenCode session history; -these controls are invocation settings, not complete account or process -isolation. +unavailable. Calls may appear in the user's OpenCode session history. During +tool-aware replay, the fixed input (`synthetic`), fixed result, and temporary +project metadata may remain there as well. These controls are invocation +settings, not complete account or process isolation. The managed scheduler stores the backend but not `--source`, `--opencode-db`, -`--opencode-path`, or `--model`. Put `transcript_source`, `opencode_db`, -`opencode_path`, and `model` in `~/.skillopt-sleep/config.json` as needed. Use -absolute database and executable paths, and verify OpenCode access when the -scheduled run uses the backend. +`--opencode-path`, `--opencode-tool-replay`, or `--model`. Put +`transcript_source`, `opencode_db`, `opencode_path`, and `model` in +`~/.skillopt-sleep/config.json` as needed. Add +`"opencode_tool_replay": true` only when the scheduled run should enable +tool-aware replay. Use absolute database and executable paths, and verify +OpenCode access when the scheduled run uses the backend. ### Cursor source and backend diff --git a/docs/sleep/README.md b/docs/sleep/README.md index 1687f2c2..d656d965 100644 --- a/docs/sleep/README.md +++ b/docs/sleep/README.md @@ -212,21 +212,27 @@ skillopt-sleep run --project "$(pwd)" \ --opencode-path /absolute/path/to/opencode --model provider/model ``` -For plain calls, SkillOpt disables project configuration, tool use, external -plugins, and configured MCP servers. It stops before the model call if it cannot -confirm that every resolved MCP server is disabled. The subprocess keeps -OpenCode's normal data directory, so calls may appear in the user's OpenCode -session history. SkillOpt sets `OPENCODE_CONFIG_CONTENT` for the child process -to define the temporary agent and disable configured MCP servers. This replaces -the user's existing value in that child process, so settings supplied only -through that value are unavailable; use file-based global configuration or -provider environment variables instead. - -Tool-aware replay and a native OpenCode plugin or command are not implemented -yet. For scheduled runs, configure the source, database, executable, and model -in `~/.skillopt-sleep/config.json` as needed; the -[CLI reference](../reference/cli.md#opencode-source-and-backend) has the full -scheduler details. +Plain calls disable project configuration, model-initiated tool invocation, +external plugins, and configured MCP servers. SkillOpt stops before the model +call if it cannot confirm that every resolved MCP server is disabled. + +Tool-aware replay is disabled by default. Enable it with +`--opencode-tool-replay` or `"opencode_tool_replay": true` for tasks whose rule +judge contains a `tool_called` check. It exposes temporary synthetic tools with +randomized names and fixed results, verifies which tools OpenCode actually +invokes, and denies all other tools. Historical tool arguments and results are +not retained or replayed. + +Both modes continue to use OpenCode's normal data directory and file-based +global configuration. Calls may therefore appear in session history, and global +custom JS/TS tools may initialize, although SkillOpt does not allow the model to +invoke them. See the +[CLI reference](../reference/cli.md#opencode-source-and-backend) for complete +configuration, history, and isolation details. + +For scheduled runs, configure the source, database, executable, and model in +`~/.skillopt-sleep/config.json` as needed. Set `opencode_tool_replay` to `true` +there to opt in to tool-aware replay. ### Cursor diff --git a/plugins/README.md b/plugins/README.md index 7bfc1ff2..31cf77b9 100644 --- a/plugins/README.md +++ b/plugins/README.md @@ -97,11 +97,14 @@ optimization. CLI, authentication, or provider access. See [the CLI reference](../docs/reference/cli.md#opencode-source-and-backend) for its retained-data boundary. -- The core `opencode` backend uses the installed OpenCode CLI for plain model - calls. It keeps the user's login and file-based global configuration - while disabling project configuration, tool use, external plugins, and - configured MCP servers for those calls. Tool-aware replay and a native - OpenCode plugin or command are not included yet. +- The core `opencode` backend uses the installed OpenCode CLI. Plain calls + disable project configuration, model-initiated tool invocation, external + plugins, and configured MCP servers. Tool-aware replay requires explicit + opt-in. It exposes only temporary synthetic tools with randomized names and + fixed results, then verifies which tools OpenCode actually invoked. See the + [CLI reference](../docs/reference/cli.md#opencode-source-and-backend) for + history and isolation details. A native OpenCode plugin or command is not + included. - Outbound prompts are not currently guaranteed to be free of secrets. Do not use a third-party provider on sensitive transcripts without reviewing the data source and the provider's retention policy. @@ -146,6 +149,7 @@ Common implemented flags include: | `--pi-path PATH` | auto-detect `pi` | select the Pi coding-agent CLI executable | | `--opencode-path PATH` | `SKILLOPT_SLEEP_OPENCODE_PATH`, then `opencode` on `PATH`/`PATHEXT` | select the OpenCode CLI executable | | `--opencode-db PATH` | `OPENCODE_DB`, `%LOCALAPPDATA%`/`%APPDATA%` (Windows), or `${XDG_DATA_HOME:-~/.local/share}/opencode/opencode.db` | select the OpenCode SQLite history database | +| `--opencode-tool-replay` | off | enable OpenCode tool-aware replay for `tool_called` checks in rule judges | | `--project PATH` | current directory | select the project and invoked harvest scope | | `--scope invoked\|all` | `invoked` | limit transcript harvesting | | `--target-skill-path PATH` | managed skill | select a specific `SKILL.md` to stage/adopt | diff --git a/plugins/openclaw/run_sleep.py b/plugins/openclaw/run_sleep.py index 9ba84f53..65872f36 100755 --- a/plugins/openclaw/run_sleep.py +++ b/plugins/openclaw/run_sleep.py @@ -44,6 +44,7 @@ def get_backend( pi_path="", cursor_path="", opencode_path="", + opencode_tool_replay=False, azure_endpoint="", project_dir="", ): @@ -57,6 +58,7 @@ def get_backend( pi_path=pi_path, cursor_path=cursor_path, opencode_path=opencode_path, + opencode_tool_replay=opencode_tool_replay, azure_endpoint=azure_endpoint, project_dir=project_dir, ) diff --git a/skillopt_sleep/__main__.py b/skillopt_sleep/__main__.py index b1c20931..0273d67c 100644 --- a/skillopt_sleep/__main__.py +++ b/skillopt_sleep/__main__.py @@ -83,6 +83,11 @@ def _add_common(p: argparse.ArgumentParser) -> None: p.add_argument("--cursor-path", default="", help="path to the Cursor Agent CLI") p.add_argument("--pi-path", default="", help="path to the Pi coding-agent CLI") p.add_argument("--opencode-path", default="", help="path to the OpenCode CLI") + p.add_argument( + "--opencode-tool-replay", + action="store_true", + help="allow controlled synthetic tools for OpenCode tool-aware replay", + ) p.add_argument("--claude-home", default="", help="override ~/.claude (also isolates state)") p.add_argument("--codex-home", default="", help="override ~/.codex for archived session harvest") p.add_argument("--cursor-home", default="", help="override ~/.cursor for Cursor session harvest") @@ -134,6 +139,8 @@ def _cfg_from_args(args, task_meta: Dict[str, Any] | None = None) -> Any: overrides["cursor_path"] = os.path.abspath(os.path.expanduser(args.cursor_path)) if getattr(args, "opencode_path", ""): overrides["opencode_path"] = os.path.abspath(os.path.expanduser(args.opencode_path)) + if getattr(args, "opencode_tool_replay", False): + overrides["opencode_tool_replay"] = True if getattr(args, "claude_home", ""): overrides["claude_home"] = os.path.abspath(args.claude_home) if getattr(args, "codex_home", ""): diff --git a/skillopt_sleep/backend.py b/skillopt_sleep/backend.py index c331a721..d2ca2260 100644 --- a/skillopt_sleep/backend.py +++ b/skillopt_sleep/backend.py @@ -23,8 +23,11 @@ import json import os import re +import secrets +import shutil import subprocess import tempfile +from dataclasses import dataclass from typing import Any, Dict, List, Optional, Tuple from skillopt_sleep.types import EditRecord, ReplayResult, TaskRecord @@ -867,39 +870,54 @@ def attempt_with_tools(self, task, skill, memory, tools): pass +_OPENCODE_SYNTHETIC_TOOL_QUERY = "synthetic" +_OPENCODE_SYNTHETIC_TOOL_RESULT = "Synthetic replay result available." +_OPENCODE_STREAM_ERRORS = { + "malformed_jsonl": "returned malformed JSONL", + "invalid_event": "returned an invalid JSONL event", + "mixed_session": "returned mixed sessions", + "error_event": "emitted an error event", + "unexpected_tool_event": "attempted unsupported tool use", + "invalid_tool_event": "returned an invalid tool event", + "unexpected_tool_id": "used an unexpected tool", + "tool_error": "reported a failed synthetic tool", + "incomplete_stream": "returned an incomplete stream", + "missing_final_text": "returned no final answer after tool use", + "empty_response": "returned an empty response", +} + + +class OpenCodeError(RuntimeError): + """An OpenCode failure with a message that is safe to expose to users.""" + + def __init__(self, message: str, *, prompt_chars: int = 0) -> None: + super().__init__(message) + self.prompt_chars = prompt_chars + + +@dataclass(frozen=True) +class _OpenCodeReplayProject: + agent_name: str + tool_mapping: Dict[str, str] + + def resolve_opencode_path(explicit: str = "") -> str: """Find the OpenCode CLI, including Windows npm ``.CMD`` launchers.""" - import shutil - - candidate = os.path.expanduser( - explicit or os.environ.get("SKILLOPT_SLEEP_OPENCODE_PATH", "") or "opencode" - ) + candidate = os.path.expanduser(explicit or os.environ.get("SKILLOPT_SLEEP_OPENCODE_PATH", "") or "opencode") resolved = shutil.which(candidate) if resolved: - # Make the result absolute before the child changes directory. return os.path.abspath(resolved) - if os.path.dirname(candidate): - # Do the same for a path-like value even if the file does not exist yet. - return os.path.abspath(candidate) - # Otherwise let the operating system find the command on PATH when it runs. - return candidate + return os.path.abspath(candidate) if os.path.dirname(candidate) else candidate -def _parse_opencode_jsonl_text(raw: str) -> Tuple[str, str]: - """Extract text and an error code from one OpenCode JSONL response.""" +def _parse_opencode_jsonl_events(raw: str, expected_tool_ids: Optional[set[str]] = None) -> Tuple[str, List[str], str]: + """Extract final text and the IDs of validated, completed tool calls from OpenCode JSONL.""" text_parts: List[str] = [] - session_id = "" + called: List[str] = [] + session_id = last_event = "" saw_step_start = False - last_known_event_type = "" - known_event_types = {"error", "reasoning", "step_finish", "step_start", "text", "tool_use"} - expected_part_types = { - "reasoning": "reasoning", - "step_finish": "step-finish", - "step_start": "step-start", - "text": "text", - "tool_use": "tool", - } - + needs_final_text = False + known = {"reasoning", "step_finish", "step_start", "text", "tool_use"} for raw_line in raw.splitlines(): line = raw_line.strip() if not line: @@ -907,46 +925,149 @@ def _parse_opencode_jsonl_text(raw: str) -> Tuple[str, str]: try: event = json.loads(line) except (ValueError, RecursionError): - return "", "malformed_jsonl" + return "", [], "malformed_jsonl" if not isinstance(event, dict): - return "", "invalid_event" - - event_type = event.get("type") - event_session_id = event.get("sessionID") + return "", [], "invalid_event" + event_type, event_session = event.get("type"), event.get("sessionID") if not isinstance(event_type, str) or not event_type: - return "", "invalid_event" - if not isinstance(event_session_id, str) or not event_session_id: - return "", "invalid_event" - if not session_id: - session_id = event_session_id - elif event_session_id != session_id: - return "", "mixed_session" - - if event_type not in known_event_types: - continue - last_known_event_type = event_type + return "", [], "invalid_event" + if not isinstance(event_session, str) or not event_session: + return "", [], "invalid_event" + if session_id and event_session != session_id: + return "", [], "mixed_session" + session_id = event_session if event_type == "error": - return "", "error_event" - - part = event.get("part") - if not isinstance(part, dict) or part.get("type") != expected_part_types[event_type]: - return "", "invalid_event" - if event_type == "tool_use": - return "", "unexpected_tool_event" - if event_type == "step_start": - saw_step_start = True - elif event_type == "text": + return "", [], "error_event" + if event_type not in known: + continue + last_event, part = event_type, event.get("part") + if event_type == "text": + if not isinstance(part, dict) or part.get("type") != "text": + return "", [], "invalid_event" text = part.get("text") if not isinstance(text, str): - return "", "invalid_event" + return "", [], "invalid_event" text_parts.append(text) - - if not saw_step_start or last_known_event_type != "step_finish": - return "", "incomplete_stream" + needs_final_text = needs_final_text and not bool(text.strip()) + elif event_type == "tool_use": + if expected_tool_ids is None: + return "", [], "unexpected_tool_event" + if not isinstance(part, dict) or part.get("type") != "tool": + return "", [], "invalid_tool_event" + tool_id, state = part.get("tool"), part.get("state") + if not isinstance(tool_id, str) or not isinstance(state, dict): + return "", [], "invalid_tool_event" + if tool_id not in expected_tool_ids: + return "", [], "unexpected_tool_id" + if state.get("status") == "error": + return "", [], "tool_error" + if ( + state.get("status") != "completed" + or state.get("input") != {"query": _OPENCODE_SYNTHETIC_TOOL_QUERY} + or state.get("output") != _OPENCODE_SYNTHETIC_TOOL_RESULT + ): + return "", [], "invalid_tool_event" + called.append(tool_id) + text_parts.clear() + needs_final_text = True + elif event_type in {"step_start", "step_finish"}: + part_type = "step-start" if event_type == "step_start" else "step-finish" + if not isinstance(part, dict) or part.get("type") != part_type: + return "", [], "invalid_event" + saw_step_start = saw_step_start or event_type == "step_start" + elif not isinstance(part, dict) or part.get("type") != "reasoning": + return "", [], "invalid_event" + if not saw_step_start or last_event != "step_finish": + return "", [], "incomplete_stream" + if needs_final_text: + return "", [], "missing_final_text" text = "\n".join(text_parts).strip() - if not text: - return "", "empty_response" - return text, "" + return (text, called, "") if text else ("", [], "empty_response") + + +def _write_exclusive_text(path: str, content: str, encoding: str = "utf-8") -> None: + with open(path, "x", encoding=encoding, newline="\n") as output: + output.write(content) + + +def _normalize_opencode_tool_names(tools: List[str]) -> List[str]: + normalized: List[str] = [] + seen: set[str] = set() + for value in tools: + if not isinstance(value, str): + raise OpenCodeError("OpenCode CLI tool replay received an invalid tool list") + name, folded = value.strip(), value.strip().casefold() + if not re.fullmatch(r"[A-Za-z0-9][A-Za-z0-9_.:/-]{0,127}", name): + raise OpenCodeError("OpenCode CLI tool replay received an invalid tool list") + if folded not in seen: + seen.add(folded) + normalized.append(name) + if not normalized or len(normalized) > 32: + raise OpenCodeError("OpenCode CLI tool replay received an invalid tool list") + return normalized + + +def _prepare_opencode_replay_project(work: str, tools: List[str]) -> _OpenCodeReplayProject: + """Create a replay project with a Git boundary, offline metadata, and randomly named JavaScript tools.""" + try: + project_id = f"skillopt-sleep-{secrets.token_hex(32)}" + git_dir = os.path.join(work, ".git") + os.makedirs(os.path.join(git_dir, "objects")) + os.makedirs(os.path.join(git_dir, "refs", "heads")) + git_files = { + "HEAD": "ref: refs/heads/main\n", + "config": "[core]\n\trepositoryformatversion = 0\n\tfilemode = false\n\tbare = false\n", + "opencode": project_id + "\n", + } + for name, content in git_files.items(): + _write_exclusive_text(os.path.join(git_dir, name), content, "ascii") + agent = f"skillopt-sleep-{secrets.token_hex(16)}" + mapping = {name: f"skillopt_replay_{secrets.token_hex(16)}" for name in tools} + if len(set(mapping.values())) != len(mapping): + raise ValueError("random tool identifier collision") + config_dir, package_name = os.path.join(work, ".opencode"), "skillopt-opencode-replay" + tool_dir = os.path.join(config_dir, "tools") + os.makedirs(tool_dir) + os.makedirs(os.path.join(config_dir, "node_modules")) + root = {"name": package_name, "dependencies": {"@opencode-ai/plugin": "0.0.0"}} + metadata = { + "package.json": {**root, "private": True}, + "package-lock.json": { + "name": package_name, + "lockfileVersion": 3, + "requires": True, + "packages": {"": root}, + }, + } + for name, payload in metadata.items(): + _write_exclusive_text(os.path.join(config_dir, name), json.dumps(payload, separators=(",", ":"))) + for logical_name, tool_id in mapping.items(): + description = f"Controlled synthetic stand-in for the {logical_name} tool." + source = ( + f"const query = {json.dumps(_OPENCODE_SYNTHETIC_TOOL_QUERY)};\n" + f"const result = {json.dumps(_OPENCODE_SYNTHETIC_TOOL_RESULT)};\n" + "export default {\n" + f" description: {json.dumps(description)},\n" + ' args: { query: { type: "string", description: "Use exactly synthetic." } },\n' + " async execute(args) {\n" + ' if (args === null || typeof args !== "object" || ' + "Object.keys(args).length !== 1 || args.query !== query) {\n" + ' throw new Error("Invalid synthetic replay input.");\n' + " }\n" + " return result;\n" + " },\n};\n" + ) + _write_exclusive_text(os.path.join(tool_dir, f"{tool_id}.js"), source) + return _OpenCodeReplayProject(agent, mapping) + except Exception: + raise OpenCodeError("OpenCode CLI tool replay could not be prepared") from None + + +def _opencode_temporary_workspace(prefix: str, error: str) -> tempfile.TemporaryDirectory: + try: + return tempfile.TemporaryDirectory(prefix=prefix, ignore_cleanup_errors=True) + except Exception: + raise OpenCodeError(error) from None class OpenCodeCliBackend(CliBackend): @@ -959,202 +1080,206 @@ def __init__( model: str = "", opencode_path: str = "", timeout: int = 180, + tool_replay: bool = False, ) -> None: super().__init__( model=model or os.environ.get("SKILLOPT_SLEEP_OPENCODE_MODEL", ""), timeout=timeout, ) self.opencode_path = resolve_opencode_path(opencode_path) + # Require the literal True so config strings cannot enable synthetic tools. + self.tool_replay = tool_replay is True - def _cached_call(self, key: str, prompt: str, *, max_tokens: int = 1024) -> str: - """Do not cache failed OpenCode calls.""" - if key in self._cache: - # A cached answer makes any earlier error irrelevant to this call. - self.last_call_error = "" - out = super()._cached_call(key, prompt, max_tokens=max_tokens) - if not out: - self._cache.pop(key, None) - return out - - def _read_mcp_disabled_statuses( + def _run_process( self, + command: List[str], env: Dict[str, str], work: str, - stage: str, - ) -> Optional[Dict[str, bool]]: - """Return whether each configured MCP server is explicitly disabled.""" + *, + operation: str, + input_text: Optional[str] = None, + report_exit_code: bool = False, + exception_action: str = "completed", + ) -> subprocess.CompletedProcess[str]: + kwargs: Dict[str, Any] = { + "capture_output": True, + "creationflags": _NO_WINDOW, + "text": True, + "encoding": "utf-8", + "errors": "replace", + "timeout": self.timeout, + "cwd": work, + "env": env, + } + if input_text is not None: + kwargs["input"] = input_text try: - proc = subprocess.run( - [self.opencode_path, "debug", "config", "--pure"], - capture_output=True, - creationflags=_NO_WINDOW, - text=True, - encoding="utf-8", - errors="replace", - timeout=self.timeout, - cwd=work, - env=env, - ) + proc = subprocess.run(command, **kwargs) except subprocess.TimeoutExpired: - self.last_call_error = f"OpenCode CLI {stage} timed out after {self.timeout}s" - return None + raise OpenCodeError(f"{operation} timed out after {self.timeout}s") from None except Exception: - self.last_call_error = f"OpenCode CLI {stage} could not be completed" - return None + raise OpenCodeError(f"{operation} could not be {exception_action}") from None + if proc.returncode: + detail = f"exited {proc.returncode}" if report_exit_code else "failed" + raise OpenCodeError(f"{operation} {detail}") + return proc - if proc.returncode != 0: - self.last_call_error = f"OpenCode CLI {stage} exited {proc.returncode}" - return None + def _read_debug_json( + self, + args: List[str], + env: Dict[str, str], + work: str, + operation: str, + report_exit_code: bool = False, + ) -> Dict[str, Any]: + proc = self._run_process( + [self.opencode_path, "debug", *args, "--pure"], + env, + work, + operation=operation, + report_exit_code=report_exit_code, + ) try: - resolved = json.loads(proc.stdout or "") + value = json.loads(proc.stdout or "") except (ValueError, RecursionError, TypeError): - self.last_call_error = f"OpenCode CLI {stage} returned invalid configuration" - return None - if not isinstance(resolved, dict): - self.last_call_error = f"OpenCode CLI {stage} returned invalid configuration" - return None + value = None + if not isinstance(value, dict): + raise OpenCodeError(f"{operation} returned invalid configuration") + return value - mcp = resolved.get("mcp", {}) - if not isinstance(mcp, dict): - self.last_call_error = f"OpenCode CLI {stage} returned invalid MCP configuration" - return None - disabled_by_name: Dict[str, bool] = {} - for name, entry in mcp.items(): - if not isinstance(name, str) or not isinstance(entry, dict): - self.last_call_error = f"OpenCode CLI {stage} returned invalid MCP configuration" - return None - disabled_by_name[name] = entry.get("enabled") is False - return disabled_by_name - - def _call(self, prompt: str, *, max_tokens: int = 1024) -> str: - import secrets - - del max_tokens - self.last_call_error = "" - # Use a per-call agent so settings from the user's default agent do not apply. - agent_name = f"skillopt-sleep-{secrets.token_hex(16)}" - work = tempfile.mkdtemp(prefix="skillopt_sleep_opencode_") - cmd = [ - self.opencode_path, - "run", - "--pure", - "--format", - "json", - "--agent", - agent_name, - "--title", - "skillopt-sleep", - "--dir", - work, - ] - if self.model: - cmd += ["--model", self.model] - - # Keep the user's login and file-based global settings. + @staticmethod + def _build_child_environment(work: str, permission: Dict[str, str], tools: bool = False) -> Dict[str, str]: env = os.environ.copy() - # The child changes directory below, so resolve relative config paths now. for key in ("OPENCODE_CONFIG", "OPENCODE_CONFIG_DIR"): - value = env.get(key) - if value: - env[key] = os.path.abspath(os.path.expanduser(value)) + if env.get(key): + env[key] = os.path.abspath(os.path.expanduser(env[key])) env.update( { "NO_COLOR": "1", "OPENCODE_DISABLE_AUTOUPDATE": "1", "OPENCODE_DISABLE_EXTERNAL_SKILLS": "1", - "OPENCODE_DISABLE_PROJECT_CONFIG": "1", + "OPENCODE_DISABLE_PROJECT_CONFIG": "0" if tools else "1", "OPENCODE_DISABLE_SHARE": "1", "OPENCODE_DISABLE_TERMINAL_TITLE": "1", - "OPENCODE_PERMISSION": '{"*":"deny"}', + "OPENCODE_PERMISSION": json.dumps(permission, separators=(",", ":")), "OPENCODE_PURE": "1", } ) - env["PWD"] = work - env.pop("OLDPWD", None) - # Define the per-call agent without changing the user's config files. - plain_config = { - "agent": { - agent_name: { - "mode": "primary", - "permission": {"*": "deny"}, - } + npm: Dict[str, str] = {} + blocked = {"OPENCODE_DIRECT_TRACE"} + if tools: + env.update({"OPENCODE_DISABLE_LSP_DOWNLOAD": "1", "OPENCODE_DISABLE_MODELS_FETCH": "1"}) + npm = { + "npm_config_audit": "false", + "npm_config_cache": os.path.join(work, ".npm-cache"), + "npm_config_fetch_retries": "0", + "npm_config_fetch_retry_maxtimeout": "100", + "npm_config_fetch_retry_mintimeout": "100", + "npm_config_fetch_timeout": "1000", + "npm_config_fund": "false", + "npm_config_offline": "true", + "npm_config_update_notifier": "false", } - } - env["OPENCODE_CONFIG_CONTENT"] = json.dumps( - plain_config, - separators=(",", ":"), - ) - try: - # Tool permissions alone do not disable configured MCP servers. Read - # the merged config, disable every server it contains, and verify the - # result before calling the model. - discovered_mcp_statuses = self._read_mcp_disabled_statuses( - env, work, "MCP discovery" - ) - if discovered_mcp_statuses is None: - return "" - plain_config["mcp"] = { - name: {"enabled": False} for name in sorted(discovered_mcp_statuses) + blocked |= {key.upper() for key in npm} | { + "GIT_COMMON_DIR", + "GIT_DIR", + "GIT_OBJECT_DIRECTORY", + "GIT_WORK_TREE", } - env["OPENCODE_CONFIG_CONTENT"] = json.dumps( - plain_config, - separators=(",", ":"), - ) - verified_mcp_statuses = self._read_mcp_disabled_statuses( - env, work, "MCP verification" - ) - if verified_mcp_statuses is None: - return "" - if any(not disabled for disabled in verified_mcp_statuses.values()): - self.last_call_error = "OpenCode CLI could not disable every configured MCP server" - return "" + env = {key: value for key, value in env.items() if key.upper() not in blocked} + env.update(npm, PWD=work) + env.pop("OLDPWD", None) + return env - try: - proc = subprocess.run( - cmd, - capture_output=True, - creationflags=_NO_WINDOW, - text=True, - encoding="utf-8", - errors="replace", - timeout=self.timeout, - cwd=work, - env=env, - input=prompt, - ) - except subprocess.TimeoutExpired: - self.last_call_error = f"OpenCode CLI timed out after {self.timeout}s" - return "" - except Exception: - self.last_call_error = "OpenCode CLI could not be executed" - return "" - finally: - try: - import shutil + def _build_run_command(self, work: str, agent: str, title: str) -> List[str]: + command = [ + self.opencode_path, + "run", + "--pure", + "--format", + "json", + "--agent", + agent, + "--title", + title, + "--dir", + work, + ] + return command + (["--model", self.model] if self.model else []) - shutil.rmtree(work, ignore_errors=True) - except Exception: - pass + def _read_mcp_config(self, env: Dict[str, str], work: str, stage: str) -> Tuple[Dict[str, Any], Dict[str, bool]]: + resolved = self._read_debug_json(["config"], env, work, f"OpenCode CLI {stage}", True) + mcp = resolved.get("mcp", {}) + if not isinstance(mcp, dict) or any( + not isinstance(name, str) or not isinstance(entry, dict) for name, entry in mcp.items() + ): + raise OpenCodeError(f"OpenCode CLI {stage} returned invalid MCP configuration") + return resolved, {name: entry.get("enabled") is False for name, entry in mcp.items()} + + def _disable_and_verify_mcp(self, env: Dict[str, str], work: str, config: Dict[str, Any]) -> None: + _resolved, discovered = self._read_mcp_config(env, work, "MCP discovery") + # Disable every discovered MCP server, including those already disabled. + effective = {**config, "mcp": {name: {"enabled": False} for name in sorted(discovered)}} + env["OPENCODE_CONFIG_CONTENT"] = json.dumps(effective, separators=(",", ":")) + verified, statuses = self._read_mcp_config(env, work, "MCP verification") + if any(not disabled for disabled in statuses.values()): + raise OpenCodeError("OpenCode CLI could not disable every configured MCP server") + if verified.get("snapshot") is not False: + raise OpenCodeError("OpenCode CLI could not disable session snapshots") + + def _verify_tool_allowlist(self, env: Dict[str, str], work: str, agent: str, expected: set[str]) -> None: + resolved = self._read_debug_json(["agent", agent], env, work, "OpenCode CLI tool permission verification") + tools = resolved.get("tools") + if not isinstance(tools, dict) or any( + not isinstance(name, str) or not isinstance(value, bool) for name, value in tools.items() + ): + raise OpenCodeError("OpenCode CLI tool permission verification returned invalid configuration") + if {name for name, enabled in tools.items() if enabled} != expected: + raise OpenCodeError("OpenCode CLI could not restrict tools to the replay allowlist") - if proc.returncode != 0: - self.last_call_error = f"OpenCode CLI exited {proc.returncode}" - return "" - text, error_code = _parse_opencode_jsonl_text(proc.stdout or "") - if error_code: - error_messages = { - "malformed_jsonl": "OpenCode CLI returned malformed JSONL", - "invalid_event": "OpenCode CLI returned an invalid JSONL event", - "mixed_session": "OpenCode CLI returned mixed sessions", - "error_event": "OpenCode CLI emitted an error event", - "unexpected_tool_event": "OpenCode CLI attempted unsupported tool use", - "incomplete_stream": "OpenCode CLI returned an incomplete stream", - "empty_response": "OpenCode CLI returned an empty response", - } - self.last_call_error = error_messages.get( - error_code, "OpenCode CLI returned an unknown JSONL error" - ) + def _cached_call(self, key: str, prompt: str, *, max_tokens: int = 1024) -> str: + """Keep failed OpenCode calls out of the cache.""" + if key in self._cache: + self.last_call_error = "" + out = super()._cached_call(key, prompt, max_tokens=max_tokens) + if not out: + self._cache.pop(key, None) + return out + + def _call(self, prompt: str, *, max_tokens: int = 1024) -> str: + del max_tokens + self.last_call_error = "" + try: + with _opencode_temporary_workspace( + "skillopt_sleep_opencode_", + "OpenCode CLI workspace could not be prepared", + ) as work: + try: + agent = f"skillopt-sleep-{secrets.token_hex(16)}" + except Exception: + raise OpenCodeError("OpenCode CLI workspace could not be prepared") from None + permission = {"*": "deny"} + config = {"snapshot": False, "agent": {agent: {"mode": "primary", "permission": permission}}} + env = self._build_child_environment(work, permission) + env["OPENCODE_CONFIG_CONTENT"] = json.dumps(config, separators=(",", ":")) + self._disable_and_verify_mcp(env, work, config) + proc = self._run_process( + self._build_run_command(work, agent, "skillopt-sleep"), + env, + work, + operation="OpenCode CLI", + input_text=prompt, + report_exit_code=True, + exception_action="executed", + ) + text, _called, error = _parse_opencode_jsonl_events(proc.stdout or "") + if error: + detail = _OPENCODE_STREAM_ERRORS.get(error, "returned an unknown JSONL error") + raise OpenCodeError(f"OpenCode CLI {detail}") + return text + except OpenCodeError as exc: + self.last_call_error = str(exc) return "" - return text def attempt_with_tools( self, @@ -1163,9 +1288,79 @@ def attempt_with_tools( memory: str, tools: List[str], ) -> Tuple[str, List[str]]: - del task, skill, memory, tools - self.last_call_error = "OpenCode CLI tool replay is not supported" - return "", [] + self.last_call_error = "" + if not self.tool_replay: + self.last_call_error = ( + "OpenCode CLI tool replay is not supported without explicit " + "opencode_tool_replay opt-in" + ) + return "", [] + try: + logical_tools = _normalize_opencode_tool_names(tools) + with _opencode_temporary_workspace( + "skillopt_sleep_opencode_tools_", + "OpenCode CLI tool replay could not be prepared", + ) as work: + project = _prepare_opencode_replay_project(work, logical_tools) + expected = set(project.tool_mapping.values()) + permission = {"*": "deny", **dict.fromkeys(expected, "allow")} + agent_config: Dict[str, Any] = {"mode": "primary", "permission": permission} + if self.model: + agent_config["model"] = self.model + config: Dict[str, Any] = { + "share": "disabled", + "autoupdate": False, + "formatter": False, + "lsp": False, + "snapshot": False, + "permission": permission, + "agent": {project.agent_name: agent_config}, + } + env = self._build_child_environment(work, permission, True) + env["OPENCODE_CONFIG_CONTENT"] = json.dumps(config, separators=(",", ":")) + self._disable_and_verify_mcp(env, work, config) + self._verify_tool_allowlist(env, work, project.agent_name, expected) + tool_lines = "\n".join( + f"- Logical tool {json.dumps(name)} is available as `{tool_id}`; when required, " + f'invoke it with {{"query":{json.dumps(_OPENCODE_SYNTHETIC_TOOL_QUERY)}}}.' + for name, tool_id in project.tool_mapping.items() + ) + prompt = ( + "Complete the task. Apply the skill and memory rules exactly, including any rule " + "requiring a tool call before answering. Treat a 'Learned preferences' block as hard " + "constraints that override earlier conflicting skill text. The tools below are controlled " + "synthetic stand-ins. Use one only when the task, skill, or memory requires its logical " + "tool; do not call a tool merely because it is listed. When required, invoke the random " + "internal ID instead of claiming that you called it. Each call returns the same fixed result.\n\n" + f"# Controlled tools\n{tool_lines}\n\n# Skill\n{skill or '(none)'}\n\n" + f"# Memory\n{memory or '(none)'}\n\n# Task\n{task.intent}\n\n{task.context_excerpt}\n\n" + "Return only the final answer text." + ) + try: + proc = self._run_process( + self._build_run_command(work, project.agent_name, "skillopt-sleep-tool-replay"), + env, + work, + operation="OpenCode CLI tool replay", + input_text=prompt, + exception_action="executed", + ) + text, called_ids, error = _parse_opencode_jsonl_events(proc.stdout or "", expected) + if error: + detail = _OPENCODE_STREAM_ERRORS.get(error, "returned an unknown JSONL error") + raise OpenCodeError(f"OpenCode CLI tool replay {detail}") + except OpenCodeError as exc: + raise OpenCodeError(str(exc), prompt_chars=len(prompt)) from None + called = set(called_ids) + called_tools = [ + name for name, tool_id in project.tool_mapping.items() if tool_id in called + ] + except OpenCodeError as exc: + self._tokens += exc.prompt_chars // 4 + self.last_call_error = str(exc) + return "", [] + self._tokens += len(prompt) // 4 + len(text) // 4 + return text, called_tools def resolve_codex_path(explicit: str = "") -> str: @@ -2305,6 +2500,7 @@ def get_backend( pi_path: str = "", cursor_path: str = "", opencode_path: str = "", + opencode_tool_replay: bool = False, azure_endpoint: str = "", project_dir: str = "", ) -> Backend: @@ -2325,7 +2521,11 @@ def get_backend( if n in {"cursor", "cursor_agent", "cursor_cli"}: return CursorCliBackend(model=model, cursor_path=cursor_path) if n in {"opencode", "opencode_cli", "opencode-cli"}: - return OpenCodeCliBackend(model=model, opencode_path=opencode_path) + return OpenCodeCliBackend( + model=model, + opencode_path=opencode_path, + tool_replay=opencode_tool_replay, + ) if n in {"handoff", "session", "file"}: # Lazy import: handoff_backend imports CliBackend from this module. from skillopt_sleep.handoff_backend import HandoffBackend @@ -2348,6 +2548,7 @@ def build_backend( pi_path: str = "", cursor_path: str = "", opencode_path: str = "", + opencode_tool_replay: bool = False, azure_endpoint: str = "", preferences: str = "", project_dir: str = "", @@ -2368,6 +2569,7 @@ def build_backend( pi_path=pi_path, cursor_path=cursor_path, opencode_path=opencode_path, + opencode_tool_replay=opencode_tool_replay, azure_endpoint=azure_endpoint, project_dir=project_dir, ) @@ -2375,11 +2577,13 @@ def build_backend( return be tgt = get_backend(target_backend or backend, model=target_model or model, codex_path=codex_path, pi_path=pi_path, cursor_path=cursor_path, - opencode_path=opencode_path, azure_endpoint=azure_endpoint, + opencode_path=opencode_path, opencode_tool_replay=opencode_tool_replay, + azure_endpoint=azure_endpoint, project_dir=project_dir) opt = get_backend(optimizer_backend or backend, model=optimizer_model or model, codex_path=codex_path, pi_path=pi_path, cursor_path=cursor_path, - opencode_path=opencode_path, azure_endpoint=azure_endpoint, + opencode_path=opencode_path, opencode_tool_replay=opencode_tool_replay, + azure_endpoint=azure_endpoint, project_dir=project_dir) opt.preferences = preferences # reflect runs on the optimizer dual = DualBackend(target=tgt, optimizer=opt) diff --git a/skillopt_sleep/config.py b/skillopt_sleep/config.py index 6aa5a716..6c279f68 100644 --- a/skillopt_sleep/config.py +++ b/skillopt_sleep/config.py @@ -60,6 +60,7 @@ "pi_path": "", # "" => use `pi` on PATH "cursor_path": "", # "" => auto-detect the Cursor Agent CLI "opencode_path": "", # "" => SKILLOPT_SLEEP_OPENCODE_PATH, then `opencode` on PATH/PATHEXT + "opencode_tool_replay": False, # explicit opt-in for OpenCode tool-aware replay "edit_budget": 4, # textual learning rate (max edits/night) "preferences": "", # free-text house rules injected into reflect as a prior "gate_metric": "mixed", # hard | soft | mixed (mixed best for tiny holdouts) diff --git a/skillopt_sleep/cycle.py b/skillopt_sleep/cycle.py index 0cbf24fe..4cdb73dc 100644 --- a/skillopt_sleep/cycle.py +++ b/skillopt_sleep/cycle.py @@ -56,6 +56,7 @@ def _make_model_key(cfg: SleepConfig) -> str: pi_path=cfg.get("pi_path", ""), cursor_path=cfg.get("cursor_path", ""), opencode_path=cfg.get("opencode_path", ""), + opencode_tool_replay=cfg.get("opencode_tool_replay", False), azure_endpoint=cfg.get("azure_endpoint", ""), project_dir=cfg.get("invoked_project", "") or os.getcwd(), ) @@ -360,6 +361,7 @@ def run_sleep_cycle( pi_path=cfg.get("pi_path", ""), cursor_path=cfg.get("cursor_path", ""), opencode_path=cfg.get("opencode_path", ""), + opencode_tool_replay=cfg.get("opencode_tool_replay", False), azure_endpoint=cfg.get("azure_endpoint", ""), preferences=cfg.get("preferences", ""), project_dir=project, @@ -392,16 +394,20 @@ def run_sleep_cycle( redact=bool(cfg.get("redact_secrets", True)), ) evidence.attach(backend, ev) + cycle_config = {k: cfg.get(k) for k in ( + "backend", "model", "optimizer_backend", "optimizer_model", + "target_backend", "target_model", "gate_mode", "gate_metric", + "gate_mixed_weight", "gate_no_regression", "edit_budget", + "holdout_fraction", + "dream_rollouts", "dream_factor", "recall_k", + "max_tasks_per_night", "lookback_hours", "llm_mine", + "evolve_skill", "evolve_memory")} + cycle_config["opencode_tool_replay"] = ( + cfg.get("opencode_tool_replay", False) is True + ) ev.log("cycle", "start", night=night, project=project, backend=backend.name, model=cfg.get("model", ""), - config={k: cfg.get(k) for k in ( - "backend", "model", "optimizer_backend", "optimizer_model", - "target_backend", "target_model", "gate_mode", "gate_metric", - "gate_mixed_weight", "gate_no_regression", "edit_budget", - "holdout_fraction", - "dream_rollouts", "dream_factor", "recall_k", - "max_tasks_per_night", "lookback_hours", "llm_mine", - "evolve_skill", "evolve_memory")}) + config=cycle_config) # ── live skill/memory docs ─────────────────────────────────────────── live_memory_path = os.path.join(project, "CLAUDE.md") @@ -659,6 +665,9 @@ def run_sleep_cycle( json_safe({ "night": night, "backend": cfg.get("backend"), + "opencode_tool_replay": ( + cfg.get("opencode_tool_replay", False) is True + ), "gate_mode": cfg.get("gate_mode"), "gate_no_regression": cfg.get("gate_no_regression", False), "n_tasks": len(tasks), diff --git a/tests/test_backend_opencode.py b/tests/test_backend_opencode.py index de06f911..71e173cf 100644 --- a/tests/test_backend_opencode.py +++ b/tests/test_backend_opencode.py @@ -6,6 +6,7 @@ import json import os import subprocess +from pathlib import Path from unittest import mock import pytest @@ -14,15 +15,19 @@ from skillopt_sleep.__main__ import _add_common, _cfg_from_args from skillopt_sleep.backend import ( _NO_WINDOW, + _OPENCODE_SYNTHETIC_TOOL_QUERY, + _OPENCODE_SYNTHETIC_TOOL_RESULT, DualBackend, MockBackend, OpenCodeCliBackend, - _parse_opencode_jsonl_text, + OpenCodeError, + _parse_opencode_jsonl_events, build_backend, get_backend, resolve_opencode_path, ) from skillopt_sleep.config import DEFAULTS, SleepConfig, load_config +from skillopt_sleep.types import TaskRecord class _FakeProc: @@ -48,7 +53,41 @@ def _success_stream(*texts: str) -> str: return "\n".join(lines) -def _resolved_mcp(*names: str, disabled: bool = False) -> str: +def _tool_event( + tool_id: str, + *, + call_id: str = "call-1", + status: str = "completed", + output: str | None = None, + query=_OPENCODE_SYNTHETIC_TOOL_QUERY, +) -> str: + state = {"status": status, "input": {"query": query}} + if output is not None: + state["output"] = output + return _event( + "tool_use", + part={"type": "tool", "tool": tool_id, "callID": call_id, "state": state}, + ) + + +def _tool_success_stream(tool_id: str, text: str = "answer") -> str: + return "\n".join( + [ + _event("step_start", part={"type": "step-start"}), + _tool_event(tool_id, output=_OPENCODE_SYNTHETIC_TOOL_RESULT), + _event("step_finish", part={"type": "step-finish"}), + _event("step_start", part={"type": "step-start"}), + _event("text", part={"type": "text", "text": text}), + _event("step_finish", part={"type": "step-finish"}), + ] + ) + + +def _resolved_mcp( + *names: str, + disabled: bool = False, + snapshot: bool | None = None, +) -> str: mcp = { name: { "type": "local", @@ -57,17 +96,74 @@ def _resolved_mcp(*names: str, disabled: bool = False) -> str: } for name in names } - return json.dumps({"mcp": mcp}) + resolved = {"mcp": mcp} + if snapshot is not None: + resolved["snapshot"] = snapshot + return json.dumps(resolved) def _successful_plain_results(*mcp_names: str, answer: str = "answer") -> list[_FakeProc]: return [ - _FakeProc(_resolved_mcp(*mcp_names)), - _FakeProc(_resolved_mcp(*mcp_names, disabled=True)), + _FakeProc(_resolved_mcp(*mcp_names, snapshot=False)), + _FakeProc(_resolved_mcp(*mcp_names, disabled=True, snapshot=False)), _FakeProc(_success_stream(answer)), ] +def _assert_replay_permissions(config, env, agent_name: str, expected_permission: dict[str, str]) -> None: + assert json.loads(env["OPENCODE_PERMISSION"]) == expected_permission + assert config["permission"] == expected_permission + assert config["agent"][agent_name]["permission"] == expected_permission + + +def _assert_controlled_tool_environment(env, work: str) -> None: + expected_npm = { + "npm_config_audit": "false", + "npm_config_cache": os.path.join(work, ".npm-cache"), + "npm_config_fetch_retries": "0", + "npm_config_fetch_retry_maxtimeout": "100", + "npm_config_fetch_retry_mintimeout": "100", + "npm_config_fetch_timeout": "1000", + "npm_config_fund": "false", + "npm_config_offline": "true", + "npm_config_update_notifier": "false", + } + for key, value in expected_npm.items(): + matches = { + candidate: candidate_value for candidate, candidate_value in env.items() if candidate.upper() == key.upper() + } + assert matches == {key: value} + + assert env["OPENCODE_DISABLE_LSP_DOWNLOAD"] == "1" + assert env["OPENCODE_DISABLE_MODELS_FETCH"] == "1" + + +def _assert_replay_project_artifacts( + work: str, + *, + project_id: str, + tool_id: str, + forbidden_text: str, +) -> None: + root = Path(work) + assert (root / ".git").is_dir() + assert (root / ".git" / "opencode").read_text(encoding="ascii") == project_id + "\n" + + tool_files = list((root / ".opencode" / "tools").glob("*.js")) + assert [path.stem for path in tool_files] == [tool_id] + assert (root / ".opencode" / "node_modules").is_dir() + + package = json.loads((root / ".opencode" / "package.json").read_text()) + lock = json.loads((root / ".opencode" / "package-lock.json").read_text()) + assert package["dependencies"] == {"@opencode-ai/plugin": "0.0.0"} + assert lock["packages"][""]["dependencies"] == package["dependencies"] + + source = tool_files[0].read_text(encoding="utf-8") + assert forbidden_text not in source + assert json.dumps(_OPENCODE_SYNTHETIC_TOOL_QUERY) in source + assert json.dumps(_OPENCODE_SYNTHETIC_TOOL_RESULT) in source + + def test_resolve_opencode_path_precedence(monkeypatch): monkeypatch.setenv("SKILLOPT_SLEEP_OPENCODE_PATH", "env-opencode") with mock.patch("shutil.which", side_effect=lambda value: os.path.abspath(f"resolved-{value}")): @@ -75,24 +171,12 @@ def test_resolve_opencode_path_precedence(monkeypatch): assert resolve_opencode_path() == os.path.abspath("resolved-env-opencode") -def test_resolve_opencode_path_falls_back_to_path_or_command(monkeypatch, tmp_path): +def test_resolve_opencode_path_falls_back_to_command(monkeypatch): monkeypatch.delenv("SKILLOPT_SLEEP_OPENCODE_PATH", raising=False) - executable = str(tmp_path / "opencode") - with mock.patch("shutil.which", return_value=executable): - assert resolve_opencode_path() == executable with mock.patch("shutil.which", return_value=None): assert resolve_opencode_path() == "opencode" -def test_resolve_opencode_path_anchors_relative_path_search_result(monkeypatch, tmp_path): - monkeypatch.chdir(tmp_path) - monkeypatch.delenv("SKILLOPT_SLEEP_OPENCODE_PATH", raising=False) - resolved = os.path.join("tools", "opencode") - - with mock.patch("shutil.which", return_value=resolved): - assert resolve_opencode_path("opencode") == os.path.abspath(resolved) - - @pytest.mark.skipif(os.name != "nt", reason="Windows PATHEXT shim behavior") def test_resolve_opencode_path_preserves_windows_cmd_shim(monkeypatch): monkeypatch.delenv("SKILLOPT_SLEEP_OPENCODE_PATH", raising=False) @@ -122,6 +206,24 @@ def test_constructor_uses_explicit_or_environment_model(monkeypatch): assert OpenCodeCliBackend().model == "env/model" +def test_backend_runtime_uses_current_mutable_settings(): + be = OpenCodeCliBackend(model="old/model", opencode_path="old-opencode", timeout=1) + be.model = "new/model" + be.opencode_path = "new-opencode" + be.timeout = 7 + + with mock.patch( + "skillopt_sleep.backend.subprocess.run", + side_effect=_successful_plain_results(), + ) as run: + assert be._call("hello") == "answer" + + assert all(call.args[0][0] == "new-opencode" for call in run.call_args_list) + assert all(call.kwargs["timeout"] == 7 for call in run.call_args_list) + command = run.call_args_list[-1].args[0] + assert command[command.index("--model") + 1] == "new/model" + + def test_parse_opencode_jsonl_collects_text_and_ignores_extensions(): raw = "\n".join( [ @@ -134,7 +236,7 @@ def test_parse_opencode_jsonl_collects_text_and_ignores_extensions(): _event("step_finish", part={"type": "step-finish"}), ] ) - assert _parse_opencode_jsonl_text(raw) == ("first\nsecond", "") + assert _parse_opencode_jsonl_events(raw) == ("first\nsecond", [], "") @pytest.mark.parametrize( @@ -154,6 +256,7 @@ def test_parse_opencode_jsonl_collects_text_and_ignores_extensions(): "mixed_session", ), (_event("error"), "error_event"), + (_event("reasoning", part={"type": "text"}), "invalid_event"), ( "\n".join( [ @@ -164,11 +267,75 @@ def test_parse_opencode_jsonl_collects_text_and_ignores_extensions(): "unexpected_tool_event", ), (_event("step_start", part={"type": "step-start"}), "incomplete_stream"), + (_event("step_finish", part={"type": "step-finish"}), "incomplete_stream"), (_success_stream(" "), "empty_response"), ], ) def test_parse_opencode_jsonl_rejects_invalid_streams(raw, expected_code): - assert _parse_opencode_jsonl_text(raw) == ("", expected_code) + assert _parse_opencode_jsonl_events(raw) == ("", [], expected_code) + + +def test_parse_opencode_tool_jsonl_accepts_only_expected_completed_tools(): + tool_id = "skillopt_replay_abc123" + raw = _tool_success_stream(tool_id, "final answer") + + assert _parse_opencode_jsonl_events(raw, {tool_id}) == ( + "final answer", + [tool_id], + "", + ) + + +@pytest.mark.parametrize( + ("raw", "expected_code"), + [ + (_tool_success_stream("other"), "unexpected_tool_id"), + ( + "\n".join( + [ + _event("step_start", part={"type": "step-start"}), + _tool_event("expected", status="error"), + ] + ), + "tool_error", + ), + ( + "\n".join( + [ + _event("step_start", part={"type": "step-start"}), + _tool_event("expected", output="unexpected result"), + _event("step_finish", part={"type": "step-finish"}), + ] + ), + "invalid_tool_event", + ), + ( + "\n".join( + [ + _event("step_start", part={"type": "step-start"}), + _tool_event( + "expected", + query="task-derived query", + output=_OPENCODE_SYNTHETIC_TOOL_RESULT, + ), + ] + ), + "invalid_tool_event", + ), + ( + "\n".join( + [ + _event("step_start", part={"type": "step-start"}), + _tool_event("expected", output=_OPENCODE_SYNTHETIC_TOOL_RESULT), + _event("step_finish", part={"type": "step-finish"}), + ] + ), + "missing_final_text", + ), + ], +) +def test_parse_opencode_tool_jsonl_rejects_unverified_tools(raw, expected_code): + assert _parse_opencode_jsonl_events(raw, {"expected"}) == ("", [], expected_code) def test_call_uses_stdin_temp_workspace_and_user_environment(monkeypatch, tmp_path): @@ -219,7 +386,6 @@ def fake_run(cmd, **kwargs): with mock.patch("skillopt_sleep.backend.subprocess.run", side_effect=fake_run): assert be._call("do the thing") == "answer" - assert len(captured) == 3 discovery_cmd, discovery_call = captured[0] verification_cmd, verification_call = captured[1] cmd, run_call = captured[2] @@ -256,7 +422,6 @@ def fake_run(cmd, **kwargs): ] agent_name = cmd[cmd.index("--agent") + 1] assert agent_name.startswith("skillopt-sleep-") - assert agent_name != "unsafe-user-agent" assert cmd[cmd.index("--title") + 1] == "skillopt-sleep" assert cmd[cmd.index("--dir") + 1] == run_call["cwd"] assert cmd[cmd.index("--model") + 1] == "provider/model" @@ -273,6 +438,7 @@ def fake_run(cmd, **kwargs): assert "mcp" not in discovered injected = json.loads(run_call["env"]["OPENCODE_CONFIG_CONTENT"]) assert injected == { + "snapshot": False, "agent": { agent_name: { "mode": "primary", @@ -285,8 +451,6 @@ def fake_run(cmd, **kwargs): }, } assert verification_call["env"]["OPENCODE_CONFIG_CONTENT"] == run_call["env"]["OPENCODE_CONFIG_CONTENT"] - assert run_call["env"]["OPENCODE_CONFIG_CONTENT"] != inline_config - assert "unsafe-user-agent" not in run_call["env"]["OPENCODE_CONFIG_CONTENT"] assert "OPENCODE_DISABLE_DEFAULT_PLUGINS" not in run_call["env"] assert not os.path.exists(run_call["cwd"]) assert be.last_call_error == "" @@ -309,7 +473,6 @@ def fake_run(cmd, **kwargs): with mock.patch("skillopt_sleep.backend.subprocess.run", side_effect=fake_run): assert be._call("hello") == "answer" - assert len(captured) == 3 assert all(env["OPENCODE_CONFIG"] == os.path.abspath(config_path) for env in captured) assert all(env["OPENCODE_CONFIG_DIR"] == os.path.abspath(config_dir) for env in captured) @@ -343,17 +506,16 @@ def test_each_call_uses_a_new_agent_name(): @pytest.mark.parametrize( - ("side_effect", "return_value", "error_fragment"), + ("run_result", "error_fragment"), [ - (subprocess.TimeoutExpired("opencode", 1), None, "timed out"), - (OSError("secret path"), None, "could not be executed"), - (None, _FakeProc("misleading", returncode=2), "exited 2"), - (None, _FakeProc("not json"), "malformed JSONL"), + (subprocess.TimeoutExpired("opencode", 1), "timed out"), + (OSError("secret path"), "could not be executed"), + (_FakeProc("misleading", returncode=2), "exited 2"), + (_FakeProc("not json"), "malformed JSONL"), ], ) -def test_call_records_process_and_protocol_failures(side_effect, return_value, error_fragment): +def test_call_records_process_and_protocol_failures(run_result, error_fragment): be = OpenCodeCliBackend(opencode_path="opencode", timeout=1) - run_result = side_effect if side_effect is not None else return_value effects = _successful_plain_results()[:2] + [run_result] with mock.patch("skillopt_sleep.backend.subprocess.run", side_effect=effects): assert be._call("hello") == "" @@ -361,26 +523,58 @@ def test_call_records_process_and_protocol_failures(side_effect, return_value, e assert "secret path" not in be.last_call_error -def test_mcp_reenabled_by_final_config_stops_before_model_call(): - discovered = _FakeProc(_resolved_mcp("global-server")) - reenabled = _FakeProc( - json.dumps( +def test_call_handles_workspace_creation_failure_without_starting_child(): + be = OpenCodeCliBackend(opencode_path="opencode") + with ( + mock.patch( + "skillopt_sleep.backend.tempfile.TemporaryDirectory", + side_effect=OSError("private workspace detail"), + ), + mock.patch("skillopt_sleep.backend.subprocess.run") as run, + ): + assert be._call("hello") == "" + + run.assert_not_called() + assert "workspace could not be prepared" in be.last_call_error + assert "private workspace detail" not in be.last_call_error + + +@pytest.mark.parametrize( + ("discovered_names", "final_mcp"), + [ + pytest.param( + ("global-server",), { - "mcp": { - "global-server": { - "type": "local", - "command": ["mcp-server"], - "enabled": True, - } + "global-server": { + "type": "local", + "command": ["mcp-server"], + "enabled": True, } - } - ) - ) + }, + id="known-server-re-enabled", + ), + pytest.param( + ("known-server",), + { + "known-server": {"enabled": False}, + "new-server": { + "type": "local", + "command": ["mcp-server"], + "enabled": True, + }, + }, + id="new-server-enabled", + ), + ], +) +def test_enabled_mcp_in_final_config_stops_before_model_call(discovered_names, final_mcp): + discovered = _FakeProc(_resolved_mcp(*discovered_names)) + verification = _FakeProc(json.dumps({"mcp": final_mcp})) be = OpenCodeCliBackend(opencode_path="opencode") with mock.patch( "skillopt_sleep.backend.subprocess.run", - side_effect=[discovered, reenabled], + side_effect=[discovered, verification], ) as run: assert be._call("hello") == "" @@ -389,33 +583,18 @@ def test_mcp_reenabled_by_final_config_stops_before_model_call(): assert "disable every configured MCP server" in be.last_call_error -def test_new_enabled_mcp_in_final_config_stops_before_model_call(): - discovered = _FakeProc(_resolved_mcp("known-server")) - verification = _FakeProc( - json.dumps( - { - "mcp": { - "known-server": {"enabled": False}, - "new-server": { - "type": "local", - "command": ["mcp-server"], - "enabled": True, - }, - } - } - ) - ) +def test_plain_replay_fails_if_snapshots_remain_enabled(): be = OpenCodeCliBackend(opencode_path="opencode") + results = [ + _FakeProc(_resolved_mcp(snapshot=False)), + _FakeProc(_resolved_mcp(snapshot=True)), + ] - with mock.patch( - "skillopt_sleep.backend.subprocess.run", - side_effect=[discovered, verification], - ) as run: + with mock.patch("skillopt_sleep.backend.subprocess.run", side_effect=results) as run: assert be._call("hello") == "" assert run.call_count == 2 - assert all(call.args[0][1:3] == ["debug", "config"] for call in run.call_args_list) - assert "disable every configured MCP server" in be.last_call_error + assert "disable session snapshots" in be.last_call_error @pytest.mark.parametrize( @@ -449,21 +628,12 @@ def test_mcp_discovery_failure_stops_before_model_call(bad_result, error_fragmen assert "private config path" not in be.last_call_error -@pytest.mark.parametrize( - "bad_verification", - [ - subprocess.TimeoutExpired("opencode", 1), - _FakeProc("", returncode=4), - _FakeProc("not json"), - _FakeProc(json.dumps({"mcp": {"server": []}})), - ], -) -def test_mcp_verification_failure_stops_before_model_call(bad_verification): +def test_mcp_verification_failure_stops_before_model_call(): be = OpenCodeCliBackend(opencode_path="opencode") with mock.patch( "skillopt_sleep.backend.subprocess.run", - side_effect=[_FakeProc(_resolved_mcp("server")), bad_verification], + side_effect=[_FakeProc(_resolved_mcp("server")), _FakeProc("not json")], ) as run: assert be._call("hello") == "" @@ -541,12 +711,335 @@ def test_cached_success_clears_stale_call_error(): assert be.last_call_error == "" -def test_attempt_with_tools_fails_without_starting_child(): +def test_attempt_with_tools_requires_explicit_opt_in_without_starting_child(): be = OpenCodeCliBackend(opencode_path="opencode") with mock.patch("skillopt_sleep.backend.subprocess.run") as run: assert be.attempt_with_tools(mock.Mock(), "skill", "memory", ["search"]) == ("", []) run.assert_not_called() - assert "not supported" in be.last_call_error + assert "opencode_tool_replay" in be.last_call_error + + +@pytest.mark.parametrize("configured", [False, 0, 1, "false", "true", None]) +def test_tool_replay_opt_in_requires_boolean_true(configured): + be = OpenCodeCliBackend( + opencode_path="opencode", + tool_replay=configured, + ) + + assert be.tool_replay is False + + +def test_attempt_with_tools_builds_isolated_random_replay_request(monkeypatch, tmp_path): + monkeypatch.setenv("OPENAI_API_KEY", "provider-secret-stays-in-env") + monkeypatch.setenv("OPENCODE_DIRECT_TRACE", "1") + for key in ("GIT_COMMON_DIR", "GIT_DIR", "GIT_OBJECT_DIRECTORY", "GIT_WORK_TREE"): + monkeypatch.setenv(key, str(tmp_path / "outside-repository")) + monkeypatch.setenv("git_dir", str(tmp_path / "lowercase-outside-repository")) + hostile_npm = { + "NPM_CONFIG_AUDIT": "true", + "NpM_CoNfIg_CaChE": str(tmp_path / "outside-npm-cache"), + "NPM_CONFIG_FETCH_RETRIES": "99", + "npm_CONFIG_FETCH_RETRY_MAXTIMEOUT": "99999", + "NPM_CONFIG_FETCH_RETRY_MINTIMEOUT": "99999", + "NPM_CONFIG_FETCH_TIMEOUT": "99999", + "nPm_CoNfIg_FuNd": "true", + "NPM_CONFIG_OFFLINE": "false", + "NPM_CONFIG_UPDATE_NOTIFIER": "true", + } + for key, value in hostile_npm.items(): + monkeypatch.setenv(key, value) + + captured = [] + project_id = "skillopt-sleep-" + "a" * 64 + agent_name = "skillopt-sleep-" + "b" * 32 + tool_id = "skillopt_replay_" + "c" * 32 + expected_permission = {"*": "deny", tool_id: "allow"} + intent = "OPENCODE_INTENT_SENTINEL_1937" + secret_context = "task-secret-value-4819" + skill = "OPENCODE_SKILL_SENTINEL_2864" + memory = "OPENCODE_MEMORY_SENTINEL_7351" + + def fake_run(cmd, **kwargs): + snapshot = dict(kwargs) + snapshot["env"] = kwargs["env"].copy() + captured.append((cmd, snapshot)) + _assert_controlled_tool_environment(kwargs["env"], kwargs["cwd"]) + if cmd[1:3] == ["debug", "config"]: + config = json.loads(kwargs["env"]["OPENCODE_CONFIG_CONTENT"]) + _assert_replay_permissions(config, kwargs["env"], agent_name, expected_permission) + disabled = bool(config.get("mcp")) + return _FakeProc( + _resolved_mcp( + "configured-server", + disabled=disabled, + snapshot=False, + ) + ) + if cmd[1:3] == ["debug", "agent"]: + config = json.loads(kwargs["env"]["OPENCODE_CONFIG_CONTENT"]) + _assert_replay_permissions(config, kwargs["env"], agent_name, expected_permission) + assert cmd[3] == agent_name + agent = config["agent"][agent_name] + assert agent["model"] == "provider/model" + return _FakeProc(json.dumps({"tools": {"bash": False, tool_id: True}})) + + work = kwargs["cwd"] + config = json.loads(kwargs["env"]["OPENCODE_CONFIG_CONTENT"]) + _assert_replay_permissions(config, kwargs["env"], agent_name, expected_permission) + assert config["snapshot"] is False + assert cmd[cmd.index("--agent") + 1] == agent_name + _assert_replay_project_artifacts( + work, + project_id=project_id, + tool_id=tool_id, + forbidden_text=secret_context, + ) + return _FakeProc(_tool_success_stream(tool_id)) + + be = OpenCodeCliBackend( + model="provider/model", + opencode_path="opencode", + timeout=41, + tool_replay=True, + ) + task = TaskRecord( + id="opencode-tool-prompt", + project=str(tmp_path), + intent=intent, + context_excerpt=secret_context, + ) + with ( + mock.patch( + "skillopt_sleep.backend.secrets.token_hex", + side_effect=["a" * 64, "b" * 32, "c" * 32], + ), + mock.patch("skillopt_sleep.backend.subprocess.run", side_effect=fake_run), + ): + response, called = be.attempt_with_tools(task, skill, memory, ["search"]) + + assert response == "answer" + assert called == ["search"] + assert be.last_call_error == "" + assert [call[0][1:3] for call in captured] == [ + ["debug", "config"], + ["debug", "config"], + ["debug", "agent"], + ["run", "--pure"], + ] + run_call = captured[-1][1] + assert run_call["env"]["OPENCODE_DISABLE_PROJECT_CONFIG"] == "0" + assert run_call["env"]["OPENCODE_PURE"] == "1" + assert run_call["env"]["OPENAI_API_KEY"] == "provider-secret-stays-in-env" + assert "OPENCODE_DIRECT_TRACE" not in run_call["env"] + for key in ("GIT_COMMON_DIR", "GIT_DIR", "GIT_OBJECT_DIRECTORY", "GIT_WORK_TREE"): + assert not any(candidate.upper() == key for candidate in run_call["env"]) + for sentinel in (intent, secret_context, skill, memory): + assert sentinel in run_call["input"] + assert '{"query":"synthetic"}' in run_call["input"] + assert "do not call a tool merely because it is listed" in run_call["input"] + assert "Learned preferences" in run_call["input"] + assert "override earlier conflicting skill text" in run_call["input"] + assert "Call every listed random internal ID" not in run_call["input"] + assert not os.path.exists(run_call["cwd"]) + + +@pytest.mark.parametrize( + "tools", + [ + pytest.param(["../search"], id="invalid-prefix"), + pytest.param([], id="empty"), + pytest.param(["search", 123], id="non-string"), + pytest.param(["x" * 129], id="too-long"), + pytest.param([f"tool-{index}" for index in range(33)], id="too-many"), + ], +) +def test_attempt_with_tools_rejects_invalid_names_before_starting_child(tools, tmp_path): + be = OpenCodeCliBackend(opencode_path="opencode", tool_replay=True) + task = TaskRecord(id="invalid-tools", project=str(tmp_path), intent="answer") + with mock.patch("skillopt_sleep.backend.subprocess.run") as run: + assert be.attempt_with_tools(task, "", "", tools) == ("", []) + run.assert_not_called() + assert "invalid tool list" in be.last_call_error + + +@pytest.mark.parametrize( + ("child_result", "expected_fragment"), + [ + pytest.param( + _FakeProc(json.dumps({"tools": {"expected": False}})), + "restrict tools", + id="expected-disabled", + ), + pytest.param( + _FakeProc(json.dumps({"tools": {"expected": True, "bash": True}})), + "restrict tools", + id="extra-enabled", + ), + pytest.param( + _FakeProc(json.dumps({"tools": {"expected": "allow"}})), + "invalid configuration", + id="non-boolean-status", + ), + pytest.param( + _FakeProc(json.dumps({"tools": []})), + "invalid configuration", + id="invalid-tools-shape", + ), + pytest.param( + subprocess.TimeoutExpired(cmd=["opencode"], timeout=1), + "timed out", + id="timeout", + ), + pytest.param(OSError("private child detail"), "could not be completed", id="spawn-error"), + pytest.param(_FakeProc("", returncode=9), "failed", id="nonzero-exit"), + pytest.param(_FakeProc("not-json"), "invalid configuration", id="invalid-json"), + ], +) +def test_tool_permission_verification_fails_closed(child_result, expected_fragment, tmp_path): + backend = OpenCodeCliBackend(opencode_path=str(tmp_path / "opencode"), timeout=1) + + with ( + mock.patch("skillopt_sleep.backend.subprocess.run", side_effect=[child_result]) as run, + pytest.raises(OpenCodeError) as error, + ): + backend._verify_tool_allowlist({}, str(tmp_path), "skillopt-sleep-agent", {"expected"}) + + run.assert_called_once() + assert expected_fragment in str(error.value) + assert "private child detail" not in str(error.value) + + +def test_tool_replay_fails_before_model_if_snapshots_remain_enabled(): + captured_work = "" + + def fake_run(cmd, **kwargs): + nonlocal captured_work + captured_work = kwargs["cwd"] + assert cmd[1:3] == ["debug", "config"] + snapshot = len(run.call_args_list) == 2 + return _FakeProc(_resolved_mcp(snapshot=snapshot)) + + be = OpenCodeCliBackend(opencode_path="opencode", tool_replay=True) + task = mock.Mock(intent="search", context_excerpt="") + with mock.patch("skillopt_sleep.backend.subprocess.run", side_effect=fake_run) as run: + assert be.attempt_with_tools(task, "", "", ["search"]) == ("", []) + + assert run.call_count == 2 + assert "disable session snapshots" in be.last_call_error + assert captured_work and not os.path.exists(captured_work) + + +@pytest.mark.parametrize( + ("run_result", "error_fragment"), + [ + (subprocess.TimeoutExpired(cmd=["opencode"], timeout=1), "timed out"), + (OSError("private run detail"), "could not be executed"), + (_FakeProc("", returncode=9), "tool replay failed"), + ], +) +def test_tool_replay_handles_model_child_failures(run_result, error_fragment): + captured_work = "" + + def fake_run(cmd, **kwargs): + nonlocal captured_work + captured_work = kwargs["cwd"] + if cmd[1:3] == ["debug", "config"]: + return _FakeProc(_resolved_mcp(snapshot=False)) + if cmd[1:3] == ["debug", "agent"]: + config = json.loads(kwargs["env"]["OPENCODE_CONFIG_CONTENT"]) + permission = config["agent"][cmd[3]]["permission"] + allowed = {name: True for name, value in permission.items() if value == "allow"} + return _FakeProc(json.dumps({"tools": allowed})) + if isinstance(run_result, BaseException): + raise run_result + return run_result + + be = OpenCodeCliBackend( + opencode_path="opencode", + timeout=1, + tool_replay=True, + ) + task = mock.Mock(intent="search", context_excerpt="private prompt detail") + with mock.patch("skillopt_sleep.backend.subprocess.run", side_effect=fake_run) as run: + assert be.attempt_with_tools(task, "", "", ["search"]) == ("", []) + + assert run.call_count == 4 + assert be.tokens_used() > 0 + assert error_fragment in be.last_call_error + assert "private" not in be.last_call_error + assert captured_work and not os.path.exists(captured_work) + + +def test_tool_replay_maps_and_deduplicates_only_completed_requested_tools(tmp_path): + search_id = "skillopt_replay_" + "c" * 32 + lookup_id = "skillopt_replay_" + "d" * 32 + + def fake_run(cmd, **kwargs): + if cmd[1:3] == ["debug", "config"]: + return _FakeProc(_resolved_mcp(snapshot=False)) + config = json.loads(kwargs["env"]["OPENCODE_CONFIG_CONTENT"]) + if cmd[1:3] == ["debug", "agent"]: + permission = config["agent"][cmd[3]]["permission"] + assert permission == {"*": "deny", search_id: "allow", lookup_id: "allow"} + return _FakeProc(json.dumps({"tools": {search_id: True, lookup_id: True}})) + + return _FakeProc( + "\n".join( + [ + _event("step_start", part={"type": "step-start"}), + _tool_event( + lookup_id, + call_id="call-1", + output=_OPENCODE_SYNTHETIC_TOOL_RESULT, + ), + _tool_event( + lookup_id, + call_id="call-2", + output=_OPENCODE_SYNTHETIC_TOOL_RESULT, + ), + _event("step_finish", part={"type": "step-finish"}), + _event("step_start", part={"type": "step-start"}), + _event("text", part={"type": "text", "text": "answer"}), + _event("step_finish", part={"type": "step-finish"}), + ] + ) + ) + + be = OpenCodeCliBackend(opencode_path="opencode", tool_replay=True) + task = TaskRecord(id="partial-tool-replay", project=str(tmp_path), intent="use lookup") + with ( + mock.patch( + "skillopt_sleep.backend.secrets.token_hex", + side_effect=["a" * 64, "b" * 32, "c" * 32, "d" * 32], + ), + mock.patch("skillopt_sleep.backend.subprocess.run", side_effect=fake_run), + ): + response, called = be.attempt_with_tools(task, "", "", ["search", "lookup"]) + + assert response == "answer" + assert called == ["lookup"] + + +def test_tool_replay_does_not_treat_a_self_report_as_a_tool_call(): + def fake_run(cmd, **kwargs): + if cmd[1:3] == ["debug", "config"]: + return _FakeProc(_resolved_mcp(snapshot=False)) + if cmd[1:3] == ["debug", "agent"]: + config = json.loads(kwargs["env"]["OPENCODE_CONFIG_CONTENT"]) + permission = config["agent"][cmd[3]]["permission"] + return _FakeProc( + json.dumps({"tools": {name: value == "allow" for name, value in permission.items() if name != "*"}}) + ) + return _FakeProc(_success_stream("TOOL_CALL: search")) + + be = OpenCodeCliBackend(opencode_path="opencode", tool_replay=True) + task = mock.Mock(intent="answer", context_excerpt="") + with mock.patch("skillopt_sleep.backend.subprocess.run", side_effect=fake_run): + assert be.attempt_with_tools(task, "", "", ["search"]) == ( + "TOOL_CALL: search", + [], + ) + assert be.last_call_error == "" def test_get_and_build_backend_route_opencode_path(): @@ -554,19 +1047,34 @@ def test_get_and_build_backend_route_opencode_path(): for alias in ("opencode", "opencode_cli", "opencode-cli", "OPENCODE"): assert isinstance(get_backend(alias), OpenCodeCliBackend) - single = build_backend(backend="opencode", opencode_path="custom-opencode") + single = build_backend( + backend="opencode", + opencode_path="custom-opencode", + opencode_tool_replay=True, + ) assert isinstance(single, OpenCodeCliBackend) assert single.opencode_path == "custom-opencode" + assert single.tool_replay is True + + stringly_enabled = build_backend( + backend="opencode", + opencode_path="custom-opencode", + opencode_tool_replay="true", + ) + assert stringly_enabled.tool_replay is False dual = build_backend( backend="mock", optimizer_backend="opencode", target_backend="opencode", opencode_path="custom-opencode", + opencode_tool_replay=True, ) assert isinstance(dual, DualBackend) assert dual.optimizer.opencode_path == "custom-opencode" assert dual.target.opencode_path == "custom-opencode" + assert dual.optimizer.tool_replay is True + assert dual.target.tool_replay is True def test_cli_makes_relative_opencode_path_absolute(monkeypatch, tmp_path): @@ -574,21 +1082,50 @@ def test_cli_makes_relative_opencode_path_absolute(monkeypatch, tmp_path): parser = argparse.ArgumentParser() _add_common(parser) relative = os.path.join("bin", "opencode") - args = parser.parse_args(["--backend", "opencode", "--opencode-path", relative]) + args = parser.parse_args( + [ + "--backend", + "opencode", + "--opencode-path", + relative, + "--opencode-tool-replay", + ] + ) monkeypatch.setattr("skillopt_sleep.config._user_config_path", lambda: None) cfg = _cfg_from_args(args) assert cfg.get("backend") == "opencode" assert cfg.get("opencode_path") == os.path.abspath(relative) + assert cfg.get("opencode_tool_replay") is True + + +def test_cli_without_tool_replay_flag_preserves_enabled_user_config(monkeypatch, tmp_path): + parser = argparse.ArgumentParser() + _add_common(parser) + args = parser.parse_args(["--backend", "opencode"]) + config_path = tmp_path / "config.json" + config_path.write_text(json.dumps({"opencode_tool_replay": True}), encoding="utf-8") + monkeypatch.setattr("skillopt_sleep.config._user_config_path", lambda: str(config_path)) + cfg = _cfg_from_args(args) + + assert cfg.get("opencode_tool_replay") is True -def test_cycle_diagnostic_build_forwards_opencode_path(): - cfg = load_config(backend="opencode", opencode_path="custom-opencode") + +def test_cycle_diagnostic_build_forwards_opencode_settings(): + cfg = load_config( + backend="opencode", + opencode_path="custom-opencode", + opencode_tool_replay=True, + ) with mock.patch("skillopt_sleep.cycle.build_backend", return_value=MockBackend()) as builder: cycle._make_model_key(cfg) + builder.assert_called_once() assert builder.call_args.kwargs["opencode_path"] == "custom-opencode" + assert builder.call_args.kwargs["opencode_tool_replay"] is True -def test_runtime_cycle_build_forwards_opencode_path(tmp_path): +@pytest.mark.parametrize("configured", [True, "true"]) +def test_runtime_cycle_build_forwards_opencode_settings(tmp_path, configured): project = tmp_path / "project" project.mkdir() cfg = SleepConfig( @@ -596,6 +1133,7 @@ def test_runtime_cycle_build_forwards_opencode_path(tmp_path): **DEFAULTS, "backend": "opencode", "opencode_path": "custom-opencode", + "opencode_tool_replay": configured, "projects": "invoked", "invoked_project": str(project), "state_dir": str(tmp_path / "state"), @@ -605,8 +1143,56 @@ def test_runtime_cycle_build_forwards_opencode_path(tmp_path): ) with mock.patch("skillopt_sleep.cycle.build_backend", return_value=MockBackend()) as builder: - outcome = cycle.run_sleep_cycle(cfg, seed_tasks=[], dry_run=True) + cycle.run_sleep_cycle(cfg, seed_tasks=[], dry_run=True) - assert outcome.report.n_tasks == 0 assert builder.call_count == 1 assert builder.call_args.kwargs["opencode_path"] == "custom-opencode" + assert builder.call_args.kwargs["opencode_tool_replay"] == configured + + +@pytest.mark.parametrize( + ("configured", "expected"), + [(True, True), ("true", False)], +) +def test_cycle_artifacts_record_only_a_boolean_tool_replay_opt_in( + tmp_path, + configured, + expected, +): + project = tmp_path / "project" + project.mkdir() + state_dir = tmp_path / "state" + cfg = SleepConfig( + data={ + **DEFAULTS, + "backend": "opencode", + "opencode_tool_replay": configured, + "projects": "invoked", + "invoked_project": str(project), + "state_dir": str(state_dir), + "claude_home": str(tmp_path / "claude-home"), + "evidence_log": True, + "evolve_skill": False, + "evolve_memory": False, + } + ) + task = TaskRecord( + id="opencode-artifacts", + project=str(project), + intent="record the resolved replay setting", + reference_kind="exact", + reference="expected response", + split="val", + ) + + with mock.patch("skillopt_sleep.cycle.build_backend", return_value=MockBackend()): + outcome = cycle.run_sleep_cycle(cfg, seed_tasks=[task]) + + staging_dir = Path(outcome.staging_dir) + records = [json.loads(line) for line in (staging_dir / "evidence.jsonl").read_text(encoding="utf-8").splitlines()] + start = next(record for record in records if record["stage"] == "cycle" and record["event"] == "start") + assert start["config"]["opencode_tool_replay"] is expected + + diagnostics_path = staging_dir / "diagnostics.json" + diagnostics = json.loads(diagnostics_path.read_text(encoding="utf-8")) + assert diagnostics["opencode_tool_replay"] is expected diff --git a/tests/test_backend_opencode_live.py b/tests/test_backend_opencode_live.py index db28e931..c4d373b9 100644 --- a/tests/test_backend_opencode_live.py +++ b/tests/test_backend_opencode_live.py @@ -1,11 +1,13 @@ """Opt-in live tests for the OpenCode CLI backend. These tests make real model calls through the user's OpenCode installation, -account, and configuration. They run unchanged on Windows and POSIX, but are -skipped unless ``SKILLOPT_TEST_REAL_OPENCODE=1`` is set. An opted-in run also -requires an explicit model in ``SKILLOPT_SLEEP_OPENCODE_MODEL`` so the -provider/model selection is explicit. They may incur provider charges and -create entries in the user's OpenCode session history. +account, and configuration. They support Windows and POSIX, but are skipped +unless ``SKILLOPT_TEST_REAL_OPENCODE=1`` is set. An opted-in run also requires +an explicit model in ``SKILLOPT_SLEEP_OPENCODE_MODEL`` so model selection is +explicit. They may incur provider charges and create entries +in the user's OpenCode session history. The tool-aware replay test also runs one +generated JavaScript tool with a fixed result inside an isolated temporary +project. """ from __future__ import annotations @@ -16,6 +18,7 @@ import shutil import subprocess import sys +import tempfile from pathlib import Path import pytest @@ -24,10 +27,10 @@ from skillopt_sleep.config import DEFAULTS, SleepConfig from skillopt_sleep.cycle import run_sleep_cycle from skillopt_sleep.evidence import read_events +from skillopt_sleep.replay import replay_one from skillopt_sleep.types import TaskRecord _LIVE_ENABLED = os.environ.get("SKILLOPT_TEST_REAL_OPENCODE", "").strip() == "1" -_PLAIN_MARKER = "SKILLOPT_OPENCODE_PLAIN_OK_7F3C" _CYCLE_MARKER = "SKILLOPT_OPENCODE_CYCLE_OK_9A6D" pytestmark = pytest.mark.skipif( @@ -54,16 +57,6 @@ def _live_settings() -> tuple[str, str]: return model, opencode_path -def _require_marker(response: str, marker: str, backend: OpenCodeCliBackend) -> None: - """Fail without copying provider output into pytest's assertion report.""" - if backend.last_call_error: - pytest.fail("the real OpenCode call failed", pytrace=False) - if not response: - pytest.fail("the real OpenCode call returned no text", pytrace=False) - if marker not in response: - pytest.fail("the real OpenCode response did not contain the requested marker", pytrace=False) - - def _add_mcp_canary(monkeypatch, tmp_path: Path) -> tuple[Path, str]: """Add a local MCP that leaves a marker if OpenCode starts it.""" mcp_name = f"skillopt-live-canary-{secrets.token_hex(8)}" @@ -130,27 +123,92 @@ def _require_mcp_canary_configured(opencode_path: str, mcp_name: str, tmp_path: pytest.fail("the real OpenCode config did not include the MCP canary", pytrace=False) -def test_real_opencode_plain_replay(monkeypatch, tmp_path): - """Exercise one public backend attempt against the real OpenCode CLI.""" +def _add_ancestor_tool_canary(monkeypatch, tmp_path: Path) -> tuple[Path, Path, list[Path]]: + """Place a load-time canary in an ancestor directory outside the replay Git boundary.""" + outer = tmp_path / "tool-project-parent" + tool_dir = outer / ".opencode" / "tools" + tool_dir.mkdir(parents=True) + marker = tmp_path / "ancestor-tool-loaded.txt" + (tool_dir / "ancestor_canary.js").write_text( + 'import { writeFileSync } from "node:fs";\n' + f"writeFileSync({json.dumps(str(marker))}, 'loaded', 'utf8');\n" + "export default {\n" + " description: 'Ancestor configuration canary',\n" + " args: {},\n" + " async execute() { return 'ancestor canary'; },\n" + "};\n", + encoding="utf-8", + ) + + replay_workspaces: list[Path] = [] + + def make_replay_workspace(prefix: str, _error: str) -> tempfile.TemporaryDirectory: + workspace = tempfile.TemporaryDirectory( + prefix=prefix, + dir=outer, + ignore_cleanup_errors=True, + ) + replay_workspaces.append(Path(workspace.name)) + return workspace + + monkeypatch.setattr( + "skillopt_sleep.backend._opencode_temporary_workspace", + make_replay_workspace, + ) + return marker, outer, replay_workspaces + + +def test_real_opencode_tool_replay(monkeypatch, tmp_path): + """Exercise one synthetic tool call through the real OpenCode CLI and verify local scoring.""" model, opencode_path = _live_settings() monkeypatch.setenv("SKILLOPT_SLEEP_PROMPTS_PATH", str(tmp_path / "no-prompt-overrides.json")) mcp_marker, mcp_name = _add_mcp_canary(monkeypatch, tmp_path) _require_mcp_canary_configured(opencode_path, mcp_name, tmp_path) + ancestor_tool_marker, ancestor_outer, replay_workspaces = _add_ancestor_tool_canary( + monkeypatch, + tmp_path, + ) - backend = OpenCodeCliBackend(model=model, opencode_path=opencode_path) + backend = OpenCodeCliBackend( + model=model, + opencode_path=opencode_path, + tool_replay=True, + ) task = TaskRecord( - id="opencode-live-plain", + id="opencode-live-tool", project=str(tmp_path), - intent=f"Reply with exactly this text and nothing else: {_PLAIN_MARKER}", - reference_kind="exact", - reference=_PLAIN_MARKER, + intent="Call the controlled search stand-in, then give a short final answer.", + reference_kind="rule", + judge={ + "kind": "rule", + "checks": [{"op": "tool_called", "arg": "search"}], + }, ) - response = backend.attempt(task, skill="", memory="") + result = replay_one( + backend, + task, + "Before answering, you MUST call the search tool.", + "", + ) + if len(replay_workspaces) != 1: + pytest.fail("the real OpenCode tool replay did not create one workspace", pytrace=False) + replay_workspace = replay_workspaces[0] + try: + replay_workspace.resolve().relative_to(ancestor_outer.resolve()) + except (OSError, ValueError): + pytest.fail("the real OpenCode tool replay escaped its canary boundary", pytrace=False) + if replay_workspace.exists(): + pytest.fail("the real OpenCode tool replay did not clean up its workspace", pytrace=False) if mcp_marker.exists(): - pytest.fail("the real OpenCode call started a configured MCP server", pytrace=False) - _require_marker(response, _PLAIN_MARKER, backend) + pytest.fail("the real OpenCode tool call started a configured MCP server", pytrace=False) + if ancestor_tool_marker.exists(): + pytest.fail("the real OpenCode tool call loaded ancestor project tools", pytrace=False) + if backend.last_call_error: + pytest.fail("the real OpenCode tool-aware replay failed", pytrace=False) + if not result.response or result.hard != 1.0 or result.tools_called != ["search"]: + pytest.fail("the real OpenCode tool invocation was not verified", pytrace=False) def test_real_opencode_cycle_smoke(monkeypatch, tmp_path): @@ -162,6 +220,8 @@ def test_real_opencode_cycle_smoke(monkeypatch, tmp_path): project.mkdir() monkeypatch.setenv("SKILLOPT_SLEEP_PROMPTS_PATH", str(tmp_path / "no-prompt-overrides.json")) monkeypatch.setenv("SKILLOPT_SLEEP_WORKERS", "1") + mcp_marker, mcp_name = _add_mcp_canary(monkeypatch, tmp_path) + _require_mcp_canary_configured(opencode_path, mcp_name, tmp_path) cfg = SleepConfig( data={ @@ -198,6 +258,8 @@ def test_real_opencode_cycle_smoke(monkeypatch, tmp_path): outcome = run_sleep_cycle(cfg, seed_tasks=[task]) + if mcp_marker.exists(): + pytest.fail("the real OpenCode cycle started a configured MCP server", pytrace=False) staging_dir = Path(outcome.staging_dir) try: staging_dir.resolve().relative_to(project.resolve()) @@ -232,18 +294,10 @@ def test_real_opencode_cycle_smoke(monkeypatch, tmp_path): pytest.fail("the live cycle diagnostics did not record a successful replay", pytrace=False) events = read_events(str(artifact_paths["evidence.jsonl"])) - uncached_model_calls = [ - event - for event in events - if event.get("stage") == "replay" and event.get("event") == "model_call" and not event.get("cache_hit") - ] model_calls = [ event for event in events - if event.get("stage") == "replay" - and event.get("event") == "model_call" - and event.get("kind") == "attempt" - and not event.get("cache_hit") + if event.get("stage") == "replay" and event.get("event") == "model_call" and not event.get("cache_hit") ] replay_results = [ event @@ -253,7 +307,7 @@ def test_real_opencode_cycle_smoke(monkeypatch, tmp_path): cycle_ends = [event for event in events if event.get("stage") == "cycle" and event.get("event") == "end"] # Keep any provider text in the temporary evidence file and out of pytest output. - if len(uncached_model_calls) != 1 or len(model_calls) != 1: + if len(model_calls) != 1 or model_calls[0].get("kind") != "attempt": pytest.fail( "the live cycle did not make exactly one uncached OpenCode model call", pytrace=False, @@ -271,5 +325,5 @@ def test_real_opencode_cycle_smoke(monkeypatch, tmp_path): pytest.fail("the live cycle did not reach completion", pytrace=False) if outcome.report.n_tasks != 1 or outcome.report.n_replayed != 1: pytest.fail("the live cycle report did not record its seeded replay", pytrace=False) - if not outcome.staging_dir or outcome.adopted or outcome.adopted_paths: + if outcome.adopted or outcome.adopted_paths: pytest.fail("the live cycle did not preserve review-before-adopt behavior", pytrace=False) diff --git a/tests/test_plugin_sync.py b/tests/test_plugin_sync.py index e49a8994..3e594e36 100644 --- a/tests/test_plugin_sync.py +++ b/tests/test_plugin_sync.py @@ -124,10 +124,28 @@ def test_openclaw_wrapper_matches_shared_backend_signature(self): self.assertNotIn("**kwargs", text) script = f""" +import inspect import runpy import sys sys.path.insert(0, {os.path.dirname(OPENCLAW_RUNNER)!r}) -runpy.run_path({OPENCLAW_RUNNER!r}, run_name="openclaw_runner_test") +runner = runpy.run_path({OPENCLAW_RUNNER!r}, run_name="openclaw_runner_test") +wrapped_get_backend = runner["get_backend"] +assert inspect.signature(wrapped_get_backend).parameters["opencode_tool_replay"].default is False + +enabled = wrapped_get_backend( + "opencode", + opencode_path="unused-opencode", + opencode_tool_replay=True, +) +assert enabled.tool_replay is True, enabled.tool_replay + +stringly_enabled = wrapped_get_backend( + "opencode", + opencode_path="unused-opencode", + opencode_tool_replay="true", +) +assert stringly_enabled.tool_replay is False, stringly_enabled.tool_replay + from skillopt_sleep.backend import build_backend backend = build_backend( backend="mock",