diff --git a/packages/cli/dist/cli.js b/packages/cli/dist/cli.js index d2adc3b..ce5efd9 100755 --- a/packages/cli/dist/cli.js +++ b/packages/cli/dist/cli.js @@ -733,6 +733,7 @@ var ENV_CODEX_THREAD_ID = "CODEX_THREAD_ID"; var HOOKS_REL_PATH = ".codex/hooks.json"; var HOOK_COMMAND2 = `npx --yes agent-note hook --agent ${AGENT_NAMES.codex}`; var TRANSCRIPT_PREVIEW_CHARS = 4096; +var NESTED_CMD_KEY_RE = /(?:^|[,{]\s*)(?:cmd|["']cmd["'])\s*:\s*$/; var SHELL_MUTATION_COMMAND_RE = /(^|[;&|]\s*)(apply_patch|cat\s+>|cp\b|install\b|mkdir\b|mv\b|npm\s+(audit\s+fix|dedupe|install|update|version)\b|perl\s+-[^\n;&|]*i|pnpm\s+(add|install|update)\b|rm\b|sed\s+-[^\n;&|]*i|tee\b|touch\b|yarn\s+(add|install|upgrade)\b)|(\s|^)(>|>>)\s*\S+/; var CODEX_HOOK_EVENTS = { sessionStart: "SessionStart", @@ -826,6 +827,79 @@ function collectCommandStrings(value, seen = /* @__PURE__ */ new Set()) { ...collectCommandStrings(value.shell, seen) ]; } +function readJavaScriptString(source, start) { + const quote = source[start]; + if (quote !== '"' && quote !== "'" && quote !== "`") return null; + let value = ""; + for (let index = start + 1; index < source.length; index += 1) { + const char = source[index]; + if (char === quote) return { value, end: index + 1 }; + if (char !== "\\") { + value += char; + continue; + } + const escaped = source[index + 1]; + if (escaped === void 0) return null; + index += 1; + if (escaped === "\n") continue; + if (escaped === "n") value += "\n"; + else if (escaped === "r") value += "\r"; + else if (escaped === "t") value += " "; + else if (escaped === "b") value += "\b"; + else if (escaped === "f") value += "\f"; + else if (escaped === "v") value += "\v"; + else if (escaped === "0") value += "\0"; + else value += escaped; + } + return null; +} +function collectNestedExecEvidence(source) { + const toolNames = []; + const toolSeen = /* @__PURE__ */ new Set(); + const stringLiterals = []; + for (let index = 0; index < source.length; index += 1) { + const char = source[index]; + const next = source[index + 1]; + if (char === "/" && next === "/") { + const lineEnd = source.indexOf("\n", index + 2); + if (lineEnd === -1) break; + index = lineEnd; + continue; + } + if (char === "/" && next === "*") { + const commentEnd = source.indexOf("*/", index + 2); + if (commentEnd === -1) break; + index = commentEnd + 1; + continue; + } + const literal = readJavaScriptString(source, index); + if (literal) { + stringLiterals.push({ value: literal.value, start: index }); + index = literal.end - 1; + continue; + } + for (const toolName of ["apply_patch", "exec_command"]) { + const callee = `tools.${toolName}`; + if (!source.startsWith(callee, index)) continue; + const previous = source[index - 1]; + if (previous && /[\w$.]/.test(previous)) continue; + let cursor2 = index + callee.length; + while (/\s/.test(source[cursor2] ?? "")) cursor2 += 1; + if (source[cursor2] !== "(") continue; + appendUnique(toolNames, toolSeen, toolName); + index += callee.length - 1; + break; + } + } + return { + toolNames, + patchInputs: toolSeen.has("apply_patch") ? stringLiterals.map(({ value }) => value).filter((value) => value.includes("*** Begin Patch")) : [], + commandInputs: toolSeen.has("exec_command") ? stringLiterals.filter(({ start }) => { + const prefix = source.slice(Math.max(0, start - 80), start); + return NESTED_CMD_KEY_RE.test(prefix); + }).map(({ value }) => value) : [] + }; +} function readTranscriptSessionId(candidate) { try { const preview = readFileSync(candidate, TEXT_ENCODING).slice(0, TRANSCRIPT_PREVIEW_CHARS); @@ -1175,6 +1249,7 @@ ${response}` : response; continue; } const toolName = typeof payload.name === "string" ? payload.name : typeof payload.call_name === "string" ? payload.call_name : void 0; + const nestedExec = toolName === "exec" && typeof payload.input === "string" ? collectNestedExecEvidence(payload.input) : null; if ((payloadType === "custom_tool_call" || payloadType === "function_call" || payloadType === "tool_use") && toolName) { appendInteractionTool(current, toolName); if (toolName === "exec_command" && [ @@ -1184,11 +1259,20 @@ ${response}` : response; appendInteractionMutationTool(current, toolName); } } - if ((payloadType === "custom_tool_call" || payloadType === "function_call") && toolName === "apply_patch") { - const patchInputs = [ - ...collectPatchStrings(payload.input), - ...collectPatchStrings(payload.arguments) - ]; + if (nestedExec) { + for (const nestedToolName of nestedExec.toolNames) { + appendInteractionTool(current, nestedToolName); + } + if (nestedExec.toolNames.includes("apply_patch")) { + appendInteractionMutationTool(current, "apply_patch"); + } + if (nestedExec.commandInputs.some(isMutatingShellCommand)) { + appendInteractionMutationTool(current, "exec_command"); + } + } + const isDirectApplyPatch = (payloadType === "custom_tool_call" || payloadType === "function_call") && toolName === "apply_patch"; + const patchInputs = isDirectApplyPatch ? [...collectPatchStrings(payload.input), ...collectPatchStrings(payload.arguments)] : nestedExec?.patchInputs ?? []; + if (isDirectApplyPatch || patchInputs.length > 0) { const files = []; const fileSeen = /* @__PURE__ */ new Set(); current.line_stats = current.line_stats ?? {}; @@ -1212,7 +1296,7 @@ ${response}` : response; if (files.length > 0) { current.files_touched = [.../* @__PURE__ */ new Set([...current.files_touched ?? [], ...files])]; } - appendInteractionMutationTool(current, toolName); + appendInteractionMutationTool(current, "apply_patch"); if (Object.keys(current.line_stats).length === 0) { delete current.line_stats; } diff --git a/packages/cli/src/agents/codex.test.ts b/packages/cli/src/agents/codex.test.ts index 8779b90..ae2fa60 100644 --- a/packages/cli/src/agents/codex.test.ts +++ b/packages/cli/src/agents/codex.test.ts @@ -38,6 +38,21 @@ function buildShellCommandTranscript(command: string): string { ); } +function buildNestedExecTranscript(source: string): string { + return ( + '{"type":"session_meta","payload":{"id":"nested-exec","cwd":"/repo"}}\n' + + '{"type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"Update nested execution"}]}}\n' + + `${JSON.stringify({ + type: "response_item", + payload: { + type: "custom_tool_call", + name: "exec", + input: source, + }, + })}\n` + ); +} + describe("codex adapter", () => { let codexHome: string; let previousCodexHome: string | undefined; @@ -200,6 +215,82 @@ describe("codex adapter", () => { }); }); + it("extracts patch evidence from nested exec tool calls", async () => { + const transcriptDir = join(codexHome, "sessions", "nested-exec-patch"); + mkdirSync(transcriptDir, { recursive: true }); + const transcriptPath = join(transcriptDir, "rollout.jsonl"); + const patch = "*** Begin Patch\n*** Update File: src/nested.ts\n@@\n-old\n+new\n*** End Patch"; + const source = + `const patch = ${JSON.stringify(patch)};\n` + + 'const status = await tools.exec_command({cmd: "git status --short", workdir: "/repo"});\n' + + "text(status.output);\n" + + "const result = await tools.apply_patch(patch);\n" + + "text(result);"; + writeFileSync(transcriptPath, buildNestedExecTranscript(source)); + + const interactions = await codex.extractInteractions(transcriptPath); + + assert.deepEqual(interactions[0].tools, ["exec", "exec_command", "apply_patch"]); + assert.deepEqual(interactions[0].mutation_tools, ["apply_patch"]); + assert.deepEqual(interactions[0].files_touched, ["src/nested.ts"]); + assert.deepEqual(interactions[0].line_stats, { + "src/nested.ts": { added: 1, deleted: 1 }, + }); + }); + + it("marks nested apply_patch as mutating when the patch text is built dynamically", async () => { + const transcriptDir = join(codexHome, "sessions", "nested-exec-dynamic-patch"); + mkdirSync(transcriptDir, { recursive: true }); + const transcriptPath = join(transcriptDir, "rollout.jsonl"); + const source = + "const patch = buildPatch(files);\n" + + "const result = await tools.apply_patch(patch);\n" + + "text(result);"; + writeFileSync(transcriptPath, buildNestedExecTranscript(source)); + + const interactions = await codex.extractInteractions(transcriptPath); + + assert.deepEqual(interactions[0].tools, ["exec", "apply_patch"]); + assert.deepEqual(interactions[0].mutation_tools, ["apply_patch"]); + assert.equal(interactions[0].files_touched, undefined); + assert.equal(interactions[0].line_stats, undefined); + }); + + it("marks nested mutating exec commands without inferring files", async () => { + const transcriptDir = join(codexHome, "sessions", "nested-exec-command"); + mkdirSync(transcriptDir, { recursive: true }); + const transcriptPath = join(transcriptDir, "rollout.jsonl"); + const source = + 'const result = await tools.exec_command({cmd: "touch generated.txt", workdir: "/repo"});\n' + + "text(result.output);"; + writeFileSync(transcriptPath, buildNestedExecTranscript(source)); + + const interactions = await codex.extractInteractions(transcriptPath); + + assert.deepEqual(interactions[0].tools, ["exec", "exec_command"]); + assert.deepEqual(interactions[0].mutation_tools, ["exec_command"]); + assert.equal(interactions[0].files_touched, undefined); + }); + + it("does not attribute read-only or quoted nested tool references", async () => { + const transcriptDir = join(codexHome, "sessions", "nested-exec-readonly"); + mkdirSync(transcriptDir, { recursive: true }); + const transcriptPath = join(transcriptDir, "rollout.jsonl"); + const source = + 'const example = "tools.apply_patch(fakePatch)";\n' + + "// tools.apply_patch(commentedPatch);\n" + + 'const result = await tools.exec_command({cmd: "git status --short", justification: "rm is not run"});\n' + + "text(example);\n" + + "text(result.output);"; + writeFileSync(transcriptPath, buildNestedExecTranscript(source)); + + const interactions = await codex.extractInteractions(transcriptPath); + + assert.deepEqual(interactions[0].tools, ["exec", "exec_command"]); + assert.equal(interactions[0].mutation_tools, undefined); + assert.equal(interactions[0].files_touched, undefined); + }); + it("marks mutating shell commands as mutation tools", async () => { const transcriptDir = join(codexHome, "sessions", "shell-mutation"); mkdirSync(transcriptDir, { recursive: true }); diff --git a/packages/cli/src/agents/codex.ts b/packages/cli/src/agents/codex.ts index 7e95060..854017d 100644 --- a/packages/cli/src/agents/codex.ts +++ b/packages/cli/src/agents/codex.ts @@ -21,6 +21,9 @@ const ENV_CODEX_THREAD_ID = "CODEX_THREAD_ID"; const HOOKS_REL_PATH = ".codex/hooks.json"; const HOOK_COMMAND = `npx --yes agent-note hook --agent ${AGENT_NAMES.codex}`; const TRANSCRIPT_PREVIEW_CHARS = 4096; +// Matches a `cmd:` (or quoted "cmd":) property key immediately preceding a +// string literal, so only shell-command argument values feed mutation detection. +const NESTED_CMD_KEY_RE = /(?:^|[,{]\s*)(?:cmd|["']cmd["'])\s*:\s*$/; const SHELL_MUTATION_COMMAND_RE = /(^|[;&|]\s*)(apply_patch|cat\s+>|cp\b|install\b|mkdir\b|mv\b|npm\s+(audit\s+fix|dedupe|install|update|version)\b|perl\s+-[^\n;&|]*i|pnpm\s+(add|install|update)\b|rm\b|sed\s+-[^\n;&|]*i|tee\b|touch\b|yarn\s+(add|install|upgrade)\b)|(\s|^)(>|>>)\s*\S+/; const CODEX_HOOK_EVENTS = { @@ -158,6 +161,105 @@ function collectCommandStrings(value: unknown, seen = new Set()): strin ]; } +type NestedExecEvidence = { + toolNames: string[]; + patchInputs: string[]; + commandInputs: string[]; +}; + +function readJavaScriptString( + source: string, + start: number, +): { value: string; end: number } | null { + const quote = source[start]; + if (quote !== '"' && quote !== "'" && quote !== "`") return null; + + let value = ""; + for (let index = start + 1; index < source.length; index += 1) { + const char = source[index]; + if (char === quote) return { value, end: index + 1 }; + if (char !== "\\") { + value += char; + continue; + } + + const escaped = source[index + 1]; + if (escaped === undefined) return null; + index += 1; + if (escaped === "\n") continue; + if (escaped === "n") value += "\n"; + else if (escaped === "r") value += "\r"; + else if (escaped === "t") value += "\t"; + else if (escaped === "b") value += "\b"; + else if (escaped === "f") value += "\f"; + else if (escaped === "v") value += "\v"; + else if (escaped === "0") value += "\0"; + else value += escaped; + } + + return null; +} + +function collectNestedExecEvidence(source: string): NestedExecEvidence { + const toolNames: string[] = []; + const toolSeen = new Set(); + const stringLiterals: Array<{ value: string; start: number }> = []; + + for (let index = 0; index < source.length; index += 1) { + const char = source[index]; + const next = source[index + 1]; + if (char === "/" && next === "/") { + const lineEnd = source.indexOf("\n", index + 2); + if (lineEnd === -1) break; + index = lineEnd; + continue; + } + if (char === "/" && next === "*") { + const commentEnd = source.indexOf("*/", index + 2); + if (commentEnd === -1) break; + index = commentEnd + 1; + continue; + } + + const literal = readJavaScriptString(source, index); + if (literal) { + stringLiterals.push({ value: literal.value, start: index }); + index = literal.end - 1; + continue; + } + + for (const toolName of ["apply_patch", "exec_command"]) { + const callee = `tools.${toolName}`; + if (!source.startsWith(callee, index)) continue; + const previous = source[index - 1]; + if (previous && /[\w$.]/.test(previous)) continue; + let cursor = index + callee.length; + while (/\s/.test(source[cursor] ?? "")) cursor += 1; + if (source[cursor] !== "(") continue; + appendUnique(toolNames, toolSeen, toolName); + index += callee.length - 1; + break; + } + } + + return { + toolNames, + patchInputs: toolSeen.has("apply_patch") + ? stringLiterals + .map(({ value }) => value) + .filter((value) => value.includes("*** Begin Patch")) + : [], + commandInputs: toolSeen.has("exec_command") + ? stringLiterals + .filter(({ start }) => { + const prefix = source.slice(Math.max(0, start - 80), start); + return NESTED_CMD_KEY_RE.test(prefix); + }) + .map(({ value }) => value) + : [], + }; +} + /** * Identify the Agent Note session inside a Codex transcript candidate. * @@ -600,6 +702,10 @@ export const codex: AgentAdapter = { : typeof payload.call_name === "string" ? payload.call_name : undefined; + const nestedExec = + toolName === "exec" && typeof payload.input === "string" + ? collectNestedExecEvidence(payload.input) + : null; if ( (payloadType === "custom_tool_call" || @@ -619,14 +725,27 @@ export const codex: AgentAdapter = { } } - if ( + if (nestedExec) { + for (const nestedToolName of nestedExec.toolNames) { + appendInteractionTool(current, nestedToolName); + } + // A nested apply_patch call is always mutating, even when the patch + // text is built dynamically and cannot be recovered from the source. + if (nestedExec.toolNames.includes("apply_patch")) { + appendInteractionMutationTool(current, "apply_patch"); + } + if (nestedExec.commandInputs.some(isMutatingShellCommand)) { + appendInteractionMutationTool(current, "exec_command"); + } + } + + const isDirectApplyPatch = (payloadType === "custom_tool_call" || payloadType === "function_call") && - toolName === "apply_patch" - ) { - const patchInputs = [ - ...collectPatchStrings(payload.input), - ...collectPatchStrings(payload.arguments), - ]; + toolName === "apply_patch"; + const patchInputs = isDirectApplyPatch + ? [...collectPatchStrings(payload.input), ...collectPatchStrings(payload.arguments)] + : (nestedExec?.patchInputs ?? []); + if (isDirectApplyPatch || patchInputs.length > 0) { const files: string[] = []; const fileSeen = new Set(); current.line_stats = current.line_stats ?? {}; @@ -653,7 +772,7 @@ export const codex: AgentAdapter = { if (files.length > 0) { current.files_touched = [...new Set([...(current.files_touched ?? []), ...files])]; } - appendInteractionMutationTool(current, toolName); + appendInteractionMutationTool(current, "apply_patch"); if (Object.keys(current.line_stats).length === 0) { delete current.line_stats; diff --git a/packages/cli/src/commands/codex.test.ts b/packages/cli/src/commands/codex.test.ts index dc3af19..7f08653 100644 --- a/packages/cli/src/commands/codex.test.ts +++ b/packages/cli/src/commands/codex.test.ts @@ -29,6 +29,50 @@ function buildRealSessionPatchTranscript(opts: { ); } +function buildNestedExecPatchTranscript(opts: { + sessionId: string; + workdir: string; + prompt: string; + response: string; + patchPath: string; +}): string { + const patch = `*** Begin Patch\n*** Add File: ${opts.patchPath}\n+nested status\n*** End Patch`; + const source = + `const patch = ${JSON.stringify(patch)};\n` + + "const result = await tools.apply_patch(patch);\n" + + "text(result);"; + return ( + `${JSON.stringify({ + timestamp: "2026-04-15T09:31:23.296Z", + type: "session_meta", + payload: { id: opts.sessionId, cwd: opts.workdir, cli_version: "0.142.5" }, + })}\n` + + `${JSON.stringify({ + timestamp: "2026-04-15T09:31:24.296Z", + type: "response_item", + payload: { + type: "message", + role: "user", + content: [{ type: "input_text", text: opts.prompt }], + }, + })}\n` + + `${JSON.stringify({ + timestamp: "2026-04-15T09:31:25.296Z", + type: "response_item", + payload: { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: opts.response }], + }, + })}\n` + + `${JSON.stringify({ + timestamp: "2026-04-15T09:31:26.296Z", + type: "response_item", + payload: { type: "custom_tool_call", name: "exec", input: source }, + })}\n` + ); +} + describe("agentnote codex", () => { let testDir: string; let testHome: string; @@ -157,6 +201,57 @@ describe("agentnote codex", () => { ); }); + it("records notes and AI ratio from nested exec patch transcripts", () => { + const sessionId = "codex-session-nested-exec"; + const transcriptDir = join(testHome, ".codex", "sessions"); + mkdirSync(transcriptDir, { recursive: true }); + const transcriptPath = join(transcriptDir, "rollout-nested-exec.jsonl"); + writeFileSync( + transcriptPath, + buildNestedExecPatchTranscript({ + sessionId, + workdir: testDir, + patchPath: "nested-exec.txt", + prompt: "Create nested-exec.txt", + response: "Creating the file through the unified executor.", + }), + ); + + runCodexHook({ + hook_event_name: "SessionStart", + session_id: sessionId, + transcript_path: transcriptPath, + model: "gpt-5-codex", + }); + runCodexHook({ + hook_event_name: "UserPromptSubmit", + session_id: sessionId, + transcript_path: transcriptPath, + prompt: "Create nested-exec.txt", + model: "gpt-5-codex", + }); + + writeFileSync(join(testDir, "nested-exec.txt"), "nested status\n"); + execSync("git add nested-exec.txt", { cwd: testDir }); + const output = execSync(`node ${cliPath} commit -m "feat: codex nested exec" 2>&1`, { + cwd: testDir, + env: { ...process.env, HOME: testHome }, + encoding: "utf-8", + }); + assert.match(output, /agent-note: 1 prompts, AI ratio 100%/, output); + + const note = JSON.parse( + execSync("git notes --ref=agentnote show HEAD", { + cwd: testDir, + encoding: "utf-8", + }), + ); + assert.equal(note.attribution.method, "line"); + assert.equal(note.attribution.ai_ratio, 100); + assert.deepEqual(note.interactions[0].files_touched, ["nested-exec.txt"]); + assert.deepEqual(note.interactions[0].tools, ["exec", "apply_patch"]); + }); + it("falls back to file attribution when transcript patch counts do not match the commit", () => { const sessionId = "codex-session-2"; const transcriptDir = join(testHome, ".codex", "sessions");