From 75023aa4742fa5ad78065cf2942c7a4fa838127d Mon Sep 17 00:00:00 2001 From: wasabeef Date: Wed, 8 Jul 2026 16:25:26 +0900 Subject: [PATCH 1/2] fix(codex): attribute nested exec tool calls from current Codex transcripts Why Codex CLI 0.142.5 records unified-executor edits as JavaScript nested inside custom_tool_call exec entries instead of direct apply_patch tool calls. The transcript parser only recognized direct tool calls, so nested edits produced no file or mutation evidence and commits were left without an Agent Note git note. User impact Codex commits made through the unified exec tool now record git notes with file attribution and an AI ratio. Nested apply_patch calls whose patch text is built dynamically still count as mutation evidence, and read-only or quoted nested references do not create false attribution. Verification npm run build --prefix packages/cli npm run typecheck --prefix packages/cli npm run lint --prefix packages/cli npm test --prefix packages/cli (491 passed) Release note: Codex edits made through the unified exec tool are now recorded with file attribution and an AI ratio. Agentnote-Session: 4a6d322d-5afe-45e3-a11b-cf15d0e740a9 --- packages/cli/dist/cli.js | 95 +++++++++++++++-- packages/cli/src/agents/codex.test.ts | 91 ++++++++++++++++ packages/cli/src/agents/codex.ts | 132 ++++++++++++++++++++++-- packages/cli/src/commands/codex.test.ts | 95 +++++++++++++++++ 4 files changed, 399 insertions(+), 14 deletions(-) diff --git a/packages/cli/dist/cli.js b/packages/cli/dist/cli.js index d2adc3b6..54e79afd 100755 --- a/packages/cli/dist/cli.js +++ b/packages/cli/dist/cli.js @@ -826,6 +826,79 @@ function collectCommandStrings(value, seen = /* @__PURE__ */ new Set()) { ...collectCommandStrings(value.shell, seen) ]; } +function readJavaScriptString(source, start) { + const quote = source[start]; + if (quote !== '"' && quote !== "'" && quote !== "`") return null; + let value = ""; + for (let index = start + 1; index < source.length; index += 1) { + const char = source[index]; + if (char === quote) return { value, end: index + 1 }; + if (char !== "\\") { + value += char; + continue; + } + const escaped = source[index + 1]; + if (escaped === void 0) return null; + index += 1; + if (escaped === "\n") continue; + if (escaped === "n") value += "\n"; + else if (escaped === "r") value += "\r"; + else if (escaped === "t") value += " "; + else if (escaped === "b") value += "\b"; + else if (escaped === "f") value += "\f"; + else if (escaped === "v") value += "\v"; + else if (escaped === "0") value += "\0"; + else value += escaped; + } + return null; +} +function collectNestedExecEvidence(source) { + const toolNames = []; + const toolSeen = /* @__PURE__ */ new Set(); + const stringLiterals = []; + for (let index = 0; index < source.length; index += 1) { + const char = source[index]; + const next = source[index + 1]; + if (char === "/" && next === "/") { + const lineEnd = source.indexOf("\n", index + 2); + if (lineEnd === -1) break; + index = lineEnd; + continue; + } + if (char === "/" && next === "*") { + const commentEnd = source.indexOf("*/", index + 2); + if (commentEnd === -1) break; + index = commentEnd + 1; + continue; + } + const literal = readJavaScriptString(source, index); + if (literal) { + stringLiterals.push({ value: literal.value, start: index }); + index = literal.end - 1; + continue; + } + for (const toolName of ["apply_patch", "exec_command"]) { + const callee = `tools.${toolName}`; + if (!source.startsWith(callee, index)) continue; + const previous = source[index - 1]; + if (previous && /[\w$.]/.test(previous)) continue; + let cursor2 = index + callee.length; + while (/\s/.test(source[cursor2] ?? "")) cursor2 += 1; + if (source[cursor2] !== "(") continue; + appendUnique(toolNames, toolSeen, toolName); + index += callee.length - 1; + break; + } + } + return { + toolNames, + patchInputs: toolSeen.has("apply_patch") ? stringLiterals.map(({ value }) => value).filter((value) => value.includes("*** Begin Patch")) : [], + commandInputs: toolSeen.has("exec_command") ? stringLiterals.filter(({ start }) => { + const prefix = source.slice(Math.max(0, start - 80), start); + return /(?:^|[,{]\s*)(?:cmd|["']cmd["'])\s*:\s*$/.test(prefix); + }).map(({ value }) => value) : [] + }; +} function readTranscriptSessionId(candidate) { try { const preview = readFileSync(candidate, TEXT_ENCODING).slice(0, TRANSCRIPT_PREVIEW_CHARS); @@ -1175,6 +1248,7 @@ ${response}` : response; continue; } const toolName = typeof payload.name === "string" ? payload.name : typeof payload.call_name === "string" ? payload.call_name : void 0; + const nestedExec = toolName === "exec" && typeof payload.input === "string" ? collectNestedExecEvidence(payload.input) : null; if ((payloadType === "custom_tool_call" || payloadType === "function_call" || payloadType === "tool_use") && toolName) { appendInteractionTool(current, toolName); if (toolName === "exec_command" && [ @@ -1184,11 +1258,20 @@ ${response}` : response; appendInteractionMutationTool(current, toolName); } } - if ((payloadType === "custom_tool_call" || payloadType === "function_call") && toolName === "apply_patch") { - const patchInputs = [ - ...collectPatchStrings(payload.input), - ...collectPatchStrings(payload.arguments) - ]; + if (nestedExec) { + for (const nestedToolName of nestedExec.toolNames) { + appendInteractionTool(current, nestedToolName); + } + if (nestedExec.toolNames.includes("apply_patch")) { + appendInteractionMutationTool(current, "apply_patch"); + } + if (nestedExec.commandInputs.some(isMutatingShellCommand)) { + appendInteractionMutationTool(current, "exec_command"); + } + } + const isDirectApplyPatch = (payloadType === "custom_tool_call" || payloadType === "function_call") && toolName === "apply_patch"; + const patchInputs = isDirectApplyPatch ? [...collectPatchStrings(payload.input), ...collectPatchStrings(payload.arguments)] : nestedExec?.patchInputs ?? []; + if (isDirectApplyPatch || patchInputs.length > 0) { const files = []; const fileSeen = /* @__PURE__ */ new Set(); current.line_stats = current.line_stats ?? {}; @@ -1212,7 +1295,7 @@ ${response}` : response; if (files.length > 0) { current.files_touched = [.../* @__PURE__ */ new Set([...current.files_touched ?? [], ...files])]; } - appendInteractionMutationTool(current, toolName); + appendInteractionMutationTool(current, "apply_patch"); if (Object.keys(current.line_stats).length === 0) { delete current.line_stats; } diff --git a/packages/cli/src/agents/codex.test.ts b/packages/cli/src/agents/codex.test.ts index 8779b90b..ae2fa608 100644 --- a/packages/cli/src/agents/codex.test.ts +++ b/packages/cli/src/agents/codex.test.ts @@ -38,6 +38,21 @@ function buildShellCommandTranscript(command: string): string { ); } +function buildNestedExecTranscript(source: string): string { + return ( + '{"type":"session_meta","payload":{"id":"nested-exec","cwd":"/repo"}}\n' + + '{"type":"response_item","payload":{"type":"message","role":"user","content":[{"type":"input_text","text":"Update nested execution"}]}}\n' + + `${JSON.stringify({ + type: "response_item", + payload: { + type: "custom_tool_call", + name: "exec", + input: source, + }, + })}\n` + ); +} + describe("codex adapter", () => { let codexHome: string; let previousCodexHome: string | undefined; @@ -200,6 +215,82 @@ describe("codex adapter", () => { }); }); + it("extracts patch evidence from nested exec tool calls", async () => { + const transcriptDir = join(codexHome, "sessions", "nested-exec-patch"); + mkdirSync(transcriptDir, { recursive: true }); + const transcriptPath = join(transcriptDir, "rollout.jsonl"); + const patch = "*** Begin Patch\n*** Update File: src/nested.ts\n@@\n-old\n+new\n*** End Patch"; + const source = + `const patch = ${JSON.stringify(patch)};\n` + + 'const status = await tools.exec_command({cmd: "git status --short", workdir: "/repo"});\n' + + "text(status.output);\n" + + "const result = await tools.apply_patch(patch);\n" + + "text(result);"; + writeFileSync(transcriptPath, buildNestedExecTranscript(source)); + + const interactions = await codex.extractInteractions(transcriptPath); + + assert.deepEqual(interactions[0].tools, ["exec", "exec_command", "apply_patch"]); + assert.deepEqual(interactions[0].mutation_tools, ["apply_patch"]); + assert.deepEqual(interactions[0].files_touched, ["src/nested.ts"]); + assert.deepEqual(interactions[0].line_stats, { + "src/nested.ts": { added: 1, deleted: 1 }, + }); + }); + + it("marks nested apply_patch as mutating when the patch text is built dynamically", async () => { + const transcriptDir = join(codexHome, "sessions", "nested-exec-dynamic-patch"); + mkdirSync(transcriptDir, { recursive: true }); + const transcriptPath = join(transcriptDir, "rollout.jsonl"); + const source = + "const patch = buildPatch(files);\n" + + "const result = await tools.apply_patch(patch);\n" + + "text(result);"; + writeFileSync(transcriptPath, buildNestedExecTranscript(source)); + + const interactions = await codex.extractInteractions(transcriptPath); + + assert.deepEqual(interactions[0].tools, ["exec", "apply_patch"]); + assert.deepEqual(interactions[0].mutation_tools, ["apply_patch"]); + assert.equal(interactions[0].files_touched, undefined); + assert.equal(interactions[0].line_stats, undefined); + }); + + it("marks nested mutating exec commands without inferring files", async () => { + const transcriptDir = join(codexHome, "sessions", "nested-exec-command"); + mkdirSync(transcriptDir, { recursive: true }); + const transcriptPath = join(transcriptDir, "rollout.jsonl"); + const source = + 'const result = await tools.exec_command({cmd: "touch generated.txt", workdir: "/repo"});\n' + + "text(result.output);"; + writeFileSync(transcriptPath, buildNestedExecTranscript(source)); + + const interactions = await codex.extractInteractions(transcriptPath); + + assert.deepEqual(interactions[0].tools, ["exec", "exec_command"]); + assert.deepEqual(interactions[0].mutation_tools, ["exec_command"]); + assert.equal(interactions[0].files_touched, undefined); + }); + + it("does not attribute read-only or quoted nested tool references", async () => { + const transcriptDir = join(codexHome, "sessions", "nested-exec-readonly"); + mkdirSync(transcriptDir, { recursive: true }); + const transcriptPath = join(transcriptDir, "rollout.jsonl"); + const source = + 'const example = "tools.apply_patch(fakePatch)";\n' + + "// tools.apply_patch(commentedPatch);\n" + + 'const result = await tools.exec_command({cmd: "git status --short", justification: "rm is not run"});\n' + + "text(example);\n" + + "text(result.output);"; + writeFileSync(transcriptPath, buildNestedExecTranscript(source)); + + const interactions = await codex.extractInteractions(transcriptPath); + + assert.deepEqual(interactions[0].tools, ["exec", "exec_command"]); + assert.equal(interactions[0].mutation_tools, undefined); + assert.equal(interactions[0].files_touched, undefined); + }); + it("marks mutating shell commands as mutation tools", async () => { const transcriptDir = join(codexHome, "sessions", "shell-mutation"); mkdirSync(transcriptDir, { recursive: true }); diff --git a/packages/cli/src/agents/codex.ts b/packages/cli/src/agents/codex.ts index 7e950600..db71a767 100644 --- a/packages/cli/src/agents/codex.ts +++ b/packages/cli/src/agents/codex.ts @@ -158,6 +158,105 @@ function collectCommandStrings(value: unknown, seen = new Set()): strin ]; } +type NestedExecEvidence = { + toolNames: string[]; + patchInputs: string[]; + commandInputs: string[]; +}; + +function readJavaScriptString( + source: string, + start: number, +): { value: string; end: number } | null { + const quote = source[start]; + if (quote !== '"' && quote !== "'" && quote !== "`") return null; + + let value = ""; + for (let index = start + 1; index < source.length; index += 1) { + const char = source[index]; + if (char === quote) return { value, end: index + 1 }; + if (char !== "\\") { + value += char; + continue; + } + + const escaped = source[index + 1]; + if (escaped === undefined) return null; + index += 1; + if (escaped === "\n") continue; + if (escaped === "n") value += "\n"; + else if (escaped === "r") value += "\r"; + else if (escaped === "t") value += "\t"; + else if (escaped === "b") value += "\b"; + else if (escaped === "f") value += "\f"; + else if (escaped === "v") value += "\v"; + else if (escaped === "0") value += "\0"; + else value += escaped; + } + + return null; +} + +function collectNestedExecEvidence(source: string): NestedExecEvidence { + const toolNames: string[] = []; + const toolSeen = new Set(); + const stringLiterals: Array<{ value: string; start: number }> = []; + + for (let index = 0; index < source.length; index += 1) { + const char = source[index]; + const next = source[index + 1]; + if (char === "/" && next === "/") { + const lineEnd = source.indexOf("\n", index + 2); + if (lineEnd === -1) break; + index = lineEnd; + continue; + } + if (char === "/" && next === "*") { + const commentEnd = source.indexOf("*/", index + 2); + if (commentEnd === -1) break; + index = commentEnd + 1; + continue; + } + + const literal = readJavaScriptString(source, index); + if (literal) { + stringLiterals.push({ value: literal.value, start: index }); + index = literal.end - 1; + continue; + } + + for (const toolName of ["apply_patch", "exec_command"]) { + const callee = `tools.${toolName}`; + if (!source.startsWith(callee, index)) continue; + const previous = source[index - 1]; + if (previous && /[\w$.]/.test(previous)) continue; + let cursor = index + callee.length; + while (/\s/.test(source[cursor] ?? "")) cursor += 1; + if (source[cursor] !== "(") continue; + appendUnique(toolNames, toolSeen, toolName); + index += callee.length - 1; + break; + } + } + + return { + toolNames, + patchInputs: toolSeen.has("apply_patch") + ? stringLiterals + .map(({ value }) => value) + .filter((value) => value.includes("*** Begin Patch")) + : [], + commandInputs: toolSeen.has("exec_command") + ? stringLiterals + .filter(({ start }) => { + const prefix = source.slice(Math.max(0, start - 80), start); + return /(?:^|[,{]\s*)(?:cmd|["']cmd["'])\s*:\s*$/.test(prefix); + }) + .map(({ value }) => value) + : [], + }; +} + /** * Identify the Agent Note session inside a Codex transcript candidate. * @@ -600,6 +699,10 @@ export const codex: AgentAdapter = { : typeof payload.call_name === "string" ? payload.call_name : undefined; + const nestedExec = + toolName === "exec" && typeof payload.input === "string" + ? collectNestedExecEvidence(payload.input) + : null; if ( (payloadType === "custom_tool_call" || @@ -619,14 +722,27 @@ export const codex: AgentAdapter = { } } - if ( + if (nestedExec) { + for (const nestedToolName of nestedExec.toolNames) { + appendInteractionTool(current, nestedToolName); + } + // A nested apply_patch call is always mutating, even when the patch + // text is built dynamically and cannot be recovered from the source. + if (nestedExec.toolNames.includes("apply_patch")) { + appendInteractionMutationTool(current, "apply_patch"); + } + if (nestedExec.commandInputs.some(isMutatingShellCommand)) { + appendInteractionMutationTool(current, "exec_command"); + } + } + + const isDirectApplyPatch = (payloadType === "custom_tool_call" || payloadType === "function_call") && - toolName === "apply_patch" - ) { - const patchInputs = [ - ...collectPatchStrings(payload.input), - ...collectPatchStrings(payload.arguments), - ]; + toolName === "apply_patch"; + const patchInputs = isDirectApplyPatch + ? [...collectPatchStrings(payload.input), ...collectPatchStrings(payload.arguments)] + : (nestedExec?.patchInputs ?? []); + if (isDirectApplyPatch || patchInputs.length > 0) { const files: string[] = []; const fileSeen = new Set(); current.line_stats = current.line_stats ?? {}; @@ -653,7 +769,7 @@ export const codex: AgentAdapter = { if (files.length > 0) { current.files_touched = [...new Set([...(current.files_touched ?? []), ...files])]; } - appendInteractionMutationTool(current, toolName); + appendInteractionMutationTool(current, "apply_patch"); if (Object.keys(current.line_stats).length === 0) { delete current.line_stats; diff --git a/packages/cli/src/commands/codex.test.ts b/packages/cli/src/commands/codex.test.ts index dc3af19d..7f086530 100644 --- a/packages/cli/src/commands/codex.test.ts +++ b/packages/cli/src/commands/codex.test.ts @@ -29,6 +29,50 @@ function buildRealSessionPatchTranscript(opts: { ); } +function buildNestedExecPatchTranscript(opts: { + sessionId: string; + workdir: string; + prompt: string; + response: string; + patchPath: string; +}): string { + const patch = `*** Begin Patch\n*** Add File: ${opts.patchPath}\n+nested status\n*** End Patch`; + const source = + `const patch = ${JSON.stringify(patch)};\n` + + "const result = await tools.apply_patch(patch);\n" + + "text(result);"; + return ( + `${JSON.stringify({ + timestamp: "2026-04-15T09:31:23.296Z", + type: "session_meta", + payload: { id: opts.sessionId, cwd: opts.workdir, cli_version: "0.142.5" }, + })}\n` + + `${JSON.stringify({ + timestamp: "2026-04-15T09:31:24.296Z", + type: "response_item", + payload: { + type: "message", + role: "user", + content: [{ type: "input_text", text: opts.prompt }], + }, + })}\n` + + `${JSON.stringify({ + timestamp: "2026-04-15T09:31:25.296Z", + type: "response_item", + payload: { + type: "message", + role: "assistant", + content: [{ type: "output_text", text: opts.response }], + }, + })}\n` + + `${JSON.stringify({ + timestamp: "2026-04-15T09:31:26.296Z", + type: "response_item", + payload: { type: "custom_tool_call", name: "exec", input: source }, + })}\n` + ); +} + describe("agentnote codex", () => { let testDir: string; let testHome: string; @@ -157,6 +201,57 @@ describe("agentnote codex", () => { ); }); + it("records notes and AI ratio from nested exec patch transcripts", () => { + const sessionId = "codex-session-nested-exec"; + const transcriptDir = join(testHome, ".codex", "sessions"); + mkdirSync(transcriptDir, { recursive: true }); + const transcriptPath = join(transcriptDir, "rollout-nested-exec.jsonl"); + writeFileSync( + transcriptPath, + buildNestedExecPatchTranscript({ + sessionId, + workdir: testDir, + patchPath: "nested-exec.txt", + prompt: "Create nested-exec.txt", + response: "Creating the file through the unified executor.", + }), + ); + + runCodexHook({ + hook_event_name: "SessionStart", + session_id: sessionId, + transcript_path: transcriptPath, + model: "gpt-5-codex", + }); + runCodexHook({ + hook_event_name: "UserPromptSubmit", + session_id: sessionId, + transcript_path: transcriptPath, + prompt: "Create nested-exec.txt", + model: "gpt-5-codex", + }); + + writeFileSync(join(testDir, "nested-exec.txt"), "nested status\n"); + execSync("git add nested-exec.txt", { cwd: testDir }); + const output = execSync(`node ${cliPath} commit -m "feat: codex nested exec" 2>&1`, { + cwd: testDir, + env: { ...process.env, HOME: testHome }, + encoding: "utf-8", + }); + assert.match(output, /agent-note: 1 prompts, AI ratio 100%/, output); + + const note = JSON.parse( + execSync("git notes --ref=agentnote show HEAD", { + cwd: testDir, + encoding: "utf-8", + }), + ); + assert.equal(note.attribution.method, "line"); + assert.equal(note.attribution.ai_ratio, 100); + assert.deepEqual(note.interactions[0].files_touched, ["nested-exec.txt"]); + assert.deepEqual(note.interactions[0].tools, ["exec", "apply_patch"]); + }); + it("falls back to file attribution when transcript patch counts do not match the commit", () => { const sessionId = "codex-session-2"; const transcriptDir = join(testHome, ".codex", "sessions"); From 41996c03a9c0c4bde3ee274ebe7dafcc777ef6dc Mon Sep 17 00:00:00 2001 From: wasabeef Date: Wed, 8 Jul 2026 16:33:39 +0900 Subject: [PATCH 2/2] refactor(codex): name the nested cmd key regex Hoist the inline mutation-evidence regex into NESTED_CMD_KEY_RE with an intent comment, matching how SHELL_MUTATION_COMMAND_RE is declared. Addresses CodeRabbit review feedback on PR #85. Release note: skip Agentnote-Session: 4a6d322d-5afe-45e3-a11b-cf15d0e740a9 --- packages/cli/dist/cli.js | 3 ++- packages/cli/src/agents/codex.ts | 5 ++++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/packages/cli/dist/cli.js b/packages/cli/dist/cli.js index 54e79afd..ce5efd9d 100755 --- a/packages/cli/dist/cli.js +++ b/packages/cli/dist/cli.js @@ -733,6 +733,7 @@ var ENV_CODEX_THREAD_ID = "CODEX_THREAD_ID"; var HOOKS_REL_PATH = ".codex/hooks.json"; var HOOK_COMMAND2 = `npx --yes agent-note hook --agent ${AGENT_NAMES.codex}`; var TRANSCRIPT_PREVIEW_CHARS = 4096; +var NESTED_CMD_KEY_RE = /(?:^|[,{]\s*)(?:cmd|["']cmd["'])\s*:\s*$/; var SHELL_MUTATION_COMMAND_RE = /(^|[;&|]\s*)(apply_patch|cat\s+>|cp\b|install\b|mkdir\b|mv\b|npm\s+(audit\s+fix|dedupe|install|update|version)\b|perl\s+-[^\n;&|]*i|pnpm\s+(add|install|update)\b|rm\b|sed\s+-[^\n;&|]*i|tee\b|touch\b|yarn\s+(add|install|upgrade)\b)|(\s|^)(>|>>)\s*\S+/; var CODEX_HOOK_EVENTS = { sessionStart: "SessionStart", @@ -895,7 +896,7 @@ function collectNestedExecEvidence(source) { patchInputs: toolSeen.has("apply_patch") ? stringLiterals.map(({ value }) => value).filter((value) => value.includes("*** Begin Patch")) : [], commandInputs: toolSeen.has("exec_command") ? stringLiterals.filter(({ start }) => { const prefix = source.slice(Math.max(0, start - 80), start); - return /(?:^|[,{]\s*)(?:cmd|["']cmd["'])\s*:\s*$/.test(prefix); + return NESTED_CMD_KEY_RE.test(prefix); }).map(({ value }) => value) : [] }; } diff --git a/packages/cli/src/agents/codex.ts b/packages/cli/src/agents/codex.ts index db71a767..854017d7 100644 --- a/packages/cli/src/agents/codex.ts +++ b/packages/cli/src/agents/codex.ts @@ -21,6 +21,9 @@ const ENV_CODEX_THREAD_ID = "CODEX_THREAD_ID"; const HOOKS_REL_PATH = ".codex/hooks.json"; const HOOK_COMMAND = `npx --yes agent-note hook --agent ${AGENT_NAMES.codex}`; const TRANSCRIPT_PREVIEW_CHARS = 4096; +// Matches a `cmd:` (or quoted "cmd":) property key immediately preceding a +// string literal, so only shell-command argument values feed mutation detection. +const NESTED_CMD_KEY_RE = /(?:^|[,{]\s*)(?:cmd|["']cmd["'])\s*:\s*$/; const SHELL_MUTATION_COMMAND_RE = /(^|[;&|]\s*)(apply_patch|cat\s+>|cp\b|install\b|mkdir\b|mv\b|npm\s+(audit\s+fix|dedupe|install|update|version)\b|perl\s+-[^\n;&|]*i|pnpm\s+(add|install|update)\b|rm\b|sed\s+-[^\n;&|]*i|tee\b|touch\b|yarn\s+(add|install|upgrade)\b)|(\s|^)(>|>>)\s*\S+/; const CODEX_HOOK_EVENTS = { @@ -250,7 +253,7 @@ function collectNestedExecEvidence(source: string): NestedExecEvidence { ? stringLiterals .filter(({ start }) => { const prefix = source.slice(Math.max(0, start - 80), start); - return /(?:^|[,{]\s*)(?:cmd|["']cmd["'])\s*:\s*$/.test(prefix); + return NESTED_CMD_KEY_RE.test(prefix); }) .map(({ value }) => value) : [],