diff --git a/packages/codev/src/__tests__/consult-truthfulness.test.ts b/packages/codev/src/__tests__/consult-truthfulness.test.ts new file mode 100644 index 000000000..10ea54fde --- /dev/null +++ b/packages/codev/src/__tests__/consult-truthfulness.test.ts @@ -0,0 +1,244 @@ +/** + * Issues #25, #35, #43 — consult saying the wrong thing when it cannot tell. + * + * #43 A bare `--type pr` failed by naming `codev/consult-types/pr-review.md`, + * a path that has never shipped in any release. The reader is being told + * to create a file; the actual fix is `--protocol`. + * #35 A 0-byte PR diff produced a prompt saying `Changed Files (0)`, and + * three lanes returned APPROVE (HIGH) on nothing. + * #25 The agy skip artifact ended with "install the CLI and sign in" + * regardless of cause — two wrong instructions for a quota wall. + */ + +import { describe, it, expect } from 'vitest'; +import * as fs from 'node:fs'; +import * as os from 'node:os'; +import * as path from 'node:path'; +import { + agyRemedy, + buildOpencodeArgs, + extractSandboxPaths, + _buildPRQuery, + _consultSandboxDirForTest as _consultSandboxDir, +} from '../commands/consult/index.js'; +import { protocolsProvidingConsultType } from '../lib/skeleton.js'; + +describe('#43: a protocol-scoped review type must name the real remedy', () => { + it('finds the protocols that actually ship pr-review.md', () => { + const owners = protocolsProvidingConsultType('pr-review.md'); + + // Whichever tier answers, the answer must be non-empty and must include the + // protocols this repo ships. If this ever returns [], the error message + // correctly falls back to the plain not-found rather than guessing. + expect(owners.length).toBeGreaterThan(0); + expect(owners).toContain('bugfix'); + expect(owners).toContain('pir'); + }); + + it('reports nothing for a review type no protocol provides', () => { + // "I could not find an owner" must be expressible. Returning a plausible + // list here would be a second wrong remedy replacing the first. + expect(protocolsProvidingConsultType('nonsense-review.md')).toEqual([]); + }); + + it('integration-review is NOT protocol-scoped, so it needs no --protocol', () => { + // The one type that legitimately resolves bare. Pinned so a future change + // that moves it under protocols/ has to notice. + expect(protocolsProvidingConsultType('integration-review.md')).toEqual([]); + }); +}); + +describe('#25: the agy skip remedy must match the actual failure', () => { + it('does NOT advise installing the CLI when the cause is a quota wall', () => { + // The CLI is installed. Saying "install it" is a detour, and saying + // "sign in again" does not refill a quota. + const remedy = agyRemedy('agy exited with code 1', 'Error: RESOURCE_EXHAUSTED: quota exceeded for model'); + + // Asserted against the INSTRUCTION, not the word: the message legitimately + // says "the CLI is installed and signed in", which is the point being made. + expect(remedy).not.toMatch(/install the cli|https:\/\/antigravity/i); + expect(remedy).toMatch(/quota|rate limit/i); + }); + + it('recognises a rate limit by HTTP status alone', () => { + expect(agyRemedy('agy exited with code 1', 'request failed: 429 Too Many Requests')) + .toMatch(/quota|rate limit/i); + }); + + it('DOES advise installing when the CLI is genuinely missing', () => { + expect(agyRemedy('agy CLI not found (install: https://antigravity.google/cli/install.sh)')) + .toMatch(/install/i); + }); + + it('advises signing in for an auth failure', () => { + const remedy = agyRemedy('agy exited with code 1', 'Error: 401 Unauthorized — no credentials found'); + + expect(remedy).toMatch(/sign in/i); + expect(remedy).not.toMatch(/install/i); + }); + + it('advises re-running for a timeout', () => { + expect(agyRemedy('agy timed out producing the review')).toMatch(/time budget|re-run/i); + }); + + it('offers NO remedy for an unrecognised failure', () => { + // The point of the whole change. A guessed remedy costs more than no + // remedy, because the reader acts on it. The caller shows agy's raw output + // instead. + expect(agyRemedy('agy exited with code 1', 'Segmentation fault')).toBe(''); + }); + + it('reads the reason as well as the tail, so a cause named either way is caught', () => { + expect(agyRemedy('agy quota exhausted', '')).toMatch(/quota|rate limit/i); + }); +}); + +describe('#44: the opencode arg vector', () => { + it('puts `--` immediately before the prompt so yargs cannot eat it', () => { + // `-f/--file` is declared `[array]` and yargs arrays are greedy. Without + // the separator the prompt becomes another filename and the lane dies with + // `Error: File not found: `. Verified live against the + // installed CLI before this test was written. + const args = buildOpencodeArgs('xai/grok-4.6', ['/tmp/s/a.md', '/tmp/s/b.diff'], 'review this'); + + expect(args[args.length - 1]).toBe('review this'); + expect(args[args.length - 2]).toBe('--'); + }); + + it('emits the separator even with NO attachments', () => { + // Harmless when empty (verified live), and making it conditional is one + // more branch that can be wrong on the path that matters. + const args = buildOpencodeArgs('xai/grok-4.6', [], 'review this'); + + expect(args[args.length - 2]).toBe('--'); + expect(args).not.toContain('-f'); + }); + + it('pairs each attachment with its own -f', () => { + const args = buildOpencodeArgs('m', ['/a', '/b'], 'p'); + const flags = args.filter(a => a === '-f'); + + expect(flags).toHaveLength(2); + expect(args[args.indexOf('-f') + 1]).toBe('/a'); + }); + + it('keeps the model flag ahead of the separator', () => { + const args = buildOpencodeArgs('xai/grok-4.6', ['/a'], 'p'); + + expect(args.indexOf('-m')).toBeLessThan(args.indexOf('--')); + expect(args[args.indexOf('-m') + 1]).toBe('xai/grok-4.6'); + }); +}); + +describe('#25: the not-found remedy must not catch a model-not-found', () => { + it('does NOT say "install the CLI" for a 404 from the provider', () => { + // The lane passes --model-id, so "model not found" in stderr is a live + // possibility. Matching it against the combined haystack produced exactly + // the wrong-remedy class this function exists to remove. + const remedy = agyRemedy('agy exited with code 1', 'Error: model not found: gemini-9-ultra'); + + expect(remedy).not.toMatch(/install the cli/i); + }); + + it('does NOT say "install the CLI" for a 404 Not Found', () => { + expect(agyRemedy('agy exited with code 1', 'request failed: 404 Not Found')) + .not.toMatch(/install the cli/i); + }); + + it('STILL says install when the reason is a genuinely missing binary', () => { + expect(agyRemedy('agy CLI not found (install: https://antigravity.google/cli/install.sh)')) + .toMatch(/install the cli/i); + }); +}); + +describe('#44: extractSandboxPaths', () => { + it('returns nothing when the sandbox was never created', () => { + // No consult artifacts this process, so there is nothing to attach. Must be + // empty, not a guess at a path that might exist. + expect(extractSandboxPaths('**Diff file**: `/var/folders/xx/codev-consult-abc/pr-1.diff`')) + .toEqual([]); + }); + + it('ignores paths outside the sandbox even when the sandbox exists', () => { + const sandbox = _consultSandboxDir(); + const inside = path.join(sandbox, 'pr-7.diff'); + fs.writeFileSync(inside, 'diff --git a/x b/x\n'); + + const text = [ + '**Diff file**: `' + inside + '`', + 'Also see `/etc/passwd` and `' + path.join(os.tmpdir(), 'elsewhere.diff') + '`', + ].join('\n'); + + expect(extractSandboxPaths(text)).toEqual([inside]); + }); + + it('skips a sandbox path that is named but does not exist', () => { + // The prompt can name a file a previous run cleaned up. Attaching a + // non-existent path would make opencode fail on a file nobody needed. + const sandbox = _consultSandboxDir(); + const missing = path.join(sandbox, 'never-written.diff'); + + expect(extractSandboxPaths('**Diff file**: `' + missing + '`')).toEqual([]); + }); + + it('deduplicates a path named more than once', () => { + const sandbox = _consultSandboxDir(); + const p1 = path.join(sandbox, 'pr-9.diff'); + fs.writeFileSync(p1, 'x'); + + expect(extractSandboxPaths('`' + p1 + '` and again `' + p1 + '`')).toEqual([p1]); + }); +}); + +describe('#25 round 2: ordinary review prose must not trigger a remedy', () => { + // `outputTail` is the last 2000 chars of stdout+stderr COMBINED, so on a + // non-zero exit after partial output it holds agy's own review writing. + // Unanchored substrings turned that prose into confident, inapplicable + // instructions — the same defect as #25, with a new trigger. + it.each([ + ['The author of this change added a guard.', 'author contains auth'], + ['see src/auth/session.ts for context', 'a path segment named auth'], + ['reviewed 4293 lines of diff', '4293 contains 429'], + ['tokens: 14015 in / 4012 out', '4012 contains 401'], + ['the login flow is unrelated to this diff', 'discussing login, not failing it'], + ])('offers no remedy for %j (%s)', (tail) => { + expect(agyRemedy('agy exited with code 1', tail)).toBe(''); + }); + + it('still catches a real rate limit stated with its status code', () => { + expect(agyRemedy('agy exited with code 1', 'request failed with status 429')) + .toMatch(/quota|rate limit/i); + }); + + it('still catches a real auth failure', () => { + expect(agyRemedy('agy exited with code 1', 'Error: unauthorized — no credentials found')) + .toMatch(/sign in/i); + }); + + it('still catches a bare quota word', () => { + expect(agyRemedy('agy exited with code 1', 'RESOURCE_EXHAUSTED: quota exceeded')) + .toMatch(/quota|rate limit/i); + }); +}); + +describe('#35: an empty PR diff must throw before anything is written', () => { + it('refuses a 0-byte diff and explains what it could be', () => { + // The one issue in this set with no coverage was the one that produced + // three APPROVE (HIGH) verdicts against nothing. + expect(() => _buildPRQuery('1', { diff: '', changedFiles: [] })) + .toThrow(/0-byte diff/); + }); + + it('names the forge-config cause, since that is what actually happened', () => { + expect(() => _buildPRQuery('1', { diff: '', changedFiles: [] })) + .toThrow(/forge config did not resolve/); + }); + + it('throws BEFORE writing, so an in-process retry gets the message not EEXIST', () => { + // `flag: 'wx'` refuses to overwrite. Checking after the write meant the + // second attempt for the same prId died with EEXIST instead of the + // explanation. + expect(() => _buildPRQuery('77', { diff: '', changedFiles: [] })).toThrow(/0-byte diff/); + expect(() => _buildPRQuery('77', { diff: '', changedFiles: [] })).toThrow(/0-byte diff/); + }); +}); diff --git a/packages/codev/src/commands/consult/index.ts b/packages/codev/src/commands/consult/index.ts index 1dc96434a..1857ce42f 100644 --- a/packages/codev/src/commands/consult/index.ts +++ b/packages/codev/src/commands/consult/index.ts @@ -14,7 +14,8 @@ import { tmpdir, homedir } from 'node:os'; import chalk from 'chalk'; import { query as claudeQuery } from '@anthropic-ai/claude-agent-sdk'; import { Codex } from '@openai/codex-sdk'; -import { readCodevFile, findWorkspaceRoot } from '../../lib/skeleton.js'; +import { readCodevFile, findWorkspaceRoot, protocolsProvidingConsultType } from '../../lib/skeleton.js'; +import { NO_REVIEW_MARKER } from '../porch/verdict.js'; import { resolveDefaultBranch } from '../../lib/default-branch.js'; import { loadConfig, findConfigSource } from '../../lib/config.js'; import { @@ -204,12 +205,30 @@ function resolveProtocolPrompt(workspaceRoot: string, protocol: string | undefin const location = protocol ? `codev/protocols/${protocol}/consult-types/${templateName}` : `codev/consult-types/${templateName}`; + + // Issue #43: naming the missing path is a remedy only if creating that file + // is the fix. For a bare `--type pr` it is not — `pr-review.md` has never + // shipped at `codev/consult-types/`, only under `protocols//`, so the + // old message pointed at a file that has never existed in any release and + // read as "create this". The actual fix is `--protocol`, and nothing said so. + if (!protocol) { + const owners = protocolsProvidingConsultType(templateName, workspaceRoot); + if (owners.length > 0) { + throw new Error( + `No bare template for --type ${type}. This review type is protocol-scoped; ` + + `pass --protocol with one of: ${owners.join(', ')}\n` + + ` e.g. consult -m -t ${type} --protocol ${owners[0]} --issue `, + ); + } + } + throw new Error(`Prompt template not found: ${location}`); } return content; } + /** * Load .env file if it exists */ @@ -944,20 +963,84 @@ export function resolveAgyBin(): string | null { return null; } +/** + * The remedy that actually applies to a given agy failure (#25). + * + * The skip artifact used to end with "install the CLI and run `agy` once to + * sign in" no matter what went wrong. For a quota-exhausted lane that is two + * wrong instructions at once: the CLI is installed, and signing in again does + * not refill a quota. An error that names a remedy which does not apply costs + * more than one that names none, because the reader acts on it. + * + * Anything unrecognised gets no remedy at all rather than a guessed one. + */ +export function agyRemedy(reason: string, outputTail = ''): string { + const haystack = `${reason}\n${outputTail}`.toLowerCase(); + + // Scoped to the REASON, not the combined haystack: agy stderr containing + // "model not found" or "404 not found" would otherwise yield "install the + // CLI" — the exact class of wrong remedy this function exists to remove. The + // lane passes `--model-id`, so a model-not-found is a live possibility. + if (/agy cli not found|enoent/.test(reason.toLowerCase())) { + return 'Install the CLI: https://antigravity.google/cli/install.sh'; + } + // Word-boundaried and context-qualified. Unanchored substrings matched + // ordinary review prose in the tail — `outputTail` is the last 2000 chars of + // stdout+stderr combined, so on a non-zero exit after partial output it holds + // agy's own writing. Measured: "reviewed 4293 lines" hit 429; "tokens: 4012 + // out" hit 401; "The author of this change" hit auth. Each produced a + // confident, inapplicable instruction — #25 in a new shape. + if (/\bquota\b|\brate.?limit(ed|s)?\b|\bresource.?exhausted\b|\btoo many requests\b|\busage limit\b|(?:status|http|code|error)\D{0,4}429\b/.test(haystack)) { + return ( + 'This is a quota/rate limit, not a configuration problem — the CLI is installed and ' + + 'signed in. Wait for the window to reset, or run this lane with a different model ' + + '(`--model-id`), or drop "gemini" from porch.consultation in .codev/config.json for now.' + ); + } + // Bare "login" / "sign in" are deliberately NOT triggers: a reviewer writing + // "the login flow is unrelated to this diff" is discussing login, not failing + // at it. Every reason that genuinely reaches this branch says `authenticat*` + // (preflight emits "authentication required" / "agy unauthenticated") or + // carries a status code. + if (/\bauthenticat(e|ed|ion|ing)\b|\bunauthenticated\b|\bcredentials?\b|\bunauthorized\b|\bpermission denied\b|(?:status|http|code|error)\D{0,4}40[13]\b/.test(haystack)) { + return 'Run `agy` once interactively to sign in.'; + } + if (haystack.includes('timed out')) { + return 'The lane exceeded its time budget. Re-run, or reduce the review scope.'; + } + return ''; +} + /** Non-blocking skip artifact: porch's verdict parser treats COMMENT as non-blocking. */ -function agySkipContent(reason: string): string { - return [ +function agySkipContent(reason: string, outputTail = ''): string { + const remedy = agyRemedy(reason, outputTail); + const lines = [ '---', 'VERDICT: COMMENT', + // #20: the machine-readable half. This artifact is WELL-FORMED — it states a + // real verdict — so nothing downstream could distinguish it from a review + // that concluded COMMENT, and `allApprove` counted it toward unanimity. A + // missing verdict cannot signal this; the lane has to declare it. + NO_REVIEW_MARKER, `SUMMARY: Gemini lane skipped — ${reason}`, 'CONFIDENCE: LOW', '---', '', `The Gemini (Antigravity \`agy\`) reviewer was skipped: ${reason}.`, - 'This is a non-blocking skip; the remaining reviewers still apply. To enable the', - 'Gemini lane, install the CLI (https://antigravity.google/cli/install.sh) and run', - '`agy` once to sign in.', - ].join('\n'); + '', + 'THIS LANE DID NOT REVIEW ANYTHING. It is recorded as a non-blocking skip so the', + 'run can continue on the remaining reviewers — that is not the same as an approval,', + 'and it should not be read as one (see issue #20).', + ]; + if (remedy) { + lines.push('', remedy); + } + // Without a recognised cause, show what agy actually said rather than + // inventing a fix. A raw tail is a lead; a wrong remedy is a detour. + if (!remedy && outputTail.trim()) { + lines.push('', `agy output (tail):`, '```', outputTail.trim(), '```'); + } + return lines.join('\n'); } /** @@ -971,6 +1054,8 @@ function agySkipContent(reason: string): string { * write with mode 0o600 / flag 'wx' to defeat symlink/clobber races. */ let _consultSandboxDir: string | null = null; +/** Test seam: the per-process sandbox dir, created on demand. */ +export function _consultSandboxDirForTest(): string { return consultSandboxDir(); } function consultSandboxDir(): string { if (!_consultSandboxDir) { _consultSandboxDir = fs.mkdtempSync(path.join(tmpdir(), 'codev-consult-')); @@ -978,6 +1063,33 @@ function consultSandboxDir(): string { return _consultSandboxDir; } +/** + * Sandbox-dir file paths a composed query text points at (#44). + * + * `composePRQueryText` embeds the diff path as `**Diff file**: \`\``. + * Rather than re-deriving that path (a second source of truth that drifts from + * the first), this reads back what the prompt actually told the model to open, + * keeping only paths inside this run's sandbox that exist on disk. + * + * Used by the opencode lane, which cannot read the sandbox and must have those + * files ATTACHED instead. + */ +export function extractSandboxPaths(queryText: string): string[] { + const sandbox = _consultSandboxDir; + if (!sandbox) return []; + const found = new Set(); + const re = /`([^`\n]+)`/g; + let m: RegExpExecArray | null; + while ((m = re.exec(queryText)) !== null) { + const candidate = m[1].trim(); + if (!candidate.startsWith(sandbox + path.sep)) continue; + try { + if (fs.statSync(candidate).isFile()) found.add(candidate); + } catch { /* named but absent — nothing to attach */ } + } + return [...found]; +} + function writeConsultOutput(outputPath: string | undefined, content: string): void { if (!outputPath || content.length === 0) return; const outputDir = path.dirname(outputPath); @@ -1249,7 +1361,10 @@ async function runAgyConsultation( : raw.includes(AGY_NONRESPONSE_MARKER) ? 'agy timed out producing the review' : 'agy produced no review output'; - const content = agySkipContent(reason); + // #25: pass the tail so the remedy is chosen from what agy actually + // said. "exited with code 1" alone cannot distinguish a quota wall from + // a missing login, and the old fixed advice assumed the latter. + const content = agySkipContent(reason, outputTail); process.stdout.write(content); writeConsultOutput(outputPath, content); recordAgyMetrics(metricsCtx, startTime, code ?? 1, reason, choice?.id ?? null); @@ -1345,6 +1460,34 @@ export function opencodeReviewHeader(choice: LaneModelChoice): string { return `_Reviewed by the opencode lane — model: \`${choice.id}\`${from}._\n\n`; } +/** + * The argv for `opencode run`. + * + * Extracted so it can be tested. It shipped broken precisely because nothing + * covered it: `opencode run` is yargs-based and `-f/--file` is declared + * `[array]`, so it GREEDILY swallows following positionals. Without a `--` + * separator the prompt is consumed as another filename and the lane dies with + * `Error: File not found: ` on every review that has an + * attachment — which, after #44, is every PR review. + * + * Verified live against the installed CLI: with two attachments the model read + * both, and with none the separator is harmless. So it is emitted + * unconditionally rather than as one more branch to get wrong. + */ +export function buildOpencodeArgs( + modelId: string, + attachments: string[], + promptArg: string, +): string[] { + return [ + ...MODEL_CONFIGS.opencode.args, + '-m', modelId, + ...attachments.flatMap(f => ['-f', f]), + '--', + promptArg, + ]; +} + /** * Run the `opencode` consult lane (`opencode run -m `). * @@ -1405,18 +1548,50 @@ export async function runOpencodeConsultation( const prompt = `${role}\n\n---\n\n${queryText}`; let tempFile: string | null = null; let promptArg = prompt; - // Large inline argv can exceed ARG_MAX (E2BIG) — write it out and point opencode at the file. - // The temp file lands in the consult sandbox dir, the same one the agy lane uses. + // Files handed to opencode via `-f`, which ATTACHES their content to the + // message rather than asking the model to go read a path (#44). + // + // opencode auto-rejects reads outside its working directory. The consult + // sandbox is an `mkdtemp` dir under the OS temp root, granted to the `agy` + // lane through `--add-dir`; opencode has no equivalent flag and got no + // equivalent grant, so every artifact placed there was unreachable to it. + // Observed live: + // + // ! permission requested: external_directory (/var/.../codev-consult-XXXX/*); auto-rejecting + // ✗ Read /var/.../codev-consult-XXXX/pr-42.diff failed + // + // Two things landed in that dir. The PR diff — so an opencode PR review + // silently read the working tree instead of the PR's head→base changes. And, + // above CLI_PROMPT_INLINE_MAX_CHARS, the ENTIRE PROMPT: the lane then held + // nothing but an instruction pointing at an unreadable path, and still + // produced output and a verdict. That is precisely the failure this lane's + // own header says it hard-fails to prevent ("a lane that quietly emits a skip + // is a lane that quietly lowers the bar"). Its guards all catch a process that + // failed; none caught a process that exited 0 with a verdict formed from + // nothing. + // + // Attaching sidesteps the permission system instead of negotiating with it. + const attachments: string[] = []; + if (prompt.length > CLI_PROMPT_INLINE_MAX_CHARS) { tempFile = path.join(consultSandboxDir(), `codev-consult-prompt-${Date.now()}.md`); fs.writeFileSync(tempFile, prompt); + attachments.push(tempFile); promptArg = [ - `Read the full consultation prompt from this file: ${tempFile}`, - 'You have file access. Read files directly from disk to review code.', - ].join('\n\n'); + 'The full consultation prompt is ATTACHED to this message. Read the attachment and', + 'follow it exactly. Do not proceed on the summary below alone.', + '', + 'You also have filesystem access to the repository for surrounding context.', + ].join('\n'); } - const args = [...MODEL_CONFIGS.opencode.args, '-m', choice.id, promptArg]; + // Attach the PR diff too, when this review has one. `queryText` names the + // path; without the attachment the model can see the name and not the bytes. + for (const diffPath of extractSandboxPaths(queryText)) { + if (!attachments.includes(diffPath)) attachments.push(diffPath); + } + + const args = buildOpencodeArgs(choice.id, attachments, promptArg); const cleanup = () => { if (tempFile && fs.existsSync(tempFile)) { @@ -2008,6 +2183,24 @@ function buildPRQuery(prId: string, localDiff?: { diff: string; changedFiles: st const diff = localDiff ? localDiff.diff : fetchPRDiff(prId); const changedFiles = localDiff ? localDiff.changedFiles : prData.changedFiles; + // Emptiness is checked BEFORE the write. `flag: 'wx'` refuses to overwrite, so + // checking after meant an in-process retry for the same prId died with EEXIST + // instead of the message that explains what actually went wrong. + const emptyDiffBytes = Buffer.byteLength(diff, 'utf-8'); + if (emptyDiffBytes === 0) { + throw new Error( + `PR #${prId} produced a 0-byte diff — refusing to run a review on nothing.\n` + + `A reviewer cannot tell an empty diff from a failed fetch, and neither can you ` + + `once three lanes have returned APPROVE.\n` + + `Likely causes:\n` + + ` - the forge config did not resolve, so 'gh' ran against a non-GitHub host ` + + `(check .codev/config.json "forge", and see issue #35)\n` + + ` - the PR genuinely has no changes\n` + + ` - the branch was already merged and the head/base diff is empty\n` + + `Verify with your forge's own diff command before re-running.`, + ); + } + // Private-per-user dir to avoid world-readable /tmp diffs + symlink/clobber // races: consultSandboxDir() is a fresh mkdtempSync dir owned by us (and the // only temp dir granted to the sandboxed agy reviewer); writeFileSync with diff --git a/packages/codev/src/commands/porch/__tests__/agy-porch-progression.test.ts b/packages/codev/src/commands/porch/__tests__/agy-porch-progression.test.ts index c71ec9b92..d12dd7d7d 100644 --- a/packages/codev/src/commands/porch/__tests__/agy-porch-progression.test.ts +++ b/packages/codev/src/commands/porch/__tests__/agy-porch-progression.test.ts @@ -131,13 +131,23 @@ describe('porch progression with a skipped agy/gemini lane (drives next())', () const res = await next(testDir, '0778'); - // Porch advanced: it requested the human `pr` gate ("All reviewers approved!"), - // NOT a rebuttal/re-iteration. The skipped lane did not block progression. + // Porch advanced: it requested the human `pr` gate, NOT a rebuttal or a + // re-iteration. The skipped lane did not block progression — that is the + // deliberate behaviour and it is unchanged. expect(res.status).toBe('gate_pending'); expect(res.gate).toBe('pr'); const subjects = (res.tasks ?? []).map(t => t.subject).join(' | '); expect(subjects).not.toMatch(/rebuttal/i); - expect((res.tasks ?? []).map(t => t.description).join('\n')).toMatch(/All reviewers approved/); + + // What DID change (#20): the description no longer says "All reviewers + // approved!" over a run where gemini never looked at the code. This is the + // sentence a human reads immediately before approving the gate, so it has + // to say what actually happened. + const description = (res.tasks ?? []).map(t => t.description).join('\n'); + expect(description).not.toMatch(/All reviewers approved/); + expect(description).toMatch(/2 of 3 lanes actually reviewed/); + expect(description).toMatch(/Did not review: gemini/); + expect(description).toMatch(/NOT as approval/); }); it('does NOT mask a genuine REQUEST_CHANGES (gemini skipped, codex blocks)', async () => { diff --git a/packages/codev/src/commands/porch/__tests__/agy-skip-progression.test.ts b/packages/codev/src/commands/porch/__tests__/agy-skip-progression.test.ts index 8f801f797..a5bdb56fa 100644 --- a/packages/codev/src/commands/porch/__tests__/agy-skip-progression.test.ts +++ b/packages/codev/src/commands/porch/__tests__/agy-skip-progression.test.ts @@ -46,10 +46,39 @@ describe('agy skip is non-blocking for porch progression', () => { expect(allApprove(reviews)).toBe(false); }); - it('skip artifact is self-describing (names the lane and the remediation)', () => { + it('skip artifact is self-describing (names the lane and a remediation that FITS)', () => { const content = _agySkipContent('authentication required'); expect(content).toMatch(/Gemini lane skipped/); expect(content).toMatch(/non-blocking/); - expect(content).toMatch(/antigravity\.google/); + + // #25: the remediation is now chosen from the cause. For an auth failure + // that is "sign in", NOT the install URL — the CLI is plainly installed if + // it got far enough to reject credentials. The old fixed text advised + // installing regardless, which for a quota wall was two wrong instructions + // at once. + expect(content).toMatch(/sign in/i); + expect(content).not.toMatch(/antigravity\.google/); + }); + + it('skip artifact declares itself unreviewed in machine-readable form (#20)', () => { + // The artifact states a real VERDICT: COMMENT, so nothing downstream could + // tell it from a review that concluded COMMENT — and COMMENT counts toward + // unanimous approval. A missing verdict cannot signal this; the lane has to + // say so itself. + const content = _agySkipContent('authentication required'); + expect(content).toMatch(/VERDICT: COMMENT/); + expect(content).toMatch(/LANE_DID_NOT_REVIEW: true/); + }); + + it('quota exhaustion gets the quota remedy, not the install or login one', () => { + const content = _agySkipContent('agy exited with code 1', 'RESOURCE_EXHAUSTED: quota exceeded'); + expect(content).toMatch(/quota|rate limit/i); + expect(content).not.toMatch(/antigravity\.google/); + }); + + it('an unrecognised failure shows agy output instead of inventing a remedy', () => { + const content = _agySkipContent('agy exited with code 1', 'Segmentation fault'); + expect(content).toMatch(/Segmentation fault/); + expect(content).not.toMatch(/sign in|antigravity\.google/i); }); }); diff --git a/packages/codev/src/commands/porch/__tests__/issue-20-lane-honesty.test.ts b/packages/codev/src/commands/porch/__tests__/issue-20-lane-honesty.test.ts new file mode 100644 index 000000000..8f509d0e8 --- /dev/null +++ b/packages/codev/src/commands/porch/__tests__/issue-20-lane-honesty.test.ts @@ -0,0 +1,200 @@ +/** + * Issue #20 — a review lane that never ran must not read as an approval. + * + * `parseVerdict` returns COMMENT both when a reviewer wrote COMMENT and when it + * wrote no verdict line at all, and `allApprove` counts COMMENT as approval. A + * skipped lane therefore joined a "unanimous" approval, and the gate message + * said "All reviewers approved!" over a run one reviewer never looked at. + * + * The blocking behaviour is deliberately UNCHANGED. A lane that is + * unauthenticated or quota-exhausted must not wedge a project — that was the + * explicit call. What was wrong was the record, not the flow control. These + * tests pin both halves: still non-blocking, no longer called an approval. + */ + +import { describe, it, expect } from 'vitest'; +import { parseVerdict, findVerdict, statedVerdict, allApprove, laneSummary } from '../verdict.js'; +import type { ReviewResult } from '../types.js'; + +/** What the agy lane writes when it skips. */ +const SKIP_ARTIFACT = `--- +VERDICT: COMMENT +SUMMARY: Gemini lane skipped — agy exited with code 1 +CONFIDENCE: LOW +--- + +The Gemini (Antigravity \`agy\`) reviewer was skipped: agy exited with code 1. + +THIS LANE DID NOT REVIEW ANYTHING. +`; + +/** A real review that happens to conclude COMMENT. */ +const REAL_COMMENT = `I read the diff and the surrounding code. Two nits, neither blocking. +The naming in the new helper could be clearer but it is correct as written. + +VERDICT: COMMENT +SUMMARY: Minor nits only. +CONFIDENCE: HIGH +`; + +/** A lane that produced prose and then died before stating a verdict. */ +const NO_VERDICT = `I'll read the diff and the surrounding code. +Reading packages/codev/src/commands/porch/index.ts... +Reading the role docs... +`; + +function review(model: string, output: string): ReviewResult { + return { + model, + verdict: parseVerdict(output), + file: `/tmp/${model}.txt`, + stated: statedVerdict(output), + }; +} + +describe('#20: telling a stated verdict from a defaulted one', () => { + it('a skip artifact and a real COMMENT parse to the SAME verdict', () => { + // This is the whole problem in one assertion. Both are COMMENT, so + // downstream code that sees only the verdict cannot tell them apart. + expect(parseVerdict(SKIP_ARTIFACT)).toBe('COMMENT'); + expect(parseVerdict(REAL_COMMENT)).toBe('COMMENT'); + }); + + it('but statedVerdict separates a verdict-less review from a real one', () => { + expect(statedVerdict(REAL_COMMENT)).toBe(true); + expect(statedVerdict(NO_VERDICT)).toBe(false); + }); + + it('a skip artifact DOES state COMMENT — it is honest about its own verdict', () => { + // Worth pinning: the artifact is not lying. The defect was downstream, + // where COMMENT was read as approval regardless of what produced it. + expect(findVerdict(SKIP_ARTIFACT)).toBe('COMMENT'); + }); +}); + +describe('#20: blocking behaviour is unchanged', () => { + it('a skipped lane still does NOT block the run', () => { + // Deliberate. An unauthenticated or quota-exhausted lane wedging every + // project is worse than the reporting bug this issue is about. + const reviews = [ + review('claude', 'x'.repeat(60) + '\nVERDICT: APPROVE\n'), + { model: 'gemini', verdict: parseVerdict(SKIP_ARTIFACT), file: '/tmp/g.txt', stated: false }, + ]; + expect(allApprove(reviews)).toBe(true); + }); + + it('REQUEST_CHANGES still blocks', () => { + const reviews = [ + review('claude', 'x'.repeat(60) + '\nVERDICT: APPROVE\n'), + review('codex', 'x'.repeat(60) + '\nVERDICT: REQUEST_CHANGES\n'), + ]; + expect(allApprove(reviews)).toBe(false); + }); +}); + +describe('#20: the sentence a human reads before approving a gate', () => { + it('does NOT claim approval when a lane produced no verdict', () => { + const reviews = [ + review('claude', 'x'.repeat(60) + '\nVERDICT: APPROVE\n'), + review('codex', 'x'.repeat(60) + '\nVERDICT: APPROVE\n'), + review('gemini', NO_VERDICT), + ]; + + const s = laneSummary(reviews); + + expect(s.ran).toBe(2); + expect(s.total).toBe(3); + expect(s.silent).toEqual(['gemini']); + expect(s.sentence).toMatch(/2 of 3/); + expect(s.sentence).toMatch(/NOT as approval/); + }); + + it('names WHICH lanes were silent, so the gap is in the record', () => { + // The gap used to be visible only as a missing file nobody was looking for. + const reviews = [ + review('claude', 'x'.repeat(60) + '\nVERDICT: APPROVE\n'), + review('codex', NO_VERDICT), + review('gemini', NO_VERDICT), + ]; + + expect(laneSummary(reviews).sentence).toMatch(/codex, gemini/); + }); + + it('says plainly that all lanes approved when they actually did', () => { + const reviews = [ + review('claude', 'x'.repeat(60) + '\nVERDICT: APPROVE\n'), + review('codex', 'x'.repeat(60) + '\nVERDICT: APPROVE\n'), + ]; + + const s = laneSummary(reviews); + expect(s.silent).toHaveLength(0); + expect(s.sentence).toMatch(/2 of 2 lanes reviewed and approved/); + }); + + it('counts a genuine COMMENT as a lane that RAN', () => { + // A reviewer that read the code and concluded "nits only" reviewed it. + // Only an absent verdict is a silent lane. + const reviews = [ + review('claude', 'x'.repeat(60) + '\nVERDICT: APPROVE\n'), + review('codex', REAL_COMMENT), + ]; + + expect(laneSummary(reviews).silent).toHaveLength(0); + }); + + it('treats an unrecorded `stated` as ran, not as silent', () => { + // Backward compatibility: reviews in existing status.yaml files predate the + // field. Absent provenance must not retroactively accuse them. + const reviews: ReviewResult[] = [ + { model: 'claude', verdict: 'APPROVE', file: '/tmp/a.txt' }, + { model: 'codex', verdict: 'COMMENT', file: '/tmp/b.txt' }, + ]; + + expect(laneSummary(reviews).silent).toHaveLength(0); + }); +}); + +describe('#20 round 2: laneSummary must look at the verdicts, not just who spoke', () => { + it('does NOT say "approved" when a lane returned REQUEST_CHANGES', () => { + // Reachable on the FORCE-ADVANCE path, entered only after REQUEST_CHANGES + // persisted to the iteration ceiling. The gate task there read + // "3 of 3 lanes reviewed and approved" directly under a force-advance + // warning — the exact sentence this issue exists to stop being false. + const reviews = [ + review('claude', 'x'.repeat(60) + '\nVERDICT: APPROVE\n'), + review('codex', 'x'.repeat(60) + '\nVERDICT: REQUEST_CHANGES\n'), + ]; + + const s = laneSummary(reviews); + expect(s.sentence).not.toMatch(/approved/); + expect(s.sentence).toMatch(/Did not approve: codex: REQUEST_CHANGES/); + }); + + it('does NOT say "approved" for a COMMENT-only run', () => { + // Every lane read the code and none of them approved it. Non-blocking is + // not the same as approval, which is the whole thesis. + const reviews = [review('claude', REAL_COMMENT), review('codex', REAL_COMMENT)]; + + expect(laneSummary(reviews).sentence).not.toMatch(/approved/); + }); + + it('reports both problems at once when a lane blocks AND another was silent', () => { + const reviews = [ + review('claude', 'x'.repeat(60) + '\nVERDICT: REQUEST_CHANGES\n'), + review('gemini', NO_VERDICT), + ]; + + const s = laneSummary(reviews); + expect(s.sentence).toMatch(/Did not approve: claude/); + expect(s.sentence).toMatch(/Did not review: gemini/); + }); + + it('still says approved when every lane really did approve', () => { + const reviews = [ + review('claude', 'x'.repeat(60) + '\nVERDICT: APPROVE\n'), + review('codex', 'x'.repeat(60) + '\nVERDICT: APPROVE\n'), + ]; + + expect(laneSummary(reviews).sentence).toMatch(/2 of 2 lanes reviewed and approved/); + }); +}); diff --git a/packages/codev/src/commands/porch/next.ts b/packages/codev/src/commands/porch/next.ts index 2b95576fd..0ca437d5a 100644 --- a/packages/codev/src/commands/porch/next.ts +++ b/packages/codev/src/commands/porch/next.ts @@ -33,7 +33,7 @@ import { allPlanPhasesComplete, } from './plan.js'; import { buildPhasePrompt } from './prompts.js'; -import { parseVerdict, allApprove } from './verdict.js'; +import { parseVerdict, allApprove, laneReviewed, laneSummary } from './verdict.js'; import { loadCheckOverrides, resolveConsultationModels } from './config.js'; import { getResolver, type ArtifactResolver } from './artifacts.js'; import { @@ -80,7 +80,9 @@ function findReviewFiles( if (fs.existsSync(filePath)) { const content = fs.readFileSync(filePath, 'utf-8'); const verdict = parseVerdict(content); - results.push({ model, verdict, file: filePath }); + // #20: record whether the lane stated this verdict or porch defaulted it, + // so a lane that never ran cannot read as part of a unanimous approval. + results.push({ model, verdict, file: filePath, stated: laneReviewed(content) }); } } @@ -852,7 +854,10 @@ async function handleVerifyApproved( tasks: [{ subject: `Request human approval: ${gateName}`, activeForm: `Requesting ${gateName} approval`, - description: `All reviewers approved!\n\nReviewer verdicts:\n${formatVerdicts(reviews)}\n\nSTOP and wait for human approval.`, + // #20: never print "All reviewers approved!" over a run where a lane + // never looked at the code. This is the sentence a human reads right + // before approving a gate. + description: `${laneSummary(reviews).sentence}\n\nReviewer verdicts:\n${formatVerdicts(reviews)}\n\nSTOP and wait for human approval.`, }], }; } @@ -966,6 +971,12 @@ async function handleOncePhase( */ function formatVerdicts(reviews: ReviewResult[]): string { return reviews - .map(r => ` ${r.model}: ${r.verdict}`) + .map(r => + // #20: mark a defaulted verdict as defaulted. `parseVerdict` returns + // COMMENT both for a reviewer that wrote COMMENT and for one that wrote + // no verdict at all, and the two are not the same evidence. + r.stated === false + ? ` ${r.model}: ${r.verdict} (LANE DID NOT REVIEW — skipped or produced no verdict)` + : ` ${r.model}: ${r.verdict}`) .join('\n'); } diff --git a/packages/codev/src/commands/porch/types.ts b/packages/codev/src/commands/porch/types.ts index f984c2c6d..6c64c3ddb 100644 --- a/packages/codev/src/commands/porch/types.ts +++ b/packages/codev/src/commands/porch/types.ts @@ -165,6 +165,16 @@ export interface ReviewResult { model: string; verdict: Verdict; file: string; // Path to review output file + /** + * Whether the reviewer stated this verdict itself (#20). + * + * `parseVerdict` returns COMMENT both when a reviewer wrote COMMENT and when + * it wrote no verdict line at all, and `allApprove` counts COMMENT as an + * approval — so a lane that never ran silently joins a unanimous approval. + * Optional for backward compatibility with existing status.yaml records; + * `undefined` means "not recorded", which is not the same as `false`. + */ + stated?: boolean; } /** diff --git a/packages/codev/src/commands/porch/verdict.ts b/packages/codev/src/commands/porch/verdict.ts index 54828371a..55d33f17b 100644 --- a/packages/codev/src/commands/porch/verdict.ts +++ b/packages/codev/src/commands/porch/verdict.ts @@ -64,9 +64,107 @@ export function parseVerdict(output: string): Verdict { * Returns true only if ALL reviewers explicitly APPROVE. * COMMENT counts as approve (non-blocking feedback). * CONSULT_ERROR and REQUEST_CHANGES block approval. + * + * Issue #20: a skipped lane stays NON-BLOCKING here on purpose. A lane that is + * unauthenticated, quota-exhausted, or absent must not wedge a project — that + * was the explicit design call. What was wrong was calling it an approval in + * the record. Use `laneSummary` for anything a human or a review file will + * read, so "3 reviewers approved" is never printed over a run where one of them + * never looked at the code. */ export function allApprove(reviews: ReviewResult[]): boolean { if (reviews.length === 0) return true; // No verification = auto-approve return reviews.every(r => r.verdict === 'APPROVE' || r.verdict === 'COMMENT'); } +/** + * Marker a lane writes to declare that it produced no review (#20). + * + * A skipped lane's artifact is WELL-FORMED — `agySkipContent` writes a real + * `VERDICT: COMMENT` line — so the absence of a verdict cannot detect it. The + * lane has to say so itself. This is a contract between codev's own skip + * writers and this parser, not an inference from prose. + */ +export const NO_REVIEW_MARKER = 'LANE_DID_NOT_REVIEW: true'; + +/** + * Did this review actually state a verdict, or did porch default one for it? + * + * `parseVerdict` collapses "the reviewer wrote COMMENT" and "the reviewer wrote + * no verdict line" into the same COMMENT, and `allApprove` counts COMMENT as an + * approval. That is how a lane that never ran becomes part of a unanimous + * approval. `findVerdict` is the distinction; this carries it to callers. + */ +export function statedVerdict(output: string): boolean { + return findVerdict(output) !== null; +} + +/** + * Did this lane actually review the code? + * + * Two ways it did not, and they look nothing alike: + * - it stated no verdict at all (crashed, truncated, produced only prose) + * - it stated a verdict on an artifact that declares itself a skip + * + * The second is the one that mattered in practice: a skipped agy lane writes a + * perfectly well-formed `VERDICT: COMMENT`, which `allApprove` counts as an + * approval. Checking only for a missing verdict misses it entirely. + */ +export function laneReviewed(output: string): boolean { + if (output.includes(NO_REVIEW_MARKER)) return false; + return statedVerdict(output); +} + +/** How many lanes actually produced a verdict, out of how many were asked. */ +export interface LaneSummary { + /** Lanes that stated a verdict of their own. */ + ran: number; + /** Lanes asked for. */ + total: number; + /** Lanes that produced no verdict of their own (skipped, crashed, empty). */ + silent: string[]; + /** One line stating exactly what happened, safe to print or commit. */ + sentence: string; +} + +/** + * Describe the lane outcome honestly (#20). + * + * "All reviewers approved!" is false whenever a lane was skipped, and it is the + * sentence a human reads before merging. This produces the sentence that is + * actually true, naming the silent lanes so the gap is visible in the record + * rather than inferred from a missing file. + */ +export function laneSummary(reviews: ReviewResult[]): LaneSummary { + const silent = reviews.filter(r => r.stated === false).map(r => r.model); + const ran = reviews.length - silent.length; + + // Only say "approved" when the lanes that ran actually said APPROVE. The + // first version counted `stated` and never looked at a verdict, so it + // asserted approval on the FORCE-ADVANCE path — reached only after + // REQUEST_CHANGES persisted to the iteration ceiling — and on a run where + // every lane returned COMMENT. This sentence is the one a human reads before + // approving a gate; it must not be the last thing in the system still saying + // "approved" about a run nobody approved. + const notApproved = reviews + .filter(r => r.stated !== false && r.verdict !== 'APPROVE') + .map(r => `${r.model}: ${r.verdict}`); + + const parts: string[] = []; + if (silent.length === 0 && notApproved.length === 0) { + parts.push(`${ran} of ${reviews.length} lanes reviewed and approved.`); + } else { + parts.push(`${ran} of ${reviews.length} lanes actually reviewed.`); + if (notApproved.length > 0) { + parts.push(`Did not approve: ${notApproved.join(', ')}.`); + } + if (silent.length > 0) { + parts.push( + `Did not review: ${silent.join(', ')} — recorded as non-blocking, NOT as approval.`, + ); + } + } + + return { ran, total: reviews.length, silent, sentence: parts.join(' ') }; +} + diff --git a/packages/codev/src/lib/skeleton.ts b/packages/codev/src/lib/skeleton.ts index 77816d47f..bc97e0634 100644 --- a/packages/codev/src/lib/skeleton.ts +++ b/packages/codev/src/lib/skeleton.ts @@ -224,6 +224,40 @@ function protocolDirs(workspaceRoot?: string): string[] { return dirs; } +/** + * Which protocols ship a given consult-type template (e.g. `pr-review.md`). + * + * Issue #43: five of the six review types exist only under + * `protocols//consult-types/`, never at the bare `codev/consult-types/`. + * A bare `--type pr` therefore fails against a path that has never shipped, and + * the fix is `--protocol`. This turns that dead end into an actionable list. + * + * Union across all four tiers, matching `listProtocolNames`. Returns an empty + * list when nothing can be read — the caller falls back to the plain + * not-found error rather than inventing a second wrong remedy. + */ +export function protocolsProvidingConsultType( + templateName: string, + workspaceRoot?: string, +): string[] { + const found = new Set(); + for (const dir of protocolDirs(workspaceRoot)) { + if (!fs.existsSync(dir)) continue; + let entries: fs.Dirent[]; + try { + entries = fs.readdirSync(dir, { withFileTypes: true }).filter(d => d.isDirectory()); + } catch { + continue; + } + for (const entry of entries) { + if (fs.existsSync(path.join(dir, entry.name, 'consult-types', templateName))) { + found.add(entry.name); + } + } + } + return [...found].sort(); +} + function readProtocolJson(filePath: string): Record | null { try { return JSON.parse(fs.readFileSync(filePath, 'utf-8')) as Record;