diff --git a/CHANGELOG.md b/CHANGELOG.md index f44be8edd..ccdcc9b65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Fixes +- Indexing a repository that contains a deeply nested C or C++ file no longer crashes partway through with no error message. A file with thousands of nested braces — parser stress tests shipped by compiler projects like clang are the usual source — could end `codegraph init` with a segmentation fault around the time the progress bar reached the file, leaving no index and no indication of which file was responsible. Such a file now indexes normally. `CODEGRAPH_KERNEL=0` was the workaround and is no longer needed. - Naming a file by its path in a `codegraph_explore` query now works reliably: the path is resolved against the index and that file is guaranteed a place at the top of the answer. Previously the path was broken into fragments — bracketed route segments like SvelteKit's `[id]` made this worst — and pieces like `page` or `runs` matched every sibling file, so the file you actually named could be crowded out of the answer entirely. A path that doesn't match any indexed file is now called out instead of silently ignored. - Plainly-worded `codegraph_explore` questions now find camelCase code: a query like "auto-scroll to bottom" can reach a function named `scrollFeedToBottom`, because query words are matched against the words inside identifiers, not just whole names. - Variables and constants now count when `codegraph_explore` picks its starting symbols, so state held in plain variables — `$state`-style variables in Svelte, for example — no longer gets overlooked. diff --git a/__tests__/parse-worker-stack.test.ts b/__tests__/parse-worker-stack.test.ts new file mode 100644 index 000000000..5129796c9 --- /dev/null +++ b/__tests__/parse-worker-stack.test.ts @@ -0,0 +1,109 @@ +/** + * Parse-worker native stack size (issue #1581). + * + * The kernel walkers recurse once per parse-tree level, so a deeply nested + * source file consumes native stack proportional to its nesting depth. Node's + * default worker stack is 4MB, and overflowing it inside the addon is a + * SIGSEGV — not a catchable JS error — which kills the whole `codegraph index` + * process, since worker threads share it. clang's + * `test/Parser/parser_overflow.c` (16,384 nested braces) triggered exactly + * this on a real repo. The pool therefore pins an explicit + * `resourceLimits.stackSizeMb`. + * + * Both arms run the parse in a CHILD PROCESS on purpose: if the fix regresses, + * the parse segfaults, and a segfault in a worker thread would take this test + * runner down with it instead of reporting a failure. A dead child is + * observable; a dead runner is not. + */ + +import { describe, it, expect } from 'vitest'; +import { spawnSync } from 'child_process'; +import * as fs from 'fs'; +import * as path from 'path'; +import { PARSE_WORKER_STACK_MB } from '../src/extraction/parse-pool'; + +const KERNEL_PATH = path.join( + __dirname, + '..', + 'codegraph-kernel', + 'prebuilds', + `${process.platform}-${process.arch}`, + 'codegraph-kernel.node' +); +const kernelBuilt = fs.existsSync(KERNEL_PATH); + +/** Nesting depth from clang's parser_overflow.c — the input seen in the wild. */ +const NESTING_DEPTH = 16_384; + +/** + * Run `body` inside a worker thread with `stackSizeMb`, in a child process. + * Returns the child's exit status: 0 when the worker completed, non-zero (or a + * signal) when it died — which is what a native stack overflow looks like. + */ +function runInWorker(stackSizeMb: number, body: string) { + const script = ` + const { Worker, isMainThread, workerData } = require('worker_threads'); + if (isMainThread) { + const w = new Worker(__filename, { + workerData: { kernelPath: ${JSON.stringify(KERNEL_PATH)}, depth: ${NESTING_DEPTH} }, + resourceLimits: { stackSizeMb: ${stackSizeMb} }, + }); + w.on('exit', (code) => process.exit(code)); + w.on('error', (e) => { console.error(e.message); process.exit(1); }); + } else { + ${body} + } + `; + const file = path.join(__dirname, `.stack-probe-${stackSizeMb}-${process.pid}.cjs`); + fs.writeFileSync(file, script); + try { + return spawnSync(process.execPath, [file], { encoding: 'utf-8' }); + } finally { + fs.rmSync(file, { force: true }); + } +} + +describe('parse worker stack size', () => { + it('is large enough for the deepest nesting seen in real repos', () => { + // ~450 bytes of native stack per tree level for c/cpp, measured on the + // ccpp walker; the observed cliff was between 8k and 10k levels at 4MB. + const bytesPerLevel = 450; + const headroom = 2; + expect(PARSE_WORKER_STACK_MB * 1024 * 1024).toBeGreaterThan( + NESTING_DEPTH * bytesPerLevel * headroom + ); + }); + + it('reaches the worker thread without shrinking its heap', () => { + const r = runInWorker( + PARSE_WORKER_STACK_MB, + ` + const { resourceLimits } = require('worker_threads'); + console.log(JSON.stringify(resourceLimits)); + ` + ); + expect(r.status).toBe(0); + const limits = JSON.parse(r.stdout.trim()); + expect(limits.stackSizeMb).toBe(PARSE_WORKER_STACK_MB); + // A partial resourceLimits must not silently cap the V8 heap — parse + // workers legitimately reach ~1.4GB RSS on a large index. + expect(limits.maxOldGenerationSizeMb).toBeGreaterThan(512); + }); + + it.skipIf(!kernelBuilt)('parses a deeply nested C file without dying', () => { + const r = runInWorker( + PARSE_WORKER_STACK_MB, + ` + const { workerData } = require('worker_threads'); + const kernel = require(workerData.kernelPath); + const d = workerData.depth; + const src = 'void foo(void) {\\n' + '{'.repeat(d) + '}'.repeat(d) + '\\n}\\n'; + const buffers = kernel.extractFile('deep.c', src, 'c'); + console.log('nodes=' + buffers.meta.readUInt32LE(4)); + ` + ); + expect(r.signal, 'worker died on a native stack overflow').toBeNull(); + expect(r.status).toBe(0); + expect(r.stdout).toContain('nodes='); + }); +}); diff --git a/src/extraction/parse-pool.ts b/src/extraction/parse-pool.ts index c0cacd216..ad90349ce 100644 --- a/src/extraction/parse-pool.ts +++ b/src/extraction/parse-pool.ts @@ -89,6 +89,23 @@ const MAX_CONCURRENT_SPAWN = 2; * orchestrator's retry pass and shouldn't trip this on a merely-crashy repo. */ const CRASH_BUDGET = 100; +/** + * Native stack for each parse worker. Node's default worker stack is 4MB (the + * main thread gets 8MB), and the kernel walkers recurse once per tree level — + * ~450 bytes of native stack per level for c/cpp — so a deeply nested source + * file overflows it. That overflow is a SIGSEGV inside the addon: it never + * becomes a JS exception, so neither the worker's catch nor the kernel's + * per-file `defer:` fallback can see it, and because worker threads share the + * process it takes the whole `codegraph index` down with no diagnostic. + * Observed on clang's `test/Parser/parser_overflow.c` (16,384 nested braces): + * segfaults at 4MB and 6MB, parses at 8MB. 16MB leaves headroom without + * committing memory — a thread stack is reserved lazily, page by page. + * + * This raises the cliff rather than removing it; a depth cap in the kernel + * walkers that raises `defer:` (so the file lands on the wasm extractor, which + * handles this input today) is the complete fix. + */ +export const PARSE_WORKER_STACK_MB = 16; /** * Resolve the pool size from the `CODEGRAPH_PARSE_WORKERS` override and the @@ -214,7 +231,8 @@ export class ParseWorkerPool { this.createWorker = opts.createWorker; } else if (opts.workerScriptPath) { const scriptPath = opts.workerScriptPath; - this.createWorker = () => new Worker(scriptPath); + this.createWorker = () => + new Worker(scriptPath, { resourceLimits: { stackSizeMb: PARSE_WORKER_STACK_MB } }); } else { throw new Error('ParseWorkerPool requires workerScriptPath or createWorker'); }