From 33ac8e62f6fbddec198fa55f64729470f9eb731f Mon Sep 17 00:00:00 2001 From: xiangyu meng <1356464784@qq.com> Date: Fri, 21 Aug 2026 17:59:23 +0800 Subject: [PATCH] fix(extraction): give parse workers a native stack the kernel walkers can't overflow MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A C/C++ file with thousands of nested braces segfaulted the whole `codegraph index` process. The kernel walkers recurse once per parse-tree level (~450 bytes of native stack per level for c/cpp), and parse workers ran on Node's default 4MB worker stack, so deep nesting overflowed it. The overflow is a SIGSEGV inside the addon rather than a JS exception, so neither parse-worker's catch nor the kernel's per-file `defer:` fallback could see it — and since worker threads share the process, it took the CLI down with no diagnostic and no index. Repro'd on llvm/llvm-project, where `clang/test/Parser/parser_overflow.c` nests 16,384 braces: SIGSEGV at 4MB and 6MB of worker stack, parses cleanly at 8MB. The pool now pins `resourceLimits.stackSizeMb` to 16MB, which is reserved lazily and so costs nothing on files of ordinary depth. This raises the cliff rather than removing it. The complete fix is a depth cap in the kernel walkers that raises `defer:`, landing such a file on the wasm extractor — which parses this input correctly today, and is why `CODEGRAPH_KERNEL=0` was a working workaround. The regression test runs the parse in a child process on purpose: if this regresses, the parse segfaults, and a segfault in a worker thread would take the test runner down instead of reporting a failure. Fixes #1581. --- CHANGELOG.md | 1 + __tests__/parse-worker-stack.test.ts | 109 +++++++++++++++++++++++++++ src/extraction/parse-pool.ts | 20 ++++- 3 files changed, 129 insertions(+), 1 deletion(-) create mode 100644 __tests__/parse-worker-stack.test.ts diff --git a/CHANGELOG.md b/CHANGELOG.md index f44be8edd..ccdcc9b65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ### Fixes +- Indexing a repository that contains a deeply nested C or C++ file no longer crashes partway through with no error message. A file with thousands of nested braces — parser stress tests shipped by compiler projects like clang are the usual source — could end `codegraph init` with a segmentation fault around the time the progress bar reached the file, leaving no index and no indication of which file was responsible. Such a file now indexes normally. `CODEGRAPH_KERNEL=0` was the workaround and is no longer needed. - Naming a file by its path in a `codegraph_explore` query now works reliably: the path is resolved against the index and that file is guaranteed a place at the top of the answer. Previously the path was broken into fragments — bracketed route segments like SvelteKit's `[id]` made this worst — and pieces like `page` or `runs` matched every sibling file, so the file you actually named could be crowded out of the answer entirely. A path that doesn't match any indexed file is now called out instead of silently ignored. - Plainly-worded `codegraph_explore` questions now find camelCase code: a query like "auto-scroll to bottom" can reach a function named `scrollFeedToBottom`, because query words are matched against the words inside identifiers, not just whole names. - Variables and constants now count when `codegraph_explore` picks its starting symbols, so state held in plain variables — `$state`-style variables in Svelte, for example — no longer gets overlooked. diff --git a/__tests__/parse-worker-stack.test.ts b/__tests__/parse-worker-stack.test.ts new file mode 100644 index 000000000..5129796c9 --- /dev/null +++ b/__tests__/parse-worker-stack.test.ts @@ -0,0 +1,109 @@ +/** + * Parse-worker native stack size (issue #1581). + * + * The kernel walkers recurse once per parse-tree level, so a deeply nested + * source file consumes native stack proportional to its nesting depth. Node's + * default worker stack is 4MB, and overflowing it inside the addon is a + * SIGSEGV — not a catchable JS error — which kills the whole `codegraph index` + * process, since worker threads share it. clang's + * `test/Parser/parser_overflow.c` (16,384 nested braces) triggered exactly + * this on a real repo. The pool therefore pins an explicit + * `resourceLimits.stackSizeMb`. + * + * Both arms run the parse in a CHILD PROCESS on purpose: if the fix regresses, + * the parse segfaults, and a segfault in a worker thread would take this test + * runner down with it instead of reporting a failure. A dead child is + * observable; a dead runner is not. + */ + +import { describe, it, expect } from 'vitest'; +import { spawnSync } from 'child_process'; +import * as fs from 'fs'; +import * as path from 'path'; +import { PARSE_WORKER_STACK_MB } from '../src/extraction/parse-pool'; + +const KERNEL_PATH = path.join( + __dirname, + '..', + 'codegraph-kernel', + 'prebuilds', + `${process.platform}-${process.arch}`, + 'codegraph-kernel.node' +); +const kernelBuilt = fs.existsSync(KERNEL_PATH); + +/** Nesting depth from clang's parser_overflow.c — the input seen in the wild. */ +const NESTING_DEPTH = 16_384; + +/** + * Run `body` inside a worker thread with `stackSizeMb`, in a child process. + * Returns the child's exit status: 0 when the worker completed, non-zero (or a + * signal) when it died — which is what a native stack overflow looks like. + */ +function runInWorker(stackSizeMb: number, body: string) { + const script = ` + const { Worker, isMainThread, workerData } = require('worker_threads'); + if (isMainThread) { + const w = new Worker(__filename, { + workerData: { kernelPath: ${JSON.stringify(KERNEL_PATH)}, depth: ${NESTING_DEPTH} }, + resourceLimits: { stackSizeMb: ${stackSizeMb} }, + }); + w.on('exit', (code) => process.exit(code)); + w.on('error', (e) => { console.error(e.message); process.exit(1); }); + } else { + ${body} + } + `; + const file = path.join(__dirname, `.stack-probe-${stackSizeMb}-${process.pid}.cjs`); + fs.writeFileSync(file, script); + try { + return spawnSync(process.execPath, [file], { encoding: 'utf-8' }); + } finally { + fs.rmSync(file, { force: true }); + } +} + +describe('parse worker stack size', () => { + it('is large enough for the deepest nesting seen in real repos', () => { + // ~450 bytes of native stack per tree level for c/cpp, measured on the + // ccpp walker; the observed cliff was between 8k and 10k levels at 4MB. + const bytesPerLevel = 450; + const headroom = 2; + expect(PARSE_WORKER_STACK_MB * 1024 * 1024).toBeGreaterThan( + NESTING_DEPTH * bytesPerLevel * headroom + ); + }); + + it('reaches the worker thread without shrinking its heap', () => { + const r = runInWorker( + PARSE_WORKER_STACK_MB, + ` + const { resourceLimits } = require('worker_threads'); + console.log(JSON.stringify(resourceLimits)); + ` + ); + expect(r.status).toBe(0); + const limits = JSON.parse(r.stdout.trim()); + expect(limits.stackSizeMb).toBe(PARSE_WORKER_STACK_MB); + // A partial resourceLimits must not silently cap the V8 heap — parse + // workers legitimately reach ~1.4GB RSS on a large index. + expect(limits.maxOldGenerationSizeMb).toBeGreaterThan(512); + }); + + it.skipIf(!kernelBuilt)('parses a deeply nested C file without dying', () => { + const r = runInWorker( + PARSE_WORKER_STACK_MB, + ` + const { workerData } = require('worker_threads'); + const kernel = require(workerData.kernelPath); + const d = workerData.depth; + const src = 'void foo(void) {\\n' + '{'.repeat(d) + '}'.repeat(d) + '\\n}\\n'; + const buffers = kernel.extractFile('deep.c', src, 'c'); + console.log('nodes=' + buffers.meta.readUInt32LE(4)); + ` + ); + expect(r.signal, 'worker died on a native stack overflow').toBeNull(); + expect(r.status).toBe(0); + expect(r.stdout).toContain('nodes='); + }); +}); diff --git a/src/extraction/parse-pool.ts b/src/extraction/parse-pool.ts index c0cacd216..ad90349ce 100644 --- a/src/extraction/parse-pool.ts +++ b/src/extraction/parse-pool.ts @@ -89,6 +89,23 @@ const MAX_CONCURRENT_SPAWN = 2; * orchestrator's retry pass and shouldn't trip this on a merely-crashy repo. */ const CRASH_BUDGET = 100; +/** + * Native stack for each parse worker. Node's default worker stack is 4MB (the + * main thread gets 8MB), and the kernel walkers recurse once per tree level — + * ~450 bytes of native stack per level for c/cpp — so a deeply nested source + * file overflows it. That overflow is a SIGSEGV inside the addon: it never + * becomes a JS exception, so neither the worker's catch nor the kernel's + * per-file `defer:` fallback can see it, and because worker threads share the + * process it takes the whole `codegraph index` down with no diagnostic. + * Observed on clang's `test/Parser/parser_overflow.c` (16,384 nested braces): + * segfaults at 4MB and 6MB, parses at 8MB. 16MB leaves headroom without + * committing memory — a thread stack is reserved lazily, page by page. + * + * This raises the cliff rather than removing it; a depth cap in the kernel + * walkers that raises `defer:` (so the file lands on the wasm extractor, which + * handles this input today) is the complete fix. + */ +export const PARSE_WORKER_STACK_MB = 16; /** * Resolve the pool size from the `CODEGRAPH_PARSE_WORKERS` override and the @@ -214,7 +231,8 @@ export class ParseWorkerPool { this.createWorker = opts.createWorker; } else if (opts.workerScriptPath) { const scriptPath = opts.workerScriptPath; - this.createWorker = () => new Worker(scriptPath); + this.createWorker = () => + new Worker(scriptPath, { resourceLimits: { stackSizeMb: PARSE_WORKER_STACK_MB } }); } else { throw new Error('ParseWorkerPool requires workerScriptPath or createWorker'); }