fix(extraction): give parse workers a native stack the kernel walkers can't overflow

CHANGELOG.md

@@ -21,6 +21,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ### Fixes
 
+- Indexing a repository that contains a deeply nested C or C++ file no longer crashes partway through with no error message. A file with thousands of nested braces — parser stress tests shipped by compiler projects like clang are the usual source — could end `codegraph init` with a segmentation fault around the time the progress bar reached the file, leaving no index and no indication of which file was responsible. Such a file now indexes normally. `CODEGRAPH_KERNEL=0` was the workaround and is no longer needed.
 - Naming a file by its path in a `codegraph_explore` query now works reliably: the path is resolved against the index and that file is guaranteed a place at the top of the answer. Previously the path was broken into fragments — bracketed route segments like SvelteKit's `[id]` made this worst — and pieces like `page` or `runs` matched every sibling file, so the file you actually named could be crowded out of the answer entirely. A path that doesn't match any indexed file is now called out instead of silently ignored.
 - Plainly-worded `codegraph_explore` questions now find camelCase code: a query like "auto-scroll to bottom" can reach a function named `scrollFeedToBottom`, because query words are matched against the words inside identifiers, not just whole names.
 - Variables and constants now count when `codegraph_explore` picks its starting symbols, so state held in plain variables — `$state`-style variables in Svelte, for example — no longer gets overlooked.

__tests__/parse-worker-stack.test.ts

@@ -0,0 +1,109 @@
+/**
+ * Parse-worker native stack size (issue #1581).
+ *
+ * The kernel walkers recurse once per parse-tree level, so a deeply nested
+ * source file consumes native stack proportional to its nesting depth. Node's
+ * default worker stack is 4MB, and overflowing it inside the addon is a
+ * SIGSEGV — not a catchable JS error — which kills the whole `codegraph index`
+ * process, since worker threads share it. clang's
+ * `test/Parser/parser_overflow.c` (16,384 nested braces) triggered exactly
+ * this on a real repo. The pool therefore pins an explicit
+ * `resourceLimits.stackSizeMb`.
+ *
+ * Both arms run the parse in a CHILD PROCESS on purpose: if the fix regresses,
+ * the parse segfaults, and a segfault in a worker thread would take this test
+ * runner down with it instead of reporting a failure. A dead child is
+ * observable; a dead runner is not.
+ */
+
+import { describe, it, expect } from 'vitest';
+import { spawnSync } from 'child_process';
+import * as fs from 'fs';
+import * as path from 'path';
+import { PARSE_WORKER_STACK_MB } from '../src/extraction/parse-pool';
+
+const KERNEL_PATH = path.join(
+  __dirname,
+  '..',
+  'codegraph-kernel',
+  'prebuilds',
+  `${process.platform}-${process.arch}`,
+  'codegraph-kernel.node'
+);
+const kernelBuilt = fs.existsSync(KERNEL_PATH);
+
+/** Nesting depth from clang's parser_overflow.c — the input seen in the wild. */
+const NESTING_DEPTH = 16_384;
+
+/**
+ * Run `body` inside a worker thread with `stackSizeMb`, in a child process.
+ * Returns the child's exit status: 0 when the worker completed, non-zero (or a
+ * signal) when it died — which is what a native stack overflow looks like.
+ */
+function runInWorker(stackSizeMb: number, body: string) {
+  const script = `
+    const { Worker, isMainThread, workerData } = require('worker_threads');
+    if (isMainThread) {
+      const w = new Worker(__filename, {
+        workerData: { kernelPath: ${JSON.stringify(KERNEL_PATH)}, depth: ${NESTING_DEPTH} },
+        resourceLimits: { stackSizeMb: ${stackSizeMb} },
+      });
+      w.on('exit', (code) => process.exit(code));
+      w.on('error', (e) => { console.error(e.message); process.exit(1); });
+    } else {
+      ${body}
+    }
+  `;
+  const file = path.join(__dirname, `.stack-probe-${stackSizeMb}-${process.pid}.cjs`);
+  fs.writeFileSync(file, script);
+  try {
+    return spawnSync(process.execPath, [file], { encoding: 'utf-8' });
+  } finally {
+    fs.rmSync(file, { force: true });
+  }
+}
+
+describe('parse worker stack size', () => {
+  it('is large enough for the deepest nesting seen in real repos', () => {
+    // ~450 bytes of native stack per tree level for c/cpp, measured on the
+    // ccpp walker; the observed cliff was between 8k and 10k levels at 4MB.
+    const bytesPerLevel = 450;
+    const headroom = 2;
+    expect(PARSE_WORKER_STACK_MB * 1024 * 1024).toBeGreaterThan(
+      NESTING_DEPTH * bytesPerLevel * headroom
+    );
+  });
+
+  it('reaches the worker thread without shrinking its heap', () => {
+    const r = runInWorker(
+      PARSE_WORKER_STACK_MB,
+      `
+      const { resourceLimits } = require('worker_threads');
+      console.log(JSON.stringify(resourceLimits));
+      `
+    );
+    expect(r.status).toBe(0);
+    const limits = JSON.parse(r.stdout.trim());
+    expect(limits.stackSizeMb).toBe(PARSE_WORKER_STACK_MB);
+    // A partial resourceLimits must not silently cap the V8 heap — parse
+    // workers legitimately reach ~1.4GB RSS on a large index.
+    expect(limits.maxOldGenerationSizeMb).toBeGreaterThan(512);
+  });
+
+  it.skipIf(!kernelBuilt)('parses a deeply nested C file without dying', () => {
+    const r = runInWorker(
+      PARSE_WORKER_STACK_MB,
+      `
+      const { workerData } = require('worker_threads');
+      const kernel = require(workerData.kernelPath);
+      const d = workerData.depth;
+      const src = 'void foo(void) {\\n' + '{'.repeat(d) + '}'.repeat(d) + '\\n}\\n';
+      const buffers = kernel.extractFile('deep.c', src, 'c');
+      console.log('nodes=' + buffers.meta.readUInt32LE(4));
+      `
+    );
+    expect(r.signal, 'worker died on a native stack overflow').toBeNull();
+    expect(r.status).toBe(0);
+    expect(r.stdout).toContain('nodes=');
+  });
+});

src/extraction/parse-pool.ts

@@ -89,6 +89,23 @@ const MAX_CONCURRENT_SPAWN = 2;
  * orchestrator's retry pass and shouldn't trip this on a merely-crashy repo.
  */
 const CRASH_BUDGET = 100;
+/**
+ * Native stack for each parse worker. Node's default worker stack is 4MB (the
+ * main thread gets 8MB), and the kernel walkers recurse once per tree level —
+ * ~450 bytes of native stack per level for c/cpp — so a deeply nested source
+ * file overflows it. That overflow is a SIGSEGV inside the addon: it never
+ * becomes a JS exception, so neither the worker's catch nor the kernel's
+ * per-file `defer:` fallback can see it, and because worker threads share the
+ * process it takes the whole `codegraph index` down with no diagnostic.
+ * Observed on clang's `test/Parser/parser_overflow.c` (16,384 nested braces):
+ * segfaults at 4MB and 6MB, parses at 8MB. 16MB leaves headroom without
+ * committing memory — a thread stack is reserved lazily, page by page.
+ *
+ * This raises the cliff rather than removing it; a depth cap in the kernel
+ * walkers that raises `defer:` (so the file lands on the wasm extractor, which
+ * handles this input today) is the complete fix.
+ */
+export const PARSE_WORKER_STACK_MB = 16;
 
 /**
  * Resolve the pool size from the `CODEGRAPH_PARSE_WORKERS` override and the
@@ -214,7 +231,8 @@ export class ParseWorkerPool {
       this.createWorker = opts.createWorker;
     } else if (opts.workerScriptPath) {
       const scriptPath = opts.workerScriptPath;
-      this.createWorker = () => new Worker(scriptPath);
+      this.createWorker = () =>
+        new Worker(scriptPath, { resourceLimits: { stackSizeMb: PARSE_WORKER_STACK_MB } });
     } else {
       throw new Error('ParseWorkerPool requires workerScriptPath or createWorker');
     }