diff --git a/CHANGELOG.md b/CHANGELOG.md index 3b2d07fb3..56f70c171 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -71,6 +71,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). - Files skipped because they are too large or repeatedly fail to parse are now recorded with the reason, so unchanged rejected files are no longer rediscovered and retried on every status check and sync — and a later successful parse of such a file replaces the record with its real symbols. Thanks @netbrah for the exceptional failure analysis behind this batch, and @danusha2345 for the fixes. (#1557) - C/C++ function-pointer analysis now bounds its compiled-pattern caches, so very large repositories can no longer exhaust the JavaScript engine's regular-expression code space during indexing. (#1559) - JSX rendering analysis now runs only on JavaScript-family files, so JSX-looking strings in C/C++ (or any other language) no longer create impossible call edges — in pure-C projects and in mixed-language monorepos alike. (#1560) +- Indexing no longer crashes the whole process — a segmentation fault with no message and no partial index — on a C/C++ (or any other) file with extremely deep nesting, such as the parser stress-test fixtures in the clang and gcc test suites or a fuzzer corpus. Such a file is now handed to the fallback parser and recorded with a parse warning while the rest of the repository indexes normally. Thanks @apollo600 for the exact diagnosis. (#1581) ## [1.5.0] - 2026-07-21 diff --git a/__tests__/kernel-deep-nesting.test.ts b/__tests__/kernel-deep-nesting.test.ts new file mode 100644 index 000000000..7529ff3b3 --- /dev/null +++ b/__tests__/kernel-deep-nesting.test.ts @@ -0,0 +1,281 @@ +/** + * Deep-nesting safety for the native kernel (#1581). + * + * The kernel's per-language walkers recurse once per AST level. tree-sitter's + * parser is iterative, so a pathologically nested file — clang's + * `clang/test/Parser/parser_overflow.c` nests 16,384 `{`; fuzzer corpora go + * deeper — parses fine and then overflowed the WALKER's native stack. A native + * overflow is uncatchable: the parse worker is a thread of the `codegraph` + * process, so the SIGSEGV killed the whole indexer with no message, no partial + * index, no per-file fallback. Worker threads get Node's 4 MiB default stack; + * the 8 MiB main thread only moved the cliff (100k levels still died). + * + * The kernel now guards its recursion against the calling thread's real stack + * bounds (codegraph-kernel/src/stack.rs) and turns an imminent overflow into + * its `defer:` routing signal, so the file takes the wasm path — whose walker + * catches its own JS `RangeError` per file and stores a partial result with a + * `parse_error`. These tests pin that contract on every default-routed + * language, on the main thread AND inside a default-sized worker, and + * end-to-end through the built CLI. + * + * Like the other kernel suites: skipped without a staged .node; CI that + * builds the kernel sets CODEGRAPH_KERNEL_EXPECT=1 so a missing binary FAILS. + */ + +import { describe, it, expect, beforeAll, beforeEach, afterEach } from 'vitest'; +import * as fs from 'fs'; +import * as os from 'os'; +import * as path from 'path'; +import { execFileSync } from 'child_process'; +import { Worker } from 'worker_threads'; +import { extractFromSource } from '../src/extraction'; +import { initGrammars, loadGrammarsForLanguages } from '../src/extraction/grammars'; +import { kernelRoutes, resetKernelForTests } from '../src/extraction/kernel'; +import type { Language } from '../src/types'; + +const REPO = path.resolve(__dirname, '..'); +const KERNEL_PATH = path.join( + REPO, + 'codegraph-kernel', + 'prebuilds', + `${process.platform}-${process.arch}`, + 'codegraph-kernel.node' +); +const kernelBuilt = fs.existsSync(KERNEL_PATH); +const expectKernel = process.env.CODEGRAPH_KERNEL_EXPECT === '1'; +const BIN = path.join(REPO, 'dist', 'bin', 'codegraph.js'); +const DIST_KERNEL = path.join(REPO, 'dist', 'extraction', 'kernel'); +const distBuilt = fs.existsSync(BIN) && fs.existsSync(path.join(DIST_KERNEL, 'index.js')); + +/** Deep enough to overflow an 8 MiB main-thread stack on every walker. */ +const PARENS_DEPTH = 60_000; +/** The reporter's exact shape: clang's parser_overflow.c nests 16,384 `{`. */ +const BRACES_DEPTH = 16_384; + +const CANDIDATES: Language[] = [ + 'typescript', 'tsx', 'javascript', 'jsx', 'java', 'python', 'go', 'c', 'cpp', + 'rust', 'csharp', 'ruby', 'php', 'swift', 'kotlin', 'r', 'lua', 'luau', 'scala', 'dart', +]; + +const EXT: Record = { + typescript: 'ts', tsx: 'tsx', javascript: 'js', jsx: 'jsx', java: 'java', python: 'py', + go: 'go', c: 'c', cpp: 'cpp', rust: 'rs', csharp: 'cs', ruby: 'rb', php: 'php', + swift: 'swift', kotlin: 'kt', r: 'R', lua: 'lua', luau: 'luau', scala: 'scala', dart: 'dart', +}; + +/** A function `f` whose body is a `depth`-deep parenthesized expression. */ +function deepParens(language: Language, depth: number): string { + const open = '('.repeat(depth); + const close = ')'.repeat(depth); + switch (language) { + case 'typescript': case 'tsx': case 'javascript': case 'jsx': + return `function f() { return ${open}1${close}; }\n`; + case 'java': + return `class A {\n int f() { return ${open}1${close}; }\n}\n`; + case 'python': + return `def f():\n return ${open}1${close}\n`; + case 'go': + return `package p\n\nfunc f() int { return ${open}1${close} }\n`; + case 'c': + return `int f(void) { return ${open}1${close}; }\n`; + case 'cpp': + return `int f() { return ${open}1${close}; }\n`; + case 'rust': + return `fn f() -> i32 { ${open}1${close} }\n`; + case 'csharp': + return `class A {\n int f() { return ${open}1${close}; }\n}\n`; + case 'ruby': + return `def f\n ${open}1${close}\nend\n`; + case 'php': + return ` Int { return ${open}1${close} }\n`; + case 'kotlin': + return `fun f(): Int { return ${open}1${close} }\n`; + case 'r': + return `f <- function() {\n ${open}1${close}\n}\n`; + case 'lua': case 'luau': + return `local function f()\n return ${open}1${close}\nend\n`; + case 'scala': + return `object A {\n def f(): Int = ${open}1${close}\n}\n`; + case 'dart': + return `int f() { return ${open}1${close}; }\n`; + default: + throw new Error(`no deep fixture for ${language}`); + } +} + +/** The reporter's repro: a C function body of `depth` nested blocks. */ +function deepBraces(depth: number): string { + return `void foo(void) {\n${'{'.repeat(depth)}${'}'.repeat(depth)}\n}\n`; +} + +const ENV_KEYS = ['CODEGRAPH_KERNEL', 'CODEGRAPH_KERNEL_LANGS', 'CODEGRAPH_KERNEL_PATH'] as const; +let savedEnv: Record; + +describe.skipIf(!kernelBuilt)('kernel deep-nesting guard (#1581)', () => { + let routed: Language[] = []; + + beforeAll(async () => { + resetKernelForTests(); + routed = CANDIDATES.filter((l) => kernelRoutes(l)); + expect(routed.length).toBeGreaterThan(0); + await initGrammars(); + await loadGrammarsForLanguages(routed); + }); + + beforeEach(() => { + savedEnv = Object.fromEntries(ENV_KEYS.map((k) => [k, process.env[k]])); + resetKernelForTests(); + }); + + afterEach(() => { + for (const k of ENV_KEYS) { + if (savedEnv[k] === undefined) delete process.env[k]; + else process.env[k] = savedEnv[k]; + } + resetKernelForTests(); + }); + + it('every default-routed language survives a 60k-deep expression on the main thread', () => { + const failures: string[] = []; + for (const language of routed) { + const file = `deep.${EXT[language]}`; + const source = deepParens(language, PARENS_DEPTH); + // The ONLY acceptable outcomes: a clean result (the thread's stack was + // big enough for the walk), or the wasm fallback's partial result with + // its parse_error. A native overflow would have killed this process. + const result = extractFromSource(file, source, language); + const fn = result.nodes.find((n) => n.name === 'f' && (n.kind === 'function' || n.kind === 'method')); + // R's wasm walker mints `f <- function()` only after walking the + // assignment's value, so its partial result for a file this deep holds + // just the file node — the same shape main's wasm-only path produces + // (verified with CODEGRAPH_KERNEL=0). Pre-existing and out of scope + // here; what this test pins for R is that the process survives. + if (!fn && language !== 'r') failures.push(`${language}: no function node 'f' (nodes=${result.nodes.map((n) => `${n.kind}:${n.name}`).join(',')})`); + for (const e of result.errors) { + if (!/Maximum call stack|parse_error|Parse error/.test(`${e.code} ${e.message}`)) { + failures.push(`${language}: unexpected error ${e.message}`); + } + } + } + expect(failures).toEqual([]); + }, 120_000); + + it("the reporter's 16,384-brace C file is indexed (partial) instead of killing the process", () => { + const result = extractFromSource('deep.c', deepBraces(BRACES_DEPTH), 'c'); + expect(result.nodes.some((n) => n.kind === 'function' && n.name === 'foo')).toBe(true); + }, 60_000); + + it('shallow files still take the kernel path (the guard never trips on normal code)', () => { + // Sanity for the perf-neutral claim: a 200-deep expression is far inside + // any thread's stack, so it must come back clean with no parse_error. + for (const language of routed) { + const result = extractFromSource(`ok.${EXT[language]}`, deepParens(language, 200), language); + expect(result.errors, language).toEqual([]); + expect(result.nodes.some((n) => n.name === 'f'), language).toBe(true); + } + }, 60_000); + + describe.skipIf(!distBuilt)('inside a default-sized (4 MiB) parse worker, through dist/', () => { + let tmp: string; + beforeEach(() => { + tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-deep-')); + }); + afterEach(() => { + fs.rmSync(tmp, { recursive: true, force: true }); + }); + + /** + * Run the kernel's raw extraction for `file` inside a Worker with Node's + * DEFAULT resourceLimits — exactly how ParseWorkerPool runs it. Resolves + * with the worker's exit code and what it reported; a native overflow + * would SIGSEGV/SIGILL this whole vitest process instead. + */ + function runInWorker(file: string, source: string, language: Language): Promise<{ exitCode: number; outcome: string }> { + const script = path.join(tmp, 'worker.cjs'); + fs.writeFileSync( + script, + [ + `const { parentPort, workerData } = require('worker_threads');`, + `const { tryKernelExtractRaw } = require(${JSON.stringify(DIST_KERNEL)});`, + `const raw = tryKernelExtractRaw(workerData.file, workerData.source, workerData.language);`, + `parentPort.postMessage(raw ? 'kernel:' + raw.counts.nodes : 'deferred');`, + ].join('\n') + ); + return new Promise((resolve, reject) => { + let outcome = 'no message'; + const w = new Worker(script, { workerData: { file, source, language } }); + w.on('message', (m: string) => { outcome = m; }); + w.on('error', reject); + w.on('exit', (exitCode) => resolve({ exitCode, outcome })); + }); + } + + it("defers the reporter's deep.c instead of crashing the worker", async () => { + const r = await runInWorker('deep.c', deepBraces(BRACES_DEPTH), 'c'); + expect(r.exitCode).toBe(0); + expect(r.outcome).toBe('deferred'); + }, 60_000); + + it('defers a 60k-deep expression in every default-routed language', async () => { + for (const language of routed) { + const r = await runInWorker(`deep.${EXT[language]}`, deepParens(language, PARENS_DEPTH), language); + expect(r.exitCode, language).toBe(0); + // Either the guard tripped (deferred) or the walk fit — never a crash. + expect(['deferred', 'kernel'].some((p) => r.outcome.startsWith(p)), `${language}: ${r.outcome}`).toBe(true); + } + }, 180_000); + + it('still extracts a normal file natively in the worker', async () => { + const r = await runInWorker('ok.c', 'int add(int a, int b) { return a + b; }\n', 'c'); + expect(r.exitCode).toBe(0); + expect(r.outcome).toMatch(/^kernel:/); + }, 30_000); + }); + + describe.skipIf(!distBuilt)('end-to-end: codegraph init on a repo holding the deep file', () => { + let tmp: string; + beforeEach(() => { + tmp = fs.mkdtempSync(path.join(os.tmpdir(), 'codegraph-deep-cli-')); + }); + afterEach(() => { + fs.rmSync(tmp, { recursive: true, force: true }); + }); + + it('exits 0 and records deep.c alongside the normal files', () => { + fs.writeFileSync(path.join(tmp, 'deep.c'), deepBraces(BRACES_DEPTH)); + fs.writeFileSync(path.join(tmp, 'ok.c'), 'int add(int a, int b) { return a + b; }\n'); + execFileSync(process.execPath, [BIN, 'init', '.'], { + cwd: tmp, + encoding: 'utf-8', + stdio: ['ignore', 'pipe', 'pipe'], + timeout: 120_000, + env: { + ...process.env, + CODEGRAPH_NO_DAEMON: '1', + CODEGRAPH_WASM_RELAUNCHED: '1', + CODEGRAPH_TELEMETRY: '0', + DO_NOT_TRACK: '1', + CODEGRAPH_NO_PROMPT_HOOK: '1', + }, + }); + const { DatabaseSync } = require('node:sqlite') as typeof import('node:sqlite'); + const db = new DatabaseSync(path.join(tmp, '.codegraph', 'codegraph.db'), { readOnly: true }); + try { + const files = (db.prepare('SELECT path FROM files ORDER BY path').all() as Array<{ path: string }>).map((r) => r.path); + expect(files).toEqual(['deep.c', 'ok.c']); + const fns = (db.prepare("SELECT name FROM nodes WHERE kind = 'function' ORDER BY name").all() as Array<{ name: string }>).map((r) => r.name); + expect(fns).toEqual(['add', 'foo']); + } finally { + db.close(); + } + }, 180_000); + }); +}); + +describe.skipIf(!expectKernel)('kernel presence (CODEGRAPH_KERNEL_EXPECT=1)', () => { + it('the staged .node exists so the deep-nesting suite actually ran', () => { + expect(kernelBuilt).toBe(true); + }); +}); diff --git a/codegraph-kernel/Cargo.lock b/codegraph-kernel/Cargo.lock index 65a70e115..a53782029 100644 --- a/codegraph-kernel/Cargo.lock +++ b/codegraph-kernel/Cargo.lock @@ -47,6 +47,7 @@ name = "codegraph-kernel" version = "0.1.0" dependencies = [ "cc", + "libc", "napi", "napi-build", "napi-derive", diff --git a/codegraph-kernel/Cargo.toml b/codegraph-kernel/Cargo.toml index 3cb15d7a4..f40626ae1 100644 --- a/codegraph-kernel/Cargo.toml +++ b/codegraph-kernel/Cargo.toml @@ -62,6 +62,12 @@ tree-sitter-luau = "=1.2.0" # kotlin grammar C (see build.rs — no kotlin crate dep is possible). tree-sitter-language = "0.1" +# Stack-bounds queries for the walker stack guard (src/stack.rs, #1581): +# pthread_getattr_np / pthread_get_stackaddr_np. Already in the lock file +# transitively; Windows uses a hand-declared kernel32 extern instead. +[target.'cfg(unix)'.dependencies] +libc = "0.2" + [build-dependencies] napi-build = "2" cc = "1" diff --git a/codegraph-kernel/src/ccpp/mod.rs b/codegraph-kernel/src/ccpp/mod.rs index 9f058f495..78877d7da 100644 --- a/codegraph-kernel/src/ccpp/mod.rs +++ b/codegraph-kernel/src/ccpp/mod.rs @@ -776,6 +776,7 @@ impl<'t> Walker<'t> { // --- visitNode ----------------------------------------------------------- fn visit_node(&mut self, node: Node<'t>) { + stack_guard!(); let kind = node.kind(); let mut skip_children = false; @@ -852,6 +853,7 @@ impl<'t> Walker<'t> { // --- extractors ---------------------------------------------------------- fn extract_function(&mut self, node: Node<'t>) { + stack_guard!(); // Receiver present (out-of-line `Cls::method` def) → method instead. if self.variant == Variant::Cpp && self.receiver_type_of(node).is_some() { self.extract_method(node); @@ -892,6 +894,7 @@ impl<'t> Walker<'t> { } fn extract_method(&mut self, node: Node<'t>) { + stack_guard!(); let receiver_type = if self.variant == Variant::Cpp { self.receiver_type_of(node) } else { None }; if !self.inside_class_like() && receiver_type.is_none() { @@ -956,6 +959,7 @@ impl<'t> Walker<'t> { /// extractClass for cpp class_specifier (skipBodilessClass, #1093). fn extract_class(&mut self, node: Node<'t>) { + stack_guard!(); let Some(body) = node.child_by_field_name("body") else { return }; let name = self.extract_name(node); let extra = Extra { @@ -976,6 +980,7 @@ impl<'t> Walker<'t> { /// Extract a struct-like declaration while preserving its semantic kind. fn extract_aggregate(&mut self, node: Node<'t>, kind: &'static str) { + stack_guard!(); let Some(body) = node.child_by_field_name("body") else { return }; let name = self.extract_name(node); let extra = Extra { @@ -995,6 +1000,7 @@ impl<'t> Walker<'t> { } fn extract_enum(&mut self, node: Node<'t>) { + stack_guard!(); let Some(body) = node.child_by_field_name("body") else { return }; let name = self.extract_name(node); let extra = Extra { @@ -1028,6 +1034,7 @@ impl<'t> Walker<'t> { /// extractTypeAlias for type_definition / alias_declaration. Returns true /// when children were consumed (typedef struct/enum bodies). fn extract_type_alias(&mut self, node: Node<'t>) -> bool { + stack_guard!(); let name = self.extract_name(node); if name == "" { return false; @@ -1502,10 +1509,12 @@ impl<'t> Walker<'t> { // --- function bodies ----------------------------------------------------- fn visit_function_body(&mut self, body: Node<'t>) { + stack_guard!(); self.visit_for_calls_and_structure(body); } fn visit_for_calls_and_structure(&mut self, node: Node<'t>) { + stack_guard!(); let kind = node.kind(); self.maybe_capture_fn_refs(node); @@ -1591,6 +1600,7 @@ impl<'t> Walker<'t> { /// grammars: base_class_clause (#1043), the field_declaration Go-embedding /// shape, and the field_declaration_list recursion that reaches it. fn extract_inheritance(&mut self, node: Node<'t>, class_row: u32) { + stack_guard!(); let extends_kind = edge_kind_index("extends").unwrap(); for i in 0..node.named_child_count() { let Some(child) = node.named_child(i) else { continue }; @@ -1703,6 +1713,7 @@ impl<'t> Walker<'t> { /// normalizeValue for cFamilySpec: bare identifiers, and the /// pointer_expression unwrap (`&fn`; `&Cls::m` keeps the qualified name). fn normalize_fn_ref_value(&mut self, v: Node<'t>, from: u32, mode: Mode, explicit_ref: bool, depth: u32) { + stack_guard!(); if depth > 4 { return; } @@ -1749,6 +1760,7 @@ impl<'t> Walker<'t> { /// scanFnRefSubtree: capture-only walk of subtrees the main walkers skip /// (variable-declaration initializers). Halts at nested functions/lambdas. fn scan_fn_ref_subtree(&mut self, node: Node<'t>, depth: u32) { + stack_guard!(); if depth > 12 { return; } diff --git a/codegraph-kernel/src/csharp.rs b/codegraph-kernel/src/csharp.rs index 4bed3a996..f370a0e54 100644 --- a/codegraph-kernel/src/csharp.rs +++ b/codegraph-kernel/src/csharp.rs @@ -500,6 +500,7 @@ impl<'t> Walker<'t> { // --- the dispatcher (visitNode, C#-relevant branches) ----------------------- fn visit_node(&mut self, node: Node<'t>) { + stack_guard!(); let kind = node.kind(); let mut skip_children = false; @@ -571,10 +572,12 @@ impl<'t> Walker<'t> { // --- visitFunctionBody ------------------------------------------------------ fn visit_function_body(&mut self, body: Node<'t>) { + stack_guard!(); self.visit_for_calls_and_structure(body); } fn visit_for_calls_and_structure(&mut self, node: Node<'t>) { + stack_guard!(); let kind = node.kind(); self.maybe_capture_fn_refs(node); @@ -626,6 +629,7 @@ impl<'t> Walker<'t> { // --- extractors -------------------------------------------------------------- fn extract_class(&mut self, node: Node<'t>) { + stack_guard!(); // skipBodilessClass unset: a bodiless `record Empty;` still mints a node. let name = self.extract_name(node); let extra = Extra { @@ -656,6 +660,7 @@ impl<'t> Walker<'t> { } fn extract_struct(&mut self, node: Node<'t>) { + stack_guard!(); // Body gate — EXCEPT C# positional records (`record struct M(…);`, // node type record_declaration), complete definitions with no body. // A bodiless `struct Fwd;` mints NO node. (#831) @@ -685,6 +690,7 @@ impl<'t> Walker<'t> { } fn extract_interface(&mut self, node: Node<'t>) { + stack_guard!(); let name = self.extract_name(node); let extra = Extra { docstring: preceding_docstring(node, self.src), @@ -703,6 +709,7 @@ impl<'t> Walker<'t> { } fn extract_enum(&mut self, node: Node<'t>) { + stack_guard!(); let Some(body) = node.child_by_field_name("body") else { return }; let name = self.extract_name(node); let extra = Extra { @@ -890,6 +897,7 @@ impl<'t> Walker<'t> { /// extractMethod (1737) — method_declaration + constructor_declaration. /// Signature is ALWAYS undefined (no getSignature hook); isAsync is real. fn extract_method(&mut self, node: Node<'t>) { + stack_guard!(); if !self.inside_class_like() { // Unreachable on non-erroring C# (top-level `void M(){}` parses as // local_function_statement; erroring files defer) — mirror the TS @@ -924,6 +932,7 @@ impl<'t> Walker<'t> { /// extractFunction — only reachable for a method outside any class /// (unreachable on non-erroring C#; kept faithful to the generic tail). fn extract_function(&mut self, node: Node<'t>) { + stack_guard!(); let name = self.extract_name(node); if name == "" { if let Some(body) = node.child_by_field_name("body") { @@ -1087,6 +1096,7 @@ impl<'t> Walker<'t> { /// (object initializers are initializer_expression), so this is /// unreachable — mirrored from the shared TS path like java.rs. fn extract_anonymous_class(&mut self, node: Node<'t>, body: Node<'t>) { + stack_guard!(); let type_node = node .child_by_field_name("constructor") .or_else(|| node.child_by_field_name("type")) @@ -1243,6 +1253,7 @@ impl<'t> Walker<'t> { /// walkCsharpTypePosition (5955). fn walk_type_position(&mut self, node: Node<'t>, from_row: u32) { + stack_guard!(); match node.kind() { "predefined_type" => {} "identifier" => { @@ -1362,6 +1373,7 @@ impl<'t> Walker<'t> { /// normalizeValue (function-ref.ts:525) for CSHARP_SPEC: bare identifiers, /// the transparent `argument` layer, and the `this.Member` special. fn normalize_fn_ref_value(&mut self, v: Node<'t>, from: u32, depth: u32) { + stack_guard!(); if depth > 4 { return; } @@ -1412,6 +1424,7 @@ impl<'t> Walker<'t> { } fn scan_fn_ref_subtree(&mut self, node: Node<'t>, depth: u32) { + stack_guard!(); if depth > 12 { return; } diff --git a/codegraph-kernel/src/dart.rs b/codegraph-kernel/src/dart.rs index a0df27ac0..748410b10 100644 --- a/codegraph-kernel/src/dart.rs +++ b/codegraph-kernel/src/dart.rs @@ -596,6 +596,7 @@ impl<'t> Walker<'t> { // --- the main walk (visitNode, tree-sitter.ts:936-1303) --------------- fn visit(&mut self, node: Node<'t>) { + stack_guard!(); // The visitNode hook (dart.ts:144-157) — the constants branch. if node.kind() == "static_final_declaration" { let mut cursor = node.walk(); @@ -669,6 +670,7 @@ impl<'t> Walker<'t> { // --- extractFunction / extractMethod (:1517 / :1737) ------------------ fn extract_function(&mut self, node: Node<'t>) { + stack_guard!(); // No receiver hook. Name first (resolveName inside extract_name). let name = self.extract_name(node); if name == "" { @@ -770,6 +772,7 @@ impl<'t> Walker<'t> { // --- extractClass (:1679) — classes, mixins, extensions --------------- fn extract_class(&mut self, node: Node<'t>) { + stack_guard!(); let resolved_body = self.resolve_body(node); // No skipBodilessClass. Anonymous `extension on String` → the name // fallback finds the ON type's type_identifier — a class named after @@ -800,6 +803,7 @@ impl<'t> Walker<'t> { // --- extractEnum (:1914) ---------------------------------------------- fn extract_enum(&mut self, node: Node<'t>) { + stack_guard!(); let body = match self.resolve_body(node) { Some(b) => b, None => return, @@ -1221,6 +1225,7 @@ impl<'t> Walker<'t> { } fn type_refs_from_subtree(&mut self, node: Node<'t>, from_row: u32) { + stack_guard!(); if node.kind() == "type_identifier" { let name = self.text(node); if !name.is_empty() && !is_builtin_type(name) { @@ -1239,6 +1244,7 @@ impl<'t> Walker<'t> { // --- visitFunctionBody (:5129-5286) — dart rows ----------------------- fn visit_body(&mut self, node: Node<'t>) { + stack_guard!(); self.maybe_capture_fn_refs(node); let kind = node.kind(); @@ -1348,6 +1354,7 @@ impl<'t> Walker<'t> { /// normalizeValue with DART_SPEC's one layer (`argument` → fan out). /// Named arguments are NOT captured (named_argument is not a layer). fn normalize_fn_ref_value(&mut self, v: Node<'t>, from: u32, depth: u32) { + stack_guard!(); if depth > 4 { return; } @@ -1378,6 +1385,7 @@ impl<'t> Walker<'t> { } fn scan_fn_ref_subtree(&mut self, node: Node<'t>, depth: u32) { + stack_guard!(); if depth > 12 { return; } diff --git a/codegraph-kernel/src/go.rs b/codegraph-kernel/src/go.rs index 71081cb6c..66bb352f1 100644 --- a/codegraph-kernel/src/go.rs +++ b/codegraph-kernel/src/go.rs @@ -380,6 +380,7 @@ impl<'t> Walker<'t> { // --- visitNode ------------------------------------------------------------ fn visit_node(&mut self, node: Node<'t>) { + stack_guard!(); let kind = node.kind(); let mut skip_children = false; @@ -417,10 +418,12 @@ impl<'t> Walker<'t> { } fn visit_function_body(&mut self, body: Node<'t>) { + stack_guard!(); self.visit_for_calls_and_structure(body); } fn visit_for_calls_and_structure(&mut self, node: Node<'t>) { + stack_guard!(); let kind = node.kind(); self.maybe_capture_fn_refs(node); @@ -448,6 +451,7 @@ impl<'t> Walker<'t> { // --- extractors -------------------------------------------------------------- fn extract_function(&mut self, node: Node<'t>) { + stack_guard!(); // (getReceiverType only matches method_declaration's receiver field — // function_declaration has none, so no reroute happens here) let name = self.extract_name(node); @@ -524,6 +528,7 @@ impl<'t> Walker<'t> { /// extractTypeAlias for Go: type_spec → struct / interface / plain alias. fn extract_type_alias(&mut self, node: Node<'t>) -> bool { + stack_guard!(); let name = self.extract_name(node); if name == "" { return false; @@ -842,6 +847,7 @@ impl<'t> Walker<'t> { /// (constraint_elem) and struct embedding (field_declaration without a /// field_identifier), plus the field_declaration_list recursion. fn extract_inheritance(&mut self, node: Node<'t>, class_row: u32) { + stack_guard!(); let extends_kind = edge_kind_index("extends").unwrap(); for i in 0..node.named_child_count() { let Some(child) = node.named_child(i) else { continue }; @@ -894,6 +900,7 @@ impl<'t> Walker<'t> { } fn extract_type_refs_from_subtree(&mut self, node: Node<'t>, from_row: u32) { + stack_guard!(); if node.kind() == "type_identifier" { let type_name = self.text(node).to_string(); if !type_name.is_empty() && !is_builtin_type(&type_name) { @@ -980,6 +987,7 @@ impl<'t> Walker<'t> { /// normalizeValue with GO_SPEC's transparent layers (literal_element, /// expression_list — both fan out to named children). fn normalize_fn_ref_value(&mut self, v: Node<'t>, from: u32, depth: u32) { + stack_guard!(); if depth > 4 { return; } @@ -1010,6 +1018,7 @@ impl<'t> Walker<'t> { } fn scan_fn_ref_subtree(&mut self, node: Node<'t>, depth: u32) { + stack_guard!(); if depth > 12 { return; } diff --git a/codegraph-kernel/src/java.rs b/codegraph-kernel/src/java.rs index d2017a093..7065ab565 100644 --- a/codegraph-kernel/src/java.rs +++ b/codegraph-kernel/src/java.rs @@ -485,6 +485,7 @@ impl<'t> Walker<'t> { // --- the dispatcher (visitNode, Java-relevant branches) ----------------------- fn visit_node(&mut self, node: Node<'t>) { + stack_guard!(); let kind = node.kind(); let mut skip_children = false; @@ -534,10 +535,12 @@ impl<'t> Walker<'t> { // --- visitFunctionBody ---------------------------------------------------------- fn visit_function_body(&mut self, body: Node<'t>) { + stack_guard!(); self.visit_for_calls_and_structure(body); } fn visit_for_calls_and_structure(&mut self, node: Node<'t>) { + stack_guard!(); let kind = node.kind(); self.maybe_capture_fn_refs(node); @@ -577,6 +580,7 @@ impl<'t> Walker<'t> { // --- extractors -------------------------------------------------------------- fn extract_class(&mut self, node: Node<'t>) { + stack_guard!(); let name = self.extract_name(node); let extra = Extra { docstring: preceding_docstring(node, self.src), @@ -600,6 +604,7 @@ impl<'t> Walker<'t> { } fn extract_method(&mut self, node: Node<'t>) { + stack_guard!(); if !self.inside_class_like() { // (object-literal parents don't exist in Java; a stray top-level // method extracts as a function, mirroring extractMethod's tail) @@ -627,6 +632,7 @@ impl<'t> Walker<'t> { /// extractFunction — only reachable for a method outside any class. fn extract_function(&mut self, node: Node<'t>) { + stack_guard!(); let name = self.extract_name(node); if name == "" { if let Some(body) = node.child_by_field_name("body") { @@ -653,6 +659,7 @@ impl<'t> Walker<'t> { } fn extract_interface(&mut self, node: Node<'t>) { + stack_guard!(); let name = self.extract_name(node); let extra = Extra { docstring: preceding_docstring(node, self.src), @@ -671,6 +678,7 @@ impl<'t> Walker<'t> { } fn extract_enum(&mut self, node: Node<'t>) { + stack_guard!(); let Some(body) = node.child_by_field_name("body") else { return }; let name = self.extract_name(node); let extra = Extra { @@ -902,6 +910,7 @@ impl<'t> Walker<'t> { /// extractAnonymousClass — `new T() { ... }`. fn extract_anonymous_class(&mut self, node: Node<'t>, body: Node<'t>) { + stack_guard!(); let type_node = node .child_by_field_name("constructor") .or_else(|| node.child_by_field_name("type")) @@ -1101,6 +1110,7 @@ impl<'t> Walker<'t> { } fn extract_type_refs_from_subtree(&mut self, node: Node<'t>, from_row: u32) { + stack_guard!(); if node.kind() == "type_identifier" { let type_name = self.text(node).to_string(); if !type_name.is_empty() && !is_builtin_type(&type_name) { @@ -1193,6 +1203,7 @@ impl<'t> Walker<'t> { } fn scan_fn_ref_subtree(&mut self, node: Node<'t>, depth: u32) { + stack_guard!(); if depth > 12 { return; } diff --git a/codegraph-kernel/src/kotlin.rs b/codegraph-kernel/src/kotlin.rs index ee47d8758..58fde4285 100644 --- a/codegraph-kernel/src/kotlin.rs +++ b/codegraph-kernel/src/kotlin.rs @@ -648,6 +648,7 @@ impl<'t> Walker<'t> { // --- the dispatcher (visitNode, Kotlin-relevant branches) ----------------------- fn visit_node(&mut self, node: Node<'t>) { + stack_guard!(); if self.try_visit_hook(node) { self.scan_fn_ref_subtree(node, 0); return; @@ -719,10 +720,12 @@ impl<'t> Walker<'t> { // --- visitFunctionBody ---------------------------------------------------------- fn visit_function_body(&mut self, body: Node<'t>) { + stack_guard!(); self.visit_for_calls_and_structure(body); } fn visit_for_calls_and_structure(&mut self, node: Node<'t>) { + stack_guard!(); let kind = node.kind(); self.maybe_capture_fn_refs(node); @@ -777,6 +780,7 @@ impl<'t> Walker<'t> { // --- extractors ------------------------------------------------------------------ fn extract_function(&mut self, node: Node<'t>) { + stack_guard!(); // getReceiverType short-circuit (1522) — extension fns at any scope. if self.receiver_type_of(node).is_some() { self.extract_method(node); @@ -810,6 +814,7 @@ impl<'t> Walker<'t> { } fn extract_method(&mut self, node: Node<'t>) { + stack_guard!(); let receiver = self.receiver_type_of(node); let name = self.extract_name(node); let qualified_override = receiver.as_ref().map(|r| format!("{r}::{name}")); @@ -861,6 +866,7 @@ impl<'t> Walker<'t> { } fn extract_class(&mut self, node: Node<'t>) { + stack_guard!(); let resolved_body = self.resolve_body(node); let name = self.extract_name(node); let extra = Extra { @@ -887,6 +893,7 @@ impl<'t> Walker<'t> { } fn extract_interface(&mut self, node: Node<'t>) { + stack_guard!(); let name = self.extract_name(node); let extra = Extra { docstring: preceding_docstring(node, self.src), @@ -905,6 +912,7 @@ impl<'t> Walker<'t> { } fn extract_enum(&mut self, node: Node<'t>) { + stack_guard!(); let Some(body) = self.resolve_body(node) else { return }; let name = self.extract_name(node); let extra = Extra { @@ -1283,6 +1291,7 @@ impl<'t> Walker<'t> { } fn normalize_fn_ref_value(&mut self, v: Node<'t>, from: u32, depth: u32) { + stack_guard!(); if depth > 4 { return; } @@ -1362,6 +1371,7 @@ impl<'t> Walker<'t> { } fn scan_fn_ref_subtree(&mut self, node: Node<'t>, depth: u32) { + stack_guard!(); if depth > 12 { return; } diff --git a/codegraph-kernel/src/lib.rs b/codegraph-kernel/src/lib.rs index d600b0cda..7411fce0f 100644 --- a/codegraph-kernel/src/lib.rs +++ b/codegraph-kernel/src/lib.rs @@ -16,6 +16,20 @@ #![deny(clippy::all)] +/// First statement of every recursive walker function (see stack.rs, #1581): +/// once the stack pointer is inside the red zone, stop descending — the +/// latched flag makes `stack::run_guarded` discard the walk and defer the +/// file to wasm. `Default::default()` covers every walker return type in use +/// (`()`, `bool`, `Option<_>`, `String`); a hook returning `false` just sends +/// its caller down the generic child walk, whose own guard returns at once. +macro_rules! stack_guard { + () => { + if $crate::stack::exhausted() { + return ::core::default::Default::default(); + } + }; +} + mod buffers; mod ccpp; mod cfnptr; @@ -33,6 +47,7 @@ mod rlang; mod ruby; mod rustlang; mod scala; +mod stack; mod swift; mod textutil; mod python; @@ -216,23 +231,28 @@ pub fn cfnptr_strip_c(text: String) -> String { #[napi] pub fn extract_file(file_path: String, content: String, language: String) -> Result { - let out = match language.as_str() { - "java" => java::extract(&file_path, &content).map_err(Error::from_reason)?, - "python" => python::extract(&file_path, &content).map_err(Error::from_reason)?, - "go" => go::extract(&file_path, &content).map_err(Error::from_reason)?, - "c" | "cpp" => ccpp::extract(&file_path, &content, &language).map_err(Error::from_reason)?, - "rust" => rustlang::extract(&file_path, &content).map_err(Error::from_reason)?, - "csharp" => csharp::extract(&file_path, &content).map_err(Error::from_reason)?, - "ruby" => ruby::extract(&file_path, &content).map_err(Error::from_reason)?, - "php" => php::extract(&file_path, &content).map_err(Error::from_reason)?, - "swift" => swift::extract(&file_path, &content).map_err(Error::from_reason)?, - "kotlin" => kotlin::extract(&file_path, &content).map_err(Error::from_reason)?, - "r" => rlang::extract(&file_path, &content).map_err(Error::from_reason)?, - "lua" | "luau" => lua::extract(&file_path, &content, &language).map_err(Error::from_reason)?, - "scala" => scala::extract(&file_path, &content).map_err(Error::from_reason)?, - "dart" => dart::extract(&file_path, &content).map_err(Error::from_reason)?, - _ => tsjs::extract(&file_path, &content, &language).map_err(Error::from_reason)?, - }; + // The whole walk runs under the stack guard (stack.rs, #1581): a file + // nested deeply enough to overflow this thread's stack comes back as a + // `defer:` error — the TS side's routine "take the wasm path" signal — + // instead of a SIGSEGV that kills the entire indexer process. + let out = stack::run_guarded(|| match language.as_str() { + "java" => java::extract(&file_path, &content), + "python" => python::extract(&file_path, &content), + "go" => go::extract(&file_path, &content), + "c" | "cpp" => ccpp::extract(&file_path, &content, &language), + "rust" => rustlang::extract(&file_path, &content), + "csharp" => csharp::extract(&file_path, &content), + "ruby" => ruby::extract(&file_path, &content), + "php" => php::extract(&file_path, &content), + "swift" => swift::extract(&file_path, &content), + "kotlin" => kotlin::extract(&file_path, &content), + "r" => rlang::extract(&file_path, &content), + "lua" | "luau" => lua::extract(&file_path, &content, &language), + "scala" => scala::extract(&file_path, &content), + "dart" => dart::extract(&file_path, &content), + _ => tsjs::extract(&file_path, &content, &language), + }) + .map_err(Error::from_reason)?; Ok(ExtractBuffers { meta: out.meta.into(), nodes: out.nodes.into(), diff --git a/codegraph-kernel/src/lua.rs b/codegraph-kernel/src/lua.rs index f9ab3ab2b..b3cd32a76 100644 --- a/codegraph-kernel/src/lua.rs +++ b/codegraph-kernel/src/lua.rs @@ -418,6 +418,7 @@ impl<'t> Walker<'t> { // --- the main walk (visitNode, tree-sitter.ts:936-1303) --------------- fn visit(&mut self, node: Node<'t>) { + stack_guard!(); let kind = node.kind(); // The visitNode hook (lua.ts:105-151) runs FIRST. @@ -487,6 +488,7 @@ impl<'t> Walker<'t> { // --- extractFunction / extractMethod (1517 / 1737) -------------------- fn extract_function(&mut self, node: Node<'t>) { + stack_guard!(); // :1522 receiver short-circuit IS the method routing. if let Some(receiver) = self.receiver_type(node) { let receiver = receiver.to_string(); @@ -522,6 +524,7 @@ impl<'t> Walker<'t> { } fn extract_method(&mut self, node: Node<'t>, receiver: String) { + stack_guard!(); let name = self.extract_name(node); let docstring = preceding_docstring(node, self.src); let signature = self.signature_of(node); @@ -654,6 +657,7 @@ impl<'t> Walker<'t> { // --- visitFunctionBody (5129-5286) — the hook-free body walk ---------- fn visit_body(&mut self, node: Node<'t>) { + stack_guard!(); // maybeCaptureFnRefs (5137) fires in the body walker too. self.maybe_capture_fn_refs(node); @@ -750,6 +754,7 @@ impl<'t> Walker<'t> { /// normalizeValue with LUA_SPEC's one transparent layer (expression_list /// fans out to named children). fn normalize_fn_ref_value(&mut self, v: Node<'t>, from: u32, depth: u32) { + stack_guard!(); if depth > 4 { return; } @@ -780,6 +785,7 @@ impl<'t> Walker<'t> { } fn scan_fn_ref_subtree(&mut self, node: Node<'t>, depth: u32) { + stack_guard!(); if depth > 12 { return; } diff --git a/codegraph-kernel/src/php.rs b/codegraph-kernel/src/php.rs index 3369e311b..13a8626b6 100644 --- a/codegraph-kernel/src/php.rs +++ b/codegraph-kernel/src/php.rs @@ -530,6 +530,7 @@ impl<'t> Walker<'t> { // --- the dispatcher (visitNode, PHP-relevant branches) ------------------------ fn visit_node(&mut self, node: Node<'t>) { + stack_guard!(); if self.try_visit_hook(node) { self.scan_fn_ref_subtree(node, 0); return; @@ -609,10 +610,12 @@ impl<'t> Walker<'t> { // --- visitFunctionBody -------------------------------------------------------- fn visit_function_body(&mut self, body: Node<'t>) { + stack_guard!(); self.visit_for_calls_and_structure(body); } fn visit_for_calls_and_structure(&mut self, node: Node<'t>) { + stack_guard!(); let kind = node.kind(); self.maybe_capture_fn_refs(node); @@ -671,6 +674,7 @@ impl<'t> Walker<'t> { // --- extractors ---------------------------------------------------------------- fn extract_function(&mut self, node: Node<'t>) { + stack_guard!(); let name = self.extract_name(node); if name == "" { if let Some(body) = node.child_by_field_name("body") { @@ -696,6 +700,7 @@ impl<'t> Walker<'t> { } fn extract_method(&mut self, node: Node<'t>) { + stack_guard!(); let name = self.extract_name(node); let extra = Extra { docstring: preceding_docstring(node, self.src), @@ -715,6 +720,7 @@ impl<'t> Walker<'t> { } fn extract_class(&mut self, node: Node<'t>, kind: &'static str) { + stack_guard!(); let name = self.extract_name(node); let extra = Extra { docstring: preceding_docstring(node, self.src), @@ -736,6 +742,7 @@ impl<'t> Walker<'t> { } fn extract_interface(&mut self, node: Node<'t>) { + stack_guard!(); let name = self.extract_name(node); let extra = Extra { docstring: preceding_docstring(node, self.src), @@ -754,6 +761,7 @@ impl<'t> Walker<'t> { } fn extract_enum(&mut self, node: Node<'t>) { + stack_guard!(); let Some(body) = node.child_by_field_name("body") else { return }; let name = self.extract_name(node); let extra = Extra { @@ -1080,6 +1088,7 @@ impl<'t> Walker<'t> { /// nests inside `anonymous_class`, so findAnonymousClassBody finds no /// DIRECT child) — mirrored from the shared TS path for shape. fn extract_anonymous_class(&mut self, node: Node<'t>, body: Node<'t>) { + stack_guard!(); let type_node = node .child_by_field_name("constructor") .or_else(|| node.child_by_field_name("type")) @@ -1206,6 +1215,7 @@ impl<'t> Walker<'t> { } fn walk_php_type_position(&mut self, node: Node<'t>, from_row: u32) { + stack_guard!(); match node.kind() { "primitive_type" => {} "name" => { @@ -1249,6 +1259,7 @@ impl<'t> Walker<'t> { } fn normalize_fn_ref_value(&mut self, v: Node<'t>, from: u32, depth: u32) { + stack_guard!(); if depth > 4 { return; } @@ -1337,6 +1348,7 @@ impl<'t> Walker<'t> { } fn scan_fn_ref_subtree(&mut self, node: Node<'t>, depth: u32) { + stack_guard!(); if depth > 12 { return; } diff --git a/codegraph-kernel/src/python.rs b/codegraph-kernel/src/python.rs index 93cb10a9d..b2397facd 100644 --- a/codegraph-kernel/src/python.rs +++ b/codegraph-kernel/src/python.rs @@ -320,6 +320,7 @@ impl<'t> Walker<'t> { // --- visitNode ------------------------------------------------------------ fn visit_node(&mut self, node: Node<'t>) { + stack_guard!(); let kind = node.kind(); let mut skip_children = false; @@ -356,10 +357,12 @@ impl<'t> Walker<'t> { } fn visit_function_body(&mut self, body: Node<'t>) { + stack_guard!(); self.visit_for_calls_and_structure(body); } fn visit_for_calls_and_structure(&mut self, node: Node<'t>) { + stack_guard!(); let kind = node.kind(); self.maybe_capture_fn_refs(node); @@ -390,6 +393,7 @@ impl<'t> Walker<'t> { // --- extractors -------------------------------------------------------------- fn extract_function(&mut self, node: Node<'t>) { + stack_guard!(); let name = self.extract_name(node); if name == "" { if let Some(body) = node.child_by_field_name("body") { @@ -414,6 +418,7 @@ impl<'t> Walker<'t> { } fn extract_method(&mut self, node: Node<'t>) { + stack_guard!(); let name = self.extract_name(node); let extra = Extra { docstring: preceding_docstring(node, self.src), @@ -431,6 +436,7 @@ impl<'t> Walker<'t> { } fn extract_class(&mut self, node: Node<'t>) { + stack_guard!(); let name = self.extract_name(node); let extra = Extra { docstring: preceding_docstring(node, self.src), @@ -789,6 +795,7 @@ impl<'t> Walker<'t> { } fn scan_fn_ref_subtree(&mut self, node: Node<'t>, depth: u32) { + stack_guard!(); if depth > 12 { return; } diff --git a/codegraph-kernel/src/rlang.rs b/codegraph-kernel/src/rlang.rs index 04be9dccb..9c85f0cf2 100644 --- a/codegraph-kernel/src/rlang.rs +++ b/codegraph-kernel/src/rlang.rs @@ -298,6 +298,7 @@ impl<'t> Walker<'t> { // else plain recursion over namedChildren in order. fn visit(&mut self, node: Node<'t>) { + stack_guard!(); if self.hook(node) { return; } @@ -313,6 +314,7 @@ impl<'t> Walker<'t> { /// The visitNode hook (r.ts:180-309). Returns true when consumed. fn hook(&mut self, node: Node<'t>) -> bool { + stack_guard!(); match node.kind() { "call" => self.hook_call(node), "binary_operator" => self.hook_binary_operator(node), @@ -321,6 +323,7 @@ impl<'t> Walker<'t> { } fn hook_call(&mut self, node: Node<'t>) -> bool { + stack_guard!(); let fname = match self.callee_name(node) { Some(f) => f, None => return false, @@ -405,6 +408,7 @@ impl<'t> Walker<'t> { } fn hook_binary_operator(&mut self, node: Node<'t>) -> bool { + stack_guard!(); let op = match node.child_by_field_name("operator") { Some(o) => self.text(o), None => return false, @@ -477,6 +481,7 @@ impl<'t> Walker<'t> { /// `list(…)` entries become methods. Non-method argument subtrees are /// NEVER visited (`representation(…)`, `signature(…)` invisible). fn extract_class_members(&mut self, class_call: Node<'t>, class_row: u32) { + stack_guard!(); let args = match class_call.child_by_field_name("arguments") { Some(a) => a, None => return, @@ -543,6 +548,7 @@ impl<'t> Walker<'t> { /// `method` node positioned at the ARGUMENT node, signature from the raw /// parameters text, body walked hook-aware inside the method scope. fn emit_method_arg(&mut self, entry: Node<'t>) { + stack_guard!(); let entry_name = match entry.child_by_field_name("name") { Some(n) => n, None => return, diff --git a/codegraph-kernel/src/ruby.rs b/codegraph-kernel/src/ruby.rs index 18d8407b1..35cd2a713 100644 --- a/codegraph-kernel/src/ruby.rs +++ b/codegraph-kernel/src/ruby.rs @@ -383,6 +383,7 @@ impl<'t> Walker<'t> { /// the source of the module multiply-capture quirk (scan runs with the /// module already POPPED, so candidates re-attribute to the outer scope). fn try_visit_hook(&mut self, node: Node<'t>) -> bool { + stack_guard!(); let kind = node.kind(); if kind == "call" && node.child_by_field_name("receiver").is_none() { if let Some(method) = node.child_by_field_name("method") { @@ -451,6 +452,7 @@ impl<'t> Walker<'t> { // --- the dispatcher (visitNode, Ruby-relevant branches) ---------------------- fn visit_node(&mut self, node: Node<'t>) { + stack_guard!(); // Language hook FIRST (tree-sitter.ts:943) — a handled subtree is // scanned for fn-ref candidates and never reaches the ladder (or the // maybeCaptureFnRefs call below). @@ -522,10 +524,12 @@ impl<'t> Walker<'t> { // --- visitFunctionBody ------------------------------------------------------ fn visit_function_body(&mut self, body: Node<'t>) { + stack_guard!(); self.visit_for_calls_and_structure(body); } fn visit_for_calls_and_structure(&mut self, node: Node<'t>) { + stack_guard!(); let kind = node.kind(); self.maybe_capture_fn_refs(node); @@ -596,6 +600,7 @@ impl<'t> Walker<'t> { // --- extractors -------------------------------------------------------------- fn extract_function(&mut self, node: Node<'t>) { + stack_guard!(); let name = self.extract_name(node); if name == "" { if let Some(body) = node.child_by_field_name("body") { @@ -619,6 +624,7 @@ impl<'t> Walker<'t> { } fn extract_method(&mut self, node: Node<'t>) { + stack_guard!(); let name = self.extract_name(node); let extra = Extra { docstring: preceding_docstring(node, self.src), @@ -634,6 +640,7 @@ impl<'t> Walker<'t> { } fn extract_class(&mut self, node: Node<'t>) { + stack_guard!(); let name = self.extract_name(node); let extra = Extra { docstring: preceding_docstring(node, self.src), @@ -872,6 +879,7 @@ impl<'t> Walker<'t> { /// qualify); `block_argument` is a transparent layer; specials are the /// `method(:sym)` call form and hook-DSL `simple_symbol`s. fn normalize_fn_ref_value(&mut self, v: Node<'t>, from: u32, depth: u32) { + stack_guard!(); if depth > 4 { return; } @@ -938,6 +946,7 @@ impl<'t> Walker<'t> { } fn scan_fn_ref_subtree(&mut self, node: Node<'t>, depth: u32) { + stack_guard!(); if depth > 12 { return; } diff --git a/codegraph-kernel/src/rustlang.rs b/codegraph-kernel/src/rustlang.rs index 16447c243..d6d15d54a 100644 --- a/codegraph-kernel/src/rustlang.rs +++ b/codegraph-kernel/src/rustlang.rs @@ -435,6 +435,7 @@ impl<'t> Walker<'t> { // --- visitNode ------------------------------------------------------------ fn visit_node(&mut self, node: Node<'t>) { + stack_guard!(); let kind = node.kind(); let mut skip_children = false; @@ -498,6 +499,7 @@ impl<'t> Walker<'t> { /// impl method's body, whose parent walk passes through the outer fn) or /// the stack top is class-like (trait members). fn extract_fn_or_method(&mut self, node: Node<'t>) { + stack_guard!(); let receiver = self.receiver_type_of(node); let as_method = receiver.is_some() || self.inside_class_like(); @@ -564,6 +566,7 @@ impl<'t> Walker<'t> { /// extractInterface — kind `trait` (interfaceKind), inheritance from /// trait_bounds, body children visited with the trait pushed. fn extract_interface(&mut self, node: Node<'t>) { + stack_guard!(); let name = self.extract_name(node); let extra = Extra { docstring: preceding_docstring(node, self.src), @@ -584,6 +587,7 @@ impl<'t> Walker<'t> { /// Extract a Rust struct or union with a body; unit structs remain skipped. fn extract_aggregate(&mut self, node: Node<'t>, kind: &'static str) { + stack_guard!(); let Some(body) = node.child_by_field_name("body") else { return }; let name = self.extract_name(node); let extra = Extra { @@ -606,6 +610,7 @@ impl<'t> Walker<'t> { /// extractEnum — body required; enum_variant children → enum_member nodes /// (name field only, payloads never walked); other children re-dispatched. fn extract_enum(&mut self, node: Node<'t>) { + stack_guard!(); let Some(body) = node.child_by_field_name("body") else { return }; let name = self.extract_name(node); let extra = Extra { @@ -700,6 +705,7 @@ impl<'t> Walker<'t> { /// getRootModule (languages/rust.ts:124). fn root_module(&self, n: Node) -> String { + stack_guard!(); let Some(first) = n.named_child(0) else { return self.text(n).to_string(); }; @@ -719,6 +725,7 @@ impl<'t> Walker<'t> { if prefix.is_empty() { seg.to_string() } else { format!("{prefix}::{seg}") } } fn collect<'t>(w: &Walker<'t>, n: Node<'t>, prefix: &str, paths: &mut Vec<(String, Node<'t>)>) { + stack_guard!(); match n.kind() { "identifier" => paths.push((join(prefix, w.text(n)), n)), "scoped_identifier" => { @@ -966,6 +973,7 @@ impl<'t> Walker<'t> { /// every field has a field_identifier), and the field_declaration_list /// recursion that reaches it. fn extract_inheritance(&mut self, node: Node<'t>, class_row: u32) { + stack_guard!(); let extends_kind = edge_kind_index("extends").unwrap(); for i in 0..node.named_child_count() { let Some(child) = node.named_child(i) else { continue }; @@ -1086,6 +1094,7 @@ impl<'t> Walker<'t> { } fn extract_type_refs_from_subtree(&mut self, node: Node<'t>, from_row: u32) { + stack_guard!(); if node.kind() == "type_identifier" { let type_name = self.text(node).to_string(); if !type_name.is_empty() && !is_builtin_type(&type_name) { @@ -1103,10 +1112,12 @@ impl<'t> Walker<'t> { // --- visitFunctionBody ----------------------------------------------------- fn visit_function_body(&mut self, body: Node<'t>) { + stack_guard!(); self.visit_for_calls_and_structure(body); } fn visit_for_calls_and_structure(&mut self, node: Node<'t>) { + stack_guard!(); let kind = node.kind(); self.maybe_capture_fn_refs(node); @@ -1262,6 +1273,7 @@ impl<'t> Walker<'t> { } fn scan_fn_ref_subtree(&mut self, node: Node<'t>, depth: u32) { + stack_guard!(); if depth > 12 { return; } diff --git a/codegraph-kernel/src/scala.rs b/codegraph-kernel/src/scala.rs index eb7a0bd3d..fc80feae7 100644 --- a/codegraph-kernel/src/scala.rs +++ b/codegraph-kernel/src/scala.rs @@ -471,6 +471,7 @@ impl<'t> Walker<'t> { /// scalaBaseTypeName (tree-sitter.ts:201-224). fn scala_base_type_name(&self, node: Option>) -> Option { + stack_guard!(); let node = node?; match node.kind() { "type_identifier" | "identifier" => Some(self.text(node).to_string()), @@ -495,6 +496,7 @@ impl<'t> Walker<'t> { /// emitScalaTypeRefs (scala.ts:27-45) — the hook's own builtin set. fn emit_scala_type_refs(&mut self, type_node: Node<'t>, from_row: u32) { + stack_guard!(); if type_node.kind() == "type_identifier" { let name = self.text(type_node); if !name.is_empty() && !is_scala_builtin(name) { @@ -529,6 +531,7 @@ impl<'t> Walker<'t> { // --- the main walk (visitNode, tree-sitter.ts:936-1303) --------------- fn visit(&mut self, node: Node<'t>) { + stack_guard!(); // The visitNode hook (scala.ts:131-198) runs FIRST. if self.hook(node) { self.scan_fn_ref_subtree(node, 0); @@ -591,6 +594,7 @@ impl<'t> Walker<'t> { /// The visitNode hook (scala.ts:131-198). Returns true when consumed. fn hook(&mut self, node: Node<'t>) -> bool { + stack_guard!(); match node.kind() { "val_definition" | "var_definition" => { let is_val = node.kind() == "val_definition"; @@ -691,6 +695,7 @@ impl<'t> Walker<'t> { // --- extractMethod → extractFunction routing (:1737 / :1517) ---------- fn extract_method_or_function(&mut self, node: Node<'t>) { + stack_guard!(); // No receiver hook, no methodsAreTopLevel: inside class-like → method, // else → function (the object/object_expression parent check never // matches scala node kinds). @@ -735,6 +740,7 @@ impl<'t> Walker<'t> { // --- extractClass (:1679) — classes, objects, traits ------------------ fn extract_class(&mut self, node: Node<'t>, kind: &'static str) { + stack_guard!(); let resolved_body = node.child_by_field_name("body"); // template_body // No skipBodilessClass — bodiless mints (scala-complete). let name = self.extract_name(node); @@ -765,6 +771,7 @@ impl<'t> Walker<'t> { // --- extractEnum (:1914) ---------------------------------------------- fn extract_enum(&mut self, node: Node<'t>) { + stack_guard!(); let body = match node.child_by_field_name("body") { Some(b) => b, None => return, // bodiless enum mints nothing @@ -812,6 +819,7 @@ impl<'t> Walker<'t> { // --- extractImport (:3170-3236) --------------------------------------- fn extract_import(&mut self, node: Node<'t>) { + stack_guard!(); let import_text = self.text(node).trim(); // extractImport hook (scala.ts:200-211): `path` field is FIRST-MATCH- // WINS → the FIRST dotted segment names the import. @@ -1133,6 +1141,7 @@ impl<'t> Walker<'t> { } fn type_refs_from_subtree(&mut self, node: Node<'t>, from_row: u32) { + stack_guard!(); if node.kind() == "type_identifier" { let name = self.text(node); if !name.is_empty() && !is_builtin_type(name) { @@ -1151,6 +1160,7 @@ impl<'t> Walker<'t> { // --- visitFunctionBody (:5129-5286) — scala rows ---------------------- fn visit_body(&mut self, node: Node<'t>) { + stack_guard!(); self.maybe_capture_fn_refs(node); let kind = node.kind(); @@ -1266,6 +1276,7 @@ impl<'t> Walker<'t> { /// normalizeValue with SCALA_SPEC's unwrap (postfix_expression → first /// named child — eta-expansion `handler _`). No layers. fn normalize_fn_ref_value(&mut self, v: Node<'t>, from: u32, depth: u32) { + stack_guard!(); if depth > 4 { return; } @@ -1294,6 +1305,7 @@ impl<'t> Walker<'t> { } fn scan_fn_ref_subtree(&mut self, node: Node<'t>, depth: u32) { + stack_guard!(); if depth > 12 { return; } diff --git a/codegraph-kernel/src/stack.rs b/codegraph-kernel/src/stack.rs new file mode 100644 index 000000000..e5b044b23 --- /dev/null +++ b/codegraph-kernel/src/stack.rs @@ -0,0 +1,273 @@ +//! Stack-budget guard for the recursive walkers (#1581). +//! +//! Every language walker recurses per AST level (`visit_node` → +//! `visit_for_calls_and_structure` → …). tree-sitter's own parser is +//! iterative, so a pathologically nested file — clang's +//! `parser_overflow.c` nests 16,384 `{`, fuzzer corpora go deeper — parses +//! fine and then overflows the WALKER's native stack. A native overflow is +//! uncatchable: the parse worker is a thread of the `codegraph` process, so +//! the SIGSEGV takes the whole indexer down with no message, no partial +//! index and no per-file fallback. Worker threads get Node's 4 MiB default +//! stack; the main thread's 8 MiB only moves the cliff (100k levels still +//! kill it). +//! +//! The guard turns "about to overflow" into the kernel's existing `defer:` +//! routing signal: `exhausted()` is checked at the top of every recursive +//! walker function (the `stack_guard!` macro in lib.rs), returns `true` once +//! the stack pointer is within `RED_ZONE` of the thread's stack limit, and +//! latches a per-thread flag. `run_guarded` wraps a whole extraction: when +//! the flag is set afterwards the result is discarded and replaced by a +//! `defer:` error, which the TS side (`src/extraction/kernel/index.ts`) +//! already treats as "this file takes the wasm path" — and the wasm walker +//! catches its own JS `RangeError` per file, so the file lands as a partial +//! result with a recorded parse error instead of a dead process. +//! +//! The per-thread stack bounds come from the OS (glibc/musl +//! `pthread_getattr_np`, macOS `pthread_get_stackaddr_np`, Win32 +//! `GetCurrentThreadStackLimits`), computed once per thread and cached, so +//! the guard is exact on the 4 MiB worker, the 8 MiB main thread and any +//! `resourceLimits.stackSizeMb` alike. Where the bounds are unavailable the +//! guard falls back to a fixed descent budget measured from the entry stack +//! pointer. Hot path: one thread-local load and one compare. + +use std::cell::Cell; + +/// Headroom kept free below the deepest walker frame: the napi return path, +/// tree-sitter's node accessors and the error formatting all still need to +/// run after the guard trips, and the frames BETWEEN two guard checks (an +/// `extract_class` between two `visit_node`s) are never more than a few KiB. +const RED_ZONE: usize = 256 * 1024; + +/// Descent budget when the OS can't report the thread's stack bounds — safe +/// on anything from Node's 4 MiB worker default upwards. +const FALLBACK_BUDGET: usize = 1024 * 1024; + +thread_local! { + /// Lowest stack-pointer value the walker may reach before the guard + /// trips. `0` = not computed yet for this thread. + static THRESHOLD: Cell = const { Cell::new(0) }; + /// `true` when the thread's threshold came from real OS bounds (fixed + /// for the thread's lifetime) rather than the per-call fallback budget. + static THRESHOLD_IS_OS: Cell = const { Cell::new(false) }; + /// Latched by `exhausted()`; read by `run_guarded` after the walk. + static OVERFLOWED: Cell = const { Cell::new(false) }; +} + +/// Approximate current stack pointer: the address of a local. Stacks grow +/// downward on every target the kernel ships for (x86_64 / aarch64). +#[inline(always)] +fn current_sp() -> usize { + let marker = 0u8; + std::hint::black_box(&marker) as *const u8 as usize +} + +/// Low (deepest) address of the calling thread's stack, from the OS. +#[cfg(target_os = "linux")] +fn os_stack_low() -> Option { + // SAFETY: plain pthread queries on the calling thread; `attr` is + // initialised by pthread_getattr_np and destroyed before returning. + unsafe { + let mut attr: libc::pthread_attr_t = std::mem::zeroed(); + if libc::pthread_getattr_np(libc::pthread_self(), &mut attr) != 0 { + return None; + } + let mut addr: *mut libc::c_void = std::ptr::null_mut(); + let mut size: libc::size_t = 0; + let rc = libc::pthread_attr_getstack(&attr, &mut addr, &mut size); + libc::pthread_attr_destroy(&mut attr); + if rc != 0 || addr.is_null() || size == 0 { + return None; + } + Some(addr as usize) + } +} + +#[cfg(target_os = "macos")] +fn os_stack_low() -> Option { + // SAFETY: plain pthread queries on the calling thread. + unsafe { + let me = libc::pthread_self(); + // pthread_get_stackaddr_np returns the HIGH end (the stack base). + let high = libc::pthread_get_stackaddr_np(me) as usize; + let size = libc::pthread_get_stacksize_np(me); + if high == 0 || size == 0 || size > high { + return None; + } + Some(high - size) + } +} + +#[cfg(windows)] +fn os_stack_low() -> Option { + #[link(name = "kernel32")] + extern "system" { + // Win8+ (the bundled Node runtime needs Win10 anyway). Reports the + // full RESERVED range; Windows commits pages on demand down to it. + fn GetCurrentThreadStackLimits(low_limit: *mut usize, high_limit: *mut usize); + } + let mut low: usize = 0; + let mut high: usize = 0; + // SAFETY: both out-pointers are valid for the duration of the call. + unsafe { GetCurrentThreadStackLimits(&mut low, &mut high) }; + if low == 0 || high <= low { + return None; + } + Some(low) +} + +#[cfg(not(any(target_os = "linux", target_os = "macos", windows)))] +fn os_stack_low() -> Option { + None +} + +/// Arm the guard for one extraction on the calling thread: clear the latch +/// and (re)compute the threshold. OS bounds are computed once per thread; +/// the fallback budget is re-anchored at every call's entry stack pointer. +pub fn begin() { + OVERFLOWED.with(|o| o.set(false)); + let cached = THRESHOLD.with(|t| t.get()); + if cached != 0 && THRESHOLD_IS_OS.with(|f| f.get()) { + return; + } + match os_stack_low() { + Some(low) => { + THRESHOLD.with(|t| t.set(low.saturating_add(RED_ZONE))); + THRESHOLD_IS_OS.with(|f| f.set(true)); + } + None => { + THRESHOLD.with(|t| t.set(current_sp().saturating_sub(FALLBACK_BUDGET).max(1))); + THRESHOLD_IS_OS.with(|f| f.set(false)); + } + } +} + +/// `true` once the walker has descended to within `RED_ZONE` of the stack +/// limit. Latches `OVERFLOWED` so `run_guarded` can discard the result. A +/// thread that never called `begin()` (a direct unit-test call) arms itself +/// lazily from the current position. +#[inline(always)] +pub fn exhausted() -> bool { + let threshold = THRESHOLD.with(|t| t.get()); + if threshold == 0 { + begin(); + return exhausted(); + } + if current_sp() < threshold { + OVERFLOWED.with(|o| o.set(true)); + true + } else { + false + } +} + +/// Whether the guard tripped since the last `begin()`. +pub fn overflowed() -> bool { + OVERFLOWED.with(|o| o.get()) +} + +/// Run one extraction under the guard. A walk that tripped the guard returns +/// a `defer:` error — the TS side's routine "take the wasm path" signal — +/// regardless of what the truncated walk produced. +pub fn run_guarded(f: impl FnOnce() -> Result) -> Result { + begin(); + let out = f(); + if overflowed() { + return Err( + "defer: nesting too deep for the native walker — wasm recovery handles it".to_string(), + ); + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + + /// 1 MiB is a quarter of Node's worker default; a guard that holds here + /// holds on every real thread. Without the guard these walks SIGSEGV the + /// test process instead of failing an assertion. + const SMALL_STACK: usize = 1 << 20; + const DEPTH: usize = 30_000; + + fn on_small_stack(f: impl FnOnce() -> T + Send + 'static) -> T { + std::thread::Builder::new() + .stack_size(SMALL_STACK) + .spawn(f) + .expect("spawn") + .join() + .expect("walker thread panicked") + } + + fn nested_parens(prefix: &str, suffix: &str) -> String { + format!("{prefix}{}1{}{suffix}", "(".repeat(DEPTH), ")".repeat(DEPTH)) + } + + #[test] + fn os_bounds_are_sane_on_this_platform() { + // Every shipped target has an OS implementation; the fallback budget + // is only for platforms the kernel is not built for. + let low = os_stack_low().expect("OS stack bounds available"); + let sp = current_sp(); + assert!(low < sp, "stack low {low:#x} must be below the current sp {sp:#x}"); + assert!(sp - low < 1 << 31, "implausible stack size {}", sp - low); + } + + #[test] + fn small_stack_reports_its_own_bounds() { + on_small_stack(|| { + let low = os_stack_low().expect("OS stack bounds available"); + let used = current_sp() - low; + // std/the OS round the requested size up a little (macOS reports + // 1,060,864 for a 1 MiB request); the point is that the bounds + // describe THIS thread's small stack, not the main thread's. + assert!( + used <= SMALL_STACK + 128 * 1024, + "used {used} is not within the {SMALL_STACK}-byte stack" + ); + }); + } + + #[test] + fn deep_braces_c_defer_instead_of_crashing() { + let src = format!("void foo(void) {{\n{}{}\n}}\n", "{".repeat(DEPTH), "}".repeat(DEPTH)); + let r = on_small_stack(move || run_guarded(|| crate::ccpp::extract("deep.c", &src, "c"))); + let err = r.err().expect("deep nesting must defer"); + assert!(err.starts_with("defer:"), "unexpected error: {err}"); + } + + type Extract = fn(&str) -> Result; + + #[test] + fn deep_parens_cpp_rust_ts_python_defer_instead_of_crashing() { + let cases: [(&str, Extract, String); 4] = [ + ("deep.cpp", |s| crate::ccpp::extract("deep.cpp", s, "cpp"), nested_parens("int f() { return ", "; }\n")), + ("deep.rs", |s| crate::rustlang::extract("deep.rs", s), nested_parens("fn f() -> i32 { ", " }\n")), + ("deep.ts", |s| crate::tsjs::extract("deep.ts", s, "typescript"), nested_parens("function f() { return ", "; }\n")), + ("deep.py", |s| crate::python::extract("deep.py", s), nested_parens("def f():\n return ", "\n")), + ]; + for (name, extract, src) in cases { + let r = on_small_stack(move || run_guarded(|| extract(&src))); + let err = r.err().unwrap_or_else(|| panic!("{name}: deep nesting must defer")); + assert!(err.starts_with("defer:"), "{name}: unexpected error: {err}"); + } + } + + #[test] + fn normal_files_are_untouched_by_the_guard() { + let src = "int add(int a, int b) { return a + b; }\nint main(void) { return add(1, 2); }\n"; + let r = on_small_stack(move || run_guarded(|| crate::ccpp::extract("ok.c", src, "c"))); + assert!(r.is_ok(), "a shallow file must not defer: {:?}", r.err()); + assert!(!overflowed()); + } + + #[test] + fn latch_resets_between_runs() { + let deep = format!("void foo(void) {{\n{}{}\n}}\n", "{".repeat(DEPTH), "}".repeat(DEPTH)); + on_small_stack(move || { + assert!(run_guarded(|| crate::ccpp::extract("deep.c", &deep, "c")).is_err()); + // The latch from the deep file must not poison the next, shallow one. + let ok = run_guarded(|| crate::ccpp::extract("ok.c", "int x;\n", "c")); + assert!(ok.is_ok(), "latch leaked into the next run: {:?}", ok.err()); + }); + } +} diff --git a/codegraph-kernel/src/swift.rs b/codegraph-kernel/src/swift.rs index e365b7197..7d46c4922 100644 --- a/codegraph-kernel/src/swift.rs +++ b/codegraph-kernel/src/swift.rs @@ -264,6 +264,7 @@ fn first_simple_identifier<'t>(node: Option>) -> Option> { /// lastNamedOfType (function-ref.ts:600): rightmost matching DESCENDANT in /// document order (deeper matches override). fn last_simple_identifier<'t>(node: Node<'t>) -> Option> { + stack_guard!(); let mut found: Option> = None; for i in 0..node.named_child_count() { let Some(child) = node.named_child(i) else { continue }; @@ -560,6 +561,7 @@ impl<'t> Walker<'t> { // --- the dispatcher (visitNode, Swift-relevant branches) ----------------------- fn visit_node(&mut self, node: Node<'t>) { + stack_guard!(); let kind = node.kind(); let mut skip_children = false; @@ -632,6 +634,7 @@ impl<'t> Walker<'t> { /// THE DEDICATED PROPERTY BRANCH (tree-sitter.ts:1113-1193, #1020). /// Returns skipChildren. fn dedicated_property_branch(&mut self, node: Node<'t>) -> bool { + stack_guard!(); let owner_row = self.top_row(); let info = self.swift_property_info(node); let mut computed_prop: Option<(u32, String)> = None; @@ -707,6 +710,7 @@ impl<'t> Walker<'t> { } fn walk_attr_args(&mut self, n: Node<'t>) { + stack_guard!(); self.extract_static_member_ref(n); for i in 0..n.named_child_count() { if let Some(c) = n.named_child(i) { @@ -718,10 +722,12 @@ impl<'t> Walker<'t> { // --- visitFunctionBody --------------------------------------------------------- fn visit_function_body(&mut self, body: Node<'t>) { + stack_guard!(); self.visit_for_calls_and_structure(body); } fn visit_for_calls_and_structure(&mut self, node: Node<'t>) { + stack_guard!(); let kind = node.kind(); self.maybe_capture_fn_refs(node); @@ -775,6 +781,7 @@ impl<'t> Walker<'t> { // --- extractors ----------------------------------------------------------------- fn extract_function(&mut self, node: Node<'t>) { + stack_guard!(); let name = self.extract_name(node); if name == "" { if let Some(body) = node.child_by_field_name("body") { @@ -802,6 +809,7 @@ impl<'t> Walker<'t> { } fn extract_method(&mut self, node: Node<'t>) { + stack_guard!(); let name = self.extract_name(node); let extra = Extra { docstring: preceding_docstring(node, self.src), @@ -823,6 +831,7 @@ impl<'t> Walker<'t> { } fn extract_class(&mut self, node: Node<'t>) { + stack_guard!(); let name = self.extract_name(node); let extra = Extra { docstring: preceding_docstring(node, self.src), @@ -845,6 +854,7 @@ impl<'t> Walker<'t> { } fn extract_struct(&mut self, node: Node<'t>) { + stack_guard!(); // Body gate (:1876) — bodiless mints nothing (record exemption is C#). let Some(body) = node.child_by_field_name("body") else { return }; let name = self.extract_name(node); @@ -866,6 +876,7 @@ impl<'t> Walker<'t> { } fn extract_enum(&mut self, node: Node<'t>) { + stack_guard!(); let Some(body) = node.child_by_field_name("body") else { return }; let name = self.extract_name(node); let extra = Extra { @@ -900,6 +911,7 @@ impl<'t> Walker<'t> { } fn extract_interface(&mut self, node: Node<'t>) { + stack_guard!(); let name = self.extract_name(node); let extra = Extra { docstring: preceding_docstring(node, self.src), @@ -1151,6 +1163,7 @@ impl<'t> Walker<'t> { } fn extract_type_refs_from_subtree(&mut self, node: Node<'t>, from_row: u32) { + stack_guard!(); if node.kind() == "type_identifier" { let type_name = self.text(node).to_string(); if !type_name.is_empty() && !is_builtin_type(&type_name) { @@ -1317,6 +1330,7 @@ impl<'t> Walker<'t> { } fn normalize_fn_ref_value(&mut self, v: Node<'t>, from: u32, depth: u32) { + stack_guard!(); if depth > 4 { return; } @@ -1381,6 +1395,7 @@ impl<'t> Walker<'t> { } fn scan_fn_ref_subtree(&mut self, node: Node<'t>, depth: u32) { + stack_guard!(); if depth > 12 { return; } diff --git a/codegraph-kernel/src/tsjs/extractors.rs b/codegraph-kernel/src/tsjs/extractors.rs index ad0440d6d..3e4814736 100644 --- a/codegraph-kernel/src/tsjs/extractors.rs +++ b/codegraph-kernel/src/tsjs/extractors.rs @@ -478,6 +478,7 @@ impl<'t> Walker<'t> { } fn find_initializer_returned_object(&self, call: Node<'t>, depth: u32) -> Option> { + stack_guard!(); if depth > 4 { return None; } @@ -499,6 +500,7 @@ impl<'t> Walker<'t> { fn function_returned_object(&self, fn_node: Node<'t>) -> Option> { fn as_object<'t>(n: Node<'t>) -> Option> { + stack_guard!(); match n.kind() { "object" | "object_expression" => Some(n), "parenthesized_expression" => { @@ -873,6 +875,7 @@ impl<'t> Walker<'t> { fn extract_ts_tuple_contract_names(&mut self, value: Node<'t>, alias_row: u32, alias_name: &str) { let mut tuples: Vec = Vec::new(); fn collect<'t>(n: Node<'t>, depth: u32, out: &mut Vec>) { + stack_guard!(); if depth > 6 { return; } @@ -1230,6 +1233,7 @@ impl<'t> Walker<'t> { // --- extractInheritance (TS/JS clauses) --------------------------------------------------- pub(super) fn extract_inheritance(&mut self, node: Node<'t>, class_row: u32) { + stack_guard!(); let extends_kind = edge_kind_index("extends").unwrap(); let implements_kind = edge_kind_index("implements").unwrap(); for i in 0..node.named_child_count() { @@ -1298,6 +1302,7 @@ impl<'t> Walker<'t> { } fn extract_type_refs_from_subtree(&mut self, node: Node<'t>, from_row: u32) { + stack_guard!(); if node.kind() == "type_identifier" { let type_name = self.text(node).to_string(); if !type_name.is_empty() && !is_builtin_type(&type_name) { diff --git a/codegraph-kernel/src/tsjs/mod.rs b/codegraph-kernel/src/tsjs/mod.rs index f61cf93e2..7559ae3f8 100644 --- a/codegraph-kernel/src/tsjs/mod.rs +++ b/codegraph-kernel/src/tsjs/mod.rs @@ -550,6 +550,7 @@ impl<'t> Walker<'t> { /// scanFnRefSubtree: capture-only walk of subtrees the main walkers skip. fn scan_fn_ref_subtree(&mut self, node: Node<'t>, depth: u32) { + stack_guard!(); if depth > 12 { return; } @@ -607,6 +608,7 @@ impl<'t> Walker<'t> { // --- the dispatcher (visitNode) -------------------------------------------- fn visit_node(&mut self, node: Node<'t>) { + stack_guard!(); let kind = node.kind(); let mut skip_children = false; @@ -686,6 +688,7 @@ impl<'t> Walker<'t> { } fn visit_for_calls_and_structure(&mut self, node: Node<'t>) { + stack_guard!(); let kind = node.kind(); self.maybe_capture_fn_refs(node); diff --git a/docs/design/rust-kernel-migration-plan.md b/docs/design/rust-kernel-migration-plan.md index 2f58bc7c6..88a80dde3 100644 --- a/docs/design/rust-kernel-migration-plan.md +++ b/docs/design/rust-kernel-migration-plan.md @@ -410,6 +410,14 @@ in a different emission order would shift rowids and change resolution — and file whose tree `has_error()` to the wasm extractor** (`defer:` signal, silent, per-file) — parity by construction on erroring files, 99.6%+ keep the fast path, and the harness fails if deferrals exceed 10% (a broken kernel can't hide). + The same signal carries a second, rarer case (#1581): the walkers recurse per + AST level, and a pathologically nested file (clang's 16,384-brace + `parser_overflow.c`, fuzzer corpora) overflowed the native stack — a SIGSEGV + that killed the whole indexer, uncatchable from JS. `src/stack.rs` now + checks the thread's real stack bounds at every recursive entry + (`stack_guard!`) and `run_guarded` turns a tripped walk into `defer:`, so + the file lands on the wasm path (which catches its own `RangeError` per + file) while the process lives. Pinned by `__tests__/kernel-deep-nesting.test.ts`. 3. **Retrieval invariants:** kernel-indexed excalidraw — `mutateElement → renderStaticScene` connects end-to-end via explore (callback + react-render + jsx hops shown); synthesized-edge families present (408 jsx-render / 46 diff --git a/src/extraction/parse-pool.ts b/src/extraction/parse-pool.ts index c0cacd216..9b2b100e4 100644 --- a/src/extraction/parse-pool.ts +++ b/src/extraction/parse-pool.ts @@ -214,6 +214,12 @@ export class ParseWorkerPool { this.createWorker = opts.createWorker; } else if (opts.workerScriptPath) { const scriptPath = opts.workerScriptPath; + // Deliberately no `resourceLimits.stackSizeMb`: a bigger worker stack + // only moves the cliff a deeply nested file falls off (#1581 — the + // 8 MiB main thread still dies at 100k levels). The native kernel + // guards its own recursion against THIS thread's real stack bounds + // (codegraph-kernel/src/stack.rs) and defers such a file to the wasm + // path, which catches its JS RangeError per file. this.createWorker = () => new Worker(scriptPath); } else { throw new Error('ParseWorkerPool requires workerScriptPath or createWorker');