Skip to content

Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.

Contain deeply nested source to the file that contains it

taeskimfdb85e6ad2e9
diff --git a/docs/architecture.md b/docs/architecture.md
index 5828153..574c500 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -60,7 +60,10 @@ source bytes; never reconstruct returned code from the tree.
Tree-sitter recognizes syntax rather than validating CPython semantics. Trees
with syntax errors use text fallback. Bare-CR Python also uses text fallback
-because retrieval coordinates count LF lines. Repository source is never executed.
+because retrieval coordinates count LF lines. So does generated source nested
+beyond what a parser handles: TypeScript that exhausts the call stack, and Python
+nested more than 50,000 levels, near where Tree-sitter queries stop matching and
+slow down sharply. Repository source is never executed.
See [parser contracts](../test/parser/README.md) and
[measurement evidence](../specs/tree-sitter/RESULTS.md).
diff --git a/packages/core/src/parser-helpers.mjs b/packages/core/src/parser-helpers.mjs
index 28bd880..e7f6f1e 100644
--- a/packages/core/src/parser-helpers.mjs
+++ b/packages/core/src/parser-helpers.mjs
@@ -23,6 +23,10 @@ async function parse(source, action, languageName = "python") {
parser.setLanguage(languages.get(languageName));
tree = parser.parse(source);
if (!tree || tree.rootNode.hasError) return null;
+ // Tree-sitter queries stop reporting matches beyond 65,535 levels of nesting and slow down
+ // sharply near that depth: 70,000 chained `+` took 8 s, 100,000 took 74 s. Treat such
+ // generated source as unparseable, like a syntax error, before any query runs.
+ if (languageName === "python" && deeperThan(tree, maxTreeDepth)) return null;
if (languageName === "python" && !validPython(tree.rootNode)) return null;
return action(tree.rootNode);
} finally {
@@ -31,6 +35,27 @@ async function parse(source, action, languageName = "python") {
}
}
const field = (n, name) => n.childForFieldName(name);
+// Well below the query engine's 65,535-level limit; ordinary source nests far less.
+const maxTreeDepth = 50_000;
+/** Visits every node with a tree cursor, which neither recurses nor allocates node objects. */
+function deeperThan(tree, limit) {
+ const cursor = tree.walk();
+ try {
+ let depth = 0;
+ for (;;) {
+ if (cursor.gotoFirstChild()) {
+ if (++depth > limit) return true;
+ continue;
+ }
+ while (!cursor.gotoNextSibling()) {
+ if (!cursor.gotoParent()) return false;
+ depth--;
+ }
+ }
+ } finally {
+ cursor.delete();
+ }
+}
function unparenthesized(node) {
while (node?.type === "parenthesized_expression")
node = node.namedChildren.find((child) => child.type !== "comment");
diff --git a/packages/core/src/source.ts b/packages/core/src/source.ts
index 47a9e17..9ddbb7c 100644
--- a/packages/core/src/source.ts
+++ b/packages/core/src/source.ts
@@ -146,13 +146,14 @@ export async function inspect(
comments = pythonComments;
mode = "python";
} else if (language === "TS" || language === "TSX" || language === "JS" || language === "JSX") {
- const file = ts.createSourceFile(
- path,
- source,
- ts.ScriptTarget.Latest,
- true,
- ts.ScriptKind[language],
- );
+ let file: ts.SourceFile;
+ try {
+ file = ts.createSourceFile(path, source, ts.ScriptTarget.Latest, true, ts.ScriptKind[language]);
+ } catch (error) {
+ // Deeply nested source can exhaust the parser's call stack: unparseable here, like a syntax error.
+ if (error instanceof RangeError) return fallback("syntax");
+ throw error;
+ }
syntaxFallback = !!(file as ts.SourceFile & { parseDiagnostics?: unknown[] }).parseDiagnostics
?.length;
const line = (position: number) => file.getLineAndCharacterOfPosition(position).line + 1;
@@ -182,15 +183,23 @@ export async function inspect(
});
};
if (!syntaxFallback) for (const statement of file.statements) add(statement);
- const visit = (node: ts.Node) => {
+ // Pre-order walk with an explicit stack: long generated expressions nest deeper than the call
+ // stack allows. Children are pushed in reverse so they are visited in source order.
+ const walk: ts.Node[] = [file];
+ while (walk.length) {
+ const node = walk.pop()!;
for (const comment of [
...(ts.getLeadingCommentRanges(source, node.getFullStart()) ?? []),
...(ts.getTrailingCommentRanges(source, node.end) ?? []),
])
comments.push({ startLine: line(comment.pos), endLine: line(comment.end - 1) });
- ts.forEachChild(node, visit);
- };
- visit(file);
+ const children: ts.Node[] = [];
+ // The callback must return undefined: a truthy result stops forEachChild early.
+ ts.forEachChild(node, (child) => {
+ children.push(child);
+ });
+ for (let index = children.length - 1; index >= 0; index--) walk.push(children[index]!);
+ }
mode = "typescript";
} else if (language === "go" || language === "rust") {
const parsed = await runParser<{ units: typeof units; comments: Range[] }>(
diff --git a/test/parser/README.md b/test/parser/README.md
index d52e1dc..bbe3f1e 100644
--- a/test/parser/README.md
+++ b/test/parser/README.md
@@ -2,6 +2,8 @@
Tests cover original source coordinates, Unicode boundaries, declaration owners,
syntax fallback and cancellation through the production inspection API. Unsupported languages retain lossless text fallback.
+Generated source nested too deeply for a parser also falls back to text, promptly
+and without affecting other files; wide files with many statements still parse.
Python helpers in `reference/` are frozen test-only oracles from the replaced
runtime. Differential fixtures compare declaration, preview, neighbourhood and
diff --git a/test/parser/source.test.ts b/test/parser/source.test.ts
index 01f7e85..a4db84b 100644
--- a/test/parser/source.test.ts
+++ b/test/parser/source.test.ts
@@ -372,3 +372,46 @@ test("same-named classes retain their own structural headers", async () => {
]);
}
});
+
+test("generated TypeScript deeper than the call stack parses or falls back to text", async () => {
+ const { sourceForUnit } = await import("../../packages/core/src/source.ts");
+ // A long flat expression nests one level per operator in the syntax tree.
+ const chain = "// generated\nexport const table = " + "1 + ".repeat(50_000) + "1;\n";
+ const parsed = await inspect(
+ { path: "chain.ts", source: chain, contentHash: "fixture" },
+ { maxUnitBytes: Buffer.byteLength(chain) },
+ );
+ assert.equal(parsed.mode, "typescript");
+ assert.deepEqual(
+ parsed.units.map(({ name, range }) => ({ name, ...range })),
+ [{ name: "table", startLine: 2, endLine: 2 }],
+ );
+ assert.deepEqual(parsed.comments, [{ startLine: 1, endLine: 1 }]);
+ const nested = "export const x = " + "(".repeat(20_000) + "1" + ")".repeat(20_000) + ";\n";
+ const snapshot = { path: "nested.ts", source: nested, contentHash: "fixture" };
+ const fallback = await inspect(snapshot, { maxUnitBytes: 3000 });
+ assert.equal(fallback.mode, "text");
+ assert.equal(fallback.fallback, "syntax");
+ assert.equal(fallback.units.map((unit) => sourceForUnit(snapshot, unit)).join(""), nested);
+});
+
+test("Python nested beyond the query depth falls back quickly while wide files parse", async () => {
+ const { sourceForUnit } = await import("../../packages/core/src/source.ts");
+ // One level per `+`: past 65,535 levels Tree-sitter queries miss matches and slow down sharply.
+ const deep = "def f():\n return 1\nvalue = " + "1+".repeat(100_000) + "1\n";
+ const snapshot = { path: "generated.py", source: deep, contentHash: "fixture" };
+ const started = performance.now();
+ const result = await inspect(snapshot, { maxUnitBytes: 3000 });
+ assert.ok(performance.now() - started < 10_000, "deep source was not rejected quickly");
+ assert.equal(result.mode, "text");
+ assert.equal(result.fallback, "syntax");
+ assert.equal(result.units.map((unit) => sourceForUnit(snapshot, unit)).join(""), deep);
+ // Many statements or elements make a wide tree, not a deep one.
+ const wide =
+ "def f():\n return 1\n" +
+ Array.from({ length: 30_000 }, (_, i) => `x${i} = [${i}, ${i}]`).join("\n") +
+ "\n";
+ const parsed = await inspect({ path: "wide.py", source: wide, contentHash: "fixture" });
+ assert.equal(parsed.mode, "python");
+ assert.equal(parsed.units[0]?.name, "f");
+});
diff --git a/test/retrieval.test.ts b/test/retrieval.test.ts
index f0ff617..3033336 100644
--- a/test/retrieval.test.ts
+++ b/test/retrieval.test.ts
@@ -615,3 +615,46 @@ testIfDocker(
}
},
);
+
+testIfDocker(
+ "unparseable generated files do not fail the search or other files' parsing",
+ async () => {
+ const root = await mkdtemp(join(tmpdir(), "jg-unparseable-"));
+ try {
+ await mkdir(join(root, "src"), { recursive: true });
+ await writeFile(
+ join(root, "src/nested.ts"),
+ "export const x = " + "(".repeat(20_000) + "1" + ")".repeat(20_000) + ";\n",
+ );
+ await writeFile(join(root, "src/generated.py"), "value = " + "1+".repeat(100_000) + "1\n");
+ await writeFile(
+ join(root, "src/events.py"),
+ "class Events:\n def record(self, name):\n return name\n",
+ );
+ await writeFile(
+ join(root, "src/events.ts"),
+ "export class Events {\n record(name: string) {\n return name;\n }\n}\n",
+ );
+ const result = await retrieve(
+ { root, query: "event recording", signal: new AbortController().signal },
+ {
+ requests: 0,
+ async evaluate(request) {
+ return Object.fromEntries(Object.keys(request.questions).map((id) => [id, 0.9]));
+ },
+ },
+ );
+ expect(result.status).toBe("complete");
+ const leads = (path: string) =>
+ result.files.find((file) => file.path === path)?.leads.map((lead) => lead.name);
+ // Healthy files keep declaration parsing; unparseable ones fall back to text.
+ expect(leads("src/events.py")).toContain("Events.record");
+ expect(leads("src/events.ts")).toContain("Events.record");
+ for (const path of ["src/nested.ts", "src/generated.py"])
+ expect(result.files.find((file) => file.path === path)?.excerpts.length).toBeGreaterThan(0);
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ },
+ 120_000,
+);