Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.
TypeScript ai-sdkclaude-codeclicode-searchcodexcoding-agentscontext-retrievaldeveloper-toolsjevsemantic-searchtypescriptvercel-ai-gateway
Contain deeply nested source to the file that contains it
taeskimfdb85e6ad2e9
diff --git a/docs/architecture.md b/docs/architecture.mdindex 5828153..574c500 100644--- a/docs/architecture.md+++ b/docs/architecture.md@@ -60,7 +60,10 @@ source bytes; never reconstruct returned code from the tree.Tree-sitter recognizes syntax rather than validating CPython semantics. Treeswith syntax errors use text fallback. Bare-CR Python also uses text fallback-because retrieval coordinates count LF lines. Repository source is never executed.+because retrieval coordinates count LF lines. So does generated source nested+beyond what a parser handles: TypeScript that exhausts the call stack, and Python+nested more than 50,000 levels, near where Tree-sitter queries stop matching and+slow down sharply. Repository source is never executed.See [parser contracts](../test/parser/README.md) and[measurement evidence](../specs/tree-sitter/RESULTS.md).diff --git a/packages/core/src/parser-helpers.mjs b/packages/core/src/parser-helpers.mjsindex 28bd880..e7f6f1e 100644--- a/packages/core/src/parser-helpers.mjs+++ b/packages/core/src/parser-helpers.mjs@@ -23,6 +23,10 @@ async function parse(source, action, languageName = "python") {parser.setLanguage(languages.get(languageName));tree = parser.parse(source);if (!tree || tree.rootNode.hasError) return null;+ // Tree-sitter queries stop reporting matches beyond 65,535 levels of nesting and slow down+ // sharply near that depth: 70,000 chained `+` took 8 s, 100,000 took 74 s. Treat such+ // generated source as unparseable, like a syntax error, before any query runs.+ if (languageName === "python" && deeperThan(tree, maxTreeDepth)) return null;if (languageName === "python" && !validPython(tree.rootNode)) return null;return action(tree.rootNode);} finally {@@ -31,6 +35,27 @@ async function parse(source, action, languageName = "python") {}}const field = (n, name) => n.childForFieldName(name);+// Well below the query engine's 65,535-level limit; ordinary source nests far less.+const maxTreeDepth = 50_000;+/** Visits every node with a tree cursor, which neither recurses nor allocates node objects. */+function deeperThan(tree, limit) {+ const cursor = tree.walk();+ try {+ let depth = 0;+ for (;;) {+ if (cursor.gotoFirstChild()) {+ if (++depth > limit) return true;+ continue;+ }+ while (!cursor.gotoNextSibling()) {+ if (!cursor.gotoParent()) return false;+ depth--;+ }+ }+ } finally {+ cursor.delete();+ }+}function unparenthesized(node) {while (node?.type === "parenthesized_expression")node = node.namedChildren.find((child) => child.type !== "comment");diff --git a/packages/core/src/source.ts b/packages/core/src/source.tsindex 47a9e17..9ddbb7c 100644--- a/packages/core/src/source.ts+++ b/packages/core/src/source.ts@@ -146,13 +146,14 @@ export async function inspect(comments = pythonComments;mode = "python";} else if (language === "TS" || language === "TSX" || language === "JS" || language === "JSX") {- const file = ts.createSourceFile(- path,- source,- ts.ScriptTarget.Latest,- true,- ts.ScriptKind[language],- );+ let file: ts.SourceFile;+ try {+ file = ts.createSourceFile(path, source, ts.ScriptTarget.Latest, true, ts.ScriptKind[language]);+ } catch (error) {+ // Deeply nested source can exhaust the parser's call stack: unparseable here, like a syntax error.+ if (error instanceof RangeError) return fallback("syntax");+ throw error;+ }syntaxFallback = !!(file as ts.SourceFile & { parseDiagnostics?: unknown[] }).parseDiagnostics?.length;const line = (position: number) => file.getLineAndCharacterOfPosition(position).line + 1;@@ -182,15 +183,23 @@ export async function inspect(});};if (!syntaxFallback) for (const statement of file.statements) add(statement);- const visit = (node: ts.Node) => {+ // Pre-order walk with an explicit stack: long generated expressions nest deeper than the call+ // stack allows. Children are pushed in reverse so they are visited in source order.+ const walk: ts.Node[] = [file];+ while (walk.length) {+ const node = walk.pop()!;for (const comment of [...(ts.getLeadingCommentRanges(source, node.getFullStart()) ?? []),...(ts.getTrailingCommentRanges(source, node.end) ?? []),])comments.push({ startLine: line(comment.pos), endLine: line(comment.end - 1) });- ts.forEachChild(node, visit);- };- visit(file);+ const children: ts.Node[] = [];+ // The callback must return undefined: a truthy result stops forEachChild early.+ ts.forEachChild(node, (child) => {+ children.push(child);+ });+ for (let index = children.length - 1; index >= 0; index--) walk.push(children[index]!);+ }mode = "typescript";} else if (language === "go" || language === "rust") {const parsed = await runParser<{ units: typeof units; comments: Range[] }>(diff --git a/test/parser/README.md b/test/parser/README.mdindex d52e1dc..bbe3f1e 100644--- a/test/parser/README.md+++ b/test/parser/README.md@@ -2,6 +2,8 @@Tests cover original source coordinates, Unicode boundaries, declaration owners,syntax fallback and cancellation through the production inspection API. Unsupported languages retain lossless text fallback.+Generated source nested too deeply for a parser also falls back to text, promptly+and without affecting other files; wide files with many statements still parse.Python helpers in `reference/` are frozen test-only oracles from the replacedruntime. Differential fixtures compare declaration, preview, neighbourhood anddiff --git a/test/parser/source.test.ts b/test/parser/source.test.tsindex 01f7e85..a4db84b 100644--- a/test/parser/source.test.ts+++ b/test/parser/source.test.ts@@ -372,3 +372,46 @@ test("same-named classes retain their own structural headers", async () => {]);}});++test("generated TypeScript deeper than the call stack parses or falls back to text", async () => {+ const { sourceForUnit } = await import("../../packages/core/src/source.ts");+ // A long flat expression nests one level per operator in the syntax tree.+ const chain = "// generated\nexport const table = " + "1 + ".repeat(50_000) + "1;\n";+ const parsed = await inspect(+ { path: "chain.ts", source: chain, contentHash: "fixture" },+ { maxUnitBytes: Buffer.byteLength(chain) },+ );+ assert.equal(parsed.mode, "typescript");+ assert.deepEqual(+ parsed.units.map(({ name, range }) => ({ name, ...range })),+ [{ name: "table", startLine: 2, endLine: 2 }],+ );+ assert.deepEqual(parsed.comments, [{ startLine: 1, endLine: 1 }]);+ const nested = "export const x = " + "(".repeat(20_000) + "1" + ")".repeat(20_000) + ";\n";+ const snapshot = { path: "nested.ts", source: nested, contentHash: "fixture" };+ const fallback = await inspect(snapshot, { maxUnitBytes: 3000 });+ assert.equal(fallback.mode, "text");+ assert.equal(fallback.fallback, "syntax");+ assert.equal(fallback.units.map((unit) => sourceForUnit(snapshot, unit)).join(""), nested);+});++test("Python nested beyond the query depth falls back quickly while wide files parse", async () => {+ const { sourceForUnit } = await import("../../packages/core/src/source.ts");+ // One level per `+`: past 65,535 levels Tree-sitter queries miss matches and slow down sharply.+ const deep = "def f():\n return 1\nvalue = " + "1+".repeat(100_000) + "1\n";+ const snapshot = { path: "generated.py", source: deep, contentHash: "fixture" };+ const started = performance.now();+ const result = await inspect(snapshot, { maxUnitBytes: 3000 });+ assert.ok(performance.now() - started < 10_000, "deep source was not rejected quickly");+ assert.equal(result.mode, "text");+ assert.equal(result.fallback, "syntax");+ assert.equal(result.units.map((unit) => sourceForUnit(snapshot, unit)).join(""), deep);+ // Many statements or elements make a wide tree, not a deep one.+ const wide =+ "def f():\n return 1\n" ++ Array.from({ length: 30_000 }, (_, i) => `x${i} = [${i}, ${i}]`).join("\n") ++ "\n";+ const parsed = await inspect({ path: "wide.py", source: wide, contentHash: "fixture" });+ assert.equal(parsed.mode, "python");+ assert.equal(parsed.units[0]?.name, "f");+});diff --git a/test/retrieval.test.ts b/test/retrieval.test.tsindex f0ff617..3033336 100644--- a/test/retrieval.test.ts+++ b/test/retrieval.test.ts@@ -615,3 +615,46 @@ testIfDocker(}},);++testIfDocker(+ "unparseable generated files do not fail the search or other files' parsing",+ async () => {+ const root = await mkdtemp(join(tmpdir(), "jg-unparseable-"));+ try {+ await mkdir(join(root, "src"), { recursive: true });+ await writeFile(+ join(root, "src/nested.ts"),+ "export const x = " + "(".repeat(20_000) + "1" + ")".repeat(20_000) + ";\n",+ );+ await writeFile(join(root, "src/generated.py"), "value = " + "1+".repeat(100_000) + "1\n");+ await writeFile(+ join(root, "src/events.py"),+ "class Events:\n def record(self, name):\n return name\n",+ );+ await writeFile(+ join(root, "src/events.ts"),+ "export class Events {\n record(name: string) {\n return name;\n }\n}\n",+ );+ const result = await retrieve(+ { root, query: "event recording", signal: new AbortController().signal },+ {+ requests: 0,+ async evaluate(request) {+ return Object.fromEntries(Object.keys(request.questions).map((id) => [id, 0.9]));+ },+ },+ );+ expect(result.status).toBe("complete");+ const leads = (path: string) =>+ result.files.find((file) => file.path === path)?.leads.map((lead) => lead.name);+ // Healthy files keep declaration parsing; unparseable ones fall back to text.+ expect(leads("src/events.py")).toContain("Events.record");+ expect(leads("src/events.ts")).toContain("Events.record");+ for (const path of ["src/nested.ts", "src/generated.py"])+ expect(result.files.find((file) => file.path === path)?.excerpts.length).toBeGreaterThan(0);+ } finally {+ await rm(root, { recursive: true, force: true });+ }+ },+ 120_000,+);