Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.
TypeScript ai-sdkclaude-codeclicode-searchcodexcoding-agentscontext-retrievaldeveloper-toolsjevsemantic-searchtypescriptvercel-ai-gateway
Match reference Python syntax fallback and TypeScript comment traversal
David Zhangffa5d7c1539c
diff --git a/packages/core/src/source.ts b/packages/core/src/source.tsindex c7188a8..2520472 100644--- a/packages/core/src/source.ts+++ b/packages/core/src/source.ts@@ -37,6 +37,66 @@ type Declaration = {parent?: Declaration;};let pythonLanguage: Promise<Language> | undefined;+function outsideReferenceSyntax(node: SyntaxNode): boolean {+ switch (node.type) {+ case "type_alias_statement":+ return true;+ case "function_definition":+ case "class_definition":+ return node.childForFieldName("type_parameters") !== null;+ case "interpolation":+ case "format_expression":+ case "format_specifier": {+ let owner = node.parent;+ while (owner && owner.type !== "string") owner = owner.parent;+ const start = owner+ ? (children(owner).find((child) => child.type === "string_start")?.text ?? "")+ : "";+ const delimiter = /^[a-z]*("""|'''|"|')/i.exec(start)?.[1];+ if (!/f/i.test(start) || !delimiter) return false;+ // Literal format text permits escaped line continuations; expression text never does.+ if (node.type === "format_specifier")+ return delimiter.length === 1 && /(?:^|[^\\])(?:\\\\)*\r?\n/.test(node.text);+ const expression = node.childForFieldName("expression")?.text ?? "";+ return (+ expression.includes(delimiter) ||+ expression.includes("\\") ||+ (delimiter.length === 1 && /[\r\n]/.test(expression))+ );+ }+ case "comment": {+ for (let parent = node.parent; parent; parent = parent.parent) {+ if (parent.type === "interpolation") return true;+ if (parent.type === "string") break;+ }+ return false;+ }+ case "print_statement":+ // A redirection-shaped print is also a valid Python 3 shift/tuple expression.+ return !children(node).some((child) => child.type === "chevron");+ case "exec_statement":+ return true;+ case "raise_statement":+ return children(node).some((child) => child.type === "expression_list");+ case "except_clause":+ return node.children.some((child) => child?.type === ",");+ case "comparison_operator":+ return node.children.some((child) => child?.type === "<>");+ case "integer":+ return /[lL]$/.test(node.text) || /^0[0-9_]*[1-9][0-9_]*$/.test(node.text);+ case "string_start":+ return node.text.startsWith("`") || /^(?:ur|ru)["']/i.test(node.text);+ case "tuple_pattern":+ case "list_pattern": {+ let parent = node.parent;+ while (parent && ["tuple_pattern", "list_pattern", "default_parameter"].includes(parent.type))+ parent = parent.parent;+ return parent?.type === "parameters" || parent?.type === "lambda_parameters";+ }+ default:+ return false;+ }+}async function parsePython(source: string) {pythonLanguage ??= (async () => {await Parser.init({@@ -59,7 +119,12 @@ async function parsePython(source: string) {while (!invalid && pending.length) {const node = pending.pop()!;const nodes = children(node);- if (node.type === "block" && !nodes.some((child) => child.type !== "comment")) invalid = true;+ // The grammar spans multiple Python versions; consumers share the frozen Python 3.11 boundary.+ if (+ outsideReferenceSyntax(node) ||+ (node.type === "block" && !nodes.some((child) => child.type !== "comment"))+ )+ invalid = true;else pending.push(...nodes);}if (invalid) {@@ -292,7 +357,7 @@ export async function inspect(...(ts.getTrailingCommentRanges(source, node.end) ?? []),])comments.push({ startLine: line(comment.pos), endLine: line(comment.end - 1) });- for (const child of node.getChildren(file)) visit(child);+ ts.forEachChild(node, visit);};visit(file);mode = "typescript";diff --git a/test/parser/README.md b/test/parser/README.mdindex 555cd62..faba469 100644--- a/test/parser/README.md+++ b/test/parser/README.md@@ -13,13 +13,24 @@ declarations, and treats function-local functions as part of the outer unit. Quepreview windows and additive neighboring-method ranges preserve the recorded Pythonpolicy. Comments stay independently available for later context expansion.-Intentional grammar differences are visible: Tree-sitter accepts Python-2-`print` syntax that the Python-3 reference rejects, and actual comment tokens include-inline comments while avoiding hashes inside strings. TypeScript comment collection-also retains comments between tokens and before closing braces. The packaged grammar is a-syntax inspector, not a Python interpreter or semantic validator. Syntax errors-reported by either language parser use text chunks. Asset loading errors are setup-failures, because silently changing behavior would hide a broken installation.+The reference language boundary is CPython 3.11: the retained Requests runtime+uses 3.11.5 and the isolated fixture oracle uses 3.11.2. The bundled grammar accepts+syntax from other versions, so one shared compatibility check rejects recognized+Python 2 forms and Python 3.12-only type parameters, type aliases and f-string+expression forms. Inspection, previews and neighboring-method selection then use+the same text fallback. Source bytes are retained; no Python executable is required+by the product. Valid reference syntax is preserved, including print-shaped shift+expressions and ordinary f-strings. The cases live in the parser tests; this is+bounded conformance coverage, not a replacement implementation of CPython's grammar.++Python context expansion preserves the reference's conservative whole-line comment+matching, including hash-prefixed lines in multiline strings. TypeScript comment+collection follows the reference AST-child traversal; it does not broaden expansion+by walking additional closing-brace and literal tokens. Syntax+errors use text chunks; asset loading errors are setup failures rather than silent+behavior changes. The whole-CLI HTTP comparison in+[the Python parity test](../reference/python-parity.test.ts) checks computed previews,+fallback questions, follow-up evidence and stdout against the actual frozen bundle.Units carry byte spans into the same immutable snapshot as their one-based lineranges. Consumers use `sourceForUnit` for request text: widening a partial unit backdiff --git a/test/parser/source.test.ts b/test/parser/source.test.tsindex a48c725..8763706 100644--- a/test/parser/source.test.ts+++ b/test/parser/source.test.ts@@ -78,7 +78,7 @@ test("query previews preserve frozen Python source windows and bytes", async ()const corpus = JSON.parse(await readFile(new URL("./fixtures/python-reference.json", import.meta.url), "utf8"),);- for (const fixture of corpus.cases.filter((f) => f.name !== "python2")) {+ for (const fixture of corpus.cases) {const preview = await pythonPreview({ path: fixture.path, source: fixture.source, contentHash: "fixture" },fixture.query,@@ -128,14 +128,82 @@ test("TS/JS uses original source coordinates, comments, and decorator-bearing me]);});-test("Python 2 syntax accepted by the packaged grammar is an explicit parser difference", async () => {- const source = 'def target():\n print "old"\n';- const result = await inspect({ path: "old.py", source, contentHash: "fixture" });- assert.equal(result.mode, "python");- assert.deepEqual(- result.units.map(({ name, range }) => ({ name, ...range })),- [{ name: "target", startLine: 1, endLine: 2 }],- );+test("syntax outside reference Python 3.11 falls back across inspection, preview and neighbors", async () => {+ const { pythonNeighborhood, pythonPreview, sourceForUnit } =+ await import("../../packages/core/src/source.ts");+ for (const source of [+ 'def target():\n print "old"\n',+ 'def target():\n exec "x=1" in globals(), locals()\n',+ "try:\n pass\nexcept Exception, e:\n pass\n",+ 'raise Error, "message", traceback\n',+ "value = left <> right\n",+ "value = 123L\n",+ "value = 0xFFl\n",+ "value = 0755\n",+ "value = 0_1\n",+ "value = `expression`\n",+ 'value = ur"text"\n',+ 'value = rU"text"\n',+ "def target((first, second)):\n return first\n",+ "def target((first, second)=(1, 2)):\n return first\n",+ "value = lambda (first, second): first\n",+ "type Alias = int\n",+ "def target[T](value: T):\n return value\n",+ 'value = f"{mapping["key"]}"\n',+ "value = f\"{'\\n'}\"\n",+ 'value = f"""{value # comment\n}"""\n',+ 'value = f"{(value +\n other)}"\n',+ "value = f'{1:{mapping['width']}}'\n",+ "value = f'{1:{\"\\n\"}}'\n",+ ]) {+ const snapshot = {+ path: "old.py",+ source: source + "# padding\n".repeat(60),+ contentHash: "fixture",+ };+ const result = await inspect(snapshot);+ assert.equal(result.mode, "text", source);+ assert.equal(result.fallback, "syntax", source);+ assert.equal(+ result.units.map((unit) => sourceForUnit(snapshot, unit)).join(""),+ snapshot.source,+ );+ assert.deepEqual(+ await pythonNeighborhood(snapshot, [{ startLine: 1, endLine: 2 }]),+ [],+ source,+ );+ const preview = await pythonPreview(snapshot, "target", 512);+ assert.equal(preview.parseUnavailable, true, source);+ assert.equal(preview.matchedDeclarations, 0, source);+ }+});++test("Python 3 equivalents and Python 2-looking strings remain parsed", async () => {+ for (const source of [+ 'print("old")\n',+ "print >> stream, value\n",+ "print +1\nprint [1,2]\n",+ "value = f\"{mapping['key']}\"\n",+ 'value = f"""{mapping["key"]}"""\n',+ 'value = f"""{(value +\n other)}"""\n',+ "value = f\"{'#value'}\"\n",+ "value = f'{1:\\\n}'\n",+ "type = int\ndef target(value: int):\n return value\n",+ 'exec("x=1", globals(), locals())\n',+ "try:\n pass\nexcept Exception as e:\n pass\n",+ "try:\n pass\nexcept (TypeError, ValueError):\n pass\n",+ 'raise Error("message") from cause\n',+ "raise (Error, value)\n",+ "value = 0o755 + 0xFF + 0b11 + 000 + 0_0\n",+ 'value = u"text" + r"raw" + rb"bytes".decode()\n',+ "def target(value=[a for (a,b) in pairs]):\n return value\n",+ "for (a,b) in pairs:\n pass\n",+ '# print "old"; exec "x"; 123L\ntext = "ur\\\"text\\\" <> `value`"\n',+ ]) {+ const result = await inspect({ path: "modern.py", source, contentHash: "fixture" });+ assert.equal(result.mode, "python", source);+ }});test("trailing Python comments stay separate from AST declaration ends", async () => {@@ -226,12 +294,16 @@ test("empty Python suites fall back instead of crashing query previews", async (assert.equal((await inspect(snapshot)).mode, "text");});-test("TS comments survive token positions without treating string contents as comments", async () => {+test("TS comment discovery follows frozen AST nodes while retaining exact source bytes", async () => {const source ='function x() {\n // end\n}\nconst x = /*important*/ 1;\nconst url="https://example.test";\nconst regexp=/https?:\\/\\//;\n';- const result = await inspect({ path: "comments.ts", source, contentHash: "fixture" });- assert.deepEqual(result.comments, [- { startLine: 2, endLine: 2 },- { startLine: 4, endLine: 4 },- ]);+ const snapshot = { path: "comments.ts", source, contentHash: "fixture" };+ const result = await inspect(snapshot);+ // The frozen forEachChild traversal does not visit the closing-brace or numeric-literal token.+ assert.deepEqual(result.comments, []);+ const { sourceForUnit } = await import("../../packages/core/src/source.ts");+ assert.equal(+ sourceForUnit(snapshot, result.units[0]!),+ source.split("\n").slice(0, 3).join("\n") + "\n",+ );});diff --git a/test/reference/python-parity.test.ts b/test/reference/python-parity.test.tsnew file mode 100644index 0000000..027624a--- /dev/null+++ b/test/reference/python-parity.test.ts@@ -0,0 +1,81 @@+import { expect } from "bun:test";+import { mkdtemp, rm, writeFile } from "node:fs/promises";+import { tmpdir } from "node:os";+import { join, resolve } from "node:path";+import { testIfDocker } from "../helpers/docker";++for (const [name, source] of [+ ["Python 2 fallback", 'def target():\n print "old"\n' + "# padding\n".repeat(2100)],+ ["Python 3.12 type alias fallback", "type Alias = int\n" + "# padding\n".repeat(2100)],+ [+ "Python 3.12 generic fallback",+ "def target[T](value: T):\n return value\n" + "# padding\n".repeat(2100),+ ],+ ["Python 3.12 f-string fallback", 'value = f"{mapping["key"]}"\n' + "# padding\n".repeat(2100)],+ [+ "oversized Python declaration",+ "def target():\n" ++ Array.from({ length: 80 }, (_, i) => ` value${i} = '${"x".repeat(400)}'\n`).join(""),+ ],+])+ testIfDocker(+ `${name} matches the frozen whole-CLI HTTP requests and stdout`,+ async () => {+ const root = await mkdtemp(join(tmpdir(), "jg-python2-parity-"));+ const query = "Find target behavior";+ const requests: string[][] = [[], []];+ let arm = 0;+ const server = Bun.serve({+ port: 0,+ async fetch(request) {+ const body = (await request.json()) as { questions: Record<string, unknown> };+ requests[arm]!.push(JSON.stringify(body));+ return Response.json({+ answers: Object.fromEntries(+ Object.keys(body.questions).map((id) => [id, { type: "boolean", probability: 0.9 }]),+ ),+ });+ },+ });+ try {+ await writeFile(join(root, "source.py"), source!);+ const outputs: string[] = [];+ for (arm = 0; arm < 2; arm++) {+ const child = Bun.spawn(+ arm === 0+ ? ["node", "/opt/jevgrep-reference.mjs", "--root", root, "--query", query]+ : [+ "node",+ process.env.JEVGREP_PARITY_CLI ?? resolve("apps/cli/dist/bin/index.js"),+ query,+ root,+ "--no-cache",+ ],+ {+ env: {+ PATH: process.env.PATH,+ AI_GATEWAY_API_KEY: "fixture",+ AI_GATEWAY_BASE_URL: `http://127.0.0.1:${server.port}/v4/ai`,+ },+ stdout: "pipe",+ stderr: "pipe",+ },+ );+ const [code, stdout, stderr] = await Promise.all([+ child.exited,+ new Response(child.stdout).text(),+ new Response(child.stderr).text(),+ ]);+ expect(code, stdout).toBe(0);+ expect(stderr).toBe("");+ outputs.push(stdout);+ }+ expect(requests[1]!.sort()).toEqual(requests[0]!.sort());+ expect(outputs[1]).toBe(outputs[0]);+ } finally {+ server.stop(true);+ await rm(root, { recursive: true, force: true });+ }+ },+ 120_000,+ );diff --git a/test/reference/selection.test.ts b/test/reference/selection.test.tsindex 708f257..0190c77 100644--- a/test/reference/selection.test.ts+++ b/test/reference/selection.test.ts@@ -88,3 +88,8 @@ test("follow-up reading leads retain first-seen order across passes", async () =d.name === "target" ? 0.9 : pass === 1 ? 0.4 : 0,);});++test("Python 2 source preserves frozen fallback questions instead of named declarations", async () => {+ const source = 'def target():\n print "old"\n' + "# padding\n".repeat(600);+ await expectParity("old.py", source, () => 0.9);+});