Skip to content

Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.

Match reference Python syntax fallback and TypeScript comment traversal

David Zhangffa5d7c1539c
diff --git a/packages/core/src/source.ts b/packages/core/src/source.ts
index c7188a8..2520472 100644
--- a/packages/core/src/source.ts
+++ b/packages/core/src/source.ts
@@ -37,6 +37,66 @@ type Declaration = {
parent?: Declaration;
};
let pythonLanguage: Promise<Language> | undefined;
+function outsideReferenceSyntax(node: SyntaxNode): boolean {
+ switch (node.type) {
+ case "type_alias_statement":
+ return true;
+ case "function_definition":
+ case "class_definition":
+ return node.childForFieldName("type_parameters") !== null;
+ case "interpolation":
+ case "format_expression":
+ case "format_specifier": {
+ let owner = node.parent;
+ while (owner && owner.type !== "string") owner = owner.parent;
+ const start = owner
+ ? (children(owner).find((child) => child.type === "string_start")?.text ?? "")
+ : "";
+ const delimiter = /^[a-z]*("""|'''|"|')/i.exec(start)?.[1];
+ if (!/f/i.test(start) || !delimiter) return false;
+ // Literal format text permits escaped line continuations; expression text never does.
+ if (node.type === "format_specifier")
+ return delimiter.length === 1 && /(?:^|[^\\])(?:\\\\)*\r?\n/.test(node.text);
+ const expression = node.childForFieldName("expression")?.text ?? "";
+ return (
+ expression.includes(delimiter) ||
+ expression.includes("\\") ||
+ (delimiter.length === 1 && /[\r\n]/.test(expression))
+ );
+ }
+ case "comment": {
+ for (let parent = node.parent; parent; parent = parent.parent) {
+ if (parent.type === "interpolation") return true;
+ if (parent.type === "string") break;
+ }
+ return false;
+ }
+ case "print_statement":
+ // A redirection-shaped print is also a valid Python 3 shift/tuple expression.
+ return !children(node).some((child) => child.type === "chevron");
+ case "exec_statement":
+ return true;
+ case "raise_statement":
+ return children(node).some((child) => child.type === "expression_list");
+ case "except_clause":
+ return node.children.some((child) => child?.type === ",");
+ case "comparison_operator":
+ return node.children.some((child) => child?.type === "<>");
+ case "integer":
+ return /[lL]$/.test(node.text) || /^0[0-9_]*[1-9][0-9_]*$/.test(node.text);
+ case "string_start":
+ return node.text.startsWith("`") || /^(?:ur|ru)["']/i.test(node.text);
+ case "tuple_pattern":
+ case "list_pattern": {
+ let parent = node.parent;
+ while (parent && ["tuple_pattern", "list_pattern", "default_parameter"].includes(parent.type))
+ parent = parent.parent;
+ return parent?.type === "parameters" || parent?.type === "lambda_parameters";
+ }
+ default:
+ return false;
+ }
+}
async function parsePython(source: string) {
pythonLanguage ??= (async () => {
await Parser.init({
@@ -59,7 +119,12 @@ async function parsePython(source: string) {
while (!invalid && pending.length) {
const node = pending.pop()!;
const nodes = children(node);
- if (node.type === "block" && !nodes.some((child) => child.type !== "comment")) invalid = true;
+ // The grammar spans multiple Python versions; consumers share the frozen Python 3.11 boundary.
+ if (
+ outsideReferenceSyntax(node) ||
+ (node.type === "block" && !nodes.some((child) => child.type !== "comment"))
+ )
+ invalid = true;
else pending.push(...nodes);
}
if (invalid) {
@@ -292,7 +357,7 @@ export async function inspect(
...(ts.getTrailingCommentRanges(source, node.end) ?? []),
])
comments.push({ startLine: line(comment.pos), endLine: line(comment.end - 1) });
- for (const child of node.getChildren(file)) visit(child);
+ ts.forEachChild(node, visit);
};
visit(file);
mode = "typescript";
diff --git a/test/parser/README.md b/test/parser/README.md
index 555cd62..faba469 100644
--- a/test/parser/README.md
+++ b/test/parser/README.md
@@ -13,13 +13,24 @@ declarations, and treats function-local functions as part of the outer unit. Que
preview windows and additive neighboring-method ranges preserve the recorded Python
policy. Comments stay independently available for later context expansion.
-Intentional grammar differences are visible: Tree-sitter accepts Python-2
-`print` syntax that the Python-3 reference rejects, and actual comment tokens include
-inline comments while avoiding hashes inside strings. TypeScript comment collection
-also retains comments between tokens and before closing braces. The packaged grammar is a
-syntax inspector, not a Python interpreter or semantic validator. Syntax errors
-reported by either language parser use text chunks. Asset loading errors are setup
-failures, because silently changing behavior would hide a broken installation.
+The reference language boundary is CPython 3.11: the retained Requests runtime
+uses 3.11.5 and the isolated fixture oracle uses 3.11.2. The bundled grammar accepts
+syntax from other versions, so one shared compatibility check rejects recognized
+Python 2 forms and Python 3.12-only type parameters, type aliases and f-string
+expression forms. Inspection, previews and neighboring-method selection then use
+the same text fallback. Source bytes are retained; no Python executable is required
+by the product. Valid reference syntax is preserved, including print-shaped shift
+expressions and ordinary f-strings. The cases live in the parser tests; this is
+bounded conformance coverage, not a replacement implementation of CPython's grammar.
+
+Python context expansion preserves the reference's conservative whole-line comment
+matching, including hash-prefixed lines in multiline strings. TypeScript comment
+collection follows the reference AST-child traversal; it does not broaden expansion
+by walking additional closing-brace and literal tokens. Syntax
+errors use text chunks; asset loading errors are setup failures rather than silent
+behavior changes. The whole-CLI HTTP comparison in
+[the Python parity test](../reference/python-parity.test.ts) checks computed previews,
+fallback questions, follow-up evidence and stdout against the actual frozen bundle.
Units carry byte spans into the same immutable snapshot as their one-based line
ranges. Consumers use `sourceForUnit` for request text: widening a partial unit back
diff --git a/test/parser/source.test.ts b/test/parser/source.test.ts
index a48c725..8763706 100644
--- a/test/parser/source.test.ts
+++ b/test/parser/source.test.ts
@@ -78,7 +78,7 @@ test("query previews preserve frozen Python source windows and bytes", async ()
const corpus = JSON.parse(
await readFile(new URL("./fixtures/python-reference.json", import.meta.url), "utf8"),
);
- for (const fixture of corpus.cases.filter((f) => f.name !== "python2")) {
+ for (const fixture of corpus.cases) {
const preview = await pythonPreview(
{ path: fixture.path, source: fixture.source, contentHash: "fixture" },
fixture.query,
@@ -128,14 +128,82 @@ test("TS/JS uses original source coordinates, comments, and decorator-bearing me
]);
});
-test("Python 2 syntax accepted by the packaged grammar is an explicit parser difference", async () => {
- const source = 'def target():\n print "old"\n';
- const result = await inspect({ path: "old.py", source, contentHash: "fixture" });
- assert.equal(result.mode, "python");
- assert.deepEqual(
- result.units.map(({ name, range }) => ({ name, ...range })),
- [{ name: "target", startLine: 1, endLine: 2 }],
- );
+test("syntax outside reference Python 3.11 falls back across inspection, preview and neighbors", async () => {
+ const { pythonNeighborhood, pythonPreview, sourceForUnit } =
+ await import("../../packages/core/src/source.ts");
+ for (const source of [
+ 'def target():\n print "old"\n',
+ 'def target():\n exec "x=1" in globals(), locals()\n',
+ "try:\n pass\nexcept Exception, e:\n pass\n",
+ 'raise Error, "message", traceback\n',
+ "value = left <> right\n",
+ "value = 123L\n",
+ "value = 0xFFl\n",
+ "value = 0755\n",
+ "value = 0_1\n",
+ "value = `expression`\n",
+ 'value = ur"text"\n',
+ 'value = rU"text"\n',
+ "def target((first, second)):\n return first\n",
+ "def target((first, second)=(1, 2)):\n return first\n",
+ "value = lambda (first, second): first\n",
+ "type Alias = int\n",
+ "def target[T](value: T):\n return value\n",
+ 'value = f"{mapping["key"]}"\n',
+ "value = f\"{'\\n'}\"\n",
+ 'value = f"""{value # comment\n}"""\n',
+ 'value = f"{(value +\n other)}"\n',
+ "value = f'{1:{mapping['width']}}'\n",
+ "value = f'{1:{\"\\n\"}}'\n",
+ ]) {
+ const snapshot = {
+ path: "old.py",
+ source: source + "# padding\n".repeat(60),
+ contentHash: "fixture",
+ };
+ const result = await inspect(snapshot);
+ assert.equal(result.mode, "text", source);
+ assert.equal(result.fallback, "syntax", source);
+ assert.equal(
+ result.units.map((unit) => sourceForUnit(snapshot, unit)).join(""),
+ snapshot.source,
+ );
+ assert.deepEqual(
+ await pythonNeighborhood(snapshot, [{ startLine: 1, endLine: 2 }]),
+ [],
+ source,
+ );
+ const preview = await pythonPreview(snapshot, "target", 512);
+ assert.equal(preview.parseUnavailable, true, source);
+ assert.equal(preview.matchedDeclarations, 0, source);
+ }
+});
+
+test("Python 3 equivalents and Python 2-looking strings remain parsed", async () => {
+ for (const source of [
+ 'print("old")\n',
+ "print >> stream, value\n",
+ "print +1\nprint [1,2]\n",
+ "value = f\"{mapping['key']}\"\n",
+ 'value = f"""{mapping["key"]}"""\n',
+ 'value = f"""{(value +\n other)}"""\n',
+ "value = f\"{'#value'}\"\n",
+ "value = f'{1:\\\n}'\n",
+ "type = int\ndef target(value: int):\n return value\n",
+ 'exec("x=1", globals(), locals())\n',
+ "try:\n pass\nexcept Exception as e:\n pass\n",
+ "try:\n pass\nexcept (TypeError, ValueError):\n pass\n",
+ 'raise Error("message") from cause\n',
+ "raise (Error, value)\n",
+ "value = 0o755 + 0xFF + 0b11 + 000 + 0_0\n",
+ 'value = u"text" + r"raw" + rb"bytes".decode()\n',
+ "def target(value=[a for (a,b) in pairs]):\n return value\n",
+ "for (a,b) in pairs:\n pass\n",
+ '# print "old"; exec "x"; 123L\ntext = "ur\\\"text\\\" <> `value`"\n',
+ ]) {
+ const result = await inspect({ path: "modern.py", source, contentHash: "fixture" });
+ assert.equal(result.mode, "python", source);
+ }
});
test("trailing Python comments stay separate from AST declaration ends", async () => {
@@ -226,12 +294,16 @@ test("empty Python suites fall back instead of crashing query previews", async (
assert.equal((await inspect(snapshot)).mode, "text");
});
-test("TS comments survive token positions without treating string contents as comments", async () => {
+test("TS comment discovery follows frozen AST nodes while retaining exact source bytes", async () => {
const source =
'function x() {\n // end\n}\nconst x = /*important*/ 1;\nconst url="https://example.test";\nconst regexp=/https?:\\/\\//;\n';
- const result = await inspect({ path: "comments.ts", source, contentHash: "fixture" });
- assert.deepEqual(result.comments, [
- { startLine: 2, endLine: 2 },
- { startLine: 4, endLine: 4 },
- ]);
+ const snapshot = { path: "comments.ts", source, contentHash: "fixture" };
+ const result = await inspect(snapshot);
+ // The frozen forEachChild traversal does not visit the closing-brace or numeric-literal token.
+ assert.deepEqual(result.comments, []);
+ const { sourceForUnit } = await import("../../packages/core/src/source.ts");
+ assert.equal(
+ sourceForUnit(snapshot, result.units[0]!),
+ source.split("\n").slice(0, 3).join("\n") + "\n",
+ );
});
diff --git a/test/reference/python-parity.test.ts b/test/reference/python-parity.test.ts
new file mode 100644
index 0000000..027624a
--- /dev/null
+++ b/test/reference/python-parity.test.ts
@@ -0,0 +1,81 @@
+import { expect } from "bun:test";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { testIfDocker } from "../helpers/docker";
+
+for (const [name, source] of [
+ ["Python 2 fallback", 'def target():\n print "old"\n' + "# padding\n".repeat(2100)],
+ ["Python 3.12 type alias fallback", "type Alias = int\n" + "# padding\n".repeat(2100)],
+ [
+ "Python 3.12 generic fallback",
+ "def target[T](value: T):\n return value\n" + "# padding\n".repeat(2100),
+ ],
+ ["Python 3.12 f-string fallback", 'value = f"{mapping["key"]}"\n' + "# padding\n".repeat(2100)],
+ [
+ "oversized Python declaration",
+ "def target():\n" +
+ Array.from({ length: 80 }, (_, i) => ` value${i} = '${"x".repeat(400)}'\n`).join(""),
+ ],
+])
+ testIfDocker(
+ `${name} matches the frozen whole-CLI HTTP requests and stdout`,
+ async () => {
+ const root = await mkdtemp(join(tmpdir(), "jg-python2-parity-"));
+ const query = "Find target behavior";
+ const requests: string[][] = [[], []];
+ let arm = 0;
+ const server = Bun.serve({
+ port: 0,
+ async fetch(request) {
+ const body = (await request.json()) as { questions: Record<string, unknown> };
+ requests[arm]!.push(JSON.stringify(body));
+ return Response.json({
+ answers: Object.fromEntries(
+ Object.keys(body.questions).map((id) => [id, { type: "boolean", probability: 0.9 }]),
+ ),
+ });
+ },
+ });
+ try {
+ await writeFile(join(root, "source.py"), source!);
+ const outputs: string[] = [];
+ for (arm = 0; arm < 2; arm++) {
+ const child = Bun.spawn(
+ arm === 0
+ ? ["node", "/opt/jevgrep-reference.mjs", "--root", root, "--query", query]
+ : [
+ "node",
+ process.env.JEVGREP_PARITY_CLI ?? resolve("apps/cli/dist/bin/index.js"),
+ query,
+ root,
+ "--no-cache",
+ ],
+ {
+ env: {
+ PATH: process.env.PATH,
+ AI_GATEWAY_API_KEY: "fixture",
+ AI_GATEWAY_BASE_URL: `http://127.0.0.1:${server.port}/v4/ai`,
+ },
+ stdout: "pipe",
+ stderr: "pipe",
+ },
+ );
+ const [code, stdout, stderr] = await Promise.all([
+ child.exited,
+ new Response(child.stdout).text(),
+ new Response(child.stderr).text(),
+ ]);
+ expect(code, stdout).toBe(0);
+ expect(stderr).toBe("");
+ outputs.push(stdout);
+ }
+ expect(requests[1]!.sort()).toEqual(requests[0]!.sort());
+ expect(outputs[1]).toBe(outputs[0]);
+ } finally {
+ server.stop(true);
+ await rm(root, { recursive: true, force: true });
+ }
+ },
+ 120_000,
+ );
diff --git a/test/reference/selection.test.ts b/test/reference/selection.test.ts
index 708f257..0190c77 100644
--- a/test/reference/selection.test.ts
+++ b/test/reference/selection.test.ts
@@ -88,3 +88,8 @@ test("follow-up reading leads retain first-seen order across passes", async () =
d.name === "target" ? 0.9 : pass === 1 ? 0.4 : 0,
);
});
+
+test("Python 2 source preserves frozen fallback questions instead of named declarations", async () => {
+ const source = 'def target():\n print "old"\n' + "# padding\n".repeat(600);
+ await expectParity("old.py", source, () => 0.9);
+});