Skip to content

Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.

Retain reference comment windows during TypeScript syntax fallback

David Zhangbcf32530bb6a
diff --git a/packages/core/src/source.ts b/packages/core/src/source.ts
index 2520472..43a2ba1 100644
--- a/packages/core/src/source.ts
+++ b/packages/core/src/source.ts
@@ -304,6 +304,7 @@ export async function inspect(
let units: { name: string; range: Range }[] = [],
comments: Range[] = [],
mode: Inspection["mode"];
+ let syntaxFallback = false;
if (/\.pyi?$/.test(path)) {
// A missing/incompatible packaged parser is a setup failure, never syntax fallback.
const tree = await parsePython(source);
@@ -324,8 +325,8 @@ export async function inspect(
? ts.ScriptKind.JS
: ts.ScriptKind.TS;
const file = ts.createSourceFile(path, source, ts.ScriptTarget.Latest, true, kind);
- if ((file as ts.SourceFile & { parseDiagnostics?: unknown[] }).parseDiagnostics?.length)
- return fallback("syntax");
+ syntaxFallback = !!(file as ts.SourceFile & { parseDiagnostics?: unknown[] }).parseDiagnostics
+ ?.length;
const line = (position: number) => file.getLineAndCharacterOfPosition(position).line + 1;
const add = (node: ts.Node, prefix = "") => {
const named = node as ts.Node & { name?: ts.Node };
@@ -350,7 +351,7 @@ export async function inspect(
},
});
};
- for (const statement of file.statements) add(statement);
+ if (!syntaxFallback) for (const statement of file.statements) add(statement);
const visit = (node: ts.Node) => {
for (const comment of [
...(ts.getLeadingCommentRanges(source, node.getFullStart()) ?? []),
@@ -365,6 +366,8 @@ export async function inspect(
comments = [...new Map(comments.map((r) => [`${r.startLine}:${r.endLine}`, r])).values()].sort(
(a, b) => a.startLine - b.startLine,
);
+ // Invalid declarations fall back to text, but comments still own the context-window boundaries.
+ if (syntaxFallback) return { ...fallback("syntax"), comments };
if (!units.length && source)
return {
units: textUnits(
diff --git a/test/parser/source.test.ts b/test/parser/source.test.ts
index 8763706..ddb4b7c 100644
--- a/test/parser/source.test.ts
+++ b/test/parser/source.test.ts
@@ -38,6 +38,18 @@ test("syntax errors and unsupported source fall back without losing source lines
}
});
+test("TypeScript syntax fallback retains block-comment ranges and exact source bytes", async () => {
+ const source =
+ "/*\n" + ("comment " + "x".repeat(70) + "\n").repeat(100) + "*/\nconst broken = ;\n";
+ const snapshot = { path: "broken.ts", source, contentHash: "fixture" };
+ const result = await inspect(snapshot, { maxUnitBytes: 3000 });
+ assert.equal(result.mode, "text");
+ assert.equal(result.fallback, "syntax");
+ assert.deepEqual(result.comments, [{ startLine: 1, endLine: 102 }]);
+ const { sourceForUnit } = await import("../../packages/core/src/source.ts");
+ assert.equal(result.units.map((unit) => sourceForUnit(snapshot, unit)).join(""), source);
+});
+
test("oversized Unicode lines are losslessly split into bounded byte spans", async () => {
const source = "é漢🙂".repeat(10) + "\r\nend";
const result = await inspect(
diff --git a/test/reference/fallback-comments.test.ts b/test/reference/fallback-comments.test.ts
new file mode 100644
index 0000000..45a1730
--- /dev/null
+++ b/test/reference/fallback-comments.test.ts
@@ -0,0 +1,84 @@
+import { expect } from "bun:test";
+import { mkdtemp, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { testIfDocker } from "../helpers/docker";
+
+testIfDocker(
+ "syntax fallback preserves frozen comment expansion through complete CLI requests and stdout",
+ async () => {
+ const root = await mkdtemp(join(tmpdir(), "jg-fallback-comments-"));
+ const source =
+ "/*\n" + ("comment " + "x".repeat(70) + "\n").repeat(100) + "*/\nconst broken = ;\n";
+ const query = "Find source behavior";
+ const requests: string[][] = [[], []];
+ let arm = 0;
+ const server = Bun.serve({
+ port: 0,
+ async fetch(request) {
+ const body = (await request.json()) as {
+ state: { declarations?: Array<{ startLine: number }>; selectedEvidence?: unknown };
+ questions: Record<string, unknown>;
+ };
+ requests[arm]!.push(JSON.stringify(body));
+ return Response.json({
+ answers: Object.fromEntries(
+ Object.keys(body.questions).map((id, index) => [
+ id,
+ {
+ type: "boolean",
+ probability: body.state.declarations
+ ? !body.state.selectedEvidence && body.state.declarations[index]?.startLine === 1
+ ? 0.9
+ : 0
+ : 0.9,
+ },
+ ]),
+ ),
+ });
+ },
+ });
+ try {
+ await writeFile(join(root, "source.ts"), source);
+ const outputs: string[] = [];
+ for (arm = 0; arm < 2; arm++) {
+ const child = Bun.spawn(
+ arm === 0
+ ? ["node", "/opt/jevgrep-reference.mjs", "--root", root, "--query", query]
+ : [
+ "node",
+ process.env.JEVGREP_PARITY_CLI ?? resolve("apps/cli/dist/bin/index.js"),
+ query,
+ root,
+ "--no-cache",
+ ],
+ {
+ env: {
+ PATH: process.env.PATH,
+ AI_GATEWAY_API_KEY: "fixture",
+ AI_GATEWAY_BASE_URL: `http://127.0.0.1:${server.port}/v4/ai`,
+ },
+ stdout: "pipe",
+ stderr: "pipe",
+ },
+ );
+ const [code, stdout, stderr] = await Promise.all([
+ child.exited,
+ new Response(child.stdout).text(),
+ new Response(child.stderr).text(),
+ ]);
+ expect(code, stdout).toBe(0);
+ expect(stderr).toBe("");
+ outputs.push(stdout);
+ }
+ expect(outputs[0]).toContain('Source block "source.ts" lines 1-104:');
+ expect(outputs[0]).toContain("103: const broken = ;");
+ expect(requests[1]!.sort()).toEqual(requests[0]!.sort());
+ expect(outputs[1]).toBe(outputs[0]);
+ } finally {
+ server.stop(true);
+ await rm(root, { recursive: true, force: true });
+ }
+ },
+ 120_000,
+);