Skip to content

Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.

Remove superlinear work on comment windows and declaration indexes

taeskimd325ee80c137
diff --git a/packages/core/src/retrieve.ts b/packages/core/src/retrieve.ts
index e584398..5ac9b96 100644
--- a/packages/core/src/retrieve.ts
+++ b/packages/core/src/retrieve.ts
@@ -353,10 +353,25 @@ export async function retrieve(input: SearchInput, evaluator: Evaluator): Promis
preview.declarations = syntax.units
.filter((unit) => !unit.partial)
.map((unit) => ({ name: unit.name, ...unit.range }));
- while (preview.declarations.length && Buffer.byteLength(JSON.stringify(preview)) > 32000) {
- preview.declarations.pop();
+ // Keep the longest prefix of declarations that fits. Serialized size grows with each entry,
+ // so a binary search finds the same prefix as removing one declaration at a time.
+ const all = preview.declarations;
+ const fits = (count: number) => {
+ preview.declarations = all.slice(0, count);
+ return Buffer.byteLength(JSON.stringify(preview)) <= 32000;
+ };
+ if (all.length && !fits(all.length)) {
+ // Removal measured every shorter prefix with the truncation flag already set.
preview.declarationIndexTruncated = true;
- }
+ let low = 0,
+ high = all.length - 1;
+ while (low < high) {
+ const middle = Math.ceil((low + high) / 2);
+ if (fits(middle)) low = middle;
+ else high = middle - 1;
+ }
+ preview.declarations = all.slice(0, low);
+ } else preview.declarations = all;
}
return preview;
}
diff --git a/packages/core/src/selection.ts b/packages/core/src/selection.ts
index 451c4c4..199262d 100644
--- a/packages/core/src/selection.ts
+++ b/packages/core/src/selection.ts
@@ -345,36 +345,43 @@ export async function selectFile(
throw error;
warn("cancelled");
}
+ // nonBlankThrough[n] counts lines 1..n that contain non-whitespace, so a gap check is O(1).
+ const nonBlankThrough = [0];
+ for (const text of lines) nonBlankThrough.push(nonBlankThrough.at(-1)! + (text.trim() ? 1 : 0));
+ const blankLines = (first: number, last: number) =>
+ last < first ||
+ nonBlankThrough[Math.min(last, lines.length)]! ===
+ nonBlankThrough[Math.min(Math.max(first, 1) - 1, lines.length)]!;
function excerptsFor(ranges: Range[], rendered: Span[]) {
const windows = ranges.map((range) => ({
startLine: Math.max(1, range.startLine - 3),
endLine: Math.min(lines.length, range.endLine + 3),
}));
for (const window of windows) {
+ // A window absorbs comments it overlaps or that border it across blank lines, until none
+ // qualify. Absorption only grows the window, so every order reaches the same closure.
+ // Ascending passes absorb chains after the window, descending passes chains before it.
+ const absorb = (comment: Range) => {
+ const qualifies =
+ (comment.startLine <= window.endLine && comment.endLine >= window.startLine) ||
+ (comment.endLine < window.startLine &&
+ blankLines(comment.endLine + 1, window.startLine - 1)) ||
+ (comment.startLine > window.endLine &&
+ blankLines(window.endLine + 1, comment.startLine - 1));
+ if (!qualifies) return false;
+ const start = Math.min(window.startLine, comment.startLine),
+ end = Math.max(window.endLine, comment.endLine);
+ if (start === window.startLine && end === window.endLine) return false;
+ window.startLine = start;
+ window.endLine = end;
+ return true;
+ };
let changed = true;
while (changed) {
changed = false;
- for (const comment of syntax.comments) {
- const before =
- comment.endLine < window.startLine &&
- lines.slice(comment.endLine, window.startLine - 1).every((line) => !line.trim());
- const after =
- comment.startLine > window.endLine &&
- lines.slice(window.endLine, comment.startLine - 1).every((line) => !line.trim());
- if (
- (comment.startLine <= window.endLine && comment.endLine >= window.startLine) ||
- before ||
- after
- ) {
- const start = Math.min(window.startLine, comment.startLine),
- end = Math.max(window.endLine, comment.endLine);
- if (start !== window.startLine || end !== window.endLine) {
- window.startLine = start;
- window.endLine = end;
- changed = true;
- }
- }
- }
+ for (const comment of syntax.comments) if (absorb(comment)) changed = true;
+ for (let index = syntax.comments.length - 1; index >= 0; index--)
+ if (absorb(syntax.comments[index]!)) changed = true;
}
let segmentStart = offsets[window.startLine - 1]!;
for (let line = window.startLine; line <= window.endLine; line++) {
diff --git a/packages/core/test/selection.test.ts b/packages/core/test/selection.test.ts
index 221fdad..da853e1 100644
--- a/packages/core/test/selection.test.ts
+++ b/packages/core/test/selection.test.ts
@@ -414,3 +414,27 @@ test("oversized class context split into blocks stays out of leads", async () =>
expect(result.file.selected.length).toBeGreaterThan(0);
expect(result.file.leads.map((lead) => lead.name)).toEqual(["Big.method"]);
});
+
+test("excerpts follow long comment blocks and stop at the first line of code", async () => {
+ for (const [path, marker, fn] of [
+ ["notes.py", "#", "def target():\n return 1\n"],
+ ["notes.ts", "//", "export function target() {\n return 1;\n}\n"],
+ ] as const) {
+ const block = Array.from({ length: 5000 }, (_, i) => `${marker} note ${i}`).join("\n");
+ const source = `${marker} unrelated\nconst_or_code = 1\n${block}\n\n${fn}\n${marker} trailing\n`;
+ const lines = source.split("\n");
+ const result = await selectFile(
+ { path, contentHash: path, source },
+ "target",
+ 0.9,
+ evaluator((d) => (d.name === "target" ? 0.9 : 0)),
+ );
+ // The whole block, separated from the function only by a blank line, joins the excerpt;
+ // the comment above the line of code does not.
+ const excerpt = result.file.excerpts[0]!;
+ expect(excerpt.range.startLine).toBe(3);
+ expect(lines[excerpt.range.startLine - 1]).toBe(`${marker} note 0`);
+ expect(excerpt.source).toContain(`${marker} trailing`);
+ expect(excerpt.source).not.toContain(`${marker} unrelated`);
+ }
+});
diff --git a/test/retrieval.test.ts b/test/retrieval.test.ts
index ea98341..282834d 100644
--- a/test/retrieval.test.ts
+++ b/test/retrieval.test.ts
@@ -746,3 +746,59 @@ for (const [layout, source, expected] of [
await rm(root, { recursive: true, force: true });
}
});
+
+testIfDocker(
+ "a large declaration index keeps its longest prefix that fits the preview",
+ async () => {
+ const root = await mkdtemp(join(tmpdir(), "jg-declaration-index-"));
+ try {
+ const count = 3000;
+ await writeFile(
+ join(root, "client.ts"),
+ Array.from(
+ { length: count },
+ (_, i) => `export function op${i}(a: number) { return a + ${i}; }\n`,
+ ).join(""),
+ );
+ const previews: Array<{ declarations?: unknown[]; declarationIndexTruncated?: boolean }> = [];
+ await retrieve(
+ { root, query: "op behavior", signal: new AbortController().signal },
+ {
+ requests: 0,
+ async evaluate(request) {
+ const state = request.state as {
+ items?: Array<{ filePreview?: (typeof previews)[number] }>;
+ };
+ for (const item of state.items ?? [])
+ if (item.filePreview) previews.push(item.filePreview);
+ return Object.fromEntries(Object.keys(request.questions).map((id) => [id, 0.1]));
+ },
+ },
+ );
+ const preview = previews[0]!;
+ const kept = preview.declarations!.length;
+ expect(preview.declarationIndexTruncated).toBe(true);
+ expect(kept).toBeGreaterThan(0);
+ expect(kept).toBeLessThan(count);
+ expect(preview.declarations).toEqual(
+ Array.from({ length: kept }, (_, i) => ({
+ name: `op${i}`,
+ startLine: i + 1,
+ endLine: i + 1,
+ })),
+ );
+ // The index keeps the longest prefix: one more declaration would not fit.
+ expect(Buffer.byteLength(JSON.stringify(preview))).toBeLessThanOrEqual(32000);
+ const longer = {
+ ...preview,
+ declarations: [
+ ...preview.declarations!,
+ { name: `op${kept}`, startLine: kept + 1, endLine: kept + 1 },
+ ],
+ };
+ expect(Buffer.byteLength(JSON.stringify(longer))).toBeGreaterThan(32000);
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ },
+);