Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.
TypeScript ai-sdkclaude-codeclicode-searchcodexcoding-agentscontext-retrievaldeveloper-toolsjevsemantic-searchtypescriptvercel-ai-gateway
Remove superlinear work on comment windows and declaration indexes
taeskimd325ee80c137
diff --git a/packages/core/src/retrieve.ts b/packages/core/src/retrieve.tsindex e584398..5ac9b96 100644--- a/packages/core/src/retrieve.ts+++ b/packages/core/src/retrieve.ts@@ -353,10 +353,25 @@ export async function retrieve(input: SearchInput, evaluator: Evaluator): Promispreview.declarations = syntax.units.filter((unit) => !unit.partial).map((unit) => ({ name: unit.name, ...unit.range }));- while (preview.declarations.length && Buffer.byteLength(JSON.stringify(preview)) > 32000) {- preview.declarations.pop();+ // Keep the longest prefix of declarations that fits. Serialized size grows with each entry,+ // so a binary search finds the same prefix as removing one declaration at a time.+ const all = preview.declarations;+ const fits = (count: number) => {+ preview.declarations = all.slice(0, count);+ return Buffer.byteLength(JSON.stringify(preview)) <= 32000;+ };+ if (all.length && !fits(all.length)) {+ // Removal measured every shorter prefix with the truncation flag already set.preview.declarationIndexTruncated = true;- }+ let low = 0,+ high = all.length - 1;+ while (low < high) {+ const middle = Math.ceil((low + high) / 2);+ if (fits(middle)) low = middle;+ else high = middle - 1;+ }+ preview.declarations = all.slice(0, low);+ } else preview.declarations = all;}return preview;}diff --git a/packages/core/src/selection.ts b/packages/core/src/selection.tsindex 451c4c4..199262d 100644--- a/packages/core/src/selection.ts+++ b/packages/core/src/selection.ts@@ -345,36 +345,43 @@ export async function selectFile(throw error;warn("cancelled");}+ // nonBlankThrough[n] counts lines 1..n that contain non-whitespace, so a gap check is O(1).+ const nonBlankThrough = [0];+ for (const text of lines) nonBlankThrough.push(nonBlankThrough.at(-1)! + (text.trim() ? 1 : 0));+ const blankLines = (first: number, last: number) =>+ last < first ||+ nonBlankThrough[Math.min(last, lines.length)]! ===+ nonBlankThrough[Math.min(Math.max(first, 1) - 1, lines.length)]!;function excerptsFor(ranges: Range[], rendered: Span[]) {const windows = ranges.map((range) => ({startLine: Math.max(1, range.startLine - 3),endLine: Math.min(lines.length, range.endLine + 3),}));for (const window of windows) {+ // A window absorbs comments it overlaps or that border it across blank lines, until none+ // qualify. Absorption only grows the window, so every order reaches the same closure.+ // Ascending passes absorb chains after the window, descending passes chains before it.+ const absorb = (comment: Range) => {+ const qualifies =+ (comment.startLine <= window.endLine && comment.endLine >= window.startLine) ||+ (comment.endLine < window.startLine &&+ blankLines(comment.endLine + 1, window.startLine - 1)) ||+ (comment.startLine > window.endLine &&+ blankLines(window.endLine + 1, comment.startLine - 1));+ if (!qualifies) return false;+ const start = Math.min(window.startLine, comment.startLine),+ end = Math.max(window.endLine, comment.endLine);+ if (start === window.startLine && end === window.endLine) return false;+ window.startLine = start;+ window.endLine = end;+ return true;+ };let changed = true;while (changed) {changed = false;- for (const comment of syntax.comments) {- const before =- comment.endLine < window.startLine &&- lines.slice(comment.endLine, window.startLine - 1).every((line) => !line.trim());- const after =- comment.startLine > window.endLine &&- lines.slice(window.endLine, comment.startLine - 1).every((line) => !line.trim());- if (- (comment.startLine <= window.endLine && comment.endLine >= window.startLine) ||- before ||- after- ) {- const start = Math.min(window.startLine, comment.startLine),- end = Math.max(window.endLine, comment.endLine);- if (start !== window.startLine || end !== window.endLine) {- window.startLine = start;- window.endLine = end;- changed = true;- }- }- }+ for (const comment of syntax.comments) if (absorb(comment)) changed = true;+ for (let index = syntax.comments.length - 1; index >= 0; index--)+ if (absorb(syntax.comments[index]!)) changed = true;}let segmentStart = offsets[window.startLine - 1]!;for (let line = window.startLine; line <= window.endLine; line++) {diff --git a/packages/core/test/selection.test.ts b/packages/core/test/selection.test.tsindex 221fdad..da853e1 100644--- a/packages/core/test/selection.test.ts+++ b/packages/core/test/selection.test.ts@@ -414,3 +414,27 @@ test("oversized class context split into blocks stays out of leads", async () =>expect(result.file.selected.length).toBeGreaterThan(0);expect(result.file.leads.map((lead) => lead.name)).toEqual(["Big.method"]);});++test("excerpts follow long comment blocks and stop at the first line of code", async () => {+ for (const [path, marker, fn] of [+ ["notes.py", "#", "def target():\n return 1\n"],+ ["notes.ts", "//", "export function target() {\n return 1;\n}\n"],+ ] as const) {+ const block = Array.from({ length: 5000 }, (_, i) => `${marker} note ${i}`).join("\n");+ const source = `${marker} unrelated\nconst_or_code = 1\n${block}\n\n${fn}\n${marker} trailing\n`;+ const lines = source.split("\n");+ const result = await selectFile(+ { path, contentHash: path, source },+ "target",+ 0.9,+ evaluator((d) => (d.name === "target" ? 0.9 : 0)),+ );+ // The whole block, separated from the function only by a blank line, joins the excerpt;+ // the comment above the line of code does not.+ const excerpt = result.file.excerpts[0]!;+ expect(excerpt.range.startLine).toBe(3);+ expect(lines[excerpt.range.startLine - 1]).toBe(`${marker} note 0`);+ expect(excerpt.source).toContain(`${marker} trailing`);+ expect(excerpt.source).not.toContain(`${marker} unrelated`);+ }+});diff --git a/test/retrieval.test.ts b/test/retrieval.test.tsindex ea98341..282834d 100644--- a/test/retrieval.test.ts+++ b/test/retrieval.test.ts@@ -746,3 +746,59 @@ for (const [layout, source, expected] of [await rm(root, { recursive: true, force: true });}});++testIfDocker(+ "a large declaration index keeps its longest prefix that fits the preview",+ async () => {+ const root = await mkdtemp(join(tmpdir(), "jg-declaration-index-"));+ try {+ const count = 3000;+ await writeFile(+ join(root, "client.ts"),+ Array.from(+ { length: count },+ (_, i) => `export function op${i}(a: number) { return a + ${i}; }\n`,+ ).join(""),+ );+ const previews: Array<{ declarations?: unknown[]; declarationIndexTruncated?: boolean }> = [];+ await retrieve(+ { root, query: "op behavior", signal: new AbortController().signal },+ {+ requests: 0,+ async evaluate(request) {+ const state = request.state as {+ items?: Array<{ filePreview?: (typeof previews)[number] }>;+ };+ for (const item of state.items ?? [])+ if (item.filePreview) previews.push(item.filePreview);+ return Object.fromEntries(Object.keys(request.questions).map((id) => [id, 0.1]));+ },+ },+ );+ const preview = previews[0]!;+ const kept = preview.declarations!.length;+ expect(preview.declarationIndexTruncated).toBe(true);+ expect(kept).toBeGreaterThan(0);+ expect(kept).toBeLessThan(count);+ expect(preview.declarations).toEqual(+ Array.from({ length: kept }, (_, i) => ({+ name: `op${i}`,+ startLine: i + 1,+ endLine: i + 1,+ })),+ );+ // The index keeps the longest prefix: one more declaration would not fit.+ expect(Buffer.byteLength(JSON.stringify(preview))).toBeLessThanOrEqual(32000);+ const longer = {+ ...preview,+ declarations: [+ ...preview.declarations!,+ { name: `op${kept}`, startLine: kept + 1, endLine: kept + 1 },+ ],+ };+ expect(Buffer.byteLength(JSON.stringify(longer))).toBeGreaterThan(32000);+ } finally {+ await rm(root, { recursive: true, force: true });+ }+ },+);