Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.
TypeScript ai-sdkclaude-codeclicode-searchcodexcoding-agentscontext-retrievaldeveloper-toolsjevsemantic-searchtypescriptvercel-ai-gateway
Prioritize lead-backed excerpts within bounded source output
David Zhang945af19478c1
diff --git a/apps/cli/src/args.ts b/apps/cli/src/args.tsindex 3b33213..6180e45 100644--- a/apps/cli/src/args.ts+++ b/apps/cli/src/args.ts@@ -1,6 +1,7 @@import { parseArgs } from "node:util";import type { SearchInput } from "@repo/core";import { CliError } from "./errors";+import { DEFAULT_MAX_SOURCE_BYTES } from "./render";export type Command =| { kind: "help" | "version" | "doctor" | "skill" | "cache-clear" }@@ -61,7 +62,7 @@ export function parseCommand(args: string[]): Command {if (!first?.trim() || positionals.length > 2 || values.stdin)throw new CliError('Usage: jg "question" [root]. Run jg --help.');const rawBudget = values["max-source-bytes"];- const maxSourceBytes = rawBudget === undefined ? 0 : Number(rawBudget);+ const maxSourceBytes = rawBudget === undefined ? DEFAULT_MAX_SOURCE_BYTES : Number(rawBudget);if (rawBudget !== undefined &&(!/^\d+$/.test(rawBudget) || !Number.isSafeInteger(maxSourceBytes))@@ -96,7 +97,7 @@ Commands:--version Show the installed versionSearch options:- --max-source-bytes N Source allocation; 0 means unlimited (default)+ --max-source-bytes N Source allocation; 0 means unlimited (default: ${DEFAULT_MAX_SOURCE_BYTES})--hidden Include hidden paths--no-ignore Disable .gitignore/.ignore patterns--include-dependencies Include dependency and build directoriesdiff --git a/apps/cli/src/render.ts b/apps/cli/src/render.tsindex f6ceef9..49fef0a 100644--- a/apps/cli/src/render.ts+++ b/apps/cli/src/render.ts@@ -1,4 +1,4 @@-import type { RetrievalResult } from "@repo/core";+import type { FileEvidence, RetrievalResult } from "@repo/core";function quote(value: string): string {return JSON.stringify(value).replace(@@ -7,24 +7,62 @@ function quote(value: string): string {);}-/** Allocate whole excerpts so a byte limit cannot invalidate their source coordinates. */-export function renderResult(result: RetrievalResult, maxSourceBytes = 0): string {- let remaining = maxSourceBytes || Infinity;+export const DEFAULT_MAX_SOURCE_BYTES = 0;++type Excerpt = FileEvidence["excerpts"][number];+function containsLead(excerpt: Excerpt, range: FileEvidence["leads"][number]["range"]): boolean {+ if (range.startLine < excerpt.range.startLine || range.endLine > excerpt.range.endLine)+ return false;+ const start = excerpt.sourceByteStart ?? excerpt.range.sourceByteStart;+ const end = excerpt.sourceByteEnd ?? excerpt.range.sourceByteEnd;+ if (start !== undefined && range.sourceByteStart !== undefined && range.sourceByteStart < start)+ return false;+ if (end !== undefined && range.sourceByteEnd !== undefined && range.sourceByteEnd > end)+ return false;+ return true;+}++/** Whole excerpts keep coordinates truthful; unlimited rendering retains its original ordering. */+export function renderResult(+ result: RetrievalResult,+ maxSourceBytes = DEFAULT_MAX_SOURCE_BYTES,+): string {const files = [...result.files].sort((a, b) => b.score - a.score || (a.path < b.path ? -1 : a.path > b.path ? 1 : 0))- .map((file) => {- let omitted = file.sourceOmitted;- const excerpts = file.excerpts.filter(({ source }) => {- const bytes = Buffer.byteLength(source);- if (bytes > remaining) {- omitted = true;- return false;- }- remaining -= bytes;- return true;- });- return { file, excerpts, omitted };- });+ .map((file) => ({ file, excerpts: file.excerpts, omitted: file.sourceOmitted }));+ if (maxSourceBytes > 0) {+ let remaining = maxSourceBytes;+ const kept = files.map(() => new Set<number>());+ const ranked = files.flatMap(({ file }, fileIndex) =>+ file.excerpts.map((excerpt, excerptIndex) => ({+ fileIndex,+ excerptIndex,+ bytes: Buffer.byteLength(excerpt.source),+ // Context snippets have no reading-lead score: this prioritizes available evidence,+ // not a complete confidence estimate for every selected or expanded source range.+ score: file.leads.reduce(+ (score, lead) =>+ lead.score > 0.5 && containsLead(excerpt, lead.range)+ ? Math.max(score, lead.score)+ : score,+ 0,+ ),+ })),+ );+ // Stable sorting uses original file/excerpt order for equal or absent lead scores.+ ranked.sort((a, b) => b.score - a.score);+ for (const candidate of ranked)+ if (candidate.bytes <= remaining) {+ kept[candidate.fileIndex]!.add(candidate.excerptIndex);+ remaining -= candidate.bytes;+ }+ for (const [index, entry] of files.entries()) {+ entry.excerpts = entry.file.excerpts.filter((_, excerptIndex) =>+ kept[index]!.has(excerptIndex),+ );+ entry.omitted ||= entry.excerpts.length !== entry.file.excerpts.length;+ }+ }const lines = [`Status: ${result.status}`,`Root: ${quote(result.root)}`,diff --git a/apps/cli/test/args.test.ts b/apps/cli/test/args.test.tsindex 0e1470a..455f377 100644--- a/apps/cli/test/args.test.ts+++ b/apps/cli/test/args.test.ts@@ -24,3 +24,27 @@ test("search accepts independent policy flags and a dash-prefixed root after --"expect(() => parseCommand(["question", "--max-source-bytes", "-1"])).toThrow();expect(() => parseCommand(["question", "--secret-mistake"])).toThrow("Unknown option");});++test("source allocation is independent of the filesystem policy used for retrieval and cache identity", () => {+ const unlimited = parseCommand([+ "question",+ "tree",+ "--hidden",+ "--no-ignore",+ "--max-source-bytes",+ "0",+ ]);+ const bounded = parseCommand([+ "question",+ "tree",+ "--hidden",+ "--no-ignore",+ "--max-source-bytes",+ "1500",+ ]);+ if (unlimited.kind !== "search" || bounded.kind !== "search") throw new Error("Expected search");+ expect(unlimited.maxSourceBytes).toBe(0);+ expect(bounded.maxSourceBytes).toBe(1500);+ expect(unlimited.policy).toEqual({ hidden: true, noIgnore: true });+ expect(bounded.policy).toEqual(unlimited.policy);+});diff --git a/apps/cli/test/render.test.ts b/apps/cli/test/render.test.tsindex a0c0ff5..995c328 100644--- a/apps/cli/test/render.test.ts+++ b/apps/cli/test/render.test.ts@@ -68,3 +68,152 @@ test("partial source carries byte coordinates without changing its bytes", () =>expect(output).toContain("partial excerpt; UTF-8 bytes [5, 7)");expect(output).toContain("\né\n");});++function allocationFixture(): RetrievalResult {+ return {+ root: "/root",+ query: "q",+ status: "complete",+ issues: [],+ counts: { requests: 1, cacheHits: 0, inspectedFiles: 1 },+ files: [+ {+ path: "model.py",+ contentHash: "hash",+ score: 1,+ roles: [],+ leads: [+ { name: "first", range: { startLine: 5, endLine: 5 }, score: 0.7 },+ { name: "second", range: { startLine: 9, endLine: 9 }, score: 0.9 },+ ],+ selected: [+ { startLine: 5, endLine: 5 },+ { startLine: 9, endLine: 9 },+ ],+ rendered: [+ { startLine: 1, endLine: 1 },+ { startLine: 5, endLine: 5 },+ { startLine: 9, endLine: 9 },+ ],+ excerpts: [+ { range: { startLine: 1, endLine: 1 }, source: "context\n" },+ { range: { startLine: 5, endLine: 5 }, source: "method_a\n" },+ { range: { startLine: 9, endLine: 9 }, source: "method_b\n" },+ ],+ sourceOmitted: false,+ },+ ],+ };+}++test("a bounded packet favors confident declarations but renders retained excerpts in source order", () => {+ const result = allocationFixture();+ const before = structuredClone(result);+ const output = renderResult(result, 18);+ expect(output).not.toContain("\ncontext\n");+ expect(output).toContain("\nmethod_a\n");+ expect(output).toContain("\nmethod_b\n");+ expect(output.indexOf("method_a")).toBeLessThan(output.indexOf("method_b"));+ expect(output).toContain("Source omitted: 1 file(s)");+ expect(output).toContain('"first": 5-5');+ expect(output).toContain('"second": 9-9');+ expect(result).toEqual(before);+});++test("unlimited output retains the complete original packet byte for byte", () => {+ expect(renderResult(allocationFixture(), 0)).toBe(+ 'Status: complete\nRoot: "/root"\nRelevant files: 1\nSource omitted: 0 file(s)\n\nReading leads (estimates; source below is evidence):\n- "model.py" — relevant; role uncertain\n "first": 5-5\n "second": 9-9\n\nSource (verbatim; ranges may end within declarations):\n\n"model.py":1-1\ncontext\n\n\n"model.py":5-5\nmethod_a\n\n\n"model.py":9-9\nmethod_b\n\n\nEnd context.\n',+ );+});++test("confidence ranking crosses file boundaries without dropping any file or lead", () => {+ const result = allocationFixture();+ result.files.push({+ ...result.files[0]!,+ path: "lower.py",+ score: 0.6,+ leads: [{ name: "decisive", range: { startLine: 20, endLine: 20 }, score: 0.95 }],+ excerpts: [{ range: { startLine: 20, endLine: 20 }, source: "decisive\n" }],+ });+ const output = renderResult(result, 9);+ expect(output).toContain('"lower.py":20-20\ndecisive\n');+ expect(output).not.toContain("\nmethod_b\n");+ expect(output).toContain('"first": 5-5');+ expect(output).toContain('"second": 9-9');+ expect(output).toContain('"decisive": 20-20');+ expect(output.indexOf('- "model.py"')).toBeLessThan(output.indexOf('- "lower.py"'));+ expect(output).toContain("Relevant files: 2");+});++test("optional or partially overlapping leads do not outrank a contained confident declaration", () => {+ const result = allocationFixture();+ result.files[0]!.leads = [+ { name: "optional_context", range: { startLine: 1, endLine: 1 }, score: 0.5 },+ { name: "overlaps_context", range: { startLine: 1, endLine: 4 }, score: 0.99 },+ { name: "first", range: { startLine: 5, endLine: 5 }, score: 0.6 },+ ];+ expect(renderResult(result, 9)).toContain('"model.py":5-5\nmethod_a\n');+});++test("partial-line ranking uses actual byte containment and counts UTF-8 bytes", () => {+ const result = allocationFixture();+ result.files[0]!.excerpts = [+ {+ range: { startLine: 1, endLine: 1 },+ source: "é",+ sourceByteStart: 0,+ sourceByteEnd: 2,+ partial: true,+ },+ {+ range: { startLine: 1, endLine: 1 },+ source: "ß",+ sourceByteStart: 2,+ sourceByteEnd: 4,+ partial: true,+ },+ ];+ result.files[0]!.leads = [+ {+ name: "first_fragment",+ range: { startLine: 1, endLine: 1, sourceByteStart: 0, sourceByteEnd: 2 },+ score: 0.6,+ },+ {+ name: "selected_fragment",+ range: { startLine: 1, endLine: 1, sourceByteStart: 2, sourceByteEnd: 4 },+ score: 0.8,+ },+ ];+ const output = renderResult(result, 2);+ expect(output).toContain("partial excerpt; UTF-8 bytes [2, 4)");+ expect(output).toContain("\nß\n");+ expect(output).not.toContain("\né\n");+ expect(renderResult(result, 1)).not.toContain("\nß\n");+});++test("a partial opposite endpoint does not demote a fully selected whole-line declaration", () => {+ for (const [source, sourceByteStart, range] of [+ ["high()\npart", 0, { startLine: 1, endLine: 1 }],+ ["part\nhigh()\n", 4, { startLine: 2, endLine: 2 }],+ ] as const) {+ const result = allocationFixture();+ result.files[0]!.excerpts = [+ {+ range: { startLine: 1, endLine: 2 },+ source,+ sourceByteStart,+ sourceByteEnd: sourceByteStart + Buffer.byteLength(source),+ partial: true,+ },+ { range: { startLine: 5, endLine: 5 }, source: "low\n" },+ ];+ result.files[0]!.leads = [+ { name: "high", range, score: 0.9 },+ { name: "low", range: { startLine: 5, endLine: 5 }, score: 0.8 },+ ];+ const output = renderResult(result, Buffer.byteLength(source));+ expect(output).toContain(source);+ expect(output).not.toContain("\nlow\n");+ }+});diff --git a/packages/core/src/types.ts b/packages/core/src/types.tsindex 1098012..a0c2a12 100644--- a/packages/core/src/types.ts+++ b/packages/core/src/types.ts@@ -38,7 +38,7 @@ export type RetrievalResult = {export type SearchInput = {root: string;query: string;- policy?: FilesystemPolicy & { maxSourceBytes?: number };+ policy?: FilesystemPolicy;signal: AbortSignal;protectedPaths?: string[];};