Skip to content

Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.

Prioritize lead-backed excerpts within bounded source output

David Zhang945af19478c1
diff --git a/apps/cli/src/args.ts b/apps/cli/src/args.ts
index 3b33213..6180e45 100644
--- a/apps/cli/src/args.ts
+++ b/apps/cli/src/args.ts
@@ -1,6 +1,7 @@
import { parseArgs } from "node:util";
import type { SearchInput } from "@repo/core";
import { CliError } from "./errors";
+import { DEFAULT_MAX_SOURCE_BYTES } from "./render";
export type Command =
| { kind: "help" | "version" | "doctor" | "skill" | "cache-clear" }
@@ -61,7 +62,7 @@ export function parseCommand(args: string[]): Command {
if (!first?.trim() || positionals.length > 2 || values.stdin)
throw new CliError('Usage: jg "question" [root]. Run jg --help.');
const rawBudget = values["max-source-bytes"];
- const maxSourceBytes = rawBudget === undefined ? 0 : Number(rawBudget);
+ const maxSourceBytes = rawBudget === undefined ? DEFAULT_MAX_SOURCE_BYTES : Number(rawBudget);
if (
rawBudget !== undefined &&
(!/^\d+$/.test(rawBudget) || !Number.isSafeInteger(maxSourceBytes))
@@ -96,7 +97,7 @@ Commands:
--version Show the installed version
Search options:
- --max-source-bytes N Source allocation; 0 means unlimited (default)
+ --max-source-bytes N Source allocation; 0 means unlimited (default: ${DEFAULT_MAX_SOURCE_BYTES})
--hidden Include hidden paths
--no-ignore Disable .gitignore/.ignore patterns
--include-dependencies Include dependency and build directories
diff --git a/apps/cli/src/render.ts b/apps/cli/src/render.ts
index f6ceef9..49fef0a 100644
--- a/apps/cli/src/render.ts
+++ b/apps/cli/src/render.ts
@@ -1,4 +1,4 @@
-import type { RetrievalResult } from "@repo/core";
+import type { FileEvidence, RetrievalResult } from "@repo/core";
function quote(value: string): string {
return JSON.stringify(value).replace(
@@ -7,24 +7,62 @@ function quote(value: string): string {
);
}
-/** Allocate whole excerpts so a byte limit cannot invalidate their source coordinates. */
-export function renderResult(result: RetrievalResult, maxSourceBytes = 0): string {
- let remaining = maxSourceBytes || Infinity;
+export const DEFAULT_MAX_SOURCE_BYTES = 0;
+
+type Excerpt = FileEvidence["excerpts"][number];
+function containsLead(excerpt: Excerpt, range: FileEvidence["leads"][number]["range"]): boolean {
+ if (range.startLine < excerpt.range.startLine || range.endLine > excerpt.range.endLine)
+ return false;
+ const start = excerpt.sourceByteStart ?? excerpt.range.sourceByteStart;
+ const end = excerpt.sourceByteEnd ?? excerpt.range.sourceByteEnd;
+ if (start !== undefined && range.sourceByteStart !== undefined && range.sourceByteStart < start)
+ return false;
+ if (end !== undefined && range.sourceByteEnd !== undefined && range.sourceByteEnd > end)
+ return false;
+ return true;
+}
+
+/** Whole excerpts keep coordinates truthful; unlimited rendering retains its original ordering. */
+export function renderResult(
+ result: RetrievalResult,
+ maxSourceBytes = DEFAULT_MAX_SOURCE_BYTES,
+): string {
const files = [...result.files]
.sort((a, b) => b.score - a.score || (a.path < b.path ? -1 : a.path > b.path ? 1 : 0))
- .map((file) => {
- let omitted = file.sourceOmitted;
- const excerpts = file.excerpts.filter(({ source }) => {
- const bytes = Buffer.byteLength(source);
- if (bytes > remaining) {
- omitted = true;
- return false;
- }
- remaining -= bytes;
- return true;
- });
- return { file, excerpts, omitted };
- });
+ .map((file) => ({ file, excerpts: file.excerpts, omitted: file.sourceOmitted }));
+ if (maxSourceBytes > 0) {
+ let remaining = maxSourceBytes;
+ const kept = files.map(() => new Set<number>());
+ const ranked = files.flatMap(({ file }, fileIndex) =>
+ file.excerpts.map((excerpt, excerptIndex) => ({
+ fileIndex,
+ excerptIndex,
+ bytes: Buffer.byteLength(excerpt.source),
+ // Context snippets have no reading-lead score: this prioritizes available evidence,
+ // not a complete confidence estimate for every selected or expanded source range.
+ score: file.leads.reduce(
+ (score, lead) =>
+ lead.score > 0.5 && containsLead(excerpt, lead.range)
+ ? Math.max(score, lead.score)
+ : score,
+ 0,
+ ),
+ })),
+ );
+ // Stable sorting uses original file/excerpt order for equal or absent lead scores.
+ ranked.sort((a, b) => b.score - a.score);
+ for (const candidate of ranked)
+ if (candidate.bytes <= remaining) {
+ kept[candidate.fileIndex]!.add(candidate.excerptIndex);
+ remaining -= candidate.bytes;
+ }
+ for (const [index, entry] of files.entries()) {
+ entry.excerpts = entry.file.excerpts.filter((_, excerptIndex) =>
+ kept[index]!.has(excerptIndex),
+ );
+ entry.omitted ||= entry.excerpts.length !== entry.file.excerpts.length;
+ }
+ }
const lines = [
`Status: ${result.status}`,
`Root: ${quote(result.root)}`,
diff --git a/apps/cli/test/args.test.ts b/apps/cli/test/args.test.ts
index 0e1470a..455f377 100644
--- a/apps/cli/test/args.test.ts
+++ b/apps/cli/test/args.test.ts
@@ -24,3 +24,27 @@ test("search accepts independent policy flags and a dash-prefixed root after --"
expect(() => parseCommand(["question", "--max-source-bytes", "-1"])).toThrow();
expect(() => parseCommand(["question", "--secret-mistake"])).toThrow("Unknown option");
});
+
+test("source allocation is independent of the filesystem policy used for retrieval and cache identity", () => {
+ const unlimited = parseCommand([
+ "question",
+ "tree",
+ "--hidden",
+ "--no-ignore",
+ "--max-source-bytes",
+ "0",
+ ]);
+ const bounded = parseCommand([
+ "question",
+ "tree",
+ "--hidden",
+ "--no-ignore",
+ "--max-source-bytes",
+ "1500",
+ ]);
+ if (unlimited.kind !== "search" || bounded.kind !== "search") throw new Error("Expected search");
+ expect(unlimited.maxSourceBytes).toBe(0);
+ expect(bounded.maxSourceBytes).toBe(1500);
+ expect(unlimited.policy).toEqual({ hidden: true, noIgnore: true });
+ expect(bounded.policy).toEqual(unlimited.policy);
+});
diff --git a/apps/cli/test/render.test.ts b/apps/cli/test/render.test.ts
index a0c0ff5..995c328 100644
--- a/apps/cli/test/render.test.ts
+++ b/apps/cli/test/render.test.ts
@@ -68,3 +68,152 @@ test("partial source carries byte coordinates without changing its bytes", () =>
expect(output).toContain("partial excerpt; UTF-8 bytes [5, 7)");
expect(output).toContain("\né\n");
});
+
+function allocationFixture(): RetrievalResult {
+ return {
+ root: "/root",
+ query: "q",
+ status: "complete",
+ issues: [],
+ counts: { requests: 1, cacheHits: 0, inspectedFiles: 1 },
+ files: [
+ {
+ path: "model.py",
+ contentHash: "hash",
+ score: 1,
+ roles: [],
+ leads: [
+ { name: "first", range: { startLine: 5, endLine: 5 }, score: 0.7 },
+ { name: "second", range: { startLine: 9, endLine: 9 }, score: 0.9 },
+ ],
+ selected: [
+ { startLine: 5, endLine: 5 },
+ { startLine: 9, endLine: 9 },
+ ],
+ rendered: [
+ { startLine: 1, endLine: 1 },
+ { startLine: 5, endLine: 5 },
+ { startLine: 9, endLine: 9 },
+ ],
+ excerpts: [
+ { range: { startLine: 1, endLine: 1 }, source: "context\n" },
+ { range: { startLine: 5, endLine: 5 }, source: "method_a\n" },
+ { range: { startLine: 9, endLine: 9 }, source: "method_b\n" },
+ ],
+ sourceOmitted: false,
+ },
+ ],
+ };
+}
+
+test("a bounded packet favors confident declarations but renders retained excerpts in source order", () => {
+ const result = allocationFixture();
+ const before = structuredClone(result);
+ const output = renderResult(result, 18);
+ expect(output).not.toContain("\ncontext\n");
+ expect(output).toContain("\nmethod_a\n");
+ expect(output).toContain("\nmethod_b\n");
+ expect(output.indexOf("method_a")).toBeLessThan(output.indexOf("method_b"));
+ expect(output).toContain("Source omitted: 1 file(s)");
+ expect(output).toContain('"first": 5-5');
+ expect(output).toContain('"second": 9-9');
+ expect(result).toEqual(before);
+});
+
+test("unlimited output retains the complete original packet byte for byte", () => {
+ expect(renderResult(allocationFixture(), 0)).toBe(
+ 'Status: complete\nRoot: "/root"\nRelevant files: 1\nSource omitted: 0 file(s)\n\nReading leads (estimates; source below is evidence):\n- "model.py" — relevant; role uncertain\n "first": 5-5\n "second": 9-9\n\nSource (verbatim; ranges may end within declarations):\n\n"model.py":1-1\ncontext\n\n\n"model.py":5-5\nmethod_a\n\n\n"model.py":9-9\nmethod_b\n\n\nEnd context.\n',
+ );
+});
+
+test("confidence ranking crosses file boundaries without dropping any file or lead", () => {
+ const result = allocationFixture();
+ result.files.push({
+ ...result.files[0]!,
+ path: "lower.py",
+ score: 0.6,
+ leads: [{ name: "decisive", range: { startLine: 20, endLine: 20 }, score: 0.95 }],
+ excerpts: [{ range: { startLine: 20, endLine: 20 }, source: "decisive\n" }],
+ });
+ const output = renderResult(result, 9);
+ expect(output).toContain('"lower.py":20-20\ndecisive\n');
+ expect(output).not.toContain("\nmethod_b\n");
+ expect(output).toContain('"first": 5-5');
+ expect(output).toContain('"second": 9-9');
+ expect(output).toContain('"decisive": 20-20');
+ expect(output.indexOf('- "model.py"')).toBeLessThan(output.indexOf('- "lower.py"'));
+ expect(output).toContain("Relevant files: 2");
+});
+
+test("optional or partially overlapping leads do not outrank a contained confident declaration", () => {
+ const result = allocationFixture();
+ result.files[0]!.leads = [
+ { name: "optional_context", range: { startLine: 1, endLine: 1 }, score: 0.5 },
+ { name: "overlaps_context", range: { startLine: 1, endLine: 4 }, score: 0.99 },
+ { name: "first", range: { startLine: 5, endLine: 5 }, score: 0.6 },
+ ];
+ expect(renderResult(result, 9)).toContain('"model.py":5-5\nmethod_a\n');
+});
+
+test("partial-line ranking uses actual byte containment and counts UTF-8 bytes", () => {
+ const result = allocationFixture();
+ result.files[0]!.excerpts = [
+ {
+ range: { startLine: 1, endLine: 1 },
+ source: "é",
+ sourceByteStart: 0,
+ sourceByteEnd: 2,
+ partial: true,
+ },
+ {
+ range: { startLine: 1, endLine: 1 },
+ source: "ß",
+ sourceByteStart: 2,
+ sourceByteEnd: 4,
+ partial: true,
+ },
+ ];
+ result.files[0]!.leads = [
+ {
+ name: "first_fragment",
+ range: { startLine: 1, endLine: 1, sourceByteStart: 0, sourceByteEnd: 2 },
+ score: 0.6,
+ },
+ {
+ name: "selected_fragment",
+ range: { startLine: 1, endLine: 1, sourceByteStart: 2, sourceByteEnd: 4 },
+ score: 0.8,
+ },
+ ];
+ const output = renderResult(result, 2);
+ expect(output).toContain("partial excerpt; UTF-8 bytes [2, 4)");
+ expect(output).toContain("\nß\n");
+ expect(output).not.toContain("\né\n");
+ expect(renderResult(result, 1)).not.toContain("\nß\n");
+});
+
+test("a partial opposite endpoint does not demote a fully selected whole-line declaration", () => {
+ for (const [source, sourceByteStart, range] of [
+ ["high()\npart", 0, { startLine: 1, endLine: 1 }],
+ ["part\nhigh()\n", 4, { startLine: 2, endLine: 2 }],
+ ] as const) {
+ const result = allocationFixture();
+ result.files[0]!.excerpts = [
+ {
+ range: { startLine: 1, endLine: 2 },
+ source,
+ sourceByteStart,
+ sourceByteEnd: sourceByteStart + Buffer.byteLength(source),
+ partial: true,
+ },
+ { range: { startLine: 5, endLine: 5 }, source: "low\n" },
+ ];
+ result.files[0]!.leads = [
+ { name: "high", range, score: 0.9 },
+ { name: "low", range: { startLine: 5, endLine: 5 }, score: 0.8 },
+ ];
+ const output = renderResult(result, Buffer.byteLength(source));
+ expect(output).toContain(source);
+ expect(output).not.toContain("\nlow\n");
+ }
+});
diff --git a/packages/core/src/types.ts b/packages/core/src/types.ts
index 1098012..a0c2a12 100644
--- a/packages/core/src/types.ts
+++ b/packages/core/src/types.ts
@@ -38,7 +38,7 @@ export type RetrievalResult = {
export type SearchInput = {
root: string;
query: string;
- policy?: FilesystemPolicy & { maxSourceBytes?: number };
+ policy?: FilesystemPolicy;
signal: AbortSignal;
protectedPaths?: string[];
};