Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.
TypeScript ai-sdkclaude-codeclicode-searchcodexcoding-agentscontext-retrievaldeveloper-toolsjevsemantic-searchtypescriptvercel-ai-gateway
fix: drain streamed discovery results on stop
David Zhang14f08be812b4
diff --git a/docs/architecture.md b/docs/architecture.mdindex e6cf85b..1b3ed20 100644--- a/docs/architecture.md+++ b/docs/architecture.md@@ -22,8 +22,9 @@ byte was found. Oversized preview requests are split into bounded source chunks.Discovery streams source previews into bounded navigation batches with backpressure,so a wide directory can begin scoring before all of its files have been read. Split-retries stay in the same queue and preserve admission order; stale donors, cancellation,-and source-generator or result-processing errors stop admission while active evaluations drain.+retries stay in the same queue and preserve admission order. Stale files are isolated+or dropped while admission continues; cancellation and terminal source or result-processing+errors stop admission while active evaluations drain.A relationship pass can recover implementations of the same class contract acrossplatforms. Directory samples and whole-file previews use the same relationshipdiff --git a/packages/core/src/retrieve.ts b/packages/core/src/retrieve.tsindex 9b9362f..bcf46c6 100644--- a/packages/core/src/retrieve.ts+++ b/packages/core/src/retrieve.ts@@ -125,14 +125,20 @@ export async function retrieve(input: SearchInput, evaluator: Evaluator): Promislet commit = 0;let failure: unknown;let budgetPaused = false;- function flush() {+ function flush(force = false) {while (commit < ordered.length && ordered[commit]!.state === "done") {const group = ordered[commit++]!;for (const result of group.results ?? []) {- results.push(result);consume(result.item, result.score);}}+ if (force) {+ for (; commit < ordered.length; commit++) {+ const group = ordered[commit]!;+ if (group.state !== "done") continue;+ for (const result of group.results ?? []) consume(result.item, result.score);+ }+ }}function split(group: Group) {const middle = Math.ceil(group.items.length / 2);@@ -185,6 +191,7 @@ export async function retrieve(input: SearchInput, evaluator: Evaluator): Promisif (score > relevanceBoundary) positiveScoresSeen++;}await confirmSource(classified);+ results.push(...classified);group.results = classified;group.state = "done";flush();@@ -229,6 +236,24 @@ export async function retrieve(input: SearchInput, evaluator: Evaluator): PromisbudgetPaused = false;pump();}+ async function drain() {+ for (;;) {+ if (failure || stop || input.signal.aborted) break;+ if (budgetPaused) {+ await settleBudget();+ continue;+ }+ pump();+ if (active.size) {+ await Promise.race(active);+ continue;+ }+ if (queued.length) continue;+ break;+ }+ await waitForProgress();+ flush(Boolean(failure || stop || input.signal.aborted));+ }async function submit(group: NavigationItem[]) {const entry: Group = { items: group, state: "queued" };ordered.push(entry);@@ -270,14 +295,14 @@ export async function retrieve(input: SearchInput, evaluator: Evaluator): Promisbatch.push(item);}if (batch.length) await submit(batch);- if (budgetPaused) await settleBudget();+ await drain();} catch (error) {failure ??= error;} finally {- await waitForProgress();+ await drain();}if (failure) throw failure;- flush();+ flush(Boolean(stop || input.signal.aborted));return results;}async function previewDirectory(path: string): Promise<DirectoryPreview | undefined> {@@ -439,6 +464,7 @@ export async function retrieve(input: SearchInput, evaluator: Evaluator): Promisconst level = directories.splice(0).map((path) => ({ path, depth: 0 }));const hashes = new Map<string, string>();async function* items(): AsyncGenerator<NavigationItem> {+ const expanded: NavigationItem[] = [];for (let index = 0; index < level.length && !stop; index++) {const current = level[index]!;if (entriesSeen >= 100_000) {@@ -498,25 +524,28 @@ export async function retrieve(input: SearchInput, evaluator: Evaluator): Promis) {for (const chunk of splitSource(source, 12_000)) {const text = sourceForUnit(source, chunk);- yield buffered(- {- path: entry.path,- kind: "file",- filePreview: {- sizeBytes: filePreview.sizeBytes,- extension: filePreview.extension,- text,- previewBytes: Buffer.byteLength(text),- truncated: true,- range: "sampled source ranges",+ expanded.push(+ buffered(+ {+ path: entry.path,+ kind: "file",+ filePreview: {+ sizeBytes: filePreview.sizeBytes,+ extension: filePreview.extension,+ text,+ previewBytes: Buffer.byteLength(text),+ truncated: true,+ range: "sampled source ranges",+ },},- },- [source],+ [source],+ ),);}} else yield item;}}+ yield* expanded;}}const classified: Array<{ item: NavigationItem; score: number }> = [];diff --git a/test/discovery.test.ts b/test/discovery.test.tsindex e6832e3..ad594b9 100644--- a/test/discovery.test.ts+++ b/test/discovery.test.ts@@ -865,8 +865,7 @@ testIfDocker({ root, query: "late match", signal: new AbortController().signal },{requests: 0,- async evaluate(request, policy) {- await policy?.beforeAttempt?.();+ async evaluate(request) {const state = request.state as {items?: Array<{ path: string; filePreview?: { text: string } }>;};@@ -981,3 +980,67 @@ testIfDocker("discovery propagates result-processing errors", async () => {await rm(root, { recursive: true, force: true });}});++testIfDocker(+ "completed navigation groups survive a terminal stop behind an unfinished split",+ async () => {+ const root = await mkdtemp(join(tmpdir(), "jg-discovery-completed-stop-"));+ try {+ for (let index = 0; index < 257; index++)+ await writeFile(join(root, `f${String(index).padStart(3, "0")}.txt`), `entry ${index}\n`);+ const result = await retrieve(+ { root, query: "completed group", signal: new AbortController().signal },+ {+ requests: 0,+ async evaluate(request) {+ const items = (request.state as Body["state"]).items;+ if (items) {+ if (items[0]?.path === "f000.txt") throw new EvaluationFailure("provider", true);+ if (items[0]?.path === "f050.txt") throw new EvaluationFailure("request-limit");+ }+ return Object.fromEntries(Object.keys(request.questions).map((id) => [id, 0.9]));+ },+ },+ );+ expect(result.status).toBe("incomplete");+ expect(result.files.map(({ path }) => path)).toContain("f025.txt");+ } finally {+ await rm(root, { recursive: true, force: true });+ }+ },+);++testIfDocker("oversized preview chunks retain main's batch ordering", async () => {+ const root = await mkdtemp(join(tmpdir(), "jg-oversized-order-"));+ const requests: Array<{ paths: string[]; bytes: number }> = [];+ try {+ await writeFile(+ join(root, "a.ts"),+ Array.from(+ { length: 1500 },+ (_, index) => `export function f${index}() { return ${index}; }\n`,+ ).join(""),+ );+ await writeFile(join(root, "b.txt"), "b");+ const result = await retrieve(+ { root, query: "context ".repeat(3000), signal: new AbortController().signal },+ {+ requests: 0,+ async evaluate(request) {+ const items = (request.state as Body["state"]).items;+ if (items)+ requests.push({+ paths: items.map((item) => item.path),+ bytes: Buffer.byteLength(JSON.stringify(request)),+ });+ return Object.fromEntries(Object.keys(request.questions).map((id) => [id, 0.9]));+ },+ },+ );+ expect(result.status).toBe("complete");+ expect(requests[0]?.paths[0]).toBe("b.txt");+ expect(requests[0]?.bytes).toBeLessThan(38_000);+ } finally {+ await rm(root, { recursive: true, force: true });+ }+});