Skip to content

Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.

fix: drain streamed discovery results on stop

David Zhang14f08be812b4
diff --git a/docs/architecture.md b/docs/architecture.md
index e6cf85b..1b3ed20 100644
--- a/docs/architecture.md
+++ b/docs/architecture.md
@@ -22,8 +22,9 @@ byte was found. Oversized preview requests are split into bounded source chunks.
Discovery streams source previews into bounded navigation batches with backpressure,
so a wide directory can begin scoring before all of its files have been read. Split
-retries stay in the same queue and preserve admission order; stale donors, cancellation,
-and source-generator or result-processing errors stop admission while active evaluations drain.
+retries stay in the same queue and preserve admission order. Stale files are isolated
+or dropped while admission continues; cancellation and terminal source or result-processing
+errors stop admission while active evaluations drain.
A relationship pass can recover implementations of the same class contract across
platforms. Directory samples and whole-file previews use the same relationship
diff --git a/packages/core/src/retrieve.ts b/packages/core/src/retrieve.ts
index 9b9362f..bcf46c6 100644
--- a/packages/core/src/retrieve.ts
+++ b/packages/core/src/retrieve.ts
@@ -125,14 +125,20 @@ export async function retrieve(input: SearchInput, evaluator: Evaluator): Promis
let commit = 0;
let failure: unknown;
let budgetPaused = false;
- function flush() {
+ function flush(force = false) {
while (commit < ordered.length && ordered[commit]!.state === "done") {
const group = ordered[commit++]!;
for (const result of group.results ?? []) {
- results.push(result);
consume(result.item, result.score);
}
}
+ if (force) {
+ for (; commit < ordered.length; commit++) {
+ const group = ordered[commit]!;
+ if (group.state !== "done") continue;
+ for (const result of group.results ?? []) consume(result.item, result.score);
+ }
+ }
}
function split(group: Group) {
const middle = Math.ceil(group.items.length / 2);
@@ -185,6 +191,7 @@ export async function retrieve(input: SearchInput, evaluator: Evaluator): Promis
if (score > relevanceBoundary) positiveScoresSeen++;
}
await confirmSource(classified);
+ results.push(...classified);
group.results = classified;
group.state = "done";
flush();
@@ -229,6 +236,24 @@ export async function retrieve(input: SearchInput, evaluator: Evaluator): Promis
budgetPaused = false;
pump();
}
+ async function drain() {
+ for (;;) {
+ if (failure || stop || input.signal.aborted) break;
+ if (budgetPaused) {
+ await settleBudget();
+ continue;
+ }
+ pump();
+ if (active.size) {
+ await Promise.race(active);
+ continue;
+ }
+ if (queued.length) continue;
+ break;
+ }
+ await waitForProgress();
+ flush(Boolean(failure || stop || input.signal.aborted));
+ }
async function submit(group: NavigationItem[]) {
const entry: Group = { items: group, state: "queued" };
ordered.push(entry);
@@ -270,14 +295,14 @@ export async function retrieve(input: SearchInput, evaluator: Evaluator): Promis
batch.push(item);
}
if (batch.length) await submit(batch);
- if (budgetPaused) await settleBudget();
+ await drain();
} catch (error) {
failure ??= error;
} finally {
- await waitForProgress();
+ await drain();
}
if (failure) throw failure;
- flush();
+ flush(Boolean(stop || input.signal.aborted));
return results;
}
async function previewDirectory(path: string): Promise<DirectoryPreview | undefined> {
@@ -439,6 +464,7 @@ export async function retrieve(input: SearchInput, evaluator: Evaluator): Promis
const level = directories.splice(0).map((path) => ({ path, depth: 0 }));
const hashes = new Map<string, string>();
async function* items(): AsyncGenerator<NavigationItem> {
+ const expanded: NavigationItem[] = [];
for (let index = 0; index < level.length && !stop; index++) {
const current = level[index]!;
if (entriesSeen >= 100_000) {
@@ -498,25 +524,28 @@ export async function retrieve(input: SearchInput, evaluator: Evaluator): Promis
) {
for (const chunk of splitSource(source, 12_000)) {
const text = sourceForUnit(source, chunk);
- yield buffered(
- {
- path: entry.path,
- kind: "file",
- filePreview: {
- sizeBytes: filePreview.sizeBytes,
- extension: filePreview.extension,
- text,
- previewBytes: Buffer.byteLength(text),
- truncated: true,
- range: "sampled source ranges",
+ expanded.push(
+ buffered(
+ {
+ path: entry.path,
+ kind: "file",
+ filePreview: {
+ sizeBytes: filePreview.sizeBytes,
+ extension: filePreview.extension,
+ text,
+ previewBytes: Buffer.byteLength(text),
+ truncated: true,
+ range: "sampled source ranges",
+ },
},
- },
- [source],
+ [source],
+ ),
);
}
} else yield item;
}
}
+ yield* expanded;
}
}
const classified: Array<{ item: NavigationItem; score: number }> = [];
diff --git a/test/discovery.test.ts b/test/discovery.test.ts
index e6832e3..ad594b9 100644
--- a/test/discovery.test.ts
+++ b/test/discovery.test.ts
@@ -865,8 +865,7 @@ testIfDocker(
{ root, query: "late match", signal: new AbortController().signal },
{
requests: 0,
- async evaluate(request, policy) {
- await policy?.beforeAttempt?.();
+ async evaluate(request) {
const state = request.state as {
items?: Array<{ path: string; filePreview?: { text: string } }>;
};
@@ -981,3 +980,67 @@ testIfDocker("discovery propagates result-processing errors", async () => {
await rm(root, { recursive: true, force: true });
}
});
+
+testIfDocker(
+ "completed navigation groups survive a terminal stop behind an unfinished split",
+ async () => {
+ const root = await mkdtemp(join(tmpdir(), "jg-discovery-completed-stop-"));
+ try {
+ for (let index = 0; index < 257; index++)
+ await writeFile(join(root, `f${String(index).padStart(3, "0")}.txt`), `entry ${index}\n`);
+ const result = await retrieve(
+ { root, query: "completed group", signal: new AbortController().signal },
+ {
+ requests: 0,
+ async evaluate(request) {
+ const items = (request.state as Body["state"]).items;
+ if (items) {
+ if (items[0]?.path === "f000.txt") throw new EvaluationFailure("provider", true);
+ if (items[0]?.path === "f050.txt") throw new EvaluationFailure("request-limit");
+ }
+ return Object.fromEntries(Object.keys(request.questions).map((id) => [id, 0.9]));
+ },
+ },
+ );
+ expect(result.status).toBe("incomplete");
+ expect(result.files.map(({ path }) => path)).toContain("f025.txt");
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ },
+);
+
+testIfDocker("oversized preview chunks retain main's batch ordering", async () => {
+ const root = await mkdtemp(join(tmpdir(), "jg-oversized-order-"));
+ const requests: Array<{ paths: string[]; bytes: number }> = [];
+ try {
+ await writeFile(
+ join(root, "a.ts"),
+ Array.from(
+ { length: 1500 },
+ (_, index) => `export function f${index}() { return ${index}; }\n`,
+ ).join(""),
+ );
+ await writeFile(join(root, "b.txt"), "b");
+ const result = await retrieve(
+ { root, query: "context ".repeat(3000), signal: new AbortController().signal },
+ {
+ requests: 0,
+ async evaluate(request) {
+ const items = (request.state as Body["state"]).items;
+ if (items)
+ requests.push({
+ paths: items.map((item) => item.path),
+ bytes: Buffer.byteLength(JSON.stringify(request)),
+ });
+ return Object.fromEntries(Object.keys(request.questions).map((id) => [id, 0.9]));
+ },
+ },
+ );
+ expect(result.status).toBe("complete");
+ expect(requests[0]?.paths[0]).toBe("b.txt");
+ expect(requests[0]?.bytes).toBeLessThan(38_000);
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+});