Skip to content

Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.

Order cross-file evidence deterministically

taeskim7a7e27dd829c
diff --git a/packages/core/src/retrieve.ts b/packages/core/src/retrieve.ts
index cfe310e..4175d8a 100644
--- a/packages/core/src/retrieve.ts
+++ b/packages/core/src/retrieve.ts
@@ -57,6 +57,7 @@ export async function retrieve(input: SearchInput, evaluator: Evaluator): Promis
const candidates = new Map<string, { path: string; contentHash: string; score: number }>();
const files = new Map<string, FileEvidence>();
const declarations = new Map<string, SelectionResult["declarations"]>();
+ const groupCounts = new Map<string, number>();
const visited = new Set<string>();
const pruned = new Map<string, NavigationItem>();
const previews = new Map<string, FilePreview>();
@@ -702,6 +703,10 @@ export async function retrieve(input: SearchInput, evaluator: Evaluator): Promis
selection.file.score = candidate.score;
files.set(candidate.path, selection.file);
declarations.set(candidate.path, selection.declarations);
+ groupCounts.set(candidate.path, selection.groups);
+ for (const entry of selection.issues)
+ if (entry.kind !== "source-invalid")
+ issue(entry.kind, entry.count, selection.providerFailure);
};
if (evidence) {
for (const candidate of ordered) {
@@ -713,8 +718,13 @@ export async function retrieve(input: SearchInput, evaluator: Evaluator): Promis
const selectEvidence = async () => {
await select();
const evidence: Evidence[] = [];
- // Donors follow selection completion order; concurrent completion can affect request context.
- for (const path of declarations.keys()) {
+ // Donor order is part of every follow-up request and its cache key. Files are selected
+ // concurrently, so completion order depends on network timing; donors use the order in
+ // which sequential selection would finish them instead: fewer groups first, then path.
+ const donors = [...declarations.keys()].sort(
+ (a, b) => groupCounts.get(a)! - groupCounts.get(b)! || (a < b ? -1 : a > b ? 1 : 0),
+ );
+ for (const path of donors) {
const candidate = candidates.get(path)!;
if (stop || input.signal.aborted) break;
if (!files.get(candidate.path)!.excerpts.length) continue;
diff --git a/packages/core/src/selection.ts b/packages/core/src/selection.ts
index 001905f..c153321 100644
--- a/packages/core/src/selection.ts
+++ b/packages/core/src/selection.ts
@@ -19,6 +19,8 @@ export const sourceConfidenceBoundary = 0.7;
export type SelectionResult = {
file: FileEvidence;
declarations: Array<Pick<SourceUnit, "name" | "range">>;
+ /** Declaration requests this file needs per pass; sequential selection finished in this order. */
+ groups: number;
issues: Array<{ kind: string; count: number }>;
providerFailure?: string;
};
@@ -409,6 +411,7 @@ export async function selectFile(
return {
file,
declarations: units.map(({ name, range }) => ({ name, range })),
+ groups: groups.length,
issues: [...issues].map(([kind, count]) => ({ kind, count })),
providerFailure,
};
diff --git a/specs/done/jevgrep/README.md b/specs/done/jevgrep/README.md
index 78ee696..64bdd14 100644
--- a/specs/done/jevgrep/README.md
+++ b/specs/done/jevgrep/README.md
@@ -60,8 +60,9 @@ order must not be normalized to make a comparison pass.
cannot be withdrawn retroactively.
- One evaluator owns retries, cancellation and actual request accounting. Cache
entries contain validated answers keyed by exact semantic request and namespace,
- never source or credentials. Completion-order changes can produce novel requests
- on a warm search; identical native requests still reuse successful answers.
+ never source or credentials. Cross-file evidence has a deterministic order, so
+ unchanged selections rebuild identical follow-up requests on a warm search, and
+ identical native requests reuse successful answers.
- Selected and rendered ranges remain distinct, and returned source is attributable
to snapshot bytes. The current renderer places source before detailed reading leads; see the [architecture](../../../docs/architecture.md). Explicit
source-budget overrides do not remove qualifying file locations.
diff --git a/specs/done/jevgrep/choices.md b/specs/done/jevgrep/choices.md
index 61d0003..ace634e 100644
--- a/specs/done/jevgrep/choices.md
+++ b/specs/done/jevgrep/choices.md
@@ -47,13 +47,18 @@ comparison. Owner: [installed runner](../../../evals/implementation/swebench/ins
### Preserve order inside requests while comparing independent requests fairly
-**When:** reference harness and source-selection integration.
-
-Files A and B both produce selected source. If B finishes first, its source enters
-the next cross-file request first. A cached answer can change which file finishes
-first, so it can legitimately produce a differently ordered later request. Sorting
-that source by filename would introduce a different input to Jev, the relevance
-classifier, even if every source byte were retained.
+**When:** reference harness and source-selection integration; donor order revised
+after measuring warm searches.
+
+Files A and B both produce selected source, and both enter the next cross-file
+request. Files are selected concurrently and used to enter in completion order.
+Most files need a single declaration group, so that order depended on provider
+timing: with jittered provider latency, warm searches resent 26 to 36 follow-up
+requests in half to all of the runs. Donors are now ordered explicitly: fewer
+declaration groups first, as sequential selection would finish them, then path.
+A warm search with unchanged selections therefore rebuilds the same follow-up
+requests. Sorting by filename alone would give Jev, the relevance classifier, a
+different input from the measured strategy, even if every source byte were retained.
Navigation has the same property one level up. A directory level can span several
concurrent navigation requests, and its decisions determine the next level's
@@ -75,9 +80,10 @@ preserve every request's internal array and question order
**Gap:** the spec did not prescribe how to control concurrent reference calls in a
fixture. **Reach:** fixture equality is not a promise of identical arrival or
-completion order in a live repository. Production retains completion order for
-cross-file evidence. **Verdict:** sound because it avoids hiding a semantic input
-change. **Confidence:** medium. Owners: [retrieval](../../../packages/core/src/retrieve.ts)
+completion order in a live repository. Where sequential selection would have
+finished files out of group-count order, such as ties or slow parsing, production
+requests differ from that order but stay repeatable. **Verdict:** sound because it
+avoids hiding a semantic input change. **Confidence:** medium. Owners: [retrieval](../../../packages/core/src/retrieve.ts)
and [reference tests](https://github.com/dzhng/jevgrep/blob/80a216bfa0bf04b2ec615ede81f7af32f1c14153/test/reference/).
### Use one child process for the bundled Python interpreter
diff --git a/test/installed.test.mjs b/test/installed.test.mjs
index 221f2e4..00a76de 100644
--- a/test/installed.test.mjs
+++ b/test/installed.test.mjs
@@ -370,7 +370,7 @@ function assertCachedRequestsAreReused(requests, before) {
for (const { raw, body } of requests.slice(before)) {
assert.ok(!seen.has(raw), "An identical successful native request bypassed the cache");
seen.add(raw);
- // Completion order affects request context, so warm reads can produce different cache keys.
+ // Tolerate reordered evidence: this checks cache reuse, not request order.
assert.ok(Array.isArray(body.state.selectedEvidence));
const evidenceContents = (value) =>
JSON.stringify(value.state.selectedEvidence.map((item) => JSON.stringify(item)).sort());
diff --git a/test/retrieval.test.ts b/test/retrieval.test.ts
index 3033336..3b61b2f 100644
--- a/test/retrieval.test.ts
+++ b/test/retrieval.test.ts
@@ -658,3 +658,52 @@ testIfDocker(
},
120_000,
);
+
+testIfDocker(
+ "cross-file evidence order does not depend on which file finishes selection first",
+ async () => {
+ const root = await mkdtemp(join(tmpdir(), "jg-donor-order-"));
+ try {
+ // Twenty declarations need three groups; one declaration needs one.
+ await writeFile(
+ join(root, "many.ts"),
+ Array.from({ length: 20 }, (_, i) => `export function many${i}() {return ${i};}\n`).join(
+ "",
+ ),
+ );
+ await writeFile(join(root, "few.ts"), "export function few() {return 0;}\n");
+ const donorOrders = async (slow: string) => {
+ const orders: string[][] = [];
+ await retrieve(
+ { root, query: "event behavior", signal: new AbortController().signal },
+ {
+ requests: 0,
+ async evaluate(request, policy) {
+ await policy?.beforeAttempt?.();
+ const state = request.state as {
+ path?: string;
+ declarations?: unknown[];
+ selectedEvidence?: Array<{ path: string }>;
+ };
+ // Delay one file's first pass so it finishes selection after the other.
+ if (state.declarations && !state.selectedEvidence && state.path === slow)
+ await new Promise((resolve) => setTimeout(resolve, 50));
+ if (state.selectedEvidence)
+ orders.push([...new Set(state.selectedEvidence.map((entry) => entry.path))]);
+ return Object.fromEntries(Object.keys(request.questions).map((id) => [id, 0.9]));
+ },
+ },
+ );
+ return orders;
+ };
+ for (const slow of ["few.ts", "many.ts"]) {
+ const orders = await donorOrders(slow);
+ expect(orders.length).toBeGreaterThan(0);
+ // Fewer declaration groups first, as sequential selection used to finish them.
+ for (const order of orders) expect(order).toEqual(["few.ts", "many.ts"]);
+ }
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ },
+);