Skip to content

Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.

Revalidate evidence at declaration boundaries and honor cached cancellation

David Zhanga23e5f0f2d80
diff --git a/packages/core/src/gateway.ts b/packages/core/src/gateway.ts
index 350864b..4ca4fd3 100644
--- a/packages/core/src/gateway.ts
+++ b/packages/core/src/gateway.ts
@@ -62,6 +62,7 @@ export function createEvaluator(options: {
return requests;
},
async evaluate(request: EvaluationRequest): Promise<Record<string, number>> {
+ if (options.signal.aborted) throw new EvaluationFailure("cancelled");
const cacheInput: CacheInput = {
request,
namespace: {
@@ -73,6 +74,7 @@ export function createEvaluator(options: {
},
};
const cached = await options.cache?.get(cacheInput);
+ if (options.signal.aborted) throw new EvaluationFailure("cancelled");
if (
cached &&
Object.keys(cached).length === Object.keys(request.questions).length &&
diff --git a/packages/core/src/retrieve.ts b/packages/core/src/retrieve.ts
index 2026a3a..adaf1e4 100644
--- a/packages/core/src/retrieve.ts
+++ b/packages/core/src/retrieve.ts
@@ -296,7 +296,7 @@ export async function retrieve(input: SearchInput, evaluator: Evaluator): Promis
excerpts: [],
sourceOmitted: false,
});
- const select = async (evidence?: Evidence[]) =>
+ const select = async (evidence?: () => Promise<Evidence[] | undefined>) =>
parallel(ordered, async (candidate) => {
const source = await unchanged(candidate);
if (!source) return;
@@ -305,7 +305,10 @@ export async function retrieve(input: SearchInput, evaluator: Evaluator): Promis
input.query,
candidate.score,
evaluator,
- evidence,
+ async () => {
+ if (!(await unchanged(candidate))) return null;
+ return { evidence: await evidence?.() };
+ },
files.get(candidate.path),
);
files.set(candidate.path, selection.file);
@@ -327,7 +330,15 @@ export async function retrieve(input: SearchInput, evaluator: Evaluator): Promis
}
if (evidence.length && Buffer.byteLength(JSON.stringify(evidence)) <= 64_000 && !stop)
- await select(evidence);
+ await select(async () => {
+ const current = new Set<string>();
+ for (const path of new Set(evidence.map((entry) => entry.path))) {
+ const candidate = candidates.get(path)!;
+ if (await unchanged(candidate)) current.add(path);
+ }
+ const fresh = evidence.filter((entry) => current.has(entry.path));
+ return fresh.length ? fresh : undefined;
+ });
await parallel(ordered, async (candidate) => {
const source = await unchanged(candidate);
if (!source) return;
diff --git a/packages/core/src/selection.ts b/packages/core/src/selection.ts
index 4e32728..d3c04e1 100644
--- a/packages/core/src/selection.ts
+++ b/packages/core/src/selection.ts
@@ -29,7 +29,7 @@ export async function selectFile(
query: string,
score: number,
evaluator: Evaluator,
- selectedEvidence?: Evidence[],
+ prepare?: () => Promise<{ evidence?: Evidence[] } | null>,
previous?: FileEvidence,
): Promise<SelectionResult> {
const issues = new Map<string, number>();
@@ -101,7 +101,15 @@ export async function selectFile(
pending.push(unit);
}
if (pending.length) groups.push(pending);
+ let invalidated = false;
for (const group of groups) {
+ const prepared = prepare ? await prepare() : {};
+ if (prepared === null) {
+ invalidated = true;
+ selected.length = 0;
+ leads.clear();
+ break;
+ }
const first = Math.max(1, group[0]!.range.startLine - 8),
last = Math.min(lines.length, group.at(-1)!.range.endLine + 8);
const oversizedContext = [...lines.slice(0, 20), ...lines.slice(first - 1, last)].some(
@@ -124,7 +132,7 @@ export async function selectFile(
snapshot.path,
context,
group.map((unit) => ({ name: unit.name, ...unit.range })),
- selectedEvidence,
+ prepared.evidence,
);
try {
const answers = await evaluator.evaluate(request);
@@ -226,7 +234,7 @@ export async function selectFile(
...(partial ? { sourceByteStart: span.start, sourceByteEnd: span.end, partial: true } : {}),
};
}),
- sourceOmitted: false,
+ sourceOmitted: invalidated,
};
return { file, issues: [...issues].map(([kind, count]) => ({ kind, count })) };
}
diff --git a/packages/core/test/selection.test.ts b/packages/core/test/selection.test.ts
index 1eadf02..6c67e05 100644
--- a/packages/core/test/selection.test.ts
+++ b/packages/core/test/selection.test.ts
@@ -96,7 +96,7 @@ test("Python neighbors are rendering context and follow-up passes do not expand
);
return 0;
}),
- evidence,
+ async () => ({ evidence }),
first.file,
);
expect(calls).toBeGreaterThan(0);
@@ -162,7 +162,7 @@ test("partial giant lines stay byte-bounded in requests, selections and additive
"TARGET",
0.8,
evaluator(() => 0),
- [{ path: "other.txt", startLine: 1, endLine: 1, source: "TARGET" }],
+ async () => ({ evidence: [{ path: "other.txt", startLine: 1, endLine: 1, source: "TARGET" }] }),
first.file,
);
expect(second.file.selected).toEqual(first.file.selected);
@@ -190,7 +190,7 @@ test("malformed group answers stay unknown without discarding previous successes
"q",
0.8,
broken,
- [{ path: "other.ts", startLine: 1, endLine: 1, source: "first()" }],
+ async () => ({ evidence: [{ path: "other.ts", startLine: 1, endLine: 1, source: "first()" }] }),
first.file,
);
expect(second.issues).toEqual([{ kind: "provider", count: 1 }]);
@@ -230,7 +230,7 @@ test("previous evidence from changed bytes is discarded explicitly", async () =>
"q",
0.9,
evaluator(() => 0),
- [],
+ async () => ({ evidence: [] }),
old.file,
);
expect(next.issues).toEqual([{ kind: "changed", count: 1 }]);
diff --git a/scripts/test-installed.sh b/scripts/test-installed.sh
index c1ec1b5..898f274 100755
--- a/scripts/test-installed.sh
+++ b/scripts/test-installed.sh
@@ -12,18 +12,39 @@ trap cleanup EXIT
platform=()
if [[ -n "${JEVGREP_TEST_PLATFORM:-}" ]]; then platform=(--platform "$JEVGREP_TEST_PLATFORM"); fi
build_args=()
-if [[ "${1:-}" == "--prebuilt" ]]; then
- if [[ $# -ne 2 || ! -f "$2" ]]; then echo "Usage: $0 [--prebuilt package.tgz]"; exit 1; fi
+package_input=""
+pattern=""
+while [[ $# -gt 0 ]]; do
+ case "$1" in
+ --prebuilt)
+ [[ $# -ge 2 && -f "$2" ]] || { echo "--prebuilt requires a package tarball"; exit 1; }
+ package_input="$2"; shift 2 ;;
+ --case)
+ [[ $# -ge 2 ]] || { echo "--case requires checkpoint, failures, or cache"; exit 1; }
+ case "$2" in
+ checkpoint) pattern="" ;;
+ failures) pattern="provider|malformed|cancellation|interrupt|stdout pipe" ;;
+ cache) pattern="cache" ;;
+ *) echo "Unknown installed test case: $2"; exit 1 ;;
+ esac
+ shift 2 ;;
+ --test-name-pattern)
+ [[ $# -ge 2 ]] || { echo "--test-name-pattern requires a pattern"; exit 1; }
+ pattern="$2"; shift 2 ;;
+ *) echo "Usage: $0 [--prebuilt package.tgz] [--case checkpoint|failures|cache] [--test-name-pattern regex]"; exit 1 ;;
+ esac
+done
+if [[ -n "$package_input" ]]; then
scratch="$(mktemp -d "${TMPDIR:-/tmp}/jevgrep-installed.XXXXXX")"
mkdir -p "$scratch/.package-input" "$scratch/test"
- cp "$2" "$scratch/.package-input/jevgrep.tgz"
+ cp "$package_input" "$scratch/.package-input/jevgrep.tgz"
cp "${JEVGREP_CANONICAL_SKILL:-$repo_root/skills/jevgrep/SKILL.md}" "$scratch/.package-input/canonical-skill.md"
cp "$repo_root/test/installed.test.mjs" "$repo_root/test/runtime.Dockerfile" "$scratch/test/"
context_dir="$scratch"
build_args=(--build-arg PACKAGE_STAGE=prebuilt)
-elif [[ $# -ne 0 ]]; then
- echo "Usage: $0 [--prebuilt package.tgz]"
- exit 1
fi
+node_args=(node --test --test-concurrency=1)
+if [[ -n "$pattern" ]]; then node_args+=(--test-name-pattern "$pattern"); fi
+node_args+=(/test/installed.test.mjs)
docker build "${platform[@]}" "${build_args[@]}" -f "$context_dir/test/runtime.Dockerfile" -t "$image" "$context_dir"
-docker run --rm --network none --read-only --tmpfs /tmp:rw,nosuid,nodev,size=256m "${platform[@]}" "$image"
+docker run --rm --network none --read-only --tmpfs /tmp:rw,nosuid,nodev,size=256m "${platform[@]}" "$image" "${node_args[@]}"
diff --git a/specs/jevgrep/assets/integration-verification.md b/specs/jevgrep/assets/integration-verification.md
index 263f4fa..35b4b7f 100644
--- a/specs/jevgrep/assets/integration-verification.md
+++ b/specs/jevgrep/assets/integration-verification.md
@@ -30,3 +30,24 @@ and permits execution on that isolated mount. Coverage was preserved.
Still unproven: live Requests/Sol checkpoint, final failure/cache requirements,
source-budget choice, ten-task frozen quality gate, Linux/amd64 and native macOS
release verification. Historical spike outcomes are not production outcomes.
+
+## Independent review follow-up
+
+`codex review --commit 3c8835f` reproduced stale donors during the follow-up pass
+and successful cache hits after cancellation. Both were confirmed with failing
+regressions. Selection now asks its caller to prepare each declaration request:
+the filesystem owner refreshes donor eligibility/hashes and checks the target
+snapshot before each group. This also fixes a separately reproduced case where a
+file ignored after its first declaration group was uploaded twice more.
+Cancellation is checked before and after asynchronous cache lookup.
+
+Focused verification: 14 retrieval/selection tests pass in Docker; four HTTP
+evaluator tests pass, including rate-limit cancellation and Retry-After recovery.
+The installed suite passed eight journeys before these last retrieval fixes;
+`bun run test:e2e -- --case failures` selects its three current failure cases
+(malformed answers, authentication failure, transient recovery) and passed.
+This is not the complete slice 05 matrix yet.
+
+A live `jg doctor` completed successfully through AI Gateway with the authorized
+Duet staging credential, loaded through its existing dotenvx mechanism. No secret
+was printed. This verifies live connectivity only; the official task remains next.
diff --git a/specs/jevgrep/choices.md b/specs/jevgrep/choices.md
index 2377524..199d773 100644
--- a/specs/jevgrep/choices.md
+++ b/specs/jevgrep/choices.md
@@ -53,11 +53,11 @@ cannot accidentally broaden a follow-up upload. Verdict: sound. Confidence: high
### Revalidate selected evidence before sharing it across files
When: installed retrieval integration. Files can change or become ignored while
-Jev evaluates other files. Before building the cross-file evidence request, check
-that each donor is still eligible and has the same content hash. Remove stale
+Jev evaluates other files. Before each declaration evaluation, check that the target and every cross-file
+donor are still eligible and have the same content hash. Remove stale
excerpts and report incomplete results while retaining admitted file locations.
-Gap: the spec required fresh snapshots but did not prescribe the cross-file
+Gap: the spec required fresh snapshots but did not prescribe the per-request
validation point. Reach: an answer can contain fewer excerpts after a concurrent
edit; it will not silently present the saved bytes as current. Verdict: sound.
Confidence: high. This is a bounded snapshot check, not an atomic filesystem lock.
diff --git a/specs/jevgrep/contracts.md b/specs/jevgrep/contracts.md
index a575c01..21ac55f 100644
--- a/specs/jevgrep/contracts.md
+++ b/specs/jevgrep/contracts.md
@@ -67,27 +67,20 @@ Keep `apps/cli` (credentials/process/rendering) and `packages/core` (retrieval)
plus existing TypeScript configuration. Test helpers remain under `test/`.
Official evaluation tools remain development-only and must not enter the package.
-```ts
-type Range = { startLine: number; endLine: number }; // one-based, inclusive
-type Snapshot = { path: string; contentHash: string; source: string };
-type SourceUnit = { id: string; name: string; range: Range };
-type Lead = { unit: SourceUnit; score: number };
-type FileEvidence = {
- path: string; contentHash: string; score: number; roles: string[];
- leads: Lead[]; selected: Range[]; rendered: Range[];
- excerpts: Array<{ range: Range; source: string }>;
- sourceOmitted: boolean;
-};
-type RetrievalResult = {
- root: string; query: string; status: 'complete' | 'incomplete' | 'interrupted';
- files: FileEvidence[];
- issues: Array<{ kind: string; count: number }>;
- counts: { requests: number; cacheHits: number; inspectedFiles: number };
-};
-// Names may change; these semantic boundaries must survive.
-retrieve(input: { root: string; query: string; policy: SearchPolicy;
- signal: AbortSignal }, dependencies: RetrievalDependencies): Promise<RetrievalResult>;
-```
+The implemented schemas live in [retrieval types](../../packages/core/src/types.ts),
+[source units](../../packages/core/src/source.ts), and
+[filesystem snapshots](../../packages/core/src/filesystem.ts). `retrieve` accepts a
+root, query, policy, cancellation signal and evaluator; it returns file evidence,
+completion status, counted issues and request/cache statistics. Cache warnings
+are separate from missing-evidence issues.
+
+A source unit has a name, inclusive one-based line range, and half-open UTF-8 byte
+coordinates in its snapshot. A reading lead carries the name, range and relevance
+score without duplicating the parser unit. File evidence distinguishes selected
+ranges from expanded rendered ranges and includes exact excerpt bytes. Partial
+long-line excerpts also carry byte coordinates, because line numbers alone cannot
+locate a fragment. Snapshot hashes bind all of those coordinates to the original
+source. Names may change; these semantic boundaries must survive.
Paths are relative to the canonical root. Output escapes control characters in
paths; source remains verbatim with attributable ranges. One snapshot owns source,
diff --git a/test/gateway.test.ts b/test/gateway.test.ts
index 86ec07e..ff3ad7b 100644
--- a/test/gateway.test.ts
+++ b/test/gateway.test.ts
@@ -65,3 +65,78 @@ test("transient failures retry within the shared request guard and never become
server.stop(true);
}
});
+
+test("cancellation of a rate-limited request prevents further attempts", async () => {
+ const controller = new AbortController();
+ let calls = 0;
+ let firstResponse!: () => void;
+ const received = new Promise<void>((resolve) => {
+ firstResponse = resolve;
+ });
+ const server = Bun.serve({
+ port: 0,
+ fetch() {
+ calls++;
+ firstResponse();
+ return Response.json(
+ { error: "rate limited" },
+ { status: 429, headers: { "retry-after": "30" } },
+ );
+ },
+ });
+ try {
+ const evaluator = createEvaluator({
+ apiKey: "fixture",
+ baseURL: `http://127.0.0.1:${server.port}`,
+ signal: controller.signal,
+ retryDelayMs: 0,
+ });
+ const result = evaluator.evaluate({
+ state: "test",
+ questions: { q: { type: "boolean", instructions: "Relevant?" } },
+ });
+ await received;
+ controller.abort();
+ await expect(result).rejects.toMatchObject({ kind: "cancelled" });
+ expect(calls).toBe(1);
+ expect(evaluator.requests).toBe(1);
+ } finally {
+ controller.abort();
+ server.stop(true);
+ }
+}, 2000);
+
+test("Retry-After delays a retry before the provider can recover", async () => {
+ const received: number[] = [];
+ const server = Bun.serve({
+ port: 0,
+ fetch() {
+ received.push(performance.now());
+ return received.length === 1
+ ? Response.json(
+ { error: "rate limited" },
+ { status: 429, headers: { "retry-after": "0.1" } },
+ )
+ : Response.json({ answers: { q: { type: "boolean", probability: 0.8 } } });
+ },
+ });
+ try {
+ const evaluator = createEvaluator({
+ apiKey: "fixture",
+ baseURL: `http://127.0.0.1:${server.port}`,
+ signal: new AbortController().signal,
+ retryDelayMs: 0,
+ });
+ expect(
+ await evaluator.evaluate({
+ state: "test",
+ questions: { q: { type: "boolean", instructions: "Relevant?" } },
+ }),
+ ).toEqual({ q: 0.8 });
+ expect(received.length).toBe(2);
+ expect(received[1]! - received[0]!).toBeGreaterThanOrEqual(95);
+ expect(evaluator.requests).toBe(2);
+ } finally {
+ server.stop(true);
+ }
+});
diff --git a/test/installed.test.mjs b/test/installed.test.mjs
index e1bf6cc..bcd2b4e 100644
--- a/test/installed.test.mjs
+++ b/test/installed.test.mjs
@@ -94,6 +94,16 @@ async function context(t, mode = "healthy") {
}
assert.ok(requests.length < 256, "Synthetic search stopped making bounded forward progress");
requests.push({ body, raw });
+ if (mode === "transient" && requests.length === 1) {
+ response.writeHead(500, { "content-type": "application/json" });
+ response.end(JSON.stringify({ error: "temporary fixture failure" }));
+ return;
+ }
+ if (mode === "unauthorized") {
+ response.writeHead(401, { "content-type": "application/json" });
+ response.end(JSON.stringify({ error: "fixture credential rejected" }));
+ return;
+ }
let probabilities;
if (Array.isArray(body.state.items)) {
assert.equal(body.state.query, query);
@@ -367,3 +377,21 @@ test("doctor uses the installed SDK while missing credentials fail cleanly", asy
assert.ok(missing.stdout.trim().length > 0);
assert.equal(fixture.requests.length, before);
});
+
+test("provider authentication failure stops immediately with fatal exit 1", async (t) => {
+ const fixture = await context(t, "unauthorized");
+ const result = await fixture.run([query]);
+ assert.equal(result.code, 1);
+ assert.equal(fixture.requests.length, 1);
+ assert.ok(!result.stdout.includes("Status: complete"));
+ assert.ok(!result.stdout.includes("py-evidence-"));
+});
+
+test("a transient provider failure retries the same request and completes installed retrieval", async (t) => {
+ const fixture = await context(t, "transient");
+ const result = await fixture.run([query]);
+ assert.equal(result.code, 0);
+ assert.match(result.stdout, /Status: complete/);
+ assert.deepEqual(fixture.requests[0].body, fixture.requests[1].body);
+ for (const [branch] of branches) assert.ok(result.stdout.includes(`py-evidence-${branch}:`));
+});
diff --git a/test/retrieval.test.ts b/test/retrieval.test.ts
index 5aecf8d..1dae3e7 100644
--- a/test/retrieval.test.ts
+++ b/test/retrieval.test.ts
@@ -155,3 +155,120 @@ testIfDocker(
}
},
);
+
+testIfDocker(
+ "donors ignored during follow-up are absent from later evaluation requests",
+ async () => {
+ const root = await mkdtemp(join(tmpdir(), "jg-followup-change-"));
+ let calls = 0;
+ let changed = false;
+ let staleUploads = 0;
+ let followups = 0;
+ try {
+ for (let index = 0; index < 12; index++) {
+ await writeFile(
+ join(root, `${index}.ts`),
+ `export function record${index}() {return true;}\n`,
+ );
+ }
+ const result = await retrieve(
+ { root, query: "record behavior", signal: new AbortController().signal },
+ {
+ get requests() {
+ return calls;
+ },
+ async evaluate(request) {
+ calls++;
+ const state = request.state as { selectedEvidence?: Array<{ path: string }> };
+ if (state.selectedEvidence) {
+ followups++;
+ if (followups > 8 && state.selectedEvidence.some((entry) => entry.path === "0.ts"))
+ staleUploads++;
+ if (!changed) {
+ changed = true;
+ await writeFile(join(root, ".ignore"), "0.ts\n");
+ }
+ }
+ return Object.fromEntries(Object.keys(request.questions).map((id) => [id, 0.9]));
+ },
+ },
+ );
+ expect(changed).toBe(true);
+ expect(staleUploads).toBe(0);
+ expect(result.status).toBe("incomplete");
+ expect(result.files.find((file) => file.path === "0.ts")?.excerpts).toEqual([]);
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ },
+);
+
+testIfDocker("an aborted evaluator cannot return a previously cached answer", async () => {
+ const { createEvaluationCache } = await import("../packages/core/src/cache");
+ const directory = await mkdtemp(join(tmpdir(), "jg-cache-abort-"));
+ const controller = new AbortController();
+ let calls = 0;
+ const server = Bun.serve({
+ port: 0,
+ fetch() {
+ calls++;
+ return Response.json({ answers: { q: { type: "boolean", probability: 0.8 } } });
+ },
+ });
+ try {
+ const evaluator = createEvaluator({
+ apiKey: "fixture",
+ baseURL: `http://127.0.0.1:${server.port}`,
+ signal: controller.signal,
+ cache: createEvaluationCache({ directory }),
+ });
+ const request = {
+ state: "test",
+ questions: { q: { type: "boolean" as const, instructions: "Relevant?" } },
+ };
+ expect(await evaluator.evaluate(request)).toEqual({ q: 0.8 });
+ expect(await evaluator.evaluate(request)).toEqual({ q: 0.8 });
+ expect(evaluator.cacheHits).toBe(1);
+ controller.abort();
+ await expect(evaluator.evaluate(request)).rejects.toMatchObject({ kind: "cancelled" });
+ expect(calls).toBe(1);
+ } finally {
+ server.stop(true);
+ await rm(directory, { recursive: true, force: true });
+ }
+});
+
+testIfDocker(
+ "a file excluded during declaration selection is not uploaded in later groups",
+ async () => {
+ const root = await mkdtemp(join(tmpdir(), "jg-group-change-"));
+ let declarations = 0;
+ try {
+ await writeFile(
+ join(root, "events.ts"),
+ Array.from({ length: 20 }, (_, i) => `export function event${i}() {return true;}\n`).join(
+ "",
+ ),
+ );
+ const result = await retrieve(
+ { root, query: "event behavior", signal: new AbortController().signal },
+ {
+ requests: 0,
+ async evaluate(request) {
+ const state = request.state as { declarations?: unknown[] };
+ if (state.declarations) {
+ declarations++;
+ if (declarations === 1) await writeFile(join(root, ".ignore"), "events.ts\n");
+ }
+ return Object.fromEntries(Object.keys(request.questions).map((id) => [id, 0.9]));
+ },
+ },
+ );
+ expect(declarations).toBe(1);
+ expect(result.status).toBe("incomplete");
+ expect(result.files[0]?.excerpts).toEqual([]);
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+ },
+);