Skip to content

Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.

Add answer-only evaluation cache with atomic publication

David Zhang5029449999c1
diff --git a/packages/core/src/cache.ts b/packages/core/src/cache.ts
new file mode 100644
index 0000000..b71082a
--- /dev/null
+++ b/packages/core/src/cache.ts
@@ -0,0 +1,258 @@
+import { createHash, randomUUID } from "node:crypto";
+import { constants } from "node:fs";
+import { chmod, lstat, mkdir, open, opendir, rename, rm, unlink } from "node:fs/promises";
+import { join, resolve } from "node:path";
+
+export type JsonValue =
+ | null
+ | boolean
+ | number
+ | string
+ | JsonValue[]
+ | { [key: string]: JsonValue };
+export type CacheInput = {
+ request: JsonValue;
+ namespace: { model: string; provider: string; policyVersion: string; [key: string]: string };
+};
+export type CacheAnswers = Record<string, number>;
+export type CacheIssue = "cache_unavailable" | "cache_corrupt" | "cache_limit";
+export type CacheOptions = {
+ /** CLI owns XDG resolution; this directory contains only Jevgrep cache data. */
+ directory: string;
+ enabled?: boolean;
+ ttlMs?: number;
+ maxBytes?: number;
+ now?: () => number;
+};
+const schema = 1;
+const entryName = /^[a-f0-9]{64}\.json$/;
+const pendingName = /^\.pending-[a-f0-9-]+$/;
+
+function validAnswers(value: unknown): value is CacheAnswers {
+ return (
+ !!value &&
+ typeof value === "object" &&
+ !Array.isArray(value) &&
+ Object.values(value).every((answer) => typeof answer === "number" && Number.isFinite(answer))
+ );
+}
+function missing(error: unknown) {
+ return (error as NodeJS.ErrnoException).code === "ENOENT";
+}
+
+/** Answer-only, best-effort storage; the evaluator must validate success before calling put. */
+export function createEvaluationCache(options: CacheOptions) {
+ const directory = resolve(options.directory);
+ const entries = join(directory, "entries");
+ const enabled = options.enabled !== false;
+ const ttlMs = options.ttlMs ?? 7 * 24 * 60 * 60 * 1000;
+ const maxBytes = options.maxBytes ?? 256 * 1024 * 1024;
+ const now = options.now ?? Date.now;
+ if (
+ !Number.isSafeInteger(ttlMs) ||
+ ttlMs <= 0 ||
+ !Number.isSafeInteger(maxBytes) ||
+ maxBytes <= 0
+ )
+ throw new Error("Invalid cache limits");
+ const maxEntryBytes = Math.min(maxBytes, 1024 * 1024);
+ const issues = new Map<CacheIssue, number>();
+ let hits = 0,
+ misses = 0;
+ function warn(kind: CacheIssue) {
+ issues.set(kind, (issues.get(kind) ?? 0) + 1);
+ }
+ function key(input: CacheInput) {
+ // Preserve request/question ordering. Only this digest, never the serialized request, reaches disk.
+ return createHash("sha256")
+ .update(JSON.stringify([schema, input.namespace, input.request]))
+ .digest("hex");
+ }
+ async function checkDirectory(path: string, create: boolean) {
+ if (create) await mkdir(path, { recursive: true, mode: 0o700 });
+ const info = await lstat(path);
+ if (!info.isDirectory() || info.isSymbolicLink())
+ throw new Error("Cache path is not a directory");
+ if (create && (info.mode & 0o777) !== 0o700) await chmod(path, 0o700);
+ }
+ async function prepare(create: boolean) {
+ await checkDirectory(directory, create);
+ await checkDirectory(entries, create);
+ }
+ async function get(input: CacheInput): Promise<CacheAnswers | undefined> {
+ if (!enabled) {
+ misses++;
+ return undefined;
+ }
+ let handle;
+ try {
+ await prepare(false);
+ handle = await open(
+ join(entries, `${key(input)}.json`),
+ constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK,
+ );
+ const info = await handle.stat();
+ if (!info.isFile() || info.size > maxEntryBytes) {
+ warn("cache_corrupt");
+ misses++;
+ return undefined;
+ }
+ const buffer = Buffer.allocUnsafe(maxEntryBytes + 1);
+ let length = 0;
+ while (length < buffer.length) {
+ const result = await handle.read(buffer, length, buffer.length - length, null);
+ if (!result.bytesRead) break;
+ length += result.bytesRead;
+ }
+ if (length > maxEntryBytes) {
+ warn("cache_corrupt");
+ misses++;
+ return undefined;
+ }
+ let value;
+ try {
+ value = JSON.parse(
+ new TextDecoder("utf-8", { fatal: true }).decode(buffer.subarray(0, length)),
+ );
+ } catch {
+ warn("cache_corrupt");
+ misses++;
+ return undefined;
+ }
+ if (
+ value?.schema !== schema ||
+ typeof value.createdAt !== "number" ||
+ !Number.isFinite(value.createdAt) ||
+ !validAnswers(value.answers)
+ ) {
+ warn("cache_corrupt");
+ misses++;
+ return undefined;
+ }
+ const age = now() - value.createdAt;
+ if (age < 0 || age >= ttlMs) {
+ misses++;
+ return undefined;
+ }
+ hits++;
+ return value.answers;
+ } catch (error) {
+ if (!missing(error)) warn("cache_unavailable");
+ misses++;
+ return undefined;
+ } finally {
+ try {
+ await handle?.close();
+ } catch {
+ warn("cache_unavailable");
+ }
+ }
+ }
+ async function remove(path: string) {
+ try {
+ await unlink(path);
+ } catch (error) {
+ if (!missing(error)) throw error;
+ }
+ }
+ async function trim() {
+ // Streaming retention bounds memory even with many tiny entries. Eviction is deliberately not LRU.
+ let retainedBytes = 0;
+ const scan = await opendir(entries);
+ for await (const entry of scan) {
+ if (!entryName.test(entry.name) && !pendingName.test(entry.name)) continue;
+ const path = join(entries, entry.name);
+ try {
+ const info = await lstat(path);
+ if (!info.isFile() || info.isSymbolicLink()) continue;
+ if (pendingName.test(entry.name)) {
+ // Abandoned publications after a crash must not accumulate forever; current writes get an hour.
+ if (Date.now() - info.mtimeMs > 60 * 60 * 1000) await remove(path);
+ continue;
+ }
+ if (retainedBytes + info.size > maxBytes) await remove(path);
+ else retainedBytes += info.size;
+ } catch (error) {
+ if (!missing(error)) warn("cache_unavailable");
+ }
+ }
+ }
+ async function cleanupDetached() {
+ const scan = await opendir(directory);
+ for await (const entry of scan) {
+ if (!/^\.cleared-[a-f0-9-]+$/.test(entry.name)) continue;
+ try {
+ await rm(join(directory, entry.name), { recursive: true, force: true });
+ } catch (error) {
+ if (!missing(error)) warn("cache_unavailable");
+ }
+ }
+ }
+ async function put(input: CacheInput, answers: CacheAnswers): Promise<void> {
+ if (!enabled) return;
+ let temporary: string | undefined;
+ let handle;
+ try {
+ if (!validAnswers(answers)) {
+ warn("cache_corrupt");
+ return;
+ }
+ const payload = JSON.stringify({ schema, createdAt: now(), answers });
+ if (Buffer.byteLength(payload) > maxEntryBytes) {
+ warn("cache_limit");
+ return;
+ }
+ const destination = join(entries, `${key(input)}.json`);
+ await prepare(true);
+ await cleanupDetached();
+ temporary = join(entries, `.pending-${randomUUID()}`);
+ handle = await open(
+ temporary,
+ constants.O_WRONLY | constants.O_CREAT | constants.O_EXCL | constants.O_NOFOLLOW,
+ 0o600,
+ );
+ await handle.writeFile(payload);
+ await handle.close();
+ handle = undefined;
+ await rename(temporary, destination);
+ temporary = undefined;
+ await trim();
+ } catch {
+ warn("cache_unavailable");
+ } finally {
+ try {
+ await handle?.close();
+ } catch {
+ warn("cache_unavailable");
+ }
+ if (temporary) {
+ try {
+ await remove(temporary);
+ } catch {
+ warn("cache_unavailable");
+ }
+ }
+ }
+ }
+ async function clear(): Promise<void> {
+ // Detach the current generation atomically. Later writers create a new one; no global lock or pause.
+ const detached = join(directory, `.cleared-${randomUUID()}`);
+ try {
+ await checkDirectory(directory, false);
+ try {
+ await checkDirectory(entries, false);
+ await rename(entries, detached);
+ } catch (error) {
+ if (!missing(error)) throw error;
+ }
+ await cleanupDetached();
+ } catch (error) {
+ if (!missing(error)) warn("cache_unavailable");
+ }
+ }
+ function stats() {
+ return { hits, misses, issues: [...issues].map(([kind, count]) => ({ kind, count })) };
+ }
+ return { get, put, clear, stats };
+}
+export type EvaluationCache = ReturnType<typeof createEvaluationCache>;
diff --git a/packages/core/test/cache.test.ts b/packages/core/test/cache.test.ts
new file mode 100644
index 0000000..1570b2b
--- /dev/null
+++ b/packages/core/test/cache.test.ts
@@ -0,0 +1,208 @@
+import { afterEach, expect, test } from "bun:test";
+import {
+ mkdtemp,
+ readdir,
+ readFile,
+ rm,
+ stat,
+ writeFile,
+ chmod,
+ mkdir,
+ symlink,
+} from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { createEvaluationCache, type CacheInput } from "../src/cache";
+
+const roots: string[] = [];
+afterEach(async () => {
+ await Promise.all(roots.splice(0).map((root) => rm(root, { recursive: true, force: true })));
+});
+async function directory() {
+ const root = await mkdtemp(join(tmpdir(), "jevgrep-cache-"));
+ roots.push(root);
+ return join(root, "cache");
+}
+const input = {
+ namespace: {
+ model: "jev",
+ provider: "gateway",
+ policyVersion: "policy1",
+ promptVersion: "prompt1",
+ },
+ request: {
+ state: {
+ source: "SOURCE_SENTINEL private code",
+ credential: "API_KEY_SENTINEL",
+ path: "source.ts",
+ },
+ questions: ["question1", "question2"],
+ },
+} satisfies CacheInput;
+
+test("exact requests reuse numeric answers without persisting source or credentials", async () => {
+ const dir = await directory();
+ const cache = createEvaluationCache({ directory: dir });
+ expect(await cache.get(input)).toBeUndefined();
+ await cache.put(input, { question1: 0.75, question2: 0 });
+ expect(await cache.get(input)).toEqual({ question1: 0.75, question2: 0 });
+ const names = await readdir(join(dir, "entries"));
+ expect(names.every((name) => /^[a-f0-9]{64}\.json$/.test(name))).toBe(true);
+ const saved = await readFile(join(dir, "entries", names[0]!), "utf8");
+ expect(saved).not.toContain("SOURCE_SENTINEL");
+ expect(saved).not.toContain("source.ts");
+ expect(saved).not.toContain("API_KEY_SENTINEL");
+ expect((await stat(join(dir, "entries", names[0]!))).mode & 0o777).toBe(0o600);
+ expect((await stat(dir)).mode & 0o777).toBe(0o700);
+ expect(cache.stats()).toEqual({ hits: 1, misses: 1, issues: [] });
+});
+
+test("source, path, query, ordered questions and namespace changes cannot reuse another request", async () => {
+ const cache = createEvaluationCache({ directory: await directory() });
+ await cache.put(input, { question1: 0.8 });
+ const variants: CacheInput[] = [
+ {
+ ...input,
+ request: {
+ ...input.request,
+ state: { ...input.request.state, source: "different same-size source" },
+ },
+ },
+ {
+ ...input,
+ request: { ...input.request, state: { ...input.request.state, path: "other.ts" } },
+ },
+ { ...input, request: { ...input.request, questions: ["question2", "question1"] } },
+ { ...input, request: { ...input.request, query: "different question" } },
+ ...["model", "provider", "policyVersion", "promptVersion"].map((name) => ({
+ ...input,
+ namespace: { ...input.namespace, [name]: "changed" },
+ })),
+ ];
+ for (const variant of variants) expect(await cache.get(variant)).toBeUndefined();
+ expect(await cache.get(input)).toEqual({ question1: 0.8 });
+});
+
+test("TTL expiry and disabled caching are misses without filesystem side effects", async () => {
+ let clock = 1000;
+ const dir = await directory();
+ const cache = createEvaluationCache({ directory: dir, ttlMs: 100, now: () => clock });
+ await cache.put(input, { question1: 0.5 });
+ clock = 1099;
+ expect(await cache.get(input)).toEqual({ question1: 0.5 });
+ clock = 1100;
+ expect(await cache.get(input)).toBeUndefined();
+ expect(cache.stats().issues).toEqual([]);
+ const disabledDir = await directory();
+ const disabled = createEvaluationCache({ directory: disabledDir, enabled: false });
+ await disabled.put(input, { question1: 0.9 });
+ expect(await disabled.get(input)).toBeUndefined();
+ expect(await stat(disabledDir).catch(() => undefined)).toBeUndefined();
+ const bypass = createEvaluationCache({
+ directory: dir,
+ enabled: false,
+ ttlMs: 100,
+ now: () => 1050,
+ });
+ expect(await bypass.get(input)).toBeUndefined();
+ await bypass.clear();
+ await bypass.clear();
+ expect(await readdir(dir)).toEqual([]);
+});
+
+test("corrupt, incompatible and unreadable entries degrade to counted misses", async () => {
+ const dir = await directory();
+ const cache = createEvaluationCache({ directory: dir });
+ await cache.put(input, { question1: 0.8 });
+ const file = join(dir, "entries", (await readdir(join(dir, "entries")))[0]!);
+ for (const contents of [
+ "{broken",
+ JSON.stringify({ schema: 1, createdAt: Date.now(), answers: { question1: "not numeric" } }),
+ JSON.stringify({ schema: 100, createdAt: Date.now(), answers: { question1: 1 } }),
+ ]) {
+ await writeFile(file, contents);
+ expect(await cache.get(input)).toBeUndefined();
+ }
+ await cache.put(input, { question1: 0.2 });
+ await chmod(file, 0);
+ try {
+ expect(await cache.get(input)).toBeUndefined();
+ } finally {
+ await chmod(file, 0o600);
+ }
+ expect(cache.stats().issues).toEqual([
+ { kind: "cache_corrupt", count: 3 },
+ { kind: "cache_unavailable", count: 1 },
+ ]);
+ expect(await cache.get(input)).toEqual({ question1: 0.2 });
+});
+
+test("concurrent writers publish complete answers and clear permits later fresh writes", async () => {
+ const dir = await directory();
+ const caches = [
+ createEvaluationCache({ directory: dir }),
+ createEvaluationCache({ directory: dir }),
+ ];
+ await Promise.all(
+ caches.map((cache, index) => cache.put(input, { question1: index, question2: 1 - index })),
+ );
+ const value = await caches[0]!.get(input);
+ expect([
+ { question1: 0, question2: 1 },
+ { question1: 1, question2: 0 },
+ ]).toContainEqual(value);
+ await Promise.all([
+ caches[0]!.clear(),
+ caches[1]!.put(input, { question1: 0.5 }),
+ caches[0]!.get(input),
+ ]);
+ await caches[1]!.put(input, { question1: 0.25 });
+ expect(await caches[0]!.get(input)).toEqual({ question1: 0.25 });
+ await caches[0]!.clear();
+ await caches[0]!.clear();
+ expect(await caches[1]!.get(input)).toBeUndefined();
+});
+
+test("byte bounds retain valid entries and oversized answers bypass persistence visibly", async () => {
+ const dir = await directory();
+ const cache = createEvaluationCache({ directory: dir, maxBytes: 180 });
+ for (let index = 0; index < 8; index++)
+ await cache.put({ ...input, request: { index } }, { question1: index / 8 });
+ const names = await readdir(join(dir, "entries"));
+ let bytes = 0;
+ for (const name of names) {
+ const file = join(dir, "entries", name);
+ bytes += (await stat(file)).size;
+ expect(JSON.parse(await readFile(file, "utf8")).answers.question1).toBeGreaterThanOrEqual(0);
+ }
+ expect(bytes).toBeLessThanOrEqual(180);
+ expect(bytes).toBeGreaterThan(0);
+ await cache.put(input, { ["large".repeat(100)]: 1 });
+ expect(await cache.get(input)).toBeUndefined();
+ expect(cache.stats().issues).toEqual([{ kind: "cache_limit", count: 1 }]);
+});
+
+test("clear removes an abandoned detached generation even when no active entries remain", async () => {
+ const dir = await directory();
+ const detached = join(dir, ".cleared-aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa");
+ await mkdir(detached, { recursive: true });
+ await writeFile(join(detached, "old.json"), "old answer");
+ const cache = createEvaluationCache({ directory: dir });
+ await cache.clear();
+ expect(await readdir(dir)).toEqual([]);
+});
+
+test("unavailable or symlinked cache locations never throw or modify the target", async () => {
+ const target = await directory();
+ await mkdir(target, { recursive: true });
+ await writeFile(join(target, "sentinel"), "DO_NOT_TOUCH");
+ const alias = await directory();
+ await symlink(target, alias);
+ const cache = createEvaluationCache({ directory: alias });
+ expect(await cache.get(input)).toBeUndefined();
+ await cache.put(input, { question1: 1 });
+ await cache.clear();
+ expect(await readFile(join(target, "sentinel"), "utf8")).toBe("DO_NOT_TOUCH");
+ expect(await readdir(target)).toEqual(["sentinel"]);
+ expect(cache.stats().issues).toEqual([{ kind: "cache_unavailable", count: 3 }]);
+});