Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.
TypeScript ai-sdkclaude-codeclicode-searchcodexcoding-agentscontext-retrievaldeveloper-toolsjevsemantic-searchtypescriptvercel-ai-gateway
Sweep the answer cache periodically instead of on every write
taeskimdc865e47097b
diff --git a/packages/core/src/cache.ts b/packages/core/src/cache.tsindex ea766dc..b323aff 100644--- a/packages/core/src/cache.ts+++ b/packages/core/src/cache.ts@@ -27,6 +27,13 @@ export type CacheOptions = {const schema = 1;const entryName = /^[a-f0-9]{64}\.json$/;const pendingName = /^\.pending-[a-f0-9-]+$/;+// Abandoned publications after a crash must not accumulate forever; current writes get an hour.+const pendingLifetimeMs = 60 * 60 * 1000;+const maintenanceIntervalMs = 60 * 60 * 1000;+/** Tiny answer files still occupy whole filesystem blocks; the byte bound counts that space. */+function allocated(size: number, blocks = 0) {+ return Math.max(Math.ceil(size / 4096) * 4096, blocks * 512);+}function validAnswers(value: unknown): value is CacheAnswers {return (@@ -155,27 +162,103 @@ export function createEvaluationCache(options: CacheOptions) {if (!missing(error)) throw error;}}- async function trim() {- // Streaming retention bounds memory even with many tiny entries. Eviction is deliberately not LRU.- let retainedBytes = 0;- const scan = await opendir(entries);- for await (const entry of scan) {- if (!entryName.test(entry.name) && !pendingName.test(entry.name)) continue;- const path = join(entries, entry.name);+ let checked = false;+ let sweeping: Promise<void> | undefined;+ /** Allocated bytes seen by the last sweep plus this process's writes since then. */+ let estimate = 0;+ // Its mtime records the last completed sweep by any process, on the cache clock.+ const stamp = join(directory, "maintained");+ async function maintenanceDue() {+ try {+ const info = await lstat(stamp);+ const age = now() - info.mtimeMs;+ return !info.isFile() || age < 0 || age >= maintenanceIntervalMs;+ } catch {+ return true;+ }+ }+ /** A directory pass runs at most once per interval across processes, or when this process's+ * writes exceed the budget. Entry age is its mtime on the cache clock, so no entry is opened.+ * Between passes the budget can be exceeded by recent writes. Eviction is deliberately not LRU:+ * reads never write, and the oldest answers go first. */+ async function maintain(): Promise<number> {+ await cleanupDetached();+ const names: string[] = [];+ for await (const entry of await opendir(entries))+ if (entryName.test(entry.name) || pendingName.test(entry.name)) names.push(entry.name);+ const retained: Array<{ name: string; mtimeMs: number; bytes: number }> = [];+ let total = 0;+ await pooled(names, async (name) => {+ const path = join(entries, name);try {const info = await lstat(path);- if (!info.isFile() || info.isSymbolicLink()) continue;- if (pendingName.test(entry.name)) {- // Abandoned publications after a crash must not accumulate forever; current writes get an hour.- if (Date.now() - info.mtimeMs > 60 * 60 * 1000) await remove(path);- continue;+ // lstat does not follow links: symlinks and special files are skipped, never touched.+ if (!info.isFile()) return;+ // Future timestamps are other writers' fresh entries or clock changes, never expiry.+ const age = now() - info.mtimeMs;+ if (pendingName.test(name)) {+ if (age > pendingLifetimeMs) await remove(path);+ } else if (age >= ttlMs) await remove(path);+ else {+ const bytes = allocated(info.size, info.blocks);+ retained.push({ name, mtimeMs: info.mtimeMs, bytes });+ total += bytes;}- if (retainedBytes + info.size > maxBytes) await remove(path);- else retainedBytes += info.size;} catch (error) {if (!missing(error)) warn("cache_unavailable");}+ });+ if (total > maxBytes) {+ // Leave headroom so the next few writes do not immediately trigger another pass.+ const target = maxBytes - Math.floor(maxBytes / 8);+ retained.sort((a, b) => a.mtimeMs - b.mtimeMs || (a.name < b.name ? -1 : 1));+ const evicted: typeof retained = [];+ for (const entry of retained) {+ if (total <= target) break;+ evicted.push(entry);+ total -= entry.bytes;+ }+ await pooled(evicted, async (entry) => {+ try {+ await remove(join(entries, entry.name));+ } catch {+ total += entry.bytes;+ warn("cache_unavailable");+ }+ });}+ return total;+ }+ /** Bounded parallel filesystem work; each task handles its own errors. */+ async function pooled<T>(items: T[], work: (item: T) => Promise<void>) {+ let next = 0;+ await Promise.all(+ Array.from({ length: Math.min(16, items.length) }, async () => {+ while (next < items.length) await work(items[next++]!);+ }),+ );+ }+ function sweep() {+ sweeping ??= (async () => {+ try {+ estimate = await maintain();+ const handle = await open(+ stamp,+ constants.O_WRONLY | constants.O_CREAT | constants.O_NOFOLLOW,+ 0o600,+ );+ try {+ await handle.utimes(now() / 1000, now() / 1000);+ } finally {+ await handle.close();+ }+ } catch (error) {+ if (!missing(error)) warn("cache_unavailable");+ } finally {+ sweeping = undefined;+ }+ })();+ return sweeping;}async function cleanupDetached() {const scan = await opendir(directory);@@ -197,14 +280,15 @@ export function createEvaluationCache(options: CacheOptions) {warn("cache_corrupt");return;}- const payload = JSON.stringify({ schema, createdAt: now(), answers });- if (Buffer.byteLength(payload) > maxEntryBytes) {+ const createdAt = now();+ const payload = JSON.stringify({ schema, createdAt, answers });+ const size = Buffer.byteLength(payload);+ if (size > maxEntryBytes || allocated(size) > maxBytes) {warn("cache_limit");return;}const destination = join(entries, `${key(input)}.json`);await prepare(true);- await cleanupDetached();temporary = join(entries, `.pending-${randomUUID()}`);handle = await open(temporary,@@ -212,11 +296,17 @@ export function createEvaluationCache(options: CacheOptions) {0o600,);await handle.writeFile(payload);+ // Maintenance reads age from mtime; keep it on the same clock as createdAt.+ await handle.utimes(createdAt / 1000, createdAt / 1000);await handle.close();handle = undefined;await rename(temporary, destination);temporary = undefined;- await trim();+ estimate += allocated(size);+ if (!checked) {+ checked = true;+ if (await maintenanceDue()) await sweep();+ } else if (estimate > maxBytes) await sweep();} catch {warn("cache_unavailable");} finally {@@ -242,9 +332,11 @@ export function createEvaluationCache(options: CacheOptions) {try {await checkDirectory(entries, false);await rename(entries, detached);+ estimate = 0;} catch (error) {if (!missing(error)) throw error;}+ await remove(stamp);await cleanupDetached();} catch (error) {if (!missing(error)) warn("cache_unavailable");diff --git a/packages/core/test/cache.test.ts b/packages/core/test/cache.test.tsindex 2230103..0625b29 100644--- a/packages/core/test/cache.test.ts+++ b/packages/core/test/cache.test.ts@@ -10,6 +10,7 @@ import {chmod,mkdir,symlink,+ utimes,} from "node:fs/promises";import { tmpdir } from "node:os";import { join } from "node:path";@@ -170,25 +171,65 @@ test("concurrent writers publish complete answers and clear permits later freshexpect(await caches[1]!.get(input)).toBeUndefined();});-test("byte bounds retain valid entries and oversized answers bypass persistence visibly", async () => {+test("byte bounds count allocated blocks, evict the oldest answers and reject oversized ones", async () => {+ let clock = Date.now();+ const budget = 3 * 4096 + 100;const dir = await directory();- const cache = createEvaluationCache({ directory: dir, maxBytes: 180 });- for (let index = 0; index < 8; index++)+ const cache = createEvaluationCache({ directory: dir, maxBytes: budget, now: () => clock });+ for (let index = 0; index < 8; index++) {+ clock += 1000;await cache.put({ ...input, request: { index } }, { question1: index / 8 });+ }const names = await readdir(join(dir, "entries"));- let bytes = 0;+ let allocatedBytes = 0;for (const name of names) {const file = join(dir, "entries", name);- bytes += (await stat(file)).size;+ const info = await stat(file);+ allocatedBytes += Math.max(Math.ceil(info.size / 4096) * 4096, info.blocks * 512);expect(JSON.parse(await readFile(file, "utf8")).answers.question1).toBeGreaterThanOrEqual(0);}- expect(bytes).toBeLessThanOrEqual(180);- expect(bytes).toBeGreaterThan(0);- await cache.put(input, { ["large".repeat(100)]: 1 });+ expect(allocatedBytes).toBeLessThanOrEqual(budget);+ expect(names.length).toBeGreaterThan(0);+ expect(await cache.get({ ...input, request: { index: 7 } })).toEqual({ question1: 7 / 8 });+ expect(await cache.get({ ...input, request: { index: 0 } })).toBeUndefined();+ await cache.put(input, { ["large".repeat(3000)]: 1 });expect(await cache.get(input)).toBeUndefined();expect(cache.stats().issues).toEqual([{ kind: "cache_limit", count: 1 }]);});+test("sweeps run at most hourly across processes and remove expired and abandoned files", async () => {+ const minute = 60 * 1000;+ const start = Date.now();+ let clock = start;+ const dir = await directory();+ const entries = join(dir, "entries");+ const options = { directory: dir, ttlMs: 45 * minute, now: () => clock };+ const writer = createEvaluationCache(options);+ await writer.put({ ...input, request: { index: 0 } }, { question1: 0 });+ const [expired] = await readdir(entries);+ clock = start + 50 * minute;+ const abandoned = ".pending-aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa";+ const hoursAgo = (clock - 120 * minute) / 1000;+ await writeFile(join(entries, abandoned), "partial");+ await utimes(join(entries, abandoned), hoursAgo, hoursAgo);+ for (let index = 1; index <= 3; index++)+ await writer.put({ ...input, request: { index } }, { question1: index });+ // Neither later writes nor another process within the hour rescan the directory.+ const early = createEvaluationCache(options);+ await early.put({ ...input, request: { index: 4 } }, { question1: 4 });+ expect(await readdir(entries)).toEqual(expect.arrayContaining([expired, abandoned]));+ expect(await writer.get({ ...input, request: { index: 0 } })).toBeUndefined();+ clock = start + 70 * minute;+ const late = createEvaluationCache(options);+ await late.put({ ...input, request: { index: 5 } }, { question1: 5 });+ const remaining = await readdir(entries);+ expect(remaining).toHaveLength(5);+ expect(remaining).not.toContain(expired);+ expect(remaining).not.toContain(abandoned);+ expect(await late.get({ ...input, request: { index: 1 } })).toEqual({ question1: 1 });+ expect([writer, early, late].flatMap((cache) => cache.stats().issues)).toEqual([]);+});+test("clear removes an abandoned detached generation even when no active entries remain", async () => {const dir = await directory();const detached = join(dir, ".cleared-aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa");diff --git a/specs/done/jevgrep/choices.md b/specs/done/jevgrep/choices.mdindex 5558876..9caad03 100644--- a/specs/done/jevgrep/choices.md+++ b/specs/done/jevgrep/choices.md@@ -142,22 +142,27 @@ search or optimal treatment of minified source. A change to question grouping isa separate retrieval-policy experiment. **Verdict:** sound under the preservationconstraint. **Confidence:** medium. Owner: [selection](../../../packages/core/src/selection.ts).-### Enforce the cache bound by scanning stored entries+### Enforce the cache bound with periodic oldest-first sweeps-**When:** cache integration and performance review.+**When:** cache integration; revised after measuring per-write scans.A new answer is ready to save while the cache is near its disk limit. The cache-publishes the complete answer atomically, then scans stored entries in filesystem-enumeration order and removes entries beyond the retained byte budget. This is-not oldest-first eviction. It does not maintain a separate persistent index or run a-background cleanup service. With many entries, repeating that scan for many new-answers adds overhead; an indexed eviction design would trade that work for-another stateful component to maintain.--**Gap:** the cache size bound was fixed but its maintenance mechanism was open.-**Reach:** storage is best effort, and large-cache write throughput is limited by-repeated scans. This mechanism makes no throughput claim. **Verdict:** sound as a-simple bounded-storage owner with a disclosed cost. **Confidence:** medium.+publishes the complete answer atomically and sets its modification time to the+answer's creation time; that write does not scan the directory. A sweep runs on a+process's first write when the shared `maintained` stamp is missing, in the future+or more than an hour old, and again when that process's own writes push its+estimate past the budget. The sweep reads metadata only. It removes expired entries+and abandoned publications, then removes the oldest answers until allocated blocks+fall to seven eighths of the budget. Tiny answers are charged whole filesystem+blocks, so the bound tracks disk use rather than payload bytes. There is still no+index or background service.++**Gap:** the cache size bound was fixed but its maintenance mechanism was open. The+first design rescanned every entry after each write, and at 40,000 entries each+write took about a second while holding a request slot. **Reach:** storage remains+best effort. Other processes' writes can exceed the budget until the next sweep,+and the write that triggers a sweep pays for one metadata pass. **Verdict:** sound+as a bounded-storage owner without per-write scans. **Confidence:** medium.Owner: [cache](../../../packages/core/src/cache.ts).## Sound — high confidence