Skip to content

Find code by asking what it does. A CLI for coding agents that uses Jev to discover relevant files and source context.

Sweep the answer cache periodically instead of on every write

taeskimdc865e47097b
diff --git a/packages/core/src/cache.ts b/packages/core/src/cache.ts
index ea766dc..b323aff 100644
--- a/packages/core/src/cache.ts
+++ b/packages/core/src/cache.ts
@@ -27,6 +27,13 @@ export type CacheOptions = {
const schema = 1;
const entryName = /^[a-f0-9]{64}\.json$/;
const pendingName = /^\.pending-[a-f0-9-]+$/;
+// Abandoned publications after a crash must not accumulate forever; current writes get an hour.
+const pendingLifetimeMs = 60 * 60 * 1000;
+const maintenanceIntervalMs = 60 * 60 * 1000;
+/** Tiny answer files still occupy whole filesystem blocks; the byte bound counts that space. */
+function allocated(size: number, blocks = 0) {
+ return Math.max(Math.ceil(size / 4096) * 4096, blocks * 512);
+}
function validAnswers(value: unknown): value is CacheAnswers {
return (
@@ -155,27 +162,103 @@ export function createEvaluationCache(options: CacheOptions) {
if (!missing(error)) throw error;
}
}
- async function trim() {
- // Streaming retention bounds memory even with many tiny entries. Eviction is deliberately not LRU.
- let retainedBytes = 0;
- const scan = await opendir(entries);
- for await (const entry of scan) {
- if (!entryName.test(entry.name) && !pendingName.test(entry.name)) continue;
- const path = join(entries, entry.name);
+ let checked = false;
+ let sweeping: Promise<void> | undefined;
+ /** Allocated bytes seen by the last sweep plus this process's writes since then. */
+ let estimate = 0;
+ // Its mtime records the last completed sweep by any process, on the cache clock.
+ const stamp = join(directory, "maintained");
+ async function maintenanceDue() {
+ try {
+ const info = await lstat(stamp);
+ const age = now() - info.mtimeMs;
+ return !info.isFile() || age < 0 || age >= maintenanceIntervalMs;
+ } catch {
+ return true;
+ }
+ }
+ /** A directory pass runs at most once per interval across processes, or when this process's
+ * writes exceed the budget. Entry age is its mtime on the cache clock, so no entry is opened.
+ * Between passes the budget can be exceeded by recent writes. Eviction is deliberately not LRU:
+ * reads never write, and the oldest answers go first. */
+ async function maintain(): Promise<number> {
+ await cleanupDetached();
+ const names: string[] = [];
+ for await (const entry of await opendir(entries))
+ if (entryName.test(entry.name) || pendingName.test(entry.name)) names.push(entry.name);
+ const retained: Array<{ name: string; mtimeMs: number; bytes: number }> = [];
+ let total = 0;
+ await pooled(names, async (name) => {
+ const path = join(entries, name);
try {
const info = await lstat(path);
- if (!info.isFile() || info.isSymbolicLink()) continue;
- if (pendingName.test(entry.name)) {
- // Abandoned publications after a crash must not accumulate forever; current writes get an hour.
- if (Date.now() - info.mtimeMs > 60 * 60 * 1000) await remove(path);
- continue;
+ // lstat does not follow links: symlinks and special files are skipped, never touched.
+ if (!info.isFile()) return;
+ // Future timestamps are other writers' fresh entries or clock changes, never expiry.
+ const age = now() - info.mtimeMs;
+ if (pendingName.test(name)) {
+ if (age > pendingLifetimeMs) await remove(path);
+ } else if (age >= ttlMs) await remove(path);
+ else {
+ const bytes = allocated(info.size, info.blocks);
+ retained.push({ name, mtimeMs: info.mtimeMs, bytes });
+ total += bytes;
}
- if (retainedBytes + info.size > maxBytes) await remove(path);
- else retainedBytes += info.size;
} catch (error) {
if (!missing(error)) warn("cache_unavailable");
}
+ });
+ if (total > maxBytes) {
+ // Leave headroom so the next few writes do not immediately trigger another pass.
+ const target = maxBytes - Math.floor(maxBytes / 8);
+ retained.sort((a, b) => a.mtimeMs - b.mtimeMs || (a.name < b.name ? -1 : 1));
+ const evicted: typeof retained = [];
+ for (const entry of retained) {
+ if (total <= target) break;
+ evicted.push(entry);
+ total -= entry.bytes;
+ }
+ await pooled(evicted, async (entry) => {
+ try {
+ await remove(join(entries, entry.name));
+ } catch {
+ total += entry.bytes;
+ warn("cache_unavailable");
+ }
+ });
}
+ return total;
+ }
+ /** Bounded parallel filesystem work; each task handles its own errors. */
+ async function pooled<T>(items: T[], work: (item: T) => Promise<void>) {
+ let next = 0;
+ await Promise.all(
+ Array.from({ length: Math.min(16, items.length) }, async () => {
+ while (next < items.length) await work(items[next++]!);
+ }),
+ );
+ }
+ function sweep() {
+ sweeping ??= (async () => {
+ try {
+ estimate = await maintain();
+ const handle = await open(
+ stamp,
+ constants.O_WRONLY | constants.O_CREAT | constants.O_NOFOLLOW,
+ 0o600,
+ );
+ try {
+ await handle.utimes(now() / 1000, now() / 1000);
+ } finally {
+ await handle.close();
+ }
+ } catch (error) {
+ if (!missing(error)) warn("cache_unavailable");
+ } finally {
+ sweeping = undefined;
+ }
+ })();
+ return sweeping;
}
async function cleanupDetached() {
const scan = await opendir(directory);
@@ -197,14 +280,15 @@ export function createEvaluationCache(options: CacheOptions) {
warn("cache_corrupt");
return;
}
- const payload = JSON.stringify({ schema, createdAt: now(), answers });
- if (Buffer.byteLength(payload) > maxEntryBytes) {
+ const createdAt = now();
+ const payload = JSON.stringify({ schema, createdAt, answers });
+ const size = Buffer.byteLength(payload);
+ if (size > maxEntryBytes || allocated(size) > maxBytes) {
warn("cache_limit");
return;
}
const destination = join(entries, `${key(input)}.json`);
await prepare(true);
- await cleanupDetached();
temporary = join(entries, `.pending-${randomUUID()}`);
handle = await open(
temporary,
@@ -212,11 +296,17 @@ export function createEvaluationCache(options: CacheOptions) {
0o600,
);
await handle.writeFile(payload);
+ // Maintenance reads age from mtime; keep it on the same clock as createdAt.
+ await handle.utimes(createdAt / 1000, createdAt / 1000);
await handle.close();
handle = undefined;
await rename(temporary, destination);
temporary = undefined;
- await trim();
+ estimate += allocated(size);
+ if (!checked) {
+ checked = true;
+ if (await maintenanceDue()) await sweep();
+ } else if (estimate > maxBytes) await sweep();
} catch {
warn("cache_unavailable");
} finally {
@@ -242,9 +332,11 @@ export function createEvaluationCache(options: CacheOptions) {
try {
await checkDirectory(entries, false);
await rename(entries, detached);
+ estimate = 0;
} catch (error) {
if (!missing(error)) throw error;
}
+ await remove(stamp);
await cleanupDetached();
} catch (error) {
if (!missing(error)) warn("cache_unavailable");
diff --git a/packages/core/test/cache.test.ts b/packages/core/test/cache.test.ts
index 2230103..0625b29 100644
--- a/packages/core/test/cache.test.ts
+++ b/packages/core/test/cache.test.ts
@@ -10,6 +10,7 @@ import {
chmod,
mkdir,
symlink,
+ utimes,
} from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";
@@ -170,25 +171,65 @@ test("concurrent writers publish complete answers and clear permits later fresh
expect(await caches[1]!.get(input)).toBeUndefined();
});
-test("byte bounds retain valid entries and oversized answers bypass persistence visibly", async () => {
+test("byte bounds count allocated blocks, evict the oldest answers and reject oversized ones", async () => {
+ let clock = Date.now();
+ const budget = 3 * 4096 + 100;
const dir = await directory();
- const cache = createEvaluationCache({ directory: dir, maxBytes: 180 });
- for (let index = 0; index < 8; index++)
+ const cache = createEvaluationCache({ directory: dir, maxBytes: budget, now: () => clock });
+ for (let index = 0; index < 8; index++) {
+ clock += 1000;
await cache.put({ ...input, request: { index } }, { question1: index / 8 });
+ }
const names = await readdir(join(dir, "entries"));
- let bytes = 0;
+ let allocatedBytes = 0;
for (const name of names) {
const file = join(dir, "entries", name);
- bytes += (await stat(file)).size;
+ const info = await stat(file);
+ allocatedBytes += Math.max(Math.ceil(info.size / 4096) * 4096, info.blocks * 512);
expect(JSON.parse(await readFile(file, "utf8")).answers.question1).toBeGreaterThanOrEqual(0);
}
- expect(bytes).toBeLessThanOrEqual(180);
- expect(bytes).toBeGreaterThan(0);
- await cache.put(input, { ["large".repeat(100)]: 1 });
+ expect(allocatedBytes).toBeLessThanOrEqual(budget);
+ expect(names.length).toBeGreaterThan(0);
+ expect(await cache.get({ ...input, request: { index: 7 } })).toEqual({ question1: 7 / 8 });
+ expect(await cache.get({ ...input, request: { index: 0 } })).toBeUndefined();
+ await cache.put(input, { ["large".repeat(3000)]: 1 });
expect(await cache.get(input)).toBeUndefined();
expect(cache.stats().issues).toEqual([{ kind: "cache_limit", count: 1 }]);
});
+test("sweeps run at most hourly across processes and remove expired and abandoned files", async () => {
+ const minute = 60 * 1000;
+ const start = Date.now();
+ let clock = start;
+ const dir = await directory();
+ const entries = join(dir, "entries");
+ const options = { directory: dir, ttlMs: 45 * minute, now: () => clock };
+ const writer = createEvaluationCache(options);
+ await writer.put({ ...input, request: { index: 0 } }, { question1: 0 });
+ const [expired] = await readdir(entries);
+ clock = start + 50 * minute;
+ const abandoned = ".pending-aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa";
+ const hoursAgo = (clock - 120 * minute) / 1000;
+ await writeFile(join(entries, abandoned), "partial");
+ await utimes(join(entries, abandoned), hoursAgo, hoursAgo);
+ for (let index = 1; index <= 3; index++)
+ await writer.put({ ...input, request: { index } }, { question1: index });
+ // Neither later writes nor another process within the hour rescan the directory.
+ const early = createEvaluationCache(options);
+ await early.put({ ...input, request: { index: 4 } }, { question1: 4 });
+ expect(await readdir(entries)).toEqual(expect.arrayContaining([expired, abandoned]));
+ expect(await writer.get({ ...input, request: { index: 0 } })).toBeUndefined();
+ clock = start + 70 * minute;
+ const late = createEvaluationCache(options);
+ await late.put({ ...input, request: { index: 5 } }, { question1: 5 });
+ const remaining = await readdir(entries);
+ expect(remaining).toHaveLength(5);
+ expect(remaining).not.toContain(expired);
+ expect(remaining).not.toContain(abandoned);
+ expect(await late.get({ ...input, request: { index: 1 } })).toEqual({ question1: 1 });
+ expect([writer, early, late].flatMap((cache) => cache.stats().issues)).toEqual([]);
+});
+
test("clear removes an abandoned detached generation even when no active entries remain", async () => {
const dir = await directory();
const detached = join(dir, ".cleared-aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa");
diff --git a/specs/done/jevgrep/choices.md b/specs/done/jevgrep/choices.md
index 5558876..9caad03 100644
--- a/specs/done/jevgrep/choices.md
+++ b/specs/done/jevgrep/choices.md
@@ -142,22 +142,27 @@ search or optimal treatment of minified source. A change to question grouping is
a separate retrieval-policy experiment. **Verdict:** sound under the preservation
constraint. **Confidence:** medium. Owner: [selection](../../../packages/core/src/selection.ts).
-### Enforce the cache bound by scanning stored entries
+### Enforce the cache bound with periodic oldest-first sweeps
-**When:** cache integration and performance review.
+**When:** cache integration; revised after measuring per-write scans.
A new answer is ready to save while the cache is near its disk limit. The cache
-publishes the complete answer atomically, then scans stored entries in filesystem
-enumeration order and removes entries beyond the retained byte budget. This is
-not oldest-first eviction. It does not maintain a separate persistent index or run a
-background cleanup service. With many entries, repeating that scan for many new
-answers adds overhead; an indexed eviction design would trade that work for
-another stateful component to maintain.
-
-**Gap:** the cache size bound was fixed but its maintenance mechanism was open.
-**Reach:** storage is best effort, and large-cache write throughput is limited by
-repeated scans. This mechanism makes no throughput claim. **Verdict:** sound as a
-simple bounded-storage owner with a disclosed cost. **Confidence:** medium.
+publishes the complete answer atomically and sets its modification time to the
+answer's creation time; that write does not scan the directory. A sweep runs on a
+process's first write when the shared `maintained` stamp is missing, in the future
+or more than an hour old, and again when that process's own writes push its
+estimate past the budget. The sweep reads metadata only. It removes expired entries
+and abandoned publications, then removes the oldest answers until allocated blocks
+fall to seven eighths of the budget. Tiny answers are charged whole filesystem
+blocks, so the bound tracks disk use rather than payload bytes. There is still no
+index or background service.
+
+**Gap:** the cache size bound was fixed but its maintenance mechanism was open. The
+first design rescanned every entry after each write, and at 40,000 entries each
+write took about a second while holding a request slot. **Reach:** storage remains
+best effort. Other processes' writes can exceed the budget until the next sweep,
+and the write that triggers a sweep pays for one metadata pass. **Verdict:** sound
+as a bounded-storage owner without per-write scans. **Confidence:** medium.
Owner: [cache](../../../packages/core/src/cache.ts).
## Sound — high confidence