Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

test(evals): preserve active IP quotas during cleanup

louis7793b09384ef
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index ee6281ba9..7446f756c 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -1078,3 +1078,22 @@ equivalent private names/prose and missing-source setup classification. Fixtures
and dependency links are added at grading time. This does not establish HTTP
authentication, accounting, actual provider availability, complete log redaction,
execution isolation or model performance.
+
+## Active IP quota retention
+
+`ai-gateway-active-ip-quota-retention` runs the actual runtime maintenance entrypoint
+and its SQL against an in-memory SQLite database through a sequential D1 adapter.
+Eight outcomes cover active counts with stale update metadata, reset-date boundaries,
+expired rows with recent metadata, repeated cleanup, other usage tiers, bounded
+backlog cleanup and preserved aggregates. The parent fails four intended outcomes
+and preserves four; the historical reference and current maintenance pass eight.
+The current usage writer also refreshes update timestamps. This case preserves
+legacy or delayed metadata handling, not a claim that every current write is stale.
+
+Run `bun test evals/coding-agent/calibrate-ip-quota-retention.test.js` for nine controls:
+broken/reference/current, unused correct code, equivalent SQL/private bindings,
+no-op cleanup, blanket removal, an unbounded batch and missing-source setup failure.
+Set `EVAL_CALIBRATION_RESULTS_DIR` to retain the per-control logs. No installed
+packages are needed, and the hidden fixture is materialized only at grading.
+These outcomes do not prove workerd/D1 concurrency or atomicity, deployed cron
+delivery, full quota admission, agent isolation or model improvement.
diff --git a/evals/coding-agent/calibrate-ip-quota-retention.test.js b/evals/coding-agent/calibrate-ip-quota-retention.test.js
new file mode 100644
index 000000000..6b5a2e442
--- /dev/null
+++ b/evals/coding-agent/calibrate-ip-quota-retention.test.js
@@ -0,0 +1,35 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { mkdtempSync, mkdirSync, readFileSync, writeFileSync, rmSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, dirname, resolve } from 'node:path';
+const repo = resolve(import.meta.dir, '../..');
+const item = JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c => c.id === 'ai-gateway-active-ip-quota-retention');
+const sourcePath = 'packages/ai-gateway/src/services/runtime-state-maintenance.ts';
+const show = ref => execFileSync('git',['show',`${ref}:${sourcePath}`],{cwd:repo,encoding:'utf8'});
+const broken=show(item.base_ref), fixed=show(item.oracle_ref), current=show('HEAD');
+const root=mkdtempSync(join(tmpdir(),'ip-quota-calibration-'));
+afterAll(()=>rmSync(root,{recursive:true,force:true}));
+function grade(name,source,unused=false) {
+ const dir=join(root,name); const write=(path,text)=>{mkdirSync(dirname(join(dir,path)),{recursive:true});writeFileSync(join(dir,path),text);};
+ if(source!==null) write(sourcePath,source);
+ if(unused) write(sourcePath.replace('.ts','.unused.ts'),fixed);
+ const fixture=item.grader.fixtures[0];write(fixture.destination_path,readFileSync(join(import.meta.dir,fixture.local_path),'utf8'));
+ const r=spawnSync(process.execPath,['--no-env-file','test',fixture.destination_path],{cwd:dir,encoding:'utf8',timeout:15000,env:{PATH:dirname(process.execPath)}});
+ if(process.env.EVAL_CALIBRATION_RESULTS_DIR){mkdirSync(process.env.EVAL_CALIBRATION_RESULTS_DIR,{recursive:true});writeFileSync(join(process.env.EVAL_CALIBRATION_RESULTS_DIR,name+'.json'),JSON.stringify({status:r.status,signal:r.signal,error:r.error?.message,stdout:r.stdout,stderr:r.stderr},null,2));}
+ expect(r.error).toBeUndefined();expect(r.signal).toBeNull();return r;
+}
+function pass(r){expect(r.status).toBe(0);expect(r.stderr).toContain('8 pass');}
+function fail(r){expect(r.status).toBe(1);expect(r.stderr).toContain('expect(received)');expect(r.stderr).not.toContain('Cannot find module');}
+test('parent loses active quotas: four failures and four preserved outcomes',()=>{const r=grade('parent',broken);fail(r);expect(r.stderr).toContain('4 fail');expect(r.stderr).toContain('4 pass');});
+test('historical reference preserves every outcome',()=>pass(grade('reference',fixed)));
+test('current maintenance preserves every outcome',()=>pass(grade('current',current)));
+test('unused correct maintenance cannot bypass actual cleanup',()=>fail(grade('unused',broken,true)));
+test('equivalent SQL case and private bindings are accepted',()=>pass(grade('equivalent',fixed.replaceAll('statements','maintenanceBatch').replaceAll('DELETE FROM','delete from').replaceAll('SELECT device_id','select device_id'))));
+test('no-op maintenance is rejected',()=>fail(grade('no-op','export async function pruneRuntimeState() {}')));
+test('blanket removal is rejected',()=>fail(grade('blanket',"export async function pruneRuntimeState(env) { await env.DB.prepare('DELETE FROM usage').run(); }")));
+test('removing the batch bound is rejected',()=>{expect(fixed).toContain('5_000');fail(grade('unbounded',fixed.replace('5_000','50_000')));});
+test('missing source remains a setup error',()=>{const r=grade('missing',null);expect(r.status).not.toBe(0);expect(r.stderr).toContain('Cannot find module');expect(r.stderr).not.toContain('expect(received)');});
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 93528d5b5..98ce660b8 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-09-27.3",
+ "dataset_version": "2026-09-27.4",
"suite_type": "regression",
"owner": "screenpipe engineering",
"policy": {
@@ -3776,6 +3776,42 @@
"destination_path": "packages/ai-gateway/node_modules"
}
]
+ },
+ {
+ "id": "ai-gateway-active-ip-quota-retention",
+ "name": "Preserve active IP quotas during runtime cleanup",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "6b92dc24ee785857a06a2d39188c8831e98e865f"
+ },
+ "base_ref": "a687dbcb28d83d0691c169fe10aa2863297a0820",
+ "oracle_ref": "6b92dc24ee785857a06a2d39188c8831e98e865f",
+ "oracle_paths": [
+ "packages/ai-gateway/src/services/runtime-state-maintenance.ts"
+ ],
+ "gate": "advisory",
+ "trigger_paths": [
+ "packages/ai-gateway/src/services/runtime-state-maintenance.ts",
+ "packages/ai-gateway/src/services/usage-tracker.ts"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "quota",
+ "preserved-behavior"
+ ],
+ "prompt": "Repair runtime-state cleanup so active IP usage counters are not erased when their update metadata is stale. For ip_tracking rows, retain last_reset dates from seven days ago onward, including the exact seven-day boundary and future dates, preserving the entire row and daily count. Remove older reset dates even when updated_at is recent. Repeating cleanup must neither reset active counts nor resurrect expired rows. Preserve the separate updated_at-based seven-day retention for free_chat_in_flight_v1, free_chat_turn_v2:* and free_chat_budget_v2:* rows, and ninety-day retention for ordinary usage. A single invocation must make progress while deleting no more than 5000 expired IP rows. Retain existing ninety-day aggregate cleanup and one-hour model-health cleanup, including recent data. Keep the public pruneRuntimeState entrypoint. Grade observable stored rows using real SQLite executing production SQL through a sequential synthetic D1 adapter, not SQL spelling, private names, statement counts or helper architecture. This case does not establish native capture, live D1 atomicity/concurrency, cron delivery, current gateway admission policy or execution isolation.",
+ "agent_timeout_seconds": 900,
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/ip-quota-retention.fixture.ts.txt",
+ "destination_path": "packages/ai-gateway/src/test/eval-ip-quota-retention.test.ts"
+ }
+ ],
+ "command": "cd packages/ai-gateway && \"${BUN_BIN:-bun}\" --no-env-file test src/test/eval-ip-quota-retention.test.ts"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/ip-quota-retention.fixture.ts.txt b/evals/coding-agent/graders/ip-quota-retention.fixture.ts.txt
new file mode 100644
index 000000000..36d542701
--- /dev/null
+++ b/evals/coding-agent/graders/ip-quota-retention.fixture.ts.txt
@@ -0,0 +1,83 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+import { afterEach, beforeEach, expect, test } from 'bun:test';
+import { Database } from 'bun:sqlite';
+import { pruneRuntimeState } from '../services/runtime-state-maintenance';
+let db: Database;
+let env: any;
+beforeEach(() => {
+ db = new Database(':memory:');
+ db.exec(`CREATE TABLE usage (device_id TEXT PRIMARY KEY, user_id TEXT, daily_count INTEGER DEFAULT 0,
+ last_reset TEXT NOT NULL, tier TEXT, updated_at TEXT, created_at TEXT, daily_cost_usd REAL DEFAULT 0);
+ CREATE TABLE cost_daily (date TEXT, requests INTEGER);
+ CREATE TABLE transcription_daily (date TEXT, requests INTEGER);
+ CREATE TABLE model_health_window (bucket_epoch INTEGER, model TEXT);
+ CREATE TABLE hosted_ai_settlements (settlement_id TEXT PRIMARY KEY, created_at TEXT);`);
+ const prepare = (sql: string) => {
+ let bindings: any[] = [];
+ const statement: any = {
+ bind(...values: any[]) { bindings = values; return statement; },
+ async run() { const result = db.query(sql).run(...bindings); return { success: true, meta: { changes: result.changes } }; },
+ async first() { return db.query(sql).get(...bindings); },
+ async all() { return { success: true, results: db.query(sql).all(...bindings) }; },
+ };
+ return statement;
+ };
+ // Real SQLite executes the production SQL. This sequential adapter is not workerd/D1 concurrency.
+ env = { DB: { prepare, batch: async (statements: any[]) => {
+ const results = []; for (const statement of statements) results.push(await statement.run()); return results;
+ } } };
+});
+afterEach(() => db.close());
+function seed(id: string, tier: string, resetDays: number, updatedDays: number, count = 41) {
+ db.query(`INSERT INTO usage(device_id,tier,last_reset,updated_at,daily_count)
+ VALUES (?, ?, date('now', ?), datetime('now', ?), ?)`).run(id, tier, `${resetDays} days`, `${updatedDays} days`, count);
+}
+const rows = () => db.query('SELECT device_id,tier,last_reset,updated_at,daily_count FROM usage ORDER BY device_id').all();
+const ids = () => rows().map((r: any) => r.device_id);
+test('active IP quota survives stale update metadata with its count intact', async () => {
+ seed('ip:active', 'ip_tracking', 0, -120, 913); const before = rows();
+ await pruneRuntimeState(env); expect(rows()).toEqual(before);
+});
+test('IP retention includes the seven-day boundary and future reset dates', async () => {
+ seed('ip:boundary', 'ip_tracking', -7, -120); seed('ip:future', 'ip_tracking', 1, -120);
+ const before = rows(); await pruneRuntimeState(env); expect(rows()).toEqual(before);
+});
+test('expired IP rows are removed despite recently touched metadata', async () => {
+ seed('ip:expired', 'ip_tracking', -8, 0); seed('ip:recent', 'ip_tracking', -6, -8);
+ await pruneRuntimeState(env); expect(ids()).toEqual(['ip:recent']);
+});
+test('maintenance replay does not reset an active quota or resurrect old rows', async () => {
+ seed('ip:active', 'ip_tracking', 0, -8, 713); seed('ip:expired', 'ip_tracking', -10, -10);
+ await pruneRuntimeState(env); const first = rows();
+ expect(ids()).toEqual(['ip:active']); expect((first[0] as any).daily_count).toBe(713);
+ await pruneRuntimeState(env); expect(rows()).toEqual(first);
+});
+test('in-flight and free-chat counters keep their updated-time retention', async () => {
+ for (const tier of ['free_chat_in_flight_v1','free_chat_turn_v2:synthetic','free_chat_budget_v2:synthetic']) {
+ seed(`${tier}:old`, tier, 0, -8); seed(`${tier}:recent`, tier, -100, -6);
+ }
+ await pruneRuntimeState(env);
+ expect(ids()).toEqual(['free_chat_budget_v2:synthetic:recent','free_chat_in_flight_v1:recent','free_chat_turn_v2:synthetic:recent']);
+});
+test('ordinary usage keeps its separate ninety-day retention and quota values', async () => {
+ seed('user:old', 'subscribed', 0, -91); seed('user:retained', 'subscribed', -100, -89, 517);
+ await pruneRuntimeState(env); expect(ids()).toEqual(['user:retained']); expect((rows()[0] as any).daily_count).toBe(517);
+});
+test('one cleanup makes bounded progress on an oversized expired IP backlog', async () => {
+ const insert = db.query(`INSERT INTO usage(device_id,tier,last_reset,updated_at) VALUES (?,'ip_tracking',date('now','-8 days'),datetime('now','-8 days'))`);
+ db.transaction(() => { for(let i=0;i<5001;i++) insert.run(`ip:old:${i}`); })();
+ seed('ip:active', 'ip_tracking', 0, 0);
+ await pruneRuntimeState(env);
+ const remaining = (db.query("SELECT COUNT(*) AS n FROM usage WHERE device_id LIKE 'ip:old:%'").get() as any).n;
+ expect(remaining).toBeGreaterThanOrEqual(1); expect(remaining).toBeLessThan(5001);
+ expect(ids()).toContain('ip:active');
+});
+test('aggregate cleanup still removes old data and retains recent data', async () => {
+ for(const table of ['cost_daily','transcription_daily']) db.exec(`INSERT INTO ${table} VALUES(date('now','-91 days'),11),(date('now','-89 days'),17)`);
+ db.exec(`INSERT INTO model_health_window VALUES((CAST(strftime('%s','now') AS INTEGER)-7200)/10,'old'),(CAST(strftime('%s','now') AS INTEGER)/10,'recent')`);
+ await pruneRuntimeState(env);
+ for(const table of ['cost_daily','transcription_daily']) expect(db.query(`SELECT requests FROM ${table}`).all()).toEqual([{requests:17}]);
+ expect(db.query('SELECT model FROM model_health_window').all()).toEqual([{model:'recent'}]);
+});