YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
test(evals): preserve active IP quotas during cleanup
louis7793b09384ef
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex ee6281ba9..7446f756c 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -1078,3 +1078,22 @@ equivalent private names/prose and missing-source setup classification. Fixturesand dependency links are added at grading time. This does not establish HTTPauthentication, accounting, actual provider availability, complete log redaction,execution isolation or model performance.++## Active IP quota retention++`ai-gateway-active-ip-quota-retention` runs the actual runtime maintenance entrypoint+and its SQL against an in-memory SQLite database through a sequential D1 adapter.+Eight outcomes cover active counts with stale update metadata, reset-date boundaries,+expired rows with recent metadata, repeated cleanup, other usage tiers, bounded+backlog cleanup and preserved aggregates. The parent fails four intended outcomes+and preserves four; the historical reference and current maintenance pass eight.+The current usage writer also refreshes update timestamps. This case preserves+legacy or delayed metadata handling, not a claim that every current write is stale.++Run `bun test evals/coding-agent/calibrate-ip-quota-retention.test.js` for nine controls:+broken/reference/current, unused correct code, equivalent SQL/private bindings,+no-op cleanup, blanket removal, an unbounded batch and missing-source setup failure.+Set `EVAL_CALIBRATION_RESULTS_DIR` to retain the per-control logs. No installed+packages are needed, and the hidden fixture is materialized only at grading.+These outcomes do not prove workerd/D1 concurrency or atomicity, deployed cron+delivery, full quota admission, agent isolation or model improvement.diff --git a/evals/coding-agent/calibrate-ip-quota-retention.test.js b/evals/coding-agent/calibrate-ip-quota-retention.test.jsnew file mode 100644index 000000000..6b5a2e442--- /dev/null+++ b/evals/coding-agent/calibrate-ip-quota-retention.test.js@@ -0,0 +1,35 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { mkdtempSync, mkdirSync, readFileSync, writeFileSync, rmSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join, dirname, resolve } from 'node:path';+const repo = resolve(import.meta.dir, '../..');+const item = JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c => c.id === 'ai-gateway-active-ip-quota-retention');+const sourcePath = 'packages/ai-gateway/src/services/runtime-state-maintenance.ts';+const show = ref => execFileSync('git',['show',`${ref}:${sourcePath}`],{cwd:repo,encoding:'utf8'});+const broken=show(item.base_ref), fixed=show(item.oracle_ref), current=show('HEAD');+const root=mkdtempSync(join(tmpdir(),'ip-quota-calibration-'));+afterAll(()=>rmSync(root,{recursive:true,force:true}));+function grade(name,source,unused=false) {+ const dir=join(root,name); const write=(path,text)=>{mkdirSync(dirname(join(dir,path)),{recursive:true});writeFileSync(join(dir,path),text);};+ if(source!==null) write(sourcePath,source);+ if(unused) write(sourcePath.replace('.ts','.unused.ts'),fixed);+ const fixture=item.grader.fixtures[0];write(fixture.destination_path,readFileSync(join(import.meta.dir,fixture.local_path),'utf8'));+ const r=spawnSync(process.execPath,['--no-env-file','test',fixture.destination_path],{cwd:dir,encoding:'utf8',timeout:15000,env:{PATH:dirname(process.execPath)}});+ if(process.env.EVAL_CALIBRATION_RESULTS_DIR){mkdirSync(process.env.EVAL_CALIBRATION_RESULTS_DIR,{recursive:true});writeFileSync(join(process.env.EVAL_CALIBRATION_RESULTS_DIR,name+'.json'),JSON.stringify({status:r.status,signal:r.signal,error:r.error?.message,stdout:r.stdout,stderr:r.stderr},null,2));}+ expect(r.error).toBeUndefined();expect(r.signal).toBeNull();return r;+}+function pass(r){expect(r.status).toBe(0);expect(r.stderr).toContain('8 pass');}+function fail(r){expect(r.status).toBe(1);expect(r.stderr).toContain('expect(received)');expect(r.stderr).not.toContain('Cannot find module');}+test('parent loses active quotas: four failures and four preserved outcomes',()=>{const r=grade('parent',broken);fail(r);expect(r.stderr).toContain('4 fail');expect(r.stderr).toContain('4 pass');});+test('historical reference preserves every outcome',()=>pass(grade('reference',fixed)));+test('current maintenance preserves every outcome',()=>pass(grade('current',current)));+test('unused correct maintenance cannot bypass actual cleanup',()=>fail(grade('unused',broken,true)));+test('equivalent SQL case and private bindings are accepted',()=>pass(grade('equivalent',fixed.replaceAll('statements','maintenanceBatch').replaceAll('DELETE FROM','delete from').replaceAll('SELECT device_id','select device_id'))));+test('no-op maintenance is rejected',()=>fail(grade('no-op','export async function pruneRuntimeState() {}')));+test('blanket removal is rejected',()=>fail(grade('blanket',"export async function pruneRuntimeState(env) { await env.DB.prepare('DELETE FROM usage').run(); }")));+test('removing the batch bound is rejected',()=>{expect(fixed).toContain('5_000');fail(grade('unbounded',fixed.replace('5_000','50_000')));});+test('missing source remains a setup error',()=>{const r=grade('missing',null);expect(r.status).not.toBe(0);expect(r.stderr).toContain('Cannot find module');expect(r.stderr).not.toContain('expect(received)');});diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex 93528d5b5..98ce660b8 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-09-27.3",+ "dataset_version": "2026-09-27.4","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -3776,6 +3776,42 @@"destination_path": "packages/ai-gateway/node_modules"}]+ },+ {+ "id": "ai-gateway-active-ip-quota-retention",+ "name": "Preserve active IP quotas during runtime cleanup",+ "source": {+ "kind": "git_regression",+ "fix_commit": "6b92dc24ee785857a06a2d39188c8831e98e865f"+ },+ "base_ref": "a687dbcb28d83d0691c169fe10aa2863297a0820",+ "oracle_ref": "6b92dc24ee785857a06a2d39188c8831e98e865f",+ "oracle_paths": [+ "packages/ai-gateway/src/services/runtime-state-maintenance.ts"+ ],+ "gate": "advisory",+ "trigger_paths": [+ "packages/ai-gateway/src/services/runtime-state-maintenance.ts",+ "packages/ai-gateway/src/services/usage-tracker.ts"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "quota",+ "preserved-behavior"+ ],+ "prompt": "Repair runtime-state cleanup so active IP usage counters are not erased when their update metadata is stale. For ip_tracking rows, retain last_reset dates from seven days ago onward, including the exact seven-day boundary and future dates, preserving the entire row and daily count. Remove older reset dates even when updated_at is recent. Repeating cleanup must neither reset active counts nor resurrect expired rows. Preserve the separate updated_at-based seven-day retention for free_chat_in_flight_v1, free_chat_turn_v2:* and free_chat_budget_v2:* rows, and ninety-day retention for ordinary usage. A single invocation must make progress while deleting no more than 5000 expired IP rows. Retain existing ninety-day aggregate cleanup and one-hour model-health cleanup, including recent data. Keep the public pruneRuntimeState entrypoint. Grade observable stored rows using real SQLite executing production SQL through a sequential synthetic D1 adapter, not SQL spelling, private names, statement counts or helper architecture. This case does not establish native capture, live D1 atomicity/concurrency, cron delivery, current gateway admission policy or execution isolation.",+ "agent_timeout_seconds": 900,+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/ip-quota-retention.fixture.ts.txt",+ "destination_path": "packages/ai-gateway/src/test/eval-ip-quota-retention.test.ts"+ }+ ],+ "command": "cd packages/ai-gateway && \"${BUN_BIN:-bun}\" --no-env-file test src/test/eval-ip-quota-retention.test.ts"+ }}]}diff --git a/evals/coding-agent/graders/ip-quota-retention.fixture.ts.txt b/evals/coding-agent/graders/ip-quota-retention.fixture.ts.txtnew file mode 100644index 000000000..36d542701--- /dev/null+++ b/evals/coding-agent/graders/ip-quota-retention.fixture.ts.txt@@ -0,0 +1,83 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++import { afterEach, beforeEach, expect, test } from 'bun:test';+import { Database } from 'bun:sqlite';+import { pruneRuntimeState } from '../services/runtime-state-maintenance';+let db: Database;+let env: any;+beforeEach(() => {+ db = new Database(':memory:');+ db.exec(`CREATE TABLE usage (device_id TEXT PRIMARY KEY, user_id TEXT, daily_count INTEGER DEFAULT 0,+ last_reset TEXT NOT NULL, tier TEXT, updated_at TEXT, created_at TEXT, daily_cost_usd REAL DEFAULT 0);+ CREATE TABLE cost_daily (date TEXT, requests INTEGER);+ CREATE TABLE transcription_daily (date TEXT, requests INTEGER);+ CREATE TABLE model_health_window (bucket_epoch INTEGER, model TEXT);+ CREATE TABLE hosted_ai_settlements (settlement_id TEXT PRIMARY KEY, created_at TEXT);`);+ const prepare = (sql: string) => {+ let bindings: any[] = [];+ const statement: any = {+ bind(...values: any[]) { bindings = values; return statement; },+ async run() { const result = db.query(sql).run(...bindings); return { success: true, meta: { changes: result.changes } }; },+ async first() { return db.query(sql).get(...bindings); },+ async all() { return { success: true, results: db.query(sql).all(...bindings) }; },+ };+ return statement;+ };+ // Real SQLite executes the production SQL. This sequential adapter is not workerd/D1 concurrency.+ env = { DB: { prepare, batch: async (statements: any[]) => {+ const results = []; for (const statement of statements) results.push(await statement.run()); return results;+ } } };+});+afterEach(() => db.close());+function seed(id: string, tier: string, resetDays: number, updatedDays: number, count = 41) {+ db.query(`INSERT INTO usage(device_id,tier,last_reset,updated_at,daily_count)+ VALUES (?, ?, date('now', ?), datetime('now', ?), ?)`).run(id, tier, `${resetDays} days`, `${updatedDays} days`, count);+}+const rows = () => db.query('SELECT device_id,tier,last_reset,updated_at,daily_count FROM usage ORDER BY device_id').all();+const ids = () => rows().map((r: any) => r.device_id);+test('active IP quota survives stale update metadata with its count intact', async () => {+ seed('ip:active', 'ip_tracking', 0, -120, 913); const before = rows();+ await pruneRuntimeState(env); expect(rows()).toEqual(before);+});+test('IP retention includes the seven-day boundary and future reset dates', async () => {+ seed('ip:boundary', 'ip_tracking', -7, -120); seed('ip:future', 'ip_tracking', 1, -120);+ const before = rows(); await pruneRuntimeState(env); expect(rows()).toEqual(before);+});+test('expired IP rows are removed despite recently touched metadata', async () => {+ seed('ip:expired', 'ip_tracking', -8, 0); seed('ip:recent', 'ip_tracking', -6, -8);+ await pruneRuntimeState(env); expect(ids()).toEqual(['ip:recent']);+});+test('maintenance replay does not reset an active quota or resurrect old rows', async () => {+ seed('ip:active', 'ip_tracking', 0, -8, 713); seed('ip:expired', 'ip_tracking', -10, -10);+ await pruneRuntimeState(env); const first = rows();+ expect(ids()).toEqual(['ip:active']); expect((first[0] as any).daily_count).toBe(713);+ await pruneRuntimeState(env); expect(rows()).toEqual(first);+});+test('in-flight and free-chat counters keep their updated-time retention', async () => {+ for (const tier of ['free_chat_in_flight_v1','free_chat_turn_v2:synthetic','free_chat_budget_v2:synthetic']) {+ seed(`${tier}:old`, tier, 0, -8); seed(`${tier}:recent`, tier, -100, -6);+ }+ await pruneRuntimeState(env);+ expect(ids()).toEqual(['free_chat_budget_v2:synthetic:recent','free_chat_in_flight_v1:recent','free_chat_turn_v2:synthetic:recent']);+});+test('ordinary usage keeps its separate ninety-day retention and quota values', async () => {+ seed('user:old', 'subscribed', 0, -91); seed('user:retained', 'subscribed', -100, -89, 517);+ await pruneRuntimeState(env); expect(ids()).toEqual(['user:retained']); expect((rows()[0] as any).daily_count).toBe(517);+});+test('one cleanup makes bounded progress on an oversized expired IP backlog', async () => {+ const insert = db.query(`INSERT INTO usage(device_id,tier,last_reset,updated_at) VALUES (?,'ip_tracking',date('now','-8 days'),datetime('now','-8 days'))`);+ db.transaction(() => { for(let i=0;i<5001;i++) insert.run(`ip:old:${i}`); })();+ seed('ip:active', 'ip_tracking', 0, 0);+ await pruneRuntimeState(env);+ const remaining = (db.query("SELECT COUNT(*) AS n FROM usage WHERE device_id LIKE 'ip:old:%'").get() as any).n;+ expect(remaining).toBeGreaterThanOrEqual(1); expect(remaining).toBeLessThan(5001);+ expect(ids()).toContain('ip:active');+});+test('aggregate cleanup still removes old data and retains recent data', async () => {+ for(const table of ['cost_daily','transcription_daily']) db.exec(`INSERT INTO ${table} VALUES(date('now','-91 days'),11),(date('now','-89 days'),17)`);+ db.exec(`INSERT INTO model_health_window VALUES((CAST(strftime('%s','now') AS INTEGER)-7200)/10,'old'),(CAST(strftime('%s','now') AS INTEGER)/10,'recent')`);+ await pruneRuntimeState(env);+ for(const table of ['cost_daily','transcription_daily']) expect(db.query(`SELECT requests FROM ${table}`).all()).toEqual([{requests:17}]);+ expect(db.query('SELECT model FROM model_health_window').all()).toEqual([{model:'recent'}]);+});