Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: verify hosted AI settlement replay and atomicity

louis6165b5e6b024
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index ef2b9fcec..23a35767a 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -992,3 +992,24 @@ classification. No dependencies or build caches are linked by this case. Fixture
are materialized at grading time. This does not establish caller fallback
selection, cancellation, live provider behavior, execution isolation or model
performance, and does not reinstate historical Argus routing policy.
+
+## Hosted-AI settlement replay
+
+`ai-gateway-settlement-replay-atomicity` exercises the existing `logCost`
+entrypoint against synthetic local workerd D1. Ten outcomes cover concurrent
+replay, independent requests, conflicting cost/ownership, atomic rollback,
+lost commit acknowledgement, zero cost and preserved legacy, unbudgeted and
+speech accounting. The parent fails six intended outcomes and preserves four;
+the reference and current source pass ten.
+
+Run `bun test evals/coding-agent/calibrate-settlement-replay.test.js` with
+Miniflare `3.20250718.3` available in `packages/ai-gateway/node_modules`.
+Ten calibration controls include an unused corrected implementation, blanket
+success, collapsed request identities, ignored cost conflicts and non-atomic
+writes. Equivalent private names pass; missing source is a setup error.
+The case checks account and aggregate effects without requiring a particular
+ledger table or helper design. Fixtures and dependency links appear only at
+grading time; dependency links do not establish agent isolation. Current product
+tests separately cover additional ledger windows. This case does not establish
+HTTP admission, hold release, every budget window, deployed migrations, actual
+provider billing or model performance.
diff --git a/evals/coding-agent/calibrate-settlement-replay.test.js b/evals/coding-agent/calibrate-settlement-replay.test.js
new file mode 100644
index 000000000..f9e5f0cbc
--- /dev/null
+++ b/evals/coding-agent/calibrate-settlement-replay.test.js
@@ -0,0 +1,39 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { dirname, join, resolve } from 'node:path';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { classifyGraderError } from './grader-outcome.mjs';
+const repo = resolve(import.meta.dir, '../..');
+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'))).cases.find(c => c.id === 'ai-gateway-settlement-replay-atomicity');
+const receipts = process.env.EVAL_CALIBRATION_RESULTS_DIR;
+if (receipts) mkdirSync(receipts, { recursive: true });
+const root = mkdtempSync(join(receipts || tmpdir(), 'settlement-calibration-')), archives = new Map();
+const tracker = 'packages/ai-gateway/src/services/cost-tracker.ts', ledger = 'packages/ai-gateway/src/services/hosted-ai-settlement-ledger.ts';
+afterAll(() => { if (!receipts) rmSync(root, { recursive: true, force: true }); });
+function replace(cwd, path, from, to) { const text = readFileSync(join(cwd, path), 'utf8'); expect(text.split(from)).toHaveLength(2); writeFileSync(join(cwd, path), text.replace(from, to)); }
+function grade(name, ref = item.oracle_ref, mutate = () => {}) {
+ const cwd = join(root, name); mkdirSync(cwd);
+ if (!archives.has(ref)) archives.set(ref, execFileSync('git', ['-c','core.commitGraph=false','archive',ref,'packages/ai-gateway/src'], { cwd:repo, maxBuffer:32*1024*1024 }));
+ execFileSync('tar',['-x','-C',cwd],{input:archives.get(ref)}); mutate(cwd);
+ writeFileSync(join(cwd,'packages/ai-gateway/src/test/eval-settlement-replay.test.ts'),readFileSync(join(import.meta.dir,'graders/settlement-replay.fixture.ts.txt')));
+ symlinkSync(join(repo,'packages/ai-gateway/node_modules'),join(cwd,'packages/ai-gateway/node_modules'),'dir');
+ const args=['test','src/test/eval-settlement-replay.test.ts','--timeout=10000'];
+ const r=spawnSync(process.execPath,args,{cwd:join(cwd,'packages/ai-gateway'),encoding:'utf8',timeout:45000,env:{PATH:dirname(process.execPath),HOME:cwd,CI:'true'}});
+ if(receipts) writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:[process.execPath,...args],cwd,status:r.status,signal:r.signal,error:r.error?.message,stdout:r.stdout,stderr:r.stderr,failure_kind:classifyGraderError(r)},null,2));
+ return r;
+}
+function fails(r) { expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); expect(r.status).toBe(1); expect(r.stderr).toContain('expect(received)'); expect(classifyGraderError(r)).toBeNull(); }
+function passes(r) { expect(r.status).toBe(0); expect(r.stderr).toContain('10 pass'); }
+test('parent fails six intended outcomes and preserves four',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stderr).toContain('6 fail');expect(r.stderr).toContain('4 pass');});
+test('reference passes all accounting outcomes',()=>passes(grade('reference')));
+test('current source passes the same outcomes',()=>passes(grade('current','HEAD')));
+test('equivalent private cost conversion binding is accepted',()=>passes(grade('equivalent',item.oracle_ref,cwd=>{const path=join(cwd,ledger);writeFileSync(path,readFileSync(path,'utf8').replaceAll('costMicroCents','scaledCost'));})));
+test('unused corrected accounting cannot hide the broken caller',()=>fails(grade('unused',item.base_ref,cwd=>{for(const [source,dest] of [[tracker,tracker+'.unused.ts'],[ledger,ledger]])writeFileSync(join(cwd,dest),execFileSync('git',['show',item.oracle_ref+':'+source],{cwd:repo}));})));
+test('blanket success without writes is rejected',()=>fails(grade('blanket',item.oracle_ref,cwd=>replace(cwd,tracker,'export async function logCost(env: Env, entry: CostLogEntry): Promise<boolean> {','export async function logCost(env: Env, entry: CostLogEntry): Promise<boolean> { return true;'))));
+test('collapsing distinct request identities is rejected',()=>fails(grade('collapsed',item.oracle_ref,cwd=>replace(cwd,tracker,'settlementId: entry.settlement_id,',"settlementId: 'constant',"))));
+test('ignoring changed cost in the replay identity is rejected',()=>fails(grade('cost-collision',item.oracle_ref,cwd=>replace(cwd,ledger,'deviceId,\n\t\t\tmicroCents,','deviceId,\n\t\t\t0,'))));
+test('separate writes without atomic rollback are rejected',()=>fails(grade('partial-write',item.oracle_ref,cwd=>replace(cwd,ledger,'const results = await env.DB.batch<SettlementReadback>(statements);','const results = []; for (const statement of statements) results.push(await statement.all<SettlementReadback>());'))));
+test('missing entrypoint is a setup error, not a baseline regression',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,tracker)));expect(r.status).toBe(1);expect(r.stderr).toContain('0 pass');expect(classifyGraderError(r)).toBe('bun_unhandled_error');});
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 3013ef6ed..9fa31cf4d 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-09-26.3",
+ "dataset_version": "2026-09-26.4",
"suite_type": "regression",
"owner": "screenpipe engineering",
"policy": {
@@ -3565,6 +3565,49 @@
],
"command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/gateway-stream-initial-error.test.js"
}
+ },
+ {
+ "id": "ai-gateway-settlement-replay-atomicity",
+ "name": "Apply replayed hosted-AI settlements once without partial accounting",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "d9ff95802e6566a7b3ea3242800aef59cacb9aa0"
+ },
+ "base_ref": "8b455607ffbaef204a6f6a0a62e973bac6211e48",
+ "oracle_ref": "d9ff95802e6566a7b3ea3242800aef59cacb9aa0",
+ "oracle_paths": [
+ "packages/ai-gateway/src/services/cost-tracker.ts",
+ "packages/ai-gateway/src/services/hosted-ai-settlement-ledger.ts"
+ ],
+ "gate": "advisory",
+ "tags": [
+ "accounting",
+ "retry",
+ "concurrency",
+ "ownership"
+ ],
+ "trigger_paths": [
+ "packages/ai-gateway/src/services/cost-tracker.ts",
+ "packages/ai-gateway/src/services/hosted-ai-settlement-ledger.ts"
+ ],
+ "prompt": "The existing hosted-AI logCost entrypoint can apply the same priced request repeatedly and report success after only part of its accounting succeeds. Repair its budgeted text accounting for calls carrying a stable settlement_id. Sequential or concurrent identical replays must return true and apply the account cost and cost_daily request/cost aggregate exactly once. Different IDs still apply independently. Reusing an applied ID with a different cost or account owner must return false without changing either account or the aggregate. A failed dependent aggregate write must return false with no partial account charge; a later successful retry applies once. If the accounting transaction committed but its acknowledgement was lost, an identical retry must succeed without another charge. Zero-cost replays record one request and no spend. Preserve existing legacy calls without settlement_id: each call still accounts independently. Preserve budgeted:false telemetry-only calls and transcription_budgeted:true calls: they retain their repeated telemetry behavior, with unbudgeted calls not charging accounts and speech calls charging only their existing separate transcription account bucket. Exercise the real logCost entrypoint with synthetic local D1 data, never production services. Any implementation meeting these outcomes is acceptable; do not depend on particular helper names or a particular idempotency-table design. This case does not prove request admission, hold release, all global/background/trial budget windows, deployment activation, or provider billing correctness. Add focused tests for these outcomes.",
+ "agent_timeout_seconds": 900,
+ "grader_dependency_links": [
+ {
+ "source_path": "packages/ai-gateway/node_modules",
+ "destination_path": "packages/ai-gateway/node_modules"
+ }
+ ],
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/settlement-replay.fixture.ts.txt",
+ "destination_path": "packages/ai-gateway/src/test/eval-settlement-replay.test.ts"
+ }
+ ],
+ "command": "cd packages/ai-gateway && \"${BUN_BIN:-bun}\" test src/test/eval-settlement-replay.test.ts --timeout=10000"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/settlement-replay.fixture.ts.txt b/evals/coding-agent/graders/settlement-replay.fixture.ts.txt
new file mode 100644
index 000000000..aeef14605
--- /dev/null
+++ b/evals/coding-agent/graders/settlement-replay.fixture.ts.txt
@@ -0,0 +1,149 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterEach, beforeEach, describe, expect, it } from 'bun:test';
+import { Miniflare } from 'miniflare';
+import runtime from 'miniflare/package.json';
+if (runtime.version !== '3.20250718.3') throw new Error('Grader requires miniflare 3.20250718.3');
+import type { Env } from '../types';
+import { logCost } from '../services/cost-tracker';
+const COST_DAILY_SCHEMA = `CREATE TABLE cost_daily (
+ date TEXT NOT NULL, tier TEXT NOT NULL, provider TEXT NOT NULL,
+ model TEXT NOT NULL, endpoint TEXT NOT NULL, stream INTEGER NOT NULL,
+ router_tier TEXT NOT NULL, requests INTEGER NOT NULL DEFAULT 0,
+ input_tokens INTEGER NOT NULL DEFAULT 0, output_tokens INTEGER NOT NULL DEFAULT 0,
+ cache_read_tokens INTEGER NOT NULL DEFAULT 0,
+ cache_creation_tokens INTEGER NOT NULL DEFAULT 0,
+ estimated_cost_usd REAL NOT NULL DEFAULT 0,
+ latency_ms_sum INTEGER NOT NULL DEFAULT 0,
+ latency_samples INTEGER NOT NULL DEFAULT 0,
+ updated_at TEXT NOT NULL DEFAULT (datetime('now')),
+ PRIMARY KEY (date, tier, provider, model, endpoint, stream, router_tier)
+) WITHOUT ROWID`;
+
+const SCHEMA = [
+ `CREATE TABLE usage (
+ device_id TEXT PRIMARY KEY, user_id TEXT, daily_count INTEGER DEFAULT 0,
+ last_reset TEXT NOT NULL, tier TEXT DEFAULT 'anonymous',
+ created_at TEXT DEFAULT (datetime('now')), updated_at TEXT DEFAULT (datetime('now')),
+ cost_day TEXT, daily_cost_usd REAL NOT NULL DEFAULT 0
+ )`,
+ COST_DAILY_SCHEMA,
+ `CREATE TABLE hosted_ai_settlements (
+ settlement_id TEXT PRIMARY KEY, fingerprint TEXT NOT NULL,
+ cost_micro_cents INTEGER NOT NULL CHECK (cost_micro_cents >= 0),
+ day TEXT NOT NULL, month_period TEXT NOT NULL, hour TEXT NOT NULL,
+ lane TEXT NOT NULL CHECK (lane IN ('interactive', 'background')),
+ hosted_ai_trial INTEGER NOT NULL CHECK (hosted_ai_trial IN (0, 1)),
+ ledger_epoch TEXT NOT NULL, applied_at TEXT,
+ created_at TEXT NOT NULL DEFAULT (datetime('now'))
+ ) WITHOUT ROWID`,
+];
+
+function entry(settlementId: string, overrides: Record<string, unknown> = {}) {
+ return {
+ settlement_id: settlementId,
+ device_id: 'settlement-test-account',
+ tier: 'business',
+ provider: 'anthropic',
+ model: 'claude-sonnet-5',
+ input_tokens: 1_000,
+ output_tokens: 100,
+ cache_read_tokens: 50,
+ cache_creation_tokens: 25,
+ estimated_cost_usd: 0.12345678,
+ endpoint: '/v1/chat/completions',
+ stream: true,
+ lane: 'interactive' as const,
+ latency_ms: 250,
+ ...overrides,
+ };
+}
+
+describe('hosted AI settlement ledger against workerd D1', () => {
+ let miniflare: Miniflare;
+ let env: Env;
+
+ beforeEach(async () => {
+ miniflare = new Miniflare({
+ compatibilityDate: '2026-01-01',
+ d1Databases: { DB: 'settlement-ledger-test' },
+ modules: true,
+ script: 'export default { fetch() { return new Response("ok"); } };',
+ });
+ const db = await miniflare.getD1Database('DB');
+ await db.batch(SCHEMA.map((statement) => db.prepare(statement)));
+ env = { DB: db as unknown as D1Database } as Env;
+ });
+
+ afterEach(async () => {
+ await miniflare.dispose();
+ });
+
+
+ const amount = async (id = 'settlement-test-account') => (await env.DB.prepare('SELECT daily_cost_usd AS cost FROM usage WHERE device_id = ?').bind(id).first())?.cost ?? 0;
+ const aggregate = async () => await env.DB.prepare('SELECT SUM(requests) AS requests, SUM(estimated_cost_usd) AS cost FROM cost_daily').first();
+ it('concurrent replay charges account and telemetry once', async () => {
+ expect((await Promise.all(Array.from({length: 12}, () => logCost(env, entry('same'))))).every(Boolean)).toBe(true);
+ expect(await amount()).toBeCloseTo(0.12345678, 8);
+ expect((await aggregate())?.requests).toBe(1);
+ expect((await aggregate())?.cost).toBeCloseTo(0.12345678, 8);
+ });
+ it('distinct requests retain both charges', async () => {
+ expect(await logCost(env, entry('one'))).toBe(true);
+ expect(await logCost(env, entry('two'))).toBe(true);
+ expect(await amount()).toBeCloseTo(0.24691356, 8);
+ expect((await aggregate())?.requests).toBe(2);
+ });
+ it('changed cost cannot reuse an applied identity', async () => {
+ expect(await logCost(env, entry('cost'))).toBe(true);
+ expect(await logCost(env, entry('cost', {estimated_cost_usd: 0.5}))).toBe(false);
+ expect(await amount()).toBeCloseTo(0.12345678, 8);
+ expect((await aggregate())?.requests).toBe(1);
+ });
+ it('changed ownership cannot reuse an applied identity', async () => {
+ expect(await logCost(env, entry('owner'))).toBe(true);
+ expect(await logCost(env, entry('owner', {device_id: 'other-synthetic-account'}))).toBe(false);
+ expect(await amount('other-synthetic-account')).toBe(0);
+ expect(await amount()).toBeCloseTo(0.12345678, 8);
+ expect((await aggregate())?.requests).toBe(1);
+ });
+ it('dependent write failure leaves no charge and supports retry', async () => {
+ await env.DB.prepare('DROP TABLE cost_daily').run();
+ expect(await logCost(env, entry('retry'))).toBe(false);
+ expect(await amount()).toBe(0);
+ await env.DB.prepare(COST_DAILY_SCHEMA).run();
+ expect(await logCost(env, entry('retry'))).toBe(true);
+ expect(await amount()).toBeCloseTo(0.12345678,8);
+ expect((await aggregate())?.requests).toBe(1);
+ });
+
+ it('an ambiguous committed response can be retried without a second charge', async () => {
+ const db = env.DB; let fail = true;
+ const unreliable = { DB: {
+ prepare: db.prepare.bind(db),
+ batch: async (...args) => { const result = await db.batch(...args); if (fail) { fail = false; throw new Error('Synthetic lost commit acknowledgement'); } return result; },
+ }} as Env;
+ expect(await logCost(unreliable, entry('ack'))).toBe(false);
+ expect(await logCost(env, entry('ack'))).toBe(true);
+ expect(await amount()).toBeCloseTo(0.12345678,8);
+ expect((await aggregate())?.requests).toBe(1);
+ });
+ it('zero cost replay records one request without inventing spend', async () => {
+ for(let i=0;i<2;i++) expect(await logCost(env,entry('zero',{estimated_cost_usd:0}))).toBe(true);
+ expect(await amount()).toBe(0); expect((await aggregate())?.requests).toBe(1);
+ });
+ it('legacy calls without settlement identity retain independent accounting', async () => {
+ for(let i=0;i<2;i++) expect(await logCost(env,entry(undefined))).toBe(true);
+ expect(await amount()).toBeCloseTo(0.24691356,8); expect((await aggregate())?.requests).toBe(2);
+ });
+ it('unbudgeted calls retain telemetry without charging the account', async () => {
+ for(let i=0;i<2;i++) expect(await logCost(env,entry('unbudgeted',{budgeted:false}))).toBe(true);
+ expect(await amount()).toBe(0); expect((await aggregate())?.requests).toBe(2);
+ });
+ it('speech accounting preserves its separate repeated-charge path', async () => {
+ for(let i=0;i<2;i++) expect(await logCost(env,entry('speech',{transcription_budgeted:true}))).toBe(true);
+ expect(await amount()).toBe(0);
+ expect(await amount('hosted-transcription-cost:day:v1:settlement-test-account')).toBeCloseTo(0.24691356,8);
+ expect((await aggregate())?.requests).toBe(2);
+ });
+});