YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
eval: cover gateway cost writes outliving responses
louis93afdd2f8500
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 9608dd2b1..6a252de48 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -1265,3 +1265,23 @@ boundary, not live account refresh, native credential persistence, recording,execution isolation or model performance.The bounded first-run preview case exercises the actual hook with synthetic native status, activity reads and clock. Its calibration command is `bun test evals/coding-agent/calibrate-bounded-preview.test.js`. This checks corpus outcomes and grader controls; it does not execute native history queries or measure agent performance.+++### Gateway response-lifetime accounting++`app-gateway-response-lifetime-cost-settlement` exercises the real HTTP handler+and daily cost writer with synthetic provider and database ports. Both JSON and+SSE responses must retain exactly one write with actual input, output, and cached+input counts through request completion. Three nearby outcomes preserve tool+responses and unauthenticated rejection. The parent loses two deferred writes;+the historical fix passes all five outcomes. Twelve calibration controls include+synchronous JSON accounting, equivalent background scheduling, detached work,+unused fixed code, missing writes, lost cached counts, reordered SQL, and a missing-module+infrastructure failure. Run `bun test evals/coding-agent/calibrate-cost-lifetime.test.js`.++The simulated lifetime ends after the handler, response body, and registered work+complete; the database operation completes on a later scheduler turn. Actual SQL executes+against an in-memory SQLite ledger; assertions inspect the resulting row. This is+not native Workers termination, live D1 durability, retry or concurrent billing+coverage. Hidden fixtures and dependency links appear only at grading time;+execution isolation and model performance remain unproven.diff --git a/evals/coding-agent/calibrate-cost-lifetime.test.js b/evals/coding-agent/calibrate-cost-lifetime.test.jsnew file mode 100644index 000000000..4ccbd28e3--- /dev/null+++ b/evals/coding-agent/calibrate-cost-lifetime.test.js@@ -0,0 +1,69 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { dirname, join, resolve } from 'node:path';+import { tmpdir } from 'node:os';+const repo = resolve(import.meta.dir, '../..');+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'))).cases.find(c => c.id === 'app-gateway-response-lifetime-cost-settlement');+const root = mkdtempSync(join(tmpdir(), 'cost-lifetime-calibration-'));+afterAll(() => rmSync(root, { recursive: true, force: true }));+const fixture = readFileSync(join(import.meta.dir, 'graders/cost-lifetime.fixture.ts.txt'));+function replaceOne(text, before, after) {+ expect(text.split(before)).toHaveLength(2);+ return text.replace(before, after);+}+function grade(name) {+ const cwd = join(root, name); mkdirSync(cwd);+ const ref = ['parent', 'unused', 'oracle-only'].includes(name) ? item.base_ref : item.oracle_ref;+ execFileSync('tar', ['-x', '-C', cwd], { input: execFileSync('git', ['archive', ref, 'packages/ai-gateway', 'crates/screenpipe-core/assets/extensions/lib'], { cwd: repo, maxBuffer: 32 * 1024 * 1024 }) });+ const src = join(cwd, 'packages/ai-gateway/src'), index = join(src, 'index.ts');+ if (name === 'unused' || name === 'oracle-only') writeFileSync(name === 'unused' ? index + '.unused.ts' : index, execFileSync('git', ['show', `${item.oracle_ref}:packages/ai-gateway/src/index.ts`], { cwd: repo }));+ let text = readFileSync(index, 'utf8');+ if (name === 'stream-detached') text = replaceOne(text, 'ctx.waitUntil(usagePromise.then(u => logCost(env, {', 'void (usagePromise.then(u => logCost(env, {');+ if (name === 'json-detached') text = replaceOne(text, 'ctx.waitUntil(settleActualOrReservedCost(', 'void (settleActualOrReservedCost(');+ if (name === 'synchronous-json') {+ text = replaceOne(text, 'ctx.waitUntil(settleActualOrReservedCost(', 'await settleActualOrReservedCost(');+ text = replaceOne(text, '\t\t\t\t\t));\n\t\t\t\t}\n\n\t\t\t\treturn attachLeaseRelease(response);', '\t\t\t\t\t);\n\t\t\t\t}\n\n\t\t\t\treturn attachLeaseRelease(response);');+ }+ if (name === 'equivalent-background') text = text.replaceAll('ctx.waitUntil(', '((task: Promise<unknown>) => ctx.waitUntil(Promise.resolve().then(() => task)))(');+ writeFileSync(index, text);+ const cost = join(src, 'services/cost-tracker.ts');+ if (name === 'cached-loss') writeFileSync(cost, replaceOne(readFileSync(cost, 'utf8'), 'nonNegativeNumber(entry.output_tokens),\n nonNegativeNumber(entry.cache_read_tokens),', 'nonNegativeNumber(entry.output_tokens),\n 0,'));+ if (name === 'no-ledger') writeFileSync(cost, replaceOne(readFileSync(cost, 'utf8'), 'await env.DB.prepare(\n `INSERT INTO cost_daily', 'await Promise.resolve(); return accumulatorRecorded;\n await env.DB.prepare(\n `INSERT INTO cost_daily'));+ if (name === 'sql-column-order') {+ let sql = replaceOne(readFileSync(cost, 'utf8'), 'requests, input_tokens, output_tokens, cache_read_tokens,', 'requests, output_tokens, input_tokens, cache_read_tokens,');+ sql = replaceOne(sql, 'nonNegativeNumber(entry.input_tokens),\n nonNegativeNumber(entry.output_tokens),', 'nonNegativeNumber(entry.output_tokens),\n nonNegativeNumber(entry.input_tokens),');+ writeFileSync(cost, sql);+ }+ if (name === 'missing') rmSync(index);+ writeFileSync(join(src, 'test/eval-cost-lifetime.test.ts'), fixture);+ symlinkSync(join(repo, 'packages/ai-gateway/node_modules'), join(cwd, 'packages/ai-gateway/node_modules'), 'dir');+ const result = spawnSync(process.execPath, ['test', 'src/test/eval-cost-lifetime.test.ts'], { cwd: join(cwd, 'packages/ai-gateway'), encoding: 'utf8', timeout: 15000, env: { PATH: dirname(process.execPath), HOME: cwd, CI: 'true' } });+ if (process.env.EVAL_CALIBRATION_RESULTS) {+ const out = resolve(process.env.EVAL_CALIBRATION_RESULTS); mkdirSync(out, { recursive: true });+ for (const ext of ['stdout', 'stderr']) writeFileSync(join(out, name + '.' + ext), result[ext] || '');+ writeFileSync(join(out, name + '.json'), JSON.stringify({ status: result.status, signal: result.signal, error: result.error?.message || null }) + '\n');+ }+ return result;+}+function fails(result, count = 2) {+ expect(result.error).toBeUndefined(); expect(result.signal).toBeNull(); expect(result.status).toBe(1);+ expect(result.stderr).toContain('expect(received)'); expect(result.stderr).not.toContain('Cannot find module');+ expect(result.stderr).toContain(`${count} fail`); expect(result.stderr).toContain(`${5 - count} pass`);+}+function passes(result) { expect(result.status).toBe(0); expect(result.stderr).toContain('5 pass'); }+test('parent loses both deferred writes and preserves three outcomes', () => fails(grade('parent')));+test('historical reference passes', () => passes(grade('reference')));+test('only the manifest oracle path is sufficient', () => passes(grade('oracle-only')));+test('unused fixed handler does not repair dispatch', () => fails(grade('unused')));+test('detached streaming accounting is rejected', () => fails(grade('stream-detached'), 1));+test('detached JSON accounting is rejected', () => fails(grade('json-detached'), 1));+test('cached-input loss is rejected', () => fails(grade('cached-loss')));+test('removing the actual daily write is rejected', () => fails(grade('no-ledger')));+test('synchronous JSON accounting is accepted', () => passes(grade('synchronous-json')));+test('equivalent background scheduling is accepted', () => passes(grade('equivalent-background')));+test('missing handler is a setup failure', () => { const r = grade('missing'); expect(r.status).not.toBe(0); expect(r.stderr).toContain('Cannot find module'); expect(r.stderr).not.toContain('expect(received)'); });++test('equivalent SQL column and binding order is accepted', () => passes(grade('sql-column-order')));diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex ba6763670..acb5bcd56 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -4303,6 +4303,44 @@"destination_path": "apps/screenpipe-app-tauri/node_modules"}]+ },+ {+ "id": "app-gateway-response-lifetime-cost-settlement",+ "name": "Retain completed gateway usage through request lifetime",+ "source": {+ "kind": "git_regression",+ "fix_commit": "7a320188f55127d91762c336483adcd3336a46a6"+ },+ "base_ref": "1f7880caf6b4c0faa0f4e9805f8002f794f4d0d2",+ "oracle_ref": "7a320188f55127d91762c336483adcd3336a46a6",+ "oracle_paths": [+ "packages/ai-gateway/src/index.ts"+ ],+ "gate": "advisory",+ "tags": [+ "regression",+ "billing",+ "gateway",+ "request-lifetime"+ ],+ "prompt": "Successful authorized chat completions can return their answer while their deferred usage recording is lost when the request lifetime ends. Repair the existing gateway HTTP handler so successful JSON and SSE completions retain exactly one daily usage record with the actual input, output, and cached-input token counts. Preserve the successful status and tool-call response body for both formats, and continue rejecting unauthenticated requests without recording cost. Work may finish before the response completes or be retained as background work; no specific helper or scheduling API is required. Tests use the real handler and cost-recording source with synthetic provider, authentication, rate-limit, and database ports. The database write completes on a later scheduler turn and survives only while the handler/body or registered execution-lifetime work remains active. External network is denied. Missing usage, provider failure, retries, quota reservations, concurrency, runtime termination behavior, live D1 durability, and model quality are outside this case. Add focused tests.",+ "agent_timeout_seconds": 900,+ "grader_dependency_links": [+ {+ "source_path": "packages/ai-gateway/node_modules",+ "destination_path": "packages/ai-gateway/node_modules"+ }+ ],+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/cost-lifetime.fixture.ts.txt",+ "destination_path": "packages/ai-gateway/src/test/eval-cost-lifetime.test.ts"+ }+ ],+ "command": "cd packages/ai-gateway && \"${BUN_BIN:-bun}\" test src/test/eval-cost-lifetime.test.ts"+ }}]}diff --git a/evals/coding-agent/graders/cost-lifetime.fixture.ts.txt b/evals/coding-agent/graders/cost-lifetime.fixture.ts.txtnew file mode 100644index 000000000..e2e96df4f--- /dev/null+++ b/evals/coding-agent/graders/cost-lifetime.fixture.ts.txt@@ -0,0 +1,137 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { expect, it, spyOn } from 'bun:test';+import { Database } from 'bun:sqlite';+import { handleRequest } from '../index';+import { OpenAIProvider } from '../providers/openai';+import type { Env, RequestBody } from '../types';++const originalFetch = globalThis.fetch;+async function exercise(stream: boolean, authorized = true) {+ const db = new Database(':memory:');+ db.exec(`CREATE TABLE cost_daily (+ date TEXT, tier TEXT, provider TEXT, model TEXT, endpoint TEXT,+ stream INTEGER, router_tier TEXT, requests INTEGER, input_tokens INTEGER,+ output_tokens INTEGER, cache_read_tokens INTEGER, cache_creation_tokens INTEGER,+ estimated_cost_usd REAL, latency_ms_sum REAL, latency_samples INTEGER,+ updated_at TEXT, PRIMARY KEY(date,tier,provider,model,endpoint,stream,router_tier)+ )`);+ const lifetime: Promise<unknown>[] = [], pending: Promise<unknown>[] = [];+ let contextOpen = true;+ let release!: () => void;+ const gate = new Promise<void>(resolve => { release = resolve; });+ const network: string[] = [];+ globalThis.fetch = (async (input: unknown) => {+ network.push(String(input));+ throw Error('unexpected external network');+ }) as typeof fetch;+ const info = spyOn(console, 'info').mockImplementation(() => {});+ const warn = spyOn(console, 'warn').mockImplementation(() => {});+ const tool = {+ id: 'synthetic-call', type: 'function',+ function: { name: 'lookup', arguments: '{"query":"synthetic"}' },+ };+ const usage = {+ prompt_tokens: 100, completion_tokens: 20,+ prompt_tokens_details: { cached_tokens: 40 },+ };+ const reply = async (body: RequestBody) => {+ const json = {+ model: body.model,+ choices: [{ message: { role: 'assistant', content: null, tool_calls: [tool] }, finish_reason: 'tool_calls' }],+ usage,+ };+ const events = [+ { choices: [{ delta: { role: 'assistant', tool_calls: [{ index: 0, ...tool }] }, finish_reason: null }] },+ { choices: [{ delta: {}, finish_reason: 'tool_calls' }] },+ { choices: [], usage },+ ];+ const sse = events.map(event => 'data: ' + JSON.stringify(event) + '\n\n').join('') + 'data: [DONE]\n\n';+ return stream+ ? new Response(sse, { headers: { 'content-type': 'text/event-stream' } })+ : Response.json(json);+ };+ const jsonSpy = spyOn(OpenAIProvider.prototype, 'createCompletion').mockImplementation(reply);+ const streamSpy = spyOn(OpenAIProvider.prototype, 'createStreamingCompletion')+ .mockImplementation(async body => (await reply(body)).body!);+ const env = {+ AI_GATEWAY_SERVICE_TOKEN: 'synthetic-service-token', OPENAI_API_KEY: 'synthetic-openai-key',+ CLOUDFLARE_AI_GATEWAY_ID: 'synthetic', CLOUDFLARE_AI_GATEWAY_TOKEN: 'synthetic-gateway-key',+ CLOUDFLARE_AI_GATEWAY_BASE_URL: 'https://gateway.ai.cloudflare.com/v1/00000000000000000000000000000000/synthetic/compat/chat/completions',+ RATE_LIMITER: {+ idFromName: (name: string) => name,+ get: () => ({ fetch: async () => Response.json({ allowed: true, standing: 'good' }) }),+ },+ DB: {+ prepare: (sql: string) => ({ bind: (...values: unknown[]) => ({+ first: async () => null,+ run: () => {+ const task = (async () => {+ if (/\bcost_daily\b/i.test(sql)) {+ await gate;+ if (contextOpen) db.query(sql).run(...values as any[]);+ }+ return { success: true };+ })();+ pending.push(task);+ return task;+ },+ }) }),+ },+ } as unknown as Env;+ // In this synthetic runtime, DB completion occurs on the next scheduler turn.+ // Synchronous completion, response-body work, and registered background work+ // can all keep the lifetime open. No particular scheduling call is asserted.+ const timer = setImmediate(release);+ try {+ const request = new Request('https://gateway.test/v1/chat/completions', {+ method: 'POST',+ headers: {+ ...(authorized ? { authorization: 'Bearer synthetic-service-token' } : {}),+ 'content-type': 'application/json',+ },+ body: JSON.stringify({+ model: 'auto', stream, messages: [{ role: 'user', content: 'Synthetic lookup' }],+ tools: [{ type: 'function', function: { name: 'lookup', parameters: { type: 'object', properties: { query: { type: 'string' } } } } }],+ max_completion_tokens: 64,+ }),+ });+ const response = await handleRequest(request, env, {+ waitUntil: (task: Promise<unknown>) => lifetime.push(task),+ passThroughOnException() {},+ } as unknown as ExecutionContext);+ const content = await response.text();+ await Promise.all(lifetime);+ contextOpen = false;+ release();+ await Promise.all(pending);+ const writes = db.query('SELECT requests, input_tokens, output_tokens, cache_read_tokens FROM cost_daily').all();+ return { status: response.status, content, network, writes };+ } finally {+ clearImmediate(timer);+ release();+ await Promise.allSettled(pending);+ info.mockRestore(); warn.mockRestore(); jsonSpy.mockRestore(); streamSpy.mockRestore();+ globalThis.fetch = originalFetch;+ db.close();+ }+}+for (const stream of [false, true]) {+ it(`retains ${stream ? 'SSE' : 'JSON'} usage through request completion`, async () => {+ const result = await exercise(stream);+ expect(result.writes).toHaveLength(1);+ expect(result.writes[0]).toEqual({ requests: 1, input_tokens: 100, output_tokens: 20, cache_read_tokens: 40 });+ });+ it(`preserves ${stream ? 'SSE' : 'JSON'} successful tool response`, async () => {+ const result = await exercise(stream);+ expect(result.status).toBe(200);+ expect(result.content).toContain('synthetic-call');+ expect(result.network).toEqual([]);+ });+}+it('unauthenticated request does not produce a cost write', async () => {+ const result = await exercise(false, false);+ expect(result.status).toBe(401);+ expect(result.writes).toEqual([]);+ expect(result.network).toEqual([]);+});