YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: verify voice transcription budgets before provider work
louis4c638a9cf2a0
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex f1a647fc4..ad46454e2 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -9,7 +9,7 @@ This is an agent eval suite, not a unit-test suite. Every case contains:- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.-The current app corpus contains 74 git-mined regressions. See+The current app corpus contains 75 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.@@ -25,6 +25,20 @@ desktop test dependencies installed. Graders and dependency links are installedonly after grading starts; those links do not establish execution isolation.No native ACP, live billing, entitlement safety or model performance is measured.+`ai-gateway-voice-transcription-budget` exercises the actual voice route, budget+readers and transcription handler with SQLite-backed D1 reads and synthetic+identity and provider ports.+The parent fails four intended budget refusals and preserves three outcomes;+the historical reference passes all seven. Exact cap boundaries refuse provider+work, while values immediately below all caps preserve the original audio bytes.+Run `bun test evals/coding-agent/calibrate-voice-transcription-budget.test.js` with+the gateway dependencies installed. Controls reject unused correct code, ignored+gates, provider work before refusal, blanket denial and fabricated transcripts,+and accept equivalent private bindings and SQL. A wrong account bucket is rejected.+Missing source is a setup failure. Telemetry writes remain inert.+This does not establish concurrent reservation safety, duration billing, anonymous+identity, realtime metering, live delivery, isolation or model capability.+`app-daily-summary-terminal-error` runs the actual summary orchestration andits internal helpers against synthetic native commands and event delivery.Thirteen outcomes cover terminal message/agent errors, original provider detail,diff --git a/evals/coding-agent/calibrate-voice-transcription-budget.test.js b/evals/coding-agent/calibrate-voice-transcription-budget.test.jsnew file mode 100644index 000000000..e3dd6bb9a--- /dev/null+++ b/evals/coding-agent/calibrate-voice-transcription-budget.test.js@@ -0,0 +1,64 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { cpSync, mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { dirname, join, resolve } from 'node:path';+import { tmpdir } from 'node:os';+const repo = resolve(import.meta.dir, '../..');+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'))).cases.find(c => c.id === 'ai-gateway-voice-transcription-budget');+const root = mkdtempSync(join(tmpdir(), 'voice-budget-calibration-'));+afterAll(() => rmSync(root, { recursive: true, force: true }));+const template = join(root, 'template');+mkdirSync(template);+execFileSync('tar', ['-x', '-C', template], { input: execFileSync('git', ['archive', item.oracle_ref, 'packages/ai-gateway/src'], { cwd: repo, maxBuffer: 16 * 1024 * 1024 }) });+const indexPath = 'packages/ai-gateway/src/index.ts';+const fixed = readFileSync(join(template, indexPath), 'utf8');+const broken = execFileSync('git', ['show', `${item.base_ref}:${indexPath}`], { cwd: repo, encoding: 'utf8' });+const graderPath = 'evals/coding-agent/graders/voice-transcription-budget.test.js';+const grader = readFileSync(join(repo, graderPath));+const marker = "if (path === '/v1/voice/transcribe' && request.method === 'POST') {";+const start = fixed.indexOf(marker), end = fixed.indexOf("if (path === '/v1/voice/query'", start);+expect(start).toBeGreaterThan(0); expect(end).toBeGreaterThan(start);+const route = fixed.slice(start, end);+function change(from, to) {+ expect(route.split(from)).toHaveLength(2);+ return fixed.slice(0, start) + route.replace(from, to) + fixed.slice(end);+}+function grade(name, source = fixed, unused = false, costSource = null) {+ const cwd = join(root, name); cpSync(template, cwd, { recursive: true });+ if (source === null) rmSync(join(cwd, indexPath)); else writeFileSync(join(cwd, indexPath), source);+ if (costSource !== null) writeFileSync(join(cwd, 'packages/ai-gateway/src/services/cost-tracker.ts'), costSource);+ if (unused) writeFileSync(join(cwd, 'unused-correct-dispatcher.ts'), fixed);+ symlinkSync(join(repo, 'packages/ai-gateway/node_modules'), join(cwd, 'packages/ai-gateway/node_modules'), 'dir');+ mkdirSync(dirname(join(cwd, graderPath)), { recursive: true }); writeFileSync(join(cwd, graderPath), grader);+ const result = spawnSync(process.execPath, ['test', graderPath], { cwd, encoding: 'utf8', timeout: 15000, env: { PATH: dirname(process.execPath) } });+ if (process.env.EVAL_CALIBRATION_RESULTS) {+ const out = resolve(process.env.EVAL_CALIBRATION_RESULTS); mkdirSync(out, { recursive: true });+ writeFileSync(join(out, name + '.stdout'), result.stdout || ''); writeFileSync(join(out, name + '.stderr'), result.stderr || '');+ writeFileSync(join(out, name + '.json'), JSON.stringify({ status: result.status, signal: result.signal, error: result.error?.message || null }) + '\n');+ }+ return result;+}+function fails(r) {+ expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); expect(r.status).toBe(1);+ expect(r.stderr).toContain('expect(received)'); expect(r.stderr).not.toContain('Cannot find module');+}+test('broken dispatcher fails four budget outcomes and preserves three', () => { const r = grade('parent', broken); fails(r); expect(r.stderr).toContain('4 fail'); expect(r.stderr).toContain('3 pass'); });+test('historical reference passes seven outcomes', () => { const r = grade('reference'); expect(r.status).toBe(0); expect(r.stderr).toContain('7 pass'); });+test('unused correct dispatcher cannot repair caller', () => fails(grade('unused', broken, true)));+test('equivalent private gate binding is accepted', () => {+ const renamed = fixed.replace('transcriptionGateResponse,', 'transcriptionGateResponse as voiceBudgetGate,').replaceAll('transcriptionGateResponse(env,', 'voiceBudgetGate(env,');+ expect(renamed).not.toBe(fixed); expect(grade('equivalent', renamed).status).toBe(0);+});+test('ignoring gate refusal is rejected', () => fails(grade('ignored', change('if (gate) return gate;', 'void gate;'))));+test('provider work before budget refusal is rejected', () => fails(grade('late', change('if (gate) return gate;', 'if (gate) { await handleVoiceTranscription(request, env); return gate; }'))));+test('blanket refusal loses preserved behavior', () => fails(grade('deny', change(marker, marker + '\n return new Response("blocked", {status:429});'))));+test('fabricated transcript without provider effect is rejected', () => fails(grade('fake', change('await handleVoiceTranscription(request, env)', 'Response.json({transcription:"synthetic transcript"})'))));+test('missing dispatcher is setup failure', () => { const r = grade('missing', null); expect(r.status).not.toBe(0); expect(r.stderr).toContain('Cannot find module'); expect(r.stderr).not.toContain('expect(received)'); });+test('equivalent SQL formatting remains accepted', () => {+ const source = readFileSync(join(template, 'packages/ai-gateway/src/services/cost-tracker.ts'), 'utf8');+ const equivalent = source.replaceAll('SELECT ', 'select ').replaceAll('SUM(daily_cost_usd)', 'sum( daily_cost_usd )');+ expect(equivalent).not.toBe(source); expect(grade('equivalent-sql', fixed, false, equivalent).status).toBe(0);+});+test('reading another account bucket is rejected', () => fails(grade('wrong-account', change('transcriptionCostIdentity(request, authResult)', '"user_unrelated"'))));diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex a43f5b98a..7dd9a9f8d 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-09-23.5",+ "dataset_version": "2026-09-24.1","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -3047,6 +3047,50 @@],"command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config vitest.config.ts eval-acp-billing-semantic.test.tsx"}+ },+ {+ "id": "ai-gateway-voice-transcription-budget",+ "name": "Enforce transcription budgets before voice provider work",+ "source": {+ "kind": "git_regression",+ "fix_commit": "dc35366d18ce0732e935cbfbdaa6093f6534406e"+ },+ "base_ref": "9064c3134d61bc3848ac3d84da24392f14d20ea5",+ "oracle_ref": "dc35366d18ce0732e935cbfbdaa6093f6534406e",+ "oracle_paths": [+ "packages/ai-gateway/src/index.ts",+ "packages/ai-gateway/src/handlers/realtime-transcription.ts",+ "packages/ai-gateway/src/services/cost-tracker.ts",+ "packages/ai-gateway/src/services/hosted-ai-cost-controls.ts",+ "packages/ai-gateway/src/services/transcription-budget.ts",+ "packages/ai-gateway/src/types.ts"+ ],+ "gate": "advisory",+ "tags": [+ "regression",+ "billing",+ "voice",+ "provider-effects",+ "budget"+ ],+ "prompt": "The hosted POST /v1/voice/transcribe route can contact its transcription provider despite exhausted spending limits. Repair the actual request boundary for an authenticated Business account in good standing. At the configured account daily, global hourly or global daily transcription cap, return 429 before any provider call. If the cost ledger cannot be read, return 503 before provider work. Use the existing cost ledger and configurable transcription limits; synthetic test caps are 103, 501 and 502 respectively, not production policy. Preserve an in-budget audio/wav request, including values immediately below every cap: send the original audio bytes to the provider exactly once and return 200 with its transcript in the transcription field. Preserve a 400 response with no provider call for application/json audio requests. Identity verification, entitlement service, rate limiter, D1 (SQLite-backed reads with inert telemetry writes) and transcription provider are synthetic ports; all other network access is refused. Any implementation achieving these outcomes is acceptable. This case does not establish atomic reservations or concurrent spending safety, duration-based billing, anonymous identity, realtime metering, live provider delivery, native behavior, agent isolation or model capability.",+ "agent_timeout_seconds": 900,+ "grader_dependency_links": [+ {+ "source_path": "packages/ai-gateway/node_modules",+ "destination_path": "packages/ai-gateway/node_modules"+ }+ ],+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/voice-transcription-budget.test.js",+ "destination_path": "evals/coding-agent/graders/voice-transcription-budget.test.js"+ }+ ],+ "command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/voice-transcription-budget.test.js"+ }}]}diff --git a/evals/coding-agent/graders/voice-transcription-budget.test.js b/evals/coding-agent/graders/voice-transcription-budget.test.jsnew file mode 100644index 000000000..579734f53--- /dev/null+++ b/evals/coding-agent/graders/voice-transcription-budget.test.js@@ -0,0 +1,86 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { beforeEach, afterEach, afterAll, expect, mock, setSystemTime, test } from 'bun:test';+import { Database } from 'bun:sqlite';+let state, serial = 0;+const originalFetch = globalThis.fetch;+mock.module('@clerk/backend', () => ({ verifyToken: async () => ({ sub: state.subject }) }));+mock.module('@deepgram/sdk', () => ({+ createClient: () => ({ listen: { prerecorded: {+ transcribeFile: async (body, options) => {+ state.provider.push({ body: Buffer.from(body), options });+ return { result: { results: { channels: [{ alternatives: [{+ transcript: 'synthetic transcript', confidence: 0.9, words: [],+ }] }] } }, error: null };+ },+ } } }),+}));+const { handleRequest } = await import('../../../packages/ai-gateway/src/index');+const { __resetAuthEntitlementCacheForTests: reset } = await import('../../../packages/ai-gateway/src/utils/auth');+const { privateCostControls } = await import('../../../packages/ai-gateway/src/test/fixtures/private-cost-controls');+const audio = Buffer.from([82, 73, 70, 70, 4, 0, 0, 0, 87, 65, 86, 69]);+function environment() {+ return {+ ...privateCostControls(), MAX_GLOBAL_HOURLY_TRANSCRIPTION_COST: '501', MAX_GLOBAL_DAILY_TRANSCRIPTION_COST: '502',+ NODE_ENV: 'production', CLERK_SECRET_KEY: 'synthetic', DEEPGRAM_API_KEY: 'synthetic',+ RATE_LIMITER: { idFromName: x => x, get: () => ({ fetch: async () => Response.json({+ allowed: true, standing: 'good', remaining: 10, reset_in: 60, tier: 'logged_in', rpm_limit: 25,+ }) }) },+ // Execute budget queries against seeded SQLite rows, accepting equivalent SQL.+ // Telemetry writes are inert ports; settlement durability is outside this case.+ DB: { prepare: sql => ({ bind: (...args) => ({+ first: async () => {+ if (state.fail) throw Error('synthetic D1 unavailable');+ return state.db.query(sql).get(...args);+ },+ all: async () => ({ results: state.db.query(sql).all(...args) }),+ run: async () => ({ success: true }),+ }) }), batch: async () => [] },+ };+}+beforeEach(() => {+ reset?.();+ setSystemTime(new Date("2026-01-02T12:00:00Z"));+ state = { subject: `user_voice_${++serial}`, provider: [], reads: [], pending: [], account: 0, hourly: 0, daily: 0, fail: false };+ state.db = new Database(':memory:');+ state.db.run('CREATE TABLE usage (device_id TEXT PRIMARY KEY, last_reset TEXT, cost_day TEXT, daily_cost_usd REAL)');+ globalThis.fetch = async input => {+ if (String(input) === 'https://screenpipe.com/api/user') return Response.json({ success: true, user: {+ clerk_id: state.subject, cloud_subscribed: true, app_entitled: true, subscription_plan: 'pro',+ entitlement: { active: true, plan: 'pro', features: { app: true, cloud: true } },+ } });+ throw Error('Unexpected network port');+ };+});+afterEach(() => { state.db.close(); });+afterAll(() => { globalThis.fetch = originalFetch; setSystemTime(); });+async function request(type = 'audio/wav') {+ const insert = state.db.query('INSERT INTO usage VALUES (?, ?, ?, ?)');+ insert.run(`hosted-transcription-cost:day:v1:${state.subject}`, '2026-01-02', '2026-01-02', state.account);+ insert.run('hosted-transcription-cost:global-hour:v1:2026-01-02T12', '2026-01-02T12', '2026-01-02T12', state.hourly);+ insert.run('hosted-transcription-cost:global-hour:v1:2026-01-02T11', '2026-01-02T11', '2026-01-02T11', Math.max(0, state.daily - state.hourly));+ const response = await handleRequest(new Request('https://gateway.example.invalid/v1/voice/transcribe', {+ method: 'POST', headers: { 'content-type': type, authorization: 'Bearer eyJ.synthetic.voice' }, body: audio,+ }), environment(), { waitUntil: promise => state.pending.push(promise), passThroughOnException: () => {} });+ await Promise.all(state.pending);+ return response;+}+async function refuse(status) {+ const response = await request(); expect(response.status).toBe(status); expect(state.provider).toEqual([]);+}+test('account exhaustion refuses before provider', async () => { state.account = 103; await refuse(429); });+test('global hour exhaustion refuses before provider', async () => { state.hourly = 501; await refuse(429); });+test('global day exhaustion refuses before provider', async () => { state.daily = 502; await refuse(429); });+test('unknown accounting refuses before provider', async () => { state.fail = true; await refuse(503); });+test('in-budget request preserves provider body and transcript', async () => {+ const response = await request(); expect(response.status).toBe(200); expect(state.provider).toHaveLength(1);+ expect(state.provider[0].body).toEqual(audio); expect(await response.json()).toMatchObject({ transcription: 'synthetic transcript' });+});+test('invalid audio type preserves refusal', async () => {+ const response = await request('application/json'); expect(response.status).toBe(400); expect(state.provider).toEqual([]);+});+test('spend immediately below every cap remains eligible', async () => {+ state.account = 102.999; state.hourly = 500.999; state.daily = 501.999;+ const response = await request(); expect(response.status).toBe(200); expect(state.provider).toHaveLength(1);+ expect(state.provider[0].body).toEqual(audio);+});