YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: cover missing model accounting fallbacks
Louis Beaumont79676913d905
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex a33ba65e4..9102a715e 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -1,5 +1,18 @@# Coding-agent regression evals+`ai-gateway-null-model-accounting` executes the complete accounting module's+existing exports without provider or database calls. Thirteen outcomes cover+absent model names and preserved unknown-string, measured/estimated token,+versioned-name, priced-model and free-model behavior. The parent fails six+missing-model outcomes and preserves seven; the historical fix and current+source pass thirteen. Run+`bun test evals/coding-agent/calibrate-null-model-accounting.test.js` for eleven+controls, including equivalent guards, unused correct code, blanket fallback,+lost free classification and missing-source setup errors. The fixture appears+only at grading and needs no dependency links. This does not execute HTTP+admission, real billing or settlement, and does not establish agent isolation+or model performance.+`app-zero-channel-downmix` compiles the actual complete converter module withsynthetic buffers. Ten outcomes cover zero-channel passthrough and preservedmono, signed multichannel, silence, empty, high-channel and partial-frame behavior.diff --git a/evals/coding-agent/calibrate-null-model-accounting.test.js b/evals/coding-agent/calibrate-null-model-accounting.test.jsnew file mode 100644index 000000000..24efb8e25--- /dev/null+++ b/evals/coding-agent/calibrate-null-model-accounting.test.js@@ -0,0 +1,54 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { dirname, join, resolve } from 'node:path';+import { tmpdir } from 'node:os';+const repo = resolve(import.meta.dir, '../..');+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'))).cases.find(c => c.id === 'ai-gateway-null-model-accounting');+const root = mkdtempSync(join(tmpdir(), 'null-model-calibration-'));+afterAll(() => rmSync(root, { recursive: true, force: true }));+const fixture = readFileSync(join(import.meta.dir, 'graders/null-model-accounting.fixture.ts.txt'));+const path = 'packages/ai-gateway/src/services/cost-tracker.ts';+function replaceOne(text, before, after) {+ expect(text.split(before)).toHaveLength(2);+ return text.replace(before, after);+}+function grade(name) {+ const cwd = join(root, name); mkdirSync(cwd);+ const ref = ['parent', 'unused', 'oracle-only'].includes(name) ? item.base_ref : name === 'current' ? 'HEAD' : item.oracle_ref;+ execFileSync('tar', ['-x', '-C', cwd], { input: execFileSync('git', ['archive', ref, 'packages/ai-gateway/src'], { cwd: repo, maxBuffer: 32 * 1024 * 1024 }) });+ const file = join(cwd, path);+ if (name === 'unused' || name === 'oracle-only') writeFileSync(name === 'unused' ? file + '.unused.ts' : file, execFileSync('git', ['show', `${item.oracle_ref}:${path}`], { cwd: repo }));+ let source = readFileSync(file, 'utf8');+ if (name === 'equivalent') source = source.replaceAll("typeof model !== 'string' || model.length === 0", "!(typeof model === 'string' && model.length > 0)");+ if (name === 'free-unknown') source = replaceOne(source, 'return pricing !== null && pricing.input === 0 && pricing.output === 0;', 'return pricing === null || (pricing.input === 0 && pricing.output === 0);');+ if (name === 'blanket-fallback') source = replaceOne(source, 'const pricing = findPricing(model);\n if (!pricing)', 'const pricing = null;\n if (!pricing)');+ if (name === 'lost-free') source = replaceOne(source, 'return pricing !== null && pricing.input === 0 && pricing.output === 0;', 'return false;');+ if (name === 'missing-provider-guard') source = replaceOne(source, "if (typeof model !== 'string' || model.length === 0) return 'unknown';", '');+ writeFileSync(file, source);+ if (name === 'missing-source') rmSync(file);+ const testPath = 'packages/ai-gateway/src/test/eval-null-model-accounting.test.ts';+ writeFileSync(join(cwd, testPath), fixture);+ const result = spawnSync(process.execPath, ['test', testPath], { cwd, encoding: 'utf8', timeout: 15000, env: { PATH: dirname(process.execPath), CI: 'true' } });+ if (process.env.EVAL_CALIBRATION_RESULTS) {+ const out = resolve(process.env.EVAL_CALIBRATION_RESULTS); mkdirSync(out, { recursive: true });+ for (const ext of ['stdout', 'stderr']) writeFileSync(join(out, name + '.' + ext), result[ext] || '');+ writeFileSync(join(out, name + '.json'), JSON.stringify({ status: result.status, signal: result.signal, error: result.error?.message || null }) + '\n');+ }+ return result;+}+function passes(r) { expect(r.error).toBeUndefined(); expect(r.status).toBe(0); expect(r.stderr).toContain('13 pass'); }+function fails(r, count) { expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); expect(r.status).toBe(1); expect(r.stderr).not.toContain('Cannot find module'); expect(r.stderr).toMatch(/expect\(received\)|TypeError: null is not an object|undefined is not an object/); expect(r.stderr).toContain(`${count} fail`); expect(r.stderr).toContain(`${13 - count} pass`); }+test('parent fails six missing-model outcomes and preserves seven', () => fails(grade('parent'), 6));+test('reference passes', () => passes(grade('reference')));+test('current source passes', () => passes(grade('current')));+test('manifest oracle path alone repairs the parent', () => passes(grade('oracle-only')));+test('unused correct source does not fix executed exports', () => fails(grade('unused'), 6));+test('equivalent guard is accepted', () => passes(grade('equivalent')));+test('unknown models marked free are rejected', () => fails(grade('free-unknown'), 5));+test('blanket fallback loses priced and free behavior', () => fails(grade('blanket-fallback'), 4));+test('removing free classification is rejected', () => fails(grade('lost-free'), 1));+test('price guard alone cannot repair provider inference', () => fails(grade('missing-provider-guard'), 2));+test('missing module remains a setup error', () => { const r = grade('missing-source'); expect(r.status).not.toBe(0); expect(r.stderr).toContain('Cannot find module'); expect(r.stderr).not.toContain('expect(received)'); });diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex ca39f34f5..b31926aec 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -5975,6 +5975,38 @@],"command": "cd packages/screenpipe-mcp && SCREENPIPE_MCP_TELEMETRY_DISABLED=1 DO_NOT_TRACK=1 node node_modules/vitest/vitest.mjs run src/eval-evidence.test.ts"}+ },+ {+ "id": "ai-gateway-null-model-accounting",+ "name": "Preserve accounting fallbacks when the model name is absent",+ "source": {+ "kind": "git_regression",+ "fix_commit": "982e1b2849b81966153cba1cd7d30e07a26ed25b"+ },+ "base_ref": "4b8326d778d7ea2ced2a80ea9cf92b63ba59795e",+ "oracle_ref": "982e1b2849b81966153cba1cd7d30e07a26ed25b",+ "oracle_paths": [+ "packages/ai-gateway/src/services/cost-tracker.ts"+ ],+ "gate": "advisory",+ "tags": [+ "regression",+ "accounting",+ "missing-input",+ "preserved-behavior"+ ],+ "prompt": "Gateway accounting crashes when the model name is absent. Repair the existing exported getModelCost, isZeroCostModel and inferProvider behavior for undefined and null model names: return the existing unknown-model estimate of 0.01 USD, false for free-model classification, and unknown for provider inference without throwing. The unknown estimate applies with either absent or measured token counts. Preserve the same behavior for empty, whitespace-only and unrecognized model strings. Preserve known-model accounting: the existing gpt-5.4-nano table uses 0.20 USD per million input tokens and 1.25 per million output tokens, gives 0.00045 for 1000 input/200 output, zero for zero tokens, and 0.001025 when both token counts are null. A lone null token count contributes zero when the other is measured. Preserve case-insensitive versioned-name matching and the longest matching price entry, OpenAI provider inference, and the existing screenpipe-event-classifier free/zero-cost/screenpipe-vllm behavior. These are repository pricing fixtures, not a claim about current external prices. Any implementation meeting the exported behavior is acceptable; do not require a specific private helper. The grader executes the complete accounting module without database or provider calls. This case does not establish HTTP admission, live billing, settlement durability, deployment, agent isolation or model capability. Add focused tests.",+ "agent_timeout_seconds": 900,+ "grader": {+ "timeout_seconds": 30,+ "fixtures": [+ {+ "local_path": "graders/null-model-accounting.fixture.ts.txt",+ "destination_path": "packages/ai-gateway/src/test/eval-null-model-accounting.test.ts"+ }+ ],+ "command": "\"${BUN_BIN:-bun}\" test packages/ai-gateway/src/test/eval-null-model-accounting.test.ts"+ }}]}diff --git a/evals/coding-agent/graders/null-model-accounting.fixture.ts.txt b/evals/coding-agent/graders/null-model-accounting.fixture.ts.txtnew file mode 100644index 000000000..d1ce5c39d--- /dev/null+++ b/evals/coding-agent/graders/null-model-accounting.fixture.ts.txt@@ -0,0 +1,48 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { expect, test } from 'bun:test';+import { getModelCost, inferProvider, isZeroCostModel } from '../services/cost-tracker';++// Exercise existing exported accounting behavior, without providers or a database.+for (const [label, model] of [['missing', undefined], ['null', null]] as const) {+ test(`${label} model uses the unknown-price estimate`, () => {+ expect(() => getModelCost(model, null, null)).not.toThrow();+ expect(getModelCost(model, null, null)).toBe(0.01);+ expect(getModelCost(model, 400, 70)).toBe(0.01);+ });+ test(`${label} model is not free`, () => {+ expect(() => isZeroCostModel(model)).not.toThrow();+ expect(isZeroCostModel(model)).toBe(false);+ });+ test(`${label} model has unknown provider`, () => {+ expect(() => inferProvider(model)).not.toThrow();+ expect(inferProvider(model)).toBe('unknown');+ });+}+for (const model of ['', ' ', 'synthetic-unlisted-model']) {+ test(`unrecognized string ${JSON.stringify(model)} preserves fallback`, () => {+ expect(getModelCost(model, null, null)).toBe(0.01);+ expect(isZeroCostModel(model)).toBe(false);+ expect(inferProvider(model)).toBe('unknown');+ });+}+test('priced model retains measured and zero-token cost', () => {+ expect(getModelCost('gpt-5.4-nano', 1000, 200)).toBeCloseTo(0.00045, 10);+ expect(getModelCost('gpt-5.4-nano', 0, 0)).toBe(0);+ expect(isZeroCostModel('gpt-5.4-nano')).toBe(false);+ expect(inferProvider('gpt-5.4-nano')).toBe('openai');+});+test('unknown token counts retain the existing estimate and one-sided counts', () => {+ expect(getModelCost('gpt-5.4-nano', null, null)).toBeCloseTo(0.001025, 10);+ expect(getModelCost('gpt-5.4-nano', null, 200)).toBeCloseTo(0.00025, 10);+ expect(getModelCost('gpt-5.4-nano', 1000, null)).toBeCloseTo(0.0002, 10);+});+test('case-insensitive versioned model retains longest pricing match', () => {+ expect(getModelCost('GPT-5.4-NANO-2026', 1000, 200)).toBeCloseTo(0.00045, 10);+ expect(inferProvider('GPT-5.4-NANO-2026')).toBe('openai');+});+test('existing free model remains free', () => {+ expect(getModelCost('screenpipe-event-classifier', null, null)).toBe(0);+ expect(isZeroCostModel('screenpipe-event-classifier')).toBe(true);+ expect(inferProvider('screenpipe-event-classifier')).toBe('screenpipe-vllm');+});