YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
test(evals): cover initial lazy streaming failures
louis394ca42b6a1a
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 6460cd966..ef2b9fcec 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -975,3 +975,20 @@ only when grading begins. These checks do not establish native notificationdisplay, completed Pipe results, policy authority to start work, multi-windowconcurrency, execution isolation or model performance. The task covers an alreadyscheduled follow-up and does not authorize activating paused work.++## Lazy streaming errors++`ai-gateway-lazy-stream-initial-error` exercises the actual OpenAI-compatible+provider with synthetic SDK iterables and inert telemetry. Initial lazy quota+and service failures reject before a successful stream is returned. Five nearby+outcomes preserve SDK creation errors, text, tool fragments and usage, empty+streams, and SSE errors after partial delivery. The parent fails the two initial+error outcomes and preserves five; the reference and current provider pass seven.++Run `bun test evals/coding-agent/calibrate-stream-initial-error.test.js` for nine+controls, including unused correct code, equivalent iterator names, lost or+repeated first chunks, incorrect later error status and missing-source setup+classification. No dependencies or build caches are linked by this case. Fixtures+are materialized at grading time. This does not establish caller fallback+selection, cancellation, live provider behavior, execution isolation or model+performance, and does not reinstate historical Argus routing policy.diff --git a/evals/coding-agent/calibrate-stream-initial-error.test.js b/evals/coding-agent/calibrate-stream-initial-error.test.jsnew file mode 100644index 000000000..209c4c7ae--- /dev/null+++ b/evals/coding-agent/calibrate-stream-initial-error.test.js@@ -0,0 +1,35 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { dirname, join, resolve } from 'node:path';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { classifyGraderError } from './grader-outcome.mjs';+const repo = resolve(import.meta.dir, '../..'), source = 'packages/ai-gateway/src/providers/openai.ts';+const fixture = 'evals/coding-agent/graders/gateway-stream-initial-error.test.js';+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'ai-gateway-lazy-stream-initial-error');+const receipts = process.env.EVAL_CALIBRATION_RESULTS_DIR;+if (receipts) mkdirSync(receipts, { recursive: true });+const root = mkdtempSync(join(receipts || tmpdir(), 'stream-initial-calibration-')), archives = new Map();+afterAll(() => { if (!receipts) rmSync(root, { recursive: true, force: true }); });+function grade(name, ref = item.oracle_ref, mutate = () => {}) {+ const cwd = join(root, name); mkdirSync(cwd);+ if (!archives.has(ref)) archives.set(ref, execFileSync('git', ['-c', 'core.commitGraph=false', 'archive', ref, 'packages/ai-gateway'], { cwd: repo, maxBuffer: 32 * 1024 * 1024 }));+ execFileSync('tar', ['-x', '-C', cwd], { input: archives.get(ref) });+ mkdirSync(dirname(join(cwd, fixture)), { recursive: true }); writeFileSync(join(cwd, fixture), readFileSync(join(repo, fixture))); mutate(cwd);+ const r = spawnSync(process.execPath, ['test', fixture], { cwd, encoding: 'utf8', timeout: 20_000, env: { PATH: dirname(process.execPath) } });+ if (receipts) writeFileSync(join(receipts, name + '.json'), JSON.stringify({ ref, command: [process.execPath, 'test', fixture], cwd, status: r.status, signal: r.signal, error: r.error?.message, stdout: r.stdout, stderr: r.stderr, failure_kind: classifyGraderError(r) }, null, 2));+ return r;+}+function change(cwd, from, to) { const path = join(cwd, source), text = readFileSync(path, 'utf8'); expect(text.split(from)).toHaveLength(2); writeFileSync(path, text.replace(from, to)); }+function fails(r) { expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); expect(r.status).toBe(1); expect(r.stderr).toContain('expect(received)'); expect(classifyGraderError(r)).toBeNull(); }+test('parent fails two intended lazy errors and preserves five neighbors', () => { const r = grade('parent', item.base_ref); fails(r); expect(r.stderr).toContain('2 fail'); expect(r.stderr).toContain('5 pass'); });+test('reference passes all seven outcomes', () => { const r = grade('reference'); expect(r.status).toBe(0); expect(r.stderr).toContain('7 pass'); });+test('current source passes the same outcomes', () => { const r = grade('current', 'HEAD'); expect(r.status).toBe(0); expect(r.stderr).toContain('7 pass'); });+test('equivalent private iterator names are accepted', () => { const r = grade('equivalent', item.oracle_ref, cwd => { const path = join(cwd, source); writeFileSync(path, readFileSync(path, 'utf8').replace(/\bfirstChunk\b/g, 'initialResult').replace(/\bprimedStream\b/g, 'forwardedStream')); }); expect(r.status).toBe(0); });+test('unused correct provider cannot hide unchanged broken dispatch', () => { const r = grade('unused-correct', item.base_ref, cwd => writeFileSync(join(cwd, 'packages/ai-gateway/src/providers/unused-correct.ts'), execFileSync('git', ['show', item.oracle_ref + ':' + source], { cwd: repo }))); fails(r); expect(r.stderr).toContain('2 fail'); });+test('dropping the first chunk is rejected', () => fails(grade('lost-first', item.oracle_ref, cwd => change(cwd, 'if (!firstChunk.done) yield firstChunk.value;', 'if (!firstChunk.done) { /* dropped */ }'))));+test('duplicating the first chunk is rejected', () => fails(grade('duplicate-first', item.oracle_ref, cwd => change(cwd, 'if (!firstChunk.done) yield firstChunk.value;', 'if (!firstChunk.done) { yield firstChunk.value; yield firstChunk.value; }'))));+test('fabricating a later error status is rejected', () => fails(grade('wrong-midstream-status', item.oracle_ref, cwd => change(cwd, "const errorStatus = error?.status || 500;", 'const errorStatus = 200;'))));+test('missing provider is a setup failure with zero executed tests', () => { const r = grade('missing-source', item.oracle_ref, cwd => rmSync(join(cwd, source))); expect(r.status).toBe(1); expect(r.stderr).toContain('0 pass'); expect(classifyGraderError(r)).toBe('bun_unhandled_error'); });diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex e8566f7fc..3013ef6ed 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-09-26.2",+ "dataset_version": "2026-09-26.3","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -3528,6 +3528,43 @@],"command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-notification-transport.config.mjs"}+ },+ {+ "id": "ai-gateway-lazy-stream-initial-error",+ "name": "Expose initial lazy-stream failures before successful stream delivery",+ "source": {+ "kind": "git_regression",+ "fix_commit": "e8ed10afed8400e943c7ab2cb9c7a18f7b83e5b6",+ "pull_request": "https://github.com/screenpipe/screenpipe/pull/5840"+ },+ "base_ref": "bfaeef6183c6b757f9779b25635d3687466f56e3",+ "oracle_ref": "e8ed10afed8400e943c7ab2cb9c7a18f7b83e5b6",+ "oracle_paths": [+ "packages/ai-gateway/src/providers/openai.ts"+ ],+ "gate": "advisory",+ "trigger_paths": [+ "packages/ai-gateway/src/providers/openai.ts"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "ai-gateway",+ "streaming",+ "error-handling"+ ],+ "prompt": "An OpenAI-compatible SDK can return a stream successfully and raise its upstream error only when iteration begins. The provider currently turns that initial failure into an already-successful stream, preventing callers from handling it before delivery. Repair the actual OpenAIProvider streaming boundary so initial quota and service-unavailable errors reject before returning a successful stream, preserving their message, status and code. Preserve SDK creation errors as well. Successful streams must deliver the first and later text or tool-call fragments exactly once and in order, preserve tool identity and argument assembly, completion reasons and usage including cached tokens, and end with one [DONE]. Empty upstream streams must complete normally without invented content. After content has begun, later errors must remain SSE errors with original message and numeric status code, network_error completion and [DONE], retaining the partial content. Preserve the input object and outgoing model/messages/stream request. Use synthetic SDK and telemetry ports and no external network. No particular iterator, private helper or priming implementation is required. Caller fallback selection, retired model-routing policy, cancellation, live provider behavior and model quality are outside this task.",+ "agent_timeout_seconds": 900,+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/gateway-stream-initial-error.test.js",+ "destination_path": "evals/coding-agent/graders/gateway-stream-initial-error.test.js"+ }+ ],+ "command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/gateway-stream-initial-error.test.js"+ }}]}diff --git a/evals/coding-agent/graders/gateway-stream-initial-error.test.js b/evals/coding-agent/graders/gateway-stream-initial-error.test.jsnew file mode 100644index 000000000..506b9e93d--- /dev/null+++ b/evals/coding-agent/graders/gateway-stream-initial-error.test.js@@ -0,0 +1,80 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, afterEach, beforeEach, expect, mock, test } from 'bun:test';+let supply, requests, network;+const originalFetch = globalThis.fetch;+mock.module('openai', () => ({ default: class {+ constructor(options) { this.baseURL = options.baseURL; }+ chat = { completions: { create: async params => { requests.push(structuredClone(params)); return supply(); } } };+} }));+mock.module('@sentry/cloudflare', () => ({ captureException() {} }));+globalThis.fetch = async () => { network++; throw Error('External network forbidden'); };+afterAll(() => { globalThis.fetch = originalFetch; });+const { OpenAIProvider } = await import('../../../packages/ai-gateway/src/providers/openai');+const input = () => ({ model: 'gpt-4o', messages: [{ role: 'user', content: 'Synthetic stream check.' }] });+const fault = (status, code) => Object.assign(new Error('Synthetic provider failure'), { status, code });+function iterable(generate) { const value = generate(); value.controller = { abort() {} }; return value; }+beforeEach(() => { requests = []; network = 0; });+afterEach(() => { expect(network).toBe(0); });+async function invoke() {+ const body = input(), snapshot = structuredClone(body);+ const stream = await new OpenAIProvider('synthetic-key').createStreamingCompletion(body);+ expect(body).toEqual(snapshot);+ expect(requests).toHaveLength(1); expect(requests[0]).toMatchObject({ model: body.model, messages: body.messages, stream: true });+ return stream;+}+async function delivered() {+ const text = await new Response(await invoke()).text();+ expect(text.split('data: [DONE]')).toHaveLength(2);+ return text.split('\n\n').filter(l => l.startsWith('data: ') && l !== 'data: [DONE]').map(l => JSON.parse(l.slice(6)));+}+for (const [status, code] of [[429, 'insufficient_quota'], [503, 'service_unavailable']]) test(`initial lazy ${status} rejects before exposing a successful stream`, async () => {+ supply = () => iterable(async function* () { throw fault(status, code); });+ await expect(invoke()).rejects.toMatchObject({ message: 'Synthetic provider failure', status, code });+});+test('SDK creation failure retains error details', async () => {+ supply = () => { throw fault(429, 'insufficient_quota'); };+ await expect(invoke()).rejects.toMatchObject({ message: 'Synthetic provider failure', status: 429, code: 'insufficient_quota' });+});+test('first and later text arrive exactly once in order', async () => {+ supply = () => iterable(async function* () {+ yield { choices: [{ delta: { content: 'first|' } }] };+ yield { choices: [{ delta: { content: 'second' } }] };+ yield { choices: [{ delta: {}, finish_reason: 'stop' }] };+ });+ const events = await delivered();+ expect(events.map(e => e.choices?.[0]?.delta?.content ?? '').join('')).toBe('first|second');+ expect(events.filter(e => e.error)).toHaveLength(0);+ expect(events.filter(e => e.choices?.[0]?.finish_reason).map(e => e.choices[0].finish_reason)).toEqual(['stop']);+});+test('first tool fragment, later arguments, completion reason and usage survive', async () => {+ supply = () => iterable(async function* () {+ yield { choices: [{ delta: { tool_calls: [{ index: 0, id: 'synthetic-call', type: 'function', function: { name: 'read', arguments: '{"path":' } }] } }] };+ yield { choices: [{ delta: { tool_calls: [{ index: 0, function: { arguments: '"sample.txt"}' } }] } }] };+ yield { choices: [{ delta: {}, finish_reason: 'tool_calls' }] };+ yield { choices: [], usage: { prompt_tokens: 23, completion_tokens: 7, total_tokens: 30, prompt_tokens_details: { cached_tokens: 9 } } };+ });+ const events = await delivered(), calls = events.flatMap(e => e.choices?.[0]?.delta?.tool_calls ?? []);+ expect(calls.filter(c => c.id).map(c => ({ id: c.id, name: c.function?.name }))).toEqual([{ id: 'synthetic-call', name: 'read' }]);+ expect(calls.map(c => c.function?.arguments ?? '').join('')).toBe('{"path":"sample.txt"}');+ expect(events.filter(e => e.choices?.[0]?.finish_reason).map(e => e.choices[0].finish_reason)).toEqual(['tool_calls']);+ expect(events.filter(e => e.usage).map(e => e.usage)).toMatchObject([{ prompt_tokens: 23, completion_tokens: 7, total_tokens: 30, prompt_tokens_details: { cached_tokens: 9 } }]);+});+test('empty upstream completes normally without invented content', async () => {+ supply = () => iterable(async function* () {});+ const events = await delivered();+ expect(events.filter(e => e.error)).toHaveLength(0);+ expect(events.map(e => e.choices?.[0]?.delta?.content ?? '').join('')).toBe('');+ expect(events.flatMap(e => e.choices?.[0]?.delta?.tool_calls ?? [])).toEqual([]);+ expect(events.filter(e => e.choices?.[0]?.finish_reason).map(e => e.choices[0].finish_reason)).toEqual(['stop']);+});+test('failure after delivered content remains an SSE error and closes', async () => {+ supply = () => iterable(async function* () {+ yield { choices: [{ delta: { content: 'partial evidence' } }] };+ throw fault(503, 'service_unavailable');+ });+ const events = await delivered();+ expect(events.map(e => e.choices?.[0]?.delta?.content ?? '').join('')).toBe('partial evidence');+ expect(events.filter(e => e.error).map(e => e.error)).toMatchObject([{ message: 'Synthetic provider failure', code: '503' }]);+ expect(events.at(-1).choices[0].finish_reason).toBe('network_error');+});