Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

test(evals): cover initial lazy streaming failures

louis394ca42b6a1a
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 6460cd966..ef2b9fcec 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -975,3 +975,20 @@ only when grading begins. These checks do not establish native notification
display, completed Pipe results, policy authority to start work, multi-window
concurrency, execution isolation or model performance. The task covers an already
scheduled follow-up and does not authorize activating paused work.
+
+## Lazy streaming errors
+
+`ai-gateway-lazy-stream-initial-error` exercises the actual OpenAI-compatible
+provider with synthetic SDK iterables and inert telemetry. Initial lazy quota
+and service failures reject before a successful stream is returned. Five nearby
+outcomes preserve SDK creation errors, text, tool fragments and usage, empty
+streams, and SSE errors after partial delivery. The parent fails the two initial
+error outcomes and preserves five; the reference and current provider pass seven.
+
+Run `bun test evals/coding-agent/calibrate-stream-initial-error.test.js` for nine
+controls, including unused correct code, equivalent iterator names, lost or
+repeated first chunks, incorrect later error status and missing-source setup
+classification. No dependencies or build caches are linked by this case. Fixtures
+are materialized at grading time. This does not establish caller fallback
+selection, cancellation, live provider behavior, execution isolation or model
+performance, and does not reinstate historical Argus routing policy.
diff --git a/evals/coding-agent/calibrate-stream-initial-error.test.js b/evals/coding-agent/calibrate-stream-initial-error.test.js
new file mode 100644
index 000000000..209c4c7ae
--- /dev/null
+++ b/evals/coding-agent/calibrate-stream-initial-error.test.js
@@ -0,0 +1,35 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { dirname, join, resolve } from 'node:path';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { classifyGraderError } from './grader-outcome.mjs';
+const repo = resolve(import.meta.dir, '../..'), source = 'packages/ai-gateway/src/providers/openai.ts';
+const fixture = 'evals/coding-agent/graders/gateway-stream-initial-error.test.js';
+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'ai-gateway-lazy-stream-initial-error');
+const receipts = process.env.EVAL_CALIBRATION_RESULTS_DIR;
+if (receipts) mkdirSync(receipts, { recursive: true });
+const root = mkdtempSync(join(receipts || tmpdir(), 'stream-initial-calibration-')), archives = new Map();
+afterAll(() => { if (!receipts) rmSync(root, { recursive: true, force: true }); });
+function grade(name, ref = item.oracle_ref, mutate = () => {}) {
+ const cwd = join(root, name); mkdirSync(cwd);
+ if (!archives.has(ref)) archives.set(ref, execFileSync('git', ['-c', 'core.commitGraph=false', 'archive', ref, 'packages/ai-gateway'], { cwd: repo, maxBuffer: 32 * 1024 * 1024 }));
+ execFileSync('tar', ['-x', '-C', cwd], { input: archives.get(ref) });
+ mkdirSync(dirname(join(cwd, fixture)), { recursive: true }); writeFileSync(join(cwd, fixture), readFileSync(join(repo, fixture))); mutate(cwd);
+ const r = spawnSync(process.execPath, ['test', fixture], { cwd, encoding: 'utf8', timeout: 20_000, env: { PATH: dirname(process.execPath) } });
+ if (receipts) writeFileSync(join(receipts, name + '.json'), JSON.stringify({ ref, command: [process.execPath, 'test', fixture], cwd, status: r.status, signal: r.signal, error: r.error?.message, stdout: r.stdout, stderr: r.stderr, failure_kind: classifyGraderError(r) }, null, 2));
+ return r;
+}
+function change(cwd, from, to) { const path = join(cwd, source), text = readFileSync(path, 'utf8'); expect(text.split(from)).toHaveLength(2); writeFileSync(path, text.replace(from, to)); }
+function fails(r) { expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); expect(r.status).toBe(1); expect(r.stderr).toContain('expect(received)'); expect(classifyGraderError(r)).toBeNull(); }
+test('parent fails two intended lazy errors and preserves five neighbors', () => { const r = grade('parent', item.base_ref); fails(r); expect(r.stderr).toContain('2 fail'); expect(r.stderr).toContain('5 pass'); });
+test('reference passes all seven outcomes', () => { const r = grade('reference'); expect(r.status).toBe(0); expect(r.stderr).toContain('7 pass'); });
+test('current source passes the same outcomes', () => { const r = grade('current', 'HEAD'); expect(r.status).toBe(0); expect(r.stderr).toContain('7 pass'); });
+test('equivalent private iterator names are accepted', () => { const r = grade('equivalent', item.oracle_ref, cwd => { const path = join(cwd, source); writeFileSync(path, readFileSync(path, 'utf8').replace(/\bfirstChunk\b/g, 'initialResult').replace(/\bprimedStream\b/g, 'forwardedStream')); }); expect(r.status).toBe(0); });
+test('unused correct provider cannot hide unchanged broken dispatch', () => { const r = grade('unused-correct', item.base_ref, cwd => writeFileSync(join(cwd, 'packages/ai-gateway/src/providers/unused-correct.ts'), execFileSync('git', ['show', item.oracle_ref + ':' + source], { cwd: repo }))); fails(r); expect(r.stderr).toContain('2 fail'); });
+test('dropping the first chunk is rejected', () => fails(grade('lost-first', item.oracle_ref, cwd => change(cwd, 'if (!firstChunk.done) yield firstChunk.value;', 'if (!firstChunk.done) { /* dropped */ }'))));
+test('duplicating the first chunk is rejected', () => fails(grade('duplicate-first', item.oracle_ref, cwd => change(cwd, 'if (!firstChunk.done) yield firstChunk.value;', 'if (!firstChunk.done) { yield firstChunk.value; yield firstChunk.value; }'))));
+test('fabricating a later error status is rejected', () => fails(grade('wrong-midstream-status', item.oracle_ref, cwd => change(cwd, "const errorStatus = error?.status || 500;", 'const errorStatus = 200;'))));
+test('missing provider is a setup failure with zero executed tests', () => { const r = grade('missing-source', item.oracle_ref, cwd => rmSync(join(cwd, source))); expect(r.status).toBe(1); expect(r.stderr).toContain('0 pass'); expect(classifyGraderError(r)).toBe('bun_unhandled_error'); });
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index e8566f7fc..3013ef6ed 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-09-26.2",
+ "dataset_version": "2026-09-26.3",
"suite_type": "regression",
"owner": "screenpipe engineering",
"policy": {
@@ -3528,6 +3528,43 @@
],
"command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-notification-transport.config.mjs"
}
+ },
+ {
+ "id": "ai-gateway-lazy-stream-initial-error",
+ "name": "Expose initial lazy-stream failures before successful stream delivery",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "e8ed10afed8400e943c7ab2cb9c7a18f7b83e5b6",
+ "pull_request": "https://github.com/screenpipe/screenpipe/pull/5840"
+ },
+ "base_ref": "bfaeef6183c6b757f9779b25635d3687466f56e3",
+ "oracle_ref": "e8ed10afed8400e943c7ab2cb9c7a18f7b83e5b6",
+ "oracle_paths": [
+ "packages/ai-gateway/src/providers/openai.ts"
+ ],
+ "gate": "advisory",
+ "trigger_paths": [
+ "packages/ai-gateway/src/providers/openai.ts"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "ai-gateway",
+ "streaming",
+ "error-handling"
+ ],
+ "prompt": "An OpenAI-compatible SDK can return a stream successfully and raise its upstream error only when iteration begins. The provider currently turns that initial failure into an already-successful stream, preventing callers from handling it before delivery. Repair the actual OpenAIProvider streaming boundary so initial quota and service-unavailable errors reject before returning a successful stream, preserving their message, status and code. Preserve SDK creation errors as well. Successful streams must deliver the first and later text or tool-call fragments exactly once and in order, preserve tool identity and argument assembly, completion reasons and usage including cached tokens, and end with one [DONE]. Empty upstream streams must complete normally without invented content. After content has begun, later errors must remain SSE errors with original message and numeric status code, network_error completion and [DONE], retaining the partial content. Preserve the input object and outgoing model/messages/stream request. Use synthetic SDK and telemetry ports and no external network. No particular iterator, private helper or priming implementation is required. Caller fallback selection, retired model-routing policy, cancellation, live provider behavior and model quality are outside this task.",
+ "agent_timeout_seconds": 900,
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/gateway-stream-initial-error.test.js",
+ "destination_path": "evals/coding-agent/graders/gateway-stream-initial-error.test.js"
+ }
+ ],
+ "command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/gateway-stream-initial-error.test.js"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/gateway-stream-initial-error.test.js b/evals/coding-agent/graders/gateway-stream-initial-error.test.js
new file mode 100644
index 000000000..506b9e93d
--- /dev/null
+++ b/evals/coding-agent/graders/gateway-stream-initial-error.test.js
@@ -0,0 +1,80 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, afterEach, beforeEach, expect, mock, test } from 'bun:test';
+let supply, requests, network;
+const originalFetch = globalThis.fetch;
+mock.module('openai', () => ({ default: class {
+ constructor(options) { this.baseURL = options.baseURL; }
+ chat = { completions: { create: async params => { requests.push(structuredClone(params)); return supply(); } } };
+} }));
+mock.module('@sentry/cloudflare', () => ({ captureException() {} }));
+globalThis.fetch = async () => { network++; throw Error('External network forbidden'); };
+afterAll(() => { globalThis.fetch = originalFetch; });
+const { OpenAIProvider } = await import('../../../packages/ai-gateway/src/providers/openai');
+const input = () => ({ model: 'gpt-4o', messages: [{ role: 'user', content: 'Synthetic stream check.' }] });
+const fault = (status, code) => Object.assign(new Error('Synthetic provider failure'), { status, code });
+function iterable(generate) { const value = generate(); value.controller = { abort() {} }; return value; }
+beforeEach(() => { requests = []; network = 0; });
+afterEach(() => { expect(network).toBe(0); });
+async function invoke() {
+ const body = input(), snapshot = structuredClone(body);
+ const stream = await new OpenAIProvider('synthetic-key').createStreamingCompletion(body);
+ expect(body).toEqual(snapshot);
+ expect(requests).toHaveLength(1); expect(requests[0]).toMatchObject({ model: body.model, messages: body.messages, stream: true });
+ return stream;
+}
+async function delivered() {
+ const text = await new Response(await invoke()).text();
+ expect(text.split('data: [DONE]')).toHaveLength(2);
+ return text.split('\n\n').filter(l => l.startsWith('data: ') && l !== 'data: [DONE]').map(l => JSON.parse(l.slice(6)));
+}
+for (const [status, code] of [[429, 'insufficient_quota'], [503, 'service_unavailable']]) test(`initial lazy ${status} rejects before exposing a successful stream`, async () => {
+ supply = () => iterable(async function* () { throw fault(status, code); });
+ await expect(invoke()).rejects.toMatchObject({ message: 'Synthetic provider failure', status, code });
+});
+test('SDK creation failure retains error details', async () => {
+ supply = () => { throw fault(429, 'insufficient_quota'); };
+ await expect(invoke()).rejects.toMatchObject({ message: 'Synthetic provider failure', status: 429, code: 'insufficient_quota' });
+});
+test('first and later text arrive exactly once in order', async () => {
+ supply = () => iterable(async function* () {
+ yield { choices: [{ delta: { content: 'first|' } }] };
+ yield { choices: [{ delta: { content: 'second' } }] };
+ yield { choices: [{ delta: {}, finish_reason: 'stop' }] };
+ });
+ const events = await delivered();
+ expect(events.map(e => e.choices?.[0]?.delta?.content ?? '').join('')).toBe('first|second');
+ expect(events.filter(e => e.error)).toHaveLength(0);
+ expect(events.filter(e => e.choices?.[0]?.finish_reason).map(e => e.choices[0].finish_reason)).toEqual(['stop']);
+});
+test('first tool fragment, later arguments, completion reason and usage survive', async () => {
+ supply = () => iterable(async function* () {
+ yield { choices: [{ delta: { tool_calls: [{ index: 0, id: 'synthetic-call', type: 'function', function: { name: 'read', arguments: '{"path":' } }] } }] };
+ yield { choices: [{ delta: { tool_calls: [{ index: 0, function: { arguments: '"sample.txt"}' } }] } }] };
+ yield { choices: [{ delta: {}, finish_reason: 'tool_calls' }] };
+ yield { choices: [], usage: { prompt_tokens: 23, completion_tokens: 7, total_tokens: 30, prompt_tokens_details: { cached_tokens: 9 } } };
+ });
+ const events = await delivered(), calls = events.flatMap(e => e.choices?.[0]?.delta?.tool_calls ?? []);
+ expect(calls.filter(c => c.id).map(c => ({ id: c.id, name: c.function?.name }))).toEqual([{ id: 'synthetic-call', name: 'read' }]);
+ expect(calls.map(c => c.function?.arguments ?? '').join('')).toBe('{"path":"sample.txt"}');
+ expect(events.filter(e => e.choices?.[0]?.finish_reason).map(e => e.choices[0].finish_reason)).toEqual(['tool_calls']);
+ expect(events.filter(e => e.usage).map(e => e.usage)).toMatchObject([{ prompt_tokens: 23, completion_tokens: 7, total_tokens: 30, prompt_tokens_details: { cached_tokens: 9 } }]);
+});
+test('empty upstream completes normally without invented content', async () => {
+ supply = () => iterable(async function* () {});
+ const events = await delivered();
+ expect(events.filter(e => e.error)).toHaveLength(0);
+ expect(events.map(e => e.choices?.[0]?.delta?.content ?? '').join('')).toBe('');
+ expect(events.flatMap(e => e.choices?.[0]?.delta?.tool_calls ?? [])).toEqual([]);
+ expect(events.filter(e => e.choices?.[0]?.finish_reason).map(e => e.choices[0].finish_reason)).toEqual(['stop']);
+});
+test('failure after delivered content remains an SSE error and closes', async () => {
+ supply = () => iterable(async function* () {
+ yield { choices: [{ delta: { content: 'partial evidence' } }] };
+ throw fault(503, 'service_unavailable');
+ });
+ const events = await delivered();
+ expect(events.map(e => e.choices?.[0]?.delta?.content ?? '').join('')).toBe('partial evidence');
+ expect(events.filter(e => e.error).map(e => e.error)).toMatchObject([{ message: 'Synthetic provider failure', code: '503' }]);
+ expect(events.at(-1).choices[0].finish_reason).toBe('network_error');
+});