Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

eval: cover reasoning-only Vertex streaming fallback

Louis Beaumont509a86ab3a59
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 3a216818d..116aee4ca 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -1,5 +1,16 @@
# Coding-agent regression evals
+`ai-gateway-vertex-stream-fallback` executes the public streaming provider with
+synthetic authentication and transport. Twelve outcomes cover reasoning-only
+finish, terminal marker, EOF, split Unicode, CRLF and empty deltas, while
+preserving ordinary answers, reasoning events, tools, usage and refusals.
+Run `bun test evals/coding-agent/calibrate-vertex-stream-fallback.test.js` for ten
+controls, including equivalent guards, disconnected correct code, duplicate
+fallback, overwritten answers, lost reasoning and missing-source setup errors.
+Hidden fixtures appear only at grading; no dependency links are needed.
+This does not test live models, multiple-choice streams, full HTTP gateway
+admission, billing, desktop rendering, execution isolation or model trials.
+
`app-browser-native-consent-navigation` mounts the actual browser sidebar,
event hook and ownership helpers with synthetic native and persistence ports.
Twelve outcomes cover native and compatible event formats, default/user tabs,
diff --git a/evals/coding-agent/calibrate-vertex-stream-fallback.test.js b/evals/coding-agent/calibrate-vertex-stream-fallback.test.js
new file mode 100644
index 000000000..a64399669
--- /dev/null
+++ b/evals/coding-agent/calibrate-vertex-stream-fallback.test.js
@@ -0,0 +1,54 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join, resolve } from 'node:path';
+import { createHash } from 'node:crypto';
+import { classifyGraderError } from './grader-outcome.mjs';
+const repo = resolve(import.meta.dir, '../..');
+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'))).cases.find(c => c.id === 'ai-gateway-vertex-stream-fallback');
+const sourcePath = item.oracle_paths[0];
+const git = (...args) => execFileSync('git', args, { cwd: repo, encoding: 'utf8' });
+const source = ref => git('show', `${ref}:${sourcePath}`);
+const parent = source(item.base_ref), fixed = source(item.oracle_ref);
+const fixture = readFileSync(join(import.meta.dir, 'graders/vertex-stream-fallback.fixture.ts.txt'));
+const hash = x => createHash('sha256').update(x).digest('hex');
+function change(from, to) { expect(fixed.split(from)).toHaveLength(2); return fixed.replace(from, to); }
+const controls = [
+ ['parent', parent, 'fail', item.base_ref],
+ ['reference', fixed, 'pass', item.base_ref],
+ ['current', source('HEAD'), 'pass', 'HEAD'],
+ ['equivalent', change('injected || contentEmitted || !reasoningBuffer || !lastTemplate', '[injected, contentEmitted, reasoningBuffer.length === 0, !lastTemplate].some(Boolean)'), 'pass', item.base_ref],
+ ['unused-correct', parent, 'fail', item.base_ref],
+ ['disconnected-transform', change('return response.body!.pipeThrough(promoteReasoningStream());','return response.body!;'), 'fail', item.base_ref],
+ ['duplicate-injection', change('injected = true;', 'injected = false;'), 'fail', item.base_ref],
+ ['overwrite-answer', change('injected || contentEmitted || !reasoningBuffer || !lastTemplate', 'injected || !reasoningBuffer || !lastTemplate'), 'fail', item.base_ref],
+ ['lose-reasoning-pass-through', fixed.replaceAll("controller.enqueue(encoder.encode(line + '\\n'));", "if (!line.includes('reasoning_content')) controller.enqueue(encoder.encode(line + '\\n'));"), 'fail', item.base_ref],
+ ['missing-source', null, 'error', item.base_ref],
+];
+for (const [name, body, expected, supportingRef] of controls) test(`Vertex stream fallback calibration: ${name}`, () => {
+ const root = mkdtempSync(join(tmpdir(), 'vertex-stream-fallback-'));
+ try {
+ const fingerprints = {};
+ for (const path of ['packages/ai-gateway/src/providers/vertex.ts', 'packages/ai-gateway/src/providers/base.ts', 'packages/ai-gateway/src/types.ts']) {
+ const bytes = git('show', `${supportingRef}:${path}`); const destination = join(root, path);
+ mkdirSync(dirname(destination), { recursive: true }); writeFileSync(destination, bytes); fingerprints[path] = hash(bytes);
+ }
+ if (body !== null) writeFileSync(join(root, sourcePath), body);
+ if (name === 'unused-correct') writeFileSync(join(root, 'unused-correct.ts'), fixed);
+ writeFileSync(join(root, 'packages/ai-gateway/eval-vertex-stream-fallback.test.ts'), fixture);
+ const result = spawnSync(process.execPath, ['--no-env-file', 'test', 'packages/ai-gateway/eval-vertex-stream-fallback.test.ts'], { cwd: root, encoding: 'utf8', timeout: 60000, maxBuffer: 4 * 1024 * 1024 });
+ const kind = classifyGraderError(result);
+ const observed = result.error || result.signal || kind ? 'error' : result.status === 0 ? 'pass' : 'fail';
+ if (process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS) {
+ const dir = resolve(process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS); mkdirSync(dir, { recursive: true });
+ writeFileSync(join(dir, `${name}.json`), JSON.stringify({ command: item.grader.command, expected, observed, exit: result.status, signal: result.signal, error: result.error?.message ?? null, error_kind: kind, supporting_ref: supportingRef, source_sha256: body === null ? null : hash(body), supporting_sha256: fingerprints, fixture_sha256: hash(fixture), stdout: result.stdout, stderr: result.stderr }, null, 2));
+ }
+ expect(result.error).toBeUndefined(); expect(result.signal).toBeNull(); expect(observed).toBe(expected);
+ if (expected === 'pass') expect(result.stderr).toContain('12 pass');
+ if (name === 'parent' || name === 'unused-correct') { expect(result.stderr).toContain('6 fail'); expect(result.stderr).toContain('6 pass'); }
+ if (name === 'missing-source') expect(kind).toBe('bun_unhandled_error');
+ } finally { rmSync(root, { recursive: true, force: true }); }
+}, 65000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 422c73244..f708d514f 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -6483,6 +6483,41 @@
"destination_path": "apps/screenpipe-app-tauri/node_modules"
}
]
+ },
+ {
+ "id": "ai-gateway-vertex-stream-fallback",
+ "name": "Keep reasoning-only Vertex streams visibly readable",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "65f7928a294cc847534cb1adfe693e8553addfa1"
+ },
+ "base_ref": "dec32b804203013868d99a87ba94b6bac61d00e9",
+ "oracle_ref": "65f7928a294cc847534cb1adfe693e8553addfa1",
+ "oracle_paths": [
+ "packages/ai-gateway/src/providers/vertex-maas.ts"
+ ],
+ "gate": "advisory",
+ "trigger_paths": [
+ "packages/ai-gateway/src/providers/vertex-maas.ts"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "streaming",
+ "provider-compatibility"
+ ],
+ "prompt": "The public Vertex MaaS streaming completion can return no visible answer when a thinking model supplies only reasoning_content. Preserve the original reasoning deltas and expose their complete text as content exactly once when the stream finishes, sends [DONE], or closes without either, provided no nonempty content was delivered. Handle empty content deltas, CRLF lines and arbitrary byte boundaries including split Unicode. Preserve response id, model and choice index; fallback content must precede an existing [DONE] marker. Ordinary content and reasoning followed by a real answer must remain exact without appended reasoning. Empty streams must not invent text. Preserve tool-call deltas, usage records, comments, malformed data lines and ordinary upstream client-error refusal. For the existing glm-4.7 endpoint, retain one completion POST, bearer authorization, user messages, zero temperature, max_tokens and stream:true without changing caller input. Use synthetic authentication and transport only; never call live providers. No particular helper or transform implementation is required. This task covers one-choice streaming transport, not non-streaming JSON parsing, multiple-choice multiplexing, live model output, billing or desktop rendering.",
+ "agent_timeout_seconds": 900,
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/vertex-stream-fallback.fixture.ts.txt",
+ "destination_path": "packages/ai-gateway/eval-vertex-stream-fallback.test.ts"
+ }
+ ],
+ "command": "bun --no-env-file test packages/ai-gateway/eval-vertex-stream-fallback.test.ts"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/vertex-stream-fallback.fixture.ts.txt b/evals/coding-agent/graders/vertex-stream-fallback.fixture.ts.txt
new file mode 100644
index 000000000..5485574ad
--- /dev/null
+++ b/evals/coding-agent/graders/vertex-stream-fallback.fixture.ts.txt
@@ -0,0 +1,53 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { beforeAll, afterEach, expect, test } from 'bun:test';
+import { VertexMaasProvider } from './src/providers/vertex-maas';
+import { resetTokenCache } from './src/providers/vertex';
+const originalFetch = globalThis.fetch;
+const encoder = new TextEncoder();
+let pem: string;
+beforeAll(async () => {
+ const key = await crypto.subtle.generateKey({name:'RSASSA-PKCS1-v1_5',modulusLength:2048,publicExponent:new Uint8Array([1,0,1]),hash:'SHA-256'},true,['sign','verify']);
+ pem='-----BEGIN PRIVATE KEY-----\n'+btoa(String.fromCharCode(...new Uint8Array(await crypto.subtle.exportKey('pkcs8',key.privateKey))))+'\n-----END PRIVATE KEY-----';
+});
+afterEach(() => { globalThis.fetch=originalFetch; resetTokenCache(); });
+function event(delta: any, finish_reason: string | null = null) {
+ return 'data: '+JSON.stringify({id:'synthetic-stream',model:'synthetic-model',choices:[{index:0,delta,finish_reason}]})+'\n\n';
+}
+async function stream(text: string, bytewise=false, status=200) {
+ resetTokenCache();let requests=0;
+ const body:any={model:'glm-4.7',messages:[{role:'user',content:'Synthetic request'}],temperature:0,max_tokens:64};
+ const before=JSON.stringify(body);
+ globalThis.fetch=(async (input:any,init:RequestInit={}) => {
+ const url=String(input);
+ if(url==='https://oauth2.googleapis.com/token')return Response.json({access_token:'synthetic-token',expires_in:3600});
+ expect(url).toBe('https://aiplatform.googleapis.com/v1/projects/synthetic-project/locations/global/endpoints/openapi/chat/completions');
+ expect(init.method).toBe('POST');expect(new Headers(init.headers).get('authorization')).toBe('Bearer synthetic-token');
+ expect(JSON.parse(String(init.body))).toMatchObject({messages:body.messages,temperature:0,max_tokens:64,stream:true});requests++;
+ const bytes=encoder.encode(text);
+ return new Response(new ReadableStream({start(c){if(bytewise){for(const byte of bytes)c.enqueue(Uint8Array.of(byte));}else c.enqueue(bytes);c.close();}}),{status});
+ }) as typeof fetch;
+ const provider=new VertexMaasProvider(JSON.stringify({project_id:'synthetic-project',client_email:'[email protected]',private_key:pem}),'synthetic-project');
+ try{return await new Response(await provider.createStreamingCompletion(body)).text();}
+ finally{expect(requests).toBe(1);expect(JSON.stringify(body)).toBe(before);}
+}
+function parsed(text:string) {
+ return text.split('\n').filter(l=>l.startsWith('data: ')&&l.trim()!=='data: [DONE]').flatMap(l=>{try{return [JSON.parse(l.slice(6))];}catch{return [];}});
+}
+function values(text:string,key:string) {return parsed(text).flatMap(x=>x.choices??[]).map(c=>c.delta?.[key]).filter(x=>typeof x==='string').join('');}
+const reason=event({reasoning_content:'Synthetic '})+event({reasoning_content:'fallback'});
+for(const [label,suffix] of [['finish',event({},'length')+'data: [DONE]\n\n'],['done','data: [DONE]\n\n'],['eof','']])test(`reasoning-only ${label} supplies one visible answer`,async()=>{
+ const out=await stream(reason+suffix);expect(values(out,'content')).toBe('Synthetic fallback');expect(values(out,'reasoning_content')).toBe('Synthetic fallback');
+ const content=parsed(out).filter(x=>x.choices?.some(c=>c.delta?.content));expect(content.length).toBeGreaterThan(0);
+ for(const x of content){expect(x.id).toBe('synthetic-stream');expect(x.model).toBe('synthetic-model');expect(x.choices[0].index).toBe(0);}
+ if(label!=='eof')expect(out.indexOf('"content":')).toBeLessThan(out.indexOf('data: [DONE]'));
+});
+test('split UTF8 bytes preserve fallback text',async()=>{const text='Synthetic café 🧪';const out=await stream(event({reasoning_content:text})+event({},'length')+'data: [DONE]\n\n',true);expect(values(out,'content')).toBe(text);expect(values(out,'reasoning_content')).toBe(text);});
+test('CRLF reasoning stream supplies fallback',async()=>{const out=await stream((reason+'data: [DONE]\n\n').replaceAll('\n','\r\n'));expect(values(out,'content')).toBe('Synthetic fallback');});
+test('empty content delta does not suppress fallback',async()=>{const out=await stream(event({content:''})+reason+event({},'length'));expect(values(out,'content')).toBe('Synthetic fallback');});
+test('ordinary content remains exact without fallback',async()=>{const out=await stream(event({content:'Visible '})+event({content:'answer'})+event({},'stop')+'data: [DONE]\n\n');expect(values(out,'content')).toBe('Visible answer');});
+test('reasoning followed by answer is not duplicated as answer',async()=>{const out=await stream(reason+event({content:'Final answer'})+event({},'stop')+'data: [DONE]\n\n');expect(values(out,'content')).toBe('Final answer');expect(values(out,'reasoning_content')).toBe('Synthetic fallback');});
+test('empty stream does not invent content',async()=>{const out=await stream(event({})+'data: [DONE]\n\n');expect(values(out,'content')).toBe('');});
+test('tool calls and usage survive without invented text',async()=>{const delta={tool_calls:[{index:0,id:'synthetic-call',type:'function',function:{name:'synthetic_tool',arguments:'{}'}}]};const usage={choices:[],usage:{completion_tokens:7}};const out=await stream(event(delta)+event({},'tool_calls')+'data: '+JSON.stringify(usage)+'\n\ndata: [DONE]\n\n');expect(parsed(out)[0].choices[0].delta).toEqual(delta);expect(parsed(out)).toContainEqual(usage);expect(values(out,'content')).toBe('');});
+test('comments and malformed data remain readable',async()=>{const text=': synthetic keepalive\n\ndata: {malformed}\n\n'+event({content:'Visible'})+'data: [DONE]\n\n';const out=await stream(text);expect(out).toContain(': synthetic keepalive');expect(out).toContain('data: {malformed}');expect(values(out,'content')).toBe('Visible');});
+test('upstream client refusal remains an error',async()=>{await expect(stream('synthetic refusal',false,400)).rejects.toThrow();});