YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
eval: cover reasoning-only Vertex streaming fallback
Louis Beaumont509a86ab3a59
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 3a216818d..116aee4ca 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -1,5 +1,16 @@# Coding-agent regression evals+`ai-gateway-vertex-stream-fallback` executes the public streaming provider with+synthetic authentication and transport. Twelve outcomes cover reasoning-only+finish, terminal marker, EOF, split Unicode, CRLF and empty deltas, while+preserving ordinary answers, reasoning events, tools, usage and refusals.+Run `bun test evals/coding-agent/calibrate-vertex-stream-fallback.test.js` for ten+controls, including equivalent guards, disconnected correct code, duplicate+fallback, overwritten answers, lost reasoning and missing-source setup errors.+Hidden fixtures appear only at grading; no dependency links are needed.+This does not test live models, multiple-choice streams, full HTTP gateway+admission, billing, desktop rendering, execution isolation or model trials.+`app-browser-native-consent-navigation` mounts the actual browser sidebar,event hook and ownership helpers with synthetic native and persistence ports.Twelve outcomes cover native and compatible event formats, default/user tabs,diff --git a/evals/coding-agent/calibrate-vertex-stream-fallback.test.js b/evals/coding-agent/calibrate-vertex-stream-fallback.test.jsnew file mode 100644index 000000000..a64399669--- /dev/null+++ b/evals/coding-agent/calibrate-vertex-stream-fallback.test.js@@ -0,0 +1,54 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { dirname, join, resolve } from 'node:path';+import { createHash } from 'node:crypto';+import { classifyGraderError } from './grader-outcome.mjs';+const repo = resolve(import.meta.dir, '../..');+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'))).cases.find(c => c.id === 'ai-gateway-vertex-stream-fallback');+const sourcePath = item.oracle_paths[0];+const git = (...args) => execFileSync('git', args, { cwd: repo, encoding: 'utf8' });+const source = ref => git('show', `${ref}:${sourcePath}`);+const parent = source(item.base_ref), fixed = source(item.oracle_ref);+const fixture = readFileSync(join(import.meta.dir, 'graders/vertex-stream-fallback.fixture.ts.txt'));+const hash = x => createHash('sha256').update(x).digest('hex');+function change(from, to) { expect(fixed.split(from)).toHaveLength(2); return fixed.replace(from, to); }+const controls = [+ ['parent', parent, 'fail', item.base_ref],+ ['reference', fixed, 'pass', item.base_ref],+ ['current', source('HEAD'), 'pass', 'HEAD'],+ ['equivalent', change('injected || contentEmitted || !reasoningBuffer || !lastTemplate', '[injected, contentEmitted, reasoningBuffer.length === 0, !lastTemplate].some(Boolean)'), 'pass', item.base_ref],+ ['unused-correct', parent, 'fail', item.base_ref],+ ['disconnected-transform', change('return response.body!.pipeThrough(promoteReasoningStream());','return response.body!;'), 'fail', item.base_ref],+ ['duplicate-injection', change('injected = true;', 'injected = false;'), 'fail', item.base_ref],+ ['overwrite-answer', change('injected || contentEmitted || !reasoningBuffer || !lastTemplate', 'injected || !reasoningBuffer || !lastTemplate'), 'fail', item.base_ref],+ ['lose-reasoning-pass-through', fixed.replaceAll("controller.enqueue(encoder.encode(line + '\\n'));", "if (!line.includes('reasoning_content')) controller.enqueue(encoder.encode(line + '\\n'));"), 'fail', item.base_ref],+ ['missing-source', null, 'error', item.base_ref],+];+for (const [name, body, expected, supportingRef] of controls) test(`Vertex stream fallback calibration: ${name}`, () => {+ const root = mkdtempSync(join(tmpdir(), 'vertex-stream-fallback-'));+ try {+ const fingerprints = {};+ for (const path of ['packages/ai-gateway/src/providers/vertex.ts', 'packages/ai-gateway/src/providers/base.ts', 'packages/ai-gateway/src/types.ts']) {+ const bytes = git('show', `${supportingRef}:${path}`); const destination = join(root, path);+ mkdirSync(dirname(destination), { recursive: true }); writeFileSync(destination, bytes); fingerprints[path] = hash(bytes);+ }+ if (body !== null) writeFileSync(join(root, sourcePath), body);+ if (name === 'unused-correct') writeFileSync(join(root, 'unused-correct.ts'), fixed);+ writeFileSync(join(root, 'packages/ai-gateway/eval-vertex-stream-fallback.test.ts'), fixture);+ const result = spawnSync(process.execPath, ['--no-env-file', 'test', 'packages/ai-gateway/eval-vertex-stream-fallback.test.ts'], { cwd: root, encoding: 'utf8', timeout: 60000, maxBuffer: 4 * 1024 * 1024 });+ const kind = classifyGraderError(result);+ const observed = result.error || result.signal || kind ? 'error' : result.status === 0 ? 'pass' : 'fail';+ if (process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS) {+ const dir = resolve(process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS); mkdirSync(dir, { recursive: true });+ writeFileSync(join(dir, `${name}.json`), JSON.stringify({ command: item.grader.command, expected, observed, exit: result.status, signal: result.signal, error: result.error?.message ?? null, error_kind: kind, supporting_ref: supportingRef, source_sha256: body === null ? null : hash(body), supporting_sha256: fingerprints, fixture_sha256: hash(fixture), stdout: result.stdout, stderr: result.stderr }, null, 2));+ }+ expect(result.error).toBeUndefined(); expect(result.signal).toBeNull(); expect(observed).toBe(expected);+ if (expected === 'pass') expect(result.stderr).toContain('12 pass');+ if (name === 'parent' || name === 'unused-correct') { expect(result.stderr).toContain('6 fail'); expect(result.stderr).toContain('6 pass'); }+ if (name === 'missing-source') expect(kind).toBe('bun_unhandled_error');+ } finally { rmSync(root, { recursive: true, force: true }); }+}, 65000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex 422c73244..f708d514f 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -6483,6 +6483,41 @@"destination_path": "apps/screenpipe-app-tauri/node_modules"}]+ },+ {+ "id": "ai-gateway-vertex-stream-fallback",+ "name": "Keep reasoning-only Vertex streams visibly readable",+ "source": {+ "kind": "git_regression",+ "fix_commit": "65f7928a294cc847534cb1adfe693e8553addfa1"+ },+ "base_ref": "dec32b804203013868d99a87ba94b6bac61d00e9",+ "oracle_ref": "65f7928a294cc847534cb1adfe693e8553addfa1",+ "oracle_paths": [+ "packages/ai-gateway/src/providers/vertex-maas.ts"+ ],+ "gate": "advisory",+ "trigger_paths": [+ "packages/ai-gateway/src/providers/vertex-maas.ts"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "streaming",+ "provider-compatibility"+ ],+ "prompt": "The public Vertex MaaS streaming completion can return no visible answer when a thinking model supplies only reasoning_content. Preserve the original reasoning deltas and expose their complete text as content exactly once when the stream finishes, sends [DONE], or closes without either, provided no nonempty content was delivered. Handle empty content deltas, CRLF lines and arbitrary byte boundaries including split Unicode. Preserve response id, model and choice index; fallback content must precede an existing [DONE] marker. Ordinary content and reasoning followed by a real answer must remain exact without appended reasoning. Empty streams must not invent text. Preserve tool-call deltas, usage records, comments, malformed data lines and ordinary upstream client-error refusal. For the existing glm-4.7 endpoint, retain one completion POST, bearer authorization, user messages, zero temperature, max_tokens and stream:true without changing caller input. Use synthetic authentication and transport only; never call live providers. No particular helper or transform implementation is required. This task covers one-choice streaming transport, not non-streaming JSON parsing, multiple-choice multiplexing, live model output, billing or desktop rendering.",+ "agent_timeout_seconds": 900,+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/vertex-stream-fallback.fixture.ts.txt",+ "destination_path": "packages/ai-gateway/eval-vertex-stream-fallback.test.ts"+ }+ ],+ "command": "bun --no-env-file test packages/ai-gateway/eval-vertex-stream-fallback.test.ts"+ }}]}diff --git a/evals/coding-agent/graders/vertex-stream-fallback.fixture.ts.txt b/evals/coding-agent/graders/vertex-stream-fallback.fixture.ts.txtnew file mode 100644index 000000000..5485574ad--- /dev/null+++ b/evals/coding-agent/graders/vertex-stream-fallback.fixture.ts.txt@@ -0,0 +1,53 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { beforeAll, afterEach, expect, test } from 'bun:test';+import { VertexMaasProvider } from './src/providers/vertex-maas';+import { resetTokenCache } from './src/providers/vertex';+const originalFetch = globalThis.fetch;+const encoder = new TextEncoder();+let pem: string;+beforeAll(async () => {+ const key = await crypto.subtle.generateKey({name:'RSASSA-PKCS1-v1_5',modulusLength:2048,publicExponent:new Uint8Array([1,0,1]),hash:'SHA-256'},true,['sign','verify']);+ pem='-----BEGIN PRIVATE KEY-----\n'+btoa(String.fromCharCode(...new Uint8Array(await crypto.subtle.exportKey('pkcs8',key.privateKey))))+'\n-----END PRIVATE KEY-----';+});+afterEach(() => { globalThis.fetch=originalFetch; resetTokenCache(); });+function event(delta: any, finish_reason: string | null = null) {+ return 'data: '+JSON.stringify({id:'synthetic-stream',model:'synthetic-model',choices:[{index:0,delta,finish_reason}]})+'\n\n';+}+async function stream(text: string, bytewise=false, status=200) {+ resetTokenCache();let requests=0;+ const body:any={model:'glm-4.7',messages:[{role:'user',content:'Synthetic request'}],temperature:0,max_tokens:64};+ const before=JSON.stringify(body);+ globalThis.fetch=(async (input:any,init:RequestInit={}) => {+ const url=String(input);+ if(url==='https://oauth2.googleapis.com/token')return Response.json({access_token:'synthetic-token',expires_in:3600});+ expect(url).toBe('https://aiplatform.googleapis.com/v1/projects/synthetic-project/locations/global/endpoints/openapi/chat/completions');+ expect(init.method).toBe('POST');expect(new Headers(init.headers).get('authorization')).toBe('Bearer synthetic-token');+ expect(JSON.parse(String(init.body))).toMatchObject({messages:body.messages,temperature:0,max_tokens:64,stream:true});requests++;+ const bytes=encoder.encode(text);+ return new Response(new ReadableStream({start(c){if(bytewise){for(const byte of bytes)c.enqueue(Uint8Array.of(byte));}else c.enqueue(bytes);c.close();}}),{status});+ }) as typeof fetch;+ const provider=new VertexMaasProvider(JSON.stringify({project_id:'synthetic-project',client_email:'[email protected]',private_key:pem}),'synthetic-project');+ try{return await new Response(await provider.createStreamingCompletion(body)).text();}+ finally{expect(requests).toBe(1);expect(JSON.stringify(body)).toBe(before);}+}+function parsed(text:string) {+ return text.split('\n').filter(l=>l.startsWith('data: ')&&l.trim()!=='data: [DONE]').flatMap(l=>{try{return [JSON.parse(l.slice(6))];}catch{return [];}});+}+function values(text:string,key:string) {return parsed(text).flatMap(x=>x.choices??[]).map(c=>c.delta?.[key]).filter(x=>typeof x==='string').join('');}+const reason=event({reasoning_content:'Synthetic '})+event({reasoning_content:'fallback'});+for(const [label,suffix] of [['finish',event({},'length')+'data: [DONE]\n\n'],['done','data: [DONE]\n\n'],['eof','']])test(`reasoning-only ${label} supplies one visible answer`,async()=>{+ const out=await stream(reason+suffix);expect(values(out,'content')).toBe('Synthetic fallback');expect(values(out,'reasoning_content')).toBe('Synthetic fallback');+ const content=parsed(out).filter(x=>x.choices?.some(c=>c.delta?.content));expect(content.length).toBeGreaterThan(0);+ for(const x of content){expect(x.id).toBe('synthetic-stream');expect(x.model).toBe('synthetic-model');expect(x.choices[0].index).toBe(0);}+ if(label!=='eof')expect(out.indexOf('"content":')).toBeLessThan(out.indexOf('data: [DONE]'));+});+test('split UTF8 bytes preserve fallback text',async()=>{const text='Synthetic café 🧪';const out=await stream(event({reasoning_content:text})+event({},'length')+'data: [DONE]\n\n',true);expect(values(out,'content')).toBe(text);expect(values(out,'reasoning_content')).toBe(text);});+test('CRLF reasoning stream supplies fallback',async()=>{const out=await stream((reason+'data: [DONE]\n\n').replaceAll('\n','\r\n'));expect(values(out,'content')).toBe('Synthetic fallback');});+test('empty content delta does not suppress fallback',async()=>{const out=await stream(event({content:''})+reason+event({},'length'));expect(values(out,'content')).toBe('Synthetic fallback');});+test('ordinary content remains exact without fallback',async()=>{const out=await stream(event({content:'Visible '})+event({content:'answer'})+event({},'stop')+'data: [DONE]\n\n');expect(values(out,'content')).toBe('Visible answer');});+test('reasoning followed by answer is not duplicated as answer',async()=>{const out=await stream(reason+event({content:'Final answer'})+event({},'stop')+'data: [DONE]\n\n');expect(values(out,'content')).toBe('Final answer');expect(values(out,'reasoning_content')).toBe('Synthetic fallback');});+test('empty stream does not invent content',async()=>{const out=await stream(event({})+'data: [DONE]\n\n');expect(values(out,'content')).toBe('');});+test('tool calls and usage survive without invented text',async()=>{const delta={tool_calls:[{index:0,id:'synthetic-call',type:'function',function:{name:'synthetic_tool',arguments:'{}'}}]};const usage={choices:[],usage:{completion_tokens:7}};const out=await stream(event(delta)+event({},'tool_calls')+'data: '+JSON.stringify(usage)+'\n\ndata: [DONE]\n\n');expect(parsed(out)[0].choices[0].delta).toEqual(delta);expect(parsed(out)).toContainEqual(usage);expect(values(out,'content')).toBe('');});+test('comments and malformed data remain readable',async()=>{const text=': synthetic keepalive\n\ndata: {malformed}\n\n'+event({content:'Visible'})+'data: [DONE]\n\n';const out=await stream(text);expect(out).toContain(': synthetic keepalive');expect(out).toContain('data: {malformed}');expect(values(out,'content')).toBe('Visible');});+test('upstream client refusal remains an error',async()=>{await expect(stream('synthetic refusal',false,400)).rejects.toThrow();});