YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: cover malformed Vertex completion response classification
Louis Beaumont6ec91848cd2d
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex b98bea9af..05916d3df 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -1898,3 +1898,7 @@ unresponsive startup stop collection as setup errors. Calibrate with### Meeting background recovery`app-meeting-background-recovery` executes the actual meeting audio reader with synthetic local API responses. Eight outcomes cover partial live plus recovered speech, ordering/capping, duplicate and distinct-device behavior, source failures, malformed rows and metadata preservation. The known fix changes only the reader for this case; native transcription and database recovery are outside its evidence boundary. Run `bun test evals/coding-agent/calibrate-meeting-background.test.js` for calibration controls.++### Vertex upstream JSON failures++The Vertex upstream JSON case executes public JSON and streaming completion methods with synthetic OAuth and completion transport. Empty or malformed successful bodies must produce status 502 while valid content, reasoning fallback, client refusal and streaming remain intact. It does not prove gateway retry behavior, live provider reliability or agent isolation.diff --git a/evals/coding-agent/calibrate-vertex-upstream-json.test.js b/evals/coding-agent/calibrate-vertex-upstream-json.test.jsnew file mode 100644index 000000000..a4be4ea3b--- /dev/null+++ b/evals/coding-agent/calibrate-vertex-upstream-json.test.js@@ -0,0 +1,55 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { dirname, join, resolve } from 'node:path';+import { createHash } from 'node:crypto';+import { classifyGraderError } from './grader-outcome.mjs';+const repo = resolve(import.meta.dir, '../..');+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'))).cases.find(c => c.id === 'ai-gateway-vertex-upstream-json');+const sourcePath = item.oracle_paths[0];+const git = (...args) => execFileSync('git', args, { cwd: repo, encoding: 'utf8' });+const source = ref => git('show', `${ref}:${sourcePath}`);+const parent = source(item.base_ref), fixed = source(item.oracle_ref);+const fixture = readFileSync(join(import.meta.dir, 'graders/vertex-upstream-json.fixture.ts.txt'));+const hash = x => createHash('sha256').update(x).digest('hex');+function change(from, to) { expect(fixed.split(from)).toHaveLength(2); return fixed.replace(from, to); }+const controls = [+ ['parent', parent, 'fail', item.base_ref],+ ['reference', fixed, 'pass', item.base_ref],+ ['current', source('HEAD'), 'pass', 'HEAD'],+ ['equivalent', change('const result = await parseVertexMaasJsonResponse(response, resolved.vertexId);', "let result; try { result = await response.json(); } catch { throw Object.assign(new Error('Synthetic upstream failure'), { status: 502 }); }"), 'pass', item.base_ref],+ ['unused-correct', parent, 'fail', item.base_ref],+ ['blanket-refusal', change('const result = await parseVertexMaasJsonResponse(response, resolved.vertexId);', "throw Object.assign(new Error('Synthetic refusal'), { status: 502 }); const result = {};"), 'fail', item.base_ref],+ ['wrong-error-status', fixed.replaceAll(', 502);', ', 500);'), 'fail', item.base_ref],+ ['wrong-client-status', change('`Vertex MaaS request failed: ${response.status} ${error}`,\n\t\t\t\tresponse.status,', '`Vertex MaaS request failed: ${response.status} ${error}`,\n\t\t\t\t502,'), 'fail', item.base_ref],+ ['lose-reasoning-fallback', change('promoteReasoningToContent(result);', '// Synthetic omission of reasoning fallback.'), 'fail', item.base_ref],+ ['missing-source', null, 'error', item.base_ref],+];+for (const [name, body, expected, supportingRef] of controls) test(`Vertex upstream JSON calibration: ${name}`, () => {+ const root = mkdtempSync(join(tmpdir(), 'vertex-upstream-json-'));+ try {+ const fingerprints = {};+ for (const path of ['packages/ai-gateway/src/providers/vertex.ts', 'packages/ai-gateway/src/providers/base.ts', 'packages/ai-gateway/src/types.ts']) {+ const bytes = git('show', `${supportingRef}:${path}`); const destination = join(root, path);+ mkdirSync(dirname(destination), { recursive: true }); writeFileSync(destination, bytes); fingerprints[path] = hash(bytes);+ }+ if (body !== null) writeFileSync(join(root, sourcePath), body);+ if (name === 'unused-correct') writeFileSync(join(root, 'unused-correct.ts'), fixed);+ writeFileSync(join(root, 'packages/ai-gateway/eval-vertex-upstream-json.test.ts'), fixture);+ const result = spawnSync(process.execPath, ['--no-env-file', 'test', 'packages/ai-gateway/eval-vertex-upstream-json.test.ts'], { cwd: root, encoding: 'utf8', timeout: 60000, maxBuffer: 4 * 1024 * 1024 });+ const kind = classifyGraderError(result);+ const observed = result.error || result.signal || kind ? 'error' : result.status === 0 ? 'pass' : 'fail';+ if (process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS) {+ const dir = resolve(process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS); mkdirSync(dir, { recursive: true });+ writeFileSync(join(dir, `${name}.json`), JSON.stringify({ command: item.grader.command, expected, observed, exit: result.status, signal: result.signal, error: result.error?.message ?? null, error_kind: kind, supporting_ref: supportingRef, source_sha256: body === null ? null : hash(body), supporting_sha256: fingerprints, fixture_sha256: hash(fixture), stdout: result.stdout, stderr: result.stderr }, null, 2));+ }+ expect(result.error).toBeUndefined(); expect(result.signal).toBeNull(); expect(observed).toBe(expected);+ if (expected === 'pass') expect(result.stderr).toContain('8 pass');+ if (name === 'parent' || name === 'unused-correct') { expect(result.stderr).toContain('4 fail'); expect(result.stderr).toContain('4 pass'); }+ if (name === 'lose-reasoning-fallback') expect(result.stderr).toContain('(fail) reasoning-only response still supplies visible content');+ if (name === 'missing-source') expect(kind).toBe('bun_unhandled_error');+ } finally { rmSync(root, { recursive: true, force: true }); }+}, 65000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex 2f48d12eb..ce93560c2 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -6288,6 +6288,41 @@],"command": "bun --no-env-file test apps/screenpipe-app-tauri/lib/utils/meeting-background.eval.test.ts"}+ },+ {+ "id": "ai-gateway-vertex-upstream-json",+ "name": "Classify malformed Vertex success bodies as upstream failures",+ "source": {+ "kind": "git_regression",+ "fix_commit": "b4563754cf43d182d4f57147a315817cf435efd6"+ },+ "base_ref": "d7952bd90b412705b1d5f6133ec75d8e61858214",+ "oracle_ref": "b4563754cf43d182d4f57147a315817cf435efd6",+ "oracle_paths": [+ "packages/ai-gateway/src/providers/vertex-maas.ts"+ ],+ "gate": "advisory",+ "trigger_paths": [+ "packages/ai-gateway/src/providers/vertex-maas.ts"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "provider-compatibility",+ "error-classification"+ ],+ "prompt": "The public Vertex MaaS JSON completion method leaks unclassified parse failures when the provider returns HTTP 200 with an empty, whitespace-only, truncated JSON or HTML response body. Report these as upstream errors carrying status 502 so callers can identify a bad gateway response. Preserve valid completion content and metadata, reasoning-only responses supplying visible content, ordinary upstream HTTP 400 refusal status and readable streaming completions. For the existing glm-4.7 endpoint, preserve POST and bearer-token authentication, user messages, zero temperature, max_tokens and the streaming flag, without mutating caller input. Each ordinary success or malformed-success response uses one completion request. Do not make valid requests fail or turn client refusals into 502 errors. Exercise the public completion methods with synthetic credentials and transport only; do not contact live providers. No particular parser, helper name, error class or error wording is required.",+ "agent_timeout_seconds": 900,+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/vertex-upstream-json.fixture.ts.txt",+ "destination_path": "packages/ai-gateway/eval-vertex-upstream-json.test.ts"+ }+ ],+ "command": "bun --no-env-file test packages/ai-gateway/eval-vertex-upstream-json.test.ts"+ }}]}diff --git a/evals/coding-agent/graders/vertex-upstream-json.fixture.ts.txt b/evals/coding-agent/graders/vertex-upstream-json.fixture.ts.txtnew file mode 100644index 000000000..62a3cce58--- /dev/null+++ b/evals/coding-agent/graders/vertex-upstream-json.fixture.ts.txt@@ -0,0 +1,52 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { beforeAll, afterEach, expect, test } from 'bun:test';+import { VertexMaasProvider } from './src/providers/vertex-maas';+import { resetTokenCache } from './src/providers/vertex';+const originalFetch = globalThis.fetch;+let pem: string;+beforeAll(async () => {+ const key = await crypto.subtle.generateKey({name:'RSASSA-PKCS1-v1_5',modulusLength:2048,publicExponent:new Uint8Array([1,0,1]),hash:'SHA-256'},true,['sign','verify']);+ pem='-----BEGIN PRIVATE KEY-----\n'+btoa(String.fromCharCode(...new Uint8Array(await crypto.subtle.exportKey('pkcs8',key.privateKey))))+'\n-----END PRIVATE KEY-----';+});+afterEach(() => { globalThis.fetch=originalFetch; resetTokenCache(); });+async function completion(response: () => Response, stream=false) {+ resetTokenCache(); let requests=0;+ const body:any={model:'glm-4.7',messages:[{role:'user',content:'Synthetic request'}],temperature:0,max_tokens:64};+ const before=JSON.stringify(body);+ globalThis.fetch=(async (input:any,init:RequestInit={}) => {+ const url=String(input);+ if(url==='https://oauth2.googleapis.com/token')return Response.json({access_token:'synthetic-token',expires_in:3600});+ expect(url).toBe('https://aiplatform.googleapis.com/v1/projects/synthetic-project/locations/global/endpoints/openapi/chat/completions');+ expect(init.method).toBe('POST');expect(new Headers(init.headers).get('authorization')).toBe('Bearer synthetic-token');+ expect(JSON.parse(String(init.body))).toMatchObject({messages:body.messages,temperature:0,max_tokens:64,stream});requests++;+ return response();+ }) as typeof fetch;+ const provider=new VertexMaasProvider(JSON.stringify({project_id:'synthetic-project',client_email:'[email protected]',private_key:pem}),'synthetic-project');+ try {return stream ? await new Response(await provider.createStreamingCompletion(body)).text() : await provider.createCompletion(body);}+ finally {expect(requests).toBe(1);expect(JSON.stringify(body)).toBe(before);}+}+for(const [label,body] of [['empty',''],['whitespace',' \n\t '],['truncated','{"choices":'],['html','<html>synthetic outage</html>']]){+ test(`${label} successful upstream body is classified as bad gateway`,async()=>{+ let failure:any;+ try{await completion(()=>new Response(body,{status:200}));}catch(error){failure=error;}+ expect(failure).toBeDefined();expect(failure.status).toBe(502);+ });+}+test('valid completion content and metadata are preserved',async()=>{+ const expected={id:'synthetic-id',choices:[{message:{role:'assistant',content:'Synthetic answer'}}],usage:{prompt_tokens:4,completion_tokens:2}};+ const response=await completion(()=>Response.json(expected)) as Response;+ expect(response.status).toBe(200);expect(await response.json()).toEqual(expected);+});+test('reasoning-only response still supplies visible content',async()=>{+ const response=await completion(()=>Response.json({choices:[{message:{content:'',reasoning_content:'Synthetic reasoning'}}]})) as Response;+ expect((await response.json()).choices[0].message.content).toBe('Synthetic reasoning');+});+test('ordinary upstream client rejection keeps its status',async()=>{+ let failure:any;try{await completion(()=>new Response('synthetic refusal',{status:400}));}catch(error){failure=error;}+ expect(failure?.status).toBe(400);+});+test('streaming completion remains readable',async()=>{+ const value=await completion(()=>new Response('data: {"choices":[{"delta":{"content":"Synthetic stream"}}]}\n\ndata: [DONE]\n\n'),true);+ expect(value).toContain('Synthetic stream');expect(value).toContain('[DONE]');+});