Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

test(evals): preserve Vertex tool calls across content shapes

Louis Beaumontfd838577f20d
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 5dcc0ca2e..2fd26676a 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -1,5 +1,17 @@
# Coding-agent regression evals
+`ai-gateway-vertex-string-tool-calls` executes the complete Vertex MaaS provider
+through its JSON and streaming completion methods with synthetic authentication
+and HTTP replies. Twelve outcomes preserve tool calls and matching results across
+string, null and array content, ordinary text/images, orphan filtering, request
+options and caller input. The parent fails eight content-shape outcomes and
+preserves four; the reference and current provider pass all twelve. Run
+`bun test evals/coding-agent/calibrate-vertex-tool-content.test.js` for nine
+controls, including equivalent code, unused correct code, lost results/images,
+input mutation and missing-source classification. The hidden fixture appears
+only at grading and uses no dependency links. This does not execute a live model,
+HTTP gateway admission, billing, agent isolation or model trials.
+
`ai-gateway-null-model-accounting` executes the complete accounting module's
existing exports without provider or database calls. Thirteen outcomes cover
absent model names and preserved unknown-string, measured/estimated token,
@@ -91,7 +103,7 @@ The hidden fixture appears only at grading and uses no dependency links.
This does not establish native recording continuity, real authentication,
account-switch race safety, browser integration, isolation or model performance.
-The current app corpus contains 142 git-mined regressions. See
+The current app corpus contains 146 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest uses this same harness.
diff --git a/evals/coding-agent/calibrate-vertex-tool-content.test.js b/evals/coding-agent/calibrate-vertex-tool-content.test.js
new file mode 100644
index 000000000..38735c9bf
--- /dev/null
+++ b/evals/coding-agent/calibrate-vertex-tool-content.test.js
@@ -0,0 +1,54 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join, resolve } from 'node:path';
+import { createHash } from 'node:crypto';
+import { classifyGraderError } from './grader-outcome.mjs';
+const repo = resolve(import.meta.dir, '../..');
+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'))).cases.find(c => c.id === 'ai-gateway-vertex-string-tool-calls');
+const sourcePath = item.oracle_paths[0];
+const git = (...args) => execFileSync('git', args, { cwd: repo, encoding: 'utf8' });
+const source = ref => git('show', `${ref}:${sourcePath}`);
+const parent = source(item.base_ref), fixed = source(item.oracle_ref);
+const fixture = readFileSync(join(import.meta.dir, 'graders/vertex-tool-content.fixture.ts.txt'));
+const hash = x => createHash('sha256').update(x).digest('hex');
+function change(from, to) { expect(fixed.split(from)).toHaveLength(2); return fixed.replace(from, to); }
+const controls = [
+ ['parent', parent, 'fail', item.base_ref],
+ ['reference', fixed, 'pass', item.base_ref],
+ ['current', source('HEAD'), 'pass', 'HEAD'],
+ ['equivalent', change("content: nonEmptyText(msg.content) ?? (topLevelToolCalls.length > 0 ? null : ''),", "content: nonEmptyText(msg.content) ? [{ type: 'text', text: msg.content }] : [],"), 'pass', item.base_ref],
+ ['unused-correct', parent, 'fail', item.base_ref],
+ ['drop-results', change('return filtered.map((msg)', "return filtered.filter((msg) => msg.role !== 'tool').map((msg)"), 'fail', item.base_ref],
+ ['lose-images', change("content.push({ type: 'image_url', image_url: { url: part.image_url.url } });", '// Synthetic preservation failure: omit image.'), 'fail', item.base_ref],
+ ['mutate-input', change('const topLevelToolCalls: any[] = [...((msg as any).tool_calls ?? [])];', "const topLevelToolCalls: any[] = [...((msg as any).tool_calls ?? [])];\n\t\tif (topLevelToolCalls[0]) topLevelToolCalls[0].function.arguments = '{}';"), 'fail', item.base_ref],
+ ['missing-source', null, 'error', item.base_ref],
+];
+for (const [name, body, expected, supportingRef] of controls) test(`Vertex tool content calibration: ${name}`, () => {
+ const root = mkdtempSync(join(tmpdir(), 'vertex-tool-content-'));
+ try {
+ const fingerprints = {};
+ for (const path of ['packages/ai-gateway/src/providers/vertex.ts', 'packages/ai-gateway/src/providers/base.ts', 'packages/ai-gateway/src/types.ts']) {
+ const bytes = git('show', `${supportingRef}:${path}`); const destination = join(root, path);
+ mkdirSync(dirname(destination), { recursive: true }); writeFileSync(destination, bytes); fingerprints[path] = hash(bytes);
+ }
+ if (body !== null) writeFileSync(join(root, sourcePath), body);
+ if (name === 'unused-correct') writeFileSync(join(root, 'unused-correct.ts'), fixed);
+ writeFileSync(join(root, 'packages/ai-gateway/eval-vertex-tool-content.test.ts'), fixture);
+ const result = spawnSync(process.execPath, ['--no-env-file', 'test', 'packages/ai-gateway/eval-vertex-tool-content.test.ts'], { cwd: root, encoding: 'utf8', timeout: 60000, maxBuffer: 4 * 1024 * 1024 });
+ const kind = classifyGraderError(result);
+ const observed = result.error || result.signal || kind ? 'error' : result.status === 0 ? 'pass' : 'fail';
+ if (process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS) {
+ const dir = resolve(process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS); mkdirSync(dir, { recursive: true });
+ writeFileSync(join(dir, `${name}.json`), JSON.stringify({ command: item.grader.command, expected, observed, exit: result.status, signal: result.signal, error: result.error?.message ?? null, error_kind: kind, supporting_ref: supportingRef, source_sha256: body === null ? null : hash(body), supporting_sha256: fingerprints, fixture_sha256: hash(fixture), stdout: result.stdout, stderr: result.stderr }, null, 2));
+ }
+ expect(result.error).toBeUndefined(); expect(result.signal).toBeNull(); expect(observed).toBe(expected);
+ if (expected === 'pass') expect(result.stderr).toContain('12 pass');
+ if (name === 'parent' || name === 'unused-correct') { expect(result.stderr).toContain('8 fail'); expect(result.stderr).toContain('4 pass'); }
+ if (name === 'lose-images') expect(result.stderr).toContain('(fail) ordinary text and image messages remain intact');
+ if (name === 'missing-source') expect(kind).toBe('bun_unhandled_error');
+ } finally { rmSync(root, { recursive: true, force: true }); }
+}, 65000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index a214f64a9..1a2b7777b 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -6056,6 +6056,41 @@
],
"command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config manual-grant.config.mjs"
}
+ },
+ {
+ "id": "ai-gateway-vertex-string-tool-calls",
+ "name": "Preserve tool-call history across Vertex message content shapes",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "85b0f37c6e7efbcdde83bfeea21b601f8da79ca7"
+ },
+ "base_ref": "9371579352e01a088bc153d287323d54601a954f",
+ "oracle_ref": "85b0f37c6e7efbcdde83bfeea21b601f8da79ca7",
+ "oracle_paths": [
+ "packages/ai-gateway/src/providers/vertex-maas.ts"
+ ],
+ "gate": "advisory",
+ "trigger_paths": [
+ "packages/ai-gateway/src/providers/vertex-maas.ts"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "tool-calling",
+ "provider-compatibility"
+ ],
+ "prompt": "Valid tool conversations sent through Vertex MaaS fail after an assistant response whose content is a string, even though it includes tool calls and the following tool outputs reference those calls. Preserve every valid call identifier, function name, serialized arguments and matching tool result in outbound JSON and streaming requests. Support empty, whitespace, null and nonempty assistant content; preserve nonempty text and accept either empty-string or null representation for absent text. Preserve array text with top-level calls, ordinary system/user/assistant text and image messages, request tools and tool choice, zero temperature and token limits, and returned response content. Preserve the existing model endpoint, POST method and bearer authentication header, and send one completion request on ordinary success. Tool outputs appearing before their matching call, or with missing, empty or unknown identifiers, must still be filtered when no missing-id call can be repaired. Preserve valid neighboring pairs and do not mutate the caller message array or nested calls. Exercise the existing public completion methods. No particular private helper or implementation is required. Use synthetic transport and credentials only; do not call a live model, change authentication policy or publish releases.",
+ "agent_timeout_seconds": 900,
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/vertex-tool-content.fixture.ts.txt",
+ "destination_path": "packages/ai-gateway/eval-vertex-tool-content.test.ts"
+ }
+ ],
+ "command": "bun --no-env-file test packages/ai-gateway/eval-vertex-tool-content.test.ts"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/vertex-tool-content.fixture.ts.txt b/evals/coding-agent/graders/vertex-tool-content.fixture.ts.txt
new file mode 100644
index 000000000..a05212061
--- /dev/null
+++ b/evals/coding-agent/graders/vertex-tool-content.fixture.ts.txt
@@ -0,0 +1,75 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterEach, beforeAll, expect, test } from 'bun:test';
+import { VertexMaasProvider } from './src/providers/vertex-maas';
+import { resetTokenCache } from './src/providers/vertex';
+const realFetch = globalThis.fetch;
+let pem: string;
+beforeAll(async () => {
+ const pair = await crypto.subtle.generateKey({ name: 'RSASSA-PKCS1-v1_5', modulusLength: 2048, publicExponent: new Uint8Array([1, 0, 1]), hash: 'SHA-256' }, true, ['sign', 'verify']);
+ pem = '-----BEGIN PRIVATE KEY-----\n' + btoa(String.fromCharCode(...new Uint8Array(await crypto.subtle.exportKey('pkcs8', pair.privateKey)))) + '\n-----END PRIVATE KEY-----';
+});
+afterEach(() => { globalThis.fetch = realFetch; resetTokenCache(); });
+const calls = [
+ { id: 'synthetic-a', type: 'function', function: { name: 'list_notes', arguments: '{"folder":"α"}' } },
+ { id: 'synthetic-b', type: 'function', function: { name: 'count_notes', arguments: '{"limit":2}' } },
+];
+async function request(messages: any[], stream: boolean) {
+ resetTokenCache(); const before = JSON.stringify(messages); const sent: any[] = [];
+ globalThis.fetch = (async (input: any, init: RequestInit = {}) => {
+ const url = String(input);
+ if (url === 'https://oauth2.googleapis.com/token') return Response.json({ access_token: 'synthetic-token', expires_in: 3600 });
+ if (url !== 'https://aiplatform.googleapis.com/v1/projects/synthetic-project/locations/global/endpoints/openapi/chat/completions') throw new Error('Unexpected synthetic transport target');
+ expect(init.method).toBe('POST');
+ expect(new Headers(init.headers).get('authorization')).toBe('Bearer synthetic-token');
+ sent.push(JSON.parse(String(init.body)));
+ return stream ? new Response('data: {"choices":[{"delta":{"content":"synthetic answer"}}]}\n\ndata: [DONE]\n\n') : Response.json({ choices: [{ message: { content: 'synthetic answer' } }] });
+ }) as typeof fetch;
+ const provider = new VertexMaasProvider(JSON.stringify({ project_id: 'synthetic-project', client_email: '[email protected]', private_key: pem }), 'synthetic-project');
+ const body: any = { model: 'glm-4.7', messages, tools: [{ type: 'function', function: { name: 'list_notes', parameters: { type: 'object' } } }], tool_choice: 'auto', temperature: 0, max_tokens: 64 };
+ const result = stream ? await new Response(await provider.createStreamingCompletion(body)).text() : await (await provider.createCompletion(body)).text();
+ expect(result).toContain('synthetic answer'); expect(sent).toHaveLength(1);
+ expect(sent[0]).toMatchObject({ stream, tools: body.tools, tool_choice: 'auto', temperature: 0, max_tokens: 64 });
+ expect(JSON.stringify(messages)).toBe(before);
+ return sent[0].messages;
+}
+function history(content: any) {
+ return [{ role: 'user', content: 'Inspect synthetic notes' }, { role: 'assistant', content, tool_calls: structuredClone(calls) }, { role: 'tool', tool_call_id: 'synthetic-a', content: 'note α' }, { role: 'tool', tool_call_id: 'synthetic-b', content: '2' }];
+}
+function contentParts(content: any): any[] {
+ if (content == null || (typeof content === 'string' && !content.trim())) return [];
+ return typeof content === 'string' ? [{ type: 'text', text: content }] : content;
+}
+function semanticMessages(messages: any[]) {
+ return messages.map(message => ({ ...message, content: contentParts(message.content) }));
+}
+function assertPair(output: any[]) {
+ expect(output).toHaveLength(4);
+ expect(semanticMessages([output[0]])).toEqual(semanticMessages([{ role: 'user', content: 'Inspect synthetic notes' }]));
+ expect(output[1]).toMatchObject({ role: 'assistant', tool_calls: calls });
+ expect(semanticMessages([output[2]])).toEqual(semanticMessages([{ role: 'tool', tool_call_id: 'synthetic-a', content: 'note α' }]));
+ expect(semanticMessages([output[3]])).toEqual(semanticMessages([{ role: 'tool', tool_call_id: 'synthetic-b', content: '2' }]));
+}
+for (const stream of [false, true]) {
+ for (const [label, content] of [['empty', ''], ['whitespace', ' '], ['null', null], ['text', 'Checking notes.']] as const) {
+ test(`${stream ? 'stream' : 'json'} preserves tool calls with ${label} content`, async () => {
+ const output = await request(history(content), stream); assertPair(output);
+ // Equivalent valid text-part arrays and strings carry the same content.
+ expect(contentParts(output[1].content)).toEqual(contentParts(content));
+ });
+ }
+ test(`${stream ? 'stream' : 'json'} preserves array content and top-level calls`, async () => {
+ const output = await request(history([{ type: 'text', text: 'Checking notes.' }]), stream); assertPair(output);
+ expect(contentParts(output[1].content)).toEqual([{ type: 'text', text: 'Checking notes.' }]);
+ });
+}
+test('ordinary text and image messages remain intact', async () => {
+ const messages = [{ role: 'system', content: 'Synthetic instruction' }, { role: 'user', content: [{ type: 'text', text: 'Describe' }, { type: 'image_url', image_url: { url: 'data:image/png;base64,c3ludGhldGlj' } }] }, { role: 'assistant', content: 'Synthetic description' }];
+ expect(semanticMessages(await request(messages, false))).toEqual(semanticMessages(messages));
+});
+test('unmatched, missing and empty tool result ids are filtered without losing valid pairs', async () => {
+ const messages = history([{ type: 'text', text: 'Checking notes.' }]);
+ messages.splice(0, 0, { role: 'tool', tool_call_id: 'synthetic-a', content: 'too early' } as any);
+ messages.push({ role: 'tool', content: 'missing' } as any, { role: 'tool', tool_call_id: '', content: 'empty' } as any, { role: 'tool', tool_call_id: 'unknown', content: 'unmatched' } as any);
+ assertPair(await request(messages, false));
+});