Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: cover outbound chat sanitation and sampling recovery

Louis Beaumont24226ca15a91
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 98c13d544..a548447d7 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -1,5 +1,14 @@
# Coding-agent regression evals
+`ai-gateway-outbound-chat-compatibility` executes the actual provider through its
+completion and streaming methods with synthetic SDK and telemetry ports. Nine
+outcomes cover response-only field removal, orphan and valid tool history,
+rejected-temperature recovery, unchanged errors and preserved text/images.
+Run `bun test evals/coding-agent/calibrate-outbound-chat.test.js` for calibration.
+Hidden fixtures appear only at grading and require no dependency links. This
+historical case does not test live SDK transport, gateway admission, other
+providers, billing, production activation, execution isolation or model trials.
+
`ai-gateway-vertex-string-tool-calls` executes the complete Vertex MaaS provider
through its JSON and streaming completion methods with synthetic authentication
and HTTP replies. Twelve outcomes preserve tool calls and matching results across
diff --git a/evals/coding-agent/calibrate-outbound-chat.test.js b/evals/coding-agent/calibrate-outbound-chat.test.js
new file mode 100644
index 000000000..e5ef23108
--- /dev/null
+++ b/evals/coding-agent/calibrate-outbound-chat.test.js
@@ -0,0 +1,57 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { dirname, join, resolve } from 'node:path';
+import { createHash } from 'node:crypto';
+import { classifyGraderError } from './grader-outcome.mjs';
+const repo = resolve(import.meta.dir, '../..');
+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'))).cases.find(c => c.id === 'ai-gateway-outbound-chat-compatibility');
+const sourcePath = item.oracle_paths[0];
+const git = (...args) => execFileSync('git', args, { cwd: repo, encoding: 'utf8' });
+const source = ref => git('show', `${ref}:${sourcePath}`);
+const parent = source(item.base_ref), fixed = source(item.oracle_ref);
+const fixture = readFileSync(join(import.meta.dir, 'graders/outbound-chat.fixture.ts.txt'));
+const hash = x => createHash('sha256').update(x).digest('hex');
+function change(from, to) { expect(fixed.split(from)).toHaveLength(2); return fixed.replace(from, to); }
+const controls = [
+ ['parent', parent, 'fail', item.base_ref],
+ ['reference', fixed, 'pass', item.base_ref],
+ ['current', source('HEAD'), 'pass', 'HEAD'],
+ ['equivalent', change('const knownToolCallIds = new Set<string>();', 'const knownToolCallIds = { ids: [] as string[], add(id: string) { this.ids.push(id); }, has(id: string) { return this.ids.includes(id); } };'), 'pass', item.base_ref],
+ ['unused-correct', parent, 'fail', item.base_ref],
+ ['response-field', change('tool_call_id: msg.tool_call_id,', 'tool_call_id: msg.tool_call_id, refusal: null,'), 'fail', item.base_ref],
+ ['lose-tool-id', change('tool_call_id: msg.tool_call_id,', 'tool_call_id: undefined,'), 'fail', item.base_ref],
+ ['retain-orphans', change("if (msg.role === 'tool' && msg.tool_call_id && !knownToolCallIds.has(msg.tool_call_id)) {", 'if (false) {'), 'fail', item.base_ref],
+ ['drop-tools', change("if (msg.role === 'tool' && msg.tool_call_id && !knownToolCallIds.has(msg.tool_call_id)) {", "if (msg.role === 'tool') {"), 'fail', item.base_ref],
+ ['server-retry', change('if (error?.status !== 400) return null;', '// incorrectly accept all errors'), 'fail', item.base_ref],
+ ['lose-image-detail', change("detail: part.image_url.detail || 'auto',", "detail: 'low',"), 'fail', item.base_ref],
+ ['nonstream-only-retry', change("const stream = await this.createWithUnsupportedParamRetry(params, (p) =>\n\t\t\tthis.client.chat.completions.create(p),\n\t\t);", "const stream = await this.client.chat.completions.create(params);"), 'fail', item.base_ref],
+ ['missing-source', null, 'error', item.base_ref],
+];
+for (const [name, body, expected, supportingRef] of controls) test(`outbound chat calibration: ${name}`, () => {
+ const root = mkdtempSync(join(tmpdir(), 'outbound-chat-'));
+ try {
+ const fingerprints = {};
+ for (const path of ['packages/ai-gateway/src/providers/base.ts', 'packages/ai-gateway/src/types.ts']) {
+ const bytes = git('show', `${supportingRef}:${path}`); const destination = join(root, path);
+ mkdirSync(dirname(destination), { recursive: true }); writeFileSync(destination, bytes); fingerprints[path] = hash(bytes);
+ }
+ if (body !== null) writeFileSync(join(root, sourcePath), body);
+ if (name === 'unused-correct') writeFileSync(join(root, 'unused-correct.ts'), fixed);
+ writeFileSync(join(root, 'packages/ai-gateway/eval-outbound-chat.test.ts'), fixture);
+ const result = spawnSync(process.execPath, ['--no-env-file', 'test', 'packages/ai-gateway/eval-outbound-chat.test.ts'], { cwd: root, encoding: 'utf8', timeout: 60000, maxBuffer: 4 * 1024 * 1024 });
+ const kind = classifyGraderError(result);
+ const observed = result.error || result.signal || kind ? 'error' : result.status === 0 ? 'pass' : 'fail';
+ if (process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS) {
+ const dir = resolve(process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS); mkdirSync(dir, { recursive: true });
+ writeFileSync(join(dir, `${name}.json`), JSON.stringify({ command: item.grader.command, expected, observed, exit: result.status, signal: result.signal, error: result.error?.message ?? null, error_kind: kind, supporting_ref: supportingRef, source_sha256: body === null ? null : hash(body), supporting_sha256: fingerprints, fixture_sha256: hash(fixture), stdout: result.stdout, stderr: result.stderr }, null, 2));
+ }
+ expect(result.error).toBeUndefined(); expect(result.signal).toBeNull(); expect(observed).toBe(expected);
+ if (expected === 'pass') expect(result.stderr).toContain('9 pass');
+ if (name === 'parent' || name === 'unused-correct') { expect(result.stderr).toContain('5 fail'); expect(result.stderr).toContain('4 pass'); }
+ if (name === 'missing-source') expect(kind).toBe('bun_unhandled_error');
+ } finally { rmSync(root, { recursive: true, force: true }); }
+}, 65000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 3f8b68ab8..f337c3c88 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -6218,6 +6218,41 @@
],
"command": "cd packages/screenpipe-mcp && bun test src/eval-package-dispatch.test.js"
}
+ },
+ {
+ "id": "ai-gateway-outbound-chat-compatibility",
+ "name": "Preserve outbound chat compatibility and recover rejected sampling",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "ee62f61129503c9400c845b9c25b88e37fc510ea"
+ },
+ "base_ref": "ece9d58bb90d88a4a232d0277bb74bbce11714ab",
+ "oracle_ref": "ee62f61129503c9400c845b9c25b88e37fc510ea",
+ "oracle_paths": [
+ "packages/ai-gateway/src/providers/openai.ts"
+ ],
+ "gate": "advisory",
+ "tags": [
+ "regression",
+ "hosted-ai",
+ "tools",
+ "recovery"
+ ],
+ "trigger_paths": [
+ "packages/ai-gateway/src/providers/openai.ts"
+ ],
+ "prompt": "Chat requests can be rejected after conversation history changes or when an upstream model refuses an explicitly supplied temperature. Repair the existing OpenAIProvider public completion and streaming methods. Outbound messages must omit the response-only refusal field, remove tool outputs whose tool_call_id has no preceding assistant call, and preserve ordinary conversation text plus valid assistant calls and their matching tool output IDs/content. When an HTTP400 Unsupported value error names the supplied temperature, recover in both streaming and non-streaming calls by retrying without that parameter; retain messages and token limits and return the resulting answer. Do not retry HTTP500 failures or errors naming a parameter that was not sent. A repeated identical temperature refusal must terminate with the original upstream error after at most one retry. Preserve ordinary text and image_url content, including image detail. Keep the existing public interfaces compatible; no private helper, field, or implementation structure is required. Grading runs the actual provider with synthetic SDK and telemetry ports and no live network. This historical task does not establish HTTP gateway admission, other providers, billing, SDK transport behavior, production activation, agent isolation or model performance. Current support for repairing several different upstream rejections is a separate contract.",
+ "agent_timeout_seconds": 900,
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/outbound-chat.fixture.ts.txt",
+ "destination_path": "packages/ai-gateway/eval-outbound-chat.test.ts"
+ }
+ ],
+ "command": "bun --no-env-file test packages/ai-gateway/eval-outbound-chat.test.ts"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/outbound-chat.fixture.ts.txt b/evals/coding-agent/graders/outbound-chat.fixture.ts.txt
new file mode 100644
index 000000000..2f1897f33
--- /dev/null
+++ b/evals/coding-agent/graders/outbound-chat.fixture.ts.txt
@@ -0,0 +1,20 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import {test,expect,mock} from 'bun:test';
+mock.module('@sentry/cloudflare',()=>({captureException:()=>{}}));
+globalThis.fetch=async()=>{throw new Error('network forbidden in diagnostic')};
+let upstream: (params: any) => Promise<any>;
+mock.module('openai',()=>({default:class { chat={completions:{create:(params:any)=>upstream(params)}}; baseURL='https://synthetic.invalid'; }}));
+const {OpenAIProvider}=await import('./src/providers/openai');
+const make=(invoke:any)=>{upstream=invoke;return new OpenAIProvider('synthetic-key')};
+const response=()=>({choices:[{message:{content:'synthetic answer',role:'assistant'}}]});
+const body:any={model:'synthetic-model',messages:[{role:'user',content:'synthetic request'}]};
+const error=(status:number,param:string)=>Object.assign(new Error(`Unsupported value: '${param}'`),{status});
+async function capture(messages:any[]){let captured:any;const p=make(async(v:any)=>{captured=structuredClone(v);return response()});expect((await (await p.createCompletion({...body,messages})).json()).choices[0].message.content).toBe('synthetic answer');return captured.messages}
+test('outbound request omits response-only refusal',async()=>{const messages=await capture(body.messages);expect(messages[0]).not.toHaveProperty('refusal')});
+test('orphan tool output is removed without losing ordinary text',async()=>{const messages=await capture([...body.messages,{role:'tool',tool_call_id:'missing',content:'orphan'},{role:'user',content:'continue'}]);expect(messages.map((m:any)=>m.content)).toEqual(['synthetic request','continue'])});
+test('valid tool-call identity survives dispatch',async()=>{const call={id:'known',type:'function',function:{name:'synthetic_tool',arguments:'{}'}};const messages=await capture([{role:'assistant',content:'',tool_calls:[call]},{role:'tool',tool_call_id:'known',content:'synthetic result'}]);expect(messages[0].tool_calls).toEqual([call]);expect(messages[1].tool_call_id).toBe('known');expect(messages[1].content).toBe('synthetic result')});
+for(const stream of [false,true])test(`unsupported supplied temperature recovers stream=${stream}`,async()=>{const calls:any[]=[];const p=make(async(v:any)=>{calls.push(structuredClone(v));if(calls.length===1)throw error(400,'temperature');return stream?(async function*(){yield{choices:[{delta:{content:'synthetic answer'}}]}})():response()});const b={...body,temperature:0.3,max_tokens:40};const output=stream?await new Response(await p.createStreamingCompletion(b)).text():await(await p.createCompletion(b)).text();expect(output).toContain('synthetic answer');expect(calls).toHaveLength(2);expect(calls[0].temperature).toBe(0.3);expect(calls[1]).not.toHaveProperty('temperature');expect(calls[1].messages).toEqual(calls[0].messages);expect(calls[1].max_tokens).toBe(40)});
+for(const [status,param]of [[500,'temperature'],[400,'top_p']] as const)test(`unrelated rejection preserves error without retry ${status}/${param}`,async()=>{let count=0;const failure=error(status,param);const p=make(async()=>{count++;throw failure});await expect(p.createCompletion({...body,temperature:0.3})).rejects.toBe(failure);expect(count).toBe(1)});
+test('persistent supplied-parameter rejection terminates',async()=>{let count=0;const failure=error(400,'temperature');const p=make(async()=>{count++;throw failure});await expect(p.createCompletion({...body,temperature:0.3})).rejects.toBe(failure);expect(count).toBeLessThanOrEqual(2)});
+test('ordinary image and text survive formatting',async()=>{const content=[{type:'text',text:'look'},{type:'image_url',image_url:{url:'data:image/png;base64,AA',detail:'high'}}];const messages=await capture([{role:'user',content}]);expect(messages[0].content).toEqual(content)});