Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

eval: preserve OpenAI streaming tool choice

Louis Beaumont1e77db8243b6
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index b53105b8c..6599f4998 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -1385,3 +1385,15 @@ not establish Home callback wiring, native recording resumption, operating-syste
capture, agent isolation or model improvement. The hidden fixture and dependency
link are installed only after the trajectory ends. No evaluated agent runs in
calibration or baseline/reference verification.
+
+## OpenAI streaming tool policy regression
+
+`app-openai-stream-tool-choice` executes the historical OpenAI provider with a synthetic SDK transport. The broken parent loses explicit tool choices on four streams and the unsupported-usage retry; seven neighboring outcomes still pass. The provider-only historical repair and current source pass all twelve checks.
+
+The grader checks caller-selected policy, tool schemas, model/messages, input preservation, retry behavior, native tool fragments, content, usage and stream termination. Eleven calibration controls include equivalent code, an unused repair, forced policies, lost schemas/fragments and a missing-source setup error.
+
+```sh
+bun test evals/coding-agent/calibrate-openai-tool-choice.test.js
+```
+
+This covers the provider boundary with synthetic transport. It does not establish model adherence, actual tool execution, gateway authentication, native Pi policy, agent isolation or model gains. The existing GLM case covers response conversion and remains separate.
diff --git a/evals/coding-agent/calibrate-openai-tool-choice.test.js b/evals/coding-agent/calibrate-openai-tool-choice.test.js
new file mode 100644
index 000000000..666cf2e9b
--- /dev/null
+++ b/evals/coding-agent/calibrate-openai-tool-choice.test.js
@@ -0,0 +1,31 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import {afterAll,expect,test} from 'bun:test';
+import {execFileSync,spawnSync} from 'node:child_process';
+import {mkdirSync,mkdtempSync,readFileSync,rmSync,writeFileSync} from 'node:fs';
+import {dirname,join,resolve} from 'node:path';
+import {tmpdir} from 'node:os';
+const repo=resolve(import.meta.dir,'../..'),item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='app-openai-stream-tool-choice');
+const provider='packages/ai-gateway/src/providers/openai.ts',fixture='evals/coding-agent/graders/openai-tool-choice.test.js';
+const root=mkdtempSync(join(tmpdir(),'openai-choice-calibration-'));afterAll(()=>rmSync(root,{recursive:true,force:true}));
+const show=(ref,path)=>execFileSync('git',['show',`${ref}:${path}`],{cwd:repo,encoding:'utf8'});
+function grade(name,ref=item.oracle_ref,mutate=()=>{}){
+ const cwd=join(root,name);mkdirSync(cwd);const archive=execFileSync('git',['archive',ref,'packages/ai-gateway'],{cwd:repo,maxBuffer:32*1024*1024});execFileSync('tar',['-x','-C',cwd],{input:archive});mkdirSync(dirname(join(cwd,fixture)),{recursive:true});writeFileSync(join(cwd,fixture),readFileSync(join(repo,fixture)));mutate(cwd);
+ const result=spawnSync(process.execPath,['test',fixture],{cwd,encoding:'utf8',timeout:20000,env:{PATH:dirname(process.execPath)}});
+ if(process.env.EVAL_CALIBRATION_OUTPUT){const o=resolve(process.env.EVAL_CALIBRATION_OUTPUT);mkdirSync(o,{recursive:true});for(const k of ['stdout','stderr'])writeFileSync(join(o,name+'.'+k),result[k]??'');writeFileSync(join(o,name+'.json'),JSON.stringify({status:result.status,signal:result.signal,error:result.error?.message??null}));} return result;
+}
+function change(cwd,path,from,to){const file=join(cwd,path),s=readFileSync(file,'utf8');expect(s.split(from)).toHaveLength(2);writeFileSync(file,s.replace(from,to));}
+function fail(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain('expect(received)');expect(r.stderr).not.toContain('Unhandled error');expect(r.stderr).not.toContain('Cannot find');}
+function pass(r){expect(r.error).toBeUndefined();expect(r.status).toBe(0);expect(r.stderr).toContain('12 pass');}
+function streamChoice(cwd,replacement){const p=join(cwd,provider),s=readFileSync(p,'utf8'),needle="tool_choice: body.tool_choice as ChatCompletionCreateParams['tool_choice'],",index=s.lastIndexOf(needle);expect(index).toBeGreaterThan(s.indexOf(needle));writeFileSync(p,s.slice(0,index)+replacement+s.slice(index+needle.length));}
+test('parent loses five streaming policies and preserves seven outcomes',()=>{const r=grade('parent',item.base_ref);fail(r);expect(r.stderr).toContain('5 fail');expect(r.stderr).toContain('7 pass');});
+test('reference passes',()=>pass(grade('reference')));
+test('current passes',()=>pass(grade('current','HEAD')));
+test('declared provider-only oracle repairs parent',()=>pass(grade('oracle-only',item.base_ref,d=>writeFileSync(join(d,provider),show(item.oracle_ref,provider)))));
+test('unused correct provider cannot repair dispatch',()=>fail(grade('unused',item.base_ref,d=>writeFileSync(join(d,provider+'.unused.ts'),show(item.oracle_ref,provider)))));
+test('equivalent conditional property passes',()=>pass(grade('equivalent',item.oracle_ref,d=>streamChoice(d,'...(body.tool_choice === undefined ? {} : { tool_choice: body.tool_choice }),'))));
+test('forcing automatic tool policy is rejected',()=>fail(grade('forced-auto',item.oracle_ref,d=>streamChoice(d,"tool_choice: 'auto',"))));
+test('forcing required tool policy is rejected',()=>fail(grade('forced-required',item.oracle_ref,d=>streamChoice(d,"tool_choice: 'required',"))));
+test('losing schemas is rejected',()=>fail(grade('lost-tools',item.oracle_ref,d=>{const p=join(d,provider);writeFileSync(p,readFileSync(p,'utf8').replaceAll("tools: body.tools as ChatCompletionCreateParams['tools'],",'tools: [],'));})));
+test('losing native stream fragments is rejected',()=>fail(grade('lost-fragments',item.oracle_ref,d=>change(d,provider,'const toolCalls = choice?.delta?.tool_calls;','const toolCalls = undefined;'))));
+test('missing provider is setup failure',()=>{const r=grade('missing',item.oracle_ref,d=>rmSync(join(d,provider)));expect(r.status).toBe(1);expect(r.stderr).toContain('Cannot find module');expect(r.stderr).toContain('0 pass');});
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 0a7df70d1..b3a06ccff 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -4647,6 +4647,42 @@
"destination_path": "apps/screenpipe-app-tauri/node_modules"
}
]
+ },
+ {
+ "id": "app-openai-stream-tool-choice",
+ "name": "Preserve explicit tool-selection policy on OpenAI streams and retries",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "33d5637dda01c704e5436f0be477effbd18c1e4b"
+ },
+ "base_ref": "45105b5ee837501d0c10020a016b16ca2c4c4083",
+ "oracle_ref": "33d5637dda01c704e5436f0be477effbd18c1e4b",
+ "oracle_paths": [
+ "packages/ai-gateway/src/providers/openai.ts"
+ ],
+ "gate": "advisory",
+ "trigger_paths": [
+ "packages/ai-gateway/src/providers/openai.ts"
+ ],
+ "tags": [
+ "regression",
+ "gateway",
+ "tool-policy",
+ "streaming",
+ "preserved-behavior"
+ ],
+ "agent_timeout_seconds": 1200,
+ "prompt": "Streaming requests through the OpenAI provider lose the caller's tool-selection policy, although ordinary completions preserve it. Repair the existing provider behavior so required, auto, none and named-function tool_choice values reach the provider unchanged on streaming and non-streaming requests. An unspecified choice must remain unspecified, including requests without tools. Preserve the selected model, messages, exact tool schemas and input object. If an OpenAI-compatible server rejects stream_options.include_usage, the supported retry must omit that option while retaining the original tools and tool choice. Preserve streamed content, native tool-call identity/name/arguments, tool_calls finish reason, prompt/completion/total/cached usage and exactly one completion terminator. Do not change routing/model policy or authentication. Use synthetic SDK transport without live API calls or executing tools; no specific implementation shape is required. This task concerns provider request/response behavior, not native Pi configuration, model adherence to the tool policy or full gateway authentication.",
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/openai-tool-choice.test.js",
+ "destination_path": "evals/coding-agent/graders/openai-tool-choice.test.js"
+ }
+ ],
+ "command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/openai-tool-choice.test.js"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/openai-tool-choice.test.js b/evals/coding-agent/graders/openai-tool-choice.test.js
new file mode 100644
index 000000000..9fcd26279
--- /dev/null
+++ b/evals/coding-agent/graders/openai-tool-choice.test.js
@@ -0,0 +1,57 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, beforeEach, expect, mock, test } from 'bun:test';
+let requests, rejectUsage;
+const originalFetch = globalThis.fetch;
+const fragment = { index: 0, id: 'synthetic-call', type: 'function', function: { name: 'read_fixture', arguments: '{"key":"sample"}' } };
+mock.module('openai', () => ({ default: class {
+ constructor(options) { this.baseURL = options.baseURL; }
+ chat = { completions: { create: async params => {
+ requests.push(structuredClone(params));
+ if (rejectUsage && params.stream_options) {
+ rejectUsage = false;
+ const error = new Error("Unknown parameter: 'stream_options.include_usage'"); error.status = 400; throw error;
+ }
+ if (!params.stream) return { choices: [{ message: { role: 'assistant', content: 'plain answer' }, finish_reason: 'stop' }] };
+ return { controller: { abort() {} }, async *[Symbol.asyncIterator]() {
+ yield { choices: [{ delta: { content: 'plain answer' }, finish_reason: null }] };
+ yield { choices: [{ delta: { tool_calls: [fragment] }, finish_reason: 'tool_calls' }] };
+ yield { choices: [], usage: { prompt_tokens: 20, completion_tokens: 3, total_tokens: 23, prompt_tokens_details: { cached_tokens: 4 } } };
+ } };
+ } } };
+} }));
+mock.module('@sentry/cloudflare', () => ({ captureException() { throw Error('Unexpected provider error'); } }));
+globalThis.fetch = async () => { throw Error('Network forbidden'); };
+afterAll(() => { globalThis.fetch = originalFetch; });
+const { OpenAIProvider } = await import('../../../packages/ai-gateway/src/providers/openai');
+const tools = [{ type: 'function', function: { name: 'read_fixture', parameters: { type: 'object', properties: { key: { type: 'string' } }, required: ['key'] } } }];
+const named = { type: 'function', function: { name: 'read_fixture' } };
+beforeEach(() => { requests = []; rejectUsage = false; });
+async function invoke(stream, choice, withTools = true) {
+ const body = { model: 'gpt-5.6-luna', messages: [{ role: 'user', content: 'Read the synthetic fixture when permitted.' }], ...(withTools ? { tools: structuredClone(tools) } : {}), ...(choice === undefined ? {} : { tool_choice: structuredClone(choice) }), stream };
+ const before = structuredClone(body), provider = new OpenAIProvider('synthetic-key');
+ const response = stream ? new Response(await provider.createStreamingCompletion(body)) : await provider.createCompletion(body);
+ const output = await response.text();
+ expect(body).toEqual(before);
+ for (const request of requests) { expect(request.model).toBe(body.model); expect(request.messages).toEqual(body.messages); expect(request.tools).toEqual(body.tools); }
+ return output;
+}
+for (const stream of [false, true]) for (const [name, choice] of [['required', 'required'], ['automatic', 'auto'], ['disabled', 'none'], ['named', named], ['unspecified', undefined]]) {
+ test(`${stream ? 'stream' : 'completion'} preserves ${name} tool policy`, async () => {
+ await invoke(stream, choice); expect(requests).toHaveLength(1); expect(requests[0].tool_choice).toEqual(choice);
+ });
+}
+test('unsupported usage retry retains the exact tool policy and tools', async () => {
+ rejectUsage = true; await invoke(true, named); expect(requests).toHaveLength(2);
+ for (const request of requests) expect(request.tool_choice).toEqual(named);
+ expect(requests[1].stream_options).toBeUndefined();
+});
+test('ordinary streams preserve content, native tool fragments, usage and one terminator', async () => {
+ const output = await invoke(true, undefined, false); expect(requests).toHaveLength(1); expect(requests[0].tool_choice).toBeUndefined();
+ expect(output.split('data: [DONE]').length - 1).toBe(1);
+ const events = output.split('\n').filter(line => line.startsWith('data: ') && line !== 'data: [DONE]').map(line => JSON.parse(line.slice(6)));
+ expect(events.map(e => e.choices?.[0]?.delta?.content || '').join('')).toBe('plain answer');
+ expect(events.flatMap(e => e.choices?.[0]?.delta?.tool_calls || [])).toEqual([fragment]);
+ expect(events.find(e => e.choices?.[0]?.finish_reason)?.choices[0].finish_reason).toBe('tool_calls');
+ expect(events.find(e => e.usage)?.usage).toMatchObject({ prompt_tokens: 20, completion_tokens: 3, total_tokens: 23, prompt_tokens_details: { cached_tokens: 4 } });
+});