Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

test(evals): cover GLM tool response conversion

louis27d75eed8f3c
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index eba796f6c..8fe5f76a8 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -9,7 +9,7 @@ This is an agent eval suite, not a unit-test suite. Every case contains:
- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;
- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.
-The current app corpus contains 51 git-mined regressions. See
+The current app corpus contains 52 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest uses this same harness.
@@ -310,3 +310,27 @@ streaming, boundary, identity-loss and preserved-client controls. A different
valid head/tail allocation is accepted. The SDK transport and telemetry are
synthetic; there is no live inference, token-budget sufficiency, encrypted
transport, gateway authentication, native persistence, model or isolation claim.
+
+
+## GLM response-to-tool conversion
+
+`ai-gateway-glm-tool-call-boundary` exercises actual GLM completion and streaming
+providers with synthetic SDK responses. Forty-two outcomes cover known XML/JSON
+calls, typed arguments, multiple unique IDs, fragmented streams, ordinary text,
+unknown names, malformed arguments, absent tools, existing native calls and
+other OpenAI providers. The broken parent fails fifteen conversions and preserves
+twenty-seven neighbors; the reference and current selected source pass all
+forty-two outcomes (374 assertions).
+
+Run `bun test evals/coding-agent/calibrate-glm-tool-calls.test.js` for fourteen
+controls. These reject unused correct providers, completion/streaming bypasses,
+unknown-name conversion, duplicate IDs, lost argument types, missing converted
+finish reasons and blanket refusal. Equivalent parser naming and additional
+valid finish metadata on unconverted completions are accepted. Missing provider
+source is a setup error. No installed dependencies are required.
+
+The SDK and telemetry ports are synthetic; GLM/OpenAI provider dispatch and
+current shared protocol are real. This does not execute tools or establish live
+model behavior, full schema validation, every malformed XML form, prompt-injection
+resistance, native permission checks, encrypted transport or agent isolation.
+These are corpus and grader checks, not model trials.
diff --git a/evals/coding-agent/calibrate-glm-tool-calls.test.js b/evals/coding-agent/calibrate-glm-tool-calls.test.js
new file mode 100644
index 000000000..76f1831b9
--- /dev/null
+++ b/evals/coding-agent/calibrate-glm-tool-calls.test.js
@@ -0,0 +1,31 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import {afterAll,expect,test} from 'bun:test';
+import {execFileSync,spawnSync} from 'node:child_process';
+import {mkdirSync,mkdtempSync,readFileSync,rmSync,writeFileSync} from 'node:fs';
+import {dirname,join,resolve} from 'node:path';
+import {tmpdir} from 'node:os';
+const repo=resolve(import.meta.dir,'../..'),item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='ai-gateway-glm-tool-call-boundary');
+const provider='packages/ai-gateway/src/providers/screenpipe-glm.ts',openai='packages/ai-gateway/src/providers/openai.ts',fixture='evals/coding-agent/graders/glm-tool-call-boundary.test.js';
+const root=mkdtempSync(join(tmpdir(),'glm-call-calibration-'));afterAll(()=>rmSync(root,{recursive:true,force:true}));
+const show=(ref,path)=>execFileSync('git',['show',`${ref}:${path}`],{cwd:repo,encoding:'utf8'});
+function grade(name,ref=item.oracle_ref,mutate=()=>{}){
+ const cwd=join(root,name);mkdirSync(cwd);const archive=execFileSync('git',['archive',ref,'packages/ai-gateway'],{cwd:repo,maxBuffer:32*1024*1024});execFileSync('tar',['-x','-C',cwd],{input:archive});mkdirSync(dirname(join(cwd,fixture)),{recursive:true});writeFileSync(join(cwd,fixture),readFileSync(join(repo,fixture)));mutate(cwd);
+ return spawnSync(process.execPath,['test',fixture],{cwd,encoding:'utf8',timeout:20000,env:{PATH:dirname(process.execPath)}});
+}
+function change(cwd,path,from,to){const file=join(cwd,path),s=readFileSync(file,'utf8');expect(s.split(from)).toHaveLength(2);writeFileSync(file,s.replace(from,to));}
+function fail(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain('expect(received)');expect(r.stderr).not.toContain('Unhandled error');expect(r.stderr).not.toContain('Cannot find');}
+test('parent fails fifteen conversions and preserves twenty-seven neighbors',()=>{const r=grade('parent',item.base_ref);fail(r);expect(r.stderr).toContain('15 fail');expect(r.stderr).toContain('27 pass');});
+test('reference passes forty-two provider outcomes',()=>{const r=grade('reference');expect(r.status).toBe(0);expect(r.stderr).toContain('42 pass');});
+test('equivalent parser name is accepted',()=>{const r=grade('rename',item.oracle_ref,cwd=>writeFileSync(join(cwd,provider),show(item.oracle_ref,provider).replaceAll('parseGlmToolCallContent','parseDeclaredResponse')));expect(r.status).toBe(0);});
+test('valid extra finish metadata on native and ordinary completions is accepted',()=>{const r=grade('finish-metadata',item.oracle_ref,cwd=>change(cwd,openai,'\t\t\tchoices: [\n\t\t\t\t{\n\t\t\t\t\tmessage: {','\t\t\tchoices: [\n\t\t\t\t{\n\t\t\t\t\tfinish_reason: response.choices[0].finish_reason,\n\t\t\t\t\tmessage: {'));expect(r.status).toBe(0);});
+test('unused correct provider cannot repair unchanged dispatch',()=>fail(grade('unused',item.base_ref,cwd=>writeFileSync(join(cwd,'packages/ai-gateway/src/providers/unused-glm.ts'),show(item.oracle_ref,provider)))));
+test('completion conversion bypass is rejected',()=>fail(grade('completion-bypass',item.oracle_ref,cwd=>change(cwd,provider,'const payload: any = await response.json();','return response;\n\t\tconst payload: any = await response.json();'))));
+test('stream conversion bypass is rejected',()=>fail(grade('stream-bypass',item.oracle_ref,cwd=>change(cwd,provider,'return normalizeGlmToolCallStream(stream, body.tools);','return stream;'))));
+test('unknown JSON tool names cannot become calls',()=>fail(grade('unknown-json',item.oracle_ref,cwd=>change(cwd,provider,"typeof name !== 'string' || !knownTools.has(name)","typeof name !== 'string'"))));
+test('unknown XML names cannot borrow a known tool schema',()=>fail(grade('unknown-xml',item.oracle_ref,cwd=>change(cwd,provider,'const tool = knownTools.get(name);','const tool = knownTools.get(name) || knownTools.values().next().value;'))));
+test('dropping conversion finish reason is rejected',()=>fail(grade('finish-missing',item.oracle_ref,cwd=>change(cwd,provider,"payload.choices[0].finish_reason = 'tool_calls';","payload.choices[0].finish_reason = 'stop';"))));
+test('duplicate IDs for multiple calls are rejected',()=>fail(grade('duplicate-ids',item.oracle_ref,cwd=>change(cwd,provider,"`call_glm_${crypto.randomUUID().replaceAll('-', '').slice(0, 24)}`","'same-call'"))));
+test('string coercion of typed arguments is rejected',()=>fail(grade('string-args',item.oracle_ref,cwd=>change(cwd,provider,'return JSON.parse(value);','return value;'))));
+test('blanket parser refusal is rejected',()=>fail(grade('deny-all',item.oracle_ref,cwd=>change(cwd,provider,"if (typeof content !== 'string' || !Array.isArray(tools)) return [];","return [];\n\tif (typeof content !== 'string' || !Array.isArray(tools)) return [];"))));
+test('missing provider is setup failure',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,provider)));expect(r.status).toBe(1);expect(r.stderr).toContain('Cannot find module');expect(r.stderr).toContain('0 pass');});
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 4a1d4ceb1..b75e755ea 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-09-20.1",
+ "dataset_version": "2026-09-20.2",
"suite_type": "regression",
"owner": "screenpipe engineering",
"policy": {
@@ -1657,6 +1657,38 @@
],
"command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/ai-gateway-glm-tool-history.test.js"
}
+ },
+ {
+ "id": "ai-gateway-glm-tool-call-boundary",
+ "name": "Convert recognized GLM tool responses without treating ordinary text as actions",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "da9ca5d1be7ef669cf7edec1d5fba7b99afc6de6"
+ },
+ "base_ref": "50a503a51e0b6f6a529aa2f4751df36ba4a756a7",
+ "oracle_ref": "da9ca5d1be7ef669cf7edec1d5fba7b99afc6de6",
+ "oracle_paths": [
+ "packages/ai-gateway/src/providers/screenpipe-glm.ts"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "tool-calls",
+ "streaming",
+ "gateway"
+ ],
+ "prompt": "GLM returns native function-call text that Pi cannot execute. Repair the actual GLM completion and streaming provider paths so schema-known leading calls become OpenAI tool calls with nonempty distinct IDs, correct function names and argument objects, no duplicated raw call text, and a tool_calls finish reason. Support tagged XML values (string, number, boolean and object types, preserving string leading zeros and Unicode), XML containing JSON arguments, flat JSON call objects, nested function objects with JSON-string arguments, and multiple XML calls. Streaming must work across one-character and seven-character content fragments and retain a single completion terminator. Forward the caller’s declared tools and tool choice and preserve input objects. Unknown tool names, prefixed explanatory prose, malformed JSON and array/null arguments remain unchanged ordinary text; no tools means no conversion. Preserve already-native tool-call IDs/arguments and other OpenAI providers. Use synthetic provider transport, without live calls or executing tools. No particular parser/helper name or ID format is required.",
+ "agent_timeout_seconds": 1200,
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/glm-tool-call-boundary.test.js",
+ "destination_path": "evals/coding-agent/graders/glm-tool-call-boundary.test.js"
+ }
+ ],
+ "command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/glm-tool-call-boundary.test.js"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/glm-tool-call-boundary.test.js b/evals/coding-agent/graders/glm-tool-call-boundary.test.js
new file mode 100644
index 000000000..a842ddf67
--- /dev/null
+++ b/evals/coding-agent/graders/glm-tool-call-boundary.test.js
@@ -0,0 +1,53 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, beforeEach, expect, mock, test } from 'bun:test';
+let requests,payload,content,native,chunkSize;
+const originalFetch=globalThis.fetch;
+mock.module('openai',()=>({default:class{
+ constructor(options){this.baseURL=options.baseURL;}
+ chat={completions:{create:async params=>{
+ requests.push(structuredClone(params));
+ if(!params.stream)return structuredClone(payload);
+ return {controller:{abort(){}},async *[Symbol.asyncIterator](){
+ if(native)yield {choices:[{delta:{tool_calls:native.map((c,index)=>({index,...c}))},finish_reason:null}]};
+ else for(let i=0;i<content.length;i+=chunkSize)yield {choices:[{delta:{content:content.slice(i,i+chunkSize)},finish_reason:null}]};
+ yield {choices:[{delta:{},finish_reason:native?'tool_calls':'stop'}]};
+ }};
+ }}};
+}}));
+mock.module('@sentry/cloudflare',()=>({captureException:()=>{throw Error('Unexpected provider error');}}));
+globalThis.fetch=async()=>{throw Error('Network forbidden');};afterAll(()=>{globalThis.fetch=originalFetch;});
+const {ScreenpipeGlmProvider}=await import('../../../packages/ai-gateway/src/providers/screenpipe-glm');
+const {OpenAIProvider}=await import('../../../packages/ai-gateway/src/providers/openai');
+const args={path:'synthetic/σ.txt',count:2,enabled:true,options:{mode:'read'},code:'001'};
+const tool={type:'function',function:{name:'read',parameters:{type:'object',properties:{path:{type:'string'},count:{type:'number'},enabled:{type:'boolean'},options:{type:'object'},code:{type:'string'}}}}};
+const xml='<tool_call>read<arg_key>path</arg_key><arg_value>synthetic/σ.txt</arg_value><arg_key>count</arg_key><arg_value>2</arg_value><arg_key>enabled</arg_key><arg_value>true</arg_value><arg_key>options</arg_key><arg_value>{"mode":"read"}</arg_value><arg_key>code</arg_key><arg_value>001</arg_value></tool_call>';
+beforeEach(()=>{requests=[];native=null;chunkSize=7;content='';});
+async function invoke(text,mode,tools=[tool],otherProvider=false,nativeCalls=null){
+ content=text;native=nativeCalls;chunkSize=mode==='stream-one'?1:7;
+ payload={id:'synthetic-completion',choices:[{message:{role:'assistant',content:native?null:text,...(native?{tool_calls:native}:{})},finish_reason:native?'tool_calls':'stop'}]};
+ const input={model:otherProvider?'gpt-4o':'glm-5.3-flash-reap50-iq3m',messages:[{role:'user',content:'Use the disclosed synthetic tool when needed.'}],...(tools===undefined?{}:{tools,tool_choice:'auto'})};
+ const before=structuredClone(input),provider=otherProvider?new OpenAIProvider('synthetic'):new ScreenpipeGlmProvider('synthetic');let result;
+ if(mode==='completion'){
+ const response=await provider.createCompletion(input);expect(response.status).toBe(200);const body=await response.json();result={calls:body.choices[0].message.tool_calls||[],text:body.choices[0].message.content||'',finishes:[body.choices[0].finish_reason]};
+ }else{
+ const stream=await provider.createStreamingCompletion(input);const raw=await new Response(stream).text();expect(raw.split('data: [DONE]').length-1).toBe(1);
+ const events=raw.split('\n').filter(s=>s.startsWith('data: ')&&s!=='data: [DONE]').map(s=>JSON.parse(s.slice(6)));
+ result={calls:events.flatMap(e=>e.choices?.[0]?.delta?.tool_calls||[]),text:events.map(e=>e.choices?.[0]?.delta?.content||'').join(''),finishes:events.map(e=>e.choices?.[0]?.finish_reason).filter(Boolean)};
+ }
+ expect(input).toEqual(before);expect(requests).toHaveLength(1);return result;
+}
+function converted(result,expected){
+ expect(result.calls).toHaveLength(expected.length);expect(result.text).toBe('');expect(result.finishes.at(-1)).toBe('tool_calls');const ids=[];
+ for(let i=0;i<expected.length;i++){const call=result.calls[i];expect(call.type).toBe('function');expect(call.function.name).toBe('read');expect(JSON.parse(call.function.arguments)).toEqual(expected[i]);expect(typeof call.id).toBe('string');expect(call.id.length).toBeGreaterThan(0);ids.push(call.id);}
+ expect(new Set(ids).size).toBe(ids.length);expect(requests[0].tools).toEqual([tool]);expect(requests[0].tool_choice).toBe('auto');
+}
+const forms=[['tagged XML',xml,[args]],['flat JSON',JSON.stringify({name:'read',arguments:args}),[args]],['nested JSON string',JSON.stringify({function:{name:'read',arguments:JSON.stringify(args)}}),[args]],['XML JSON','<tool_call>read'+JSON.stringify(args)+'</tool_call>',[args]],['multiple XML',xml+'\n<tool_call>read{"path":"second.txt"}</tool_call>',[args,{path:'second.txt'}]]];
+const ordinary=['<tool_call>unknown{}','Here is an example: '+xml,JSON.stringify({name:'unknown',arguments:{}}),'{invalid json',JSON.stringify({name:'read',arguments:[]}),JSON.stringify({name:'read',arguments:null})];
+for(const mode of ['completion','stream-one','stream-seven']){
+ for(const [name,text,expected] of forms)test(`${mode}: converts ${name} through actual provider`,async()=>converted(await invoke(text,mode),expected));
+ for(const text of ordinary)test(`${mode}: unrecognized content stays text ${text.slice(0,35)}`,async()=>{const r=await invoke(text,mode);expect(r.calls).toEqual([]);expect(r.text).toBe(text);if(mode!=='completion')expect(r.finishes.at(-1)).toBe('stop');});
+ test(`${mode}: absent tools do not turn text into executable calls`,async()=>{const r=await invoke(xml,mode,[]);expect(r.calls).toEqual([]);expect(r.text).toBe(xml);});
+ test(`${mode}: native provider tool calls keep their identity and arguments`,async()=>{const calls=[{id:'native_read_fixture',type:'function',function:{name:'read',arguments:JSON.stringify(args)}}];const r=await invoke('',mode,[tool],false,calls);expect(r.calls).toHaveLength(1);expect(r.calls[0]).toMatchObject(calls[0]);if(mode!=='completion')expect(r.finishes.at(-1)).toBe('tool_calls');});
+ test(`${mode}: other OpenAI provider preserves identical ordinary content`,async()=>{const r=await invoke(xml,mode,[tool],true);expect(r.calls).toEqual([]);expect(r.text).toBe(xml);});
+}