YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
eval: cover image cache boundaries in gateway provider
louise167f18c2d25
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 3040d00b7..ea7672549 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -9,7 +9,7 @@ This is an agent eval suite, not a unit-test suite. Every case contains:- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.-The current app corpus contains 57 git-mined regressions. See+The current app corpus contains 58 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.@@ -460,3 +460,22 @@ wording or prove model compliance, native database locking, OS-wide filesystemisolation, real CLI/keychain discovery, package publication or other platforms.The observed IO ports are synthetic, not a universal access sandbox. Productstartup tests with supplied credentials do not exercise the missing-key fallback.++## Image cache fields at the provider boundary++`ai-gateway-image-cache-boundary` executes the real OpenAI provider methods+against a synthetic SDK that snapshots each submitted request. Fourteen outcomes+cover both streaming modes, three supported image input shapes, image-only+history, text cache boundaries, disabled history caching and older models.+The parent has eight intended failures and six preserved passes; the historical+reference and selected current code pass all fourteen. Returned content is also+checked, so suppressing provider calls cannot manufacture a pass.++Run `bun test evals/coding-agent/calibrate-image-cache.test.js`. Calibration+rejects an unused fix, either caller reintroducing image fields, dropped images+and disabled caching, while accepting equivalent internal predicate naming.+Missing source remains an import error. Only the provider source is the oracle.++The SDK and error-reporting ports are synthetic; no model or network calls run.+This does not prove upstream API acceptance, gateway authorization/billing,+audio/file/refusal normalization, model quality or trial isolation.diff --git a/evals/coding-agent/calibrate-image-cache.test.js b/evals/coding-agent/calibrate-image-cache.test.jsnew file mode 100644index 000000000..6c652a456--- /dev/null+++ b/evals/coding-agent/calibrate-image-cache.test.js@@ -0,0 +1,32 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import {afterAll,expect,test} from 'bun:test';+import {execFileSync,spawnSync} from 'node:child_process';+import {mkdirSync,mkdtempSync,readFileSync,rmSync,writeFileSync} from 'node:fs';+import {dirname,join,resolve} from 'node:path';+import {tmpdir} from 'node:os';+const repo=resolve(import.meta.dir,'../..'),root=mkdtempSync(join(tmpdir(),'image-cache-calibration-'));+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='ai-gateway-image-cache-boundary');+const source='packages/ai-gateway/src/providers/openai.ts';+const show=ref=>execFileSync('git',['show',`${ref}:${source}`],{cwd:repo,encoding:'utf8'}),fixed=show(item.oracle_ref),broken=show(item.base_ref);+afterAll(()=>rmSync(root,{recursive:true,force:true}));+function grade(name,body,unused){+ const cwd=join(root,name);mkdirSync(cwd);execFileSync('tar',['-xf','-','-C',cwd],{input:execFileSync('git',['archive',item.base_ref,'packages/ai-gateway/src'],{cwd:repo,maxBuffer:32*1024*1024})});+ if(body===null)rmSync(join(cwd,source));else writeFileSync(join(cwd,source),body);+ if(unused)writeFileSync(join(cwd,source+'.unused.ts'),unused);+ for(const f of item.grader.fixtures){const dest=join(cwd,f.destination_path);mkdirSync(dirname(dest),{recursive:true});writeFileSync(dest,readFileSync(join(import.meta.dir,f.local_path)));}+ return spawnSync(process.execPath,['test',item.grader.fixtures[0].destination_path],{cwd,encoding:'utf8',timeout:15000,env:{PATH:process.env.PATH,HOME:cwd,NO_COLOR:'1'}});+}+function fails(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain('expect(received)');expect(r.stderr).not.toMatch(/Cannot find|ModuleNotFound|SyntaxError/);}+function replace(s,a,b){expect(s.split(a)).toHaveLength(2);return s.replace(a,b);}+test('parent fails eight image boundaries and preserves six outcomes',()=>{const r=grade('parent',broken);fails(r);expect(r.stderr).toContain('8 fail');expect(r.stderr).toContain('6 pass');});+test('reference passes all fourteen provider outcomes',()=>{const r=grade('reference',fixed);expect(r.status).toBe(0);expect(r.stderr).toContain('14 pass');});+test('equivalent predicate naming passes',()=>expect(grade('renamed',fixed.replaceAll('isCacheableContentPart','canCacheTextPart')).status).toBe(0));+test('unused correct provider cannot rescue the broken entrypoint',()=>fails(grade('unused',broken,fixed)));+for(const stream of [false,true])test(`reintroducing image markers in ${stream?'streaming':'nonstreaming'} is rejected`,()=>{+ const target='await applyGpt56PromptCaching(params, body.gpt56HistoryCacheEligible === true);';expect(fixed.split(target)).toHaveLength(3);+ const body=fixed.replaceAll(target,target+` if(params.stream === ${stream} && body.gpt56HistoryCacheEligible) for(const m of params.messages) if(Array.isArray(m.content)) for(const p of m.content) if(p.type==='image_url') (p as any).prompt_cache_breakpoint={mode:'explicit'};`);fails(grade('readd-'+stream,body));+});+test('dropping images from completion requests is rejected',()=>fails(grade('drop',replace(fixed,'const messages = this.formatMessages(body.messages);','const messages = this.formatMessages(body.messages).map(m=>({...m,content:Array.isArray(m.content)?m.content.filter(p=>p.type!=="image_url"):m.content}));'))));+test('turning off all caching is rejected',()=>fails(grade('disable',replace(fixed,'function markLastCacheableContent(message: ChatCompletionMessageParam): boolean {','function markLastCacheableContent(message: ChatCompletionMessageParam): boolean { return false;'))));+test('missing provider is an import error',()=>{const r=grade('missing',null);expect(r.status).toBe(1);expect(r.stderr).toMatch(/Cannot find|ModuleNotFound/);expect(r.stderr).not.toContain('expect(received)');});diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex 2b6c5b657..4c3934879 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-09-20.8",+ "dataset_version": "2026-09-20.9","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -1885,6 +1885,38 @@],"command": "cd packages/screenpipe-mcp && SCREENPIPE_DISABLE_TELEMETRY=1 \"${BUN_BIN:-bun}\" run test -- src/eval-db-boundary.test.ts"}+ },+ {+ "id": "ai-gateway-image-cache-boundary",+ "name": "Keep image payloads free of text-only prompt cache markers",+ "source": {+ "kind": "git_regression",+ "fix_commit": "8696fbe7851f82f1bc1164929e860021790e928c"+ },+ "base_ref": "89ca135f648990b52655d5e21f3d334bb72ae3cc",+ "oracle_ref": "8696fbe7851f82f1bc1164929e860021790e928c",+ "oracle_paths": [+ "packages/ai-gateway/src/providers/openai.ts"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "ai-gateway",+ "media",+ "provider-compatibility"+ ],+ "prompt": "Explicit GPT-5.6 prompt caching can attach text-only cache breakpoint fields to image payloads and cause provider rejection. Fix the actual OpenAI provider request path for both streaming and non-streaming calls. Keep image data, detail and order intact through existing image_url, native image and base64-image normalization; do not add cache fields to image blocks or drop images to avoid rejection. Keep an instruction breakpoint and mark the last nonempty text in eligible multipart history, including text that precedes trailing images; image-only history should stay unchanged without a history marker. Preserve the final-text boundary, disabled history caching and older models without explicit cache fields. Successful provider content must still reach the caller. Verify sent request snapshots and returned responses with a synthetic SDK; do not make real model calls. Audio/file/refusal normalization is outside this task.",+ "agent_timeout_seconds": 1200,+ "grader": {+ "timeout_seconds": 90,+ "fixtures": [+ {+ "local_path": "graders/openai-media-cache.test.js",+ "destination_path": "evals/coding-agent/graders/openai-media-cache.test.js"+ }+ ],+ "command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/openai-media-cache.test.js"+ }}]}diff --git a/evals/coding-agent/graders/openai-media-cache.test.js b/evals/coding-agent/graders/openai-media-cache.test.jsnew file mode 100644index 000000000..a2e84209c--- /dev/null+++ b/evals/coding-agent/graders/openai-media-cache.test.js@@ -0,0 +1,49 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import {expect,test,mock,beforeEach} from 'bun:test';+const sent=[];+mock.module('openai',()=>({default:class OpenAI {+ chat={completions:{create:async params=>{+ sent.push(structuredClone(params));+ if(params.stream)return {controller:{abort(){}},async *[Symbol.asyncIterator](){yield {id:'synthetic',object:'chat.completion.chunk',created:1,model:params.model,choices:[{index:0,delta:{content:'synthetic completion'},finish_reason:null}]};yield {id:'synthetic',object:'chat.completion.chunk',created:1,model:params.model,choices:[{index:0,delta:{},finish_reason:'stop'}]};}};+ return {id:'synthetic',object:'chat.completion',created:1,model:params.model,choices:[{index:0,message:{role:'assistant',content:'synthetic completion'},finish_reason:'stop'}],usage:{prompt_tokens:3,completion_tokens:2,total_tokens:5}};+ }}};+}}));+mock.module('@sentry/cloudflare',()=>({captureException(){}}));+const {OpenAIProvider}=await import('../../../packages/ai-gateway/src/providers/openai.ts');+beforeEach(()=>{sent.length=0;});+const marker={mode:'explicit'};+const media=[+ {label:'image_url',input:{type:'image_url',image_url:{url:'data:image/png;base64,AA',detail:'high'}},expected:{type:'image_url',image_url:{url:'data:image/png;base64,AA',detail:'high'}}},+ {label:'native image',input:{type:'image',data:'AA',mimeType:'image/png'},expected:{type:'image_url',image_url:{url:'data:image/png;base64,AA',detail:'auto'}}},+ {label:'base64 image',input:{type:'image',source:{type:'base64',media_type:'image/png',data:'BB'}},expected:{type:'image_url',image_url:{url:'data:image/png;base64,BB',detail:'auto'}}},+];+async function call(stream,content,{history=true,model='gpt-5.6-luna'}={}){+ const provider=new OpenAIProvider('synthetic-test-key');+ const body={model,gpt56HistoryCacheEligible:history,messages:[{role:'system',content:'stable synthetic instruction'},{role:'user',content}],max_tokens:100};+ if(stream){const output=await provider.createStreamingCompletion(body);expect(await new Response(output).text()).toContain('synthetic completion');}+ else expect(JSON.stringify(await (await provider.createCompletion(body)).json())).toContain('synthetic completion');+ expect(sent).toHaveLength(1);return sent[0];+}+for(const stream of [false,true]){+ for(const part of media)test(`${stream?'stream':'completion'} keeps cache markers off ${part.label} and preserves preceding text caching`,async()=>{+ const params=await call(stream,[{type:'text',text:'inspect synthetic attachment'},structuredClone(part.input),{type:'text',text:''}]);+ expect(params.messages[1].content[1]).toEqual(part.expected);+ expect(params.messages[1].content[0].prompt_cache_breakpoint).toEqual(marker);+ expect(params.messages[1].content[2]).toEqual({type:'text',text:''});+ expect(params.messages[0].content[0].prompt_cache_breakpoint).toEqual(marker);+ });+ test(`${stream?'stream':'completion'} preserves media-only history without marking or dropping it`,async()=>{+ const params=await call(stream,media.map(p=>structuredClone(p.input)));expect(params.messages[1].content).toEqual(media.map(p=>p.expected));expect(params.messages[0].content[0].prompt_cache_breakpoint).toEqual(marker);+ });+ test(`${stream?'stream':'completion'} preserves the final nonempty text boundary`,async()=>{+ const params=await call(stream,[{type:'text',text:'earlier'},structuredClone(media[0].input),{type:'text',text:'latest'}]);+ expect(params.messages[1].content[0].prompt_cache_breakpoint).toBeUndefined();expect(params.messages[1].content[1]).toEqual(media[0].expected);expect(params.messages[1].content[2]).toEqual({type:'text',text:'latest',prompt_cache_breakpoint:marker});+ });+ test(`${stream?'stream':'completion'} preserves disabled history caching`,async()=>{+ const content=[{type:'text',text:'ordinary'},...media.map(p=>structuredClone(p.input))];const params=await call(stream,content,{history:false});expect(params.messages[1].content).toEqual([{type:"text",text:"ordinary"},...media.map(p=>p.expected)]);expect(params.messages[0].content[0].prompt_cache_breakpoint).toEqual(marker);+ });+ test(`${stream?'stream':'completion'} leaves older models without explicit cache fields`,async()=>{+ const content=[{type:'text',text:'ordinary'},...media.map(p=>structuredClone(p.input))];const params=await call(stream,content,{model:'gpt-5.4'});expect(params.messages[1].content).toEqual([{type:"text",text:"ordinary"},...media.map(p=>p.expected)]);expect(params.messages[0].content).toBe('stable synthetic instruction');expect(params.prompt_cache_options).toBeUndefined();expect(params.prompt_cache_key).toBeUndefined();+ });+}