YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: preserve embedded Codex image imports
Louis Beaumontfed2c61e10dd
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex bc22ded6c..cbb41d953 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -67,7 +67,7 @@ The hidden fixture appears only at grading and uses no dependency links.This does not establish native recording continuity, real authentication,account-switch race safety, browser integration, isolation or model performance.-The current app corpus contains 140 git-mined regressions. See+The current app corpus contains 141 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.@@ -1781,3 +1781,16 @@ preserves four; the historical fix and current source pass eight. Runcorrect, broken, disconnected, equivalent and preserved-behavior controls.This does not verify actual transcript discovery, durable native writes,composer selection, enterprise policy, provider execution, isolation or model gains.++## Codex embedded image imports++`app-codex-embedded-image-import` executes the actual importer and parser with+synthetic filesystem, persistence and event ports. Eleven outcomes cover ordered+embedded images, image-only messages, missing/changed image refreshes, transport+markup, unchanged records, rejected sources and preserved conversation state.+The parent fails five attachment outcomes and preserves six; the historical fix+and current source pass eleven. Run+`bun test evals/coding-agent/calibrate-codex-images.test.js` for calibration.+Fixtures and dependency links appear only at grading. This does not establish+image decoding/rendering, path-only attachment handling, real discovery, native+durability, enforced isolation or model performance.diff --git a/evals/coding-agent/calibrate-codex-images.test.js b/evals/coding-agent/calibrate-codex-images.test.jsnew file mode 100644index 000000000..d316c0698--- /dev/null+++ b/evals/coding-agent/calibrate-codex-images.test.js@@ -0,0 +1,39 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import {afterAll,expect,test} from 'bun:test';+import {execFileSync,spawnSync} from 'node:child_process';+import {mkdirSync,mkdtempSync,readFileSync,writeFileSync,symlinkSync,rmSync} from 'node:fs';+import {dirname,join,resolve} from 'node:path';+import {tmpdir} from 'node:os';+const repo=resolve(import.meta.dir,'../..'),app='apps/screenpipe-app-tauri';+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='app-codex-embedded-image-import');+const root=mkdtempSync(join(tmpdir(),'codex-images-calibration-'));+const receipts=process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;+const parser=`${app}/lib/chat/external-chat-parser.ts`,importer=`${app}/lib/chat/external-chat-import.ts`;+const paths=[parser,importer,`${app}/lib/chat-storage.ts`];+afterAll(()=>rmSync(root,{recursive:true,force:true}));+function replace(cwd,file,from,to){const p=join(cwd,file),s=readFileSync(p,'utf8');expect(s.split(from)).toHaveLength(2);writeFileSync(p,s.replace(from,to));}+function grade(name,ref=item.oracle_ref,mutate=()=>{}){+ const cwd=join(root,name);mkdirSync(cwd);+ // Exact historical sources only. External dependencies are linked at grading.+ for(const path of paths){const dest=join(cwd,path);mkdirSync(dirname(dest),{recursive:true});writeFileSync(dest,execFileSync('git',['show',`${ref}:${path}`],{cwd:repo}));}+ for(const f of item.grader.fixtures){const dest=join(cwd,f.destination_path);mkdirSync(dirname(dest),{recursive:true});writeFileSync(dest,readFileSync(join(import.meta.dir,f.local_path)));}+ for(const link of item.grader_dependency_links)symlinkSync(join(repo,link.source_path),join(cwd,link.destination_path),'dir');+ mutate(cwd);+ const r=spawnSync('/bin/bash',['-c',item.grader.command],{cwd,encoding:'utf8',timeout:60000,maxBuffer:4*1024*1024,env:{PATH:process.env.PATH,HOME:cwd,CI:'true',TZ:'UTC',NO_COLOR:'1'}});+ if(receipts){mkdirSync(receipts,{recursive:true});writeFileSync(join(receipts,name+'.stdout'),r.stdout||'');writeFileSync(join(receipts,name+'.stderr'),r.stderr||'');writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message},null,2));}+ expect(r.error).toBeUndefined();expect(r.signal).toBeNull();return r;+}+function passes(r){expect(r.status).toBe(0);expect(r.stdout).toContain('11 passed');expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import/);}+function fails(r){expect(r.status).toBe(1);expect(r.stderr).toMatch(/AssertionError|Error: expect\(/);expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Cannot find module/);}+test('parent fails five attachment outcomes and preserves six',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('5 failed | 6 passed');},60000);+test('historical reference passes eleven',()=>passes(grade('reference')),60000);+test('current source passes eleven',()=>passes(grade('current','HEAD')),60000);+test('equivalent private helper name passes',()=>passes(grade('equivalent',item.oracle_ref,cwd=>{const p=join(cwd,parser);writeFileSync(p,readFileSync(p,'utf8').replaceAll('codexImages','readEmbeddedImages'));})),60000);+test('unused fixed parser cannot hide the broken importer',()=>fails(grade('bypass',item.base_ref,cwd=>{writeFileSync(join(cwd,`${app}/unused-parser.ts`),execFileSync('git',['show',`${item.oracle_ref}:${parser}`],{cwd:repo}));})),60000);+test('fixed parser with old sync comparison fails refresh',()=>fails(grade('old-comparison',item.oracle_ref,cwd=>{writeFileSync(join(cwd,importer),execFileSync('git',['show',`${item.base_ref}:${importer}`],{cwd:repo}));})),60000);+test('dropping image-only messages is rejected',()=>fails(grade('image-only',item.oracle_ref,cwd=>{replace(cwd,parser,'if (!userText && images.length === 0) continue;','if (!userText) continue;');})),60000);+test('unsafe image admission is rejected',()=>fails(grade('unsafe',item.oracle_ref,cwd=>{replace(cwd,parser,'? url : undefined;','? url : url;');})),60000);+test('retaining transport envelopes is rejected',()=>fails(grade('transport',item.oracle_ref,cwd=>{replace(cwd,parser,'const userText = codexImageText(cleanCodexUserText(text), imageParts);','const userText = cleanCodexUserText(text);');})),60000);+test('losing user conversation state is rejected',()=>fails(grade('preserved',item.oracle_ref,cwd=>{replace(cwd,importer,'...existing,','');})),60000);+test('missing parser remains infrastructure failure',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,parser)));expect(r.status).toBe(1);expect(r.stdout+r.stderr).toMatch(/Failed to resolve import|Failed to load url|Cannot find module/);expect(r.stderr).not.toMatch(/AssertionError/);},60000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex f590fb190..48dbc4b0e 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-10-07-imported-agent-settings",+ "dataset_version": "2026-10-08-codex-embedded-images","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -5841,6 +5841,54 @@],"command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-import-settings.config.mjs"}+ },+ {+ "id": "app-codex-embedded-image-import",+ "name": "Preserve embedded image attachments during Codex transcript import",+ "source": {+ "kind": "git_regression",+ "fix_commit": "a8ef9734ad2e852593b787c6272a2a11b85e12b4"+ },+ "base_ref": "21b59f57bd16e20196349287de7cbc2da99262eb",+ "oracle_ref": "a8ef9734ad2e852593b787c6272a2a11b85e12b4",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/lib/chat/external-chat-parser.ts",+ "apps/screenpipe-app-tauri/lib/chat/external-chat-import.ts"+ ],+ "trigger_paths": [+ "apps/screenpipe-app-tauri/lib/chat/external-chat-parser.ts",+ "apps/screenpipe-app-tauri/lib/chat/external-chat-import.ts"+ ],+ "gate": "advisory",+ "tags": [+ "regression",+ "escaped-failure",+ "chat-import",+ "images",+ "preserved-behavior"+ ],+ "prompt": "Codex transcript imports lose embedded image attachments. Repair the actual import-and-save flow so accepted base64 raster image data reaches the existing saved-message images contract in original order, alongside user text. Retain image-only user messages. Automatic sync must persist missing or changed image bytes even if text, timestamps and image count are unchanged, while avoiding writes for fully unchanged text or image conversations. Matching trailing image transport envelopes must not appear in the saved user text or title; preserve quoted transport syntax as authored text. Reject remote image URLs, non-raster data such as HTML/SVG, and malformed base64 syntax from saved attachments without losing the accompanying text. Preserve chosen titles, presets, pins, local continuation messages and ordinary Claude text imports. Use the existing importer and conversation interfaces; no particular implementation is required. Grading invokes actual import-and-save code with synthetic filesystem, storage and event ports. Path-only attachments, renderer behavior, image decoding, real transcript discovery, native durability, execution isolation and provider access are separate.",+ "agent_timeout_seconds": 900,+ "grader_dependency_links": [+ {+ "source_path": "apps/screenpipe-app-tauri/node_modules",+ "destination_path": "apps/screenpipe-app-tauri/node_modules"+ }+ ],+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/codex-images.fixture.ts.txt",+ "destination_path": "apps/screenpipe-app-tauri/eval-codex-images.test.ts"+ },+ {+ "local_path": "graders/codex-images.config.mjs",+ "destination_path": "apps/screenpipe-app-tauri/eval-codex-images.config.mjs"+ }+ ],+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-codex-images.config.mjs"+ }}]}diff --git a/evals/coding-agent/graders/codex-images.config.mjs b/evals/coding-agent/graders/codex-images.config.mjsnew file mode 100644index 000000000..f783be7a8--- /dev/null+++ b/evals/coding-agent/graders/codex-images.config.mjs@@ -0,0 +1,3 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import {defineConfig} from 'vitest/config';import {resolve} from 'node:path';export default defineConfig({resolve:{alias:{'@':resolve(process.cwd())}},test:{environment:'node',include:['eval-codex-images.test.ts'],pool:'forks',minWorkers:1,maxWorkers:1}});diff --git a/evals/coding-agent/graders/codex-images.fixture.ts.txt b/evals/coding-agent/graders/codex-images.fixture.ts.txtnew file mode 100644index 000000000..41f407c07--- /dev/null+++ b/evals/coding-agent/graders/codex-images.fixture.ts.txt@@ -0,0 +1,29 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import {beforeEach,expect,it,vi} from 'vitest';+const io=vi.hoisted(()=>({read:vi.fn(),stat:vi.fn(),load:vi.fn(),save:vi.fn(),emit:vi.fn()}));+vi.mock('@tauri-apps/api/event',()=>({emit:io.emit}));+vi.mock('@tauri-apps/api/path',()=>({homeDir:vi.fn(),join:vi.fn()}));+vi.mock('@tauri-apps/plugin-fs',()=>({exists:vi.fn(),readDir:vi.fn(),readTextFile:io.read,stat:io.stat}));+vi.mock('@/lib/chat-storage',()=>({deleteConversationFile:vi.fn(),loadConversationFile:io.load,saveConversationFile:io.save,invalidateConversationListCache:vi.fn()}));+import {importExternalChatHistory} from '@/lib/chat/external-chat-import';+const at=Date.parse('2026-08-01T12:00:00Z');+const red='data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVQIHWP4z8DwHwAFgAI/ScLttAAAAABJRU5ErkJggg==';+const blue='data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVQIHWP4z/D/PwAH/wL+XJr2AAAAAElFTkSuQmCC';+const text='Review the synthetic tiles';+function codex(content=text,images:string[]=[]){return JSON.stringify({type:'response_item',timestamp:at,payload:{type:'message',id:'u1',role:'user',content:[{type:'input_text',text:content},...images.map(image_url=>({type:'input_image',image_url}))]}});}+function run(source='codex',skipUnchanged=true){return importExternalChatHistory([{source:source as any,path:'/fixture/synthetic.jsonl',sourceId:'synthetic-images',modifiedAt:at,size:1024}],{skipUnchanged});}+function saved(){return io.save.mock.calls.at(-1)?.[0];}+async function existing(images:string[]=[]){io.read.mockResolvedValue(codex());await run();const prior=structuredClone(saved());prior.messages[0].images=images;io.load.mockResolvedValue(prior);io.save.mockClear();return prior;}+beforeEach(()=>{vi.resetAllMocks();io.stat.mockResolvedValue({size:1024});io.load.mockResolvedValue(null);io.save.mockResolvedValue(undefined);io.emit.mockResolvedValue(undefined);});+it('persists ordered embedded images alongside the actual user text',async()=>{io.read.mockResolvedValue(codex(text,[red,blue]));expect(await run()).toMatchObject({imported:1,failed:0});expect(saved().messages[0]).toMatchObject({content:text,images:[red,blue]});});+it('retains an image-only user message',async()=>{io.read.mockResolvedValue(codex('',[red]));expect(await run()).toMatchObject({imported:1,skipped:0,failed:0});expect(saved().messages).toHaveLength(1);expect(saved().messages[0]).toMatchObject({role:'user',content:'',images:[red]});});+it('refreshes missing images when text and timestamps have not changed',async()=>{await existing();io.read.mockResolvedValue(codex(text,[red]));expect(await run()).toMatchObject({updated:1,skipped:0,failed:0});expect(saved().messages[0].images).toEqual([red]);});+it('refreshes changed image data even when the image count is unchanged',async()=>{await existing([red]);io.read.mockResolvedValue(codex(text,[blue]));expect(await run()).toMatchObject({updated:1,skipped:0,failed:0});expect(saved().messages[0].images).toEqual([blue]);});+it('removes matching trailing transport markup while preserving text and title',async()=>{io.read.mockResolvedValue(codex(`${text}\n<image name=[Image #1] path="/fixture/tile.png">\n</image>`,[red]));await run();expect(saved().messages[0].content).toBe(text);expect(saved().title).toBe(text);expect(saved().messages[0].images).toEqual([red]);});+it('leaves a completely unchanged text conversation unwritten',async()=>{await existing();expect(await run()).toMatchObject({skipped:1,updated:0,failed:0});expect(io.save).not.toHaveBeenCalled();});+it('leaves unchanged images unwritten',async()=>{await existing([red]);io.read.mockResolvedValue(codex(text,[red]));expect(await run()).toMatchObject({skipped:1,updated:0,failed:0});expect(io.save).not.toHaveBeenCalled();});+it('keeps unsupported or remote image sources out of saved attachments',async()=>{io.read.mockResolvedValue(codex(text,['https://example.invalid/tracker.png','data:text/html;base64,PHNjcmlwdD4=','data:image/svg+xml;base64,PHN2Zz4=','data:image/png;base64,%%%']));expect(await run()).toMatchObject({imported:1,failed:0});expect(saved().messages[0].content).toBe(text);expect(saved().messages[0].images??[]).toEqual([]);});+it('preserves quoted transport syntax as user-authored text',async()=>{const quoted='Example:\n```xml\n<image name=[Image #1] path="/fixture/tile.png">\n```';io.read.mockResolvedValue(codex(quoted));await run();expect(saved().messages[0].content).toBe(quoted);});+it('preserves chosen title, preset, pin and a local continuation when refreshing',async()=>{const prior=await existing();Object.assign(prior,{title:'My chosen title',titleSource:'user',presetId:'chosen-preset',pinned:true});prior.messages.push({id:'local-reply',role:'assistant',content:'Local continuation',timestamp:at+100});io.load.mockResolvedValue(prior);await run('codex',false);expect(saved()).toMatchObject({title:'My chosen title',titleSource:'user',presetId:'chosen-preset',pinned:true});expect(saved().messages.at(-1)).toMatchObject({id:'local-reply',content:'Local continuation'});});+it('preserves ordinary Claude text import',async()=>{io.read.mockResolvedValue(JSON.stringify({type:'user',timestamp:at,sessionId:'synthetic-images',uuid:'u1',message:{role:'user',content:'Synthetic Claude text'}}));expect(await run('claude-code')).toMatchObject({imported:1,failed:0});expect(saved().messages[0].content).toBe('Synthetic Claude text');});