YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
test(evals): verify settings persistence before restart
louisa47ec6bf1908
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 7446f756c..2f9fdfd55 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -13,6 +13,18 @@ The current app corpus contains 80 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.+`app-settings-restart-persistence` runs the actual settings provider, write queue+and update banner with synthetic React, storage and native restart ports. Four+outcomes cover a delayed save, another edit during the drain, failed-save refusal+with recovery, and an idle restart. The parent fails three and preserves one;+the reference and current source pass four. Run+`bun test evals/coding-agent/calibrate-settings-restart.test.js` for nine controls,+including unawaited drain, swallowed failures, unused correct code, blanket+restart denial and missing-source classification. The fixture is installed only+when grading begins. This checks the persisted snapshot at the macOS/Linux+handoff, not OS durability, encryption, native installation, Windows behavior,+cross-window synchronization, isolation or model performance.+`app-pipe-store-load-error` renders the actual store component with syntheticnetwork, cache and native ports. Ten outcomes distinguish payload, HTTP andnetwork failures from genuine empty catalogs and verify retries to populateddiff --git a/evals/coding-agent/calibrate-settings-restart.test.js b/evals/coding-agent/calibrate-settings-restart.test.jsnew file mode 100644index 000000000..f2dfced2a--- /dev/null+++ b/evals/coding-agent/calibrate-settings-restart.test.js@@ -0,0 +1,28 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { dirname, join, resolve } from 'node:path';+import { tmpdir } from 'node:os';+const repo=resolve(import.meta.dir,'../..'),prefix='apps/screenpipe-app-tauri/';+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='app-settings-restart-persistence');+const paths=['tsconfig.json','lib/hooks/use-settings.tsx','lib/ai-preset-deletion.ts','lib/active-ai-preset.ts','lib/chat-storage.ts','lib/chat-dedup.ts','lib/chat-merge.ts','lib/chat/ephemeral-side-conversation.ts','lib/chat/external-chat-parser.ts','lib/hooks/managed-settings.ts','lib/desktop-remote-control.ts','lib/app-entitlement.ts','lib/telemetry-env.ts','lib/utils/font-size.ts','lib/live-views/onboarding-activation.ts','lib/utils/sidebar-nav-layout.ts','lib/utils/chat-title.ts','lib/auth-guard.tsx','lib/utils/tauri.ts','lib/hooks/use-is-enterprise-build.ts','components/settings/settings-write-queue.ts','lib/free-plan-retention.ts','components/update-banner.tsx'];+function sources(ref){const available=new Set(execFileSync('git',['ls-tree','-r','--name-only',ref,prefix],{cwd:repo,encoding:'utf8'}).trim().split('\n'));return Object.fromEntries(paths.filter(p=>available.has(prefix+p)).map(p=>[p,execFileSync('git',['show',`${ref}:${prefix}${p}`],{cwd:repo,encoding:'utf8'})]));}+const broken=sources(item.base_ref),fixed=sources(item.oracle_ref);+const fixture='evals/coding-agent/graders/settings-restart-persistence.test.js',grader=readFileSync(join(repo,fixture));+const root=mkdtempSync(join(tmpdir(),'settings-restart-calibration-'));afterAll(()=>rmSync(root,{recursive:true,force:true}));+function grade(name,sources){const cwd=join(root,name);for(const [path,source] of Object.entries(sources)){const file=join(cwd,prefix,path);mkdirSync(dirname(file),{recursive:true});writeFileSync(file,source);}mkdirSync(dirname(join(cwd,fixture)),{recursive:true});writeFileSync(join(cwd,fixture),grader);return spawnSync(process.execPath,['test','../../'+fixture],{cwd:join(cwd,prefix),encoding:'utf8',timeout:30_000,env:{PATH:dirname(process.execPath)}});}+function fails(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain('expect(received)');expect(r.stderr).not.toContain('Cannot find module');expect(r.stderr).not.toContain('Unhandled error');}+function replace(source,old,next){expect(source.split(old)).toHaveLength(2);return source.replace(old,next);}++const settings='lib/hooks/use-settings.tsx',banner='components/update-banner.tsx';+test('parent fails three persistence outcomes and preserves idle restart',()=>{const r=grade('parent',broken);fails(r);expect(r.stderr).toContain('3 fail');expect(r.stderr).toContain('1 pass');});+test('reference passes all four outcomes',()=>{const r=grade('reference',fixed);expect(r.status).toBe(0);expect(r.stderr).toContain('4 pass');});+test('equivalent private handler name is accepted',()=>expect(grade('renamed',{...fixed,[banner]:fixed[banner].replaceAll('handleUpdate','requestUpdateRestart')}).status).toBe(0));+test('unawaited drain cannot authorize early restart',()=>fails(grade('unawaited',{...fixed,[banner]:replace(fixed[banner],'await flushPendingSettingsWrites();','void flushPendingSettingsWrites().catch(() => {});')})));+test('an unused correct banner cannot hide the broken call path',()=>fails(grade('unused',{...broken,'components/unused-correct-banner.tsx':fixed[banner]})));+test('ignoring failed persistence is rejected',()=>fails(grade('ignore-error',{...fixed,[banner]:replace(fixed[banner],'await flushPendingSettingsWrites();','await flushPendingSettingsWrites().catch(() => {});')})));+test('blanket restart refusal violates preserved idle behavior',()=>fails(grade('deny-all',{...fixed,[banner]:replace(fixed[banner],'const handleUpdate = async () => {','const handleUpdate = async () => { return;')})));+test('dropping settings mutations is rejected',()=>fails(grade('drop-writes',{...fixed,[settings]:replace(fixed[settings],'await settingsStore.set(updates);','await Promise.resolve();')})));+test('missing actual settings source remains setup error',()=>{const files={...fixed};delete files[settings];const r=grade('missing',files);expect(r.status).toBe(1);expect(r.stderr).toContain('Cannot find module');expect(r.stderr).toContain('0 pass');});diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex 98ce660b8..6b526f312 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-09-27.4",+ "dataset_version": "2026-09-27.5","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -3812,6 +3812,39 @@],"command": "cd packages/ai-gateway && \"${BUN_BIN:-bun}\" --no-env-file test src/test/eval-ip-quota-retention.test.ts"}+ },+ {+ "id": "app-settings-restart-persistence",+ "name": "Persist accepted settings before update restart",+ "source": {+ "kind": "git_regression",+ "fix_commit": "46402fe054406730ef291fc7de18a7ded6f975ce"+ },+ "base_ref": "4e7b00fdb6317594150daefacb60d69eab8baaba",+ "oracle_ref": "46402fe054406730ef291fc7de18a7ded6f975ce",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/lib/hooks/use-settings.tsx",+ "apps/screenpipe-app-tauri/components/update-banner.tsx"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "settings",+ "update",+ "persistence"+ ],+ "prompt": "A user enables Auto-update and immediately chooses Restart to update, but the restarted app can retain the old setting. Repair the actual settings persistence and update-banner flow so accepted writes finish saving before the macOS/Linux native restart handoff. Include another settings edit accepted while the first save is pending: both changes must be saved before the handoff. A failed save must prevent the handoff and leave the banner usable; a later successful save must allow a retry. Restart with no pending writes must still proceed and preserve already saved Auto-update and font-size settings. No particular helper names, queue structure or private implementation is required. Native installation, OS persistence guarantees, encryption, Windows update behavior and cross-window synchronization are outside this task.",+ "agent_timeout_seconds": 1200,+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/settings-restart-persistence.test.js",+ "destination_path": "evals/coding-agent/graders/settings-restart-persistence.test.js"+ }+ ],+ "command": "cd apps/screenpipe-app-tauri && \"${BUN_BIN:-bun}\" test ../../evals/coding-agent/graders/settings-restart-persistence.test.js"+ }}]}diff --git a/evals/coding-agent/graders/settings-restart-persistence.test.js b/evals/coding-agent/graders/settings-restart-persistence.test.jsnew file mode 100644index 000000000..3c64212e0--- /dev/null+++ b/evals/coding-agent/graders/settings-restart-persistence.test.js@@ -0,0 +1,96 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { beforeEach, afterEach, expect, mock, test } from 'bun:test';+import { posix } from 'node:path';+import { existsSync } from 'node:fs';+function mockAppModule(specifier,factory){+ mock.module(specifier,factory);+ const stem=new URL('../../../apps/screenpipe-app-tauri/'+specifier.slice(2),import.meta.url).pathname;+ for(const suffix of ['.ts','.tsx','.js'])if(existsSync(stem+suffix)){mock.module(stem+suffix,factory);break;}+}+let s;+const clone=x=>structuredClone(x);+const storage=new Map();+globalThis.window={localStorage:{getItem:k=>storage.get(k)??null,setItem:(k,v)=>storage.set(k,String(v)),removeItem:k=>storage.delete(k)},location:{pathname:'/settings'}};+globalThis.localStorage=window.localStorage;+globalThis.fetch=async()=>{s.unexpected.push('network');throw Error('Network forbidden');};+const react={createContext:()=>({Provider:'provider'}),createElement:(type,props,...children)=>({type,props:{...props,children}}),useState:value=>[typeof value==='function'?value():value,v=>{if(typeof v!=="function")s.rendered=v;}],useEffect:()=>{},useRef:value=>({current:value}),useContext:()=>undefined};+mock.module('react',()=>({...react,default:react}));+mock.module('react/jsx-dev-runtime',()=>({jsxDEV:(type,props)=>({type,props})}));+mock.module('react/jsx-runtime',()=>({jsx:(type,props)=>({type,props}),jsxs:(type,props)=>({type,props})}));+mock.module('posthog-js',()=>({default:{capture:()=>{},identify:()=>{}}}));+mock.module('@tauri-apps/api/core',()=>({invoke:async(name)=>{s.unexpected.push('invoke:'+name);throw Error('Unexpected native command');}}));+mock.module('@tauri-apps/api/path',()=>({homeDir:async()=>'/synthetic',join:async(...parts)=>posix.join(...parts)}));+mock.module('@tauri-apps/api/app',()=>({getVersion:async()=>'0.0.0'}));+mock.module('@tauri-apps/plugin-os',()=>({platform:()=>'macos',arch:()=>'aarch64'}));+mock.module('@tauri-apps/api/event',()=>({emit:async(name)=>{s.events.push(name);},listen:async()=>()=>{}}));+const store={get:async()=>clone(s.settings),set:async(k,value)=>{s.settings=clone(value);s.order.push('settings:set');},save:async()=>{if(s.saveGate)await s.saveGate;if(s.failSettings)throw Error('synthetic settings failure');s.diskSettings=clone(s.settings);s.order.push('settings:save');},onKeyChange:()=>()=>{}};+mock.module('@tauri-apps/plugin-store',()=>({Store:{load:async()=>store}}));+mockAppModule('@/lib/utils/tauri',()=>({commands:{restartForUpdate:async()=>{s.handoffs.push(clone(s.diskSettings));return {status:'ok',data:'proceed'};},getScreenpipeBaseDir:async()=>({status:'ok',data:'/synthetic'}),getChatsDir:async()=>({status:'ok',data:'/synthetic/chats'}),reencryptStore:async()=>{},getCloudToken:async()=>({status:'ok',data:null}),setCloudToken:async()=>({status:"ok",data:null})}}));+mockAppModule('@/lib/analytics-id',()=>({cacheAnalyticsId:()=>{},cacheAnalyticsEnabled:()=>{}}));+mockAppModule('@/lib/analytics/settings-change',()=>({captureSettingsChange:()=>{}}));+mockAppModule('@/lib/analytics/onboarding-h1-follow-up',()=>({captureOnboardingH1FollowUp:()=>{}}));+mockAppModule('@/lib/auth-guard',()=>({installAuthInterceptor:()=>()=>{}}));+mockAppModule('@/lib/hooks/use-is-enterprise-build',()=>({isResolvedConsumerBuild:async()=>true}));+mockAppModule('@/lib/web-url',()=>({screenpipeWebUrl:path=>'https://screenpipe.invalid'+path}));+mockAppModule('@/lib/browser-state-cache',()=>({deleteCachedBrowserState:async()=>{}}));+mockAppModule('@/lib/stores/chat-store',()=>({useChatStore:{getState:()=>({sessions:s.sessions,actions:{patch:(id,update)=>Object.assign(s.sessions[id],update)}})}}));+mockAppModule('@/lib/api',()=>({localFetch:async(path,init)=>{+ s.requests.push({path,init});if(path==='/pipes'){if(s.failDiscovery)return Response.json({error:'unavailable'},{status:503});return Response.json({data:s.pipes});}+ const pipe=s.pipes.find(p=>path===`/pipes/${encodeURIComponent(p.config.name)}/config`);if(!pipe){s.unexpected.push(path);throw Error('Unknown task');}+ if(s.failPipe===pipe.config.name)return Response.json({success:false},{status:503});Object.assign(pipe.config,JSON.parse(init.body));s.order.push('pipe:'+pipe.config.name);return Response.json({success:true});+}}));+mock.module('@tauri-apps/plugin-fs',()=>({+ exists:async path=>path==='/synthetic/chats'||s.files.has(path),mkdir:async()=>{},+ readTextFile:async path=>{if(!s.files.has(path))throw Error('missing file');const text=s.files.get(path);if(s.advanceSelectionOnRead){s.advanceSelectionOnRead=false;const newer=JSON.parse(text);newer.presetId='c';newer.rev++;s.files.set(path,JSON.stringify(newer));}return text;},+ writeTextFile:async(path,text)=>{if(s.failChat)throw Error('synthetic chat failure');s.files.set(path,text);},+ rename:async(from,to)=>{s.files.set(to,s.files.get(from));s.files.delete(from);s.order.push('chat:'+to);},remove:async path=>{s.files.delete(path);},+ readDir:async()=>[...s.files.keys()].filter(p=>p.endsWith('.json')).map(p=>({name:posix.basename(p),isFile:true,isDirectory:false})),+ stat:async()=>({mtime:new Date(100),isFile:true,isDirectory:false})+}));+++mock.module('zustand',()=>({create:factory=>{let state;const set=patch=>Object.assign(state,patch);state=factory(set);const hook=()=>state;hook.getState=()=>state;return hook;}}));+mock.module('lucide-react',()=>({Sparkles:'sparkles',X:'x'}));+mock.module('gt-react',()=>({useGT:()=>s=>s,msg:s=>s}));+mock.module('@tauri-apps/plugin-process',()=>({relaunch:async()=>{throw Error('Unexpected relaunch');}}));+mock.module('@tauri-apps/plugin-updater',()=>({check:async()=>{throw Error('Unexpected update download');}}));+mockAppModule('@/components/ui/button',()=>({Button:'button'}));+mockAppModule('@/components/ui/use-toast',()=>({useToast:()=>({toast:x=>s.toasts.push(x)})}));+mockAppModule('@/lib/utils',()=>({cn:(...args)=>args.join(' ')}));+mockAppModule('@/lib/enterprise-auth-recovery',()=>({enterpriseUpdateAuthHeaders:()=>({})}));+const settingsModule=await import('../../../apps/screenpipe-app-tauri/lib/hooks/use-settings');+const banner=await import('../../../apps/screenpipe-app-tauri/components/update-banner');+let api;+const tick=()=>new Promise(resolve=>setTimeout(resolve,0));+const deferred=()=>{let resolve;const promise=new Promise(r=>resolve=r);return {promise,resolve};};+function text(node){if(typeof node==='string')return node;if(Array.isArray(node))return node.map(text).join(' ');return node?.props ? text(node.props.children) : '';}+function restartButton(node=banner.UpdateBanner({compact:true})){+ if(Array.isArray(node)){for(const child of node){const found=restartButton(child);if(found)return found;}return null;}+ if(node?.type==='button' && /restart.*update|update.*restart/i.test(text(node)))return node;+ return node?.props?.children ? restartButton(node.props.children) : null;+}+function click(){const button=restartButton();expect(button).not.toBeNull();expect(button.props.disabled).not.toBe(true);return button.props.onClick();}+beforeEach(async()=>{+ storage.clear();s={settings:{...clone(settingsModule.createDefaultSettingsObject()),deviceId:'synthetic-device',user:null,autoUpdate:false},diskSettings:null,files:new Map(),sessions:{},pipes:[],requests:[],events:[],unexpected:[],order:[],rendered:null,handoffs:[],toasts:[]};+ s.diskSettings=clone(s.settings);api=settingsModule.SettingsProvider({children:null}).props.value;+ await api.reloadStore();await api.updateSettings({autoUpdate:false});+ Object.assign(banner.useUpdateBanner.getState(),{isVisible:true,updateInfo:{version:'9.9.9',body:'synthetic'},isInstalling:false,authRequired:null});+});+afterEach(()=>expect(s.unexpected).toEqual([]));+test('accepted settings persist before restart handoff',async()=>{+ const gate=deferred();s.saveGate=gate.promise;const write=api.updateSettings({autoUpdate:true});const restart=click();+ try {await tick();expect(s.handoffs).toEqual([]);} finally {gate.resolve();await write;await restart;}+ expect(s.handoffs).toHaveLength(1);expect(s.handoffs[0].autoUpdate).toBe(true);+});+test('a second accepted edit during drain also persists before restart',async()=>{+ const gate=deferred();s.saveGate=gate.promise;const first=api.updateSettings({autoUpdate:true});const restart=click();await tick();const second=api.updateSettings({fontSize:'18px'});+ gate.resolve();await Promise.all([first,second,restart]);expect(s.handoffs).toHaveLength(1);expect(s.handoffs[0].autoUpdate).toBe(true);expect(s.handoffs[0].fontSize).toBe('18px');+});+test('failed persistence refuses restart and allows recovery',async()=>{+ s.failSettings=true;const write=api.updateSettings({autoUpdate:true}).catch(e=>e);await write;await click();expect(s.handoffs).toEqual([]);expect(restartButton().props.disabled).not.toBe(true);+ s.failSettings=false;await api.updateSettings({autoUpdate:true});await click();expect(s.handoffs).toHaveLength(1);expect(s.handoffs[0].autoUpdate).toBe(true);+});+test('idle restart preserves already saved settings',async()=>{+ await api.updateSettings({autoUpdate:true,fontSize:'18px'});await click();expect(s.handoffs).toHaveLength(1);expect(s.handoffs[0].autoUpdate).toBe(true);expect(s.handoffs[0].fontSize).toBe('18px');+});