Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

test(evals): verify settings persistence before restart

louisa47ec6bf1908
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 7446f756c..2f9fdfd55 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -13,6 +13,18 @@ The current app corpus contains 80 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest uses this same harness.
+`app-settings-restart-persistence` runs the actual settings provider, write queue
+and update banner with synthetic React, storage and native restart ports. Four
+outcomes cover a delayed save, another edit during the drain, failed-save refusal
+with recovery, and an idle restart. The parent fails three and preserves one;
+the reference and current source pass four. Run
+`bun test evals/coding-agent/calibrate-settings-restart.test.js` for nine controls,
+including unawaited drain, swallowed failures, unused correct code, blanket
+restart denial and missing-source classification. The fixture is installed only
+when grading begins. This checks the persisted snapshot at the macOS/Linux
+handoff, not OS durability, encryption, native installation, Windows behavior,
+cross-window synchronization, isolation or model performance.
+
`app-pipe-store-load-error` renders the actual store component with synthetic
network, cache and native ports. Ten outcomes distinguish payload, HTTP and
network failures from genuine empty catalogs and verify retries to populated
diff --git a/evals/coding-agent/calibrate-settings-restart.test.js b/evals/coding-agent/calibrate-settings-restart.test.js
new file mode 100644
index 000000000..f2dfced2a
--- /dev/null
+++ b/evals/coding-agent/calibrate-settings-restart.test.js
@@ -0,0 +1,28 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { tmpdir } from 'node:os';
+const repo=resolve(import.meta.dir,'../..'),prefix='apps/screenpipe-app-tauri/';
+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='app-settings-restart-persistence');
+const paths=['tsconfig.json','lib/hooks/use-settings.tsx','lib/ai-preset-deletion.ts','lib/active-ai-preset.ts','lib/chat-storage.ts','lib/chat-dedup.ts','lib/chat-merge.ts','lib/chat/ephemeral-side-conversation.ts','lib/chat/external-chat-parser.ts','lib/hooks/managed-settings.ts','lib/desktop-remote-control.ts','lib/app-entitlement.ts','lib/telemetry-env.ts','lib/utils/font-size.ts','lib/live-views/onboarding-activation.ts','lib/utils/sidebar-nav-layout.ts','lib/utils/chat-title.ts','lib/auth-guard.tsx','lib/utils/tauri.ts','lib/hooks/use-is-enterprise-build.ts','components/settings/settings-write-queue.ts','lib/free-plan-retention.ts','components/update-banner.tsx'];
+function sources(ref){const available=new Set(execFileSync('git',['ls-tree','-r','--name-only',ref,prefix],{cwd:repo,encoding:'utf8'}).trim().split('\n'));return Object.fromEntries(paths.filter(p=>available.has(prefix+p)).map(p=>[p,execFileSync('git',['show',`${ref}:${prefix}${p}`],{cwd:repo,encoding:'utf8'})]));}
+const broken=sources(item.base_ref),fixed=sources(item.oracle_ref);
+const fixture='evals/coding-agent/graders/settings-restart-persistence.test.js',grader=readFileSync(join(repo,fixture));
+const root=mkdtempSync(join(tmpdir(),'settings-restart-calibration-'));afterAll(()=>rmSync(root,{recursive:true,force:true}));
+function grade(name,sources){const cwd=join(root,name);for(const [path,source] of Object.entries(sources)){const file=join(cwd,prefix,path);mkdirSync(dirname(file),{recursive:true});writeFileSync(file,source);}mkdirSync(dirname(join(cwd,fixture)),{recursive:true});writeFileSync(join(cwd,fixture),grader);return spawnSync(process.execPath,['test','../../'+fixture],{cwd:join(cwd,prefix),encoding:'utf8',timeout:30_000,env:{PATH:dirname(process.execPath)}});}
+function fails(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain('expect(received)');expect(r.stderr).not.toContain('Cannot find module');expect(r.stderr).not.toContain('Unhandled error');}
+function replace(source,old,next){expect(source.split(old)).toHaveLength(2);return source.replace(old,next);}
+
+const settings='lib/hooks/use-settings.tsx',banner='components/update-banner.tsx';
+test('parent fails three persistence outcomes and preserves idle restart',()=>{const r=grade('parent',broken);fails(r);expect(r.stderr).toContain('3 fail');expect(r.stderr).toContain('1 pass');});
+test('reference passes all four outcomes',()=>{const r=grade('reference',fixed);expect(r.status).toBe(0);expect(r.stderr).toContain('4 pass');});
+test('equivalent private handler name is accepted',()=>expect(grade('renamed',{...fixed,[banner]:fixed[banner].replaceAll('handleUpdate','requestUpdateRestart')}).status).toBe(0));
+test('unawaited drain cannot authorize early restart',()=>fails(grade('unawaited',{...fixed,[banner]:replace(fixed[banner],'await flushPendingSettingsWrites();','void flushPendingSettingsWrites().catch(() => {});')})));
+test('an unused correct banner cannot hide the broken call path',()=>fails(grade('unused',{...broken,'components/unused-correct-banner.tsx':fixed[banner]})));
+test('ignoring failed persistence is rejected',()=>fails(grade('ignore-error',{...fixed,[banner]:replace(fixed[banner],'await flushPendingSettingsWrites();','await flushPendingSettingsWrites().catch(() => {});')})));
+test('blanket restart refusal violates preserved idle behavior',()=>fails(grade('deny-all',{...fixed,[banner]:replace(fixed[banner],'const handleUpdate = async () => {','const handleUpdate = async () => { return;')})));
+test('dropping settings mutations is rejected',()=>fails(grade('drop-writes',{...fixed,[settings]:replace(fixed[settings],'await settingsStore.set(updates);','await Promise.resolve();')})));
+test('missing actual settings source remains setup error',()=>{const files={...fixed};delete files[settings];const r=grade('missing',files);expect(r.status).toBe(1);expect(r.stderr).toContain('Cannot find module');expect(r.stderr).toContain('0 pass');});
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 98ce660b8..6b526f312 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-09-27.4",
+ "dataset_version": "2026-09-27.5",
"suite_type": "regression",
"owner": "screenpipe engineering",
"policy": {
@@ -3812,6 +3812,39 @@
],
"command": "cd packages/ai-gateway && \"${BUN_BIN:-bun}\" --no-env-file test src/test/eval-ip-quota-retention.test.ts"
}
+ },
+ {
+ "id": "app-settings-restart-persistence",
+ "name": "Persist accepted settings before update restart",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "46402fe054406730ef291fc7de18a7ded6f975ce"
+ },
+ "base_ref": "4e7b00fdb6317594150daefacb60d69eab8baaba",
+ "oracle_ref": "46402fe054406730ef291fc7de18a7ded6f975ce",
+ "oracle_paths": [
+ "apps/screenpipe-app-tauri/lib/hooks/use-settings.tsx",
+ "apps/screenpipe-app-tauri/components/update-banner.tsx"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "settings",
+ "update",
+ "persistence"
+ ],
+ "prompt": "A user enables Auto-update and immediately chooses Restart to update, but the restarted app can retain the old setting. Repair the actual settings persistence and update-banner flow so accepted writes finish saving before the macOS/Linux native restart handoff. Include another settings edit accepted while the first save is pending: both changes must be saved before the handoff. A failed save must prevent the handoff and leave the banner usable; a later successful save must allow a retry. Restart with no pending writes must still proceed and preserve already saved Auto-update and font-size settings. No particular helper names, queue structure or private implementation is required. Native installation, OS persistence guarantees, encryption, Windows update behavior and cross-window synchronization are outside this task.",
+ "agent_timeout_seconds": 1200,
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/settings-restart-persistence.test.js",
+ "destination_path": "evals/coding-agent/graders/settings-restart-persistence.test.js"
+ }
+ ],
+ "command": "cd apps/screenpipe-app-tauri && \"${BUN_BIN:-bun}\" test ../../evals/coding-agent/graders/settings-restart-persistence.test.js"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/settings-restart-persistence.test.js b/evals/coding-agent/graders/settings-restart-persistence.test.js
new file mode 100644
index 000000000..3c64212e0
--- /dev/null
+++ b/evals/coding-agent/graders/settings-restart-persistence.test.js
@@ -0,0 +1,96 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { beforeEach, afterEach, expect, mock, test } from 'bun:test';
+import { posix } from 'node:path';
+import { existsSync } from 'node:fs';
+function mockAppModule(specifier,factory){
+ mock.module(specifier,factory);
+ const stem=new URL('../../../apps/screenpipe-app-tauri/'+specifier.slice(2),import.meta.url).pathname;
+ for(const suffix of ['.ts','.tsx','.js'])if(existsSync(stem+suffix)){mock.module(stem+suffix,factory);break;}
+}
+let s;
+const clone=x=>structuredClone(x);
+const storage=new Map();
+globalThis.window={localStorage:{getItem:k=>storage.get(k)??null,setItem:(k,v)=>storage.set(k,String(v)),removeItem:k=>storage.delete(k)},location:{pathname:'/settings'}};
+globalThis.localStorage=window.localStorage;
+globalThis.fetch=async()=>{s.unexpected.push('network');throw Error('Network forbidden');};
+const react={createContext:()=>({Provider:'provider'}),createElement:(type,props,...children)=>({type,props:{...props,children}}),useState:value=>[typeof value==='function'?value():value,v=>{if(typeof v!=="function")s.rendered=v;}],useEffect:()=>{},useRef:value=>({current:value}),useContext:()=>undefined};
+mock.module('react',()=>({...react,default:react}));
+mock.module('react/jsx-dev-runtime',()=>({jsxDEV:(type,props)=>({type,props})}));
+mock.module('react/jsx-runtime',()=>({jsx:(type,props)=>({type,props}),jsxs:(type,props)=>({type,props})}));
+mock.module('posthog-js',()=>({default:{capture:()=>{},identify:()=>{}}}));
+mock.module('@tauri-apps/api/core',()=>({invoke:async(name)=>{s.unexpected.push('invoke:'+name);throw Error('Unexpected native command');}}));
+mock.module('@tauri-apps/api/path',()=>({homeDir:async()=>'/synthetic',join:async(...parts)=>posix.join(...parts)}));
+mock.module('@tauri-apps/api/app',()=>({getVersion:async()=>'0.0.0'}));
+mock.module('@tauri-apps/plugin-os',()=>({platform:()=>'macos',arch:()=>'aarch64'}));
+mock.module('@tauri-apps/api/event',()=>({emit:async(name)=>{s.events.push(name);},listen:async()=>()=>{}}));
+const store={get:async()=>clone(s.settings),set:async(k,value)=>{s.settings=clone(value);s.order.push('settings:set');},save:async()=>{if(s.saveGate)await s.saveGate;if(s.failSettings)throw Error('synthetic settings failure');s.diskSettings=clone(s.settings);s.order.push('settings:save');},onKeyChange:()=>()=>{}};
+mock.module('@tauri-apps/plugin-store',()=>({Store:{load:async()=>store}}));
+mockAppModule('@/lib/utils/tauri',()=>({commands:{restartForUpdate:async()=>{s.handoffs.push(clone(s.diskSettings));return {status:'ok',data:'proceed'};},getScreenpipeBaseDir:async()=>({status:'ok',data:'/synthetic'}),getChatsDir:async()=>({status:'ok',data:'/synthetic/chats'}),reencryptStore:async()=>{},getCloudToken:async()=>({status:'ok',data:null}),setCloudToken:async()=>({status:"ok",data:null})}}));
+mockAppModule('@/lib/analytics-id',()=>({cacheAnalyticsId:()=>{},cacheAnalyticsEnabled:()=>{}}));
+mockAppModule('@/lib/analytics/settings-change',()=>({captureSettingsChange:()=>{}}));
+mockAppModule('@/lib/analytics/onboarding-h1-follow-up',()=>({captureOnboardingH1FollowUp:()=>{}}));
+mockAppModule('@/lib/auth-guard',()=>({installAuthInterceptor:()=>()=>{}}));
+mockAppModule('@/lib/hooks/use-is-enterprise-build',()=>({isResolvedConsumerBuild:async()=>true}));
+mockAppModule('@/lib/web-url',()=>({screenpipeWebUrl:path=>'https://screenpipe.invalid'+path}));
+mockAppModule('@/lib/browser-state-cache',()=>({deleteCachedBrowserState:async()=>{}}));
+mockAppModule('@/lib/stores/chat-store',()=>({useChatStore:{getState:()=>({sessions:s.sessions,actions:{patch:(id,update)=>Object.assign(s.sessions[id],update)}})}}));
+mockAppModule('@/lib/api',()=>({localFetch:async(path,init)=>{
+ s.requests.push({path,init});if(path==='/pipes'){if(s.failDiscovery)return Response.json({error:'unavailable'},{status:503});return Response.json({data:s.pipes});}
+ const pipe=s.pipes.find(p=>path===`/pipes/${encodeURIComponent(p.config.name)}/config`);if(!pipe){s.unexpected.push(path);throw Error('Unknown task');}
+ if(s.failPipe===pipe.config.name)return Response.json({success:false},{status:503});Object.assign(pipe.config,JSON.parse(init.body));s.order.push('pipe:'+pipe.config.name);return Response.json({success:true});
+}}));
+mock.module('@tauri-apps/plugin-fs',()=>({
+ exists:async path=>path==='/synthetic/chats'||s.files.has(path),mkdir:async()=>{},
+ readTextFile:async path=>{if(!s.files.has(path))throw Error('missing file');const text=s.files.get(path);if(s.advanceSelectionOnRead){s.advanceSelectionOnRead=false;const newer=JSON.parse(text);newer.presetId='c';newer.rev++;s.files.set(path,JSON.stringify(newer));}return text;},
+ writeTextFile:async(path,text)=>{if(s.failChat)throw Error('synthetic chat failure');s.files.set(path,text);},
+ rename:async(from,to)=>{s.files.set(to,s.files.get(from));s.files.delete(from);s.order.push('chat:'+to);},remove:async path=>{s.files.delete(path);},
+ readDir:async()=>[...s.files.keys()].filter(p=>p.endsWith('.json')).map(p=>({name:posix.basename(p),isFile:true,isDirectory:false})),
+ stat:async()=>({mtime:new Date(100),isFile:true,isDirectory:false})
+}));
+
+
+mock.module('zustand',()=>({create:factory=>{let state;const set=patch=>Object.assign(state,patch);state=factory(set);const hook=()=>state;hook.getState=()=>state;return hook;}}));
+mock.module('lucide-react',()=>({Sparkles:'sparkles',X:'x'}));
+mock.module('gt-react',()=>({useGT:()=>s=>s,msg:s=>s}));
+mock.module('@tauri-apps/plugin-process',()=>({relaunch:async()=>{throw Error('Unexpected relaunch');}}));
+mock.module('@tauri-apps/plugin-updater',()=>({check:async()=>{throw Error('Unexpected update download');}}));
+mockAppModule('@/components/ui/button',()=>({Button:'button'}));
+mockAppModule('@/components/ui/use-toast',()=>({useToast:()=>({toast:x=>s.toasts.push(x)})}));
+mockAppModule('@/lib/utils',()=>({cn:(...args)=>args.join(' ')}));
+mockAppModule('@/lib/enterprise-auth-recovery',()=>({enterpriseUpdateAuthHeaders:()=>({})}));
+const settingsModule=await import('../../../apps/screenpipe-app-tauri/lib/hooks/use-settings');
+const banner=await import('../../../apps/screenpipe-app-tauri/components/update-banner');
+let api;
+const tick=()=>new Promise(resolve=>setTimeout(resolve,0));
+const deferred=()=>{let resolve;const promise=new Promise(r=>resolve=r);return {promise,resolve};};
+function text(node){if(typeof node==='string')return node;if(Array.isArray(node))return node.map(text).join(' ');return node?.props ? text(node.props.children) : '';}
+function restartButton(node=banner.UpdateBanner({compact:true})){
+ if(Array.isArray(node)){for(const child of node){const found=restartButton(child);if(found)return found;}return null;}
+ if(node?.type==='button' && /restart.*update|update.*restart/i.test(text(node)))return node;
+ return node?.props?.children ? restartButton(node.props.children) : null;
+}
+function click(){const button=restartButton();expect(button).not.toBeNull();expect(button.props.disabled).not.toBe(true);return button.props.onClick();}
+beforeEach(async()=>{
+ storage.clear();s={settings:{...clone(settingsModule.createDefaultSettingsObject()),deviceId:'synthetic-device',user:null,autoUpdate:false},diskSettings:null,files:new Map(),sessions:{},pipes:[],requests:[],events:[],unexpected:[],order:[],rendered:null,handoffs:[],toasts:[]};
+ s.diskSettings=clone(s.settings);api=settingsModule.SettingsProvider({children:null}).props.value;
+ await api.reloadStore();await api.updateSettings({autoUpdate:false});
+ Object.assign(banner.useUpdateBanner.getState(),{isVisible:true,updateInfo:{version:'9.9.9',body:'synthetic'},isInstalling:false,authRequired:null});
+});
+afterEach(()=>expect(s.unexpected).toEqual([]));
+test('accepted settings persist before restart handoff',async()=>{
+ const gate=deferred();s.saveGate=gate.promise;const write=api.updateSettings({autoUpdate:true});const restart=click();
+ try {await tick();expect(s.handoffs).toEqual([]);} finally {gate.resolve();await write;await restart;}
+ expect(s.handoffs).toHaveLength(1);expect(s.handoffs[0].autoUpdate).toBe(true);
+});
+test('a second accepted edit during drain also persists before restart',async()=>{
+ const gate=deferred();s.saveGate=gate.promise;const first=api.updateSettings({autoUpdate:true});const restart=click();await tick();const second=api.updateSettings({fontSize:'18px'});
+ gate.resolve();await Promise.all([first,second,restart]);expect(s.handoffs).toHaveLength(1);expect(s.handoffs[0].autoUpdate).toBe(true);expect(s.handoffs[0].fontSize).toBe('18px');
+});
+test('failed persistence refuses restart and allows recovery',async()=>{
+ s.failSettings=true;const write=api.updateSettings({autoUpdate:true}).catch(e=>e);await write;await click();expect(s.handoffs).toEqual([]);expect(restartButton().props.disabled).not.toBe(true);
+ s.failSettings=false;await api.updateSettings({autoUpdate:true});await click();expect(s.handoffs).toHaveLength(1);expect(s.handoffs[0].autoUpdate).toBe(true);
+});
+test('idle restart preserves already saved settings',async()=>{
+ await api.updateSettings({autoUpdate:true,fontSize:'18px'});await click();expect(s.handoffs).toHaveLength(1);expect(s.handoffs[0].autoUpdate).toBe(true);expect(s.handoffs[0].fontSize).toBe('18px');
+});