Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

test(evals): verify stale consumer policy through settings persistence

louis5134f208c1ae
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 23a35767a..45d269373 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -13,6 +13,16 @@ The current app corpus contains 80 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest uses this same harness.
+`app-consumer-stale-managed-settings` runs actual settings read, write and reset
+operations, the write queue and build-authority gate against synthetic native,
+store, React rendering and external ports. Confirmed consumer builds release stale
+policy; enterprise builds and failed IPC preserve it. Twelve outcomes include
+ordinary edits without cached policy. The parent fails three consumer outcomes
+and preserves nine; the reference and current source pass twelve.
+Run `bun test evals/coding-agent/calibrate-stale-policy.test.js` for calibration.
+The task concerns persisted frontend choices, not actual native recording,
+migration authority, store encryption, IPC delivery, isolation or model quality.
+
`app-card-ask-login-hydration` runs the actual provider, controller and dialog
with synthetic external ports. Nine outcomes cover initial and delayed account
loading, post-mount sign-in, once-per-install persistence and suppressed prompts.
diff --git a/evals/coding-agent/calibrate-stale-policy.test.js b/evals/coding-agent/calibrate-stale-policy.test.js
new file mode 100644
index 000000000..e4b6c0500
--- /dev/null
+++ b/evals/coding-agent/calibrate-stale-policy.test.js
@@ -0,0 +1,30 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { tmpdir } from 'node:os';
+const repo=resolve(import.meta.dir,'../..'),prefix='apps/screenpipe-app-tauri/';
+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='app-consumer-stale-managed-settings');
+const paths=['tsconfig.json','lib/hooks/use-settings.tsx','lib/ai-preset-deletion.ts','lib/active-ai-preset.ts','lib/chat-storage.ts','lib/chat-dedup.ts','lib/chat-merge.ts','lib/chat/ephemeral-side-conversation.ts','lib/chat/external-chat-parser.ts','lib/hooks/managed-settings.ts','lib/desktop-remote-control.ts','lib/app-entitlement.ts','lib/free-plan-retention.ts','lib/telemetry-env.ts','lib/utils/font-size.ts','lib/live-views/onboarding-activation.ts','lib/utils/sidebar-nav-layout.ts','lib/utils/chat-title.ts','lib/auth-guard.tsx','lib/utils/tauri.ts','lib/hooks/use-is-enterprise-build.ts','components/settings/settings-write-queue.ts'];
+function sources(ref){const available=new Set(execFileSync('git',['ls-tree','-r','--name-only',ref,prefix],{cwd:repo,encoding:'utf8'}).trim().split('\n'));return Object.fromEntries(paths.filter(p=>available.has(prefix+p)).map(p=>[p,execFileSync('git',['show',`${ref}:${prefix}${p}`],{cwd:repo,encoding:'utf8'})]));}
+const broken=sources(item.base_ref),fixed=sources(item.oracle_ref),current=sources('HEAD');
+const fixtures=['evals/coding-agent/graders/stale-policy.test.js','evals/coding-agent/graders/stale-policy.mjs'];
+const root=mkdtempSync(join(tmpdir(),'stale-policy-calibration-'));afterAll(()=>rmSync(root,{recursive:true,force:true}));
+function grade(name,src){const cwd=join(root,name);for(const [path,source] of Object.entries(src)){const f=join(cwd,prefix,path);mkdirSync(dirname(f),{recursive:true});writeFileSync(f,source);}for(const path of fixtures){const f=join(cwd,path);mkdirSync(dirname(f),{recursive:true});writeFileSync(f,readFileSync(join(repo,path)));}const r=spawnSync(process.execPath,['../../'+fixtures[1]],{cwd:join(cwd,prefix),encoding:'utf8',timeout:45_000,env:{PATH:dirname(process.execPath)}});if(process.env.EVAL_CALIBRATION_RESULTS_DIR){mkdirSync(process.env.EVAL_CALIBRATION_RESULTS_DIR,{recursive:true});writeFileSync(join(process.env.EVAL_CALIBRATION_RESULTS_DIR,name+'.json'),JSON.stringify({status:r.status,signal:r.signal,error:r.error?.message,stdout:r.stdout,stderr:r.stderr},null,2)+'\n');}return r;}
+function fails(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain('expect(received)');expect(r.stderr).not.toContain('Cannot find');expect(r.stderr).not.toContain('TypeError');expect(r.stderr).not.toContain('Unhandled error');}
+function passes(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(0);expect(r.stderr.match(/4 pass/g)).toHaveLength(3);expect(r.stderr).not.toContain('Unhandled error');}
+function replace(s,old,next){expect(s.split(old)).toHaveLength(2);return s.replace(old,next);}
+const settings='lib/hooks/use-settings.tsx',authority='lib/hooks/use-is-enterprise-build.ts';
+test('parent fails three consumer outcomes and preserves nine',()=>{const r=grade('parent',broken);fails(r);expect(r.stderr).toContain('3 fail');expect(r.stderr.match(/4 pass/g)).toHaveLength(2);expect(r.stderr).toContain('1 pass');},60000);
+test('reference passes twelve caller outcomes',()=>passes(grade('reference',fixed)),60000);
+test('current source passes the same persistence contract',()=>passes(grade('current',current)),60000);
+test('equivalent private bindings pass',()=>passes(grade('equivalent',{...fixed,[settings]:fixed[settings].replaceAll('activeManagedValues','effectivePolicyForBuild'),[authority]:fixed[authority].replaceAll('cachedResult','confirmedBuild')})),60000);
+test('unused correct store does not repair real entry point',()=>fails(grade('unused',{...broken,'unused-settings.tsx':fixed[settings]})),60000);
+test('adding only the correct authority helper cannot fix persistence',()=>fails(grade('helper-only',{...broken,[authority]:fixed[authority]})),60000);
+test('blanket consumer classification fails managed preservation',()=>fails(grade('blanket',{...fixed,[settings]:replace(fixed[settings],'return (await isResolvedConsumerBuild()) ? undefined : managed;','return undefined;')})),60000);
+test('failed IPC cannot release managed policy',()=>fails(grade('fail-open',{...fixed,[authority]:replace(fixed[authority],' } catch {\n return false;\n }\n}\n\n/**\n * Tri-state',' } catch {\n return true;\n }\n}\n\n/**\n * Tri-state')})),60000);
+test('dropping durable save fails persistence outcomes',()=>fails(grade('no-save',{...fixed,[settings]:replace(fixed[settings],'\tawait store.save();','\t// no durable save')})),60000);
+test('blanket update refusal fails ordinary edits',()=>fails(grade('no-edit',{...fixed,[settings]:replace(fixed[settings],'const updateSettings = async (updates: Partial<Settings>) => {','const updateSettings = async (updates: Partial<Settings>) => { return;')})),60000);
+test('missing entry point remains an import error',()=>{const src={...fixed};delete src[settings];const r=grade('missing',src);expect(r.status).toBe(1);expect(r.stderr).toContain('Cannot find module');expect(r.stderr.match(/0 pass/g)).toHaveLength(3);expect(r.stderr).not.toContain('expect(received)');},60000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 9fa31cf4d..ab12022e3 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-09-26.4",
+ "dataset_version": "2026-09-26.5",
"suite_type": "regression",
"owner": "screenpipe engineering",
"policy": {
@@ -3608,6 +3608,42 @@
],
"command": "cd packages/ai-gateway && \"${BUN_BIN:-bun}\" test src/test/eval-settlement-replay.test.ts --timeout=10000"
}
+ },
+ {
+ "id": "app-consumer-stale-managed-settings",
+ "name": "Release stale managed settings only on confirmed consumer builds",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "9712ee210f7efc7f85e98bcc1e80c4c4173450dc"
+ },
+ "base_ref": "335ca8d842d1ac0311c038a72a5c3bf084081340",
+ "oracle_ref": "9712ee210f7efc7f85e98bcc1e80c4c4173450dc",
+ "oracle_paths": [
+ "apps/screenpipe-app-tauri/lib/hooks/use-settings.tsx",
+ "apps/screenpipe-app-tauri/lib/hooks/use-is-enterprise-build.ts"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "privacy",
+ "settings"
+ ],
+ "prompt": "Consumer and enterprise installations can share a settings store. A confirmed consumer build currently keeps enforcing stale enterpriseManagedSettings, silently undoing personal choices. Repair actual settings loading, updating and resetting so an authoritative native consumer result releases the stale managed values and durably removes that cached policy. Loading must preserve the existing personal screenshot and font choices. Writes must persist new choices, survive reload and retain unrelated settings. Reset must restore ordinary defaults without restoring the stale policy. An enterprise build or failed native build check must continue enforcing and retaining managed screenshot and font values on read, write and reset; unrelated settings must remain editable. With no cached policy, ordinary edits must still persist in all three situations. Exercise the real settings provider and store actions, their queued disk-write completion and reload using synthetic native, storage and external ports. Do not perform native builds, network operations or actual user-file writes. This case checks frontend persistence and its build-authority gate; it does not authorize recording, migration, changes to native policy ownership or real device enforcement. No helper name or implementation structure is required.",
+ "agent_timeout_seconds": 1200,
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/stale-policy.test.js",
+ "destination_path": "evals/coding-agent/graders/stale-policy.test.js"
+ },
+ {
+ "local_path": "graders/stale-policy.mjs",
+ "destination_path": "evals/coding-agent/graders/stale-policy.mjs"
+ }
+ ],
+ "command": "cd apps/screenpipe-app-tauri && \"${BUN_BIN:-bun}\" ../../evals/coding-agent/graders/stale-policy.mjs"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/stale-policy.mjs b/evals/coding-agent/graders/stale-policy.mjs
new file mode 100644
index 000000000..0f7184859
--- /dev/null
+++ b/evals/coding-agent/graders/stale-policy.mjs
@@ -0,0 +1,13 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { spawnSync } from 'node:child_process';
+import { dirname, resolve } from 'node:path';
+const fixture=resolve(import.meta.dir,'stale-policy.test.js');
+let status=0;
+for(const mode of ['consumer','enterprise','error']) {
+ const result=spawnSync(process.execPath,['test',fixture],{cwd:process.cwd(),encoding:'utf8',timeout:30_000,env:{PATH:dirname(process.execPath),POLICY_MODE:mode}});
+ process.stdout.write(result.stdout??'');process.stderr.write(result.stderr??'');
+ if(result.error||result.signal){console.error('Grader subprocess error:',result.error?.message??result.signal);process.exit(127);}
+ if(result.status!==0)status=result.status??127;
+}
+process.exit(status);
diff --git a/evals/coding-agent/graders/stale-policy.test.js b/evals/coding-agent/graders/stale-policy.test.js
new file mode 100644
index 000000000..8449b2501
--- /dev/null
+++ b/evals/coding-agent/graders/stale-policy.test.js
@@ -0,0 +1,95 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { beforeEach, afterEach, expect, mock, test } from 'bun:test';
+import { posix } from 'node:path';
+import { existsSync } from 'node:fs';
+function mockAppModule(specifier,factory){
+ mock.module(specifier,factory);
+ const stem=new URL('../../../apps/screenpipe-app-tauri/'+specifier.slice(2),import.meta.url).pathname;
+ for(const suffix of ['.ts','.tsx','.js'])if(existsSync(stem+suffix)){mock.module(stem+suffix,factory);break;}
+}
+let s;
+const mode=process.env.POLICY_MODE;
+if(!['consumer','enterprise','error'].includes(mode))throw Error('Missing calibration mode');
+const isConsumer=mode==='consumer';
+const clone=x=>structuredClone(x);
+const storage=new Map();
+globalThis.window={localStorage:{getItem:k=>storage.get(k)??null,setItem:(k,v)=>storage.set(k,String(v)),removeItem:k=>storage.delete(k)},location:{pathname:'/settings'}};
+globalThis.localStorage=window.localStorage;
+globalThis.fetch=async()=>{s.unexpected.push('network');throw Error('Network forbidden');};
+const react={createContext:()=>({Provider:'provider'}),createElement:(type,props,...children)=>({type,props:{...props,children}}),useState:value=>[typeof value==='function'?value():value,v=>{if(typeof v!=="function")s.rendered=v;}],useEffect:()=>{},useRef:value=>({current:value}),useContext:()=>undefined};
+mock.module('react',()=>({...react,default:react}));
+mock.module('react/jsx-dev-runtime',()=>({jsxDEV:(type,props)=>({type,props})}));
+mock.module('react/jsx-runtime',()=>({jsx:(type,props)=>({type,props}),jsxs:(type,props)=>({type,props})}));
+mock.module('gt-react',()=>({msg:s=>s}));
+mock.module('posthog-js',()=>({default:{capture:()=>{},identify:()=>{}}}));
+mock.module('@tauri-apps/api/core',()=>({invoke:async(name)=>{s.unexpected.push('invoke:'+name);throw Error('Unexpected native command');}}));
+mock.module('@tauri-apps/api/path',()=>({homeDir:async()=>'/synthetic',join:async(...parts)=>posix.join(...parts)}));
+mock.module('@tauri-apps/api/app',()=>({getVersion:async()=>'0.0.0'}));
+mock.module('@tauri-apps/plugin-os',()=>({platform:()=>'macos'}));
+mock.module('@tauri-apps/api/event',()=>({emit:async(name)=>{s.events.push(name);},listen:async()=>()=>{}}));
+const store={get:async()=>clone(s.settings),set:async(k,value)=>{s.settings=clone(value);s.order.push('settings:set');},save:async()=>{if(s.failSettings)throw Error('synthetic settings failure');s.diskSettings=clone(s.settings);s.order.push('settings:save');},onKeyChange:()=>()=>{}};
+mock.module('@tauri-apps/plugin-store',()=>({Store:{load:async()=>store}}));
+mockAppModule('@/lib/utils/tauri',()=>({commands:{isEnterpriseBuildCmd:async()=>{if(mode==='error')throw Error('synthetic IPC failure');return mode==='enterprise';},getScreenpipeBaseDir:async()=>({status:'ok',data:'/synthetic'}),getChatsDir:async()=>({status:'ok',data:'/synthetic/chats'}),reencryptStore:async()=>{},getCloudToken:async()=>({status:'ok',data:null}),setCloudToken:async()=>({status:"ok",data:null})}}));
+mockAppModule('@/lib/analytics-id',()=>({cacheAnalyticsId:()=>{},cacheAnalyticsEnabled:()=>{}}));
+mockAppModule('@/lib/analytics/settings-change',()=>({captureSettingsChange:()=>{}}));
+mockAppModule('@/lib/analytics/onboarding-h1-follow-up',()=>({captureOnboardingH1FollowUp:()=>{}}));
+mockAppModule('@/lib/auth-guard',()=>({installAuthInterceptor:()=>()=>{}}));
+
+mockAppModule('@/lib/web-url',()=>({screenpipeWebUrl:path=>'https://screenpipe.invalid'+path}));
+mockAppModule('@/lib/browser-state-cache',()=>({deleteCachedBrowserState:async()=>{}}));
+mockAppModule('@/lib/stores/chat-store',()=>({useChatStore:{getState:()=>({sessions:s.sessions,actions:{patch:(id,update)=>Object.assign(s.sessions[id],update)}})}}));
+mockAppModule('@/lib/api',()=>({configureApi:()=>{},localFetch:async(path,init)=>{
+ s.requests.push({path,init});if(path==='/pipes'){if(s.failDiscovery)return Response.json({error:'unavailable'},{status:503});return Response.json({data:s.pipes});}
+ const pipe=s.pipes.find(p=>path===`/pipes/${encodeURIComponent(p.config.name)}/config`);if(!pipe){s.unexpected.push(path);throw Error('Unknown task');}
+ if(s.failPipe===pipe.config.name)return Response.json({success:false},{status:503});Object.assign(pipe.config,JSON.parse(init.body));s.order.push('pipe:'+pipe.config.name);return Response.json({success:true});
+}}));
+mock.module('@tauri-apps/plugin-fs',()=>({
+ exists:async path=>path==='/synthetic/chats'||s.files.has(path),mkdir:async()=>{},
+ readTextFile:async path=>{if(!s.files.has(path))throw Error('missing file');const text=s.files.get(path);if(s.advanceSelectionOnRead){s.advanceSelectionOnRead=false;const newer=JSON.parse(text);newer.presetId='c';newer.rev++;s.files.set(path,JSON.stringify(newer));}return text;},
+ writeTextFile:async(path,text)=>{if(s.failChat)throw Error('synthetic chat failure');s.files.set(path,text);},
+ rename:async(from,to)=>{s.files.set(to,s.files.get(from));s.files.delete(from);s.order.push('chat:'+to);},remove:async path=>{s.files.delete(path);},
+ readDir:async()=>[...s.files.keys()].filter(p=>p.endsWith('.json')).map(p=>({name:posix.basename(p),isFile:true,isDirectory:false})),
+ stat:async()=>({mtime:new Date(100),isFile:true,isDirectory:false})
+}));
+
+const settingsModule=await import('../../../apps/screenpipe-app-tauri/lib/hooks/use-settings');
+let api;
+const local=()=>({id:'local-model',provider:'native-ollama',model:'synthetic-local',url:'http://localhost:11434',defaultPreset:true,prompt:'preserved instructions',maxContextChars:128000});
+const user=subscribed=>({id:'synthetic-account',email:'[email protected]',cloud_subscribed:subscribed});
+beforeEach(()=>{
+ storage.clear();s={settings:{...clone(settingsModule.createDefaultSettingsObject()),deviceId:'synthetic-device',user:null,aiPresets:[local()]},diskSettings:null,files:new Map(),sessions:{},pipes:[],requests:[],events:[],unexpected:[],order:[],rendered:null};
+ s.diskSettings=clone(s.settings);api=settingsModule.SettingsProvider({children:null}).props.value;
+});
+afterEach(()=>expect(s.unexpected).toEqual([]));
+
+
+const policy={disableScreenshots:true,fontSize:'20px'};
+function seed(){s.settings.enterpriseManagedSettings=clone(policy);s.settings.disableScreenshots=false;s.settings.fontSize='18px';s.settings.port=3845;s.diskSettings=clone(s.settings);}
+async function settle(){await settingsModule.flushPendingSettingsWrites();await api.reloadStore();}
+test('read reconciles cached policy according to authoritative build',async()=>{
+ seed();await api.reloadStore();
+ expect(s.rendered.disableScreenshots).toBe(!isConsumer);
+ expect(s.rendered.fontSize).toBe(isConsumer?'18px':'20px');
+ expect(s.settings.enterpriseManagedSettings).toEqual(isConsumer?undefined:policy);
+ expect(s.diskSettings.enterpriseManagedSettings).toEqual(isConsumer?undefined:policy);
+ expect(s.settings.port).toBe(3845);
+},30000);
+test('write persists chosen consumer values while managed builds retain locks',async()=>{
+ seed();await api.updateSettings({disableScreenshots:false,fontSize:'16px',port:3846});await settle();
+ expect(s.settings.disableScreenshots).toBe(!isConsumer);expect(s.diskSettings.disableScreenshots).toBe(!isConsumer);
+ expect(s.rendered.fontSize).toBe(isConsumer?'16px':'20px');expect(s.diskSettings.port).toBe(3846);
+ expect(s.diskSettings.enterpriseManagedSettings).toEqual(isConsumer?undefined:policy);
+},30000);
+test('reset releases only obsolete consumer policy and preserves managed policy',async()=>{
+ seed();await api.resetSettings();await settle();
+ expect(s.diskSettings.disableScreenshots).toBe(isConsumer?settingsModule.createDefaultSettingsObject().disableScreenshots:true);
+ expect(s.diskSettings.fontSize).toBe(isConsumer?settingsModule.createDefaultSettingsObject().fontSize:'20px');
+ expect(s.diskSettings.enterpriseManagedSettings).toEqual(isConsumer?undefined:policy);
+ expect(s.diskSettings.port).toBe(settingsModule.createDefaultSettingsObject().port);
+},30000);
+test('settings without a cached policy preserve ordinary edits',async()=>{
+ delete s.settings.enterpriseManagedSettings;await api.updateSettings({disableScreenshots:true,fontSize:'20px',port:3847});await settle();
+ expect(s.diskSettings.disableScreenshots).toBe(true);expect(s.diskSettings.fontSize).toBe('20px');expect(s.diskSettings.port).toBe(3847);
+ expect(s.diskSettings.enterpriseManagedSettings).toBeUndefined();
+},30000);