YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: preserve local-only presets through account refresh
louis413c3d759474
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 02aff84b8..0365200d0 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -399,3 +399,23 @@ names pass; missing source remains setup failure. The grader is installed onlyafter an evaluated trajectory. Settings, managed policy, analytics and nativecommands are synthetic. Native auth, recorder durability, persisted enterprisepolicy, live downloads and execution isolation are outside this UI evidence.+++## Local-only preset preservation++`app-local-preset-preservation` executes the actual settings provider, shared+settings store and write queue against synthetic native storage and account data.+Twelve outcomes cover cold loads, repeated reloads, sign-in/subscription/sign-out+transitions, malformed saved data, invalid mutations, mixed configurations and+preserved explicit edits. The parent fails eight assertions and preserves four+outcomes; the historical reference and selected current code pass twelve.+Only the settings module is applied as the oracle.++Run `bun test evals/coding-agent/calibrate-local-presets.test.js` to check known+correct/broken, unused-helper, cloud-seeding, account-reset, silent-repair and+blanket-refusal controls, equivalent helper naming and missing-source errors.+The standalone grader needs no installed packages. React hooks and native,+account, event and analytics ports are synthetic; mount effects do not execute.+It does not prove rendered deletion controls, live account refresh, native+durability, enforced isolation or model performance. Preset dependency deletion+recovery remains a separate case.diff --git a/evals/coding-agent/calibrate-local-presets.test.js b/evals/coding-agent/calibrate-local-presets.test.jsnew file mode 100644index 000000000..53bf17461--- /dev/null+++ b/evals/coding-agent/calibrate-local-presets.test.js@@ -0,0 +1,31 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { dirname, join, resolve } from 'node:path';+import { tmpdir } from 'node:os';+const repo=resolve(import.meta.dir,'../..'),prefix='apps/screenpipe-app-tauri/';+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='app-local-preset-preservation');+const paths=['tsconfig.json','lib/hooks/use-settings.tsx','lib/ai-preset-deletion.ts','lib/active-ai-preset.ts','lib/chat-storage.ts','lib/chat-dedup.ts','lib/chat-merge.ts','lib/chat/ephemeral-side-conversation.ts','lib/chat/external-chat-parser.ts','lib/hooks/managed-settings.ts','lib/desktop-remote-control.ts','lib/app-entitlement.ts','lib/telemetry-env.ts','lib/utils/font-size.ts','lib/live-views/onboarding-activation.ts','lib/utils/sidebar-nav-layout.ts','lib/utils/chat-title.ts','lib/auth-guard.tsx','lib/utils/tauri.ts','lib/hooks/use-is-enterprise-build.ts','components/settings/settings-write-queue.ts'];+function sources(ref){const available=new Set(execFileSync('git',['ls-tree','-r','--name-only',ref,prefix],{cwd:repo,encoding:'utf8'}).trim().split('\n'));return Object.fromEntries(paths.filter(p=>available.has(prefix+p)).map(p=>[p,execFileSync('git',['show',`${ref}:${prefix}${p}`],{cwd:repo,encoding:'utf8'})]));}+const broken=sources(item.base_ref),fixed=sources(item.oracle_ref);+const fixture='evals/coding-agent/graders/local-preset-preservation.test.js',grader=readFileSync(join(repo,fixture));+const root=mkdtempSync(join(tmpdir(),'local-presets-calibration-'));afterAll(()=>rmSync(root,{recursive:true,force:true}));+function grade(name,sources){const cwd=join(root,name);for(const [path,source] of Object.entries(sources)){const file=join(cwd,prefix,path);mkdirSync(dirname(file),{recursive:true});writeFileSync(file,source);}mkdirSync(dirname(join(cwd,fixture)),{recursive:true});writeFileSync(join(cwd,fixture),grader);return spawnSync(process.execPath,['test','../../'+fixture],{cwd:join(cwd,prefix),encoding:'utf8',timeout:30_000,env:{PATH:dirname(process.execPath)}});}+function fails(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain('expect(received)');expect(r.stderr).not.toContain('Cannot find module');expect(r.stderr).not.toContain('Unhandled error');}+function replace(source,old,next){expect(source.split(old)).toHaveLength(2);return source.replace(old,next);}++const settings='lib/hooks/use-settings.tsx';+test('parent reproduces settings failures and preserves neighbors',()=>{const r=grade('parent',broken);fails(r);expect(r.stderr).toContain('8 fail');expect(r.stderr).toContain('4 pass');});+test('reference passes persistence and recovery outcomes',()=>{const r=grade('reference',fixed);expect(r.status).toBe(0);expect(r.stderr).toContain('12 pass');});+test('equivalent validation helper naming is accepted',()=>expect(grade('renamed',{...fixed,[settings]:fixed[settings].replaceAll('assertValidAiPresetUpdate','checkPresetShape')}).status).toBe(0));+test('correct unused normalization cannot hide a broken persistence path',()=>{+ const start=fixed[settings].indexOf('export function normalizeSettingsArrays('),end=fixed[settings].indexOf('// Store singleton',start);+ const unused=fixed[settings].slice(start,end).replaceAll('normalizeSettingsArrays','unusedNormalization').replaceAll('assertValidAiPresetUpdate','unusedValidation');fails(grade('unused',{...broken,[settings]:broken[settings]+'\n'+unused}));+});+test('reintroduced cloud seeding is rejected',()=>fails(grade('cloud-seed',{...fixed,[settings]:replace(fixed[settings],'let needsUpdate = normalizeSettingsArrays(settings);','let needsUpdate = normalizeSettingsArrays(settings); if (!settings.aiPresets.some(p => p.provider === "screenpipe-cloud")) { settings.aiPresets.unshift(makeDefaultPresets(false)[0]); needsUpdate = true; }')})));+test('silently repairing invalid writes instead of rejecting is rejected',()=>fails(grade('repair-write',{...fixed,[settings]:fixed[settings].replace('assertValidAiPresetUpdate(updates);','if ("aiPresets" in updates && (!Array.isArray(updates.aiPresets) || !updates.aiPresets.length)) updates.aiPresets = makeDefaultPresets(false);')})));+test('account changes cannot replace local settings with defaults',()=>fails(grade('account-defaults',{...fixed,[settings]:replace(fixed[settings],'if ("user" in value) {','if ("user" in value) { newSettings.aiPresets = makeDefaultPresets(false);')})));+test('blanket update refusal fails preserved edits',()=>{const r=grade('deny-all',{...fixed,[settings]:replace(fixed[settings],'assertValidAiPresetUpdate(updates);','throw new Error("synthetic refusal");')});expect(r.status).toBe(1);expect(r.signal).toBeNull();expect(r.stderr).toContain("3 fail");expect(r.stderr).toContain("9 pass");expect(r.stderr).toContain("error: synthetic refusal");expect(r.stderr).not.toContain("Unhandled error");});+test('missing actual settings module is setup failure',()=>{const source={...fixed};delete source[settings];const r=grade('missing',source);expect(r.status).toBe(1);expect(r.stderr).toContain('Cannot find module');expect(r.stderr).toContain('0 pass');});diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex df69e67ac..6b9408cab 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-09-20.5",+ "dataset_version": "2026-09-20.6","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -1809,6 +1809,38 @@],"command": "cd apps/screenpipe-app-tauri && NODE_OPTIONS=--localstorage-file=.eval-localstorage ./node_modules/.bin/vitest run --config vitest.config.ts components/restricted-onboarding.eval.test.tsx --maxWorkers=1 --minWorkers=1"}+ },+ {+ "id": "app-local-preset-preservation",+ "name": "Preserve local-only presets through settings and account refresh",+ "source": {+ "kind": "git_regression",+ "fix_commit": "5c3661a0b979e5cd6204ac20fd8872a274efe47a"+ },+ "base_ref": "45778f74227cbff7a83f65265e834b5985f7e72b",+ "oracle_ref": "5c3661a0b979e5cd6204ac20fd8872a274efe47a",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/lib/hooks/use-settings.tsx"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "settings",+ "account",+ "local-model"+ ],+ "prompt": "A local-only AI preset configuration unexpectedly gains a cloud preset when settings reload or the account changes. Fix the actual settings persistence path so existing nonempty configurations, including custom model, prompt, URL, default selection and ordering, survive cold loads, repeated reloads, sign-in, subscription changes and sign-out unchanged. Explicit valid preset edits and unrelated settings changes must still persist, and intentional mixed cloud/local configurations must remain intact. Persisted missing, malformed or empty preset lists must recover usable defaults. Explicit empty or malformed preset updates must reject before any persistence or events, leaving prior settings intact. Preserve existing managed-policy behavior. No particular helper or internal structure is required. Rendered deletion controls, native durability and account network verification are outside this task.",+ "agent_timeout_seconds": 1200,+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/local-preset-preservation.test.js",+ "destination_path": "evals/coding-agent/graders/local-preset-preservation.test.js"+ }+ ],+ "command": "cd apps/screenpipe-app-tauri && \"${BUN_BIN:-bun}\" test ../../evals/coding-agent/graders/local-preset-preservation.test.js"+ }}]}diff --git a/evals/coding-agent/graders/local-preset-preservation.test.js b/evals/coding-agent/graders/local-preset-preservation.test.jsnew file mode 100644index 000000000..e7040cf9c--- /dev/null+++ b/evals/coding-agent/graders/local-preset-preservation.test.js@@ -0,0 +1,78 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { beforeEach, afterEach, expect, mock, test } from 'bun:test';+import { posix } from 'node:path';+import { existsSync } from 'node:fs';+function mockAppModule(specifier,factory){+ mock.module(specifier,factory);+ const stem=new URL('../../../apps/screenpipe-app-tauri/'+specifier.slice(2),import.meta.url).pathname;+ for(const suffix of ['.ts','.tsx','.js'])if(existsSync(stem+suffix)){mock.module(stem+suffix,factory);break;}+}+let s;+const clone=x=>structuredClone(x);+const storage=new Map();+globalThis.window={localStorage:{getItem:k=>storage.get(k)??null,setItem:(k,v)=>storage.set(k,String(v)),removeItem:k=>storage.delete(k)},location:{pathname:'/settings'}};+globalThis.localStorage=window.localStorage;+globalThis.fetch=async()=>{s.unexpected.push('network');throw Error('Network forbidden');};+const react={createContext:()=>({Provider:'provider'}),createElement:(type,props,...children)=>({type,props:{...props,children}}),useState:value=>[typeof value==='function'?value():value,v=>{if(typeof v!=="function")s.rendered=v;}],useEffect:()=>{},useRef:value=>({current:value}),useContext:()=>undefined};+mock.module('react',()=>({...react,default:react}));+mock.module('react/jsx-dev-runtime',()=>({jsxDEV:(type,props)=>({type,props})}));+mock.module('react/jsx-runtime',()=>({jsx:(type,props)=>({type,props}),jsxs:(type,props)=>({type,props})}));+mock.module('posthog-js',()=>({default:{capture:()=>{},identify:()=>{}}}));+mock.module('@tauri-apps/api/core',()=>({invoke:async(name)=>{s.unexpected.push('invoke:'+name);throw Error('Unexpected native command');}}));+mock.module('@tauri-apps/api/path',()=>({homeDir:async()=>'/synthetic',join:async(...parts)=>posix.join(...parts)}));+mock.module('@tauri-apps/api/app',()=>({getVersion:async()=>'0.0.0'}));+mock.module('@tauri-apps/plugin-os',()=>({platform:()=>'macos'}));+mock.module('@tauri-apps/api/event',()=>({emit:async(name)=>{s.events.push(name);},listen:async()=>()=>{}}));+const store={get:async()=>clone(s.settings),set:async(k,value)=>{s.settings=clone(value);s.order.push('settings:set');},save:async()=>{if(s.failSettings)throw Error('synthetic settings failure');s.diskSettings=clone(s.settings);s.order.push('settings:save');},onKeyChange:()=>()=>{}};+mock.module('@tauri-apps/plugin-store',()=>({Store:{load:async()=>store}}));+mockAppModule('@/lib/utils/tauri',()=>({commands:{getScreenpipeBaseDir:async()=>({status:'ok',data:'/synthetic'}),getChatsDir:async()=>({status:'ok',data:'/synthetic/chats'}),reencryptStore:async()=>{},getCloudToken:async()=>({status:'ok',data:null}),setCloudToken:async()=>({status:"ok",data:null})}}));+mockAppModule('@/lib/analytics-id',()=>({cacheAnalyticsId:()=>{},cacheAnalyticsEnabled:()=>{}}));+mockAppModule('@/lib/analytics/settings-change',()=>({captureSettingsChange:()=>{}}));+mockAppModule('@/lib/analytics/onboarding-h1-follow-up',()=>({captureOnboardingH1FollowUp:()=>{}}));+mockAppModule('@/lib/auth-guard',()=>({installAuthInterceptor:()=>()=>{}}));+mockAppModule('@/lib/hooks/use-is-enterprise-build',()=>({isResolvedConsumerBuild:async()=>true}));+mockAppModule('@/lib/web-url',()=>({screenpipeWebUrl:path=>'https://screenpipe.invalid'+path}));+mockAppModule('@/lib/browser-state-cache',()=>({deleteCachedBrowserState:async()=>{}}));+mockAppModule('@/lib/stores/chat-store',()=>({useChatStore:{getState:()=>({sessions:s.sessions,actions:{patch:(id,update)=>Object.assign(s.sessions[id],update)}})}}));+mockAppModule('@/lib/api',()=>({localFetch:async(path,init)=>{+ s.requests.push({path,init});if(path==='/pipes'){if(s.failDiscovery)return Response.json({error:'unavailable'},{status:503});return Response.json({data:s.pipes});}+ const pipe=s.pipes.find(p=>path===`/pipes/${encodeURIComponent(p.config.name)}/config`);if(!pipe){s.unexpected.push(path);throw Error('Unknown task');}+ if(s.failPipe===pipe.config.name)return Response.json({success:false},{status:503});Object.assign(pipe.config,JSON.parse(init.body));s.order.push('pipe:'+pipe.config.name);return Response.json({success:true});+}}));+mock.module('@tauri-apps/plugin-fs',()=>({+ exists:async path=>path==='/synthetic/chats'||s.files.has(path),mkdir:async()=>{},+ readTextFile:async path=>{if(!s.files.has(path))throw Error('missing file');const text=s.files.get(path);if(s.advanceSelectionOnRead){s.advanceSelectionOnRead=false;const newer=JSON.parse(text);newer.presetId='c';newer.rev++;s.files.set(path,JSON.stringify(newer));}return text;},+ writeTextFile:async(path,text)=>{if(s.failChat)throw Error('synthetic chat failure');s.files.set(path,text);},+ rename:async(from,to)=>{s.files.set(to,s.files.get(from));s.files.delete(from);s.order.push('chat:'+to);},remove:async path=>{s.files.delete(path);},+ readDir:async()=>[...s.files.keys()].filter(p=>p.endsWith('.json')).map(p=>({name:posix.basename(p),isFile:true,isDirectory:false})),+ stat:async()=>({mtime:new Date(100),isFile:true,isDirectory:false})+}));++const settingsModule=await import('../../../apps/screenpipe-app-tauri/lib/hooks/use-settings');+let api;+const local=()=>({id:'local-model',provider:'native-ollama',model:'synthetic-local',url:'http://localhost:11434',defaultPreset:true,prompt:'preserved instructions',maxContextChars:128000});+const user=subscribed=>({id:'synthetic-account',email:'[email protected]',cloud_subscribed:subscribed});+beforeEach(()=>{+ storage.clear();s={settings:{...clone(settingsModule.createDefaultSettingsObject()),deviceId:'synthetic-device',user:null,aiPresets:[local()]},diskSettings:null,files:new Map(),sessions:{},pipes:[],requests:[],events:[],unexpected:[],order:[],rendered:null};+ s.diskSettings=clone(s.settings);api=settingsModule.SettingsProvider({children:null}).props.value;+});+afterEach(()=>expect(s.unexpected).toEqual([]));+for(const account of [null,false,true])test(`cold settings load preserves local-only configuration for account ${account}`,async()=>{+ s.settings.user=account===null?null:user(account);const expected=clone(s.settings.aiPresets);await api.reloadStore();expect(s.rendered.aiPresets).toEqual(expected);expect(s.settings.aiPresets).toEqual(expected);await api.reloadStore();expect(s.rendered.aiPresets).toEqual(expected);+});+test('sign-in subscription refresh and sign-out retain the chosen local presets',async()=>{+ const expected=clone(s.settings.aiPresets);for(const account of [user(false),user(true),user(false),null]){await api.updateSettings({user:account});await api.reloadStore();expect(s.settings.aiPresets).toEqual(expected);expect(s.rendered.aiPresets).toEqual(expected);}+});+for(const invalid of [[],null,'malformed'])test(`invalid preset mutation ${JSON.stringify(invalid)} rejects before persistence`,async()=>{+ await api.reloadStore();const before=clone(s.settings);s.order=[];s.events=[];await expect(api.updateSettings({aiPresets:invalid})).rejects.toThrow();expect(s.settings).toEqual(before);expect(s.order).toEqual([]);expect(s.events).toEqual([]);+});+for(const malformed of [[],null,'malformed'])test(`cold malformed presets ${JSON.stringify(malformed)} recover usable defaults`,async()=>{+ s.settings.aiPresets=malformed;await api.reloadStore();expect(Array.isArray(s.rendered.aiPresets)).toBe(true);expect(s.rendered.aiPresets.length).toBeGreaterThan(0);expect(s.rendered.aiPresets.some(p=>p.defaultPreset&&p.model&&p.provider)).toBe(true);await api.reloadStore();expect(s.rendered.aiPresets).toEqual(s.settings.aiPresets);+});+test('intentional mixed cloud and local presets survive reload and unrelated saves',async()=>{+ const cloud={id:'chosen-cloud',provider:'screenpipe-cloud',model:'synthetic-cloud',defaultPreset:false,prompt:'cloud instructions',maxContextChars:64000};s.settings.aiPresets.push(cloud);const expected=clone(s.settings.aiPresets);await api.reloadStore();await api.updateSettings({fontSize:'18px'});await api.reloadStore();expect(s.rendered.aiPresets).toEqual(expected);expect(s.settings.fontSize).toBe('18px');+});+test('explicit preset edits and ordering persist without cloud insertion',async()=>{+ const a=local(),b={...local(),id:'second-local',model:'different-local',defaultPreset:false};await api.updateSettings({aiPresets:[b,a]});await api.reloadStore();expect(s.settings.aiPresets).toEqual([b,a]);expect(s.rendered.aiPresets).toEqual([b,a]);+});