Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

eval: preserve explicit analytics and falsy settings on reload

Louis Beaumontb8504992cd70
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index fac597207..176559578 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -1,5 +1,17 @@
# Coding-agent regression evals
+`app-historical-analytics-false-persistence` mounts the historical settings hook
+with synthetic native storage. Fourteen outcomes cover explicit false, zero and
+empty strings, accepted updates across remount, missing/null defaults, ordinary
+values, empty collections and a custom data directory. The exact parent loses
+ten outcomes and preserves four; the complete historical fix passes fourteen.
+Run `bun test evals/coding-agent/calibrate-settings-falsy.test.js` for controls.
+Fixtures and third-party dependency links appear only at grading. The legacy
+scalar fields/default rules are historical, not current-schema requirements.
+Current provider opt-out, opt-in and managed-policy boundaries were checked
+separately. This does not prove native disk durability, real process restart,
+execution isolation or model improvement.
+
`app-meeting-speaker-source` executes the actual transcript renderer and full
meeting Markdown builder with synthetic chunks. Twelve outcomes distinguish
per-stream generic labels while preserving explicit names, grouping, the gap
diff --git a/evals/coding-agent/calibrate-settings-falsy.test.js b/evals/coding-agent/calibrate-settings-falsy.test.js
new file mode 100644
index 000000000..378aa02f3
--- /dev/null
+++ b/evals/coding-agent/calibrate-settings-falsy.test.js
@@ -0,0 +1,35 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { dirname, join, resolve } from 'node:path';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+const repo=resolve(import.meta.dir,'../..');
+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='app-historical-analytics-false-persistence');
+const hook='screenpipe-app-tauri/lib/hooks/use-settings.tsx';
+const root=mkdtempSync(join(tmpdir(),'settings-falsy-calibration-'));
+const receipts=process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;
+afterAll(()=>rmSync(root,{recursive:true,force:true}));
+const old='storedValue === null || storedValue === undefined \n ? defaultSettingsObject[key as keyof Settings]\n : storedValue';
+function replace(cwd,from,to){const p=join(cwd,hook),text=readFileSync(p,'utf8');expect(text.split(from)).toHaveLength(2);writeFileSync(p,text.replace(from,to));}
+function grade(name,ref=item.oracle_ref,mutate=()=>{}){
+ const cwd=join(root,name);mkdirSync(cwd);mkdirSync(dirname(join(cwd,hook)),{recursive:true});
+ writeFileSync(join(cwd,hook),execFileSync('git',['show',`${ref}:${hook}`],{cwd:repo}));
+ for(const f of item.grader.fixtures){const dest=join(cwd,f.destination_path);mkdirSync(dirname(dest),{recursive:true});writeFileSync(dest,readFileSync(join(import.meta.dir,f.local_path)));}
+ symlinkSync(join(repo,'apps/screenpipe-app-tauri/node_modules'),join(cwd,'screenpipe-app-tauri/node_modules'),'dir');mutate(cwd);
+ const r=spawnSync('/bin/bash',['-c',item.grader.command],{cwd,encoding:'utf8',timeout:60000,maxBuffer:4*1024*1024,env:{PATH:process.env.PATH,HOME:cwd,CI:'true',TZ:'UTC',NO_COLOR:'1'}});
+ if(receipts){mkdirSync(receipts,{recursive:true});for(const ext of ['stdout','stderr'])writeFileSync(join(receipts,`${name}.${ext}`),r[ext]||'');writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message},null,2));}
+ expect(r.error).toBeUndefined();expect(r.signal).toBeNull();return r;
+}
+function passes(r){expect(r.status).toBe(0);expect(r.stdout).toContain('14 passed');expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught/);}
+function fails(r){expect(r.status).toBe(1);expect(r.stderr).toContain('AssertionError');expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/);}
+test('exact parent fails ten outcomes and preserves four',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('10 failed | 4 passed');},60000);
+test('complete historical fix passes fourteen outcomes',()=>passes(grade('reference')),60000);
+test('equivalent nullish fallback passes',()=>passes(grade('equivalent',item.oracle_ref,cwd=>replace(cwd,old,'storedValue ?? defaultSettingsObject[key as keyof Settings]'))),60000);
+test('unused correct hook cannot hide broken active caller',()=>fails(grade('unused',item.base_ref,cwd=>writeFileSync(join(cwd,'unused-correct.tsx'),execFileSync('git',['show',`${item.oracle_ref}:${hook}`],{cwd:repo})))),60000);
+test('blanket defaults lose explicit values',()=>fails(grade('blanket-defaults',item.oracle_ref,cwd=>replace(cwd,old,'defaultSettingsObject[key as keyof Settings]'))),60000);
+test('missing defaults fail preserved behavior',()=>fails(grade('missing-defaults',item.oracle_ref,cwd=>replace(cwd,old,'storedValue'))),60000);
+test('discarded updates fail reload checks',()=>fails(grade('lost-update',item.oracle_ref,cwd=>replace(cwd,'await store!.set(key, value);','void value;'))),60000);
+test('no-op package script cannot bypass outcome grading',()=>{const mutate=cwd=>writeFileSync(join(cwd,'screenpipe-app-tauri/package.json'),JSON.stringify({scripts:{test:'true'}}));passes(grade('noop-reference',item.oracle_ref,mutate));fails(grade('noop-parent',item.base_ref,mutate));},60000);
+test('missing hook is classified as setup failure',()=>{const r=grade('missing-source',item.oracle_ref,cwd=>rmSync(join(cwd,hook)));expect(r.status).toBe(1);expect(r.stdout+r.stderr).toMatch(/Failed to resolve import|Failed to load url/);expect(r.stderr).not.toContain('AssertionError');},60000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index fd3f84d63..faa5192d4 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -6398,6 +6398,49 @@
],
"command": "bun --no-env-file test apps/screenpipe-app-tauri/lib/utils/meeting-speaker-source.eval.test.ts"
}
+ },
+ {
+ "id": "app-historical-analytics-false-persistence",
+ "name": "Preserve explicit falsy preferences across historical settings reload",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "648c89abeb770ab5a2630fa90d9fcd80e1bbc813"
+ },
+ "base_ref": "500c8fc3e658fd8045e19f35b814dc310b5c65a7",
+ "oracle_ref": "648c89abeb770ab5a2630fa90d9fcd80e1bbc813",
+ "oracle_paths": [
+ "screenpipe-app-tauri/lib/hooks/use-settings.tsx"
+ ],
+ "gate": "advisory",
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "privacy",
+ "settings",
+ "historical-contract"
+ ],
+ "prompt": "In this historical app revision, loading saved settings replaces explicit false, zero and empty-string choices with defaults. Repair the actual useSettings hook while retaining its public interface. Loading must preserve false analyticsEnabled, enableFrameCache and isFirstTimeUser; zero fps and aiMaxContextChars; and empty aiModel and customPrompt. Accepted analyticsEnabled=false, fps=0 and customPrompt=\"\" updates must be visible immediately and survive unmount/remount from the synthetic store, preserving unrelated port and model values. Absent or null fields must retain this revision's defaults, including analytics enabled and macOS fps 0.2 and apple-native OCR. Loading explicit preferences, ordinary values, empty ignoredWindows or absent fields must not write the store. Ordinary values and empty ignoredWindows must be preserved. Custom dataDir must still be returned by getDataDir. Tests mount the real React hook with synthetic native storage, platform and path ports; no network or native builds are needed. The storage port verifies accepted values across hook reload, not physical disk durability or native process restart. These scalar fields and default rules belong to this historical revision; do not apply them to a later settings schema. No particular private helper or implementation path is required.",
+ "agent_timeout_seconds": 900,
+ "grader_dependency_links": [
+ {
+ "source_path": "apps/screenpipe-app-tauri/node_modules",
+ "destination_path": "screenpipe-app-tauri/node_modules"
+ }
+ ],
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/settings-falsy.fixture.tsx.txt",
+ "destination_path": "screenpipe-app-tauri/lib/hooks/eval-settings-falsy.test.tsx"
+ },
+ {
+ "local_path": "graders/settings-falsy.config.mjs",
+ "destination_path": "screenpipe-app-tauri/eval-settings-falsy.config.mjs"
+ }
+ ],
+ "command": "cd screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-settings-falsy.config.mjs"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/settings-falsy.config.mjs b/evals/coding-agent/graders/settings-falsy.config.mjs
new file mode 100644
index 000000000..05d14404f
--- /dev/null
+++ b/evals/coding-agent/graders/settings-falsy.config.mjs
@@ -0,0 +1,3 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+export default {test:{environment:'jsdom',include:['lib/hooks/eval-settings-falsy.test.tsx'],pool:'forks',maxWorkers:1,minWorkers:1}};
diff --git a/evals/coding-agent/graders/settings-falsy.fixture.tsx.txt b/evals/coding-agent/graders/settings-falsy.fixture.tsx.txt
new file mode 100644
index 000000000..89c704edc
--- /dev/null
+++ b/evals/coding-agent/graders/settings-falsy.fixture.tsx.txt
@@ -0,0 +1,22 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import {test,expect,vi,beforeEach,afterEach} from 'vitest';
+import {renderHook,act,waitFor,cleanup} from '@testing-library/react';
+const s=vi.hoisted(()=>({values:{} as Record<string,any>,writes:[]as any[],saves:0}));
+vi.mock('@tauri-apps/plugin-store',()=>({createStore:async()=>({get:async(k:string)=>structuredClone(s.values[k]),set:async(k:string,v:any)=>{s.writes.push([k,structuredClone(v)]);s.values[k]=structuredClone(v)},save:async()=>{s.saves++}})}));
+vi.mock('@tauri-apps/api/path',()=>({localDataDir:async()=>'/synthetic/data',join:async(...v:string[])=>v.join('/'),homeDir:async()=>'/synthetic/home'}));
+vi.mock('@tauri-apps/plugin-os',()=>({platform:()=>'macos'}));
+vi.mock('posthog-js',()=>({default:{identify:vi.fn()}}));
+import {useSettings}from './use-settings';
+beforeEach(()=>{s.values={};s.writes=[];s.saves=0;vi.stubGlobal('fetch',()=>{throw Error('network forbidden')})});afterEach(()=>{cleanup();vi.unstubAllGlobals()});
+async function loaded(){const h=renderHook(()=>useSettings());await waitFor(()=>expect(h.result.current.settings.isLoading).toBe(false));return h}
+for(const[key,value]of Object.entries({analyticsEnabled:false,enableFrameCache:false,isFirstTimeUser:false,fps:0,aiMaxContextChars:0,aiModel:'',customPrompt:''})){
+ test(`loading preserves explicit ${key}=${JSON.stringify(value)}`,async()=>{s.values={[key]:value};const h=await loaded();expect(h.result.current.settings[key]).toBe(value);expect(s.writes).toEqual([])})
+}
+test('missing fields use defaults and platform defaults',async()=>{const h=await loaded();expect(h.result.current.settings.analyticsEnabled).toBe(true);expect(h.result.current.settings.fps).toBe(0.2);expect(h.result.current.settings.ocrEngine).toBe('apple-native');expect(s.writes).toEqual([])});
+test('null fields use defaults',async()=>{s.values={analyticsEnabled:null,fps:null};const h=await loaded();expect(h.result.current.settings.analyticsEnabled).toBe(true);expect(h.result.current.settings.fps).toBe(0.2)});
+test('ordinary values and empty collections remain intact',async()=>{s.values={fps:2,aiModel:'synthetic-model',devMode:true,ignoredWindows:[]};const h=await loaded();expect(h.result.current.settings).toMatchObject(s.values)});
+for(const[key,value,old]of [['analyticsEnabled',false,true],['fps',0,2],['customPrompt','','synthetic prior']]as const){
+ test(`accepted ${key} update survives hook remount and preserves unrelated preference`,async()=>{s.values={[key]:old,port:4242,aiModel:'synthetic-model'};const h=await loaded();await act(async()=>{await h.result.current.updateSettings({[key]:value})});expect(h.result.current.settings[key]).toBe(value);h.unmount();const reloaded=await loaded();expect(reloaded.result.current.settings[key]).toBe(value);expect(reloaded.result.current.settings.port).toBe(4242);expect(reloaded.result.current.settings.aiModel).toBe('synthetic-model')})
+}
+test('custom data directory remains available',async()=>{s.values={dataDir:'/synthetic/custom'};const h=await loaded();expect(await h.result.current.getDataDir()).toBe('/synthetic/custom')});