YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: keep sign-out authoritative over pending account refresh
Louis Beaumontae84199d25c4
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 3030f6869..f93eba9f1 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -67,7 +67,7 @@ The hidden fixture appears only at grading and uses no dependency links.This does not establish native recording continuity, real authentication,account-switch race safety, browser integration, isolation or model performance.-The current app corpus contains 138 git-mined regressions. See+The current app corpus contains 139 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.@@ -1755,3 +1755,16 @@ for broken/correct/equivalent, unused-source, disconnected-caller, refusal,preserved-behavior and missing-source controls. Fixtures and runtime links appearonly at grading. Native file access/decoding, playback, Markdown filtering,cache policy, enforced execution isolation and model performance are unverified.++## Sign-out during account refresh++`app-auth-logout-pending-load` mounts the actual SettingsProvider with synthetic+HTTP, native event/storage/token, analytics and translation ports. Six outcomes+cover local and peer-window sign-out, fresh sign-in, explicit verification,+post-sign-out sign-in and unrelated preference updates. The parent restores a+signed-out session in two cases while preserving four; the historical fix and+current provider pass six. Run `bun test evals/coding-agent/calibrate-auth-logout.test.js`+for correct, broken, equivalent, unused-source, refusal and preserved controls.+This differs from `app-auth-verification-schedule`, which mocks the provider and+checks AuthGuard scheduling. Actual multi-window delivery, native credential+durability, provider revocation, enforced isolation and model gains are unverified.diff --git a/evals/coding-agent/calibrate-auth-logout.test.js b/evals/coding-agent/calibrate-auth-logout.test.jsnew file mode 100644index 000000000..97792ee44--- /dev/null+++ b/evals/coding-agent/calibrate-auth-logout.test.js@@ -0,0 +1,53 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import {afterAll,expect,test} from 'bun:test';+import {execFileSync,spawnSync} from 'node:child_process';+import {mkdirSync,mkdtempSync,readFileSync,writeFileSync,symlinkSync,rmSync} from 'node:fs';+import {dirname,join,resolve} from 'node:path';+import {tmpdir} from 'node:os';+const repo=resolve(import.meta.dir,'../..'),app='apps/screenpipe-app-tauri';+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='app-auth-logout-pending-load');+const root=mkdtempSync(join(tmpdir(),'auth-logout-calibration-'));+const receipts=process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;+const provider=`${app}/lib/hooks/use-settings.tsx`;+afterAll(()=>rmSync(root,{recursive:true,force:true}));+function replace(cwd,file,from,to){const p=join(cwd,file),s=readFileSync(p,'utf8');expect(s.split(from)).toHaveLength(2);writeFileSync(p,s.replace(from,to));}+function grade(name,ref=item.oracle_ref,mutate=()=>{}){+ const cwd=join(root,name);mkdirSync(cwd);+ // Exact historical sources only. External dependencies are linked at grading.+ const paths=[`${app}/lib`,`${app}/types`,`${app}/components`,`${app}/tsconfig.json`].filter(path=>execFileSync('git',['ls-tree',ref,'--',path],{cwd:repo,encoding:'utf8'}).trim());+ const archive=execFileSync('git',['archive',ref,'--',...paths],{cwd:repo,maxBuffer:64*1024*1024});+ execFileSync('tar',['-x','-C',cwd],{input:archive});+ for(const f of item.grader.fixtures){const dest=join(cwd,f.destination_path);mkdirSync(dirname(dest),{recursive:true});writeFileSync(dest,readFileSync(join(import.meta.dir,f.local_path)));}+ for(const link of item.grader_dependency_links)symlinkSync(join(repo,link.source_path),join(cwd,link.destination_path),'dir');+ mutate(cwd);+ const r=spawnSync('/bin/bash',['-c',item.grader.command],{cwd,encoding:'utf8',timeout:60000,maxBuffer:4*1024*1024,env:{PATH:process.env.PATH,HOME:cwd,CI:'true',TZ:'UTC',NO_COLOR:'1'}});+ if(receipts){mkdirSync(receipts,{recursive:true});writeFileSync(join(receipts,name+'.stdout'),r.stdout||'');writeFileSync(join(receipts,name+'.stderr'),r.stderr||'');writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message},null,2));}+ expect(r.error).toBeUndefined();expect(r.signal).toBeNull();return r;+}+function passes(r){expect(r.status).toBe(0);expect(r.stdout).toContain('6 passed');expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import/);}+function fails(r){expect(r.status).toBe(1);expect(r.stderr).toMatch(/AssertionError|Error: expect\(/);expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Cannot find module/);}+test('parent restores two signed-out sessions and preserves four outcomes',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('2 failed | 4 passed');},60000);+test('historical fix passes',()=>passes(grade('reference')),60000);+test('current provider passes',()=>passes(grade('current','HEAD')),60000);+test('equivalent private state name passes',()=>passes(grade('equivalent',item.oracle_ref,cwd=>{+ const p=join(cwd,provider);writeFileSync(p,readFileSync(p,'utf8').replaceAll('authGenerationRef','sessionEpoch'));+})),60000);+test('unused corrected provider cannot mask active broken code',()=>fails(grade('unused',item.base_ref,cwd=>{+ writeFileSync(join(cwd,`${app}/unused-provider.tsx`),execFileSync('git',['show',`${item.oracle_ref}:${provider}`],{cwd:repo}));+})),60000);+test('disconnected response fence is rejected',()=>fails(grade('disconnected',item.oracle_ref,cwd=>{+ replace(cwd,provider,'if (authGenerationRef.current !== generation) {','if (false) {');+})),60000);+test('ignoring peer notifications is rejected',()=>fails(grade('peer',item.oracle_ref,cwd=>{+ replace(cwd,provider,'listen("screenpipe-auth-signout",','listen("unrelated-event",');+})),60000);+test('blanket account refusal is rejected',()=>fails(grade('refusal',item.oracle_ref,cwd=>{+ replace(cwd,provider,'const generation = authGenerationRef.current;','return; const generation = authGenerationRef.current;');+})),60000);+test('unrelated settings changes must not invalidate sign-in',()=>fails(grade('preferences',item.oracle_ref,cwd=>{+ replace(cwd,provider,'if ("user" in updates && !updates.user) {','if (true) {');+})),60000);+test('missing provider is an infrastructure failure',()=>{+ const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,provider)));expect(r.status).toBe(1);expect(r.stdout+r.stderr).toMatch(/Failed to resolve import|Failed to load url/);expect(r.stderr).not.toMatch(/AssertionError/);+},60000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex c813f1b94..c21704cca 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-10-07-media-file-url",+ "dataset_version": "2026-10-07-auth-logout","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -5744,6 +5744,55 @@],"command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-media-file-url.config.mjs"}+ },+ {+ "id": "app-auth-logout-pending-load",+ "name": "Keep sign-out authoritative over a pending account refresh",+ "source": {+ "kind": "git_regression",+ "fix_commit": "b19f9c2ae6a55994f5b1057c11ed89c7522e9304"+ },+ "base_ref": "32a3db993ea7ac0dc71a7805f20f875ca1281567",+ "oracle_ref": "b19f9c2ae6a55994f5b1057c11ed89c7522e9304",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/lib/hooks/use-settings.tsx"+ ],+ "trigger_paths": [+ "apps/screenpipe-app-tauri/lib/hooks/use-settings.tsx"+ ],+ "gate": "advisory",+ "tags": [+ "regression",+ "escaped-failure",+ "authentication",+ "preserved-behavior"+ ],+ "prompt": "A slow successful account refresh can restore a session after the user signs out. Repair the actual SettingsProvider account-loading flow. Once this window explicitly clears its user, a previously pending account response must not persist that user or reinstall its native cloud token. The same must hold when another window clears the shared user and delivers the existing screenpipe-auth-signout notification. Preserve successful fresh sign-in: save the returned identity and pass the supplied token to the existing native token setter. Preserve explicit verification requests, a fresh sign-in begun after completed sign-out, and successful pending refreshes across unrelated preference changes. Keep the existing provider, account-request and native command contracts compatible. No particular implementation is required. Grading mounts the provider with synthetic HTTP, native event/storage/token and translation ports; real multi-window event delivery, native credential durability, identity-provider revocation and execution isolation are separate.",+ "agent_timeout_seconds": 900,+ "grader_dependency_links": [+ {+ "source_path": "apps/screenpipe-app-tauri/node_modules",+ "destination_path": "apps/screenpipe-app-tauri/node_modules"+ }+ ],+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/auth-logout.fixture.tsx.txt",+ "destination_path": "apps/screenpipe-app-tauri/eval-auth-logout.test.tsx"+ },+ {+ "local_path": "graders/auth-logout.config.mjs",+ "destination_path": "apps/screenpipe-app-tauri/eval-auth-logout.config.mjs"+ },+ {+ "local_path": "graders/auth-logout-translation.fixture.ts.txt",+ "destination_path": "apps/screenpipe-app-tauri/eval-auth-translation.ts"+ }+ ],+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-auth-logout.config.mjs"+ }}]}diff --git a/evals/coding-agent/graders/auth-logout-translation.fixture.ts.txt b/evals/coding-agent/graders/auth-logout-translation.fixture.ts.txtnew file mode 100644index 000000000..9d4cc5e88--- /dev/null+++ b/evals/coding-agent/graders/auth-logout-translation.fixture.ts.txt@@ -0,0 +1,3 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+export const msg=(s:string)=>s;export const useGT=()=>((s:string)=>s);export const useMessages=()=>((s:string)=>s);export const T=({children}:any)=>children;diff --git a/evals/coding-agent/graders/auth-logout.config.mjs b/evals/coding-agent/graders/auth-logout.config.mjsnew file mode 100644index 000000000..26adfcaad--- /dev/null+++ b/evals/coding-agent/graders/auth-logout.config.mjs@@ -0,0 +1,3 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import {defineConfig} from 'vitest/config';import {resolve} from 'node:path';export default defineConfig({resolve:{alias:{'@':resolve(process.cwd()),'gt-react':resolve(process.cwd(),'eval-auth-translation.ts')}},test:{environment:'jsdom',include:['eval-auth-logout.test.tsx'],pool:'forks',maxWorkers:1,minWorkers:1},esbuild:{jsx:'automatic'}});diff --git a/evals/coding-agent/graders/auth-logout.fixture.tsx.txt b/evals/coding-agent/graders/auth-logout.fixture.tsx.txtnew file mode 100644index 000000000..6212855d6--- /dev/null+++ b/evals/coding-agent/graders/auth-logout.fixture.tsx.txt@@ -0,0 +1,36 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import React from 'react';+import {render,act,waitFor,cleanup} from '@testing-library/react';+import {it,expect,vi,afterEach,beforeEach} from 'vitest';+const state=vi.hoisted(()=>({settings:null as any, runtime:Promise.resolve(false),version:Promise.resolve('0.0.0'),identify:vi.fn(),alias:vi.fn(),optOut:vi.fn(),unexpected:[] as string[],listeners:new Map<string,Function>(),native:vi.fn()}));+vi.mock('posthog-js',()=>({default:{identify:state.identify,alias:state.alias,opt_out_capturing:state.optOut,capture:vi.fn()}}));+vi.mock('@tauri-apps/api/core',()=>({invoke:(name:string)=>{if(name==='is_telemetry_disabled_by_env')return state.runtime;state.unexpected.push(name);throw Error('unexpected native command');}}));+vi.mock('@tauri-apps/api/app',()=>({getVersion:()=>state.version}));+vi.mock('@tauri-apps/api/path',()=>({homeDir:async()=>'/synthetic'}));+vi.mock('@tauri-apps/plugin-os',()=>({platform:()=>'macos',arch:()=>'aarch64'}));+vi.mock('@tauri-apps/api/event',()=>({emit:vi.fn(async(name:string)=>{state.listeners.get(name)?.({payload:null})}),listen:vi.fn(async(name:string,callback:Function)=>{state.listeners.set(name,callback);return ()=>state.listeners.delete(name)})}));+vi.mock('@tauri-apps/plugin-store',()=>({Store:{load:async()=>({get:async()=>structuredClone(state.settings),set:async(_k:string,v:any)=>{state.settings=structuredClone(v)},save:async()=>{},onKeyChange:async()=>()=>{}})}}));+vi.mock('@/lib/utils/tauri',()=>({commands:{getScreenpipeBaseDir:async()=>({status:'ok',data:'/synthetic'}),reencryptStore:async()=>{},getCloudToken:async()=>null,setCloudToken:async(token:any)=>{state.native(token);return {status:'ok'}},setBrowserCookieAccessState:async()=>{}}}));+vi.mock('@/lib/auth-guard',()=>({installAuthInterceptor:()=>()=>{}}));+vi.mock('@/lib/analytics-id',()=>({cacheAnalyticsId:()=>{},cacheAnalyticsEnabled:()=>{}}));+vi.mock('@/lib/analytics/settings-change',()=>({captureSettingsChange:()=>{}}));+vi.mock('@/lib/api',()=>({configureApi:()=>{},refreshApiConfig:async()=>{},localFetch:async()=>{throw Error('unexpected network')}}));+vi.mock('@/lib/hooks/use-is-enterprise-build',()=>({isResolvedConsumerBuild:async()=>true,resolveEnterpriseBuild:async()=>false}));++let api:any;+const account={id:'synthetic-user',clerk_id:'synthetic-clerk',email:'[email protected]',cloud_subscribed:false,subscription_status:'none',billing_plan:'free',local_plan_policy:'free',plan:'free'};+const response=()=>({ok:true,status:200,json:async()=>({success:true,user:account})});+let release!:(value:any)=>void;+let calls:any[];+beforeEach(()=>{vi.resetModules();vi.clearAllMocks();state.listeners.clear();state.unexpected=[];state.runtime=Promise.resolve(false);state.version=Promise.resolve('0.0.0');localStorage.clear();calls=[];vi.stubGlobal('fetch',vi.fn(async(url:any,opts:any)=>{calls.push({url,body:JSON.parse(opts.body)});if(calls.length===1)return new Promise(r=>release=r);return response()}));});+afterEach(()=>{cleanup();vi.unstubAllGlobals();expect(state.unexpected).toEqual([])});+async function mount(){const m=await import('@/lib/hooks/use-settings');state.settings={...m.createDefaultSettingsObject(),deviceId:'synthetic-device',analyticsId:'synthetic-install',analyticsEnabled:false,user:null,_proCloudMigrationDone:true,_presetsSeededForUser:true};function Observer(){api=m.useSettings();return null};render(<m.SettingsProvider><Observer/></m.SettingsProvider>);await waitFor(()=>expect(api.isSettingsLoaded).toBe(true));}+async function start(verify=false){let pending!:Promise<any>;await act(async()=>{pending=api.loadUser('synthetic-token',verify)});expect(calls.length).toBe(1);return {pending};}+it('late successful account response cannot undo explicit sign-out',async()=>{await mount();const {pending}=await start();await act(async()=>{await api.updateSettings({user:null})});state.native.mockClear();await act(async()=>{release(response());await pending});expect(state.settings.user).toBeNull();expect(state.native).not.toHaveBeenCalledWith('synthetic-token');});+it('peer sign-out fences this window pending account request',async()=>{await mount();const {pending}=await start();await act(async()=>{state.settings.user=null;state.listeners.get('screenpipe-auth-signout')?.({payload:null})});state.native.mockClear();await act(async()=>{release(response());await pending});expect(state.settings.user).toBeNull();expect(state.native).not.toHaveBeenCalledWith('synthetic-token');});+it('fresh successful sign-in persists identity and native token',async()=>{await mount();const {pending}=await start();await act(async()=>{release(response());await pending});expect(state.settings.user?.id).toBe('synthetic-user');expect(state.native).toHaveBeenCalledWith('synthetic-token');});+it('explicit verification flag reaches the account request',async()=>{await mount();const {pending}=await start(true);await act(async()=>{release(response());await pending});expect(calls[0].body).toEqual({token:'synthetic-token',verify:true});expect(state.settings.user?.id).toBe('synthetic-user');});+it('fresh request after a completed sign-out may sign in',async()=>{await mount();await act(async()=>{await api.updateSettings({user:null})});const {pending}=await start();await act(async()=>{release(response());await pending});expect(state.settings.user?.id).toBe('synthetic-user');expect(state.native).toHaveBeenCalledWith('synthetic-token');});++it('unrelated preference update does not invalidate a fresh account response',async()=>{await mount();const {pending}=await start();await act(async()=>{await api.updateSettings({analyticsEnabled:false})});await act(async()=>{release(response());await pending});expect(state.settings.user?.id).toBe('synthetic-user');expect(state.native).toHaveBeenCalledWith('synthetic-token');});