YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: cover auth verification scheduling across settings updates
louis9a7fb97375e1
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex b98a6caf5..d919cbf2b 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -1247,3 +1247,19 @@ equivalent private bindings and missing-source classification. Hidden fixturesand dependency links appear only at grading. This checks the React/native-commandboundary; it does not establish native capture, permission delivery, full UIunmount/skip recovery, migration authority, agent isolation or model performance.++## Session verification schedule++`app-auth-verification-schedule` runs the actual React auth guard with synthetic+settings, native credential and clock ports. Twelve outcomes cover stable startup+and periodic checks, callback refresh, token changes, sign-out, focus cooldown,+hidden windows, unmount and session-expiry ownership. The parent fails three+scheduling outcomes and preserves nine; reference and current source pass twelve.++Run `bun test evals/coding-agent/calibrate-auth-schedule.test.js` for thirteen+controls. A recursive timer implementation passes; unused fixed code, disabled+checks, stale callbacks, lost token resets, cross-account logout and missing native+credential cleanup fail. Missing source is a collection error. Fixtures and+dependencies are installed only when grading starts. This checks the React port+boundary, not live account refresh, native credential persistence, recording,+execution isolation or model performance.diff --git a/evals/coding-agent/calibrate-auth-schedule.test.js b/evals/coding-agent/calibrate-auth-schedule.test.jsnew file mode 100644index 000000000..87ef92a15--- /dev/null+++ b/evals/coding-agent/calibrate-auth-schedule.test.js@@ -0,0 +1,50 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { dirname, join, resolve } from 'node:path';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { classifyGraderError } from './grader-outcome.mjs';+const repo=resolve(import.meta.dir,'../..'), app='apps/screenpipe-app-tauri';+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='app-auth-verification-schedule');+const source=`${app}/lib/auth-guard.tsx`;+const root=mkdtempSync(join(tmpdir(),'auth-schedule-calibration-')), archives=new Map();+const receipts=process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;+afterAll(()=>rmSync(root,{recursive:true,force:true}));+function replace(cwd,file,from,to){const p=join(cwd,file),s=readFileSync(p,'utf8');expect(s.split(from)).toHaveLength(2);writeFileSync(p,s.replace(from,to));}+function grade(name,ref=item.oracle_ref,mutate=()=>{}){+ const cwd=join(root,name);mkdirSync(cwd);+ if(!archives.has(ref)){+ const children=execFileSync('git',['ls-tree','--name-only',`${ref}:${app}`],{cwd:repo,encoding:'utf8'}).trim().split('\n');+ const paths=children.filter(n=>!['src-tauri','public','e2e','.e2e'].includes(n)&&!n.startsWith('.env')).map(n=>`${app}/${n}`);+ archives.set(ref,execFileSync('git',['archive',ref,...paths],{cwd:repo,maxBuffer:128*1024*1024}));+ }+ execFileSync('tar',['-x','-C',cwd],{input:archives.get(ref)});+ for(const f of item.grader.fixtures){const dest=join(cwd,f.destination_path);mkdirSync(dirname(dest),{recursive:true});writeFileSync(dest,readFileSync(join(import.meta.dir,f.local_path)));}+ symlinkSync(join(repo,app,'node_modules'),join(cwd,app,'node_modules'),'dir');mutate(cwd);+ const r=spawnSync('/bin/bash',['-c',item.grader.command],{cwd,encoding:'utf8',timeout:60000,maxBuffer:4*1024*1024,env:{PATH:process.env.PATH,HOME:cwd,CI:'true',TZ:'UTC',NO_COLOR:'1'}});+ const kind=classifyGraderError(r);+ if(receipts){mkdirSync(receipts,{recursive:true});writeFileSync(join(receipts,name+'.stdout'),r.stdout||'');writeFileSync(join(receipts,name+'.stderr'),r.stderr||'');writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message,error_kind:kind},null,2));}+ expect(r.error).toBeUndefined();expect(r.signal).toBeNull();return {...r,kind};+}+function passes(r){expect(r.status).toBe(0);expect(r.stdout).toContain('12 passed');expect(r.kind).toBeNull();}+function fails(r){expect(r.status).toBe(1);expect(r.kind).toBeNull();expect(r.stdout).toMatch(/Tests\s+[1-9]\d* failed/);}+test('parent fails scheduling and preserves nine outcomes',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('3 failed | 9 passed');},60000);+test('reference passes all outcomes',()=>passes(grade('reference')),60000);+test('current source passes all outcomes',()=>passes(grade('current','HEAD')),60000);+test('equivalent recursive timer preserves the schedule',()=>passes(grade('equivalent',item.oracle_ref,cwd=>{+ replace(cwd,source,'const interval = setInterval(verify, CHECK_INTERVAL_MS);','let interval: ReturnType<typeof setTimeout>; const tick = () => { verify(); interval = setTimeout(tick, CHECK_INTERVAL_MS); }; interval = setTimeout(tick, CHECK_INTERVAL_MS);');+ replace(cwd,source,'clearInterval(interval);','clearTimeout(interval);');+})),60000);+test('unused fixed module cannot bypass the active broken guard',()=>fails(grade('unused',item.oracle_ref,cwd=>{+ writeFileSync(join(cwd,source+'.unused.txt'),readFileSync(join(cwd,source)));writeFileSync(join(cwd,source),execFileSync('git',['show',`${item.base_ref}:${source}`],{cwd:repo}));+})),60000);+test('blanket verification suppression fails',()=>fails(grade('disabled',item.oracle_ref,cwd=>replace(cwd,source,'await loadUser(token);','return;'))),60000);+test('lost periodic verification fails',()=>fails(grade('periodic',item.oracle_ref,cwd=>replace(cwd,source,'setInterval(verify, CHECK_INTERVAL_MS)','setInterval(() => {}, CHECK_INTERVAL_MS)'))),60000);+test('stale refresh callback fails',()=>fails(grade('stale',item.oracle_ref,cwd=>replace(cwd,source,'verifyTokenRef.current = verifyToken;','// retain stale callback'))),60000);+test('missing account reset fails',()=>fails(grade('token-reset',item.oracle_ref,cwd=>replace(cwd,source,'}, [settings.user?.token]);','}, []);'))),60000);+test('stale-account rejection cannot clear current login',()=>fails(grade('old-401',item.oracle_ref,cwd=>replace(cwd,source,'if (tokenRef.current !== token) return;',''))),60000);+test('permission failure cannot clear account',()=>fails(grade('403',item.oracle_ref,cwd=>replace(cwd,source,'if (msg.includes(" 401 ")) {','if (msg.includes(" 401 ") || msg.includes(" 403 ")) {'))),60000);+test('missing native credential cleanup fails',()=>fails(grade('native',item.oracle_ref,cwd=>replace(cwd,source,'await commands.setCloudToken(null);',''))),60000);+test('missing source remains collection failure',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,source)));expect(r.status).toBe(1);expect(r.kind).toBe('vitest_collection_error');},60000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex 1f9873056..eedc5a739 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-09-29",+ "dataset_version": "2026-09-29.7","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -4222,6 +4222,49 @@],"command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config vitest.config.ts components/onboarding/eval-capture-recovery.test.tsx --maxWorkers=1 --minWorkers=1"}+ },+ {+ "id": "app-auth-verification-schedule",+ "name": "Keep session verification on schedule across account refreshes",+ "source": {+ "kind": "git_regression",+ "fix_commit": "89c104ce80f4dbebe23323d6dabab2fc873ecd0c"+ },+ "base_ref": "b3ac026dbef83a15dc6f93deb98883609ea1320d",+ "oracle_ref": "89c104ce80f4dbebe23323d6dabab2fc873ecd0c",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/lib/auth-guard.tsx"+ ],+ "gate": "advisory",+ "trigger_paths": [+ "apps/screenpipe-app-tauri/lib/auth-guard.tsx",+ "apps/screenpipe-app-tauri/lib/hooks/use-settings.tsx"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "auth",+ "scheduling",+ "session-preservation"+ ],+ "prompt": "Account refreshes and unrelated settings renders can repeatedly restart or postpone desktop session verification. Keep the startup check at five seconds and periodic checks every ten minutes from the start of each token session, even when refresh callbacks change. Use the latest refresh callback. Changing the account token must start a fresh schedule and cancel the old one. Stop checks on sign-out and unmount, while preserving rendered children. Preserve eager verification on visible focus or visibility changes with a 30-second cooldown, and suppress it while hidden. A current-session 401 must clear the full saved account and native cloud token. Permission errors and transient network failures must preserve the account, and an old account's delayed 401 must never clear a newly selected account.",+ "agent_timeout_seconds": 900,+ "grader_dependency_links": [+ {+ "source_path": "apps/screenpipe-app-tauri/node_modules",+ "destination_path": "apps/screenpipe-app-tauri/node_modules"+ }+ ],+ "grader": {+ "timeout_seconds": 90,+ "fixtures": [+ {+ "local_path": "graders/auth-schedule.fixture.tsx.txt",+ "destination_path": "apps/screenpipe-app-tauri/lib/auth-schedule.eval.test.tsx"+ }+ ],+ "command": "cd apps/screenpipe-app-tauri && bun x vitest run --config vitest.config.ts lib/auth-schedule.eval.test.tsx --maxWorkers=1 --minWorkers=1"+ }}]}diff --git a/evals/coding-agent/graders/auth-schedule.fixture.tsx.txt b/evals/coding-agent/graders/auth-schedule.fixture.tsx.txtnew file mode 100644index 000000000..130037681--- /dev/null+++ b/evals/coding-agent/graders/auth-schedule.fixture.tsx.txt@@ -0,0 +1,64 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import React from 'react';+import {afterEach, beforeEach, expect, test, vi} from 'vitest';+import {act, cleanup, fireEvent, render} from '@testing-library/react';+const ports=vi.hoisted(()=>({user:null as any,refresh:vi.fn(),save:vi.fn(),native:vi.fn()}));+vi.mock('@/lib/hooks/use-settings',()=>({useSettings:()=>{+ const refresh=ports.refresh;+ return {settings:{user:ports.user},loadUser:(token:string)=>refresh(token),updateSettings:(v:unknown)=>ports.save(v)};+}}));+vi.mock('@/lib/utils/tauri',()=>({commands:{setCloudToken:(v:unknown)=>ports.native(v)}}));+vi.mock('posthog-js',()=>({default:{capture:vi.fn()}}));+vi.mock('@/components/ui/use-toast',()=>({toast:vi.fn()}));+vi.mock('@/components/ui/toast',()=>({ToastAction:()=>null}));+vi.mock('gt-react',()=>({T:({children}:any)=>children,useGT:()=>((s:string)=>s)}));+vi.mock('@/lib/web-url',()=>({PROD_WEB_BASE:'https://screenpipe.com',screenpipeWebBase:()=> 'https://screenpipe.com',screenpipeWebUrl:()=> 'https://screenpipe.com/login'}));+import {AuthGuard} from '@/lib/auth-guard';+const tree=()=> <AuthGuard><span>Account content</span></AuthGuard>;+const advance=async(ms:number)=>{await act(async()=>{await vi.advanceTimersByTimeAsync(ms);});};+const focus=async()=>{await act(async()=>{fireEvent(window,new Event('focus'));});};+const visibility=(value:string)=>Object.defineProperty(document,'visibilityState',{configurable:true,value});+const originalVisibility=Object.getOwnPropertyDescriptor(document,'visibilityState');+beforeEach(()=>{vi.useFakeTimers();vi.setSystemTime(new Date('2026-01-01T00:00:00Z'));ports.user={token:'synthetic-session-a'};ports.refresh=vi.fn().mockResolvedValue(undefined);ports.save.mockReset().mockResolvedValue(undefined);ports.native.mockReset().mockResolvedValue(undefined);visibility('visible');});+afterEach(()=>{cleanup();vi.useRealTimers();if(originalVisibility)Object.defineProperty(document,'visibilityState',originalVisibility);else delete (document as any).visibilityState;});+test('successful account refresh keeps the original startup and periodic schedule',async()=>{+ const view=render(tree());ports.refresh.mockImplementation(async()=>{ports.user={...ports.user};view.rerender(tree());});+ await advance(4999);expect(ports.refresh).toHaveBeenCalledTimes(0);+ await advance(1);expect(ports.refresh).toHaveBeenCalledTimes(1);+ await advance(595000);expect(ports.refresh).toHaveBeenCalledTimes(2);+ await advance(5000);expect(ports.refresh).toHaveBeenCalledTimes(2);+ expect(ports.refresh.mock.calls.every(([token])=>token==='synthetic-session-a')).toBe(true);+});+test('an unrelated render cannot postpone the startup deadline',async()=>{+ const view=render(tree());await advance(4000);view.rerender(tree());await advance(1000);expect(ports.refresh).toHaveBeenCalledTimes(1);+});+test('scheduled verification uses the latest account refresh callback',async()=>{+ const previous=ports.refresh;const view=render(tree());await advance(4000);ports.refresh=vi.fn().mockResolvedValue(undefined);view.rerender(tree());await advance(1000);+ expect(previous).not.toHaveBeenCalled();expect(ports.refresh).toHaveBeenCalledWith('synthetic-session-a');+});+test('account switch starts its own schedule without stale scheduled requests',async()=>{+ const view=render(tree());await advance(4000);ports.user={token:'synthetic-session-b'};view.rerender(tree());await advance(1000);expect(ports.refresh).not.toHaveBeenCalled();+ await advance(4000);expect(ports.refresh).toHaveBeenCalledTimes(1);expect(ports.refresh).toHaveBeenLastCalledWith('synthetic-session-b');+});+test('sign-out stops verification without hiding children',async()=>{+ const view=render(tree());await advance(5000);ports.user=null;view.rerender(tree());await advance(600000);await focus();expect(ports.refresh).toHaveBeenCalledTimes(1);expect(view.getByText('Account content')).toBeTruthy();+});+test('visible focus respects the 30 second cooldown boundary',async()=>{+ render(tree());await advance(5000);await focus();expect(ports.refresh).toHaveBeenCalledTimes(1);await advance(29999);await focus();expect(ports.refresh).toHaveBeenCalledTimes(1);await advance(1);await focus();expect(ports.refresh).toHaveBeenCalledTimes(2);+});+test('hidden visibility changes preserve the session without refreshing',async()=>{+ render(tree());visibility('hidden');await act(async()=>{fireEvent(document,new Event('visibilitychange'));});expect(ports.refresh).not.toHaveBeenCalled();expect(ports.save).not.toHaveBeenCalled();visibility('visible');await act(async()=>{fireEvent(document,new Event('visibilitychange'));});expect(ports.refresh).toHaveBeenCalledTimes(1);+});+test('unmount removes timers and focus listeners',async()=>{+ const view=render(tree());view.unmount();await advance(600000);await focus();expect(ports.refresh).not.toHaveBeenCalled();+});+test('current-session 401 clears the full account and native cloud token',async()=>{+ ports.refresh.mockRejectedValue(new Error('failed to verify token: 401 Unauthorized'));render(tree());await focus();expect(ports.save).toHaveBeenCalledWith({user:null});expect(ports.native).toHaveBeenCalledWith(null);+});+for(const reason of ['failed to verify token: 403 Forbidden','network unavailable'])test('preserves account on '+reason,async()=>{+ ports.refresh.mockRejectedValue(new Error(reason));render(tree());await focus();expect(ports.refresh).toHaveBeenCalledTimes(1);expect(ports.save).not.toHaveBeenCalled();expect(ports.native).not.toHaveBeenCalled();+});+test('old-account delayed 401 cannot clear the newly selected account',async()=>{+ let reject!: (error:Error)=>void;ports.refresh.mockImplementationOnce(()=>new Promise((_resolve,rejectRequest)=>{reject=rejectRequest;}));const view=render(tree());await focus();ports.user={token:'synthetic-session-b'};view.rerender(tree());await act(async()=>{reject(new Error('failed to verify token: 401 Unauthorized'));});expect(ports.save).not.toHaveBeenCalled();expect(ports.native).not.toHaveBeenCalled();+});