YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
test(evals): verify onboarding notification transport outcomes
louise5ac60c7a40a
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex f34081610..6460cd966 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -957,3 +957,21 @@ suppression, dropped annotations and blocked acceptance; equivalent private nameand test attributes pass. Missing source is a setup error. Hidden fixtures anddependencies are installed only at grading. This does not establish nativestorage, browser geometry, agent isolation or model performance.++## Onboarding notification transport++`app-onboarding-notification-transport` exercises the existing follow-up+orchestration, activation storage, time helper and app-control client with+synthetic native and transport ports. Six outcomes cover runtime-port delivery,+configuration fallback, refusal followed by retry, replay suppression, failed+engine dispatch, future deadlines and missing views. It observes the default+transport rather than injecting a replacement notification helper. The historical+parent fails three delivery outcomes and preserves three; the reference and+current source pass all six.++Run `bun test evals/coding-agent/calibrate-notification-transport.test.js` with+desktop dependencies installed. Fixtures and dependency links are materialized+only when grading begins. These checks do not establish native notification+display, completed Pipe results, policy authority to start work, multi-window+concurrency, execution isolation or model performance. The task covers an already+scheduled follow-up and does not authorize activating paused work.diff --git a/evals/coding-agent/calibrate-notification-transport.test.js b/evals/coding-agent/calibrate-notification-transport.test.jsnew file mode 100644index 000000000..66e3aef05--- /dev/null+++ b/evals/coding-agent/calibrate-notification-transport.test.js@@ -0,0 +1,49 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import {expect,test} from 'bun:test';+import {classifyGraderError} from './grader-outcome.mjs';+import {execFileSync,spawnSync} from 'node:child_process';+import {mkdtempSync,mkdirSync,readFileSync,writeFileSync,symlinkSync,realpathSync,rmSync} from 'node:fs';+import {tmpdir} from 'node:os';+import {resolve,join,dirname} from 'node:path';+const repo=resolve(import.meta.dir,'../..'), prefix='apps/screenpipe-app-tauri/';+const fix='35caddb4562d3bd15fdb0aa3f7928ce20dfed93c',parent='a7d74fb9adf6c9c286ce9cfa33b2e8c9c62135bc';+const files=['lib/live-views/onboarding-follow-up.ts','lib/live-views/onboarding-activation.ts','lib/live-views/time-range.ts','lib/analytics/onboarding-h1-follow-up.ts','lib/notifications/app-server.ts','lib/api.ts','lib/utils/tauri.ts'];+const source='lib/live-views/onboarding-follow-up.ts',client='lib/notifications/app-server.ts';+const runtime=realpathSync(join(repo,prefix,'node_modules'));+const fixtures=[['notification-transport.fixture.ts.txt','eval-notification-transport.test.ts'],['notification-transport.config.mjs','eval-notification-transport.config.mjs'],['notification-localization.fixture.ts.txt','eval-notification-localization.ts']];+function replace(s,from,to){if(!s.includes(from))throw Error('Control mutation drift: '+from);return s.replaceAll(from,to)}+const controls=[+ {name:'parent',ref:parent,passed:3,failed:3},+ {name:'reference',ref:fix,passed:6,failed:0},+ {name:'unused-correct-client',ref:parent,passed:3,failed:3,extra:true},+ {name:'equivalent-private-names',ref:fix,passed:6,failed:0,client:s=>replace(replace(s,'getAppServerBaseUrl','resolveControlAddress'),'appServerBaseUrl','controlAddress')},+ {name:'success-without-delivery',ref:fix,passed:3,failed:3,client:s=>replace(s,'return fetch(`${baseUrl}${normalizedPath}`, init);','return {ok:true} as Response;')},+ {name:'lost-sent-receipt',ref:fix,passed:4,failed:2,source:s=>replace(s,' markFollowUpSent(activation.viewId, now);',' // intentionally omitted durable receipt')},+ {name:'wrong-engine-endpoint',ref:fix,passed:4,failed:2,client:s=>replace(s,'http://localhost:${config.port || 11435}','http://localhost:3030')},+ {name:'missing-source',ref:fix,passed:0,failed:0,missing:true},+];+for(const control of controls)test(control.name,()=>{+ const root=mkdtempSync(join(tmpdir(),'notification-calibration-')),app=join(root,prefix);+ try{+ for(const path of files){+ if(path===client&&control.ref===parent&&!control.extra)continue;+ const ref=path===client&&control.extra?fix:control.ref;+ let text=execFileSync('git',['show',ref+':'+prefix+path],{cwd:repo,encoding:'utf8'});+ if(path===source&&control.source)text=control.source(text);+ if(path===client&&control.client)text=control.client(text);+ const dest=join(app,path);mkdirSync(dirname(dest),{recursive:true});writeFileSync(dest,text);+ }+ if(control.missing)rmSync(join(app,source));+ for(const [from,to]of fixtures)writeFileSync(join(app,to),readFileSync(join(repo,'evals/coding-agent/graders',from)));+ symlinkSync(runtime,join(app,'node_modules'),'dir');+ const result=spawnSync(process.env.NODE_BIN||'node',['node_modules/vitest/vitest.mjs','run','--config','eval-notification-transport.config.mjs','--reporter=default','--reporter=json','--outputFile=result.json'],{cwd:app,encoding:'utf8',timeout:20000});+ const report=JSON.parse(readFileSync(join(app,'result.json'),'utf8'));+ if(process.env.CALIBRATION_OUTPUT_DIR){const out=process.env.CALIBRATION_OUTPUT_DIR;mkdirSync(out,{recursive:true});writeFileSync(join(out,control.name+'.json'),JSON.stringify({control:control.name,ref:control.ref,exit_code:result.status,signal:result.signal,stdout:result.stdout,stderr:result.stderr,report},null,2)+'\n')}+ expect(result.error).toBeUndefined();expect(result.signal).toBeNull();+ expect(report.numPassedTests).toBe(control.passed);expect(report.numFailedTests).toBe(control.failed);+ expect(result.status===0).toBe(control.failed===0&&!control.missing);+ if(control.missing){expect(classifyGraderError(result)).toBe('vitest_collection_error');expect(report.numTotalTests).toBe(0);expect(JSON.stringify(report)).toMatch(/Failed to load|Cannot find|resolve/)}+ else expect(report.numTotalTests).toBe(6);+ }finally{rmSync(root,{recursive:true,force:true})}+},30000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex 260b15769..e8566f7fc 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-09-26.1",+ "dataset_version": "2026-09-26.2","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -3475,6 +3475,59 @@],"command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-canvas-outcome.config.mjs"}+ },+ {+ "id": "app-onboarding-notification-transport",+ "name": "Deliver onboarding follow-ups through the app-control server",+ "source": {+ "kind": "git_regression",+ "fix_commit": "35caddb4562d3bd15fdb0aa3f7928ce20dfed93c",+ "pull_request": "https://github.com/screenpipe/screenpipe/pull/5844"+ },+ "base_ref": "a7d74fb9adf6c9c286ce9cfa33b2e8c9c62135bc",+ "oracle_ref": "35caddb4562d3bd15fdb0aa3f7928ce20dfed93c",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/lib/live-views/onboarding-follow-up.ts",+ "apps/screenpipe-app-tauri/lib/notifications/app-server.ts"+ ],+ "gate": "advisory",+ "trigger_paths": [+ "apps/screenpipe-app-tauri/lib/live-views/onboarding-follow-up.ts",+ "apps/screenpipe-app-tauri/lib/notifications/app-server.ts"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "notifications",+ "retry",+ "transport"+ ],+ "prompt": "An already scheduled onboarding follow-up dispatches its permitted dashboard work but never delivers its notification because notification requests go to the recording engine. Repair the existing runDueOnboardingLiveViewFollowUp default transport. Keep Pipe runs on the engine transport, deduplicate shared Pipe bindings, and POST notifications to the app-control server using its native runtime port; use localhost:11435 if native configuration lookup fails. Preserve the notification link to the originating Live View. Persist sent status only after notification acceptance, and suppress later duplicate delivery. A rejected notification must remain scheduled for a later retry that can recover and persist sent status. If all engine dispatches fail, schedule a retry without sending a notification. A future deadline or a missing view must perform no transport work. Use synthetic native configuration, storage and network ports. This task fixes delivery of an existing scheduled follow-up; it does not authorize new scheduling, activation of paused work, claim completed Pipe results, or change notification wording or notification policy. Native display, engine completion, multi-window concurrency, recording and live service behavior are outside scope.",+ "agent_timeout_seconds": 900,+ "grader_dependency_links": [+ {+ "source_path": "apps/screenpipe-app-tauri/node_modules",+ "destination_path": "apps/screenpipe-app-tauri/node_modules"+ }+ ],+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/notification-transport.fixture.ts.txt",+ "destination_path": "apps/screenpipe-app-tauri/eval-notification-transport.test.ts"+ },+ {+ "local_path": "graders/notification-transport.config.mjs",+ "destination_path": "apps/screenpipe-app-tauri/eval-notification-transport.config.mjs"+ },+ {+ "local_path": "graders/notification-localization.fixture.ts.txt",+ "destination_path": "apps/screenpipe-app-tauri/eval-notification-localization.ts"+ }+ ],+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-notification-transport.config.mjs"+ }}]}diff --git a/evals/coding-agent/graders/notification-localization.fixture.ts.txt b/evals/coding-agent/graders/notification-localization.fixture.ts.txtnew file mode 100644index 000000000..43ae53d77--- /dev/null+++ b/evals/coding-agent/graders/notification-localization.fixture.ts.txt@@ -0,0 +1,3 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+export const msg=(x:string)=>x;diff --git a/evals/coding-agent/graders/notification-transport.config.mjs b/evals/coding-agent/graders/notification-transport.config.mjsnew file mode 100644index 000000000..ffb509034--- /dev/null+++ b/evals/coding-agent/graders/notification-transport.config.mjs@@ -0,0 +1,3 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import {defineConfig} from 'vitest/config';import {fileURLToPath} from 'node:url';export default defineConfig({resolve:{alias:{'@':fileURLToPath(new URL('.',import.meta.url)),'gt-react':fileURLToPath(new URL('./eval-notification-localization.ts',import.meta.url))}},test:{environment:'node',include:['eval-notification-transport.test.ts'],pool:'forks',poolOptions:{forks:{singleFork:true}}}});diff --git a/evals/coding-agent/graders/notification-transport.fixture.ts.txt b/evals/coding-agent/graders/notification-transport.fixture.ts.txtnew file mode 100644index 000000000..9888b26df--- /dev/null+++ b/evals/coding-agent/graders/notification-transport.fixture.ts.txt@@ -0,0 +1,67 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import {beforeEach, afterEach, expect, test, vi} from 'vitest';+const ports = vi.hoisted(()=>({invoke:vi.fn(), list:vi.fn(), capture:vi.fn()}));+vi.mock('@tauri-apps/api/core',()=>({invoke:ports.invoke}));+vi.mock('@/lib/utils/tauri',()=>({commands:{listBrainViews:ports.list}}));+vi.mock('posthog-js',()=>({default:{capture:ports.capture}}));+vi.mock('@/lib/api',()=>({localFetch:(path:string,init?:RequestInit)=>fetch('http://localhost:3030'+path,init)}));+import {runDueOnboardingLiveViewFollowUp as publicBoundary} from './lib/live-views/onboarding-follow-up';+if(typeof publicBoundary!=='function')throw Error('Missing production follow-up boundary');+const values=new Map<string,string>();+const storage={getItem:(k:string)=>values.get(k)??null,setItem:(k:string,v:string)=>values.set(k,String(v)),removeItem:(k:string)=>values.delete(k),clear:()=>values.clear()};+const view={id:'synthetic-view',title:'Project notes',timeRange:'today',slots:[{id:'a',binding:{pipeName:'summary'}},{id:'b',binding:{pipeName:'summary'}}]};+let activation:any,follow:any,requests:Array<{url:string,body:any}>,controlStatus:number,engineStatus:number,delivered:any[];+let now=new Date('2026-08-01T12:00:00Z');+beforeEach(async()=>{+ vi.resetModules();vi.clearAllMocks();values.clear();requests=[];delivered=[];controlStatus=200;engineStatus=200;+ vi.useFakeTimers();vi.setSystemTime(now);+ vi.stubGlobal('window',{localStorage:storage,dispatchEvent:()=>true});+ vi.stubGlobal('CustomEvent',class {constructor(public type:string,public detail:any){}});+ ports.invoke.mockResolvedValue({port:18435});ports.list.mockResolvedValue({status:'ok',data:[view]});+ vi.stubGlobal('fetch',vi.fn(async(input:any,init:any)=>{+ const url=String(input),body=init?.body?JSON.parse(init.body):null;requests.push({url,body});+ const target=new URL(url);let status=404;+ if(init?.method!=='POST')return {ok:false,status:405};+ if(target.origin==='http://localhost:3030'&&target.pathname==='/pipes/summary/run')status=engineStatus;+ if(['http://localhost:18435','http://localhost:11435'].includes(target.origin)&&target.pathname==='/notify') {status=controlStatus;if(status===200)delivered.push(body);}+ return {ok:status===200,status};+ }));+ activation=await import('./lib/live-views/onboarding-activation');follow=await import('./lib/live-views/onboarding-follow-up');+ activation.startOnboardingLiveViewActivation(view.id,'work_memory',{goal:'Find project notes'});+ activation.markOnboardingLiveViewSetupReady(view.id);+ activation.updateOnboardingLiveViewFollowUp(view.id,(f:any)=>({...f,dueAt:now.toISOString()}));+});+afterEach(()=>{vi.useRealTimers();vi.unstubAllGlobals()});+test('default transport delivers through runtime control port and suppresses replay',async()=>{+ await follow.runDueOnboardingLiveViewFollowUp({now:()=>now});+ expect(delivered).toHaveLength(1);+ expect(requests.filter(x=>x.url==='http://localhost:3030/pipes/summary/run')).toHaveLength(1);+ expect(delivered[0].source_url).toBe('screenpipe://live-view/synthetic-view');+ expect(activation.getOnboardingLiveViewActivation(view.id).followUp.status).toBe('sent');+ await follow.runDueOnboardingLiveViewFollowUp({now:()=>now});expect(delivered).toHaveLength(1);+});+test('native config lookup failure still reaches default control port',async()=>{+ ports.invoke.mockRejectedValue(new Error('synthetic native config unavailable'));+ await follow.runDueOnboardingLiveViewFollowUp({now:()=>now});+ expect(delivered).toHaveLength(1);expect(requests.some(x=>x.url==='http://localhost:11435/notify')).toBe(true);+});+test('refused notification remains retryable and later recovery persists success',async()=>{+ controlStatus=503;await follow.runDueOnboardingLiveViewFollowUp({now:()=>now});expect(delivered).toHaveLength(0);+ expect(activation.getOnboardingLiveViewActivation(view.id).followUp.status).toBe('scheduled');+ controlStatus=200;const later=new Date(now.getTime()+6*60*1000);vi.setSystemTime(later);+ await follow.runDueOnboardingLiveViewFollowUp({now:()=>later});expect(delivered).toHaveLength(1);+ expect(activation.getOnboardingLiveViewActivation(view.id).followUp.status).toBe('sent');+});+test('no notification is delivered when engine dispatch fails',async()=>{+ engineStatus=503;await follow.runDueOnboardingLiveViewFollowUp({now:()=>now});expect(delivered).toHaveLength(0);+ expect(requests.some(x=>x.url.endsWith('/notify'))).toBe(false);+ expect(activation.getOnboardingLiveViewActivation(view.id).followUp.status).toBe('scheduled');+});+test('future deadline performs no transport work',async()=>{+ activation.updateOnboardingLiveViewFollowUp(view.id,(f:any)=>({...f,dueAt:new Date(now.getTime()+60000).toISOString()}));+ await follow.runDueOnboardingLiveViewFollowUp({now:()=>now});expect(requests).toHaveLength(0);+});+test('missing view performs no transport work',async()=>{+ ports.list.mockResolvedValue({status:'ok',data:[]});await follow.runDueOnboardingLiveViewFollowUp({now:()=>now});expect(requests).toHaveLength(0);+});