YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: cover unavailable native calendar polling
Louis Beaumontc5caaf4d0bb2
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex b413ce394..d37ec8e48 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -1626,3 +1626,16 @@ bypass, equivalent, preserved-behavior and setup-error controls. Hidden fixturesand dependency links appear only after the trajectory. This does not establishnative persistence, encryption, real account access, provider delivery, executionisolation or model improvement.++The `app-native-calendar-unavailable-poll` case executes the actual frontend+calendar module with synthetic HTTP and native-command ports. Nine outcomes+cover repeated unavailable polls, explicit disconnected responses, stale or+unknown status recovery, connected empty calendars, authorized HTTP failures,+and preserved Google/ICS providers. The parent fails two intended outcomes and+preserves seven; reference and current source pass nine.+Run `bun test evals/coding-agent/calibrate-calendar-unavailable.test.js` for eleven+controls, including equivalent implementations, unused repairs, blanket native+suppression and missing-module setup errors. Only the frontend calendar module+is applied from the historical fix. Native endpoint mappings, OS permissions,+OAuth refresh, publisher backoff, enforced agent isolation and model performance+remain outside this evidence. Fixtures and dependencies are installed at grading.diff --git a/evals/coding-agent/calibrate-calendar-unavailable.test.js b/evals/coding-agent/calibrate-calendar-unavailable.test.jsnew file mode 100644index 000000000..0a1074afc--- /dev/null+++ b/evals/coding-agent/calibrate-calendar-unavailable.test.js@@ -0,0 +1,39 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { dirname, join, resolve } from 'node:path';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+const repo=resolve(import.meta.dir,'../..'), app='apps/screenpipe-app-tauri';+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='app-native-calendar-unavailable-poll');+const source=`${app}/lib/utils/calendar.ts`, root=mkdtempSync(join(tmpdir(),'calendar-calibration-'));+const receipts=process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;+afterAll(()=>rmSync(root,{recursive:true,force:true}));+function replace(cwd,from,to){const p=join(cwd,source),text=readFileSync(p,'utf8');expect(text.split(from)).toHaveLength(2);writeFileSync(p,text.replace(from,to));}+function grade(name,ref=item.oracle_ref,mutate=()=>{}){+ const cwd=join(root,name);mkdirSync(cwd);+ // Only the actual historical module and mocked port modules are extracted.+ // No agent runs; hidden fixtures and dependencies are installed for grading.+ for(const file of [source,`${app}/lib/api.ts`,`${app}/lib/utils/tauri.ts`]){+ const dest=join(cwd,file);mkdirSync(dirname(dest),{recursive:true});writeFileSync(dest,execFileSync('git',['show',`${ref}:${file}`],{cwd:repo,maxBuffer:8*1024*1024}));+ }+ for(const f of item.grader.fixtures){const dest=join(cwd,f.destination_path);mkdirSync(dirname(dest),{recursive:true});writeFileSync(dest,readFileSync(join(import.meta.dir,f.local_path)));}+ symlinkSync(join(repo,app,'node_modules'),join(cwd,app,'node_modules'),'dir');mutate(cwd);+ const r=spawnSync('/bin/bash',['-c',item.grader.command],{cwd,encoding:'utf8',timeout:60000,maxBuffer:4*1024*1024,env:{PATH:process.env.PATH,HOME:cwd,CI:'true',TZ:'UTC',NO_COLOR:'1'}});+ if(receipts){mkdirSync(receipts,{recursive:true});writeFileSync(join(receipts,name+'.stdout'),r.stdout||'');writeFileSync(join(receipts,name+'.stderr'),r.stderr||'');writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message},null,2));}+ expect(r.error).toBeUndefined();expect(r.signal).toBeNull();return r;+}+function passes(r){expect(r.status).toBe(0);expect(r.stdout).toContain('9 passed');expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught/);}+function fails(r){expect(r.status).toBe(1);expect(r.stdout).toContain('failed');expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/);}+test('parent has two intended failures and seven preserved passes',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('2 failed | 7 passed');},60000);+test('reference passes every outcome',()=>passes(grade('reference')),60000);+test('current source passes every outcome',()=>passes(grade('current','HEAD')),60000);+test('unused corrected source cannot repair the actual caller',()=>{const r=grade('unused',item.base_ref,cwd=>writeFileSync(join(cwd,app,'unused-calendar.ts'),execFileSync('git',['show',`${item.oracle_ref}:${source}`],{cwd:repo})));fails(r);expect(r.stdout).toContain('2 failed | 7 passed');},60000);+test('equivalent private helper names pass',()=>passes(grade('equivalent',item.oracle_ref,cwd=>{const p=join(cwd,source);writeFileSync(p,readFileSync(p,'utf8').replaceAll('fetchNativeProvider','readSystemEvents').replaceAll('fetchNativeCalendar','requestSystemEvents'));})),60000);+test('equivalent availability condition passes',()=>passes(grade('equivalent-condition',item.oracle_ref,cwd=>replace(cwd,'if (statusKnown && !statusAvailable) {','if (statusKnown === true && statusAvailable === false) {'))),60000);+test('blanket native suppression loses healthy recovery',()=>fails(grade('blanket',item.oracle_ref,cwd=>replace(cwd,'if (statusKnown && !statusAvailable) {','if (true) {'))),60000);+test('probing known unavailable status is rejected',()=>fails(grade('always-probe',item.oracle_ref,cwd=>replace(cwd,'if (statusKnown && !statusAvailable) {','if (false) {'))),60000);+test('ignoring explicit disconnected response is rejected',()=>fails(grade('ignore-disconnected',item.oracle_ref,cwd=>replace(cwd,'if (body.connected === false) return null;','void body.connected;'))),60000);+test('dropping another provider loses preserved events',()=>fails(grade('drop-ics',item.oracle_ref,cwd=>replace(cwd,'fetchIcsProvider(hoursBack, hoursAhead),','Promise.resolve({source: "ics" as const, connected:false, ok:true, events:[]}),'))),60000);+test('missing module is setup failure rather than intended behavior',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,source)));expect(r.status).toBe(1);expect(r.stdout+r.stderr).toMatch(/Failed to resolve import|Failed to load url/);},60000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex 74579c035..66ed2f77c 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -5351,6 +5351,51 @@],"command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-manual-sync.config.mjs"}+ },+ {+ "id": "app-native-calendar-unavailable-poll",+ "name": "Keep unavailable native calendars disconnected without repeated HTTP probes",+ "source": {+ "kind": "git_regression",+ "fix_commit": "f5b1f3d29246bdcb0bdb3f156721b80c71f367e5"+ },+ "base_ref": "166bdc1dd78a7767ceabe20495c2faad0e7ef866",+ "oracle_ref": "f5b1f3d29246bdcb0bdb3f156721b80c71f367e5",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/lib/utils/calendar.ts"+ ],+ "gate": "advisory",+ "trigger_paths": [+ "apps/screenpipe-app-tauri/lib/utils/calendar.ts"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "calendar",+ "provider-availability"+ ],+ "prompt": "The upcoming-calendar frontend poller repeatedly requests native events on machines whose native calendar is known to be unavailable, and can present an unavailable provider as connected. Keep the public fetchUpcomingCalendarSnapshot API. Across repeated polls, a successful native status reporting unavailable must produce no native events request and no native connection or failure entry. If status is unknown and the HTTP response explicitly reports connected:false with empty data, keep native disconnected and without a failed-source entry. Preserve HTTP recovery when native status cannot be obtained or returns an error, and when the calendar is available but its authorization status is stale: a successful event response must still appear as connected, healthy native events. Forward caller hoursBack and hoursAhead to native HTTP queries. An authorized native calendar with a successful empty response stays connected and healthy; its HTTP failure stays connected and failed. Preserve disconnected Google accounts making no events request, both connected Google accounts contributing their events, and enabled ICS events while native is unavailable. Preserve event titles and provider source labels. Tests execute the actual frontend module with synthetic HTTP and native-command ports. Native HTTP implementation, OS permissions, OAuth token refresh and publisher retry timing are outside this task.",+ "agent_timeout_seconds": 900,+ "grader_dependency_links": [+ {+ "source_path": "apps/screenpipe-app-tauri/node_modules",+ "destination_path": "apps/screenpipe-app-tauri/node_modules"+ }+ ],+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/calendar-unavailable.fixture.ts.txt",+ "destination_path": "apps/screenpipe-app-tauri/eval-calendar-unavailable.test.ts"+ },+ {+ "local_path": "graders/calendar-unavailable.config.mjs",+ "destination_path": "apps/screenpipe-app-tauri/eval-calendar-unavailable.config.mjs"+ }+ ],+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-calendar-unavailable.config.mjs"+ }}]}diff --git a/evals/coding-agent/graders/calendar-unavailable.config.mjs b/evals/coding-agent/graders/calendar-unavailable.config.mjsnew file mode 100644index 000000000..0bdb4b0c6--- /dev/null+++ b/evals/coding-agent/graders/calendar-unavailable.config.mjs@@ -0,0 +1,8 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { defineConfig } from 'vitest/config';+import path from 'node:path';+export default defineConfig({+ test:{include:['eval-calendar-unavailable.test.ts'],environment:'node',pool:'forks',poolOptions:{forks:{singleFork:true}}},+ resolve:{alias:{'@':path.resolve('.')}},+});diff --git a/evals/coding-agent/graders/calendar-unavailable.fixture.ts.txt b/evals/coding-agent/graders/calendar-unavailable.fixture.ts.txtnew file mode 100644index 000000000..616f296f4--- /dev/null+++ b/evals/coding-agent/graders/calendar-unavailable.fixture.ts.txt@@ -0,0 +1,75 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { beforeEach, expect, test, vi } from 'vitest';+const ports = vi.hoisted(() => ({ localFetch: vi.fn(), commands: {+ calendarStatus: vi.fn(), oauthStatus: vi.fn(), oauthListInstances: vi.fn(),+ icsCalendarGetEntries: vi.fn(), icsCalendarGetUpcoming: vi.fn(),+}}));+vi.mock('@/lib/api', () => ({localFetch: ports.localFetch}));+vi.mock('@/lib/utils/tauri', () => ({commands: ports.commands}));+import { fetchUpcomingCalendarSnapshot } from './lib/utils/calendar';+const ok = (data: unknown) => ({status:'ok', data});+const event = (title='Synthetic meeting') => ({id:title,title,start:'2030-01-02T10:00:00Z',end:'2030-01-02T11:00:00Z'});+const nativeStatus = (available:boolean, authorized=false, calendarCount=0) =>+ ports.commands.calendarStatus.mockResolvedValue(ok({available, authorized, calendarCount}));+const nativeCalls = () => ports.localFetch.mock.calls.filter(([url])=>url.startsWith('/connections/calendar/events?'));+const response = (body: unknown, success=true) => ({ok:success,json:async()=>body});+beforeEach(()=>{+ vi.resetAllMocks(); nativeStatus(false);+ ports.commands.oauthStatus.mockResolvedValue(ok({connected:false}));+ ports.commands.oauthListInstances.mockResolvedValue(ok([]));+ ports.commands.icsCalendarGetEntries.mockResolvedValue(ok([]));+ ports.commands.icsCalendarGetUpcoming.mockResolvedValue(ok([]));+ ports.localFetch.mockResolvedValue(response({},false));+});+test('repeated known-unavailable polls make no native request and stay disconnected',async()=>{+ for(let i=0;i<2;i++) {+ const s=await fetchUpcomingCalendarSnapshot();+ expect(s.connectedSources).toEqual([]);expect(s.failedSources).toEqual([]);expect(s.events).toEqual([]);+ }+ expect(nativeCalls()).toHaveLength(0);+});+test('unknown native status and explicit disconnected HTTP body stay disconnected',async()=>{+ ports.commands.calendarStatus.mockRejectedValue(new Error('synthetic status unavailable'));+ ports.localFetch.mockResolvedValue(response({connected:false,data:[]}));+ const s=await fetchUpcomingCalendarSnapshot();+ expect(nativeCalls()).toHaveLength(1);expect(s.connectedSources).toEqual([]);expect(s.failedSources).toEqual([]);expect(s.events).toEqual([]);+});+test('available but stale unauthorized status still recovers events through HTTP',async()=>{+ nativeStatus(true);ports.localFetch.mockResolvedValue(response({data:[event()]}));+ const s=await fetchUpcomingCalendarSnapshot({hoursBack:2,hoursAhead:6});+ expect(nativeCalls()).toHaveLength(1);+ const url=new URL(nativeCalls()[0][0],'http://localhost');expect(url.searchParams.get('hours_back')).toBe('2');expect(url.searchParams.get('hours_ahead')).toBe('6');+ expect(s.connectedSources).toEqual(['native']);expect(s.failedSources).toEqual([]);expect(s.events).toMatchObject([{title:'Synthetic meeting',source:'native'}]);+});+test('authorized empty native calendar remains connected and healthy',async()=>{+ nativeStatus(true,true,1);ports.localFetch.mockResolvedValue(response({data:[]}));+ const s=await fetchUpcomingCalendarSnapshot();expect(s.connectedSources).toEqual(['native']);expect(s.failedSources).toEqual([]);expect(s.events).toEqual([]);+});+test('unknown native status may recover a successful nonempty HTTP result',async()=>{+ ports.commands.calendarStatus.mockResolvedValue({status:'error',error:'synthetic status unavailable'});+ ports.localFetch.mockResolvedValue(response({data:[event()]}));+ const s=await fetchUpcomingCalendarSnapshot();expect(s.connectedSources).toEqual(['native']);expect(s.failedSources).toEqual([]);expect(s.events).toHaveLength(1);+});+test('authorized native HTTP failure remains a connected failed source',async()=>{+ nativeStatus(true,true,1);const s=await fetchUpcomingCalendarSnapshot();+ expect(nativeCalls()).toHaveLength(1);expect(s.connectedSources).toEqual(['native']);expect(s.failedSources).toEqual(['native']);expect(s.events).toEqual([]);+});+test('disconnected Google account makes no Google request',async()=>{+ await fetchUpcomingCalendarSnapshot();expect(ports.localFetch.mock.calls.filter(([url])=>url.includes('/google-calendar/'))).toHaveLength(0);+});+test('both connected Google accounts retain their events with unavailable native calendar',async()=>{+ ports.commands.oauthListInstances.mockResolvedValue(ok([{instance:'[email protected]'},{instance:'[email protected]'}]));+ ports.commands.oauthStatus.mockResolvedValue(ok({connected:true}));+ ports.localFetch.mockImplementation(async(raw:string)=>{+ const url=new URL(raw,'http://localhost');+ return url.pathname==='/connections/google-calendar/events'?response([event(url.searchParams.get('instance')!)]):response({},false);+ });+ const s=await fetchUpcomingCalendarSnapshot();expect(s.connectedSources).toEqual(['google']);expect(s.failedSources).toEqual([]);+ expect(s.events.map(e=>e.title).sort()).toEqual(['[email protected]','[email protected]']);expect(s.events.every(e=>e.source==='google')).toBe(true);+});+test('enabled ICS events survive unavailable native calendar',async()=>{+ ports.commands.icsCalendarGetEntries.mockResolvedValue(ok([{id:'synthetic-feed',enabled:true}]));+ ports.commands.icsCalendarGetUpcoming.mockResolvedValue(ok([event('Synthetic ICS')]));+ const s=await fetchUpcomingCalendarSnapshot();expect(s.connectedSources).toEqual(['ics']);expect(s.failedSources).toEqual([]);expect(s.events).toMatchObject([{title:'Synthetic ICS',source:'ics'}]);+});