YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
eval: cover bounded first-run preview polling
louis24c2002e6ea9
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex d919cbf2b..9608dd2b1 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -1263,3 +1263,5 @@ credential cleanup fail. Missing source is a collection error. Fixtures anddependencies are installed only when grading starts. This checks the React portboundary, not live account refresh, native credential persistence, recording,execution isolation or model performance.++The bounded first-run preview case exercises the actual hook with synthetic native status, activity reads and clock. Its calibration command is `bun test evals/coding-agent/calibrate-bounded-preview.test.js`. This checks corpus outcomes and grader controls; it does not execute native history queries or measure agent performance.diff --git a/evals/coding-agent/calibrate-bounded-preview.test.js b/evals/coding-agent/calibrate-bounded-preview.test.jsnew file mode 100644index 000000000..7191897e7--- /dev/null+++ b/evals/coding-agent/calibrate-bounded-preview.test.js@@ -0,0 +1,42 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { dirname, join, resolve } from 'node:path';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { classifyGraderError } from './grader-outcome.mjs';+const repo=resolve(import.meta.dir,'../..'), app='apps/screenpipe-app-tauri';+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='app-bounded-first-run-preview');+const source=`${app}/lib/first-run/use-learning-window.ts`;+const root=mkdtempSync(join(tmpdir(),'bounded-preview-calibration-')), archives=new Map();+const receipts=process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;+afterAll(()=>rmSync(root,{recursive:true,force:true}));+function replace(cwd,file,from,to){const p=join(cwd,file),s=readFileSync(p,'utf8');expect(s.split(from)).toHaveLength(2);writeFileSync(p,s.replace(from,to));}+function grade(name,ref=item.oracle_ref,mutate=()=>{}){+ const cwd=join(root,name);mkdirSync(cwd);+ if(!archives.has(ref)){+ const children=execFileSync('git',['ls-tree','--name-only',`${ref}:${app}`],{cwd:repo,encoding:'utf8'}).trim().split('\n');+ const paths=children.filter(n=>!['src-tauri','public','e2e','.e2e'].includes(n)&&!n.startsWith('.env')).map(n=>`${app}/${n}`);+ archives.set(ref,execFileSync('git',['archive',ref,...paths],{cwd:repo,maxBuffer:128*1024*1024}));+ }+ execFileSync('tar',['-x','-C',cwd],{input:archives.get(ref)});+ for(const f of item.grader.fixtures){const dest=join(cwd,f.destination_path);mkdirSync(dirname(dest),{recursive:true});writeFileSync(dest,readFileSync(join(import.meta.dir,f.local_path)));}+ symlinkSync(join(repo,app,'node_modules'),join(cwd,app,'node_modules'),'dir');mutate(cwd);+ const r=spawnSync('/bin/bash',['-c',item.grader.command],{cwd,encoding:'utf8',timeout:60000,maxBuffer:4*1024*1024,env:{PATH:process.env.PATH,HOME:cwd,CI:'true',TZ:'UTC',NO_COLOR:'1'}});+ const kind=classifyGraderError(r);+ if(receipts){mkdirSync(receipts,{recursive:true});writeFileSync(join(receipts,name+'.stdout'),r.stdout||'');writeFileSync(join(receipts,name+'.stderr'),r.stderr||'');writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message,error_kind:kind},null,2));}+ expect(r.error).toBeUndefined();expect(r.signal).toBeNull();return {...r,kind};+}+function passes(r){expect(r.status).toBe(0);expect(r.stdout).toContain('18 passed');expect(r.kind).toBeNull();}+function fails(r){expect(r.status).toBe(1);expect(r.kind).toBeNull();expect(r.stdout).toMatch(/Tests\s+[1-9]\d* failed/);expect(r.stderr).not.toContain('TypeError:');}+test('parent fails intended polling and cancellation outcomes',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('passed');},60000);+test('reference passes all outcomes',()=>passes(grade('reference')),60000);+test('current source passes all outcomes',()=>passes(grade('current','HEAD')),60000);+test('equivalent deadline comparison passes',()=>passes(grade('equivalent',item.oracle_ref,cwd=>replace(cwd,source,'remaining <= 0','remaining < 1'))),60000);+test('unused fixed module cannot repair the active parent hook',()=>fails(grade('unused',item.base_ref,cwd=>writeFileSync(join(cwd,source+'.unused.txt'),execFileSync('git',['show',`${item.oracle_ref}:${source}`],{cwd:repo})))),60000);+test('blanket suppression loses valid preview',()=>fails(grade('disabled',item.oracle_ref,cwd=>replace(cwd,source,'void refresh();','// preview disabled'))),60000);+test('missing backoff fails',()=>fails(grade('backoff',item.oracle_ref,cwd=>replace(cwd,source,'Math.min(delay * 2, LEARNING_WINDOW_CEILING_MS)','LEARNING_POLL_INTERVAL_MS'))),60000);+test('missing cancellation fails',()=>fails(grade('cancel',item.oracle_ref,cwd=>{const f=join(cwd,source);writeFileSync(f,readFileSync(f,'utf8').replaceAll('controller.abort();','/* no cancellation */'));})),60000);+test('lost persisted dismissal fails preserved behavior',()=>fails(grade('dismissal',item.oracle_ref,cwd=>replace(cwd,source,' setState(markLearningDone());\n }, [state.phase',' setState(readLearningWindow());\n }, [state.phase'))),60000);+test('missing source remains setup failure',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,source)));expect(r.status).toBe(1);expect(r.kind).toBe('vitest_collection_error');},60000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex eedc5a739..ba6763670 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -4265,6 +4265,44 @@],"command": "cd apps/screenpipe-app-tauri && bun x vitest run --config vitest.config.ts lib/auth-schedule.eval.test.tsx --maxWorkers=1 --minWorkers=1"}+ },+ {+ "id": "app-bounded-first-run-preview",+ "name": "Bound native-authorized first-run activity previews",+ "source": {+ "kind": "git_regression",+ "fix_commit": "dd652a6dc5db63832e8b7b913af8151c502608cc"+ },+ "base_ref": "34fb4aec4995c3b0874db8565c198df9283968e0",+ "oracle_ref": "dd652a6dc5db63832e8b7b913af8151c502608cc",+ "tags": [+ "regression",+ "escaped-failure",+ "onboarding",+ "polling",+ "cancellation"+ ],+ "prompt": "The first-run activity preview can repeatedly query history from restored browser state, overlap slow reads and continue after native summary generation takes over. Fix the actual learning-window hook. Preview reads require a completed native onboarding status in its learning phase and the native start time. Restored state alone, pending native status, idle/writing phases and an expired window must not authorize reads. Keep one read in flight; after a successful completion, wait the existing three-second preview interval before the next read. A null read must back off to six seconds, then twelve seconds for the next consecutive null result. The existing learning deadline must not restart after navigation or reload. Cancel pending reads when the hook unmounts, the existing deadline expires or native generation starts writing, and ignore late results without changing the preview or starting more reads. Successful previews must still display their returned app counts and minutes and use the native time window. Preserve native ready-chat restoration, unread/open summaries across reload, explicit dismissal across remount, native writing projection and the summary-first trial paywall transition only after rendering. A restored summary behind an already persisted treatment paywall must remain saved. Tests exercise the actual hook and storage helpers with synthetic native status/event, analytics, clock and activity-read ports. Native recording, SQL history queries and server generation are outside this task. Add or update focused tests as needed.",+ "agent_timeout_seconds": 900,+ "grader": {+ "timeout_seconds": 90,+ "fixtures": [+ {+ "local_path": "graders/bounded-preview.fixture.ts.txt",+ "destination_path": "apps/screenpipe-app-tauri/lib/first-run/eval-bounded-preview.test.ts"+ }+ ],+ "command": "cd apps/screenpipe-app-tauri && bun x --no-install vitest run --config vitest.config.ts lib/first-run/eval-bounded-preview.test.ts --maxWorkers=1 --minWorkers=1"+ },+ "oracle_paths": [+ "apps/screenpipe-app-tauri/lib/first-run/use-learning-window.ts"+ ],+ "grader_dependency_links": [+ {+ "source_path": "apps/screenpipe-app-tauri/node_modules",+ "destination_path": "apps/screenpipe-app-tauri/node_modules"+ }+ ]}]}diff --git a/evals/coding-agent/graders/bounded-preview.fixture.ts.txt b/evals/coding-agent/graders/bounded-preview.fixture.ts.txtnew file mode 100644index 000000000..ef08919f9--- /dev/null+++ b/evals/coding-agent/graders/bounded-preview.fixture.ts.txt@@ -0,0 +1,337 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++import { act, renderHook, waitFor } from "@testing-library/react";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";++const mocks = vi.hoisted(() => ({+ getOnboardingStatus: vi.fn(),+ setOnboardingStep: vi.fn(async () => undefined),+ capture: vi.fn(),+ fetchRecentActivity: vi.fn().mockResolvedValue(null),+}));++vi.mock("posthog-js", () => ({ default: { capture: mocks.capture } }));+vi.mock("@tauri-apps/api/event", () => ({+ listen: vi.fn(async () => () => {}),+}));+vi.mock("@/lib/utils/tauri", () => ({+ commands: {+ getOnboardingStatus: mocks.getOnboardingStatus,+ setOnboardingStep: mocks.setOnboardingStep,+ },+}));+vi.mock("@/lib/first-run/recent-activity", () => ({+ fetchRecentActivity: mocks.fetchRecentActivity,+}));++import { useLearningWindow } from "./use-learning-window";+import {+ LEARNING_WINDOW_CEILING_MS,+ beginLearningWindow,+ markLearningReady,+ markLearningReadyShown,+ markLearningSummaryOpened,+ readLearningWindow,+} from "./learning-window";++function nativeStatus(+ phase: string,+ chatId: string | null = null,+ currentStep: string | null = null,+) {+ return {+ status: "ok" as const,+ data: {+ isCompleted: true,+ completedAt: "2026-08-27T07:00:00.000Z",+ currentStep,+ firstRunSummaryPhase: phase,+ firstRunSummaryStartedAt: "2026-08-27T07:00:00.000Z",+ firstRunSummaryChatId: chatId,+ firstRunSummaryNotificationSentAt: null,+ firstRunSummaryError: null,+ firstRunSummaryTelemetryVersion: 2,+ },+ };+}++beforeEach(() => {+ if (!window.localStorage) {+ const values = new Map<string, string>();+ Object.defineProperty(window, "localStorage", {+ configurable: true,+ value: {+ clear: () => values.clear(),+ getItem: (key: string) => values.get(key) ?? null,+ setItem: (key: string, value: string) => values.set(key, value),+ removeItem: (key: string) => values.delete(key),+ },+ });+ }+ window.localStorage.clear();+ vi.clearAllMocks();+ mocks.fetchRecentActivity.mockResolvedValue(null);+});++describe("native first-run summary projection", () => {+ it("shows the native writing phase without starting an agent in React", async () => {+ mocks.getOnboardingStatus.mockResolvedValue(nativeStatus("writing"));+ const { result } = renderHook(() => useLearningWindow());++ await waitFor(() => expect(result.current.phase).toBe("writing"));+ expect(mocks.getOnboardingStatus).toHaveBeenCalled();+ });++ it("rehydrates the durable native chat after a webview reload", async () => {+ mocks.getOnboardingStatus.mockResolvedValue(+ nativeStatus("ready", "first-run-native-chat"),+ );+ const { result } = renderHook(() => useLearningWindow());++ await waitFor(() => expect(result.current.phase).toBe("ready"));+ expect(result.current.chatId).toBe("first-run-native-chat");+ expect(mocks.capture).not.toHaveBeenCalledWith(+ "first_run_learning_resolved",+ expect.anything(),+ );+ });++ it("preserves an unread ready card even if an older version stamped it shown", async () => {+ beginLearningWindow();+ markLearningReady("first-run-native-chat");+ markLearningReadyShown("2026-08-27T07:05:00.000Z");+ mocks.getOnboardingStatus.mockResolvedValue(+ nativeStatus("ready", "first-run-native-chat"),+ );++ const { result } = renderHook(() => useLearningWindow());++ await waitFor(() => expect(mocks.getOnboardingStatus).toHaveBeenCalled());+ expect(result.current.phase).toBe("ready");+ expect(readLearningWindow().chatId).toBe("first-run-native-chat");+ });++ it("preserves an open request across remounts until the summary renders", async () => {+ mocks.getOnboardingStatus.mockResolvedValue(+ nativeStatus("ready", "first-run-native-chat"),+ );+ const first = renderHook(() => useLearningWindow());+ await waitFor(() => expect(first.result.current.phase).toBe("ready"));+ act(() => first.result.current.markSummaryOpened());+ first.unmount();++ const restored = renderHook(() => useLearningWindow());+ expect(restored.result.current.phase).toBe("ready");+ await act(async () => restored.result.current.markSummaryRendered());+ expect(readLearningWindow().phase).toBe("done");+ expect(mocks.setOnboardingStep).not.toHaveBeenCalled();+ restored.unmount();++ const consumed = renderHook(() => useLearningWindow());+ await act(async () => {});+ expect(consumed.result.current.phase).toBe("done");+ });++ it("keeps an explicit dismissal retired after remounting", async () => {+ mocks.getOnboardingStatus.mockResolvedValue(+ nativeStatus("ready", "first-run-native-chat"),+ );+ const first = renderHook(() => useLearningWindow());+ await waitFor(() => expect(first.result.current.phase).toBe("ready"));+ act(() => first.result.current.dismiss());+ first.unmount();++ const restored = renderHook(() => useLearningWindow());+ await act(async () => {});+ expect(restored.result.current.phase).toBe("done");+ });++ it("retains a treatment summary when it renders before the initial status poll", async () => {+ beginLearningWindow();+ markLearningReady("first-run-native-chat");+ markLearningSummaryOpened();+ mocks.getOnboardingStatus+ .mockReturnValueOnce(new Promise(() => {}))+ .mockResolvedValue(nativeStatus("ready", "first-run-native-chat", "trial-activation-v1-summary"));+ const { result } = renderHook(() => useLearningWindow());++ await act(async () => result.current.markSummaryRendered());++ expect(result.current.activationState).toBe("paywall");+ expect(readLearningWindow().phase).toBe("ready");+ expect(readLearningWindow().summaryOpenedAt).not.toBeNull();+ });++ it("retains the saved summary behind a persisted treatment paywall", async () => {+ beginLearningWindow();+ markLearningReady("first-run-native-chat");+ markLearningSummaryOpened();+ mocks.getOnboardingStatus.mockResolvedValue(+ nativeStatus("ready", "first-run-native-chat", "trial-activation-v1-paywall"),+ );+ const { result } = renderHook(() => useLearningWindow());++ await act(async () => result.current.markSummaryRendered());++ expect(readLearningWindow().phase).toBe("ready");+ expect(mocks.setOnboardingStep).not.toHaveBeenCalled();+ });++ it("persists the paywall only after the treatment summary reports a render", async () => {+ mocks.getOnboardingStatus.mockResolvedValue(+ nativeStatus(+ "ready",+ "first-run-native-chat",+ "trial-activation-v1-summary",+ ),+ );+ const { result } = renderHook(() => useLearningWindow());++ await waitFor(() => expect(result.current.activationState).toBe("summary"));+ await act(async () => result.current.markSummaryRendered());++ expect(mocks.setOnboardingStep).toHaveBeenCalledWith(+ "trial-activation-v1-paywall",+ );+ expect(mocks.capture).toHaveBeenCalledWith(+ "first_run_summary_rendered",+ expect.objectContaining({ variant: "summary_first" }),+ );+ expect(result.current.activationState).toBe("paywall");+ });+});++// Slow/erroring reads and stale storage are the battery-drain boundary: the+// preview must never turn native generation retries into full-history polling.+describe("bounded first-run preview", () => {+ const startedAt = "2026-09-29T12:00:00.000Z";+ beforeEach(() => {+ vi.useFakeTimers();+ vi.setSystemTime(new Date(startedAt));+ mocks.getOnboardingStatus.mockResolvedValue({+ ...nativeStatus("learning"),+ data: { ...nativeStatus("learning").data, firstRunSummaryStartedAt: startedAt },+ });+ });+ afterEach(() => { vi.clearAllTimers(); vi.useRealTimers(); });++ it("waits for native authorization before reading restored state", async () => {+ beginLearningWindow(startedAt, true);+ mocks.getOnboardingStatus.mockReturnValue(new Promise(() => {}));+ const hook = renderHook(() => useLearningWindow());+ await act(async () => {});+ await act(async () => { await vi.advanceTimersByTimeAsync(15000); });+ expect(mocks.fetchRecentActivity).not.toHaveBeenCalled();+ hook.unmount();+ });++ it("waits for a slow read to finish and cancels it on unmount", async () => {+ let finish!: (value: unknown) => void;+ mocks.fetchRecentActivity.mockImplementation(() => new Promise((resolve) => { finish = resolve; }));+ const hook = renderHook(() => useLearningWindow());+ await act(async () => {});+ await act(async () => { await vi.advanceTimersByTimeAsync(15000); });+ expect(mocks.fetchRecentActivity).toHaveBeenCalledTimes(1);+ const signal = mocks.fetchRecentActivity.mock.calls[0]?.[1]?.signal;+ expect(signal?.aborted).toBe(false);+ await act(async () => { finish({ apps: [] }); });+ await act(async () => { await vi.advanceTimersByTimeAsync(2999); });+ expect(mocks.fetchRecentActivity).toHaveBeenCalledTimes(1);+ await act(async () => { await vi.advanceTimersByTimeAsync(1); });+ expect(mocks.fetchRecentActivity).toHaveBeenCalledTimes(2);+ hook.unmount();+ expect(signal?.aborted).toBe(true);+ await act(async () => { finish({ apps: [] }); await vi.advanceTimersByTimeAsync(15000); });+ expect(mocks.fetchRecentActivity).toHaveBeenCalledTimes(2);+ });++ it("backs off unsuccessful reads instead of retrying each tick", async () => {+ const hook = renderHook(() => useLearningWindow());+ await act(async () => {});+ await act(async () => { await vi.advanceTimersByTimeAsync(5999); });+ expect(mocks.fetchRecentActivity).toHaveBeenCalledTimes(1);+ await act(async () => { await vi.advanceTimersByTimeAsync(1); });+ expect(mocks.fetchRecentActivity).toHaveBeenCalledTimes(2);+ await act(async () => { await vi.advanceTimersByTimeAsync(11999); });+ expect(mocks.fetchRecentActivity).toHaveBeenCalledTimes(2);+ await act(async () => { await vi.advanceTimersByTimeAsync(1); });+ expect(mocks.fetchRecentActivity).toHaveBeenCalledTimes(3);+ hook.unmount();+ });++ it.each(["idle", "writing"])("does not poll while the native owner is %s", async (phase) => {+ beginLearningWindow(startedAt, true);+ mocks.getOnboardingStatus.mockResolvedValue(nativeStatus(phase));+ const hook = renderHook(() => useLearningWindow());+ await act(async () => {});+ await act(async () => { await vi.advanceTimersByTimeAsync(15000); });+ expect(mocks.fetchRecentActivity).not.toHaveBeenCalled();+ if (phase === "idle") expect(hook.result.current.phase).toBe("idle");+ hook.unmount();+ });++ it("does not query an expired learning window after restart", async () => {+ const oldStart = "2026-09-22T07:43:10.000Z";+ beginLearningWindow(oldStart, true);+ mocks.getOnboardingStatus.mockResolvedValue({+ ...nativeStatus("learning"),+ data: { ...nativeStatus("learning").data, firstRunSummaryStartedAt: oldStart },+ });+ const hook = renderHook(() => useLearningWindow());+ await act(async () => {});+ await act(async () => { await vi.advanceTimersByTimeAsync(15000); });+ expect(mocks.fetchRecentActivity).not.toHaveBeenCalled();+ expect(hook.result.current.remainingMs).toBe(0);+ hook.unmount();+ });++ it("aborts an unfinished preview at the existing learning deadline", async () => {+ mocks.fetchRecentActivity.mockImplementation(() => new Promise(() => {}));+ const hook = renderHook(() => useLearningWindow());+ await act(async () => {});+ await act(async () => { await vi.advanceTimersByTimeAsync(LEARNING_WINDOW_CEILING_MS); });+ expect(mocks.fetchRecentActivity).toHaveBeenCalledTimes(1);+ expect(mocks.fetchRecentActivity.mock.calls[0]?.[1]?.signal?.aborted).toBe(true);+ expect(hook.result.current.remainingMs).toBe(0);+ hook.unmount();+ });++ it("cancels the preview when native generation starts writing", async () => {+ mocks.fetchRecentActivity.mockImplementation(() => new Promise(() => {}));+ const hook = renderHook(() => useLearningWindow());+ await act(async () => {});+ await act(async () => { await vi.advanceTimersByTimeAsync(0); });+ expect(mocks.fetchRecentActivity).toHaveBeenCalledTimes(1);+ const signal = mocks.fetchRecentActivity.mock.calls[0]?.[1]?.signal;+ mocks.getOnboardingStatus.mockResolvedValue(nativeStatus("writing"));+ await act(async () => { await vi.advanceTimersByTimeAsync(1000); });+ expect(signal?.aborted).toBe(true);+ expect(hook.result.current.phase).toBe("writing");+ hook.unmount();+ });+ it("preserves a successful preview and queries its native time window", async () => {+ mocks.fetchRecentActivity.mockResolvedValue({ apps: [{ name: "Synthetic editor", frame_count: 4, minutes: 2 }] });+ const hook = renderHook(() => useLearningWindow());+ await act(async () => {});+ expect(mocks.fetchRecentActivity.mock.calls[0]?.[0]).toBe(startedAt);+ expect(hook.result.current.capturedApps).toEqual([expect.objectContaining({ name: "Synthetic editor", frameCount: 4, activeMinutes: 2 })]);+ await act(async () => { await vi.advanceTimersByTimeAsync(3000); });+ expect(mocks.fetchRecentActivity).toHaveBeenCalledTimes(2);+ hook.unmount();+ });++ it("does not publish a slow preview after its lifetime has ended", async () => {+ let finish!: (value: unknown) => void;+ mocks.fetchRecentActivity.mockImplementation(() => new Promise(resolve => { finish = resolve; }));+ const hook = renderHook(() => useLearningWindow());+ await act(async () => {});+ await act(async () => { await vi.advanceTimersByTimeAsync(LEARNING_WINDOW_CEILING_MS); });+ expect(typeof finish).toBe("function");+ await act(async () => { finish({ apps: [{ name: "Expired result", frame_count: 1, minutes: 1 }] }); });+ expect(hook.result.current.capturedApps).toEqual([]);+ expect(hook.result.current.remainingMs).toBe(0);+ hook.unmount();+ });++});