YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: cover login prompts after account hydration
louis0f4600f4d9a9
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex f6eaab684..bc0cee93d 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -9,10 +9,23 @@ This is an agent eval suite, not a unit-test suite. Every case contains:- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.-The current app corpus contains 78 git-mined regressions. See+The current app corpus contains 79 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.+`app-card-ask-login-hydration` runs the actual provider, controller and dialog+with synthetic external ports. Nine outcomes cover initial and delayed account+loading, post-mount sign-in, once-per-install persistence and suppressed prompts.+The parent fails two intended outcomes and preserves seven; the reference passes+nine. Run `bun test evals/coding-agent/calibrate-card-login.test.js` with desktop+test dependencies installed for eight controls, including unused correct code,+equivalent private names/test attributes, lost suppression or persistence, and+missing-source setup classification. Current product provider tests separately+pass twelve outcomes with current rollout controls. This historical configured+flow does not authorize enabling a rollout. Graders and dependency links are+installed only when grading starts; no live checkout, native behavior, execution+isolation or model capability is established.+`app-business-capacity-plan-picker` drives the actual account settings componentwith synthetic native, settings and telemetry ports. Eight outcomes cover visiblecapacity plans and prices, exact billing targets, current-plan labeling, the nextdiff --git a/evals/coding-agent/calibrate-card-login.test.js b/evals/coding-agent/calibrate-card-login.test.jsnew file mode 100644index 000000000..6b336fb06--- /dev/null+++ b/evals/coding-agent/calibrate-card-login.test.js@@ -0,0 +1,52 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { dirname, join, resolve } from 'node:path';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+const repo = resolve(import.meta.dir, '../..'), app = 'apps/screenpipe-app-tauri';+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'app-card-ask-login-hydration');+const provider = `${app}/components/card-ask-provider.tsx`, hook = `${app}/lib/hooks/use-card-ask.tsx`, gating = `${app}/lib/card-ask/gating.ts`;+const root = mkdtempSync(join(tmpdir(), 'card-login-calibration-')), archives = new Map();+const receipts = process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;+afterAll(() => rmSync(root, {recursive: true, force: true}));+function replace(cwd, file, from, to) {+ const p = join(cwd, file), text = readFileSync(p, 'utf8');+ expect(text.split(from)).toHaveLength(2);+ writeFileSync(p, text.replace(from, to));+}+function grade(name, ref = item.oracle_ref, mutate = () => {}) {+ const cwd = join(root, name); mkdirSync(cwd);+ if (!archives.has(ref)) {+ // Archive historical frontend source/config, never node_modules, credentials,+ // native build outputs or future source. No evaluated agent runs here.+ const children = execFileSync('git', ['ls-tree', '--name-only', `${ref}:${app}`], {cwd: repo, encoding: 'utf8'}).trim().split('\n');+ const paths = children.filter(n => !['src-tauri', 'public', 'e2e', '.e2e'].includes(n) && !n.startsWith('.env')).map(n => `${app}/${n}`);++ archives.set(ref, execFileSync('git', ['archive', ref, ...paths], {cwd: repo, maxBuffer: 128 * 1024 * 1024}));+ }+ execFileSync('tar', ['-x', '-C', cwd], {input: archives.get(ref)});+ for (const f of item.grader.fixtures) { const dest = join(cwd, f.destination_path); mkdirSync(dirname(dest), {recursive:true}); writeFileSync(dest, readFileSync(join(import.meta.dir, f.local_path))); }+ symlinkSync(join(repo, app, 'node_modules'), join(cwd, app, 'node_modules'), 'dir');+ mutate(cwd);+ const r = spawnSync('/bin/bash', ['-c', item.grader.command], {cwd, encoding:'utf8', timeout:60000, maxBuffer:4*1024*1024, env:{PATH:process.env.PATH, HOME:cwd, CI:'true', TZ:'UTC', NO_COLOR:'1'}});+ if (receipts) { mkdirSync(receipts,{recursive:true}); writeFileSync(join(receipts, name+'.stdout'), r.stdout||''); writeFileSync(join(receipts,name+'.stderr'),r.stderr||''); writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message},null,2)); }+ expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); return r;+}+function passes(r) { expect(r.status).toBe(0); expect(r.stdout).toContain('9 passed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught/); }+function fails(r) { expect(r.status).toBe(1); expect(r.stdout).toContain('failed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/); }+test('parent loses delayed account prompts and preserves seven outcomes',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('2 failed | 7 passed');},60000);+test('reference passes all nine outcomes',()=>passes(grade('reference')),60000);+test('equivalent private names and removed test ids pass',()=>passes(grade('equivalent',item.oracle_ref,cwd=>{+ for (const file of [provider,hook]) { const p=join(cwd,file);writeFileSync(p,readFileSync(p,'utf8').replaceAll('activeTrigger','displayTrigger')); }+ const p=join(cwd,`${app}/components/card-ask-modal.tsx`);writeFileSync(p,readFileSync(p,'utf8').replaceAll('data-testid=','data-eval-neutral='));+})),60000);+test('unused correct provider does not hide broken active provider',()=>fails(grade('unused',item.oracle_ref,cwd=>{+ writeFileSync(join(cwd,`${app}/components/unused-card-provider.tsx`),readFileSync(join(cwd,provider)));+ writeFileSync(join(cwd,provider),execFileSync('git',['show',`${item.base_ref}:${provider}`],{cwd:repo}));+})),60000);+test('blanket login silence fails',()=>fails(grade('silent',item.oracle_ref,cwd=>replace(cwd,provider,'emitCardAskTrigger("login");','void 0;'))),60000);+test('payment-method suppression remains required',()=>fails(grade('payer',item.oracle_ref,cwd=>replace(cwd,gating,'if (user.has_payment_method === true) return false;','if (user.has_payment_method === true) return true;'))),60000);+test('lost persistent once-only behavior fails',()=>fails(grade('repeat',item.oracle_ref,cwd=>replace(cwd,hook,'JSON.stringify(shownRef.current),','JSON.stringify([]),'))),60000);+test('missing provider is setup error',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,provider)));expect(r.status).toBe(1);expect(r.stdout+r.stderr).toMatch(/Failed to resolve import|Failed to load url/);},60000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex a12011cd1..56fa70c79 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-09-24.1",+ "dataset_version": "2026-09-25.1","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -3223,6 +3223,48 @@"trigger_paths": ["crates/screenpipe-core/assets/extensions/screenpipe-permissions.ts"]+ },+ {+ "id": "app-card-ask-login-hydration",+ "name": "Preserve an eligible login prompt after account hydration",+ "source": {+ "kind": "git_regression",+ "fix_commit": "1eb83b93c716ca27d7c91bbd74f886ff8e429872"+ },+ "base_ref": "36d84b89d54597c1a672eaa00bbfc653f1873ffc",+ "oracle_ref": "1eb83b93c716ca27d7c91bbd74f886ff8e429872",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/components/card-ask-provider.tsx"+ ],+ "gate": "advisory",+ "trigger_paths": [+ "apps/screenpipe-app-tauri/components/card-ask-provider.tsx",+ "apps/screenpipe-app-tauri/lib/hooks/use-card-ask.tsx"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "billing",+ "account-hydration"+ ],+ "prompt": "The Home login card prompt can disappear for an entire session when its timing variant resolves before account settings or when a user signs in after Home mounts. For the configured at_login variant, show the existing dialog once an eligible signed-in account without a payment method is available, whether loaded initially or later. Keep it hidden until settings and account identity are available. Preserve once-per-install suppression across unmount and remount with persisted browser storage, suppression for accounts with a payment method, silence in control, silence for a timing variant that does not own login, and silence while the variant is unresolved. Preserve the public CardAskProvider component interface. No particular effect dependencies, private names, data-test attributes, storage representation or modal wording are required. Exercise real provider/controller/dialog behavior with synthetic settings, feature-flag, native-platform and telemetry ports. This is the historical configured login flow; it does not authorize enabling any current rollout or changing payment, recording or native behavior.",+ "agent_timeout_seconds": 900,+ "grader_dependency_links": [+ {+ "source_path": "apps/screenpipe-app-tauri/node_modules",+ "destination_path": "apps/screenpipe-app-tauri/node_modules"+ }+ ],+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/card-login-hydration.fixture.tsx.txt",+ "destination_path": "apps/screenpipe-app-tauri/components/__tests__/eval-card-login-hydration.test.tsx"+ }+ ],+ "command": "cd apps/screenpipe-app-tauri && NODE_OPTIONS=--localstorage-file=.eval-localstorage node node_modules/vitest/vitest.mjs run --config vitest.config.ts components/__tests__/eval-card-login-hydration.test.tsx --maxWorkers=1 --minWorkers=1"+ }}]}diff --git a/evals/coding-agent/graders/card-login-hydration.fixture.tsx.txt b/evals/coding-agent/graders/card-login-hydration.fixture.tsx.txtnew file mode 100644index 000000000..dc1a36de9--- /dev/null+++ b/evals/coding-agent/graders/card-login-hydration.fixture.tsx.txt@@ -0,0 +1,143 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++import { render, screen } from "@testing-library/react";+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";+import {+ CARD_ASK_ARM_STORAGE_KEY,+} from "@/lib/card-ask/gating";+import { resetCardAskTriggerBus } from "@/lib/card-ask/trigger-bus";++++let flagVariant: string | undefined = "at_login";+let settingsState: { settings: any; isSettingsLoaded: boolean };++vi.mock("posthog-js/react", () => ({+ useFeatureFlagVariantKey: () => flagVariant,+}));++vi.mock("@/lib/hooks/use-settings", () => ({+ useSettings: () => settingsState,+}));++vi.mock("@tauri-apps/plugin-os", () => ({ platform: () => "macos" }));++vi.mock("posthog-js", () => ({ default: { capture: vi.fn() } }));++vi.mock("@/lib/open-external-url", () => ({+ openExternalUrl: vi.fn(async () => {}),+}));++import { CardAskProvider } from "@/components/card-ask-provider";++const localStorageValues = new Map<string, string>();+const localStorageMock = {+ getItem: (key: string) => localStorageValues.get(key) ?? null,+ setItem: (key: string, value: string) => {+ localStorageValues.set(key, value);+ },+ removeItem: (key: string) => {+ localStorageValues.delete(key);+ },+ clear: () => {+ localStorageValues.clear();+ },+};+++const cardlessUser = {+ id: "synthetic-user",+ email: "[email protected]",+ has_payment_method: false,+};++function reset() {+ localStorageValues.clear();+ Object.defineProperty(window, "localStorage", {+ configurable: true,+ value: localStorageMock,+ });+ resetCardAskTriggerBus();+ flagVariant = "at_login";+ settingsState = { settings: { user: cardlessUser }, isSettingsLoaded: true };+}++beforeEach(reset);+afterEach(reset);++function modal() {+ return screen.queryByRole("dialog");+}++describe("CardAskProvider login trigger", () => {+ it("shows the login ask for an eligible user in at_login", () => {+ render(<CardAskProvider />);+ expect(modal()).not.toBeNull();+ });++ it("still shows it when settings finish loading after the arm resolves", () => {+ settingsState = { settings: { user: null }, isSettingsLoaded: false };+ const { rerender } = render(<CardAskProvider />);+ expect(modal()).toBeNull();++ settingsState = { settings: { user: cardlessUser }, isSettingsLoaded: true };+ rerender(<CardAskProvider />);+ expect(modal()).not.toBeNull();+ });++ it("shows it when the user signs in after Home is already mounted", () => {+ settingsState = { settings: { user: null }, isSettingsLoaded: true };+ const { rerender } = render(<CardAskProvider />);+ expect(modal()).toBeNull();++ settingsState = { settings: { user: cardlessUser }, isSettingsLoaded: true };+ rerender(<CardAskProvider />);+ expect(modal()).not.toBeNull();+ });++ it("re-emitting on every settings change still asks only once", () => {+ const first = render(<CardAskProvider />);+ expect(modal()).not.toBeNull();+ first.unmount();++ resetCardAskTriggerBus();+ render(<CardAskProvider />);+ expect(modal()).toBeNull();+ });++ it("stays silent for a user with a card on file", () => {+ settingsState = {+ settings: { user: { ...cardlessUser, has_payment_method: true } },+ isSettingsLoaded: true,+ };+ render(<CardAskProvider />);+ expect(modal()).toBeNull();+ });++ it("stays silent in control even once settings load", () => {+ flagVariant = "control";+ settingsState = { settings: { user: null }, isSettingsLoaded: false };+ const { rerender } = render(<CardAskProvider />);+ settingsState = { settings: { user: cardlessUser }, isSettingsLoaded: true };+ rerender(<CardAskProvider />);+ expect(modal()).toBeNull();+ });++ it("does not fire login for an arm that does not own it", () => {+ flagVariant = "at_limit";+ localStorageValues.set(CARD_ASK_ARM_STORAGE_KEY, "at_limit");+ render(<CardAskProvider />);+ expect(modal()).toBeNull();+ });+});+it("stays silent without a signed-in account", () => {+ settingsState = { settings: { user: null }, isSettingsLoaded: true };+ render(<CardAskProvider />);+ expect(modal()).toBeNull();+});+it("stays silent while the timing variant is unresolved", () => {+ flagVariant = undefined;+ render(<CardAskProvider />);+ expect(modal()).toBeNull();+});