Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: cover login prompts after account hydration

louis0f4600f4d9a9
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index f6eaab684..bc0cee93d 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -9,10 +9,23 @@ This is an agent eval suite, not a unit-test suite. Every case contains:
- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;
- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.
-The current app corpus contains 78 git-mined regressions. See
+The current app corpus contains 79 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest uses this same harness.
+`app-card-ask-login-hydration` runs the actual provider, controller and dialog
+with synthetic external ports. Nine outcomes cover initial and delayed account
+loading, post-mount sign-in, once-per-install persistence and suppressed prompts.
+The parent fails two intended outcomes and preserves seven; the reference passes
+nine. Run `bun test evals/coding-agent/calibrate-card-login.test.js` with desktop
+test dependencies installed for eight controls, including unused correct code,
+equivalent private names/test attributes, lost suppression or persistence, and
+missing-source setup classification. Current product provider tests separately
+pass twelve outcomes with current rollout controls. This historical configured
+flow does not authorize enabling a rollout. Graders and dependency links are
+installed only when grading starts; no live checkout, native behavior, execution
+isolation or model capability is established.
+
`app-business-capacity-plan-picker` drives the actual account settings component
with synthetic native, settings and telemetry ports. Eight outcomes cover visible
capacity plans and prices, exact billing targets, current-plan labeling, the next
diff --git a/evals/coding-agent/calibrate-card-login.test.js b/evals/coding-agent/calibrate-card-login.test.js
new file mode 100644
index 000000000..6b336fb06
--- /dev/null
+++ b/evals/coding-agent/calibrate-card-login.test.js
@@ -0,0 +1,52 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { dirname, join, resolve } from 'node:path';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+const repo = resolve(import.meta.dir, '../..'), app = 'apps/screenpipe-app-tauri';
+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'app-card-ask-login-hydration');
+const provider = `${app}/components/card-ask-provider.tsx`, hook = `${app}/lib/hooks/use-card-ask.tsx`, gating = `${app}/lib/card-ask/gating.ts`;
+const root = mkdtempSync(join(tmpdir(), 'card-login-calibration-')), archives = new Map();
+const receipts = process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;
+afterAll(() => rmSync(root, {recursive: true, force: true}));
+function replace(cwd, file, from, to) {
+ const p = join(cwd, file), text = readFileSync(p, 'utf8');
+ expect(text.split(from)).toHaveLength(2);
+ writeFileSync(p, text.replace(from, to));
+}
+function grade(name, ref = item.oracle_ref, mutate = () => {}) {
+ const cwd = join(root, name); mkdirSync(cwd);
+ if (!archives.has(ref)) {
+ // Archive historical frontend source/config, never node_modules, credentials,
+ // native build outputs or future source. No evaluated agent runs here.
+ const children = execFileSync('git', ['ls-tree', '--name-only', `${ref}:${app}`], {cwd: repo, encoding: 'utf8'}).trim().split('\n');
+ const paths = children.filter(n => !['src-tauri', 'public', 'e2e', '.e2e'].includes(n) && !n.startsWith('.env')).map(n => `${app}/${n}`);
+
+ archives.set(ref, execFileSync('git', ['archive', ref, ...paths], {cwd: repo, maxBuffer: 128 * 1024 * 1024}));
+ }
+ execFileSync('tar', ['-x', '-C', cwd], {input: archives.get(ref)});
+ for (const f of item.grader.fixtures) { const dest = join(cwd, f.destination_path); mkdirSync(dirname(dest), {recursive:true}); writeFileSync(dest, readFileSync(join(import.meta.dir, f.local_path))); }
+ symlinkSync(join(repo, app, 'node_modules'), join(cwd, app, 'node_modules'), 'dir');
+ mutate(cwd);
+ const r = spawnSync('/bin/bash', ['-c', item.grader.command], {cwd, encoding:'utf8', timeout:60000, maxBuffer:4*1024*1024, env:{PATH:process.env.PATH, HOME:cwd, CI:'true', TZ:'UTC', NO_COLOR:'1'}});
+ if (receipts) { mkdirSync(receipts,{recursive:true}); writeFileSync(join(receipts, name+'.stdout'), r.stdout||''); writeFileSync(join(receipts,name+'.stderr'),r.stderr||''); writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message},null,2)); }
+ expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); return r;
+}
+function passes(r) { expect(r.status).toBe(0); expect(r.stdout).toContain('9 passed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught/); }
+function fails(r) { expect(r.status).toBe(1); expect(r.stdout).toContain('failed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/); }
+test('parent loses delayed account prompts and preserves seven outcomes',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('2 failed | 7 passed');},60000);
+test('reference passes all nine outcomes',()=>passes(grade('reference')),60000);
+test('equivalent private names and removed test ids pass',()=>passes(grade('equivalent',item.oracle_ref,cwd=>{
+ for (const file of [provider,hook]) { const p=join(cwd,file);writeFileSync(p,readFileSync(p,'utf8').replaceAll('activeTrigger','displayTrigger')); }
+ const p=join(cwd,`${app}/components/card-ask-modal.tsx`);writeFileSync(p,readFileSync(p,'utf8').replaceAll('data-testid=','data-eval-neutral='));
+})),60000);
+test('unused correct provider does not hide broken active provider',()=>fails(grade('unused',item.oracle_ref,cwd=>{
+ writeFileSync(join(cwd,`${app}/components/unused-card-provider.tsx`),readFileSync(join(cwd,provider)));
+ writeFileSync(join(cwd,provider),execFileSync('git',['show',`${item.base_ref}:${provider}`],{cwd:repo}));
+})),60000);
+test('blanket login silence fails',()=>fails(grade('silent',item.oracle_ref,cwd=>replace(cwd,provider,'emitCardAskTrigger("login");','void 0;'))),60000);
+test('payment-method suppression remains required',()=>fails(grade('payer',item.oracle_ref,cwd=>replace(cwd,gating,'if (user.has_payment_method === true) return false;','if (user.has_payment_method === true) return true;'))),60000);
+test('lost persistent once-only behavior fails',()=>fails(grade('repeat',item.oracle_ref,cwd=>replace(cwd,hook,'JSON.stringify(shownRef.current),','JSON.stringify([]),'))),60000);
+test('missing provider is setup error',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,provider)));expect(r.status).toBe(1);expect(r.stdout+r.stderr).toMatch(/Failed to resolve import|Failed to load url/);},60000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index a12011cd1..56fa70c79 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-09-24.1",
+ "dataset_version": "2026-09-25.1",
"suite_type": "regression",
"owner": "screenpipe engineering",
"policy": {
@@ -3223,6 +3223,48 @@
"trigger_paths": [
"crates/screenpipe-core/assets/extensions/screenpipe-permissions.ts"
]
+ },
+ {
+ "id": "app-card-ask-login-hydration",
+ "name": "Preserve an eligible login prompt after account hydration",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "1eb83b93c716ca27d7c91bbd74f886ff8e429872"
+ },
+ "base_ref": "36d84b89d54597c1a672eaa00bbfc653f1873ffc",
+ "oracle_ref": "1eb83b93c716ca27d7c91bbd74f886ff8e429872",
+ "oracle_paths": [
+ "apps/screenpipe-app-tauri/components/card-ask-provider.tsx"
+ ],
+ "gate": "advisory",
+ "trigger_paths": [
+ "apps/screenpipe-app-tauri/components/card-ask-provider.tsx",
+ "apps/screenpipe-app-tauri/lib/hooks/use-card-ask.tsx"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "billing",
+ "account-hydration"
+ ],
+ "prompt": "The Home login card prompt can disappear for an entire session when its timing variant resolves before account settings or when a user signs in after Home mounts. For the configured at_login variant, show the existing dialog once an eligible signed-in account without a payment method is available, whether loaded initially or later. Keep it hidden until settings and account identity are available. Preserve once-per-install suppression across unmount and remount with persisted browser storage, suppression for accounts with a payment method, silence in control, silence for a timing variant that does not own login, and silence while the variant is unresolved. Preserve the public CardAskProvider component interface. No particular effect dependencies, private names, data-test attributes, storage representation or modal wording are required. Exercise real provider/controller/dialog behavior with synthetic settings, feature-flag, native-platform and telemetry ports. This is the historical configured login flow; it does not authorize enabling any current rollout or changing payment, recording or native behavior.",
+ "agent_timeout_seconds": 900,
+ "grader_dependency_links": [
+ {
+ "source_path": "apps/screenpipe-app-tauri/node_modules",
+ "destination_path": "apps/screenpipe-app-tauri/node_modules"
+ }
+ ],
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/card-login-hydration.fixture.tsx.txt",
+ "destination_path": "apps/screenpipe-app-tauri/components/__tests__/eval-card-login-hydration.test.tsx"
+ }
+ ],
+ "command": "cd apps/screenpipe-app-tauri && NODE_OPTIONS=--localstorage-file=.eval-localstorage node node_modules/vitest/vitest.mjs run --config vitest.config.ts components/__tests__/eval-card-login-hydration.test.tsx --maxWorkers=1 --minWorkers=1"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/card-login-hydration.fixture.tsx.txt b/evals/coding-agent/graders/card-login-hydration.fixture.tsx.txt
new file mode 100644
index 000000000..dc1a36de9
--- /dev/null
+++ b/evals/coding-agent/graders/card-login-hydration.fixture.tsx.txt
@@ -0,0 +1,143 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+import { render, screen } from "@testing-library/react";
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import {
+ CARD_ASK_ARM_STORAGE_KEY,
+} from "@/lib/card-ask/gating";
+import { resetCardAskTriggerBus } from "@/lib/card-ask/trigger-bus";
+
+
+
+let flagVariant: string | undefined = "at_login";
+let settingsState: { settings: any; isSettingsLoaded: boolean };
+
+vi.mock("posthog-js/react", () => ({
+ useFeatureFlagVariantKey: () => flagVariant,
+}));
+
+vi.mock("@/lib/hooks/use-settings", () => ({
+ useSettings: () => settingsState,
+}));
+
+vi.mock("@tauri-apps/plugin-os", () => ({ platform: () => "macos" }));
+
+vi.mock("posthog-js", () => ({ default: { capture: vi.fn() } }));
+
+vi.mock("@/lib/open-external-url", () => ({
+ openExternalUrl: vi.fn(async () => {}),
+}));
+
+import { CardAskProvider } from "@/components/card-ask-provider";
+
+const localStorageValues = new Map<string, string>();
+const localStorageMock = {
+ getItem: (key: string) => localStorageValues.get(key) ?? null,
+ setItem: (key: string, value: string) => {
+ localStorageValues.set(key, value);
+ },
+ removeItem: (key: string) => {
+ localStorageValues.delete(key);
+ },
+ clear: () => {
+ localStorageValues.clear();
+ },
+};
+
+
+const cardlessUser = {
+ id: "synthetic-user",
+ email: "[email protected]",
+ has_payment_method: false,
+};
+
+function reset() {
+ localStorageValues.clear();
+ Object.defineProperty(window, "localStorage", {
+ configurable: true,
+ value: localStorageMock,
+ });
+ resetCardAskTriggerBus();
+ flagVariant = "at_login";
+ settingsState = { settings: { user: cardlessUser }, isSettingsLoaded: true };
+}
+
+beforeEach(reset);
+afterEach(reset);
+
+function modal() {
+ return screen.queryByRole("dialog");
+}
+
+describe("CardAskProvider login trigger", () => {
+ it("shows the login ask for an eligible user in at_login", () => {
+ render(<CardAskProvider />);
+ expect(modal()).not.toBeNull();
+ });
+
+ it("still shows it when settings finish loading after the arm resolves", () => {
+ settingsState = { settings: { user: null }, isSettingsLoaded: false };
+ const { rerender } = render(<CardAskProvider />);
+ expect(modal()).toBeNull();
+
+ settingsState = { settings: { user: cardlessUser }, isSettingsLoaded: true };
+ rerender(<CardAskProvider />);
+ expect(modal()).not.toBeNull();
+ });
+
+ it("shows it when the user signs in after Home is already mounted", () => {
+ settingsState = { settings: { user: null }, isSettingsLoaded: true };
+ const { rerender } = render(<CardAskProvider />);
+ expect(modal()).toBeNull();
+
+ settingsState = { settings: { user: cardlessUser }, isSettingsLoaded: true };
+ rerender(<CardAskProvider />);
+ expect(modal()).not.toBeNull();
+ });
+
+ it("re-emitting on every settings change still asks only once", () => {
+ const first = render(<CardAskProvider />);
+ expect(modal()).not.toBeNull();
+ first.unmount();
+
+ resetCardAskTriggerBus();
+ render(<CardAskProvider />);
+ expect(modal()).toBeNull();
+ });
+
+ it("stays silent for a user with a card on file", () => {
+ settingsState = {
+ settings: { user: { ...cardlessUser, has_payment_method: true } },
+ isSettingsLoaded: true,
+ };
+ render(<CardAskProvider />);
+ expect(modal()).toBeNull();
+ });
+
+ it("stays silent in control even once settings load", () => {
+ flagVariant = "control";
+ settingsState = { settings: { user: null }, isSettingsLoaded: false };
+ const { rerender } = render(<CardAskProvider />);
+ settingsState = { settings: { user: cardlessUser }, isSettingsLoaded: true };
+ rerender(<CardAskProvider />);
+ expect(modal()).toBeNull();
+ });
+
+ it("does not fire login for an arm that does not own it", () => {
+ flagVariant = "at_limit";
+ localStorageValues.set(CARD_ASK_ARM_STORAGE_KEY, "at_limit");
+ render(<CardAskProvider />);
+ expect(modal()).toBeNull();
+ });
+});
+it("stays silent without a signed-in account", () => {
+ settingsState = { settings: { user: null }, isSettingsLoaded: true };
+ render(<CardAskProvider />);
+ expect(modal()).toBeNull();
+});
+it("stays silent while the timing variant is unresolved", () => {
+ flagVariant = undefined;
+ render(<CardAskProvider />);
+ expect(modal()).toBeNull();
+});