Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

test(evals): cover capacity plan selection outcomes

louis84c6ccc842ab
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index b83b6ffe3..c82530768 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -9,10 +9,25 @@ This is an agent eval suite, not a unit-test suite. Every case contains:
- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;
- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.
-The current app corpus contains 76 git-mined regressions. See
+The current app corpus contains 77 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest uses this same harness.
+`app-business-capacity-plan-picker` drives the actual account settings component
+with synthetic native, settings and telemetry ports. Eight outcomes cover visible
+capacity plans and prices, exact billing targets, current-plan labeling, the next
+capacity step and preserved login/account navigation. The parent fails six
+intended outcomes and preserves two; the historical fix passes all eight. The
+same outcomes pass on current source using its offline localization test setup.
+Run `bun test evals/coding-agent/calibrate-capacity-plan-picker.test.js` with the
+desktop test dependencies installed. Nine controls include incorrect targets,
+missing navigation, an unused correct module, blanket current-plan labels, lost
+account navigation, equivalent helper/test-id names and query ordering, and
+missing-source setup classification. The grader does not require test IDs or
+private helper names. Fixtures and dependency links are installed only at grading.
+This does not establish live billing, backend proration, native behavior, agent
+isolation or model capability.
+
`app-terminal-quota-retry` exercises the actual foreground event hook with
synthetic native commands and event delivery. Seven event outcomes cover terminal
quota/model retry cancellation, finalization, retained transient retries, delayed
diff --git a/evals/coding-agent/calibrate-capacity-plan-picker.test.js b/evals/coding-agent/calibrate-capacity-plan-picker.test.js
new file mode 100644
index 000000000..69a5bc41a
--- /dev/null
+++ b/evals/coding-agent/calibrate-capacity-plan-picker.test.js
@@ -0,0 +1,53 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { dirname, join, resolve } from 'node:path';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+const repo = resolve(import.meta.dir, '../..'), app = 'apps/screenpipe-app-tauri';
+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'app-business-capacity-plan-picker');
+const options = `${app}/components/settings/account-plan-options.tsx`, section = `${app}/components/settings/account-section.tsx`;
+const root = mkdtempSync(join(tmpdir(), 'capacity-plan-calibration-')), archives = new Map();
+const receipts = process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;
+afterAll(() => rmSync(root, {recursive: true, force: true}));
+function replace(cwd, file, from, to) {
+ const p = join(cwd, file), text = readFileSync(p, 'utf8');
+ expect(text.split(from)).toHaveLength(2);
+ writeFileSync(p, text.replace(from, to));
+}
+function grade(name, ref = item.oracle_ref, mutate = () => {}) {
+ const cwd = join(root, name); mkdirSync(cwd);
+ if (!archives.has(ref)) {
+ // Archive historical frontend source/config, never node_modules, credentials,
+ // native build outputs or future source. No evaluated agent runs here.
+ const children = execFileSync('git', ['ls-tree', '--name-only', `${ref}:${app}`], {cwd: repo, encoding: 'utf8'}).trim().split('\n');
+ const paths = children.filter(n => !['src-tauri', 'public', 'e2e', '.e2e'].includes(n) && !n.startsWith('.env')).map(n => `${app}/${n}`);
+
+ archives.set(ref, execFileSync('git', ['archive', ref, ...paths], {cwd: repo, maxBuffer: 128 * 1024 * 1024}));
+ }
+ execFileSync('tar', ['-x', '-C', cwd], {input: archives.get(ref)});
+ for (const f of item.grader.fixtures) { const dest = join(cwd, f.destination_path); mkdirSync(dirname(dest), {recursive:true}); writeFileSync(dest, readFileSync(join(import.meta.dir, f.local_path))); }
+ symlinkSync(join(repo, app, 'node_modules'), join(cwd, app, 'node_modules'), 'dir');
+ mutate(cwd);
+ const r = spawnSync('/bin/bash', ['-c', item.grader.command], {cwd, encoding:'utf8', timeout:60000, maxBuffer:4*1024*1024, env:{PATH:process.env.PATH, HOME:cwd, CI:'true', TZ:'UTC', NO_COLOR:'1'}});
+ if (receipts) { mkdirSync(receipts,{recursive:true}); writeFileSync(join(receipts, name+'.stdout'), r.stdout||''); writeFileSync(join(receipts,name+'.stderr'),r.stderr||''); writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message},null,2)); }
+ expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); return r;
+}
+function passes(r) { expect(r.status).toBe(0); expect(r.stdout).toContain('8 passed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught/); }
+function fails(r) { expect(r.status).toBe(1); expect(r.stdout).toContain('failed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/); }
+test('parent fails capacity outcomes and preserves login/account outcomes',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('6 failed | 2 passed');},60000);
+test('reference passes all eight outcomes',()=>passes(grade('reference')),60000);
+test('equivalent helper names, test ids and query ordering pass',()=>passes(grade('equivalent',item.oracle_ref,cwd=>{
+ for(const file of [options,section]) { const f=join(cwd,file);writeFileSync(f,readFileSync(f,'utf8').replaceAll('openCapacityBilling','openPlanBilling').replaceAll('data-testid=','data-eval-neutral=')); }
+ replace(cwd,section,'billingUrl.searchParams.set("target_plan", targetPlan);\n billingUrl.searchParams.set("interval", "month");','billingUrl.searchParams.set("interval", "month");\n billingUrl.searchParams.set("target_plan", targetPlan);');
+})),60000);
+test('unused correct module cannot hide broken active presentation',()=>fails(grade('unused',item.oracle_ref,cwd=>{
+ writeFileSync(join(cwd,`${app}/components/settings/unused-correct-plans.tsx`),readFileSync(join(cwd,options)));
+ for(const f of [options,section])writeFileSync(join(cwd,f),execFileSync('git',['show',`${item.base_ref}:${f}`],{cwd:repo}));
+})),60000);
+test('wrong outgoing billing target fails',()=>fails(grade('wrong-target',item.oracle_ref,cwd=>replace(cwd,section,'const billingUrl = new URL(BILLING_URL);\n billingUrl.searchParams.set("target_plan", targetPlan);','const billingUrl = new URL(BILLING_URL);\n billingUrl.searchParams.set("target_plan", "pro");'))),60000);
+test('missing navigation effect fails',()=>fails(grade('no-navigation',item.oracle_ref,cwd=>replace(cwd,section,'await openExternalUrl(billingUrl.toString());\n };\n\n const selectPlan','void billingUrl;\n };\n\n const selectPlan'))),60000);
+test('all plans marked current fails',()=>fails(grade('all-current',item.oracle_ref,cwd=>replace(cwd,options,'const isCurrent = plan.id === current;','const isCurrent = true;'))),60000);
+test('lost preserved web account navigation fails',()=>fails(grade('lost-account',item.oracle_ref,cwd=>replace(cwd,section,'openExternalUrl(ACCOUNT_URL)','Promise.resolve()'))),60000);
+test('missing source is setup error',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,options)));expect(r.status).toBe(1);expect(r.stdout+r.stderr).toMatch(/Failed to resolve import|Failed to load url/);},60000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 966ac6e11..b01028f0e 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -3141,6 +3141,53 @@
"apps/screenpipe-app-tauri/lib/utils/ai-endpoint-url.ts",
"apps/screenpipe-app-tauri/lib/utils/validation.ts"
]
+ },
+ {
+ "id": "app-business-capacity-plan-picker",
+ "name": "Expose capacity plans and route each selection to its billing target",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "9d7866f0eec57826f3af7660e4c2a224e3f53dbc"
+ },
+ "base_ref": "e86a6d71a79ba302f9a8e6a765e1d4d1342404c3",
+ "oracle_ref": "9d7866f0eec57826f3af7660e4c2a224e3f53dbc",
+ "oracle_paths": [
+ "apps/screenpipe-app-tauri/components/settings/account-plan-options.tsx",
+ "apps/screenpipe-app-tauri/components/settings/account-section.tsx"
+ ],
+ "gate": "advisory",
+ "trigger_paths": [
+ "apps/screenpipe-app-tauri/components/settings/account-plan-options.tsx",
+ "apps/screenpipe-app-tauri/components/settings/account-section.tsx"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "billing",
+ "plan-picker"
+ ],
+ "prompt": "Account settings hides Business Max and Ultra behind an unset feature flag. Make all five plan choices visible without enabling a feature flag: Free $0, Basic $25, Business $50, Business Max $100 and Business Ultra $200 per month. Describe Max and Ultra as 2x and 4x daily query/request capacity respectively, retaining the same 400 credits and frontier models, not extra credit allowances. A signed-in Business owner selecting Max must open the screenpipe.com/account/billing page with target_plan=pro_max and interval=month; a Max owner selecting Ultra must use target_plan=pro_ultra and interval=month. These capacity changes must not start a new in-app checkout. Mark each Max or Ultra owner on the matching priced plan as current, without a purchase button for that current plan or a current marker on ordinary Business. Recommend Max as the next step for Business, Ultra for Max, and no next step above Ultra or for team/enterprise accounts. Preserve signed-out stale-account gating: an account without a token, even with a stale cloud subscription flag, must show sign-in guidance and no active subscription. Signed-in subscribers must retain their active account display and working web-account link. Preserve the public AccountSection interface. Exact layout, test ids, CSS, internal helper names, analytics names and URL query ordering are not prescribed. Use synthetic native/provider ports only. This covers account settings presentation and outgoing billing navigation, not live financial operations, backend proration, native behavior or model performance.",
+ "agent_timeout_seconds": 900,
+ "grader_dependency_links": [
+ {
+ "source_path": "apps/screenpipe-app-tauri/node_modules",
+ "destination_path": "apps/screenpipe-app-tauri/node_modules"
+ }
+ ],
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/capacity-plan-picker.fixture.tsx.txt",
+ "destination_path": "apps/screenpipe-app-tauri/eval-capacity-plan-picker.test.tsx"
+ },
+ {
+ "local_path": "graders/capacity-plan-picker.config.mjs",
+ "destination_path": "apps/screenpipe-app-tauri/eval-capacity-plan-picker.config.mjs"
+ }
+ ],
+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-capacity-plan-picker.config.mjs"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/capacity-plan-picker.config.mjs b/evals/coding-agent/graders/capacity-plan-picker.config.mjs
new file mode 100644
index 000000000..d1ba97a70
--- /dev/null
+++ b/evals/coding-agent/graders/capacity-plan-picker.config.mjs
@@ -0,0 +1,12 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { defineConfig } from 'vitest/config';
+import path from 'node:path';
+export default defineConfig({
+ esbuild: { jsx: 'automatic' },
+ test: { include: ['eval-capacity-plan-picker.test.tsx'], environment: 'jsdom',
+ setupFiles: ['@testing-library/jest-dom/vitest'],
+ pool: 'forks', poolOptions: { forks: { singleFork: true } } },
+ resolve: { alias: { '@': path.resolve('.') } },
+ css: { postcss: { plugins: [] } },
+});
diff --git a/evals/coding-agent/graders/capacity-plan-picker.fixture.tsx.txt b/evals/coding-agent/graders/capacity-plan-picker.fixture.tsx.txt
new file mode 100644
index 000000000..a9b071704
--- /dev/null
+++ b/evals/coding-agent/graders/capacity-plan-picker.fixture.tsx.txt
@@ -0,0 +1,179 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterEach, describe, expect, it, vi } from "vitest";
+import {
+ act,
+ cleanup,
+ fireEvent,
+ render,
+ screen,
+ waitFor,
+ within,
+} from "@testing-library/react";
+
+const mocks = vi.hoisted(() => ({
+ state: { user: null as any },
+ updateSettings: vi.fn().mockResolvedValue(undefined),
+ loadUser: vi.fn().mockResolvedValue(undefined),
+ openLoginWindow: vi.fn().mockResolvedValue(undefined),
+ piUpdateConfig: vi.fn().mockResolvedValue(undefined),
+ capture: vi.fn(),
+ openUrl: vi.fn().mockResolvedValue(undefined),
+ eventHandlers: new Map<string, (event: any) => unknown>(),
+ listen: vi.fn(async (event: string, handler: (event: any) => unknown) => {
+ mocks.eventHandlers.set(event, handler);
+ return () => mocks.eventHandlers.delete(event);
+ }),
+}));
+
+// AccountSection reads everything through useSettings + the tauri `commands`
+// object; swap `mocks.state.user` per case. Keep `@/lib/app-entitlement` REAL —
+// `isSignedInCloudSubscriber` is the gate under test.
+vi.mock("@/lib/hooks/use-settings", () => ({
+ useSettings: () => ({
+ settings: {
+ user: mocks.state.user,
+ pipeSyncEnabled: false,
+ memoriesSyncEnabled: false,
+ connectionSyncEnabled: false,
+ },
+ updateSettings: mocks.updateSettings,
+ loadUser: mocks.loadUser,
+ }),
+}));
+
+vi.mock("@/lib/hooks/use-health-check", () => ({
+ useHealthCheck: () => ({ isServerDown: false }),
+}));
+
+vi.mock("@/lib/hooks/use-managed-policy", () => ({
+ useManagedPolicy: () => ({ isManagedDeployment: false }),
+}));
+
+vi.mock("@/lib/utils/tauri", () => ({
+ commands: {
+ openLoginWindow: mocks.openLoginWindow,
+ piUpdateConfig: mocks.piUpdateConfig,
+ },
+}));
+
+vi.mock("@/components/ui/use-toast", () => ({ toast: vi.fn() }));
+vi.mock("@/lib/api", () => ({ localFetch: vi.fn() }));
+vi.mock("posthog-js", () => ({ default: { capture: mocks.capture } }));
+
+// Tauri plugins the effect wires up on mount — keep them inert.
+vi.mock("@tauri-apps/plugin-shell", () => ({ open: mocks.openUrl }));
+vi.mock("@tauri-apps/plugin-deep-link", () => ({
+ onOpenUrl: vi.fn().mockResolvedValue(() => {}),
+}));
+vi.mock("@tauri-apps/api/event", () => ({
+ listen: mocks.listen,
+}));
+
+// ReferralCard pulls its own data deps; it is irrelevant to the gate.
+vi.mock("@/components/settings/referral-card", () => ({
+ ReferralCard: () => null,
+}));
+vi.mock("@/components/settings/business-upgrade-card", () => ({
+ BusinessUpgradeCard: ({
+ onContinue,
+ }: {
+ onContinue: (selection: any) => void;
+ }) => (
+ <button
+ type="button"
+ onClick={() =>
+ onContinue({
+ interval: "month",
+ offerVersion: "business-desktop-v1",
+ experimentKey: "desktop-business-upgrade-offer",
+ experimentVariant: "control",
+ source: "app-account-section",
+ })
+ }
+ >
+ upgrade to business
+ </button>
+ ),
+}));
+
+import { AccountSection } from "@/components/settings/account-section";
+
+// Current source adds translated strings; identity translation is an inert port.
+vi.mock("gt-react", () => ({ useGT: () => (text: string) => text }));
+const blockedFetch = vi.fn(async () => { throw new Error("Unexpected network request"); });
+function user(plan = "pro", token: string | null = "synthetic-token") {
+ mocks.state.user = { id: "synthetic-user", email: "[email protected]", token,
+ cloud_subscribed: true, subscription_plan: plan };
+}
+// Find the smallest visible grouping containing the plan name and price. No
+// class, test id, internal card key, helper name or DOM depth is prescribed.
+function card(name: string, price: number): HTMLElement {
+ const label = screen.getByText(new RegExp(`^${name}$`, "i"));
+ let el = label.parentElement;
+ while (el && !within(el).queryByText(`$${price}`)) el = el.parentElement;
+ expect(el, `visible ${name} price grouping`).not.toBeNull();
+ return el!;
+}
+function billing(target: string) {
+ expect(mocks.openUrl).toHaveBeenCalledTimes(1);
+ const url = new URL(mocks.openUrl.mock.calls[0][0]);
+ expect(url.origin).toBe("https://screenpipe.com");
+ expect(url.pathname).toBe("/account/billing");
+ expect(url.searchParams.get("target_plan")).toBe(target);
+ expect(url.searchParams.get("interval")).toBe("month");
+ expect(blockedFetch).not.toHaveBeenCalled();
+}
+afterEach(() => { cleanup(); vi.clearAllMocks(); vi.unstubAllGlobals();
+ vi.unstubAllEnvs(); mocks.eventHandlers.clear(); mocks.state.user = null; });
+function mount(plan = "pro") {
+ vi.stubGlobal("fetch", blockedFetch);
+ vi.stubEnv("NEXT_PUBLIC_BUSINESS_POWER_PLANS_ENABLED", "");
+ user(plan); return render(<AccountSection />);
+}
+describe("capacity plan outcomes", () => {
+ it("exposes all five named prices without a feature flag", () => {
+ mount();
+ for (const [name, price] of [["free",0],["basic",25],["business",50],["business max",100],["business ultra",200]] as const)
+ expect(card(name,price)).toBeVisible();
+ expect(card("business max",100).textContent).toMatch(/2x.*quer/i);
+ expect(card("business ultra",200).textContent).toMatch(/4x.*quer/i);
+ for (const [name,price] of [["business max",100],["business ultra",200]] as const) {
+ expect(card(name,price).textContent).toMatch(/400.*credits/i);
+ expect(card(name,price).textContent).toMatch(/frontier.*models/i);
+ }
+ });
+ it("selects Max through website billing without starting checkout", () => {
+ mount(); fireEvent.click(within(card("business max",100)).getByRole("button")); billing("pro_max");
+ });
+ it("selects Ultra from Max through website billing", () => {
+ mount("pro_max"); fireEvent.click(within(card("business ultra",200)).getByRole("button")); billing("pro_ultra");
+ });
+ it.each([["pro_max","business max",100],["pro_ultra","business ultra",200]] as const)("marks the actual %s plan current", (plan,name,price) => {
+ mount(plan); const active=card(name,price);
+ expect(within(active).getAllByText(/^(current|your plan)$/i).length).toBeGreaterThan(0);
+ expect(within(card("business",50)).queryByText(/^(current|your plan)$/i)).toBeNull();
+ expect(within(active).queryByRole("button")).toBeNull();
+ });
+ it("recommends the immediate capacity step and none above or off ladder", () => {
+ const view=mount();
+ expect(within(card("business max",100)).getByText(/next step/i)).toBeVisible();
+ user("pro_max");view.rerender(<AccountSection />);
+ expect(within(card("business ultra",200)).getByText(/next step/i)).toBeVisible();
+ for (const plan of ["pro_ultra","team","enterprise"]) {
+ user(plan);view.rerender(<AccountSection />);expect(screen.queryByText(/next step/i)).toBeNull();
+ }
+ });
+ it("preserves signed-out stale-account login gating", () => {
+ vi.stubGlobal("fetch",blockedFetch);user("pro",null);render(<AccountSection />);
+ expect(screen.getByText(/sign in to screenpipe/i)).toBeVisible();
+ expect(screen.getByText(/not logged in/i)).toBeVisible();
+ expect(screen.queryByText(/^active$/i)).toBeNull();
+ });
+ it("preserves signed-in account access and the account link", () => {
+ mount();expect(screen.getByText(/^active$/i)).toBeVisible();
+ expect(screen.getByText(/logged in as [email protected]/i)).toBeVisible();
+ fireEvent.click(screen.getByRole("button",{name:/web account/i}));
+ expect(mocks.openUrl).toHaveBeenCalledWith("https://screenpipe.com/account");
+ });
+});