YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
test(evals): cover capacity plan selection outcomes
louis84c6ccc842ab
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex b83b6ffe3..c82530768 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -9,10 +9,25 @@ This is an agent eval suite, not a unit-test suite. Every case contains:- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.-The current app corpus contains 76 git-mined regressions. See+The current app corpus contains 77 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.+`app-business-capacity-plan-picker` drives the actual account settings component+with synthetic native, settings and telemetry ports. Eight outcomes cover visible+capacity plans and prices, exact billing targets, current-plan labeling, the next+capacity step and preserved login/account navigation. The parent fails six+intended outcomes and preserves two; the historical fix passes all eight. The+same outcomes pass on current source using its offline localization test setup.+Run `bun test evals/coding-agent/calibrate-capacity-plan-picker.test.js` with the+desktop test dependencies installed. Nine controls include incorrect targets,+missing navigation, an unused correct module, blanket current-plan labels, lost+account navigation, equivalent helper/test-id names and query ordering, and+missing-source setup classification. The grader does not require test IDs or+private helper names. Fixtures and dependency links are installed only at grading.+This does not establish live billing, backend proration, native behavior, agent+isolation or model capability.+`app-terminal-quota-retry` exercises the actual foreground event hook withsynthetic native commands and event delivery. Seven event outcomes cover terminalquota/model retry cancellation, finalization, retained transient retries, delayeddiff --git a/evals/coding-agent/calibrate-capacity-plan-picker.test.js b/evals/coding-agent/calibrate-capacity-plan-picker.test.jsnew file mode 100644index 000000000..69a5bc41a--- /dev/null+++ b/evals/coding-agent/calibrate-capacity-plan-picker.test.js@@ -0,0 +1,53 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { dirname, join, resolve } from 'node:path';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+const repo = resolve(import.meta.dir, '../..'), app = 'apps/screenpipe-app-tauri';+const item = JSON.parse(readFileSync(join(import.meta.dir, 'cases.json'), 'utf8')).cases.find(c => c.id === 'app-business-capacity-plan-picker');+const options = `${app}/components/settings/account-plan-options.tsx`, section = `${app}/components/settings/account-section.tsx`;+const root = mkdtempSync(join(tmpdir(), 'capacity-plan-calibration-')), archives = new Map();+const receipts = process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;+afterAll(() => rmSync(root, {recursive: true, force: true}));+function replace(cwd, file, from, to) {+ const p = join(cwd, file), text = readFileSync(p, 'utf8');+ expect(text.split(from)).toHaveLength(2);+ writeFileSync(p, text.replace(from, to));+}+function grade(name, ref = item.oracle_ref, mutate = () => {}) {+ const cwd = join(root, name); mkdirSync(cwd);+ if (!archives.has(ref)) {+ // Archive historical frontend source/config, never node_modules, credentials,+ // native build outputs or future source. No evaluated agent runs here.+ const children = execFileSync('git', ['ls-tree', '--name-only', `${ref}:${app}`], {cwd: repo, encoding: 'utf8'}).trim().split('\n');+ const paths = children.filter(n => !['src-tauri', 'public', 'e2e', '.e2e'].includes(n) && !n.startsWith('.env')).map(n => `${app}/${n}`);++ archives.set(ref, execFileSync('git', ['archive', ref, ...paths], {cwd: repo, maxBuffer: 128 * 1024 * 1024}));+ }+ execFileSync('tar', ['-x', '-C', cwd], {input: archives.get(ref)});+ for (const f of item.grader.fixtures) { const dest = join(cwd, f.destination_path); mkdirSync(dirname(dest), {recursive:true}); writeFileSync(dest, readFileSync(join(import.meta.dir, f.local_path))); }+ symlinkSync(join(repo, app, 'node_modules'), join(cwd, app, 'node_modules'), 'dir');+ mutate(cwd);+ const r = spawnSync('/bin/bash', ['-c', item.grader.command], {cwd, encoding:'utf8', timeout:60000, maxBuffer:4*1024*1024, env:{PATH:process.env.PATH, HOME:cwd, CI:'true', TZ:'UTC', NO_COLOR:'1'}});+ if (receipts) { mkdirSync(receipts,{recursive:true}); writeFileSync(join(receipts, name+'.stdout'), r.stdout||''); writeFileSync(join(receipts,name+'.stderr'),r.stderr||''); writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message},null,2)); }+ expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); return r;+}+function passes(r) { expect(r.status).toBe(0); expect(r.stdout).toContain('8 passed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught/); }+function fails(r) { expect(r.status).toBe(1); expect(r.stdout).toContain('failed'); expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Failed to load url|Cannot find module/); }+test('parent fails capacity outcomes and preserves login/account outcomes',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('6 failed | 2 passed');},60000);+test('reference passes all eight outcomes',()=>passes(grade('reference')),60000);+test('equivalent helper names, test ids and query ordering pass',()=>passes(grade('equivalent',item.oracle_ref,cwd=>{+ for(const file of [options,section]) { const f=join(cwd,file);writeFileSync(f,readFileSync(f,'utf8').replaceAll('openCapacityBilling','openPlanBilling').replaceAll('data-testid=','data-eval-neutral=')); }+ replace(cwd,section,'billingUrl.searchParams.set("target_plan", targetPlan);\n billingUrl.searchParams.set("interval", "month");','billingUrl.searchParams.set("interval", "month");\n billingUrl.searchParams.set("target_plan", targetPlan);');+})),60000);+test('unused correct module cannot hide broken active presentation',()=>fails(grade('unused',item.oracle_ref,cwd=>{+ writeFileSync(join(cwd,`${app}/components/settings/unused-correct-plans.tsx`),readFileSync(join(cwd,options)));+ for(const f of [options,section])writeFileSync(join(cwd,f),execFileSync('git',['show',`${item.base_ref}:${f}`],{cwd:repo}));+})),60000);+test('wrong outgoing billing target fails',()=>fails(grade('wrong-target',item.oracle_ref,cwd=>replace(cwd,section,'const billingUrl = new URL(BILLING_URL);\n billingUrl.searchParams.set("target_plan", targetPlan);','const billingUrl = new URL(BILLING_URL);\n billingUrl.searchParams.set("target_plan", "pro");'))),60000);+test('missing navigation effect fails',()=>fails(grade('no-navigation',item.oracle_ref,cwd=>replace(cwd,section,'await openExternalUrl(billingUrl.toString());\n };\n\n const selectPlan','void billingUrl;\n };\n\n const selectPlan'))),60000);+test('all plans marked current fails',()=>fails(grade('all-current',item.oracle_ref,cwd=>replace(cwd,options,'const isCurrent = plan.id === current;','const isCurrent = true;'))),60000);+test('lost preserved web account navigation fails',()=>fails(grade('lost-account',item.oracle_ref,cwd=>replace(cwd,section,'openExternalUrl(ACCOUNT_URL)','Promise.resolve()'))),60000);+test('missing source is setup error',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,options)));expect(r.status).toBe(1);expect(r.stdout+r.stderr).toMatch(/Failed to resolve import|Failed to load url/);},60000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex 966ac6e11..b01028f0e 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -3141,6 +3141,53 @@"apps/screenpipe-app-tauri/lib/utils/ai-endpoint-url.ts","apps/screenpipe-app-tauri/lib/utils/validation.ts"]+ },+ {+ "id": "app-business-capacity-plan-picker",+ "name": "Expose capacity plans and route each selection to its billing target",+ "source": {+ "kind": "git_regression",+ "fix_commit": "9d7866f0eec57826f3af7660e4c2a224e3f53dbc"+ },+ "base_ref": "e86a6d71a79ba302f9a8e6a765e1d4d1342404c3",+ "oracle_ref": "9d7866f0eec57826f3af7660e4c2a224e3f53dbc",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/components/settings/account-plan-options.tsx",+ "apps/screenpipe-app-tauri/components/settings/account-section.tsx"+ ],+ "gate": "advisory",+ "trigger_paths": [+ "apps/screenpipe-app-tauri/components/settings/account-plan-options.tsx",+ "apps/screenpipe-app-tauri/components/settings/account-section.tsx"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "billing",+ "plan-picker"+ ],+ "prompt": "Account settings hides Business Max and Ultra behind an unset feature flag. Make all five plan choices visible without enabling a feature flag: Free $0, Basic $25, Business $50, Business Max $100 and Business Ultra $200 per month. Describe Max and Ultra as 2x and 4x daily query/request capacity respectively, retaining the same 400 credits and frontier models, not extra credit allowances. A signed-in Business owner selecting Max must open the screenpipe.com/account/billing page with target_plan=pro_max and interval=month; a Max owner selecting Ultra must use target_plan=pro_ultra and interval=month. These capacity changes must not start a new in-app checkout. Mark each Max or Ultra owner on the matching priced plan as current, without a purchase button for that current plan or a current marker on ordinary Business. Recommend Max as the next step for Business, Ultra for Max, and no next step above Ultra or for team/enterprise accounts. Preserve signed-out stale-account gating: an account without a token, even with a stale cloud subscription flag, must show sign-in guidance and no active subscription. Signed-in subscribers must retain their active account display and working web-account link. Preserve the public AccountSection interface. Exact layout, test ids, CSS, internal helper names, analytics names and URL query ordering are not prescribed. Use synthetic native/provider ports only. This covers account settings presentation and outgoing billing navigation, not live financial operations, backend proration, native behavior or model performance.",+ "agent_timeout_seconds": 900,+ "grader_dependency_links": [+ {+ "source_path": "apps/screenpipe-app-tauri/node_modules",+ "destination_path": "apps/screenpipe-app-tauri/node_modules"+ }+ ],+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/capacity-plan-picker.fixture.tsx.txt",+ "destination_path": "apps/screenpipe-app-tauri/eval-capacity-plan-picker.test.tsx"+ },+ {+ "local_path": "graders/capacity-plan-picker.config.mjs",+ "destination_path": "apps/screenpipe-app-tauri/eval-capacity-plan-picker.config.mjs"+ }+ ],+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config eval-capacity-plan-picker.config.mjs"+ }}]}diff --git a/evals/coding-agent/graders/capacity-plan-picker.config.mjs b/evals/coding-agent/graders/capacity-plan-picker.config.mjsnew file mode 100644index 000000000..d1ba97a70--- /dev/null+++ b/evals/coding-agent/graders/capacity-plan-picker.config.mjs@@ -0,0 +1,12 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { defineConfig } from 'vitest/config';+import path from 'node:path';+export default defineConfig({+ esbuild: { jsx: 'automatic' },+ test: { include: ['eval-capacity-plan-picker.test.tsx'], environment: 'jsdom',+ setupFiles: ['@testing-library/jest-dom/vitest'],+ pool: 'forks', poolOptions: { forks: { singleFork: true } } },+ resolve: { alias: { '@': path.resolve('.') } },+ css: { postcss: { plugins: [] } },+});diff --git a/evals/coding-agent/graders/capacity-plan-picker.fixture.tsx.txt b/evals/coding-agent/graders/capacity-plan-picker.fixture.tsx.txtnew file mode 100644index 000000000..a9b071704--- /dev/null+++ b/evals/coding-agent/graders/capacity-plan-picker.fixture.tsx.txt@@ -0,0 +1,179 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterEach, describe, expect, it, vi } from "vitest";+import {+ act,+ cleanup,+ fireEvent,+ render,+ screen,+ waitFor,+ within,+} from "@testing-library/react";++const mocks = vi.hoisted(() => ({+ state: { user: null as any },+ updateSettings: vi.fn().mockResolvedValue(undefined),+ loadUser: vi.fn().mockResolvedValue(undefined),+ openLoginWindow: vi.fn().mockResolvedValue(undefined),+ piUpdateConfig: vi.fn().mockResolvedValue(undefined),+ capture: vi.fn(),+ openUrl: vi.fn().mockResolvedValue(undefined),+ eventHandlers: new Map<string, (event: any) => unknown>(),+ listen: vi.fn(async (event: string, handler: (event: any) => unknown) => {+ mocks.eventHandlers.set(event, handler);+ return () => mocks.eventHandlers.delete(event);+ }),+}));++// AccountSection reads everything through useSettings + the tauri `commands`+// object; swap `mocks.state.user` per case. Keep `@/lib/app-entitlement` REAL —+// `isSignedInCloudSubscriber` is the gate under test.+vi.mock("@/lib/hooks/use-settings", () => ({+ useSettings: () => ({+ settings: {+ user: mocks.state.user,+ pipeSyncEnabled: false,+ memoriesSyncEnabled: false,+ connectionSyncEnabled: false,+ },+ updateSettings: mocks.updateSettings,+ loadUser: mocks.loadUser,+ }),+}));++vi.mock("@/lib/hooks/use-health-check", () => ({+ useHealthCheck: () => ({ isServerDown: false }),+}));++vi.mock("@/lib/hooks/use-managed-policy", () => ({+ useManagedPolicy: () => ({ isManagedDeployment: false }),+}));++vi.mock("@/lib/utils/tauri", () => ({+ commands: {+ openLoginWindow: mocks.openLoginWindow,+ piUpdateConfig: mocks.piUpdateConfig,+ },+}));++vi.mock("@/components/ui/use-toast", () => ({ toast: vi.fn() }));+vi.mock("@/lib/api", () => ({ localFetch: vi.fn() }));+vi.mock("posthog-js", () => ({ default: { capture: mocks.capture } }));++// Tauri plugins the effect wires up on mount — keep them inert.+vi.mock("@tauri-apps/plugin-shell", () => ({ open: mocks.openUrl }));+vi.mock("@tauri-apps/plugin-deep-link", () => ({+ onOpenUrl: vi.fn().mockResolvedValue(() => {}),+}));+vi.mock("@tauri-apps/api/event", () => ({+ listen: mocks.listen,+}));++// ReferralCard pulls its own data deps; it is irrelevant to the gate.+vi.mock("@/components/settings/referral-card", () => ({+ ReferralCard: () => null,+}));+vi.mock("@/components/settings/business-upgrade-card", () => ({+ BusinessUpgradeCard: ({+ onContinue,+ }: {+ onContinue: (selection: any) => void;+ }) => (+ <button+ type="button"+ onClick={() =>+ onContinue({+ interval: "month",+ offerVersion: "business-desktop-v1",+ experimentKey: "desktop-business-upgrade-offer",+ experimentVariant: "control",+ source: "app-account-section",+ })+ }+ >+ upgrade to business+ </button>+ ),+}));++import { AccountSection } from "@/components/settings/account-section";++// Current source adds translated strings; identity translation is an inert port.+vi.mock("gt-react", () => ({ useGT: () => (text: string) => text }));+const blockedFetch = vi.fn(async () => { throw new Error("Unexpected network request"); });+function user(plan = "pro", token: string | null = "synthetic-token") {+ mocks.state.user = { id: "synthetic-user", email: "[email protected]", token,+ cloud_subscribed: true, subscription_plan: plan };+}+// Find the smallest visible grouping containing the plan name and price. No+// class, test id, internal card key, helper name or DOM depth is prescribed.+function card(name: string, price: number): HTMLElement {+ const label = screen.getByText(new RegExp(`^${name}$`, "i"));+ let el = label.parentElement;+ while (el && !within(el).queryByText(`$${price}`)) el = el.parentElement;+ expect(el, `visible ${name} price grouping`).not.toBeNull();+ return el!;+}+function billing(target: string) {+ expect(mocks.openUrl).toHaveBeenCalledTimes(1);+ const url = new URL(mocks.openUrl.mock.calls[0][0]);+ expect(url.origin).toBe("https://screenpipe.com");+ expect(url.pathname).toBe("/account/billing");+ expect(url.searchParams.get("target_plan")).toBe(target);+ expect(url.searchParams.get("interval")).toBe("month");+ expect(blockedFetch).not.toHaveBeenCalled();+}+afterEach(() => { cleanup(); vi.clearAllMocks(); vi.unstubAllGlobals();+ vi.unstubAllEnvs(); mocks.eventHandlers.clear(); mocks.state.user = null; });+function mount(plan = "pro") {+ vi.stubGlobal("fetch", blockedFetch);+ vi.stubEnv("NEXT_PUBLIC_BUSINESS_POWER_PLANS_ENABLED", "");+ user(plan); return render(<AccountSection />);+}+describe("capacity plan outcomes", () => {+ it("exposes all five named prices without a feature flag", () => {+ mount();+ for (const [name, price] of [["free",0],["basic",25],["business",50],["business max",100],["business ultra",200]] as const)+ expect(card(name,price)).toBeVisible();+ expect(card("business max",100).textContent).toMatch(/2x.*quer/i);+ expect(card("business ultra",200).textContent).toMatch(/4x.*quer/i);+ for (const [name,price] of [["business max",100],["business ultra",200]] as const) {+ expect(card(name,price).textContent).toMatch(/400.*credits/i);+ expect(card(name,price).textContent).toMatch(/frontier.*models/i);+ }+ });+ it("selects Max through website billing without starting checkout", () => {+ mount(); fireEvent.click(within(card("business max",100)).getByRole("button")); billing("pro_max");+ });+ it("selects Ultra from Max through website billing", () => {+ mount("pro_max"); fireEvent.click(within(card("business ultra",200)).getByRole("button")); billing("pro_ultra");+ });+ it.each([["pro_max","business max",100],["pro_ultra","business ultra",200]] as const)("marks the actual %s plan current", (plan,name,price) => {+ mount(plan); const active=card(name,price);+ expect(within(active).getAllByText(/^(current|your plan)$/i).length).toBeGreaterThan(0);+ expect(within(card("business",50)).queryByText(/^(current|your plan)$/i)).toBeNull();+ expect(within(active).queryByRole("button")).toBeNull();+ });+ it("recommends the immediate capacity step and none above or off ladder", () => {+ const view=mount();+ expect(within(card("business max",100)).getByText(/next step/i)).toBeVisible();+ user("pro_max");view.rerender(<AccountSection />);+ expect(within(card("business ultra",200)).getByText(/next step/i)).toBeVisible();+ for (const plan of ["pro_ultra","team","enterprise"]) {+ user(plan);view.rerender(<AccountSection />);expect(screen.queryByText(/next step/i)).toBeNull();+ }+ });+ it("preserves signed-out stale-account login gating", () => {+ vi.stubGlobal("fetch",blockedFetch);user("pro",null);render(<AccountSection />);+ expect(screen.getByText(/sign in to screenpipe/i)).toBeVisible();+ expect(screen.getByText(/not logged in/i)).toBeVisible();+ expect(screen.queryByText(/^active$/i)).toBeNull();+ });+ it("preserves signed-in account access and the account link", () => {+ mount();expect(screen.getByText(/^active$/i)).toBeVisible();+ expect(screen.getByText(/logged in as [email protected]/i)).toBeVisible();+ fireEvent.click(screen.getByRole("button",{name:/web account/i}));+ expect(mocks.openUrl).toHaveBeenCalledWith("https://screenpipe.com/account");+ });+});