Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: grade trial upgrades by account and saved state outcomes

louis1f99c3926751
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index b6e99a47d..8eac17d02 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -1208,3 +1208,25 @@ establish native rollout authority, cohort/stop controls, recovery-copy deletion
policy, all interruption paths, live recording continuity, execution isolation
or model performance. See [the rollout review](MIGRATION-ROLLOUT-REVIEW.md) for the
separate authority requirements.
+
+## Trial state after a paid upgrade
+
+`app-trial-upgrade-expiration` runs the historical Account component with a fixed
+clock and synthetic settings, billing, entitlement-refresh and native event ports.
+Nine outcomes cover stale trial notice suppression, saved expiry and entitlement
+after confirmed checkout, unconfirmed checkout preservation, manual-trial billing,
+Basic account behavior and signed-out/tokenless accounts. The parent fails two
+intended outcomes and preserves seven; the reference passes all nine.
+
+Run `bun test evals/coding-agent/calibrate-trial-upgrade.test.js` for nine controls.
+Equivalent DOM attributes and split persistence writes pass. Unused correct code,
+blanket countdown suppression, retained expiry, missing refresh and unconfirmed
+activation fail. Missing active source is a Vitest collection error. The grader
+checks visible account behavior and the settings port's final snapshot, without
+requiring test IDs or a particular sequence of persistence calls.
+
+Only the Account, expiration component and entitlement helper are oracle paths.
+Home integration, real settings durability, live billing/refresh delivery and
+current full application integration are outside this case. Dependencies and the
+fixture are materialized only for grading. No model trial or enforced-isolation
+claim follows from this verification.
diff --git a/evals/coding-agent/calibrate-trial-upgrade.test.js b/evals/coding-agent/calibrate-trial-upgrade.test.js
new file mode 100644
index 000000000..be5a1b82a
--- /dev/null
+++ b/evals/coding-agent/calibrate-trial-upgrade.test.js
@@ -0,0 +1,46 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { dirname, join, resolve } from 'node:path';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, symlinkSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { classifyGraderError } from './grader-outcome.mjs';
+const repo=resolve(import.meta.dir,'../..'), app='apps/screenpipe-app-tauri';
+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='app-trial-upgrade-expiration');
+const account=`${app}/components/settings/account-section.tsx`, notice=`${app}/components/plan-expiration-notice.tsx`;
+const root=mkdtempSync(join(tmpdir(),'trial-upgrade-calibration-')), archives=new Map();
+const receipts=process.env.SCREENPIPE_EVAL_CALIBRATION_RECEIPTS;
+afterAll(()=>rmSync(root,{recursive:true,force:true}));
+function replace(cwd,file,from,to){const p=join(cwd,file),s=readFileSync(p,'utf8');expect(s.split(from)).toHaveLength(2);writeFileSync(p,s.replace(from,to));}
+function grade(name,ref=item.oracle_ref,mutate=()=>{}){
+ const cwd=join(root,name);mkdirSync(cwd);
+ if(!archives.has(ref)){
+ const children=execFileSync('git',['ls-tree','--name-only',`${ref}:${app}`],{cwd:repo,encoding:'utf8'}).trim().split('\n');
+ const paths=children.filter(n=>!['src-tauri','public','e2e','.e2e'].includes(n)&&!n.startsWith('.env')).map(n=>`${app}/${n}`);
+ archives.set(ref,execFileSync('git',['archive',ref,...paths],{cwd:repo,maxBuffer:128*1024*1024}));
+ }
+ execFileSync('tar',['-x','-C',cwd],{input:archives.get(ref)});
+ for(const f of item.grader.fixtures){const dest=join(cwd,f.destination_path);mkdirSync(dirname(dest),{recursive:true});writeFileSync(dest,readFileSync(join(import.meta.dir,f.local_path)));}
+ symlinkSync(join(repo,app,'node_modules'),join(cwd,app,'node_modules'),'dir');mutate(cwd);
+ const r=spawnSync('/bin/bash',['-c',item.grader.command],{cwd,encoding:'utf8',timeout:60000,maxBuffer:4*1024*1024,env:{PATH:process.env.PATH,HOME:cwd,CI:'true',TZ:'UTC',NO_COLOR:'1'}});
+ const kind=classifyGraderError(r);
+ if(receipts){mkdirSync(receipts,{recursive:true});writeFileSync(join(receipts,name+'.stdout'),r.stdout||'');writeFileSync(join(receipts,name+'.stderr'),r.stderr||'');writeFileSync(join(receipts,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit_code:r.status,signal:r.signal,error:r.error?.message,error_kind:kind},null,2));}
+ expect(r.error).toBeUndefined();expect(r.signal).toBeNull();return {...r,kind};
+}
+function passes(r){expect(r.status).toBe(0);expect(r.stdout).toContain('9 passed');expect(r.kind).toBeNull();}
+function fails(r){expect(r.status).toBe(1);expect(r.kind).toBeNull();expect(r.stdout).toMatch(/Tests\s+[1-9]\d* failed/);}
+test('parent keeps stale trial UI and saved state; seven outcomes preserved',()=>{const r=grade('parent',item.base_ref);fails(r);expect(r.stdout).toContain('2 failed | 7 passed');},60000);
+test('reference passes nine account outcomes',()=>passes(grade('reference')),60000);
+test('equivalent DOM attributes and separate persistence writes pass',()=>passes(grade('equivalent',item.oracle_ref,cwd=>{
+ for(const file of [account,notice]){const p=join(cwd,file);writeFileSync(p,readFileSync(p,'utf8').replaceAll('data-testid=','data-neutral-marker='));}
+ replace(cwd,account,'await updateSettings({\n user: {','await updateSettings({ user: { ...settings.user, plan_expires_at: null } });\n await updateSettings({\n user: {');
+})),60000);
+test('unused correct files do not hide broken active code',()=>fails(grade('unused',item.oracle_ref,cwd=>{
+ for(const file of item.oracle_paths){writeFileSync(join(cwd,file+'.unused.txt'),readFileSync(join(cwd,file)));writeFileSync(join(cwd,file),execFileSync('git',['show',`${item.base_ref}:${file}`],{cwd:repo}));}
+})),60000);
+test('blanket countdown suppression fails preserved manual trial',()=>fails(grade('silent',item.oracle_ref,cwd=>replace(cwd,notice,'if (hasBillingSubscriptionEntitlement(user)) return null;','return null;'))),60000);
+test('retained saved expiry fails after activation',()=>fails(grade('stale-saved',item.oracle_ref,cwd=>replace(cwd,account,'plan_expires_at: null,','plan_expires_at: settings.user.plan_expires_at,'))),60000);
+test('missing entitlement refresh fails saved state',()=>fails(grade('no-refresh',item.oracle_ref,cwd=>replace(cwd,account,'await loadUser(settings.user.token, true);','void 0;'))),60000);
+test('unconfirmed checkout cannot clear manual grant',()=>fails(grade('unconfirmed',item.oracle_ref,cwd=>replace(cwd,account,'if (isActive) {','if (true) {'))),60000);
+test('missing active source is collection error',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,account)));expect(r.status).toBe(1);expect(r.kind).toBe('vitest_collection_error');},60000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index c95a9f0e2..1dc9f088f 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-09-28.1",
+ "dataset_version": "2026-09-29",
"suite_type": "regression",
"owner": "screenpipe engineering",
"policy": {
@@ -4147,6 +4147,44 @@
],
"command": "cargo test --offline --locked -p screenpipe-db --test eval_migration_unicode -- --test-threads=1"
}
+ },
+ {
+ "id": "app-trial-upgrade-expiration",
+ "name": "Paid upgrade replaces stale trial state",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "29f4477d049b37b295184e50016a4c16c31fa452"
+ },
+ "base_ref": "e57c61cd78ffd6bf6519af5e08cfcca668826066",
+ "oracle_ref": "29f4477d049b37b295184e50016a4c16c31fa452",
+ "oracle_paths": [
+ "apps/screenpipe-app-tauri/components/settings/account-section.tsx",
+ "apps/screenpipe-app-tauri/components/plan-expiration-notice.tsx",
+ "apps/screenpipe-app-tauri/lib/app-entitlement.ts"
+ ],
+ "gate": "advisory",
+ "tags": [
+ "trial",
+ "entitlement"
+ ],
+ "prompt": "Account settings can continue showing a profile-trial countdown after billing activates a subscription. Make the displayed and saved account state agree with the verified subscription entitlement. A subscription entitlement must suppress leftover profile-trial expiry, while an active manual trial must retain its countdown and billing navigation. After checkout is confirmed active, persist the cleared trial expiry and refreshed subscription entitlement, preserving account identity and the signed-in token. An unconfirmed checkout must retain the manual trial and expiry. Preserve the active Business account and its cloud settings, the active Basic badge with cloud controls locked, login-first behavior without an active badge or enabled cloud controls for signed-out or tokenless accounts, and routing existing Basic upgrades to billing without a duplicate checkout. Use synthetic service responses for tests. The task concerns Account settings; do not change product policy or native capture.",
+ "agent_timeout_seconds": 1200,
+ "grader_dependency_links": [
+ {
+ "source_path": "apps/screenpipe-app-tauri/node_modules",
+ "destination_path": "apps/screenpipe-app-tauri/node_modules"
+ }
+ ],
+ "grader": {
+ "timeout_seconds": 180,
+ "fixtures": [
+ {
+ "local_path": "graders/trial-upgrade.fixture.tsx.txt",
+ "destination_path": "apps/screenpipe-app-tauri/components/settings/__tests__/eval-trial-upgrade.test.tsx"
+ }
+ ],
+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config vitest.config.ts components/settings/__tests__/eval-trial-upgrade.test.tsx --maxWorkers=1 --minWorkers=1"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/trial-upgrade.fixture.tsx.txt b/evals/coding-agent/graders/trial-upgrade.fixture.tsx.txt
new file mode 100644
index 000000000..cb8ebca1a
--- /dev/null
+++ b/evals/coding-agent/graders/trial-upgrade.fixture.tsx.txt
@@ -0,0 +1,306 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+import { afterEach, describe, expect, it, vi } from "vitest";
+import {
+ act,
+ cleanup,
+ fireEvent,
+ render,
+ screen,
+} from "@testing-library/react";
+
+const mocks = vi.hoisted(() => ({
+ state: { user: null as any },
+ updateSettings: vi.fn(async (patch: any) => {
+ if (Object.hasOwn(patch, "user")) mocks.state.user = structuredClone(patch.user);
+ }),
+ loadUser: vi.fn(async (token: string) => {
+ if (token !== "tok") throw new Error("synthetic session mismatch");
+ mocks.state.user = { ...mocks.state.user, entitlement: { active: true, plan: "pro", source: "subscription", status: "active" } };
+ }),
+ openLoginWindow: vi.fn().mockResolvedValue(undefined),
+ piUpdateConfig: vi.fn().mockResolvedValue(undefined),
+ capture: vi.fn(),
+ openUrl: vi.fn().mockResolvedValue(undefined),
+ eventHandlers: new Map<string, (event: any) => unknown>(),
+ listen: vi.fn(
+ async (event: string, handler: (event: any) => unknown) => {
+ mocks.eventHandlers.set(event, handler);
+ return () => mocks.eventHandlers.delete(event);
+ },
+ ),
+}));
+
+// AccountSection reads everything through useSettings + the tauri `commands`
+// object; swap `mocks.state.user` per case. Keep `@/lib/app-entitlement` REAL —
+// `isSignedInCloudSubscriber` is the gate under test.
+vi.mock("@/lib/hooks/use-settings", () => ({
+ useSettings: () => ({
+ settings: {
+ user: mocks.state.user,
+ pipeSyncEnabled: false,
+ memoriesSyncEnabled: false,
+ connectionSyncEnabled: false,
+ },
+ updateSettings: mocks.updateSettings,
+ loadUser: mocks.loadUser,
+ }),
+}));
+
+vi.mock("@/lib/hooks/use-health-check", () => ({
+ useHealthCheck: () => ({ isServerDown: false }),
+}));
+
+vi.mock("@/lib/hooks/use-managed-policy", () => ({
+ useManagedPolicy: () => ({ isManagedDeployment: false }),
+}));
+
+vi.mock("@/lib/utils/tauri", () => ({
+ commands: {
+ openLoginWindow: mocks.openLoginWindow,
+ piUpdateConfig: mocks.piUpdateConfig,
+ },
+}));
+
+vi.mock("@/components/ui/use-toast", () => ({ toast: vi.fn() }));
+vi.mock("@/lib/api", () => ({ localFetch: vi.fn() }));
+vi.mock("posthog-js", () => ({ default: { capture: mocks.capture } }));
+
+// Tauri plugins the effect wires up on mount — keep them inert.
+vi.mock("@tauri-apps/plugin-shell", () => ({ open: mocks.openUrl }));
+vi.mock("@tauri-apps/plugin-deep-link", () => ({
+ onOpenUrl: vi.fn().mockResolvedValue(() => {}),
+}));
+vi.mock("@tauri-apps/api/event", () => ({
+ listen: mocks.listen,
+}));
+
+// ReferralCard pulls its own data deps; it is irrelevant to the gate.
+vi.mock("@/components/settings/referral-card", () => ({ ReferralCard: () => null }));
+
+import { AccountSection } from "../account-section";
+
+// Locked upsell toggles may remain visible; only enabled cloud controls grant access.
+const cloudSettings = () => screen.queryAllByRole("switch").filter(control => !(control as HTMLButtonElement).disabled);
+
+function loginStatus(): string {
+ return (screen.getByText(/^(logged in as|not logged in)/i).textContent || "").toLowerCase();
+}
+
+describe("AccountSection subscription/login gating", () => {
+ afterEach(() => {
+ cleanup();
+ vi.useRealTimers();
+ vi.clearAllMocks();
+ vi.unstubAllGlobals();
+ mocks.eventHandlers.clear();
+ mocks.state.user = null;
+ });
+
+ it("hides the active plan card for a tokenless stale shell (the bug)", () => {
+ // store.bin kept cloud_subscribed:true (+ id) but the token failed to
+ // hydrate. The header says "not logged in"; the card must agree.
+ mocks.state.user = {
+ id: "u1",
+ email: "[email protected]",
+ token: null,
+ cloud_subscribed: true,
+ subscription_plan: "pro",
+ };
+
+ render(<AccountSection />);
+
+ expect(loginStatus()).toContain("not logged in");
+ expect(screen.queryByText(/^active$/i)).not.toBeInTheDocument();
+ // The core assertion: no "Business · active" card under a "not logged in"
+ // header.
+ expect(cloudSettings()).toHaveLength(0);
+ // It falls through to the login-first layout instead.
+ expect(screen.getByText(/sign in to screenpipe/i)).toBeInTheDocument();
+ });
+
+ it("shows the active plan card for a real signed-in cloud subscriber", () => {
+ mocks.state.user = {
+ id: "u1",
+ email: "[email protected]",
+ token: "tok",
+ cloud_subscribed: true,
+ subscription_plan: "pro",
+ };
+
+ render(<AccountSection />);
+
+ expect(loginStatus()).toContain("logged in as [email protected]");
+ expect(cloudSettings()).not.toHaveLength(0);
+ expect(screen.getByRole("heading", { name: "Screenpipe Business" })).toBeInTheDocument();
+ expect(screen.getByText("active")).toBeInTheDocument();
+ });
+
+ it("opens website billing before a profile-granted Business plan expires", () => {
+ vi.useFakeTimers();
+ vi.setSystemTime(new Date("2026-07-21T12:00:00.000Z"));
+ mocks.state.user = {
+ id: "u1",
+ email: "[email protected]",
+ token: "tok",
+ cloud_subscribed: true,
+ subscription_plan: "pro",
+ plan_expires_at: "2026-08-04T12:00:00.000Z",
+ entitlement: {
+ active: true,
+ plan: "pro",
+ source: "manual",
+ status: "active",
+ },
+ };
+
+ render(<AccountSection />);
+ fireEvent.click(screen.getByRole("button", { name: /manage subscription/i }));
+
+ expect(screen.getByText("Business plan ends in 14 days")).toBeInTheDocument();
+ expect(mocks.openUrl).toHaveBeenCalledWith("https://screenpipe.com/account/billing");
+ });
+
+ it("hides a stale trial end date after the paid subscription starts", () => {
+ vi.useFakeTimers();
+ vi.setSystemTime(new Date("2026-07-21T12:00:00.000Z"));
+ mocks.state.user = {
+ id: "u1",
+ email: "[email protected]",
+ token: "tok",
+ cloud_subscribed: true,
+ subscription_plan: "pro",
+ plan_expires_at: "2026-08-04T12:00:00.000Z",
+ entitlement: {
+ active: true,
+ plan: "pro",
+ source: "subscription",
+ status: "active",
+ },
+ };
+
+ render(<AccountSection />);
+
+ expect(
+ screen.queryByText(/plan ends in/i),
+ ).not.toBeInTheDocument();
+ expect(
+ screen.queryByText("Business plan ends in 14 days"),
+ ).not.toBeInTheDocument();
+ });
+
+ it.each([true, false])("persists verified checkout activation and preserves unconfirmed trial (active=%s)", async (active) => {
+ vi.useFakeTimers();
+ vi.setSystemTime(new Date("2026-07-21T12:00:00.000Z"));
+ mocks.state.user = {
+ id: "u1",
+ email: "[email protected]",
+ token: "tok",
+ cloud_subscribed: true,
+ subscription_plan: "pro",
+ plan_expires_at: "2026-08-04T12:00:00.000Z",
+ entitlement: {
+ active: true,
+ plan: "pro",
+ source: "manual",
+ status: "active",
+ },
+ };
+ vi.stubGlobal(
+ "fetch",
+ vi
+ .fn()
+ .mockResolvedValueOnce(
+ new Response(
+ JSON.stringify({ url: "https://checkout.stripe.test/session" }),
+ { status: 200, headers: { "Content-Type": "application/json" } },
+ ),
+ )
+ .mockResolvedValueOnce(
+ new Response(
+ JSON.stringify({
+ hasSubscription: active,
+ subscription: { status: active ? "active" : "incomplete" },
+ }),
+ { status: 200, headers: { "Content-Type": "application/json" } },
+ ),
+ ),
+ );
+
+ render(<AccountSection />);
+ const trayUpgrade = mocks.eventHandlers.get("tray-upgrade");
+ expect(trayUpgrade).toBeDefined();
+
+ await act(async () => {
+ await trayUpgrade?.({ event: "tray-upgrade", id: 1, payload: null });
+ await vi.advanceTimersByTimeAsync(2000);
+ });
+
+ // The settings port persists complete user snapshots. Assert its final
+ // state, allowing changes to the number, ordering and shape of writes.
+ expect(mocks.state.user).toMatchObject({
+ id: "u1", token: "tok", email: "[email protected]",
+ cloud_subscribed: true, subscription_plan: "pro",
+ plan_expires_at: active ? null : "2026-08-04T12:00:00.000Z",
+ entitlement: { source: active ? "subscription" : "manual", active: true },
+ });
+ });
+
+ it("does not regress the logged-in Basic plan badge (token, no cloud)", () => {
+ // A paying Basic/standard owner is logged in but not cloud_subscribed: they
+ // still get an "active" named-plan badge, but NOT the cloud sync card.
+ mocks.state.user = {
+ id: "u1",
+ email: "[email protected]",
+ token: "tok",
+ cloud_subscribed: false,
+ subscription_plan: "standard",
+ };
+
+ render(<AccountSection />);
+
+ expect(loginStatus()).toContain("logged in as [email protected]");
+ expect(cloudSettings()).toHaveLength(0);
+ // Branch-3 named-plan badge still renders for the paying Basic user.
+ expect(screen.getByText("active")).toBeInTheDocument();
+ });
+
+ it("sends existing Basic subscribers to billing instead of creating a second checkout", () => {
+ const checkoutFetch = vi.fn().mockResolvedValue(
+ new Response(JSON.stringify({ url: "https://checkout.stripe.test/session" }), {
+ status: 200,
+ headers: { "Content-Type": "application/json" },
+ }),
+ );
+ vi.stubGlobal("fetch", checkoutFetch);
+ mocks.state.user = {
+ id: "u1",
+ email: "[email protected]",
+ token: "tok",
+ cloud_subscribed: false,
+ app_entitled: true,
+ subscription_plan: "standard",
+ };
+
+ render(<AccountSection />);
+ fireEvent.click(screen.getByRole("button", { name: /upgrade to business/i }));
+
+ expect(checkoutFetch).not.toHaveBeenCalledWith(
+ expect.stringContaining("/api/subscription/checkout"),
+ expect.anything(),
+ );
+ expect(mocks.openUrl).toHaveBeenCalledWith("https://screenpipe.com/account/billing");
+ });
+
+ it("shows the login-first layout for a signed-out free user", () => {
+ mocks.state.user = { token: null, cloud_subscribed: false };
+
+ render(<AccountSection />);
+
+ expect(loginStatus()).toContain("not logged in");
+ expect(screen.queryByText(/^active$/i)).not.toBeInTheDocument();
+ expect(cloudSettings()).toHaveLength(0);
+ expect(screen.getByText(/sign in to screenpipe/i)).toBeInTheDocument();
+ });
+});