Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: cover fixed-term manual grant payment eligibility

Louis Beaumont1de77ff5411d
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 9102a715e..bde7a9305 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -1824,3 +1824,7 @@ durability, enforced isolation or model performance.
`app-mcp-search-source-evidence` exercises the actual HTTP MCP server and SDK client against a synthetic loopback backend. It checks frame/chunk references, input-event text, bounded and explicitly unlimited reads, pagination, missing IDs, empty results and upstream refusal. The broken parent fails five outcomes while preserving four; the historical reference passes nine.
Run `bun test evals/coding-agent/calibrate-mcp-search-evidence.test.js` to check the historical contrast, current source, equivalent implementation and broken/bypass controls. Dependencies are linked only for grading. This does not establish stdio/native telemetry, disconnect cleanup, live authentication, model performance or enforced host isolation. The existing activity-summary case tests another public tool.
+
+## Fixed-term manual grants
+
+`app-manual-grant-payment-prompts` executes the actual trial and contextual-card eligibility functions with synthetic accounts and a frozen clock. The parent fails four grant outcomes and preserves eighteen; the source-only reference and current source pass all twenty-two. Signup trials, ordinary payers, enterprise accounts, expired/invalid evidence and eligibility refusals remain covered. Run `bun test evals/coding-agent/calibrate-manual-grant.test.js` for correct, broken, equivalent, disconnected-caller, blanket-exemption and missing-source controls. Fixtures and dependencies appear only at grading. This does not establish rendered UI, native recording, live billing, identity-provider verification, enforced agent isolation or model performance.
diff --git a/evals/coding-agent/calibrate-manual-grant.test.js b/evals/coding-agent/calibrate-manual-grant.test.js
new file mode 100644
index 000000000..28f113515
--- /dev/null
+++ b/evals/coding-agent/calibrate-manual-grant.test.js
@@ -0,0 +1,37 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import {afterAll,expect,test} from 'bun:test';
+import {execFileSync,spawnSync} from 'node:child_process';
+import {mkdirSync,mkdtempSync,readFileSync,writeFileSync,symlinkSync,rmSync} from 'node:fs';
+import {dirname,join,resolve} from 'node:path';
+import {tmpdir} from 'node:os';
+const repo=resolve(import.meta.dir,'../..'),app='apps/screenpipe-app-tauri';
+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='app-manual-grant-payment-prompts');
+const root=mkdtempSync(join(tmpdir(),'manual-grant-calibration-'));
+const entitlement=`${app}/lib/app-entitlement.ts`,trial=`${app}/lib/first-run/trial-activation.ts`,card=`${app}/lib/card-ask/gating.ts`;
+afterAll(()=>rmSync(root,{recursive:true,force:true}));
+const show=(ref,path)=>execFileSync('git',['show',`${ref}:${path}`],{cwd:repo});
+function edit(cwd,path,from,to){const f=join(cwd,path),s=readFileSync(f,'utf8');expect(s.split(from)).toHaveLength(2);writeFileSync(f,s.replace(from,to));}
+function grade(name,ref=item.oracle_ref,mutate=()=>{}){
+ const cwd=join(root,name);mkdirSync(cwd);
+ execFileSync('tar',['-x','-C',cwd],{input:execFileSync('git',['archive',ref,'--',`${app}/lib`,`${app}/tsconfig.json`],{cwd:repo,maxBuffer:64*1024*1024})});
+ for(const f of item.grader.fixtures){const dest=join(cwd,f.destination_path);mkdirSync(dirname(dest),{recursive:true});writeFileSync(dest,readFileSync(join(import.meta.dir,f.local_path)));}
+ for(const link of item.grader_dependency_links)symlinkSync(join(repo,link.source_path),join(cwd,link.destination_path),'dir');
+ mutate(cwd);
+ const r=spawnSync('/bin/bash',['-c',item.grader.command],{cwd,encoding:'utf8',timeout:60000,maxBuffer:4*1024*1024,env:{PATH:process.env.PATH,HOME:cwd,CI:'true',TZ:'UTC',NO_COLOR:'1'}});
+ if(process.env.EVAL_CALIBRATION_RESULTS){const out=resolve(process.env.EVAL_CALIBRATION_RESULTS);mkdirSync(out,{recursive:true});writeFileSync(join(out,name+'.stdout'),r.stdout||'');writeFileSync(join(out,name+'.stderr'),r.stderr||'');writeFileSync(join(out,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit:r.status,signal:r.signal,error:r.error?.message??null},null,2));}
+ expect(r.error).toBeUndefined();expect(r.signal).toBeNull();return r;
+}
+function passes(r){expect(r.status).toBe(0);expect(r.stdout).toContain('22 passed');expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import/);}
+function fails(r){expect(r.status).toBe(1);expect(r.stderr).toMatch(/AssertionError/);expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Cannot find module/);}
+test('parent fails four grant outcomes and preserves eighteen',()=>{const r=grade('known_broken',item.base_ref);fails(r);expect(r.stdout).toContain('4 failed | 18 passed');},60000);
+test('historical fix passes',()=>passes(grade('known_correct')),60000);
+test('current source passes',()=>passes(grade('current','HEAD')),60000);
+test('declared source-only oracle repairs parent',()=>passes(grade('oracle_only',item.base_ref,cwd=>{for(const p of item.oracle_paths)writeFileSync(join(cwd,p),show(item.oracle_ref,p));})),60000);
+test('equivalent period comparison passes',()=>passes(grade('preserved_behavior',item.oracle_ref,cwd=>edit(cwd,entitlement,'periodEnd > Date.now()','Date.now() < periodEnd'))),60000);
+test('unused correct source cannot mask broken callers',()=>fails(grade('bypass',item.base_ref,cwd=>writeFileSync(join(cwd,`${app}/lib/unused-entitlement.ts`),show(item.oracle_ref,entitlement)))),60000);
+test('blanket grant exemption fails',()=>fails(grade('blanket',item.oracle_ref,cwd=>edit(cwd,entitlement,'const periodEnd = parseEntitlementTime(entitlement?.current_period_end);','return true;\n const periodEnd = parseEntitlementTime(entitlement?.current_period_end);'))),60000);
+test('repairing only trial eligibility fails card behavior',()=>fails(grade('trial_only',item.oracle_ref,cwd=>edit(cwd,card,'if (hasActiveManualSubscriptionGrant(user)) return false;',''))),60000);
+test('repairing only card eligibility fails trial behavior',()=>fails(grade('card_only',item.oracle_ref,cwd=>edit(cwd,trial,'if (hasActiveManualSubscriptionGrant(user)) return true;',''))),60000);
+test('expired grant exemption fails',()=>fails(grade('expired',item.oracle_ref,cwd=>edit(cwd,entitlement,'periodEnd > Date.now()','true'))),60000);
+test('missing source is setup error, not behavior evidence',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,trial)));expect(r.status).toBe(1);expect(r.stdout+r.stderr).toMatch(/Failed to resolve import|Failed to load url/);expect(r.stderr).not.toMatch(/AssertionError/);},60000);
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index b31926aec..daf6a240d 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-10-09-mcp-search-evidence",
+ "dataset_version": "2026-10-09-manual-grant",
"suite_type": "regression",
"owner": "screenpipe engineering",
"policy": {
@@ -6007,6 +6007,55 @@
],
"command": "\"${BUN_BIN:-bun}\" test packages/ai-gateway/src/test/eval-null-model-accounting.test.ts"
}
+ },
+ {
+ "id": "app-manual-grant-payment-prompts",
+ "name": "Honor active fixed-term grants across trial and card eligibility",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "b1c2736e8c0d6218be4d8053c84a3920912956cb"
+ },
+ "base_ref": "2670901fe575db4715830fdaa9e39512356d32c0",
+ "oracle_ref": "b1c2736e8c0d6218be4d8053c84a3920912956cb",
+ "oracle_paths": [
+ "apps/screenpipe-app-tauri/lib/app-entitlement.ts",
+ "apps/screenpipe-app-tauri/lib/card-ask/gating.ts",
+ "apps/screenpipe-app-tauri/lib/first-run/trial-activation.ts"
+ ],
+ "trigger_paths": [
+ "apps/screenpipe-app-tauri/lib/app-entitlement.ts",
+ "apps/screenpipe-app-tauri/lib/card-ask/gating.ts",
+ "apps/screenpipe-app-tauri/lib/first-run/trial-activation.ts"
+ ],
+ "gate": "advisory",
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "entitlement",
+ "preserved-behavior"
+ ],
+ "prompt": "An account with verified, active fixed-term manual app access and no payment card is still sent through trial activation and contextual card requests. Repair the existing account eligibility flow so a consistent paid plan with fresh entitlement evidence, active app feature and a future subscription period end exempts the account from trial activation, new-trial enrollment and contextual card requests. Accept whitespace/case-normalized manual source values. Profile signup trials use expires_at rather than a subscription period end and must retain their ordinary trial/card eligibility. Do not exempt inactive, expired (including exactly at the deadline), invalid or absent period ends, stale checked evidence without grace, missing app features, non-manual or conflicting unpaid source evidence, inconsistent plan truth or explicit app denial. Preserve card-backed subscription and enterprise-account exemptions, signed-out refusal, settings-loading refusal and existing-install trial refusal. Keep existing public interfaces compatible; no particular helper or implementation is required. Grading exercises actual eligibility functions with synthetic accounts and a frozen clock, not rendered UI, native recording, live billing, provider authentication or execution isolation.",
+ "agent_timeout_seconds": 900,
+ "grader_dependency_links": [
+ {
+ "source_path": "apps/screenpipe-app-tauri/node_modules",
+ "destination_path": "apps/screenpipe-app-tauri/node_modules"
+ }
+ ],
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/manual-grant.fixture.ts.txt",
+ "destination_path": "apps/screenpipe-app-tauri/lib/first-run/manual-grant.eval.test.ts"
+ },
+ {
+ "local_path": "graders/manual-grant.config.mjs",
+ "destination_path": "apps/screenpipe-app-tauri/manual-grant.config.mjs"
+ }
+ ],
+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config manual-grant.config.mjs"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/manual-grant.config.mjs b/evals/coding-agent/graders/manual-grant.config.mjs
new file mode 100644
index 000000000..2b1c6811a
--- /dev/null
+++ b/evals/coding-agent/graders/manual-grant.config.mjs
@@ -0,0 +1,5 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import {defineConfig} from "vitest/config";
+import {fileURLToPath} from "node:url";
+export default defineConfig({resolve:{alias:{"@":fileURLToPath(new URL(".",import.meta.url))}},test:{environment:"node",include:["lib/first-run/manual-grant.eval.test.ts"],maxWorkers:1,minWorkers:1}});
diff --git a/evals/coding-agent/graders/manual-grant.fixture.ts.txt b/evals/coding-agent/graders/manual-grant.fixture.ts.txt
new file mode 100644
index 000000000..256a1da67
--- /dev/null
+++ b/evals/coding-agent/graders/manual-grant.fixture.ts.txt
@@ -0,0 +1,87 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+import { describe, expect, it, beforeEach, afterEach, vi } from "vitest";
+import type { AppEntitlement, AppUser } from "@/lib/app-entitlement";
+import { isCardAskEligible } from "@/lib/card-ask/gating";
+import { bypassesTrialActivation, isTrialActivationEligible } from "./trial-activation";
+
+function participant(overrides: Partial<AppEntitlement> = {}): AppUser {
+ return {
+ id: "synthetic-participant",
+ token: "synthetic-token",
+ subscription_plan: "pro",
+ cloud_subscribed: true,
+ app_entitled: true,
+ has_payment_method: false,
+ entitlement_source: "manual",
+ plan_expires_at: null,
+ entitlement: {
+ source: "manual",
+ plan: "pro",
+ active: true,
+ status: "active",
+ current_period_end: new Date(Date.now() + 30 * 86400000).toISOString(),
+ expires_at: null,
+ checked_at: new Date().toISOString(),
+ features: { app: true, cloud: true },
+ ...overrides,
+ },
+ } as AppUser;
+}
+
+describe("cardless participant grants", () => {
+ it("bypasses trial activation during a valid fixed-term grant", () => {
+ expect(bypassesTrialActivation(participant())).toBe(true);
+ });
+ it("does not enroll a valid grant holder in a trial", () => {
+ expect(isTrialActivationEligible(true, participant(), false)).toBe(false);
+ });
+ it("does not offer a contextual card prompt during a valid grant", () => {
+ expect(isCardAskEligible(participant(), true)).toBe(false);
+ });
+
+ it("keeps profile signup trials in the existing card flow", () => {
+ const expires = new Date(Date.now() + 7 * 86400000).toISOString();
+ const user = { ...participant({ current_period_end: null, expires_at: expires }), plan_expires_at: expires };
+ expect(bypassesTrialActivation(user)).toBe(false);
+ expect(isCardAskEligible(user, true)).toBe(true);
+ });
+
+ it.each<Partial<AppEntitlement>>([
+ { active: false },
+ { current_period_end: new Date(0).toISOString() },
+ { current_period_end: "invalid" },
+ { current_period_end: null },
+ { checked_at: new Date(0).toISOString() },
+ { features: { app: false, cloud: false } },
+ { source: "none" },
+ ])("does not exempt invalid or expired grants: %j", (overrides) => {
+ expect(bypassesTrialActivation(participant(overrides))).toBe(false);
+ expect(isCardAskEligible(participant(overrides), true)).toBe(true);
+ });
+});
+
+beforeEach(() => { vi.useFakeTimers(); vi.setSystemTime(new Date("2026-10-09T18:00:00Z")); });
+afterEach(() => vi.useRealTimers());
+it("keeps ordinary paid card holders exempt", () => {
+ const user={...participant({source:"subscription"}),entitlement_source:"subscription",has_payment_method:true};
+ expect(bypassesTrialActivation(user)).toBe(true); expect(isTrialActivationEligible(true,user,false)).toBe(false); expect(isCardAskEligible(user,true)).toBe(false);
+});
+it("keeps enterprise accounts exempt", () => {
+ const user={...participant(),enterprise_account:{team_id:"synthetic-team"}};
+ expect(bypassesTrialActivation(user)).toBe(true); expect(isCardAskEligible(user,true)).toBe(false);
+});
+it("keeps signed-out users out of both enrollments", () => {
+ expect(isTrialActivationEligible(true,null,false)).toBe(false); expect(isCardAskEligible(null,true)).toBe(false);
+});
+it("preserves settings loading refusal", () => {expect(isCardAskEligible(participant({current_period_end:null}),false)).toBe(false);});
+it("preserves existing-install trial refusal", () => {expect(isTrialActivationEligible(false,participant({current_period_end:null}),false)).toBe(false);});
+it("preserves signup trial enrollment", () => {expect(isTrialActivationEligible(true,participant({current_period_end:null,expires_at:new Date(Date.now()+86400000).toISOString()}),false)).toBe(true);});
+it("does not treat a conflicting unpaid source as a grant", () => {
+ const user={...participant(),entitlement_source:"none"}; expect(bypassesTrialActivation(user)).toBe(false);expect(isCardAskEligible(user,true)).toBe(true);
+});
+it("does not exempt inconsistent plan truth", () => {const user={...participant(),subscription_plan:"none"};expect(bypassesTrialActivation(user)).toBe(false);expect(isCardAskEligible(user,true)).toBe(true);});
+it("does not exempt explicit app refusal", () => {const user={...participant(),app_entitled:false};expect(bypassesTrialActivation(user)).toBe(false);expect(isCardAskEligible(user,true)).toBe(true);});
+it("ends the exemption exactly at the granted deadline", () => {const user=participant({current_period_end:new Date(Date.now()).toISOString()});expect(bypassesTrialActivation(user)).toBe(false);expect(isCardAskEligible(user,true)).toBe(true);});
+it("accepts normalized manual source evidence", () => {const user={...participant({source:" MANUAL "}),entitlement_source:" Manual "};expect(bypassesTrialActivation(user)).toBe(true);expect(isCardAskEligible(user,true)).toBe(false);});