YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: cover fixed-term manual grant payment eligibility
Louis Beaumont1de77ff5411d
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 9102a715e..bde7a9305 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -1824,3 +1824,7 @@ durability, enforced isolation or model performance.`app-mcp-search-source-evidence` exercises the actual HTTP MCP server and SDK client against a synthetic loopback backend. It checks frame/chunk references, input-event text, bounded and explicitly unlimited reads, pagination, missing IDs, empty results and upstream refusal. The broken parent fails five outcomes while preserving four; the historical reference passes nine.Run `bun test evals/coding-agent/calibrate-mcp-search-evidence.test.js` to check the historical contrast, current source, equivalent implementation and broken/bypass controls. Dependencies are linked only for grading. This does not establish stdio/native telemetry, disconnect cleanup, live authentication, model performance or enforced host isolation. The existing activity-summary case tests another public tool.++## Fixed-term manual grants++`app-manual-grant-payment-prompts` executes the actual trial and contextual-card eligibility functions with synthetic accounts and a frozen clock. The parent fails four grant outcomes and preserves eighteen; the source-only reference and current source pass all twenty-two. Signup trials, ordinary payers, enterprise accounts, expired/invalid evidence and eligibility refusals remain covered. Run `bun test evals/coding-agent/calibrate-manual-grant.test.js` for correct, broken, equivalent, disconnected-caller, blanket-exemption and missing-source controls. Fixtures and dependencies appear only at grading. This does not establish rendered UI, native recording, live billing, identity-provider verification, enforced agent isolation or model performance.diff --git a/evals/coding-agent/calibrate-manual-grant.test.js b/evals/coding-agent/calibrate-manual-grant.test.jsnew file mode 100644index 000000000..28f113515--- /dev/null+++ b/evals/coding-agent/calibrate-manual-grant.test.js@@ -0,0 +1,37 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import {afterAll,expect,test} from 'bun:test';+import {execFileSync,spawnSync} from 'node:child_process';+import {mkdirSync,mkdtempSync,readFileSync,writeFileSync,symlinkSync,rmSync} from 'node:fs';+import {dirname,join,resolve} from 'node:path';+import {tmpdir} from 'node:os';+const repo=resolve(import.meta.dir,'../..'),app='apps/screenpipe-app-tauri';+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='app-manual-grant-payment-prompts');+const root=mkdtempSync(join(tmpdir(),'manual-grant-calibration-'));+const entitlement=`${app}/lib/app-entitlement.ts`,trial=`${app}/lib/first-run/trial-activation.ts`,card=`${app}/lib/card-ask/gating.ts`;+afterAll(()=>rmSync(root,{recursive:true,force:true}));+const show=(ref,path)=>execFileSync('git',['show',`${ref}:${path}`],{cwd:repo});+function edit(cwd,path,from,to){const f=join(cwd,path),s=readFileSync(f,'utf8');expect(s.split(from)).toHaveLength(2);writeFileSync(f,s.replace(from,to));}+function grade(name,ref=item.oracle_ref,mutate=()=>{}){+ const cwd=join(root,name);mkdirSync(cwd);+ execFileSync('tar',['-x','-C',cwd],{input:execFileSync('git',['archive',ref,'--',`${app}/lib`,`${app}/tsconfig.json`],{cwd:repo,maxBuffer:64*1024*1024})});+ for(const f of item.grader.fixtures){const dest=join(cwd,f.destination_path);mkdirSync(dirname(dest),{recursive:true});writeFileSync(dest,readFileSync(join(import.meta.dir,f.local_path)));}+ for(const link of item.grader_dependency_links)symlinkSync(join(repo,link.source_path),join(cwd,link.destination_path),'dir');+ mutate(cwd);+ const r=spawnSync('/bin/bash',['-c',item.grader.command],{cwd,encoding:'utf8',timeout:60000,maxBuffer:4*1024*1024,env:{PATH:process.env.PATH,HOME:cwd,CI:'true',TZ:'UTC',NO_COLOR:'1'}});+ if(process.env.EVAL_CALIBRATION_RESULTS){const out=resolve(process.env.EVAL_CALIBRATION_RESULTS);mkdirSync(out,{recursive:true});writeFileSync(join(out,name+'.stdout'),r.stdout||'');writeFileSync(join(out,name+'.stderr'),r.stderr||'');writeFileSync(join(out,name+'.json'),JSON.stringify({ref,command:item.grader.command,exit:r.status,signal:r.signal,error:r.error?.message??null},null,2));}+ expect(r.error).toBeUndefined();expect(r.signal).toBeNull();return r;+}+function passes(r){expect(r.status).toBe(0);expect(r.stdout).toContain('22 passed');expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import/);}+function fails(r){expect(r.status).toBe(1);expect(r.stderr).toMatch(/AssertionError/);expect(r.stdout+r.stderr).not.toMatch(/Unhandled|Uncaught|Failed to resolve import|Cannot find module/);}+test('parent fails four grant outcomes and preserves eighteen',()=>{const r=grade('known_broken',item.base_ref);fails(r);expect(r.stdout).toContain('4 failed | 18 passed');},60000);+test('historical fix passes',()=>passes(grade('known_correct')),60000);+test('current source passes',()=>passes(grade('current','HEAD')),60000);+test('declared source-only oracle repairs parent',()=>passes(grade('oracle_only',item.base_ref,cwd=>{for(const p of item.oracle_paths)writeFileSync(join(cwd,p),show(item.oracle_ref,p));})),60000);+test('equivalent period comparison passes',()=>passes(grade('preserved_behavior',item.oracle_ref,cwd=>edit(cwd,entitlement,'periodEnd > Date.now()','Date.now() < periodEnd'))),60000);+test('unused correct source cannot mask broken callers',()=>fails(grade('bypass',item.base_ref,cwd=>writeFileSync(join(cwd,`${app}/lib/unused-entitlement.ts`),show(item.oracle_ref,entitlement)))),60000);+test('blanket grant exemption fails',()=>fails(grade('blanket',item.oracle_ref,cwd=>edit(cwd,entitlement,'const periodEnd = parseEntitlementTime(entitlement?.current_period_end);','return true;\n const periodEnd = parseEntitlementTime(entitlement?.current_period_end);'))),60000);+test('repairing only trial eligibility fails card behavior',()=>fails(grade('trial_only',item.oracle_ref,cwd=>edit(cwd,card,'if (hasActiveManualSubscriptionGrant(user)) return false;',''))),60000);+test('repairing only card eligibility fails trial behavior',()=>fails(grade('card_only',item.oracle_ref,cwd=>edit(cwd,trial,'if (hasActiveManualSubscriptionGrant(user)) return true;',''))),60000);+test('expired grant exemption fails',()=>fails(grade('expired',item.oracle_ref,cwd=>edit(cwd,entitlement,'periodEnd > Date.now()','true'))),60000);+test('missing source is setup error, not behavior evidence',()=>{const r=grade('missing',item.oracle_ref,cwd=>rmSync(join(cwd,trial)));expect(r.status).toBe(1);expect(r.stdout+r.stderr).toMatch(/Failed to resolve import|Failed to load url/);expect(r.stderr).not.toMatch(/AssertionError/);},60000);diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex b31926aec..daf6a240d 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-10-09-mcp-search-evidence",+ "dataset_version": "2026-10-09-manual-grant","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -6007,6 +6007,55 @@],"command": "\"${BUN_BIN:-bun}\" test packages/ai-gateway/src/test/eval-null-model-accounting.test.ts"}+ },+ {+ "id": "app-manual-grant-payment-prompts",+ "name": "Honor active fixed-term grants across trial and card eligibility",+ "source": {+ "kind": "git_regression",+ "fix_commit": "b1c2736e8c0d6218be4d8053c84a3920912956cb"+ },+ "base_ref": "2670901fe575db4715830fdaa9e39512356d32c0",+ "oracle_ref": "b1c2736e8c0d6218be4d8053c84a3920912956cb",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/lib/app-entitlement.ts",+ "apps/screenpipe-app-tauri/lib/card-ask/gating.ts",+ "apps/screenpipe-app-tauri/lib/first-run/trial-activation.ts"+ ],+ "trigger_paths": [+ "apps/screenpipe-app-tauri/lib/app-entitlement.ts",+ "apps/screenpipe-app-tauri/lib/card-ask/gating.ts",+ "apps/screenpipe-app-tauri/lib/first-run/trial-activation.ts"+ ],+ "gate": "advisory",+ "tags": [+ "regression",+ "escaped-failure",+ "entitlement",+ "preserved-behavior"+ ],+ "prompt": "An account with verified, active fixed-term manual app access and no payment card is still sent through trial activation and contextual card requests. Repair the existing account eligibility flow so a consistent paid plan with fresh entitlement evidence, active app feature and a future subscription period end exempts the account from trial activation, new-trial enrollment and contextual card requests. Accept whitespace/case-normalized manual source values. Profile signup trials use expires_at rather than a subscription period end and must retain their ordinary trial/card eligibility. Do not exempt inactive, expired (including exactly at the deadline), invalid or absent period ends, stale checked evidence without grace, missing app features, non-manual or conflicting unpaid source evidence, inconsistent plan truth or explicit app denial. Preserve card-backed subscription and enterprise-account exemptions, signed-out refusal, settings-loading refusal and existing-install trial refusal. Keep existing public interfaces compatible; no particular helper or implementation is required. Grading exercises actual eligibility functions with synthetic accounts and a frozen clock, not rendered UI, native recording, live billing, provider authentication or execution isolation.",+ "agent_timeout_seconds": 900,+ "grader_dependency_links": [+ {+ "source_path": "apps/screenpipe-app-tauri/node_modules",+ "destination_path": "apps/screenpipe-app-tauri/node_modules"+ }+ ],+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/manual-grant.fixture.ts.txt",+ "destination_path": "apps/screenpipe-app-tauri/lib/first-run/manual-grant.eval.test.ts"+ },+ {+ "local_path": "graders/manual-grant.config.mjs",+ "destination_path": "apps/screenpipe-app-tauri/manual-grant.config.mjs"+ }+ ],+ "command": "cd apps/screenpipe-app-tauri && node node_modules/vitest/vitest.mjs run --config manual-grant.config.mjs"+ }}]}diff --git a/evals/coding-agent/graders/manual-grant.config.mjs b/evals/coding-agent/graders/manual-grant.config.mjsnew file mode 100644index 000000000..2b1c6811a--- /dev/null+++ b/evals/coding-agent/graders/manual-grant.config.mjs@@ -0,0 +1,5 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import {defineConfig} from "vitest/config";+import {fileURLToPath} from "node:url";+export default defineConfig({resolve:{alias:{"@":fileURLToPath(new URL(".",import.meta.url))}},test:{environment:"node",include:["lib/first-run/manual-grant.eval.test.ts"],maxWorkers:1,minWorkers:1}});diff --git a/evals/coding-agent/graders/manual-grant.fixture.ts.txt b/evals/coding-agent/graders/manual-grant.fixture.ts.txtnew file mode 100644index 000000000..256a1da67--- /dev/null+++ b/evals/coding-agent/graders/manual-grant.fixture.ts.txt@@ -0,0 +1,87 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++import { describe, expect, it, beforeEach, afterEach, vi } from "vitest";+import type { AppEntitlement, AppUser } from "@/lib/app-entitlement";+import { isCardAskEligible } from "@/lib/card-ask/gating";+import { bypassesTrialActivation, isTrialActivationEligible } from "./trial-activation";++function participant(overrides: Partial<AppEntitlement> = {}): AppUser {+ return {+ id: "synthetic-participant",+ token: "synthetic-token",+ subscription_plan: "pro",+ cloud_subscribed: true,+ app_entitled: true,+ has_payment_method: false,+ entitlement_source: "manual",+ plan_expires_at: null,+ entitlement: {+ source: "manual",+ plan: "pro",+ active: true,+ status: "active",+ current_period_end: new Date(Date.now() + 30 * 86400000).toISOString(),+ expires_at: null,+ checked_at: new Date().toISOString(),+ features: { app: true, cloud: true },+ ...overrides,+ },+ } as AppUser;+}++describe("cardless participant grants", () => {+ it("bypasses trial activation during a valid fixed-term grant", () => {+ expect(bypassesTrialActivation(participant())).toBe(true);+ });+ it("does not enroll a valid grant holder in a trial", () => {+ expect(isTrialActivationEligible(true, participant(), false)).toBe(false);+ });+ it("does not offer a contextual card prompt during a valid grant", () => {+ expect(isCardAskEligible(participant(), true)).toBe(false);+ });++ it("keeps profile signup trials in the existing card flow", () => {+ const expires = new Date(Date.now() + 7 * 86400000).toISOString();+ const user = { ...participant({ current_period_end: null, expires_at: expires }), plan_expires_at: expires };+ expect(bypassesTrialActivation(user)).toBe(false);+ expect(isCardAskEligible(user, true)).toBe(true);+ });++ it.each<Partial<AppEntitlement>>([+ { active: false },+ { current_period_end: new Date(0).toISOString() },+ { current_period_end: "invalid" },+ { current_period_end: null },+ { checked_at: new Date(0).toISOString() },+ { features: { app: false, cloud: false } },+ { source: "none" },+ ])("does not exempt invalid or expired grants: %j", (overrides) => {+ expect(bypassesTrialActivation(participant(overrides))).toBe(false);+ expect(isCardAskEligible(participant(overrides), true)).toBe(true);+ });+});++beforeEach(() => { vi.useFakeTimers(); vi.setSystemTime(new Date("2026-10-09T18:00:00Z")); });+afterEach(() => vi.useRealTimers());+it("keeps ordinary paid card holders exempt", () => {+ const user={...participant({source:"subscription"}),entitlement_source:"subscription",has_payment_method:true};+ expect(bypassesTrialActivation(user)).toBe(true); expect(isTrialActivationEligible(true,user,false)).toBe(false); expect(isCardAskEligible(user,true)).toBe(false);+});+it("keeps enterprise accounts exempt", () => {+ const user={...participant(),enterprise_account:{team_id:"synthetic-team"}};+ expect(bypassesTrialActivation(user)).toBe(true); expect(isCardAskEligible(user,true)).toBe(false);+});+it("keeps signed-out users out of both enrollments", () => {+ expect(isTrialActivationEligible(true,null,false)).toBe(false); expect(isCardAskEligible(null,true)).toBe(false);+});+it("preserves settings loading refusal", () => {expect(isCardAskEligible(participant({current_period_end:null}),false)).toBe(false);});+it("preserves existing-install trial refusal", () => {expect(isTrialActivationEligible(false,participant({current_period_end:null}),false)).toBe(false);});+it("preserves signup trial enrollment", () => {expect(isTrialActivationEligible(true,participant({current_period_end:null,expires_at:new Date(Date.now()+86400000).toISOString()}),false)).toBe(true);});+it("does not treat a conflicting unpaid source as a grant", () => {+ const user={...participant(),entitlement_source:"none"}; expect(bypassesTrialActivation(user)).toBe(false);expect(isCardAskEligible(user,true)).toBe(true);+});+it("does not exempt inconsistent plan truth", () => {const user={...participant(),subscription_plan:"none"};expect(bypassesTrialActivation(user)).toBe(false);expect(isCardAskEligible(user,true)).toBe(true);});+it("does not exempt explicit app refusal", () => {const user={...participant(),app_entitled:false};expect(bypassesTrialActivation(user)).toBe(false);expect(isCardAskEligible(user,true)).toBe(true);});+it("ends the exemption exactly at the granted deadline", () => {const user=participant({current_period_end:new Date(Date.now()).toISOString()});expect(bypassesTrialActivation(user)).toBe(false);expect(isCardAskEligible(user,true)).toBe(true);});+it("accepts normalized manual source evidence", () => {const user={...participant({source:" MANUAL "}),entitlement_source:" Manual "};expect(bypassesTrialActivation(user)).toBe(true);expect(isCardAskEligible(user,true)).toBe(false);});