Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

test(evals): cover complete free entitlement evidence

louisd6ef918a045b
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index bc0cee93d..63a5029ea 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -9,7 +9,7 @@ This is an agent eval suite, not a unit-test suite. Every case contains:
- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;
- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.
-The current app corpus contains 79 git-mined regressions. See
+The current app corpus contains 80 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest uses this same harness.
@@ -898,3 +898,20 @@ permitted writes and reads. No commands are executed. This does not establish
complete shell analysis, native enforcement, symlink safety or agent isolation.
Run `bun test evals/coding-agent/calibrate-pipe-write-hook.test.js` for historical,
current, equivalent-implementation, unused-correct and mutation controls.
+
+## Complete free entitlement evidence
+
+`app-complete-free-entitlement-evidence` executes the real account normalizer and
+policy interfaces against synthetic account values with a fixed clock. Complete
+legacy denial becomes verified free; partial newer evidence remains unknown.
+Twenty-six outcomes preserve offline free policy, identity/token boundaries,
+explicit denial and valid paid plans. The parent fails ten intended assertions
+and preserves sixteen; the historical fix and current source pass all 26.
+
+Run `bun test evals/coding-agent/calibrate-free-evidence.test.js` for ten controls,
+including unused correct code, equivalent private names, missing cloud/feature
+evidence, whitespace authentication and unsupported sources. Missing source is
+a setup error. Only URL configuration is mocked. The companion paid-plan-label
+case covers a separate omission invariant. Fixtures are installed at grading;
+no UI/native recording, live provider verification, agent isolation or model
+improvement is established.
diff --git a/evals/coding-agent/calibrate-free-evidence.test.js b/evals/coding-agent/calibrate-free-evidence.test.js
new file mode 100644
index 000000000..cb2bafd7f
--- /dev/null
+++ b/evals/coding-agent/calibrate-free-evidence.test.js
@@ -0,0 +1,33 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from 'bun:test';
+import { execFileSync, spawnSync } from 'node:child_process';
+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { dirname, join, resolve } from 'node:path';
+import { tmpdir } from 'node:os';
+const repo=resolve(import.meta.dir,'../..');
+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='app-complete-free-entitlement-evidence');
+const sourcePath='apps/screenpipe-app-tauri/lib/app-entitlement.ts';
+const graderPath='evals/coding-agent/graders/free-evidence.test.ts';
+const show=ref=>execFileSync('git',['show',`${ref}:${sourcePath}`],{cwd:repo,encoding:'utf8'});
+const fixed=show(item.oracle_ref),broken=show(item.base_ref),grader=readFileSync(join(repo,graderPath),'utf8');
+const root=process.env.EVAL_CALIBRATION_DIR?resolve(process.env.EVAL_CALIBRATION_DIR):mkdtempSync(join(tmpdir(),'free-evidence-'));
+mkdirSync(root,{recursive:true});afterAll(()=>{if(!process.env.EVAL_CALIBRATION_DIR)rmSync(root,{recursive:true,force:true});});
+function grade(name,source,unused=false){
+ const cwd=join(root,name);for(const [p,s] of Object.entries({[graderPath]:grader,...(source===null?{}:{[sourcePath]:source}),...(unused?{'apps/screenpipe-app-tauri/lib/unused-correct.ts':fixed}:{})})){const f=join(cwd,p);mkdirSync(dirname(f),{recursive:true});writeFileSync(f,s);}
+ const r=spawnSync(process.execPath,['test',graderPath],{cwd,encoding:'utf8',timeout:30000,env:{PATH:dirname(process.execPath)}});
+ if(process.env.EVAL_CALIBRATION_DIR){writeFileSync(join(root,name+'.stdout'),r.stdout||'');writeFileSync(join(root,name+'.stderr'),r.stderr||'');writeFileSync(join(root,name+'.json'),JSON.stringify({exit_code:r.status,signal:r.signal,error:r.error?.message??null}));}return r;
+}
+function failure(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain('expect(received)');expect(r.stderr).not.toContain('Cannot find module');expect(r.stderr).not.toContain('Unhandled error');expect(r.stderr).toMatch(/[1-9][0-9]* pass/);}
+function success(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(0);expect(r.stderr).toContain('26 pass');}
+function change(from,to){expect(fixed.split(from)).toHaveLength(2);return fixed.replace(from,to);}
+test('known broken parent fails behavior and preserves neighbors',()=>failure(grade('parent',broken)));
+test('historical fix passes every outcome',()=>success(grade('reference',fixed)));
+test('equivalent private names are accepted',()=>success(grade('equivalent',fixed.replaceAll('legacyVerifiedFree','completeOldDenial').replaceAll('hasVerifiedFreePlan','freeEvidenceValid'))));
+test('unused correct implementation cannot mask broken exports',()=>failure(grade('unused',broken,true)));
+test('legacy denial cannot be ignored',()=>failure(grade('no-legacy',change('legacyVerifiedFree ||','false ||'))));
+test('missing cloud truth cannot be accepted',()=>failure(grade('missing-cloud',change('user.cloud_subscribed !== false','user.cloud_subscribed === true'))));
+test('app feature evidence cannot be omitted',()=>failure(grade('missing-feature',change('typeof entitlement.features?.app !== "boolean" ||','false ||').replace('entitlement.features.cloud === true','entitlement.features?.cloud === true'))));
+test('whitespace token cannot authenticate',()=>failure(grade('whitespace',change('user.token.trim().length > 0','user.token.length > 0'))));
+test('unknown free source cannot be trusted',()=>failure(grade('unknown-source',change('source !== "none" ||','["manual", "enterprise", "lifetime", "dev"].includes(source) ||'))));
+test('missing source is setup failure',()=>{const r=grade('missing',null);expect(r.status).toBe(1);expect(r.stderr).toContain('Cannot find module');expect(r.stderr).toContain('0 pass');});
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 56fa70c79..0f93e14cd 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-09-25.1",
+ "dataset_version": "2026-09-25.3",
"suite_type": "regression",
"owner": "screenpipe engineering",
"policy": {
@@ -3265,6 +3265,41 @@
],
"command": "cd apps/screenpipe-app-tauri && NODE_OPTIONS=--localstorage-file=.eval-localstorage node node_modules/vitest/vitest.mjs run --config vitest.config.ts components/__tests__/eval-card-login-hydration.test.tsx --maxWorkers=1 --minWorkers=1"
}
+ },
+ {
+ "id": "app-complete-free-entitlement-evidence",
+ "name": "Accept complete legacy free evidence while rejecting partial account truth",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "e3adc22d1fbd54a0637a3f2fae7ccb379f72ebd0"
+ },
+ "base_ref": "4cc85bdf22649300f41b2e67d492de649ac124cb",
+ "oracle_ref": "e3adc22d1fbd54a0637a3f2fae7ccb379f72ebd0",
+ "oracle_paths": [
+ "apps/screenpipe-app-tauri/lib/app-entitlement.ts"
+ ],
+ "gate": "advisory",
+ "trigger_paths": [
+ "apps/screenpipe-app-tauri/lib/app-entitlement.ts"
+ ],
+ "tags": [
+ "entitlement",
+ "identity",
+ "free-policy",
+ "normalization"
+ ],
+ "prompt": "An authenticated older API response containing a stable account identity and cloud_subscribed=false, with app_entitled, subscription_plan and entitlement omitted, is incorrectly classified as unknown. Normalize that complete legacy denial into verified-free policy, preserving account identity and the supplied session token, without granting paid app access. Partial newer evidence (plan-only or no explicit app-feature boolean) stays unknown; explicit null newer fields must not be treated as legacy omissions. Complete newer free evidence requires explicit cloud_subscribed=false, matching none plan labels, source none, boolean active and app feature evidence, a non-future checked_at, no cloud feature and no future grace. Either true or false app-feature/active values may describe a complete free account; old verified-free evidence remains free offline beyond 72 hours. Missing/null/true cloud denial and subscription,unknown,manual,enterprise,lifetime or dev sources cannot establish free policy. Empty, whitespace-only and null tokens cannot authenticate cached free policy, though cached free classification itself remains valid. Require a nonblank id or clerk_id; preserve clerk identity fallback. Preserve explicit app denial overriding stale paid labels and valid fresh standard,pro and lifetime paid classification. Keep the public normalization/classification interfaces; no private helper name or implementation shape is required. This task covers pure classification, not native recording, UI gates, provider verification or production entitlement safety.",
+ "agent_timeout_seconds": 900,
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/free-evidence.test.ts",
+ "destination_path": "evals/coding-agent/graders/free-evidence.test.ts"
+ }
+ ],
+ "command": "bun test ./evals/coding-agent/graders/free-evidence.test.ts"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/free-evidence.test.ts b/evals/coding-agent/graders/free-evidence.test.ts
new file mode 100644
index 000000000..214cec99b
--- /dev/null
+++ b/evals/coding-agent/graders/free-evidence.test.ts
@@ -0,0 +1,48 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterEach, beforeEach, expect, mock, setSystemTime, test } from "bun:test";
+mock.module("@/lib/web-url", () => ({ screenpipeWebUrl: (p: string) => `https://fixture.invalid${p}` }));
+const { normalizeAppUser, getLocalPlanPolicy, isAuthenticatedFreeUser, hasVerifiedPaidPlan, hasAppEntitlement } = await import("../../../apps/screenpipe-app-tauri/lib/app-entitlement");
+const NOW = new Date("2026-06-05T12:00:00Z");
+const oldGate = process.env.NEXT_PUBLIC_SCREENPIPE_FORCE_BILLING_GATE;
+beforeEach(() => { setSystemTime(NOW); process.env.NEXT_PUBLIC_SCREENPIPE_FORCE_BILLING_GATE = "true"; });
+afterEach(() => { setSystemTime(); if (oldGate === undefined) delete process.env.NEXT_PUBLIC_SCREENPIPE_FORCE_BILLING_GATE; else process.env.NEXT_PUBLIC_SCREENPIPE_FORCE_BILLING_GATE = oldGate; });
+const legacy = () => ({ id: "fixture-owner", cloud_subscribed: false });
+const free = () => ({ id: "fixture-owner", token: "fixture-token", cloud_subscribed: false, subscription_plan: "none", entitlement: { active: true, plan: "none", source: "none", checked_at: NOW.toISOString(), features: { app: true, cloud: false } } });
+test("complete legacy denial normalizes to authenticated free without paid app access", () => {
+ const u = normalizeAppUser(legacy(), "fixture-token");
+ expect(getLocalPlanPolicy(u)).toBe("verified-free"); expect(isAuthenticatedFreeUser(u)).toBe(true);
+ expect(hasVerifiedPaidPlan(u)).toBe(false); expect(hasAppEntitlement(u)).toBe(false);
+ expect(u.id).toBe("fixture-owner"); expect(u.token).toBe("fixture-token");
+});
+for (const entitlement of [{ plan: "none" }, { active: false, plan: "none", source: "none" }]) {
+ test(`partial newer evidence stays unknown: ${JSON.stringify(entitlement)}`, () => {
+ const u=normalizeAppUser({...legacy(),entitlement},"fixture-token");expect(getLocalPlanPolicy(u)).toBe("unknown");expect(isAuthenticatedFreeUser(u)).toBe(false);
+ });
+}
+for (const field of ["app_entitled", "subscription_plan", "entitlement"]) {
+ test(`explicit null ${field} is not a legacy omission`, () => { const u=normalizeAppUser({...legacy(),[field]:null},"fixture-token");expect(getLocalPlanPolicy(u)).toBe("unknown"); });
+}
+for (const token of ["", " ", null]) {
+ test(`token ${JSON.stringify(token)} cannot authenticate cached free policy`, () => { const u={...free(),token};expect(getLocalPlanPolicy(u)).toBe("verified-free");expect(isAuthenticatedFreeUser(u as any)).toBe(false); });
+}
+for (const active of [true,false]) {
+ test(`complete newer free evidence with app=${active} remains free offline`, () => { const u=free();u.entitlement={...u.entitlement,active,checked_at:"2026-05-01T00:00:00Z",features:{app:active,cloud:false}};expect(getLocalPlanPolicy(u)).toBe("verified-free");expect(isAuthenticatedFreeUser(u)).toBe(true); });
+}
+for (const cloud of [undefined,null,true]) {
+ test(`cloud denial must be explicit: ${cloud}`, () => {expect(getLocalPlanPolicy({...free(),cloud_subscribed:cloud} as any)).toBe("unknown");});
+}
+for (const source of ["subscription","unexpected","manual","enterprise","lifetime","dev"]) {
+ test(`source ${source} cannot establish free policy`, () => {const u=free();u.entitlement.source=source;expect(getLocalPlanPolicy(u)).toBe("unknown");});
+}
+test("cloud feature, future timestamps and grace do not establish free policy", () => {
+ for(const patch of [{features:{app:true,cloud:true}},{checked_at:"2027-01-01T00:00:00Z"},{grace_until:"2026-06-06T12:00:00Z"}]) { const u=free();expect(getLocalPlanPolicy({...u,entitlement:{...u.entitlement,...patch}})).toBe("unknown"); }
+});
+test("stable identity is required and clerk fallback stays valid",()=>{
+ expect(getLocalPlanPolicy(normalizeAppUser({...legacy(),id:" "},"fixture-token"))).toBe("unknown");
+ const u=normalizeAppUser({...legacy(),id:" ",clerk_id:"fixture-clerk"},"fixture-token");expect(getLocalPlanPolicy(u)).toBe("verified-free");expect(u.clerk_id).toBe("fixture-clerk");
+});
+test("explicit denial still overrides stale paid label",()=>{const u=normalizeAppUser({...legacy(),app_entitled:false,subscription_plan:"standard"},"fixture-token");expect(getLocalPlanPolicy(u)).toBe("verified-free");expect(hasVerifiedPaidPlan(u)).toBe(false);});
+for(const plan of ["standard","pro","lifetime"]) {
+ test(`preserve valid ${plan} paid policy`,()=>{const u=normalizeAppUser({id:"fixture-owner",app_entitled:true,cloud_subscribed:plan==="pro",subscription_plan:plan,entitlement:{active:true,plan,source:plan==="lifetime"?"lifetime":"subscription",checked_at:NOW.toISOString(),features:{app:true,cloud:plan==="pro"}}},"fixture-token");expect(getLocalPlanPolicy(u)).toBe("verified-paid");expect(hasVerifiedPaidPlan(u)).toBe(true);expect(isAuthenticatedFreeUser(u)).toBe(false);});
+}