YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
test(evals): cover complete free entitlement evidence
louisd6ef918a045b
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex bc0cee93d..63a5029ea 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -9,7 +9,7 @@ This is an agent eval suite, not a unit-test suite. Every case contains:- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.-The current app corpus contains 79 git-mined regressions. See+The current app corpus contains 80 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.@@ -898,3 +898,20 @@ permitted writes and reads. No commands are executed. This does not establishcomplete shell analysis, native enforcement, symlink safety or agent isolation.Run `bun test evals/coding-agent/calibrate-pipe-write-hook.test.js` for historical,current, equivalent-implementation, unused-correct and mutation controls.++## Complete free entitlement evidence++`app-complete-free-entitlement-evidence` executes the real account normalizer and+policy interfaces against synthetic account values with a fixed clock. Complete+legacy denial becomes verified free; partial newer evidence remains unknown.+Twenty-six outcomes preserve offline free policy, identity/token boundaries,+explicit denial and valid paid plans. The parent fails ten intended assertions+and preserves sixteen; the historical fix and current source pass all 26.++Run `bun test evals/coding-agent/calibrate-free-evidence.test.js` for ten controls,+including unused correct code, equivalent private names, missing cloud/feature+evidence, whitespace authentication and unsupported sources. Missing source is+a setup error. Only URL configuration is mocked. The companion paid-plan-label+case covers a separate omission invariant. Fixtures are installed at grading;+no UI/native recording, live provider verification, agent isolation or model+improvement is established.diff --git a/evals/coding-agent/calibrate-free-evidence.test.js b/evals/coding-agent/calibrate-free-evidence.test.jsnew file mode 100644index 000000000..cb2bafd7f--- /dev/null+++ b/evals/coding-agent/calibrate-free-evidence.test.js@@ -0,0 +1,33 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterAll, expect, test } from 'bun:test';+import { execFileSync, spawnSync } from 'node:child_process';+import { mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { dirname, join, resolve } from 'node:path';+import { tmpdir } from 'node:os';+const repo=resolve(import.meta.dir,'../..');+const item=JSON.parse(readFileSync(join(import.meta.dir,'cases.json'),'utf8')).cases.find(c=>c.id==='app-complete-free-entitlement-evidence');+const sourcePath='apps/screenpipe-app-tauri/lib/app-entitlement.ts';+const graderPath='evals/coding-agent/graders/free-evidence.test.ts';+const show=ref=>execFileSync('git',['show',`${ref}:${sourcePath}`],{cwd:repo,encoding:'utf8'});+const fixed=show(item.oracle_ref),broken=show(item.base_ref),grader=readFileSync(join(repo,graderPath),'utf8');+const root=process.env.EVAL_CALIBRATION_DIR?resolve(process.env.EVAL_CALIBRATION_DIR):mkdtempSync(join(tmpdir(),'free-evidence-'));+mkdirSync(root,{recursive:true});afterAll(()=>{if(!process.env.EVAL_CALIBRATION_DIR)rmSync(root,{recursive:true,force:true});});+function grade(name,source,unused=false){+ const cwd=join(root,name);for(const [p,s] of Object.entries({[graderPath]:grader,...(source===null?{}:{[sourcePath]:source}),...(unused?{'apps/screenpipe-app-tauri/lib/unused-correct.ts':fixed}:{})})){const f=join(cwd,p);mkdirSync(dirname(f),{recursive:true});writeFileSync(f,s);}+ const r=spawnSync(process.execPath,['test',graderPath],{cwd,encoding:'utf8',timeout:30000,env:{PATH:dirname(process.execPath)}});+ if(process.env.EVAL_CALIBRATION_DIR){writeFileSync(join(root,name+'.stdout'),r.stdout||'');writeFileSync(join(root,name+'.stderr'),r.stderr||'');writeFileSync(join(root,name+'.json'),JSON.stringify({exit_code:r.status,signal:r.signal,error:r.error?.message??null}));}return r;+}+function failure(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(1);expect(r.stderr).toContain('expect(received)');expect(r.stderr).not.toContain('Cannot find module');expect(r.stderr).not.toContain('Unhandled error');expect(r.stderr).toMatch(/[1-9][0-9]* pass/);}+function success(r){expect(r.error).toBeUndefined();expect(r.signal).toBeNull();expect(r.status).toBe(0);expect(r.stderr).toContain('26 pass');}+function change(from,to){expect(fixed.split(from)).toHaveLength(2);return fixed.replace(from,to);}+test('known broken parent fails behavior and preserves neighbors',()=>failure(grade('parent',broken)));+test('historical fix passes every outcome',()=>success(grade('reference',fixed)));+test('equivalent private names are accepted',()=>success(grade('equivalent',fixed.replaceAll('legacyVerifiedFree','completeOldDenial').replaceAll('hasVerifiedFreePlan','freeEvidenceValid'))));+test('unused correct implementation cannot mask broken exports',()=>failure(grade('unused',broken,true)));+test('legacy denial cannot be ignored',()=>failure(grade('no-legacy',change('legacyVerifiedFree ||','false ||'))));+test('missing cloud truth cannot be accepted',()=>failure(grade('missing-cloud',change('user.cloud_subscribed !== false','user.cloud_subscribed === true'))));+test('app feature evidence cannot be omitted',()=>failure(grade('missing-feature',change('typeof entitlement.features?.app !== "boolean" ||','false ||').replace('entitlement.features.cloud === true','entitlement.features?.cloud === true'))));+test('whitespace token cannot authenticate',()=>failure(grade('whitespace',change('user.token.trim().length > 0','user.token.length > 0'))));+test('unknown free source cannot be trusted',()=>failure(grade('unknown-source',change('source !== "none" ||','["manual", "enterprise", "lifetime", "dev"].includes(source) ||'))));+test('missing source is setup failure',()=>{const r=grade('missing',null);expect(r.status).toBe(1);expect(r.stderr).toContain('Cannot find module');expect(r.stderr).toContain('0 pass');});diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex 56fa70c79..0f93e14cd 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-09-25.1",+ "dataset_version": "2026-09-25.3","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -3265,6 +3265,41 @@],"command": "cd apps/screenpipe-app-tauri && NODE_OPTIONS=--localstorage-file=.eval-localstorage node node_modules/vitest/vitest.mjs run --config vitest.config.ts components/__tests__/eval-card-login-hydration.test.tsx --maxWorkers=1 --minWorkers=1"}+ },+ {+ "id": "app-complete-free-entitlement-evidence",+ "name": "Accept complete legacy free evidence while rejecting partial account truth",+ "source": {+ "kind": "git_regression",+ "fix_commit": "e3adc22d1fbd54a0637a3f2fae7ccb379f72ebd0"+ },+ "base_ref": "4cc85bdf22649300f41b2e67d492de649ac124cb",+ "oracle_ref": "e3adc22d1fbd54a0637a3f2fae7ccb379f72ebd0",+ "oracle_paths": [+ "apps/screenpipe-app-tauri/lib/app-entitlement.ts"+ ],+ "gate": "advisory",+ "trigger_paths": [+ "apps/screenpipe-app-tauri/lib/app-entitlement.ts"+ ],+ "tags": [+ "entitlement",+ "identity",+ "free-policy",+ "normalization"+ ],+ "prompt": "An authenticated older API response containing a stable account identity and cloud_subscribed=false, with app_entitled, subscription_plan and entitlement omitted, is incorrectly classified as unknown. Normalize that complete legacy denial into verified-free policy, preserving account identity and the supplied session token, without granting paid app access. Partial newer evidence (plan-only or no explicit app-feature boolean) stays unknown; explicit null newer fields must not be treated as legacy omissions. Complete newer free evidence requires explicit cloud_subscribed=false, matching none plan labels, source none, boolean active and app feature evidence, a non-future checked_at, no cloud feature and no future grace. Either true or false app-feature/active values may describe a complete free account; old verified-free evidence remains free offline beyond 72 hours. Missing/null/true cloud denial and subscription,unknown,manual,enterprise,lifetime or dev sources cannot establish free policy. Empty, whitespace-only and null tokens cannot authenticate cached free policy, though cached free classification itself remains valid. Require a nonblank id or clerk_id; preserve clerk identity fallback. Preserve explicit app denial overriding stale paid labels and valid fresh standard,pro and lifetime paid classification. Keep the public normalization/classification interfaces; no private helper name or implementation shape is required. This task covers pure classification, not native recording, UI gates, provider verification or production entitlement safety.",+ "agent_timeout_seconds": 900,+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/free-evidence.test.ts",+ "destination_path": "evals/coding-agent/graders/free-evidence.test.ts"+ }+ ],+ "command": "bun test ./evals/coding-agent/graders/free-evidence.test.ts"+ }}]}diff --git a/evals/coding-agent/graders/free-evidence.test.ts b/evals/coding-agent/graders/free-evidence.test.tsnew file mode 100644index 000000000..214cec99b--- /dev/null+++ b/evals/coding-agent/graders/free-evidence.test.ts@@ -0,0 +1,48 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com+import { afterEach, beforeEach, expect, mock, setSystemTime, test } from "bun:test";+mock.module("@/lib/web-url", () => ({ screenpipeWebUrl: (p: string) => `https://fixture.invalid${p}` }));+const { normalizeAppUser, getLocalPlanPolicy, isAuthenticatedFreeUser, hasVerifiedPaidPlan, hasAppEntitlement } = await import("../../../apps/screenpipe-app-tauri/lib/app-entitlement");+const NOW = new Date("2026-06-05T12:00:00Z");+const oldGate = process.env.NEXT_PUBLIC_SCREENPIPE_FORCE_BILLING_GATE;+beforeEach(() => { setSystemTime(NOW); process.env.NEXT_PUBLIC_SCREENPIPE_FORCE_BILLING_GATE = "true"; });+afterEach(() => { setSystemTime(); if (oldGate === undefined) delete process.env.NEXT_PUBLIC_SCREENPIPE_FORCE_BILLING_GATE; else process.env.NEXT_PUBLIC_SCREENPIPE_FORCE_BILLING_GATE = oldGate; });+const legacy = () => ({ id: "fixture-owner", cloud_subscribed: false });+const free = () => ({ id: "fixture-owner", token: "fixture-token", cloud_subscribed: false, subscription_plan: "none", entitlement: { active: true, plan: "none", source: "none", checked_at: NOW.toISOString(), features: { app: true, cloud: false } } });+test("complete legacy denial normalizes to authenticated free without paid app access", () => {+ const u = normalizeAppUser(legacy(), "fixture-token");+ expect(getLocalPlanPolicy(u)).toBe("verified-free"); expect(isAuthenticatedFreeUser(u)).toBe(true);+ expect(hasVerifiedPaidPlan(u)).toBe(false); expect(hasAppEntitlement(u)).toBe(false);+ expect(u.id).toBe("fixture-owner"); expect(u.token).toBe("fixture-token");+});+for (const entitlement of [{ plan: "none" }, { active: false, plan: "none", source: "none" }]) {+ test(`partial newer evidence stays unknown: ${JSON.stringify(entitlement)}`, () => {+ const u=normalizeAppUser({...legacy(),entitlement},"fixture-token");expect(getLocalPlanPolicy(u)).toBe("unknown");expect(isAuthenticatedFreeUser(u)).toBe(false);+ });+}+for (const field of ["app_entitled", "subscription_plan", "entitlement"]) {+ test(`explicit null ${field} is not a legacy omission`, () => { const u=normalizeAppUser({...legacy(),[field]:null},"fixture-token");expect(getLocalPlanPolicy(u)).toBe("unknown"); });+}+for (const token of ["", " ", null]) {+ test(`token ${JSON.stringify(token)} cannot authenticate cached free policy`, () => { const u={...free(),token};expect(getLocalPlanPolicy(u)).toBe("verified-free");expect(isAuthenticatedFreeUser(u as any)).toBe(false); });+}+for (const active of [true,false]) {+ test(`complete newer free evidence with app=${active} remains free offline`, () => { const u=free();u.entitlement={...u.entitlement,active,checked_at:"2026-05-01T00:00:00Z",features:{app:active,cloud:false}};expect(getLocalPlanPolicy(u)).toBe("verified-free");expect(isAuthenticatedFreeUser(u)).toBe(true); });+}+for (const cloud of [undefined,null,true]) {+ test(`cloud denial must be explicit: ${cloud}`, () => {expect(getLocalPlanPolicy({...free(),cloud_subscribed:cloud} as any)).toBe("unknown");});+}+for (const source of ["subscription","unexpected","manual","enterprise","lifetime","dev"]) {+ test(`source ${source} cannot establish free policy`, () => {const u=free();u.entitlement.source=source;expect(getLocalPlanPolicy(u)).toBe("unknown");});+}+test("cloud feature, future timestamps and grace do not establish free policy", () => {+ for(const patch of [{features:{app:true,cloud:true}},{checked_at:"2027-01-01T00:00:00Z"},{grace_until:"2026-06-06T12:00:00Z"}]) { const u=free();expect(getLocalPlanPolicy({...u,entitlement:{...u.entitlement,...patch}})).toBe("unknown"); }+});+test("stable identity is required and clerk fallback stays valid",()=>{+ expect(getLocalPlanPolicy(normalizeAppUser({...legacy(),id:" "},"fixture-token"))).toBe("unknown");+ const u=normalizeAppUser({...legacy(),id:" ",clerk_id:"fixture-clerk"},"fixture-token");expect(getLocalPlanPolicy(u)).toBe("verified-free");expect(u.clerk_id).toBe("fixture-clerk");+});+test("explicit denial still overrides stale paid label",()=>{const u=normalizeAppUser({...legacy(),app_entitled:false,subscription_plan:"standard"},"fixture-token");expect(getLocalPlanPolicy(u)).toBe("verified-free");expect(hasVerifiedPaidPlan(u)).toBe(false);});+for(const plan of ["standard","pro","lifetime"]) {+ test(`preserve valid ${plan} paid policy`,()=>{const u=normalizeAppUser({id:"fixture-owner",app_entitled:true,cloud_subscribed:plan==="pro",subscription_plan:plan,entitlement:{active:true,plan,source:plan==="lifetime"?"lifetime":"subscription",checked_at:NOW.toISOString(),features:{app:true,cloud:plan==="pro"}}},"fixture-token");expect(getLocalPlanPolicy(u)).toBe("verified-paid");expect(hasVerifiedPaidPlan(u)).toBe(true);expect(isAuthenticatedFreeUser(u)).toBe(false);});+}