Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: preserve verified entitlement plan identity

Louis Beaumont31e2e48a6804
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index ef9074ea5..34a0b846f 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -9,7 +9,7 @@ This is an agent eval suite, not a unit-test suite. Every case contains:
- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;
- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.
-The current app corpus contains 42 git-mined regressions. See
+The current app corpus contains 43 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest uses this same harness.
@@ -73,3 +73,5 @@ one harness implementation.
Do not turn capability scores into a release gate after one run. Establish matched-environment repeated baselines first. Regression cases intended to block should target reliable `pass^k`, not a lucky `pass@k`.
The app-provider-outage-vs-limit case uses standalone Bun tests without installed package dependencies. `BUN_BIN` may select an explicit Bun binary. It checks provider presentation semantics, not live model availability.
+
+The app-entitlement-plan-consistency case uses a fixed clock and standalone Bun tests to verify historical account normalization and downstream plan policy. Its URL configuration port is synthetic; it does not test gate rendering, billing connectivity, telemetry or recording.
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index 325d01a0b..e5b4f3355 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-09-14.1",
+ "dataset_version": "2026-09-18.1",
"suite_type": "regression",
"owner": "screenpipe engineering",
"policy": {
@@ -1321,6 +1321,43 @@
],
"command": "\"${BUN_BIN:-bun}\" test ./evals/coding-agent/graders/app-provider-outage-vs-limit.test.ts"
}
+ },
+ {
+ "id": "app-entitlement-plan-consistency",
+ "name": "Preserve verified entitlement plans when the API omits a plan label",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "6ce45931c33a89cf27675e947bea3abec3d80159",
+ "pull_request": "https://github.com/screenpipe/screenpipe/pull/5781"
+ },
+ "base_ref": "f0e9248ed600ee83e84e216e026ff98b6eead0e7",
+ "oracle_ref": "6ce45931c33a89cf27675e947bea3abec3d80159",
+ "oracle_paths": [
+ "apps/screenpipe-app-tauri/lib/app-entitlement.ts"
+ ],
+ "gate": "advisory",
+ "trigger_paths": [
+ "apps/screenpipe-app-tauri/lib/app-entitlement.ts"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "entitlement",
+ "normalization",
+ "preserved-behavior"
+ ],
+ "prompt": "A paid account becomes unknown to local plan policy when an API response omits subscription_plan or returns it as null, even though the response includes a fresh, active entitlement for the correct plan. Preserve valid lifetime, pro_max and pro_ultra plan identity and verified paid consumer access when that top-level label is absent; cloud access must still follow the entitlement. Keep the normalized account identity and supplied session token. An explicitly supplied label, including an empty string or a conflicting plan, retains precedence; conflicting or empty evidence must not grant paid access. Explicit app denial with no cloud subscription must retain free policy. Preserve boolean-only legacy standard/pro fallbacks, and ignore null, whitespace-only or non-string entitlement plan labels as fallback candidates without treating malformed evidence as verified paid. Stale, future-dated or inactive non-lifetime evidence and accounts without stable identity must remain unknown. Enterprise entitlements must not become consumer subscriptions. Keep the existing normalization and policy interfaces. This task concerns pure account classification, not gate rendering, telemetry, live billing or recorder behavior.",
+ "agent_timeout_seconds": 900,
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/app-entitlement-plan-consistency.test.ts",
+ "destination_path": "evals/coding-agent/graders/app-entitlement-plan-consistency.test.ts"
+ }
+ ],
+ "command": "\"${BUN_BIN:-bun}\" test ./evals/coding-agent/graders/app-entitlement-plan-consistency.test.ts"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/app-entitlement-plan-consistency.test.ts b/evals/coding-agent/graders/app-entitlement-plan-consistency.test.ts
new file mode 100644
index 000000000..75ac1fc05
--- /dev/null
+++ b/evals/coding-agent/graders/app-entitlement-plan-consistency.test.ts
@@ -0,0 +1,126 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+import { afterEach, beforeEach, expect, mock, setSystemTime, test } from "bun:test";
+
+// Only URL configuration is substituted. Account normalization and policy
+// classification execute the historical product module, with a fixed clock.
+mock.module("@/lib/web-url", () => ({
+ screenpipeWebUrl: (path: string) => `https://fixture.invalid${path}`,
+}));
+const {
+ normalizeAppUser,
+ getLocalPlanPolicy,
+ hasConsumerAppSubscription,
+ hasCloudEntitlement,
+} = await import("../../../apps/screenpipe-app-tauri/lib/app-entitlement");
+
+const NOW = new Date("2026-06-05T12:00:00.000Z");
+const originalForceGate = process.env.NEXT_PUBLIC_SCREENPIPE_FORCE_BILLING_GATE;
+beforeEach(() => {
+ setSystemTime(NOW);
+ process.env.NEXT_PUBLIC_SCREENPIPE_FORCE_BILLING_GATE = "true";
+});
+afterEach(() => {
+ setSystemTime();
+ if (originalForceGate === undefined) delete process.env.NEXT_PUBLIC_SCREENPIPE_FORCE_BILLING_GATE;
+ else process.env.NEXT_PUBLIC_SCREENPIPE_FORCE_BILLING_GATE = originalForceGate;
+});
+
+function account(plan = "pro_ultra", cloud = true) {
+ return {
+ id: "synthetic-account",
+ app_entitled: true,
+ cloud_subscribed: cloud,
+ entitlement: {
+ active: true,
+ plan,
+ source: plan === "lifetime" ? "lifetime" : "subscription",
+ checked_at: NOW.toISOString(),
+ features: { app: true, cloud },
+ },
+ };
+}
+
+for (const [plan, cloud] of [["lifetime", false], ["pro_max", true], ["pro_ultra", true]] as const) {
+ for (const absent of ["omitted", "null"] as const) {
+ test(`${plan} stays verified when top-level plan is ${absent}`, () => {
+ const raw = account(plan, cloud);
+ const normalized = normalizeAppUser(absent === "null" ? { ...raw, subscription_plan: null } : raw, "synthetic-token");
+ expect(getLocalPlanPolicy(normalized)).toBe("verified-paid");
+ expect(normalized.subscription_plan).toBe(plan);
+ expect(normalized.id).toBe("synthetic-account");
+ expect(normalized.token).toBe("synthetic-token");
+ expect(hasConsumerAppSubscription(normalized)).toBe(true);
+ expect(hasCloudEntitlement(normalized)).toBe(cloud);
+ });
+ }
+}
+
+test("explicit conflicting server plan retains precedence and stays unknown", () => {
+ const normalized = normalizeAppUser({ ...account(), subscription_plan: "standard" }, "synthetic-token");
+ expect(normalized.subscription_plan).toBe("standard");
+ expect(getLocalPlanPolicy(normalized)).toBe("unknown");
+ expect(hasConsumerAppSubscription(normalized)).toBe(false);
+});
+
+test("an explicit empty plan is not treated as an omitted plan", () => {
+ const normalized = normalizeAppUser({ ...account(), subscription_plan: "" }, "synthetic-token");
+ expect(normalized.subscription_plan).toBe("");
+ expect(getLocalPlanPolicy(normalized)).toBe("unknown");
+});
+
+test("explicit app denial without cloud subscription remains free", () => {
+ const normalized = normalizeAppUser({ ...account("lifetime", false), app_entitled: false }, "synthetic-token");
+ expect(normalized.subscription_plan).toBe("none");
+ expect(getLocalPlanPolicy(normalized)).toBe("verified-free");
+ expect(hasConsumerAppSubscription(normalized)).toBe(false);
+ expect(hasCloudEntitlement(normalized)).toBe(false);
+});
+
+for (const cloud of [false, true]) {
+ test(`legacy boolean-only evidence keeps its ${cloud ? "pro" : "standard"} fallback`, () => {
+ const normalized = normalizeAppUser({ id: "synthetic-account", app_entitled: true, cloud_subscribed: cloud }, "synthetic-token");
+ expect(normalized.subscription_plan).toBe(cloud ? "pro" : "standard");
+ expect(getLocalPlanPolicy(normalized)).toBe("verified-paid");
+ expect(hasConsumerAppSubscription(normalized)).toBe(true);
+ expect(hasCloudEntitlement(normalized)).toBe(cloud);
+ });
+}
+
+for (const invalidPlan of [null, " ", 42]) {
+ test(`malformed entitlement plan ${JSON.stringify(invalidPlan)} cannot establish paid policy`, () => {
+ const raw = account();
+ const normalized = normalizeAppUser({ ...raw, entitlement: { ...raw.entitlement, plan: invalidPlan } }, "synthetic-token");
+ expect(normalized.subscription_plan).toBe("pro");
+ expect(getLocalPlanPolicy(normalized)).toBe("unknown");
+ expect(hasConsumerAppSubscription(normalized)).toBe(false);
+ });
+}
+
+for (const invalidEvidence of ["stale", "future", "inactive"] as const) {
+ test(`${invalidEvidence} non-lifetime evidence remains unknown after normalization`, () => {
+ const raw = account();
+ if (invalidEvidence === "stale") raw.entitlement.checked_at = "2026-06-01T12:00:00.000Z";
+ if (invalidEvidence === "future") raw.entitlement.checked_at = "2026-06-06T12:00:00.000Z";
+ if (invalidEvidence === "inactive") raw.entitlement.active = false;
+ const normalized = normalizeAppUser(raw, "synthetic-token");
+ expect(getLocalPlanPolicy(normalized)).toBe("unknown");
+ expect(hasConsumerAppSubscription(normalized)).toBe(false);
+ expect(hasCloudEntitlement(normalized)).toBe(false);
+ });
+}
+
+test("enterprise evidence is not a consumer subscription", () => {
+ const raw = account("enterprise");
+ raw.entitlement.source = "enterprise";
+ const normalized = normalizeAppUser({ ...raw, subscription_plan: "enterprise" }, "synthetic-token");
+ expect(getLocalPlanPolicy(normalized)).toBe("verified-paid");
+ expect(hasConsumerAppSubscription(normalized)).toBe(false);
+});
+
+test("missing stable identity cannot establish verified paid policy", () => {
+ const normalized = normalizeAppUser({ ...account(), id: "" }, "synthetic-token");
+ expect(getLocalPlanPolicy(normalized)).toBe("unknown");
+ expect(hasConsumerAppSubscription(normalized)).toBe(false);
+});