Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

evals: require verified gateway identity before paid access

Louis Beaumont04eb89ec104e
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 34a0b846f..448d4f0e5 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -9,7 +9,7 @@ This is an agent eval suite, not a unit-test suite. Every case contains:
- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;
- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.
-The current app corpus contains 43 git-mined regressions. See
+The current app corpus contains 44 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest uses this same harness.
@@ -75,3 +75,23 @@ Do not turn capability scores into a release gate after one run. Establish match
The app-provider-outage-vs-limit case uses standalone Bun tests without installed package dependencies. `BUN_BIN` may select an explicit Bun binary. It checks provider presentation semantics, not live model availability.
The app-entitlement-plan-consistency case uses a fixed clock and standalone Bun tests to verify historical account normalization and downstream plan policy. Its URL configuration port is synthetic; it does not test gate rendering, billing connectivity, telemetry or recording.
+
+The ai-gateway-verified-identity case invokes the historical `validateAuth`
+boundary with synthetic Clerk verification and website/database response ports.
+It rejects public account identifiers as credentials, malformed verifier subjects
+and unsuccessful legacy responses while preserving legitimate authenticated and
+anonymous behavior. Only `auth.ts` from the fixing commit is applied as the
+oracle; the same commit's route and native-settings changes are outside this
+case. No JWT cryptography, live account, billing, or full gateway route behavior
+is exercised.
+
+Calibrate the identity grader with:
+
+```bash
+bun test evals/coding-agent/calibrate-gateway-identity.test.js
+```
+
+The five controls require the historical contrast, accept equivalent helper
+renaming, reject an identifier-trusting fast path, and reject blanket anonymous
+fallback. These are grader controls, not model trials. The July 10 historical
+case does not require machine-service-token support introduced afterward.
diff --git a/evals/coding-agent/calibrate-gateway-identity.test.js b/evals/coding-agent/calibrate-gateway-identity.test.js
new file mode 100644
index 000000000..f31f87e2e
--- /dev/null
+++ b/evals/coding-agent/calibrate-gateway-identity.test.js
@@ -0,0 +1,70 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+// Local deterministic grader calibration; no model or external provider call.
+import { afterAll, expect, test } from "bun:test";
+import { execFileSync, spawnSync } from "node:child_process";
+import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import { tmpdir } from "node:os";
+
+const repo = resolve(import.meta.dir, "../..");
+const manifest = JSON.parse(readFileSync(join(import.meta.dir, "cases.json"), "utf8"));
+const item = manifest.cases.find(item => item.id === "ai-gateway-verified-identity");
+const authPath = "packages/ai-gateway/src/utils/auth.ts";
+const subscriptionPath = "packages/ai-gateway/src/utils/subscription.ts";
+const show = (ref, path) => execFileSync("git", ["-c", "core.commitGraph=false", "show", `${ref}:${path}`], {
+ cwd: repo, encoding: "utf8",
+});
+const broken = show(item.base_ref, authPath);
+const fixed = show(item.oracle_ref, authPath);
+const subscription = show(item.base_ref, subscriptionPath);
+const root = mkdtempSync(join(tmpdir(), "gateway-identity-calibration-"));
+afterAll(() => rmSync(root, { recursive: true, force: true }));
+function grade(name, source) {
+ const workspace = join(root, name);
+ const write = (path, contents) => {
+ mkdirSync(dirname(join(workspace, path)), { recursive: true });
+ writeFileSync(join(workspace, path), contents);
+ };
+ write(authPath, source); write(subscriptionPath, subscription);
+ const fixture = item.grader.fixtures[0];
+ const destination = join(workspace, fixture.destination_path);
+ mkdirSync(dirname(destination), { recursive: true });
+ copyFileSync(join(import.meta.dir, fixture.local_path), destination);
+ return spawnSync(process.execPath, ["test", fixture.destination_path], {
+ cwd: workspace, encoding: "utf8", timeout: 15_000,
+ env: { PATH: dirname(process.execPath) },
+ });
+}
+function behaviorFailure(result) {
+ expect(result.error).toBeUndefined(); expect(result.signal).toBeNull();
+ expect(result.status).toBe(1); expect(result.stderr).toContain("expect(received)");
+ expect(result.stderr).not.toContain("Cannot find module");
+}
+test("broken parent fails identity assertions with 11 preserved passes", () => {
+ const result = grade("parent", broken); behaviorFailure(result);
+ expect(result.stderr).toContain("11 pass"); expect(result.stderr).toContain("17 fail");
+});
+test("known correct reference needs no further changes", () => {
+ const result = grade("reference", fixed);
+ expect(result.status).toBe(0); expect(result.stderr).toContain("28 pass");
+});
+test("equivalent helper renaming remains valid", () => {
+ expect(fixed).toContain("validateSubscriptionWithId");
+ const result = grade("renamed", fixed.replaceAll("validateSubscriptionWithId", "lookupVerifiedSubscription"));
+ expect(result.status).toBe(0); expect(result.stderr).toContain("28 pass");
+});
+test("a raw-identifier fast path cannot bypass the verified boundary", () => {
+ const anchor = "const token = authHeader.split(' ')[1];";
+ expect(fixed.split(anchor)).toHaveLength(2);
+ const bypass = `\n if (token.startsWith('user_') || /^[0-9a-f-]{36}$/.test(token)) {\n return { isValid: true, tier: 'logged_in', userId: token, deviceId: token };\n }`;
+ const result = grade("identity-bypass", fixed.replace(anchor, anchor + bypass));
+ behaviorFailure(result); expect(result.stderr).toContain("8 fail");
+});
+test("blanket anonymous fallback fails legitimate authenticated access", () => {
+ const anchor = "const authHeader = request.headers.get('Authorization');";
+ expect(fixed.split(anchor)).toHaveLength(2);
+ const deny = "\n return { isValid: true, tier: 'anonymous', deviceId: headerDeviceId };";
+ behaviorFailure(grade("deny-authenticated", fixed.replace(anchor, anchor + deny)));
+});
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index e5b4f3355..2b4b3ea81 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -1,7 +1,7 @@
{
"schema_version": 1,
"suite": "screenpipe-app-coding-agent-regressions",
- "dataset_version": "2026-09-18.1",
+ "dataset_version": "2026-09-18.2",
"suite_type": "regression",
"owner": "screenpipe engineering",
"policy": {
@@ -1358,6 +1358,38 @@
],
"command": "\"${BUN_BIN:-bun}\" test ./evals/coding-agent/graders/app-entitlement-plan-consistency.test.ts"
}
+ },
+ {
+ "id": "ai-gateway-verified-identity",
+ "name": "Require verified account ownership before authenticated gateway tiers",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "252b96c168e79a6e377f307cf1e54423d17e0db8"
+ },
+ "base_ref": "041f111810b334ac3667006907f1dc5b511273df",
+ "oracle_ref": "252b96c168e79a6e377f307cf1e54423d17e0db8",
+ "oracle_paths": [
+ "packages/ai-gateway/src/utils/auth.ts"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "security",
+ "identity",
+ "entitlement"
+ ],
+ "prompt": "The gateway can treat a public database UUID or Clerk user ID supplied as a Bearer or Token credential as an authenticated account, even when no token verifier established ownership. Repair the authentication boundary so those public identifiers cannot grant logged-in or subscribed tiers or trigger account/subscription lookups. Preserve anonymous access and device identity for absent, invalid, or unproven credentials; preserve verified Clerk and successfully validated legacy website tokens at their proper logged-in/subscribed tier and canonical identity. A verifier result requires a nonempty string subject; a legacy website HTTP 200 alone is not proof of success and requires success=true with a user identity. Missing, rejected and unavailable verification must not manufacture an identity. Use synthetic credentials only. This task concerns validateAuth behavior; no particular helper structure is required.",
+ "agent_timeout_seconds": 900,
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/gateway-verified-identity.test.js",
+ "destination_path": "evals/coding-agent/graders/gateway-verified-identity.test.js"
+ }
+ ],
+ "command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/gateway-verified-identity.test.js"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/gateway-verified-identity.test.js b/evals/coding-agent/graders/gateway-verified-identity.test.js
new file mode 100644
index 000000000..f0b76da31
--- /dev/null
+++ b/evals/coding-agent/graders/gateway-verified-identity.test.js
@@ -0,0 +1,130 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+
+// Execute the gateway auth boundary; Clerk and website/database responses are
+// synthetic ports. This does not test JWT cryptography or live billing.
+import { beforeEach, expect, mock, test } from "bun:test";
+
+const UUID = "11111111-1111-4111-8111-111111111111";
+const SUBJECT = "user_fixtureVerified";
+const VALID_TOKEN = "eyJ.fixture.verified";
+const LEGACY_TOKEN = "eyJ.fixture.legacy";
+let state;
+mock.module("@clerk/backend", () => ({ verifyToken: async (token) => {
+ state.verifications.push(token);
+ if (state.acceptClerk && token === VALID_TOKEN) return { sub: state.subject };
+ throw new Error("synthetic verification rejected");
+} }));
+globalThis.fetch = async (input) => {
+ const url = new URL(String(input));
+ state.requests.push(url.href);
+ if (state.networkFailure) throw new Error("synthetic provider unavailable");
+ if (url.origin === "https://db.fixture.invalid") {
+ if (url.pathname === "/rest/v1/users") {
+ return Response.json([{ id: UUID, clerk_id: SUBJECT }]);
+ }
+ if (url.pathname === "/rest/v1/cloud_subscriptions") {
+ return Response.json(state.paid ? [{ id: "sub_fixture" }] : []);
+ }
+ }
+ if (url.href === "https://screenpipe.com/api/user") {
+ return Response.json(state.website, { status: state.websiteStatus });
+ }
+ state.unexpected.push(url.origin + url.pathname);
+ throw new Error("Unexpected network port; live access is forbidden");
+};
+const auth = await import("../../../packages/ai-gateway/src/utils/auth");
+const env = {
+ NODE_ENV: "production",
+ CLERK_SECRET_KEY: "synthetic-verifier-key",
+ SUPABASE_URL: "https://db.fixture.invalid",
+ SUPABASE_ANON_KEY: "synthetic-db-key",
+ AI_GATEWAY_SERVICE_TOKEN: "synthetic-machine-credential",
+};
+function request(token, scheme = "Bearer") {
+ return new Request("https://gateway.fixture.invalid/v1/usage", { headers: {
+ "X-Device-Id": "device_fixture",
+ ...(token ? { Authorization: `${scheme} ${token}` } : {}),
+ } });
+}
+function entitledUser(paid) {
+ const plan = paid ? "pro" : "standard";
+ return { success: true, user: { id: UUID, clerk_id: SUBJECT,
+ cloud_subscribed: paid, app_entitled: true, subscription_plan: plan,
+ entitlement: { active: true, plan, features: { app: true, cloud: paid } } } };
+}
+beforeEach(() => {
+ state = { acceptClerk: false, subject: SUBJECT, paid: false, websiteStatus: 200,
+ website: { success: false }, networkFailure: false, requests: [], verifications: [], unexpected: [] };
+ // Optional cache cleanup allows the same boundary assertions to be checked
+ // on current source too; historical implementations need no such helper.
+ auth.__resetAuthEntitlementCacheForTests?.();
+});
+async function result(token, scheme) {
+ const value = await auth.validateAuth(request(token, scheme), env);
+ expect(state.unexpected).toEqual([]);
+ return value;
+}
+async function anonymous(token, scheme) {
+ const value = await result(token, scheme);
+ expect(value).toMatchObject({ isValid: true, tier: "anonymous", deviceId: "device_fixture" });
+ expect(value.userId).toBeUndefined();
+ return value;
+}
+
+test("absent credentials retain anonymous access without identity lookup", async () => {
+ await anonymous(); expect(state.requests).toEqual([]); expect(state.verifications).toEqual([]);
+});
+for (const scheme of ["Bearer", "Token"]) {
+ for (const token of [UUID, "user_fixturePublic"]) {
+ for (const paid of [false, true]) {
+ test(`${scheme} public ${token === UUID ? "UUID" : "Clerk ID"}, paid=${paid}, cannot authenticate`, async () => {
+ state.paid = paid;
+ await anonymous(token, scheme);
+ expect(state.requests).toEqual([]);
+ });
+ }
+ }
+ for (const paid of [false, true]) {
+ test(`${scheme} verified caller retains ${paid ? "subscribed" : "logged_in"} access`, async () => {
+ state.acceptClerk = true; state.paid = paid; state.website = entitledUser(paid);
+ expect(await result(VALID_TOKEN, scheme)).toMatchObject({ isValid: true,
+ tier: paid ? "subscribed" : "logged_in", userId: SUBJECT, deviceId: SUBJECT });
+ expect(state.verifications).toContain(VALID_TOKEN);
+ });
+ }
+}
+for (const [name, subject] of [["missing", undefined], ["null", null], ["empty", ""], ["numeric", 42], ["object", {}]]) {
+ test(`verified-provider response with ${name} subject cannot authenticate`, async () => {
+ state.acceptClerk = true; state.subject = subject; await anonymous(VALID_TOKEN);
+ });
+}
+for (const [name, website] of [
+ ["unsuccessful", { success: false, user: { clerk_id: SUBJECT, cloud_subscribed: true } }],
+ ["missing success", { user: { clerk_id: SUBJECT, cloud_subscribed: true } }],
+ ["missing user", { success: true }],
+ ["missing identity", { success: true, user: { cloud_subscribed: true } }],
+]) {
+ test(`HTTP 200 website response with ${name} cannot authenticate`, async () => {
+ state.website = website; await anonymous(LEGACY_TOKEN);
+ });
+}
+for (const paid of [false, true]) {
+ test(`validated legacy token retains ${paid ? "subscribed" : "logged_in"} access`, async () => {
+ state.website = entitledUser(paid);
+ expect(await result(LEGACY_TOKEN)).toMatchObject({ isValid: true,
+ tier: paid ? "subscribed" : "logged_in", userId: SUBJECT, deviceId: SUBJECT });
+ });
+}
+test("unavailable legacy validation does not manufacture an identity", async () => {
+ state.networkFailure = true; await anonymous(LEGACY_TOKEN);
+});
+test("rejected legacy HTTP response cannot grant access", async () => {
+ state.website = entitledUser(true); state.websiteStatus = 401; await anonymous(LEGACY_TOKEN);
+});
+test("arbitrary token remains anonymous", async () => {
+ await anonymous("synthetic-invalid-credential"); expect(state.requests).toEqual([]);
+});
+test("unrecognized long credential cannot authenticate", async () => {
+ await anonymous(env.AI_GATEWAY_SERVICE_TOKEN + "-wrong");
+});