YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context
Rust agentsagiaiai-memoryaudio-recordingcomputer-visionhermeshermes-agentllmlocal-ailocal-firstmachine-learningmcpmultimodalopenclawprivacyrewindscreen-recordingspeech-to-textycombinator
evals: require verified gateway identity before paid access
Louis Beaumont04eb89ec104e
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.mdindex 34a0b846f..448d4f0e5 100644--- a/evals/coding-agent/README.md+++ b/evals/coding-agent/README.md@@ -9,7 +9,7 @@ This is an agent eval suite, not a unit-test suite. Every case contains:- saved prompt, transcript, candidate patch, grader output, runtime fingerprint, and result;- repeated-trial reporting with success rate, `pass@k`, and `pass^k`.-The current app corpus contains 43 git-mined regressions. See+The current app corpus contains 44 git-mined regressions. See[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, andhistory-mining workflow. The companion website manifest uses this same harness.@@ -75,3 +75,23 @@ Do not turn capability scores into a release gate after one run. Establish matchThe app-provider-outage-vs-limit case uses standalone Bun tests without installed package dependencies. `BUN_BIN` may select an explicit Bun binary. It checks provider presentation semantics, not live model availability.The app-entitlement-plan-consistency case uses a fixed clock and standalone Bun tests to verify historical account normalization and downstream plan policy. Its URL configuration port is synthetic; it does not test gate rendering, billing connectivity, telemetry or recording.++The ai-gateway-verified-identity case invokes the historical `validateAuth`+boundary with synthetic Clerk verification and website/database response ports.+It rejects public account identifiers as credentials, malformed verifier subjects+and unsuccessful legacy responses while preserving legitimate authenticated and+anonymous behavior. Only `auth.ts` from the fixing commit is applied as the+oracle; the same commit's route and native-settings changes are outside this+case. No JWT cryptography, live account, billing, or full gateway route behavior+is exercised.++Calibrate the identity grader with:++```bash+bun test evals/coding-agent/calibrate-gateway-identity.test.js+```++The five controls require the historical contrast, accept equivalent helper+renaming, reject an identifier-trusting fast path, and reject blanket anonymous+fallback. These are grader controls, not model trials. The July 10 historical+case does not require machine-service-token support introduced afterward.diff --git a/evals/coding-agent/calibrate-gateway-identity.test.js b/evals/coding-agent/calibrate-gateway-identity.test.jsnew file mode 100644index 000000000..f31f87e2e--- /dev/null+++ b/evals/coding-agent/calibrate-gateway-identity.test.js@@ -0,0 +1,70 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++// Local deterministic grader calibration; no model or external provider call.+import { afterAll, expect, test } from "bun:test";+import { execFileSync, spawnSync } from "node:child_process";+import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";+import { dirname, join, resolve } from "node:path";+import { tmpdir } from "node:os";++const repo = resolve(import.meta.dir, "../..");+const manifest = JSON.parse(readFileSync(join(import.meta.dir, "cases.json"), "utf8"));+const item = manifest.cases.find(item => item.id === "ai-gateway-verified-identity");+const authPath = "packages/ai-gateway/src/utils/auth.ts";+const subscriptionPath = "packages/ai-gateway/src/utils/subscription.ts";+const show = (ref, path) => execFileSync("git", ["-c", "core.commitGraph=false", "show", `${ref}:${path}`], {+ cwd: repo, encoding: "utf8",+});+const broken = show(item.base_ref, authPath);+const fixed = show(item.oracle_ref, authPath);+const subscription = show(item.base_ref, subscriptionPath);+const root = mkdtempSync(join(tmpdir(), "gateway-identity-calibration-"));+afterAll(() => rmSync(root, { recursive: true, force: true }));+function grade(name, source) {+ const workspace = join(root, name);+ const write = (path, contents) => {+ mkdirSync(dirname(join(workspace, path)), { recursive: true });+ writeFileSync(join(workspace, path), contents);+ };+ write(authPath, source); write(subscriptionPath, subscription);+ const fixture = item.grader.fixtures[0];+ const destination = join(workspace, fixture.destination_path);+ mkdirSync(dirname(destination), { recursive: true });+ copyFileSync(join(import.meta.dir, fixture.local_path), destination);+ return spawnSync(process.execPath, ["test", fixture.destination_path], {+ cwd: workspace, encoding: "utf8", timeout: 15_000,+ env: { PATH: dirname(process.execPath) },+ });+}+function behaviorFailure(result) {+ expect(result.error).toBeUndefined(); expect(result.signal).toBeNull();+ expect(result.status).toBe(1); expect(result.stderr).toContain("expect(received)");+ expect(result.stderr).not.toContain("Cannot find module");+}+test("broken parent fails identity assertions with 11 preserved passes", () => {+ const result = grade("parent", broken); behaviorFailure(result);+ expect(result.stderr).toContain("11 pass"); expect(result.stderr).toContain("17 fail");+});+test("known correct reference needs no further changes", () => {+ const result = grade("reference", fixed);+ expect(result.status).toBe(0); expect(result.stderr).toContain("28 pass");+});+test("equivalent helper renaming remains valid", () => {+ expect(fixed).toContain("validateSubscriptionWithId");+ const result = grade("renamed", fixed.replaceAll("validateSubscriptionWithId", "lookupVerifiedSubscription"));+ expect(result.status).toBe(0); expect(result.stderr).toContain("28 pass");+});+test("a raw-identifier fast path cannot bypass the verified boundary", () => {+ const anchor = "const token = authHeader.split(' ')[1];";+ expect(fixed.split(anchor)).toHaveLength(2);+ const bypass = `\n if (token.startsWith('user_') || /^[0-9a-f-]{36}$/.test(token)) {\n return { isValid: true, tier: 'logged_in', userId: token, deviceId: token };\n }`;+ const result = grade("identity-bypass", fixed.replace(anchor, anchor + bypass));+ behaviorFailure(result); expect(result.stderr).toContain("8 fail");+});+test("blanket anonymous fallback fails legitimate authenticated access", () => {+ const anchor = "const authHeader = request.headers.get('Authorization');";+ expect(fixed.split(anchor)).toHaveLength(2);+ const deny = "\n return { isValid: true, tier: 'anonymous', deviceId: headerDeviceId };";+ behaviorFailure(grade("deny-authenticated", fixed.replace(anchor, anchor + deny)));+});diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.jsonindex e5b4f3355..2b4b3ea81 100644--- a/evals/coding-agent/cases.json+++ b/evals/coding-agent/cases.json@@ -1,7 +1,7 @@{"schema_version": 1,"suite": "screenpipe-app-coding-agent-regressions",- "dataset_version": "2026-09-18.1",+ "dataset_version": "2026-09-18.2","suite_type": "regression","owner": "screenpipe engineering","policy": {@@ -1358,6 +1358,38 @@],"command": "\"${BUN_BIN:-bun}\" test ./evals/coding-agent/graders/app-entitlement-plan-consistency.test.ts"}+ },+ {+ "id": "ai-gateway-verified-identity",+ "name": "Require verified account ownership before authenticated gateway tiers",+ "source": {+ "kind": "git_regression",+ "fix_commit": "252b96c168e79a6e377f307cf1e54423d17e0db8"+ },+ "base_ref": "041f111810b334ac3667006907f1dc5b511273df",+ "oracle_ref": "252b96c168e79a6e377f307cf1e54423d17e0db8",+ "oracle_paths": [+ "packages/ai-gateway/src/utils/auth.ts"+ ],+ "tags": [+ "regression",+ "escaped-failure",+ "security",+ "identity",+ "entitlement"+ ],+ "prompt": "The gateway can treat a public database UUID or Clerk user ID supplied as a Bearer or Token credential as an authenticated account, even when no token verifier established ownership. Repair the authentication boundary so those public identifiers cannot grant logged-in or subscribed tiers or trigger account/subscription lookups. Preserve anonymous access and device identity for absent, invalid, or unproven credentials; preserve verified Clerk and successfully validated legacy website tokens at their proper logged-in/subscribed tier and canonical identity. A verifier result requires a nonempty string subject; a legacy website HTTP 200 alone is not proof of success and requires success=true with a user identity. Missing, rejected and unavailable verification must not manufacture an identity. Use synthetic credentials only. This task concerns validateAuth behavior; no particular helper structure is required.",+ "agent_timeout_seconds": 900,+ "grader": {+ "timeout_seconds": 60,+ "fixtures": [+ {+ "local_path": "graders/gateway-verified-identity.test.js",+ "destination_path": "evals/coding-agent/graders/gateway-verified-identity.test.js"+ }+ ],+ "command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/gateway-verified-identity.test.js"+ }}]}diff --git a/evals/coding-agent/graders/gateway-verified-identity.test.js b/evals/coding-agent/graders/gateway-verified-identity.test.jsnew file mode 100644index 000000000..f0b76da31--- /dev/null+++ b/evals/coding-agent/graders/gateway-verified-identity.test.js@@ -0,0 +1,130 @@+// screenpipe — AI that knows everything you've seen, said, or heard+// https://screenpipe.com++// Execute the gateway auth boundary; Clerk and website/database responses are+// synthetic ports. This does not test JWT cryptography or live billing.+import { beforeEach, expect, mock, test } from "bun:test";++const UUID = "11111111-1111-4111-8111-111111111111";+const SUBJECT = "user_fixtureVerified";+const VALID_TOKEN = "eyJ.fixture.verified";+const LEGACY_TOKEN = "eyJ.fixture.legacy";+let state;+mock.module("@clerk/backend", () => ({ verifyToken: async (token) => {+ state.verifications.push(token);+ if (state.acceptClerk && token === VALID_TOKEN) return { sub: state.subject };+ throw new Error("synthetic verification rejected");+} }));+globalThis.fetch = async (input) => {+ const url = new URL(String(input));+ state.requests.push(url.href);+ if (state.networkFailure) throw new Error("synthetic provider unavailable");+ if (url.origin === "https://db.fixture.invalid") {+ if (url.pathname === "/rest/v1/users") {+ return Response.json([{ id: UUID, clerk_id: SUBJECT }]);+ }+ if (url.pathname === "/rest/v1/cloud_subscriptions") {+ return Response.json(state.paid ? [{ id: "sub_fixture" }] : []);+ }+ }+ if (url.href === "https://screenpipe.com/api/user") {+ return Response.json(state.website, { status: state.websiteStatus });+ }+ state.unexpected.push(url.origin + url.pathname);+ throw new Error("Unexpected network port; live access is forbidden");+};+const auth = await import("../../../packages/ai-gateway/src/utils/auth");+const env = {+ NODE_ENV: "production",+ CLERK_SECRET_KEY: "synthetic-verifier-key",+ SUPABASE_URL: "https://db.fixture.invalid",+ SUPABASE_ANON_KEY: "synthetic-db-key",+ AI_GATEWAY_SERVICE_TOKEN: "synthetic-machine-credential",+};+function request(token, scheme = "Bearer") {+ return new Request("https://gateway.fixture.invalid/v1/usage", { headers: {+ "X-Device-Id": "device_fixture",+ ...(token ? { Authorization: `${scheme} ${token}` } : {}),+ } });+}+function entitledUser(paid) {+ const plan = paid ? "pro" : "standard";+ return { success: true, user: { id: UUID, clerk_id: SUBJECT,+ cloud_subscribed: paid, app_entitled: true, subscription_plan: plan,+ entitlement: { active: true, plan, features: { app: true, cloud: paid } } } };+}+beforeEach(() => {+ state = { acceptClerk: false, subject: SUBJECT, paid: false, websiteStatus: 200,+ website: { success: false }, networkFailure: false, requests: [], verifications: [], unexpected: [] };+ // Optional cache cleanup allows the same boundary assertions to be checked+ // on current source too; historical implementations need no such helper.+ auth.__resetAuthEntitlementCacheForTests?.();+});+async function result(token, scheme) {+ const value = await auth.validateAuth(request(token, scheme), env);+ expect(state.unexpected).toEqual([]);+ return value;+}+async function anonymous(token, scheme) {+ const value = await result(token, scheme);+ expect(value).toMatchObject({ isValid: true, tier: "anonymous", deviceId: "device_fixture" });+ expect(value.userId).toBeUndefined();+ return value;+}++test("absent credentials retain anonymous access without identity lookup", async () => {+ await anonymous(); expect(state.requests).toEqual([]); expect(state.verifications).toEqual([]);+});+for (const scheme of ["Bearer", "Token"]) {+ for (const token of [UUID, "user_fixturePublic"]) {+ for (const paid of [false, true]) {+ test(`${scheme} public ${token === UUID ? "UUID" : "Clerk ID"}, paid=${paid}, cannot authenticate`, async () => {+ state.paid = paid;+ await anonymous(token, scheme);+ expect(state.requests).toEqual([]);+ });+ }+ }+ for (const paid of [false, true]) {+ test(`${scheme} verified caller retains ${paid ? "subscribed" : "logged_in"} access`, async () => {+ state.acceptClerk = true; state.paid = paid; state.website = entitledUser(paid);+ expect(await result(VALID_TOKEN, scheme)).toMatchObject({ isValid: true,+ tier: paid ? "subscribed" : "logged_in", userId: SUBJECT, deviceId: SUBJECT });+ expect(state.verifications).toContain(VALID_TOKEN);+ });+ }+}+for (const [name, subject] of [["missing", undefined], ["null", null], ["empty", ""], ["numeric", 42], ["object", {}]]) {+ test(`verified-provider response with ${name} subject cannot authenticate`, async () => {+ state.acceptClerk = true; state.subject = subject; await anonymous(VALID_TOKEN);+ });+}+for (const [name, website] of [+ ["unsuccessful", { success: false, user: { clerk_id: SUBJECT, cloud_subscribed: true } }],+ ["missing success", { user: { clerk_id: SUBJECT, cloud_subscribed: true } }],+ ["missing user", { success: true }],+ ["missing identity", { success: true, user: { cloud_subscribed: true } }],+]) {+ test(`HTTP 200 website response with ${name} cannot authenticate`, async () => {+ state.website = website; await anonymous(LEGACY_TOKEN);+ });+}+for (const paid of [false, true]) {+ test(`validated legacy token retains ${paid ? "subscribed" : "logged_in"} access`, async () => {+ state.website = entitledUser(paid);+ expect(await result(LEGACY_TOKEN)).toMatchObject({ isValid: true,+ tier: paid ? "subscribed" : "logged_in", userId: SUBJECT, deviceId: SUBJECT });+ });+}+test("unavailable legacy validation does not manufacture an identity", async () => {+ state.networkFailure = true; await anonymous(LEGACY_TOKEN);+});+test("rejected legacy HTTP response cannot grant access", async () => {+ state.website = entitledUser(true); state.websiteStatus = 401; await anonymous(LEGACY_TOKEN);+});+test("arbitrary token remains anonymous", async () => {+ await anonymous("synthetic-invalid-credential"); expect(state.requests).toEqual([]);+});+test("unrecognized long credential cannot authenticate", async () => {+ await anonymous(env.AI_GATEWAY_SERVICE_TOKEN + "-wrong");+});