Skip to content

YC (S26) | Open Computer History | Continuously record your company computer work, map your workflows, help you find work worth automating, and power your agents' context

eval: keep gateway authentication logs private

Louis Beaumontc0444550ebc1
diff --git a/evals/coding-agent/README.md b/evals/coding-agent/README.md
index 6599f4998..d1950ddf7 100644
--- a/evals/coding-agent/README.md
+++ b/evals/coding-agent/README.md
@@ -13,6 +13,21 @@ The current app corpus contains 80 git-mined regressions. See
[DESIGN.md](./DESIGN.md) for the Anthropic guidance, source contract, and
history-mining workflow. The companion website manifest uses this same harness.
+`ai-gateway-auth-log-privacy` runs the actual authentication entry points with
+synthetic Clerk and account-service replies. Eight checks reject credentials,
+account identifiers, emails and upstream error details in console output;
+four nearby outcomes preserve verified and legacy access and anonymous fallback.
+The broken parent fails eight checks and preserves four. The auth-only historical
+fix and current source pass all twelve. Run
+`bun test evals/coding-agent/calibrate-auth-log-privacy.test.js` for eleven
+controls, including equivalent diagnostics, silence, unused correct source,
+raw-value logging, blanket access/denial and missing-source classification.
+Fixtures are installed only when grading begins. This does not test real JWT
+cryptography, the HTTP router, external log sinks, native model policy, execution
+isolation or model capability. Public-identifier authentication from the historical
+source is superseded and is not required by this case.
+
+
`app-permission-recovery-audio-policy` renders permission recovery, onboarding
and the status banner with synthetic settings and native command ports. Fourteen
outcomes cover audio-off policy, loading, policy changes during delayed completion,
diff --git a/evals/coding-agent/calibrate-auth-log-privacy.test.js b/evals/coding-agent/calibrate-auth-log-privacy.test.js
new file mode 100644
index 000000000..c0ebda469
--- /dev/null
+++ b/evals/coding-agent/calibrate-auth-log-privacy.test.js
@@ -0,0 +1,64 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+import { afterAll, expect, test } from "bun:test";
+import { execFileSync, spawnSync } from "node:child_process";
+import { copyFileSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs";
+import { dirname, join, resolve } from "node:path";
+import { tmpdir } from "node:os";
+const repo = resolve(import.meta.dir, "../..");
+const item = JSON.parse(readFileSync(join(import.meta.dir, "cases.json"), "utf8")).cases.find(x => x.id === "ai-gateway-auth-log-privacy");
+const path = "packages/ai-gateway/src/utils/auth.ts";
+const dependency = "packages/ai-gateway/src/utils/subscription.ts";
+const show = (ref, p) => execFileSync("git", ["show", `${ref}:${p}`], { cwd: repo, encoding: "utf8" });
+const broken = show(item.base_ref, path), fixed = show(item.oracle_ref, path);
+const root = mkdtempSync(join(tmpdir(), "auth-log-calibration-"));
+afterAll(() => rmSync(root, { recursive: true, force: true }));
+function grade(name, source, extra) {
+ const workspace = join(root, name);
+ const write = (p, data) => { mkdirSync(dirname(join(workspace, p)), { recursive: true }); writeFileSync(join(workspace, p), data); };
+ if (source !== null) write(path, source);
+ write(dependency, show(item.base_ref, dependency));
+ if (extra) write("packages/ai-gateway/src/utils/unused-repair.ts", extra);
+ const fixture = item.grader.fixtures[0];
+ mkdirSync(dirname(join(workspace, fixture.destination_path)), { recursive: true });
+ copyFileSync(join(import.meta.dir, fixture.local_path), join(workspace, fixture.destination_path));
+ return spawnSync(process.execPath, ["test", fixture.destination_path], { cwd: workspace, encoding: "utf8", timeout: 15000, env: { PATH: dirname(process.execPath) } });
+}
+function failure(r) {
+ expect(r.error).toBeUndefined(); expect(r.signal).toBeNull(); expect(r.status).toBe(1);
+ expect(r.stderr).toContain("expect(received)"); expect(r.stderr).not.toContain("Cannot find module");
+}
+function patch(anchor, replacement) { expect(fixed.split(anchor)).toHaveLength(2); return fixed.replace(anchor, replacement); }
+test("broken parent fails eight privacy checks and preserves four outcomes", () => {
+ const r = grade("parent", broken); failure(r); expect(r.stderr).toContain("8 fail"); expect(r.stderr).toContain("4 pass");
+});
+test("historical auth-only reference passes all twelve outcomes", () => {
+ const r = grade("reference", fixed); expect(r.status).toBe(0); expect(r.stderr).toContain("12 pass");
+});
+test("equivalent generic diagnostics and renamed private helper are accepted", () => {
+ const r = grade("equivalent", fixed.replaceAll("validateScreenpipeToken", "validateLegacyCredential").replaceAll("clerk verification failed", "verification unavailable"));
+ expect(r.status).toBe(0);
+});
+test("silencing diagnostics preserves authentication", () => {
+ const r = grade("silent", fixed.replace(/^\s*console\.(log|error)\([^\n]*\);$/gm, "")); expect(r.status).toBe(0);
+});
+test("unused correct source cannot hide the active leaking module", () => { failure(grade("unused", broken, fixed)); });
+test("logging an accepted raw token is rejected", () => {
+ failure(grade("raw-token", patch("const payload = await verifyToken(token, {", "console.warn(token);\n const payload = await verifyToken(token, {")));
+});
+test("logging a provider Error object is rejected", () => {
+ failure(grade("raw-error", patch("} catch {\n // Never log", "} catch (error) {\n console.error(error);\n // Never log")));
+});
+test("logging legacy email or canonical account ID is rejected", () => {
+ failure(grade("email", patch("const userData = data.user;", "const userData = data.user; console.info(userData?.email);")));
+ failure(grade("subject", patch("const resolvedUserId = clerkResult.userId || token;", "const resolvedUserId = clerkResult.userId || token; console.debug(resolvedUserId);")));
+});
+test("quiet blanket denial cannot replace authentication", () => {
+ failure(grade("deny", patch("const authHeader = request.headers.get('Authorization');", "const authHeader = request.headers.get('Authorization'); return { isValid: true, tier: 'anonymous', deviceId: headerDeviceId };")));
+});
+test("quiet blanket paid access cannot replace rejection", () => {
+ failure(grade("grant", patch("const authHeader = request.headers.get('Authorization');", "const authHeader = request.headers.get('Authorization'); return { isValid: true, tier: 'subscribed', userId: 'user_SyntheticSubjectCanary', deviceId: 'user_SyntheticSubjectCanary' };")));
+});
+test("missing auth source is a setup error", () => {
+ const r = grade("missing", null); expect(r.status).not.toBe(0); expect(r.stderr).toContain("Cannot find module"); expect(r.stderr).not.toContain("expect(received)");
+});
diff --git a/evals/coding-agent/cases.json b/evals/coding-agent/cases.json
index b3a06ccff..17ae8d4c4 100644
--- a/evals/coding-agent/cases.json
+++ b/evals/coding-agent/cases.json
@@ -4683,6 +4683,37 @@
],
"command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/openai-tool-choice.test.js"
}
+ },
+ {
+ "id": "ai-gateway-auth-log-privacy",
+ "name": "Keep gateway authentication credentials and identities out of logs",
+ "source": {
+ "kind": "git_regression",
+ "fix_commit": "33d5637dda01c704e5436f0be477effbd18c1e4b"
+ },
+ "base_ref": "45105b5ee837501d0c10020a016b16ca2c4c4083",
+ "oracle_ref": "33d5637dda01c704e5436f0be477effbd18c1e4b",
+ "oracle_paths": [
+ "packages/ai-gateway/src/utils/auth.ts"
+ ],
+ "tags": [
+ "regression",
+ "escaped-failure",
+ "privacy",
+ "authentication"
+ ],
+ "prompt": "Gateway authentication writes credentials and account details to operational logs. Repair verifyClerkToken and validateAuth so successful and rejected verification, verified-account lookup, legacy website-token validation and provider exceptions do not log raw credentials, account IDs, email addresses, secret keys or upstream error details. Generic diagnostics or silence are both acceptable. Preserve actual token verification with the supplied verifier key, successful verified and legacy paid/free account tiers and canonical identity, Bearer and Token support, and anonymous access with the supplied device identity when credentials are missing, invalid, rejected or unavailable. Missing credentials must not trigger verification or account lookup. Use synthetic credentials only. This task does not require preserving public-identifier authentication or any particular log message or implementation structure.",
+ "agent_timeout_seconds": 900,
+ "grader": {
+ "timeout_seconds": 60,
+ "fixtures": [
+ {
+ "local_path": "graders/gateway-auth-log-privacy.test.js",
+ "destination_path": "evals/coding-agent/graders/gateway-auth-log-privacy.test.js"
+ }
+ ],
+ "command": "\"${BUN_BIN:-bun}\" test evals/coding-agent/graders/gateway-auth-log-privacy.test.js"
+ }
}
]
}
diff --git a/evals/coding-agent/graders/gateway-auth-log-privacy.test.js b/evals/coding-agent/graders/gateway-auth-log-privacy.test.js
new file mode 100644
index 000000000..6ced2731b
--- /dev/null
+++ b/evals/coding-agent/graders/gateway-auth-log-privacy.test.js
@@ -0,0 +1,97 @@
+// screenpipe — AI that knows everything you've seen, said, or heard
+// https://screenpipe.com
+// Actual auth entry points with synthetic verifier and network ports.
+import { afterEach, beforeEach, expect, mock, test } from "bun:test";
+import { inspect } from "node:util";
+const JWT = "eyJ.syntheticCredentialCanary.signature";
+const SUBJECT = "user_SyntheticSubjectCanary";
+const EMAIL = "[email protected]";
+const UUID = "11111111-1111-4111-8111-111111111111";
+const ERROR = "synthetic-upstream-detail-canary";
+const env = { NODE_ENV: "production", CLERK_SECRET_KEY: "synthetic-secret-key",
+ SUPABASE_URL: "https://db.fixture.invalid", SUPABASE_ANON_KEY: "synthetic-db-key" };
+let state;
+mock.module("@clerk/backend", () => ({ verifyToken: async (token, options) => {
+ state.verifications.push({ token, options });
+ if (state.accept) return { sub: SUBJECT };
+ throw new Error(ERROR);
+} }));
+globalThis.fetch = async (input) => {
+ const url = new URL(String(input)); state.requests.push(url.href);
+ if (state.networkFailure) throw new Error(ERROR);
+ if (url.origin === env.SUPABASE_URL) {
+ if (url.pathname === "/rest/v1/users") return Response.json([{ id: UUID, clerk_id: SUBJECT }]);
+ if (url.pathname === "/rest/v1/cloud_subscriptions") return Response.json(state.paid ? [{ id: "synthetic-subscription" }] : []);
+ }
+ if (url.href === "https://screenpipe.com/api/user") {
+ const plan = state.paid ? "pro" : "standard";
+ return Response.json({ success: true, user: { id: UUID, clerk_id: SUBJECT, email: EMAIL,
+ cloud_subscribed: state.paid, app_entitled: true, subscription_plan: plan,
+ entitlement: { active: true, plan, features: { app: true, cloud: state.paid } } } },
+ { status: state.websiteStatus });
+ }
+ state.unexpected.push(url.origin + url.pathname);
+ throw new Error("Unexpected synthetic port");
+};
+const auth = await import("../../../packages/ai-gateway/src/utils/auth");
+const originals = Object.fromEntries(["log", "error", "warn", "info", "debug"].map(k => [k, console[k]]));
+beforeEach(() => {
+ state = { accept: false, paid: false, websiteStatus: 200, networkFailure: false,
+ requests: [], verifications: [], unexpected: [], logs: [] };
+ auth.__resetAuthEntitlementCacheForTests?.();
+ for (const k of Object.keys(originals)) console[k] = (...args) => state.logs.push(args.map(v => inspect(v, { depth: 8 })).join(" "));
+});
+afterEach(() => { for (const [k, value] of Object.entries(originals)) console[k] = value; });
+function request(token = JWT, scheme = "Bearer") {
+ return new Request("https://gateway.fixture.invalid/v1/usage", { headers: {
+ "X-Device-Id": "synthetic-device", ...(token ? { Authorization: `${scheme} ${token}` } : {}) } });
+}
+async function result(token = JWT, scheme) {
+ const value = await auth.validateAuth(request(token, scheme), env);
+ expect(state.unexpected).toEqual([]); return value;
+}
+function privateLogs() {
+ const output = state.logs.join("\n");
+ for (const secret of [JWT, SUBJECT, EMAIL, UUID, ERROR, env.CLERK_SECRET_KEY, env.SUPABASE_ANON_KEY]) expect(output).not.toContain(secret);
+}
+function identity(value, paid) {
+ expect(value).toMatchObject({ isValid: true, userId: SUBJECT, deviceId: SUBJECT,
+ tier: paid ? "subscribed" : "logged_in" });
+}
+function anonymous(value) {
+ expect(value).toMatchObject({ isValid: true, tier: "anonymous", deviceId: "synthetic-device" });
+ expect(value.userId).toBeUndefined();
+}
+for (const accept of [true, false]) test(`Clerk verification preserves result and private logs, accepted=${accept}`, async () => {
+ state.accept = accept;
+ expect(await auth.verifyClerkToken(env, JWT)).toEqual(accept ? { valid: true, userId: SUBJECT } : { valid: false });
+ expect(state.verifications).toEqual([{ token: JWT, options: { secretKey: env.CLERK_SECRET_KEY } }]);
+ privateLogs();
+});
+for (const accept of [true, false]) for (const paid of [true, false]) test(`${accept ? "verified" : "legacy"} identity keeps tier and private logs, paid=${paid}`, async () => {
+ state.accept = accept; state.paid = paid; identity(await result(), paid);
+ expect(state.verifications.length).toBeGreaterThan(0);
+ expect(state.requests.length).toBeGreaterThan(0); privateLogs();
+});
+test("provider exception keeps anonymous fallback and private logs", async () => {
+ state.networkFailure = true; anonymous(await result());
+ expect(state.requests).toContain("https://screenpipe.com/api/user"); privateLogs();
+});
+test("rejected website response keeps anonymous fallback and private logs", async () => {
+ state.websiteStatus = 401; anonymous(await result()); privateLogs();
+});
+test("missing credentials preserve device identity without verifier or network", async () => {
+ anonymous(await result(null)); expect(state.verifications).toEqual([]); expect(state.requests).toEqual([]); privateLogs();
+});
+test("Bearer and Token retain verified paid and free identity", async () => {
+ state.accept = true;
+ for (const scheme of ["Bearer", "Token"]) for (const paid of [true, false]) {
+ auth.__resetAuthEntitlementCacheForTests?.(); state.paid = paid; identity(await result(JWT, scheme), paid);
+ }
+});
+test("legacy validation retains authenticated paid and free identity", async () => {
+ for (const paid of [true, false]) { state.paid = paid; identity(await result(), paid); }
+});
+test("invalid credential preserves anonymous fallback", async () => {
+ anonymous(await result("synthetic-invalid-credential")); expect(state.requests).toEqual([]);
+});