Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): log every auth denial as one canonical auth/denied line (#5008)

Gergő Móricze3986ebd0ad7
diff --git a/apps/api/src/__tests__/routes/check-credits-middleware.test.ts b/apps/api/src/__tests__/routes/check-credits-middleware.test.ts
index 4e809206f..54f79dbf7 100644
--- a/apps/api/src/__tests__/routes/check-credits-middleware.test.ts
+++ b/apps/api/src/__tests__/routes/check-credits-middleware.test.ts
@@ -255,6 +255,52 @@ describe("checkCreditsMiddleware – Autumn overage handling", () => {
});
});
+describe("checkCreditsMiddleware – agent-key denials", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ });
+
+ function buildSponsoredReq(status: "blocked" | "pending") {
+ return buildReq({
+ acuc: {
+ team_id: "team_test",
+ api_key_id: 42,
+ _agentSponsor: {
+ status,
+ verification_deadline: new Date(Date.now() - 1000).toISOString(),
+ email: "[email protected]",
+ },
+ },
+ });
+ }
+
+ it.each([
+ ["blocked", "agent_key_blocked"],
+ ["pending", "agent_key_verification_expired"],
+ ] as const)(
+ "logs a %s agent key as auth/denied before the 403",
+ async (status, reason) => {
+ const { res } = await runMiddleware(buildSponsoredReq(status));
+
+ expect(res.status).toHaveBeenCalledWith(403);
+ expect(logger.warn).toHaveBeenCalledTimes(1);
+ expect(logger.warn).toHaveBeenCalledWith("Request denied", {
+ canonicalLog: "auth/denied",
+ reason,
+ status: 403,
+ method: undefined,
+ route: "/v1/crawl",
+ teamId: "team_test",
+ apiKeyId: 42,
+ });
+ expect(JSON.stringify(vi.mocked(logger.warn).mock.calls)).not.toContain(
+ );
+ expect(checkCreditsMock).not.toHaveBeenCalled();
+ },
+ );
+});
+
describe("checkCreditsMiddleware – unverified agent-key 50-credit cap", () => {
beforeEach(() => {
vi.clearAllMocks();
diff --git a/apps/api/src/__tests__/snips/v2/auth-denied.test.ts b/apps/api/src/__tests__/snips/v2/auth-denied.test.ts
new file mode 100644
index 000000000..bb7b012e3
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/auth-denied.test.ts
@@ -0,0 +1,24 @@
+import request from "supertest";
+import { describeIf, TEST_API_URL, TEST_PRODUCTION } from "../lib";
+
+// Denials write an auth/denied log line; the response must stay the same.
+describeIf(TEST_PRODUCTION)("Auth denial responses", () => {
+ it.each([
+ ["no token after the scheme", "Bearer", "Unauthorized: Token missing"],
+ ["a malformed key", "Bearer not-a-real-key", "Unauthorized: Invalid token"],
+ [
+ "an unknown key",
+ `Bearer fc-${crypto.randomUUID().replace(/-/g, "")}`,
+ "Unauthorized: Invalid token",
+ ],
+ ])("returns the same 401 body for %s", async (_, authorization, error) => {
+ const response = await request(TEST_API_URL)
+ .post("/v2/scrape")
+ .set("Authorization", authorization)
+ .set("Content-Type", "application/json")
+ .send({ url: "https://example.com" });
+
+ expect(response.statusCode).toBe(401);
+ expect(response.body).toEqual({ success: false, error });
+ });
+});
diff --git a/apps/api/src/controllers/__tests__/auth.test.ts b/apps/api/src/controllers/__tests__/auth.test.ts
index 569bdc25c..5a6293462 100644
--- a/apps/api/src/controllers/__tests__/auth.test.ts
+++ b/apps/api/src/controllers/__tests__/auth.test.ts
@@ -1535,6 +1535,207 @@ describe("authenticateUser", () => {
});
});
+ describe("auth/denied log", () => {
+ const unknownKey = "fc-3c9a0d5e7b1f4a2c8d6e9f0a1b2c3d4e";
+ const clientIp = "198.51.100.23";
+
+ const deniedLines = () =>
+ (vi.mocked(logger.warn).mock.calls as unknown as [string, any][])
+ .filter(([, meta]) => meta?.canonicalLog === "auth/denied")
+ .map(([message, meta]) => ({ message, meta }));
+
+ beforeEach(() => {
+ config.USE_DB_AUTHENTICATION = true;
+ vi.mocked(getValue).mockResolvedValue(null);
+ vi.mocked(authCreditUsageChunk).mockResolvedValue([]);
+ vi.mocked(redlock.using).mockImplementation(
+ async (_keys, _ttl, _options, fn) => fn({ aborted: false } as never),
+ );
+ vi.spyOn(logger, "warn").mockImplementation(() => logger);
+ });
+
+ it.each([
+ ["missing_credentials", {}, "Unauthorized"],
+ [
+ "malformed_authorization",
+ { authorization: "Bearer" },
+ "Unauthorized: Token missing",
+ ],
+ [
+ "malformed_key",
+ { authorization: "Bearer not-a-real-key" },
+ "Unauthorized: Invalid token",
+ ],
+ [
+ "unknown_key",
+ { authorization: `Bearer ${unknownKey}` },
+ "Unauthorized: Invalid token",
+ ],
+ ] as const)(
+ "logs one %s line and returns the same 401",
+ async (reason, headers, error) => {
+ const auth = await authenticateUser(
+ {
+ method: "POST",
+ baseUrl: "/v2",
+ path: "/scrape",
+ route: { path: "/scrape" },
+ headers,
+ socket: { remoteAddress: clientIp },
+ },
+ {},
+ RateLimiterMode.Scrape,
+ );
+
+ expect(auth).toEqual({ success: false, error, status: 401 });
+ expect(deniedLines()).toEqual([
+ {
+ message: "Request denied",
+ meta: {
+ canonicalLog: "auth/denied",
+ reason,
+ status: 401,
+ method: "POST",
+ route: "/v2/scrape",
+ },
+ },
+ ]);
+ },
+ );
+
+ it("redacts the admin secret from an admin route", async () => {
+ const originalBullAuthKey = config.BULL_AUTH_KEY;
+ config.BULL_AUTH_KEY = "bull-admin-secret";
+ try {
+ await authenticateUser(
+ {
+ method: "POST",
+ baseUrl: "",
+ path: "/admin/bull-admin-secret/crawl-monitor",
+ route: { path: "/admin/bull-admin-secret/crawl-monitor" },
+ headers: { authorization: `Bearer ${unknownKey}` },
+ socket: { remoteAddress: clientIp },
+ },
+ {},
+ RateLimiterMode.Crawl,
+ );
+ } finally {
+ config.BULL_AUTH_KEY = originalBullAuthKey;
+ }
+
+ expect(deniedLines()).toEqual([
+ expect.objectContaining({
+ meta: expect.objectContaining({
+ reason: "unknown_key",
+ route: "/admin/:bullAuthKey/crawl-monitor",
+ }),
+ }),
+ ]);
+ expect(JSON.stringify(deniedLines())).not.toContain("bull-admin-secret");
+ });
+
+ it("names the team and key id of a banned team's key", async () => {
+ vi.mocked(authCreditUsageChunk).mockResolvedValue([
+ {
+ api_key: "00000000-0000-4000-8000-000000000000",
+ api_key_id: 9,
+ team_id: "team-banned",
+ org_id: "org-1",
+ is_banned: true,
+ flags: null,
+ },
+ ]);
+
+ const auth = await authenticateUser(
+ {
+ headers: {
+ authorization: "Bearer 00000000-0000-4000-8000-000000000000",
+ },
+ socket: { remoteAddress: clientIp },
+ },
+ {},
+ RateLimiterMode.Scrape,
+ );
+
+ expect(auth).toEqual(expect.objectContaining({ status: 403 }));
+ expect(deniedLines()).toEqual([
+ {
+ message: "Request denied",
+ meta: {
+ canonicalLog: "auth/denied",
+ reason: "team_banned",
+ status: 403,
+ teamId: "team-banned",
+ apiKeyId: 9,
+ },
+ },
+ ]);
+ });
+
+ it("never logs the credential, the Authorization header, or the client IP", async () => {
+ await authenticateUser(
+ {
+ headers: {
+ authorization: `Bearer ${unknownKey}`,
+ "x-forwarded-for": clientIp,
+ },
+ socket: { remoteAddress: clientIp },
+ ip: clientIp,
+ },
+ {},
+ RateLimiterMode.Scrape,
+ );
+
+ const logged = JSON.stringify(deniedLines());
+ expect(deniedLines()).toHaveLength(1);
+ expect(logged).not.toContain(unknownKey.slice(3, 11));
+ expect(logged).not.toContain(unknownKey.slice(-8));
+ expect(logged).not.toContain("Bearer");
+ expect(logged).not.toContain(clientIp);
+ });
+
+ it("logs a suspicious keyless IP as denied but not a keyless quota 429", async () => {
+ vi.mocked(isKeylessConfigured).mockReturnValue(true);
+ vi.mocked(isKeylessIpSuspicious).mockResolvedValueOnce(true);
+ const keylessRequest = () => ({
+ headers: {},
+ socket: { remoteAddress: "203.0.113.8" },
+ });
+
+ const suspicious = await authenticateUser(
+ keylessRequest(),
+ {},
+ RateLimiterMode.Scrape,
+ { allowKeyless: true },
+ );
+ vi.mocked(consumeKeylessRequest).mockResolvedValue({
+ ok: false,
+ reason: "requests",
+ requestsUsed: 10,
+ creditsUsed: 2,
+ });
+ const limited = await authenticateUser(
+ keylessRequest(),
+ {},
+ RateLimiterMode.Scrape,
+ { allowKeyless: true },
+ );
+
+ expect(suspicious).toEqual(expect.objectContaining({ status: 403 }));
+ expect(limited).toEqual(expect.objectContaining({ status: 429 }));
+ expect(deniedLines()).toEqual([
+ {
+ message: "Request denied",
+ meta: {
+ canonicalLog: "auth/denied",
+ reason: "keyless_ip_suspicious",
+ status: 403,
+ },
+ },
+ ]);
+ });
+ });
+
it("clears purpose-qualified and legacy ACUC cache entries", async () => {
await clearACUC("api-key");
diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.ts
index 940fe85b0..3c96bb48b 100644
--- a/apps/api/src/controllers/auth.ts
+++ b/apps/api/src/controllers/auth.ts
@@ -28,6 +28,7 @@ import { isKeylessIpSuspicious } from "../lib/spur";
import { keylessAuthTotal } from "../lib/keyless-metrics";
import { checkIpRestriction } from "../lib/ip-restriction";
import { checkKeyEndpointRestriction } from "../lib/key-restriction";
+import { type AuthDenialReason, logAuthDenied } from "../lib/auth-denied-log";
import { deleteKey, getValue, setValue } from "../services/redis";
import { redlock } from "../services/redlock";
import { db, dbRr } from "../db/connection";
@@ -531,6 +532,16 @@ export async function clearACUCTeam(team_id: string): Promise<void> {
await deleteKey(`acuc_team_${team_id}`);
}
+function denied(
+ req,
+ reason: AuthDenialReason,
+ response: Extract<AuthResponse, { success: false }>,
+ key?: AuthCreditUsageChunk,
+): AuthResponse {
+ logAuthDenied(req, response.status, reason, key);
+ return response;
+}
+
// Both prompts end the sentence after the URL with a space, so a copied link
// never picks up the period.
function keylessEndpointNotAvailableMessage(signupUrl: string): string {
@@ -585,7 +596,9 @@ async function handleKeylessAuth(
// The keyless tier is off unless BOTH limits are configured (even to 0). When
// unconfigured we behave exactly as before — a generic 401 — and don't reveal
// that the tier exists.
- if (!isKeylessConfigured()) return unauthorized;
+ if (!isKeylessConfigured()) {
+ return denied(req, "missing_credentials", unauthorized);
+ }
// Configured, but this endpoint isn't part of the keyless tier: tell the user
// they need a key (with the signup nudge) rather than a bare "Unauthorized".
@@ -604,12 +617,12 @@ async function handleKeylessAuth(
401,
signupRef,
);
- return {
+ return denied(req, "missing_credentials", {
success: false,
error: keylessEndpointNotAvailableMessage(url),
status: 401,
signupUrl: url,
- };
+ });
}
const origin = req.body?.origin;
@@ -625,7 +638,9 @@ async function handleKeylessAuth(
// Only a valid IPv4 identity gets keyless: IPv6 is too cheap to rotate for a
// per-IP cap to mean anything, and malformed/forwarded values must not be
// usable as arbitrary limiter buckets. Anything else falls through to 401.
- if (!isKeylessIpEligible(ip)) return unauthorized;
+ if (!isKeylessIpEligible(ip)) {
+ return denied(req, "keyless_ip_ineligible", unauthorized);
+ }
// Canonicalize `::ffff:`-mapped IPv4 so a client gets one Spur cache entry,
// one quota bucket, and one team id regardless of how the socket reported it.
@@ -658,14 +673,14 @@ async function handleKeylessAuth(
reason: "suspicious",
...(signupRef ? { signupRef } : {}),
});
- return {
+ return denied(req, "keyless_ip_suspicious", {
success: false,
error: keylessSuspiciousIpMessage(url),
status: 403,
// Tell agents where to find the key/signup flow they now need.
agentAuthDiscovery: true,
signupUrl: url,
- };
+ });
}
const teamId = keylessTeamId(ip);
@@ -697,7 +712,7 @@ async function handleKeylessAuth(
teamId,
error,
});
- return unauthorized;
+ return denied(req, "keyless_limiter_unavailable", unauthorized);
}
const baseLog = {
canonicalLog: "keyless/consume",
@@ -824,11 +839,11 @@ async function supaAuthenticateUser(
}
const token = authHeader.split(" ")[1]; // Extract the token from "Bearer <token>"
if (!token) {
- return {
+ return denied(req, "malformed_authorization", {
success: false,
error: "Unauthorized: Token missing",
status: 401,
- };
+ });
}
const incomingIP = (req.headers["x-preview-ip"] ||
@@ -871,11 +886,11 @@ async function supaAuthenticateUser(
config.MCP_DELEGATED_CREDENTIAL_SECRET,
);
if (!delegation) {
- return {
+ return denied(req, "invalid_mcp_credential", {
success: false,
error: "Unauthorized: Invalid token",
status: 401,
- };
+ });
}
const resolvedApi = parseApi(delegation.api_key);
@@ -887,11 +902,11 @@ async function supaAuthenticateUser(
"hosted_mcp_oauth",
);
if (chunk === null) {
- return {
+ return denied(req, "unknown_key", {
success: false,
error: "Unauthorized: Invalid token",
status: 401,
- };
+ });
}
teamId = chunk.team_id;
@@ -917,21 +932,21 @@ async function supaAuthenticateUser(
throw error;
}
if (!introspection) {
- return {
+ return denied(req, "invalid_oauth_token", {
success: false,
error: "Unauthorized: Invalid or expired OAuth token",
status: 401,
- };
+ });
}
if (
introspection.credential_purpose !== undefined &&
introspection.credential_purpose !== "general"
) {
- return {
+ return denied(req, "oauth_purpose_mismatch", {
success: false,
error: "Unauthorized: Invalid token",
status: 401,
- };
+ });
}
// Use the resolved fc- API key to get the normal ACUC chunk
@@ -945,18 +960,23 @@ async function supaAuthenticateUser(
);
if (chunk === null) {
- return {
+ return denied(req, "unknown_key", {
success: false,
error: "Unauthorized: Invalid token",
status: 401,
- };
+ });
}
if (chunk.team_id !== introspection.team_id) {
- return {
- success: false,
- error: "Unauthorized: Invalid token",
- status: 401,
- };
+ return denied(
+ req,
+ "oauth_team_mismatch",
+ {
+ success: false,
+ error: "Unauthorized: Invalid token",
+ status: 401,
+ },
+ chunk,
+ );
}
teamId = chunk.team_id;
@@ -972,11 +992,11 @@ async function supaAuthenticateUser(
} else {
normalizedApi = parseApi(token);
if (!normalizedApiIsUuid(normalizedApi)) {
- return {
+ return denied(req, "malformed_key", {
success: false,
error: "Unauthorized: Invalid token",
status: 401,
- };
+ });
}
chunk = await getACUC(normalizedApi, false, true, RateLimiterMode.Scrape);
@@ -999,11 +1019,11 @@ async function supaAuthenticateUser(
}
if (chunk === null) {
- return {
+ return denied(req, "unknown_key", {
success: false,
error: "Unauthorized: Invalid token",
status: 401,
- };
+ });
}
teamId = chunk.team_id;
@@ -1024,12 +1044,17 @@ async function supaAuthenticateUser(
// and never reads that field, so bans went unenforced. auth_chunk_2 surfaces
// teams.banned as is_banned and we deny it explicitly.
if (chunk?.is_banned) {
- return {
- success: false,
- error:
- "Unauthorized: This account has been banned. Contact [email protected] if you believe this is a mistake.",
- status: 403,
- };
+ return denied(
+ req,
+ "team_banned",
+ {
+ success: false,
+ error:
+ "Unauthorized: This account has been banned. Contact [email protected] if you believe this is a mistake.",
+ status: 403,
+ },
+ chunk,
+ );
}
if (chunk?.flags?.ipRestriction) {
@@ -1039,11 +1064,12 @@ async function supaAuthenticateUser(
chunk.flags,
);
if (!ipCheck.allowed) {
- return {
- success: false,
- error: ipCheck.error,
- status: ipCheck.status,
- };
+ return denied(
+ req,
+ "ip_restricted",
+ { success: false, error: ipCheck.error, status: ipCheck.status },
+ chunk,
+ );
}
}
@@ -1056,11 +1082,16 @@ async function supaAuthenticateUser(
chunk.flags,
);
if (!endpointCheck.allowed) {
- return {
- success: false,
- error: endpointCheck.error,
- status: endpointCheck.status,
- };
+ return denied(
+ req,
+ "endpoint_restricted",
+ {
+ success: false,
+ error: endpointCheck.error,
+ status: endpointCheck.status,
+ },
+ chunk,
+ );
}
}
diff --git a/apps/api/src/lib/auth-denied-log.ts b/apps/api/src/lib/auth-denied-log.ts
new file mode 100644
index 000000000..72a094e47
--- /dev/null
+++ b/apps/api/src/lib/auth-denied-log.ts
@@ -0,0 +1,58 @@
+import type { Request } from "express";
+import { config } from "../config";
+import type { AuthCreditUsageChunk } from "../controllers/v1/types";
+import { getRoutePattern } from "./http-metrics";
+import { logger } from "./logger";
+
+export type AuthDenialReason =
+ | "missing_credentials"
+ | "malformed_authorization"
+ | "malformed_key"
+ | "unknown_key"
+ | "invalid_mcp_credential"
+ | "invalid_oauth_token"
+ | "oauth_purpose_mismatch"
+ | "oauth_team_mismatch"
+ | "team_banned"
+ | "ip_restricted"
+ | "endpoint_restricted"
+ | "keyless_ip_ineligible"
+ | "keyless_ip_suspicious"
+ | "keyless_limiter_unavailable"
+ | "agent_key_blocked"
+ | "agent_key_verification_expired";
+
+// Admin routes carry BULL_AUTH_KEY as a path segment.
+function routeOf(req: Request): string | undefined {
+ if (typeof req.path !== "string") return undefined;
+ return `${req.baseUrl ?? ""}${getRoutePattern(req)}`
+ .split("/")
+ .map(segment =>
+ config.BULL_AUTH_KEY && segment === config.BULL_AUTH_KEY
+ ? ":bullAuthKey"
+ : segment,
+ )
+ .join("/");
+}
+
+/**
+ * Writes the single `auth/denied` line for a request refused access. Pass `key`
+ * only when it came from a resolved API key: preview and keyless chunks carry
+ * the client IP in their team id. Never log the credential or the client IP.
+ */
+export function logAuthDenied(
+ req: Request,
+ status: number,
+ reason: AuthDenialReason,
+ key?: Pick<AuthCreditUsageChunk, "team_id" | "api_key_id">,
+): void {
+ logger.warn("Request denied", {
+ canonicalLog: "auth/denied",
+ reason,
+ status,
+ method: req.method,
+ route: routeOf(req),
+ teamId: key?.team_id,
+ apiKeyId: key?.api_key_id,
+ });
+}
diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.ts
index ca0b81f99..55c930cb6 100644
--- a/apps/api/src/routes/shared.ts
+++ b/apps/api/src/routes/shared.ts
@@ -13,6 +13,7 @@ import { createIdempotencyKey } from "../services/idempotency/create";
import { validateIdempotencyKey } from "../services/idempotency/validate";
import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist";
import { logger } from "../lib/logger";
+import { logAuthDenied } from "../lib/auth-denied-log";
import {
httpRequestDurationSeconds,
getRoutePattern,
@@ -65,6 +66,7 @@ export function checkCreditsMiddleware(
const sponsor = req.acuc._agentSponsor;
if (sponsor.status === "blocked") {
+ logAuthDenied(req, 403, "agent_key_blocked", req.acuc);
return res.status(403).json({
success: false,
error: "This API key has been blocked by the account holder.",
@@ -74,6 +76,7 @@ export function checkCreditsMiddleware(
if (sponsor.status === "pending") {
const deadline = new Date(sponsor.verification_deadline);
if (deadline < new Date()) {
+ logAuthDenied(req, 403, "agent_key_verification_expired", req.acuc);
return res.status(403).json({
success: false,
error: "sponsor_verification_expired",