Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): log every auth denial as one canonical auth/denied line (#5008)
Gergő Móricze3986ebd0ad7
diff --git a/apps/api/src/__tests__/routes/check-credits-middleware.test.ts b/apps/api/src/__tests__/routes/check-credits-middleware.test.tsindex 4e809206f..54f79dbf7 100644--- a/apps/api/src/__tests__/routes/check-credits-middleware.test.ts+++ b/apps/api/src/__tests__/routes/check-credits-middleware.test.ts@@ -255,6 +255,52 @@ describe("checkCreditsMiddleware – Autumn overage handling", () => {});});+describe("checkCreditsMiddleware – agent-key denials", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ });++ function buildSponsoredReq(status: "blocked" | "pending") {+ return buildReq({+ acuc: {+ team_id: "team_test",+ api_key_id: 42,+ _agentSponsor: {+ status,+ verification_deadline: new Date(Date.now() - 1000).toISOString(),+ email: "[email protected]",+ },+ },+ });+ }++ it.each([+ ["blocked", "agent_key_blocked"],+ ["pending", "agent_key_verification_expired"],+ ] as const)(+ "logs a %s agent key as auth/denied before the 403",+ async (status, reason) => {+ const { res } = await runMiddleware(buildSponsoredReq(status));++ expect(res.status).toHaveBeenCalledWith(403);+ expect(logger.warn).toHaveBeenCalledTimes(1);+ expect(logger.warn).toHaveBeenCalledWith("Request denied", {+ canonicalLog: "auth/denied",+ reason,+ status: 403,+ method: undefined,+ route: "/v1/crawl",+ teamId: "team_test",+ apiKeyId: 42,+ });+ expect(JSON.stringify(vi.mocked(logger.warn).mock.calls)).not.toContain(+ "[email protected]",+ );+ expect(checkCreditsMock).not.toHaveBeenCalled();+ },+ );+});+describe("checkCreditsMiddleware – unverified agent-key 50-credit cap", () => {beforeEach(() => {vi.clearAllMocks();diff --git a/apps/api/src/__tests__/snips/v2/auth-denied.test.ts b/apps/api/src/__tests__/snips/v2/auth-denied.test.tsnew file mode 100644index 000000000..bb7b012e3--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/auth-denied.test.ts@@ -0,0 +1,24 @@+import request from "supertest";+import { describeIf, TEST_API_URL, TEST_PRODUCTION } from "../lib";++// Denials write an auth/denied log line; the response must stay the same.+describeIf(TEST_PRODUCTION)("Auth denial responses", () => {+ it.each([+ ["no token after the scheme", "Bearer", "Unauthorized: Token missing"],+ ["a malformed key", "Bearer not-a-real-key", "Unauthorized: Invalid token"],+ [+ "an unknown key",+ `Bearer fc-${crypto.randomUUID().replace(/-/g, "")}`,+ "Unauthorized: Invalid token",+ ],+ ])("returns the same 401 body for %s", async (_, authorization, error) => {+ const response = await request(TEST_API_URL)+ .post("/v2/scrape")+ .set("Authorization", authorization)+ .set("Content-Type", "application/json")+ .send({ url: "https://example.com" });++ expect(response.statusCode).toBe(401);+ expect(response.body).toEqual({ success: false, error });+ });+});diff --git a/apps/api/src/controllers/__tests__/auth.test.ts b/apps/api/src/controllers/__tests__/auth.test.tsindex 569bdc25c..5a6293462 100644--- a/apps/api/src/controllers/__tests__/auth.test.ts+++ b/apps/api/src/controllers/__tests__/auth.test.ts@@ -1535,6 +1535,207 @@ describe("authenticateUser", () => {});});+ describe("auth/denied log", () => {+ const unknownKey = "fc-3c9a0d5e7b1f4a2c8d6e9f0a1b2c3d4e";+ const clientIp = "198.51.100.23";++ const deniedLines = () =>+ (vi.mocked(logger.warn).mock.calls as unknown as [string, any][])+ .filter(([, meta]) => meta?.canonicalLog === "auth/denied")+ .map(([message, meta]) => ({ message, meta }));++ beforeEach(() => {+ config.USE_DB_AUTHENTICATION = true;+ vi.mocked(getValue).mockResolvedValue(null);+ vi.mocked(authCreditUsageChunk).mockResolvedValue([]);+ vi.mocked(redlock.using).mockImplementation(+ async (_keys, _ttl, _options, fn) => fn({ aborted: false } as never),+ );+ vi.spyOn(logger, "warn").mockImplementation(() => logger);+ });++ it.each([+ ["missing_credentials", {}, "Unauthorized"],+ [+ "malformed_authorization",+ { authorization: "Bearer" },+ "Unauthorized: Token missing",+ ],+ [+ "malformed_key",+ { authorization: "Bearer not-a-real-key" },+ "Unauthorized: Invalid token",+ ],+ [+ "unknown_key",+ { authorization: `Bearer ${unknownKey}` },+ "Unauthorized: Invalid token",+ ],+ ] as const)(+ "logs one %s line and returns the same 401",+ async (reason, headers, error) => {+ const auth = await authenticateUser(+ {+ method: "POST",+ baseUrl: "/v2",+ path: "/scrape",+ route: { path: "/scrape" },+ headers,+ socket: { remoteAddress: clientIp },+ },+ {},+ RateLimiterMode.Scrape,+ );++ expect(auth).toEqual({ success: false, error, status: 401 });+ expect(deniedLines()).toEqual([+ {+ message: "Request denied",+ meta: {+ canonicalLog: "auth/denied",+ reason,+ status: 401,+ method: "POST",+ route: "/v2/scrape",+ },+ },+ ]);+ },+ );++ it("redacts the admin secret from an admin route", async () => {+ const originalBullAuthKey = config.BULL_AUTH_KEY;+ config.BULL_AUTH_KEY = "bull-admin-secret";+ try {+ await authenticateUser(+ {+ method: "POST",+ baseUrl: "",+ path: "/admin/bull-admin-secret/crawl-monitor",+ route: { path: "/admin/bull-admin-secret/crawl-monitor" },+ headers: { authorization: `Bearer ${unknownKey}` },+ socket: { remoteAddress: clientIp },+ },+ {},+ RateLimiterMode.Crawl,+ );+ } finally {+ config.BULL_AUTH_KEY = originalBullAuthKey;+ }++ expect(deniedLines()).toEqual([+ expect.objectContaining({+ meta: expect.objectContaining({+ reason: "unknown_key",+ route: "/admin/:bullAuthKey/crawl-monitor",+ }),+ }),+ ]);+ expect(JSON.stringify(deniedLines())).not.toContain("bull-admin-secret");+ });++ it("names the team and key id of a banned team's key", async () => {+ vi.mocked(authCreditUsageChunk).mockResolvedValue([+ {+ api_key: "00000000-0000-4000-8000-000000000000",+ api_key_id: 9,+ team_id: "team-banned",+ org_id: "org-1",+ is_banned: true,+ flags: null,+ },+ ]);++ const auth = await authenticateUser(+ {+ headers: {+ authorization: "Bearer 00000000-0000-4000-8000-000000000000",+ },+ socket: { remoteAddress: clientIp },+ },+ {},+ RateLimiterMode.Scrape,+ );++ expect(auth).toEqual(expect.objectContaining({ status: 403 }));+ expect(deniedLines()).toEqual([+ {+ message: "Request denied",+ meta: {+ canonicalLog: "auth/denied",+ reason: "team_banned",+ status: 403,+ teamId: "team-banned",+ apiKeyId: 9,+ },+ },+ ]);+ });++ it("never logs the credential, the Authorization header, or the client IP", async () => {+ await authenticateUser(+ {+ headers: {+ authorization: `Bearer ${unknownKey}`,+ "x-forwarded-for": clientIp,+ },+ socket: { remoteAddress: clientIp },+ ip: clientIp,+ },+ {},+ RateLimiterMode.Scrape,+ );++ const logged = JSON.stringify(deniedLines());+ expect(deniedLines()).toHaveLength(1);+ expect(logged).not.toContain(unknownKey.slice(3, 11));+ expect(logged).not.toContain(unknownKey.slice(-8));+ expect(logged).not.toContain("Bearer");+ expect(logged).not.toContain(clientIp);+ });++ it("logs a suspicious keyless IP as denied but not a keyless quota 429", async () => {+ vi.mocked(isKeylessConfigured).mockReturnValue(true);+ vi.mocked(isKeylessIpSuspicious).mockResolvedValueOnce(true);+ const keylessRequest = () => ({+ headers: {},+ socket: { remoteAddress: "203.0.113.8" },+ });++ const suspicious = await authenticateUser(+ keylessRequest(),+ {},+ RateLimiterMode.Scrape,+ { allowKeyless: true },+ );+ vi.mocked(consumeKeylessRequest).mockResolvedValue({+ ok: false,+ reason: "requests",+ requestsUsed: 10,+ creditsUsed: 2,+ });+ const limited = await authenticateUser(+ keylessRequest(),+ {},+ RateLimiterMode.Scrape,+ { allowKeyless: true },+ );++ expect(suspicious).toEqual(expect.objectContaining({ status: 403 }));+ expect(limited).toEqual(expect.objectContaining({ status: 429 }));+ expect(deniedLines()).toEqual([+ {+ message: "Request denied",+ meta: {+ canonicalLog: "auth/denied",+ reason: "keyless_ip_suspicious",+ status: 403,+ },+ },+ ]);+ });+ });+it("clears purpose-qualified and legacy ACUC cache entries", async () => {await clearACUC("api-key");diff --git a/apps/api/src/controllers/auth.ts b/apps/api/src/controllers/auth.tsindex 940fe85b0..3c96bb48b 100644--- a/apps/api/src/controllers/auth.ts+++ b/apps/api/src/controllers/auth.ts@@ -28,6 +28,7 @@ import { isKeylessIpSuspicious } from "../lib/spur";import { keylessAuthTotal } from "../lib/keyless-metrics";import { checkIpRestriction } from "../lib/ip-restriction";import { checkKeyEndpointRestriction } from "../lib/key-restriction";+import { type AuthDenialReason, logAuthDenied } from "../lib/auth-denied-log";import { deleteKey, getValue, setValue } from "../services/redis";import { redlock } from "../services/redlock";import { db, dbRr } from "../db/connection";@@ -531,6 +532,16 @@ export async function clearACUCTeam(team_id: string): Promise<void> {await deleteKey(`acuc_team_${team_id}`);}+function denied(+ req,+ reason: AuthDenialReason,+ response: Extract<AuthResponse, { success: false }>,+ key?: AuthCreditUsageChunk,+): AuthResponse {+ logAuthDenied(req, response.status, reason, key);+ return response;+}+// Both prompts end the sentence after the URL with a space, so a copied link// never picks up the period.function keylessEndpointNotAvailableMessage(signupUrl: string): string {@@ -585,7 +596,9 @@ async function handleKeylessAuth(// The keyless tier is off unless BOTH limits are configured (even to 0). When// unconfigured we behave exactly as before — a generic 401 — and don't reveal// that the tier exists.- if (!isKeylessConfigured()) return unauthorized;+ if (!isKeylessConfigured()) {+ return denied(req, "missing_credentials", unauthorized);+ }// Configured, but this endpoint isn't part of the keyless tier: tell the user// they need a key (with the signup nudge) rather than a bare "Unauthorized".@@ -604,12 +617,12 @@ async function handleKeylessAuth(401,signupRef,);- return {+ return denied(req, "missing_credentials", {success: false,error: keylessEndpointNotAvailableMessage(url),status: 401,signupUrl: url,- };+ });}const origin = req.body?.origin;@@ -625,7 +638,9 @@ async function handleKeylessAuth(// Only a valid IPv4 identity gets keyless: IPv6 is too cheap to rotate for a// per-IP cap to mean anything, and malformed/forwarded values must not be// usable as arbitrary limiter buckets. Anything else falls through to 401.- if (!isKeylessIpEligible(ip)) return unauthorized;+ if (!isKeylessIpEligible(ip)) {+ return denied(req, "keyless_ip_ineligible", unauthorized);+ }// Canonicalize `::ffff:`-mapped IPv4 so a client gets one Spur cache entry,// one quota bucket, and one team id regardless of how the socket reported it.@@ -658,14 +673,14 @@ async function handleKeylessAuth(reason: "suspicious",...(signupRef ? { signupRef } : {}),});- return {+ return denied(req, "keyless_ip_suspicious", {success: false,error: keylessSuspiciousIpMessage(url),status: 403,// Tell agents where to find the key/signup flow they now need.agentAuthDiscovery: true,signupUrl: url,- };+ });}const teamId = keylessTeamId(ip);@@ -697,7 +712,7 @@ async function handleKeylessAuth(teamId,error,});- return unauthorized;+ return denied(req, "keyless_limiter_unavailable", unauthorized);}const baseLog = {canonicalLog: "keyless/consume",@@ -824,11 +839,11 @@ async function supaAuthenticateUser(}const token = authHeader.split(" ")[1]; // Extract the token from "Bearer <token>"if (!token) {- return {+ return denied(req, "malformed_authorization", {success: false,error: "Unauthorized: Token missing",status: 401,- };+ });}const incomingIP = (req.headers["x-preview-ip"] ||@@ -871,11 +886,11 @@ async function supaAuthenticateUser(config.MCP_DELEGATED_CREDENTIAL_SECRET,);if (!delegation) {- return {+ return denied(req, "invalid_mcp_credential", {success: false,error: "Unauthorized: Invalid token",status: 401,- };+ });}const resolvedApi = parseApi(delegation.api_key);@@ -887,11 +902,11 @@ async function supaAuthenticateUser("hosted_mcp_oauth",);if (chunk === null) {- return {+ return denied(req, "unknown_key", {success: false,error: "Unauthorized: Invalid token",status: 401,- };+ });}teamId = chunk.team_id;@@ -917,21 +932,21 @@ async function supaAuthenticateUser(throw error;}if (!introspection) {- return {+ return denied(req, "invalid_oauth_token", {success: false,error: "Unauthorized: Invalid or expired OAuth token",status: 401,- };+ });}if (introspection.credential_purpose !== undefined &&introspection.credential_purpose !== "general") {- return {+ return denied(req, "oauth_purpose_mismatch", {success: false,error: "Unauthorized: Invalid token",status: 401,- };+ });}// Use the resolved fc- API key to get the normal ACUC chunk@@ -945,18 +960,23 @@ async function supaAuthenticateUser();if (chunk === null) {- return {+ return denied(req, "unknown_key", {success: false,error: "Unauthorized: Invalid token",status: 401,- };+ });}if (chunk.team_id !== introspection.team_id) {- return {- success: false,- error: "Unauthorized: Invalid token",- status: 401,- };+ return denied(+ req,+ "oauth_team_mismatch",+ {+ success: false,+ error: "Unauthorized: Invalid token",+ status: 401,+ },+ chunk,+ );}teamId = chunk.team_id;@@ -972,11 +992,11 @@ async function supaAuthenticateUser(} else {normalizedApi = parseApi(token);if (!normalizedApiIsUuid(normalizedApi)) {- return {+ return denied(req, "malformed_key", {success: false,error: "Unauthorized: Invalid token",status: 401,- };+ });}chunk = await getACUC(normalizedApi, false, true, RateLimiterMode.Scrape);@@ -999,11 +1019,11 @@ async function supaAuthenticateUser(}if (chunk === null) {- return {+ return denied(req, "unknown_key", {success: false,error: "Unauthorized: Invalid token",status: 401,- };+ });}teamId = chunk.team_id;@@ -1024,12 +1044,17 @@ async function supaAuthenticateUser(// and never reads that field, so bans went unenforced. auth_chunk_2 surfaces// teams.banned as is_banned and we deny it explicitly.if (chunk?.is_banned) {- return {- success: false,- error:- "Unauthorized: This account has been banned. Contact [email protected] if you believe this is a mistake.",- status: 403,- };+ return denied(+ req,+ "team_banned",+ {+ success: false,+ error:+ "Unauthorized: This account has been banned. Contact [email protected] if you believe this is a mistake.",+ status: 403,+ },+ chunk,+ );}if (chunk?.flags?.ipRestriction) {@@ -1039,11 +1064,12 @@ async function supaAuthenticateUser(chunk.flags,);if (!ipCheck.allowed) {- return {- success: false,- error: ipCheck.error,- status: ipCheck.status,- };+ return denied(+ req,+ "ip_restricted",+ { success: false, error: ipCheck.error, status: ipCheck.status },+ chunk,+ );}}@@ -1056,11 +1082,16 @@ async function supaAuthenticateUser(chunk.flags,);if (!endpointCheck.allowed) {- return {- success: false,- error: endpointCheck.error,- status: endpointCheck.status,- };+ return denied(+ req,+ "endpoint_restricted",+ {+ success: false,+ error: endpointCheck.error,+ status: endpointCheck.status,+ },+ chunk,+ );}}diff --git a/apps/api/src/lib/auth-denied-log.ts b/apps/api/src/lib/auth-denied-log.tsnew file mode 100644index 000000000..72a094e47--- /dev/null+++ b/apps/api/src/lib/auth-denied-log.ts@@ -0,0 +1,58 @@+import type { Request } from "express";+import { config } from "../config";+import type { AuthCreditUsageChunk } from "../controllers/v1/types";+import { getRoutePattern } from "./http-metrics";+import { logger } from "./logger";++export type AuthDenialReason =+ | "missing_credentials"+ | "malformed_authorization"+ | "malformed_key"+ | "unknown_key"+ | "invalid_mcp_credential"+ | "invalid_oauth_token"+ | "oauth_purpose_mismatch"+ | "oauth_team_mismatch"+ | "team_banned"+ | "ip_restricted"+ | "endpoint_restricted"+ | "keyless_ip_ineligible"+ | "keyless_ip_suspicious"+ | "keyless_limiter_unavailable"+ | "agent_key_blocked"+ | "agent_key_verification_expired";++// Admin routes carry BULL_AUTH_KEY as a path segment.+function routeOf(req: Request): string | undefined {+ if (typeof req.path !== "string") return undefined;+ return `${req.baseUrl ?? ""}${getRoutePattern(req)}`+ .split("/")+ .map(segment =>+ config.BULL_AUTH_KEY && segment === config.BULL_AUTH_KEY+ ? ":bullAuthKey"+ : segment,+ )+ .join("/");+}++/**+ * Writes the single `auth/denied` line for a request refused access. Pass `key`+ * only when it came from a resolved API key: preview and keyless chunks carry+ * the client IP in their team id. Never log the credential or the client IP.+ */+export function logAuthDenied(+ req: Request,+ status: number,+ reason: AuthDenialReason,+ key?: Pick<AuthCreditUsageChunk, "team_id" | "api_key_id">,+): void {+ logger.warn("Request denied", {+ canonicalLog: "auth/denied",+ reason,+ status,+ method: req.method,+ route: routeOf(req),+ teamId: key?.team_id,+ apiKeyId: key?.api_key_id,+ });+}diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.tsindex ca0b81f99..55c930cb6 100644--- a/apps/api/src/routes/shared.ts+++ b/apps/api/src/routes/shared.ts@@ -13,6 +13,7 @@ import { createIdempotencyKey } from "../services/idempotency/create";import { validateIdempotencyKey } from "../services/idempotency/validate";import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist";import { logger } from "../lib/logger";+import { logAuthDenied } from "../lib/auth-denied-log";import {httpRequestDurationSeconds,getRoutePattern,@@ -65,6 +66,7 @@ export function checkCreditsMiddleware(const sponsor = req.acuc._agentSponsor;if (sponsor.status === "blocked") {+ logAuthDenied(req, 403, "agent_key_blocked", req.acuc);return res.status(403).json({success: false,error: "This API key has been blocked by the account holder.",@@ -74,6 +76,7 @@ export function checkCreditsMiddleware(if (sponsor.status === "pending") {const deadline = new Date(sponsor.verification_deadline);if (deadline < new Date()) {+ logAuthDenied(req, 403, "agent_key_verification_expired", req.acuc);return res.status(403).json({success: false,error: "sponsor_verification_expired",