Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): add WWW-Authenticate header on auth 401 for agent discovery

Max Loffgrene66b9162dc4f
diff --git a/apps/api/src/__tests__/snips/v2/agent-auth-discovery.test.ts b/apps/api/src/__tests__/snips/v2/agent-auth-discovery.test.ts
new file mode 100644
index 000000000..264306d23
--- /dev/null
+++ b/apps/api/src/__tests__/snips/v2/agent-auth-discovery.test.ts
@@ -0,0 +1,67 @@
+import { config } from "../../../config";
+import { describeIf, TEST_API_URL, TEST_PRODUCTION } from "../lib";
+import request from "supertest";
+
+const CANONICAL_PRM_URL =
+ "https://www.firecrawl.dev/.well-known/oauth-protected-resource";
+
+const EXPECTED_WWW_AUTHENTICATE = `Bearer resource_metadata="${CANONICAL_PRM_URL}"`;
+
+describeIf(TEST_PRODUCTION)("Agent auth discovery (WWW-Authenticate)", () => {
+ beforeAll(() => {
+ config.USE_DB_AUTHENTICATION = true;
+ });
+
+ afterAll(() => {
+ delete config.USE_DB_AUTHENTICATION;
+ });
+
+ it("defaults AGENT_AUTH_RESOURCE_METADATA_URL to canonical PRM", () => {
+ expect(config.AGENT_AUTH_RESOURCE_METADATA_URL).toBe(CANONICAL_PRM_URL);
+ });
+
+ it.concurrent(
+ "returns WWW-Authenticate on 401 without Authorization",
+ async () => {
+ const response = await request(TEST_API_URL)
+ .post("/v2/scrape")
+ .set("Content-Type", "application/json")
+ .send({ url: "https://example.com" });
+
+ expect(response.statusCode).toBe(401);
+ expect(response.headers["www-authenticate"]).toBe(
+ EXPECTED_WWW_AUTHENTICATE,
+ );
+ },
+ );
+
+ it.concurrent(
+ "returns WWW-Authenticate on 401 with invalid API key",
+ async () => {
+ const response = await request(TEST_API_URL)
+ .post("/v2/scrape")
+ .set("Authorization", "Bearer invalid-api-key")
+ .set("Content-Type", "application/json")
+ .send({ url: "https://example.com" });
+
+ expect(response.statusCode).toBe(401);
+ expect(response.headers["www-authenticate"]).toBe(
+ EXPECTED_WWW_AUTHENTICATE,
+ );
+ },
+ );
+
+ it.concurrent(
+ "does not return WWW-Authenticate on successful auth",
+ async () => {
+ const response = await request(TEST_API_URL)
+ .post("/v2/scrape")
+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)
+ .set("Content-Type", "application/json")
+ .send({ url: "https://example.com" });
+
+ expect(response.statusCode).toBe(200);
+ expect(response.headers["www-authenticate"]).toBeUndefined();
+ },
+ );
+});
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index a94821ae4..181578950 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -70,6 +70,11 @@ const configSchema = z.object({
OAUTH_INTROSPECT_URL: z.string().optional(),
OAUTH_INTROSPECT_SECRET: z.string().optional(),
+ // Agent auth discovery (RFC 9728 WWW-Authenticate on 401)
+ AGENT_AUTH_RESOURCE_METADATA_URL: z
+ .url()
+ .default("https://www.firecrawl.dev/.well-known/oauth-protected-resource"),
+
// Database & Storage
POSTGRES_HOST: z.string().default("localhost"),
POSTGRES_PORT: z.string().default("5432"),
diff --git a/apps/api/src/controllers/v0/crawl-cancel.ts b/apps/api/src/controllers/v0/crawl-cancel.ts
index 91fc10927..b091009f1 100644
--- a/apps/api/src/controllers/v0/crawl-cancel.ts
+++ b/apps/api/src/controllers/v0/crawl-cancel.ts
@@ -8,12 +8,14 @@ import { configDotenv } from "dotenv";
import { redisEvictConnection } from "../../../src/services/redis";
import { crawlGroup } from "../../services/worker/nuq";
import { getScrapeZDR } from "../../lib/zdr-helpers";
+import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
configDotenv();
export async function crawlCancelController(req: Request, res: Response) {
try {
const auth = await authenticateUser(req, res, RateLimiterMode.CrawlStatus);
if (!auth.success) {
+ if (auth.status === 401) applyAgentAuthDiscoveryHeader(res);
return res.status(auth.status).json({ error: auth.error });
}
diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.ts
index 3c18b3169..0ce22a4af 100644
--- a/apps/api/src/controllers/v0/crawl-status.ts
+++ b/apps/api/src/controllers/v0/crawl-status.ts
@@ -14,6 +14,7 @@ import { getJobFromGCS } from "../../lib/gcs-jobs";
import { scrapeQueue, NuQJob } from "../../services/worker/nuq";
import { includesFormat } from "../../lib/format-utils";
import { getScrapeZDR } from "../../lib/zdr-helpers";
+import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
configDotenv();
async function getJobs(
@@ -94,6 +95,7 @@ export async function crawlStatusController(req: Request, res: Response) {
try {
const auth = await authenticateUser(req, res, RateLimiterMode.CrawlStatus);
if (!auth.success) {
+ if (auth.status === 401) applyAgentAuthDiscoveryHeader(res);
return res.status(auth.status).json({ error: auth.error });
}
diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.ts
index cf026605e..f532c135b 100644
--- a/apps/api/src/controllers/v0/crawl.ts
+++ b/apps/api/src/controllers/v0/crawl.ts
@@ -36,11 +36,13 @@ import { isSelfHosted } from "../../lib/deployment";
import { crawlGroup } from "../../services/worker/nuq";
import { logRequest } from "../../services/logging/log_job";
import { getScrapeZDR } from "../../lib/zdr-helpers";
+import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
export async function crawlController(req: Request, res: Response) {
try {
const auth = await authenticateUser(req, res, RateLimiterMode.Crawl);
if (!auth.success) {
+ if (auth.status === 401) applyAgentAuthDiscoveryHeader(res);
return res.status(auth.status).json({ error: auth.error });
}
diff --git a/apps/api/src/controllers/v0/keyAuth.ts b/apps/api/src/controllers/v0/keyAuth.ts
index fd8e33127..f4ec3df31 100644
--- a/apps/api/src/controllers/v0/keyAuth.ts
+++ b/apps/api/src/controllers/v0/keyAuth.ts
@@ -5,12 +5,14 @@ import { authenticateUser } from "../auth";
import { redisEvictConnection } from "../../../src/services/redis";
import { logger } from "../../lib/logger";
import { getScrapeZDR } from "../../lib/zdr-helpers";
+import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
export const keyAuthController = async (req: Request, res: Response) => {
try {
// make sure to authenticate user first, Bearer <token>
const auth = await authenticateUser(req, res);
if (!auth.success) {
+ if (auth.status === 401) applyAgentAuthDiscoveryHeader(res);
return res.status(auth.status).json({ error: auth.error });
}
diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.ts
index da7ae3acf..e4fd44656 100644
--- a/apps/api/src/controllers/v0/scrape.ts
+++ b/apps/api/src/controllers/v0/scrape.ts
@@ -26,6 +26,7 @@ import { scrapeQueue } from "../../services/worker/nuq";
import { getErrorContactMessage } from "../../lib/deployment";
import { logRequest } from "../../services/logging/log_job";
import { getScrapeZDR } from "../../lib/zdr-helpers";
+import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
async function scrapeHelper(
jobId: string,
@@ -186,6 +187,7 @@ export async function scrapeController(req: Request, res: Response) {
// make sure to authenticate user first, Bearer <token>
const auth = await authenticateUser(req, res, RateLimiterMode.Scrape);
if (!auth.success) {
+ if (auth.status === 401) applyAgentAuthDiscoveryHeader(res);
return res.status(auth.status).json({ error: auth.error });
}
diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.ts
index 79e175133..83b4d5410 100644
--- a/apps/api/src/controllers/v0/search.ts
+++ b/apps/api/src/controllers/v0/search.ts
@@ -25,6 +25,7 @@ import { ScrapeJobTimeoutError } from "../../lib/error";
import { scrapeQueue } from "../../services/worker/nuq";
import { defaultOrigin } from "../../lib/default-values";
import { getSearchZDR } from "../../lib/zdr-helpers";
+import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
async function searchHelper(
jobId: string,
@@ -180,6 +181,7 @@ export async function searchController(req: Request, res: Response) {
// make sure to authenticate user first, Bearer <token>
const auth = await authenticateUser(req, res, RateLimiterMode.Search);
if (!auth.success) {
+ if (auth.status === 401) applyAgentAuthDiscoveryHeader(res);
return res.status(auth.status).json({ error: auth.error });
}
const { team_id, chunk } = auth;
diff --git a/apps/api/src/lib/agent-auth-discovery.test.ts b/apps/api/src/lib/agent-auth-discovery.test.ts
new file mode 100644
index 000000000..bef3b6ec2
--- /dev/null
+++ b/apps/api/src/lib/agent-auth-discovery.test.ts
@@ -0,0 +1,59 @@
+import { Response } from "express";
+import {
+ applyAgentAuthDiscoveryHeader,
+ getAgentAuthResourceMetadataUrl,
+ shouldEmitAgentAuthDiscoveryHeader,
+} from "./agent-auth-discovery";
+import { config } from "../config";
+
+const CANONICAL_PRM_URL =
+ "https://www.firecrawl.dev/.well-known/oauth-protected-resource";
+
+describe("agent-auth-discovery", () => {
+ it("defaults AGENT_AUTH_RESOURCE_METADATA_URL to canonical PRM", () => {
+ expect(config.AGENT_AUTH_RESOURCE_METADATA_URL).toBe(CANONICAL_PRM_URL);
+ expect(getAgentAuthResourceMetadataUrl()).toBe(CANONICAL_PRM_URL);
+ });
+
+ it("sets WWW-Authenticate when cloud auth is enabled", () => {
+ const previous = config.USE_DB_AUTHENTICATION;
+ config.USE_DB_AUTHENTICATION = true;
+
+ const headers: Record<string, string> = {};
+ const res = {
+ setHeader(name: string, value: string) {
+ headers[name] = value;
+ },
+ } as unknown as Response;
+
+ try {
+ applyAgentAuthDiscoveryHeader(res);
+ expect(headers["WWW-Authenticate"]).toBe(
+ `Bearer resource_metadata="${CANONICAL_PRM_URL}"`,
+ );
+ expect(shouldEmitAgentAuthDiscoveryHeader()).toBe(true);
+ } finally {
+ config.USE_DB_AUTHENTICATION = previous;
+ }
+ });
+
+ it("skips WWW-Authenticate on self-hosted", () => {
+ const previous = config.USE_DB_AUTHENTICATION;
+ config.USE_DB_AUTHENTICATION = false;
+
+ const headers: Record<string, string> = {};
+ const res = {
+ setHeader(name: string, value: string) {
+ headers[name] = value;
+ },
+ } as unknown as Response;
+
+ try {
+ applyAgentAuthDiscoveryHeader(res);
+ expect(headers["WWW-Authenticate"]).toBeUndefined();
+ expect(shouldEmitAgentAuthDiscoveryHeader()).toBe(false);
+ } finally {
+ config.USE_DB_AUTHENTICATION = previous;
+ }
+ });
+});
diff --git a/apps/api/src/lib/agent-auth-discovery.ts b/apps/api/src/lib/agent-auth-discovery.ts
new file mode 100644
index 000000000..9a19f3d0a
--- /dev/null
+++ b/apps/api/src/lib/agent-auth-discovery.ts
@@ -0,0 +1,17 @@
+import { Response } from "express";
+import { config } from "../config";
+import { isSelfHosted } from "./deployment";
+
+export function getAgentAuthResourceMetadataUrl(): string {
+ return config.AGENT_AUTH_RESOURCE_METADATA_URL;
+}
+
+export function shouldEmitAgentAuthDiscoveryHeader(): boolean {
+ return !isSelfHosted();
+}
+
+export function applyAgentAuthDiscoveryHeader(res: Response): void {
+ if (!shouldEmitAgentAuthDiscoveryHeader()) return;
+ const url = getAgentAuthResourceMetadataUrl();
+ res.setHeader("WWW-Authenticate", `Bearer resource_metadata="${url}"`);
+}
diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.ts
index 180d25c7d..8c092e427 100644
--- a/apps/api/src/routes/shared.ts
+++ b/apps/api/src/routes/shared.ts
@@ -8,6 +8,7 @@ import {
} from "../controllers/v1/types";
import { RateLimiterMode } from "../types";
import { authenticateUser } from "../controllers/auth";
+import { applyAgentAuthDiscoveryHeader } from "../lib/agent-auth-discovery";
import { createIdempotencyKey } from "../services/idempotency/create";
import { validateIdempotencyKey } from "../services/idempotency/validate";
import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist";
@@ -215,6 +216,7 @@ export function authMiddleware(
if (!auth.success) {
if (!res.headersSent) {
+ if (auth.status === 401) applyAgentAuthDiscoveryHeader(res);
return res
.status(auth.status)
.json({ success: false, error: auth.error });