Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): add WWW-Authenticate header on auth 401 for agent discovery
Max Loffgrene66b9162dc4f
diff --git a/apps/api/src/__tests__/snips/v2/agent-auth-discovery.test.ts b/apps/api/src/__tests__/snips/v2/agent-auth-discovery.test.tsnew file mode 100644index 000000000..264306d23--- /dev/null+++ b/apps/api/src/__tests__/snips/v2/agent-auth-discovery.test.ts@@ -0,0 +1,67 @@+import { config } from "../../../config";+import { describeIf, TEST_API_URL, TEST_PRODUCTION } from "../lib";+import request from "supertest";++const CANONICAL_PRM_URL =+ "https://www.firecrawl.dev/.well-known/oauth-protected-resource";++const EXPECTED_WWW_AUTHENTICATE = `Bearer resource_metadata="${CANONICAL_PRM_URL}"`;++describeIf(TEST_PRODUCTION)("Agent auth discovery (WWW-Authenticate)", () => {+ beforeAll(() => {+ config.USE_DB_AUTHENTICATION = true;+ });++ afterAll(() => {+ delete config.USE_DB_AUTHENTICATION;+ });++ it("defaults AGENT_AUTH_RESOURCE_METADATA_URL to canonical PRM", () => {+ expect(config.AGENT_AUTH_RESOURCE_METADATA_URL).toBe(CANONICAL_PRM_URL);+ });++ it.concurrent(+ "returns WWW-Authenticate on 401 without Authorization",+ async () => {+ const response = await request(TEST_API_URL)+ .post("/v2/scrape")+ .set("Content-Type", "application/json")+ .send({ url: "https://example.com" });++ expect(response.statusCode).toBe(401);+ expect(response.headers["www-authenticate"]).toBe(+ EXPECTED_WWW_AUTHENTICATE,+ );+ },+ );++ it.concurrent(+ "returns WWW-Authenticate on 401 with invalid API key",+ async () => {+ const response = await request(TEST_API_URL)+ .post("/v2/scrape")+ .set("Authorization", "Bearer invalid-api-key")+ .set("Content-Type", "application/json")+ .send({ url: "https://example.com" });++ expect(response.statusCode).toBe(401);+ expect(response.headers["www-authenticate"]).toBe(+ EXPECTED_WWW_AUTHENTICATE,+ );+ },+ );++ it.concurrent(+ "does not return WWW-Authenticate on successful auth",+ async () => {+ const response = await request(TEST_API_URL)+ .post("/v2/scrape")+ .set("Authorization", `Bearer ${config.TEST_API_KEY}`)+ .set("Content-Type", "application/json")+ .send({ url: "https://example.com" });++ expect(response.statusCode).toBe(200);+ expect(response.headers["www-authenticate"]).toBeUndefined();+ },+ );+});diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex a94821ae4..181578950 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -70,6 +70,11 @@ const configSchema = z.object({OAUTH_INTROSPECT_URL: z.string().optional(),OAUTH_INTROSPECT_SECRET: z.string().optional(),+ // Agent auth discovery (RFC 9728 WWW-Authenticate on 401)+ AGENT_AUTH_RESOURCE_METADATA_URL: z+ .url()+ .default("https://www.firecrawl.dev/.well-known/oauth-protected-resource"),+// Database & StoragePOSTGRES_HOST: z.string().default("localhost"),POSTGRES_PORT: z.string().default("5432"),diff --git a/apps/api/src/controllers/v0/crawl-cancel.ts b/apps/api/src/controllers/v0/crawl-cancel.tsindex 91fc10927..b091009f1 100644--- a/apps/api/src/controllers/v0/crawl-cancel.ts+++ b/apps/api/src/controllers/v0/crawl-cancel.ts@@ -8,12 +8,14 @@ import { configDotenv } from "dotenv";import { redisEvictConnection } from "../../../src/services/redis";import { crawlGroup } from "../../services/worker/nuq";import { getScrapeZDR } from "../../lib/zdr-helpers";+import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";configDotenv();export async function crawlCancelController(req: Request, res: Response) {try {const auth = await authenticateUser(req, res, RateLimiterMode.CrawlStatus);if (!auth.success) {+ if (auth.status === 401) applyAgentAuthDiscoveryHeader(res);return res.status(auth.status).json({ error: auth.error });}diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.tsindex 3c18b3169..0ce22a4af 100644--- a/apps/api/src/controllers/v0/crawl-status.ts+++ b/apps/api/src/controllers/v0/crawl-status.ts@@ -14,6 +14,7 @@ import { getJobFromGCS } from "../../lib/gcs-jobs";import { scrapeQueue, NuQJob } from "../../services/worker/nuq";import { includesFormat } from "../../lib/format-utils";import { getScrapeZDR } from "../../lib/zdr-helpers";+import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";configDotenv();async function getJobs(@@ -94,6 +95,7 @@ export async function crawlStatusController(req: Request, res: Response) {try {const auth = await authenticateUser(req, res, RateLimiterMode.CrawlStatus);if (!auth.success) {+ if (auth.status === 401) applyAgentAuthDiscoveryHeader(res);return res.status(auth.status).json({ error: auth.error });}diff --git a/apps/api/src/controllers/v0/crawl.ts b/apps/api/src/controllers/v0/crawl.tsindex cf026605e..f532c135b 100644--- a/apps/api/src/controllers/v0/crawl.ts+++ b/apps/api/src/controllers/v0/crawl.ts@@ -36,11 +36,13 @@ import { isSelfHosted } from "../../lib/deployment";import { crawlGroup } from "../../services/worker/nuq";import { logRequest } from "../../services/logging/log_job";import { getScrapeZDR } from "../../lib/zdr-helpers";+import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";export async function crawlController(req: Request, res: Response) {try {const auth = await authenticateUser(req, res, RateLimiterMode.Crawl);if (!auth.success) {+ if (auth.status === 401) applyAgentAuthDiscoveryHeader(res);return res.status(auth.status).json({ error: auth.error });}diff --git a/apps/api/src/controllers/v0/keyAuth.ts b/apps/api/src/controllers/v0/keyAuth.tsindex fd8e33127..f4ec3df31 100644--- a/apps/api/src/controllers/v0/keyAuth.ts+++ b/apps/api/src/controllers/v0/keyAuth.ts@@ -5,12 +5,14 @@ import { authenticateUser } from "../auth";import { redisEvictConnection } from "../../../src/services/redis";import { logger } from "../../lib/logger";import { getScrapeZDR } from "../../lib/zdr-helpers";+import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";export const keyAuthController = async (req: Request, res: Response) => {try {// make sure to authenticate user first, Bearer <token>const auth = await authenticateUser(req, res);if (!auth.success) {+ if (auth.status === 401) applyAgentAuthDiscoveryHeader(res);return res.status(auth.status).json({ error: auth.error });}diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex da7ae3acf..e4fd44656 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -26,6 +26,7 @@ import { scrapeQueue } from "../../services/worker/nuq";import { getErrorContactMessage } from "../../lib/deployment";import { logRequest } from "../../services/logging/log_job";import { getScrapeZDR } from "../../lib/zdr-helpers";+import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";async function scrapeHelper(jobId: string,@@ -186,6 +187,7 @@ export async function scrapeController(req: Request, res: Response) {// make sure to authenticate user first, Bearer <token>const auth = await authenticateUser(req, res, RateLimiterMode.Scrape);if (!auth.success) {+ if (auth.status === 401) applyAgentAuthDiscoveryHeader(res);return res.status(auth.status).json({ error: auth.error });}diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.tsindex 79e175133..83b4d5410 100644--- a/apps/api/src/controllers/v0/search.ts+++ b/apps/api/src/controllers/v0/search.ts@@ -25,6 +25,7 @@ import { ScrapeJobTimeoutError } from "../../lib/error";import { scrapeQueue } from "../../services/worker/nuq";import { defaultOrigin } from "../../lib/default-values";import { getSearchZDR } from "../../lib/zdr-helpers";+import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";async function searchHelper(jobId: string,@@ -180,6 +181,7 @@ export async function searchController(req: Request, res: Response) {// make sure to authenticate user first, Bearer <token>const auth = await authenticateUser(req, res, RateLimiterMode.Search);if (!auth.success) {+ if (auth.status === 401) applyAgentAuthDiscoveryHeader(res);return res.status(auth.status).json({ error: auth.error });}const { team_id, chunk } = auth;diff --git a/apps/api/src/lib/agent-auth-discovery.test.ts b/apps/api/src/lib/agent-auth-discovery.test.tsnew file mode 100644index 000000000..bef3b6ec2--- /dev/null+++ b/apps/api/src/lib/agent-auth-discovery.test.ts@@ -0,0 +1,59 @@+import { Response } from "express";+import {+ applyAgentAuthDiscoveryHeader,+ getAgentAuthResourceMetadataUrl,+ shouldEmitAgentAuthDiscoveryHeader,+} from "./agent-auth-discovery";+import { config } from "../config";++const CANONICAL_PRM_URL =+ "https://www.firecrawl.dev/.well-known/oauth-protected-resource";++describe("agent-auth-discovery", () => {+ it("defaults AGENT_AUTH_RESOURCE_METADATA_URL to canonical PRM", () => {+ expect(config.AGENT_AUTH_RESOURCE_METADATA_URL).toBe(CANONICAL_PRM_URL);+ expect(getAgentAuthResourceMetadataUrl()).toBe(CANONICAL_PRM_URL);+ });++ it("sets WWW-Authenticate when cloud auth is enabled", () => {+ const previous = config.USE_DB_AUTHENTICATION;+ config.USE_DB_AUTHENTICATION = true;++ const headers: Record<string, string> = {};+ const res = {+ setHeader(name: string, value: string) {+ headers[name] = value;+ },+ } as unknown as Response;++ try {+ applyAgentAuthDiscoveryHeader(res);+ expect(headers["WWW-Authenticate"]).toBe(+ `Bearer resource_metadata="${CANONICAL_PRM_URL}"`,+ );+ expect(shouldEmitAgentAuthDiscoveryHeader()).toBe(true);+ } finally {+ config.USE_DB_AUTHENTICATION = previous;+ }+ });++ it("skips WWW-Authenticate on self-hosted", () => {+ const previous = config.USE_DB_AUTHENTICATION;+ config.USE_DB_AUTHENTICATION = false;++ const headers: Record<string, string> = {};+ const res = {+ setHeader(name: string, value: string) {+ headers[name] = value;+ },+ } as unknown as Response;++ try {+ applyAgentAuthDiscoveryHeader(res);+ expect(headers["WWW-Authenticate"]).toBeUndefined();+ expect(shouldEmitAgentAuthDiscoveryHeader()).toBe(false);+ } finally {+ config.USE_DB_AUTHENTICATION = previous;+ }+ });+});diff --git a/apps/api/src/lib/agent-auth-discovery.ts b/apps/api/src/lib/agent-auth-discovery.tsnew file mode 100644index 000000000..9a19f3d0a--- /dev/null+++ b/apps/api/src/lib/agent-auth-discovery.ts@@ -0,0 +1,17 @@+import { Response } from "express";+import { config } from "../config";+import { isSelfHosted } from "./deployment";++export function getAgentAuthResourceMetadataUrl(): string {+ return config.AGENT_AUTH_RESOURCE_METADATA_URL;+}++export function shouldEmitAgentAuthDiscoveryHeader(): boolean {+ return !isSelfHosted();+}++export function applyAgentAuthDiscoveryHeader(res: Response): void {+ if (!shouldEmitAgentAuthDiscoveryHeader()) return;+ const url = getAgentAuthResourceMetadataUrl();+ res.setHeader("WWW-Authenticate", `Bearer resource_metadata="${url}"`);+}diff --git a/apps/api/src/routes/shared.ts b/apps/api/src/routes/shared.tsindex 180d25c7d..8c092e427 100644--- a/apps/api/src/routes/shared.ts+++ b/apps/api/src/routes/shared.ts@@ -8,6 +8,7 @@ import {} from "../controllers/v1/types";import { RateLimiterMode } from "../types";import { authenticateUser } from "../controllers/auth";+import { applyAgentAuthDiscoveryHeader } from "../lib/agent-auth-discovery";import { createIdempotencyKey } from "../services/idempotency/create";import { validateIdempotencyKey } from "../services/idempotency/validate";import { isUrlBlocked } from "../scraper/WebScraper/utils/blocklist";@@ -215,6 +216,7 @@ export function authMiddleware(if (!auth.success) {if (!res.headersSent) {+ if (auth.status === 401) applyAgentAuthDiscoveryHeader(res);return res.status(auth.status).json({ success: false, error: auth.error });