Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
api: header-authenticated feng-check and crawl-monitor admin routes (#5379)
David Tippett5228ce8ac720
diff --git a/apps/api/src/__tests__/routes/admin-auth.routes.test.ts b/apps/api/src/__tests__/routes/admin-auth.routes.test.tsindex 5c22419dc..53e19e2dd 100644--- a/apps/api/src/__tests__/routes/admin-auth.routes.test.ts+++ b/apps/api/src/__tests__/routes/admin-auth.routes.test.ts@@ -1,7 +1,11 @@import express from "express";import request from "supertest";import { config } from "../../config";-import { adminAuth } from "../../lib/admin-auth";+import {+ ADMIN_KEY_HEADER,+ adminAuth,+ adminKeyHeaderAuth,+} from "../../lib/admin-auth";import { logger } from "../../lib/logger";const CURRENT = "current-admin-key-0001";@@ -22,6 +26,14 @@ function appWithAdminAuth() {return app;}+function appWithAdminKeyHeaderAuth() {+ const app = express();+ app.post("/admin/ops/crawl-monitor", adminKeyHeaderAuth, (req, res) =>+ res.status(200).json({ authorization: req.headers.authorization }),+ );+ return app;+}+describe("adminAuth", () => {const original = {BULL_AUTH_KEY: config.BULL_AUTH_KEY,@@ -99,3 +111,58 @@ describe("adminAuth", () => {expect(response.status).toBe(401);});});++describe("adminKeyHeaderAuth", () => {+ const original = {+ BULL_AUTH_KEY: config.BULL_AUTH_KEY,+ BULL_AUTH_KEY_PREVIOUS: config.BULL_AUTH_KEY_PREVIOUS,+ };++ beforeEach(() => {+ config.BULL_AUTH_KEY = CURRENT;+ config.BULL_AUTH_KEY_PREVIOUS = PREVIOUS;+ });++ afterEach(() => {+ config.BULL_AUTH_KEY = original.BULL_AUTH_KEY;+ config.BULL_AUTH_KEY_PREVIOUS = original.BULL_AUTH_KEY_PREVIOUS;+ vi.restoreAllMocks();+ });++ it("accepts either key and leaves Authorization for the team key", async () => {+ for (const [key, matched] of [+ [CURRENT, "current"],+ [PREVIOUS, "previous"],+ ]) {+ const info = vi.spyOn(logger, "info");+ const response = await request(appWithAdminKeyHeaderAuth())+ .post("/admin/ops/crawl-monitor")+ .set(ADMIN_KEY_HEADER, key)+ .set("Authorization", "Bearer fc-team-key");++ expect(response.status).toBe(200);+ expect(response.body).toEqual({ authorization: "Bearer fc-team-key" });+ expect(matchedKeys(info)).toEqual([matched]);+ expect(JSON.stringify(info.mock.calls)).not.toContain(key);+ info.mockRestore();+ }+ });++ it("rejects a missing or wrong key, including one sent as a bearer", async () => {+ const app = appWithAdminKeyHeaderAuth();+ const cases: Array<Record<string, string>> = [+ {},+ { Authorization: `Bearer ${CURRENT}` },+ { [ADMIN_KEY_HEADER]: `${CURRENT}x` },+ { [ADMIN_KEY_HEADER]: "" },+ ];++ for (const headers of cases) {+ const response = await request(app)+ .post("/admin/ops/crawl-monitor")+ .set(headers);+ expect(response.status).toBe(401);+ expect(response.body).toEqual({ success: false, error: "Unauthorized" });+ }+ });+});diff --git a/apps/api/src/lib/admin-auth.ts b/apps/api/src/lib/admin-auth.tsindex 5e9682aad..45c2f38b5 100644--- a/apps/api/src/lib/admin-auth.ts+++ b/apps/api/src/lib/admin-auth.ts@@ -23,12 +23,15 @@ export function secretsMatch(return left.length === right.length && crypto.timingSafeEqual(left, right);}-/**- * Accepts `Authorization: Bearer` matching BULL_AUTH_KEY, or BULL_AUTH_KEY_PREVIOUS- * while callers move to a new key. Logs which key matched, never its value.- */-export function adminAuth(req: Request, res: Response, next: NextFunction) {- const provided = bearerToken(req.headers.authorization);+export const ADMIN_KEY_HEADER = "x-firecrawl-admin-key";++function authorizeAdminKey(+ provided: string | null,+ missing: boolean,+ req: Request,+ res: Response,+ next: NextFunction,+) {const current = secretsMatch(provided, config.BULL_AUTH_KEY);const previous = secretsMatch(provided, config.BULL_AUTH_KEY_PREVIOUS);@@ -36,7 +39,7 @@ export function adminAuth(req: Request, res: Response, next: NextFunction) {logAuthDenied(req,401,- !req.headers.authorization+ missing? "missing_credentials": provided? "unknown_key"@@ -52,3 +55,31 @@ export function adminAuth(req: Request, res: Response, next: NextFunction) {});next();}++/**+ * Accepts `Authorization: Bearer` matching BULL_AUTH_KEY, or BULL_AUTH_KEY_PREVIOUS+ * while callers move to a new key. Logs which key matched, never its value.+ */+export function adminAuth(req: Request, res: Response, next: NextFunction) {+ authorizeAdminKey(+ bearerToken(req.headers.authorization),+ !req.headers.authorization,+ req,+ res,+ next,+ );+}++/**+ * Same keys as adminAuth, read from ADMIN_KEY_HEADER, for admin routes whose+ * Authorization header carries a team API key.+ */+export function adminKeyHeaderAuth(+ req: Request,+ res: Response,+ next: NextFunction,+) {+ const value = req.headers[ADMIN_KEY_HEADER];+ const provided = (Array.isArray(value) ? value[0] : value) || null;+ authorizeAdminKey(provided, !provided, req, res, next);+}diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.tsindex fc0afc275..5f349ca26 100644--- a/apps/api/src/routes/admin.ts+++ b/apps/api/src/routes/admin.ts@@ -22,7 +22,12 @@ import {handleIntegrationAdminRotateProxy,handleIntegrationAdminValidateProxy,} from "../lib/admin-integration-integrations-proxy";-import { adminAuth, bearerToken, secretsMatch } from "../lib/admin-auth";+import {+ adminAuth,+ adminKeyHeaderAuth,+ bearerToken,+ secretsMatch,+} from "../lib/admin-auth";import { logger } from "../lib/logger";import { RateLimiterMode } from "../types";import { authMiddleware, checkCreditsMiddleware, wrap } from "./shared";@@ -130,6 +135,8 @@ if (config.BULL_AUTH_KEY) {opsRouter.get("/autumn-health", autumnHealthController);+ opsRouter.get("/feng-check", wrap(checkFireEngine));+opsRouter.post("/acuc-cache-clear", wrap(acucCacheClearController));opsRouter.post(@@ -147,6 +154,14 @@ if (config.BULL_AUTH_KEY) {wrap(concurrencyQueueBackfillController),);+ adminRouter.post(+ "/admin/ops/crawl-monitor",+ adminKeyHeaderAuth,+ authMiddleware(RateLimiterMode.Crawl),+ checkCreditsMiddleware(2),+ wrap(crawlMonitorController),+ );+adminRouter.use("/admin/ops", opsRouter);}