Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
api: header-authenticated admin routes and an optional internal admin listener (#5319)
David Tippett970e0975783f
diff --git a/SELF_HOST.md b/SELF_HOST.mdindex d37ca4108..9fbec7812 100644--- a/SELF_HOST.md+++ b/SELF_HOST.md@@ -70,6 +70,40 @@ security, availability, capacity, upgrades, data retention, and compliance.Treat the Kubernetes and Helm examples as versioned starting points, not asevidence that these production decisions have been made for you.+## Admin routes++The key-protected routes on the main API port, `/admin/ops/...` and+`/admin/<key>/...`, exist only when `BULL_AUTH_KEY` is set. The+internal listener is separate: it starts whenever `ADMIN_INTERNAL_PORT` is set+and needs no key. The root Compose file passes `BULL_AUTH_KEY` alone; set the+other variables in the API container's environment.++| Variable | Default | Effect |+| --- | --- | --- |+| `BULL_AUTH_KEY` | unset | Turns on the key-protected admin routes. Routes under `/admin/ops/` accept `Authorization: Bearer <BULL_AUTH_KEY>`. |+| `BULL_AUTH_KEY_PREVIOUS` | unset | A second accepted bearer value, so callers can move to a new key without downtime. Clear it once every caller sends the new key. |+| `BULL_AUTH_KEY_PATH_ROUTES` | `true` | Keeps the routes that take the key as a path segment, `/admin/<key>/...`, including the queue UI at `/admin/<key>/queues`. Set `false` once callers use `/admin/ops/` or the internal listener. |+| `ADMIN_INTERNAL_PORT` | unset | Starts a second listener on this port that serves the routes below with no key. Never publish this port. |+| `ADMIN_INTERNAL_HOST` | `localhost` | Address the internal listener binds to. Bind a wider address only behind a network policy that limits who can reach the port. |++Header-authenticated routes: `GET /admin/ops/redis-health`,+`GET /admin/ops/autumn-health`, `POST /admin/ops/acuc-cache-clear`,+`POST /admin/ops/ip-restriction-cache-clear`,+`POST /admin/ops/key-restriction-cache-clear` and+`POST /admin/ops/concurrency-queue-backfill`.++Internal listener routes, none of which need a key: the queue UI at+`/admin/queues`; metrics at `GET /admin/metrics`, `GET /admin/nuq-metrics`,+`GET /admin/nuq-fdb-metrics` and `GET /admin/index-queue-prometheus`; and+`GET /admin/redis-health`, `GET /admin/autumn-health`, `GET /admin/feng-check`,+`GET /admin/precrawl` (queues a precrawl run), `POST /admin/fsearch` (runs a+search) and `POST /admin/crawl-monitor` (still requires an API key and credits).+Anyone who can reach the port can use them.++The `firecrawl_admin_path_route_requests_total` metric counts requests to the+`/admin/<key>/...` routes, so you can confirm nothing still uses them before+turning them off.+Stuck? Open a[self-host issue template](https://github.com/firecrawl/firecrawl/issues/new?template=self_host_issue.md)or join the [Firecrawl Discord community](https://discord.gg/firecrawl).diff --git a/apps/api/.env.example b/apps/api/.env.exampleindex 345b57ade..42e93f74c 100644--- a/apps/api/.env.example+++ b/apps/api/.env.example@@ -90,6 +90,14 @@ EXTRACTION_PROVIDER_TEAM_IDS=# EXTRACTION_PROVIDER_INPUT_COST_PER_MTOK=0.07# EXTRACTION_PROVIDER_OUTPUT_COST_PER_MTOK=0.2BULL_AUTH_KEY=@+# optional second bearer value accepted on /admin/ops/* while callers move to a new BULL_AUTH_KEY+# BULL_AUTH_KEY_PREVIOUS=+# set to false to turn off the /admin/<BULL_AUTH_KEY>/... routes once callers use /admin/ops/*+# BULL_AUTH_KEY_PATH_ROUTES=true+# optional port for an internal listener serving the queue UI, metrics and diagnostic routes with no key; keep it private+# ADMIN_INTERNAL_PORT=+# address the internal listener binds to; widen it only behind a network policy+# ADMIN_INTERNAL_HOST=localhost# set if you have a llamaparse key you'd like to use to parse pdfsLLAMAPARSE_API_KEY=# set if you'd like to send slack server health status messagesdiff --git a/apps/api/src/__tests__/routes/admin-auth.routes.test.ts b/apps/api/src/__tests__/routes/admin-auth.routes.test.tsnew file mode 100644index 000000000..5c22419dc--- /dev/null+++ b/apps/api/src/__tests__/routes/admin-auth.routes.test.ts@@ -0,0 +1,101 @@+import express from "express";+import request from "supertest";+import { config } from "../../config";+import { adminAuth } from "../../lib/admin-auth";+import { logger } from "../../lib/logger";++const CURRENT = "current-admin-key-0001";+const PREVIOUS = "previous-admin-key-0002";++function matchedKeys(spy: { mock: { calls: unknown[][] } }) {+ return spy.mock.calls+ .map(call => (call[1] as { matchedKey?: string } | undefined)?.matchedKey)+ .filter(Boolean);+}++function appWithAdminAuth() {+ const app = express();+ const ops = express.Router();+ ops.use(adminAuth);+ ops.get("/redis-health", (_req, res) => res.status(200).json({ ok: true }));+ app.use("/admin/ops", ops);+ return app;+}++describe("adminAuth", () => {+ const original = {+ BULL_AUTH_KEY: config.BULL_AUTH_KEY,+ BULL_AUTH_KEY_PREVIOUS: config.BULL_AUTH_KEY_PREVIOUS,+ };++ beforeEach(() => {+ config.BULL_AUTH_KEY = CURRENT;+ config.BULL_AUTH_KEY_PREVIOUS = undefined;+ });++ afterEach(() => {+ config.BULL_AUTH_KEY = original.BULL_AUTH_KEY;+ config.BULL_AUTH_KEY_PREVIOUS = original.BULL_AUTH_KEY_PREVIOUS;+ vi.restoreAllMocks();+ });++ it("accepts the current key and logs which key matched", async () => {+ const info = vi.spyOn(logger, "info");++ const response = await request(appWithAdminAuth())+ .get("/admin/ops/redis-health")+ .set("Authorization", `Bearer ${CURRENT}`);++ expect(response.status).toBe(200);+ expect(matchedKeys(info)).toEqual(["current"]);+ expect(JSON.stringify(info.mock.calls)).not.toContain(CURRENT);+ });++ it("accepts the previous key when it is set", async () => {+ config.BULL_AUTH_KEY_PREVIOUS = PREVIOUS;+ const info = vi.spyOn(logger, "info");++ const response = await request(appWithAdminAuth())+ .get("/admin/ops/redis-health")+ .set("Authorization", `Bearer ${PREVIOUS}`);++ expect(response.status).toBe(200);+ expect(matchedKeys(info)).toEqual(["previous"]);+ expect(JSON.stringify(info.mock.calls)).not.toContain(PREVIOUS);+ });++ it("rejects the previous key once it is cleared", async () => {+ const response = await request(appWithAdminAuth())+ .get("/admin/ops/redis-health")+ .set("Authorization", `Bearer ${PREVIOUS}`);++ expect(response.status).toBe(401);+ });++ it("rejects a missing, malformed, wrong or wrong-length key", async () => {+ const app = appWithAdminAuth();+ const headers = [+ undefined,+ CURRENT,+ `Basic ${CURRENT}`,+ "Bearer ",+ `Bearer ${CURRENT.slice(0, -1)}x`,+ `Bearer ${CURRENT}extra`,+ ];++ for (const header of headers) {+ const req = request(app).get("/admin/ops/redis-health");+ const response = await (header ? req.set("Authorization", header) : req);+ expect(response.status).toBe(401);+ expect(response.body).toEqual({ success: false, error: "Unauthorized" });+ }+ });++ it("does not expose the key in the URL path", async () => {+ const response = await request(appWithAdminAuth()).get(+ `/admin/ops/${CURRENT}/redis-health`,+ );++ expect(response.status).toBe(401);+ });+});diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex bbefcb89e..cf4751b0d 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -231,6 +231,12 @@ const configSchema = z.object({// API Keys & AuthenticationBULL_AUTH_KEY: z.string().optional(),+ BULL_AUTH_KEY_PREVIOUS: emptyStringAsUndefined(z.string()),+ BULL_AUTH_KEY_PATH_ROUTES: emptyStringAsDefault(z.stringbool().default(true)),+ ADMIN_INTERNAL_PORT: emptyStringAsUndefined(+ z.coerce.number().int().positive().max(65535),+ ),+ ADMIN_INTERNAL_HOST: emptyStringAsDefault(z.string().default("localhost")),S2S_FIRECRAWL_INTEGRATIONS_TO_FIRECRAWL_API_KEY: emptyStringAsUndefined(z.string().trim().min(1),),diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex 3d5a5e20a..735aedcb6 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -14,7 +14,11 @@ import { v0Router } from "./routes/v0";import { startNodeRuntimeMetrics } from "./lib/node-runtime-metrics";import os from "os";import { logger } from "./lib/logger";-import { adminRouter } from "./routes/admin";+import {+ adminInternalRouter,+ adminRouter,+ countAdminPathRoute,+} from "./routes/admin";import http from "node:http";import https from "node:https";import { v1Router } from "./routes/v1";@@ -98,23 +102,29 @@ if (config.EXPRESS_TRUST_PROXY) {const serverAdapter = new ExpressAdapter();+const bullBoardQueues = [+ new BullMQAdapter(getGenerateLlmsTxtQueue()),+ new BullMQAdapter(getDeepResearchQueue()),+ new BullMQAdapter(getBillingQueue()),+ new BullMQAdapter(getPrecrawlQueue()),+ // SIEM audit delivery runs on RabbitMQ; inspect it in the broker's+ // management UI, not here.+];+const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({- queues: [- new BullMQAdapter(getGenerateLlmsTxtQueue()),- new BullMQAdapter(getDeepResearchQueue()),- new BullMQAdapter(getBillingQueue()),- new BullMQAdapter(getPrecrawlQueue()),- // SIEM audit delivery runs on RabbitMQ; inspect it in the broker's- // management UI, not here.- ],+ queues: bullBoardQueues,serverAdapter: serverAdapter,});-if (config.BULL_AUTH_KEY) {- serverAdapter.setBasePath(`/admin/${config.BULL_AUTH_KEY}/queues`);- app.use(`/admin/${config.BULL_AUTH_KEY}/queues`, serverAdapter.getRouter());-} else {+if (!config.BULL_AUTH_KEY) {logger.warn("BULL_AUTH_KEY is not set; admin routes are disabled.");+} else if (config.BULL_AUTH_KEY_PATH_ROUTES) {+ serverAdapter.setBasePath(`/admin/${config.BULL_AUTH_KEY}/queues`);+ app.use(+ `/admin/${config.BULL_AUTH_KEY}/queues`,+ countAdminPathRoute("queues"),+ serverAdapter.getRouter(),+ );}app.get("/", (_, res) => {@@ -139,6 +149,51 @@ app.use(adminRouter);const DEFAULT_PORT = config.PORT;const HOST = config.HOST;+// The internal listener has no key check: keep its port off public ingress.+function startAdminInternalServer(port: number) {+ const adminApp = express();+ adminApp.disable("x-powered-by");+ adminApp.use(bodyParser.json({ limit: "10mb" }));++ const internalBullBoard = new ExpressAdapter();+ internalBullBoard.setBasePath("/admin/queues");+ createBullBoard({+ queues: bullBoardQueues,+ serverAdapter: internalBullBoard,+ });+ adminApp.use("/admin/queues", internalBullBoard.getRouter());++ adminApp.use(adminInternalRouter);+ adminApp.use(notFoundHandler);+ adminApp.use(+ (err: unknown, req: Request, res: Response, _next: NextFunction) => {+ logger.error("Internal admin request failed", {+ error: err,+ path: req.path,+ });+ res.status(500).json({ success: false, error: "Internal server error" });+ },+ );++ const host = config.ADMIN_INTERNAL_HOST;+ return adminApp.listen(port, host, (error?: Error) => {+ if (error) {+ logger.error("Failed to start internal admin server", {+ error,+ port,+ host,+ });+ throw error;+ }++ logger.info("Internal admin server listening", {+ pid: process.pid,+ port,+ host,+ });+ });+}+async function startServer(port = DEFAULT_PORT) {startNodeRuntimeMetrics();try {@@ -161,6 +216,11 @@ async function startServer(port = DEFAULT_PORT) {logger.info(`Worker ${process.pid} listening on port ${port}`);});+ const adminInternalServer =+ config.ADMIN_INTERNAL_PORT !== undefined+ ? startAdminInternalServer(config.ADMIN_INTERNAL_PORT)+ : undefined;+const exitHandler = async () => {logger.info("SIGTERM signal received: closing HTTP server");if (config.IS_KUBERNETES) {@@ -168,7 +228,13 @@ async function startServer(port = DEFAULT_PORT) {logger.info("Waiting 60s for GCE load balancer drain timeout");await new Promise(resolve => setTimeout(resolve, 60000));}- server.close(() => {+ const adminInternalClosed = new Promise<void>(resolve =>+ adminInternalServer+ ? adminInternalServer.close(() => resolve())+ : resolve(),+ );+ server.close(async () => {+ await adminInternalClosed;logger.info("Server closed.");nuqShutdown().finally(() => {shutdownWebhookQueue().finally(() => {diff --git a/apps/api/src/lib/admin-auth.ts b/apps/api/src/lib/admin-auth.tsnew file mode 100644index 000000000..5e9682aad--- /dev/null+++ b/apps/api/src/lib/admin-auth.ts@@ -0,0 +1,54 @@+import crypto from "node:crypto";+import type { NextFunction, Request, Response } from "express";+import { config } from "../config";+import { logAuthDenied } from "./auth-denied-log";+import { logger } from "./logger";++export function bearerToken(+ value: string | string[] | undefined,+): string | null {+ const header = Array.isArray(value) ? value[0] : value;+ return header?.slice(0, 7).toLowerCase() === "bearer "+ ? header.slice(7)+ : null;+}++export function secretsMatch(+ provided: string | null,+ expected?: string,+): boolean {+ if (!provided || !expected?.trim()) return false;+ const left = Buffer.from(provided);+ const right = Buffer.from(expected);+ return left.length === right.length && crypto.timingSafeEqual(left, right);+}++/**+ * Accepts `Authorization: Bearer` matching BULL_AUTH_KEY, or BULL_AUTH_KEY_PREVIOUS+ * while callers move to a new key. Logs which key matched, never its value.+ */+export function adminAuth(req: Request, res: Response, next: NextFunction) {+ const provided = bearerToken(req.headers.authorization);+ const current = secretsMatch(provided, config.BULL_AUTH_KEY);+ const previous = secretsMatch(provided, config.BULL_AUTH_KEY_PREVIOUS);++ if (!current && !previous) {+ logAuthDenied(+ req,+ 401,+ !req.headers.authorization+ ? "missing_credentials"+ : provided+ ? "unknown_key"+ : "malformed_authorization",+ );+ return res.status(401).json({ success: false, error: "Unauthorized" });+ }++ logger.info("Admin request authenticated", {+ method: req.method,+ path: `${req.baseUrl}${req.path}`,+ matchedKey: current ? "current" : "previous",+ });+ next();+}diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.tsindex f4386561d..fc0afc275 100644--- a/apps/api/src/routes/admin.ts+++ b/apps/api/src/routes/admin.ts@@ -1,5 +1,5 @@import express, { NextFunction, Request, Response } from "express";-import crypto from "node:crypto";+import { Counter } from "prom-client";import { config } from "../config";import { acucCacheClearController } from "../controllers/v0/admin/acuc-cache-clear";import { autumnHealthController } from "../controllers/v0/admin/autumn-health";@@ -22,16 +22,42 @@ import {handleIntegrationAdminRotateProxy,handleIntegrationAdminValidateProxy,} from "../lib/admin-integration-integrations-proxy";+import { adminAuth, bearerToken, secretsMatch } from "../lib/admin-auth";import { logger } from "../lib/logger";import { RateLimiterMode } from "../types";import { authMiddleware, checkCreditsMiddleware, wrap } from "./shared";export const adminRouter = express.Router();-if (config.BULL_AUTH_KEY) {- adminRouter.get(`/admin/${config.BULL_AUTH_KEY}/redis-health`, redisHealthController);+// Fixed labels only: never the request path, which carries the key.+const adminPathRouteRequests = new Counter({+ name: "firecrawl_admin_path_route_requests_total",+ help: "Requests to admin routes that take the key as a path segment",+ labelNames: ["surface"],+});- adminRouter.get(`/admin/${config.BULL_AUTH_KEY}/autumn-health`, autumnHealthController);+export function countAdminPathRoute(surface: "queues" | "routes") {+ return (_req: Request, _res: Response, next: NextFunction) => {+ adminPathRouteRequests.inc({ surface });+ next();+ };+}++if (config.BULL_AUTH_KEY && config.BULL_AUTH_KEY_PATH_ROUTES) {+ adminRouter.use(+ `/admin/${config.BULL_AUTH_KEY}`,+ countAdminPathRoute("routes"),+ );++ adminRouter.get(+ `/admin/${config.BULL_AUTH_KEY}/redis-health`,+ redisHealthController,+ );++ adminRouter.get(+ `/admin/${config.BULL_AUTH_KEY}/autumn-health`,+ autumnHealthController,+ );adminRouter.post(`/admin/${config.BULL_AUTH_KEY}/acuc-cache-clear`,@@ -48,22 +74,40 @@ if (config.BULL_AUTH_KEY) {wrap(keyRestrictionCacheClearController),);- adminRouter.get(`/admin/${config.BULL_AUTH_KEY}/feng-check`, wrap(checkFireEngine));+ adminRouter.get(+ `/admin/${config.BULL_AUTH_KEY}/feng-check`,+ wrap(checkFireEngine),+ );adminRouter.get(`/admin/${config.BULL_AUTH_KEY}/index-queue-prometheus`,wrap(indexQueuePrometheus),);- adminRouter.get(`/admin/${config.BULL_AUTH_KEY}/precrawl`, wrap(triggerPrecrawl));+ adminRouter.get(+ `/admin/${config.BULL_AUTH_KEY}/precrawl`,+ wrap(triggerPrecrawl),+ );- adminRouter.get(`/admin/${config.BULL_AUTH_KEY}/metrics`, wrap(metricsController));+ adminRouter.get(+ `/admin/${config.BULL_AUTH_KEY}/metrics`,+ wrap(metricsController),+ );- adminRouter.get(`/admin/${config.BULL_AUTH_KEY}/nuq-metrics`, wrap(nuqMetricsController));+ adminRouter.get(+ `/admin/${config.BULL_AUTH_KEY}/nuq-metrics`,+ wrap(nuqMetricsController),+ );- adminRouter.get(`/admin/${config.BULL_AUTH_KEY}/nuq-fdb-metrics`, wrap(nuqFdbMetricsController));+ adminRouter.get(+ `/admin/${config.BULL_AUTH_KEY}/nuq-fdb-metrics`,+ wrap(nuqFdbMetricsController),+ );- adminRouter.post(`/admin/${config.BULL_AUTH_KEY}/fsearch`, wrap(realtimeSearchController));+ adminRouter.post(+ `/admin/${config.BULL_AUTH_KEY}/fsearch`,+ wrap(realtimeSearchController),+ );adminRouter.post(`/admin/${config.BULL_AUTH_KEY}/concurrency-queue-backfill`,@@ -78,20 +122,40 @@ if (config.BULL_AUTH_KEY) {);}-if (config.S2S_FIRECRAWL_INTEGRATIONS_TO_FIRECRAWL_API_KEY) {- function bearerToken(value: string | string[] | undefined): string | null {- const header = Array.isArray(value) ? value[0] : value;- return header?.startsWith("Bearer ") ? header.slice(7) : null;- }+if (config.BULL_AUTH_KEY) {+ const opsRouter = express.Router();+ opsRouter.use(adminAuth);- function secretsMatch(provided: string | null, expected?: string): boolean {- if (!provided || !expected) return false;- const left = Buffer.from(provided);- const right = Buffer.from(expected);- return left.length === right.length && crypto.timingSafeEqual(left, right);- }+ opsRouter.get("/redis-health", redisHealthController);++ opsRouter.get("/autumn-health", autumnHealthController);++ opsRouter.post("/acuc-cache-clear", wrap(acucCacheClearController));++ opsRouter.post(+ "/ip-restriction-cache-clear",+ wrap(ipRestrictionCacheClearController),+ );++ opsRouter.post(+ "/key-restriction-cache-clear",+ wrap(keyRestrictionCacheClearController),+ );++ opsRouter.post(+ "/concurrency-queue-backfill",+ wrap(concurrencyQueueBackfillController),+ );++ adminRouter.use("/admin/ops", opsRouter);+}- function firecrawlIntegrationsMiddleware(req: Request, res: Response, next: NextFunction) {+if (config.S2S_FIRECRAWL_INTEGRATIONS_TO_FIRECRAWL_API_KEY) {+ function firecrawlIntegrationsMiddleware(+ req: Request,+ res: Response,+ next: NextFunction,+ ) {if (!secretsMatch(bearerToken(req.headers.authorization),@@ -100,7 +164,9 @@ if (config.S2S_FIRECRAWL_INTEGRATIONS_TO_FIRECRAWL_API_KEY) {) {return res.status(401).json({ error: "Unauthorized" });}- logger.info(`firecrawl-integrations service calling ${req.method} ${req.path}`);+ logger.info(+ `firecrawl-integrations service calling ${req.method} ${req.path}`,+ );next();}@@ -111,8 +177,51 @@ if (config.S2S_FIRECRAWL_INTEGRATIONS_TO_FIRECRAWL_API_KEY) {);}-adminRouter.post(`/admin/integration/create-user`, wrap(handleIntegrationAdminCreateUserProxy));+adminRouter.post(+ `/admin/integration/create-user`,+ wrap(handleIntegrationAdminCreateUserProxy),+);++adminRouter.post(+ `/admin/integration/validate-api-key`,+ wrap(handleIntegrationAdminValidateProxy),+);++adminRouter.post(+ `/admin/integration/rotate-api-key`,+ wrap(handleIntegrationAdminRotateProxy),+);++/** Served only by the ADMIN_INTERNAL_PORT listener, without a key. */+export const adminInternalRouter = express.Router();++adminInternalRouter.get("/admin/redis-health", redisHealthController);++adminInternalRouter.get("/admin/autumn-health", autumnHealthController);++adminInternalRouter.get("/admin/feng-check", wrap(checkFireEngine));++adminInternalRouter.get(+ "/admin/index-queue-prometheus",+ wrap(indexQueuePrometheus),+);++adminInternalRouter.get("/admin/precrawl", wrap(triggerPrecrawl));++adminInternalRouter.get("/admin/metrics", wrap(metricsController));++adminInternalRouter.get("/admin/nuq-metrics", wrap(nuqMetricsController));++adminInternalRouter.get(+ "/admin/nuq-fdb-metrics",+ wrap(nuqFdbMetricsController),+);-adminRouter.post(`/admin/integration/validate-api-key`, wrap(handleIntegrationAdminValidateProxy));+adminInternalRouter.post("/admin/fsearch", wrap(realtimeSearchController));-adminRouter.post(`/admin/integration/rotate-api-key`, wrap(handleIntegrationAdminRotateProxy));+adminInternalRouter.post(+ "/admin/crawl-monitor",+ authMiddleware(RateLimiterMode.Crawl),+ checkCreditsMiddleware(2),+ wrap(crawlMonitorController),+);