Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Add firecrawl-integrations admin s2s routes (#4344)
Erik Engervall907b8f0fafa5
diff --git a/apps/api/.env.example b/apps/api/.env.exampleindex 9554111c9..1e69032a3 100644--- a/apps/api/.env.example+++ b/apps/api/.env.example@@ -70,6 +70,9 @@ POSTHOG_API_KEY=# set if you'd like to send posthog events like job logsPOSTHOG_HOST=+# Service to service key for firecrawl-integrations+S2S_FIRECRAWL_INTEGRATIONS_TO_FIRECRAWL_API_KEY=+STRIPE_PRICE_ID_STANDARD=STRIPE_PRICE_ID_SCALE=STRIPE_PRICE_ID_STARTER=diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 0db084bbb..fdb5d2b37 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -5,28 +5,23 @@ import { getMcpActionLogConfigErrors } from "./lib/mcp-action-log-config";/* Codecs */const delimitedList = (separator = ",") => {return z.codec(z.string(), z.array(z.string()), {- decode: str => (str ? str.split(separator).map(s => s.trim()) : []),- encode: arr => arr.join(separator),+ decode: (str) => (str ? str.split(separator).map((s) => s.trim()) : []),+ encode: (arr) => arr.join(separator),});};const emptyStringAsUndefined = <T extends z.ZodTypeAny>(schema: T) =>- z.preprocess(value => (value === "" ? undefined : value), schema.optional());+ z.preprocess((value) => (value === "" ? undefined : value), schema.optional());const emptyStringAsDefault = <T extends z.ZodTypeAny>(schema: T) =>- z.preprocess(value => (value === "" ? undefined : value), schema);+ z.preprocess((value) => (value === "" ? undefined : value), schema);-const RESEARCH_PAPER_OPERATIONS = [- "search",- "inspect",- "read",- "similar",-] as const;+const RESEARCH_PAPER_OPERATIONS = ["search", "inspect", "read", "similar"] as const;export type ResearchPaperOperation = (typeof RESEARCH_PAPER_OPERATIONS)[number];const researchKeylessDisabled = z.preprocess(- value => {+ (value) => {if (typeof value !== "string") return value;const raw = value.trim().toLowerCase();if (raw === "") return undefined;@@ -36,12 +31,10 @@ const researchKeylessDisabled = z.preprocess(}return raw.split(",")- .map(operation => operation.trim())+ .map((operation) => operation.trim()).filter(Boolean);},- z- .array(z.enum(RESEARCH_PAPER_OPERATIONS))- .default([...RESEARCH_PAPER_OPERATIONS]),+ z.array(z.enum(RESEARCH_PAPER_OPERATIONS)).default([...RESEARCH_PAPER_OPERATIONS]),);const containsLoneSurrogate = (value: string): boolean => {@@ -74,11 +67,11 @@ const configSchema = z.object({SUPPORT_AGENT_URL: z.string().url().optional(),SUPPORT_AGENT_VERCEL_BYPASS_SECRET: z.string().optional(),FIREBRAIN_TRACKS_URL: z.preprocess(- v => (typeof v === "string" && v.trim() === "" ? undefined : v),+ (v) => (typeof v === "string" && v.trim() === "" ? undefined : v),z.string().url().optional(),),FIREBRAIN_TRACKS_API_KEY: z.preprocess(- v => (typeof v === "string" && v.trim() === "" ? undefined : v),+ (v) => (typeof v === "string" && v.trim() === "" ? undefined : v),z.string().trim().optional(),),RESEARCH_PROXY_URL: z.string().url().optional(),@@ -115,21 +108,14 @@ const configSchema = z.object({// Google Web Risk. An unset key disables the provider (lookups then fail// per the org's failurePolicy).GOOGLE_WEB_RISK_API_KEY: z.string().optional(),- GOOGLE_WEB_RISK_API_URL: z- .string()- .url()- .default("https://webrisk.googleapis.com"),+ GOOGLE_WEB_RISK_API_URL: z.string().url().default("https://webrisk.googleapis.com"),// Google Web Risk Update API sync tuning. ZDR: "normal" mode checks run// against a locally synced hash-prefix database (threatLists:computeDiff)// instead of sending URLs to Google, and verdicts are never persisted.//// Floor for how often threatLists:computeDiff may run per list. Google's// recommendedNextDiff is respected when it is later than this floor.- THREAT_LIST_SYNC_MIN_INTERVAL_SECONDS: z.coerce- .number()- .int()- .positive()- .default(60),+ THREAT_LIST_SYNC_MIN_INTERVAL_SECONDS: z.coerce.number().int().positive().default(60),// A synced threat list older than this is treated as unavailable// (provider-failure semantics → the org's failurePolicy decides).THREAT_LIST_STALENESS_SECONDS: z.coerce@@ -152,6 +138,7 @@ const configSchema = z.object({// API Keys & AuthenticationBULL_AUTH_KEY: z.string().optional(),+ S2S_FIRECRAWL_INTEGRATIONS_TO_FIRECRAWL_API_KEY: emptyStringAsUndefined(z.string().trim().min(1)),OPENAI_API_KEY: z.string().optional(),OPENAI_BASE_URL: z.string().optional(),OPENROUTER_API_KEY: z.string().optional(),@@ -164,11 +151,7 @@ const configSchema = z.object({SEARCH_PREVIEW_TOKEN: z.string().optional(),SEARCH_SERVICE_API_SECRET: z.string().optional(),SEARCH_FEEDBACK_MAX_AGE_SEC: z.coerce.number().int().positive().default(120),- SEARCH_FEEDBACK_DAILY_CAP_CREDITS: z.coerce- .number()- .int()- .nonnegative()- .default(100),+ SEARCH_FEEDBACK_DAILY_CAP_CREDITS: z.coerce.number().int().nonnegative().default(100),FEEDBACK_MAX_AGE_SEC: z.coerce.number().int().positive().default(120),FEEDBACK_DAILY_CAP_CREDITS: z.coerce.number().int().nonnegative().default(50),FEEDBACK_REFUND_ENABLED: z.stringbool().default(true),@@ -208,17 +191,11 @@ const configSchema = z.object({NUQ_RABBITMQ_URL: z.string().optional(),FDB_CLUSTER_FILE: emptyStringAsUndefined(z.string()),NUQ_BACKEND: emptyStringAsUndefined(z.enum(["pg", "fdb"])),- NUQ_FDB_READY_SHARDS: emptyStringAsDefault(- z.coerce.number().int().positive().default(2048),- ),+ NUQ_FDB_READY_SHARDS: emptyStringAsDefault(z.coerce.number().int().positive().default(2048)),// 1 = strict (priority, FIFO) promotion order per team; raise for teams with// extreme finish rates at the cost of approximate cross-shard ordering- NUQ_FDB_TEAM_PENDING_SHARDS: emptyStringAsDefault(- z.coerce.number().int().positive().default(1),- ),- NUQ_FDB_TIME_BUCKETS: emptyStringAsDefault(- z.coerce.number().int().positive().default(16),- ),+ NUQ_FDB_TEAM_PENDING_SHARDS: emptyStringAsDefault(z.coerce.number().int().positive().default(1)),+ NUQ_FDB_TIME_BUCKETS: emptyStringAsDefault(z.coerce.number().int().positive().default(16)),// Google Cloud StorageGCS_BUCKET_NAME: z.string().optional(),@@ -440,10 +417,10 @@ const configSchema = z.object({EXTRACT_V3_BETA_URL: z.string().optional(),AGENT_INTEROP_SECRET: z.string()- .refine(value => value.trim().length > 0, {+ .refine((value) => value.trim().length > 0, {error: "AGENT_INTEROP_SECRET must not be blank",})- .refine(value => !containsLoneSurrogate(value), {+ .refine((value) => !containsLoneSurrogate(value), {error: "AGENT_INTEROP_SECRET must not contain lone surrogates",}).optional(),diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.tsindex c25937ad6..f4386561d 100644--- a/apps/api/src/routes/admin.ts+++ b/apps/api/src/routes/admin.ts@@ -1,41 +1,37 @@-import express from "express";+import express, { NextFunction, Request, Response } from "express";+import crypto from "node:crypto";import { config } from "../config";-import { redisHealthController } from "../controllers/v0/admin/redis-health";-import { autumnHealthController } from "../controllers/v0/admin/autumn-health";-import { authMiddleware, checkCreditsMiddleware, wrap } from "./shared";import { acucCacheClearController } from "../controllers/v0/admin/acuc-cache-clear";-import { ipRestrictionCacheClearController } from "../controllers/v0/admin/ip-restriction-cache-clear";-import { keyRestrictionCacheClearController } from "../controllers/v0/admin/key-restriction-cache-clear";+import { autumnHealthController } from "../controllers/v0/admin/autumn-health";import { checkFireEngine } from "../controllers/v0/admin/check-fire-engine";+import { concurrencyQueueBackfillController } from "../controllers/v0/admin/concurrency-queue-backfill";+import { crawlMonitorController } from "../controllers/v0/admin/crawl-monitor";import { indexQueuePrometheus } from "../controllers/v0/admin/index-queue-prometheus";-import { triggerPrecrawl } from "../controllers/v0/admin/precrawl";+import { ipRestrictionCacheClearController } from "../controllers/v0/admin/ip-restriction-cache-clear";+import { keyRestrictionCacheClearController } from "../controllers/v0/admin/key-restriction-cache-clear";import {metricsController,nuqFdbMetricsController,nuqMetricsController,} from "../controllers/v0/admin/metrics";+import { triggerPrecrawl } from "../controllers/v0/admin/precrawl";+import { redisHealthController } from "../controllers/v0/admin/redis-health";import { realtimeSearchController } from "../controllers/v2/f-search";-import { concurrencyQueueBackfillController } from "../controllers/v0/admin/concurrency-queue-backfill";-import { crawlMonitorController } from "../controllers/v0/admin/crawl-monitor";import {handleIntegrationAdminCreateUserProxy,handleIntegrationAdminRotateProxy,handleIntegrationAdminValidateProxy,} from "../lib/admin-integration-integrations-proxy";+import { logger } from "../lib/logger";import { RateLimiterMode } from "../types";+import { authMiddleware, checkCreditsMiddleware, wrap } from "./shared";export const adminRouter = express.Router();if (config.BULL_AUTH_KEY) {- adminRouter.get(- `/admin/${config.BULL_AUTH_KEY}/redis-health`,- redisHealthController,- );+ adminRouter.get(`/admin/${config.BULL_AUTH_KEY}/redis-health`, redisHealthController);- adminRouter.get(- `/admin/${config.BULL_AUTH_KEY}/autumn-health`,- autumnHealthController,- );+ adminRouter.get(`/admin/${config.BULL_AUTH_KEY}/autumn-health`, autumnHealthController);adminRouter.post(`/admin/${config.BULL_AUTH_KEY}/acuc-cache-clear`,@@ -52,40 +48,22 @@ if (config.BULL_AUTH_KEY) {wrap(keyRestrictionCacheClearController),);- adminRouter.get(- `/admin/${config.BULL_AUTH_KEY}/feng-check`,- wrap(checkFireEngine),- );+ adminRouter.get(`/admin/${config.BULL_AUTH_KEY}/feng-check`, wrap(checkFireEngine));adminRouter.get(`/admin/${config.BULL_AUTH_KEY}/index-queue-prometheus`,wrap(indexQueuePrometheus),);- adminRouter.get(- `/admin/${config.BULL_AUTH_KEY}/precrawl`,- wrap(triggerPrecrawl),- );+ adminRouter.get(`/admin/${config.BULL_AUTH_KEY}/precrawl`, wrap(triggerPrecrawl));- adminRouter.get(- `/admin/${config.BULL_AUTH_KEY}/metrics`,- wrap(metricsController),- );+ adminRouter.get(`/admin/${config.BULL_AUTH_KEY}/metrics`, wrap(metricsController));- adminRouter.get(- `/admin/${config.BULL_AUTH_KEY}/nuq-metrics`,- wrap(nuqMetricsController),- );+ adminRouter.get(`/admin/${config.BULL_AUTH_KEY}/nuq-metrics`, wrap(nuqMetricsController));- adminRouter.get(- `/admin/${config.BULL_AUTH_KEY}/nuq-fdb-metrics`,- wrap(nuqFdbMetricsController),- );+ adminRouter.get(`/admin/${config.BULL_AUTH_KEY}/nuq-fdb-metrics`, wrap(nuqFdbMetricsController));- adminRouter.post(- `/admin/${config.BULL_AUTH_KEY}/fsearch`,- wrap(realtimeSearchController),- );+ adminRouter.post(`/admin/${config.BULL_AUTH_KEY}/fsearch`, wrap(realtimeSearchController));adminRouter.post(`/admin/${config.BULL_AUTH_KEY}/concurrency-queue-backfill`,@@ -100,17 +78,41 @@ if (config.BULL_AUTH_KEY) {);}-adminRouter.post(- `/admin/integration/create-user`,- wrap(handleIntegrationAdminCreateUserProxy),-);+if (config.S2S_FIRECRAWL_INTEGRATIONS_TO_FIRECRAWL_API_KEY) {+ function bearerToken(value: string | string[] | undefined): string | null {+ const header = Array.isArray(value) ? value[0] : value;+ return header?.startsWith("Bearer ") ? header.slice(7) : null;+ }++ function secretsMatch(provided: string | null, expected?: string): boolean {+ if (!provided || !expected) return false;+ const left = Buffer.from(provided);+ const right = Buffer.from(expected);+ return left.length === right.length && crypto.timingSafeEqual(left, right);+ }++ function firecrawlIntegrationsMiddleware(req: Request, res: Response, next: NextFunction) {+ if (+ !secretsMatch(+ bearerToken(req.headers.authorization),+ config.S2S_FIRECRAWL_INTEGRATIONS_TO_FIRECRAWL_API_KEY,+ )+ ) {+ return res.status(401).json({ error: "Unauthorized" });+ }+ logger.info(`firecrawl-integrations service calling ${req.method} ${req.path}`);+ next();+ }++ adminRouter.post(+ "/admin/acuc-cache-clear",+ firecrawlIntegrationsMiddleware,+ wrap(acucCacheClearController),+ );+}++adminRouter.post(`/admin/integration/create-user`, wrap(handleIntegrationAdminCreateUserProxy));-adminRouter.post(- `/admin/integration/validate-api-key`,- wrap(handleIntegrationAdminValidateProxy),-);+adminRouter.post(`/admin/integration/validate-api-key`, wrap(handleIntegrationAdminValidateProxy));-adminRouter.post(- `/admin/integration/rotate-api-key`,- wrap(handleIntegrationAdminRotateProxy),-);+adminRouter.post(`/admin/integration/rotate-api-key`, wrap(handleIntegrationAdminRotateProxy));