Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Add firecrawl-integrations admin s2s routes (#4344)

Erik Engervall907b8f0fafa5
diff --git a/apps/api/.env.example b/apps/api/.env.example
index 9554111c9..1e69032a3 100644
--- a/apps/api/.env.example
+++ b/apps/api/.env.example
@@ -70,6 +70,9 @@ POSTHOG_API_KEY=
# set if you'd like to send posthog events like job logs
POSTHOG_HOST=
+# Service to service key for firecrawl-integrations
+S2S_FIRECRAWL_INTEGRATIONS_TO_FIRECRAWL_API_KEY=
+
STRIPE_PRICE_ID_STANDARD=
STRIPE_PRICE_ID_SCALE=
STRIPE_PRICE_ID_STARTER=
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 0db084bbb..fdb5d2b37 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -5,28 +5,23 @@ import { getMcpActionLogConfigErrors } from "./lib/mcp-action-log-config";
/* Codecs */
const delimitedList = (separator = ",") => {
return z.codec(z.string(), z.array(z.string()), {
- decode: str => (str ? str.split(separator).map(s => s.trim()) : []),
- encode: arr => arr.join(separator),
+ decode: (str) => (str ? str.split(separator).map((s) => s.trim()) : []),
+ encode: (arr) => arr.join(separator),
});
};
const emptyStringAsUndefined = <T extends z.ZodTypeAny>(schema: T) =>
- z.preprocess(value => (value === "" ? undefined : value), schema.optional());
+ z.preprocess((value) => (value === "" ? undefined : value), schema.optional());
const emptyStringAsDefault = <T extends z.ZodTypeAny>(schema: T) =>
- z.preprocess(value => (value === "" ? undefined : value), schema);
+ z.preprocess((value) => (value === "" ? undefined : value), schema);
-const RESEARCH_PAPER_OPERATIONS = [
- "search",
- "inspect",
- "read",
- "similar",
-] as const;
+const RESEARCH_PAPER_OPERATIONS = ["search", "inspect", "read", "similar"] as const;
export type ResearchPaperOperation = (typeof RESEARCH_PAPER_OPERATIONS)[number];
const researchKeylessDisabled = z.preprocess(
- value => {
+ (value) => {
if (typeof value !== "string") return value;
const raw = value.trim().toLowerCase();
if (raw === "") return undefined;
@@ -36,12 +31,10 @@ const researchKeylessDisabled = z.preprocess(
}
return raw
.split(",")
- .map(operation => operation.trim())
+ .map((operation) => operation.trim())
.filter(Boolean);
},
- z
- .array(z.enum(RESEARCH_PAPER_OPERATIONS))
- .default([...RESEARCH_PAPER_OPERATIONS]),
+ z.array(z.enum(RESEARCH_PAPER_OPERATIONS)).default([...RESEARCH_PAPER_OPERATIONS]),
);
const containsLoneSurrogate = (value: string): boolean => {
@@ -74,11 +67,11 @@ const configSchema = z.object({
SUPPORT_AGENT_URL: z.string().url().optional(),
SUPPORT_AGENT_VERCEL_BYPASS_SECRET: z.string().optional(),
FIREBRAIN_TRACKS_URL: z.preprocess(
- v => (typeof v === "string" && v.trim() === "" ? undefined : v),
+ (v) => (typeof v === "string" && v.trim() === "" ? undefined : v),
z.string().url().optional(),
),
FIREBRAIN_TRACKS_API_KEY: z.preprocess(
- v => (typeof v === "string" && v.trim() === "" ? undefined : v),
+ (v) => (typeof v === "string" && v.trim() === "" ? undefined : v),
z.string().trim().optional(),
),
RESEARCH_PROXY_URL: z.string().url().optional(),
@@ -115,21 +108,14 @@ const configSchema = z.object({
// Google Web Risk. An unset key disables the provider (lookups then fail
// per the org's failurePolicy).
GOOGLE_WEB_RISK_API_KEY: z.string().optional(),
- GOOGLE_WEB_RISK_API_URL: z
- .string()
- .url()
- .default("https://webrisk.googleapis.com"),
+ GOOGLE_WEB_RISK_API_URL: z.string().url().default("https://webrisk.googleapis.com"),
// Google Web Risk Update API sync tuning. ZDR: "normal" mode checks run
// against a locally synced hash-prefix database (threatLists:computeDiff)
// instead of sending URLs to Google, and verdicts are never persisted.
//
// Floor for how often threatLists:computeDiff may run per list. Google's
// recommendedNextDiff is respected when it is later than this floor.
- THREAT_LIST_SYNC_MIN_INTERVAL_SECONDS: z.coerce
- .number()
- .int()
- .positive()
- .default(60),
+ THREAT_LIST_SYNC_MIN_INTERVAL_SECONDS: z.coerce.number().int().positive().default(60),
// A synced threat list older than this is treated as unavailable
// (provider-failure semantics → the org's failurePolicy decides).
THREAT_LIST_STALENESS_SECONDS: z.coerce
@@ -152,6 +138,7 @@ const configSchema = z.object({
// API Keys & Authentication
BULL_AUTH_KEY: z.string().optional(),
+ S2S_FIRECRAWL_INTEGRATIONS_TO_FIRECRAWL_API_KEY: emptyStringAsUndefined(z.string().trim().min(1)),
OPENAI_API_KEY: z.string().optional(),
OPENAI_BASE_URL: z.string().optional(),
OPENROUTER_API_KEY: z.string().optional(),
@@ -164,11 +151,7 @@ const configSchema = z.object({
SEARCH_PREVIEW_TOKEN: z.string().optional(),
SEARCH_SERVICE_API_SECRET: z.string().optional(),
SEARCH_FEEDBACK_MAX_AGE_SEC: z.coerce.number().int().positive().default(120),
- SEARCH_FEEDBACK_DAILY_CAP_CREDITS: z.coerce
- .number()
- .int()
- .nonnegative()
- .default(100),
+ SEARCH_FEEDBACK_DAILY_CAP_CREDITS: z.coerce.number().int().nonnegative().default(100),
FEEDBACK_MAX_AGE_SEC: z.coerce.number().int().positive().default(120),
FEEDBACK_DAILY_CAP_CREDITS: z.coerce.number().int().nonnegative().default(50),
FEEDBACK_REFUND_ENABLED: z.stringbool().default(true),
@@ -208,17 +191,11 @@ const configSchema = z.object({
NUQ_RABBITMQ_URL: z.string().optional(),
FDB_CLUSTER_FILE: emptyStringAsUndefined(z.string()),
NUQ_BACKEND: emptyStringAsUndefined(z.enum(["pg", "fdb"])),
- NUQ_FDB_READY_SHARDS: emptyStringAsDefault(
- z.coerce.number().int().positive().default(2048),
- ),
+ NUQ_FDB_READY_SHARDS: emptyStringAsDefault(z.coerce.number().int().positive().default(2048)),
// 1 = strict (priority, FIFO) promotion order per team; raise for teams with
// extreme finish rates at the cost of approximate cross-shard ordering
- NUQ_FDB_TEAM_PENDING_SHARDS: emptyStringAsDefault(
- z.coerce.number().int().positive().default(1),
- ),
- NUQ_FDB_TIME_BUCKETS: emptyStringAsDefault(
- z.coerce.number().int().positive().default(16),
- ),
+ NUQ_FDB_TEAM_PENDING_SHARDS: emptyStringAsDefault(z.coerce.number().int().positive().default(1)),
+ NUQ_FDB_TIME_BUCKETS: emptyStringAsDefault(z.coerce.number().int().positive().default(16)),
// Google Cloud Storage
GCS_BUCKET_NAME: z.string().optional(),
@@ -440,10 +417,10 @@ const configSchema = z.object({
EXTRACT_V3_BETA_URL: z.string().optional(),
AGENT_INTEROP_SECRET: z
.string()
- .refine(value => value.trim().length > 0, {
+ .refine((value) => value.trim().length > 0, {
error: "AGENT_INTEROP_SECRET must not be blank",
})
- .refine(value => !containsLoneSurrogate(value), {
+ .refine((value) => !containsLoneSurrogate(value), {
error: "AGENT_INTEROP_SECRET must not contain lone surrogates",
})
.optional(),
diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.ts
index c25937ad6..f4386561d 100644
--- a/apps/api/src/routes/admin.ts
+++ b/apps/api/src/routes/admin.ts
@@ -1,41 +1,37 @@
-import express from "express";
+import express, { NextFunction, Request, Response } from "express";
+import crypto from "node:crypto";
import { config } from "../config";
-import { redisHealthController } from "../controllers/v0/admin/redis-health";
-import { autumnHealthController } from "../controllers/v0/admin/autumn-health";
-import { authMiddleware, checkCreditsMiddleware, wrap } from "./shared";
import { acucCacheClearController } from "../controllers/v0/admin/acuc-cache-clear";
-import { ipRestrictionCacheClearController } from "../controllers/v0/admin/ip-restriction-cache-clear";
-import { keyRestrictionCacheClearController } from "../controllers/v0/admin/key-restriction-cache-clear";
+import { autumnHealthController } from "../controllers/v0/admin/autumn-health";
import { checkFireEngine } from "../controllers/v0/admin/check-fire-engine";
+import { concurrencyQueueBackfillController } from "../controllers/v0/admin/concurrency-queue-backfill";
+import { crawlMonitorController } from "../controllers/v0/admin/crawl-monitor";
import { indexQueuePrometheus } from "../controllers/v0/admin/index-queue-prometheus";
-import { triggerPrecrawl } from "../controllers/v0/admin/precrawl";
+import { ipRestrictionCacheClearController } from "../controllers/v0/admin/ip-restriction-cache-clear";
+import { keyRestrictionCacheClearController } from "../controllers/v0/admin/key-restriction-cache-clear";
import {
metricsController,
nuqFdbMetricsController,
nuqMetricsController,
} from "../controllers/v0/admin/metrics";
+import { triggerPrecrawl } from "../controllers/v0/admin/precrawl";
+import { redisHealthController } from "../controllers/v0/admin/redis-health";
import { realtimeSearchController } from "../controllers/v2/f-search";
-import { concurrencyQueueBackfillController } from "../controllers/v0/admin/concurrency-queue-backfill";
-import { crawlMonitorController } from "../controllers/v0/admin/crawl-monitor";
import {
handleIntegrationAdminCreateUserProxy,
handleIntegrationAdminRotateProxy,
handleIntegrationAdminValidateProxy,
} from "../lib/admin-integration-integrations-proxy";
+import { logger } from "../lib/logger";
import { RateLimiterMode } from "../types";
+import { authMiddleware, checkCreditsMiddleware, wrap } from "./shared";
export const adminRouter = express.Router();
if (config.BULL_AUTH_KEY) {
- adminRouter.get(
- `/admin/${config.BULL_AUTH_KEY}/redis-health`,
- redisHealthController,
- );
+ adminRouter.get(`/admin/${config.BULL_AUTH_KEY}/redis-health`, redisHealthController);
- adminRouter.get(
- `/admin/${config.BULL_AUTH_KEY}/autumn-health`,
- autumnHealthController,
- );
+ adminRouter.get(`/admin/${config.BULL_AUTH_KEY}/autumn-health`, autumnHealthController);
adminRouter.post(
`/admin/${config.BULL_AUTH_KEY}/acuc-cache-clear`,
@@ -52,40 +48,22 @@ if (config.BULL_AUTH_KEY) {
wrap(keyRestrictionCacheClearController),
);
- adminRouter.get(
- `/admin/${config.BULL_AUTH_KEY}/feng-check`,
- wrap(checkFireEngine),
- );
+ adminRouter.get(`/admin/${config.BULL_AUTH_KEY}/feng-check`, wrap(checkFireEngine));
adminRouter.get(
`/admin/${config.BULL_AUTH_KEY}/index-queue-prometheus`,
wrap(indexQueuePrometheus),
);
- adminRouter.get(
- `/admin/${config.BULL_AUTH_KEY}/precrawl`,
- wrap(triggerPrecrawl),
- );
+ adminRouter.get(`/admin/${config.BULL_AUTH_KEY}/precrawl`, wrap(triggerPrecrawl));
- adminRouter.get(
- `/admin/${config.BULL_AUTH_KEY}/metrics`,
- wrap(metricsController),
- );
+ adminRouter.get(`/admin/${config.BULL_AUTH_KEY}/metrics`, wrap(metricsController));
- adminRouter.get(
- `/admin/${config.BULL_AUTH_KEY}/nuq-metrics`,
- wrap(nuqMetricsController),
- );
+ adminRouter.get(`/admin/${config.BULL_AUTH_KEY}/nuq-metrics`, wrap(nuqMetricsController));
- adminRouter.get(
- `/admin/${config.BULL_AUTH_KEY}/nuq-fdb-metrics`,
- wrap(nuqFdbMetricsController),
- );
+ adminRouter.get(`/admin/${config.BULL_AUTH_KEY}/nuq-fdb-metrics`, wrap(nuqFdbMetricsController));
- adminRouter.post(
- `/admin/${config.BULL_AUTH_KEY}/fsearch`,
- wrap(realtimeSearchController),
- );
+ adminRouter.post(`/admin/${config.BULL_AUTH_KEY}/fsearch`, wrap(realtimeSearchController));
adminRouter.post(
`/admin/${config.BULL_AUTH_KEY}/concurrency-queue-backfill`,
@@ -100,17 +78,41 @@ if (config.BULL_AUTH_KEY) {
);
}
-adminRouter.post(
- `/admin/integration/create-user`,
- wrap(handleIntegrationAdminCreateUserProxy),
-);
+if (config.S2S_FIRECRAWL_INTEGRATIONS_TO_FIRECRAWL_API_KEY) {
+ function bearerToken(value: string | string[] | undefined): string | null {
+ const header = Array.isArray(value) ? value[0] : value;
+ return header?.startsWith("Bearer ") ? header.slice(7) : null;
+ }
+
+ function secretsMatch(provided: string | null, expected?: string): boolean {
+ if (!provided || !expected) return false;
+ const left = Buffer.from(provided);
+ const right = Buffer.from(expected);
+ return left.length === right.length && crypto.timingSafeEqual(left, right);
+ }
+
+ function firecrawlIntegrationsMiddleware(req: Request, res: Response, next: NextFunction) {
+ if (
+ !secretsMatch(
+ bearerToken(req.headers.authorization),
+ config.S2S_FIRECRAWL_INTEGRATIONS_TO_FIRECRAWL_API_KEY,
+ )
+ ) {
+ return res.status(401).json({ error: "Unauthorized" });
+ }
+ logger.info(`firecrawl-integrations service calling ${req.method} ${req.path}`);
+ next();
+ }
+
+ adminRouter.post(
+ "/admin/acuc-cache-clear",
+ firecrawlIntegrationsMiddleware,
+ wrap(acucCacheClearController),
+ );
+}
+
+adminRouter.post(`/admin/integration/create-user`, wrap(handleIntegrationAdminCreateUserProxy));
-adminRouter.post(
- `/admin/integration/validate-api-key`,
- wrap(handleIntegrationAdminValidateProxy),
-);
+adminRouter.post(`/admin/integration/validate-api-key`, wrap(handleIntegrationAdminValidateProxy));
-adminRouter.post(
- `/admin/integration/rotate-api-key`,
- wrap(handleIntegrationAdminRotateProxy),
-);
+adminRouter.post(`/admin/integration/rotate-api-key`, wrap(handleIntegrationAdminRotateProxy));