Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
import "dotenv/config";
import { config } from "./config";
import { shutdownTracing } from "./otel";
import express, { NextFunction, Request, Response } from "express";
import bodyParser from "body-parser";
import cors from "cors";
import {
getGenerateLlmsTxtQueue,
getDeepResearchQueue,
getBillingQueue,
getPrecrawlQueue,
getFirebillTrackRetryQueue,
} from "./services/queue-service";
import { v0Router } from "./routes/v0";
import { livenessController, readinessController } from "./controllers/health";
import { startNodeRuntimeMetrics } from "./lib/node-runtime-metrics";
import os from "os";
import { logger } from "./lib/logger";
import {
adminInternalRouter,
adminRouter,
countAdminPathRoute,
} from "./routes/admin";
import http from "node:http";
import https from "node:https";
import { v1Router } from "./routes/v1";
import expressWs from "express-ws";
import { ErrorResponse, RequestWithMaybeACUC } from "./controllers/v1/types";
import { ZodError } from "zod";
import { QueueFullError } from "./lib/queue-full-error";
import { v7 as uuidv7 } from "uuid";
import { cacheableLookup } from "./scraper/scrapeURL/lib/cacheableLookup";
import { v2Router } from "./routes/v2";
import { exchangeRouter } from "./routes/exchange";
import { labsRouter } from "./routes/labs";
import { registerMcpActionLogIngestRoute } from "./routes/mcp-action-logs";
import { registerHostedBashRoute } from "./controllers/v2/bash";
import { nuqShutdown } from "./services/worker/nuq";
import { getErrorContactMessage } from "./lib/deployment";
import { initializeBlocklist } from "./scraper/WebScraper/utils/blocklist";
import { warmExchangeCatalog } from "./lib/exchange";
import { initializeEngineForcing } from "./scraper/WebScraper/utils/engine-forcing";
import responseTime from "response-time";
import { shutdownWebhookQueue } from "./services/webhook";
import { shutdownIndexerQueue } from "./services/indexing/indexer-queue";
import { isKeylessConfigured } from "./lib/keyless";
import {
drainZdrRequestLogs,
shutdownPubSubLogging,
} from "./services/logging/log_job";
import { notFoundHandler } from "./lib/not-found";
const { createBullBoard } = require("@bull-board/api");
const { BullMQAdapter } = require("@bull-board/api/bullMQAdapter");
const { ExpressAdapter } = require("@bull-board/express");
const numCPUs = config.ENV === "local" ? 2 : os.cpus().length;
logger.info(`Number of CPUs: ${numCPUs} available`);
logger.info("Network info dump", {
networkInterfaces: os.networkInterfaces(),
});
if (isKeylessConfigured() && !config.KEYLESS_CONVERSION_HMAC_SECRET) {
logger.warn(
"Keyless conversion cohort logging is disabled: set KEYLESS_CONVERSION_HMAC_SECRET to enable privacy-safe quota-to-account measurement",
);
}
// Install cacheable lookup for all other requests
cacheableLookup.install(http.globalAgent);
cacheableLookup.install(https.globalAgent);
// Initialize Express with WebSocket support
const expressApp = express();
const ws = expressWs(expressApp);
const app = ws.app;
global.isProduction = config.IS_PRODUCTION;
// Capture the exact request bytes so integrations that sign the raw payload
// (e.g. Slack's X-Slack-Signature) can verify it after body parsing. Typed with
// the node http types body-parser's `verify` hook expects.
const captureRawBody = (
req: http.IncomingMessage,
_res: http.ServerResponse,
buf: Buffer,
) => {
if (buf && buf.length) {
(req as http.IncomingMessage & { rawBody?: Buffer }).rawBody = buf;
}
};
registerMcpActionLogIngestRoute(app);
registerHostedBashRoute(app);
app.use(bodyParser.urlencoded({ extended: true, verify: captureRawBody }));
app.use(bodyParser.json({ limit: "10mb", verify: captureRawBody }));
app.use(cors()); // Add this line to enable CORS
app.use(responseTime());
app.disable("x-powered-by");
if (config.EXPRESS_TRUST_PROXY) {
app.set("trust proxy", config.EXPRESS_TRUST_PROXY);
}
const serverAdapter = new ExpressAdapter();
const bullBoardQueues = [
new BullMQAdapter(getGenerateLlmsTxtQueue()),
new BullMQAdapter(getDeepResearchQueue()),
new BullMQAdapter(getBillingQueue()),
new BullMQAdapter(getPrecrawlQueue()),
new BullMQAdapter(getFirebillTrackRetryQueue()),
// SIEM audit delivery runs on RabbitMQ; inspect it in the broker's
// management UI, not here.
];
const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({
queues: bullBoardQueues,
serverAdapter: serverAdapter,
});
if (!config.BULL_AUTH_KEY) {
logger.warn("BULL_AUTH_KEY is not set; admin routes are disabled.");
} else if (config.BULL_AUTH_KEY_PATH_ROUTES) {
serverAdapter.setBasePath(`/admin/${config.BULL_AUTH_KEY}/queues`);
app.use(
`/admin/${config.BULL_AUTH_KEY}/queues`,
countAdminPathRoute("queues"),
serverAdapter.getRouter(),
);
}
app.get("/", (_, res) => {
res.json({
message: "Firecrawl API",
documentation_url: "https://docs.firecrawl.dev",
});
});
app.get("/e2e-test", (_, res) => {
res.status(200).send("OK");
});
app.get("/health/liveness", livenessController);
app.get("/health/readiness", readinessController);
// register router
app.use(v0Router);
app.use("/v1", v1Router);
app.use("/v2", v2Router);
app.use("/labs", labsRouter);
app.use("/exchange", exchangeRouter);
app.use(adminRouter);
const DEFAULT_PORT = config.PORT;
const HOST = config.HOST;
// The internal listener has no key check: keep its port off public ingress.
function startAdminInternalServer(port: number) {
const adminApp = express();
adminApp.disable("x-powered-by");
adminApp.use(bodyParser.json({ limit: "10mb" }));
const internalBullBoard = new ExpressAdapter();
internalBullBoard.setBasePath("/admin/queues");
createBullBoard({
queues: bullBoardQueues,
serverAdapter: internalBullBoard,
});
adminApp.use("/admin/queues", internalBullBoard.getRouter());
adminApp.use(adminInternalRouter);
adminApp.use(notFoundHandler);
adminApp.use(
(err: unknown, req: Request, res: Response, _next: NextFunction) => {
logger.error("Internal admin request failed", {
error: err,
path: req.path,
});
res.status(500).json({ success: false, error: "Internal server error" });
},
);
const host = config.ADMIN_INTERNAL_HOST;
return adminApp.listen(port, host, (error?: Error) => {
if (error) {
logger.error("Failed to start internal admin server", {
error,
port,
host,
});
throw error;
}
logger.info("Internal admin server listening", {
pid: process.pid,
port,
host,
});
});
}
async function startServer(port = DEFAULT_PORT) {
startNodeRuntimeMetrics();
try {
await initializeBlocklist();
initializeEngineForcing();
warmExchangeCatalog();
} catch (error) {
logger.error("Failed to initialize API startup dependencies", {
error,
});
throw error;
}
const server = app.listen(Number(port), HOST, (error?: Error) => {
if (error) {
logger.error("Failed to start HTTP server", { error, port, host: HOST });
throw error;
}
logger.info(`Worker ${process.pid} listening on port ${port}`);
});
const adminInternalServer =
config.ADMIN_INTERNAL_PORT !== undefined
? startAdminInternalServer(config.ADMIN_INTERNAL_PORT)
: undefined;
const exitHandler = async () => {
logger.info("SIGTERM signal received: closing HTTP server");
if (config.IS_KUBERNETES) {
// Account for GCE load balancer drain timeout
logger.info("Waiting 60s for GCE load balancer drain timeout");
await new Promise(resolve => setTimeout(resolve, 60000));
}
const adminInternalClosed = new Promise<void>(resolve =>
adminInternalServer
? adminInternalServer.close(() => resolve())
: resolve(),
);
server.close(async () => {
await adminInternalClosed;
logger.info("Server closed.");
await drainZdrRequestLogs();
nuqShutdown().finally(() => {
shutdownWebhookQueue().finally(() => {
shutdownIndexerQueue().finally(async () => {
await shutdownPubSubLogging();
await shutdownTracing();
logger.info("NUQ shutdown complete");
process.exit(0);
});
});
});
});
};
if (require.main === module) {
process.on("SIGTERM", exitHandler);
process.on("SIGINT", exitHandler);
}
return server;
}
if (require.main === module) {
startServer().catch(error => {
logger.error("Failed to start server", { error });
process.exit(1);
});
}
app.get("/is-production", (req, res) => {
res.send({ isProduction: global.isProduction });
});
// Terminal handler for unmatched paths. Must stay after every route and before
// the error middleware below, or Express falls back to its default HTML page.
app.use(notFoundHandler);
app.use(
(
err: unknown,
req: Request<{}, ErrorResponse, undefined>,
res: Response<ErrorResponse>,
next: NextFunction,
) => {
if (err instanceof QueueFullError) {
res.status(429).json({
success: false,
error: err.message,
});
} else if (err instanceof ZodError) {
// In zod v4, ZodError uses 'issues' instead of 'errors'
const issues = err.issues;
if (
Array.isArray(issues) &&
issues.find(x => x.message === "URL uses unsupported protocol")
) {
logger.warn("Unsupported protocol error: " + JSON.stringify(req.body));
}
// Check for unrecognized_keys errors and replace with custom message
const hasUnrecognizedKeys = issues.some(
e => e.code === "unrecognized_keys",
);
const strictMessage =
"Unrecognized key in body -- please review the v2 API documentation for request body changes";
const customErrorMessage = hasUnrecognizedKeys
? strictMessage
: issues.length > 0 && issues[0].code === "custom"
? issues[0].message
: "Bad Request";
res.status(400).json({
success: false,
code: "BAD_REQUEST",
error: customErrorMessage,
details: issues,
});
} else {
next(err);
}
},
);
app.use(
(
err: unknown,
req: RequestWithMaybeACUC<{}, ErrorResponse, undefined>,
res: Response<ErrorResponse>,
next: NextFunction,
) => {
if (
err instanceof SyntaxError &&
"status" in err &&
err.status === 400 &&
"body" in err
) {
return res.status(400).json({
success: false,
code: "BAD_REQUEST_INVALID_JSON",
error: "Bad request, malformed JSON",
});
}
if (
typeof err === "object" &&
err !== null &&
"status" in err &&
(err as { status?: number }).status === 413
) {
return res.status(413).json({
success: false,
error: "Request body is too large",
});
}
const id = uuidv7();
logger.error(
"Error occurred in request! (" + req.path + ") -- ID " + id + " -- ",
{
error: err,
errorId: id,
path: req.path,
teamId: req.acuc?.team_id,
team_id: req.acuc?.team_id,
},
);
res.status(500).json({
success: false,
code: "UNKNOWN_ERROR",
error: getErrorContactMessage(id),
});
},
);
logger.info(`Worker ${process.pid} started`);