Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api/admin/crawl-monitor): add endpoint for monitoring crawl system (#2518)

Gergő Móricza444c84114b9
diff --git a/apps/api/src/controllers/v0/admin/crawl-monitor.ts b/apps/api/src/controllers/v0/admin/crawl-monitor.ts
new file mode 100644
index 000000000..7a6f8d5bf
--- /dev/null
+++ b/apps/api/src/controllers/v0/admin/crawl-monitor.ts
@@ -0,0 +1,127 @@
+import { Response } from "express";
+import { logger as _logger } from "../../../lib/logger";
+import {
+ crawlerOptions,
+ RequestWithAuth,
+ scrapeOptions,
+ toV0CrawlerOptions,
+} from "../../v2/types";
+import { v7 as uuidv7 } from "uuid";
+import { logRequest } from "../../../services/logging/log_job";
+import {
+ crawlToCrawler,
+ markCrawlActive,
+ saveCrawl,
+ StoredCrawl,
+} from "../../../lib/crawl-redis";
+import { config } from "../../../config";
+import { crawlGroup } from "../../../services/worker/nuq";
+import { _addScrapeJobToBullMQ } from "../../../services/queue-jobs";
+
+type ResponseType = {
+ ok: boolean;
+ crawlId: string;
+};
+
+export async function crawlMonitorController(
+ req: RequestWithAuth<{}, undefined, ResponseType>,
+ res: Response<ResponseType>,
+) {
+ const id = uuidv7();
+
+ const logger = _logger.child({
+ module: "crawl-monitor",
+ method: "crawlMonitorController",
+ crawlId: id,
+ });
+
+ logger.debug("Crawl monitor " + id + " starting", {
+ account: req.account,
+ });
+
+ await logRequest({
+ id,
+ kind: "crawl",
+ api_version: "v2",
+ team_id: req.auth.team_id,
+ origin: "api",
+ integration: null,
+ target_hint: "https://firecrawl.dev",
+ zeroDataRetention: false,
+ });
+
+ const sc: StoredCrawl = {
+ originUrl: "https://firecrawl.dev",
+ crawlerOptions: toV0CrawlerOptions(crawlerOptions.parse({ limit: 2 })),
+ scrapeOptions: scrapeOptions.parse({}),
+ internalOptions: {
+ disableSmartWaitCache: true,
+ teamId: req.auth.team_id,
+ saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,
+ zeroDataRetention: false,
+ },
+ team_id: req.auth.team_id,
+ createdAt: Date.now(),
+ maxConcurrency: undefined,
+ zeroDataRetention: false,
+ };
+
+ const crawler = crawlToCrawler(id, sc, req.acuc?.flags ?? null);
+
+ try {
+ sc.robots = await crawler.getRobotsTxt(false);
+ } catch (e) {
+ logger.debug("Failed to get robots.txt (this is probably fine!)", {
+ error: e,
+ });
+ }
+
+ await crawlGroup.addGroup(
+ id,
+ sc.team_id,
+ (req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,
+ );
+
+ await saveCrawl(id, sc);
+
+ await markCrawlActive(id);
+
+ await _addScrapeJobToBullMQ(
+ {
+ url: "https://firecrawl.dev",
+ mode: "kickoff" as const,
+ team_id: req.auth.team_id,
+ crawlerOptions: crawlerOptions.parse({ limit: 2 }),
+ scrapeOptions: sc.scrapeOptions,
+ internalOptions: sc.internalOptions,
+ origin: "api",
+ integration: null,
+ crawl_id: id,
+ webhook: undefined,
+ v1: true,
+ zeroDataRetention: false,
+ apiKeyId: req.acuc?.api_key_id ?? null,
+ },
+ uuidv7(),
+ );
+
+ const startTime = Date.now();
+
+ while (Date.now() - startTime < 60000) {
+ const group = await crawlGroup.getGroup(id);
+ if (group?.status === "completed") {
+ logger.debug("Crawl completed");
+ return res.status(200).json({
+ ok: true,
+ crawlId: id,
+ });
+ }
+ await new Promise(resolve => setTimeout(resolve, 1000));
+ }
+
+ logger.warn("Crawl timed out");
+ return res.status(500).json({
+ ok: false,
+ crawlId: id,
+ });
+}
diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.ts
index c5de2d7c5..e3df36cee 100644
--- a/apps/api/src/controllers/v2/types.ts
+++ b/apps/api/src/controllers/v2/types.ts
@@ -794,7 +794,7 @@ export type BatchScrapeRequestInput = Omit<
zeroDataRetention?: boolean;
};
-const crawlerOptions = z.strictObject({
+export const crawlerOptions = z.strictObject({
includePaths: z.string().array().prefault([]),
excludePaths: z.string().array().prefault([]),
maxDiscoveryDepth: z.number().optional(),
diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.ts
index cc6839595..85ef71bb9 100644
--- a/apps/api/src/routes/admin.ts
+++ b/apps/api/src/routes/admin.ts
@@ -1,7 +1,7 @@
import express from "express";
import { config } from "../config";
import { redisHealthController } from "../controllers/v0/admin/redis-health";
-import { wrap } from "./shared";
+import { authMiddleware, checkCreditsMiddleware, wrap } from "./shared";
import { acucCacheClearController } from "../controllers/v0/admin/acuc-cache-clear";
import { checkFireEngine } from "../controllers/v0/admin/check-fire-engine";
import { cclogController } from "../controllers/v0/admin/cclog";
@@ -16,6 +16,8 @@ import { realtimeSearchController } from "../controllers/v2/f-search";
import { concurrencyQueueBackfillController } from "../controllers/v0/admin/concurrency-queue-backfill";
import { integCreateUserController } from "../controllers/v0/admin/create-user";
import { integValidateApiKeyController } from "../controllers/v0/admin/validate-api-key";
+import { crawlMonitorController } from "../controllers/v0/admin/crawl-monitor";
+import { RateLimiterMode } from "../types";
export const adminRouter = express.Router();
@@ -71,6 +73,13 @@ adminRouter.post(
wrap(concurrencyQueueBackfillController),
);
+adminRouter.post(
+ `/admin/${config.BULL_AUTH_KEY}/crawl-monitor`,
+ authMiddleware(RateLimiterMode.Crawl),
+ checkCreditsMiddleware(2),
+ wrap(crawlMonitorController),
+);
+
adminRouter.post(
`/admin/integration/create-user`,
wrap(integCreateUserController),