Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api/admin/crawl-monitor): add endpoint for monitoring crawl system (#2518)
Gergő Móricza444c84114b9
diff --git a/apps/api/src/controllers/v0/admin/crawl-monitor.ts b/apps/api/src/controllers/v0/admin/crawl-monitor.tsnew file mode 100644index 000000000..7a6f8d5bf--- /dev/null+++ b/apps/api/src/controllers/v0/admin/crawl-monitor.ts@@ -0,0 +1,127 @@+import { Response } from "express";+import { logger as _logger } from "../../../lib/logger";+import {+ crawlerOptions,+ RequestWithAuth,+ scrapeOptions,+ toV0CrawlerOptions,+} from "../../v2/types";+import { v7 as uuidv7 } from "uuid";+import { logRequest } from "../../../services/logging/log_job";+import {+ crawlToCrawler,+ markCrawlActive,+ saveCrawl,+ StoredCrawl,+} from "../../../lib/crawl-redis";+import { config } from "../../../config";+import { crawlGroup } from "../../../services/worker/nuq";+import { _addScrapeJobToBullMQ } from "../../../services/queue-jobs";++type ResponseType = {+ ok: boolean;+ crawlId: string;+};++export async function crawlMonitorController(+ req: RequestWithAuth<{}, undefined, ResponseType>,+ res: Response<ResponseType>,+) {+ const id = uuidv7();++ const logger = _logger.child({+ module: "crawl-monitor",+ method: "crawlMonitorController",+ crawlId: id,+ });++ logger.debug("Crawl monitor " + id + " starting", {+ account: req.account,+ });++ await logRequest({+ id,+ kind: "crawl",+ api_version: "v2",+ team_id: req.auth.team_id,+ origin: "api",+ integration: null,+ target_hint: "https://firecrawl.dev",+ zeroDataRetention: false,+ });++ const sc: StoredCrawl = {+ originUrl: "https://firecrawl.dev",+ crawlerOptions: toV0CrawlerOptions(crawlerOptions.parse({ limit: 2 })),+ scrapeOptions: scrapeOptions.parse({}),+ internalOptions: {+ disableSmartWaitCache: true,+ teamId: req.auth.team_id,+ saveScrapeResultToGCS: config.GCS_FIRE_ENGINE_BUCKET_NAME ? true : false,+ zeroDataRetention: false,+ },+ team_id: req.auth.team_id,+ createdAt: Date.now(),+ maxConcurrency: undefined,+ zeroDataRetention: false,+ };++ const crawler = crawlToCrawler(id, sc, req.acuc?.flags ?? null);++ try {+ sc.robots = await crawler.getRobotsTxt(false);+ } catch (e) {+ logger.debug("Failed to get robots.txt (this is probably fine!)", {+ error: e,+ });+ }++ await crawlGroup.addGroup(+ id,+ sc.team_id,+ (req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,+ );++ await saveCrawl(id, sc);++ await markCrawlActive(id);++ await _addScrapeJobToBullMQ(+ {+ url: "https://firecrawl.dev",+ mode: "kickoff" as const,+ team_id: req.auth.team_id,+ crawlerOptions: crawlerOptions.parse({ limit: 2 }),+ scrapeOptions: sc.scrapeOptions,+ internalOptions: sc.internalOptions,+ origin: "api",+ integration: null,+ crawl_id: id,+ webhook: undefined,+ v1: true,+ zeroDataRetention: false,+ apiKeyId: req.acuc?.api_key_id ?? null,+ },+ uuidv7(),+ );++ const startTime = Date.now();++ while (Date.now() - startTime < 60000) {+ const group = await crawlGroup.getGroup(id);+ if (group?.status === "completed") {+ logger.debug("Crawl completed");+ return res.status(200).json({+ ok: true,+ crawlId: id,+ });+ }+ await new Promise(resolve => setTimeout(resolve, 1000));+ }++ logger.warn("Crawl timed out");+ return res.status(500).json({+ ok: false,+ crawlId: id,+ });+}diff --git a/apps/api/src/controllers/v2/types.ts b/apps/api/src/controllers/v2/types.tsindex c5de2d7c5..e3df36cee 100644--- a/apps/api/src/controllers/v2/types.ts+++ b/apps/api/src/controllers/v2/types.ts@@ -794,7 +794,7 @@ export type BatchScrapeRequestInput = Omit<zeroDataRetention?: boolean;};-const crawlerOptions = z.strictObject({+export const crawlerOptions = z.strictObject({includePaths: z.string().array().prefault([]),excludePaths: z.string().array().prefault([]),maxDiscoveryDepth: z.number().optional(),diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.tsindex cc6839595..85ef71bb9 100644--- a/apps/api/src/routes/admin.ts+++ b/apps/api/src/routes/admin.ts@@ -1,7 +1,7 @@import express from "express";import { config } from "../config";import { redisHealthController } from "../controllers/v0/admin/redis-health";-import { wrap } from "./shared";+import { authMiddleware, checkCreditsMiddleware, wrap } from "./shared";import { acucCacheClearController } from "../controllers/v0/admin/acuc-cache-clear";import { checkFireEngine } from "../controllers/v0/admin/check-fire-engine";import { cclogController } from "../controllers/v0/admin/cclog";@@ -16,6 +16,8 @@ import { realtimeSearchController } from "../controllers/v2/f-search";import { concurrencyQueueBackfillController } from "../controllers/v0/admin/concurrency-queue-backfill";import { integCreateUserController } from "../controllers/v0/admin/create-user";import { integValidateApiKeyController } from "../controllers/v0/admin/validate-api-key";+import { crawlMonitorController } from "../controllers/v0/admin/crawl-monitor";+import { RateLimiterMode } from "../types";export const adminRouter = express.Router();@@ -71,6 +73,13 @@ adminRouter.post(wrap(concurrencyQueueBackfillController),);+adminRouter.post(+ `/admin/${config.BULL_AUTH_KEY}/crawl-monitor`,+ authMiddleware(RateLimiterMode.Crawl),+ checkCreditsMiddleware(2),+ wrap(crawlMonitorController),+);+adminRouter.post(`/admin/integration/create-user`,wrap(integCreateUserController),