Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): route v0 fdb crawl status reads
mogery70ba9e7e53bd
diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.tsindex f63525bf2..173675c2e 100644--- a/apps/api/src/controllers/v0/crawl-status.ts+++ b/apps/api/src/controllers/v0/crawl-status.ts@@ -4,14 +4,22 @@ import { authenticateUser } from "../auth";import { RateLimiterMode } from "../../../src/types";import { redisEvictConnection } from "../../../src/services/redis";import { logger } from "../../../src/lib/logger";-import { getCrawl, getCrawlJobs } from "../../../src/lib/crawl-redis";+import {+ getCrawl,+ getCrawlJobs,+ type StoredCrawl,+} from "../../../src/lib/crawl-redis";import { supabaseGetScrapesByRequestId } from "../../../src/lib/supabase-jobs";import * as Sentry from "@sentry/node";import { configDotenv } from "dotenv";import { toLegacyDocument } from "../v1/types";import type { DBScrape, PseudoJob } from "../v1/crawl-status";import { getJobFromGCS } from "../../lib/gcs-jobs";-import { scrapeQueue, NuQJob } from "../../services/worker/nuq-router";+import {+ scrapeQueue,+ NuQJob,+ crawlGroup,+} from "../../services/worker/nuq-router";import { includesFormat } from "../../lib/format-utils";import { getScrapeZDR } from "../../lib/zdr-helpers";import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";@@ -91,6 +99,77 @@ async function getJobs(return jobs;}+async function getFdbCrawlStatus(crawlId: string, sc: StoredCrawl) {+ const group = await crawlGroup.getGroup(crawlId);+ const numericStats = await scrapeQueue.getGroupNumericStats(+ crawlId,+ logger.child({ module: "v0-crawl-status", backend: "fdb" }),+ );++ const completed = numericStats.completed ?? 0;+ const total =+ completed ++ (numericStats.active ?? 0) ++ (numericStats.queued ?? 0) ++ (numericStats.backlog ?? 0);++ const doneJobs =+ completed > 0+ ? await scrapeQueue.getCrawlJobsForListing(+ crawlId,+ completed,+ 0,+ logger.child({ module: "v0-crawl-status", backend: "fdb" }),+ )+ : [];++ const data = doneJobs+ .filter(+ x => x.failedReason !== "Concurreny limit hit" && x.returnvalue !== null,+ )+ .map(x =>+ Array.isArray(x.returnvalue) ? x.returnvalue[0] : x.returnvalue,+ );++ const firstScrapeOptions =+ doneJobs.length > 0 && "scrapeOptions" in doneJobs[0].data+ ? doneJobs[0].data.scrapeOptions+ : undefined;++ if (+ firstScrapeOptions?.formats &&+ !includesFormat(firstScrapeOptions.formats, "rawHtml")+ ) {+ data.forEach(item => {+ if (item) {+ delete item.rawHtml;+ }+ });+ }++ const jobStatus = sc.cancelled+ ? "failed"+ : group?.status === "completed"+ ? "completed"+ : "active";++ return {+ status: jobStatus,+ current: completed,+ total,+ data:+ jobStatus === "completed"+ ? data.map(x => toLegacyDocument(x, sc.internalOptions))+ : null,+ partial_data:+ jobStatus === "completed"+ ? []+ : data+ .filter(x => x !== null)+ .map(x => toLegacyDocument(x, sc.internalOptions)),+ };+}+export async function crawlStatusController(req: Request, res: Response) {try {const auth = await authenticateUser(req, res, RateLimiterMode.CrawlStatus);@@ -135,6 +214,11 @@ export async function crawlStatusController(req: Request, res: Response) {if (sc.team_id !== team_id) {return res.status(403).json({ error: "Forbidden" });}++ if (sc.queueBackend === "fdb") {+ return res.json(await getFdbCrawlStatus(req.params.jobId, sc));+ }+let jobIDs = await getCrawlJobs(req.params.jobId);let jobs = await getJobs(req.params.jobId, jobIDs);let jobStatuses = jobs.map(x => x.status);