Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(crawl-status): refactor to work after a redis flush (#1664)
Gergő Móriczebc1de9d606b
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex 4d9969357..f8c5fd874 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -135,68 +135,172 @@ export async function crawlStatusController(res: Response<CrawlStatusResponse>,isBatch = false,) {+ const start =+ typeof req.query.skip === "string" ? parseInt(req.query.skip, 10) : 0;+ const end =+ typeof req.query.limit === "string"+ ? start + parseInt(req.query.limit, 10) - 1+ : undefined;+const sc = await getCrawl(req.params.jobId);- if (!sc) {- return res.status(404).json({ success: false, error: "Job not found" });- }- if (sc.team_id !== req.auth.team_id) {- return res.status(403).json({ success: false, error: "Forbidden" });- }+ let status: Exclude<CrawlStatusResponse, ErrorResponse>["status"];+ let doneJobsLength: number;+ let doneJobsOrder: string[];+ let totalCount: number;+ let creditsUsed: number;- const start =- typeof req.query.skip === "string" ? parseInt(req.query.skip, 10) : 0;- const end =- typeof req.query.limit === "string"- ? start + parseInt(req.query.limit, 10) - 1- : undefined;-- let jobIDs = await getCrawlJobs(req.params.jobId);- let jobStatuses = await Promise.all(- jobIDs.map(- async (x) => [x, await getScrapeQueue().getJobState(x)] as const,- ),- );+ if (sc) {+ if (sc.team_id !== req.auth.team_id) {+ return res.status(403).json({ success: false, error: "Forbidden" });+ }++ let jobIDs = await getCrawlJobs(req.params.jobId);+ let jobStatuses = await Promise.all(+ jobIDs.map(+ async (x) => [x, await getScrapeQueue().getJobState(x)] as const,+ ),+ );++ const teamThrottledJobsSet = await getConcurrencyLimitedJobs(req.auth.team_id);+ const crawlThrottledJobsSet = sc.crawlerOptions?.delay ? await getCrawlConcurrencyLimitedJobs(req.params.jobId) : new Set();+ const throttledJobsSet = new Set([...teamThrottledJobsSet, ...crawlThrottledJobsSet]);++ const validJobStatuses: [string, JobState | "unknown"][] = [];+ const validJobIDs: string[] = [];++ for (const [id, status] of jobStatuses) {+ if (throttledJobsSet.has(id)) {+ validJobStatuses.push([id, "prioritized"]);+ validJobIDs.push(id);+ } else if (+ status !== "failed" &&+ status !== "unknown"+ ) {+ validJobStatuses.push([id, status]);+ validJobIDs.push(id);+ }+ }++ status =+ sc.cancelled+ ? "cancelled"+ : validJobStatuses.every((x) => x[1] === "completed") &&+ (sc.crawlerOptions+ ? await isCrawlKickoffFinished(req.params.jobId)+ : true)+ ? "completed"+ : "scraping";++ // Use validJobIDs instead of jobIDs for further processing+ jobIDs = validJobIDs;++ doneJobsLength = await getDoneJobsOrderedLength(req.params.jobId);+ doneJobsOrder = await getDoneJobsOrdered(+ req.params.jobId,+ start,+ end ?? -1,+ );++ totalCount = jobIDs.length;++ if (totalCount === 0 && process.env.USE_DB_AUTHENTICATION === "true") {+ const x = await supabase_rr_service+ .from('firecrawl_jobs')+ .select('*', { count: 'exact', head: true })+ .eq("crawl_id", req.params.jobId)+ .eq("success", true)++ totalCount = x.count ?? 0;+ }- const teamThrottledJobsSet = await getConcurrencyLimitedJobs(req.auth.team_id);- const crawlThrottledJobsSet = sc.crawlerOptions?.delay ? await getCrawlConcurrencyLimitedJobs(req.params.jobId) : new Set();- const throttledJobsSet = new Set([...teamThrottledJobsSet, ...crawlThrottledJobsSet]);+ creditsUsed = totalCount * (+ sc.scrapeOptions?.extract+ ? 5+ : 1+ )+ } else if (process.env.USE_DB_AUTHENTICATION === "true") {+ const scrapeJobCount = await supabase_rr_service+ .from("firecrawl_jobs")+ .select("*", { count: "exact", head: true })+ .eq("crawl_id", req.params.jobId)+ .eq("team_id", req.auth.team_id)+ .eq("success", true)+ .throwOnError();++ const crawlJobQuery = await supabase_rr_service+ .from("firecrawl_jobs")+ .select("*")+ .eq("job_id", req.params.jobId)+ .limit(1)+ .throwOnError();++ if (!crawlJobQuery.data || crawlJobQuery.data.length === 0) {+ if (scrapeJobCount.count === 0) {+ return res.status(404).json({ success: false, error: "Job not found" });+ } else {+ status = "completed"; // fake completed to cut the losses+ }+ } else {+ status = crawlJobQuery.data[0].success ? "completed" : "failed";+ }++ const crawlJob = crawlJobQuery.data?.[0];- const validJobStatuses: [string, JobState | "unknown"][] = [];- const validJobIDs: string[] = [];+ if (crawlJob && crawlJob.team_id !== req.auth.team_id) {+ return res.status(403).json({ success: false, error: "Forbidden" });+ }- for (const [id, status] of jobStatuses) {- if (throttledJobsSet.has(id)) {- validJobStatuses.push([id, "prioritized"]);- validJobIDs.push(id);- } else if (- status !== "failed" &&- status !== "unknown"+ if (+ crawlJob+ && new Date().valueOf() - new Date(crawlJob.date_added).valueOf() > 24 * 60 * 60 * 1000) {- validJobStatuses.push([id, status]);- validJobIDs.push(id);+ return res.status(404).json({ success: false, error: "Job expired" });}- }- const status: Exclude<CrawlStatusResponse, ErrorResponse>["status"] =- sc.cancelled- ? "cancelled"- : validJobStatuses.every((x) => x[1] === "completed") &&- (sc.crawlerOptions- ? await isCrawlKickoffFinished(req.params.jobId)- : true)- ? "completed"- : "scraping";-- // Use validJobIDs instead of jobIDs for further processing- jobIDs = validJobIDs;-- const doneJobsLength = await getDoneJobsOrderedLength(req.params.jobId);- const doneJobsOrder = await getDoneJobsOrdered(- req.params.jobId,- start,- end ?? -1,- );+ doneJobsLength = scrapeJobCount.count!;+ doneJobsOrder = [];++ const step = 1000;+ let i = 0;+ while (true) {+ const rangeStart = start + (i * step);+ let rangeEnd = start + ((i + 1) * step);+ if (end !== undefined) {+ rangeEnd = Math.min(end, rangeEnd);+ }++ const currentJobs = await supabase_rr_service+ .from("firecrawl_jobs")+ .select("job_id")+ .eq("crawl_id", req.params.jobId)+ .eq("team_id", req.acuc.team_id)+ .order("date_added", { ascending: true })+ .range(rangeStart, rangeEnd)+ .throwOnError();++ const rangeLength = rangeEnd - rangeStart;++ const data = currentJobs.data ?? [];++ doneJobsOrder.push(...data.map(x => x.job_id));++ if (data.length < rangeLength) {+ break;+ }++ if (rangeEnd === end) {+ break;+ }++ i+++ }++ totalCount = scrapeJobCount.count ?? 0;+ creditsUsed = crawlJob?.credits_billed ?? totalCount;+ } else {+ return res.status(404).json({ success: false, error: "Job not found" });+ }let doneJobs: PseudoJob<any>[] = [];@@ -265,28 +369,12 @@ export async function crawlStatusController(nextURL.searchParams.set("limit", req.query.limit);}- let totalCount = jobIDs.length;-- if (totalCount === 0 && process.env.USE_DB_AUTHENTICATION === "true") {- const x = await supabase_rr_service- .from('firecrawl_jobs')- .select('*', { count: 'exact', head: true })- .eq("crawl_id", req.params.jobId)- .eq("success", true)-- totalCount = x.count ?? 0;- }-res.status(200).json({success: true,status,completed: doneJobsLength,total: totalCount,- creditsUsed: totalCount * (- sc.scrapeOptions?.extract- ? 5- : 1- ),+ creditsUsed,expiresAt: (await getCrawlExpiry(req.params.jobId)).toISOString(),next:status !== "scraping" && start + data.length === doneJobsLength // if there's not gonna be any documents after this