Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api/crawl-status): rescue stuck and dead crawls
Gergő Móricz0594c2b3a664
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex 07b079847..8da033d48 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -180,24 +180,39 @@ export async function crawlStatusController(.eq("job_id", req.params.jobId).limit(1);+ let crawlAdded: Date;+ let crawlTeamId: string;+if (crawlJobError || !crawlJobs || crawlJobs.length === 0) {- logger.error("Error getting crawl job", { error: crawlJobError });- throw new Error("Error getting crawl job", { cause: crawlJobError });- }+ const { data: scrapeJobs, error: scrapeJobError } =+ await supabase_rr_service+ .from("firecrawl_jobs")+ .select("*")+ .eq("crawl_id", req.params.jobId)+ .order("date_added", { ascending: false })+ .limit(1);++ if (scrapeJobError || !scrapeJobs || scrapeJobs.length === 0) {+ return res.status(404).json({ success: false, error: "Job not found" });+ }- const crawlJob = crawlJobs[0];+ const scrapeJob = scrapeJobs[0];+ crawlAdded = new Date(scrapeJob.date_added); // use last scrape as date added for crawl+ crawlTeamId = scrapeJob.team_id;+ } else {+ const crawlJob = crawlJobs[0];+ crawlAdded = new Date(crawlJob.date_added);+ crawlTeamId = crawlJob.team_id;+ }- if (crawlJob.team_id !== req.auth.team_id) {+ if (crawlTeamId !== req.auth.team_id) {return res.status(404).json({ success: false, error: "Job not found" });}const crawlTtlHours = req.acuc?.flags?.crawlTtlHours ?? 24;const crawlTtlMs = crawlTtlHours * 60 * 60 * 1000;- if (- new Date().valueOf() - new Date(crawlJob.date_added).valueOf() >- crawlTtlMs- ) {+ if (new Date().valueOf() - crawlAdded.valueOf() > crawlTtlMs) {return res.status(404).json({ success: false, error: "Job expired" });}} else {diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.tsindex 098b9cbdc..03255e85e 100644--- a/apps/api/src/controllers/v2/crawl-status.ts+++ b/apps/api/src/controllers/v2/crawl-status.ts@@ -198,28 +198,39 @@ export async function crawlStatusController(.eq("job_id", req.params.jobId).limit(1);- if (crawlJobError) {- logger.error("Error getting crawl job", { error: crawlJobError });- throw new Error("Error getting crawl job", { cause: crawlJobError });- }+ let crawlAdded: Date;+ let crawlTeamId: string;++ if (crawlJobError || !crawlJobs || crawlJobs.length === 0) {+ const { data: scrapeJobs, error: scrapeJobError } =+ await supabase_rr_service+ .from("firecrawl_jobs")+ .select("*")+ .eq("crawl_id", req.params.jobId)+ .order("date_added", { ascending: false })+ .limit(1);++ if (scrapeJobError || !scrapeJobs || scrapeJobs.length === 0) {+ return res.status(404).json({ success: false, error: "Job not found" });+ }- if (!crawlJobs || crawlJobs.length === 0) {- return res.status(404).json({ success: false, error: "Job not found" });+ const scrapeJob = scrapeJobs[0];+ crawlAdded = new Date(scrapeJob.date_added); // use last scrape as date added for crawl+ crawlTeamId = scrapeJob.team_id;+ } else {+ const crawlJob = crawlJobs[0];+ crawlAdded = new Date(crawlJob.date_added);+ crawlTeamId = crawlJob.team_id;}- const crawlJob = crawlJobs[0];-- if (crawlJob.team_id !== req.auth.team_id) {+ if (crawlTeamId !== req.auth.team_id) {return res.status(404).json({ success: false, error: "Job not found" });}const crawlTtlHours = req.acuc?.flags?.crawlTtlHours ?? 24;const crawlTtlMs = crawlTtlHours * 60 * 60 * 1000;- if (- new Date().valueOf() - new Date(crawlJob.date_added).valueOf() >- crawlTtlMs- ) {+ if (new Date().valueOf() - crawlAdded.valueOf() > crawlTtlMs) {return res.status(404).json({ success: false, error: "Job expired" });}} else {