Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(crawl-status): keep working even if jobs are ejected from bullmq (#1799)
Gergő Móricz69b32ebfe6a8
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex 809369cb7..3a655cb24 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -162,6 +162,42 @@ export async function crawlStatusController(),);+ if (jobStatuses.filter((x) => x[1] === "unknown").length > 0 && process.env.USE_DB_AUTHENTICATION === "true") {+ for (let rangeStart = 0;; rangeStart += 1000) {+ const rangeEnd = Math.min(rangeStart + 1000, jobIDs.length);++ const currentJobs = await supabase_service+ .from("firecrawl_jobs")+ .select("success, job_id")+ .eq("crawl_id", req.params.jobId)+ .order("date_added", { ascending: true })+ .range(rangeStart, rangeEnd)+ .throwOnError();++ const rangeLength = rangeEnd - rangeStart;++ if (currentJobs.error) {+ logger.error("Error getting current jobs", { error: currentJobs.error });+ continue;+ }++ const data = currentJobs.data ?? [];++ for (const job of data) {+ let index = jobStatuses.findIndex((x) => x[0] === job.job_id);+ if (index !== -1) {+ jobStatuses[index] = [job.job_id, job.success ? "completed" : "failed"];+ } else {+ jobStatuses.push([job.job_id, job.success ? "completed" : "failed"]);+ }+ }++ if (data.length < rangeLength) {+ break;+ }+ }+ }+const throttledJobsSet = new Set(await getConcurrencyLimitedJobs(req.auth.team_id));const activeJobsSet = new Set(await getCrawlConcurrencyLimitActiveJobs(req.params.jobId));diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex f59640c72..ce4b09475 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -769,11 +769,13 @@ const workerFun = async (}async function afterJobDone(job: Job<any, any, string>) {- if (job.id) {- runningJobs.delete(job.id);+ try {+ await concurrentJobDone(job);+ } finally {+ if (job.id) {+ runningJobs.delete(job.id);+ }}-- await concurrentJobDone(job);}if (job.data && job.data.sentry && Sentry.isInitialized()) {