Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(crawl-status): keep working even if jobs are ejected from bullmq (#1799)

Gergő Móricz69b32ebfe6a8
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts
index 809369cb7..3a655cb24 100644
--- a/apps/api/src/controllers/v1/crawl-status.ts
+++ b/apps/api/src/controllers/v1/crawl-status.ts
@@ -162,6 +162,42 @@ export async function crawlStatusController(
),
);
+ if (jobStatuses.filter((x) => x[1] === "unknown").length > 0 && process.env.USE_DB_AUTHENTICATION === "true") {
+ for (let rangeStart = 0;; rangeStart += 1000) {
+ const rangeEnd = Math.min(rangeStart + 1000, jobIDs.length);
+
+ const currentJobs = await supabase_service
+ .from("firecrawl_jobs")
+ .select("success, job_id")
+ .eq("crawl_id", req.params.jobId)
+ .order("date_added", { ascending: true })
+ .range(rangeStart, rangeEnd)
+ .throwOnError();
+
+ const rangeLength = rangeEnd - rangeStart;
+
+ if (currentJobs.error) {
+ logger.error("Error getting current jobs", { error: currentJobs.error });
+ continue;
+ }
+
+ const data = currentJobs.data ?? [];
+
+ for (const job of data) {
+ let index = jobStatuses.findIndex((x) => x[0] === job.job_id);
+ if (index !== -1) {
+ jobStatuses[index] = [job.job_id, job.success ? "completed" : "failed"];
+ } else {
+ jobStatuses.push([job.job_id, job.success ? "completed" : "failed"]);
+ }
+ }
+
+ if (data.length < rangeLength) {
+ break;
+ }
+ }
+ }
+
const throttledJobsSet = new Set(await getConcurrencyLimitedJobs(req.auth.team_id));
const activeJobsSet = new Set(await getCrawlConcurrencyLimitActiveJobs(req.params.jobId));
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index f59640c72..ce4b09475 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -769,11 +769,13 @@ const workerFun = async (
}
async function afterJobDone(job: Job<any, any, string>) {
- if (job.id) {
- runningJobs.delete(job.id);
+ try {
+ await concurrentJobDone(job);
+ } finally {
+ if (job.id) {
+ runningJobs.delete(job.id);
+ }
}
-
- await concurrentJobDone(job);
}
if (job.data && job.data.sentry && Sentry.isInitialized()) {