Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(queue-worker): clean up stalled jobs to not get crawls stuck (#1838)

Gergő Móricz30347c84ec3c
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 73f334e76..15045a86a 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -13,7 +13,7 @@ import {
import { startWebScraperPipeline } from "../main/runWebScraper";
import { callWebhook } from "./webhook";
import { logJob } from "./logging/log_job";
-import { Job, Queue } from "bullmq";
+import { Job, Queue, QueueEvents } from "bullmq";
import { logger as _logger } from "../lib/logger";
import { Worker } from "bullmq";
import systemMonitor from "./system-monitor";
@@ -1629,6 +1629,33 @@ app.listen(workerPort, () => {
});
(async () => {
+ async function failedListener(args: { jobId: string; failedReason: string; prev?: string | undefined; }) {
+ if (args.failedReason === "job stalled more than allowable limit") {
+ let logger = _logger.child({ jobId: args.jobId, scrapeId: args.jobId, module: "queue-worker", method: "failedListener" });
+ const job = await getScrapeQueue().getJob(args.jobId);
+ if (job && job.data.crawl_id) {
+ logger = logger.child({ crawlId: job.data.crawl_id });
+ logger.warn("Job stalled more than allowable limit");
+
+ const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;
+
+ logger.debug("Declaring job as done...");
+ await addCrawlJobDone(job.data.crawl_id, job.id, false, logger);
+ await redisEvictConnection.srem(
+ "crawl:" + job.data.crawl_id + ":visited_unique",
+ normalizeURL(job.data.url, sc),
+ );
+
+ await finishCrawlIfNeeded(job, sc);
+ } else {
+ logger.warn("Job stalled more than allowable limit");
+ }
+ }
+ }
+
+ const scrapeQueueEvents = new QueueEvents(getScrapeQueue().name, { connection: redisConnection });
+ scrapeQueueEvents.on("failed", failedListener);
+
await Promise.all([
workerFun(getScrapeQueue(), processJobInternal),
workerFun(getExtractQueue(), processExtractJobInternal),
@@ -1642,6 +1669,7 @@ app.listen(workerPort, () => {
await new Promise((resolve) => setTimeout(resolve, 500));
}
+ await scrapeQueueEvents.close();
console.log("All jobs finished. Worker out!");
process.exit(0);
-})();
+})();
\ No newline at end of file