Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(queue-worker): clean up stalled jobs to not get crawls stuck (#1838)
Gergő Móricz30347c84ec3c
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 73f334e76..15045a86a 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -13,7 +13,7 @@ import {import { startWebScraperPipeline } from "../main/runWebScraper";import { callWebhook } from "./webhook";import { logJob } from "./logging/log_job";-import { Job, Queue } from "bullmq";+import { Job, Queue, QueueEvents } from "bullmq";import { logger as _logger } from "../lib/logger";import { Worker } from "bullmq";import systemMonitor from "./system-monitor";@@ -1629,6 +1629,33 @@ app.listen(workerPort, () => {});(async () => {+ async function failedListener(args: { jobId: string; failedReason: string; prev?: string | undefined; }) {+ if (args.failedReason === "job stalled more than allowable limit") {+ let logger = _logger.child({ jobId: args.jobId, scrapeId: args.jobId, module: "queue-worker", method: "failedListener" });+ const job = await getScrapeQueue().getJob(args.jobId);+ if (job && job.data.crawl_id) {+ logger = logger.child({ crawlId: job.data.crawl_id });+ logger.warn("Job stalled more than allowable limit");++ const sc = (await getCrawl(job.data.crawl_id)) as StoredCrawl;++ logger.debug("Declaring job as done...");+ await addCrawlJobDone(job.data.crawl_id, job.id, false, logger);+ await redisEvictConnection.srem(+ "crawl:" + job.data.crawl_id + ":visited_unique",+ normalizeURL(job.data.url, sc),+ );++ await finishCrawlIfNeeded(job, sc);+ } else {+ logger.warn("Job stalled more than allowable limit");+ }+ }+ }++ const scrapeQueueEvents = new QueueEvents(getScrapeQueue().name, { connection: redisConnection });+ scrapeQueueEvents.on("failed", failedListener);+await Promise.all([workerFun(getScrapeQueue(), processJobInternal),workerFun(getExtractQueue(), processExtractJobInternal),@@ -1642,6 +1669,7 @@ app.listen(workerPort, () => {await new Promise((resolve) => setTimeout(resolve, 500));}+ await scrapeQueueEvents.close();console.log("All jobs finished. Worker out!");process.exit(0);-})();+})();\ No newline at end of file