Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(precrawl): use NuQ crawlGroup

Gergő Móricz31611dbabd40
diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.ts
index 74b5ad358..b474a0e40 100644
--- a/apps/api/src/services/indexing/index-worker.ts
+++ b/apps/api/src/services/indexing/index-worker.ts
@@ -37,6 +37,7 @@ import { StoredCrawl, crawlToCrawler, saveCrawl } from "../../lib/crawl-redis";
import { addScrapeJob } from "../queue-jobs";
import { BullMQOtel } from "bullmq-otel";
import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";
+import { crawlGroup, scrapeQueue } from "../worker/nuq";
const workerLockDuration = Number(process.env.WORKER_LOCK_DURATION) || 60000;
const workerStalledCheckInterval =
@@ -490,6 +491,13 @@ const processPrecrawlJob = async (token: string, job: Job) => {
// });
// }
+ await crawlGroup.addGroup(crawlId, [
+ {
+ queue: scrapeQueue,
+ maxConcurrency: sc.maxConcurrency ?? undefined,
+ },
+ ]);
+
await saveCrawl(crawlId, sc);
await addScrapeJob(