Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(precrawl): use NuQ crawlGroup
Gergő Móricz31611dbabd40
diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex 74b5ad358..b474a0e40 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -37,6 +37,7 @@ import { StoredCrawl, crawlToCrawler, saveCrawl } from "../../lib/crawl-redis";import { addScrapeJob } from "../queue-jobs";import { BullMQOtel } from "bullmq-otel";import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";+import { crawlGroup, scrapeQueue } from "../worker/nuq";const workerLockDuration = Number(process.env.WORKER_LOCK_DURATION) || 60000;const workerStalledCheckInterval =@@ -490,6 +491,13 @@ const processPrecrawlJob = async (token: string, job: Job) => {// });// }+ await crawlGroup.addGroup(crawlId, [+ {+ queue: scrapeQueue,+ maxConcurrency: sc.maxConcurrency ?? undefined,+ },+ ]);+await saveCrawl(crawlId, sc);await addScrapeJob(