Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(index-worker); precrawl stalling
Gergő Móricze2b27dc7cf3f
diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex a041da939..bbefa3240 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -7,6 +7,7 @@ import {redisConnection,getBillingQueue,getPrecrawlQueue,+ precrawlQueueName,} from "../queue-service";import { processBillingBatch, queueBillingOperation, startBillingBatchProcessing } from "../billing/batch_billing";import systemMonitor from "../system-monitor";@@ -219,7 +220,7 @@ const workerFun = async (queue: Queue, jobProcessor: (token: string, job: Job) =connection: redisConnection,lockDuration: workerLockDuration,stalledInterval: workerStalledCheckInterval,- maxStalledCount: 10,+ maxStalledCount: queue.name === precrawlQueueName ? 0 : 10,});worker.startStalledCheckTimer();