Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Nick: increasing multiplexity
Nicolasbda0105402aa
diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.tsindex b86a4ac7f..c1f834ac3 100644--- a/apps/api/src/services/queue-service.ts+++ b/apps/api/src/services/queue-service.ts@@ -13,7 +13,7 @@ let billingQueue: Queue;let precrawlQueue: Queue;export const redisConnection = new IORedis(process.env.REDIS_URL!, {- maxRetriesPerRequest: null,+ maxRetriesPerRequest: null});redisConnection.on("reconnecting", () => logger.warn("Redis reconnecting"));@@ -33,6 +33,18 @@ export const scrapeQueueNames = ["{scrapeQueue1}","{scrapeQueue2}","{scrapeQueue3}",+ "{scrapeQueue4}",+ "{scrapeQueue5}",+ "{scrapeQueue6}",+ "{scrapeQueue7}",+ "{scrapeQueue8}",+ "{scrapeQueue9}",+ "{scrapeQueue10}",+ "{scrapeQueue11}",+ "{scrapeQueue12}",+ "{scrapeQueue13}",+ "{scrapeQueue14}",+ "{scrapeQueue15}",];export function uuidToQueueNo(id: string) {diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 20b42a524..656bae24d 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -335,10 +335,10 @@ const separateWorkerFun = (): Worker => {const worker = new Worker(queue.name, path, {connection: redisConnection,- lockDuration: 30 * 1000, // 30 seconds- stalledInterval: 30 * 1000, // 30 seconds+ lockDuration: 60 * 1000, // 60 seconds (increased from 30)+ stalledInterval: 45 * 1000, // 45 seconds (increased from 30)maxStalledCount: 10, // 10 times- concurrency: Math.ceil(6 / scrapeQueueNames.length), // from k8s setup+ concurrency: Math.ceil(12 / scrapeQueueNames.length), // increased base from 6 to 12 for better throughput with 16 queuesuseWorkerThreads: true,});@@ -353,8 +353,8 @@ const workerFun = async (const worker = new Worker(queue.name, null, {connection: redisConnection,- lockDuration: 30 * 1000, // 30 seconds- stalledInterval: 30 * 1000, // 30 seconds+ lockDuration: 60 * 1000, // 60 seconds (increased from 30)+ stalledInterval: 45 * 1000, // 45 seconds (increased from 30)maxStalledCount: 10, // 10 times});