Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Revert "Nick: increasing multiplexity"
Nicolaseb08174cf91a
diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.tsindex c1f834ac3..b86a4ac7f 100644--- a/apps/api/src/services/queue-service.ts+++ b/apps/api/src/services/queue-service.ts@@ -13,7 +13,7 @@ let billingQueue: Queue;let precrawlQueue: Queue;export const redisConnection = new IORedis(process.env.REDIS_URL!, {- maxRetriesPerRequest: null+ maxRetriesPerRequest: null,});redisConnection.on("reconnecting", () => logger.warn("Redis reconnecting"));@@ -33,18 +33,6 @@ export const scrapeQueueNames = ["{scrapeQueue1}","{scrapeQueue2}","{scrapeQueue3}",- "{scrapeQueue4}",- "{scrapeQueue5}",- "{scrapeQueue6}",- "{scrapeQueue7}",- "{scrapeQueue8}",- "{scrapeQueue9}",- "{scrapeQueue10}",- "{scrapeQueue11}",- "{scrapeQueue12}",- "{scrapeQueue13}",- "{scrapeQueue14}",- "{scrapeQueue15}",];export function uuidToQueueNo(id: string) {diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 656bae24d..20b42a524 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -335,10 +335,10 @@ const separateWorkerFun = (): Worker => {const worker = new Worker(queue.name, path, {connection: redisConnection,- lockDuration: 60 * 1000, // 60 seconds (increased from 30)- stalledInterval: 45 * 1000, // 45 seconds (increased from 30)+ lockDuration: 30 * 1000, // 30 seconds+ stalledInterval: 30 * 1000, // 30 secondsmaxStalledCount: 10, // 10 times- concurrency: Math.ceil(12 / scrapeQueueNames.length), // increased base from 6 to 12 for better throughput with 16 queues+ concurrency: Math.ceil(6 / scrapeQueueNames.length), // from k8s setupuseWorkerThreads: true,});@@ -353,8 +353,8 @@ const workerFun = async (const worker = new Worker(queue.name, null, {connection: redisConnection,- lockDuration: 60 * 1000, // 60 seconds (increased from 30)- stalledInterval: 45 * 1000, // 45 seconds (increased from 30)+ lockDuration: 30 * 1000, // 30 seconds+ stalledInterval: 30 * 1000, // 30 secondsmaxStalledCount: 10, // 10 times});