Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Nick: increasing multiplexity

Nicolasbda0105402aa
diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.ts
index b86a4ac7f..c1f834ac3 100644
--- a/apps/api/src/services/queue-service.ts
+++ b/apps/api/src/services/queue-service.ts
@@ -13,7 +13,7 @@ let billingQueue: Queue;
let precrawlQueue: Queue;
export const redisConnection = new IORedis(process.env.REDIS_URL!, {
- maxRetriesPerRequest: null,
+ maxRetriesPerRequest: null
});
redisConnection.on("reconnecting", () => logger.warn("Redis reconnecting"));
@@ -33,6 +33,18 @@ export const scrapeQueueNames = [
"{scrapeQueue1}",
"{scrapeQueue2}",
"{scrapeQueue3}",
+ "{scrapeQueue4}",
+ "{scrapeQueue5}",
+ "{scrapeQueue6}",
+ "{scrapeQueue7}",
+ "{scrapeQueue8}",
+ "{scrapeQueue9}",
+ "{scrapeQueue10}",
+ "{scrapeQueue11}",
+ "{scrapeQueue12}",
+ "{scrapeQueue13}",
+ "{scrapeQueue14}",
+ "{scrapeQueue15}",
];
export function uuidToQueueNo(id: string) {
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 20b42a524..656bae24d 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -335,10 +335,10 @@ const separateWorkerFun = (
): Worker => {
const worker = new Worker(queue.name, path, {
connection: redisConnection,
- lockDuration: 30 * 1000, // 30 seconds
- stalledInterval: 30 * 1000, // 30 seconds
+ lockDuration: 60 * 1000, // 60 seconds (increased from 30)
+ stalledInterval: 45 * 1000, // 45 seconds (increased from 30)
maxStalledCount: 10, // 10 times
- concurrency: Math.ceil(6 / scrapeQueueNames.length), // from k8s setup
+ concurrency: Math.ceil(12 / scrapeQueueNames.length), // increased base from 6 to 12 for better throughput with 16 queues
useWorkerThreads: true,
});
@@ -353,8 +353,8 @@ const workerFun = async (
const worker = new Worker(queue.name, null, {
connection: redisConnection,
- lockDuration: 30 * 1000, // 30 seconds
- stalledInterval: 30 * 1000, // 30 seconds
+ lockDuration: 60 * 1000, // 60 seconds (increased from 30)
+ stalledInterval: 45 * 1000, // 45 seconds (increased from 30)
maxStalledCount: 10, // 10 times
});