Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

Revert "Nick: increasing multiplexity"

Nicolas5991fe08ee5c
diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.ts
index c1f834ac3..b86a4ac7f 100644
--- a/apps/api/src/services/queue-service.ts
+++ b/apps/api/src/services/queue-service.ts
@@ -13,7 +13,7 @@ let billingQueue: Queue;
let precrawlQueue: Queue;
export const redisConnection = new IORedis(process.env.REDIS_URL!, {
- maxRetriesPerRequest: null
+ maxRetriesPerRequest: null,
});
redisConnection.on("reconnecting", () => logger.warn("Redis reconnecting"));
@@ -33,18 +33,6 @@ export const scrapeQueueNames = [
"{scrapeQueue1}",
"{scrapeQueue2}",
"{scrapeQueue3}",
- "{scrapeQueue4}",
- "{scrapeQueue5}",
- "{scrapeQueue6}",
- "{scrapeQueue7}",
- "{scrapeQueue8}",
- "{scrapeQueue9}",
- "{scrapeQueue10}",
- "{scrapeQueue11}",
- "{scrapeQueue12}",
- "{scrapeQueue13}",
- "{scrapeQueue14}",
- "{scrapeQueue15}",
];
export function uuidToQueueNo(id: string) {
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 656bae24d..20b42a524 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -335,10 +335,10 @@ const separateWorkerFun = (
): Worker => {
const worker = new Worker(queue.name, path, {
connection: redisConnection,
- lockDuration: 60 * 1000, // 60 seconds (increased from 30)
- stalledInterval: 45 * 1000, // 45 seconds (increased from 30)
+ lockDuration: 30 * 1000, // 30 seconds
+ stalledInterval: 30 * 1000, // 30 seconds
maxStalledCount: 10, // 10 times
- concurrency: Math.ceil(12 / scrapeQueueNames.length), // increased base from 6 to 12 for better throughput with 16 queues
+ concurrency: Math.ceil(6 / scrapeQueueNames.length), // from k8s setup
useWorkerThreads: true,
});
@@ -353,8 +353,8 @@ const workerFun = async (
const worker = new Worker(queue.name, null, {
connection: redisConnection,
- lockDuration: 60 * 1000, // 60 seconds (increased from 30)
- stalledInterval: 45 * 1000, // 45 seconds (increased from 30)
+ lockDuration: 30 * 1000, // 30 seconds
+ stalledInterval: 30 * 1000, // 30 seconds
maxStalledCount: 10, // 10 times
});