Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix batch sizes

Gergő Móriczbb2cc103a6c6
diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.ts
index a35be9908..9d5d56542 100644
--- a/apps/api/src/services/index.ts
+++ b/apps/api/src/services/index.ts
@@ -208,7 +208,7 @@ export function generateDomainSplits(hostname: string): string[] {
}
const INDEX_INSERT_QUEUE_KEY = "index-insert-queue";
-const INDEX_INSERT_BATCH_SIZE = 1000;
+const INDEX_INSERT_BATCH_SIZE = 100;
export async function addIndexInsertJob(data: any) {
await redisEvictConnection.rpush(INDEX_INSERT_QUEUE_KEY, JSON.stringify(data));
@@ -238,7 +238,7 @@ export async function getIndexInsertQueueLength(): Promise<number> {
}
const INDEX_RF_INSERT_QUEUE_KEY = "index-rf-insert-queue";
-const INDEX_RF_INSERT_BATCH_SIZE = 1000;
+const INDEX_RF_INSERT_BATCH_SIZE = 100;
export async function addIndexRFInsertJob(data: any) {
await redisEvictConnection.rpush(INDEX_RF_INSERT_QUEUE_KEY, JSON.stringify(data));
diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.ts
index 737fd0999..579657ded 100644
--- a/apps/api/src/services/indexing/index-worker.ts
+++ b/apps/api/src/services/indexing/index-worker.ts
@@ -191,7 +191,7 @@ const workerFun = async (queue: Queue, jobProcessor: (token: string, job: Job) =
process.exit(0);
};
-const INDEX_INSERT_INTERVAL = 15000;
+const INDEX_INSERT_INTERVAL = 3000;
const WEBHOOK_INSERT_INTERVAL = 15000;
// Start the workers