Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix batch sizes
Gergő Móriczbb2cc103a6c6
diff --git a/apps/api/src/services/index.ts b/apps/api/src/services/index.tsindex a35be9908..9d5d56542 100644--- a/apps/api/src/services/index.ts+++ b/apps/api/src/services/index.ts@@ -208,7 +208,7 @@ export function generateDomainSplits(hostname: string): string[] {}const INDEX_INSERT_QUEUE_KEY = "index-insert-queue";-const INDEX_INSERT_BATCH_SIZE = 1000;+const INDEX_INSERT_BATCH_SIZE = 100;export async function addIndexInsertJob(data: any) {await redisEvictConnection.rpush(INDEX_INSERT_QUEUE_KEY, JSON.stringify(data));@@ -238,7 +238,7 @@ export async function getIndexInsertQueueLength(): Promise<number> {}const INDEX_RF_INSERT_QUEUE_KEY = "index-rf-insert-queue";-const INDEX_RF_INSERT_BATCH_SIZE = 1000;+const INDEX_RF_INSERT_BATCH_SIZE = 100;export async function addIndexRFInsertJob(data: any) {await redisEvictConnection.rpush(INDEX_RF_INSERT_QUEUE_KEY, JSON.stringify(data));diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex 737fd0999..579657ded 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -191,7 +191,7 @@ const workerFun = async (queue: Queue, jobProcessor: (token: string, job: Job) =process.exit(0);};-const INDEX_INSERT_INTERVAL = 15000;+const INDEX_INSERT_INTERVAL = 3000;const WEBHOOK_INSERT_INTERVAL = 15000;// Start the workers