Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Update crawl-maps-index.ts
Nicolasbaa2f9476593
diff --git a/apps/api/src/services/indexing/crawl-maps-index.ts b/apps/api/src/services/indexing/crawl-maps-index.tsindex 4cccd763d..2687e8147 100644--- a/apps/api/src/services/indexing/crawl-maps-index.ts+++ b/apps/api/src/services/indexing/crawl-maps-index.ts@@ -8,8 +8,8 @@ import { redisConnection } from "../queue-service";const BATCH_KEY = "crawl_maps_batch";const BATCH_LOCK_KEY = "crawl_maps_batch_lock";-const BATCH_SIZE = 10;-const BATCH_TIMEOUT = 10000; // 10 seconds+const BATCH_SIZE = 20;+const BATCH_TIMEOUT = 20000; // 10 secondsconst LOCK_TIMEOUT = 30000; // 30 secondsinterface CrawlMapOperation {