Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(concurrency-limit): allow for big crawls
Gergő Móricz1e7d42e8afce
diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.tsindex 7bab02f90..8fa87cb27 100644--- a/apps/api/src/lib/concurrency-limit.ts+++ b/apps/api/src/lib/concurrency-limit.ts@@ -10,7 +10,7 @@ const constructQueueKey = (team_id: string) =>export function calculateJobTimeToRun(job: ConcurrencyLimitedJob): number {- let jobTimeToRun = 2 * 60 * 1000;+ let jobTimeToRun = 86400000; // 24h (crawl)if (job.data.scrapeOptions) {if (job.data.scrapeOptions.timeout) {