Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(concurrency-limit): allow for big crawls

Gergő Móricz1e7d42e8afce
diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.ts
index 7bab02f90..8fa87cb27 100644
--- a/apps/api/src/lib/concurrency-limit.ts
+++ b/apps/api/src/lib/concurrency-limit.ts
@@ -10,7 +10,7 @@ const constructQueueKey = (team_id: string) =>
export function calculateJobTimeToRun(
job: ConcurrencyLimitedJob
): number {
- let jobTimeToRun = 2 * 60 * 1000;
+ let jobTimeToRun = 86400000; // 24h (crawl)
if (job.data.scrapeOptions) {
if (job.data.scrapeOptions.timeout) {