Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(concurrency-limit): scale! (#2071)

Gergő Móriczeb969be3c8dc
diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.ts
index 3fc996dec..14093d2d0 100644
--- a/apps/api/src/lib/concurrency-limit.ts
+++ b/apps/api/src/lib/concurrency-limit.ts
@@ -142,17 +142,17 @@ async function getNextConcurrentJob(teamId: string, i = 0): Promise<{
job: ConcurrencyLimitedJob;
timeout: number;
} | null> {
- let finalJob: {
+ let finalJobs: {
job: ConcurrencyLimitedJob;
_member: string;
timeout: number;
- } | null = null;
+ }[] = [];
const crawlCache = new Map<string, StoredCrawl>();
let cursor: string = "0";
- while (true) {
- const scanResult = await redisEvictConnection.zscan(constructQueueKey(teamId), cursor, "COUNT", 1);
+ do {
+ const scanResult = await redisEvictConnection.zscan(constructQueueKey(teamId), cursor, "COUNT", 20);
cursor = scanResult[0];
const results = scanResult[1];
@@ -172,7 +172,7 @@ async function getNextConcurrentJob(teamId: string, i = 0): Promise<{
const maxCrawlConcurrency = sc === null
? null
- : (typeof sc.crawlerOptions?.delay === "number")
+ : (typeof sc.crawlerOptions?.delay === "number" && sc.crawlerOptions.delay > 0)
? 1
: sc.maxConcurrency ?? null;
@@ -181,50 +181,46 @@ async function getNextConcurrentJob(teamId: string, i = 0): Promise<{
const currentActiveConcurrency = (await getCrawlConcurrencyLimitActiveJobs(res.job.data.crawl_id)).length;
if (currentActiveConcurrency < maxCrawlConcurrency) {
// If we're under the max concurrency limit, we can run the job
- finalJob = res;
+ finalJobs.push(res);
}
} else {
// If the crawl has no max concurrency limit, we can run the job
- finalJob = res;
+ finalJobs.push(res);
}
} else {
// If the job is not associated with a crawl ID, we can run the job
- finalJob = res;
+ finalJobs.push(res);
}
-
- if (finalJob !== null) {
+ }
+ } while (finalJobs.length === 0 && cursor !== "0");
+
+ let finalJob: typeof finalJobs[number] | null = null;
+ if (finalJobs.length > 0) {
+ for (const job of finalJobs) {
+ const res = await redisEvictConnection.zrem(constructQueueKey(teamId), job._member);
+ if (res !== 0) {
+ finalJob = job;
break;
}
}
- if (finalJob !== null) {
- break;
- }
-
- if (cursor === "0") {
- break;
- }
- }
-
- if (finalJob !== null) {
- const res = await redisEvictConnection.zrem(constructQueueKey(teamId), finalJob._member);
- if (res === 0) {
+ if (finalJob === null) {
// It's normal for this to happen, but if it happens too many times, we should log a warning
- if (i > 15) {
- logger.warn("Failed to remove job from concurrency limit queue", {
+ if (i > 100) {
+ logger.error("Failed to remove job from concurrency limit queue, hard bailing", {
teamId,
- jobId: finalJob.job.id,
- zeroDataRetention: finalJob.job.data?.zeroDataRetention,
+ jobIds: finalJobs.map(x => x.job.id),
+ zeroDataRetention: finalJobs.some(x => x.job.data?.zeroDataRetention),
i
});
- } else if (i > 100) {
- logger.error("Failed to remove job from concurrency limit queue, hard bailing", {
+ return null;
+ } else if (i > 15) {
+ logger.warn("Failed to remove job from concurrency limit queue", {
teamId,
- jobId: finalJob.job.id,
- zeroDataRetention: finalJob.job.data?.zeroDataRetention,
+ jobIds: finalJobs.map(x => x.job.id),
+ zeroDataRetention: finalJobs.some(x => x.job.data?.zeroDataRetention),
i
});
- return null;
}
return await new Promise((resolve, reject) => setTimeout(() => {