Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(queue-worker/afterJobDone): improved ccq insert logic (#1595)

Gergő Móricz3df687e4db20
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 79ab6c699..2738b8ee6 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -54,6 +54,7 @@ import { scrapeOptions } from "../controllers/v1/types";
import {
cleanOldConcurrencyLimitEntries,
cleanOldCrawlConcurrencyLimitEntries,
+ getConcurrencyLimitActiveJobs,
pushConcurrencyLimitActiveJob,
pushCrawlConcurrencyLimitActiveJob,
removeConcurrencyLimitActiveJob,
@@ -86,6 +87,7 @@ import http from "http";
import https from "https";
import { cacheableLookup } from "../scraper/scrapeURL/lib/cacheableLookup";
import { robustFetch } from "../scraper/scrapeURL/lib/fetch";
+import { RateLimiterMode } from "../types";
configDotenv();
@@ -805,31 +807,37 @@ const workerFun = async (
}
if (job.id && job.data && job.data.team_id) {
- await removeConcurrencyLimitActiveJob(job.data.team_id, job.id);
- cleanOldConcurrencyLimitEntries(job.data.team_id);
-
- // No need to check if we're under the limit here -- if the current job is finished,
- // we are 1 under the limit, assuming the job insertion logic never over-inserts. - MG
- const nextJob = await takeConcurrencyLimitedJob(job.data.team_id);
- if (nextJob !== null) {
- await pushConcurrencyLimitActiveJob(
- job.data.team_id,
- nextJob.id,
- 60 * 1000,
- ); // 60s initial timeout
+ const maxConcurrency = (await getACUCTeam(job.data.team_id, false, true, job.data.is_extract ? RateLimiterMode.Extract : RateLimiterMode.Crawl))?.concurrency ?? 2;
- await queue.add(
- nextJob.id,
- {
- ...nextJob.data,
- concurrencyLimitHit: true,
- },
- {
- ...nextJob.opts,
- jobId: nextJob.id,
- priority: nextJob.priority,
- },
- );
+ await removeConcurrencyLimitActiveJob(job.data.team_id, job.id);
+ await cleanOldConcurrencyLimitEntries(job.data.team_id);
+
+ // Check if we're under the concurrency limit before adding a new job
+ const currentActiveConcurrency = (await getConcurrencyLimitActiveJobs(job.data.team_id)).length;
+ const concurrencyLimited = currentActiveConcurrency >= maxConcurrency;
+
+ if (!concurrencyLimited) {
+ const nextJob = await takeConcurrencyLimitedJob(job.data.team_id);
+ if (nextJob !== null) {
+ await pushConcurrencyLimitActiveJob(
+ job.data.team_id,
+ nextJob.id,
+ 60 * 1000,
+ ); // 60s initial timeout
+
+ await queue.add(
+ nextJob.id,
+ {
+ ...nextJob.data,
+ concurrencyLimitHit: true,
+ },
+ {
+ ...nextJob.opts,
+ jobId: nextJob.id,
+ priority: nextJob.priority,
+ },
+ );
+ }
}
}
}