Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
refactor(api): move job data out of concurrency queue into own keys (#2875)
tomsideguide9119dd9554d1
diff --git a/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts b/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.tsindex 21a7dcdc8..99e61c128 100644--- a/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts+++ b/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts@@ -47,7 +47,7 @@ export async function concurrencyQueueBackfillController(// zscan returns [member1, score1, member2, score2, ...]// Only parse members (even indices), skip scores (odd indices)for (let i = 0; i < results.length; i += 2) {- queuedJobIDs.add(JSON.parse(results[i]).id);+ queuedJobIDs.add(results[i]);}} while (cursor !== "0");diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.tsindex 301fc53ff..8008d15b8 100644--- a/apps/api/src/lib/concurrency-limit.ts+++ b/apps/api/src/lib/concurrency-limit.ts@@ -10,6 +10,8 @@ const constructKey = (team_id: string) => "concurrency-limiter:" + team_id;const constructQueueKey = (team_id: string) =>"concurrency-limit-queue:" + team_id;+const constructJobKey = (jobId: string) => "cq-job:" + jobId;+const constructCrawlKey = (crawl_id: string) =>"crawl-concurrency-limiter:" + crawl_id;@@ -86,15 +88,22 @@ export async function pushConcurrencyLimitedJob(now: number = Date.now(),) {const queueKey = constructQueueKey(team_id);- await getRedisConnection().zadd(queueKey, now + timeout, JSON.stringify(job));- await getRedisConnection().sadd("concurrency-limit-queues", queueKey);+ const jobKey = constructJobKey(job.id);+ const redis = getRedisConnection();++ if (timeout === Infinity) {+ await redis.set(jobKey, JSON.stringify(job), "EX", 172800); // 48h+ } else {+ await redis.set(jobKey, JSON.stringify(job), "PX", timeout);+ }++ await redis.zadd(queueKey, now + timeout, job.id);+ await redis.sadd("concurrency-limit-queues", queueKey);}export async function getConcurrencyLimitedJobs(team_id: string) {return new Set(- (await getRedisConnection().zrange(constructQueueKey(team_id), 0, -1)).map(- x => JSON.parse(x).id,- ),+ await getRedisConnection().zrange(constructQueueKey(team_id), 0, -1),);}@@ -172,24 +181,37 @@ async function getNextConcurrentJob(}[] = [];const crawlCache = new Map<string, StoredCrawl>();- let cursor: string = "0";+ const queueKey = constructQueueKey(teamId);+ const redis = getRedisConnection();+ let offset = 0;do {- const scanResult = await getRedisConnection().zscan(- constructQueueKey(teamId),- cursor,- "COUNT",+ const members = await redis.zrangebyscore(+ queueKey,+ Date.now(),+ "+inf",+ "LIMIT",+ offset,20,);- cursor = scanResult[0];- const results = scanResult[1];- for (let i = 0; i < results.length; i += 2) {+ if (members.length === 0) break;+ offset += members.length;++ for (const member of members) {+ const jobData = await redis.get(constructJobKey(member));+ if (jobData === null) {+ // TTL expired - remove orphaned sorted set entry+ await redis.zrem(queueKey, member);+ offset--;+ continue;+ }+ const job: ConcurrencyLimitedJob = JSON.parse(jobData);+const res = {- job: JSON.parse(results[i]),- _member: results[i],- timeout:- results[i + 1] === "inf" ? Infinity : parseFloat(results[i + 1]),+ job,+ _member: member,+ timeout: Infinity,};// If the job is associated with a crawl ID, we need to check if the crawl has a max concurrency limit@@ -227,7 +249,7 @@ async function getNextConcurrentJob(finalJobs.push(res);}}- } while (finalJobs.length === 0 && cursor !== "0");+ } while (finalJobs.length === 0);let finalJob: (typeof finalJobs)[number] | null = null;if (finalJobs.length > 0) {@@ -237,6 +259,7 @@ async function getNextConcurrentJob(job._member,);if (res !== 0) {+ await getRedisConnection().del(constructJobKey(job._member));finalJob = job;break;}@@ -298,6 +321,7 @@ export async function concurrentJobDone(job: NuQJob<any>) {if (job.id && job.data && job.data.team_id) {await removeConcurrencyLimitActiveJob(job.data.team_id, job.id);await cleanOldConcurrencyLimitEntries(job.data.team_id);+ await cleanOldConcurrencyLimitedJobs(job.data.team_id);if (job.data.crawl_id) {await removeCrawlConcurrencyLimitActiveJob(job.data.crawl_id, job.id);