Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): pipeline/batch concurrency queue (#2928)

tomsideguideccf8555002bd
diff --git a/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts b/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts
index 99e61c128..ec73effb3 100644
--- a/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts
+++ b/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts
@@ -3,7 +3,7 @@ import { Request, Response } from "express";
import { getRedisConnection } from "../../../services/queue-service";
import { scrapeQueue } from "../../../services/worker/nuq";
import {
- pushConcurrencyLimitedJob,
+ pushConcurrencyLimitedJobs,
pushConcurrencyLimitActiveJob,
getConcurrencyLimitActiveJobs,
} from "../../../lib/concurrency-limit";
@@ -95,16 +95,18 @@ export async function concurrencyQueueBackfillController(
}
}
- for (const job of jobsToQueue) {
- await pushConcurrencyLimitedJob(
+ if (jobsToQueue.length > 0) {
+ await pushConcurrencyLimitedJobs(
ownerId,
- {
- id: job.id,
- data: job.data,
- priority: job.priority,
- listenable: job.listenChannelId !== undefined,
- },
- Infinity,
+ jobsToQueue.map(job => ({
+ job: {
+ id: job.id,
+ data: job.data,
+ priority: job.priority,
+ listenable: job.listenChannelId !== undefined,
+ },
+ timeout: Infinity,
+ })),
);
}
diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.ts
index 5ceeb2b2a..2af5dd10d 100644
--- a/apps/api/src/lib/concurrency-limit.ts
+++ b/apps/api/src/lib/concurrency-limit.ts
@@ -87,18 +87,36 @@ export async function pushConcurrencyLimitedJob(
timeout: number,
now: number = Date.now(),
) {
+ await pushConcurrencyLimitedJobs(team_id, [{ job, timeout }], now);
+}
+
+export async function pushConcurrencyLimitedJobs(
+ team_id: string,
+ jobs: { job: ConcurrencyLimitedJob; timeout: number }[],
+ now: number = Date.now(),
+) {
+ if (jobs.length === 0) {
+ return;
+ }
+
const queueKey = constructQueueKey(team_id);
- const jobKey = constructJobKey(job.id);
const redis = getRedisConnection();
-
- if (timeout === Infinity) {
- await redis.set(jobKey, JSON.stringify(job), "EX", 172800); // 48h
- } else {
- await redis.set(jobKey, JSON.stringify(job), "PX", timeout);
+ const pipeline = redis.pipeline();
+ const zaddArgs: (string | number)[] = [];
+
+ for (const { job, timeout } of jobs) {
+ const jobKey = constructJobKey(job.id);
+ if (timeout === Infinity) {
+ pipeline.set(jobKey, JSON.stringify(job), "EX", 172800); // 48h
+ } else {
+ pipeline.set(jobKey, JSON.stringify(job), "PX", timeout);
+ }
+ zaddArgs.push(now + timeout, job.id);
}
- await redis.zadd(queueKey, now + timeout, job.id);
- await redis.sadd("concurrency-limit-queues", queueKey);
+ pipeline.zadd(queueKey, ...zaddArgs);
+ pipeline.sadd("concurrency-limit-queues", queueKey);
+ await pipeline.exec();
}
export async function getConcurrencyLimitedJobs(team_id: string) {
@@ -245,11 +263,11 @@ async function getNextConcurrentJob(teamId: string): Promise<{
} finally {
// Re-add crawl-blocked jobs so they can be picked up later
if (crawlBlocked.length > 0) {
- const pipeline = redis.pipeline();
+ const zaddArgs: (string | number)[] = [];
for (const { member, score } of crawlBlocked) {
- pipeline.zadd(queueKey, score, member);
+ zaddArgs.push(score, member);
}
- await pipeline.exec();
+ await redis.zadd(queueKey, ...zaddArgs);
}
}
}
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts
index 54de69cb9..6ed5dbf03 100644
--- a/apps/api/src/services/queue-jobs.ts
+++ b/apps/api/src/services/queue-jobs.ts
@@ -7,6 +7,7 @@ import {
getCrawlConcurrencyLimitActiveJobs,
pushConcurrencyLimitActiveJob,
pushConcurrencyLimitedJob,
+ pushConcurrencyLimitedJobs,
pushCrawlConcurrencyLimitActiveJob,
} from "../lib/concurrency-limit";
import { logger as _logger } from "../lib/logger";
@@ -106,19 +107,34 @@ async function _addScrapeJobsToConcurrencyQueue(
})),
);
+ const jobsByTeam = new Map<
+ string,
+ {
+ job: { id: string; data: any; priority: number; listenable: boolean };
+ timeout: number;
+ }[]
+ >();
+
for (const job of jobs) {
- await pushConcurrencyLimitedJob(
- job.data.team_id,
- {
+ const teamId = job.data.team_id as string;
+ if (!jobsByTeam.has(teamId)) {
+ jobsByTeam.set(teamId, []);
+ }
+ jobsByTeam.get(teamId)!.push({
+ job: {
id: job.jobId,
data: job.data,
priority: job.priority,
listenable: job.listenable ?? false,
},
- job.data.crawl_id
+ timeout: job.data.crawl_id
? Infinity
: (job.data.scrapeOptions?.timeout ?? 60 * 1000),
- );
+ });
+ }
+
+ for (const [teamId, teamJobs] of jobsByTeam) {
+ await pushConcurrencyLimitedJobs(teamId, teamJobs);
}
}