Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): pipeline/batch concurrency queue (#2928)
tomsideguideccf8555002bd
diff --git a/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts b/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.tsindex 99e61c128..ec73effb3 100644--- a/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts+++ b/apps/api/src/controllers/v0/admin/concurrency-queue-backfill.ts@@ -3,7 +3,7 @@ import { Request, Response } from "express";import { getRedisConnection } from "../../../services/queue-service";import { scrapeQueue } from "../../../services/worker/nuq";import {- pushConcurrencyLimitedJob,+ pushConcurrencyLimitedJobs,pushConcurrencyLimitActiveJob,getConcurrencyLimitActiveJobs,} from "../../../lib/concurrency-limit";@@ -95,16 +95,18 @@ export async function concurrencyQueueBackfillController(}}- for (const job of jobsToQueue) {- await pushConcurrencyLimitedJob(+ if (jobsToQueue.length > 0) {+ await pushConcurrencyLimitedJobs(ownerId,- {- id: job.id,- data: job.data,- priority: job.priority,- listenable: job.listenChannelId !== undefined,- },- Infinity,+ jobsToQueue.map(job => ({+ job: {+ id: job.id,+ data: job.data,+ priority: job.priority,+ listenable: job.listenChannelId !== undefined,+ },+ timeout: Infinity,+ })),);}diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.tsindex 5ceeb2b2a..2af5dd10d 100644--- a/apps/api/src/lib/concurrency-limit.ts+++ b/apps/api/src/lib/concurrency-limit.ts@@ -87,18 +87,36 @@ export async function pushConcurrencyLimitedJob(timeout: number,now: number = Date.now(),) {+ await pushConcurrencyLimitedJobs(team_id, [{ job, timeout }], now);+}++export async function pushConcurrencyLimitedJobs(+ team_id: string,+ jobs: { job: ConcurrencyLimitedJob; timeout: number }[],+ now: number = Date.now(),+) {+ if (jobs.length === 0) {+ return;+ }+const queueKey = constructQueueKey(team_id);- const jobKey = constructJobKey(job.id);const redis = getRedisConnection();-- if (timeout === Infinity) {- await redis.set(jobKey, JSON.stringify(job), "EX", 172800); // 48h- } else {- await redis.set(jobKey, JSON.stringify(job), "PX", timeout);+ const pipeline = redis.pipeline();+ const zaddArgs: (string | number)[] = [];++ for (const { job, timeout } of jobs) {+ const jobKey = constructJobKey(job.id);+ if (timeout === Infinity) {+ pipeline.set(jobKey, JSON.stringify(job), "EX", 172800); // 48h+ } else {+ pipeline.set(jobKey, JSON.stringify(job), "PX", timeout);+ }+ zaddArgs.push(now + timeout, job.id);}- await redis.zadd(queueKey, now + timeout, job.id);- await redis.sadd("concurrency-limit-queues", queueKey);+ pipeline.zadd(queueKey, ...zaddArgs);+ pipeline.sadd("concurrency-limit-queues", queueKey);+ await pipeline.exec();}export async function getConcurrencyLimitedJobs(team_id: string) {@@ -245,11 +263,11 @@ async function getNextConcurrentJob(teamId: string): Promise<{} finally {// Re-add crawl-blocked jobs so they can be picked up laterif (crawlBlocked.length > 0) {- const pipeline = redis.pipeline();+ const zaddArgs: (string | number)[] = [];for (const { member, score } of crawlBlocked) {- pipeline.zadd(queueKey, score, member);+ zaddArgs.push(score, member);}- await pipeline.exec();+ await redis.zadd(queueKey, ...zaddArgs);}}}diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex 54de69cb9..6ed5dbf03 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -7,6 +7,7 @@ import {getCrawlConcurrencyLimitActiveJobs,pushConcurrencyLimitActiveJob,pushConcurrencyLimitedJob,+ pushConcurrencyLimitedJobs,pushCrawlConcurrencyLimitActiveJob,} from "../lib/concurrency-limit";import { logger as _logger } from "../lib/logger";@@ -106,19 +107,34 @@ async function _addScrapeJobsToConcurrencyQueue(})),);+ const jobsByTeam = new Map<+ string,+ {+ job: { id: string; data: any; priority: number; listenable: boolean };+ timeout: number;+ }[]+ >();+for (const job of jobs) {- await pushConcurrencyLimitedJob(- job.data.team_id,- {+ const teamId = job.data.team_id as string;+ if (!jobsByTeam.has(teamId)) {+ jobsByTeam.set(teamId, []);+ }+ jobsByTeam.get(teamId)!.push({+ job: {id: job.jobId,data: job.data,priority: job.priority,listenable: job.listenable ?? false,},- job.data.crawl_id+ timeout: job.data.crawl_id? Infinity: (job.data.scrapeOptions?.timeout ?? 60 * 1000),- );+ });+ }++ for (const [teamId, teamJobs] of jobsByTeam) {+ await pushConcurrencyLimitedJobs(teamId, teamJobs);}}