Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat: queue multiplexing (#1902)
Gergő Móricze6a61e3d19c5
diff --git a/apps/api/src/controllers/v0/admin/queue.ts b/apps/api/src/controllers/v0/admin/queue.tsindex d7d9c0897..f775e2c81 100644--- a/apps/api/src/controllers/v0/admin/queue.ts+++ b/apps/api/src/controllers/v0/admin/queue.ts@@ -12,7 +12,7 @@ export async function cleanBefore24hCompleteJobsController() {logger.info("🐂 Cleaning jobs older than 24h");try {- const scrapeQueue = getScrapeQueue();+ const scrapeQueue = getScrapeQueue(0);const batchSize = 10;const numberOfBatches = 9; // Adjust based on your needsconst completedJobsPromises: Promise<Job[]>[] = [];@@ -70,7 +70,7 @@ export async function checkQueuesController(req: Request, res: Response) {// Use this as a "health check" that way we dont destroy the serverexport async function queuesController(req: Request, res: Response) {try {- const scrapeQueue = getScrapeQueue();+ const scrapeQueue = getScrapeQueue(0);const [webScraperActive] = await Promise.all([scrapeQueue.getActiveCount(),@@ -93,7 +93,7 @@ export async function autoscalerController(req: Request, res: Response) {const maxNumberOfMachines = 80;const minNumberOfMachines = 20;- const scrapeQueue = getScrapeQueue();+ const scrapeQueue = getScrapeQueue(0);const [webScraperActive, webScraperWaiting, webScraperPriority] =await Promise.all([diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.tsindex 9f6e5b7bb..6fef4b64c 100644--- a/apps/api/src/controllers/v0/crawl-status.ts+++ b/apps/api/src/controllers/v0/crawl-status.ts@@ -1,7 +1,7 @@import { Request, Response } from "express";import { authenticateUser } from "../auth";import { RateLimiterMode } from "../../../src/types";-import { getScrapeQueue } from "../../../src/services/queue-service";+import { getScrapeQueue, uuidToQueueNo } from "../../../src/services/queue-service";import { redisEvictConnection } from "../../../src/services/redis";import { logger } from "../../../src/lib/logger";import { getCrawl, getCrawlJobs } from "../../../src/lib/crawl-redis";@@ -16,7 +16,7 @@ configDotenv();export async function getJobs(crawlId: string, ids: string[]): Promise<PseudoJob<any>[]> {const [bullJobs, dbJobs, gcsJobs] = await Promise.all([- Promise.all(ids.map((x) => getScrapeQueue().getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,+ Promise.all(ids.map((x) => getScrapeQueue(uuidToQueueNo(x)).getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,process.env.USE_DB_AUTHENTICATION === "true" ? await supabaseGetJobsByCrawlId(crawlId) : [],process.env.GCS_BUCKET_NAME ? Promise.all(ids.map(async (x) => ({ id: x, job: await getJobFromGCS(x) }))).then(x => x.filter(x => x.job)) as Promise<({ id: string, job: any | null })[]> : [],]);diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex d4bd73bbc..c1a9d57a4 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -21,7 +21,7 @@ import {defaultOrigin,} from "../../lib/default-values";import { addScrapeJob, waitForJob } from "../../services/queue-jobs";-import { getScrapeQueue } from "../../services/queue-service";+import { getScrapeQueue, uuidToQueueNo } from "../../services/queue-service";import { redisEvictConnection } from "../../../src/services/redis";import { v4 as uuidv4 } from "uuid";import { logger } from "../../lib/logger";@@ -128,7 +128,7 @@ export async function scrapeHelper(return err;}- await getScrapeQueue().remove(jobId);+ await getScrapeQueue(uuidToQueueNo(jobId)).remove(jobId);if (!doc) {console.error("!!! PANIC DOC IS", doc);diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.tsindex aaff178c2..35233fd3b 100644--- a/apps/api/src/controllers/v0/search.ts+++ b/apps/api/src/controllers/v0/search.ts@@ -11,7 +11,7 @@ import { search } from "../../search";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import { v4 as uuidv4 } from "uuid";import { logger } from "../../lib/logger";-import { getScrapeQueue } from "../../services/queue-service";+import { getScrapeQueue, uuidToQueueNo } from "../../services/queue-service";import { redisEvictConnection } from "../../../src/services/redis";import { addScrapeJob, waitForJob } from "../../services/queue-jobs";import * as Sentry from "@sentry/node";@@ -137,8 +137,7 @@ export async function searchHelper(return { success: true, error: "No search results found", returnCode: 200 };}- const sq = getScrapeQueue();- await Promise.all(jobDatas.map((x) => sq.remove(x.opts.jobId)));+ await Promise.all(jobDatas.map((x) => getScrapeQueue(uuidToQueueNo(x.opts.jobId)).remove(x.opts.jobId)));// make sure doc.content is not emptyconst filteredDocs = docs.filter(diff --git a/apps/api/src/controllers/v1/crawl-errors.ts b/apps/api/src/controllers/v1/crawl-errors.tsindex d8d029ee6..57eb9eb21 100644--- a/apps/api/src/controllers/v1/crawl-errors.ts+++ b/apps/api/src/controllers/v1/crawl-errors.ts@@ -8,14 +8,14 @@ import {getCrawl,getCrawlJobs,} from "../../lib/crawl-redis";-import { getScrapeQueue } from "../../services/queue-service";+import { getScrapeQueue, uuidToQueueNo } from "../../services/queue-service";import { redisEvictConnection } from "../../../src/services/redis";import { configDotenv } from "dotenv";import { Job } from "bullmq";configDotenv();export async function getJob(id: string) {- const job = await getScrapeQueue().getJob(id);+ const job = await getScrapeQueue(uuidToQueueNo(id)).getJob(id);if (!job) return job;return job;@@ -23,7 +23,7 @@ export async function getJob(id: string) {export async function getJobs(ids: string[]) {const jobs: (Job & { id: string })[] = (- await Promise.all(ids.map((x) => getScrapeQueue().getJob(x)))+ await Promise.all(ids.map((x) => getScrapeQueue(uuidToQueueNo(x)).getJob(x)))).filter((x) => x) as (Job & { id: string })[];return jobs;@@ -44,7 +44,7 @@ export async function crawlErrorsController(let jobStatuses = await Promise.all((await getCrawlJobs(req.params.jobId)).map(- async (x) => [x, await getScrapeQueue().getJobState(x)] as const,+ async (x) => [x, await getScrapeQueue(uuidToQueueNo(x)).getJobState(x)] as const,),);diff --git a/apps/api/src/controllers/v1/crawl-status-ws.ts b/apps/api/src/controllers/v1/crawl-status-ws.tsindex 93e1fc9b1..3ec7e61d3 100644--- a/apps/api/src/controllers/v1/crawl-status-ws.ts+++ b/apps/api/src/controllers/v1/crawl-status-ws.ts@@ -20,7 +20,7 @@ import {isCrawlFinished,isCrawlFinishedLocked,} from "../../lib/crawl-redis";-import { getScrapeQueue } from "../../services/queue-service";+import { getScrapeQueue, uuidToQueueNo } from "../../services/queue-service";import { getJob, getJobs } from "./crawl-status";import * as Sentry from "@sentry/node";import { Job, JobState } from "bullmq";@@ -89,12 +89,10 @@ async function crawlStatusWS(const notDoneJobIDs = jobIDs.filter((x) => !doneJobIDs.includes(x));- const queue = getScrapeQueue();-const jobStatuses = await Promise.all(notDoneJobIDs.map(async (x) => [x,- await queue.getJobState(x),+ await getScrapeQueue(uuidToQueueNo(x)).getJobState(x),]),);const newlyDoneJobIDs: string[] = jobStatuses@@ -126,11 +124,9 @@ async function crawlStatusWS(let jobIDs = await getCrawlJobs(req.params.jobId);- const queue = getScrapeQueue();-let jobStatuses = await Promise.all(jobIDs.map(- async (x) => [x, await queue.getJobState(x)] as const,+ async (x) => [x, await getScrapeQueue(uuidToQueueNo(x)).getJobState(x)] as const,),);diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex 4da7112cf..faf7d3037 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -13,7 +13,7 @@ import {getDoneJobsOrderedLength,isCrawlKickoffFinished,} from "../../lib/crawl-redis";-import { getScrapeQueue } from "../../services/queue-service";+import { getScrapeQueue, uuidToQueueNo } from "../../services/queue-service";import {supabaseGetJobById,supabaseGetJobsById,@@ -42,7 +42,7 @@ export type DBJob = { docs: any, success: boolean, page_options: any, date_addedexport async function getJob(id: string): Promise<PseudoJob<any> | null> {const [bullJob, dbJob, gcsJob] = await Promise.all([- getScrapeQueue().getJob(id),+ getScrapeQueue(uuidToQueueNo(id)).getJob(id),(process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobById(id) : null) as Promise<DBJob | null>,(process.env.GCS_BUCKET_NAME ? getJobFromGCS(id) : null) as Promise<any | null>,]);@@ -74,7 +74,7 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {const [bullJobs, dbJobs, gcsJobs] = await Promise.all([- Promise.all(ids.map((x) => getScrapeQueue().getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,+ Promise.all(ids.map((x) => getScrapeQueue(uuidToQueueNo(x)).getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobsById(ids) : [],process.env.GCS_BUCKET_NAME ? Promise.all(ids.map(async (x) => ({ id: x, job: await getJobFromGCS(x) }))).then(x => x.filter(x => x.job)) as Promise<({ id: string, job: any | null })[]> : [],]);@@ -159,7 +159,7 @@ export async function crawlStatusController(let jobIDs = await getCrawlJobs(req.params.jobId);let jobStatuses = await Promise.all(jobIDs.map(- async (x) => [x, await getScrapeQueue().getJobState(x)] as const,+ async (x) => [x, await getScrapeQueue(uuidToQueueNo(x)).getJobState(x)] as const,),);diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex 658b89273..17008b8b5 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -10,7 +10,7 @@ import {import { v4 as uuidv4 } from "uuid";import { addScrapeJob, waitForJob } from "../../services/queue-jobs";import { getJobPriority } from "../../lib/job-priority";-import { getScrapeQueue } from "../../services/queue-service";+import { getScrapeQueue, uuidToQueueNo } from "../../services/queue-service";export async function scrapeController(req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,@@ -105,7 +105,7 @@ export async function scrapeController(});if (zeroDataRetention) {- await getScrapeQueue().remove(jobId);+ await getScrapeQueue(uuidToQueueNo(jobId)).remove(jobId);}if (@@ -126,7 +126,7 @@ export async function scrapeController(logger.info("Done with waitForJob");- await getScrapeQueue().remove(jobId);+ await getScrapeQueue(uuidToQueueNo(jobId)).remove(jobId);logger.info("Removed job from queue");diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex 82efbcfb3..37bc3499c 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -15,7 +15,7 @@ import { addScrapeJob, waitForJob } from "../../services/queue-jobs";import { logJob } from "../../services/logging/log_job";import { getJobPriority } from "../../lib/job-priority";import { Mode } from "../../types";-import { getScrapeQueue } from "../../services/queue-service";+import { getScrapeQueue, uuidToQueueNo } from "../../services/queue-service";import { search } from "../../search";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";import * as Sentry from "@sentry/node";@@ -133,7 +133,7 @@ async function scrapeSearchResult(teamId: options.teamId,origin: options.origin,});- await getScrapeQueue().remove(jobId);+ await getScrapeQueue(uuidToQueueNo(jobId)).remove(jobId);const document = {title: searchResult.title,diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex b6fc1e3fa..cbfaeb5c2 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -11,6 +11,7 @@ import {getDeepResearchQueue,getBillingQueue,getPrecrawlQueue,+ scrapeQueueNames,} from "./services/queue-service";import { v0Router } from "./routes/v0";import os from "os";@@ -55,7 +56,7 @@ serverAdapter.setBasePath(`/admin/${process.env.BULL_AUTH_KEY}/queues`);const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({queues: [- new BullMQAdapter(getScrapeQueue()),+ ...scrapeQueueNames.map((_, i) => new BullMQAdapter(getScrapeQueue(i))),new BullMQAdapter(getExtractQueue()),new BullMQAdapter(getGenerateLlmsTxtQueue()),new BullMQAdapter(getDeepResearchQueue()),@@ -119,8 +120,7 @@ if (require.main === module) {app.get(`/serverHealthCheck`, async (req, res) => {try {- const scrapeQueue = getScrapeQueue();- const [waitingJobs] = await Promise.all([scrapeQueue.getWaitingCount()]);+ const waitingJobs = (await Promise.all(scrapeQueueNames.map((_, i) => getScrapeQueue(i).getWaitingCount()))).reduce((acc, curr) => acc + curr, 0);const noWaitingJobs = waitingJobs === 0;// 200 if no active jobs, 503 if there are active jobsreturn res.status(noWaitingJobs ? 200 : 500).json({@@ -139,10 +139,7 @@ app.get("/serverHealthCheck/notify", async (req, res) => {const timeout = 60000; // 1 minute // The timeout value for the check in millisecondsconst getWaitingJobsCount = async () => {- const scrapeQueue = getScrapeQueue();- const [waitingJobsCount] = await Promise.all([- scrapeQueue.getWaitingCount(),- ]);+ const waitingJobsCount = (await Promise.all(scrapeQueueNames.map((_, i) => getScrapeQueue(i).getWaitingCount()))).reduce((acc, curr) => acc + curr, 0);return waitingJobsCount;};diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.tsindex 022f3523b..4ab8f9467 100644--- a/apps/api/src/lib/concurrency-limit.ts+++ b/apps/api/src/lib/concurrency-limit.ts@@ -3,7 +3,7 @@ import { redisEvictConnection } from "../services/redis";import type { Job, JobsOptions } from "bullmq";import { getACUCTeam } from "../controllers/auth";import { getCrawl, StoredCrawl } from "./crawl-redis";-import { getScrapeQueue } from "../services/queue-service";+import { getScrapeQueue, uuidToQueueNo } from "../services/queue-service";import { logger } from "./logger";const constructKey = (team_id: string) => "concurrency-limiter:" + team_id;@@ -258,7 +258,7 @@ export async function concurrentJobDone(job: Job) {}}- (await getScrapeQueue()).add(+ (await getScrapeQueue(uuidToQueueNo(nextJob.job.id))).add(nextJob.job.id,{...nextJob.job.data,diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.tsindex 083dd2628..16719bb04 100644--- a/apps/api/src/lib/extract/document-scraper.ts+++ b/apps/api/src/lib/extract/document-scraper.ts@@ -1,6 +1,6 @@import { Document, ScrapeOptions, TeamFlags, URLTrace, scrapeOptions } from "../../controllers/v1/types";import { logger } from "../logger";-import { getScrapeQueue } from "../../services/queue-service";+import { getScrapeQueue, uuidToQueueNo } from "../../services/queue-service";import { waitForJob } from "../../services/queue-jobs";import { addScrapeJob } from "../../services/queue-jobs";import { getJobPriority } from "../job-priority";@@ -68,7 +68,7 @@ export async function scrapeDocument(const doc = await waitForJob(jobId, timeout);- await getScrapeQueue().remove(jobId);+ await getScrapeQueue(uuidToQueueNo(jobId)).remove(jobId);if (trace) {trace.timing.completedAt = new Date().toISOString();diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.tsindex c6ecca291..185d882db 100644--- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts+++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts@@ -1,6 +1,6 @@import { Document, ScrapeOptions, TeamFlags, URLTrace, scrapeOptions } from "../../../controllers/v1/types";import { logger } from "../../logger";-import { getScrapeQueue } from "../../../services/queue-service";+import { getScrapeQueue, uuidToQueueNo } from "../../../services/queue-service";import { waitForJob } from "../../../services/queue-jobs";import { addScrapeJob } from "../../../services/queue-jobs";import { getJobPriority } from "../../job-priority";@@ -65,7 +65,7 @@ export async function scrapeDocument_F0();const doc = await waitForJob(jobId, timeout);- await getScrapeQueue().remove(jobId);+ await getScrapeQueue(uuidToQueueNo(jobId)).remove(jobId);if (trace) {trace.timing.completedAt = new Date().toISOString();diff --git a/apps/api/src/services/alerts/index.ts b/apps/api/src/services/alerts/index.tsindex 44f2b8a04..c257f8d63 100644--- a/apps/api/src/services/alerts/index.ts+++ b/apps/api/src/services/alerts/index.ts@@ -1,5 +1,5 @@import { logger } from "../../../src/lib/logger";-import { getScrapeQueue } from "../queue-service";+import { getScrapeQueue, scrapeQueueNames } from "../queue-service";import { sendSlackWebhook } from "./slack";export async function checkAlerts() {@@ -13,8 +13,7 @@ export async function checkAlerts() {logger.info("Initializing alerts");const checkActiveJobs = async () => {try {- const scrapeQueue = getScrapeQueue();- const activeJobs = await scrapeQueue.getActiveCount();+ const activeJobs = (await Promise.all(scrapeQueueNames.map((_, i) => getScrapeQueue(i).getActiveCount()))).reduce((acc, curr) => acc + curr, 0);if (activeJobs > Number(process.env.ALERT_NUM_ACTIVE_JOBS)) {logger.warn(`Alert: Number of active jobs is over ${process.env.ALERT_NUM_ACTIVE_JOBS}. Current active jobs: ${activeJobs}.`,@@ -34,8 +33,7 @@ export async function checkAlerts() {};const checkWaitingQueue = async () => {- const scrapeQueue = getScrapeQueue();- const waitingJobs = await scrapeQueue.getWaitingCount();+ const waitingJobs = (await Promise.all(scrapeQueueNames.map((_, i) => getScrapeQueue(i).getWaitingCount()))).reduce((acc, curr) => acc + curr, 0);if (waitingJobs > Number(process.env.ALERT_NUM_WAITING_JOBS)) {logger.warn(diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex d6947659a..ef8976b72 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -1,4 +1,4 @@-import { getScrapeQueue, getScrapeQueueEvents } from "./queue-service";+import { getScrapeQueue, getScrapeQueueEvents, uuidToQueueNo } from "./queue-service";import { v4 as uuidv4 } from "uuid";import { NotificationType, RateLimiterMode, WebScraperOptions } from "../types";import * as Sentry from "@sentry/node";@@ -70,7 +70,7 @@ export async function _addScrapeJobToBullMQ(}}- return await getScrapeQueue().add(jobId, webScraperOptions, {+ return await getScrapeQueue(uuidToQueueNo(jobId)).add(jobId, webScraperOptions, {...options,priority: jobPriority,jobId,@@ -320,7 +320,7 @@ export async function waitForJob(logger: Logger = _logger,): Promise<Document> {const start = Date.now();- const queue = getScrapeQueue();+ const queue = getScrapeQueue(uuidToQueueNo(typeof _job == "string" ? _job : _job.id!));let job: Job | undefined = typeof _job == "string" ? await queue.getJob(_job) : _job;while (job === undefined) {logger.debug("Waiting for job to be created");@@ -331,7 +331,7 @@ export async function waitForJob(}}let doc: Document = await Promise.race([- job.waitUntilFinished(getScrapeQueueEvents(), timeout - (Date.now() - start)),+ job.waitUntilFinished(getScrapeQueueEvents(uuidToQueueNo(job.id!)), timeout - (Date.now() - start)),new Promise((resolve, reject) => {setTimeout(() => {reject(new Error("Job wait "));diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.tsindex ab184efb6..b86a4ac7f 100644--- a/apps/api/src/services/queue-service.ts+++ b/apps/api/src/services/queue-service.ts@@ -4,11 +4,9 @@ import IORedis from "ioredis";export type QueueFunction = () => Queue<any, any, string, any, any, string>;-let scrapeQueue: Queue;-let scrapeQueueEvents: QueueEvents;+let scrapeQueues: Queue[] = [];+let scrapeQueueEvents: QueueEvents[] = [];let extractQueue: Queue;-let loggingQueue: Queue;-let indexQueue: Queue;let deepResearchQueue: Queue;let generateLlmsTxtQueue: Queue;let billingQueue: Queue;@@ -21,7 +19,6 @@ export const redisConnection = new IORedis(process.env.REDIS_URL!, {redisConnection.on("reconnecting", () => logger.warn("Redis reconnecting"));redisConnection.on("error", (err) => logger.warn("Redis error", { err }));-export const scrapeQueueName = "{scrapeQueue}";export const extractQueueName = "{extractQueue}";export const loggingQueueName = "{loggingQueue}";export const indexQueueName = "{indexQueue}";@@ -30,9 +27,21 @@ export const deepResearchQueueName = "{deepResearchQueue}";export const billingQueueName = "{billingQueue}";export const precrawlQueueName = "{precrawlQueue}";-export function getScrapeQueue() {- if (!scrapeQueue) {- scrapeQueue = new Queue(scrapeQueueName, {+// Length of this array must evenly divide 16.+export const scrapeQueueNames = [+ "{scrapeQueue0}",+ "{scrapeQueue1}",+ "{scrapeQueue2}",+ "{scrapeQueue3}",+];++export function uuidToQueueNo(id: string) {+ return parseInt(id[0], 16) % scrapeQueueNames.length;+}++export function getScrapeQueue(i: number) {+ if (!scrapeQueues[i]) {+ scrapeQueues[i] = new Queue(scrapeQueueNames[i], {connection: redisConnection,defaultJobOptions: {removeOnComplete: {@@ -44,19 +53,19 @@ export function getScrapeQueue() {},});}- return scrapeQueue;+ return scrapeQueues[i];}-export function getScrapeQueueEvents() {- if (!scrapeQueueEvents) {- scrapeQueueEvents = new QueueEvents(scrapeQueueName, {+export function getScrapeQueueEvents(i: number) {+ if (!scrapeQueueEvents[i]) {+ scrapeQueueEvents[i] = new QueueEvents(scrapeQueueNames[i], {connection: new IORedis(process.env.REDIS_URL!, {maxRetriesPerRequest: null,}),});}- return scrapeQueueEvents;+ return scrapeQueueEvents[i];}export function getExtractQueue() {diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex ce3abece4..20b42a524 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -7,7 +7,9 @@ import {getDeepResearchQueue,redisConnection,getGenerateLlmsTxtQueue,- scrapeQueueName,+ uuidToQueueNo,+ scrapeQueueNames,+ getScrapeQueueEvents,} from "./queue-service";import { Job, Queue, QueueEvents } from "bullmq";import { logger as _logger } from "../lib/logger";@@ -336,7 +338,7 @@ const separateWorkerFun = (lockDuration: 30 * 1000, // 30 secondsstalledInterval: 30 * 1000, // 30 secondsmaxStalledCount: 10, // 10 times- concurrency: 6, // from k8s setup+ concurrency: Math.ceil(6 / scrapeQueueNames.length), // from k8s setupuseWorkerThreads: true,});@@ -470,7 +472,7 @@ app.listen(workerPort, () => {return;}- const job = await getScrapeQueue().getJob(args.jobId);+ const job = await getScrapeQueue(uuidToQueueNo(args.jobId)).getJob(args.jobId);let logger = _logger.child({ jobId: args.jobId, scrapeId: args.jobId, module: "queue-worker", method: "failedListener", zeroDataRetention: job?.data.zeroDataRetention });if (job && job.data.crawl_id) {@@ -502,11 +504,12 @@ app.listen(workerPort, () => {}}- const scrapeQueueEvents = new QueueEvents(scrapeQueueName, { connection: redisConnection });- scrapeQueueEvents.on("failed", failedListener);+ scrapeQueueNames.forEach((_, i) => {+ getScrapeQueueEvents(i).on("failed", failedListener);+ });const results = await Promise.all([- separateWorkerFun(getScrapeQueue(), path.join(__dirname, "worker", "scrape-worker.js")),+ ...scrapeQueueNames.map((_, i) => separateWorkerFun(getScrapeQueue(i), path.join(__dirname, "worker", "scrape-worker.js"))),workerFun(getExtractQueue(), processExtractJobInternal),workerFun(getDeepResearchQueue(), processDeepResearchJobInternal),workerFun(getGenerateLlmsTxtQueue(), processGenerateLlmsTxtJobInternal),@@ -524,12 +527,12 @@ app.listen(workerPort, () => {setInterval(async () => {_logger.debug("Currently running jobs", {jobs: (await Promise.all([...runningJobs].map(async (jobId) => {- return await getScrapeQueue().getJob(jobId);+ return await getScrapeQueue(uuidToQueueNo(jobId)).getJob(jobId);}))).filter(x => x && !x.data?.zeroDataRetention),});}, 1000);- await scrapeQueueEvents.close();+ await Promise.all(scrapeQueueNames.map((_, i) => getScrapeQueueEvents(i).close()));console.log("All jobs finished. Worker out!");process.exit(0);})();\ No newline at end of file