Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat: queue multiplexing (#1902)

Gergő Móricze6a61e3d19c5
diff --git a/apps/api/src/controllers/v0/admin/queue.ts b/apps/api/src/controllers/v0/admin/queue.ts
index d7d9c0897..f775e2c81 100644
--- a/apps/api/src/controllers/v0/admin/queue.ts
+++ b/apps/api/src/controllers/v0/admin/queue.ts
@@ -12,7 +12,7 @@ export async function cleanBefore24hCompleteJobsController(
) {
logger.info("🐂 Cleaning jobs older than 24h");
try {
- const scrapeQueue = getScrapeQueue();
+ const scrapeQueue = getScrapeQueue(0);
const batchSize = 10;
const numberOfBatches = 9; // Adjust based on your needs
const completedJobsPromises: Promise<Job[]>[] = [];
@@ -70,7 +70,7 @@ export async function checkQueuesController(req: Request, res: Response) {
// Use this as a "health check" that way we dont destroy the server
export async function queuesController(req: Request, res: Response) {
try {
- const scrapeQueue = getScrapeQueue();
+ const scrapeQueue = getScrapeQueue(0);
const [webScraperActive] = await Promise.all([
scrapeQueue.getActiveCount(),
@@ -93,7 +93,7 @@ export async function autoscalerController(req: Request, res: Response) {
const maxNumberOfMachines = 80;
const minNumberOfMachines = 20;
- const scrapeQueue = getScrapeQueue();
+ const scrapeQueue = getScrapeQueue(0);
const [webScraperActive, webScraperWaiting, webScraperPriority] =
await Promise.all([
diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.ts
index 9f6e5b7bb..6fef4b64c 100644
--- a/apps/api/src/controllers/v0/crawl-status.ts
+++ b/apps/api/src/controllers/v0/crawl-status.ts
@@ -1,7 +1,7 @@
import { Request, Response } from "express";
import { authenticateUser } from "../auth";
import { RateLimiterMode } from "../../../src/types";
-import { getScrapeQueue } from "../../../src/services/queue-service";
+import { getScrapeQueue, uuidToQueueNo } from "../../../src/services/queue-service";
import { redisEvictConnection } from "../../../src/services/redis";
import { logger } from "../../../src/lib/logger";
import { getCrawl, getCrawlJobs } from "../../../src/lib/crawl-redis";
@@ -16,7 +16,7 @@ configDotenv();
export async function getJobs(crawlId: string, ids: string[]): Promise<PseudoJob<any>[]> {
const [bullJobs, dbJobs, gcsJobs] = await Promise.all([
- Promise.all(ids.map((x) => getScrapeQueue().getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,
+ Promise.all(ids.map((x) => getScrapeQueue(uuidToQueueNo(x)).getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,
process.env.USE_DB_AUTHENTICATION === "true" ? await supabaseGetJobsByCrawlId(crawlId) : [],
process.env.GCS_BUCKET_NAME ? Promise.all(ids.map(async (x) => ({ id: x, job: await getJobFromGCS(x) }))).then(x => x.filter(x => x.job)) as Promise<({ id: string, job: any | null })[]> : [],
]);
diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.ts
index d4bd73bbc..c1a9d57a4 100644
--- a/apps/api/src/controllers/v0/scrape.ts
+++ b/apps/api/src/controllers/v0/scrape.ts
@@ -21,7 +21,7 @@ import {
defaultOrigin,
} from "../../lib/default-values";
import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
-import { getScrapeQueue } from "../../services/queue-service";
+import { getScrapeQueue, uuidToQueueNo } from "../../services/queue-service";
import { redisEvictConnection } from "../../../src/services/redis";
import { v4 as uuidv4 } from "uuid";
import { logger } from "../../lib/logger";
@@ -128,7 +128,7 @@ export async function scrapeHelper(
return err;
}
- await getScrapeQueue().remove(jobId);
+ await getScrapeQueue(uuidToQueueNo(jobId)).remove(jobId);
if (!doc) {
console.error("!!! PANIC DOC IS", doc);
diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.ts
index aaff178c2..35233fd3b 100644
--- a/apps/api/src/controllers/v0/search.ts
+++ b/apps/api/src/controllers/v0/search.ts
@@ -11,7 +11,7 @@ import { search } from "../../search";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import { v4 as uuidv4 } from "uuid";
import { logger } from "../../lib/logger";
-import { getScrapeQueue } from "../../services/queue-service";
+import { getScrapeQueue, uuidToQueueNo } from "../../services/queue-service";
import { redisEvictConnection } from "../../../src/services/redis";
import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import * as Sentry from "@sentry/node";
@@ -137,8 +137,7 @@ export async function searchHelper(
return { success: true, error: "No search results found", returnCode: 200 };
}
- const sq = getScrapeQueue();
- await Promise.all(jobDatas.map((x) => sq.remove(x.opts.jobId)));
+ await Promise.all(jobDatas.map((x) => getScrapeQueue(uuidToQueueNo(x.opts.jobId)).remove(x.opts.jobId)));
// make sure doc.content is not empty
const filteredDocs = docs.filter(
diff --git a/apps/api/src/controllers/v1/crawl-errors.ts b/apps/api/src/controllers/v1/crawl-errors.ts
index d8d029ee6..57eb9eb21 100644
--- a/apps/api/src/controllers/v1/crawl-errors.ts
+++ b/apps/api/src/controllers/v1/crawl-errors.ts
@@ -8,14 +8,14 @@ import {
getCrawl,
getCrawlJobs,
} from "../../lib/crawl-redis";
-import { getScrapeQueue } from "../../services/queue-service";
+import { getScrapeQueue, uuidToQueueNo } from "../../services/queue-service";
import { redisEvictConnection } from "../../../src/services/redis";
import { configDotenv } from "dotenv";
import { Job } from "bullmq";
configDotenv();
export async function getJob(id: string) {
- const job = await getScrapeQueue().getJob(id);
+ const job = await getScrapeQueue(uuidToQueueNo(id)).getJob(id);
if (!job) return job;
return job;
@@ -23,7 +23,7 @@ export async function getJob(id: string) {
export async function getJobs(ids: string[]) {
const jobs: (Job & { id: string })[] = (
- await Promise.all(ids.map((x) => getScrapeQueue().getJob(x)))
+ await Promise.all(ids.map((x) => getScrapeQueue(uuidToQueueNo(x)).getJob(x)))
).filter((x) => x) as (Job & { id: string })[];
return jobs;
@@ -44,7 +44,7 @@ export async function crawlErrorsController(
let jobStatuses = await Promise.all(
(await getCrawlJobs(req.params.jobId)).map(
- async (x) => [x, await getScrapeQueue().getJobState(x)] as const,
+ async (x) => [x, await getScrapeQueue(uuidToQueueNo(x)).getJobState(x)] as const,
),
);
diff --git a/apps/api/src/controllers/v1/crawl-status-ws.ts b/apps/api/src/controllers/v1/crawl-status-ws.ts
index 93e1fc9b1..3ec7e61d3 100644
--- a/apps/api/src/controllers/v1/crawl-status-ws.ts
+++ b/apps/api/src/controllers/v1/crawl-status-ws.ts
@@ -20,7 +20,7 @@ import {
isCrawlFinished,
isCrawlFinishedLocked,
} from "../../lib/crawl-redis";
-import { getScrapeQueue } from "../../services/queue-service";
+import { getScrapeQueue, uuidToQueueNo } from "../../services/queue-service";
import { getJob, getJobs } from "./crawl-status";
import * as Sentry from "@sentry/node";
import { Job, JobState } from "bullmq";
@@ -89,12 +89,10 @@ async function crawlStatusWS(
const notDoneJobIDs = jobIDs.filter((x) => !doneJobIDs.includes(x));
- const queue = getScrapeQueue();
-
const jobStatuses = await Promise.all(
notDoneJobIDs.map(async (x) => [
x,
- await queue.getJobState(x),
+ await getScrapeQueue(uuidToQueueNo(x)).getJobState(x),
]),
);
const newlyDoneJobIDs: string[] = jobStatuses
@@ -126,11 +124,9 @@ async function crawlStatusWS(
let jobIDs = await getCrawlJobs(req.params.jobId);
- const queue = getScrapeQueue();
-
let jobStatuses = await Promise.all(
jobIDs.map(
- async (x) => [x, await queue.getJobState(x)] as const,
+ async (x) => [x, await getScrapeQueue(uuidToQueueNo(x)).getJobState(x)] as const,
),
);
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts
index 4da7112cf..faf7d3037 100644
--- a/apps/api/src/controllers/v1/crawl-status.ts
+++ b/apps/api/src/controllers/v1/crawl-status.ts
@@ -13,7 +13,7 @@ import {
getDoneJobsOrderedLength,
isCrawlKickoffFinished,
} from "../../lib/crawl-redis";
-import { getScrapeQueue } from "../../services/queue-service";
+import { getScrapeQueue, uuidToQueueNo } from "../../services/queue-service";
import {
supabaseGetJobById,
supabaseGetJobsById,
@@ -42,7 +42,7 @@ export type DBJob = { docs: any, success: boolean, page_options: any, date_added
export async function getJob(id: string): Promise<PseudoJob<any> | null> {
const [bullJob, dbJob, gcsJob] = await Promise.all([
- getScrapeQueue().getJob(id),
+ getScrapeQueue(uuidToQueueNo(id)).getJob(id),
(process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobById(id) : null) as Promise<DBJob | null>,
(process.env.GCS_BUCKET_NAME ? getJobFromGCS(id) : null) as Promise<any | null>,
]);
@@ -74,7 +74,7 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {
export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {
const [bullJobs, dbJobs, gcsJobs] = await Promise.all([
- Promise.all(ids.map((x) => getScrapeQueue().getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,
+ Promise.all(ids.map((x) => getScrapeQueue(uuidToQueueNo(x)).getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,
process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobsById(ids) : [],
process.env.GCS_BUCKET_NAME ? Promise.all(ids.map(async (x) => ({ id: x, job: await getJobFromGCS(x) }))).then(x => x.filter(x => x.job)) as Promise<({ id: string, job: any | null })[]> : [],
]);
@@ -159,7 +159,7 @@ export async function crawlStatusController(
let jobIDs = await getCrawlJobs(req.params.jobId);
let jobStatuses = await Promise.all(
jobIDs.map(
- async (x) => [x, await getScrapeQueue().getJobState(x)] as const,
+ async (x) => [x, await getScrapeQueue(uuidToQueueNo(x)).getJobState(x)] as const,
),
);
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index 658b89273..17008b8b5 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -10,7 +10,7 @@ import {
import { v4 as uuidv4 } from "uuid";
import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import { getJobPriority } from "../../lib/job-priority";
-import { getScrapeQueue } from "../../services/queue-service";
+import { getScrapeQueue, uuidToQueueNo } from "../../services/queue-service";
export async function scrapeController(
req: RequestWithAuth<{}, ScrapeResponse, ScrapeRequest>,
@@ -105,7 +105,7 @@ export async function scrapeController(
});
if (zeroDataRetention) {
- await getScrapeQueue().remove(jobId);
+ await getScrapeQueue(uuidToQueueNo(jobId)).remove(jobId);
}
if (
@@ -126,7 +126,7 @@ export async function scrapeController(
logger.info("Done with waitForJob");
- await getScrapeQueue().remove(jobId);
+ await getScrapeQueue(uuidToQueueNo(jobId)).remove(jobId);
logger.info("Removed job from queue");
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index 82efbcfb3..37bc3499c 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -15,7 +15,7 @@ import { addScrapeJob, waitForJob } from "../../services/queue-jobs";
import { logJob } from "../../services/logging/log_job";
import { getJobPriority } from "../../lib/job-priority";
import { Mode } from "../../types";
-import { getScrapeQueue } from "../../services/queue-service";
+import { getScrapeQueue, uuidToQueueNo } from "../../services/queue-service";
import { search } from "../../search";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
import * as Sentry from "@sentry/node";
@@ -133,7 +133,7 @@ async function scrapeSearchResult(
teamId: options.teamId,
origin: options.origin,
});
- await getScrapeQueue().remove(jobId);
+ await getScrapeQueue(uuidToQueueNo(jobId)).remove(jobId);
const document = {
title: searchResult.title,
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index b6fc1e3fa..cbfaeb5c2 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -11,6 +11,7 @@ import {
getDeepResearchQueue,
getBillingQueue,
getPrecrawlQueue,
+ scrapeQueueNames,
} from "./services/queue-service";
import { v0Router } from "./routes/v0";
import os from "os";
@@ -55,7 +56,7 @@ serverAdapter.setBasePath(`/admin/${process.env.BULL_AUTH_KEY}/queues`);
const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({
queues: [
- new BullMQAdapter(getScrapeQueue()),
+ ...scrapeQueueNames.map((_, i) => new BullMQAdapter(getScrapeQueue(i))),
new BullMQAdapter(getExtractQueue()),
new BullMQAdapter(getGenerateLlmsTxtQueue()),
new BullMQAdapter(getDeepResearchQueue()),
@@ -119,8 +120,7 @@ if (require.main === module) {
app.get(`/serverHealthCheck`, async (req, res) => {
try {
- const scrapeQueue = getScrapeQueue();
- const [waitingJobs] = await Promise.all([scrapeQueue.getWaitingCount()]);
+ const waitingJobs = (await Promise.all(scrapeQueueNames.map((_, i) => getScrapeQueue(i).getWaitingCount()))).reduce((acc, curr) => acc + curr, 0);
const noWaitingJobs = waitingJobs === 0;
// 200 if no active jobs, 503 if there are active jobs
return res.status(noWaitingJobs ? 200 : 500).json({
@@ -139,10 +139,7 @@ app.get("/serverHealthCheck/notify", async (req, res) => {
const timeout = 60000; // 1 minute // The timeout value for the check in milliseconds
const getWaitingJobsCount = async () => {
- const scrapeQueue = getScrapeQueue();
- const [waitingJobsCount] = await Promise.all([
- scrapeQueue.getWaitingCount(),
- ]);
+ const waitingJobsCount = (await Promise.all(scrapeQueueNames.map((_, i) => getScrapeQueue(i).getWaitingCount()))).reduce((acc, curr) => acc + curr, 0);
return waitingJobsCount;
};
diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.ts
index 022f3523b..4ab8f9467 100644
--- a/apps/api/src/lib/concurrency-limit.ts
+++ b/apps/api/src/lib/concurrency-limit.ts
@@ -3,7 +3,7 @@ import { redisEvictConnection } from "../services/redis";
import type { Job, JobsOptions } from "bullmq";
import { getACUCTeam } from "../controllers/auth";
import { getCrawl, StoredCrawl } from "./crawl-redis";
-import { getScrapeQueue } from "../services/queue-service";
+import { getScrapeQueue, uuidToQueueNo } from "../services/queue-service";
import { logger } from "./logger";
const constructKey = (team_id: string) => "concurrency-limiter:" + team_id;
@@ -258,7 +258,7 @@ export async function concurrentJobDone(job: Job) {
}
}
- (await getScrapeQueue()).add(
+ (await getScrapeQueue(uuidToQueueNo(nextJob.job.id))).add(
nextJob.job.id,
{
...nextJob.job.data,
diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.ts
index 083dd2628..16719bb04 100644
--- a/apps/api/src/lib/extract/document-scraper.ts
+++ b/apps/api/src/lib/extract/document-scraper.ts
@@ -1,6 +1,6 @@
import { Document, ScrapeOptions, TeamFlags, URLTrace, scrapeOptions } from "../../controllers/v1/types";
import { logger } from "../logger";
-import { getScrapeQueue } from "../../services/queue-service";
+import { getScrapeQueue, uuidToQueueNo } from "../../services/queue-service";
import { waitForJob } from "../../services/queue-jobs";
import { addScrapeJob } from "../../services/queue-jobs";
import { getJobPriority } from "../job-priority";
@@ -68,7 +68,7 @@ export async function scrapeDocument(
const doc = await waitForJob(jobId, timeout);
- await getScrapeQueue().remove(jobId);
+ await getScrapeQueue(uuidToQueueNo(jobId)).remove(jobId);
if (trace) {
trace.timing.completedAt = new Date().toISOString();
diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
index c6ecca291..185d882db 100644
--- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
@@ -1,6 +1,6 @@
import { Document, ScrapeOptions, TeamFlags, URLTrace, scrapeOptions } from "../../../controllers/v1/types";
import { logger } from "../../logger";
-import { getScrapeQueue } from "../../../services/queue-service";
+import { getScrapeQueue, uuidToQueueNo } from "../../../services/queue-service";
import { waitForJob } from "../../../services/queue-jobs";
import { addScrapeJob } from "../../../services/queue-jobs";
import { getJobPriority } from "../../job-priority";
@@ -65,7 +65,7 @@ export async function scrapeDocument_F0(
);
const doc = await waitForJob(jobId, timeout);
- await getScrapeQueue().remove(jobId);
+ await getScrapeQueue(uuidToQueueNo(jobId)).remove(jobId);
if (trace) {
trace.timing.completedAt = new Date().toISOString();
diff --git a/apps/api/src/services/alerts/index.ts b/apps/api/src/services/alerts/index.ts
index 44f2b8a04..c257f8d63 100644
--- a/apps/api/src/services/alerts/index.ts
+++ b/apps/api/src/services/alerts/index.ts
@@ -1,5 +1,5 @@
import { logger } from "../../../src/lib/logger";
-import { getScrapeQueue } from "../queue-service";
+import { getScrapeQueue, scrapeQueueNames } from "../queue-service";
import { sendSlackWebhook } from "./slack";
export async function checkAlerts() {
@@ -13,8 +13,7 @@ export async function checkAlerts() {
logger.info("Initializing alerts");
const checkActiveJobs = async () => {
try {
- const scrapeQueue = getScrapeQueue();
- const activeJobs = await scrapeQueue.getActiveCount();
+ const activeJobs = (await Promise.all(scrapeQueueNames.map((_, i) => getScrapeQueue(i).getActiveCount()))).reduce((acc, curr) => acc + curr, 0);
if (activeJobs > Number(process.env.ALERT_NUM_ACTIVE_JOBS)) {
logger.warn(
`Alert: Number of active jobs is over ${process.env.ALERT_NUM_ACTIVE_JOBS}. Current active jobs: ${activeJobs}.`,
@@ -34,8 +33,7 @@ export async function checkAlerts() {
};
const checkWaitingQueue = async () => {
- const scrapeQueue = getScrapeQueue();
- const waitingJobs = await scrapeQueue.getWaitingCount();
+ const waitingJobs = (await Promise.all(scrapeQueueNames.map((_, i) => getScrapeQueue(i).getWaitingCount()))).reduce((acc, curr) => acc + curr, 0);
if (waitingJobs > Number(process.env.ALERT_NUM_WAITING_JOBS)) {
logger.warn(
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts
index d6947659a..ef8976b72 100644
--- a/apps/api/src/services/queue-jobs.ts
+++ b/apps/api/src/services/queue-jobs.ts
@@ -1,4 +1,4 @@
-import { getScrapeQueue, getScrapeQueueEvents } from "./queue-service";
+import { getScrapeQueue, getScrapeQueueEvents, uuidToQueueNo } from "./queue-service";
import { v4 as uuidv4 } from "uuid";
import { NotificationType, RateLimiterMode, WebScraperOptions } from "../types";
import * as Sentry from "@sentry/node";
@@ -70,7 +70,7 @@ export async function _addScrapeJobToBullMQ(
}
}
- return await getScrapeQueue().add(jobId, webScraperOptions, {
+ return await getScrapeQueue(uuidToQueueNo(jobId)).add(jobId, webScraperOptions, {
...options,
priority: jobPriority,
jobId,
@@ -320,7 +320,7 @@ export async function waitForJob(
logger: Logger = _logger,
): Promise<Document> {
const start = Date.now();
- const queue = getScrapeQueue();
+ const queue = getScrapeQueue(uuidToQueueNo(typeof _job == "string" ? _job : _job.id!));
let job: Job | undefined = typeof _job == "string" ? await queue.getJob(_job) : _job;
while (job === undefined) {
logger.debug("Waiting for job to be created");
@@ -331,7 +331,7 @@ export async function waitForJob(
}
}
let doc: Document = await Promise.race([
- job.waitUntilFinished(getScrapeQueueEvents(), timeout - (Date.now() - start)),
+ job.waitUntilFinished(getScrapeQueueEvents(uuidToQueueNo(job.id!)), timeout - (Date.now() - start)),
new Promise((resolve, reject) => {
setTimeout(() => {
reject(new Error("Job wait "));
diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.ts
index ab184efb6..b86a4ac7f 100644
--- a/apps/api/src/services/queue-service.ts
+++ b/apps/api/src/services/queue-service.ts
@@ -4,11 +4,9 @@ import IORedis from "ioredis";
export type QueueFunction = () => Queue<any, any, string, any, any, string>;
-let scrapeQueue: Queue;
-let scrapeQueueEvents: QueueEvents;
+let scrapeQueues: Queue[] = [];
+let scrapeQueueEvents: QueueEvents[] = [];
let extractQueue: Queue;
-let loggingQueue: Queue;
-let indexQueue: Queue;
let deepResearchQueue: Queue;
let generateLlmsTxtQueue: Queue;
let billingQueue: Queue;
@@ -21,7 +19,6 @@ export const redisConnection = new IORedis(process.env.REDIS_URL!, {
redisConnection.on("reconnecting", () => logger.warn("Redis reconnecting"));
redisConnection.on("error", (err) => logger.warn("Redis error", { err }));
-export const scrapeQueueName = "{scrapeQueue}";
export const extractQueueName = "{extractQueue}";
export const loggingQueueName = "{loggingQueue}";
export const indexQueueName = "{indexQueue}";
@@ -30,9 +27,21 @@ export const deepResearchQueueName = "{deepResearchQueue}";
export const billingQueueName = "{billingQueue}";
export const precrawlQueueName = "{precrawlQueue}";
-export function getScrapeQueue() {
- if (!scrapeQueue) {
- scrapeQueue = new Queue(scrapeQueueName, {
+// Length of this array must evenly divide 16.
+export const scrapeQueueNames = [
+ "{scrapeQueue0}",
+ "{scrapeQueue1}",
+ "{scrapeQueue2}",
+ "{scrapeQueue3}",
+];
+
+export function uuidToQueueNo(id: string) {
+ return parseInt(id[0], 16) % scrapeQueueNames.length;
+}
+
+export function getScrapeQueue(i: number) {
+ if (!scrapeQueues[i]) {
+ scrapeQueues[i] = new Queue(scrapeQueueNames[i], {
connection: redisConnection,
defaultJobOptions: {
removeOnComplete: {
@@ -44,19 +53,19 @@ export function getScrapeQueue() {
},
});
}
- return scrapeQueue;
+ return scrapeQueues[i];
}
-export function getScrapeQueueEvents() {
- if (!scrapeQueueEvents) {
- scrapeQueueEvents = new QueueEvents(scrapeQueueName, {
+export function getScrapeQueueEvents(i: number) {
+ if (!scrapeQueueEvents[i]) {
+ scrapeQueueEvents[i] = new QueueEvents(scrapeQueueNames[i], {
connection: new IORedis(process.env.REDIS_URL!, {
maxRetriesPerRequest: null,
}),
});
}
- return scrapeQueueEvents;
+ return scrapeQueueEvents[i];
}
export function getExtractQueue() {
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index ce3abece4..20b42a524 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -7,7 +7,9 @@ import {
getDeepResearchQueue,
redisConnection,
getGenerateLlmsTxtQueue,
- scrapeQueueName,
+ uuidToQueueNo,
+ scrapeQueueNames,
+ getScrapeQueueEvents,
} from "./queue-service";
import { Job, Queue, QueueEvents } from "bullmq";
import { logger as _logger } from "../lib/logger";
@@ -336,7 +338,7 @@ const separateWorkerFun = (
lockDuration: 30 * 1000, // 30 seconds
stalledInterval: 30 * 1000, // 30 seconds
maxStalledCount: 10, // 10 times
- concurrency: 6, // from k8s setup
+ concurrency: Math.ceil(6 / scrapeQueueNames.length), // from k8s setup
useWorkerThreads: true,
});
@@ -470,7 +472,7 @@ app.listen(workerPort, () => {
return;
}
- const job = await getScrapeQueue().getJob(args.jobId);
+ const job = await getScrapeQueue(uuidToQueueNo(args.jobId)).getJob(args.jobId);
let logger = _logger.child({ jobId: args.jobId, scrapeId: args.jobId, module: "queue-worker", method: "failedListener", zeroDataRetention: job?.data.zeroDataRetention });
if (job && job.data.crawl_id) {
@@ -502,11 +504,12 @@ app.listen(workerPort, () => {
}
}
- const scrapeQueueEvents = new QueueEvents(scrapeQueueName, { connection: redisConnection });
- scrapeQueueEvents.on("failed", failedListener);
+ scrapeQueueNames.forEach((_, i) => {
+ getScrapeQueueEvents(i).on("failed", failedListener);
+ });
const results = await Promise.all([
- separateWorkerFun(getScrapeQueue(), path.join(__dirname, "worker", "scrape-worker.js")),
+ ...scrapeQueueNames.map((_, i) => separateWorkerFun(getScrapeQueue(i), path.join(__dirname, "worker", "scrape-worker.js"))),
workerFun(getExtractQueue(), processExtractJobInternal),
workerFun(getDeepResearchQueue(), processDeepResearchJobInternal),
workerFun(getGenerateLlmsTxtQueue(), processGenerateLlmsTxtJobInternal),
@@ -524,12 +527,12 @@ app.listen(workerPort, () => {
setInterval(async () => {
_logger.debug("Currently running jobs", {
jobs: (await Promise.all([...runningJobs].map(async (jobId) => {
- return await getScrapeQueue().getJob(jobId);
+ return await getScrapeQueue(uuidToQueueNo(jobId)).getJob(jobId);
}))).filter(x => x && !x.data?.zeroDataRetention),
});
}, 1000);
- await scrapeQueueEvents.close();
+ await Promise.all(scrapeQueueNames.map((_, i) => getScrapeQueueEvents(i).close()));
console.log("All jobs finished. Worker out!");
process.exit(0);
})();
\ No newline at end of file