Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

feat(api): queue multiplexing

GergΕ‘ MΓ³ricz23b079052eb4
diff --git a/apps/api/src/controllers/v0/admin/queue.ts b/apps/api/src/controllers/v0/admin/queue.ts
index d7d9c0897..4d80918fc 100644
--- a/apps/api/src/controllers/v0/admin/queue.ts
+++ b/apps/api/src/controllers/v0/admin/queue.ts
@@ -2,61 +2,10 @@ import { Request, Response } from "express";
import { Job } from "bullmq";
import { logger } from "../../../lib/logger";
-import { getScrapeQueue } from "../../../services/queue-service";
+import { getScrapeQueue, getScrapeQueueByIndex, queueMultiplexWidth } from "../../../services/queue-service";
import { checkAlerts } from "../../../services/alerts";
import { sendSlackWebhook } from "../../../services/alerts/slack";
-export async function cleanBefore24hCompleteJobsController(
- req: Request,
- res: Response,
-) {
- logger.info("πŸ‚ Cleaning jobs older than 24h");
- try {
- const scrapeQueue = getScrapeQueue();
- const batchSize = 10;
- const numberOfBatches = 9; // Adjust based on your needs
- const completedJobsPromises: Promise<Job[]>[] = [];
- for (let i = 0; i < numberOfBatches; i++) {
- completedJobsPromises.push(
- scrapeQueue.getJobs(
- ["completed"],
- i * batchSize,
- i * batchSize + batchSize,
- true,
- ),
- );
- }
- const completedJobs: Job[] = (
- await Promise.all(completedJobsPromises)
- ).flat();
- const before24hJobs =
- completedJobs.filter(
- (job) =>
- job.finishedOn !== undefined &&
- job.finishedOn < Date.now() - 24 * 60 * 60 * 1000,
- ) || [];
-
- let count = 0;
-
- if (!before24hJobs) {
- return res.status(200).send(`No jobs to remove.`);
- }
-
- for (const job of before24hJobs) {
- try {
- await job.remove();
- count++;
- } catch (jobError) {
- logger.error(`πŸ‚ Failed to remove job with ID ${job.id}: ${jobError}`);
- }
- }
- return res.status(200).send(`Removed ${count} completed jobs.`);
- } catch (error) {
- logger.error(`πŸ‚ Failed to clean last 24h complete jobs: ${error}`);
- return res.status(500).send("Failed to clean jobs");
- }
-}
-
export async function checkQueuesController(req: Request, res: Response) {
try {
await checkAlerts();
@@ -70,11 +19,11 @@ export async function checkQueuesController(req: Request, res: Response) {
// Use this as a "health check" that way we dont destroy the server
export async function queuesController(req: Request, res: Response) {
try {
- const scrapeQueue = getScrapeQueue();
+ const scrapeQueues = new Array(queueMultiplexWidth)
+ .fill(0)
+ .map((_, index) => getScrapeQueueByIndex(index));
- const [webScraperActive] = await Promise.all([
- scrapeQueue.getActiveCount(),
- ]);
+ const webScraperActive = (await Promise.all(scrapeQueues.map((queue) => queue.getActiveCount()))).reduce((a, b) => a + b, 0);
const noActiveJobs = webScraperActive === 0;
// 200 if no active jobs, 503 if there are active jobs
@@ -87,115 +36,3 @@ export async function queuesController(req: Request, res: Response) {
return res.status(500).json({ error: error.message });
}
}
-
-export async function autoscalerController(req: Request, res: Response) {
- try {
- const maxNumberOfMachines = 80;
- const minNumberOfMachines = 20;
-
- const scrapeQueue = getScrapeQueue();
-
- const [webScraperActive, webScraperWaiting, webScraperPriority] =
- await Promise.all([
- scrapeQueue.getActiveCount(),
- scrapeQueue.getWaitingCount(),
- scrapeQueue.getPrioritizedCount(),
- ]);
-
- let waitingAndPriorityCount = webScraperWaiting + webScraperPriority;
-
- // get number of machines active
- const request = await fetch(
- "https://api.machines.dev/v1/apps/firecrawl-scraper-js/machines",
- {
- headers: {
- Authorization: `Bearer ${process.env.FLY_API_TOKEN}`,
- },
- },
- );
- const machines = await request.json();
-
- // Only worker machines
- const activeMachines = machines.filter(
- (machine) =>
- (machine.state === "started" ||
- machine.state === "starting" ||
- machine.state === "replacing") &&
- machine.config.env["FLY_PROCESS_GROUP"] === "worker",
- ).length;
-
- let targetMachineCount = activeMachines;
-
- const baseScaleUp = 10;
- // Slow scale down
- const baseScaleDown = 2;
-
- // Scale up logic
- if (webScraperActive > 9000 || waitingAndPriorityCount > 2000) {
- targetMachineCount = Math.min(
- maxNumberOfMachines,
- activeMachines + baseScaleUp * 3,
- );
- } else if (webScraperActive > 5000 || waitingAndPriorityCount > 1000) {
- targetMachineCount = Math.min(
- maxNumberOfMachines,
- activeMachines + baseScaleUp * 2,
- );
- } else if (webScraperActive > 1000 || waitingAndPriorityCount > 500) {
- targetMachineCount = Math.min(
- maxNumberOfMachines,
- activeMachines + baseScaleUp,
- );
- }
-
- // Scale down logic
- if (webScraperActive < 100 && waitingAndPriorityCount < 50) {
- targetMachineCount = Math.max(
- minNumberOfMachines,
- activeMachines - baseScaleDown * 3,
- );
- } else if (webScraperActive < 500 && waitingAndPriorityCount < 200) {
- targetMachineCount = Math.max(
- minNumberOfMachines,
- activeMachines - baseScaleDown * 2,
- );
- } else if (webScraperActive < 1000 && waitingAndPriorityCount < 500) {
- targetMachineCount = Math.max(
- minNumberOfMachines,
- activeMachines - baseScaleDown,
- );
- }
-
- if (targetMachineCount !== activeMachines) {
- logger.info(
- `πŸ‚ Scaling from ${activeMachines} to ${targetMachineCount} - ${webScraperActive} active, ${webScraperWaiting} waiting`,
- );
-
- if (targetMachineCount > activeMachines) {
- sendSlackWebhook(
- `πŸ‚ Scaling from ${activeMachines} to ${targetMachineCount} - ${webScraperActive} active, ${webScraperWaiting} waiting - Current DateTime: ${new Date().toISOString()}`,
- false,
- process.env.SLACK_AUTOSCALER ?? "",
- );
- } else {
- sendSlackWebhook(
- `πŸ‚ Scaling from ${activeMachines} to ${targetMachineCount} - ${webScraperActive} active, ${webScraperWaiting} waiting - Current DateTime: ${new Date().toISOString()}`,
- false,
- process.env.SLACK_AUTOSCALER ?? "",
- );
- }
- return res.status(200).json({
- mode: "scale-descale",
- count: targetMachineCount,
- });
- }
-
- return res.status(200).json({
- mode: "normal",
- count: activeMachines,
- });
- } catch (error) {
- logger.error(error);
- return res.status(500).send("Failed to initialize autoscaler");
- }
-}
diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.ts
index 843eb56d1..d76f5ebfa 100644
--- a/apps/api/src/controllers/v0/crawl-status.ts
+++ b/apps/api/src/controllers/v0/crawl-status.ts
@@ -16,7 +16,7 @@ configDotenv();
export async function getJobs(crawlId: string, ids: string[]): Promise<PseudoJob<any>[]> {
const [bullJobs, dbJobs, gcsJobs] = await Promise.all([
- Promise.all(ids.map((x) => getScrapeQueue().getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,
+ Promise.all(ids.map((x) => getScrapeQueue(x).getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,
process.env.USE_DB_AUTHENTICATION === "true" ? await supabaseGetJobsByCrawlId(crawlId) : [],
process.env.GCS_BUCKET_NAME ? Promise.all(ids.map(async (x) => ({ id: x, job: await getJobFromGCS(x) }))).then(x => x.filter(x => x.job)) as Promise<({ id: string, job: any | null })[]> : [],
]);
diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.ts
index 8435da31e..d4c622bcc 100644
--- a/apps/api/src/controllers/v0/scrape.ts
+++ b/apps/api/src/controllers/v0/scrape.ts
@@ -127,7 +127,7 @@ export async function scrapeHelper(
return err;
}
- await getScrapeQueue().remove(jobId);
+ await getScrapeQueue(jobId).remove(jobId);
if (!doc) {
console.error("!!! PANIC DOC IS", doc);
diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.ts
index 5ad0f4dff..45d98d531 100644
--- a/apps/api/src/controllers/v0/search.ts
+++ b/apps/api/src/controllers/v0/search.ts
@@ -139,8 +139,7 @@ export async function searchHelper(
return { success: true, error: "No search results found", returnCode: 200 };
}
- const sq = getScrapeQueue();
- await Promise.all(jobDatas.map((x) => sq.remove(x.opts.jobId)));
+ await Promise.all(jobDatas.map((x) => getScrapeQueue(x.opts.jobId).remove(x.opts.jobId)));
// make sure doc.content is not empty
const filteredDocs = docs.filter(
diff --git a/apps/api/src/controllers/v1/crawl-errors.ts b/apps/api/src/controllers/v1/crawl-errors.ts
index 45084c741..85fb491a4 100644
--- a/apps/api/src/controllers/v1/crawl-errors.ts
+++ b/apps/api/src/controllers/v1/crawl-errors.ts
@@ -19,7 +19,7 @@ import { TransportableError } from "../../lib/error";
configDotenv();
export async function getJob(id: string) {
- const job = await getScrapeQueue().getJob(id);
+ const job = await getScrapeQueue(id).getJob(id);
if (!job) return job;
return job;
@@ -27,7 +27,7 @@ export async function getJob(id: string) {
export async function getJobs(ids: string[]) {
const jobs: (Job & { id: string })[] = (
- await Promise.all(ids.map((x) => getScrapeQueue().getJob(x)))
+ await Promise.all(ids.map((x) => getScrapeQueue(x).getJob(x)))
).filter((x) => x) as (Job & { id: string })[];
return jobs;
@@ -46,7 +46,7 @@ export async function crawlErrorsController(
let jobStatuses = await Promise.all(
(await getCrawlJobs(req.params.jobId)).map(
- async (x) => [x, await getScrapeQueue().getJobState(x)] as const,
+ async (x) => [x, await getScrapeQueue(x).getJobState(x)] as const,
),
);
diff --git a/apps/api/src/controllers/v1/crawl-status-ws.ts b/apps/api/src/controllers/v1/crawl-status-ws.ts
index 190ca5209..a82d49706 100644
--- a/apps/api/src/controllers/v1/crawl-status-ws.ts
+++ b/apps/api/src/controllers/v1/crawl-status-ws.ts
@@ -88,12 +88,10 @@ async function crawlStatusWS(
const notDoneJobIDs = jobIDs.filter((x) => !doneJobIDs.includes(x));
- const queue = getScrapeQueue();
-
const jobStatuses = await Promise.all(
notDoneJobIDs.map(async (x) => [
x,
- await queue.getJobState(x),
+ await getScrapeQueue(x).getJobState(x),
]),
);
const newlyDoneJobIDs: string[] = jobStatuses
@@ -125,11 +123,9 @@ async function crawlStatusWS(
let jobIDs = await getCrawlJobs(req.params.jobId);
- const queue = getScrapeQueue();
-
let jobStatuses = await Promise.all(
jobIDs.map(
- async (x) => [x, await queue.getJobState(x)] as const,
+ async (x) => [x, await getScrapeQueue(x).getJobState(x)] as const,
),
);
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts
index 85a305f54..01f02e9a5 100644
--- a/apps/api/src/controllers/v1/crawl-status.ts
+++ b/apps/api/src/controllers/v1/crawl-status.ts
@@ -41,7 +41,7 @@ export type DBJob = { docs: any, success: boolean, page_options: any, date_added
export async function getJob(id: string): Promise<PseudoJob<any> | null> {
const [bullJob, dbJob, gcsJob] = await Promise.all([
- getScrapeQueue().getJob(id),
+ getScrapeQueue(id).getJob(id),
(process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobById(id) : null) as Promise<DBJob | null>,
(process.env.GCS_BUCKET_NAME ? getJobFromGCS(id) : null) as Promise<any | null>,
]);
@@ -73,7 +73,7 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {
export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {
const [bullJobs, dbJobs, gcsJobs] = await Promise.all([
- Promise.all(ids.map((x) => getScrapeQueue().getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,
+ Promise.all(ids.map((x) => getScrapeQueue(x).getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,
process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobsById(ids) : [],
process.env.GCS_BUCKET_NAME ? Promise.all(ids.map(async (x) => ({ id: x, job: await getJobFromGCS(x) }))).then(x => x.filter(x => x.job)) as Promise<({ id: string, job: any | null })[]> : [],
]);
@@ -315,7 +315,7 @@ export async function crawlStatusController(
const bytesLimit = 10485760; // 10 MiB in bytes
for (const jobId of doneJobs) {
- const job = await getScrapeQueue().getJob(jobId);
+ const job = await getScrapeQueue(jobId).getJob(jobId);
const state = await job?.getState();
if (state === "failed") {
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index b5b82ca43..b0444053c 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -105,7 +105,7 @@ export async function scrapeController(
});
if (zeroDataRetention) {
- await getScrapeQueue().remove(jobId);
+ await getScrapeQueue(jobId).remove(jobId);
}
if (e instanceof TransportableError) {
@@ -125,7 +125,7 @@ export async function scrapeController(
logger.info("Done with waitForJob");
- await getScrapeQueue().remove(jobId);
+ await getScrapeQueue(jobId).remove(jobId);
logger.info("Removed job from queue");
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index f185ce4df..eabc6c6de 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -138,7 +138,7 @@ async function scrapeSearchResult(
teamId: options.teamId,
origin: options.origin,
});
- await getScrapeQueue().remove(jobId);
+ await getScrapeQueue(jobId).remove(jobId);
const document = {
title: searchResult.title,
diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.ts
index ff0a66659..371aeb758 100644
--- a/apps/api/src/controllers/v1/x402-search.ts
+++ b/apps/api/src/controllers/v1/x402-search.ts
@@ -133,7 +133,7 @@ async function scrapeX402SearchResult(
teamId: options.teamId,
origin: options.origin,
});
- await getScrapeQueue().remove(jobId);
+ await getScrapeQueue(jobId).remove(jobId);
const document = {
title: searchResult.title,
diff --git a/apps/api/src/controllers/v2/crawl-errors.ts b/apps/api/src/controllers/v2/crawl-errors.ts
index 45084c741..85fb491a4 100644
--- a/apps/api/src/controllers/v2/crawl-errors.ts
+++ b/apps/api/src/controllers/v2/crawl-errors.ts
@@ -19,7 +19,7 @@ import { TransportableError } from "../../lib/error";
configDotenv();
export async function getJob(id: string) {
- const job = await getScrapeQueue().getJob(id);
+ const job = await getScrapeQueue(id).getJob(id);
if (!job) return job;
return job;
@@ -27,7 +27,7 @@ export async function getJob(id: string) {
export async function getJobs(ids: string[]) {
const jobs: (Job & { id: string })[] = (
- await Promise.all(ids.map((x) => getScrapeQueue().getJob(x)))
+ await Promise.all(ids.map((x) => getScrapeQueue(x).getJob(x)))
).filter((x) => x) as (Job & { id: string })[];
return jobs;
@@ -46,7 +46,7 @@ export async function crawlErrorsController(
let jobStatuses = await Promise.all(
(await getCrawlJobs(req.params.jobId)).map(
- async (x) => [x, await getScrapeQueue().getJobState(x)] as const,
+ async (x) => [x, await getScrapeQueue(x).getJobState(x)] as const,
),
);
diff --git a/apps/api/src/controllers/v2/crawl-status-ws.ts b/apps/api/src/controllers/v2/crawl-status-ws.ts
index 190ca5209..a82d49706 100644
--- a/apps/api/src/controllers/v2/crawl-status-ws.ts
+++ b/apps/api/src/controllers/v2/crawl-status-ws.ts
@@ -88,12 +88,10 @@ async function crawlStatusWS(
const notDoneJobIDs = jobIDs.filter((x) => !doneJobIDs.includes(x));
- const queue = getScrapeQueue();
-
const jobStatuses = await Promise.all(
notDoneJobIDs.map(async (x) => [
x,
- await queue.getJobState(x),
+ await getScrapeQueue(x).getJobState(x),
]),
);
const newlyDoneJobIDs: string[] = jobStatuses
@@ -125,11 +123,9 @@ async function crawlStatusWS(
let jobIDs = await getCrawlJobs(req.params.jobId);
- const queue = getScrapeQueue();
-
let jobStatuses = await Promise.all(
jobIDs.map(
- async (x) => [x, await queue.getJobState(x)] as const,
+ async (x) => [x, await getScrapeQueue(x).getJobState(x)] as const,
),
);
diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.ts
index 34e3fcbd3..e8af8fd9b 100644
--- a/apps/api/src/controllers/v2/crawl-status.ts
+++ b/apps/api/src/controllers/v2/crawl-status.ts
@@ -41,7 +41,7 @@ export type DBJob = { docs: any, success: boolean, page_options: any, date_added
export async function getJob(id: string): Promise<PseudoJob<any> | null> {
const [bullJob, dbJob, gcsJob] = await Promise.all([
- getScrapeQueue().getJob(id),
+ getScrapeQueue(id).getJob(id),
(process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobById(id) : null) as Promise<DBJob | null>,
(process.env.GCS_BUCKET_NAME ? getJobFromGCS(id) : null) as Promise<any | null>,
]);
@@ -73,7 +73,7 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {
export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {
const [bullJobs, dbJobs, gcsJobs] = await Promise.all([
- Promise.all(ids.map((x) => getScrapeQueue().getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,
+ Promise.all(ids.map((x) => getScrapeQueue(x).getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,
process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobsById(ids) : [],
process.env.GCS_BUCKET_NAME ? Promise.all(ids.map(async (x) => ({ id: x, job: await getJobFromGCS(x) }))).then(x => x.filter(x => x.job)) as Promise<({ id: string, job: any | null })[]> : [],
]);
@@ -315,7 +315,7 @@ export async function crawlStatusController(
const bytesLimit = 10485760; // 10 MiB in bytes
for (const jobId of doneJobs) {
- const job = await getScrapeQueue().getJob(jobId);
+ const job = await getScrapeQueue(jobId).getJob(jobId);
const state = await job?.getState();
if (state === "failed") {
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index b34fe5a31..c7e268043 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -106,7 +106,7 @@ export async function scrapeController(
});
if (zeroDataRetention) {
- await getScrapeQueue().remove(jobId);
+ await getScrapeQueue(jobId).remove(jobId);
}
if (e instanceof TransportableError) {
@@ -123,7 +123,7 @@ export async function scrapeController(
}
}
- await getScrapeQueue().remove(jobId);
+ await getScrapeQueue(jobId).remove(jobId);
if (!hasFormatOfType(req.body.formats, "rawHtml")) {
if (doc && doc.rawHtml) {
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index 158ff149d..9129e7166 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -95,7 +95,7 @@ async function scrapeSearchResult(
teamId: options.teamId,
origin: options.origin,
});
- await getScrapeQueue().remove(jobId);
+ await getScrapeQueue(jobId).remove(jobId);
const document = {
title: searchResult.title,
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index eb16f451a..e1ffdc0be 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -6,11 +6,12 @@ import bodyParser from "body-parser";
import cors from "cors";
import {
getExtractQueue,
- getScrapeQueue,
getGenerateLlmsTxtQueue,
getDeepResearchQueue,
getBillingQueue,
getPrecrawlQueue,
+ getScrapeQueueByIndex,
+ queueMultiplexWidth,
} from "./services/queue-service";
import { v0Router } from "./routes/v0";
import os from "os";
@@ -86,7 +87,7 @@ serverAdapter.setBasePath(`/admin/${process.env.BULL_AUTH_KEY}/queues`);
const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({
queues: [
- new BullMQAdapter(getScrapeQueue()),
+ ...new Array(queueMultiplexWidth).fill(0).map((_, index) => new BullMQAdapter(getScrapeQueueByIndex(index))),
new BullMQAdapter(getExtractQueue()),
new BullMQAdapter(getGenerateLlmsTxtQueue()),
new BullMQAdapter(getDeepResearchQueue()),
@@ -159,9 +160,11 @@ if (require.main === module) {
app.get(`/serverHealthCheck`, async (req, res) => {
try {
- const scrapeQueue = getScrapeQueue();
- const [waitingJobs] = await Promise.all([scrapeQueue.getWaitingCount()]);
- const noWaitingJobs = waitingJobs === 0;
+ const scrapeQueues = new Array(queueMultiplexWidth)
+ .fill(0)
+ .map((_, index) => getScrapeQueueByIndex(index));
+ const waitingJobs = await Promise.all(scrapeQueues.map((queue) => queue.getWaitingCount()));
+ const noWaitingJobs = waitingJobs.reduce((a, b) => a + b, 0) === 0;
// 200 if no active jobs, 503 if there are active jobs
return res.status(noWaitingJobs ? 200 : 500).json({
waitingJobs,
@@ -179,12 +182,12 @@ app.get("/serverHealthCheck/notify", async (req, res) => {
const timeout = 60000; // 1 minute // The timeout value for the check in milliseconds
const getWaitingJobsCount = async () => {
- const scrapeQueue = getScrapeQueue();
- const [waitingJobsCount] = await Promise.all([
- scrapeQueue.getWaitingCount(),
- ]);
+ const scrapeQueues = new Array(queueMultiplexWidth)
+ .fill(0)
+ .map((_, index) => getScrapeQueueByIndex(index));
+ const waitingJobsCount = await Promise.all(scrapeQueues.map((queue) => queue.getWaitingCount()));
- return waitingJobsCount;
+ return waitingJobsCount.reduce((a, b) => a + b, 0);
};
res.status(200).json({ message: "Check initiated" });
@@ -296,12 +299,3 @@ app.use(
);
logger.info(`Worker ${process.pid} started`);
-// const sq = getScrapeQueue();
-
-// sq.on("waiting", j => ScrapeEvents.logJobEvent(j, "waiting"));
-// sq.on("active", j => ScrapeEvents.logJobEvent(j, "active"));
-// sq.on("completed", j => ScrapeEvents.logJobEvent(j, "completed"));
-// sq.on("paused", j => ScrapeEvents.logJobEvent(j, "paused"));
-// sq.on("resumed", j => ScrapeEvents.logJobEvent(j, "resumed"));
-// sq.on("removed", j => ScrapeEvents.logJobEvent(j, "removed"));
-//
diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.ts
index 52fc2fbdb..eba623603 100644
--- a/apps/api/src/lib/concurrency-limit.ts
+++ b/apps/api/src/lib/concurrency-limit.ts
@@ -274,7 +274,7 @@ export async function concurrentJobDone(job: Job) {
}
}
- (await getScrapeQueue()).add(
+ (await getScrapeQueue(nextJob.job.id)).add(
nextJob.job.id,
{
...nextJob.job.data,
diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.ts
index 1b84733b2..0913615f1 100644
--- a/apps/api/src/lib/extract/document-scraper.ts
+++ b/apps/api/src/lib/extract/document-scraper.ts
@@ -66,7 +66,7 @@ export async function scrapeDocument(
const doc = await waitForJob(jobId, timeout);
- await getScrapeQueue().remove(jobId);
+ await getScrapeQueue(jobId).remove(jobId);
if (trace) {
trace.timing.completedAt = new Date().toISOString();
diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
index a4b524a64..5e2629744 100644
--- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
@@ -66,7 +66,7 @@ export async function scrapeDocument_F0(
);
const doc = await waitForJob(jobId, timeout);
- await getScrapeQueue().remove(jobId);
+ await getScrapeQueue(jobId).remove(jobId);
if (trace) {
trace.timing.completedAt = new Date().toISOString();
diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.ts
index 25f0880ad..51a713e3e 100644
--- a/apps/api/src/routes/admin.ts
+++ b/apps/api/src/routes/admin.ts
@@ -1,9 +1,7 @@
import express from "express";
import { redisHealthController } from "../controllers/v0/admin/redis-health";
import {
- autoscalerController,
checkQueuesController,
- cleanBefore24hCompleteJobsController,
queuesController,
} from "../controllers/v0/admin/queue";
import { wrap } from "./shared";
@@ -22,11 +20,6 @@ adminRouter.get(
redisHealthController,
);
-adminRouter.get(
- `/admin/${process.env.BULL_AUTH_KEY}/clean-before-24h-complete-jobs`,
- cleanBefore24hCompleteJobsController,
-);
-
adminRouter.get(
`/admin/${process.env.BULL_AUTH_KEY}/check-queues`,
checkQueuesController,
@@ -34,11 +27,6 @@ adminRouter.get(
adminRouter.get(`/admin/${process.env.BULL_AUTH_KEY}/queues`, queuesController);
-adminRouter.get(
- `/admin/${process.env.BULL_AUTH_KEY}/autoscaler`,
- autoscalerController,
-);
-
adminRouter.post(
`/admin/${process.env.BULL_AUTH_KEY}/acuc-cache-clear`,
wrap(acucCacheClearController),
diff --git a/apps/api/src/services/alerts/index.ts b/apps/api/src/services/alerts/index.ts
index 44f2b8a04..b7a9a9455 100644
--- a/apps/api/src/services/alerts/index.ts
+++ b/apps/api/src/services/alerts/index.ts
@@ -1,5 +1,5 @@
import { logger } from "../../../src/lib/logger";
-import { getScrapeQueue } from "../queue-service";
+import { getScrapeQueueByIndex, queueMultiplexWidth } from "../queue-service";
import { sendSlackWebhook } from "./slack";
export async function checkAlerts() {
@@ -13,9 +13,11 @@ export async function checkAlerts() {
logger.info("Initializing alerts");
const checkActiveJobs = async () => {
try {
- const scrapeQueue = getScrapeQueue();
- const activeJobs = await scrapeQueue.getActiveCount();
- if (activeJobs > Number(process.env.ALERT_NUM_ACTIVE_JOBS)) {
+ const scrapeQueues = new Array(queueMultiplexWidth)
+ .fill(0)
+ .map((_, index) => getScrapeQueueByIndex(index));
+ const activeJobs = await Promise.all(scrapeQueues.map((queue) => queue.getActiveCount()));
+ if (activeJobs.reduce((a, b) => a + b, 0) > Number(process.env.ALERT_NUM_ACTIVE_JOBS)) {
logger.warn(
`Alert: Number of active jobs is over ${process.env.ALERT_NUM_ACTIVE_JOBS}. Current active jobs: ${activeJobs}.`,
);
@@ -34,10 +36,12 @@ export async function checkAlerts() {
};
const checkWaitingQueue = async () => {
- const scrapeQueue = getScrapeQueue();
- const waitingJobs = await scrapeQueue.getWaitingCount();
+ const scrapeQueues = new Array(queueMultiplexWidth)
+ .fill(0)
+ .map((_, index) => getScrapeQueueByIndex(index));
+ const waitingJobs = await Promise.all(scrapeQueues.map((queue) => queue.getWaitingCount()));
- if (waitingJobs > Number(process.env.ALERT_NUM_WAITING_JOBS)) {
+ if (waitingJobs.reduce((a, b) => a + b, 0) > Number(process.env.ALERT_NUM_WAITING_JOBS)) {
logger.warn(
`Alert: Number of waiting jobs is over ${process.env.ALERT_NUM_WAITING_JOBS}. Current waiting jobs: ${waitingJobs}.`,
);
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts
index a4a5ab922..3ee4a9282 100644
--- a/apps/api/src/services/queue-jobs.ts
+++ b/apps/api/src/services/queue-jobs.ts
@@ -72,7 +72,7 @@ export async function _addScrapeJobToBullMQ(
}
}
- return await getScrapeQueue().add(jobId, webScraperOptions, {
+ return await getScrapeQueue(jobId).add(jobId, webScraperOptions, {
...options,
priority: jobPriority,
jobId,
@@ -322,7 +322,7 @@ export async function waitForJob(
logger: Logger = _logger,
): Promise<Document> {
const start = Date.now();
- const queue = getScrapeQueue();
+ const queue = getScrapeQueue(typeof _job === "string" ? _job : (_job.id as string));
let job: Job | undefined = typeof _job == "string" ? await queue.getJob(_job) : _job;
while (job === undefined) {
logger.debug("Waiting for job to be created");
@@ -335,7 +335,7 @@ export async function waitForJob(
let doc: Document;
try {
doc = await Promise.race([
- job.waitUntilFinished(getScrapeQueueEvents(), timeout ?? 180000),
+ job.waitUntilFinished(getScrapeQueueEvents(job.id as string), timeout ?? 180000),
timeout !== null ? new Promise((resolve, reject) => {
setTimeout(() => {
reject(new ScrapeJobTimeoutError("Scrape timed out" + (typeof _job === "string" ? " after waiting in the concurrency limit queue" : "")));
diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.ts
index 6041b9ec8..8a56a6caa 100644
--- a/apps/api/src/services/queue-service.ts
+++ b/apps/api/src/services/queue-service.ts
@@ -5,8 +5,8 @@ import { BullMQOtel } from "bullmq-otel";
export type QueueFunction = () => Queue<any, any, string, any, any, string>;
-let scrapeQueue: Queue;
-let scrapeQueueEvents: QueueEvents;
+let scrapeQueues: Queue[] = [];
+let scrapeQueueEvents: QueueEvents[] = [];
let extractQueue: Queue;
let loggingQueue: Queue;
let indexQueue: Queue;
@@ -29,7 +29,6 @@ export function getRedisConnection(): IORedis {
return redisConnection;
}
-export const scrapeQueueName = "{scrapeQueue}";
export const extractQueueName = "{extractQueue}";
export const loggingQueueName = "{loggingQueue}";
export const indexQueueName = "{indexQueue}";
@@ -38,9 +37,21 @@ export const deepResearchQueueName = "{deepResearchQueue}";
export const billingQueueName = "{billingQueue}";
export const precrawlQueueName = "{precrawlQueue}";
-export function getScrapeQueue() {
- if (!scrapeQueue) {
- scrapeQueue = new Queue(scrapeQueueName, {
+export const queueMultiplexWidth = process.env.QUEUE_MULTIPLEX_WIDTH ? parseInt(process.env.QUEUE_MULTIPLEX_WIDTH) : 1;
+
+function uuidToQueueIndex(uuid: string) {
+ const queueIndex = parseInt(uuid[0] ?? "0", 16) % queueMultiplexWidth;
+ return queueIndex;
+}
+
+export function getScrapeQueue(uuid: string) {
+ const queueIndex = uuidToQueueIndex(uuid);
+ return getScrapeQueueByIndex(queueIndex);
+}
+
+export function getScrapeQueueByIndex(index: number) {
+ if (!scrapeQueues[index]) {
+ scrapeQueues[index] = new Queue(`{scrapeQueue${index === 0 ? "" : index}}`, {
connection: getRedisConnection(),
defaultJobOptions: {
removeOnComplete: {
@@ -53,17 +64,22 @@ export function getScrapeQueue() {
telemetry: new BullMQOtel("firecrawl-bullmq"),
});
}
- return scrapeQueue;
+ return scrapeQueues[index];
+}
+
+export function getScrapeQueueEvents(uuid: string) {
+ const queueIndex = uuidToQueueIndex(uuid);
+ return getScrapeQueueEventsByIndex(queueIndex);
}
-export function getScrapeQueueEvents() {
- if (!scrapeQueueEvents) {
- scrapeQueueEvents = new QueueEvents(scrapeQueueName, {
+export function getScrapeQueueEventsByIndex(index: number) {
+ if (!scrapeQueueEvents[index]) {
+ scrapeQueueEvents[index] = new QueueEvents(`{scrapeQueue${index === 0 ? "" : index}}`, {
connection: getRedisConnection(),
});
}
- return scrapeQueueEvents;
+ return scrapeQueueEvents[index];
}
export function getExtractQueue() {
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 32a154186..3ee0928ed 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -6,8 +6,10 @@ import {
getExtractQueue,
getDeepResearchQueue,
getGenerateLlmsTxtQueue,
- scrapeQueueName,
getRedisConnection,
+ queueMultiplexWidth,
+ getScrapeQueueEventsByIndex,
+ getScrapeQueueByIndex,
} from "./queue-service";
import { Job, Queue, QueueEvents } from "bullmq";
import { logger as _logger } from "../lib/logger";
@@ -508,9 +510,11 @@ app.listen(workerPort, () => {
_logger.info(`Liveness endpoint is running on port ${workerPort}`);
});
+const myScrapeQueue = Math.floor(Math.random() * queueMultiplexWidth);
+
(async () => {
async function failedListener(args: { jobId: string; failedReason: string; prev?: string | undefined; }) {
- const job = await getScrapeQueue().getJob(args.jobId);
+ const job = await getScrapeQueueByIndex(myScrapeQueue).getJob(args.jobId);
if (job && job.data.crawl_id) {
await redisEvictConnection.srem("crawl:" + job.data.crawl_id + ":jobs_qualified", args.jobId);
@@ -559,12 +563,14 @@ app.listen(workerPort, () => {
}
}
- const scrapeQueueEvents = new QueueEvents(scrapeQueueName, { connection: getRedisConnection() });
- scrapeQueueEvents.on("failed", failedListener);
+ for (let i = 0; i < queueMultiplexWidth; i++) {
+ const scrapeQueueEvents = getScrapeQueueEventsByIndex(i);
+ scrapeQueueEvents.on("failed", failedListener);
+ }
const results = await Promise.all([
separateWorkerFun(
- getScrapeQueue(),
+ getScrapeQueueByIndex(myScrapeQueue),
path.join(__dirname, "worker", "scrape-worker.js"),
),
workerFun(getExtractQueue(), processExtractJobInternal),
@@ -586,14 +592,14 @@ app.listen(workerPort, () => {
jobs: (
await Promise.all(
[...runningJobs].map(async (jobId) => {
- return await getScrapeQueue().getJob(jobId);
+ return await getScrapeQueueByIndex(myScrapeQueue).getJob(jobId);
}),
)
).filter((x) => x && !x.data?.zeroDataRetention),
});
}, 1000);
- await scrapeQueueEvents.close();
+ await getScrapeQueueEventsByIndex(myScrapeQueue).close();
console.log("All jobs finished. Worker out!");
if (otelSdk) {
await otelSdk.shutdown();