Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): queue multiplexing
GergΕ MΓ³ricz23b079052eb4
diff --git a/apps/api/src/controllers/v0/admin/queue.ts b/apps/api/src/controllers/v0/admin/queue.tsindex d7d9c0897..4d80918fc 100644--- a/apps/api/src/controllers/v0/admin/queue.ts+++ b/apps/api/src/controllers/v0/admin/queue.ts@@ -2,61 +2,10 @@ import { Request, Response } from "express";import { Job } from "bullmq";import { logger } from "../../../lib/logger";-import { getScrapeQueue } from "../../../services/queue-service";+import { getScrapeQueue, getScrapeQueueByIndex, queueMultiplexWidth } from "../../../services/queue-service";import { checkAlerts } from "../../../services/alerts";import { sendSlackWebhook } from "../../../services/alerts/slack";-export async function cleanBefore24hCompleteJobsController(- req: Request,- res: Response,-) {- logger.info("π Cleaning jobs older than 24h");- try {- const scrapeQueue = getScrapeQueue();- const batchSize = 10;- const numberOfBatches = 9; // Adjust based on your needs- const completedJobsPromises: Promise<Job[]>[] = [];- for (let i = 0; i < numberOfBatches; i++) {- completedJobsPromises.push(- scrapeQueue.getJobs(- ["completed"],- i * batchSize,- i * batchSize + batchSize,- true,- ),- );- }- const completedJobs: Job[] = (- await Promise.all(completedJobsPromises)- ).flat();- const before24hJobs =- completedJobs.filter(- (job) =>- job.finishedOn !== undefined &&- job.finishedOn < Date.now() - 24 * 60 * 60 * 1000,- ) || [];-- let count = 0;-- if (!before24hJobs) {- return res.status(200).send(`No jobs to remove.`);- }-- for (const job of before24hJobs) {- try {- await job.remove();- count++;- } catch (jobError) {- logger.error(`π Failed to remove job with ID ${job.id}: ${jobError}`);- }- }- return res.status(200).send(`Removed ${count} completed jobs.`);- } catch (error) {- logger.error(`π Failed to clean last 24h complete jobs: ${error}`);- return res.status(500).send("Failed to clean jobs");- }-}-export async function checkQueuesController(req: Request, res: Response) {try {await checkAlerts();@@ -70,11 +19,11 @@ export async function checkQueuesController(req: Request, res: Response) {// Use this as a "health check" that way we dont destroy the serverexport async function queuesController(req: Request, res: Response) {try {- const scrapeQueue = getScrapeQueue();+ const scrapeQueues = new Array(queueMultiplexWidth)+ .fill(0)+ .map((_, index) => getScrapeQueueByIndex(index));- const [webScraperActive] = await Promise.all([- scrapeQueue.getActiveCount(),- ]);+ const webScraperActive = (await Promise.all(scrapeQueues.map((queue) => queue.getActiveCount()))).reduce((a, b) => a + b, 0);const noActiveJobs = webScraperActive === 0;// 200 if no active jobs, 503 if there are active jobs@@ -87,115 +36,3 @@ export async function queuesController(req: Request, res: Response) {return res.status(500).json({ error: error.message });}}--export async function autoscalerController(req: Request, res: Response) {- try {- const maxNumberOfMachines = 80;- const minNumberOfMachines = 20;-- const scrapeQueue = getScrapeQueue();-- const [webScraperActive, webScraperWaiting, webScraperPriority] =- await Promise.all([- scrapeQueue.getActiveCount(),- scrapeQueue.getWaitingCount(),- scrapeQueue.getPrioritizedCount(),- ]);-- let waitingAndPriorityCount = webScraperWaiting + webScraperPriority;-- // get number of machines active- const request = await fetch(- "https://api.machines.dev/v1/apps/firecrawl-scraper-js/machines",- {- headers: {- Authorization: `Bearer ${process.env.FLY_API_TOKEN}`,- },- },- );- const machines = await request.json();-- // Only worker machines- const activeMachines = machines.filter(- (machine) =>- (machine.state === "started" ||- machine.state === "starting" ||- machine.state === "replacing") &&- machine.config.env["FLY_PROCESS_GROUP"] === "worker",- ).length;-- let targetMachineCount = activeMachines;-- const baseScaleUp = 10;- // Slow scale down- const baseScaleDown = 2;-- // Scale up logic- if (webScraperActive > 9000 || waitingAndPriorityCount > 2000) {- targetMachineCount = Math.min(- maxNumberOfMachines,- activeMachines + baseScaleUp * 3,- );- } else if (webScraperActive > 5000 || waitingAndPriorityCount > 1000) {- targetMachineCount = Math.min(- maxNumberOfMachines,- activeMachines + baseScaleUp * 2,- );- } else if (webScraperActive > 1000 || waitingAndPriorityCount > 500) {- targetMachineCount = Math.min(- maxNumberOfMachines,- activeMachines + baseScaleUp,- );- }-- // Scale down logic- if (webScraperActive < 100 && waitingAndPriorityCount < 50) {- targetMachineCount = Math.max(- minNumberOfMachines,- activeMachines - baseScaleDown * 3,- );- } else if (webScraperActive < 500 && waitingAndPriorityCount < 200) {- targetMachineCount = Math.max(- minNumberOfMachines,- activeMachines - baseScaleDown * 2,- );- } else if (webScraperActive < 1000 && waitingAndPriorityCount < 500) {- targetMachineCount = Math.max(- minNumberOfMachines,- activeMachines - baseScaleDown,- );- }-- if (targetMachineCount !== activeMachines) {- logger.info(- `π Scaling from ${activeMachines} to ${targetMachineCount} - ${webScraperActive} active, ${webScraperWaiting} waiting`,- );-- if (targetMachineCount > activeMachines) {- sendSlackWebhook(- `π Scaling from ${activeMachines} to ${targetMachineCount} - ${webScraperActive} active, ${webScraperWaiting} waiting - Current DateTime: ${new Date().toISOString()}`,- false,- process.env.SLACK_AUTOSCALER ?? "",- );- } else {- sendSlackWebhook(- `π Scaling from ${activeMachines} to ${targetMachineCount} - ${webScraperActive} active, ${webScraperWaiting} waiting - Current DateTime: ${new Date().toISOString()}`,- false,- process.env.SLACK_AUTOSCALER ?? "",- );- }- return res.status(200).json({- mode: "scale-descale",- count: targetMachineCount,- });- }-- return res.status(200).json({- mode: "normal",- count: activeMachines,- });- } catch (error) {- logger.error(error);- return res.status(500).send("Failed to initialize autoscaler");- }-}diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.tsindex 843eb56d1..d76f5ebfa 100644--- a/apps/api/src/controllers/v0/crawl-status.ts+++ b/apps/api/src/controllers/v0/crawl-status.ts@@ -16,7 +16,7 @@ configDotenv();export async function getJobs(crawlId: string, ids: string[]): Promise<PseudoJob<any>[]> {const [bullJobs, dbJobs, gcsJobs] = await Promise.all([- Promise.all(ids.map((x) => getScrapeQueue().getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,+ Promise.all(ids.map((x) => getScrapeQueue(x).getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,process.env.USE_DB_AUTHENTICATION === "true" ? await supabaseGetJobsByCrawlId(crawlId) : [],process.env.GCS_BUCKET_NAME ? Promise.all(ids.map(async (x) => ({ id: x, job: await getJobFromGCS(x) }))).then(x => x.filter(x => x.job)) as Promise<({ id: string, job: any | null })[]> : [],]);diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex 8435da31e..d4c622bcc 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -127,7 +127,7 @@ export async function scrapeHelper(return err;}- await getScrapeQueue().remove(jobId);+ await getScrapeQueue(jobId).remove(jobId);if (!doc) {console.error("!!! PANIC DOC IS", doc);diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.tsindex 5ad0f4dff..45d98d531 100644--- a/apps/api/src/controllers/v0/search.ts+++ b/apps/api/src/controllers/v0/search.ts@@ -139,8 +139,7 @@ export async function searchHelper(return { success: true, error: "No search results found", returnCode: 200 };}- const sq = getScrapeQueue();- await Promise.all(jobDatas.map((x) => sq.remove(x.opts.jobId)));+ await Promise.all(jobDatas.map((x) => getScrapeQueue(x.opts.jobId).remove(x.opts.jobId)));// make sure doc.content is not emptyconst filteredDocs = docs.filter(diff --git a/apps/api/src/controllers/v1/crawl-errors.ts b/apps/api/src/controllers/v1/crawl-errors.tsindex 45084c741..85fb491a4 100644--- a/apps/api/src/controllers/v1/crawl-errors.ts+++ b/apps/api/src/controllers/v1/crawl-errors.ts@@ -19,7 +19,7 @@ import { TransportableError } from "../../lib/error";configDotenv();export async function getJob(id: string) {- const job = await getScrapeQueue().getJob(id);+ const job = await getScrapeQueue(id).getJob(id);if (!job) return job;return job;@@ -27,7 +27,7 @@ export async function getJob(id: string) {export async function getJobs(ids: string[]) {const jobs: (Job & { id: string })[] = (- await Promise.all(ids.map((x) => getScrapeQueue().getJob(x)))+ await Promise.all(ids.map((x) => getScrapeQueue(x).getJob(x)))).filter((x) => x) as (Job & { id: string })[];return jobs;@@ -46,7 +46,7 @@ export async function crawlErrorsController(let jobStatuses = await Promise.all((await getCrawlJobs(req.params.jobId)).map(- async (x) => [x, await getScrapeQueue().getJobState(x)] as const,+ async (x) => [x, await getScrapeQueue(x).getJobState(x)] as const,),);diff --git a/apps/api/src/controllers/v1/crawl-status-ws.ts b/apps/api/src/controllers/v1/crawl-status-ws.tsindex 190ca5209..a82d49706 100644--- a/apps/api/src/controllers/v1/crawl-status-ws.ts+++ b/apps/api/src/controllers/v1/crawl-status-ws.ts@@ -88,12 +88,10 @@ async function crawlStatusWS(const notDoneJobIDs = jobIDs.filter((x) => !doneJobIDs.includes(x));- const queue = getScrapeQueue();-const jobStatuses = await Promise.all(notDoneJobIDs.map(async (x) => [x,- await queue.getJobState(x),+ await getScrapeQueue(x).getJobState(x),]),);const newlyDoneJobIDs: string[] = jobStatuses@@ -125,11 +123,9 @@ async function crawlStatusWS(let jobIDs = await getCrawlJobs(req.params.jobId);- const queue = getScrapeQueue();-let jobStatuses = await Promise.all(jobIDs.map(- async (x) => [x, await queue.getJobState(x)] as const,+ async (x) => [x, await getScrapeQueue(x).getJobState(x)] as const,),);diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex 85a305f54..01f02e9a5 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -41,7 +41,7 @@ export type DBJob = { docs: any, success: boolean, page_options: any, date_addedexport async function getJob(id: string): Promise<PseudoJob<any> | null> {const [bullJob, dbJob, gcsJob] = await Promise.all([- getScrapeQueue().getJob(id),+ getScrapeQueue(id).getJob(id),(process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobById(id) : null) as Promise<DBJob | null>,(process.env.GCS_BUCKET_NAME ? getJobFromGCS(id) : null) as Promise<any | null>,]);@@ -73,7 +73,7 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {const [bullJobs, dbJobs, gcsJobs] = await Promise.all([- Promise.all(ids.map((x) => getScrapeQueue().getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,+ Promise.all(ids.map((x) => getScrapeQueue(x).getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobsById(ids) : [],process.env.GCS_BUCKET_NAME ? Promise.all(ids.map(async (x) => ({ id: x, job: await getJobFromGCS(x) }))).then(x => x.filter(x => x.job)) as Promise<({ id: string, job: any | null })[]> : [],]);@@ -315,7 +315,7 @@ export async function crawlStatusController(const bytesLimit = 10485760; // 10 MiB in bytesfor (const jobId of doneJobs) {- const job = await getScrapeQueue().getJob(jobId);+ const job = await getScrapeQueue(jobId).getJob(jobId);const state = await job?.getState();if (state === "failed") {diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex b5b82ca43..b0444053c 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -105,7 +105,7 @@ export async function scrapeController(});if (zeroDataRetention) {- await getScrapeQueue().remove(jobId);+ await getScrapeQueue(jobId).remove(jobId);}if (e instanceof TransportableError) {@@ -125,7 +125,7 @@ export async function scrapeController(logger.info("Done with waitForJob");- await getScrapeQueue().remove(jobId);+ await getScrapeQueue(jobId).remove(jobId);logger.info("Removed job from queue");diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex f185ce4df..eabc6c6de 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -138,7 +138,7 @@ async function scrapeSearchResult(teamId: options.teamId,origin: options.origin,});- await getScrapeQueue().remove(jobId);+ await getScrapeQueue(jobId).remove(jobId);const document = {title: searchResult.title,diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.tsindex ff0a66659..371aeb758 100644--- a/apps/api/src/controllers/v1/x402-search.ts+++ b/apps/api/src/controllers/v1/x402-search.ts@@ -133,7 +133,7 @@ async function scrapeX402SearchResult(teamId: options.teamId,origin: options.origin,});- await getScrapeQueue().remove(jobId);+ await getScrapeQueue(jobId).remove(jobId);const document = {title: searchResult.title,diff --git a/apps/api/src/controllers/v2/crawl-errors.ts b/apps/api/src/controllers/v2/crawl-errors.tsindex 45084c741..85fb491a4 100644--- a/apps/api/src/controllers/v2/crawl-errors.ts+++ b/apps/api/src/controllers/v2/crawl-errors.ts@@ -19,7 +19,7 @@ import { TransportableError } from "../../lib/error";configDotenv();export async function getJob(id: string) {- const job = await getScrapeQueue().getJob(id);+ const job = await getScrapeQueue(id).getJob(id);if (!job) return job;return job;@@ -27,7 +27,7 @@ export async function getJob(id: string) {export async function getJobs(ids: string[]) {const jobs: (Job & { id: string })[] = (- await Promise.all(ids.map((x) => getScrapeQueue().getJob(x)))+ await Promise.all(ids.map((x) => getScrapeQueue(x).getJob(x)))).filter((x) => x) as (Job & { id: string })[];return jobs;@@ -46,7 +46,7 @@ export async function crawlErrorsController(let jobStatuses = await Promise.all((await getCrawlJobs(req.params.jobId)).map(- async (x) => [x, await getScrapeQueue().getJobState(x)] as const,+ async (x) => [x, await getScrapeQueue(x).getJobState(x)] as const,),);diff --git a/apps/api/src/controllers/v2/crawl-status-ws.ts b/apps/api/src/controllers/v2/crawl-status-ws.tsindex 190ca5209..a82d49706 100644--- a/apps/api/src/controllers/v2/crawl-status-ws.ts+++ b/apps/api/src/controllers/v2/crawl-status-ws.ts@@ -88,12 +88,10 @@ async function crawlStatusWS(const notDoneJobIDs = jobIDs.filter((x) => !doneJobIDs.includes(x));- const queue = getScrapeQueue();-const jobStatuses = await Promise.all(notDoneJobIDs.map(async (x) => [x,- await queue.getJobState(x),+ await getScrapeQueue(x).getJobState(x),]),);const newlyDoneJobIDs: string[] = jobStatuses@@ -125,11 +123,9 @@ async function crawlStatusWS(let jobIDs = await getCrawlJobs(req.params.jobId);- const queue = getScrapeQueue();-let jobStatuses = await Promise.all(jobIDs.map(- async (x) => [x, await queue.getJobState(x)] as const,+ async (x) => [x, await getScrapeQueue(x).getJobState(x)] as const,),);diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.tsindex 34e3fcbd3..e8af8fd9b 100644--- a/apps/api/src/controllers/v2/crawl-status.ts+++ b/apps/api/src/controllers/v2/crawl-status.ts@@ -41,7 +41,7 @@ export type DBJob = { docs: any, success: boolean, page_options: any, date_addedexport async function getJob(id: string): Promise<PseudoJob<any> | null> {const [bullJob, dbJob, gcsJob] = await Promise.all([- getScrapeQueue().getJob(id),+ getScrapeQueue(id).getJob(id),(process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobById(id) : null) as Promise<DBJob | null>,(process.env.GCS_BUCKET_NAME ? getJobFromGCS(id) : null) as Promise<any | null>,]);@@ -73,7 +73,7 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {const [bullJobs, dbJobs, gcsJobs] = await Promise.all([- Promise.all(ids.map((x) => getScrapeQueue().getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,+ Promise.all(ids.map((x) => getScrapeQueue(x).getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobsById(ids) : [],process.env.GCS_BUCKET_NAME ? Promise.all(ids.map(async (x) => ({ id: x, job: await getJobFromGCS(x) }))).then(x => x.filter(x => x.job)) as Promise<({ id: string, job: any | null })[]> : [],]);@@ -315,7 +315,7 @@ export async function crawlStatusController(const bytesLimit = 10485760; // 10 MiB in bytesfor (const jobId of doneJobs) {- const job = await getScrapeQueue().getJob(jobId);+ const job = await getScrapeQueue(jobId).getJob(jobId);const state = await job?.getState();if (state === "failed") {diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex b34fe5a31..c7e268043 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -106,7 +106,7 @@ export async function scrapeController(});if (zeroDataRetention) {- await getScrapeQueue().remove(jobId);+ await getScrapeQueue(jobId).remove(jobId);}if (e instanceof TransportableError) {@@ -123,7 +123,7 @@ export async function scrapeController(}}- await getScrapeQueue().remove(jobId);+ await getScrapeQueue(jobId).remove(jobId);if (!hasFormatOfType(req.body.formats, "rawHtml")) {if (doc && doc.rawHtml) {diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 158ff149d..9129e7166 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -95,7 +95,7 @@ async function scrapeSearchResult(teamId: options.teamId,origin: options.origin,});- await getScrapeQueue().remove(jobId);+ await getScrapeQueue(jobId).remove(jobId);const document = {title: searchResult.title,diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex eb16f451a..e1ffdc0be 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -6,11 +6,12 @@ import bodyParser from "body-parser";import cors from "cors";import {getExtractQueue,- getScrapeQueue,getGenerateLlmsTxtQueue,getDeepResearchQueue,getBillingQueue,getPrecrawlQueue,+ getScrapeQueueByIndex,+ queueMultiplexWidth,} from "./services/queue-service";import { v0Router } from "./routes/v0";import os from "os";@@ -86,7 +87,7 @@ serverAdapter.setBasePath(`/admin/${process.env.BULL_AUTH_KEY}/queues`);const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({queues: [- new BullMQAdapter(getScrapeQueue()),+ ...new Array(queueMultiplexWidth).fill(0).map((_, index) => new BullMQAdapter(getScrapeQueueByIndex(index))),new BullMQAdapter(getExtractQueue()),new BullMQAdapter(getGenerateLlmsTxtQueue()),new BullMQAdapter(getDeepResearchQueue()),@@ -159,9 +160,11 @@ if (require.main === module) {app.get(`/serverHealthCheck`, async (req, res) => {try {- const scrapeQueue = getScrapeQueue();- const [waitingJobs] = await Promise.all([scrapeQueue.getWaitingCount()]);- const noWaitingJobs = waitingJobs === 0;+ const scrapeQueues = new Array(queueMultiplexWidth)+ .fill(0)+ .map((_, index) => getScrapeQueueByIndex(index));+ const waitingJobs = await Promise.all(scrapeQueues.map((queue) => queue.getWaitingCount()));+ const noWaitingJobs = waitingJobs.reduce((a, b) => a + b, 0) === 0;// 200 if no active jobs, 503 if there are active jobsreturn res.status(noWaitingJobs ? 200 : 500).json({waitingJobs,@@ -179,12 +182,12 @@ app.get("/serverHealthCheck/notify", async (req, res) => {const timeout = 60000; // 1 minute // The timeout value for the check in millisecondsconst getWaitingJobsCount = async () => {- const scrapeQueue = getScrapeQueue();- const [waitingJobsCount] = await Promise.all([- scrapeQueue.getWaitingCount(),- ]);+ const scrapeQueues = new Array(queueMultiplexWidth)+ .fill(0)+ .map((_, index) => getScrapeQueueByIndex(index));+ const waitingJobsCount = await Promise.all(scrapeQueues.map((queue) => queue.getWaitingCount()));- return waitingJobsCount;+ return waitingJobsCount.reduce((a, b) => a + b, 0);};res.status(200).json({ message: "Check initiated" });@@ -296,12 +299,3 @@ app.use();logger.info(`Worker ${process.pid} started`);-// const sq = getScrapeQueue();--// sq.on("waiting", j => ScrapeEvents.logJobEvent(j, "waiting"));-// sq.on("active", j => ScrapeEvents.logJobEvent(j, "active"));-// sq.on("completed", j => ScrapeEvents.logJobEvent(j, "completed"));-// sq.on("paused", j => ScrapeEvents.logJobEvent(j, "paused"));-// sq.on("resumed", j => ScrapeEvents.logJobEvent(j, "resumed"));-// sq.on("removed", j => ScrapeEvents.logJobEvent(j, "removed"));-//diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.tsindex 52fc2fbdb..eba623603 100644--- a/apps/api/src/lib/concurrency-limit.ts+++ b/apps/api/src/lib/concurrency-limit.ts@@ -274,7 +274,7 @@ export async function concurrentJobDone(job: Job) {}}- (await getScrapeQueue()).add(+ (await getScrapeQueue(nextJob.job.id)).add(nextJob.job.id,{...nextJob.job.data,diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.tsindex 1b84733b2..0913615f1 100644--- a/apps/api/src/lib/extract/document-scraper.ts+++ b/apps/api/src/lib/extract/document-scraper.ts@@ -66,7 +66,7 @@ export async function scrapeDocument(const doc = await waitForJob(jobId, timeout);- await getScrapeQueue().remove(jobId);+ await getScrapeQueue(jobId).remove(jobId);if (trace) {trace.timing.completedAt = new Date().toISOString();diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.tsindex a4b524a64..5e2629744 100644--- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts+++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts@@ -66,7 +66,7 @@ export async function scrapeDocument_F0();const doc = await waitForJob(jobId, timeout);- await getScrapeQueue().remove(jobId);+ await getScrapeQueue(jobId).remove(jobId);if (trace) {trace.timing.completedAt = new Date().toISOString();diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.tsindex 25f0880ad..51a713e3e 100644--- a/apps/api/src/routes/admin.ts+++ b/apps/api/src/routes/admin.ts@@ -1,9 +1,7 @@import express from "express";import { redisHealthController } from "../controllers/v0/admin/redis-health";import {- autoscalerController,checkQueuesController,- cleanBefore24hCompleteJobsController,queuesController,} from "../controllers/v0/admin/queue";import { wrap } from "./shared";@@ -22,11 +20,6 @@ adminRouter.get(redisHealthController,);-adminRouter.get(- `/admin/${process.env.BULL_AUTH_KEY}/clean-before-24h-complete-jobs`,- cleanBefore24hCompleteJobsController,-);-adminRouter.get(`/admin/${process.env.BULL_AUTH_KEY}/check-queues`,checkQueuesController,@@ -34,11 +27,6 @@ adminRouter.get(adminRouter.get(`/admin/${process.env.BULL_AUTH_KEY}/queues`, queuesController);-adminRouter.get(- `/admin/${process.env.BULL_AUTH_KEY}/autoscaler`,- autoscalerController,-);-adminRouter.post(`/admin/${process.env.BULL_AUTH_KEY}/acuc-cache-clear`,wrap(acucCacheClearController),diff --git a/apps/api/src/services/alerts/index.ts b/apps/api/src/services/alerts/index.tsindex 44f2b8a04..b7a9a9455 100644--- a/apps/api/src/services/alerts/index.ts+++ b/apps/api/src/services/alerts/index.ts@@ -1,5 +1,5 @@import { logger } from "../../../src/lib/logger";-import { getScrapeQueue } from "../queue-service";+import { getScrapeQueueByIndex, queueMultiplexWidth } from "../queue-service";import { sendSlackWebhook } from "./slack";export async function checkAlerts() {@@ -13,9 +13,11 @@ export async function checkAlerts() {logger.info("Initializing alerts");const checkActiveJobs = async () => {try {- const scrapeQueue = getScrapeQueue();- const activeJobs = await scrapeQueue.getActiveCount();- if (activeJobs > Number(process.env.ALERT_NUM_ACTIVE_JOBS)) {+ const scrapeQueues = new Array(queueMultiplexWidth)+ .fill(0)+ .map((_, index) => getScrapeQueueByIndex(index));+ const activeJobs = await Promise.all(scrapeQueues.map((queue) => queue.getActiveCount()));+ if (activeJobs.reduce((a, b) => a + b, 0) > Number(process.env.ALERT_NUM_ACTIVE_JOBS)) {logger.warn(`Alert: Number of active jobs is over ${process.env.ALERT_NUM_ACTIVE_JOBS}. Current active jobs: ${activeJobs}.`,);@@ -34,10 +36,12 @@ export async function checkAlerts() {};const checkWaitingQueue = async () => {- const scrapeQueue = getScrapeQueue();- const waitingJobs = await scrapeQueue.getWaitingCount();+ const scrapeQueues = new Array(queueMultiplexWidth)+ .fill(0)+ .map((_, index) => getScrapeQueueByIndex(index));+ const waitingJobs = await Promise.all(scrapeQueues.map((queue) => queue.getWaitingCount()));- if (waitingJobs > Number(process.env.ALERT_NUM_WAITING_JOBS)) {+ if (waitingJobs.reduce((a, b) => a + b, 0) > Number(process.env.ALERT_NUM_WAITING_JOBS)) {logger.warn(`Alert: Number of waiting jobs is over ${process.env.ALERT_NUM_WAITING_JOBS}. Current waiting jobs: ${waitingJobs}.`,);diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex a4a5ab922..3ee4a9282 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -72,7 +72,7 @@ export async function _addScrapeJobToBullMQ(}}- return await getScrapeQueue().add(jobId, webScraperOptions, {+ return await getScrapeQueue(jobId).add(jobId, webScraperOptions, {...options,priority: jobPriority,jobId,@@ -322,7 +322,7 @@ export async function waitForJob(logger: Logger = _logger,): Promise<Document> {const start = Date.now();- const queue = getScrapeQueue();+ const queue = getScrapeQueue(typeof _job === "string" ? _job : (_job.id as string));let job: Job | undefined = typeof _job == "string" ? await queue.getJob(_job) : _job;while (job === undefined) {logger.debug("Waiting for job to be created");@@ -335,7 +335,7 @@ export async function waitForJob(let doc: Document;try {doc = await Promise.race([- job.waitUntilFinished(getScrapeQueueEvents(), timeout ?? 180000),+ job.waitUntilFinished(getScrapeQueueEvents(job.id as string), timeout ?? 180000),timeout !== null ? new Promise((resolve, reject) => {setTimeout(() => {reject(new ScrapeJobTimeoutError("Scrape timed out" + (typeof _job === "string" ? " after waiting in the concurrency limit queue" : "")));diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.tsindex 6041b9ec8..8a56a6caa 100644--- a/apps/api/src/services/queue-service.ts+++ b/apps/api/src/services/queue-service.ts@@ -5,8 +5,8 @@ import { BullMQOtel } from "bullmq-otel";export type QueueFunction = () => Queue<any, any, string, any, any, string>;-let scrapeQueue: Queue;-let scrapeQueueEvents: QueueEvents;+let scrapeQueues: Queue[] = [];+let scrapeQueueEvents: QueueEvents[] = [];let extractQueue: Queue;let loggingQueue: Queue;let indexQueue: Queue;@@ -29,7 +29,6 @@ export function getRedisConnection(): IORedis {return redisConnection;}-export const scrapeQueueName = "{scrapeQueue}";export const extractQueueName = "{extractQueue}";export const loggingQueueName = "{loggingQueue}";export const indexQueueName = "{indexQueue}";@@ -38,9 +37,21 @@ export const deepResearchQueueName = "{deepResearchQueue}";export const billingQueueName = "{billingQueue}";export const precrawlQueueName = "{precrawlQueue}";-export function getScrapeQueue() {- if (!scrapeQueue) {- scrapeQueue = new Queue(scrapeQueueName, {+export const queueMultiplexWidth = process.env.QUEUE_MULTIPLEX_WIDTH ? parseInt(process.env.QUEUE_MULTIPLEX_WIDTH) : 1;++function uuidToQueueIndex(uuid: string) {+ const queueIndex = parseInt(uuid[0] ?? "0", 16) % queueMultiplexWidth;+ return queueIndex;+}++export function getScrapeQueue(uuid: string) {+ const queueIndex = uuidToQueueIndex(uuid);+ return getScrapeQueueByIndex(queueIndex);+}++export function getScrapeQueueByIndex(index: number) {+ if (!scrapeQueues[index]) {+ scrapeQueues[index] = new Queue(`{scrapeQueue${index === 0 ? "" : index}}`, {connection: getRedisConnection(),defaultJobOptions: {removeOnComplete: {@@ -53,17 +64,22 @@ export function getScrapeQueue() {telemetry: new BullMQOtel("firecrawl-bullmq"),});}- return scrapeQueue;+ return scrapeQueues[index];+}++export function getScrapeQueueEvents(uuid: string) {+ const queueIndex = uuidToQueueIndex(uuid);+ return getScrapeQueueEventsByIndex(queueIndex);}-export function getScrapeQueueEvents() {- if (!scrapeQueueEvents) {- scrapeQueueEvents = new QueueEvents(scrapeQueueName, {+export function getScrapeQueueEventsByIndex(index: number) {+ if (!scrapeQueueEvents[index]) {+ scrapeQueueEvents[index] = new QueueEvents(`{scrapeQueue${index === 0 ? "" : index}}`, {connection: getRedisConnection(),});}- return scrapeQueueEvents;+ return scrapeQueueEvents[index];}export function getExtractQueue() {diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 32a154186..3ee0928ed 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -6,8 +6,10 @@ import {getExtractQueue,getDeepResearchQueue,getGenerateLlmsTxtQueue,- scrapeQueueName,getRedisConnection,+ queueMultiplexWidth,+ getScrapeQueueEventsByIndex,+ getScrapeQueueByIndex,} from "./queue-service";import { Job, Queue, QueueEvents } from "bullmq";import { logger as _logger } from "../lib/logger";@@ -508,9 +510,11 @@ app.listen(workerPort, () => {_logger.info(`Liveness endpoint is running on port ${workerPort}`);});+const myScrapeQueue = Math.floor(Math.random() * queueMultiplexWidth);+(async () => {async function failedListener(args: { jobId: string; failedReason: string; prev?: string | undefined; }) {- const job = await getScrapeQueue().getJob(args.jobId);+ const job = await getScrapeQueueByIndex(myScrapeQueue).getJob(args.jobId);if (job && job.data.crawl_id) {await redisEvictConnection.srem("crawl:" + job.data.crawl_id + ":jobs_qualified", args.jobId);@@ -559,12 +563,14 @@ app.listen(workerPort, () => {}}- const scrapeQueueEvents = new QueueEvents(scrapeQueueName, { connection: getRedisConnection() });- scrapeQueueEvents.on("failed", failedListener);+ for (let i = 0; i < queueMultiplexWidth; i++) {+ const scrapeQueueEvents = getScrapeQueueEventsByIndex(i);+ scrapeQueueEvents.on("failed", failedListener);+ }const results = await Promise.all([separateWorkerFun(- getScrapeQueue(),+ getScrapeQueueByIndex(myScrapeQueue),path.join(__dirname, "worker", "scrape-worker.js"),),workerFun(getExtractQueue(), processExtractJobInternal),@@ -586,14 +592,14 @@ app.listen(workerPort, () => {jobs: (await Promise.all([...runningJobs].map(async (jobId) => {- return await getScrapeQueue().getJob(jobId);+ return await getScrapeQueueByIndex(myScrapeQueue).getJob(jobId);}),)).filter((x) => x && !x.data?.zeroDataRetention),});}, 1000);- await scrapeQueueEvents.close();+ await getScrapeQueueEventsByIndex(myScrapeQueue).close();console.log("All jobs finished. Worker out!");if (otelSdk) {await otelSdk.shutdown();