Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. π₯
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
Revert "Merge pull request #1982 from firecrawl/mog/v2-queue-multiplex"
GergΕ MΓ³ricz0d4c0e0e7ef2
diff --git a/apps/api/src/controllers/v0/admin/queue.ts b/apps/api/src/controllers/v0/admin/queue.tsindex 4d80918fc..d7d9c0897 100644--- a/apps/api/src/controllers/v0/admin/queue.ts+++ b/apps/api/src/controllers/v0/admin/queue.ts@@ -2,10 +2,61 @@ import { Request, Response } from "express";import { Job } from "bullmq";import { logger } from "../../../lib/logger";-import { getScrapeQueue, getScrapeQueueByIndex, queueMultiplexWidth } from "../../../services/queue-service";+import { getScrapeQueue } from "../../../services/queue-service";import { checkAlerts } from "../../../services/alerts";import { sendSlackWebhook } from "../../../services/alerts/slack";+export async function cleanBefore24hCompleteJobsController(+ req: Request,+ res: Response,+) {+ logger.info("π Cleaning jobs older than 24h");+ try {+ const scrapeQueue = getScrapeQueue();+ const batchSize = 10;+ const numberOfBatches = 9; // Adjust based on your needs+ const completedJobsPromises: Promise<Job[]>[] = [];+ for (let i = 0; i < numberOfBatches; i++) {+ completedJobsPromises.push(+ scrapeQueue.getJobs(+ ["completed"],+ i * batchSize,+ i * batchSize + batchSize,+ true,+ ),+ );+ }+ const completedJobs: Job[] = (+ await Promise.all(completedJobsPromises)+ ).flat();+ const before24hJobs =+ completedJobs.filter(+ (job) =>+ job.finishedOn !== undefined &&+ job.finishedOn < Date.now() - 24 * 60 * 60 * 1000,+ ) || [];++ let count = 0;++ if (!before24hJobs) {+ return res.status(200).send(`No jobs to remove.`);+ }++ for (const job of before24hJobs) {+ try {+ await job.remove();+ count++;+ } catch (jobError) {+ logger.error(`π Failed to remove job with ID ${job.id}: ${jobError}`);+ }+ }+ return res.status(200).send(`Removed ${count} completed jobs.`);+ } catch (error) {+ logger.error(`π Failed to clean last 24h complete jobs: ${error}`);+ return res.status(500).send("Failed to clean jobs");+ }+}+export async function checkQueuesController(req: Request, res: Response) {try {await checkAlerts();@@ -19,11 +70,11 @@ export async function checkQueuesController(req: Request, res: Response) {// Use this as a "health check" that way we dont destroy the serverexport async function queuesController(req: Request, res: Response) {try {- const scrapeQueues = new Array(queueMultiplexWidth)- .fill(0)- .map((_, index) => getScrapeQueueByIndex(index));+ const scrapeQueue = getScrapeQueue();- const webScraperActive = (await Promise.all(scrapeQueues.map((queue) => queue.getActiveCount()))).reduce((a, b) => a + b, 0);+ const [webScraperActive] = await Promise.all([+ scrapeQueue.getActiveCount(),+ ]);const noActiveJobs = webScraperActive === 0;// 200 if no active jobs, 503 if there are active jobs@@ -36,3 +87,115 @@ export async function queuesController(req: Request, res: Response) {return res.status(500).json({ error: error.message });}}++export async function autoscalerController(req: Request, res: Response) {+ try {+ const maxNumberOfMachines = 80;+ const minNumberOfMachines = 20;++ const scrapeQueue = getScrapeQueue();++ const [webScraperActive, webScraperWaiting, webScraperPriority] =+ await Promise.all([+ scrapeQueue.getActiveCount(),+ scrapeQueue.getWaitingCount(),+ scrapeQueue.getPrioritizedCount(),+ ]);++ let waitingAndPriorityCount = webScraperWaiting + webScraperPriority;++ // get number of machines active+ const request = await fetch(+ "https://api.machines.dev/v1/apps/firecrawl-scraper-js/machines",+ {+ headers: {+ Authorization: `Bearer ${process.env.FLY_API_TOKEN}`,+ },+ },+ );+ const machines = await request.json();++ // Only worker machines+ const activeMachines = machines.filter(+ (machine) =>+ (machine.state === "started" ||+ machine.state === "starting" ||+ machine.state === "replacing") &&+ machine.config.env["FLY_PROCESS_GROUP"] === "worker",+ ).length;++ let targetMachineCount = activeMachines;++ const baseScaleUp = 10;+ // Slow scale down+ const baseScaleDown = 2;++ // Scale up logic+ if (webScraperActive > 9000 || waitingAndPriorityCount > 2000) {+ targetMachineCount = Math.min(+ maxNumberOfMachines,+ activeMachines + baseScaleUp * 3,+ );+ } else if (webScraperActive > 5000 || waitingAndPriorityCount > 1000) {+ targetMachineCount = Math.min(+ maxNumberOfMachines,+ activeMachines + baseScaleUp * 2,+ );+ } else if (webScraperActive > 1000 || waitingAndPriorityCount > 500) {+ targetMachineCount = Math.min(+ maxNumberOfMachines,+ activeMachines + baseScaleUp,+ );+ }++ // Scale down logic+ if (webScraperActive < 100 && waitingAndPriorityCount < 50) {+ targetMachineCount = Math.max(+ minNumberOfMachines,+ activeMachines - baseScaleDown * 3,+ );+ } else if (webScraperActive < 500 && waitingAndPriorityCount < 200) {+ targetMachineCount = Math.max(+ minNumberOfMachines,+ activeMachines - baseScaleDown * 2,+ );+ } else if (webScraperActive < 1000 && waitingAndPriorityCount < 500) {+ targetMachineCount = Math.max(+ minNumberOfMachines,+ activeMachines - baseScaleDown,+ );+ }++ if (targetMachineCount !== activeMachines) {+ logger.info(+ `π Scaling from ${activeMachines} to ${targetMachineCount} - ${webScraperActive} active, ${webScraperWaiting} waiting`,+ );++ if (targetMachineCount > activeMachines) {+ sendSlackWebhook(+ `π Scaling from ${activeMachines} to ${targetMachineCount} - ${webScraperActive} active, ${webScraperWaiting} waiting - Current DateTime: ${new Date().toISOString()}`,+ false,+ process.env.SLACK_AUTOSCALER ?? "",+ );+ } else {+ sendSlackWebhook(+ `π Scaling from ${activeMachines} to ${targetMachineCount} - ${webScraperActive} active, ${webScraperWaiting} waiting - Current DateTime: ${new Date().toISOString()}`,+ false,+ process.env.SLACK_AUTOSCALER ?? "",+ );+ }+ return res.status(200).json({+ mode: "scale-descale",+ count: targetMachineCount,+ });+ }++ return res.status(200).json({+ mode: "normal",+ count: activeMachines,+ });+ } catch (error) {+ logger.error(error);+ return res.status(500).send("Failed to initialize autoscaler");+ }+}diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.tsindex d76f5ebfa..843eb56d1 100644--- a/apps/api/src/controllers/v0/crawl-status.ts+++ b/apps/api/src/controllers/v0/crawl-status.ts@@ -16,7 +16,7 @@ configDotenv();export async function getJobs(crawlId: string, ids: string[]): Promise<PseudoJob<any>[]> {const [bullJobs, dbJobs, gcsJobs] = await Promise.all([- Promise.all(ids.map((x) => getScrapeQueue(x).getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,+ Promise.all(ids.map((x) => getScrapeQueue().getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,process.env.USE_DB_AUTHENTICATION === "true" ? await supabaseGetJobsByCrawlId(crawlId) : [],process.env.GCS_BUCKET_NAME ? Promise.all(ids.map(async (x) => ({ id: x, job: await getJobFromGCS(x) }))).then(x => x.filter(x => x.job)) as Promise<({ id: string, job: any | null })[]> : [],]);diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.tsindex d4c622bcc..8435da31e 100644--- a/apps/api/src/controllers/v0/scrape.ts+++ b/apps/api/src/controllers/v0/scrape.ts@@ -127,7 +127,7 @@ export async function scrapeHelper(return err;}- await getScrapeQueue(jobId).remove(jobId);+ await getScrapeQueue().remove(jobId);if (!doc) {console.error("!!! PANIC DOC IS", doc);diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.tsindex 45d98d531..5ad0f4dff 100644--- a/apps/api/src/controllers/v0/search.ts+++ b/apps/api/src/controllers/v0/search.ts@@ -139,7 +139,8 @@ export async function searchHelper(return { success: true, error: "No search results found", returnCode: 200 };}- await Promise.all(jobDatas.map((x) => getScrapeQueue(x.opts.jobId).remove(x.opts.jobId)));+ const sq = getScrapeQueue();+ await Promise.all(jobDatas.map((x) => sq.remove(x.opts.jobId)));// make sure doc.content is not emptyconst filteredDocs = docs.filter(diff --git a/apps/api/src/controllers/v1/crawl-errors.ts b/apps/api/src/controllers/v1/crawl-errors.tsindex 85fb491a4..45084c741 100644--- a/apps/api/src/controllers/v1/crawl-errors.ts+++ b/apps/api/src/controllers/v1/crawl-errors.ts@@ -19,7 +19,7 @@ import { TransportableError } from "../../lib/error";configDotenv();export async function getJob(id: string) {- const job = await getScrapeQueue(id).getJob(id);+ const job = await getScrapeQueue().getJob(id);if (!job) return job;return job;@@ -27,7 +27,7 @@ export async function getJob(id: string) {export async function getJobs(ids: string[]) {const jobs: (Job & { id: string })[] = (- await Promise.all(ids.map((x) => getScrapeQueue(x).getJob(x)))+ await Promise.all(ids.map((x) => getScrapeQueue().getJob(x)))).filter((x) => x) as (Job & { id: string })[];return jobs;@@ -46,7 +46,7 @@ export async function crawlErrorsController(let jobStatuses = await Promise.all((await getCrawlJobs(req.params.jobId)).map(- async (x) => [x, await getScrapeQueue(x).getJobState(x)] as const,+ async (x) => [x, await getScrapeQueue().getJobState(x)] as const,),);diff --git a/apps/api/src/controllers/v1/crawl-status-ws.ts b/apps/api/src/controllers/v1/crawl-status-ws.tsindex a82d49706..190ca5209 100644--- a/apps/api/src/controllers/v1/crawl-status-ws.ts+++ b/apps/api/src/controllers/v1/crawl-status-ws.ts@@ -88,10 +88,12 @@ async function crawlStatusWS(const notDoneJobIDs = jobIDs.filter((x) => !doneJobIDs.includes(x));+ const queue = getScrapeQueue();+const jobStatuses = await Promise.all(notDoneJobIDs.map(async (x) => [x,- await getScrapeQueue(x).getJobState(x),+ await queue.getJobState(x),]),);const newlyDoneJobIDs: string[] = jobStatuses@@ -123,9 +125,11 @@ async function crawlStatusWS(let jobIDs = await getCrawlJobs(req.params.jobId);+ const queue = getScrapeQueue();+let jobStatuses = await Promise.all(jobIDs.map(- async (x) => [x, await getScrapeQueue(x).getJobState(x)] as const,+ async (x) => [x, await queue.getJobState(x)] as const,),);diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex 01f02e9a5..85a305f54 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -41,7 +41,7 @@ export type DBJob = { docs: any, success: boolean, page_options: any, date_addedexport async function getJob(id: string): Promise<PseudoJob<any> | null> {const [bullJob, dbJob, gcsJob] = await Promise.all([- getScrapeQueue(id).getJob(id),+ getScrapeQueue().getJob(id),(process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobById(id) : null) as Promise<DBJob | null>,(process.env.GCS_BUCKET_NAME ? getJobFromGCS(id) : null) as Promise<any | null>,]);@@ -73,7 +73,7 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {const [bullJobs, dbJobs, gcsJobs] = await Promise.all([- Promise.all(ids.map((x) => getScrapeQueue(x).getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,+ Promise.all(ids.map((x) => getScrapeQueue().getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobsById(ids) : [],process.env.GCS_BUCKET_NAME ? Promise.all(ids.map(async (x) => ({ id: x, job: await getJobFromGCS(x) }))).then(x => x.filter(x => x.job)) as Promise<({ id: string, job: any | null })[]> : [],]);@@ -315,7 +315,7 @@ export async function crawlStatusController(const bytesLimit = 10485760; // 10 MiB in bytesfor (const jobId of doneJobs) {- const job = await getScrapeQueue(jobId).getJob(jobId);+ const job = await getScrapeQueue().getJob(jobId);const state = await job?.getState();if (state === "failed") {diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.tsindex b0444053c..b5b82ca43 100644--- a/apps/api/src/controllers/v1/scrape.ts+++ b/apps/api/src/controllers/v1/scrape.ts@@ -105,7 +105,7 @@ export async function scrapeController(});if (zeroDataRetention) {- await getScrapeQueue(jobId).remove(jobId);+ await getScrapeQueue().remove(jobId);}if (e instanceof TransportableError) {@@ -125,7 +125,7 @@ export async function scrapeController(logger.info("Done with waitForJob");- await getScrapeQueue(jobId).remove(jobId);+ await getScrapeQueue().remove(jobId);logger.info("Removed job from queue");diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex eabc6c6de..f185ce4df 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -138,7 +138,7 @@ async function scrapeSearchResult(teamId: options.teamId,origin: options.origin,});- await getScrapeQueue(jobId).remove(jobId);+ await getScrapeQueue().remove(jobId);const document = {title: searchResult.title,diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.tsindex 371aeb758..ff0a66659 100644--- a/apps/api/src/controllers/v1/x402-search.ts+++ b/apps/api/src/controllers/v1/x402-search.ts@@ -133,7 +133,7 @@ async function scrapeX402SearchResult(teamId: options.teamId,origin: options.origin,});- await getScrapeQueue(jobId).remove(jobId);+ await getScrapeQueue().remove(jobId);const document = {title: searchResult.title,diff --git a/apps/api/src/controllers/v2/crawl-errors.ts b/apps/api/src/controllers/v2/crawl-errors.tsindex 85fb491a4..45084c741 100644--- a/apps/api/src/controllers/v2/crawl-errors.ts+++ b/apps/api/src/controllers/v2/crawl-errors.ts@@ -19,7 +19,7 @@ import { TransportableError } from "../../lib/error";configDotenv();export async function getJob(id: string) {- const job = await getScrapeQueue(id).getJob(id);+ const job = await getScrapeQueue().getJob(id);if (!job) return job;return job;@@ -27,7 +27,7 @@ export async function getJob(id: string) {export async function getJobs(ids: string[]) {const jobs: (Job & { id: string })[] = (- await Promise.all(ids.map((x) => getScrapeQueue(x).getJob(x)))+ await Promise.all(ids.map((x) => getScrapeQueue().getJob(x)))).filter((x) => x) as (Job & { id: string })[];return jobs;@@ -46,7 +46,7 @@ export async function crawlErrorsController(let jobStatuses = await Promise.all((await getCrawlJobs(req.params.jobId)).map(- async (x) => [x, await getScrapeQueue(x).getJobState(x)] as const,+ async (x) => [x, await getScrapeQueue().getJobState(x)] as const,),);diff --git a/apps/api/src/controllers/v2/crawl-status-ws.ts b/apps/api/src/controllers/v2/crawl-status-ws.tsindex a82d49706..190ca5209 100644--- a/apps/api/src/controllers/v2/crawl-status-ws.ts+++ b/apps/api/src/controllers/v2/crawl-status-ws.ts@@ -88,10 +88,12 @@ async function crawlStatusWS(const notDoneJobIDs = jobIDs.filter((x) => !doneJobIDs.includes(x));+ const queue = getScrapeQueue();+const jobStatuses = await Promise.all(notDoneJobIDs.map(async (x) => [x,- await getScrapeQueue(x).getJobState(x),+ await queue.getJobState(x),]),);const newlyDoneJobIDs: string[] = jobStatuses@@ -123,9 +125,11 @@ async function crawlStatusWS(let jobIDs = await getCrawlJobs(req.params.jobId);+ const queue = getScrapeQueue();+let jobStatuses = await Promise.all(jobIDs.map(- async (x) => [x, await getScrapeQueue(x).getJobState(x)] as const,+ async (x) => [x, await queue.getJobState(x)] as const,),);diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.tsindex e8af8fd9b..34e3fcbd3 100644--- a/apps/api/src/controllers/v2/crawl-status.ts+++ b/apps/api/src/controllers/v2/crawl-status.ts@@ -41,7 +41,7 @@ export type DBJob = { docs: any, success: boolean, page_options: any, date_addedexport async function getJob(id: string): Promise<PseudoJob<any> | null> {const [bullJob, dbJob, gcsJob] = await Promise.all([- getScrapeQueue(id).getJob(id),+ getScrapeQueue().getJob(id),(process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobById(id) : null) as Promise<DBJob | null>,(process.env.GCS_BUCKET_NAME ? getJobFromGCS(id) : null) as Promise<any | null>,]);@@ -73,7 +73,7 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {const [bullJobs, dbJobs, gcsJobs] = await Promise.all([- Promise.all(ids.map((x) => getScrapeQueue(x).getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,+ Promise.all(ids.map((x) => getScrapeQueue().getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobsById(ids) : [],process.env.GCS_BUCKET_NAME ? Promise.all(ids.map(async (x) => ({ id: x, job: await getJobFromGCS(x) }))).then(x => x.filter(x => x.job)) as Promise<({ id: string, job: any | null })[]> : [],]);@@ -315,7 +315,7 @@ export async function crawlStatusController(const bytesLimit = 10485760; // 10 MiB in bytesfor (const jobId of doneJobs) {- const job = await getScrapeQueue(jobId).getJob(jobId);+ const job = await getScrapeQueue().getJob(jobId);const state = await job?.getState();if (state === "failed") {diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.tsindex c7e268043..b34fe5a31 100644--- a/apps/api/src/controllers/v2/scrape.ts+++ b/apps/api/src/controllers/v2/scrape.ts@@ -106,7 +106,7 @@ export async function scrapeController(});if (zeroDataRetention) {- await getScrapeQueue(jobId).remove(jobId);+ await getScrapeQueue().remove(jobId);}if (e instanceof TransportableError) {@@ -123,7 +123,7 @@ export async function scrapeController(}}- await getScrapeQueue(jobId).remove(jobId);+ await getScrapeQueue().remove(jobId);if (!hasFormatOfType(req.body.formats, "rawHtml")) {if (doc && doc.rawHtml) {diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 9129e7166..158ff149d 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -95,7 +95,7 @@ async function scrapeSearchResult(teamId: options.teamId,origin: options.origin,});- await getScrapeQueue(jobId).remove(jobId);+ await getScrapeQueue().remove(jobId);const document = {title: searchResult.title,diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex e1ffdc0be..eb16f451a 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -6,12 +6,11 @@ import bodyParser from "body-parser";import cors from "cors";import {getExtractQueue,+ getScrapeQueue,getGenerateLlmsTxtQueue,getDeepResearchQueue,getBillingQueue,getPrecrawlQueue,- getScrapeQueueByIndex,- queueMultiplexWidth,} from "./services/queue-service";import { v0Router } from "./routes/v0";import os from "os";@@ -87,7 +86,7 @@ serverAdapter.setBasePath(`/admin/${process.env.BULL_AUTH_KEY}/queues`);const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({queues: [- ...new Array(queueMultiplexWidth).fill(0).map((_, index) => new BullMQAdapter(getScrapeQueueByIndex(index))),+ new BullMQAdapter(getScrapeQueue()),new BullMQAdapter(getExtractQueue()),new BullMQAdapter(getGenerateLlmsTxtQueue()),new BullMQAdapter(getDeepResearchQueue()),@@ -160,11 +159,9 @@ if (require.main === module) {app.get(`/serverHealthCheck`, async (req, res) => {try {- const scrapeQueues = new Array(queueMultiplexWidth)- .fill(0)- .map((_, index) => getScrapeQueueByIndex(index));- const waitingJobs = await Promise.all(scrapeQueues.map((queue) => queue.getWaitingCount()));- const noWaitingJobs = waitingJobs.reduce((a, b) => a + b, 0) === 0;+ const scrapeQueue = getScrapeQueue();+ const [waitingJobs] = await Promise.all([scrapeQueue.getWaitingCount()]);+ const noWaitingJobs = waitingJobs === 0;// 200 if no active jobs, 503 if there are active jobsreturn res.status(noWaitingJobs ? 200 : 500).json({waitingJobs,@@ -182,12 +179,12 @@ app.get("/serverHealthCheck/notify", async (req, res) => {const timeout = 60000; // 1 minute // The timeout value for the check in millisecondsconst getWaitingJobsCount = async () => {- const scrapeQueues = new Array(queueMultiplexWidth)- .fill(0)- .map((_, index) => getScrapeQueueByIndex(index));- const waitingJobsCount = await Promise.all(scrapeQueues.map((queue) => queue.getWaitingCount()));+ const scrapeQueue = getScrapeQueue();+ const [waitingJobsCount] = await Promise.all([+ scrapeQueue.getWaitingCount(),+ ]);- return waitingJobsCount.reduce((a, b) => a + b, 0);+ return waitingJobsCount;};res.status(200).json({ message: "Check initiated" });@@ -299,3 +296,12 @@ app.use();logger.info(`Worker ${process.pid} started`);+// const sq = getScrapeQueue();++// sq.on("waiting", j => ScrapeEvents.logJobEvent(j, "waiting"));+// sq.on("active", j => ScrapeEvents.logJobEvent(j, "active"));+// sq.on("completed", j => ScrapeEvents.logJobEvent(j, "completed"));+// sq.on("paused", j => ScrapeEvents.logJobEvent(j, "paused"));+// sq.on("resumed", j => ScrapeEvents.logJobEvent(j, "resumed"));+// sq.on("removed", j => ScrapeEvents.logJobEvent(j, "removed"));+//diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.tsindex eba623603..52fc2fbdb 100644--- a/apps/api/src/lib/concurrency-limit.ts+++ b/apps/api/src/lib/concurrency-limit.ts@@ -274,7 +274,7 @@ export async function concurrentJobDone(job: Job) {}}- (await getScrapeQueue(nextJob.job.id)).add(+ (await getScrapeQueue()).add(nextJob.job.id,{...nextJob.job.data,diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.tsindex 0913615f1..1b84733b2 100644--- a/apps/api/src/lib/extract/document-scraper.ts+++ b/apps/api/src/lib/extract/document-scraper.ts@@ -66,7 +66,7 @@ export async function scrapeDocument(const doc = await waitForJob(jobId, timeout);- await getScrapeQueue(jobId).remove(jobId);+ await getScrapeQueue().remove(jobId);if (trace) {trace.timing.completedAt = new Date().toISOString();diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.tsindex 5e2629744..a4b524a64 100644--- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts+++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts@@ -66,7 +66,7 @@ export async function scrapeDocument_F0();const doc = await waitForJob(jobId, timeout);- await getScrapeQueue(jobId).remove(jobId);+ await getScrapeQueue().remove(jobId);if (trace) {trace.timing.completedAt = new Date().toISOString();diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.tsindex 51a713e3e..25f0880ad 100644--- a/apps/api/src/routes/admin.ts+++ b/apps/api/src/routes/admin.ts@@ -1,7 +1,9 @@import express from "express";import { redisHealthController } from "../controllers/v0/admin/redis-health";import {+ autoscalerController,checkQueuesController,+ cleanBefore24hCompleteJobsController,queuesController,} from "../controllers/v0/admin/queue";import { wrap } from "./shared";@@ -20,6 +22,11 @@ adminRouter.get(redisHealthController,);+adminRouter.get(+ `/admin/${process.env.BULL_AUTH_KEY}/clean-before-24h-complete-jobs`,+ cleanBefore24hCompleteJobsController,+);+adminRouter.get(`/admin/${process.env.BULL_AUTH_KEY}/check-queues`,checkQueuesController,@@ -27,6 +34,11 @@ adminRouter.get(adminRouter.get(`/admin/${process.env.BULL_AUTH_KEY}/queues`, queuesController);+adminRouter.get(+ `/admin/${process.env.BULL_AUTH_KEY}/autoscaler`,+ autoscalerController,+);+adminRouter.post(`/admin/${process.env.BULL_AUTH_KEY}/acuc-cache-clear`,wrap(acucCacheClearController),diff --git a/apps/api/src/services/alerts/index.ts b/apps/api/src/services/alerts/index.tsindex b7a9a9455..44f2b8a04 100644--- a/apps/api/src/services/alerts/index.ts+++ b/apps/api/src/services/alerts/index.ts@@ -1,5 +1,5 @@import { logger } from "../../../src/lib/logger";-import { getScrapeQueueByIndex, queueMultiplexWidth } from "../queue-service";+import { getScrapeQueue } from "../queue-service";import { sendSlackWebhook } from "./slack";export async function checkAlerts() {@@ -13,11 +13,9 @@ export async function checkAlerts() {logger.info("Initializing alerts");const checkActiveJobs = async () => {try {- const scrapeQueues = new Array(queueMultiplexWidth)- .fill(0)- .map((_, index) => getScrapeQueueByIndex(index));- const activeJobs = await Promise.all(scrapeQueues.map((queue) => queue.getActiveCount()));- if (activeJobs.reduce((a, b) => a + b, 0) > Number(process.env.ALERT_NUM_ACTIVE_JOBS)) {+ const scrapeQueue = getScrapeQueue();+ const activeJobs = await scrapeQueue.getActiveCount();+ if (activeJobs > Number(process.env.ALERT_NUM_ACTIVE_JOBS)) {logger.warn(`Alert: Number of active jobs is over ${process.env.ALERT_NUM_ACTIVE_JOBS}. Current active jobs: ${activeJobs}.`,);@@ -36,12 +34,10 @@ export async function checkAlerts() {};const checkWaitingQueue = async () => {- const scrapeQueues = new Array(queueMultiplexWidth)- .fill(0)- .map((_, index) => getScrapeQueueByIndex(index));- const waitingJobs = await Promise.all(scrapeQueues.map((queue) => queue.getWaitingCount()));+ const scrapeQueue = getScrapeQueue();+ const waitingJobs = await scrapeQueue.getWaitingCount();- if (waitingJobs.reduce((a, b) => a + b, 0) > Number(process.env.ALERT_NUM_WAITING_JOBS)) {+ if (waitingJobs > Number(process.env.ALERT_NUM_WAITING_JOBS)) {logger.warn(`Alert: Number of waiting jobs is over ${process.env.ALERT_NUM_WAITING_JOBS}. Current waiting jobs: ${waitingJobs}.`,);diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.tsindex 3ee4a9282..a4a5ab922 100644--- a/apps/api/src/services/queue-jobs.ts+++ b/apps/api/src/services/queue-jobs.ts@@ -72,7 +72,7 @@ export async function _addScrapeJobToBullMQ(}}- return await getScrapeQueue(jobId).add(jobId, webScraperOptions, {+ return await getScrapeQueue().add(jobId, webScraperOptions, {...options,priority: jobPriority,jobId,@@ -322,7 +322,7 @@ export async function waitForJob(logger: Logger = _logger,): Promise<Document> {const start = Date.now();- const queue = getScrapeQueue(typeof _job === "string" ? _job : (_job.id as string));+ const queue = getScrapeQueue();let job: Job | undefined = typeof _job == "string" ? await queue.getJob(_job) : _job;while (job === undefined) {logger.debug("Waiting for job to be created");@@ -335,7 +335,7 @@ export async function waitForJob(let doc: Document;try {doc = await Promise.race([- job.waitUntilFinished(getScrapeQueueEvents(job.id as string), timeout ?? 180000),+ job.waitUntilFinished(getScrapeQueueEvents(), timeout ?? 180000),timeout !== null ? new Promise((resolve, reject) => {setTimeout(() => {reject(new ScrapeJobTimeoutError("Scrape timed out" + (typeof _job === "string" ? " after waiting in the concurrency limit queue" : "")));diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.tsindex 8a56a6caa..6041b9ec8 100644--- a/apps/api/src/services/queue-service.ts+++ b/apps/api/src/services/queue-service.ts@@ -5,8 +5,8 @@ import { BullMQOtel } from "bullmq-otel";export type QueueFunction = () => Queue<any, any, string, any, any, string>;-let scrapeQueues: Queue[] = [];-let scrapeQueueEvents: QueueEvents[] = [];+let scrapeQueue: Queue;+let scrapeQueueEvents: QueueEvents;let extractQueue: Queue;let loggingQueue: Queue;let indexQueue: Queue;@@ -29,6 +29,7 @@ export function getRedisConnection(): IORedis {return redisConnection;}+export const scrapeQueueName = "{scrapeQueue}";export const extractQueueName = "{extractQueue}";export const loggingQueueName = "{loggingQueue}";export const indexQueueName = "{indexQueue}";@@ -37,21 +38,9 @@ export const deepResearchQueueName = "{deepResearchQueue}";export const billingQueueName = "{billingQueue}";export const precrawlQueueName = "{precrawlQueue}";-export const queueMultiplexWidth = process.env.QUEUE_MULTIPLEX_WIDTH ? parseInt(process.env.QUEUE_MULTIPLEX_WIDTH) : 1;--function uuidToQueueIndex(uuid: string) {- const queueIndex = parseInt(uuid[0] ?? "0", 16) % queueMultiplexWidth;- return queueIndex;-}--export function getScrapeQueue(uuid: string) {- const queueIndex = uuidToQueueIndex(uuid);- return getScrapeQueueByIndex(queueIndex);-}--export function getScrapeQueueByIndex(index: number) {- if (!scrapeQueues[index]) {- scrapeQueues[index] = new Queue(`{scrapeQueue${index === 0 ? "" : index}}`, {+export function getScrapeQueue() {+ if (!scrapeQueue) {+ scrapeQueue = new Queue(scrapeQueueName, {connection: getRedisConnection(),defaultJobOptions: {removeOnComplete: {@@ -64,22 +53,17 @@ export function getScrapeQueueByIndex(index: number) {telemetry: new BullMQOtel("firecrawl-bullmq"),});}- return scrapeQueues[index];-}--export function getScrapeQueueEvents(uuid: string) {- const queueIndex = uuidToQueueIndex(uuid);- return getScrapeQueueEventsByIndex(queueIndex);+ return scrapeQueue;}-export function getScrapeQueueEventsByIndex(index: number) {- if (!scrapeQueueEvents[index]) {- scrapeQueueEvents[index] = new QueueEvents(`{scrapeQueue${index === 0 ? "" : index}}`, {+export function getScrapeQueueEvents() {+ if (!scrapeQueueEvents) {+ scrapeQueueEvents = new QueueEvents(scrapeQueueName, {connection: getRedisConnection(),});}- return scrapeQueueEvents[index];+ return scrapeQueueEvents;}export function getExtractQueue() {diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex e7b0b2e53..32a154186 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -6,10 +6,8 @@ import {getExtractQueue,getDeepResearchQueue,getGenerateLlmsTxtQueue,+ scrapeQueueName,getRedisConnection,- queueMultiplexWidth,- getScrapeQueueEventsByIndex,- getScrapeQueueByIndex,} from "./queue-service";import { Job, Queue, QueueEvents } from "bullmq";import { logger as _logger } from "../lib/logger";@@ -510,15 +508,9 @@ app.listen(workerPort, () => {_logger.info(`Liveness endpoint is running on port ${workerPort}`);});-const myScrapeQueue = process.env.QUEUE_MULTIPLEX_INDEX !== undefined- ? parseInt(process.env.QUEUE_MULTIPLEX_INDEX)- : Math.floor(Math.random() * queueMultiplexWidth);--_logger.info(`Using scrape queue ${myScrapeQueue}`);-(async () => {async function failedListener(args: { jobId: string; failedReason: string; prev?: string | undefined; }) {- const job = await getScrapeQueueByIndex(myScrapeQueue).getJob(args.jobId);+ const job = await getScrapeQueue().getJob(args.jobId);if (job && job.data.crawl_id) {await redisEvictConnection.srem("crawl:" + job.data.crawl_id + ":jobs_qualified", args.jobId);@@ -567,14 +559,12 @@ _logger.info(`Using scrape queue ${myScrapeQueue}`);}}- for (let i = 0; i < queueMultiplexWidth; i++) {- const scrapeQueueEvents = getScrapeQueueEventsByIndex(i);- scrapeQueueEvents.on("failed", failedListener);- }+ const scrapeQueueEvents = new QueueEvents(scrapeQueueName, { connection: getRedisConnection() });+ scrapeQueueEvents.on("failed", failedListener);const results = await Promise.all([separateWorkerFun(- getScrapeQueueByIndex(myScrapeQueue),+ getScrapeQueue(),path.join(__dirname, "worker", "scrape-worker.js"),),workerFun(getExtractQueue(), processExtractJobInternal),@@ -596,14 +586,14 @@ _logger.info(`Using scrape queue ${myScrapeQueue}`);jobs: (await Promise.all([...runningJobs].map(async (jobId) => {- return await getScrapeQueueByIndex(myScrapeQueue).getJob(jobId);+ return await getScrapeQueue().getJob(jobId);}),)).filter((x) => x && !x.data?.zeroDataRetention),});}, 1000);- await getScrapeQueueEventsByIndex(myScrapeQueue).close();+ await scrapeQueueEvents.close();console.log("All jobs finished. Worker out!");if (otelSdk) {await otelSdk.shutdown();