Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. πŸ”₯

Revert "Merge pull request #1982 from firecrawl/mog/v2-queue-multiplex"

GergΕ‘ MΓ³ricz0d4c0e0e7ef2
diff --git a/apps/api/src/controllers/v0/admin/queue.ts b/apps/api/src/controllers/v0/admin/queue.ts
index 4d80918fc..d7d9c0897 100644
--- a/apps/api/src/controllers/v0/admin/queue.ts
+++ b/apps/api/src/controllers/v0/admin/queue.ts
@@ -2,10 +2,61 @@ import { Request, Response } from "express";
import { Job } from "bullmq";
import { logger } from "../../../lib/logger";
-import { getScrapeQueue, getScrapeQueueByIndex, queueMultiplexWidth } from "../../../services/queue-service";
+import { getScrapeQueue } from "../../../services/queue-service";
import { checkAlerts } from "../../../services/alerts";
import { sendSlackWebhook } from "../../../services/alerts/slack";
+export async function cleanBefore24hCompleteJobsController(
+ req: Request,
+ res: Response,
+) {
+ logger.info("πŸ‚ Cleaning jobs older than 24h");
+ try {
+ const scrapeQueue = getScrapeQueue();
+ const batchSize = 10;
+ const numberOfBatches = 9; // Adjust based on your needs
+ const completedJobsPromises: Promise<Job[]>[] = [];
+ for (let i = 0; i < numberOfBatches; i++) {
+ completedJobsPromises.push(
+ scrapeQueue.getJobs(
+ ["completed"],
+ i * batchSize,
+ i * batchSize + batchSize,
+ true,
+ ),
+ );
+ }
+ const completedJobs: Job[] = (
+ await Promise.all(completedJobsPromises)
+ ).flat();
+ const before24hJobs =
+ completedJobs.filter(
+ (job) =>
+ job.finishedOn !== undefined &&
+ job.finishedOn < Date.now() - 24 * 60 * 60 * 1000,
+ ) || [];
+
+ let count = 0;
+
+ if (!before24hJobs) {
+ return res.status(200).send(`No jobs to remove.`);
+ }
+
+ for (const job of before24hJobs) {
+ try {
+ await job.remove();
+ count++;
+ } catch (jobError) {
+ logger.error(`πŸ‚ Failed to remove job with ID ${job.id}: ${jobError}`);
+ }
+ }
+ return res.status(200).send(`Removed ${count} completed jobs.`);
+ } catch (error) {
+ logger.error(`πŸ‚ Failed to clean last 24h complete jobs: ${error}`);
+ return res.status(500).send("Failed to clean jobs");
+ }
+}
+
export async function checkQueuesController(req: Request, res: Response) {
try {
await checkAlerts();
@@ -19,11 +70,11 @@ export async function checkQueuesController(req: Request, res: Response) {
// Use this as a "health check" that way we dont destroy the server
export async function queuesController(req: Request, res: Response) {
try {
- const scrapeQueues = new Array(queueMultiplexWidth)
- .fill(0)
- .map((_, index) => getScrapeQueueByIndex(index));
+ const scrapeQueue = getScrapeQueue();
- const webScraperActive = (await Promise.all(scrapeQueues.map((queue) => queue.getActiveCount()))).reduce((a, b) => a + b, 0);
+ const [webScraperActive] = await Promise.all([
+ scrapeQueue.getActiveCount(),
+ ]);
const noActiveJobs = webScraperActive === 0;
// 200 if no active jobs, 503 if there are active jobs
@@ -36,3 +87,115 @@ export async function queuesController(req: Request, res: Response) {
return res.status(500).json({ error: error.message });
}
}
+
+export async function autoscalerController(req: Request, res: Response) {
+ try {
+ const maxNumberOfMachines = 80;
+ const minNumberOfMachines = 20;
+
+ const scrapeQueue = getScrapeQueue();
+
+ const [webScraperActive, webScraperWaiting, webScraperPriority] =
+ await Promise.all([
+ scrapeQueue.getActiveCount(),
+ scrapeQueue.getWaitingCount(),
+ scrapeQueue.getPrioritizedCount(),
+ ]);
+
+ let waitingAndPriorityCount = webScraperWaiting + webScraperPriority;
+
+ // get number of machines active
+ const request = await fetch(
+ "https://api.machines.dev/v1/apps/firecrawl-scraper-js/machines",
+ {
+ headers: {
+ Authorization: `Bearer ${process.env.FLY_API_TOKEN}`,
+ },
+ },
+ );
+ const machines = await request.json();
+
+ // Only worker machines
+ const activeMachines = machines.filter(
+ (machine) =>
+ (machine.state === "started" ||
+ machine.state === "starting" ||
+ machine.state === "replacing") &&
+ machine.config.env["FLY_PROCESS_GROUP"] === "worker",
+ ).length;
+
+ let targetMachineCount = activeMachines;
+
+ const baseScaleUp = 10;
+ // Slow scale down
+ const baseScaleDown = 2;
+
+ // Scale up logic
+ if (webScraperActive > 9000 || waitingAndPriorityCount > 2000) {
+ targetMachineCount = Math.min(
+ maxNumberOfMachines,
+ activeMachines + baseScaleUp * 3,
+ );
+ } else if (webScraperActive > 5000 || waitingAndPriorityCount > 1000) {
+ targetMachineCount = Math.min(
+ maxNumberOfMachines,
+ activeMachines + baseScaleUp * 2,
+ );
+ } else if (webScraperActive > 1000 || waitingAndPriorityCount > 500) {
+ targetMachineCount = Math.min(
+ maxNumberOfMachines,
+ activeMachines + baseScaleUp,
+ );
+ }
+
+ // Scale down logic
+ if (webScraperActive < 100 && waitingAndPriorityCount < 50) {
+ targetMachineCount = Math.max(
+ minNumberOfMachines,
+ activeMachines - baseScaleDown * 3,
+ );
+ } else if (webScraperActive < 500 && waitingAndPriorityCount < 200) {
+ targetMachineCount = Math.max(
+ minNumberOfMachines,
+ activeMachines - baseScaleDown * 2,
+ );
+ } else if (webScraperActive < 1000 && waitingAndPriorityCount < 500) {
+ targetMachineCount = Math.max(
+ minNumberOfMachines,
+ activeMachines - baseScaleDown,
+ );
+ }
+
+ if (targetMachineCount !== activeMachines) {
+ logger.info(
+ `πŸ‚ Scaling from ${activeMachines} to ${targetMachineCount} - ${webScraperActive} active, ${webScraperWaiting} waiting`,
+ );
+
+ if (targetMachineCount > activeMachines) {
+ sendSlackWebhook(
+ `πŸ‚ Scaling from ${activeMachines} to ${targetMachineCount} - ${webScraperActive} active, ${webScraperWaiting} waiting - Current DateTime: ${new Date().toISOString()}`,
+ false,
+ process.env.SLACK_AUTOSCALER ?? "",
+ );
+ } else {
+ sendSlackWebhook(
+ `πŸ‚ Scaling from ${activeMachines} to ${targetMachineCount} - ${webScraperActive} active, ${webScraperWaiting} waiting - Current DateTime: ${new Date().toISOString()}`,
+ false,
+ process.env.SLACK_AUTOSCALER ?? "",
+ );
+ }
+ return res.status(200).json({
+ mode: "scale-descale",
+ count: targetMachineCount,
+ });
+ }
+
+ return res.status(200).json({
+ mode: "normal",
+ count: activeMachines,
+ });
+ } catch (error) {
+ logger.error(error);
+ return res.status(500).send("Failed to initialize autoscaler");
+ }
+}
diff --git a/apps/api/src/controllers/v0/crawl-status.ts b/apps/api/src/controllers/v0/crawl-status.ts
index d76f5ebfa..843eb56d1 100644
--- a/apps/api/src/controllers/v0/crawl-status.ts
+++ b/apps/api/src/controllers/v0/crawl-status.ts
@@ -16,7 +16,7 @@ configDotenv();
export async function getJobs(crawlId: string, ids: string[]): Promise<PseudoJob<any>[]> {
const [bullJobs, dbJobs, gcsJobs] = await Promise.all([
- Promise.all(ids.map((x) => getScrapeQueue(x).getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,
+ Promise.all(ids.map((x) => getScrapeQueue().getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,
process.env.USE_DB_AUTHENTICATION === "true" ? await supabaseGetJobsByCrawlId(crawlId) : [],
process.env.GCS_BUCKET_NAME ? Promise.all(ids.map(async (x) => ({ id: x, job: await getJobFromGCS(x) }))).then(x => x.filter(x => x.job)) as Promise<({ id: string, job: any | null })[]> : [],
]);
diff --git a/apps/api/src/controllers/v0/scrape.ts b/apps/api/src/controllers/v0/scrape.ts
index d4c622bcc..8435da31e 100644
--- a/apps/api/src/controllers/v0/scrape.ts
+++ b/apps/api/src/controllers/v0/scrape.ts
@@ -127,7 +127,7 @@ export async function scrapeHelper(
return err;
}
- await getScrapeQueue(jobId).remove(jobId);
+ await getScrapeQueue().remove(jobId);
if (!doc) {
console.error("!!! PANIC DOC IS", doc);
diff --git a/apps/api/src/controllers/v0/search.ts b/apps/api/src/controllers/v0/search.ts
index 45d98d531..5ad0f4dff 100644
--- a/apps/api/src/controllers/v0/search.ts
+++ b/apps/api/src/controllers/v0/search.ts
@@ -139,7 +139,8 @@ export async function searchHelper(
return { success: true, error: "No search results found", returnCode: 200 };
}
- await Promise.all(jobDatas.map((x) => getScrapeQueue(x.opts.jobId).remove(x.opts.jobId)));
+ const sq = getScrapeQueue();
+ await Promise.all(jobDatas.map((x) => sq.remove(x.opts.jobId)));
// make sure doc.content is not empty
const filteredDocs = docs.filter(
diff --git a/apps/api/src/controllers/v1/crawl-errors.ts b/apps/api/src/controllers/v1/crawl-errors.ts
index 85fb491a4..45084c741 100644
--- a/apps/api/src/controllers/v1/crawl-errors.ts
+++ b/apps/api/src/controllers/v1/crawl-errors.ts
@@ -19,7 +19,7 @@ import { TransportableError } from "../../lib/error";
configDotenv();
export async function getJob(id: string) {
- const job = await getScrapeQueue(id).getJob(id);
+ const job = await getScrapeQueue().getJob(id);
if (!job) return job;
return job;
@@ -27,7 +27,7 @@ export async function getJob(id: string) {
export async function getJobs(ids: string[]) {
const jobs: (Job & { id: string })[] = (
- await Promise.all(ids.map((x) => getScrapeQueue(x).getJob(x)))
+ await Promise.all(ids.map((x) => getScrapeQueue().getJob(x)))
).filter((x) => x) as (Job & { id: string })[];
return jobs;
@@ -46,7 +46,7 @@ export async function crawlErrorsController(
let jobStatuses = await Promise.all(
(await getCrawlJobs(req.params.jobId)).map(
- async (x) => [x, await getScrapeQueue(x).getJobState(x)] as const,
+ async (x) => [x, await getScrapeQueue().getJobState(x)] as const,
),
);
diff --git a/apps/api/src/controllers/v1/crawl-status-ws.ts b/apps/api/src/controllers/v1/crawl-status-ws.ts
index a82d49706..190ca5209 100644
--- a/apps/api/src/controllers/v1/crawl-status-ws.ts
+++ b/apps/api/src/controllers/v1/crawl-status-ws.ts
@@ -88,10 +88,12 @@ async function crawlStatusWS(
const notDoneJobIDs = jobIDs.filter((x) => !doneJobIDs.includes(x));
+ const queue = getScrapeQueue();
+
const jobStatuses = await Promise.all(
notDoneJobIDs.map(async (x) => [
x,
- await getScrapeQueue(x).getJobState(x),
+ await queue.getJobState(x),
]),
);
const newlyDoneJobIDs: string[] = jobStatuses
@@ -123,9 +125,11 @@ async function crawlStatusWS(
let jobIDs = await getCrawlJobs(req.params.jobId);
+ const queue = getScrapeQueue();
+
let jobStatuses = await Promise.all(
jobIDs.map(
- async (x) => [x, await getScrapeQueue(x).getJobState(x)] as const,
+ async (x) => [x, await queue.getJobState(x)] as const,
),
);
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts
index 01f02e9a5..85a305f54 100644
--- a/apps/api/src/controllers/v1/crawl-status.ts
+++ b/apps/api/src/controllers/v1/crawl-status.ts
@@ -41,7 +41,7 @@ export type DBJob = { docs: any, success: boolean, page_options: any, date_added
export async function getJob(id: string): Promise<PseudoJob<any> | null> {
const [bullJob, dbJob, gcsJob] = await Promise.all([
- getScrapeQueue(id).getJob(id),
+ getScrapeQueue().getJob(id),
(process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobById(id) : null) as Promise<DBJob | null>,
(process.env.GCS_BUCKET_NAME ? getJobFromGCS(id) : null) as Promise<any | null>,
]);
@@ -73,7 +73,7 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {
export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {
const [bullJobs, dbJobs, gcsJobs] = await Promise.all([
- Promise.all(ids.map((x) => getScrapeQueue(x).getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,
+ Promise.all(ids.map((x) => getScrapeQueue().getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,
process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobsById(ids) : [],
process.env.GCS_BUCKET_NAME ? Promise.all(ids.map(async (x) => ({ id: x, job: await getJobFromGCS(x) }))).then(x => x.filter(x => x.job)) as Promise<({ id: string, job: any | null })[]> : [],
]);
@@ -315,7 +315,7 @@ export async function crawlStatusController(
const bytesLimit = 10485760; // 10 MiB in bytes
for (const jobId of doneJobs) {
- const job = await getScrapeQueue(jobId).getJob(jobId);
+ const job = await getScrapeQueue().getJob(jobId);
const state = await job?.getState();
if (state === "failed") {
diff --git a/apps/api/src/controllers/v1/scrape.ts b/apps/api/src/controllers/v1/scrape.ts
index b0444053c..b5b82ca43 100644
--- a/apps/api/src/controllers/v1/scrape.ts
+++ b/apps/api/src/controllers/v1/scrape.ts
@@ -105,7 +105,7 @@ export async function scrapeController(
});
if (zeroDataRetention) {
- await getScrapeQueue(jobId).remove(jobId);
+ await getScrapeQueue().remove(jobId);
}
if (e instanceof TransportableError) {
@@ -125,7 +125,7 @@ export async function scrapeController(
logger.info("Done with waitForJob");
- await getScrapeQueue(jobId).remove(jobId);
+ await getScrapeQueue().remove(jobId);
logger.info("Removed job from queue");
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index eabc6c6de..f185ce4df 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -138,7 +138,7 @@ async function scrapeSearchResult(
teamId: options.teamId,
origin: options.origin,
});
- await getScrapeQueue(jobId).remove(jobId);
+ await getScrapeQueue().remove(jobId);
const document = {
title: searchResult.title,
diff --git a/apps/api/src/controllers/v1/x402-search.ts b/apps/api/src/controllers/v1/x402-search.ts
index 371aeb758..ff0a66659 100644
--- a/apps/api/src/controllers/v1/x402-search.ts
+++ b/apps/api/src/controllers/v1/x402-search.ts
@@ -133,7 +133,7 @@ async function scrapeX402SearchResult(
teamId: options.teamId,
origin: options.origin,
});
- await getScrapeQueue(jobId).remove(jobId);
+ await getScrapeQueue().remove(jobId);
const document = {
title: searchResult.title,
diff --git a/apps/api/src/controllers/v2/crawl-errors.ts b/apps/api/src/controllers/v2/crawl-errors.ts
index 85fb491a4..45084c741 100644
--- a/apps/api/src/controllers/v2/crawl-errors.ts
+++ b/apps/api/src/controllers/v2/crawl-errors.ts
@@ -19,7 +19,7 @@ import { TransportableError } from "../../lib/error";
configDotenv();
export async function getJob(id: string) {
- const job = await getScrapeQueue(id).getJob(id);
+ const job = await getScrapeQueue().getJob(id);
if (!job) return job;
return job;
@@ -27,7 +27,7 @@ export async function getJob(id: string) {
export async function getJobs(ids: string[]) {
const jobs: (Job & { id: string })[] = (
- await Promise.all(ids.map((x) => getScrapeQueue(x).getJob(x)))
+ await Promise.all(ids.map((x) => getScrapeQueue().getJob(x)))
).filter((x) => x) as (Job & { id: string })[];
return jobs;
@@ -46,7 +46,7 @@ export async function crawlErrorsController(
let jobStatuses = await Promise.all(
(await getCrawlJobs(req.params.jobId)).map(
- async (x) => [x, await getScrapeQueue(x).getJobState(x)] as const,
+ async (x) => [x, await getScrapeQueue().getJobState(x)] as const,
),
);
diff --git a/apps/api/src/controllers/v2/crawl-status-ws.ts b/apps/api/src/controllers/v2/crawl-status-ws.ts
index a82d49706..190ca5209 100644
--- a/apps/api/src/controllers/v2/crawl-status-ws.ts
+++ b/apps/api/src/controllers/v2/crawl-status-ws.ts
@@ -88,10 +88,12 @@ async function crawlStatusWS(
const notDoneJobIDs = jobIDs.filter((x) => !doneJobIDs.includes(x));
+ const queue = getScrapeQueue();
+
const jobStatuses = await Promise.all(
notDoneJobIDs.map(async (x) => [
x,
- await getScrapeQueue(x).getJobState(x),
+ await queue.getJobState(x),
]),
);
const newlyDoneJobIDs: string[] = jobStatuses
@@ -123,9 +125,11 @@ async function crawlStatusWS(
let jobIDs = await getCrawlJobs(req.params.jobId);
+ const queue = getScrapeQueue();
+
let jobStatuses = await Promise.all(
jobIDs.map(
- async (x) => [x, await getScrapeQueue(x).getJobState(x)] as const,
+ async (x) => [x, await queue.getJobState(x)] as const,
),
);
diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.ts
index e8af8fd9b..34e3fcbd3 100644
--- a/apps/api/src/controllers/v2/crawl-status.ts
+++ b/apps/api/src/controllers/v2/crawl-status.ts
@@ -41,7 +41,7 @@ export type DBJob = { docs: any, success: boolean, page_options: any, date_added
export async function getJob(id: string): Promise<PseudoJob<any> | null> {
const [bullJob, dbJob, gcsJob] = await Promise.all([
- getScrapeQueue(id).getJob(id),
+ getScrapeQueue().getJob(id),
(process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobById(id) : null) as Promise<DBJob | null>,
(process.env.GCS_BUCKET_NAME ? getJobFromGCS(id) : null) as Promise<any | null>,
]);
@@ -73,7 +73,7 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {
export async function getJobs(ids: string[]): Promise<PseudoJob<any>[]> {
const [bullJobs, dbJobs, gcsJobs] = await Promise.all([
- Promise.all(ids.map((x) => getScrapeQueue(x).getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,
+ Promise.all(ids.map((x) => getScrapeQueue().getJob(x))).then(x => x.filter(x => x)) as Promise<(Job<any, any, string> & { id: string })[]>,
process.env.USE_DB_AUTHENTICATION === "true" ? supabaseGetJobsById(ids) : [],
process.env.GCS_BUCKET_NAME ? Promise.all(ids.map(async (x) => ({ id: x, job: await getJobFromGCS(x) }))).then(x => x.filter(x => x.job)) as Promise<({ id: string, job: any | null })[]> : [],
]);
@@ -315,7 +315,7 @@ export async function crawlStatusController(
const bytesLimit = 10485760; // 10 MiB in bytes
for (const jobId of doneJobs) {
- const job = await getScrapeQueue(jobId).getJob(jobId);
+ const job = await getScrapeQueue().getJob(jobId);
const state = await job?.getState();
if (state === "failed") {
diff --git a/apps/api/src/controllers/v2/scrape.ts b/apps/api/src/controllers/v2/scrape.ts
index c7e268043..b34fe5a31 100644
--- a/apps/api/src/controllers/v2/scrape.ts
+++ b/apps/api/src/controllers/v2/scrape.ts
@@ -106,7 +106,7 @@ export async function scrapeController(
});
if (zeroDataRetention) {
- await getScrapeQueue(jobId).remove(jobId);
+ await getScrapeQueue().remove(jobId);
}
if (e instanceof TransportableError) {
@@ -123,7 +123,7 @@ export async function scrapeController(
}
}
- await getScrapeQueue(jobId).remove(jobId);
+ await getScrapeQueue().remove(jobId);
if (!hasFormatOfType(req.body.formats, "rawHtml")) {
if (doc && doc.rawHtml) {
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index 9129e7166..158ff149d 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -95,7 +95,7 @@ async function scrapeSearchResult(
teamId: options.teamId,
origin: options.origin,
});
- await getScrapeQueue(jobId).remove(jobId);
+ await getScrapeQueue().remove(jobId);
const document = {
title: searchResult.title,
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index e1ffdc0be..eb16f451a 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -6,12 +6,11 @@ import bodyParser from "body-parser";
import cors from "cors";
import {
getExtractQueue,
+ getScrapeQueue,
getGenerateLlmsTxtQueue,
getDeepResearchQueue,
getBillingQueue,
getPrecrawlQueue,
- getScrapeQueueByIndex,
- queueMultiplexWidth,
} from "./services/queue-service";
import { v0Router } from "./routes/v0";
import os from "os";
@@ -87,7 +86,7 @@ serverAdapter.setBasePath(`/admin/${process.env.BULL_AUTH_KEY}/queues`);
const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({
queues: [
- ...new Array(queueMultiplexWidth).fill(0).map((_, index) => new BullMQAdapter(getScrapeQueueByIndex(index))),
+ new BullMQAdapter(getScrapeQueue()),
new BullMQAdapter(getExtractQueue()),
new BullMQAdapter(getGenerateLlmsTxtQueue()),
new BullMQAdapter(getDeepResearchQueue()),
@@ -160,11 +159,9 @@ if (require.main === module) {
app.get(`/serverHealthCheck`, async (req, res) => {
try {
- const scrapeQueues = new Array(queueMultiplexWidth)
- .fill(0)
- .map((_, index) => getScrapeQueueByIndex(index));
- const waitingJobs = await Promise.all(scrapeQueues.map((queue) => queue.getWaitingCount()));
- const noWaitingJobs = waitingJobs.reduce((a, b) => a + b, 0) === 0;
+ const scrapeQueue = getScrapeQueue();
+ const [waitingJobs] = await Promise.all([scrapeQueue.getWaitingCount()]);
+ const noWaitingJobs = waitingJobs === 0;
// 200 if no active jobs, 503 if there are active jobs
return res.status(noWaitingJobs ? 200 : 500).json({
waitingJobs,
@@ -182,12 +179,12 @@ app.get("/serverHealthCheck/notify", async (req, res) => {
const timeout = 60000; // 1 minute // The timeout value for the check in milliseconds
const getWaitingJobsCount = async () => {
- const scrapeQueues = new Array(queueMultiplexWidth)
- .fill(0)
- .map((_, index) => getScrapeQueueByIndex(index));
- const waitingJobsCount = await Promise.all(scrapeQueues.map((queue) => queue.getWaitingCount()));
+ const scrapeQueue = getScrapeQueue();
+ const [waitingJobsCount] = await Promise.all([
+ scrapeQueue.getWaitingCount(),
+ ]);
- return waitingJobsCount.reduce((a, b) => a + b, 0);
+ return waitingJobsCount;
};
res.status(200).json({ message: "Check initiated" });
@@ -299,3 +296,12 @@ app.use(
);
logger.info(`Worker ${process.pid} started`);
+// const sq = getScrapeQueue();
+
+// sq.on("waiting", j => ScrapeEvents.logJobEvent(j, "waiting"));
+// sq.on("active", j => ScrapeEvents.logJobEvent(j, "active"));
+// sq.on("completed", j => ScrapeEvents.logJobEvent(j, "completed"));
+// sq.on("paused", j => ScrapeEvents.logJobEvent(j, "paused"));
+// sq.on("resumed", j => ScrapeEvents.logJobEvent(j, "resumed"));
+// sq.on("removed", j => ScrapeEvents.logJobEvent(j, "removed"));
+//
diff --git a/apps/api/src/lib/concurrency-limit.ts b/apps/api/src/lib/concurrency-limit.ts
index eba623603..52fc2fbdb 100644
--- a/apps/api/src/lib/concurrency-limit.ts
+++ b/apps/api/src/lib/concurrency-limit.ts
@@ -274,7 +274,7 @@ export async function concurrentJobDone(job: Job) {
}
}
- (await getScrapeQueue(nextJob.job.id)).add(
+ (await getScrapeQueue()).add(
nextJob.job.id,
{
...nextJob.job.data,
diff --git a/apps/api/src/lib/extract/document-scraper.ts b/apps/api/src/lib/extract/document-scraper.ts
index 0913615f1..1b84733b2 100644
--- a/apps/api/src/lib/extract/document-scraper.ts
+++ b/apps/api/src/lib/extract/document-scraper.ts
@@ -66,7 +66,7 @@ export async function scrapeDocument(
const doc = await waitForJob(jobId, timeout);
- await getScrapeQueue(jobId).remove(jobId);
+ await getScrapeQueue().remove(jobId);
if (trace) {
trace.timing.completedAt = new Date().toISOString();
diff --git a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
index 5e2629744..a4b524a64 100644
--- a/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
+++ b/apps/api/src/lib/extract/fire-0/document-scraper-f0.ts
@@ -66,7 +66,7 @@ export async function scrapeDocument_F0(
);
const doc = await waitForJob(jobId, timeout);
- await getScrapeQueue(jobId).remove(jobId);
+ await getScrapeQueue().remove(jobId);
if (trace) {
trace.timing.completedAt = new Date().toISOString();
diff --git a/apps/api/src/routes/admin.ts b/apps/api/src/routes/admin.ts
index 51a713e3e..25f0880ad 100644
--- a/apps/api/src/routes/admin.ts
+++ b/apps/api/src/routes/admin.ts
@@ -1,7 +1,9 @@
import express from "express";
import { redisHealthController } from "../controllers/v0/admin/redis-health";
import {
+ autoscalerController,
checkQueuesController,
+ cleanBefore24hCompleteJobsController,
queuesController,
} from "../controllers/v0/admin/queue";
import { wrap } from "./shared";
@@ -20,6 +22,11 @@ adminRouter.get(
redisHealthController,
);
+adminRouter.get(
+ `/admin/${process.env.BULL_AUTH_KEY}/clean-before-24h-complete-jobs`,
+ cleanBefore24hCompleteJobsController,
+);
+
adminRouter.get(
`/admin/${process.env.BULL_AUTH_KEY}/check-queues`,
checkQueuesController,
@@ -27,6 +34,11 @@ adminRouter.get(
adminRouter.get(`/admin/${process.env.BULL_AUTH_KEY}/queues`, queuesController);
+adminRouter.get(
+ `/admin/${process.env.BULL_AUTH_KEY}/autoscaler`,
+ autoscalerController,
+);
+
adminRouter.post(
`/admin/${process.env.BULL_AUTH_KEY}/acuc-cache-clear`,
wrap(acucCacheClearController),
diff --git a/apps/api/src/services/alerts/index.ts b/apps/api/src/services/alerts/index.ts
index b7a9a9455..44f2b8a04 100644
--- a/apps/api/src/services/alerts/index.ts
+++ b/apps/api/src/services/alerts/index.ts
@@ -1,5 +1,5 @@
import { logger } from "../../../src/lib/logger";
-import { getScrapeQueueByIndex, queueMultiplexWidth } from "../queue-service";
+import { getScrapeQueue } from "../queue-service";
import { sendSlackWebhook } from "./slack";
export async function checkAlerts() {
@@ -13,11 +13,9 @@ export async function checkAlerts() {
logger.info("Initializing alerts");
const checkActiveJobs = async () => {
try {
- const scrapeQueues = new Array(queueMultiplexWidth)
- .fill(0)
- .map((_, index) => getScrapeQueueByIndex(index));
- const activeJobs = await Promise.all(scrapeQueues.map((queue) => queue.getActiveCount()));
- if (activeJobs.reduce((a, b) => a + b, 0) > Number(process.env.ALERT_NUM_ACTIVE_JOBS)) {
+ const scrapeQueue = getScrapeQueue();
+ const activeJobs = await scrapeQueue.getActiveCount();
+ if (activeJobs > Number(process.env.ALERT_NUM_ACTIVE_JOBS)) {
logger.warn(
`Alert: Number of active jobs is over ${process.env.ALERT_NUM_ACTIVE_JOBS}. Current active jobs: ${activeJobs}.`,
);
@@ -36,12 +34,10 @@ export async function checkAlerts() {
};
const checkWaitingQueue = async () => {
- const scrapeQueues = new Array(queueMultiplexWidth)
- .fill(0)
- .map((_, index) => getScrapeQueueByIndex(index));
- const waitingJobs = await Promise.all(scrapeQueues.map((queue) => queue.getWaitingCount()));
+ const scrapeQueue = getScrapeQueue();
+ const waitingJobs = await scrapeQueue.getWaitingCount();
- if (waitingJobs.reduce((a, b) => a + b, 0) > Number(process.env.ALERT_NUM_WAITING_JOBS)) {
+ if (waitingJobs > Number(process.env.ALERT_NUM_WAITING_JOBS)) {
logger.warn(
`Alert: Number of waiting jobs is over ${process.env.ALERT_NUM_WAITING_JOBS}. Current waiting jobs: ${waitingJobs}.`,
);
diff --git a/apps/api/src/services/queue-jobs.ts b/apps/api/src/services/queue-jobs.ts
index 3ee4a9282..a4a5ab922 100644
--- a/apps/api/src/services/queue-jobs.ts
+++ b/apps/api/src/services/queue-jobs.ts
@@ -72,7 +72,7 @@ export async function _addScrapeJobToBullMQ(
}
}
- return await getScrapeQueue(jobId).add(jobId, webScraperOptions, {
+ return await getScrapeQueue().add(jobId, webScraperOptions, {
...options,
priority: jobPriority,
jobId,
@@ -322,7 +322,7 @@ export async function waitForJob(
logger: Logger = _logger,
): Promise<Document> {
const start = Date.now();
- const queue = getScrapeQueue(typeof _job === "string" ? _job : (_job.id as string));
+ const queue = getScrapeQueue();
let job: Job | undefined = typeof _job == "string" ? await queue.getJob(_job) : _job;
while (job === undefined) {
logger.debug("Waiting for job to be created");
@@ -335,7 +335,7 @@ export async function waitForJob(
let doc: Document;
try {
doc = await Promise.race([
- job.waitUntilFinished(getScrapeQueueEvents(job.id as string), timeout ?? 180000),
+ job.waitUntilFinished(getScrapeQueueEvents(), timeout ?? 180000),
timeout !== null ? new Promise((resolve, reject) => {
setTimeout(() => {
reject(new ScrapeJobTimeoutError("Scrape timed out" + (typeof _job === "string" ? " after waiting in the concurrency limit queue" : "")));
diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.ts
index 8a56a6caa..6041b9ec8 100644
--- a/apps/api/src/services/queue-service.ts
+++ b/apps/api/src/services/queue-service.ts
@@ -5,8 +5,8 @@ import { BullMQOtel } from "bullmq-otel";
export type QueueFunction = () => Queue<any, any, string, any, any, string>;
-let scrapeQueues: Queue[] = [];
-let scrapeQueueEvents: QueueEvents[] = [];
+let scrapeQueue: Queue;
+let scrapeQueueEvents: QueueEvents;
let extractQueue: Queue;
let loggingQueue: Queue;
let indexQueue: Queue;
@@ -29,6 +29,7 @@ export function getRedisConnection(): IORedis {
return redisConnection;
}
+export const scrapeQueueName = "{scrapeQueue}";
export const extractQueueName = "{extractQueue}";
export const loggingQueueName = "{loggingQueue}";
export const indexQueueName = "{indexQueue}";
@@ -37,21 +38,9 @@ export const deepResearchQueueName = "{deepResearchQueue}";
export const billingQueueName = "{billingQueue}";
export const precrawlQueueName = "{precrawlQueue}";
-export const queueMultiplexWidth = process.env.QUEUE_MULTIPLEX_WIDTH ? parseInt(process.env.QUEUE_MULTIPLEX_WIDTH) : 1;
-
-function uuidToQueueIndex(uuid: string) {
- const queueIndex = parseInt(uuid[0] ?? "0", 16) % queueMultiplexWidth;
- return queueIndex;
-}
-
-export function getScrapeQueue(uuid: string) {
- const queueIndex = uuidToQueueIndex(uuid);
- return getScrapeQueueByIndex(queueIndex);
-}
-
-export function getScrapeQueueByIndex(index: number) {
- if (!scrapeQueues[index]) {
- scrapeQueues[index] = new Queue(`{scrapeQueue${index === 0 ? "" : index}}`, {
+export function getScrapeQueue() {
+ if (!scrapeQueue) {
+ scrapeQueue = new Queue(scrapeQueueName, {
connection: getRedisConnection(),
defaultJobOptions: {
removeOnComplete: {
@@ -64,22 +53,17 @@ export function getScrapeQueueByIndex(index: number) {
telemetry: new BullMQOtel("firecrawl-bullmq"),
});
}
- return scrapeQueues[index];
-}
-
-export function getScrapeQueueEvents(uuid: string) {
- const queueIndex = uuidToQueueIndex(uuid);
- return getScrapeQueueEventsByIndex(queueIndex);
+ return scrapeQueue;
}
-export function getScrapeQueueEventsByIndex(index: number) {
- if (!scrapeQueueEvents[index]) {
- scrapeQueueEvents[index] = new QueueEvents(`{scrapeQueue${index === 0 ? "" : index}}`, {
+export function getScrapeQueueEvents() {
+ if (!scrapeQueueEvents) {
+ scrapeQueueEvents = new QueueEvents(scrapeQueueName, {
connection: getRedisConnection(),
});
}
- return scrapeQueueEvents[index];
+ return scrapeQueueEvents;
}
export function getExtractQueue() {
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index e7b0b2e53..32a154186 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -6,10 +6,8 @@ import {
getExtractQueue,
getDeepResearchQueue,
getGenerateLlmsTxtQueue,
+ scrapeQueueName,
getRedisConnection,
- queueMultiplexWidth,
- getScrapeQueueEventsByIndex,
- getScrapeQueueByIndex,
} from "./queue-service";
import { Job, Queue, QueueEvents } from "bullmq";
import { logger as _logger } from "../lib/logger";
@@ -510,15 +508,9 @@ app.listen(workerPort, () => {
_logger.info(`Liveness endpoint is running on port ${workerPort}`);
});
-const myScrapeQueue = process.env.QUEUE_MULTIPLEX_INDEX !== undefined
- ? parseInt(process.env.QUEUE_MULTIPLEX_INDEX)
- : Math.floor(Math.random() * queueMultiplexWidth);
-
-_logger.info(`Using scrape queue ${myScrapeQueue}`);
-
(async () => {
async function failedListener(args: { jobId: string; failedReason: string; prev?: string | undefined; }) {
- const job = await getScrapeQueueByIndex(myScrapeQueue).getJob(args.jobId);
+ const job = await getScrapeQueue().getJob(args.jobId);
if (job && job.data.crawl_id) {
await redisEvictConnection.srem("crawl:" + job.data.crawl_id + ":jobs_qualified", args.jobId);
@@ -567,14 +559,12 @@ _logger.info(`Using scrape queue ${myScrapeQueue}`);
}
}
- for (let i = 0; i < queueMultiplexWidth; i++) {
- const scrapeQueueEvents = getScrapeQueueEventsByIndex(i);
- scrapeQueueEvents.on("failed", failedListener);
- }
+ const scrapeQueueEvents = new QueueEvents(scrapeQueueName, { connection: getRedisConnection() });
+ scrapeQueueEvents.on("failed", failedListener);
const results = await Promise.all([
separateWorkerFun(
- getScrapeQueueByIndex(myScrapeQueue),
+ getScrapeQueue(),
path.join(__dirname, "worker", "scrape-worker.js"),
),
workerFun(getExtractQueue(), processExtractJobInternal),
@@ -596,14 +586,14 @@ _logger.info(`Using scrape queue ${myScrapeQueue}`);
jobs: (
await Promise.all(
[...runningJobs].map(async (jobId) => {
- return await getScrapeQueueByIndex(myScrapeQueue).getJob(jobId);
+ return await getScrapeQueue().getJob(jobId);
}),
)
).filter((x) => x && !x.data?.zeroDataRetention),
});
}, 1000);
- await getScrapeQueueEventsByIndex(myScrapeQueue).close();
+ await scrapeQueueEvents.close();
console.log("All jobs finished. Worker out!");
if (otelSdk) {
await otelSdk.shutdown();