Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

move crawlFinishQueue to regular worker

Gergő Móricz6754f3bea7ef
diff --git a/apps/api/src/services/extract-worker.ts b/apps/api/src/services/extract-worker.ts
index 57132fbe7..c0a435478 100644
--- a/apps/api/src/services/extract-worker.ts
+++ b/apps/api/src/services/extract-worker.ts
@@ -20,10 +20,6 @@ import { robustFetch } from "../scraper/scrapeURL/lib/fetch";
import { BullMQOtel } from "bullmq-otel";
import { getErrorContactMessage } from "../lib/deployment";
import { initializeBlocklist } from "../scraper/WebScraper/utils/blocklist";
-import { crawlFinishedQueue, NuQJob, scrapeQueue } from "./worker/nuq";
-import { getCrawl } from "../lib/crawl-redis";
-import type { Logger } from "winston";
-import { finishCrawlSuper } from "./worker/crawl-logic";
configDotenv();
@@ -253,123 +249,6 @@ const workerFun = async (
}
};
-async function processFinishCrawlJobInternal(_job: NuQJob, logger: Logger) {
- const job = await crawlFinishedQueue.getJob(_job.id);
-
- if (!job) {
- throw new Error("crawlFinish job disappeared");
- }
-
- if (!job.groupId) {
- throw new Error("crawlFinish job with no groupId");
- }
-
- if (!job.ownerId) {
- throw new Error("crawlFinish job with no ownerId");
- }
-
- const sc = await getCrawl(job.groupId);
-
- if (!sc) {
- throw new Error("crawlFinish job with sc expired");
- }
-
- const anyJob = await scrapeQueue.getGroupAnyJob(job.groupId, job.ownerId);
-
- if (!anyJob) {
- throw new Error("crawlFinish couldn't find anyJob");
- }
-
- await finishCrawlSuper(anyJob);
-}
-
-const crawlFinishWorker = async () => {
- const __logger = _logger.child({
- module: "extract-worker",
- method: "crawlFinishWorker",
- });
-
- let noJobTimeout = 1500;
-
- while (!isShuttingDown) {
- const job = await crawlFinishedQueue.getJobToProcess();
-
- if (job === null) {
- __logger.info("No jobs to process", { module: "nuq/metrics" });
- await new Promise(resolve => setTimeout(resolve, noJobTimeout));
- if (!process.env.NUQ_RABBITMQ_URL) {
- noJobTimeout = Math.min(noJobTimeout * 2, 10000);
- }
- continue;
- }
-
- noJobTimeout = 500;
-
- const logger = __logger.child({
- zeroDataRetention: job.data?.zeroDataRetention ?? false,
- crawlId: job.groupId,
- });
-
- logger.info("Acquired job");
-
- const lockRenewInterval = setInterval(async () => {
- logger.info("Renewing lock");
- if (!(await crawlFinishedQueue.renewLock(job.id, job.lock!, logger))) {
- logger.warn("Failed to renew lock");
- clearInterval(lockRenewInterval);
- return;
- }
- logger.info("Renewed lock");
- }, 15000);
-
- let processResult:
- | {
- ok: true;
- data: Awaited<ReturnType<typeof processFinishCrawlJobInternal>>;
- }
- | { ok: false; error: any };
-
- try {
- processResult = {
- ok: true,
- data: await processFinishCrawlJobInternal(job, logger),
- };
- } catch (error) {
- processResult = { ok: false, error };
- }
-
- clearInterval(lockRenewInterval);
-
- if (processResult.ok) {
- if (
- !(await crawlFinishedQueue.jobFinish(
- job.id,
- job.lock!,
- processResult.data,
- logger,
- ))
- ) {
- logger.warn("Could not update job status");
- }
- } else {
- if (
- !(await crawlFinishedQueue.jobFail(
- job.id,
- job.lock!,
- processResult.error instanceof Error
- ? processResult.error.message
- : typeof processResult.error === "string"
- ? processResult.error
- : JSON.stringify(processResult.error),
- logger,
- ))
- ) {
- logger.warn("Could not update job status");
- }
- }
- }
-};
-
// Start all workers
const app = Express();
@@ -418,10 +297,7 @@ app.listen(workerPort, () => {
process.exit(1);
});
- await Promise.all([
- workerFun(getExtractQueue(), processExtractJobInternal),
- crawlFinishWorker(),
- ]);
+ await Promise.all([workerFun(getExtractQueue(), processExtractJobInternal)]);
_logger.info("All workers exited. Waiting for all jobs to finish...");
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index 89914d092..2df91f172 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -20,6 +20,9 @@ import Express from "express";
import { robustFetch } from "../scraper/scrapeURL/lib/fetch";
import { BullMQOtel } from "bullmq-otel";
import { initializeBlocklist } from "../scraper/WebScraper/utils/blocklist";
+import { crawlFinishedQueue, NuQJob, scrapeQueue } from "./worker/nuq";
+import { finishCrawlSuper } from "./worker/crawl-logic";
+import { getCrawl } from "../lib/crawl-redis";
configDotenv();
@@ -191,6 +194,36 @@ const processGenerateLlmsTxtJobInternal = async (
}
};
+async function processFinishCrawlJobInternal(_job: NuQJob) {
+ const job = await crawlFinishedQueue.getJob(_job.id);
+
+ if (!job) {
+ throw new Error("crawlFinish job disappeared");
+ }
+
+ if (!job.groupId) {
+ throw new Error("crawlFinish job with no groupId");
+ }
+
+ if (!job.ownerId) {
+ throw new Error("crawlFinish job with no ownerId");
+ }
+
+ const sc = await getCrawl(job.groupId);
+
+ if (!sc) {
+ throw new Error("crawlFinish job with sc expired");
+ }
+
+ const anyJob = await scrapeQueue.getGroupAnyJob(job.groupId, job.ownerId);
+
+ if (!anyJob) {
+ throw new Error("crawlFinish couldn't find anyJob");
+ }
+
+ await finishCrawlSuper(anyJob);
+}
+
let isShuttingDown = false;
let isWorkerStalled = false;
@@ -275,6 +308,93 @@ const workerFun = async (
}
};
+const crawlFinishWorker = async () => {
+ const __logger = _logger.child({
+ module: "extract-worker",
+ method: "crawlFinishWorker",
+ });
+
+ let noJobTimeout = 1500;
+
+ while (!isShuttingDown) {
+ const job = await crawlFinishedQueue.getJobToProcess();
+
+ if (job === null) {
+ __logger.info("No jobs to process", { module: "nuq/metrics" });
+ await new Promise(resolve => setTimeout(resolve, noJobTimeout));
+ if (!process.env.NUQ_RABBITMQ_URL) {
+ noJobTimeout = Math.min(noJobTimeout * 2, 10000);
+ }
+ continue;
+ }
+
+ noJobTimeout = 500;
+
+ const logger = __logger.child({
+ zeroDataRetention: job.data?.zeroDataRetention ?? false,
+ crawlId: job.groupId,
+ });
+
+ logger.info("Acquired job");
+
+ const lockRenewInterval = setInterval(async () => {
+ logger.info("Renewing lock");
+ if (!(await crawlFinishedQueue.renewLock(job.id, job.lock!, logger))) {
+ logger.warn("Failed to renew lock");
+ clearInterval(lockRenewInterval);
+ return;
+ }
+ logger.info("Renewed lock");
+ }, 15000);
+
+ let processResult:
+ | {
+ ok: true;
+ data: Awaited<ReturnType<typeof processFinishCrawlJobInternal>>;
+ }
+ | { ok: false; error: any };
+
+ try {
+ processResult = {
+ ok: true,
+ data: await processFinishCrawlJobInternal(job),
+ };
+ } catch (error) {
+ processResult = { ok: false, error };
+ }
+
+ clearInterval(lockRenewInterval);
+
+ if (processResult.ok) {
+ if (
+ !(await crawlFinishedQueue.jobFinish(
+ job.id,
+ job.lock!,
+ processResult.data,
+ logger,
+ ))
+ ) {
+ logger.warn("Could not update job status");
+ }
+ } else {
+ if (
+ !(await crawlFinishedQueue.jobFail(
+ job.id,
+ job.lock!,
+ processResult.error instanceof Error
+ ? processResult.error.message
+ : typeof processResult.error === "string"
+ ? processResult.error
+ : JSON.stringify(processResult.error),
+ logger,
+ ))
+ ) {
+ logger.warn("Could not update job status");
+ }
+ }
+ }
+};
+
// Start all workers
const app = Express();
@@ -326,6 +446,7 @@ app.listen(workerPort, () => {
await Promise.all([
workerFun(getDeepResearchQueue(), processDeepResearchJobInternal),
workerFun(getGenerateLlmsTxtQueue(), processGenerateLlmsTxtJobInternal),
+ crawlFinishWorker(),
]);
_logger.info("All workers exited. Waiting for all jobs to finish...");