Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
move crawlFinishQueue to regular worker
Gergő Móricz6754f3bea7ef
diff --git a/apps/api/src/services/extract-worker.ts b/apps/api/src/services/extract-worker.tsindex 57132fbe7..c0a435478 100644--- a/apps/api/src/services/extract-worker.ts+++ b/apps/api/src/services/extract-worker.ts@@ -20,10 +20,6 @@ import { robustFetch } from "../scraper/scrapeURL/lib/fetch";import { BullMQOtel } from "bullmq-otel";import { getErrorContactMessage } from "../lib/deployment";import { initializeBlocklist } from "../scraper/WebScraper/utils/blocklist";-import { crawlFinishedQueue, NuQJob, scrapeQueue } from "./worker/nuq";-import { getCrawl } from "../lib/crawl-redis";-import type { Logger } from "winston";-import { finishCrawlSuper } from "./worker/crawl-logic";configDotenv();@@ -253,123 +249,6 @@ const workerFun = async (}};-async function processFinishCrawlJobInternal(_job: NuQJob, logger: Logger) {- const job = await crawlFinishedQueue.getJob(_job.id);-- if (!job) {- throw new Error("crawlFinish job disappeared");- }-- if (!job.groupId) {- throw new Error("crawlFinish job with no groupId");- }-- if (!job.ownerId) {- throw new Error("crawlFinish job with no ownerId");- }-- const sc = await getCrawl(job.groupId);-- if (!sc) {- throw new Error("crawlFinish job with sc expired");- }-- const anyJob = await scrapeQueue.getGroupAnyJob(job.groupId, job.ownerId);-- if (!anyJob) {- throw new Error("crawlFinish couldn't find anyJob");- }-- await finishCrawlSuper(anyJob);-}--const crawlFinishWorker = async () => {- const __logger = _logger.child({- module: "extract-worker",- method: "crawlFinishWorker",- });-- let noJobTimeout = 1500;-- while (!isShuttingDown) {- const job = await crawlFinishedQueue.getJobToProcess();-- if (job === null) {- __logger.info("No jobs to process", { module: "nuq/metrics" });- await new Promise(resolve => setTimeout(resolve, noJobTimeout));- if (!process.env.NUQ_RABBITMQ_URL) {- noJobTimeout = Math.min(noJobTimeout * 2, 10000);- }- continue;- }-- noJobTimeout = 500;-- const logger = __logger.child({- zeroDataRetention: job.data?.zeroDataRetention ?? false,- crawlId: job.groupId,- });-- logger.info("Acquired job");-- const lockRenewInterval = setInterval(async () => {- logger.info("Renewing lock");- if (!(await crawlFinishedQueue.renewLock(job.id, job.lock!, logger))) {- logger.warn("Failed to renew lock");- clearInterval(lockRenewInterval);- return;- }- logger.info("Renewed lock");- }, 15000);-- let processResult:- | {- ok: true;- data: Awaited<ReturnType<typeof processFinishCrawlJobInternal>>;- }- | { ok: false; error: any };-- try {- processResult = {- ok: true,- data: await processFinishCrawlJobInternal(job, logger),- };- } catch (error) {- processResult = { ok: false, error };- }-- clearInterval(lockRenewInterval);-- if (processResult.ok) {- if (- !(await crawlFinishedQueue.jobFinish(- job.id,- job.lock!,- processResult.data,- logger,- ))- ) {- logger.warn("Could not update job status");- }- } else {- if (- !(await crawlFinishedQueue.jobFail(- job.id,- job.lock!,- processResult.error instanceof Error- ? processResult.error.message- : typeof processResult.error === "string"- ? processResult.error- : JSON.stringify(processResult.error),- logger,- ))- ) {- logger.warn("Could not update job status");- }- }- }-};-// Start all workersconst app = Express();@@ -418,10 +297,7 @@ app.listen(workerPort, () => {process.exit(1);});- await Promise.all([- workerFun(getExtractQueue(), processExtractJobInternal),- crawlFinishWorker(),- ]);+ await Promise.all([workerFun(getExtractQueue(), processExtractJobInternal)]);_logger.info("All workers exited. Waiting for all jobs to finish...");diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex 89914d092..2df91f172 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -20,6 +20,9 @@ import Express from "express";import { robustFetch } from "../scraper/scrapeURL/lib/fetch";import { BullMQOtel } from "bullmq-otel";import { initializeBlocklist } from "../scraper/WebScraper/utils/blocklist";+import { crawlFinishedQueue, NuQJob, scrapeQueue } from "./worker/nuq";+import { finishCrawlSuper } from "./worker/crawl-logic";+import { getCrawl } from "../lib/crawl-redis";configDotenv();@@ -191,6 +194,36 @@ const processGenerateLlmsTxtJobInternal = async (}};+async function processFinishCrawlJobInternal(_job: NuQJob) {+ const job = await crawlFinishedQueue.getJob(_job.id);++ if (!job) {+ throw new Error("crawlFinish job disappeared");+ }++ if (!job.groupId) {+ throw new Error("crawlFinish job with no groupId");+ }++ if (!job.ownerId) {+ throw new Error("crawlFinish job with no ownerId");+ }++ const sc = await getCrawl(job.groupId);++ if (!sc) {+ throw new Error("crawlFinish job with sc expired");+ }++ const anyJob = await scrapeQueue.getGroupAnyJob(job.groupId, job.ownerId);++ if (!anyJob) {+ throw new Error("crawlFinish couldn't find anyJob");+ }++ await finishCrawlSuper(anyJob);+}+let isShuttingDown = false;let isWorkerStalled = false;@@ -275,6 +308,93 @@ const workerFun = async (}};+const crawlFinishWorker = async () => {+ const __logger = _logger.child({+ module: "extract-worker",+ method: "crawlFinishWorker",+ });++ let noJobTimeout = 1500;++ while (!isShuttingDown) {+ const job = await crawlFinishedQueue.getJobToProcess();++ if (job === null) {+ __logger.info("No jobs to process", { module: "nuq/metrics" });+ await new Promise(resolve => setTimeout(resolve, noJobTimeout));+ if (!process.env.NUQ_RABBITMQ_URL) {+ noJobTimeout = Math.min(noJobTimeout * 2, 10000);+ }+ continue;+ }++ noJobTimeout = 500;++ const logger = __logger.child({+ zeroDataRetention: job.data?.zeroDataRetention ?? false,+ crawlId: job.groupId,+ });++ logger.info("Acquired job");++ const lockRenewInterval = setInterval(async () => {+ logger.info("Renewing lock");+ if (!(await crawlFinishedQueue.renewLock(job.id, job.lock!, logger))) {+ logger.warn("Failed to renew lock");+ clearInterval(lockRenewInterval);+ return;+ }+ logger.info("Renewed lock");+ }, 15000);++ let processResult:+ | {+ ok: true;+ data: Awaited<ReturnType<typeof processFinishCrawlJobInternal>>;+ }+ | { ok: false; error: any };++ try {+ processResult = {+ ok: true,+ data: await processFinishCrawlJobInternal(job),+ };+ } catch (error) {+ processResult = { ok: false, error };+ }++ clearInterval(lockRenewInterval);++ if (processResult.ok) {+ if (+ !(await crawlFinishedQueue.jobFinish(+ job.id,+ job.lock!,+ processResult.data,+ logger,+ ))+ ) {+ logger.warn("Could not update job status");+ }+ } else {+ if (+ !(await crawlFinishedQueue.jobFail(+ job.id,+ job.lock!,+ processResult.error instanceof Error+ ? processResult.error.message+ : typeof processResult.error === "string"+ ? processResult.error+ : JSON.stringify(processResult.error),+ logger,+ ))+ ) {+ logger.warn("Could not update job status");+ }+ }+ }+};+// Start all workersconst app = Express();@@ -326,6 +446,7 @@ app.listen(workerPort, () => {await Promise.all([workerFun(getDeepResearchQueue(), processDeepResearchJobInternal),workerFun(getGenerateLlmsTxtQueue(), processGenerateLlmsTxtJobInternal),+ crawlFinishWorker(),]);_logger.info("All workers exited. Waiting for all jobs to finish...");