Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): separate NuQ FDB worker (#3779)
Gergő Móricz9e520616625e
diff --git a/apps/api/package.json b/apps/api/package.jsonindex 3af53eef9..8118851e0 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -25,6 +25,8 @@"worker:production": "node dist/src/services/queue-worker.js","nuq-worker": "tsc-watch --onSuccess \"node dist/src/services/worker/nuq-worker.js\"","nuq-worker:production": "node dist/src/services/worker/nuq-worker.js",+ "nuq-fdb-worker": "tsc-watch --onSuccess \"node dist/src/services/worker/nuq-fdb-worker.js\"",+ "nuq-fdb-worker:production": "node dist/src/services/worker/nuq-fdb-worker.js","nuq-prefetch-worker": "tsc-watch --onSuccess \"node dist/src/services/worker/nuq-prefetch-worker.js\"","nuq-prefetch-worker:production": "node dist/src/services/worker/nuq-prefetch-worker.js","nuq-reconciler-worker": "tsc-watch --onSuccess \"node dist/src/services/worker/nuq-reconciler-worker.js\"",diff --git a/apps/api/src/controllers/v1/queue-status.ts b/apps/api/src/controllers/v1/queue-status.tsindex 0f2b72316..eac91e419 100644--- a/apps/api/src/controllers/v1/queue-status.ts+++ b/apps/api/src/controllers/v1/queue-status.ts@@ -3,7 +3,7 @@ import { getACUCTeam } from "../auth";import { AuthCreditUsageChunkFromTeam, RequestWithAuth } from "./types";import { Response } from "express";import { getRedisConnection } from "../../services/queue-service";-import { fdbQueueEnabled } from "../../services/worker/nuq-router";+import { isFdbTeam } from "../../services/worker/nuq-router";import {nuqFdbHealthCheck,scrapeQueueFdb,@@ -57,7 +57,7 @@ export async function queueStatusController(let queuedJobsOfTeam = await getConcurrencyQueueJobsCount(req.auth.team_id);// during the FDB migration a team can have load on both ledgers- if (fdbQueueEnabled()) {+ if (await isFdbTeam(req.auth.team_id)) {try {if (await nuqFdbHealthCheck(FDB_OPTIONAL_COUNT_TIMEOUT_MS)) {const [fdbActive, fdbPending] = await Promise.all([diff --git a/apps/api/src/controllers/v2/queue-status.ts b/apps/api/src/controllers/v2/queue-status.tsindex ea95e8343..297e625b5 100644--- a/apps/api/src/controllers/v2/queue-status.ts+++ b/apps/api/src/controllers/v2/queue-status.ts@@ -4,7 +4,7 @@ import { RequestWithAuth } from "./types";import { AuthCreditUsageChunkFromTeam } from "../v1/types";import { Response } from "express";import { getRedisConnection } from "../../services/queue-service";-import { fdbQueueEnabled } from "../../services/worker/nuq-router";+import { isFdbTeam } from "../../services/worker/nuq-router";import {nuqFdbHealthCheck,scrapeQueueFdb,@@ -58,7 +58,7 @@ export async function queueStatusController(let queuedJobsOfTeam = await getConcurrencyQueueJobsCount(req.auth.team_id);// during the FDB migration a team can have load on both ledgers- if (fdbQueueEnabled()) {+ if (await isFdbTeam(req.auth.team_id)) {try {if (await nuqFdbHealthCheck(FDB_OPTIONAL_COUNT_TIMEOUT_MS)) {const [fdbActive, fdbPending] = await Promise.all([diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.tsindex f5a778f29..32d115848 100644--- a/apps/api/src/harness.ts+++ b/apps/api/src/harness.ts@@ -864,14 +864,14 @@ async function startServices(command?: string[]): Promise<Services> {const nuqWorkers = Array.from({ length: NUQ_WORKER_COUNT }, (_, i) =>execForward(- `nuq-worker-${i}`,+ `${config.NUQ_BACKEND === "fdb" ? "nuq-fdb-worker" : "nuq-worker"}-${i}`,process.argv[2] === "--start-docker"- ? "node dist/src/services/worker/nuq-worker.js"- : "pnpm nuq-worker:production",+ ? `node dist/src/services/worker/${config.NUQ_BACKEND === "fdb" ? "nuq-fdb-worker" : "nuq-worker"}.js`+ : `pnpm ${config.NUQ_BACKEND === "fdb" ? "nuq-fdb-worker" : "nuq-worker"}:production`,{NUQ_WORKER_PORT: String(NUQ_WORKER_START_PORT + i),NUQ_REDUCE_NOISE: "true",- NUQ_POD_NAME: `nuq-worker-${i}`,+ NUQ_POD_NAME: `${config.NUQ_BACKEND === "fdb" ? "nuq-fdb-worker" : "nuq-worker"}-${i}`,},),);diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex cdce794cc..a96a70961 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -21,7 +21,7 @@ import Express from "express";import { robustFetch } from "../scraper/scrapeURL/lib/fetch";import { initializeBlocklist } from "../scraper/WebScraper/utils/blocklist";import { initializeEngineForcing } from "../scraper/WebScraper/utils/engine-forcing";-import { crawlFinishedQueue, NuQJob, scrapeQueue } from "./worker/nuq-router";+import { crawlFinishedQueue, NuQJob, scrapeQueue } from "./worker/nuq";import { finishCrawlSuper } from "./worker/crawl-logic";import { getCrawl } from "../lib/crawl-redis";import { TransportableError } from "../lib/error";diff --git a/apps/api/src/services/worker/nuq-fdb-worker.ts b/apps/api/src/services/worker/nuq-fdb-worker.tsnew file mode 100644index 000000000..f51104bc2--- /dev/null+++ b/apps/api/src/services/worker/nuq-fdb-worker.ts@@ -0,0 +1,158 @@+import "dotenv/config";+import "../sentry";+import { setSentryServiceTag } from "../sentry";+import { config } from "../../config";+import { logger as _logger } from "../../lib/logger";+import { getCrawl } from "../../lib/crawl-redis";+import { finishCrawlSuper } from "./crawl-logic";+import {+ crawlFinishedQueueFdb,+ getNuqFdbSweeper,+ nuqFdbHealthCheck,+ scrapeQueueFdb,+} from "./nuq-fdb";+import { runNuqWorker } from "./nuq-worker-runner";+import type { NuQJob } from "./nuq";++async function processFinishCrawlJobInternal(_job: NuQJob) {+ const job = await crawlFinishedQueueFdb.getJob(_job.id);++ if (!job) {+ throw new Error("crawlFinish job disappeared");+ }++ if (!job.groupId) {+ throw new Error("crawlFinish job with no groupId");+ }++ if (!job.ownerId) {+ throw new Error("crawlFinish job with no ownerId");+ }++ const sc = await getCrawl(job.groupId);++ if (!sc) {+ throw new Error("crawlFinish job with sc expired");+ }++ const anyJob = await scrapeQueueFdb.getGroupAnyJob(job.groupId, job.ownerId);++ if (!anyJob) {+ throw new Error("crawlFinish couldn't find anyJob");+ }++ await finishCrawlSuper(anyJob as any);+}++function startCrawlFinishedLoop() {+ let shuttingDown = false;++ const loop = (async () => {+ let noJobTimeout = 1500;++ while (!shuttingDown) {+ const job = await crawlFinishedQueueFdb.getJobToProcess();++ if (job === null) {+ await new Promise(resolve => setTimeout(resolve, noJobTimeout));+ if (!config.NUQ_RABBITMQ_URL) {+ noJobTimeout = Math.min(noJobTimeout * 2, 10000);+ }+ continue;+ }++ noJobTimeout = 500;++ const logger = _logger.child({+ module: "nuq-fdb-worker",+ method: "crawlFinishedLoop",+ jobId: job.id,+ crawlId: job.groupId,+ });++ logger.info("Acquired crawl finished job");++ const lockRenewInterval = setInterval(async () => {+ try {+ logger.info("Renewing crawl finished lock");+ if (+ !(await crawlFinishedQueueFdb.renewLock(job.id, job.lock!, logger))+ ) {+ logger.warn("Failed to renew crawl finished lock");+ clearInterval(lockRenewInterval);+ }+ } catch (error) {+ logger.warn("Failed to renew crawl finished lock", { error });+ clearInterval(lockRenewInterval);+ }+ }, 15000);++ try {+ await processFinishCrawlJobInternal(job as any);+ if (+ !(await crawlFinishedQueueFdb.jobFinish(+ job.id,+ job.lock!,+ null,+ logger,+ ))+ ) {+ logger.warn("Could not update crawl finished job status");+ }+ } catch (error) {+ logger.error("Crawl finished job failed", { error });+ if (+ !(await crawlFinishedQueueFdb.jobFail(+ job.id,+ job.lock!,+ error instanceof Error ? error.message : JSON.stringify(error),+ logger,+ ))+ ) {+ logger.warn("Could not update crawl finished job status");+ }+ } finally {+ clearInterval(lockRenewInterval);+ }+ }+ })();++ const done = loop.catch(error => {+ _logger.error("Crawl finished loop stopped unexpectedly", {+ module: "nuq-fdb-worker",+ error,+ });+ });++ return {+ stop() {+ shuttingDown = true;+ },+ done,+ };+}++(async () => {+ setSentryServiceTag("nuq-fdb-worker");++ let crawlFinishedLoop: ReturnType<typeof startCrawlFinishedLoop> | null =+ null;++ await runNuqWorker({+ serviceName: "nuq-fdb-worker",+ queue: scrapeQueueFdb as any,+ healthCheck: () => nuqFdbHealthCheck(),+ beforeStart: () => {+ getNuqFdbSweeper().start();+ crawlFinishedLoop = startCrawlFinishedLoop();+ },+ beforeShutdown: async () => {+ crawlFinishedLoop?.stop();+ try {+ await crawlFinishedLoop?.done;+ } finally {+ getNuqFdbSweeper().stop();+ }+ },+ });+})();diff --git a/apps/api/src/services/worker/nuq-router.ts b/apps/api/src/services/worker/nuq-router.tsindex d736bb47d..fb0e14265 100644--- a/apps/api/src/services/worker/nuq-router.ts+++ b/apps/api/src/services/worker/nuq-router.ts@@ -38,9 +38,10 @@ import {// - new crawls: team flag (TeamFlags.nuqFdb) or NUQ_BACKEND=fdb decides; the// choice is pinned in StoredCrawl.queueBackend so a crawl never spans// backends-// - reads: probe FDB first (cheap point reads), fall back to PG-// - workers: dual-poll FDB first; in-flight jobs are tagged with their-// backend so renew/finish/fail route correctly+// - reads: stored crawl/job backend markers decide the backend; unmarked jobs+// default to PG so FDB outages do not affect non-FDB traffic+// - workers: production workers consume PG and FDB via separate entrypoints;+// this class still tracks in-flight backend for direct/router test consumersexport type QueueBackend = "pg" | "fdb";@@ -111,6 +112,7 @@ export async function resolveNewGroupBackend(async function getCrawlQueueBackend(crawlId: string,): Promise<QueueBackend | null> {+ if (fdbForced()) return "fdb";const raw = await redisEvictConnection.get("crawl:" + crawlId);if (!raw) return null;try {@@ -121,6 +123,28 @@ async function getCrawlQueueBackend(}}+const jobBackendKey = (jobId: string) => `nuq:job_backend:${jobId}`;++async function markJobBackend(+ jobId: string,+ backend: QueueBackend,+): Promise<void> {+ if (fdbForced()) return;+ await redisEvictConnection.set(+ jobBackendKey(jobId),+ backend,+ "EX",+ 24 * 60 * 60,+ );+}++async function getJobQueueBackend(jobId: string): Promise<QueueBackend> {+ if (fdbForced()) return "fdb";+ return (await redisEvictConnection.get(jobBackendKey(jobId))) === "fdb"+ ? "fdb"+ : "pg";+}+// Which backend a job belongs to at enqueue time. Crawl jobs follow their// crawl's pinned backend; standalone jobs follow the team flag.export async function resolveJobBackend(@@ -187,7 +211,7 @@ export async function getCombinedTeamActiveCount(teamId: string,): Promise<number> {const redisCount = await getConcurrencyLimitActiveJobsCount(teamId);- if (!fdbQueueEnabled()) return redisCount;+ if (!(await isFdbTeam(teamId))) return redisCount;try {return (redisCount +@@ -255,6 +279,20 @@ export async function fdbEnqueueScrapeJobs();const tagged = results.map(r => tagFdbJob(r as NuQJob<ScrapeJobData>));+ const markerResults = await Promise.allSettled(+ tagged.map(job => markJobBackend(job.id, "fdb")),+ );+ const markerFailures = markerResults.filter(+ (r): r is PromiseRejectedResult => r.status === "rejected",+ );+ if (markerFailures.length > 0) {+ _logger.warn("Failed to mark some FDB job backends", {+ module: "nuq-router",+ failed: markerFailures.length,+ total: markerResults.length,+ errors: markerFailures.map(r => r.reason),+ });+ }return {jobs: tagged,backloggedCount: tagged.filter(j => j.status === "backlog").length,@@ -361,15 +399,9 @@ class RoutedScrapeQueue {id: string,logger: Logger = _logger,): Promise<NuQJob<ScrapeJobData> | null> {- if (fdbQueueEnabled()) {- try {- const job = await optionalFdb(() => scrapeQueueFdb.getJob(id, logger));- if (job) return tagFdbJob(job as NuQJob<ScrapeJobData>);- if (fdbForced()) return null;- } catch (error) {- if (fdbForced()) throw error;- logFdbFallback(logger, "scrape.getJob", error);- }+ if ((await getJobQueueBackend(id)) === "fdb") {+ const job = await optionalFdb(() => scrapeQueueFdb.getJob(id, logger));+ return job ? tagFdbJob(job as NuQJob<ScrapeJobData>) : null;}return scrapeQueuePg.getJob(id, logger);}@@ -378,21 +410,18 @@ class RoutedScrapeQueue {ids: string[],logger: Logger = _logger,): Promise<NuQJob<ScrapeJobData>[]> {- if (!fdbQueueEnabled()) return scrapeQueuePg.getJobs(ids, logger);- let fdbJobs: NuQFdbJob<ScrapeJobData>[] = [];- try {- fdbJobs = await optionalFdb(() => scrapeQueueFdb.getJobs(ids, logger));- } catch (error) {- if (fdbForced()) throw error;- logFdbFallback(logger, "scrape.getJobs", error);- return scrapeQueuePg.getJobs(ids, logger);- }- if (fdbForced())- return fdbJobs.map(j => tagFdbJob(j as NuQJob<ScrapeJobData>));- const found = new Set(fdbJobs.map(j => j.id));- const missing = ids.filter(id => !found.has(id));- const pgJobs =- missing.length > 0 ? await scrapeQueuePg.getJobs(missing, logger) : [];+ if (ids.length === 0) return [];+ const backends = await Promise.all(ids.map(id => getJobQueueBackend(id)));+ const fdbIds = ids.filter((_, i) => backends[i] === "fdb");+ const pgIds = ids.filter((_, i) => backends[i] === "pg");+ const [fdbJobs, pgJobs] = await Promise.all([+ fdbIds.length > 0+ ? optionalFdb(() => scrapeQueueFdb.getJobs(fdbIds, logger))+ : Promise.resolve([] as NuQFdbJob<ScrapeJobData>[]),+ pgIds.length > 0+ ? scrapeQueuePg.getJobs(pgIds, logger)+ : Promise.resolve([] as NuQJob<ScrapeJobData>[]),+ ]);const byId = new Map<string, NuQJob<ScrapeJobData>>();for (const j of fdbJobs)byId.set(j.id, tagFdbJob(j as NuQJob<ScrapeJobData>));@@ -420,16 +449,9 @@ class RoutedScrapeQueue {}private async isFdbGroup(groupId: string): Promise<boolean> {- if (!fdbQueueEnabled()) return false;- try {- return (- (await optionalFdb(() => crawlGroupFdb.getGroup(groupId))) !== null- );- } catch (error) {- if (fdbForced()) throw error;- logFdbFallback(_logger, "scrape.isFdbGroup", error);- return false;- }+ const backend = await getCrawlQueueBackend(groupId);+ if (backend) return backend === "fdb";+ return fdbForced();}public async getGroupAnyJob(@@ -474,18 +496,10 @@ class RoutedScrapeQueue {}public async removeJob(id: string, logger: Logger = _logger): Promise<void> {- if (fdbQueueEnabled()) {- try {- if (await optionalFdb(() => scrapeQueueFdb.hasJob(id))) {- await optionalFdb(() => scrapeQueueFdb.removeJob(id, logger));- return;- }- } catch (error) {- if (fdbForced()) throw error;- logFdbFallback(logger, "scrape.removeJob", error);- }+ if ((await getJobQueueBackend(id)) === "fdb") {+ await optionalFdb(() => scrapeQueueFdb.removeJob(id, logger));+ return;}- if (fdbForced()) return;await scrapeQueuePg.removeJob(id, logger);}@@ -503,19 +517,9 @@ class RoutedScrapeQueue {timeout: number | null,logger: Logger = _logger,): Promise<T> {- if (fdbQueueEnabled()) {- try {- if (await optionalFdb(() => scrapeQueueFdb.hasJob(id))) {- return optionalFdb(() =>- scrapeQueueFdb.waitForJob(id, timeout, logger),- );- }- } catch (error) {- if (fdbForced()) throw error;- logFdbFallback(logger, "scrape.waitForJob", error);- }+ if ((await getJobQueueBackend(id)) === "fdb") {+ return optionalFdb(() => scrapeQueueFdb.waitForJob(id, timeout, logger));}- if (fdbForced()) throw new Error("Job not found");return scrapeQueuePg.waitForJob(id, timeout, logger) as Promise<T>;}@@ -618,17 +622,11 @@ class RoutedCrawlFinishedQueue {id: string,logger: Logger = _logger,): Promise<NuQJob<any> | null> {- if (fdbQueueEnabled()) {- try {- const job = await optionalFdb(() =>- crawlFinishedQueueFdb.getJob(id, logger),- );- if (job) return tagFdbJob(job as NuQJob<any>);- if (fdbForced()) return null;- } catch (error) {- if (fdbForced()) throw error;- logFdbFallback(logger, "crawlFinished.getJob", error);- }+ if ((await getJobQueueBackend(id)) === "fdb") {+ const job = await optionalFdb(() =>+ crawlFinishedQueueFdb.getJob(id, logger),+ );+ return job ? tagFdbJob(job as NuQJob<any>) : null;}return crawlFinishedQueuePg.getJob(id, logger);}@@ -670,15 +668,11 @@ class RoutedCrawlGroup {id: string,logger: Logger = _logger,): Promise<NuQJobGroupInstance | null> {- if (fdbQueueEnabled()) {- try {- const g = await optionalFdb(() => crawlGroupFdb.getGroup(id, logger));- if (g) return g as NuQJobGroupInstance;- if (fdbForced()) return null;- } catch (error) {- if (fdbForced()) throw error;- logFdbFallback(logger, "crawlGroup.getGroup", error);- }+ const backend = await getCrawlQueueBackend(id);+ if (backend === "fdb" || (!backend && fdbForced())) {+ return (await optionalFdb(() =>+ crawlGroupFdb.getGroup(id, logger),+ )) as NuQJobGroupInstance | null;}return crawlGroupPg.getGroup(id, logger);}@@ -687,7 +681,7 @@ class RoutedCrawlGroup {ownerId: string,logger: Logger = _logger,): Promise<NuQJobGroupInstance[]> {- if (!fdbQueueEnabled()) {+ if (!(await isFdbTeam(ownerId))) {return crawlGroupPg.getOngoingByOwner(ownerId, logger);}let fdb: NuQJobGroupInstance[] = [];@@ -715,7 +709,8 @@ class RoutedCrawlGroup {id: string,logger: Logger = _logger,): Promise<boolean> {- if (!fdbQueueEnabled()) return false;+ const backend = await getCrawlQueueBackend(id);+ if (backend !== "fdb" && !(backend === null && fdbForced())) return false;try {return await optionalFdb(() => crawlGroupFdb.cancelGroup(id, logger));} catch (error) {diff --git a/apps/api/src/services/worker/nuq-worker-runner.ts b/apps/api/src/services/worker/nuq-worker-runner.tsnew file mode 100644index 000000000..9d62cfb67--- /dev/null+++ b/apps/api/src/services/worker/nuq-worker-runner.ts@@ -0,0 +1,201 @@+import { config } from "../../config";+import { logger as _logger } from "../../lib/logger";+import { jobDurationSeconds } from "../../lib/job-metrics";+import { processJobInternal } from "./scrape-worker";+import { NuQJob } from "./nuq";+import { register } from "prom-client";+import Express from "express";+import { initializeBlocklist } from "../../scraper/WebScraper/utils/blocklist";+import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-forcing";++export type WorkerQueue = {+ getJobToProcess(logger?: any): Promise<NuQJob<any, any> | null>;+ renewLock(id: string, lock: string, logger?: any): Promise<boolean>;+ jobFinish(+ id: string,+ lock: string,+ returnvalue: any | null,+ logger?: any,+ ): Promise<boolean>;+ jobFail(+ id: string,+ lock: string,+ failedReason: string,+ logger?: any,+ ): Promise<boolean>;+};++async function withTimeout<T>(+ promise: Promise<T>,+ timeoutMs: number,+ label: string,+): Promise<T> {+ let timer: NodeJS.Timeout | undefined;+ try {+ return await Promise.race([+ promise,+ new Promise<T>((_, reject) => {+ timer = setTimeout(+ () => reject(new Error(`${label} timed out after ${timeoutMs}ms`)),+ timeoutMs,+ );+ }),+ ]);+ } finally {+ if (timer) clearTimeout(timer);+ }+}++export async function runNuqWorker(options: {+ serviceName: string;+ queue: WorkerQueue;+ healthCheck: () => Promise<boolean>;+ metrics?: () => string | Promise<string>;+ beforeStart?: () => void | Promise<void>;+ beforeShutdown?: () => void | Promise<void>;+ shutdown?: () => void | Promise<void>;+}) {+ try {+ await initializeBlocklist();+ initializeEngineForcing();+ await options.beforeStart?.();+ } catch (error) {+ _logger.error("Failed to initialize NuQ worker", {+ module: options.serviceName,+ error,+ });+ process.exit(1);+ }++ let isShuttingDown = false;++ const app = Express();++ app.get("/metrics", async (_, res) => {+ const localMetrics = options.metrics ? await options.metrics() : "";+ res+ .contentType("text/plain")+ .send(localMetrics + "\n" + (await register.metrics()));+ });+ app.get("/health", async (_, res) => {+ try {+ if (await withTimeout(options.healthCheck(), 1000, "NuQ health check")) {+ res.status(200).send("OK");+ } else {+ res.status(500).send("Not OK");+ }+ } catch (error) {+ _logger.warn("NuQ worker health check failed", {+ module: options.serviceName,+ error,+ });+ res.status(500).send("Not OK");+ }+ });++ const server = app.listen(config.NUQ_WORKER_PORT, () => {+ _logger.info("NuQ worker metrics server started", {+ module: options.serviceName,+ });+ });++ function shutdown() {+ isShuttingDown = true;+ }++ process.on("SIGINT", shutdown);+ process.on("SIGTERM", shutdown);++ let noJobTimeout = 1500;++ while (!isShuttingDown) {+ const job = await options.queue.getJobToProcess();++ if (job === null) {+ _logger.info("No jobs to process", { module: "nuq/metrics" });+ await new Promise(resolve => setTimeout(resolve, noJobTimeout));+ if (!config.NUQ_RABBITMQ_URL) {+ noJobTimeout = Math.min(noJobTimeout * 2, 10000);+ }+ continue;+ }++ noJobTimeout = 500;++ const logger = _logger.child({+ module: options.serviceName,+ scrapeId: job.id,+ zeroDataRetention: job.data?.zeroDataRetention ?? false,+ });++ logger.info("Acquired job");++ const lockRenewInterval = setInterval(async () => {+ try {+ logger.info("Renewing lock");+ if (!(await options.queue.renewLock(job.id, job.lock!, logger))) {+ logger.warn("Failed to renew lock");+ clearInterval(lockRenewInterval);+ return;+ }+ logger.info("Renewed lock");+ } catch (error) {+ logger.warn("Failed to renew lock", { error });+ clearInterval(lockRenewInterval);+ }+ }, 15000);++ let processResult:+ | { ok: true; data: Awaited<ReturnType<typeof processJobInternal>> }+ | { ok: false; error: any };++ const endJobTimer = jobDurationSeconds.startTimer({ type: job.data.mode });++ try {+ processResult = { ok: true, data: await processJobInternal(job) };+ } catch (error) {+ processResult = { ok: false, error };+ }++ clearInterval(lockRenewInterval);++ if (processResult.ok) {+ endJobTimer({ status: "success" });+ if (+ !(await options.queue.jobFinish(+ job.id,+ job.lock!,+ processResult.data,+ logger,+ ))+ ) {+ logger.warn("Could not update job status");+ }+ } else {+ endJobTimer({ status: "failed" });+ if (+ !(await options.queue.jobFail(+ job.id,+ job.lock!,+ processResult.error instanceof Error+ ? processResult.error.message+ : typeof processResult.error === "string"+ ? processResult.error+ : JSON.stringify(processResult.error),+ logger,+ ))+ ) {+ logger.warn("Could not update job status");+ }+ }+ }++ _logger.info("NuQ worker shutting down", { module: options.serviceName });++ server.close(async () => {+ await options.beforeShutdown?.();+ await options.shutdown?.();+ _logger.info("NuQ worker shut down", { module: options.serviceName });+ process.exit(0);+ });+}diff --git a/apps/api/src/services/worker/nuq-worker.ts b/apps/api/src/services/worker/nuq-worker.tsindex 05d977188..984ba3f9d 100644--- a/apps/api/src/services/worker/nuq-worker.ts+++ b/apps/api/src/services/worker/nuq-worker.ts@@ -1,177 +1,21 @@import "dotenv/config";-import { config } from "../../config";import "../sentry";import { setSentryServiceTag } from "../sentry";-import { logger as _logger } from "../../lib/logger";-import { processJobInternal } from "./scrape-worker";import {nuqGetLocalMetrics,nuqHealthCheck,- scrapeQueue as scrapeQueuePg,+ scrapeQueue,} from "./nuq";-import { scrapeQueue, fdbQueueEnabled } from "./nuq-router";-import { getNuqFdbSweeper, nuqFdbHealthCheck } from "./nuq-fdb";-import { jobDurationSeconds } from "../../lib/job-metrics";-import { register } from "prom-client";-import Express from "express";-import { _ } from "ajv";-import { initializeBlocklist } from "../../scraper/WebScraper/utils/blocklist";-import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-forcing";+import { runNuqWorker } from "./nuq-worker-runner";(async () => {setSentryServiceTag("nuq-worker");- try {- await initializeBlocklist();- initializeEngineForcing();- } catch (error) {- _logger.error("Failed to initialize blocklist and engine forcing", {- error,- });- process.exit(1);- }-- let isShuttingDown = false;-- const app = Express();-- app.get("/metrics", async (_, res) =>- res- .contentType("text/plain")- .send(nuqGetLocalMetrics() + "\n" + (await register.metrics())),- );- app.get("/health", async (_, res) => {- const pgHealthy = await nuqHealthCheck();- const fdbHealthy =- config.NUQ_BACKEND !== "fdb" || (await nuqFdbHealthCheck());- if (pgHealthy && fdbHealthy) {- res.status(200).send("OK");- } else {- res.status(500).send("Not OK");- }- });-- const server = app.listen(config.NUQ_WORKER_PORT, () => {- _logger.info("NuQ worker metrics server started");- });-- function shutdown() {- isShuttingDown = true;- }-- if (require.main === module) {- process.on("SIGINT", shutdown);- process.on("SIGTERM", shutdown);- }-- let noJobTimeout = 1500;-- // the FDB backend has no pg_cron: one worker at a time holds the sweeper- // lease and runs lease/timeout/group sweeps for everyone- if (fdbQueueEnabled()) {- try {- getNuqFdbSweeper().start();- } catch (error) {- if (config.NUQ_BACKEND === "fdb") throw error;- _logger.warn("Failed to start FDB sweeper, continuing with PG", {- module: "nuq-worker",- error,- });- }- }-- while (!isShuttingDown) {- const job = await scrapeQueue.getJobToProcess();-- if (job === null) {- _logger.info("No jobs to process", { module: "nuq/metrics" });- await new Promise(resolve => setTimeout(resolve, noJobTimeout));- if (!config.NUQ_RABBITMQ_URL) {- noJobTimeout = Math.min(noJobTimeout * 2, 10000);- }- continue;- }-- noJobTimeout = 500;-- const logger = _logger.child({- module: "nuq-worker",- scrapeId: job.id,- zeroDataRetention: job.data?.zeroDataRetention ?? false,- });-- logger.info("Acquired job");-- const lockRenewInterval = setInterval(async () => {- logger.info("Renewing lock");- if (!(await scrapeQueue.renewLock(job.id, job.lock!, logger))) {- logger.warn("Failed to renew lock");- clearInterval(lockRenewInterval);- return;- }- logger.info("Renewed lock");- }, 15000);-- let processResult:- | { ok: true; data: Awaited<ReturnType<typeof processJobInternal>> }- | { ok: false; error: any };-- const endJobTimer = jobDurationSeconds.startTimer({ type: job.data.mode });-- try {- processResult = { ok: true, data: await processJobInternal(job) };- } catch (error) {- processResult = { ok: false, error };- }-- clearInterval(lockRenewInterval);-- if (processResult.ok) {- endJobTimer({ status: "success" });- if (- !(await scrapeQueue.jobFinish(- job.id,- job.lock!,- processResult.data,- logger,- ))- ) {- logger.warn("Could not update job status");- }- } else {- endJobTimer({ status: "failed" });- if (- !(await scrapeQueue.jobFail(- job.id,- job.lock!,- processResult.error instanceof Error- ? processResult.error.message- : typeof processResult.error === "string"- ? processResult.error- : JSON.stringify(processResult.error),- logger,- ))- ) {- logger.warn("Could not update job status");- }- }- }-- _logger.info("NuQ worker shutting down");-- server.close(async () => {- if (fdbQueueEnabled()) {- try {- getNuqFdbSweeper().stop();- } catch (error) {- _logger.warn("Failed to stop FDB sweeper", {- module: "nuq-worker",- error,- });- }- }- await scrapeQueuePg.shutdown();- _logger.info("NuQ worker shut down");- process.exit(0);+ await runNuqWorker({+ serviceName: "nuq-worker",+ queue: scrapeQueue,+ healthCheck: nuqHealthCheck,+ metrics: nuqGetLocalMetrics,+ shutdown: () => scrapeQueue.shutdown(),});})();