Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): separate NuQ FDB worker (#3779)

Gergő Móricz9e520616625e
diff --git a/apps/api/package.json b/apps/api/package.json
index 3af53eef9..8118851e0 100644
--- a/apps/api/package.json
+++ b/apps/api/package.json
@@ -25,6 +25,8 @@
"worker:production": "node dist/src/services/queue-worker.js",
"nuq-worker": "tsc-watch --onSuccess \"node dist/src/services/worker/nuq-worker.js\"",
"nuq-worker:production": "node dist/src/services/worker/nuq-worker.js",
+ "nuq-fdb-worker": "tsc-watch --onSuccess \"node dist/src/services/worker/nuq-fdb-worker.js\"",
+ "nuq-fdb-worker:production": "node dist/src/services/worker/nuq-fdb-worker.js",
"nuq-prefetch-worker": "tsc-watch --onSuccess \"node dist/src/services/worker/nuq-prefetch-worker.js\"",
"nuq-prefetch-worker:production": "node dist/src/services/worker/nuq-prefetch-worker.js",
"nuq-reconciler-worker": "tsc-watch --onSuccess \"node dist/src/services/worker/nuq-reconciler-worker.js\"",
diff --git a/apps/api/src/controllers/v1/queue-status.ts b/apps/api/src/controllers/v1/queue-status.ts
index 0f2b72316..eac91e419 100644
--- a/apps/api/src/controllers/v1/queue-status.ts
+++ b/apps/api/src/controllers/v1/queue-status.ts
@@ -3,7 +3,7 @@ import { getACUCTeam } from "../auth";
import { AuthCreditUsageChunkFromTeam, RequestWithAuth } from "./types";
import { Response } from "express";
import { getRedisConnection } from "../../services/queue-service";
-import { fdbQueueEnabled } from "../../services/worker/nuq-router";
+import { isFdbTeam } from "../../services/worker/nuq-router";
import {
nuqFdbHealthCheck,
scrapeQueueFdb,
@@ -57,7 +57,7 @@ export async function queueStatusController(
let queuedJobsOfTeam = await getConcurrencyQueueJobsCount(req.auth.team_id);
// during the FDB migration a team can have load on both ledgers
- if (fdbQueueEnabled()) {
+ if (await isFdbTeam(req.auth.team_id)) {
try {
if (await nuqFdbHealthCheck(FDB_OPTIONAL_COUNT_TIMEOUT_MS)) {
const [fdbActive, fdbPending] = await Promise.all([
diff --git a/apps/api/src/controllers/v2/queue-status.ts b/apps/api/src/controllers/v2/queue-status.ts
index ea95e8343..297e625b5 100644
--- a/apps/api/src/controllers/v2/queue-status.ts
+++ b/apps/api/src/controllers/v2/queue-status.ts
@@ -4,7 +4,7 @@ import { RequestWithAuth } from "./types";
import { AuthCreditUsageChunkFromTeam } from "../v1/types";
import { Response } from "express";
import { getRedisConnection } from "../../services/queue-service";
-import { fdbQueueEnabled } from "../../services/worker/nuq-router";
+import { isFdbTeam } from "../../services/worker/nuq-router";
import {
nuqFdbHealthCheck,
scrapeQueueFdb,
@@ -58,7 +58,7 @@ export async function queueStatusController(
let queuedJobsOfTeam = await getConcurrencyQueueJobsCount(req.auth.team_id);
// during the FDB migration a team can have load on both ledgers
- if (fdbQueueEnabled()) {
+ if (await isFdbTeam(req.auth.team_id)) {
try {
if (await nuqFdbHealthCheck(FDB_OPTIONAL_COUNT_TIMEOUT_MS)) {
const [fdbActive, fdbPending] = await Promise.all([
diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.ts
index f5a778f29..32d115848 100644
--- a/apps/api/src/harness.ts
+++ b/apps/api/src/harness.ts
@@ -864,14 +864,14 @@ async function startServices(command?: string[]): Promise<Services> {
const nuqWorkers = Array.from({ length: NUQ_WORKER_COUNT }, (_, i) =>
execForward(
- `nuq-worker-${i}`,
+ `${config.NUQ_BACKEND === "fdb" ? "nuq-fdb-worker" : "nuq-worker"}-${i}`,
process.argv[2] === "--start-docker"
- ? "node dist/src/services/worker/nuq-worker.js"
- : "pnpm nuq-worker:production",
+ ? `node dist/src/services/worker/${config.NUQ_BACKEND === "fdb" ? "nuq-fdb-worker" : "nuq-worker"}.js`
+ : `pnpm ${config.NUQ_BACKEND === "fdb" ? "nuq-fdb-worker" : "nuq-worker"}:production`,
{
NUQ_WORKER_PORT: String(NUQ_WORKER_START_PORT + i),
NUQ_REDUCE_NOISE: "true",
- NUQ_POD_NAME: `nuq-worker-${i}`,
+ NUQ_POD_NAME: `${config.NUQ_BACKEND === "fdb" ? "nuq-fdb-worker" : "nuq-worker"}-${i}`,
},
),
);
diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
index cdce794cc..a96a70961 100644
--- a/apps/api/src/services/queue-worker.ts
+++ b/apps/api/src/services/queue-worker.ts
@@ -21,7 +21,7 @@ import Express from "express";
import { robustFetch } from "../scraper/scrapeURL/lib/fetch";
import { initializeBlocklist } from "../scraper/WebScraper/utils/blocklist";
import { initializeEngineForcing } from "../scraper/WebScraper/utils/engine-forcing";
-import { crawlFinishedQueue, NuQJob, scrapeQueue } from "./worker/nuq-router";
+import { crawlFinishedQueue, NuQJob, scrapeQueue } from "./worker/nuq";
import { finishCrawlSuper } from "./worker/crawl-logic";
import { getCrawl } from "../lib/crawl-redis";
import { TransportableError } from "../lib/error";
diff --git a/apps/api/src/services/worker/nuq-fdb-worker.ts b/apps/api/src/services/worker/nuq-fdb-worker.ts
new file mode 100644
index 000000000..f51104bc2
--- /dev/null
+++ b/apps/api/src/services/worker/nuq-fdb-worker.ts
@@ -0,0 +1,158 @@
+import "dotenv/config";
+import "../sentry";
+import { setSentryServiceTag } from "../sentry";
+import { config } from "../../config";
+import { logger as _logger } from "../../lib/logger";
+import { getCrawl } from "../../lib/crawl-redis";
+import { finishCrawlSuper } from "./crawl-logic";
+import {
+ crawlFinishedQueueFdb,
+ getNuqFdbSweeper,
+ nuqFdbHealthCheck,
+ scrapeQueueFdb,
+} from "./nuq-fdb";
+import { runNuqWorker } from "./nuq-worker-runner";
+import type { NuQJob } from "./nuq";
+
+async function processFinishCrawlJobInternal(_job: NuQJob) {
+ const job = await crawlFinishedQueueFdb.getJob(_job.id);
+
+ if (!job) {
+ throw new Error("crawlFinish job disappeared");
+ }
+
+ if (!job.groupId) {
+ throw new Error("crawlFinish job with no groupId");
+ }
+
+ if (!job.ownerId) {
+ throw new Error("crawlFinish job with no ownerId");
+ }
+
+ const sc = await getCrawl(job.groupId);
+
+ if (!sc) {
+ throw new Error("crawlFinish job with sc expired");
+ }
+
+ const anyJob = await scrapeQueueFdb.getGroupAnyJob(job.groupId, job.ownerId);
+
+ if (!anyJob) {
+ throw new Error("crawlFinish couldn't find anyJob");
+ }
+
+ await finishCrawlSuper(anyJob as any);
+}
+
+function startCrawlFinishedLoop() {
+ let shuttingDown = false;
+
+ const loop = (async () => {
+ let noJobTimeout = 1500;
+
+ while (!shuttingDown) {
+ const job = await crawlFinishedQueueFdb.getJobToProcess();
+
+ if (job === null) {
+ await new Promise(resolve => setTimeout(resolve, noJobTimeout));
+ if (!config.NUQ_RABBITMQ_URL) {
+ noJobTimeout = Math.min(noJobTimeout * 2, 10000);
+ }
+ continue;
+ }
+
+ noJobTimeout = 500;
+
+ const logger = _logger.child({
+ module: "nuq-fdb-worker",
+ method: "crawlFinishedLoop",
+ jobId: job.id,
+ crawlId: job.groupId,
+ });
+
+ logger.info("Acquired crawl finished job");
+
+ const lockRenewInterval = setInterval(async () => {
+ try {
+ logger.info("Renewing crawl finished lock");
+ if (
+ !(await crawlFinishedQueueFdb.renewLock(job.id, job.lock!, logger))
+ ) {
+ logger.warn("Failed to renew crawl finished lock");
+ clearInterval(lockRenewInterval);
+ }
+ } catch (error) {
+ logger.warn("Failed to renew crawl finished lock", { error });
+ clearInterval(lockRenewInterval);
+ }
+ }, 15000);
+
+ try {
+ await processFinishCrawlJobInternal(job as any);
+ if (
+ !(await crawlFinishedQueueFdb.jobFinish(
+ job.id,
+ job.lock!,
+ null,
+ logger,
+ ))
+ ) {
+ logger.warn("Could not update crawl finished job status");
+ }
+ } catch (error) {
+ logger.error("Crawl finished job failed", { error });
+ if (
+ !(await crawlFinishedQueueFdb.jobFail(
+ job.id,
+ job.lock!,
+ error instanceof Error ? error.message : JSON.stringify(error),
+ logger,
+ ))
+ ) {
+ logger.warn("Could not update crawl finished job status");
+ }
+ } finally {
+ clearInterval(lockRenewInterval);
+ }
+ }
+ })();
+
+ const done = loop.catch(error => {
+ _logger.error("Crawl finished loop stopped unexpectedly", {
+ module: "nuq-fdb-worker",
+ error,
+ });
+ });
+
+ return {
+ stop() {
+ shuttingDown = true;
+ },
+ done,
+ };
+}
+
+(async () => {
+ setSentryServiceTag("nuq-fdb-worker");
+
+ let crawlFinishedLoop: ReturnType<typeof startCrawlFinishedLoop> | null =
+ null;
+
+ await runNuqWorker({
+ serviceName: "nuq-fdb-worker",
+ queue: scrapeQueueFdb as any,
+ healthCheck: () => nuqFdbHealthCheck(),
+ beforeStart: () => {
+ getNuqFdbSweeper().start();
+ crawlFinishedLoop = startCrawlFinishedLoop();
+ },
+ beforeShutdown: async () => {
+ crawlFinishedLoop?.stop();
+ try {
+ await crawlFinishedLoop?.done;
+ } finally {
+ getNuqFdbSweeper().stop();
+ }
+ },
+ });
+})();
diff --git a/apps/api/src/services/worker/nuq-router.ts b/apps/api/src/services/worker/nuq-router.ts
index d736bb47d..fb0e14265 100644
--- a/apps/api/src/services/worker/nuq-router.ts
+++ b/apps/api/src/services/worker/nuq-router.ts
@@ -38,9 +38,10 @@ import {
// - new crawls: team flag (TeamFlags.nuqFdb) or NUQ_BACKEND=fdb decides; the
// choice is pinned in StoredCrawl.queueBackend so a crawl never spans
// backends
-// - reads: probe FDB first (cheap point reads), fall back to PG
-// - workers: dual-poll FDB first; in-flight jobs are tagged with their
-// backend so renew/finish/fail route correctly
+// - reads: stored crawl/job backend markers decide the backend; unmarked jobs
+// default to PG so FDB outages do not affect non-FDB traffic
+// - workers: production workers consume PG and FDB via separate entrypoints;
+// this class still tracks in-flight backend for direct/router test consumers
export type QueueBackend = "pg" | "fdb";
@@ -111,6 +112,7 @@ export async function resolveNewGroupBackend(
async function getCrawlQueueBackend(
crawlId: string,
): Promise<QueueBackend | null> {
+ if (fdbForced()) return "fdb";
const raw = await redisEvictConnection.get("crawl:" + crawlId);
if (!raw) return null;
try {
@@ -121,6 +123,28 @@ async function getCrawlQueueBackend(
}
}
+const jobBackendKey = (jobId: string) => `nuq:job_backend:${jobId}`;
+
+async function markJobBackend(
+ jobId: string,
+ backend: QueueBackend,
+): Promise<void> {
+ if (fdbForced()) return;
+ await redisEvictConnection.set(
+ jobBackendKey(jobId),
+ backend,
+ "EX",
+ 24 * 60 * 60,
+ );
+}
+
+async function getJobQueueBackend(jobId: string): Promise<QueueBackend> {
+ if (fdbForced()) return "fdb";
+ return (await redisEvictConnection.get(jobBackendKey(jobId))) === "fdb"
+ ? "fdb"
+ : "pg";
+}
+
// Which backend a job belongs to at enqueue time. Crawl jobs follow their
// crawl's pinned backend; standalone jobs follow the team flag.
export async function resolveJobBackend(
@@ -187,7 +211,7 @@ export async function getCombinedTeamActiveCount(
teamId: string,
): Promise<number> {
const redisCount = await getConcurrencyLimitActiveJobsCount(teamId);
- if (!fdbQueueEnabled()) return redisCount;
+ if (!(await isFdbTeam(teamId))) return redisCount;
try {
return (
redisCount +
@@ -255,6 +279,20 @@ export async function fdbEnqueueScrapeJobs(
);
const tagged = results.map(r => tagFdbJob(r as NuQJob<ScrapeJobData>));
+ const markerResults = await Promise.allSettled(
+ tagged.map(job => markJobBackend(job.id, "fdb")),
+ );
+ const markerFailures = markerResults.filter(
+ (r): r is PromiseRejectedResult => r.status === "rejected",
+ );
+ if (markerFailures.length > 0) {
+ _logger.warn("Failed to mark some FDB job backends", {
+ module: "nuq-router",
+ failed: markerFailures.length,
+ total: markerResults.length,
+ errors: markerFailures.map(r => r.reason),
+ });
+ }
return {
jobs: tagged,
backloggedCount: tagged.filter(j => j.status === "backlog").length,
@@ -361,15 +399,9 @@ class RoutedScrapeQueue {
id: string,
logger: Logger = _logger,
): Promise<NuQJob<ScrapeJobData> | null> {
- if (fdbQueueEnabled()) {
- try {
- const job = await optionalFdb(() => scrapeQueueFdb.getJob(id, logger));
- if (job) return tagFdbJob(job as NuQJob<ScrapeJobData>);
- if (fdbForced()) return null;
- } catch (error) {
- if (fdbForced()) throw error;
- logFdbFallback(logger, "scrape.getJob", error);
- }
+ if ((await getJobQueueBackend(id)) === "fdb") {
+ const job = await optionalFdb(() => scrapeQueueFdb.getJob(id, logger));
+ return job ? tagFdbJob(job as NuQJob<ScrapeJobData>) : null;
}
return scrapeQueuePg.getJob(id, logger);
}
@@ -378,21 +410,18 @@ class RoutedScrapeQueue {
ids: string[],
logger: Logger = _logger,
): Promise<NuQJob<ScrapeJobData>[]> {
- if (!fdbQueueEnabled()) return scrapeQueuePg.getJobs(ids, logger);
- let fdbJobs: NuQFdbJob<ScrapeJobData>[] = [];
- try {
- fdbJobs = await optionalFdb(() => scrapeQueueFdb.getJobs(ids, logger));
- } catch (error) {
- if (fdbForced()) throw error;
- logFdbFallback(logger, "scrape.getJobs", error);
- return scrapeQueuePg.getJobs(ids, logger);
- }
- if (fdbForced())
- return fdbJobs.map(j => tagFdbJob(j as NuQJob<ScrapeJobData>));
- const found = new Set(fdbJobs.map(j => j.id));
- const missing = ids.filter(id => !found.has(id));
- const pgJobs =
- missing.length > 0 ? await scrapeQueuePg.getJobs(missing, logger) : [];
+ if (ids.length === 0) return [];
+ const backends = await Promise.all(ids.map(id => getJobQueueBackend(id)));
+ const fdbIds = ids.filter((_, i) => backends[i] === "fdb");
+ const pgIds = ids.filter((_, i) => backends[i] === "pg");
+ const [fdbJobs, pgJobs] = await Promise.all([
+ fdbIds.length > 0
+ ? optionalFdb(() => scrapeQueueFdb.getJobs(fdbIds, logger))
+ : Promise.resolve([] as NuQFdbJob<ScrapeJobData>[]),
+ pgIds.length > 0
+ ? scrapeQueuePg.getJobs(pgIds, logger)
+ : Promise.resolve([] as NuQJob<ScrapeJobData>[]),
+ ]);
const byId = new Map<string, NuQJob<ScrapeJobData>>();
for (const j of fdbJobs)
byId.set(j.id, tagFdbJob(j as NuQJob<ScrapeJobData>));
@@ -420,16 +449,9 @@ class RoutedScrapeQueue {
}
private async isFdbGroup(groupId: string): Promise<boolean> {
- if (!fdbQueueEnabled()) return false;
- try {
- return (
- (await optionalFdb(() => crawlGroupFdb.getGroup(groupId))) !== null
- );
- } catch (error) {
- if (fdbForced()) throw error;
- logFdbFallback(_logger, "scrape.isFdbGroup", error);
- return false;
- }
+ const backend = await getCrawlQueueBackend(groupId);
+ if (backend) return backend === "fdb";
+ return fdbForced();
}
public async getGroupAnyJob(
@@ -474,18 +496,10 @@ class RoutedScrapeQueue {
}
public async removeJob(id: string, logger: Logger = _logger): Promise<void> {
- if (fdbQueueEnabled()) {
- try {
- if (await optionalFdb(() => scrapeQueueFdb.hasJob(id))) {
- await optionalFdb(() => scrapeQueueFdb.removeJob(id, logger));
- return;
- }
- } catch (error) {
- if (fdbForced()) throw error;
- logFdbFallback(logger, "scrape.removeJob", error);
- }
+ if ((await getJobQueueBackend(id)) === "fdb") {
+ await optionalFdb(() => scrapeQueueFdb.removeJob(id, logger));
+ return;
}
- if (fdbForced()) return;
await scrapeQueuePg.removeJob(id, logger);
}
@@ -503,19 +517,9 @@ class RoutedScrapeQueue {
timeout: number | null,
logger: Logger = _logger,
): Promise<T> {
- if (fdbQueueEnabled()) {
- try {
- if (await optionalFdb(() => scrapeQueueFdb.hasJob(id))) {
- return optionalFdb(() =>
- scrapeQueueFdb.waitForJob(id, timeout, logger),
- );
- }
- } catch (error) {
- if (fdbForced()) throw error;
- logFdbFallback(logger, "scrape.waitForJob", error);
- }
+ if ((await getJobQueueBackend(id)) === "fdb") {
+ return optionalFdb(() => scrapeQueueFdb.waitForJob(id, timeout, logger));
}
- if (fdbForced()) throw new Error("Job not found");
return scrapeQueuePg.waitForJob(id, timeout, logger) as Promise<T>;
}
@@ -618,17 +622,11 @@ class RoutedCrawlFinishedQueue {
id: string,
logger: Logger = _logger,
): Promise<NuQJob<any> | null> {
- if (fdbQueueEnabled()) {
- try {
- const job = await optionalFdb(() =>
- crawlFinishedQueueFdb.getJob(id, logger),
- );
- if (job) return tagFdbJob(job as NuQJob<any>);
- if (fdbForced()) return null;
- } catch (error) {
- if (fdbForced()) throw error;
- logFdbFallback(logger, "crawlFinished.getJob", error);
- }
+ if ((await getJobQueueBackend(id)) === "fdb") {
+ const job = await optionalFdb(() =>
+ crawlFinishedQueueFdb.getJob(id, logger),
+ );
+ return job ? tagFdbJob(job as NuQJob<any>) : null;
}
return crawlFinishedQueuePg.getJob(id, logger);
}
@@ -670,15 +668,11 @@ class RoutedCrawlGroup {
id: string,
logger: Logger = _logger,
): Promise<NuQJobGroupInstance | null> {
- if (fdbQueueEnabled()) {
- try {
- const g = await optionalFdb(() => crawlGroupFdb.getGroup(id, logger));
- if (g) return g as NuQJobGroupInstance;
- if (fdbForced()) return null;
- } catch (error) {
- if (fdbForced()) throw error;
- logFdbFallback(logger, "crawlGroup.getGroup", error);
- }
+ const backend = await getCrawlQueueBackend(id);
+ if (backend === "fdb" || (!backend && fdbForced())) {
+ return (await optionalFdb(() =>
+ crawlGroupFdb.getGroup(id, logger),
+ )) as NuQJobGroupInstance | null;
}
return crawlGroupPg.getGroup(id, logger);
}
@@ -687,7 +681,7 @@ class RoutedCrawlGroup {
ownerId: string,
logger: Logger = _logger,
): Promise<NuQJobGroupInstance[]> {
- if (!fdbQueueEnabled()) {
+ if (!(await isFdbTeam(ownerId))) {
return crawlGroupPg.getOngoingByOwner(ownerId, logger);
}
let fdb: NuQJobGroupInstance[] = [];
@@ -715,7 +709,8 @@ class RoutedCrawlGroup {
id: string,
logger: Logger = _logger,
): Promise<boolean> {
- if (!fdbQueueEnabled()) return false;
+ const backend = await getCrawlQueueBackend(id);
+ if (backend !== "fdb" && !(backend === null && fdbForced())) return false;
try {
return await optionalFdb(() => crawlGroupFdb.cancelGroup(id, logger));
} catch (error) {
diff --git a/apps/api/src/services/worker/nuq-worker-runner.ts b/apps/api/src/services/worker/nuq-worker-runner.ts
new file mode 100644
index 000000000..9d62cfb67
--- /dev/null
+++ b/apps/api/src/services/worker/nuq-worker-runner.ts
@@ -0,0 +1,201 @@
+import { config } from "../../config";
+import { logger as _logger } from "../../lib/logger";
+import { jobDurationSeconds } from "../../lib/job-metrics";
+import { processJobInternal } from "./scrape-worker";
+import { NuQJob } from "./nuq";
+import { register } from "prom-client";
+import Express from "express";
+import { initializeBlocklist } from "../../scraper/WebScraper/utils/blocklist";
+import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-forcing";
+
+export type WorkerQueue = {
+ getJobToProcess(logger?: any): Promise<NuQJob<any, any> | null>;
+ renewLock(id: string, lock: string, logger?: any): Promise<boolean>;
+ jobFinish(
+ id: string,
+ lock: string,
+ returnvalue: any | null,
+ logger?: any,
+ ): Promise<boolean>;
+ jobFail(
+ id: string,
+ lock: string,
+ failedReason: string,
+ logger?: any,
+ ): Promise<boolean>;
+};
+
+async function withTimeout<T>(
+ promise: Promise<T>,
+ timeoutMs: number,
+ label: string,
+): Promise<T> {
+ let timer: NodeJS.Timeout | undefined;
+ try {
+ return await Promise.race([
+ promise,
+ new Promise<T>((_, reject) => {
+ timer = setTimeout(
+ () => reject(new Error(`${label} timed out after ${timeoutMs}ms`)),
+ timeoutMs,
+ );
+ }),
+ ]);
+ } finally {
+ if (timer) clearTimeout(timer);
+ }
+}
+
+export async function runNuqWorker(options: {
+ serviceName: string;
+ queue: WorkerQueue;
+ healthCheck: () => Promise<boolean>;
+ metrics?: () => string | Promise<string>;
+ beforeStart?: () => void | Promise<void>;
+ beforeShutdown?: () => void | Promise<void>;
+ shutdown?: () => void | Promise<void>;
+}) {
+ try {
+ await initializeBlocklist();
+ initializeEngineForcing();
+ await options.beforeStart?.();
+ } catch (error) {
+ _logger.error("Failed to initialize NuQ worker", {
+ module: options.serviceName,
+ error,
+ });
+ process.exit(1);
+ }
+
+ let isShuttingDown = false;
+
+ const app = Express();
+
+ app.get("/metrics", async (_, res) => {
+ const localMetrics = options.metrics ? await options.metrics() : "";
+ res
+ .contentType("text/plain")
+ .send(localMetrics + "\n" + (await register.metrics()));
+ });
+ app.get("/health", async (_, res) => {
+ try {
+ if (await withTimeout(options.healthCheck(), 1000, "NuQ health check")) {
+ res.status(200).send("OK");
+ } else {
+ res.status(500).send("Not OK");
+ }
+ } catch (error) {
+ _logger.warn("NuQ worker health check failed", {
+ module: options.serviceName,
+ error,
+ });
+ res.status(500).send("Not OK");
+ }
+ });
+
+ const server = app.listen(config.NUQ_WORKER_PORT, () => {
+ _logger.info("NuQ worker metrics server started", {
+ module: options.serviceName,
+ });
+ });
+
+ function shutdown() {
+ isShuttingDown = true;
+ }
+
+ process.on("SIGINT", shutdown);
+ process.on("SIGTERM", shutdown);
+
+ let noJobTimeout = 1500;
+
+ while (!isShuttingDown) {
+ const job = await options.queue.getJobToProcess();
+
+ if (job === null) {
+ _logger.info("No jobs to process", { module: "nuq/metrics" });
+ await new Promise(resolve => setTimeout(resolve, noJobTimeout));
+ if (!config.NUQ_RABBITMQ_URL) {
+ noJobTimeout = Math.min(noJobTimeout * 2, 10000);
+ }
+ continue;
+ }
+
+ noJobTimeout = 500;
+
+ const logger = _logger.child({
+ module: options.serviceName,
+ scrapeId: job.id,
+ zeroDataRetention: job.data?.zeroDataRetention ?? false,
+ });
+
+ logger.info("Acquired job");
+
+ const lockRenewInterval = setInterval(async () => {
+ try {
+ logger.info("Renewing lock");
+ if (!(await options.queue.renewLock(job.id, job.lock!, logger))) {
+ logger.warn("Failed to renew lock");
+ clearInterval(lockRenewInterval);
+ return;
+ }
+ logger.info("Renewed lock");
+ } catch (error) {
+ logger.warn("Failed to renew lock", { error });
+ clearInterval(lockRenewInterval);
+ }
+ }, 15000);
+
+ let processResult:
+ | { ok: true; data: Awaited<ReturnType<typeof processJobInternal>> }
+ | { ok: false; error: any };
+
+ const endJobTimer = jobDurationSeconds.startTimer({ type: job.data.mode });
+
+ try {
+ processResult = { ok: true, data: await processJobInternal(job) };
+ } catch (error) {
+ processResult = { ok: false, error };
+ }
+
+ clearInterval(lockRenewInterval);
+
+ if (processResult.ok) {
+ endJobTimer({ status: "success" });
+ if (
+ !(await options.queue.jobFinish(
+ job.id,
+ job.lock!,
+ processResult.data,
+ logger,
+ ))
+ ) {
+ logger.warn("Could not update job status");
+ }
+ } else {
+ endJobTimer({ status: "failed" });
+ if (
+ !(await options.queue.jobFail(
+ job.id,
+ job.lock!,
+ processResult.error instanceof Error
+ ? processResult.error.message
+ : typeof processResult.error === "string"
+ ? processResult.error
+ : JSON.stringify(processResult.error),
+ logger,
+ ))
+ ) {
+ logger.warn("Could not update job status");
+ }
+ }
+ }
+
+ _logger.info("NuQ worker shutting down", { module: options.serviceName });
+
+ server.close(async () => {
+ await options.beforeShutdown?.();
+ await options.shutdown?.();
+ _logger.info("NuQ worker shut down", { module: options.serviceName });
+ process.exit(0);
+ });
+}
diff --git a/apps/api/src/services/worker/nuq-worker.ts b/apps/api/src/services/worker/nuq-worker.ts
index 05d977188..984ba3f9d 100644
--- a/apps/api/src/services/worker/nuq-worker.ts
+++ b/apps/api/src/services/worker/nuq-worker.ts
@@ -1,177 +1,21 @@
import "dotenv/config";
-import { config } from "../../config";
import "../sentry";
import { setSentryServiceTag } from "../sentry";
-import { logger as _logger } from "../../lib/logger";
-import { processJobInternal } from "./scrape-worker";
import {
nuqGetLocalMetrics,
nuqHealthCheck,
- scrapeQueue as scrapeQueuePg,
+ scrapeQueue,
} from "./nuq";
-import { scrapeQueue, fdbQueueEnabled } from "./nuq-router";
-import { getNuqFdbSweeper, nuqFdbHealthCheck } from "./nuq-fdb";
-import { jobDurationSeconds } from "../../lib/job-metrics";
-import { register } from "prom-client";
-import Express from "express";
-import { _ } from "ajv";
-import { initializeBlocklist } from "../../scraper/WebScraper/utils/blocklist";
-import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-forcing";
+import { runNuqWorker } from "./nuq-worker-runner";
(async () => {
setSentryServiceTag("nuq-worker");
- try {
- await initializeBlocklist();
- initializeEngineForcing();
- } catch (error) {
- _logger.error("Failed to initialize blocklist and engine forcing", {
- error,
- });
- process.exit(1);
- }
-
- let isShuttingDown = false;
-
- const app = Express();
-
- app.get("/metrics", async (_, res) =>
- res
- .contentType("text/plain")
- .send(nuqGetLocalMetrics() + "\n" + (await register.metrics())),
- );
- app.get("/health", async (_, res) => {
- const pgHealthy = await nuqHealthCheck();
- const fdbHealthy =
- config.NUQ_BACKEND !== "fdb" || (await nuqFdbHealthCheck());
- if (pgHealthy && fdbHealthy) {
- res.status(200).send("OK");
- } else {
- res.status(500).send("Not OK");
- }
- });
-
- const server = app.listen(config.NUQ_WORKER_PORT, () => {
- _logger.info("NuQ worker metrics server started");
- });
-
- function shutdown() {
- isShuttingDown = true;
- }
-
- if (require.main === module) {
- process.on("SIGINT", shutdown);
- process.on("SIGTERM", shutdown);
- }
-
- let noJobTimeout = 1500;
-
- // the FDB backend has no pg_cron: one worker at a time holds the sweeper
- // lease and runs lease/timeout/group sweeps for everyone
- if (fdbQueueEnabled()) {
- try {
- getNuqFdbSweeper().start();
- } catch (error) {
- if (config.NUQ_BACKEND === "fdb") throw error;
- _logger.warn("Failed to start FDB sweeper, continuing with PG", {
- module: "nuq-worker",
- error,
- });
- }
- }
-
- while (!isShuttingDown) {
- const job = await scrapeQueue.getJobToProcess();
-
- if (job === null) {
- _logger.info("No jobs to process", { module: "nuq/metrics" });
- await new Promise(resolve => setTimeout(resolve, noJobTimeout));
- if (!config.NUQ_RABBITMQ_URL) {
- noJobTimeout = Math.min(noJobTimeout * 2, 10000);
- }
- continue;
- }
-
- noJobTimeout = 500;
-
- const logger = _logger.child({
- module: "nuq-worker",
- scrapeId: job.id,
- zeroDataRetention: job.data?.zeroDataRetention ?? false,
- });
-
- logger.info("Acquired job");
-
- const lockRenewInterval = setInterval(async () => {
- logger.info("Renewing lock");
- if (!(await scrapeQueue.renewLock(job.id, job.lock!, logger))) {
- logger.warn("Failed to renew lock");
- clearInterval(lockRenewInterval);
- return;
- }
- logger.info("Renewed lock");
- }, 15000);
-
- let processResult:
- | { ok: true; data: Awaited<ReturnType<typeof processJobInternal>> }
- | { ok: false; error: any };
-
- const endJobTimer = jobDurationSeconds.startTimer({ type: job.data.mode });
-
- try {
- processResult = { ok: true, data: await processJobInternal(job) };
- } catch (error) {
- processResult = { ok: false, error };
- }
-
- clearInterval(lockRenewInterval);
-
- if (processResult.ok) {
- endJobTimer({ status: "success" });
- if (
- !(await scrapeQueue.jobFinish(
- job.id,
- job.lock!,
- processResult.data,
- logger,
- ))
- ) {
- logger.warn("Could not update job status");
- }
- } else {
- endJobTimer({ status: "failed" });
- if (
- !(await scrapeQueue.jobFail(
- job.id,
- job.lock!,
- processResult.error instanceof Error
- ? processResult.error.message
- : typeof processResult.error === "string"
- ? processResult.error
- : JSON.stringify(processResult.error),
- logger,
- ))
- ) {
- logger.warn("Could not update job status");
- }
- }
- }
-
- _logger.info("NuQ worker shutting down");
-
- server.close(async () => {
- if (fdbQueueEnabled()) {
- try {
- getNuqFdbSweeper().stop();
- } catch (error) {
- _logger.warn("Failed to stop FDB sweeper", {
- module: "nuq-worker",
- error,
- });
- }
- }
- await scrapeQueuePg.shutdown();
- _logger.info("NuQ worker shut down");
- process.exit(0);
+ await runNuqWorker({
+ serviceName: "nuq-worker",
+ queue: scrapeQueue,
+ healthCheck: nuqHealthCheck,
+ metrics: nuqGetLocalMetrics,
+ shutdown: () => scrapeQueue.shutdown(),
});
})();