Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(extract): rabbitmq + poison pill handling (#2581)
Gergő Móricz11f51eda2f2c
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex b5fe8c083..6eb8a5c98 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -31,6 +31,10 @@ jobs:image: redisports:- 6379:6379+ rabbitmq:+ image: rabbitmq+ ports:+ - 5672:5672env:HOST: 0.0.0.0TEST_SUITE_SELF_HOSTED: true@@ -42,6 +46,7 @@ jobs:PROXY_USERNAME: ${{ matrix.proxy == 'proxy' && secrets.PROXY_USERNAME || '' }}PROXY_PASSWORD: ${{ matrix.proxy == 'proxy' && secrets.PROXY_PASSWORD || '' }}NUQ_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres+ NUQ_RABBITMQ_URL: amqp://localhost:5672USE_GO_MARKDOWN_PARSER: trueALLOW_LOCAL_WEBHOOKS: trueFIRECRAWL_LOG_TO_FILE: truediff --git a/apps/api/src/controllers/v1/extract-status.ts b/apps/api/src/controllers/v1/extract-status.tsindex c1c73e345..9a73737a7 100644--- a/apps/api/src/controllers/v1/extract-status.ts+++ b/apps/api/src/controllers/v1/extract-status.ts@@ -1,67 +1,29 @@import { Response } from "express";import { config } from "../../config";import { RequestWithAuth } from "./types";-import { getExtract, getExtractExpiry } from "../../lib/extract/extract-redis";-import { getExtractQueue } from "../../services/queue-service";-import { ExtractResult } from "../../lib/extract/extraction-service";+import {+ getExtract,+ getExtractExpiry,+ getExtractResult,+} from "../../lib/extract/extract-redis";import { supabaseGetExtractByIdDirect } from "../../lib/supabase-jobs";-import { JobState } from "bullmq";import { logger as _logger } from "../../lib/logger";--type DBExtract = {- id: string;- is_successful: boolean;- options: any;- created_at: any;- error: string | null;- team_id: string;-};--type ExtractPseudoJob<T> = {- id: string;- getState: () => Promise<JobState | "unknown"> | JobState | "unknown";- returnvalue: T | null;- timestamp: number;- data: {- scrapeOptions: any;- teamId?: string;- };- failedReason?: string;-};--async function getExtractJob(- id: string,-): Promise<ExtractPseudoJob<ExtractResult> | null> {- const [bullJob, dbExtract] = await Promise.all([- getExtractQueue().getJob(id),- (config.USE_DB_AUTHENTICATION- ? supabaseGetExtractByIdDirect(id)- : null) as Promise<DBExtract | null>,- ]);-- if (!bullJob && !dbExtract) return null;-- // Extract results are stored in GCS, not in the DB- const data = bullJob?.returnvalue?.data;-- const job: ExtractPseudoJob<any> = {- id,- getState: dbExtract- ? () => (dbExtract.is_successful ? "completed" : "failed")- : () => bullJob!.getState(),- returnvalue: data,- data: {- scrapeOptions: bullJob ? bullJob.data.scrapeOptions : dbExtract!.options,- teamId: bullJob ? bullJob.data.teamId : dbExtract!.team_id,- },- timestamp: bullJob- ? bullJob.timestamp- : new Date(dbExtract!.created_at).valueOf(),- failedReason:- (bullJob ? bullJob.failedReason : dbExtract!.error) || undefined,- };-- return job;+import { getJobFromGCS } from "../../lib/gcs-jobs";++async function getExtractData(id: string): Promise<any> {+ // Try GCS first if configured+ if (config.GCS_BUCKET_NAME) {+ const gcsData = await getJobFromGCS(id);+ if (gcsData) {+ return Array.isArray(gcsData) ? gcsData[0] : gcsData;+ }+ }+ // Fallback to Redis+ const redisData = await getExtractResult(id);+ if (redisData) {+ return Array.isArray(redisData) ? redisData[0] : redisData;+ }+ return [];}export async function extractStatusController(@@ -75,10 +37,10 @@ export async function extractStatusController(extractId: req.params.jobId,});+ // Get extract status from Redis (for in-progress jobs)const extract = await getExtract(req.params.jobId);- let status = extract?.status;-+ // Check team ownership from Redisif (extract && extract.team_id !== req.auth.team_id) {return res.status(404).json({success: false,@@ -86,49 +48,63 @@ export async function extractStatusController(});}- let data: ExtractResult | [] = [];-- if (!extract || extract.status === "completed") {- const jobData = await getExtractJob(req.params.jobId);- if (- (!jobData && !extract) ||- (jobData && jobData.data.teamId !== req.auth.team_id)- ) {- logger.warn("Extract job was not found");- return res.status(404).json({- success: false,- error: "Extract job not found",- });- }+ // If not in Redis, check the database for completed jobs+ if (!extract) {+ if (config.USE_DB_AUTHENTICATION) {+ const dbExtract = await supabaseGetExtractByIdDirect(req.params.jobId);+ if (!dbExtract) {+ logger.warn("Extract job was not found");+ return res.status(404).json({+ success: false,+ error: "Extract job not found",+ });+ }- if (jobData) {- const jobStatus = await jobData.getState();+ if (dbExtract.team_id !== req.auth.team_id) {+ return res.status(404).json({+ success: false,+ error: "Extract job not found",+ });+ }- if (jobStatus === "completed") {- status = "completed";- } else if (jobStatus === "failed") {- status = "failed";- } else {- status = "processing";+ let data: any = [];+ if (dbExtract.is_successful) {+ data = await getExtractData(req.params.jobId);}- }- if (!jobData?.returnvalue) {- // if we got in the split-second where the redis is updated but the bull isn't- // just pretend it's still processing - MG- status = "processing";- } else {- data = jobData.returnvalue ?? [];+ // Return DB-based status+ return res.status(200).json({+ success: dbExtract.is_successful,+ data,+ status: dbExtract.is_successful ? "completed" : "failed",+ error: dbExtract.error || undefined,+ expiresAt: new Date(+ new Date(dbExtract.created_at).getTime() + 1000 * 60 * 60 * 24,+ ).toISOString(),+ });}++ logger.warn("Extract job was not found");+ return res.status(404).json({+ success: false,+ error: "Extract job not found",+ });+ }++ // Get result data if completed+ let data: any = [];+ if (extract.status === "completed") {+ data = await getExtractData(req.params.jobId);}+ // Return Redis-based statusreturn res.status(200).json({- success: status === "failed" ? false : true,+ success: extract.status === "failed" ? false : true,data,- status,+ status: extract.status,error: (() => {- if (typeof extract?.error === "string") return extract.error;- if (extract?.error && typeof extract.error === "object") {+ if (typeof extract.error === "string") return extract.error;+ if (extract.error && typeof extract.error === "object") {return typeof extract.error.message === "string"? extract.error.message: typeof extract.error.error === "string"@@ -138,12 +114,12 @@ export async function extractStatusController(return undefined;})(),expiresAt: (await getExtractExpiry(req.params.jobId)).toISOString(),- steps: extract?.showSteps ? extract.steps : undefined,- llmUsage: extract?.showLLMUsage ? extract.llmUsage : undefined,- sources: extract?.showSources ? extract.sources : undefined,- costTracking: extract?.showCostTracking ? extract.costTracking : undefined,- sessionIds: extract?.sessionIds ? extract.sessionIds : undefined,- tokensUsed: extract?.tokensBilled ? extract.tokensBilled : undefined,- creditsUsed: extract?.creditsBilled ? extract.creditsBilled : undefined,+ steps: extract.showSteps ? extract.steps : undefined,+ llmUsage: extract.showLLMUsage ? extract.llmUsage : undefined,+ sources: extract.showSources ? extract.sources : undefined,+ costTracking: extract.showCostTracking ? extract.costTracking : undefined,+ sessionIds: extract.sessionIds ? extract.sessionIds : undefined,+ tokensUsed: extract.tokensBilled ? extract.tokensBilled : undefined,+ creditsUsed: extract.creditsBilled ? extract.creditsBilled : undefined,});}diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.tsindex f291699aa..9f909df53 100644--- a/apps/api/src/controllers/v1/extract.ts+++ b/apps/api/src/controllers/v1/extract.ts@@ -6,7 +6,7 @@ import {extractRequestSchema,ExtractResponse,} from "./types";-import { getExtractQueue } from "../../services/queue-service";+import { addExtractJobToQueue } from "../../services/queue-service";import { saveExtract } from "../../lib/extract/extract-redis";import { getTeamIdSyncB } from "../../lib/extract/team-id-sync";import {@@ -209,9 +209,7 @@ export async function extractController(zeroDataRetention: req.acuc?.flags?.forceZDR,});- await getExtractQueue().add(extractId, jobData, {- jobId: extractId,- });+ await addExtractJobToQueue(extractId, jobData);return res.status(200).json({success: true,diff --git a/apps/api/src/controllers/v2/extract-status.ts b/apps/api/src/controllers/v2/extract-status.tsindex 3d28dac5a..d13714104 100644--- a/apps/api/src/controllers/v2/extract-status.ts+++ b/apps/api/src/controllers/v2/extract-status.ts@@ -1,171 +1,146 @@import { Response } from "express";import { config } from "../../config";import { RequestWithAuth } from "./types";-import { getExtract, getExtractExpiry } from "../../lib/extract/extract-redis";-import { getExtractQueue } from "../../services/queue-service";-import { ExtractResult } from "../../lib/extract/extraction-service";+import {+ getExtract,+ getExtractExpiry,+ getExtractResult,+} from "../../lib/extract/extract-redis";import {supabaseGetAgentByIdDirect,supabaseGetExtractByIdDirect,supabaseGetExtractRequestByIdDirect,} from "../../lib/supabase-jobs";-import { JobState } from "bullmq";import { logger as _logger } from "../../lib/logger";import { getJobFromGCS } from "../../lib/gcs-jobs";-type DBExtract = {- id: string;- is_successful: boolean;- options: any;- created_at: any;- error: string | null;- team_id: string;-};--type ExtractPseudoJob<T> = {- id: string;- getState: () => Promise<JobState | "unknown"> | JobState | "unknown";- returnvalue: T | null;- timestamp: number;- data: {- scrapeOptions: any;- teamId?: string;- };- failedReason?: string;-};--async function getExtractJob(- id: string,-): Promise<ExtractPseudoJob<ExtractResult> | null> {- const [bullJob, gcsJob, dbExtract] = await Promise.all([- getExtractQueue().getJob(id),- (config.GCS_BUCKET_NAME ? getJobFromGCS(id) : null) as Promise<any | null>,- (config.USE_DB_AUTHENTICATION- ? supabaseGetExtractByIdDirect(id)- : null) as Promise<DBExtract | null>,- ]);-- if (!bullJob && !dbExtract) return null;-- // Extract results are stored in GCS, not in the DB- let data = gcsJob ?? bullJob?.returnvalue?.data;- if (gcsJob === null && data) {- _logger.warn("GCS Job not found", {- jobId: id,- });+async function getExtractData(id: string): Promise<any> {+ // Try GCS first if configured+ if (config.GCS_BUCKET_NAME) {+ const gcsData = await getJobFromGCS(id);+ if (gcsData) {+ return Array.isArray(gcsData) ? gcsData[0] : gcsData;+ }}- if (Array.isArray(data)) data = data[0];-- const job: ExtractPseudoJob<any> = {- id,- getState: bullJob- ? bullJob.getState.bind(bullJob)- : () => (dbExtract!.is_successful ? "completed" : "failed"),- returnvalue: data,- data: {- scrapeOptions: bullJob ? bullJob.data.scrapeOptions : dbExtract!.options,- teamId: bullJob ? bullJob.data.teamId : dbExtract!.team_id,- },- timestamp: bullJob- ? bullJob.timestamp- : new Date(dbExtract!.created_at).valueOf(),- failedReason:- (bullJob ? bullJob.failedReason : dbExtract!.error) || undefined,- };-- return job;+ // Fallback to Redis+ const redisData = await getExtractResult(id);+ if (redisData) {+ return Array.isArray(redisData) ? redisData[0] : redisData;+ }+ return [];}export async function extractStatusController(req: RequestWithAuth<{ jobId: string }, any, any>,res: Response,) {- const extractRequest = await supabaseGetExtractRequestByIdDirect(- req.params.jobId,- );+ const extractRequest = config.USE_DB_AUTHENTICATION+ ? await supabaseGetExtractRequestByIdDirect(req.params.jobId)+ : null;+ if (config.USE_DB_AUTHENTICATION) {+ if (!extractRequest || extractRequest.team_id !== req.auth.team_id) {+ return res.status(404).json({+ success: false,+ error: "Extract job not found",+ });+ }- if (!extractRequest || extractRequest.team_id !== req.auth.team_id) {- return res.status(404).json({- success: false,- error: "Extract job not found",- });+ if (extractRequest.kind === "agent") {+ const agent = await supabaseGetAgentByIdDirect(req.params.jobId);++ let data: any = undefined;+ if (agent?.is_successful) {+ data = await getJobFromGCS(agent.id);+ }++ return res.status(200).json({+ success: true,+ status: !agent+ ? "processing"+ : agent.is_successful+ ? "completed"+ : "failed",+ error: agent?.error || undefined,+ data,+ expiresAt: new Date(+ new Date(agent?.created_at ?? extractRequest.created_at).getTime() ++ 1000 * 60 * 60 * 24,+ ).toISOString(),+ creditsUsed: agent?.credits_cost,+ });+ }}- if (extractRequest.kind === "agent") {- const agent = await supabaseGetAgentByIdDirect(req.params.jobId);+ // Get extract status from Redis (for in-progress jobs)+ const redisExtract = await getExtract(req.params.jobId);- let data: any = undefined;- if (agent?.is_successful) {- data = await getJobFromGCS(agent.id);+ // If not in Redis, check the database for completed jobs+ if (!redisExtract) {+ if (config.USE_DB_AUTHENTICATION) {+ const dbExtract = await supabaseGetExtractByIdDirect(req.params.jobId);+ if (dbExtract) {+ // Get result data+ let data: any = [];+ if (dbExtract.is_successful) {+ data = await getExtractData(req.params.jobId);+ }++ return res.status(200).json({+ success: dbExtract.is_successful,+ data,+ status: dbExtract.is_successful ? "completed" : "failed",+ error: dbExtract.error || undefined,+ expiresAt: new Date(+ new Date(dbExtract.created_at).getTime() + 1000 * 60 * 60 * 24,+ ).toISOString(),+ });+ }}+ // Fall back to extractRequest inforeturn res.status(200).json({success: true,- status: !agent- ? "processing"- : agent.is_successful- ? "completed"- : "failed",- error: agent?.error || undefined,- data,+ data: [],+ status: "processing",expiresAt: new Date(- new Date(agent?.created_at ?? extractRequest.created_at).getTime() +- 1000 * 60 * 60 * 24,+ new Date(extractRequest.created_at).getTime() + 1000 * 60 * 60 * 24,).toISOString(),- creditsUsed: agent?.credits_cost,});}- let data: ExtractResult | [] = [];- let status: string = "processing";-- const redisExtract = await getExtract(req.params.jobId);- const jobData = await getExtractJob(req.params.jobId);-- console.log("jobData", jobData);-- if (jobData) {- const jobStatus = await jobData.getState();-- if (jobStatus === "completed") {- status = "completed";- } else if (jobStatus === "failed") {- status = "failed";- } else {- status = "processing";- }- }-- if (!jobData?.returnvalue) {- // if we got in the split-second where the redis is updated but the bull isn't- // just pretend it's still processing - MG- status = "processing";- } else {- data = jobData.returnvalue ?? [];+ // Get result data if completed+ let data: any = [];+ if (redisExtract.status === "completed") {+ data = await getExtractData(req.params.jobId);}return res.status(200).json({- success: status === "failed" ? false : true,+ success: redisExtract.status === "failed" ? false : true,data,- status,- error: jobData?.failedReason,- expiresAt: redisExtract- ? (await getExtractExpiry(req.params.jobId)).toISOString()- : new Date(- new Date(jobData?.timestamp ?? extractRequest.created_at).getTime() +- 1000 * 60 * 60 * 24,- ).toISOString(),- steps: redisExtract?.showSteps ? redisExtract.steps : undefined,- llmUsage: redisExtract?.showLLMUsage ? redisExtract.llmUsage : undefined,- sources: redisExtract?.showSources ? redisExtract.sources : undefined,- costTracking: redisExtract?.showCostTracking+ status: redisExtract.status,+ error: (() => {+ if (typeof redisExtract.error === "string") return redisExtract.error;+ if (redisExtract.error && typeof redisExtract.error === "object") {+ return typeof redisExtract.error.message === "string"+ ? redisExtract.error.message+ : typeof redisExtract.error.error === "string"+ ? redisExtract.error.error+ : JSON.stringify(redisExtract.error);+ }+ return undefined;+ })(),+ expiresAt: (await getExtractExpiry(req.params.jobId)).toISOString(),+ steps: redisExtract.showSteps ? redisExtract.steps : undefined,+ llmUsage: redisExtract.showLLMUsage ? redisExtract.llmUsage : undefined,+ sources: redisExtract.showSources ? redisExtract.sources : undefined,+ costTracking: redisExtract.showCostTracking? redisExtract.costTracking: undefined,- sessionIds: redisExtract?.sessionIds ? redisExtract.sessionIds : undefined,- tokensUsed: redisExtract?.tokensBilled+ sessionIds: redisExtract.sessionIds ? redisExtract.sessionIds : undefined,+ tokensUsed: redisExtract.tokensBilled? redisExtract.tokensBilled: undefined,- creditsUsed: redisExtract?.creditsBilled+ creditsUsed: redisExtract.creditsBilled? redisExtract.creditsBilled: undefined,});diff --git a/apps/api/src/controllers/v2/extract.ts b/apps/api/src/controllers/v2/extract.tsindex d2ee361ee..dccb34bdf 100644--- a/apps/api/src/controllers/v2/extract.ts+++ b/apps/api/src/controllers/v2/extract.ts@@ -6,7 +6,7 @@ import {extractRequestSchema,ExtractResponse,} from "./types";-import { getExtractQueue } from "../../services/queue-service";+import { addExtractJobToQueue } from "../../services/queue-service";import { saveExtract } from "../../lib/extract/extract-redis";import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";@@ -158,8 +158,9 @@ export async function extractController(zeroDataRetention: req.acuc?.flags?.forceZDR,});- await getExtractQueue().add(extractId, jobData, {- jobId: extractId,+ await addExtractJobToQueue(extractId, {+ ...jobData,+ apiKeyId: req.acuc?.api_key_id ?? undefined,});return res.status(200).json({diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.tsindex ccfbed79f..71dda3794 100644--- a/apps/api/src/harness.ts+++ b/apps/api/src/harness.ts@@ -14,6 +14,10 @@ let nuqPostgresContainer: {containerName: string;containerRuntime: string;} | null = null;+let nuqRabbitMQContainer: {+ containerName: string;+ containerRuntime: string;+} | null = null;// Get the monorepo root (apps/api/dist/src -> ../../../..)// __dirname is available in CommonJS (which this compiles to)@@ -37,6 +41,10 @@ interface Services {containerName: string;containerRuntime: string;};+ nuqRabbitMQ?: {+ containerName: string;+ containerRuntime: string;+ };}const colors = {@@ -558,6 +566,106 @@ async function setupNuqPostgres(): Promise<Services["nuqPostgres"]> {return containerInfo;}+async function startNuqRabbitMQContainer(+ runtime: string,+ containerName: string,+): Promise<void> {+ logger.info(`Starting RabbitMQ container: ${containerName}`);+ const start = execForward(+ `${runtime}@start`,+ `${runtime} run -d --name ${containerName} -p 5672:5672 -p 15672:15672 rabbitmq:3-management`,+ );+ await start.promise;+ logger.success(`RabbitMQ container started: ${containerName}`);+}++async function waitForRabbitMQ(+ host: string,+ port: number,+ timeoutMs: number = config.HARNESS_STARTUP_TIMEOUT_MS,+): Promise<void> {+ logger.info("Waiting for RabbitMQ to be ready...");+ const start = Date.now();++ while (Date.now() - start < timeoutMs) {+ try {+ // Try to connect to RabbitMQ using amqplib+ const amqp = await import("amqplib");+ const connection = await amqp.connect(`amqp://${host}:${port}`, {+ timeout: 2000,+ });+ await connection.close();++ logger.success("RabbitMQ is ready");+ return;+ } catch (e) {+ // Not ready yet, wait and retry+ await new Promise(resolve => setTimeout(resolve, 500));+ }+ }++ throw new Error(`RabbitMQ did not become ready within ${timeoutMs}ms`);+}++async function setupNuqRabbitMQ(): Promise<Services["nuqRabbitMQ"]> {+ // If NUQ_RABBITMQ_URL is already set, respect it (user's explicit choice)+ if (config.NUQ_RABBITMQ_URL) {+ logger.info("NUQ_RABBITMQ_URL is set, skipping container management");+ return undefined;+ }++ // Check if we're running in docker-compose (POSTGRES_HOST is set and not localhost)+ const isDockerCompose = POSTGRES_HOST !== "localhost";++ if (isDockerCompose) {+ // Running in docker-compose: RabbitMQ should be configured externally+ logger.info(+ "Running in docker-compose, skipping RabbitMQ container management",+ );+ return undefined;+ }++ // Running locally: manage container+ logger.section("Setting up NUQ RabbitMQ container");++ const runtime = await detectContainerRuntime();+ if (!runtime) {+ throw new Error(+ "Neither Docker nor Podman found. Please install Docker/Podman or set NUQ_RABBITMQ_URL manually.",+ );+ }++ logger.success(`Using container runtime: ${runtime}`);++ const containerName = "firecrawl-nuq-rabbitmq";++ // Stop and remove any existing container+ await stopAndRemoveContainer(runtime, containerName);++ // Start the container (using stock image, no build needed)+ await startNuqRabbitMQContainer(runtime, containerName);++ // Wait for RabbitMQ to be ready+ await waitForRabbitMQ("localhost", 5672);++ // Set environment variables for the services+ const rabbitUrl = "amqp://localhost:5672";+ config.NUQ_RABBITMQ_URL = rabbitUrl;+ process.env.NUQ_RABBITMQ_URL = rabbitUrl;++ logger.success("NUQ RabbitMQ container is ready");++ const containerInfo = {+ containerName,+ containerRuntime: runtime,+ };++ // Store globally for graceful shutdown+ nuqRabbitMQContainer = containerInfo;++ return containerInfo;+}+async function installDependencies() {logger.section("Installing dependencies");@@ -604,6 +712,9 @@ async function startServices(command?: string[]): Promise<Services> {// Setup NUQ PostgreSQL container if neededconst nuqPostgres = await setupNuqPostgres();+ // Setup NUQ RabbitMQ container if needed+ const nuqRabbitMQ = await setupNuqRabbitMQ();+logger.section("Starting services");const api = execForward(@@ -655,20 +766,18 @@ async function startServices(command?: string[]): Promise<Services> {),);- const nuqPrefetchWorker = config.NUQ_RABBITMQ_URL- ? execForward(- "nuq-prefetch-worker",- process.argv[2] === "--start-docker"- ? "node dist/src/services/worker/nuq-prefetch-worker.js"- : "pnpm nuq-prefetch-worker:production",- {- NUQ_PREFETCH_WORKER_PORT: String(NUQ_PREFETCH_WORKER_PORT),- NUQ_REDUCE_NOISE: "true",- NUQ_POD_NAME: "nuq-prefetch-worker-0",- NUQ_PREFETCH_REPLICAS: String(1),- },- )- : undefined;+ const nuqPrefetchWorker = execForward(+ "nuq-prefetch-worker",+ process.argv[2] === "--start-docker"+ ? "node dist/src/services/worker/nuq-prefetch-worker.js"+ : "pnpm nuq-prefetch-worker:production",+ {+ NUQ_PREFETCH_WORKER_PORT: String(NUQ_PREFETCH_WORKER_PORT),+ NUQ_REDUCE_NOISE: "true",+ NUQ_POD_NAME: "nuq-prefetch-worker-0",+ NUQ_PREFETCH_REPLICAS: String(1),+ },+ );const indexWorker = config.USE_DB_AUTHENTICATION? execForward(@@ -708,6 +817,7 @@ async function startServices(command?: string[]): Promise<Services> {extractWorker,command: commandProcess,nuqPostgres,+ nuqRabbitMQ,};}@@ -741,6 +851,16 @@ async function stopDevelopmentServices(services: Services) {);logger.success("NUQ PostgreSQL container stopped");}++ // Stop and remove NUQ RabbitMQ container if it was started by harness+ if (services.nuqRabbitMQ) {+ logger.info("Stopping NUQ RabbitMQ container");+ await stopAndRemoveContainer(+ services.nuqRabbitMQ.containerRuntime,+ services.nuqRabbitMQ.containerName,+ );+ logger.success("NUQ RabbitMQ container stopped");+ }}async function runDevMode(): Promise<void> {@@ -904,6 +1024,17 @@ async function gracefulShutdown() {nuqPostgresContainer = null;}+ // Stop and remove NUQ RabbitMQ container if it was started by harness+ if (nuqRabbitMQContainer) {+ logger.info("Stopping NUQ RabbitMQ container");+ await stopAndRemoveContainer(+ nuqRabbitMQContainer.containerRuntime,+ nuqRabbitMQContainer.containerName,+ );+ logger.success("NUQ RabbitMQ container stopped");+ nuqRabbitMQContainer = null;+ }+logger.success("All processes terminated");}diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex a30ca811c..57450a5a5 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -7,7 +7,6 @@ import express, { NextFunction, Request, Response } from "express";import bodyParser from "body-parser";import cors from "cors";import {- getExtractQueue,getGenerateLlmsTxtQueue,getDeepResearchQueue,getBillingQueue,@@ -81,7 +80,6 @@ serverAdapter.setBasePath(`/admin/${config.BULL_AUTH_KEY}/queues`);const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({queues: [- new BullMQAdapter(getExtractQueue()),new BullMQAdapter(getGenerateLlmsTxtQueue()),new BullMQAdapter(getDeepResearchQueue()),new BullMQAdapter(getBillingQueue()),diff --git a/apps/api/src/lib/extract/extract-redis.ts b/apps/api/src/lib/extract/extract-redis.tsindex 0ab06ab0d..28a3faa34 100644--- a/apps/api/src/lib/extract/extract-redis.ts+++ b/apps/api/src/lib/extract/extract-redis.ts@@ -144,3 +144,24 @@ export async function getExtractExpiry(id: string): Promise<Date> {d.setMilliseconds(0);return d;}++// Result data storage (fallback when GCS is not configured)+const EXTRACT_RESULT_TTL = 24 * 60 * 60; // 24 hours++export async function saveExtractResult(+ id: string,+ result: any,+): Promise<void> {+ _logger.debug("Saving extract result " + id + " to Redis...");+ await redisEvictConnection.set(+ "extract_result:" + id,+ JSON.stringify(result),+ "EX",+ EXTRACT_RESULT_TTL,+ );+}++export async function getExtractResult(id: string): Promise<any | null> {+ const x = await redisEvictConnection.get("extract_result:" + id);+ return x ? JSON.parse(x) : null;+}diff --git a/apps/api/src/services/extract-queue.ts b/apps/api/src/services/extract-queue.tsnew file mode 100644index 000000000..f7c518a3c--- /dev/null+++ b/apps/api/src/services/extract-queue.ts@@ -0,0 +1,165 @@+import amqp from "amqplib";+import { config } from "../config";+import { logger as _logger } from "../lib/logger";++const EXTRACT_QUEUE = "extract.jobs";+const EXTRACT_DLX = "extract.dlx";+const EXTRACT_DLQ = "extract.dlq";++export type ExtractJobData = {+ extractId: string;+ request: any;+ teamId: string;+ subId?: string | null;+ apiKeyId?: number | null;+ agent?: any;+ createdAt: number;+};++let connection: amqp.ChannelModel | null = null;+let channel: amqp.Channel | null = null;++async function getChannel(): Promise<amqp.Channel> {+ if (channel) return channel;++ const url = config.NUQ_RABBITMQ_URL;+ if (!url) {+ throw new Error("NUQ_RABBITMQ_URL is not configured");+ }++ connection = await amqp.connect(url);+ channel = await connection.createChannel();++ // Set up the dead letter exchange+ await channel.assertExchange(EXTRACT_DLX, "direct", { durable: true });++ // Set up the dead letter queue+ await channel.assertQueue(EXTRACT_DLQ, {+ durable: true,+ arguments: {+ "x-queue-type": "quorum",+ },+ });+ await channel.bindQueue(EXTRACT_DLQ, EXTRACT_DLX, EXTRACT_QUEUE);++ // Set up the main queue with DLX - no retries (messages go straight to DLQ on reject/crash)+ await channel.assertQueue(EXTRACT_QUEUE, {+ durable: true,+ arguments: {+ "x-queue-type": "quorum",+ "x-dead-letter-exchange": EXTRACT_DLX,+ "x-dead-letter-routing-key": EXTRACT_QUEUE,+ "x-delivery-limit": 1,+ },+ });++ connection.on("close", () => {+ _logger.warn("Extract queue connection closed");+ connection = null;+ channel = null;+ });++ connection.on("error", err => {+ _logger.error("Extract queue connection error", { error: err });+ });++ return channel;+}++export async function addExtractJob(+ extractId: string,+ data: ExtractJobData,+): Promise<void> {+ const ch = await getChannel();+ ch.sendToQueue(EXTRACT_QUEUE, Buffer.from(JSON.stringify(data)), {+ persistent: true,+ messageId: extractId,+ });+ _logger.info("Extract job added to queue", { extractId });+}++export async function consumeExtractJobs(+ handler: (+ data: ExtractJobData,+ ack: () => void,+ nack: () => void,+ ) => Promise<void>,+): Promise<void> {+ const ch = await getChannel();+ await ch.prefetch(1);++ await ch.consume(+ EXTRACT_QUEUE,+ async msg => {+ if (!msg) return;++ const data = JSON.parse(msg.content.toString()) as ExtractJobData;+ const logger = _logger.child({+ module: "extract-queue",+ extractId: data.extractId,+ });++ logger.info("Processing extract job");++ try {+ await handler(+ data,+ () => ch.ack(msg),+ () => ch.nack(msg, false, false), // Don't requeue - send to DLX+ );+ } catch (error) {+ logger.error("Extract job handler threw an error", { error });+ // Don't requeue - send to DLX+ ch.nack(msg, false, false);+ }+ },+ { noAck: false },+ );++ _logger.info("Started consuming extract jobs");+}++export async function consumeExtractDLQ(+ handler: (data: ExtractJobData) => Promise<void>,+): Promise<void> {+ const ch = await getChannel();+ await ch.prefetch(1);++ await ch.consume(+ EXTRACT_DLQ,+ async msg => {+ if (!msg) return;++ const data = JSON.parse(msg.content.toString()) as ExtractJobData;+ const logger = _logger.child({+ module: "extract-dlq",+ extractId: data.extractId,+ });++ logger.info("Processing dead-lettered extract job");++ try {+ await handler(data);+ ch.ack(msg);+ } catch (error) {+ logger.error("DLQ handler threw an error, requeueing", { error });+ // Requeue DLQ messages on error so we don't lose them+ ch.nack(msg, false, true);+ }+ },+ { noAck: false },+ );++ _logger.info("Started consuming extract DLQ");+}++export async function shutdownExtractQueue(): Promise<void> {+ if (channel) {+ await channel.close();+ channel = null;+ }+ if (connection) {+ await connection.close();+ connection = null;+ }+}diff --git a/apps/api/src/services/extract-worker.ts b/apps/api/src/services/extract-worker.tsindex 19b2f31fc..5013457fe 100644--- a/apps/api/src/services/extract-worker.ts+++ b/apps/api/src/services/extract-worker.ts@@ -3,11 +3,7 @@ import { config } from "../config";import "./sentry";import { setSentryServiceTag } from "./sentry";import * as Sentry from "@sentry/node";-import { getExtractQueue, getRedisConnection } from "./queue-service";-import { Job, Queue, Worker } from "bullmq";import { logger as _logger } from "../lib/logger";-import systemMonitor from "./system-monitor";-import { v7 as uuidv7 } from "uuid";import { configDotenv } from "dotenv";import {ExtractResult,@@ -17,46 +13,36 @@ import { updateExtract } from "../lib/extract/extract-redis";import { performExtraction_F0 } from "../lib/extract/fire-0/extraction-service-f0";import { createWebhookSender, WebhookEvent } from "./webhook";import Express from "express";-import { robustFetch } from "../scraper/scrapeURL/lib/fetch";import { getErrorContactMessage } from "../lib/deployment";import { TransportableError } from "../lib/error";import { initializeBlocklist } from "../scraper/WebScraper/utils/blocklist";import { initializeEngineForcing } from "../scraper/WebScraper/utils/engine-forcing";+import {+ consumeExtractJobs,+ consumeExtractDLQ,+ shutdownExtractQueue,+ ExtractJobData,+} from "./extract-queue";+import { logExtract } from "./logging/log_job";configDotenv();-const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));--const jobLockExtendInterval = config.JOB_LOCK_EXTEND_INTERVAL;-const jobLockExtensionTime = config.JOB_LOCK_EXTENSION_TIME;--const cantAcceptConnectionInterval = config.CANT_ACCEPT_CONNECTION_INTERVAL;-const connectionMonitorInterval = config.CONNECTION_MONITOR_INTERVAL;-const gotJobInterval = config.CONNECTION_MONITOR_INTERVAL;--const runningJobs: Set<string> = new Set();--const processExtractJobInternal = async (- token: string,- job: Job & { id: string },+const processExtractJob = async (+ data: ExtractJobData,+ ack: () => void,+ nack: () => void,) => {const logger = _logger.child({module: "extract-worker",- method: "processJobInternal",- jobId: job.id,- extractId: job.data.extractId,- teamId: job.data?.teamId ?? undefined,+ method: "processExtractJob",+ extractId: data.extractId,+ teamId: data.teamId,});- const extendLockInterval = setInterval(async () => {- logger.info(`🔄 Worker extending lock on job ${job.id}`);- await job.extendLock(token, jobLockExtensionTime);- }, jobLockExtendInterval);-const sender = await createWebhookSender({- teamId: job.data.teamId,- jobId: job.data.extractId,- webhook: job.data.request.webhook,+ teamId: data.teamId,+ jobId: data.extractId,+ webhook: data.request.webhook,v0: false,});@@ -69,34 +55,25 @@ const processExtractJobInternal = async (let result: ExtractResult | null = null;- const model = job.data.request.agent?.model;- if (- job.data.request.agent &&- model &&- model.toLowerCase().includes("fire-1")- ) {- result = await performExtraction(job.data.extractId, {- request: job.data.request,- teamId: job.data.teamId,- subId: job.data.subId,- apiKeyId: job.data.apiKeyId,+ const model = data.request.agent?.model;+ if (data.request.agent && model && model.toLowerCase().includes("fire-1")) {+ result = await performExtraction(data.extractId, {+ request: data.request,+ teamId: data.teamId,+ subId: data.subId ?? undefined,+ apiKeyId: data.apiKeyId ?? null,});} else {- result = await performExtraction_F0(job.data.extractId, {- request: job.data.request,- teamId: job.data.teamId,- subId: job.data.subId,- apiKeyId: job.data.apiKeyId,+ result = await performExtraction_F0(data.extractId, {+ request: data.request,+ teamId: data.teamId,+ subId: data.subId ?? undefined,+ apiKeyId: data.apiKeyId ?? null,});}- // result = await performExtraction_F0(job.data.extractId, {- // request: job.data.request,- // teamId: job.data.teamId,- // subId: job.data.subId,- // });if (result && result.success) {- await updateExtract(job.data.extractId, {+ await updateExtract(data.extractId, {status: "completed",llmUsage: result.llmUsage,sources: result.sources,@@ -104,8 +81,6 @@ const processExtractJobInternal = async (creditsBilled: result.creditsBilled,});- await job.moveToCompleted(result, token, false);-if (sender) {sender.send(WebhookEvent.EXTRACT_COMPLETED, {success: true,@@ -113,192 +88,142 @@ const processExtractJobInternal = async (});}- return result;+ ack();+ return;} else {- await updateExtract(job.data.extractId, {+ await updateExtract(data.extractId, {status: "failed",- error: result?.error ?? getErrorContactMessage(job.data.extractId),+ error: result?.error ?? getErrorContactMessage(data.extractId),});- await job.moveToCompleted(result, token, false);-if (sender) {sender.send(WebhookEvent.EXTRACT_FAILED, {success: false,- error: result?.error ?? getErrorContactMessage(job.data.extractId),+ error: result?.error ?? getErrorContactMessage(data.extractId),});}- return result;+ ack();+ return;}} catch (error) {- logger.error(`🚫 Job errored ${job.id} - ${error}`, { error });+ logger.error(`🚫 Extract job errored ${data.extractId} - ${error}`, {+ error,+ });// Filter out TransportableErrors (flow control)if (!(error instanceof TransportableError)) {Sentry.captureException(error, {data: {- job: job.id,+ extractId: data.extractId,},});}- await updateExtract(job.data.extractId, {+ await updateExtract(data.extractId, {status: "failed",- error: error.error ?? error ?? getErrorContactMessage(job.data.extractId),+ error:+ (error as any)?.error ??+ (error as any)?.message ??+ getErrorContactMessage(data.extractId),});- try {- await job.moveToFailed(error, token, false);- } catch (e) {- logger.log("Failed to move job to failed state in BullMQ", { error });- }-if (sender) {sender.send(WebhookEvent.EXTRACT_FAILED, {success: false,error:- (error as any)?.message ?? getErrorContactMessage(job.data.extractId),+ (error as any)?.message ?? getErrorContactMessage(data.extractId),});}- return {- success: false,- error: error.error ?? error ?? getErrorContactMessage(job.data.extractId),- };- // throw error;- } finally {- clearInterval(extendLockInterval);+ // Ack the message even on error - we've handled it and updated the DB+ // Only nack if we want to send to DLX (which we don't for handled errors)+ ack();}};-let isShuttingDown = false;-let isWorkerStalled = false;--if (require.main === module) {- process.on("SIGINT", () => {- _logger.debug("Received SIGINT. Shutting down gracefully...");- isShuttingDown = true;- });-- process.on("SIGTERM", () => {- _logger.debug("Received SIGTERM. Shutting down gracefully...");- isShuttingDown = true;+const processDLQJob = async (data: ExtractJobData) => {+ const logger = _logger.child({+ module: "extract-dlq",+ extractId: data.extractId,+ teamId: data.teamId,});-}-let cantAcceptConnectionCount = 0;--const workerFun = async (- queue: Queue,- processJobInternal: (token: string, job: Job) => Promise<any>,-) => {- const logger = _logger.child({ module: "queue-worker", method: "workerFun" });+ logger.error("Processing crashed extract job from DLQ");- const worker = new Worker(queue.name, null, {- connection: getRedisConnection(),- lockDuration: 60 * 1000, // 60 seconds- stalledInterval: 60 * 1000, // 60 seconds- maxStalledCount: 10, // 10 times+ // Update the extract status to failed in the database+ await updateExtract(data.extractId, {+ status: "failed",+ error:+ "Extract job crashed unexpectedly. Please try again or contact support if the issue persists.",});- worker.startStalledCheckTimer();-- const monitor = await systemMonitor;-- while (true) {- if (isShuttingDown) {- _logger.info("No longer accepting new jobs. SIGINT");- break;- }- const token = uuidv7();- const canAcceptConnection = await monitor.acceptConnection();- if (!canAcceptConnection) {- console.log("Can't accept connection due to RAM/CPU load");- logger.info("Can't accept connection due to RAM/CPU load");- cantAcceptConnectionCount++;-- isWorkerStalled = cantAcceptConnectionCount >= 25;-- if (isWorkerStalled) {- logger.error("WORKER STALLED", {- cpuUsage: await monitor.checkCpuUsage(),- memoryUsage: await monitor.checkMemoryUsage(),- });- }-- await sleep(cantAcceptConnectionInterval); // more sleep- continue;- } else if (!currentLiveness) {- logger.info("Not accepting jobs because the liveness check failed");-- await sleep(cantAcceptConnectionInterval);- continue;- } else {- cantAcceptConnectionCount = 0;- }-- const job = await worker.getNextJob(token);- if (job) {- if (job.id) {- runningJobs.add(job.id);- }+ await logExtract({+ id: data.extractId,+ request_id: data.extractId,+ urls: data.request.urls,+ team_id: data.teamId,+ options: data.request,+ model_kind: data.request.agent?.model ?? "fire-0",+ credits_cost: 0,+ is_successful: false,+ error:+ "Extract job crashed unexpectedly. Please try again or contact support if the issue persists.",+ cost_tracking: undefined,+ });- processJobInternal(token, job).finally(() => {- if (job.id) {- runningJobs.delete(job.id);- }- });+ // Send webhook notification+ const sender = await createWebhookSender({+ teamId: data.teamId,+ jobId: data.extractId,+ webhook: data.request.webhook,+ v0: false,+ });- await sleep(gotJobInterval);- } else {- await sleep(connectionMonitorInterval);- }+ if (sender) {+ sender.send(WebhookEvent.EXTRACT_FAILED, {+ success: false,+ error:+ "Extract job crashed unexpectedly. Please try again or contact support if the issue persists.",+ });}++ logger.info("DLQ job processed - extract marked as failed");};-// Start all workers+// Start the workerconst app = Express();-let currentLiveness: boolean = true;--app.get("/liveness", (req, res) => {- _logger.info("Liveness endpoint hit");- if (config.USE_DB_AUTHENTICATION) {- // networking check for Kubernetes environments- const host = config.FIRECRAWL_APP_HOST;- const port = config.FIRECRAWL_APP_PORT;- const scheme = config.FIRECRAWL_APP_SCHEME;-- robustFetch({- url: `${scheme}://${host}:${port}`,- method: "GET",- mock: null,- logger: _logger,- abort: AbortSignal.timeout(5000),- ignoreResponse: true,- useCacheableLookup: false,- })- .then(() => {- currentLiveness = true;- res.status(200).json({ ok: true });- })- .catch(e => {- _logger.error("WORKER NETWORKING CHECK FAILED", { error: e });- currentLiveness = false;- res.status(500).json({ ok: false });- });- } else {- currentLiveness = true;- res.status(200).json({ ok: true });- }+app.get("/health", (req, res) => {+ res.status(200).json({ ok: true });});const workerPort = config.EXTRACT_WORKER_PORT || config.PORT;app.listen(workerPort, () => {- _logger.info(`Liveness endpoint is running on port ${workerPort}`);+ _logger.info(+ `Extract worker health endpoint is running on port ${workerPort}`,+ );});+async function shutdown() {+ _logger.info("Shutting down extract worker...");+ await shutdownExtractQueue();+ _logger.info("Extract worker shut down");+ process.exit(0);+}++if (require.main === module) {+ process.on("SIGINT", () => {+ _logger.debug("Received SIGINT. Shutting down gracefully...");+ shutdown();+ });++ process.on("SIGTERM", () => {+ _logger.debug("Received SIGTERM. Shutting down gracefully...");+ shutdown();+ });+}+(async () => {setSentryServiceTag("extract-worker");@@ -309,14 +234,13 @@ app.listen(workerPort, () => {initializeEngineForcing();- await Promise.all([workerFun(getExtractQueue(), processExtractJobInternal)]);+ _logger.info("Starting extract worker with RabbitMQ...");- _logger.info("All workers exited. Waiting for all jobs to finish...");+ // Start consuming from both the main queue and the DLQ+ await Promise.all([+ consumeExtractJobs(processExtractJob),+ consumeExtractDLQ(processDLQJob),+ ]);- while (runningJobs.size > 0) {- await new Promise(resolve => setTimeout(resolve, 500));- }-- _logger.info("All jobs finished. Shutting down...");- process.exit(0);+ _logger.info("Extract worker started, consuming from RabbitMQ");})();diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex 7d26584ce..02a5325ff 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -16,6 +16,7 @@ import { hasFormatOfType } from "../../lib/format-utils";import type { Document, ScrapeOptions } from "../../controllers/v2/types";import type { CostTracking } from "../../lib/cost-tracking";import type { Logger } from "winston";+import { saveExtractResult } from "../../lib/extract/extract-redis";configDotenv();const previewTeamId = "3adefd26-77ec-5968-8dcf-c94b5630d1de";@@ -490,7 +491,12 @@ export async function logExtract();if (extract.result) {- await saveExtractToGCS(extract);+ if (config.GCS_BUCKET_NAME) {+ await saveExtractToGCS(extract);+ } else {+ // Fallback: save result to Redis with 24h TTL when GCS is not configured+ await saveExtractResult(extract.id, extract.result);+ }}}diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.tsindex 465ec6761..04dc0b263 100644--- a/apps/api/src/services/queue-service.ts+++ b/apps/api/src/services/queue-service.ts@@ -3,8 +3,8 @@ import { config } from "../config";import { logger } from "../lib/logger";import IORedis from "ioredis";import type { DeepResearchServiceOptions } from "../lib/deep-research/deep-research-service";+import { addExtractJob, ExtractJobData } from "./extract-queue";-let extractQueue: Queue;let loggingQueue: Queue;let indexQueue: Queue;let deepResearchQueue: Queue;@@ -25,27 +25,16 @@ export function getRedisConnection(): IORedis {return redisConnection;}-const extractQueueName = "{extractQueue}";const generateLlmsTxtQueueName = "{generateLlmsTxtQueue}";const deepResearchQueueName = "{deepResearchQueue}";const billingQueueName = "{billingQueue}";export const precrawlQueueName = "{precrawlQueue}";-export function getExtractQueue() {- if (!extractQueue) {- extractQueue = new Queue(extractQueueName, {- connection: getRedisConnection(),- defaultJobOptions: {- removeOnComplete: {- age: 90000, // 25 hours- },- removeOnFail: {- age: 90000, // 25 hours- },- },- });- }- return extractQueue;+export async function addExtractJobToQueue(+ extractId: string,+ data: ExtractJobData,+): Promise<void> {+ await addExtractJob(extractId, data);}export function getGenerateLlmsTxtQueue() {diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.tsindex 12e366c11..6d347e09c 100644--- a/apps/api/src/services/worker/nuq.ts+++ b/apps/api/src/services/worker/nuq.ts@@ -264,37 +264,59 @@ class NuQ<JobData = any, JobReturnValue = any> {connection: amqp.ChannelModel;channel: amqp.Channel;} | null = null;+ private senderStarting = false;private async startSender() {- if (this.sender || this.shuttingDown) return;+ if (this.sender || this.shuttingDown || this.senderStarting) return;+ this.senderStarting = true;- if (config.NUQ_RABBITMQ_URL) {- const connection = await amqp.connect(config.NUQ_RABBITMQ_URL);- const channel = await connection.createChannel();- await channel.assertQueue(this.queueName + ".prefetch", {- durable: true,- arguments: {- "x-queue-type": "quorum",- "x-max-length": 20000,- },- });+ try {+ if (config.NUQ_RABBITMQ_URL) {+ const connection = await amqp.connect(config.NUQ_RABBITMQ_URL);+ const channel = await connection.createChannel();+ await channel.assertQueue(this.queueName + ".prefetch", {+ durable: true,+ arguments: {+ "x-queue-type": "quorum",+ "x-max-length": 20000,+ },+ });- this.sender = {- type: "rabbitmq",- connection,- channel,- };+ this.sender = {+ type: "rabbitmq",+ connection,+ channel,+ };- channel.on("close", () => {- logger.info("NuQ sender channel closed", { module: "nuq/rabbitmq" });- connection.close().catch(() => {});- this.sender = null;- });+ channel.on("close", () => {+ logger.info("NuQ sender channel closed", { module: "nuq/rabbitmq" });+ connection.close().catch(() => {});+ this.sender = null;+ });- connection.on("close", () => {- logger.info("NuQ sender connection closed", { module: "nuq/rabbitmq" });- this.sender = null;- });+ channel.on("error", err => {+ logger.error("NuQ sender channel error", {+ module: "nuq/rabbitmq",+ err,+ });+ });++ connection.on("close", () => {+ logger.info("NuQ sender connection closed", {+ module: "nuq/rabbitmq",+ });+ this.sender = null;+ });++ connection.on("error", err => {+ logger.error("NuQ sender connection error", {+ module: "nuq/rabbitmq",+ err,+ });+ });+ }+ } finally {+ this.senderStarting = false;}}@@ -1199,14 +1221,23 @@ class NuQ<JobData = any, JobReturnValue = any> {await this.startSender();if (this.sender) {- const job = await this.sender.channel.get(- this.queueName + ".prefetch",- { noAck: true },- );- if (job !== false) {- return this.rowToJob(JSON.parse(job.content.toString()));- } else {- return null;+ try {+ const job = await this.sender.channel.get(+ this.queueName + ".prefetch",+ { noAck: true },+ );+ if (job !== false) {+ return this.rowToJob(JSON.parse(job.content.toString()));+ } else {+ return null;+ }+ } catch (err) {+ logger.warn("NuQ sender get failed, falling back to postgres", {+ module: "nuq/rabbitmq",+ err,+ });+ // Reset sender so it can be re-established on next call+ this.sender = null;}} else {logger.warn("NuQ sender not started, falling back to postgres", {diff --git a/docker-compose.yaml b/docker-compose.yamlindex 13b57b2f8..b66cbec0c 100644--- a/docker-compose.yaml+++ b/docker-compose.yaml@@ -92,10 +92,12 @@ services:PORT: ${INTERNAL_PORT:-3002}EXTRACT_WORKER_PORT: ${EXTRACT_WORKER_PORT:-3004}WORKER_PORT: ${WORKER_PORT:-3005}+ NUQ_RABBITMQ_URL: amqp://rabbitmq:5672ENV: localdepends_on:- redis- playwright-service+ - rabbitmqports:- "${PORT:-3002}:${INTERNAL_PORT:-3002}"command: node dist/src/harness.js --start-docker@@ -121,6 +123,18 @@ services:max-size: "5m"max-file: "2"compress: "true"++ rabbitmq:+ image: rabbitmq:3-management+ networks:+ - backend+ command: rabbitmq-server+ logging:+ driver: "json-file"+ options:+ max-size: "5m"+ max-file: "2"+ compress: "true"nuq-postgres:build: apps/nuq-postgres