Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(extract): rabbitmq + poison pill handling (#2581)

Gergő Móricz11f51eda2f2c
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml
index b5fe8c083..6eb8a5c98 100644
--- a/.github/workflows/test-server.yml
+++ b/.github/workflows/test-server.yml
@@ -31,6 +31,10 @@ jobs:
image: redis
ports:
- 6379:6379
+ rabbitmq:
+ image: rabbitmq
+ ports:
+ - 5672:5672
env:
HOST: 0.0.0.0
TEST_SUITE_SELF_HOSTED: true
@@ -42,6 +46,7 @@ jobs:
PROXY_USERNAME: ${{ matrix.proxy == 'proxy' && secrets.PROXY_USERNAME || '' }}
PROXY_PASSWORD: ${{ matrix.proxy == 'proxy' && secrets.PROXY_PASSWORD || '' }}
NUQ_DATABASE_URL: postgres://postgres:postgres@localhost:5432/postgres
+ NUQ_RABBITMQ_URL: amqp://localhost:5672
USE_GO_MARKDOWN_PARSER: true
ALLOW_LOCAL_WEBHOOKS: true
FIRECRAWL_LOG_TO_FILE: true
diff --git a/apps/api/src/controllers/v1/extract-status.ts b/apps/api/src/controllers/v1/extract-status.ts
index c1c73e345..9a73737a7 100644
--- a/apps/api/src/controllers/v1/extract-status.ts
+++ b/apps/api/src/controllers/v1/extract-status.ts
@@ -1,67 +1,29 @@
import { Response } from "express";
import { config } from "../../config";
import { RequestWithAuth } from "./types";
-import { getExtract, getExtractExpiry } from "../../lib/extract/extract-redis";
-import { getExtractQueue } from "../../services/queue-service";
-import { ExtractResult } from "../../lib/extract/extraction-service";
+import {
+ getExtract,
+ getExtractExpiry,
+ getExtractResult,
+} from "../../lib/extract/extract-redis";
import { supabaseGetExtractByIdDirect } from "../../lib/supabase-jobs";
-import { JobState } from "bullmq";
import { logger as _logger } from "../../lib/logger";
-
-type DBExtract = {
- id: string;
- is_successful: boolean;
- options: any;
- created_at: any;
- error: string | null;
- team_id: string;
-};
-
-type ExtractPseudoJob<T> = {
- id: string;
- getState: () => Promise<JobState | "unknown"> | JobState | "unknown";
- returnvalue: T | null;
- timestamp: number;
- data: {
- scrapeOptions: any;
- teamId?: string;
- };
- failedReason?: string;
-};
-
-async function getExtractJob(
- id: string,
-): Promise<ExtractPseudoJob<ExtractResult> | null> {
- const [bullJob, dbExtract] = await Promise.all([
- getExtractQueue().getJob(id),
- (config.USE_DB_AUTHENTICATION
- ? supabaseGetExtractByIdDirect(id)
- : null) as Promise<DBExtract | null>,
- ]);
-
- if (!bullJob && !dbExtract) return null;
-
- // Extract results are stored in GCS, not in the DB
- const data = bullJob?.returnvalue?.data;
-
- const job: ExtractPseudoJob<any> = {
- id,
- getState: dbExtract
- ? () => (dbExtract.is_successful ? "completed" : "failed")
- : () => bullJob!.getState(),
- returnvalue: data,
- data: {
- scrapeOptions: bullJob ? bullJob.data.scrapeOptions : dbExtract!.options,
- teamId: bullJob ? bullJob.data.teamId : dbExtract!.team_id,
- },
- timestamp: bullJob
- ? bullJob.timestamp
- : new Date(dbExtract!.created_at).valueOf(),
- failedReason:
- (bullJob ? bullJob.failedReason : dbExtract!.error) || undefined,
- };
-
- return job;
+import { getJobFromGCS } from "../../lib/gcs-jobs";
+
+async function getExtractData(id: string): Promise<any> {
+ // Try GCS first if configured
+ if (config.GCS_BUCKET_NAME) {
+ const gcsData = await getJobFromGCS(id);
+ if (gcsData) {
+ return Array.isArray(gcsData) ? gcsData[0] : gcsData;
+ }
+ }
+ // Fallback to Redis
+ const redisData = await getExtractResult(id);
+ if (redisData) {
+ return Array.isArray(redisData) ? redisData[0] : redisData;
+ }
+ return [];
}
export async function extractStatusController(
@@ -75,10 +37,10 @@ export async function extractStatusController(
extractId: req.params.jobId,
});
+ // Get extract status from Redis (for in-progress jobs)
const extract = await getExtract(req.params.jobId);
- let status = extract?.status;
-
+ // Check team ownership from Redis
if (extract && extract.team_id !== req.auth.team_id) {
return res.status(404).json({
success: false,
@@ -86,49 +48,63 @@ export async function extractStatusController(
});
}
- let data: ExtractResult | [] = [];
-
- if (!extract || extract.status === "completed") {
- const jobData = await getExtractJob(req.params.jobId);
- if (
- (!jobData && !extract) ||
- (jobData && jobData.data.teamId !== req.auth.team_id)
- ) {
- logger.warn("Extract job was not found");
- return res.status(404).json({
- success: false,
- error: "Extract job not found",
- });
- }
+ // If not in Redis, check the database for completed jobs
+ if (!extract) {
+ if (config.USE_DB_AUTHENTICATION) {
+ const dbExtract = await supabaseGetExtractByIdDirect(req.params.jobId);
+ if (!dbExtract) {
+ logger.warn("Extract job was not found");
+ return res.status(404).json({
+ success: false,
+ error: "Extract job not found",
+ });
+ }
- if (jobData) {
- const jobStatus = await jobData.getState();
+ if (dbExtract.team_id !== req.auth.team_id) {
+ return res.status(404).json({
+ success: false,
+ error: "Extract job not found",
+ });
+ }
- if (jobStatus === "completed") {
- status = "completed";
- } else if (jobStatus === "failed") {
- status = "failed";
- } else {
- status = "processing";
+ let data: any = [];
+ if (dbExtract.is_successful) {
+ data = await getExtractData(req.params.jobId);
}
- }
- if (!jobData?.returnvalue) {
- // if we got in the split-second where the redis is updated but the bull isn't
- // just pretend it's still processing - MG
- status = "processing";
- } else {
- data = jobData.returnvalue ?? [];
+ // Return DB-based status
+ return res.status(200).json({
+ success: dbExtract.is_successful,
+ data,
+ status: dbExtract.is_successful ? "completed" : "failed",
+ error: dbExtract.error || undefined,
+ expiresAt: new Date(
+ new Date(dbExtract.created_at).getTime() + 1000 * 60 * 60 * 24,
+ ).toISOString(),
+ });
}
+
+ logger.warn("Extract job was not found");
+ return res.status(404).json({
+ success: false,
+ error: "Extract job not found",
+ });
+ }
+
+ // Get result data if completed
+ let data: any = [];
+ if (extract.status === "completed") {
+ data = await getExtractData(req.params.jobId);
}
+ // Return Redis-based status
return res.status(200).json({
- success: status === "failed" ? false : true,
+ success: extract.status === "failed" ? false : true,
data,
- status,
+ status: extract.status,
error: (() => {
- if (typeof extract?.error === "string") return extract.error;
- if (extract?.error && typeof extract.error === "object") {
+ if (typeof extract.error === "string") return extract.error;
+ if (extract.error && typeof extract.error === "object") {
return typeof extract.error.message === "string"
? extract.error.message
: typeof extract.error.error === "string"
@@ -138,12 +114,12 @@ export async function extractStatusController(
return undefined;
})(),
expiresAt: (await getExtractExpiry(req.params.jobId)).toISOString(),
- steps: extract?.showSteps ? extract.steps : undefined,
- llmUsage: extract?.showLLMUsage ? extract.llmUsage : undefined,
- sources: extract?.showSources ? extract.sources : undefined,
- costTracking: extract?.showCostTracking ? extract.costTracking : undefined,
- sessionIds: extract?.sessionIds ? extract.sessionIds : undefined,
- tokensUsed: extract?.tokensBilled ? extract.tokensBilled : undefined,
- creditsUsed: extract?.creditsBilled ? extract.creditsBilled : undefined,
+ steps: extract.showSteps ? extract.steps : undefined,
+ llmUsage: extract.showLLMUsage ? extract.llmUsage : undefined,
+ sources: extract.showSources ? extract.sources : undefined,
+ costTracking: extract.showCostTracking ? extract.costTracking : undefined,
+ sessionIds: extract.sessionIds ? extract.sessionIds : undefined,
+ tokensUsed: extract.tokensBilled ? extract.tokensBilled : undefined,
+ creditsUsed: extract.creditsBilled ? extract.creditsBilled : undefined,
});
}
diff --git a/apps/api/src/controllers/v1/extract.ts b/apps/api/src/controllers/v1/extract.ts
index f291699aa..9f909df53 100644
--- a/apps/api/src/controllers/v1/extract.ts
+++ b/apps/api/src/controllers/v1/extract.ts
@@ -6,7 +6,7 @@ import {
extractRequestSchema,
ExtractResponse,
} from "./types";
-import { getExtractQueue } from "../../services/queue-service";
+import { addExtractJobToQueue } from "../../services/queue-service";
import { saveExtract } from "../../lib/extract/extract-redis";
import { getTeamIdSyncB } from "../../lib/extract/team-id-sync";
import {
@@ -209,9 +209,7 @@ export async function extractController(
zeroDataRetention: req.acuc?.flags?.forceZDR,
});
- await getExtractQueue().add(extractId, jobData, {
- jobId: extractId,
- });
+ await addExtractJobToQueue(extractId, jobData);
return res.status(200).json({
success: true,
diff --git a/apps/api/src/controllers/v2/extract-status.ts b/apps/api/src/controllers/v2/extract-status.ts
index 3d28dac5a..d13714104 100644
--- a/apps/api/src/controllers/v2/extract-status.ts
+++ b/apps/api/src/controllers/v2/extract-status.ts
@@ -1,171 +1,146 @@
import { Response } from "express";
import { config } from "../../config";
import { RequestWithAuth } from "./types";
-import { getExtract, getExtractExpiry } from "../../lib/extract/extract-redis";
-import { getExtractQueue } from "../../services/queue-service";
-import { ExtractResult } from "../../lib/extract/extraction-service";
+import {
+ getExtract,
+ getExtractExpiry,
+ getExtractResult,
+} from "../../lib/extract/extract-redis";
import {
supabaseGetAgentByIdDirect,
supabaseGetExtractByIdDirect,
supabaseGetExtractRequestByIdDirect,
} from "../../lib/supabase-jobs";
-import { JobState } from "bullmq";
import { logger as _logger } from "../../lib/logger";
import { getJobFromGCS } from "../../lib/gcs-jobs";
-type DBExtract = {
- id: string;
- is_successful: boolean;
- options: any;
- created_at: any;
- error: string | null;
- team_id: string;
-};
-
-type ExtractPseudoJob<T> = {
- id: string;
- getState: () => Promise<JobState | "unknown"> | JobState | "unknown";
- returnvalue: T | null;
- timestamp: number;
- data: {
- scrapeOptions: any;
- teamId?: string;
- };
- failedReason?: string;
-};
-
-async function getExtractJob(
- id: string,
-): Promise<ExtractPseudoJob<ExtractResult> | null> {
- const [bullJob, gcsJob, dbExtract] = await Promise.all([
- getExtractQueue().getJob(id),
- (config.GCS_BUCKET_NAME ? getJobFromGCS(id) : null) as Promise<any | null>,
- (config.USE_DB_AUTHENTICATION
- ? supabaseGetExtractByIdDirect(id)
- : null) as Promise<DBExtract | null>,
- ]);
-
- if (!bullJob && !dbExtract) return null;
-
- // Extract results are stored in GCS, not in the DB
- let data = gcsJob ?? bullJob?.returnvalue?.data;
- if (gcsJob === null && data) {
- _logger.warn("GCS Job not found", {
- jobId: id,
- });
+async function getExtractData(id: string): Promise<any> {
+ // Try GCS first if configured
+ if (config.GCS_BUCKET_NAME) {
+ const gcsData = await getJobFromGCS(id);
+ if (gcsData) {
+ return Array.isArray(gcsData) ? gcsData[0] : gcsData;
+ }
}
- if (Array.isArray(data)) data = data[0];
-
- const job: ExtractPseudoJob<any> = {
- id,
- getState: bullJob
- ? bullJob.getState.bind(bullJob)
- : () => (dbExtract!.is_successful ? "completed" : "failed"),
- returnvalue: data,
- data: {
- scrapeOptions: bullJob ? bullJob.data.scrapeOptions : dbExtract!.options,
- teamId: bullJob ? bullJob.data.teamId : dbExtract!.team_id,
- },
- timestamp: bullJob
- ? bullJob.timestamp
- : new Date(dbExtract!.created_at).valueOf(),
- failedReason:
- (bullJob ? bullJob.failedReason : dbExtract!.error) || undefined,
- };
-
- return job;
+ // Fallback to Redis
+ const redisData = await getExtractResult(id);
+ if (redisData) {
+ return Array.isArray(redisData) ? redisData[0] : redisData;
+ }
+ return [];
}
export async function extractStatusController(
req: RequestWithAuth<{ jobId: string }, any, any>,
res: Response,
) {
- const extractRequest = await supabaseGetExtractRequestByIdDirect(
- req.params.jobId,
- );
+ const extractRequest = config.USE_DB_AUTHENTICATION
+ ? await supabaseGetExtractRequestByIdDirect(req.params.jobId)
+ : null;
+ if (config.USE_DB_AUTHENTICATION) {
+ if (!extractRequest || extractRequest.team_id !== req.auth.team_id) {
+ return res.status(404).json({
+ success: false,
+ error: "Extract job not found",
+ });
+ }
- if (!extractRequest || extractRequest.team_id !== req.auth.team_id) {
- return res.status(404).json({
- success: false,
- error: "Extract job not found",
- });
+ if (extractRequest.kind === "agent") {
+ const agent = await supabaseGetAgentByIdDirect(req.params.jobId);
+
+ let data: any = undefined;
+ if (agent?.is_successful) {
+ data = await getJobFromGCS(agent.id);
+ }
+
+ return res.status(200).json({
+ success: true,
+ status: !agent
+ ? "processing"
+ : agent.is_successful
+ ? "completed"
+ : "failed",
+ error: agent?.error || undefined,
+ data,
+ expiresAt: new Date(
+ new Date(agent?.created_at ?? extractRequest.created_at).getTime() +
+ 1000 * 60 * 60 * 24,
+ ).toISOString(),
+ creditsUsed: agent?.credits_cost,
+ });
+ }
}
- if (extractRequest.kind === "agent") {
- const agent = await supabaseGetAgentByIdDirect(req.params.jobId);
+ // Get extract status from Redis (for in-progress jobs)
+ const redisExtract = await getExtract(req.params.jobId);
- let data: any = undefined;
- if (agent?.is_successful) {
- data = await getJobFromGCS(agent.id);
+ // If not in Redis, check the database for completed jobs
+ if (!redisExtract) {
+ if (config.USE_DB_AUTHENTICATION) {
+ const dbExtract = await supabaseGetExtractByIdDirect(req.params.jobId);
+ if (dbExtract) {
+ // Get result data
+ let data: any = [];
+ if (dbExtract.is_successful) {
+ data = await getExtractData(req.params.jobId);
+ }
+
+ return res.status(200).json({
+ success: dbExtract.is_successful,
+ data,
+ status: dbExtract.is_successful ? "completed" : "failed",
+ error: dbExtract.error || undefined,
+ expiresAt: new Date(
+ new Date(dbExtract.created_at).getTime() + 1000 * 60 * 60 * 24,
+ ).toISOString(),
+ });
+ }
}
+ // Fall back to extractRequest info
return res.status(200).json({
success: true,
- status: !agent
- ? "processing"
- : agent.is_successful
- ? "completed"
- : "failed",
- error: agent?.error || undefined,
- data,
+ data: [],
+ status: "processing",
expiresAt: new Date(
- new Date(agent?.created_at ?? extractRequest.created_at).getTime() +
- 1000 * 60 * 60 * 24,
+ new Date(extractRequest.created_at).getTime() + 1000 * 60 * 60 * 24,
).toISOString(),
- creditsUsed: agent?.credits_cost,
});
}
- let data: ExtractResult | [] = [];
- let status: string = "processing";
-
- const redisExtract = await getExtract(req.params.jobId);
- const jobData = await getExtractJob(req.params.jobId);
-
- console.log("jobData", jobData);
-
- if (jobData) {
- const jobStatus = await jobData.getState();
-
- if (jobStatus === "completed") {
- status = "completed";
- } else if (jobStatus === "failed") {
- status = "failed";
- } else {
- status = "processing";
- }
- }
-
- if (!jobData?.returnvalue) {
- // if we got in the split-second where the redis is updated but the bull isn't
- // just pretend it's still processing - MG
- status = "processing";
- } else {
- data = jobData.returnvalue ?? [];
+ // Get result data if completed
+ let data: any = [];
+ if (redisExtract.status === "completed") {
+ data = await getExtractData(req.params.jobId);
}
return res.status(200).json({
- success: status === "failed" ? false : true,
+ success: redisExtract.status === "failed" ? false : true,
data,
- status,
- error: jobData?.failedReason,
- expiresAt: redisExtract
- ? (await getExtractExpiry(req.params.jobId)).toISOString()
- : new Date(
- new Date(jobData?.timestamp ?? extractRequest.created_at).getTime() +
- 1000 * 60 * 60 * 24,
- ).toISOString(),
- steps: redisExtract?.showSteps ? redisExtract.steps : undefined,
- llmUsage: redisExtract?.showLLMUsage ? redisExtract.llmUsage : undefined,
- sources: redisExtract?.showSources ? redisExtract.sources : undefined,
- costTracking: redisExtract?.showCostTracking
+ status: redisExtract.status,
+ error: (() => {
+ if (typeof redisExtract.error === "string") return redisExtract.error;
+ if (redisExtract.error && typeof redisExtract.error === "object") {
+ return typeof redisExtract.error.message === "string"
+ ? redisExtract.error.message
+ : typeof redisExtract.error.error === "string"
+ ? redisExtract.error.error
+ : JSON.stringify(redisExtract.error);
+ }
+ return undefined;
+ })(),
+ expiresAt: (await getExtractExpiry(req.params.jobId)).toISOString(),
+ steps: redisExtract.showSteps ? redisExtract.steps : undefined,
+ llmUsage: redisExtract.showLLMUsage ? redisExtract.llmUsage : undefined,
+ sources: redisExtract.showSources ? redisExtract.sources : undefined,
+ costTracking: redisExtract.showCostTracking
? redisExtract.costTracking
: undefined,
- sessionIds: redisExtract?.sessionIds ? redisExtract.sessionIds : undefined,
- tokensUsed: redisExtract?.tokensBilled
+ sessionIds: redisExtract.sessionIds ? redisExtract.sessionIds : undefined,
+ tokensUsed: redisExtract.tokensBilled
? redisExtract.tokensBilled
: undefined,
- creditsUsed: redisExtract?.creditsBilled
+ creditsUsed: redisExtract.creditsBilled
? redisExtract.creditsBilled
: undefined,
});
diff --git a/apps/api/src/controllers/v2/extract.ts b/apps/api/src/controllers/v2/extract.ts
index d2ee361ee..dccb34bdf 100644
--- a/apps/api/src/controllers/v2/extract.ts
+++ b/apps/api/src/controllers/v2/extract.ts
@@ -6,7 +6,7 @@ import {
extractRequestSchema,
ExtractResponse,
} from "./types";
-import { getExtractQueue } from "../../services/queue-service";
+import { addExtractJobToQueue } from "../../services/queue-service";
import { saveExtract } from "../../lib/extract/extract-redis";
import { BLOCKLISTED_URL_MESSAGE } from "../../lib/strings";
import { isUrlBlocked } from "../../scraper/WebScraper/utils/blocklist";
@@ -158,8 +158,9 @@ export async function extractController(
zeroDataRetention: req.acuc?.flags?.forceZDR,
});
- await getExtractQueue().add(extractId, jobData, {
- jobId: extractId,
+ await addExtractJobToQueue(extractId, {
+ ...jobData,
+ apiKeyId: req.acuc?.api_key_id ?? undefined,
});
return res.status(200).json({
diff --git a/apps/api/src/harness.ts b/apps/api/src/harness.ts
index ccfbed79f..71dda3794 100644
--- a/apps/api/src/harness.ts
+++ b/apps/api/src/harness.ts
@@ -14,6 +14,10 @@ let nuqPostgresContainer: {
containerName: string;
containerRuntime: string;
} | null = null;
+let nuqRabbitMQContainer: {
+ containerName: string;
+ containerRuntime: string;
+} | null = null;
// Get the monorepo root (apps/api/dist/src -> ../../../..)
// __dirname is available in CommonJS (which this compiles to)
@@ -37,6 +41,10 @@ interface Services {
containerName: string;
containerRuntime: string;
};
+ nuqRabbitMQ?: {
+ containerName: string;
+ containerRuntime: string;
+ };
}
const colors = {
@@ -558,6 +566,106 @@ async function setupNuqPostgres(): Promise<Services["nuqPostgres"]> {
return containerInfo;
}
+async function startNuqRabbitMQContainer(
+ runtime: string,
+ containerName: string,
+): Promise<void> {
+ logger.info(`Starting RabbitMQ container: ${containerName}`);
+ const start = execForward(
+ `${runtime}@start`,
+ `${runtime} run -d --name ${containerName} -p 5672:5672 -p 15672:15672 rabbitmq:3-management`,
+ );
+ await start.promise;
+ logger.success(`RabbitMQ container started: ${containerName}`);
+}
+
+async function waitForRabbitMQ(
+ host: string,
+ port: number,
+ timeoutMs: number = config.HARNESS_STARTUP_TIMEOUT_MS,
+): Promise<void> {
+ logger.info("Waiting for RabbitMQ to be ready...");
+ const start = Date.now();
+
+ while (Date.now() - start < timeoutMs) {
+ try {
+ // Try to connect to RabbitMQ using amqplib
+ const amqp = await import("amqplib");
+ const connection = await amqp.connect(`amqp://${host}:${port}`, {
+ timeout: 2000,
+ });
+ await connection.close();
+
+ logger.success("RabbitMQ is ready");
+ return;
+ } catch (e) {
+ // Not ready yet, wait and retry
+ await new Promise(resolve => setTimeout(resolve, 500));
+ }
+ }
+
+ throw new Error(`RabbitMQ did not become ready within ${timeoutMs}ms`);
+}
+
+async function setupNuqRabbitMQ(): Promise<Services["nuqRabbitMQ"]> {
+ // If NUQ_RABBITMQ_URL is already set, respect it (user's explicit choice)
+ if (config.NUQ_RABBITMQ_URL) {
+ logger.info("NUQ_RABBITMQ_URL is set, skipping container management");
+ return undefined;
+ }
+
+ // Check if we're running in docker-compose (POSTGRES_HOST is set and not localhost)
+ const isDockerCompose = POSTGRES_HOST !== "localhost";
+
+ if (isDockerCompose) {
+ // Running in docker-compose: RabbitMQ should be configured externally
+ logger.info(
+ "Running in docker-compose, skipping RabbitMQ container management",
+ );
+ return undefined;
+ }
+
+ // Running locally: manage container
+ logger.section("Setting up NUQ RabbitMQ container");
+
+ const runtime = await detectContainerRuntime();
+ if (!runtime) {
+ throw new Error(
+ "Neither Docker nor Podman found. Please install Docker/Podman or set NUQ_RABBITMQ_URL manually.",
+ );
+ }
+
+ logger.success(`Using container runtime: ${runtime}`);
+
+ const containerName = "firecrawl-nuq-rabbitmq";
+
+ // Stop and remove any existing container
+ await stopAndRemoveContainer(runtime, containerName);
+
+ // Start the container (using stock image, no build needed)
+ await startNuqRabbitMQContainer(runtime, containerName);
+
+ // Wait for RabbitMQ to be ready
+ await waitForRabbitMQ("localhost", 5672);
+
+ // Set environment variables for the services
+ const rabbitUrl = "amqp://localhost:5672";
+ config.NUQ_RABBITMQ_URL = rabbitUrl;
+ process.env.NUQ_RABBITMQ_URL = rabbitUrl;
+
+ logger.success("NUQ RabbitMQ container is ready");
+
+ const containerInfo = {
+ containerName,
+ containerRuntime: runtime,
+ };
+
+ // Store globally for graceful shutdown
+ nuqRabbitMQContainer = containerInfo;
+
+ return containerInfo;
+}
+
async function installDependencies() {
logger.section("Installing dependencies");
@@ -604,6 +712,9 @@ async function startServices(command?: string[]): Promise<Services> {
// Setup NUQ PostgreSQL container if needed
const nuqPostgres = await setupNuqPostgres();
+ // Setup NUQ RabbitMQ container if needed
+ const nuqRabbitMQ = await setupNuqRabbitMQ();
+
logger.section("Starting services");
const api = execForward(
@@ -655,20 +766,18 @@ async function startServices(command?: string[]): Promise<Services> {
),
);
- const nuqPrefetchWorker = config.NUQ_RABBITMQ_URL
- ? execForward(
- "nuq-prefetch-worker",
- process.argv[2] === "--start-docker"
- ? "node dist/src/services/worker/nuq-prefetch-worker.js"
- : "pnpm nuq-prefetch-worker:production",
- {
- NUQ_PREFETCH_WORKER_PORT: String(NUQ_PREFETCH_WORKER_PORT),
- NUQ_REDUCE_NOISE: "true",
- NUQ_POD_NAME: "nuq-prefetch-worker-0",
- NUQ_PREFETCH_REPLICAS: String(1),
- },
- )
- : undefined;
+ const nuqPrefetchWorker = execForward(
+ "nuq-prefetch-worker",
+ process.argv[2] === "--start-docker"
+ ? "node dist/src/services/worker/nuq-prefetch-worker.js"
+ : "pnpm nuq-prefetch-worker:production",
+ {
+ NUQ_PREFETCH_WORKER_PORT: String(NUQ_PREFETCH_WORKER_PORT),
+ NUQ_REDUCE_NOISE: "true",
+ NUQ_POD_NAME: "nuq-prefetch-worker-0",
+ NUQ_PREFETCH_REPLICAS: String(1),
+ },
+ );
const indexWorker = config.USE_DB_AUTHENTICATION
? execForward(
@@ -708,6 +817,7 @@ async function startServices(command?: string[]): Promise<Services> {
extractWorker,
command: commandProcess,
nuqPostgres,
+ nuqRabbitMQ,
};
}
@@ -741,6 +851,16 @@ async function stopDevelopmentServices(services: Services) {
);
logger.success("NUQ PostgreSQL container stopped");
}
+
+ // Stop and remove NUQ RabbitMQ container if it was started by harness
+ if (services.nuqRabbitMQ) {
+ logger.info("Stopping NUQ RabbitMQ container");
+ await stopAndRemoveContainer(
+ services.nuqRabbitMQ.containerRuntime,
+ services.nuqRabbitMQ.containerName,
+ );
+ logger.success("NUQ RabbitMQ container stopped");
+ }
}
async function runDevMode(): Promise<void> {
@@ -904,6 +1024,17 @@ async function gracefulShutdown() {
nuqPostgresContainer = null;
}
+ // Stop and remove NUQ RabbitMQ container if it was started by harness
+ if (nuqRabbitMQContainer) {
+ logger.info("Stopping NUQ RabbitMQ container");
+ await stopAndRemoveContainer(
+ nuqRabbitMQContainer.containerRuntime,
+ nuqRabbitMQContainer.containerName,
+ );
+ logger.success("NUQ RabbitMQ container stopped");
+ nuqRabbitMQContainer = null;
+ }
+
logger.success("All processes terminated");
}
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index a30ca811c..57450a5a5 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -7,7 +7,6 @@ import express, { NextFunction, Request, Response } from "express";
import bodyParser from "body-parser";
import cors from "cors";
import {
- getExtractQueue,
getGenerateLlmsTxtQueue,
getDeepResearchQueue,
getBillingQueue,
@@ -81,7 +80,6 @@ serverAdapter.setBasePath(`/admin/${config.BULL_AUTH_KEY}/queues`);
const { addQueue, removeQueue, setQueues, replaceQueues } = createBullBoard({
queues: [
- new BullMQAdapter(getExtractQueue()),
new BullMQAdapter(getGenerateLlmsTxtQueue()),
new BullMQAdapter(getDeepResearchQueue()),
new BullMQAdapter(getBillingQueue()),
diff --git a/apps/api/src/lib/extract/extract-redis.ts b/apps/api/src/lib/extract/extract-redis.ts
index 0ab06ab0d..28a3faa34 100644
--- a/apps/api/src/lib/extract/extract-redis.ts
+++ b/apps/api/src/lib/extract/extract-redis.ts
@@ -144,3 +144,24 @@ export async function getExtractExpiry(id: string): Promise<Date> {
d.setMilliseconds(0);
return d;
}
+
+// Result data storage (fallback when GCS is not configured)
+const EXTRACT_RESULT_TTL = 24 * 60 * 60; // 24 hours
+
+export async function saveExtractResult(
+ id: string,
+ result: any,
+): Promise<void> {
+ _logger.debug("Saving extract result " + id + " to Redis...");
+ await redisEvictConnection.set(
+ "extract_result:" + id,
+ JSON.stringify(result),
+ "EX",
+ EXTRACT_RESULT_TTL,
+ );
+}
+
+export async function getExtractResult(id: string): Promise<any | null> {
+ const x = await redisEvictConnection.get("extract_result:" + id);
+ return x ? JSON.parse(x) : null;
+}
diff --git a/apps/api/src/services/extract-queue.ts b/apps/api/src/services/extract-queue.ts
new file mode 100644
index 000000000..f7c518a3c
--- /dev/null
+++ b/apps/api/src/services/extract-queue.ts
@@ -0,0 +1,165 @@
+import amqp from "amqplib";
+import { config } from "../config";
+import { logger as _logger } from "../lib/logger";
+
+const EXTRACT_QUEUE = "extract.jobs";
+const EXTRACT_DLX = "extract.dlx";
+const EXTRACT_DLQ = "extract.dlq";
+
+export type ExtractJobData = {
+ extractId: string;
+ request: any;
+ teamId: string;
+ subId?: string | null;
+ apiKeyId?: number | null;
+ agent?: any;
+ createdAt: number;
+};
+
+let connection: amqp.ChannelModel | null = null;
+let channel: amqp.Channel | null = null;
+
+async function getChannel(): Promise<amqp.Channel> {
+ if (channel) return channel;
+
+ const url = config.NUQ_RABBITMQ_URL;
+ if (!url) {
+ throw new Error("NUQ_RABBITMQ_URL is not configured");
+ }
+
+ connection = await amqp.connect(url);
+ channel = await connection.createChannel();
+
+ // Set up the dead letter exchange
+ await channel.assertExchange(EXTRACT_DLX, "direct", { durable: true });
+
+ // Set up the dead letter queue
+ await channel.assertQueue(EXTRACT_DLQ, {
+ durable: true,
+ arguments: {
+ "x-queue-type": "quorum",
+ },
+ });
+ await channel.bindQueue(EXTRACT_DLQ, EXTRACT_DLX, EXTRACT_QUEUE);
+
+ // Set up the main queue with DLX - no retries (messages go straight to DLQ on reject/crash)
+ await channel.assertQueue(EXTRACT_QUEUE, {
+ durable: true,
+ arguments: {
+ "x-queue-type": "quorum",
+ "x-dead-letter-exchange": EXTRACT_DLX,
+ "x-dead-letter-routing-key": EXTRACT_QUEUE,
+ "x-delivery-limit": 1,
+ },
+ });
+
+ connection.on("close", () => {
+ _logger.warn("Extract queue connection closed");
+ connection = null;
+ channel = null;
+ });
+
+ connection.on("error", err => {
+ _logger.error("Extract queue connection error", { error: err });
+ });
+
+ return channel;
+}
+
+export async function addExtractJob(
+ extractId: string,
+ data: ExtractJobData,
+): Promise<void> {
+ const ch = await getChannel();
+ ch.sendToQueue(EXTRACT_QUEUE, Buffer.from(JSON.stringify(data)), {
+ persistent: true,
+ messageId: extractId,
+ });
+ _logger.info("Extract job added to queue", { extractId });
+}
+
+export async function consumeExtractJobs(
+ handler: (
+ data: ExtractJobData,
+ ack: () => void,
+ nack: () => void,
+ ) => Promise<void>,
+): Promise<void> {
+ const ch = await getChannel();
+ await ch.prefetch(1);
+
+ await ch.consume(
+ EXTRACT_QUEUE,
+ async msg => {
+ if (!msg) return;
+
+ const data = JSON.parse(msg.content.toString()) as ExtractJobData;
+ const logger = _logger.child({
+ module: "extract-queue",
+ extractId: data.extractId,
+ });
+
+ logger.info("Processing extract job");
+
+ try {
+ await handler(
+ data,
+ () => ch.ack(msg),
+ () => ch.nack(msg, false, false), // Don't requeue - send to DLX
+ );
+ } catch (error) {
+ logger.error("Extract job handler threw an error", { error });
+ // Don't requeue - send to DLX
+ ch.nack(msg, false, false);
+ }
+ },
+ { noAck: false },
+ );
+
+ _logger.info("Started consuming extract jobs");
+}
+
+export async function consumeExtractDLQ(
+ handler: (data: ExtractJobData) => Promise<void>,
+): Promise<void> {
+ const ch = await getChannel();
+ await ch.prefetch(1);
+
+ await ch.consume(
+ EXTRACT_DLQ,
+ async msg => {
+ if (!msg) return;
+
+ const data = JSON.parse(msg.content.toString()) as ExtractJobData;
+ const logger = _logger.child({
+ module: "extract-dlq",
+ extractId: data.extractId,
+ });
+
+ logger.info("Processing dead-lettered extract job");
+
+ try {
+ await handler(data);
+ ch.ack(msg);
+ } catch (error) {
+ logger.error("DLQ handler threw an error, requeueing", { error });
+ // Requeue DLQ messages on error so we don't lose them
+ ch.nack(msg, false, true);
+ }
+ },
+ { noAck: false },
+ );
+
+ _logger.info("Started consuming extract DLQ");
+}
+
+export async function shutdownExtractQueue(): Promise<void> {
+ if (channel) {
+ await channel.close();
+ channel = null;
+ }
+ if (connection) {
+ await connection.close();
+ connection = null;
+ }
+}
diff --git a/apps/api/src/services/extract-worker.ts b/apps/api/src/services/extract-worker.ts
index 19b2f31fc..5013457fe 100644
--- a/apps/api/src/services/extract-worker.ts
+++ b/apps/api/src/services/extract-worker.ts
@@ -3,11 +3,7 @@ import { config } from "../config";
import "./sentry";
import { setSentryServiceTag } from "./sentry";
import * as Sentry from "@sentry/node";
-import { getExtractQueue, getRedisConnection } from "./queue-service";
-import { Job, Queue, Worker } from "bullmq";
import { logger as _logger } from "../lib/logger";
-import systemMonitor from "./system-monitor";
-import { v7 as uuidv7 } from "uuid";
import { configDotenv } from "dotenv";
import {
ExtractResult,
@@ -17,46 +13,36 @@ import { updateExtract } from "../lib/extract/extract-redis";
import { performExtraction_F0 } from "../lib/extract/fire-0/extraction-service-f0";
import { createWebhookSender, WebhookEvent } from "./webhook";
import Express from "express";
-import { robustFetch } from "../scraper/scrapeURL/lib/fetch";
import { getErrorContactMessage } from "../lib/deployment";
import { TransportableError } from "../lib/error";
import { initializeBlocklist } from "../scraper/WebScraper/utils/blocklist";
import { initializeEngineForcing } from "../scraper/WebScraper/utils/engine-forcing";
+import {
+ consumeExtractJobs,
+ consumeExtractDLQ,
+ shutdownExtractQueue,
+ ExtractJobData,
+} from "./extract-queue";
+import { logExtract } from "./logging/log_job";
configDotenv();
-const sleep = (ms: number) => new Promise(resolve => setTimeout(resolve, ms));
-
-const jobLockExtendInterval = config.JOB_LOCK_EXTEND_INTERVAL;
-const jobLockExtensionTime = config.JOB_LOCK_EXTENSION_TIME;
-
-const cantAcceptConnectionInterval = config.CANT_ACCEPT_CONNECTION_INTERVAL;
-const connectionMonitorInterval = config.CONNECTION_MONITOR_INTERVAL;
-const gotJobInterval = config.CONNECTION_MONITOR_INTERVAL;
-
-const runningJobs: Set<string> = new Set();
-
-const processExtractJobInternal = async (
- token: string,
- job: Job & { id: string },
+const processExtractJob = async (
+ data: ExtractJobData,
+ ack: () => void,
+ nack: () => void,
) => {
const logger = _logger.child({
module: "extract-worker",
- method: "processJobInternal",
- jobId: job.id,
- extractId: job.data.extractId,
- teamId: job.data?.teamId ?? undefined,
+ method: "processExtractJob",
+ extractId: data.extractId,
+ teamId: data.teamId,
});
- const extendLockInterval = setInterval(async () => {
- logger.info(`🔄 Worker extending lock on job ${job.id}`);
- await job.extendLock(token, jobLockExtensionTime);
- }, jobLockExtendInterval);
-
const sender = await createWebhookSender({
- teamId: job.data.teamId,
- jobId: job.data.extractId,
- webhook: job.data.request.webhook,
+ teamId: data.teamId,
+ jobId: data.extractId,
+ webhook: data.request.webhook,
v0: false,
});
@@ -69,34 +55,25 @@ const processExtractJobInternal = async (
let result: ExtractResult | null = null;
- const model = job.data.request.agent?.model;
- if (
- job.data.request.agent &&
- model &&
- model.toLowerCase().includes("fire-1")
- ) {
- result = await performExtraction(job.data.extractId, {
- request: job.data.request,
- teamId: job.data.teamId,
- subId: job.data.subId,
- apiKeyId: job.data.apiKeyId,
+ const model = data.request.agent?.model;
+ if (data.request.agent && model && model.toLowerCase().includes("fire-1")) {
+ result = await performExtraction(data.extractId, {
+ request: data.request,
+ teamId: data.teamId,
+ subId: data.subId ?? undefined,
+ apiKeyId: data.apiKeyId ?? null,
});
} else {
- result = await performExtraction_F0(job.data.extractId, {
- request: job.data.request,
- teamId: job.data.teamId,
- subId: job.data.subId,
- apiKeyId: job.data.apiKeyId,
+ result = await performExtraction_F0(data.extractId, {
+ request: data.request,
+ teamId: data.teamId,
+ subId: data.subId ?? undefined,
+ apiKeyId: data.apiKeyId ?? null,
});
}
- // result = await performExtraction_F0(job.data.extractId, {
- // request: job.data.request,
- // teamId: job.data.teamId,
- // subId: job.data.subId,
- // });
if (result && result.success) {
- await updateExtract(job.data.extractId, {
+ await updateExtract(data.extractId, {
status: "completed",
llmUsage: result.llmUsage,
sources: result.sources,
@@ -104,8 +81,6 @@ const processExtractJobInternal = async (
creditsBilled: result.creditsBilled,
});
- await job.moveToCompleted(result, token, false);
-
if (sender) {
sender.send(WebhookEvent.EXTRACT_COMPLETED, {
success: true,
@@ -113,192 +88,142 @@ const processExtractJobInternal = async (
});
}
- return result;
+ ack();
+ return;
} else {
- await updateExtract(job.data.extractId, {
+ await updateExtract(data.extractId, {
status: "failed",
- error: result?.error ?? getErrorContactMessage(job.data.extractId),
+ error: result?.error ?? getErrorContactMessage(data.extractId),
});
- await job.moveToCompleted(result, token, false);
-
if (sender) {
sender.send(WebhookEvent.EXTRACT_FAILED, {
success: false,
- error: result?.error ?? getErrorContactMessage(job.data.extractId),
+ error: result?.error ?? getErrorContactMessage(data.extractId),
});
}
- return result;
+ ack();
+ return;
}
} catch (error) {
- logger.error(`🚫 Job errored ${job.id} - ${error}`, { error });
+ logger.error(`🚫 Extract job errored ${data.extractId} - ${error}`, {
+ error,
+ });
// Filter out TransportableErrors (flow control)
if (!(error instanceof TransportableError)) {
Sentry.captureException(error, {
data: {
- job: job.id,
+ extractId: data.extractId,
},
});
}
- await updateExtract(job.data.extractId, {
+ await updateExtract(data.extractId, {
status: "failed",
- error: error.error ?? error ?? getErrorContactMessage(job.data.extractId),
+ error:
+ (error as any)?.error ??
+ (error as any)?.message ??
+ getErrorContactMessage(data.extractId),
});
- try {
- await job.moveToFailed(error, token, false);
- } catch (e) {
- logger.log("Failed to move job to failed state in BullMQ", { error });
- }
-
if (sender) {
sender.send(WebhookEvent.EXTRACT_FAILED, {
success: false,
error:
- (error as any)?.message ?? getErrorContactMessage(job.data.extractId),
+ (error as any)?.message ?? getErrorContactMessage(data.extractId),
});
}
- return {
- success: false,
- error: error.error ?? error ?? getErrorContactMessage(job.data.extractId),
- };
- // throw error;
- } finally {
- clearInterval(extendLockInterval);
+ // Ack the message even on error - we've handled it and updated the DB
+ // Only nack if we want to send to DLX (which we don't for handled errors)
+ ack();
}
};
-let isShuttingDown = false;
-let isWorkerStalled = false;
-
-if (require.main === module) {
- process.on("SIGINT", () => {
- _logger.debug("Received SIGINT. Shutting down gracefully...");
- isShuttingDown = true;
- });
-
- process.on("SIGTERM", () => {
- _logger.debug("Received SIGTERM. Shutting down gracefully...");
- isShuttingDown = true;
+const processDLQJob = async (data: ExtractJobData) => {
+ const logger = _logger.child({
+ module: "extract-dlq",
+ extractId: data.extractId,
+ teamId: data.teamId,
});
-}
-let cantAcceptConnectionCount = 0;
-
-const workerFun = async (
- queue: Queue,
- processJobInternal: (token: string, job: Job) => Promise<any>,
-) => {
- const logger = _logger.child({ module: "queue-worker", method: "workerFun" });
+ logger.error("Processing crashed extract job from DLQ");
- const worker = new Worker(queue.name, null, {
- connection: getRedisConnection(),
- lockDuration: 60 * 1000, // 60 seconds
- stalledInterval: 60 * 1000, // 60 seconds
- maxStalledCount: 10, // 10 times
+ // Update the extract status to failed in the database
+ await updateExtract(data.extractId, {
+ status: "failed",
+ error:
+ "Extract job crashed unexpectedly. Please try again or contact support if the issue persists.",
});
- worker.startStalledCheckTimer();
-
- const monitor = await systemMonitor;
-
- while (true) {
- if (isShuttingDown) {
- _logger.info("No longer accepting new jobs. SIGINT");
- break;
- }
- const token = uuidv7();
- const canAcceptConnection = await monitor.acceptConnection();
- if (!canAcceptConnection) {
- console.log("Can't accept connection due to RAM/CPU load");
- logger.info("Can't accept connection due to RAM/CPU load");
- cantAcceptConnectionCount++;
-
- isWorkerStalled = cantAcceptConnectionCount >= 25;
-
- if (isWorkerStalled) {
- logger.error("WORKER STALLED", {
- cpuUsage: await monitor.checkCpuUsage(),
- memoryUsage: await monitor.checkMemoryUsage(),
- });
- }
-
- await sleep(cantAcceptConnectionInterval); // more sleep
- continue;
- } else if (!currentLiveness) {
- logger.info("Not accepting jobs because the liveness check failed");
-
- await sleep(cantAcceptConnectionInterval);
- continue;
- } else {
- cantAcceptConnectionCount = 0;
- }
-
- const job = await worker.getNextJob(token);
- if (job) {
- if (job.id) {
- runningJobs.add(job.id);
- }
+ await logExtract({
+ id: data.extractId,
+ request_id: data.extractId,
+ urls: data.request.urls,
+ team_id: data.teamId,
+ options: data.request,
+ model_kind: data.request.agent?.model ?? "fire-0",
+ credits_cost: 0,
+ is_successful: false,
+ error:
+ "Extract job crashed unexpectedly. Please try again or contact support if the issue persists.",
+ cost_tracking: undefined,
+ });
- processJobInternal(token, job).finally(() => {
- if (job.id) {
- runningJobs.delete(job.id);
- }
- });
+ // Send webhook notification
+ const sender = await createWebhookSender({
+ teamId: data.teamId,
+ jobId: data.extractId,
+ webhook: data.request.webhook,
+ v0: false,
+ });
- await sleep(gotJobInterval);
- } else {
- await sleep(connectionMonitorInterval);
- }
+ if (sender) {
+ sender.send(WebhookEvent.EXTRACT_FAILED, {
+ success: false,
+ error:
+ "Extract job crashed unexpectedly. Please try again or contact support if the issue persists.",
+ });
}
+
+ logger.info("DLQ job processed - extract marked as failed");
};
-// Start all workers
+// Start the worker
const app = Express();
-let currentLiveness: boolean = true;
-
-app.get("/liveness", (req, res) => {
- _logger.info("Liveness endpoint hit");
- if (config.USE_DB_AUTHENTICATION) {
- // networking check for Kubernetes environments
- const host = config.FIRECRAWL_APP_HOST;
- const port = config.FIRECRAWL_APP_PORT;
- const scheme = config.FIRECRAWL_APP_SCHEME;
-
- robustFetch({
- url: `${scheme}://${host}:${port}`,
- method: "GET",
- mock: null,
- logger: _logger,
- abort: AbortSignal.timeout(5000),
- ignoreResponse: true,
- useCacheableLookup: false,
- })
- .then(() => {
- currentLiveness = true;
- res.status(200).json({ ok: true });
- })
- .catch(e => {
- _logger.error("WORKER NETWORKING CHECK FAILED", { error: e });
- currentLiveness = false;
- res.status(500).json({ ok: false });
- });
- } else {
- currentLiveness = true;
- res.status(200).json({ ok: true });
- }
+app.get("/health", (req, res) => {
+ res.status(200).json({ ok: true });
});
const workerPort = config.EXTRACT_WORKER_PORT || config.PORT;
app.listen(workerPort, () => {
- _logger.info(`Liveness endpoint is running on port ${workerPort}`);
+ _logger.info(
+ `Extract worker health endpoint is running on port ${workerPort}`,
+ );
});
+async function shutdown() {
+ _logger.info("Shutting down extract worker...");
+ await shutdownExtractQueue();
+ _logger.info("Extract worker shut down");
+ process.exit(0);
+}
+
+if (require.main === module) {
+ process.on("SIGINT", () => {
+ _logger.debug("Received SIGINT. Shutting down gracefully...");
+ shutdown();
+ });
+
+ process.on("SIGTERM", () => {
+ _logger.debug("Received SIGTERM. Shutting down gracefully...");
+ shutdown();
+ });
+}
+
(async () => {
setSentryServiceTag("extract-worker");
@@ -309,14 +234,13 @@ app.listen(workerPort, () => {
initializeEngineForcing();
- await Promise.all([workerFun(getExtractQueue(), processExtractJobInternal)]);
+ _logger.info("Starting extract worker with RabbitMQ...");
- _logger.info("All workers exited. Waiting for all jobs to finish...");
+ // Start consuming from both the main queue and the DLQ
+ await Promise.all([
+ consumeExtractJobs(processExtractJob),
+ consumeExtractDLQ(processDLQJob),
+ ]);
- while (runningJobs.size > 0) {
- await new Promise(resolve => setTimeout(resolve, 500));
- }
-
- _logger.info("All jobs finished. Shutting down...");
- process.exit(0);
+ _logger.info("Extract worker started, consuming from RabbitMQ");
})();
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index 7d26584ce..02a5325ff 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -16,6 +16,7 @@ import { hasFormatOfType } from "../../lib/format-utils";
import type { Document, ScrapeOptions } from "../../controllers/v2/types";
import type { CostTracking } from "../../lib/cost-tracking";
import type { Logger } from "winston";
+import { saveExtractResult } from "../../lib/extract/extract-redis";
configDotenv();
const previewTeamId = "3adefd26-77ec-5968-8dcf-c94b5630d1de";
@@ -490,7 +491,12 @@ export async function logExtract(
);
if (extract.result) {
- await saveExtractToGCS(extract);
+ if (config.GCS_BUCKET_NAME) {
+ await saveExtractToGCS(extract);
+ } else {
+ // Fallback: save result to Redis with 24h TTL when GCS is not configured
+ await saveExtractResult(extract.id, extract.result);
+ }
}
}
diff --git a/apps/api/src/services/queue-service.ts b/apps/api/src/services/queue-service.ts
index 465ec6761..04dc0b263 100644
--- a/apps/api/src/services/queue-service.ts
+++ b/apps/api/src/services/queue-service.ts
@@ -3,8 +3,8 @@ import { config } from "../config";
import { logger } from "../lib/logger";
import IORedis from "ioredis";
import type { DeepResearchServiceOptions } from "../lib/deep-research/deep-research-service";
+import { addExtractJob, ExtractJobData } from "./extract-queue";
-let extractQueue: Queue;
let loggingQueue: Queue;
let indexQueue: Queue;
let deepResearchQueue: Queue;
@@ -25,27 +25,16 @@ export function getRedisConnection(): IORedis {
return redisConnection;
}
-const extractQueueName = "{extractQueue}";
const generateLlmsTxtQueueName = "{generateLlmsTxtQueue}";
const deepResearchQueueName = "{deepResearchQueue}";
const billingQueueName = "{billingQueue}";
export const precrawlQueueName = "{precrawlQueue}";
-export function getExtractQueue() {
- if (!extractQueue) {
- extractQueue = new Queue(extractQueueName, {
- connection: getRedisConnection(),
- defaultJobOptions: {
- removeOnComplete: {
- age: 90000, // 25 hours
- },
- removeOnFail: {
- age: 90000, // 25 hours
- },
- },
- });
- }
- return extractQueue;
+export async function addExtractJobToQueue(
+ extractId: string,
+ data: ExtractJobData,
+): Promise<void> {
+ await addExtractJob(extractId, data);
}
export function getGenerateLlmsTxtQueue() {
diff --git a/apps/api/src/services/worker/nuq.ts b/apps/api/src/services/worker/nuq.ts
index 12e366c11..6d347e09c 100644
--- a/apps/api/src/services/worker/nuq.ts
+++ b/apps/api/src/services/worker/nuq.ts
@@ -264,37 +264,59 @@ class NuQ<JobData = any, JobReturnValue = any> {
connection: amqp.ChannelModel;
channel: amqp.Channel;
} | null = null;
+ private senderStarting = false;
private async startSender() {
- if (this.sender || this.shuttingDown) return;
+ if (this.sender || this.shuttingDown || this.senderStarting) return;
+ this.senderStarting = true;
- if (config.NUQ_RABBITMQ_URL) {
- const connection = await amqp.connect(config.NUQ_RABBITMQ_URL);
- const channel = await connection.createChannel();
- await channel.assertQueue(this.queueName + ".prefetch", {
- durable: true,
- arguments: {
- "x-queue-type": "quorum",
- "x-max-length": 20000,
- },
- });
+ try {
+ if (config.NUQ_RABBITMQ_URL) {
+ const connection = await amqp.connect(config.NUQ_RABBITMQ_URL);
+ const channel = await connection.createChannel();
+ await channel.assertQueue(this.queueName + ".prefetch", {
+ durable: true,
+ arguments: {
+ "x-queue-type": "quorum",
+ "x-max-length": 20000,
+ },
+ });
- this.sender = {
- type: "rabbitmq",
- connection,
- channel,
- };
+ this.sender = {
+ type: "rabbitmq",
+ connection,
+ channel,
+ };
- channel.on("close", () => {
- logger.info("NuQ sender channel closed", { module: "nuq/rabbitmq" });
- connection.close().catch(() => {});
- this.sender = null;
- });
+ channel.on("close", () => {
+ logger.info("NuQ sender channel closed", { module: "nuq/rabbitmq" });
+ connection.close().catch(() => {});
+ this.sender = null;
+ });
- connection.on("close", () => {
- logger.info("NuQ sender connection closed", { module: "nuq/rabbitmq" });
- this.sender = null;
- });
+ channel.on("error", err => {
+ logger.error("NuQ sender channel error", {
+ module: "nuq/rabbitmq",
+ err,
+ });
+ });
+
+ connection.on("close", () => {
+ logger.info("NuQ sender connection closed", {
+ module: "nuq/rabbitmq",
+ });
+ this.sender = null;
+ });
+
+ connection.on("error", err => {
+ logger.error("NuQ sender connection error", {
+ module: "nuq/rabbitmq",
+ err,
+ });
+ });
+ }
+ } finally {
+ this.senderStarting = false;
}
}
@@ -1199,14 +1221,23 @@ class NuQ<JobData = any, JobReturnValue = any> {
await this.startSender();
if (this.sender) {
- const job = await this.sender.channel.get(
- this.queueName + ".prefetch",
- { noAck: true },
- );
- if (job !== false) {
- return this.rowToJob(JSON.parse(job.content.toString()));
- } else {
- return null;
+ try {
+ const job = await this.sender.channel.get(
+ this.queueName + ".prefetch",
+ { noAck: true },
+ );
+ if (job !== false) {
+ return this.rowToJob(JSON.parse(job.content.toString()));
+ } else {
+ return null;
+ }
+ } catch (err) {
+ logger.warn("NuQ sender get failed, falling back to postgres", {
+ module: "nuq/rabbitmq",
+ err,
+ });
+ // Reset sender so it can be re-established on next call
+ this.sender = null;
}
} else {
logger.warn("NuQ sender not started, falling back to postgres", {
diff --git a/docker-compose.yaml b/docker-compose.yaml
index 13b57b2f8..b66cbec0c 100644
--- a/docker-compose.yaml
+++ b/docker-compose.yaml
@@ -92,10 +92,12 @@ services:
PORT: ${INTERNAL_PORT:-3002}
EXTRACT_WORKER_PORT: ${EXTRACT_WORKER_PORT:-3004}
WORKER_PORT: ${WORKER_PORT:-3005}
+ NUQ_RABBITMQ_URL: amqp://rabbitmq:5672
ENV: local
depends_on:
- redis
- playwright-service
+ - rabbitmq
ports:
- "${PORT:-3002}:${INTERNAL_PORT:-3002}"
command: node dist/src/harness.js --start-docker
@@ -121,6 +123,18 @@ services:
max-size: "5m"
max-file: "2"
compress: "true"
+
+ rabbitmq:
+ image: rabbitmq:3-management
+ networks:
+ - backend
+ command: rabbitmq-server
+ logging:
+ driver: "json-file"
+ options:
+ max-size: "5m"
+ max-file: "2"
+ compress: "true"
nuq-postgres:
build: apps/nuq-postgres