Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): schedule a 24h delete for each ZDR blob and bound cleanup retries (#5352)

Gergő Móriczd00d4483ea8a
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.yml
index ead32ad7e..1da4c1340 100644
--- a/.github/workflows/test-server.yml
+++ b/.github/workflows/test-server.yml
@@ -416,6 +416,17 @@ jobs:
--reporter=default
working-directory: apps/api
+ - name: Run ZDR cleanup regression tests
+ if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'
+ run: >-
+ pnpm exec vitest run
+ src/lib/zdr-queue.test.ts
+ src/lib/zdrcleaner.test.ts
+ src/lib/gcs-jobs.test.ts
+ src/services/logging/log_job.test.ts
+ --reporter=default
+ working-directory: apps/api
+
- name: Run country check route regression tests
if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'
run: pnpm exec vitest run src/__tests__/routes/country-check.routes.test.ts --reporter=default
diff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.ts
index fa7f50dd4..64c1ddae6 100644
--- a/apps/api/src/controllers/v2/map.ts
+++ b/apps/api/src/controllers/v2/map.ts
@@ -173,7 +173,7 @@ export async function mapController(
},
results: avgrabResults,
credits_cost: creditsCost,
- zeroDataRetention: false,
+ zeroDataRetention,
}).catch(error => {
logger.error(
`Failed to log job for team ${req.auth.team_id}: ${error}`,
diff --git a/apps/api/src/lib/gcs-jobs.test.ts b/apps/api/src/lib/gcs-jobs.test.ts
new file mode 100644
index 000000000..c09b344eb
--- /dev/null
+++ b/apps/api/src/lib/gcs-jobs.test.ts
@@ -0,0 +1,128 @@
+import { vi } from "vitest";
+import type { LoggedScrape } from "../services/logging/log_job";
+
+const { save, remove, enqueueZdrCleanupJob, spans } = vi.hoisted(() => {
+ const spans: Array<{ name: string; attributes: Record<string, unknown> }> =
+ [];
+ return {
+ save: vi.fn(async () => {}),
+ remove: vi.fn(async (): Promise<[unknown]> => [{ statusCode: 204 }]),
+ enqueueZdrCleanupJob: vi.fn(async () => {}),
+ spans,
+ };
+});
+
+vi.mock("@google-cloud/storage", () => ({
+ ApiError: class extends Error {
+ code?: number;
+ },
+ Storage: class {
+ bucket() {
+ return { file: () => ({ save, delete: remove }) };
+ }
+ },
+}));
+vi.mock("../config", () => ({
+ config: { GCS_BUCKET_NAME: "bucket", USE_DB_AUTHENTICATION: true },
+}));
+vi.mock("./zdr-queue", () => ({ enqueueZdrCleanupJob }));
+vi.mock("./otel-tracer", () => ({
+ withSpan: async (name: string, fn: (span: any) => any) => {
+ const span = { attributes: {} };
+ spans.push({ name, attributes: span.attributes });
+ return fn(span);
+ },
+ setSpanAttributes: (
+ span: { attributes: Record<string, unknown> },
+ attributes: object,
+ ) => Object.assign(span.attributes, attributes),
+}));
+vi.mock("./logger", () => {
+ const logger: any = {
+ child: vi.fn(() => logger),
+ debug: vi.fn(),
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ };
+ return { logger };
+});
+
+import { removeJobFromGCS, saveScrapeToGCS } from "./gcs-jobs";
+
+const scrapeId = "019e6f45-7778-727d-adf0-0abe9d5062b6";
+
+function scrape(zeroDataRetention: boolean): LoggedScrape {
+ return {
+ id: scrapeId,
+ request_id: "019e6f45-7778-727d-adf0-0abe9d5062b7",
+ url: "https://example.com",
+ is_successful: true,
+ doc: { markdown: "content", metadata: {} },
+ time_taken: 1,
+ team_id: "team-id",
+ options: {},
+ credits_cost: 1,
+ skipNuq: false,
+ zeroDataRetention,
+ } as unknown as LoggedScrape;
+}
+
+describe("GCS job blobs", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ spans.length = 0;
+ });
+
+ it("schedules a ZDR blob's delete before writing it", async () => {
+ await saveScrapeToGCS(scrape(true));
+
+ expect(enqueueZdrCleanupJob).toHaveBeenCalledWith({ blobId: scrapeId });
+ expect(save).toHaveBeenCalledOnce();
+ expect(enqueueZdrCleanupJob.mock.invocationCallOrder[0]).toBeLessThan(
+ save.mock.invocationCallOrder[0],
+ );
+ });
+
+ it("does not schedule a delete for a non-ZDR blob", async () => {
+ await saveScrapeToGCS(scrape(false));
+
+ expect(enqueueZdrCleanupJob).not.toHaveBeenCalled();
+ expect(save).toHaveBeenCalledOnce();
+ });
+
+ it("does not write a ZDR blob when its delete could not be scheduled", async () => {
+ enqueueZdrCleanupJob.mockRejectedValueOnce(
+ new Error("RabbitMQ unavailable"),
+ );
+
+ await expect(saveScrapeToGCS(scrape(true))).rejects.toThrow(
+ "RabbitMQ unavailable",
+ );
+ expect(save).not.toHaveBeenCalled();
+ });
+
+ it("records whether a removed blob still existed", async () => {
+ await removeJobFromGCS("blob-found");
+ remove.mockResolvedValueOnce([{ statusCode: 404 }]);
+ await removeJobFromGCS("blob-missing");
+
+ expect(remove).toHaveBeenCalledWith({ ignoreNotFound: true });
+ expect(
+ spans
+ .filter(span => span.name === "firecrawl-gcs-remove-job")
+ .map(span => span.attributes),
+ ).toEqual([
+ expect.objectContaining({
+ "job.id": "blob-found",
+ "gcs.delete_successful": true,
+ "gcs.job_not_found": false,
+ }),
+ expect.objectContaining({
+ "job.id": "blob-missing",
+ "gcs.delete_successful": true,
+ "gcs.job_not_found": true,
+ }),
+ ]);
+ });
+});
diff --git a/apps/api/src/lib/gcs-jobs.ts b/apps/api/src/lib/gcs-jobs.ts
index 92d86f179..eedabb8b6 100644
--- a/apps/api/src/lib/gcs-jobs.ts
+++ b/apps/api/src/lib/gcs-jobs.ts
@@ -11,6 +11,7 @@ import type {
LoggedSearch,
} from "../services/logging/log_job";
import { config } from "../config";
+import { enqueueZdrCleanupJob } from "./zdr-queue";
import crypto from "crypto";
import { Logger } from "winston";
@@ -108,6 +109,11 @@ async function saveJobToGCS(params: {
return;
}
+ // Never write a ZDR blob without its delete already scheduled.
+ if (params.zeroDataRetention && config.USE_DB_AUTHENTICATION === true) {
+ await enqueueZdrCleanupJob({ blobId: params.id });
+ }
+
const bucket = storageManualRetries.bucket(config.GCS_BUCKET_NAME);
const blob = bucket.file(filename);
@@ -443,10 +449,14 @@ export async function removeJobFromGCS(
const blob = bucket.file(idToFilename(jobId));
try {
- await blob.delete({
+ const [response] = await blob.delete({
ignoreNotFound: true,
});
- setSpanAttributes(span, { "gcs.delete_successful": true });
+ setSpanAttributes(span, {
+ "gcs.delete_successful": true,
+ "gcs.job_not_found":
+ (response as { statusCode?: number } | undefined)?.statusCode === 404,
+ });
} catch (error) {
if (
error instanceof ApiError &&
diff --git a/apps/api/src/lib/zdr-queue.test.ts b/apps/api/src/lib/zdr-queue.test.ts
index 2183540e1..4c52dfe16 100644
--- a/apps/api/src/lib/zdr-queue.test.ts
+++ b/apps/api/src/lib/zdr-queue.test.ts
@@ -94,13 +94,17 @@ vi.mock("./otel-tracer", () => ({
recordSpanException,
}));
-function message(body: unknown) {
+function message(
+ body: unknown,
+ headers?: Record<string, unknown>,
+ redelivered = false,
+) {
return {
content: Buffer.from(
typeof body === "string" ? body : JSON.stringify(body),
),
- fields: {},
- properties: {},
+ fields: { redelivered },
+ properties: { headers },
};
}
@@ -118,7 +122,7 @@ describe("ZDR cleanup queue", () => {
"./zdr-queue.js"
);
- await enqueueZdrCleanupJob("request-1");
+ await enqueueZdrCleanupJob({ requestId: "request-1" });
expect(connect).toHaveBeenCalledWith(
"amqp://rabbitmq",
@@ -135,6 +139,7 @@ describe("ZDR cleanup queue", () => {
correlationId: expect.any(String),
timestamp: expect.any(Number),
mandatory: true,
+ headers: { "zdr-attempt": 1 },
}),
expect.any(Function),
);
@@ -151,6 +156,27 @@ describe("ZDR cleanup queue", () => {
await shutdownZdrQueue();
});
+ it("publishes a per-blob job to the same 24h delay queue", async () => {
+ const { enqueueZdrCleanupJob, shutdownZdrQueue } = await import(
+ "./zdr-queue.js"
+ );
+
+ await enqueueZdrCleanupJob({ blobId: "blob-1" });
+
+ expect(publishChannel.sendToQueue).toHaveBeenCalledWith(
+ "zdr.cleanup.delayed.24h",
+ Buffer.from(JSON.stringify({ blobId: "blob-1" })),
+ expect.objectContaining({
+ persistent: true,
+ messageId: "blob-1",
+ mandatory: true,
+ headers: { "zdr-attempt": 1 },
+ }),
+ expect.any(Function),
+ );
+ await shutdownZdrQueue();
+ });
+
it("retries a mandatory publish returned as unroutable", async () => {
vi.useFakeTimers();
try {
@@ -172,7 +198,7 @@ describe("ZDR cleanup queue", () => {
"./zdr-queue.js"
);
- const enqueue = enqueueZdrCleanupJob("request-returned");
+ const enqueue = enqueueZdrCleanupJob({ requestId: "request-returned" });
await vi.runAllTimersAsync();
await enqueue;
@@ -188,7 +214,7 @@ describe("ZDR cleanup queue", () => {
"./zdr-queue.js"
);
- await enqueueZdrCleanupJob("request-2");
+ await enqueueZdrCleanupJob({ requestId: "request-2" });
const queueDeclarations = new Map(
(publishChannel.assertQueue as Mock).mock.calls.map(([name, options]) => [
@@ -213,12 +239,25 @@ describe("ZDR cleanup queue", () => {
"x-queue-type": "quorum",
"x-overflow": "reject-publish",
"x-delivery-limit": -1,
+ "x-dead-letter-exchange": "zdr.cleanup",
+ "x-dead-letter-routing-key": "dead",
+ },
+ });
+ expect(queueDeclarations.get("zdr.cleanup.retry")).toMatchObject({
+ durable: true,
+ arguments: {
+ "x-queue-type": "quorum",
+ "x-overflow": "reject-publish",
+ "x-message-ttl": 300_000,
+ "x-dead-letter-strategy": "at-least-once",
+ "x-dead-letter-exchange": "zdr.cleanup",
+ "x-dead-letter-routing-key": "ready",
},
});
await shutdownZdrQueue();
});
- it("acks completed jobs and requeues failures without a delivery cap", async () => {
+ it("hands request-level and per-blob jobs to the handler and acks them", async () => {
const { consumeZdrCleanupJobs, shutdownZdrQueue } = await import(
"./zdr-queue.js"
);
@@ -226,30 +265,180 @@ describe("ZDR cleanup queue", () => {
await consumeZdrCleanupJobs(handler);
const consume = consumerCallbacks[0];
- const completed = message({ requestId: "request-3" });
- await consume(completed);
- expect(handler).toHaveBeenCalledWith({ requestId: "request-3" });
- expect(consumeChannel.ack).toHaveBeenCalledWith(completed);
- expect(spans).toContainEqual({
- name: "zdr.rabbitmq.consume",
- attributes: expect.objectContaining({
- "messaging.system": "rabbitmq",
+ const requestJob = message({ requestId: "request-3" });
+ await consume(requestJob);
+ const blobJob = message({ blobId: "blob-3" });
+ await consume(blobJob);
+
+ expect(handler.mock.calls).toEqual([
+ [{ requestId: "request-3" }],
+ [{ blobId: "blob-3" }],
+ ]);
+ expect(consumeChannel.ack).toHaveBeenCalledWith(requestJob);
+ expect(consumeChannel.ack).toHaveBeenCalledWith(blobJob);
+ expect(consumeChannel.nack).not.toHaveBeenCalled();
+ const consumeSpans = spans.filter(
+ span => span.name === "zdr.rabbitmq.consume",
+ );
+ expect(consumeSpans.map(span => span.attributes)).toEqual([
+ expect.objectContaining({
"messaging.destination.name": "zdr.cleanup.ready",
"messaging.operation.name": "process",
"zdr.request_id": "request-3",
+ "zdr.attempt": 1,
"zdr.rabbitmq.outcome": "acknowledged",
}),
+ expect.objectContaining({
+ "zdr.blob_id": "blob-3",
+ "zdr.attempt": 1,
+ "zdr.rabbitmq.outcome": "acknowledged",
+ }),
+ ]);
+ expect(consumeSpans[1].attributes).not.toHaveProperty("zdr.request_id");
+ await shutdownZdrQueue();
+ });
+
+ it("schedules a delayed retry instead of requeueing a failed job", async () => {
+ const { consumeZdrCleanupJobs, shutdownZdrQueue } = await import(
+ "./zdr-queue.js"
+ );
+ const handler = vi.fn(async () => {
+ throw new Error("GCS unavailable");
});
+ await consumeZdrCleanupJobs(handler);
+ const consume = consumerCallbacks[0];
- handler.mockRejectedValueOnce(new Error("GCS unavailable"));
- const failed = message({ requestId: "request-4" });
- await consume(failed);
- expect(consumeChannel.nack).toHaveBeenCalledWith(failed, false, true);
+ const firstFailure = message({ blobId: "blob-4" });
+ await consume(firstFailure);
+ const laterFailure = message(
+ { requestId: "request-4" },
+ { "zdr-attempt": 5 },
+ );
+ await consume(laterFailure);
+
+ expect(publishChannel.sendToQueue).toHaveBeenNthCalledWith(
+ 1,
+ "zdr.cleanup.retry",
+ Buffer.from(JSON.stringify({ blobId: "blob-4" })),
+ expect.objectContaining({
+ persistent: true,
+ messageId: "blob-4",
+ mandatory: true,
+ headers: { "zdr-attempt": 2 },
+ }),
+ expect.any(Function),
+ );
+ expect(publishChannel.sendToQueue).toHaveBeenNthCalledWith(
+ 2,
+ "zdr.cleanup.retry",
+ Buffer.from(JSON.stringify({ requestId: "request-4" })),
+ expect.objectContaining({ headers: { "zdr-attempt": 6 } }),
+ expect.any(Function),
+ );
+ expect(consumeChannel.ack).toHaveBeenCalledWith(firstFailure);
+ expect(consumeChannel.ack).toHaveBeenCalledWith(laterFailure);
+ expect(consumeChannel.nack).not.toHaveBeenCalled();
+ expect(recordSpanException).toHaveBeenCalledTimes(2);
+ expect(spans).toContainEqual({
+ name: "zdr.rabbitmq.consume",
+ attributes: expect.objectContaining({
+ "zdr.blob_id": "blob-4",
+ "zdr.rabbitmq.outcome": "retry_scheduled",
+ }),
+ });
+ await shutdownZdrQueue();
+ });
+
+ it("dead-letters a job that fails its last attempt", async () => {
+ const { consumeZdrCleanupJobs, shutdownZdrQueue } = await import(
+ "./zdr-queue.js"
+ );
+ const handler = vi.fn(async () => {
+ throw new Error("GCS unavailable");
+ });
+ await consumeZdrCleanupJobs(handler);
+ const consume = consumerCallbacks[0];
+
+ const exhausted = message({ blobId: "blob-5" }, { "zdr-attempt": 24 });
+ await consume(exhausted);
+
+ expect(handler).toHaveBeenCalledOnce();
+ expect(consumeChannel.nack).toHaveBeenCalledWith(exhausted, false, false);
+ expect(consumeChannel.ack).not.toHaveBeenCalled();
+ expect(publishChannel.sendToQueue).not.toHaveBeenCalled();
+ expect(spans).toContainEqual({
+ name: "zdr.rabbitmq.consume",
+ attributes: expect.objectContaining({
+ "zdr.blob_id": "blob-5",
+ "zdr.attempt": 24,
+ "zdr.rabbitmq.outcome": "dead_lettered",
+ }),
+ });
+ await shutdownZdrQueue();
+ });
+
+ it("counts an unsettled redelivery as a failed attempt", async () => {
+ const { consumeZdrCleanupJobs, shutdownZdrQueue } = await import(
+ "./zdr-queue.js"
+ );
+ const handler = vi.fn(async () => {});
+ await consumeZdrCleanupJobs(handler);
+ const consume = consumerCallbacks[0];
+
+ const redelivered = message(
+ { blobId: "blob-6" },
+ { "zdr-attempt": 3 },
+ true,
+ );
+ await consume(redelivered);
+ const lastRedelivery = message(
+ { blobId: "blob-7" },
+ { "zdr-attempt": 24 },
+ true,
+ );
+ await consume(lastRedelivery);
+
+ expect(handler).not.toHaveBeenCalled();
+ expect(publishChannel.sendToQueue).toHaveBeenCalledOnce();
+ expect(publishChannel.sendToQueue).toHaveBeenCalledWith(
+ "zdr.cleanup.retry",
+ Buffer.from(JSON.stringify({ blobId: "blob-6" })),
+ expect.objectContaining({ headers: { "zdr-attempt": 4 } }),
+ expect.any(Function),
+ );
+ expect(consumeChannel.ack).toHaveBeenCalledWith(redelivered);
+ expect(consumeChannel.nack).toHaveBeenCalledWith(
+ lastRedelivery,
+ false,
+ false,
+ );
+ await shutdownZdrQueue();
+ });
+
+ it("dead-letters malformed and invalid jobs without calling the handler", async () => {
+ const { consumeZdrCleanupJobs, shutdownZdrQueue } = await import(
+ "./zdr-queue.js"
+ );
+ const handler = vi.fn(async () => {});
+ await consumeZdrCleanupJobs(handler);
+ const consume = consumerCallbacks[0];
const malformed = message("not-json");
+ const emptyBlob = message({ blobId: "" });
+ const unknownShape = message({ jobId: "job-1" });
await consume(malformed);
+ await consume(emptyBlob);
+ await consume(unknownShape);
+
+ expect(handler).not.toHaveBeenCalled();
expect(consumeChannel.nack).toHaveBeenCalledWith(malformed, false, false);
- expect(recordSpanException).toHaveBeenCalled();
+ expect(consumeChannel.nack).toHaveBeenCalledWith(emptyBlob, false, false);
+ expect(consumeChannel.nack).toHaveBeenCalledWith(
+ unknownShape,
+ false,
+ false,
+ );
+ expect(recordSpanException).toHaveBeenCalledTimes(3);
await shutdownZdrQueue();
});
});
diff --git a/apps/api/src/lib/zdr-queue.ts b/apps/api/src/lib/zdr-queue.ts
index 189a9d978..e662beccd 100644
--- a/apps/api/src/lib/zdr-queue.ts
+++ b/apps/api/src/lib/zdr-queue.ts
@@ -12,6 +12,7 @@ const DELAY_MS = 24 * 60 * 60 * 1000;
const EXCHANGE = "zdr.cleanup";
const DELAY_QUEUE = "zdr.cleanup.delayed.24h";
const READY_QUEUE = "zdr.cleanup.ready";
+const RETRY_QUEUE = "zdr.cleanup.retry";
const DLQ = "zdr.cleanup.dlq";
const READY_ROUTING_KEY = "ready";
const DLQ_ROUTING_KEY = "dead";
@@ -21,12 +22,14 @@ const RECONNECT_MAX_DELAY_MS = 30_000;
// well inside the API pod's shutdown grace.
const PUBLISH_MAX_ATTEMPTS = 5;
const PUBLISH_ATTEMPT_TIMEOUT_MS = 5_000;
+const RETRY_DELAY_MS = 5 * 60 * 1000;
+const MAX_ATTEMPTS = 24;
+const ATTEMPT_HEADER = "zdr-attempt";
const logger = _logger.child({ module: "zdr-queue" });
-type ZdrCleanupJob = {
- requestId: string;
-};
+// A request-level job resolves its blobs from ClickHouse; a blob job names one blob.
+export type ZdrCleanupJob = { requestId: string } | { blobId: string };
type CleanupHandler = (job: ZdrCleanupJob) => Promise<void>;
@@ -82,6 +85,18 @@ async function assertTopology(ch: amqp.Channel): Promise<void> {
"x-dead-letter-routing-key": READY_ROUTING_KEY,
},
});
+
+ await ch.assertQueue(RETRY_QUEUE, {
+ durable: true,
+ arguments: {
+ "x-queue-type": "quorum",
+ "x-overflow": "reject-publish",
+ "x-message-ttl": RETRY_DELAY_MS,
+ "x-dead-letter-strategy": "at-least-once",
+ "x-dead-letter-exchange": EXCHANGE,
+ "x-dead-letter-routing-key": READY_ROUTING_KEY,
+ },
+ });
}
function resetCachedState(): void {
@@ -229,9 +244,15 @@ async function getConsumeChannel(): Promise<amqp.Channel> {
return consumeChannelPromise;
}
+function jobId(job: ZdrCleanupJob): string {
+ return "blobId" in job ? job.blobId : job.requestId;
+}
+
function publishConfirmed(
ch: amqp.ConfirmChannel,
+ queue: string,
job: ZdrCleanupJob,
+ attempt: number,
): Promise<void> {
const correlationId = randomUUID();
const publish = { returned: false };
@@ -240,15 +261,16 @@ function publishConfirmed(
return new Promise((resolve, reject) => {
ch.sendToQueue(
- DELAY_QUEUE,
+ queue,
Buffer.from(JSON.stringify(job)),
{
persistent: true,
contentType: "application/json",
- messageId: job.requestId,
+ messageId: jobId(job),
correlationId,
timestamp: Math.floor(Date.now() / 1000),
mandatory: true,
+ headers: { [ATTEMPT_HEADER]: attempt },
},
error => {
setImmediate(() => {
@@ -263,25 +285,27 @@ function publishConfirmed(
});
}
-export async function enqueueZdrCleanupJob(requestId: string): Promise<void> {
- if (!config.NUQ_RABBITMQ_URL) {
- throw new Error("NUQ_RABBITMQ_URL is not configured");
- }
-
+async function publishUntilConfirmed(
+ queue: string,
+ job: ZdrCleanupJob,
+ attempt: number,
+): Promise<void> {
let delayMs = RECONNECT_BASE_DELAY_MS;
- for (let attempt = 1; !closed; attempt++) {
+ for (let publishAttempt = 1; !closed; publishAttempt++) {
let attemptTimer: NodeJS.Timeout | undefined;
try {
await withSpan("zdr.rabbitmq.publish", async span => {
setSpanAttributes(span, {
"messaging.system": "rabbitmq",
- "messaging.destination.name": DELAY_QUEUE,
+ "messaging.destination.name": queue,
"messaging.operation.name": "publish",
- "messaging.message.id": requestId,
+ "messaging.message.id": jobId(job),
"zdr.rabbitmq.retry_delay_ms": delayMs,
});
await Promise.race([
- getPublishChannel().then(ch => publishConfirmed(ch, { requestId })),
+ getPublishChannel().then(ch =>
+ publishConfirmed(ch, queue, job, attempt),
+ ),
new Promise<never>((_, reject) => {
attemptTimer = setTimeout(
() => reject(new Error("ZDR publish attempt timed out")),
@@ -295,19 +319,21 @@ export async function enqueueZdrCleanupJob(requestId: string): Promise<void> {
});
return;
} catch (error) {
- if (attempt >= PUBLISH_MAX_ATTEMPTS) {
+ if (publishAttempt >= PUBLISH_MAX_ATTEMPTS) {
logger.error("Failed to enqueue ZDR cleanup job; giving up", {
error,
- requestId,
- attempts: attempt,
+ queue,
+ messageId: jobId(job),
+ attempts: publishAttempt,
});
throw new Error(
- `ZDR cleanup job not confirmed after ${attempt} attempts`,
+ `ZDR cleanup job not confirmed after ${publishAttempt} attempts`,
);
}
logger.error("Failed to durably enqueue ZDR cleanup job; retrying", {
error,
- requestId,
+ queue,
+ messageId: jobId(job),
delayMs,
});
await publishChannel?.close().catch(() => {});
@@ -323,13 +349,23 @@ export async function enqueueZdrCleanupJob(requestId: string): Promise<void> {
throw new Error("ZDR queue closed before the cleanup job was confirmed");
}
+/** Schedules a cleanup job 24 hours out; throws if the broker never confirms it. */
+export async function enqueueZdrCleanupJob(job: ZdrCleanupJob): Promise<void> {
+ if (!config.NUQ_RABBITMQ_URL) {
+ throw new Error("NUQ_RABBITMQ_URL is not configured");
+ }
+ await publishUntilConfirmed(DELAY_QUEUE, job, 1);
+}
+
function isZdrCleanupJob(value: unknown): value is ZdrCleanupJob {
- return (
- typeof value === "object" &&
- value !== null &&
- typeof (value as Partial<ZdrCleanupJob>).requestId === "string" &&
- (value as ZdrCleanupJob).requestId.length > 0
- );
+ if (typeof value !== "object" || value === null) return false;
+ const id =
+ "blobId" in value
+ ? value.blobId
+ : "requestId" in value
+ ? value.requestId
+ : undefined;
+ return typeof id === "string" && id.length > 0;
}
async function subscribe(
@@ -346,13 +382,14 @@ async function subscribe(
scheduleReconnect();
return;
}
+ const messageId = msg.properties.messageId;
try {
await withSpan("zdr.rabbitmq.consume", async span => {
setSpanAttributes(span, {
"messaging.system": "rabbitmq",
"messaging.destination.name": READY_QUEUE,
"messaging.operation.name": "process",
- "messaging.message.id": msg.properties.messageId,
+ "messaging.message.id": messageId,
"messaging.rabbitmq.delivery_tag": msg.fields.deliveryTag,
"messaging.rabbitmq.redelivered": msg.fields.redelivered,
});
@@ -381,21 +418,59 @@ async function subscribe(
return;
}
+ const attempt = Number(msg.properties.headers?.[ATTEMPT_HEADER]) || 1;
setSpanAttributes(span, {
- "zdr.request_id": job.requestId,
+ ...("blobId" in job
+ ? { "zdr.blob_id": job.blobId }
+ : { "zdr.request_id": job.requestId }),
+ "zdr.attempt": attempt,
});
- await handler(job);
+
+ try {
+ // An unsettled earlier delivery (worker died, channel closed) counts as a failed attempt.
+ if (msg.fields.redelivered) {
+ throw new Error("ZDR cleanup job was redelivered unsettled");
+ }
+ await handler(job);
+ } catch (error) {
+ recordSpanException(span, error);
+ if (attempt >= MAX_ATTEMPTS) {
+ logger.error(
+ "ZDR cleanup job failed its last attempt; dead-lettering",
+ {
+ error,
+ messageId,
+ attempt,
+ },
+ );
+ ch.nack(msg, false, false);
+ setSpanAttributes(span, {
+ "zdr.rabbitmq.outcome": "dead_lettered",
+ });
+ return;
+ }
+ logger.warn("ZDR cleanup job failed; scheduling a retry", {
+ error,
+ messageId,
+ attempt,
+ retryDelayMs: RETRY_DELAY_MS,
+ });
+ await publishUntilConfirmed(RETRY_QUEUE, job, attempt + 1);
+ ch.ack(msg);
+ setSpanAttributes(span, {
+ "zdr.rabbitmq.outcome": "retry_scheduled",
+ });
+ return;
+ }
+
ch.ack(msg);
setSpanAttributes(span, {
"zdr.rabbitmq.outcome": "acknowledged",
});
});
} catch (error) {
- logger.error("ZDR cleanup job failed; requeueing", {
- error,
- messageId: msg.properties.messageId,
- });
- ch.nack(msg, false, true);
+ // Left unacked, the broker redelivers it once this channel closes.
+ logger.error("Could not settle ZDR cleanup job", { error, messageId });
}
},
{ noAck: false },
diff --git a/apps/api/src/lib/zdrcleaner.test.ts b/apps/api/src/lib/zdrcleaner.test.ts
index bb0156f54..bc6632e90 100644
--- a/apps/api/src/lib/zdrcleaner.test.ts
+++ b/apps/api/src/lib/zdrcleaner.test.ts
@@ -43,7 +43,7 @@ vi.mock("./logger", () => {
});
vi.mock("./otel-tracer", () => ({ withSpan, setSpanAttributes }));
-import { cleanZdrRequest } from "./zdrcleaner";
+import { cleanZdrJob, cleanZdrRequest } from "./zdrcleaner";
import { logger } from "./logger";
function mockChildren(ids: string[]) {
@@ -142,3 +142,41 @@ describe("ZDR cleaner", () => {
expect(removeJobFromGCS).not.toHaveBeenCalled();
});
});
+
+describe("ZDR cleanup job dispatch", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ client.current = { query };
+ removeJobFromGCS.mockResolvedValue(undefined);
+ spans.length = 0;
+ });
+
+ it("deletes a per-blob job's blob without reading ClickHouse", async () => {
+ await cleanZdrJob({ blobId: "blob-1" });
+
+ expect(removeJobFromGCS.mock.calls.map(([id]) => id)).toEqual(["blob-1"]);
+ expect(query).not.toHaveBeenCalled();
+ });
+
+ it("propagates a per-blob delete failure so the job is retried", async () => {
+ removeJobFromGCS.mockRejectedValueOnce(new Error("GCS unavailable"));
+
+ await expect(cleanZdrJob({ blobId: "blob-2" })).rejects.toThrow(
+ "GCS unavailable",
+ );
+ expect(query).not.toHaveBeenCalled();
+ });
+
+ it("still resolves a request-level job's blobs from ClickHouse", async () => {
+ mockChildren(["scrape-blob"]);
+
+ await cleanZdrJob({ requestId: "request-5" });
+
+ expect(query.mock.calls[0][0]).toMatchObject({
+ query_params: { requestId: "request-5" },
+ });
+ expect(removeJobFromGCS.mock.calls.map(([id]) => id)).toEqual([
+ "scrape-blob",
+ ]);
+ });
+});
diff --git a/apps/api/src/lib/zdrcleaner.ts b/apps/api/src/lib/zdrcleaner.ts
index 2b3f1b55d..d48fe5f87 100644
--- a/apps/api/src/lib/zdrcleaner.ts
+++ b/apps/api/src/lib/zdrcleaner.ts
@@ -4,6 +4,7 @@ import { removeJobFromGCS } from "./gcs-jobs";
import { logger as _logger } from "./logger";
import { config } from "../config";
import { setSpanAttributes, withSpan } from "./otel-tracer";
+import type { ZdrCleanupJob } from "./zdr-queue";
export async function sendZdrHeartbeat() {
if (config.ZDRCLEANER_HEARTBEAT_URL) {
@@ -87,3 +88,11 @@ export async function cleanZdrRequest(requestId: string): Promise<void> {
setSpanAttributes(span, { "zdr.cleanup.outcome": "completed" });
});
}
+
+export async function cleanZdrJob(job: ZdrCleanupJob): Promise<void> {
+ if ("blobId" in job) {
+ await removeJobFromGCS(job.blobId);
+ return;
+ }
+ await cleanZdrRequest(job.requestId);
+}
diff --git a/apps/api/src/services/logging/log_job.test.ts b/apps/api/src/services/logging/log_job.test.ts
index d9210d8c7..08736704a 100644
--- a/apps/api/src/services/logging/log_job.test.ts
+++ b/apps/api/src/services/logging/log_job.test.ts
@@ -667,9 +667,9 @@ describe("logRequest", () => {
zeroDataRetention: true,
});
- expect(enqueueZdrCleanupJob).toHaveBeenCalledWith(
- "019e6f45-7778-727d-adf0-0abe9d5062b6",
- );
+ expect(enqueueZdrCleanupJob).toHaveBeenCalledWith({
+ requestId: "019e6f45-7778-727d-adf0-0abe9d5062b6",
+ });
expect(enqueueZdrCleanupJob.mock.invocationCallOrder[0]).toBeLessThan(
publishMessage.mock.invocationCallOrder[0],
);
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index c5532183d..09e37cabc 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -570,7 +570,7 @@ async function logRequestInternal(request: LoggedRequest) {
}
if (request.zeroDataRetention && config.USE_DB_AUTHENTICATION === true) {
- await enqueueZdrCleanupJob(request.id);
+ await enqueueZdrCleanupJob({ requestId: request.id });
}
await robustInsert(
diff --git a/apps/api/src/services/zdr-worker.ts b/apps/api/src/services/zdr-worker.ts
index 05cdc9e38..6eb113bf7 100644
--- a/apps/api/src/services/zdr-worker.ts
+++ b/apps/api/src/services/zdr-worker.ts
@@ -1,7 +1,7 @@
import "dotenv/config";
import { shutdownTracing } from "../otel";
import { logger } from "../lib/logger";
-import { cleanZdrRequest, sendZdrHeartbeat } from "../lib/zdrcleaner";
+import { cleanZdrJob, sendZdrHeartbeat } from "../lib/zdrcleaner";
import { consumeZdrCleanupJobs, shutdownZdrQueue } from "../lib/zdr-queue";
let isShuttingDown = false;
@@ -17,7 +17,7 @@ process.on("SIGTERM", () => {
});
(async () => {
- await consumeZdrCleanupJobs(job => cleanZdrRequest(job.requestId));
+ await consumeZdrCleanupJobs(cleanZdrJob);
// Cleanup is driven entirely by the delayed RabbitMQ queue; this loop only
// keeps the liveness heartbeat going while the consumer runs.