Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): schedule a 24h delete for each ZDR blob and bound cleanup retries (#5352)
Gergő Móriczd00d4483ea8a
diff --git a/.github/workflows/test-server.yml b/.github/workflows/test-server.ymlindex ead32ad7e..1da4c1340 100644--- a/.github/workflows/test-server.yml+++ b/.github/workflows/test-server.yml@@ -416,6 +416,17 @@ jobs:--reporter=defaultworking-directory: apps/api+ - name: Run ZDR cleanup regression tests+ if: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'+ run: >-+ pnpm exec vitest run+ src/lib/zdr-queue.test.ts+ src/lib/zdrcleaner.test.ts+ src/lib/gcs-jobs.test.ts+ src/services/logging/log_job.test.ts+ --reporter=default+ working-directory: apps/api+- name: Run country check route regression testsif: matrix.engine == 'fetch' && matrix.proxy == 'no-proxy' && matrix.nuq == 'postgres'run: pnpm exec vitest run src/__tests__/routes/country-check.routes.test.ts --reporter=defaultdiff --git a/apps/api/src/controllers/v2/map.ts b/apps/api/src/controllers/v2/map.tsindex fa7f50dd4..64c1ddae6 100644--- a/apps/api/src/controllers/v2/map.ts+++ b/apps/api/src/controllers/v2/map.ts@@ -173,7 +173,7 @@ export async function mapController(},results: avgrabResults,credits_cost: creditsCost,- zeroDataRetention: false,+ zeroDataRetention,}).catch(error => {logger.error(`Failed to log job for team ${req.auth.team_id}: ${error}`,diff --git a/apps/api/src/lib/gcs-jobs.test.ts b/apps/api/src/lib/gcs-jobs.test.tsnew file mode 100644index 000000000..c09b344eb--- /dev/null+++ b/apps/api/src/lib/gcs-jobs.test.ts@@ -0,0 +1,128 @@+import { vi } from "vitest";+import type { LoggedScrape } from "../services/logging/log_job";++const { save, remove, enqueueZdrCleanupJob, spans } = vi.hoisted(() => {+ const spans: Array<{ name: string; attributes: Record<string, unknown> }> =+ [];+ return {+ save: vi.fn(async () => {}),+ remove: vi.fn(async (): Promise<[unknown]> => [{ statusCode: 204 }]),+ enqueueZdrCleanupJob: vi.fn(async () => {}),+ spans,+ };+});++vi.mock("@google-cloud/storage", () => ({+ ApiError: class extends Error {+ code?: number;+ },+ Storage: class {+ bucket() {+ return { file: () => ({ save, delete: remove }) };+ }+ },+}));+vi.mock("../config", () => ({+ config: { GCS_BUCKET_NAME: "bucket", USE_DB_AUTHENTICATION: true },+}));+vi.mock("./zdr-queue", () => ({ enqueueZdrCleanupJob }));+vi.mock("./otel-tracer", () => ({+ withSpan: async (name: string, fn: (span: any) => any) => {+ const span = { attributes: {} };+ spans.push({ name, attributes: span.attributes });+ return fn(span);+ },+ setSpanAttributes: (+ span: { attributes: Record<string, unknown> },+ attributes: object,+ ) => Object.assign(span.attributes, attributes),+}));+vi.mock("./logger", () => {+ const logger: any = {+ child: vi.fn(() => logger),+ debug: vi.fn(),+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ };+ return { logger };+});++import { removeJobFromGCS, saveScrapeToGCS } from "./gcs-jobs";++const scrapeId = "019e6f45-7778-727d-adf0-0abe9d5062b6";++function scrape(zeroDataRetention: boolean): LoggedScrape {+ return {+ id: scrapeId,+ request_id: "019e6f45-7778-727d-adf0-0abe9d5062b7",+ url: "https://example.com",+ is_successful: true,+ doc: { markdown: "content", metadata: {} },+ time_taken: 1,+ team_id: "team-id",+ options: {},+ credits_cost: 1,+ skipNuq: false,+ zeroDataRetention,+ } as unknown as LoggedScrape;+}++describe("GCS job blobs", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ spans.length = 0;+ });++ it("schedules a ZDR blob's delete before writing it", async () => {+ await saveScrapeToGCS(scrape(true));++ expect(enqueueZdrCleanupJob).toHaveBeenCalledWith({ blobId: scrapeId });+ expect(save).toHaveBeenCalledOnce();+ expect(enqueueZdrCleanupJob.mock.invocationCallOrder[0]).toBeLessThan(+ save.mock.invocationCallOrder[0],+ );+ });++ it("does not schedule a delete for a non-ZDR blob", async () => {+ await saveScrapeToGCS(scrape(false));++ expect(enqueueZdrCleanupJob).not.toHaveBeenCalled();+ expect(save).toHaveBeenCalledOnce();+ });++ it("does not write a ZDR blob when its delete could not be scheduled", async () => {+ enqueueZdrCleanupJob.mockRejectedValueOnce(+ new Error("RabbitMQ unavailable"),+ );++ await expect(saveScrapeToGCS(scrape(true))).rejects.toThrow(+ "RabbitMQ unavailable",+ );+ expect(save).not.toHaveBeenCalled();+ });++ it("records whether a removed blob still existed", async () => {+ await removeJobFromGCS("blob-found");+ remove.mockResolvedValueOnce([{ statusCode: 404 }]);+ await removeJobFromGCS("blob-missing");++ expect(remove).toHaveBeenCalledWith({ ignoreNotFound: true });+ expect(+ spans+ .filter(span => span.name === "firecrawl-gcs-remove-job")+ .map(span => span.attributes),+ ).toEqual([+ expect.objectContaining({+ "job.id": "blob-found",+ "gcs.delete_successful": true,+ "gcs.job_not_found": false,+ }),+ expect.objectContaining({+ "job.id": "blob-missing",+ "gcs.delete_successful": true,+ "gcs.job_not_found": true,+ }),+ ]);+ });+});diff --git a/apps/api/src/lib/gcs-jobs.ts b/apps/api/src/lib/gcs-jobs.tsindex 92d86f179..eedabb8b6 100644--- a/apps/api/src/lib/gcs-jobs.ts+++ b/apps/api/src/lib/gcs-jobs.ts@@ -11,6 +11,7 @@ import type {LoggedSearch,} from "../services/logging/log_job";import { config } from "../config";+import { enqueueZdrCleanupJob } from "./zdr-queue";import crypto from "crypto";import { Logger } from "winston";@@ -108,6 +109,11 @@ async function saveJobToGCS(params: {return;}+ // Never write a ZDR blob without its delete already scheduled.+ if (params.zeroDataRetention && config.USE_DB_AUTHENTICATION === true) {+ await enqueueZdrCleanupJob({ blobId: params.id });+ }+const bucket = storageManualRetries.bucket(config.GCS_BUCKET_NAME);const blob = bucket.file(filename);@@ -443,10 +449,14 @@ export async function removeJobFromGCS(const blob = bucket.file(idToFilename(jobId));try {- await blob.delete({+ const [response] = await blob.delete({ignoreNotFound: true,});- setSpanAttributes(span, { "gcs.delete_successful": true });+ setSpanAttributes(span, {+ "gcs.delete_successful": true,+ "gcs.job_not_found":+ (response as { statusCode?: number } | undefined)?.statusCode === 404,+ });} catch (error) {if (error instanceof ApiError &&diff --git a/apps/api/src/lib/zdr-queue.test.ts b/apps/api/src/lib/zdr-queue.test.tsindex 2183540e1..4c52dfe16 100644--- a/apps/api/src/lib/zdr-queue.test.ts+++ b/apps/api/src/lib/zdr-queue.test.ts@@ -94,13 +94,17 @@ vi.mock("./otel-tracer", () => ({recordSpanException,}));-function message(body: unknown) {+function message(+ body: unknown,+ headers?: Record<string, unknown>,+ redelivered = false,+) {return {content: Buffer.from(typeof body === "string" ? body : JSON.stringify(body),),- fields: {},- properties: {},+ fields: { redelivered },+ properties: { headers },};}@@ -118,7 +122,7 @@ describe("ZDR cleanup queue", () => {"./zdr-queue.js");- await enqueueZdrCleanupJob("request-1");+ await enqueueZdrCleanupJob({ requestId: "request-1" });expect(connect).toHaveBeenCalledWith("amqp://rabbitmq",@@ -135,6 +139,7 @@ describe("ZDR cleanup queue", () => {correlationId: expect.any(String),timestamp: expect.any(Number),mandatory: true,+ headers: { "zdr-attempt": 1 },}),expect.any(Function),);@@ -151,6 +156,27 @@ describe("ZDR cleanup queue", () => {await shutdownZdrQueue();});+ it("publishes a per-blob job to the same 24h delay queue", async () => {+ const { enqueueZdrCleanupJob, shutdownZdrQueue } = await import(+ "./zdr-queue.js"+ );++ await enqueueZdrCleanupJob({ blobId: "blob-1" });++ expect(publishChannel.sendToQueue).toHaveBeenCalledWith(+ "zdr.cleanup.delayed.24h",+ Buffer.from(JSON.stringify({ blobId: "blob-1" })),+ expect.objectContaining({+ persistent: true,+ messageId: "blob-1",+ mandatory: true,+ headers: { "zdr-attempt": 1 },+ }),+ expect.any(Function),+ );+ await shutdownZdrQueue();+ });+it("retries a mandatory publish returned as unroutable", async () => {vi.useFakeTimers();try {@@ -172,7 +198,7 @@ describe("ZDR cleanup queue", () => {"./zdr-queue.js");- const enqueue = enqueueZdrCleanupJob("request-returned");+ const enqueue = enqueueZdrCleanupJob({ requestId: "request-returned" });await vi.runAllTimersAsync();await enqueue;@@ -188,7 +214,7 @@ describe("ZDR cleanup queue", () => {"./zdr-queue.js");- await enqueueZdrCleanupJob("request-2");+ await enqueueZdrCleanupJob({ requestId: "request-2" });const queueDeclarations = new Map((publishChannel.assertQueue as Mock).mock.calls.map(([name, options]) => [@@ -213,12 +239,25 @@ describe("ZDR cleanup queue", () => {"x-queue-type": "quorum","x-overflow": "reject-publish","x-delivery-limit": -1,+ "x-dead-letter-exchange": "zdr.cleanup",+ "x-dead-letter-routing-key": "dead",+ },+ });+ expect(queueDeclarations.get("zdr.cleanup.retry")).toMatchObject({+ durable: true,+ arguments: {+ "x-queue-type": "quorum",+ "x-overflow": "reject-publish",+ "x-message-ttl": 300_000,+ "x-dead-letter-strategy": "at-least-once",+ "x-dead-letter-exchange": "zdr.cleanup",+ "x-dead-letter-routing-key": "ready",},});await shutdownZdrQueue();});- it("acks completed jobs and requeues failures without a delivery cap", async () => {+ it("hands request-level and per-blob jobs to the handler and acks them", async () => {const { consumeZdrCleanupJobs, shutdownZdrQueue } = await import("./zdr-queue.js");@@ -226,30 +265,180 @@ describe("ZDR cleanup queue", () => {await consumeZdrCleanupJobs(handler);const consume = consumerCallbacks[0];- const completed = message({ requestId: "request-3" });- await consume(completed);- expect(handler).toHaveBeenCalledWith({ requestId: "request-3" });- expect(consumeChannel.ack).toHaveBeenCalledWith(completed);- expect(spans).toContainEqual({- name: "zdr.rabbitmq.consume",- attributes: expect.objectContaining({- "messaging.system": "rabbitmq",+ const requestJob = message({ requestId: "request-3" });+ await consume(requestJob);+ const blobJob = message({ blobId: "blob-3" });+ await consume(blobJob);++ expect(handler.mock.calls).toEqual([+ [{ requestId: "request-3" }],+ [{ blobId: "blob-3" }],+ ]);+ expect(consumeChannel.ack).toHaveBeenCalledWith(requestJob);+ expect(consumeChannel.ack).toHaveBeenCalledWith(blobJob);+ expect(consumeChannel.nack).not.toHaveBeenCalled();+ const consumeSpans = spans.filter(+ span => span.name === "zdr.rabbitmq.consume",+ );+ expect(consumeSpans.map(span => span.attributes)).toEqual([+ expect.objectContaining({"messaging.destination.name": "zdr.cleanup.ready","messaging.operation.name": "process","zdr.request_id": "request-3",+ "zdr.attempt": 1,"zdr.rabbitmq.outcome": "acknowledged",}),+ expect.objectContaining({+ "zdr.blob_id": "blob-3",+ "zdr.attempt": 1,+ "zdr.rabbitmq.outcome": "acknowledged",+ }),+ ]);+ expect(consumeSpans[1].attributes).not.toHaveProperty("zdr.request_id");+ await shutdownZdrQueue();+ });++ it("schedules a delayed retry instead of requeueing a failed job", async () => {+ const { consumeZdrCleanupJobs, shutdownZdrQueue } = await import(+ "./zdr-queue.js"+ );+ const handler = vi.fn(async () => {+ throw new Error("GCS unavailable");});+ await consumeZdrCleanupJobs(handler);+ const consume = consumerCallbacks[0];- handler.mockRejectedValueOnce(new Error("GCS unavailable"));- const failed = message({ requestId: "request-4" });- await consume(failed);- expect(consumeChannel.nack).toHaveBeenCalledWith(failed, false, true);+ const firstFailure = message({ blobId: "blob-4" });+ await consume(firstFailure);+ const laterFailure = message(+ { requestId: "request-4" },+ { "zdr-attempt": 5 },+ );+ await consume(laterFailure);++ expect(publishChannel.sendToQueue).toHaveBeenNthCalledWith(+ 1,+ "zdr.cleanup.retry",+ Buffer.from(JSON.stringify({ blobId: "blob-4" })),+ expect.objectContaining({+ persistent: true,+ messageId: "blob-4",+ mandatory: true,+ headers: { "zdr-attempt": 2 },+ }),+ expect.any(Function),+ );+ expect(publishChannel.sendToQueue).toHaveBeenNthCalledWith(+ 2,+ "zdr.cleanup.retry",+ Buffer.from(JSON.stringify({ requestId: "request-4" })),+ expect.objectContaining({ headers: { "zdr-attempt": 6 } }),+ expect.any(Function),+ );+ expect(consumeChannel.ack).toHaveBeenCalledWith(firstFailure);+ expect(consumeChannel.ack).toHaveBeenCalledWith(laterFailure);+ expect(consumeChannel.nack).not.toHaveBeenCalled();+ expect(recordSpanException).toHaveBeenCalledTimes(2);+ expect(spans).toContainEqual({+ name: "zdr.rabbitmq.consume",+ attributes: expect.objectContaining({+ "zdr.blob_id": "blob-4",+ "zdr.rabbitmq.outcome": "retry_scheduled",+ }),+ });+ await shutdownZdrQueue();+ });++ it("dead-letters a job that fails its last attempt", async () => {+ const { consumeZdrCleanupJobs, shutdownZdrQueue } = await import(+ "./zdr-queue.js"+ );+ const handler = vi.fn(async () => {+ throw new Error("GCS unavailable");+ });+ await consumeZdrCleanupJobs(handler);+ const consume = consumerCallbacks[0];++ const exhausted = message({ blobId: "blob-5" }, { "zdr-attempt": 24 });+ await consume(exhausted);++ expect(handler).toHaveBeenCalledOnce();+ expect(consumeChannel.nack).toHaveBeenCalledWith(exhausted, false, false);+ expect(consumeChannel.ack).not.toHaveBeenCalled();+ expect(publishChannel.sendToQueue).not.toHaveBeenCalled();+ expect(spans).toContainEqual({+ name: "zdr.rabbitmq.consume",+ attributes: expect.objectContaining({+ "zdr.blob_id": "blob-5",+ "zdr.attempt": 24,+ "zdr.rabbitmq.outcome": "dead_lettered",+ }),+ });+ await shutdownZdrQueue();+ });++ it("counts an unsettled redelivery as a failed attempt", async () => {+ const { consumeZdrCleanupJobs, shutdownZdrQueue } = await import(+ "./zdr-queue.js"+ );+ const handler = vi.fn(async () => {});+ await consumeZdrCleanupJobs(handler);+ const consume = consumerCallbacks[0];++ const redelivered = message(+ { blobId: "blob-6" },+ { "zdr-attempt": 3 },+ true,+ );+ await consume(redelivered);+ const lastRedelivery = message(+ { blobId: "blob-7" },+ { "zdr-attempt": 24 },+ true,+ );+ await consume(lastRedelivery);++ expect(handler).not.toHaveBeenCalled();+ expect(publishChannel.sendToQueue).toHaveBeenCalledOnce();+ expect(publishChannel.sendToQueue).toHaveBeenCalledWith(+ "zdr.cleanup.retry",+ Buffer.from(JSON.stringify({ blobId: "blob-6" })),+ expect.objectContaining({ headers: { "zdr-attempt": 4 } }),+ expect.any(Function),+ );+ expect(consumeChannel.ack).toHaveBeenCalledWith(redelivered);+ expect(consumeChannel.nack).toHaveBeenCalledWith(+ lastRedelivery,+ false,+ false,+ );+ await shutdownZdrQueue();+ });++ it("dead-letters malformed and invalid jobs without calling the handler", async () => {+ const { consumeZdrCleanupJobs, shutdownZdrQueue } = await import(+ "./zdr-queue.js"+ );+ const handler = vi.fn(async () => {});+ await consumeZdrCleanupJobs(handler);+ const consume = consumerCallbacks[0];const malformed = message("not-json");+ const emptyBlob = message({ blobId: "" });+ const unknownShape = message({ jobId: "job-1" });await consume(malformed);+ await consume(emptyBlob);+ await consume(unknownShape);++ expect(handler).not.toHaveBeenCalled();expect(consumeChannel.nack).toHaveBeenCalledWith(malformed, false, false);- expect(recordSpanException).toHaveBeenCalled();+ expect(consumeChannel.nack).toHaveBeenCalledWith(emptyBlob, false, false);+ expect(consumeChannel.nack).toHaveBeenCalledWith(+ unknownShape,+ false,+ false,+ );+ expect(recordSpanException).toHaveBeenCalledTimes(3);await shutdownZdrQueue();});});diff --git a/apps/api/src/lib/zdr-queue.ts b/apps/api/src/lib/zdr-queue.tsindex 189a9d978..e662beccd 100644--- a/apps/api/src/lib/zdr-queue.ts+++ b/apps/api/src/lib/zdr-queue.ts@@ -12,6 +12,7 @@ const DELAY_MS = 24 * 60 * 60 * 1000;const EXCHANGE = "zdr.cleanup";const DELAY_QUEUE = "zdr.cleanup.delayed.24h";const READY_QUEUE = "zdr.cleanup.ready";+const RETRY_QUEUE = "zdr.cleanup.retry";const DLQ = "zdr.cleanup.dlq";const READY_ROUTING_KEY = "ready";const DLQ_ROUTING_KEY = "dead";@@ -21,12 +22,14 @@ const RECONNECT_MAX_DELAY_MS = 30_000;// well inside the API pod's shutdown grace.const PUBLISH_MAX_ATTEMPTS = 5;const PUBLISH_ATTEMPT_TIMEOUT_MS = 5_000;+const RETRY_DELAY_MS = 5 * 60 * 1000;+const MAX_ATTEMPTS = 24;+const ATTEMPT_HEADER = "zdr-attempt";const logger = _logger.child({ module: "zdr-queue" });-type ZdrCleanupJob = {- requestId: string;-};+// A request-level job resolves its blobs from ClickHouse; a blob job names one blob.+export type ZdrCleanupJob = { requestId: string } | { blobId: string };type CleanupHandler = (job: ZdrCleanupJob) => Promise<void>;@@ -82,6 +85,18 @@ async function assertTopology(ch: amqp.Channel): Promise<void> {"x-dead-letter-routing-key": READY_ROUTING_KEY,},});++ await ch.assertQueue(RETRY_QUEUE, {+ durable: true,+ arguments: {+ "x-queue-type": "quorum",+ "x-overflow": "reject-publish",+ "x-message-ttl": RETRY_DELAY_MS,+ "x-dead-letter-strategy": "at-least-once",+ "x-dead-letter-exchange": EXCHANGE,+ "x-dead-letter-routing-key": READY_ROUTING_KEY,+ },+ });}function resetCachedState(): void {@@ -229,9 +244,15 @@ async function getConsumeChannel(): Promise<amqp.Channel> {return consumeChannelPromise;}+function jobId(job: ZdrCleanupJob): string {+ return "blobId" in job ? job.blobId : job.requestId;+}+function publishConfirmed(ch: amqp.ConfirmChannel,+ queue: string,job: ZdrCleanupJob,+ attempt: number,): Promise<void> {const correlationId = randomUUID();const publish = { returned: false };@@ -240,15 +261,16 @@ function publishConfirmed(return new Promise((resolve, reject) => {ch.sendToQueue(- DELAY_QUEUE,+ queue,Buffer.from(JSON.stringify(job)),{persistent: true,contentType: "application/json",- messageId: job.requestId,+ messageId: jobId(job),correlationId,timestamp: Math.floor(Date.now() / 1000),mandatory: true,+ headers: { [ATTEMPT_HEADER]: attempt },},error => {setImmediate(() => {@@ -263,25 +285,27 @@ function publishConfirmed(});}-export async function enqueueZdrCleanupJob(requestId: string): Promise<void> {- if (!config.NUQ_RABBITMQ_URL) {- throw new Error("NUQ_RABBITMQ_URL is not configured");- }-+async function publishUntilConfirmed(+ queue: string,+ job: ZdrCleanupJob,+ attempt: number,+): Promise<void> {let delayMs = RECONNECT_BASE_DELAY_MS;- for (let attempt = 1; !closed; attempt++) {+ for (let publishAttempt = 1; !closed; publishAttempt++) {let attemptTimer: NodeJS.Timeout | undefined;try {await withSpan("zdr.rabbitmq.publish", async span => {setSpanAttributes(span, {"messaging.system": "rabbitmq",- "messaging.destination.name": DELAY_QUEUE,+ "messaging.destination.name": queue,"messaging.operation.name": "publish",- "messaging.message.id": requestId,+ "messaging.message.id": jobId(job),"zdr.rabbitmq.retry_delay_ms": delayMs,});await Promise.race([- getPublishChannel().then(ch => publishConfirmed(ch, { requestId })),+ getPublishChannel().then(ch =>+ publishConfirmed(ch, queue, job, attempt),+ ),new Promise<never>((_, reject) => {attemptTimer = setTimeout(() => reject(new Error("ZDR publish attempt timed out")),@@ -295,19 +319,21 @@ export async function enqueueZdrCleanupJob(requestId: string): Promise<void> {});return;} catch (error) {- if (attempt >= PUBLISH_MAX_ATTEMPTS) {+ if (publishAttempt >= PUBLISH_MAX_ATTEMPTS) {logger.error("Failed to enqueue ZDR cleanup job; giving up", {error,- requestId,- attempts: attempt,+ queue,+ messageId: jobId(job),+ attempts: publishAttempt,});throw new Error(- `ZDR cleanup job not confirmed after ${attempt} attempts`,+ `ZDR cleanup job not confirmed after ${publishAttempt} attempts`,);}logger.error("Failed to durably enqueue ZDR cleanup job; retrying", {error,- requestId,+ queue,+ messageId: jobId(job),delayMs,});await publishChannel?.close().catch(() => {});@@ -323,13 +349,23 @@ export async function enqueueZdrCleanupJob(requestId: string): Promise<void> {throw new Error("ZDR queue closed before the cleanup job was confirmed");}+/** Schedules a cleanup job 24 hours out; throws if the broker never confirms it. */+export async function enqueueZdrCleanupJob(job: ZdrCleanupJob): Promise<void> {+ if (!config.NUQ_RABBITMQ_URL) {+ throw new Error("NUQ_RABBITMQ_URL is not configured");+ }+ await publishUntilConfirmed(DELAY_QUEUE, job, 1);+}+function isZdrCleanupJob(value: unknown): value is ZdrCleanupJob {- return (- typeof value === "object" &&- value !== null &&- typeof (value as Partial<ZdrCleanupJob>).requestId === "string" &&- (value as ZdrCleanupJob).requestId.length > 0- );+ if (typeof value !== "object" || value === null) return false;+ const id =+ "blobId" in value+ ? value.blobId+ : "requestId" in value+ ? value.requestId+ : undefined;+ return typeof id === "string" && id.length > 0;}async function subscribe(@@ -346,13 +382,14 @@ async function subscribe(scheduleReconnect();return;}+ const messageId = msg.properties.messageId;try {await withSpan("zdr.rabbitmq.consume", async span => {setSpanAttributes(span, {"messaging.system": "rabbitmq","messaging.destination.name": READY_QUEUE,"messaging.operation.name": "process",- "messaging.message.id": msg.properties.messageId,+ "messaging.message.id": messageId,"messaging.rabbitmq.delivery_tag": msg.fields.deliveryTag,"messaging.rabbitmq.redelivered": msg.fields.redelivered,});@@ -381,21 +418,59 @@ async function subscribe(return;}+ const attempt = Number(msg.properties.headers?.[ATTEMPT_HEADER]) || 1;setSpanAttributes(span, {- "zdr.request_id": job.requestId,+ ...("blobId" in job+ ? { "zdr.blob_id": job.blobId }+ : { "zdr.request_id": job.requestId }),+ "zdr.attempt": attempt,});- await handler(job);++ try {+ // An unsettled earlier delivery (worker died, channel closed) counts as a failed attempt.+ if (msg.fields.redelivered) {+ throw new Error("ZDR cleanup job was redelivered unsettled");+ }+ await handler(job);+ } catch (error) {+ recordSpanException(span, error);+ if (attempt >= MAX_ATTEMPTS) {+ logger.error(+ "ZDR cleanup job failed its last attempt; dead-lettering",+ {+ error,+ messageId,+ attempt,+ },+ );+ ch.nack(msg, false, false);+ setSpanAttributes(span, {+ "zdr.rabbitmq.outcome": "dead_lettered",+ });+ return;+ }+ logger.warn("ZDR cleanup job failed; scheduling a retry", {+ error,+ messageId,+ attempt,+ retryDelayMs: RETRY_DELAY_MS,+ });+ await publishUntilConfirmed(RETRY_QUEUE, job, attempt + 1);+ ch.ack(msg);+ setSpanAttributes(span, {+ "zdr.rabbitmq.outcome": "retry_scheduled",+ });+ return;+ }+ch.ack(msg);setSpanAttributes(span, {"zdr.rabbitmq.outcome": "acknowledged",});});} catch (error) {- logger.error("ZDR cleanup job failed; requeueing", {- error,- messageId: msg.properties.messageId,- });- ch.nack(msg, false, true);+ // Left unacked, the broker redelivers it once this channel closes.+ logger.error("Could not settle ZDR cleanup job", { error, messageId });}},{ noAck: false },diff --git a/apps/api/src/lib/zdrcleaner.test.ts b/apps/api/src/lib/zdrcleaner.test.tsindex bb0156f54..bc6632e90 100644--- a/apps/api/src/lib/zdrcleaner.test.ts+++ b/apps/api/src/lib/zdrcleaner.test.ts@@ -43,7 +43,7 @@ vi.mock("./logger", () => {});vi.mock("./otel-tracer", () => ({ withSpan, setSpanAttributes }));-import { cleanZdrRequest } from "./zdrcleaner";+import { cleanZdrJob, cleanZdrRequest } from "./zdrcleaner";import { logger } from "./logger";function mockChildren(ids: string[]) {@@ -142,3 +142,41 @@ describe("ZDR cleaner", () => {expect(removeJobFromGCS).not.toHaveBeenCalled();});});++describe("ZDR cleanup job dispatch", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ client.current = { query };+ removeJobFromGCS.mockResolvedValue(undefined);+ spans.length = 0;+ });++ it("deletes a per-blob job's blob without reading ClickHouse", async () => {+ await cleanZdrJob({ blobId: "blob-1" });++ expect(removeJobFromGCS.mock.calls.map(([id]) => id)).toEqual(["blob-1"]);+ expect(query).not.toHaveBeenCalled();+ });++ it("propagates a per-blob delete failure so the job is retried", async () => {+ removeJobFromGCS.mockRejectedValueOnce(new Error("GCS unavailable"));++ await expect(cleanZdrJob({ blobId: "blob-2" })).rejects.toThrow(+ "GCS unavailable",+ );+ expect(query).not.toHaveBeenCalled();+ });++ it("still resolves a request-level job's blobs from ClickHouse", async () => {+ mockChildren(["scrape-blob"]);++ await cleanZdrJob({ requestId: "request-5" });++ expect(query.mock.calls[0][0]).toMatchObject({+ query_params: { requestId: "request-5" },+ });+ expect(removeJobFromGCS.mock.calls.map(([id]) => id)).toEqual([+ "scrape-blob",+ ]);+ });+});diff --git a/apps/api/src/lib/zdrcleaner.ts b/apps/api/src/lib/zdrcleaner.tsindex 2b3f1b55d..d48fe5f87 100644--- a/apps/api/src/lib/zdrcleaner.ts+++ b/apps/api/src/lib/zdrcleaner.ts@@ -4,6 +4,7 @@ import { removeJobFromGCS } from "./gcs-jobs";import { logger as _logger } from "./logger";import { config } from "../config";import { setSpanAttributes, withSpan } from "./otel-tracer";+import type { ZdrCleanupJob } from "./zdr-queue";export async function sendZdrHeartbeat() {if (config.ZDRCLEANER_HEARTBEAT_URL) {@@ -87,3 +88,11 @@ export async function cleanZdrRequest(requestId: string): Promise<void> {setSpanAttributes(span, { "zdr.cleanup.outcome": "completed" });});}++export async function cleanZdrJob(job: ZdrCleanupJob): Promise<void> {+ if ("blobId" in job) {+ await removeJobFromGCS(job.blobId);+ return;+ }+ await cleanZdrRequest(job.requestId);+}diff --git a/apps/api/src/services/logging/log_job.test.ts b/apps/api/src/services/logging/log_job.test.tsindex d9210d8c7..08736704a 100644--- a/apps/api/src/services/logging/log_job.test.ts+++ b/apps/api/src/services/logging/log_job.test.ts@@ -667,9 +667,9 @@ describe("logRequest", () => {zeroDataRetention: true,});- expect(enqueueZdrCleanupJob).toHaveBeenCalledWith(- "019e6f45-7778-727d-adf0-0abe9d5062b6",- );+ expect(enqueueZdrCleanupJob).toHaveBeenCalledWith({+ requestId: "019e6f45-7778-727d-adf0-0abe9d5062b6",+ });expect(enqueueZdrCleanupJob.mock.invocationCallOrder[0]).toBeLessThan(publishMessage.mock.invocationCallOrder[0],);diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex c5532183d..09e37cabc 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -570,7 +570,7 @@ async function logRequestInternal(request: LoggedRequest) {}if (request.zeroDataRetention && config.USE_DB_AUTHENTICATION === true) {- await enqueueZdrCleanupJob(request.id);+ await enqueueZdrCleanupJob({ requestId: request.id });}await robustInsert(diff --git a/apps/api/src/services/zdr-worker.ts b/apps/api/src/services/zdr-worker.tsindex 05cdc9e38..6eb113bf7 100644--- a/apps/api/src/services/zdr-worker.ts+++ b/apps/api/src/services/zdr-worker.ts@@ -1,7 +1,7 @@import "dotenv/config";import { shutdownTracing } from "../otel";import { logger } from "../lib/logger";-import { cleanZdrRequest, sendZdrHeartbeat } from "../lib/zdrcleaner";+import { cleanZdrJob, sendZdrHeartbeat } from "../lib/zdrcleaner";import { consumeZdrCleanupJobs, shutdownZdrQueue } from "../lib/zdr-queue";let isShuttingDown = false;@@ -17,7 +17,7 @@ process.on("SIGTERM", () => {});(async () => {- await consumeZdrCleanupJobs(job => cleanZdrRequest(job.requestId));+ await consumeZdrCleanupJobs(cleanZdrJob);// Cleanup is driven entirely by the delayed RabbitMQ queue; this loop only// keeps the liveness heartbeat going while the consumer runs.