Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): move ZDR cleanup scheduling to RabbitMQ (#4650)

Gergő Móricz90fd89cdcef8
diff --git a/apps/api/src/lib/zdr-queue.test.ts b/apps/api/src/lib/zdr-queue.test.ts
new file mode 100644
index 000000000..2183540e1
--- /dev/null
+++ b/apps/api/src/lib/zdr-queue.test.ts
@@ -0,0 +1,255 @@
+import { vi, type Mock } from "vitest";
+
+const {
+ connect,
+ connection,
+ publishChannel,
+ consumeChannel,
+ consumerCallbacks,
+ spans,
+ withSpan,
+ setSpanAttributes,
+ recordSpanException,
+ publishEvents,
+} = vi.hoisted(() => {
+ const consumerCallbacks: Array<(message: any) => Promise<void>> = [];
+ const publishEvents = new Map<string, (...args: any[]) => void>();
+ const spans: Array<{ name: string; attributes: Record<string, unknown> }> =
+ [];
+ const withSpan = vi.fn(async (name: string, fn: (span: any) => any) => {
+ const span = { attributes: {} };
+ spans.push({ name, attributes: span.attributes });
+ return fn(span);
+ });
+ const setSpanAttributes = vi.fn(
+ (span: { attributes: Record<string, unknown> }, attributes: object) => {
+ Object.assign(span.attributes, attributes);
+ },
+ );
+ const publishChannel: any = {
+ assertExchange: vi.fn(async () => {}),
+ assertQueue: vi.fn(async () => ({})),
+ bindQueue: vi.fn(async () => {}),
+ sendToQueue: vi.fn(
+ (_queue: string, _content: Buffer, _options: any, callback: any) => {
+ callback(null);
+ return true;
+ },
+ ),
+ close: vi.fn(async () => {}),
+ on: vi.fn((event: string, handler: (...args: any[]) => void) => {
+ publishEvents.set(event, handler);
+ }),
+ };
+ const consumeChannel: any = {
+ assertExchange: vi.fn(async () => {}),
+ assertQueue: vi.fn(async () => ({})),
+ bindQueue: vi.fn(async () => {}),
+ prefetch: vi.fn(async () => {}),
+ consume: vi.fn(async (_queue: string, callback: any) => {
+ consumerCallbacks.push(callback);
+ return { consumerTag: "zdr-test" };
+ }),
+ ack: vi.fn(),
+ nack: vi.fn(),
+ close: vi.fn(async () => {}),
+ on: vi.fn(),
+ };
+ const connection: any = {
+ createConfirmChannel: vi.fn(async () => publishChannel),
+ createChannel: vi.fn(async () => consumeChannel),
+ close: vi.fn(async () => {}),
+ on: vi.fn(),
+ };
+ return {
+ connect: vi.fn(async () => connection),
+ connection,
+ publishChannel,
+ consumeChannel,
+ consumerCallbacks,
+ spans,
+ withSpan,
+ setSpanAttributes,
+ recordSpanException: vi.fn(),
+ publishEvents,
+ };
+});
+
+vi.mock("amqplib", () => ({ default: { connect } }));
+vi.mock("../config", () => ({
+ config: { NUQ_RABBITMQ_URL: "amqp://rabbitmq" },
+}));
+vi.mock("./logger", () => {
+ const logger: any = {
+ child: vi.fn(() => logger),
+ info: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ };
+ return { logger };
+});
+vi.mock("./otel-tracer", () => ({
+ withSpan,
+ setSpanAttributes,
+ recordSpanException,
+}));
+
+function message(body: unknown) {
+ return {
+ content: Buffer.from(
+ typeof body === "string" ? body : JSON.stringify(body),
+ ),
+ fields: {},
+ properties: {},
+ };
+}
+
+describe("ZDR cleanup queue", () => {
+ beforeEach(() => {
+ vi.resetModules();
+ vi.clearAllMocks();
+ consumerCallbacks.length = 0;
+ publishEvents.clear();
+ spans.length = 0;
+ });
+
+ it("publishes persistent jobs only after a broker confirm", async () => {
+ const { enqueueZdrCleanupJob, shutdownZdrQueue } = await import(
+ "./zdr-queue.js"
+ );
+
+ await enqueueZdrCleanupJob("request-1");
+
+ expect(connect).toHaveBeenCalledWith(
+ "amqp://rabbitmq",
+ expect.objectContaining({
+ clientProperties: { connection_name: "zdr-cleanup" },
+ }),
+ );
+ expect(publishChannel.sendToQueue).toHaveBeenCalledWith(
+ "zdr.cleanup.delayed.24h",
+ Buffer.from(JSON.stringify({ requestId: "request-1" })),
+ expect.objectContaining({
+ persistent: true,
+ messageId: "request-1",
+ correlationId: expect.any(String),
+ timestamp: expect.any(Number),
+ mandatory: true,
+ }),
+ expect.any(Function),
+ );
+ expect(spans).toContainEqual({
+ name: "zdr.rabbitmq.publish",
+ attributes: expect.objectContaining({
+ "messaging.system": "rabbitmq",
+ "messaging.destination.name": "zdr.cleanup.delayed.24h",
+ "messaging.operation.name": "publish",
+ "messaging.message.id": "request-1",
+ "zdr.rabbitmq.outcome": "confirmed",
+ }),
+ });
+ await shutdownZdrQueue();
+ });
+
+ it("retries a mandatory publish returned as unroutable", async () => {
+ vi.useFakeTimers();
+ try {
+ publishChannel.sendToQueue.mockImplementationOnce(
+ (
+ _queue: string,
+ _content: Buffer,
+ options: { correlationId: string },
+ callback: (error: Error | null) => void,
+ ) => {
+ publishEvents.get("return")?.({
+ properties: { correlationId: options.correlationId },
+ });
+ callback(null);
+ return true;
+ },
+ );
+ const { enqueueZdrCleanupJob, shutdownZdrQueue } = await import(
+ "./zdr-queue.js"
+ );
+
+ const enqueue = enqueueZdrCleanupJob("request-returned");
+ await vi.runAllTimersAsync();
+ await enqueue;
+
+ expect(publishChannel.sendToQueue).toHaveBeenCalledTimes(2);
+ await shutdownZdrQueue();
+ } finally {
+ vi.useRealTimers();
+ }
+ });
+
+ it("uses replicated queues and at-least-once delayed delivery", async () => {
+ const { enqueueZdrCleanupJob, shutdownZdrQueue } = await import(
+ "./zdr-queue.js"
+ );
+
+ await enqueueZdrCleanupJob("request-2");
+
+ const queueDeclarations = new Map(
+ (publishChannel.assertQueue as Mock).mock.calls.map(([name, options]) => [
+ name,
+ options,
+ ]),
+ );
+ expect(queueDeclarations.get("zdr.cleanup.delayed.24h")).toMatchObject({
+ durable: true,
+ arguments: {
+ "x-queue-type": "quorum",
+ "x-overflow": "reject-publish",
+ "x-message-ttl": 86_400_000,
+ "x-dead-letter-strategy": "at-least-once",
+ "x-dead-letter-exchange": "zdr.cleanup",
+ "x-dead-letter-routing-key": "ready",
+ },
+ });
+ expect(queueDeclarations.get("zdr.cleanup.ready")).toMatchObject({
+ durable: true,
+ arguments: {
+ "x-queue-type": "quorum",
+ "x-overflow": "reject-publish",
+ "x-delivery-limit": -1,
+ },
+ });
+ await shutdownZdrQueue();
+ });
+
+ it("acks completed jobs and requeues failures without a delivery cap", async () => {
+ const { consumeZdrCleanupJobs, shutdownZdrQueue } = await import(
+ "./zdr-queue.js"
+ );
+ const handler = vi.fn(async () => {});
+ await consumeZdrCleanupJobs(handler);
+ const consume = consumerCallbacks[0];
+
+ const completed = message({ requestId: "request-3" });
+ await consume(completed);
+ expect(handler).toHaveBeenCalledWith({ requestId: "request-3" });
+ expect(consumeChannel.ack).toHaveBeenCalledWith(completed);
+ expect(spans).toContainEqual({
+ name: "zdr.rabbitmq.consume",
+ attributes: expect.objectContaining({
+ "messaging.system": "rabbitmq",
+ "messaging.destination.name": "zdr.cleanup.ready",
+ "messaging.operation.name": "process",
+ "zdr.request_id": "request-3",
+ "zdr.rabbitmq.outcome": "acknowledged",
+ }),
+ });
+
+ handler.mockRejectedValueOnce(new Error("GCS unavailable"));
+ const failed = message({ requestId: "request-4" });
+ await consume(failed);
+ expect(consumeChannel.nack).toHaveBeenCalledWith(failed, false, true);
+
+ const malformed = message("not-json");
+ await consume(malformed);
+ expect(consumeChannel.nack).toHaveBeenCalledWith(malformed, false, false);
+ expect(recordSpanException).toHaveBeenCalled();
+ await shutdownZdrQueue();
+ });
+});
diff --git a/apps/api/src/lib/zdr-queue.ts b/apps/api/src/lib/zdr-queue.ts
new file mode 100644
index 000000000..b873ff2be
--- /dev/null
+++ b/apps/api/src/lib/zdr-queue.ts
@@ -0,0 +1,414 @@
+import amqp from "amqplib";
+import { randomUUID } from "node:crypto";
+import { config } from "../config";
+import { logger as _logger } from "./logger";
+import {
+ recordSpanException,
+ setSpanAttributes,
+ withSpan,
+} from "./otel-tracer";
+
+const DELAY_MS = 24 * 60 * 60 * 1000;
+const EXCHANGE = "zdr.cleanup";
+const DELAY_QUEUE = "zdr.cleanup.delayed.24h";
+const READY_QUEUE = "zdr.cleanup.ready";
+const DLQ = "zdr.cleanup.dlq";
+const READY_ROUTING_KEY = "ready";
+const DLQ_ROUTING_KEY = "dead";
+const RECONNECT_BASE_DELAY_MS = 1_000;
+const RECONNECT_MAX_DELAY_MS = 30_000;
+
+const logger = _logger.child({ module: "zdr-queue" });
+
+type ZdrCleanupJob = {
+ requestId: string;
+};
+
+type CleanupHandler = (job: ZdrCleanupJob) => Promise<void>;
+
+let connection: amqp.ChannelModel | null = null;
+let connectionPromise: Promise<amqp.ChannelModel> | null = null;
+let publishChannel: amqp.ConfirmChannel | null = null;
+let publishChannelPromise: Promise<amqp.ConfirmChannel> | null = null;
+let consumeChannel: amqp.Channel | null = null;
+let consumeChannelPromise: Promise<amqp.Channel> | null = null;
+let registeredHandler: CleanupHandler | null = null;
+let subscribed = false;
+let reconnectTimer: NodeJS.Timeout | null = null;
+let reconnectDelayMs = RECONNECT_BASE_DELAY_MS;
+let closed = false;
+const pendingPublishes = new WeakMap<
+ amqp.ConfirmChannel,
+ Map<string, { returned: boolean }>
+>();
+
+async function assertTopology(ch: amqp.Channel): Promise<void> {
+ await ch.assertExchange(EXCHANGE, "direct", { durable: true });
+
+ await ch.assertQueue(DLQ, {
+ durable: true,
+ arguments: {
+ "x-queue-type": "quorum",
+ "x-overflow": "reject-publish",
+ },
+ });
+ await ch.bindQueue(DLQ, EXCHANGE, DLQ_ROUTING_KEY);
+
+ await ch.assertQueue(READY_QUEUE, {
+ durable: true,
+ arguments: {
+ "x-queue-type": "quorum",
+ "x-overflow": "reject-publish",
+ "x-delivery-limit": -1,
+ "x-dead-letter-strategy": "at-least-once",
+ "x-dead-letter-exchange": EXCHANGE,
+ "x-dead-letter-routing-key": DLQ_ROUTING_KEY,
+ },
+ });
+ await ch.bindQueue(READY_QUEUE, EXCHANGE, READY_ROUTING_KEY);
+
+ await ch.assertQueue(DELAY_QUEUE, {
+ durable: true,
+ arguments: {
+ "x-queue-type": "quorum",
+ "x-overflow": "reject-publish",
+ "x-message-ttl": DELAY_MS,
+ "x-dead-letter-strategy": "at-least-once",
+ "x-dead-letter-exchange": EXCHANGE,
+ "x-dead-letter-routing-key": READY_ROUTING_KEY,
+ },
+ });
+}
+
+function resetCachedState(): void {
+ connection = null;
+ connectionPromise = null;
+ publishChannel = null;
+ publishChannelPromise = null;
+ consumeChannel = null;
+ consumeChannelPromise = null;
+ subscribed = false;
+}
+
+function handleConnectionDrop(reason: string, error?: unknown): void {
+ if (connection || publishChannel || consumeChannel) {
+ logger.warn("ZDR queue connection dropped", { reason, error });
+ }
+ resetCachedState();
+ if (registeredHandler && !closed) scheduleReconnect();
+}
+
+function scheduleReconnect(): void {
+ if (reconnectTimer || closed) return;
+
+ const delayMs = reconnectDelayMs;
+ reconnectTimer = setTimeout(async () => {
+ reconnectTimer = null;
+ try {
+ if (registeredHandler) {
+ const ch = await getConsumeChannel();
+ if (closed) {
+ await ch.close().catch(() => {});
+ await connection?.close().catch(() => {});
+ resetCachedState();
+ return;
+ }
+ await subscribe(ch, registeredHandler);
+ }
+ reconnectDelayMs = RECONNECT_BASE_DELAY_MS;
+ logger.info("ZDR queue reconnected; consumer re-subscribed");
+ } catch (error) {
+ logger.error("ZDR queue reconnect failed", { error, delayMs });
+ resetCachedState();
+ reconnectDelayMs = Math.min(delayMs * 2, RECONNECT_MAX_DELAY_MS);
+ scheduleReconnect();
+ }
+ }, delayMs);
+ reconnectTimer.unref?.();
+}
+
+async function getConnection(): Promise<amqp.ChannelModel> {
+ if (connection) return connection;
+ if (!connectionPromise) {
+ connectionPromise = amqp
+ .connect(config.NUQ_RABBITMQ_URL!, {
+ clientProperties: { connection_name: "zdr-cleanup" },
+ })
+ .then(conn => {
+ if (closed) {
+ void conn.close().catch(() => {});
+ throw new Error("ZDR queue closed while connecting");
+ }
+ connection = conn;
+ conn.on("close", () => handleConnectionDrop("connection closed"));
+ conn.on("error", error =>
+ logger.error("ZDR queue connection error", { error }),
+ );
+ return conn;
+ })
+ .catch(error => {
+ connectionPromise = null;
+ throw error;
+ });
+ }
+ return connectionPromise;
+}
+
+async function createPublishChannel(): Promise<amqp.ConfirmChannel> {
+ const conn = await getConnection();
+ const ch = await conn.createConfirmChannel();
+ await assertTopology(ch);
+ ch.on("close", () => {
+ if (publishChannel === ch) {
+ publishChannel = null;
+ publishChannelPromise = null;
+ }
+ });
+ ch.on("error", error => logger.error("ZDR publish channel error", { error }));
+ const publishes = new Map<string, { returned: boolean }>();
+ pendingPublishes.set(ch, publishes);
+ ch.on("return", msg => {
+ const correlationId = msg.properties.correlationId;
+ if (correlationId) {
+ const publish = publishes.get(correlationId);
+ if (publish) publish.returned = true;
+ }
+ });
+ return ch;
+}
+
+async function createConsumeChannel(): Promise<amqp.Channel> {
+ const conn = await getConnection();
+ const ch = await conn.createChannel();
+ await assertTopology(ch);
+ ch.on("close", () => {
+ if (consumeChannel === ch) {
+ consumeChannel = null;
+ consumeChannelPromise = null;
+ subscribed = false;
+ if (!closed) scheduleReconnect();
+ }
+ });
+ ch.on("error", error => logger.error("ZDR consume channel error", { error }));
+ return ch;
+}
+
+async function getPublishChannel(): Promise<amqp.ConfirmChannel> {
+ if (publishChannel) return publishChannel;
+ if (!publishChannelPromise) {
+ publishChannelPromise = createPublishChannel()
+ .then(ch => {
+ publishChannel = ch;
+ return ch;
+ })
+ .catch(error => {
+ publishChannelPromise = null;
+ throw error;
+ });
+ }
+ return publishChannelPromise;
+}
+
+async function getConsumeChannel(): Promise<amqp.Channel> {
+ if (consumeChannel) return consumeChannel;
+ if (!consumeChannelPromise) {
+ consumeChannelPromise = createConsumeChannel()
+ .then(ch => {
+ consumeChannel = ch;
+ return ch;
+ })
+ .catch(error => {
+ consumeChannelPromise = null;
+ throw error;
+ });
+ }
+ return consumeChannelPromise;
+}
+
+function publishConfirmed(
+ ch: amqp.ConfirmChannel,
+ job: ZdrCleanupJob,
+): Promise<void> {
+ const correlationId = randomUUID();
+ const publish = { returned: false };
+ const publishes = pendingPublishes.get(ch);
+ publishes?.set(correlationId, publish);
+
+ return new Promise((resolve, reject) => {
+ ch.sendToQueue(
+ DELAY_QUEUE,
+ Buffer.from(JSON.stringify(job)),
+ {
+ persistent: true,
+ contentType: "application/json",
+ messageId: job.requestId,
+ correlationId,
+ timestamp: Math.floor(Date.now() / 1000),
+ mandatory: true,
+ },
+ error => {
+ setImmediate(() => {
+ publishes?.delete(correlationId);
+ if (error) reject(error);
+ else if (publish.returned) {
+ reject(new Error("RabbitMQ returned unroutable ZDR cleanup job"));
+ } else resolve();
+ });
+ },
+ );
+ });
+}
+
+export async function enqueueZdrCleanupJob(requestId: string): Promise<void> {
+ if (!config.NUQ_RABBITMQ_URL) {
+ throw new Error("NUQ_RABBITMQ_URL is not configured");
+ }
+
+ let delayMs = RECONNECT_BASE_DELAY_MS;
+ while (!closed) {
+ try {
+ await withSpan("zdr.rabbitmq.publish", async span => {
+ setSpanAttributes(span, {
+ "messaging.system": "rabbitmq",
+ "messaging.destination.name": DELAY_QUEUE,
+ "messaging.operation.name": "publish",
+ "messaging.message.id": requestId,
+ "zdr.rabbitmq.retry_delay_ms": delayMs,
+ });
+ const ch = await getPublishChannel();
+ await publishConfirmed(ch, { requestId });
+ setSpanAttributes(span, {
+ "zdr.rabbitmq.outcome": "confirmed",
+ });
+ });
+ return;
+ } catch (error) {
+ logger.error("Failed to durably enqueue ZDR cleanup job; retrying", {
+ error,
+ requestId,
+ delayMs,
+ });
+ await publishChannel?.close().catch(() => {});
+ publishChannel = null;
+ publishChannelPromise = null;
+ await new Promise(resolve => setTimeout(resolve, delayMs));
+ delayMs = Math.min(delayMs * 2, RECONNECT_MAX_DELAY_MS);
+ }
+ }
+
+ throw new Error("ZDR queue closed before the cleanup job was confirmed");
+}
+
+function isZdrCleanupJob(value: unknown): value is ZdrCleanupJob {
+ return (
+ typeof value === "object" &&
+ value !== null &&
+ typeof (value as Partial<ZdrCleanupJob>).requestId === "string" &&
+ (value as ZdrCleanupJob).requestId.length > 0
+ );
+}
+
+async function subscribe(
+ ch: amqp.Channel,
+ handler: CleanupHandler,
+): Promise<void> {
+ if (subscribed) return;
+ await ch.prefetch(10);
+ await ch.consume(
+ READY_QUEUE,
+ async msg => {
+ if (!msg) {
+ subscribed = false;
+ scheduleReconnect();
+ return;
+ }
+ try {
+ await withSpan("zdr.rabbitmq.consume", async span => {
+ setSpanAttributes(span, {
+ "messaging.system": "rabbitmq",
+ "messaging.destination.name": READY_QUEUE,
+ "messaging.operation.name": "process",
+ "messaging.message.id": msg.properties.messageId,
+ "messaging.rabbitmq.delivery_tag": msg.fields.deliveryTag,
+ "messaging.rabbitmq.redelivered": msg.fields.redelivered,
+ });
+
+ let job: unknown;
+ try {
+ job = JSON.parse(msg.content.toString());
+ } catch (error) {
+ recordSpanException(span, error);
+ setSpanAttributes(span, {
+ "zdr.rabbitmq.outcome": "dead_lettered",
+ });
+ logger.error("Dead-lettering malformed ZDR cleanup job", { error });
+ ch.nack(msg, false, false);
+ return;
+ }
+
+ if (!isZdrCleanupJob(job)) {
+ const error = new Error("Invalid ZDR cleanup job");
+ recordSpanException(span, error);
+ setSpanAttributes(span, {
+ "zdr.rabbitmq.outcome": "dead_lettered",
+ });
+ logger.error("Dead-lettering invalid ZDR cleanup job", { job });
+ ch.nack(msg, false, false);
+ return;
+ }
+
+ setSpanAttributes(span, {
+ "zdr.request_id": job.requestId,
+ });
+ await handler(job);
+ ch.ack(msg);
+ setSpanAttributes(span, {
+ "zdr.rabbitmq.outcome": "acknowledged",
+ });
+ });
+ } catch (error) {
+ logger.error("ZDR cleanup job failed; requeueing", {
+ error,
+ messageId: msg.properties.messageId,
+ });
+ ch.nack(msg, false, true);
+ }
+ },
+ { noAck: false },
+ );
+ subscribed = true;
+ logger.info("Started consuming ZDR cleanup jobs");
+}
+
+export async function consumeZdrCleanupJobs(
+ handler: CleanupHandler,
+): Promise<void> {
+ registeredHandler = handler;
+ closed = false;
+ try {
+ const ch = await getConsumeChannel();
+ if (closed) {
+ await ch.close().catch(() => {});
+ await connection?.close().catch(() => {});
+ resetCachedState();
+ return;
+ }
+ await subscribe(ch, handler);
+ } catch (error) {
+ if (closed) return;
+ logger.error("Failed to start ZDR queue consumer; retrying", { error });
+ resetCachedState();
+ scheduleReconnect();
+ }
+}
+
+export async function shutdownZdrQueue(): Promise<void> {
+ closed = true;
+ registeredHandler = null;
+ if (reconnectTimer) clearTimeout(reconnectTimer);
+ reconnectTimer = null;
+ await publishChannel?.close().catch(() => {});
+ await consumeChannel?.close().catch(() => {});
+ await connection?.close().catch(() => {});
+ resetCachedState();
+}
diff --git a/apps/api/src/lib/zdrcleaner.test.ts b/apps/api/src/lib/zdrcleaner.test.ts
new file mode 100644
index 000000000..396bae881
--- /dev/null
+++ b/apps/api/src/lib/zdrcleaner.test.ts
@@ -0,0 +1,151 @@
+import { vi } from "vitest";
+
+const {
+ tables,
+ select,
+ update,
+ set,
+ where,
+ getZdrCleanupBatch,
+ removeJobFromGCS,
+ spans,
+ withSpan,
+ setSpanAttributes,
+} = vi.hoisted(() => {
+ const makeTable = (name: string) => ({
+ name,
+ id: `${name}.id`,
+ request_id: `${name}.request_id`,
+ });
+ const tables = {
+ requests: makeTable("requests"),
+ scrapes: makeTable("scrapes"),
+ searches: makeTable("searches"),
+ extracts: makeTable("extracts"),
+ maps: makeTable("maps"),
+ llmstxts: makeTable("llmstxts"),
+ deep_researches: makeTable("deep_researches"),
+ };
+ const where = vi.fn(async () => {});
+ const set = vi.fn(() => ({ where }));
+ const update = vi.fn(() => ({ set }));
+ const select = vi.fn(() => ({
+ from: (table: { name: string }) => ({
+ where: vi.fn(async () => [{ id: `${table.name}-blob` }]),
+ }),
+ }));
+ const spans: Array<{ name: string; attributes: Record<string, unknown> }> =
+ [];
+ const withSpan = vi.fn(async (name: string, fn: (span: any) => any) => {
+ const span = { attributes: {} };
+ spans.push({ name, attributes: span.attributes });
+ return fn(span);
+ });
+ const setSpanAttributes = vi.fn(
+ (span: { attributes: Record<string, unknown> }, attributes: object) => {
+ Object.assign(span.attributes, attributes);
+ },
+ );
+ return {
+ tables,
+ select,
+ update,
+ set,
+ where,
+ getZdrCleanupBatch: vi.fn<
+ (limit: number) => Promise<Array<{ request_id: string; ids: string[] }>>
+ >(async () => []),
+ removeJobFromGCS: vi.fn<(id: string) => Promise<void>>(async () => {}),
+ spans,
+ withSpan,
+ setSpanAttributes,
+ };
+});
+
+vi.mock("drizzle-orm", () => ({
+ eq: vi.fn(() => "eq"),
+ inArray: vi.fn(() => "inArray"),
+}));
+vi.mock("../db/connection", () => ({ db: { select, update } }));
+vi.mock("../db/schema", () => tables);
+vi.mock("../db/rpc", () => ({ getZdrCleanupBatch }));
+vi.mock("./gcs-jobs", () => ({ removeJobFromGCS }));
+vi.mock("../config", () => ({ config: {} }));
+vi.mock("./logger", () => {
+ const logger: any = {
+ child: vi.fn(() => logger),
+ debug: vi.fn(),
+ warn: vi.fn(),
+ error: vi.fn(),
+ };
+ return { logger };
+});
+vi.mock("./otel-tracer", () => ({ withSpan, setSpanAttributes }));
+
+import { cleanZdrRequest, zdrcleaner } from "./zdrcleaner";
+
+describe("ZDR cleaner", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ getZdrCleanupBatch.mockResolvedValue([]);
+ removeJobFromGCS.mockResolvedValue(undefined);
+ spans.length = 0;
+ });
+
+ it("loads and removes every blob type for a queued request", async () => {
+ await cleanZdrRequest("request-1");
+
+ expect(select).toHaveBeenCalledTimes(6);
+ expect(removeJobFromGCS.mock.calls.map(([id]) => id)).toEqual([
+ "scrapes-blob",
+ "searches-blob",
+ "extracts-blob",
+ "maps-blob",
+ "llmstxts-blob",
+ "deep_researches-blob",
+ ]);
+ expect(spans).toEqual(
+ expect.arrayContaining([
+ {
+ name: "zdr.cleanup.request",
+ attributes: expect.objectContaining({
+ "zdr.request_id": "request-1",
+ "zdr.blob_count": 6,
+ "zdr.cleanup.outcome": "completed",
+ }),
+ },
+ {
+ name: "zdr.postgres.read_blobs",
+ attributes: expect.objectContaining({
+ "db.system": "postgresql",
+ "db.operation.name": "select",
+ "db.response.returned_rows": 6,
+ }),
+ },
+ ]),
+ );
+ });
+
+ it("fails a queued request if any blob could not be removed", async () => {
+ removeJobFromGCS.mockRejectedValueOnce(new Error("GCS unavailable"));
+
+ await expect(cleanZdrRequest("request-2")).rejects.toThrow(
+ "Failed to remove 1 blobs for ZDR request request-2",
+ );
+ });
+
+ it("keeps draining and clearing legacy PostgreSQL schedules", async () => {
+ getZdrCleanupBatch.mockResolvedValueOnce([
+ { request_id: "legacy-1", ids: ["blob-1"] },
+ { request_id: "legacy-2", ids: ["blob-2"] },
+ ]);
+ removeJobFromGCS.mockImplementation(async id => {
+ if (id === "blob-2") throw new Error("GCS unavailable");
+ });
+
+ await zdrcleaner();
+
+ expect(set).toHaveBeenCalledWith({ dr_clean_by: null });
+ expect(where).toHaveBeenCalledOnce();
+ });
+});
diff --git a/apps/api/src/lib/zdrcleaner.ts b/apps/api/src/lib/zdrcleaner.ts
index 8ab72ecf4..029ac9377 100644
--- a/apps/api/src/lib/zdrcleaner.ts
+++ b/apps/api/src/lib/zdrcleaner.ts
@@ -1,11 +1,12 @@
import "dotenv/config";
-import { inArray } from "drizzle-orm";
+import { eq, inArray } from "drizzle-orm";
import { db } from "../db/connection";
import * as schema from "../db/schema";
import { getZdrCleanupBatch } from "../db/rpc";
import { removeJobFromGCS } from "./gcs-jobs";
import { logger as _logger } from "./logger";
import { config } from "../config";
+import { setSpanAttributes, withSpan } from "./otel-tracer";
async function sendHeartbeat() {
if (config.ZDRCLEANER_HEARTBEAT_URL) {
@@ -13,6 +14,86 @@ async function sendHeartbeat() {
}
}
+async function removeBlobs(blobIds: string[]): Promise<unknown[]> {
+ const logger = _logger.child({
+ module: "zdrcleaner",
+ method: "removeBlobs",
+ zeroDataRetention: true,
+ });
+ const results = await Promise.allSettled(
+ blobIds.map(blobId => removeJobFromGCS(blobId, logger)),
+ );
+ return results.flatMap(result =>
+ result.status === "rejected" ? [result.reason] : [],
+ );
+}
+
+async function getRequestBlobIds(requestId: string): Promise<string[]> {
+ return withSpan("zdr.postgres.read_blobs", async span => {
+ setSpanAttributes(span, {
+ "db.system": "postgresql",
+ "db.operation.name": "select",
+ "zdr.request_id": requestId,
+ });
+ const [scrapes, searches, extracts, maps, llmstxts, deepResearches] =
+ await Promise.all([
+ db
+ .select({ id: schema.scrapes.id })
+ .from(schema.scrapes)
+ .where(eq(schema.scrapes.request_id, requestId)),
+ db
+ .select({ id: schema.searches.id })
+ .from(schema.searches)
+ .where(eq(schema.searches.request_id, requestId)),
+ db
+ .select({ id: schema.extracts.id })
+ .from(schema.extracts)
+ .where(eq(schema.extracts.request_id, requestId)),
+ db
+ .select({ id: schema.maps.id })
+ .from(schema.maps)
+ .where(eq(schema.maps.request_id, requestId)),
+ db
+ .select({ id: schema.llmstxts.id })
+ .from(schema.llmstxts)
+ .where(eq(schema.llmstxts.request_id, requestId)),
+ db
+ .select({ id: schema.deep_researches.id })
+ .from(schema.deep_researches)
+ .where(eq(schema.deep_researches.request_id, requestId)),
+ ]);
+
+ const blobIds = [
+ ...scrapes,
+ ...searches,
+ ...extracts,
+ ...maps,
+ ...llmstxts,
+ ...deepResearches,
+ ].map(row => row.id);
+ setSpanAttributes(span, {
+ "db.response.returned_rows": blobIds.length,
+ });
+ return blobIds;
+ });
+}
+
+export async function cleanZdrRequest(requestId: string): Promise<void> {
+ await withSpan("zdr.cleanup.request", async span => {
+ setSpanAttributes(span, { "zdr.request_id": requestId });
+ const blobIds = await getRequestBlobIds(requestId);
+ setSpanAttributes(span, { "zdr.blob_count": blobIds.length });
+ const errors = await removeBlobs(blobIds);
+ if (errors.length > 0) {
+ throw new AggregateError(
+ errors,
+ `Failed to remove ${errors.length} blobs for ZDR request ${requestId}`,
+ );
+ }
+ setSpanAttributes(span, { "zdr.cleanup.outcome": "completed" });
+ });
+}
+
export async function zdrcleaner() {
const logger = _logger.child({
module: "zdrcleaner",
@@ -38,43 +119,17 @@ export async function zdrcleaner() {
return;
}
- const requestBlobs = new Map<string, Set<string>>();
- const successfulBlobs = new Set<string>();
-
- // Track all blobs for each request before processing
- for (const row of rows) {
- requestBlobs.set(row.request_id, new Set(row.ids));
- }
-
- // Process in batches of 50 for GCS cleanup
const deleteErrors: any[] = [];
+ const fullyCleanedRequests: string[] = [];
for (let j = 0; j < Math.ceil(rows.length / 50); j++) {
const batch = rows.slice(j * 50, (j + 1) * 50);
- await Promise.allSettled(
- batch
- .flatMap(x => x.ids)
- .map(async blob_id => {
- try {
- await removeJobFromGCS(
- blob_id,
- logger.child({ zeroDataRetention: true }),
- );
- successfulBlobs.add(blob_id);
- } catch (error) {
- deleteErrors.push(error);
- }
- }),
- );
- }
-
- const fullyCleanedRequests: string[] = [];
- for (const [requestId, blobIds] of requestBlobs) {
- const allBlobsCleaned = [...blobIds].every(blobId =>
- successfulBlobs.has(blobId),
+ await Promise.all(
+ batch.map(async row => {
+ const errors = await removeBlobs(row.ids);
+ if (errors.length === 0) fullyCleanedRequests.push(row.request_id);
+ else deleteErrors.push(...errors);
+ }),
);
- if (allBlobsCleaned) {
- fullyCleanedRequests.push(requestId);
- }
}
const updateErrors: any[] = [];
diff --git a/apps/api/src/services/logging/log_job.test.ts b/apps/api/src/services/logging/log_job.test.ts
index 131e2d952..b5f6fe84b 100644
--- a/apps/api/src/services/logging/log_job.test.ts
+++ b/apps/api/src/services/logging/log_job.test.ts
@@ -18,6 +18,7 @@ const {
withSpan,
setSpanAttributes,
spans,
+ enqueueZdrCleanupJob,
} = vi.hoisted(() => {
const logger: any = {
info: vi.fn(),
@@ -63,6 +64,7 @@ const {
withSpan,
setSpanAttributes: vi.fn(),
spans,
+ enqueueZdrCleanupJob: vi.fn(async () => {}),
};
});
@@ -121,6 +123,10 @@ vi.mock("../../lib/gcs-jobs", () => ({
saveSearchToGCS: vi.fn(),
}));
+vi.mock("../../lib/zdr-queue", () => ({
+ enqueueZdrCleanupJob,
+}));
+
vi.mock("../../lib/extract/extract-redis", () => ({
saveExtractResult: vi.fn(),
}));
@@ -335,6 +341,32 @@ describe("logRequest", () => {
);
});
+ it("durably queues ZDR cleanup before writing the request", async () => {
+ await logRequest({
+ ...makeRequest("op_integration_42"),
+ zeroDataRetention: true,
+ });
+
+ expect(enqueueZdrCleanupJob).toHaveBeenCalledWith(
+ "019e6f45-7778-727d-adf0-0abe9d5062b6",
+ );
+ expect(enqueueZdrCleanupJob.mock.invocationCallOrder[0]).toBeLessThan(
+ values.mock.invocationCallOrder[0],
+ );
+ expect(values.mock.calls[0][0]).not.toHaveProperty("dr_clean_by");
+ });
+
+ it("does not write a ZDR request unless its cleanup job was confirmed", async () => {
+ enqueueZdrCleanupJob.mockRejectedValueOnce(
+ new Error("RabbitMQ unavailable"),
+ );
+
+ await expect(
+ logRequest({ ...makeRequest(null), zeroDataRetention: true }),
+ ).rejects.toThrow("RabbitMQ unavailable");
+ expect(values).not.toHaveBeenCalled();
+ });
+
it("keeps the database write when Pub/Sub fails", async () => {
publishMessage.mockRejectedValueOnce(new Error("Pub/Sub unavailable"));
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index 6b1e7e853..86cf0fea7 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -2,6 +2,7 @@ import { db } from "../../db/connection";
import * as schema from "../../db/schema";
import { changeTrackingInsertScrape } from "../../lib/change-tracking-store";
import { config } from "../../config";
+import { enqueueZdrCleanupJob } from "../../lib/zdr-queue";
import "dotenv/config";
import { logger as _logger } from "../../lib/logger";
import { EXTERNAL_REQUEST_ID_MAX_BYTES } from "../../lib/external-request-id";
@@ -572,6 +573,10 @@ async function logRequestInternal(request: LoggedRequest) {
}
}
+ if (request.zeroDataRetention && config.USE_DB_AUTHENTICATION === true) {
+ await enqueueZdrCleanupJob(request.id);
+ }
+
await robustInsert(
"requests",
{
@@ -582,13 +587,10 @@ async function logRequestInternal(request: LoggedRequest) {
origin: sanitizedOrigin,
integration: sanitizedIntegration,
target_hint: sanitizedTargetHint,
- dr_clean_by: request.zeroDataRetention
- ? new Date(Date.now() + 24 * 60 * 60 * 1000)
- : null,
api_key_id: request.api_key_id ?? null,
// Not redacted under zero data retention: it is the caller's own
// operation id (attribution it asked for), not customer content — and
- // the row is cleaned at dr_clean_by regardless.
+ // the row's blobs are cleaned by the ZDR queue regardless.
external_request_id: boundedExternalRequestId(
sanitizeString(request.external_request_id ?? null),
logger,
diff --git a/apps/api/src/services/zdr-worker.ts b/apps/api/src/services/zdr-worker.ts
index 1ed439d7d..7702f9f7d 100644
--- a/apps/api/src/services/zdr-worker.ts
+++ b/apps/api/src/services/zdr-worker.ts
@@ -1,7 +1,8 @@
import "dotenv/config";
import { shutdownTracing } from "../otel";
import { logger } from "../lib/logger";
-import { zdrcleaner } from "../lib/zdrcleaner";
+import { cleanZdrRequest, zdrcleaner } from "../lib/zdrcleaner";
+import { consumeZdrCleanupJobs, shutdownZdrQueue } from "../lib/zdr-queue";
let isShuttingDown = false;
@@ -16,10 +17,13 @@ process.on("SIGTERM", () => {
});
(async () => {
+ await consumeZdrCleanupJobs(job => cleanZdrRequest(job.requestId));
+
while (!isShuttingDown) {
await zdrcleaner();
}
+ await shutdownZdrQueue();
await shutdownTracing();
logger.info("zdr-worker exiting");
process.exit(0);