Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): move ZDR cleanup scheduling to RabbitMQ (#4650)
Gergő Móricz90fd89cdcef8
diff --git a/apps/api/src/lib/zdr-queue.test.ts b/apps/api/src/lib/zdr-queue.test.tsnew file mode 100644index 000000000..2183540e1--- /dev/null+++ b/apps/api/src/lib/zdr-queue.test.ts@@ -0,0 +1,255 @@+import { vi, type Mock } from "vitest";++const {+ connect,+ connection,+ publishChannel,+ consumeChannel,+ consumerCallbacks,+ spans,+ withSpan,+ setSpanAttributes,+ recordSpanException,+ publishEvents,+} = vi.hoisted(() => {+ const consumerCallbacks: Array<(message: any) => Promise<void>> = [];+ const publishEvents = new Map<string, (...args: any[]) => void>();+ const spans: Array<{ name: string; attributes: Record<string, unknown> }> =+ [];+ const withSpan = vi.fn(async (name: string, fn: (span: any) => any) => {+ const span = { attributes: {} };+ spans.push({ name, attributes: span.attributes });+ return fn(span);+ });+ const setSpanAttributes = vi.fn(+ (span: { attributes: Record<string, unknown> }, attributes: object) => {+ Object.assign(span.attributes, attributes);+ },+ );+ const publishChannel: any = {+ assertExchange: vi.fn(async () => {}),+ assertQueue: vi.fn(async () => ({})),+ bindQueue: vi.fn(async () => {}),+ sendToQueue: vi.fn(+ (_queue: string, _content: Buffer, _options: any, callback: any) => {+ callback(null);+ return true;+ },+ ),+ close: vi.fn(async () => {}),+ on: vi.fn((event: string, handler: (...args: any[]) => void) => {+ publishEvents.set(event, handler);+ }),+ };+ const consumeChannel: any = {+ assertExchange: vi.fn(async () => {}),+ assertQueue: vi.fn(async () => ({})),+ bindQueue: vi.fn(async () => {}),+ prefetch: vi.fn(async () => {}),+ consume: vi.fn(async (_queue: string, callback: any) => {+ consumerCallbacks.push(callback);+ return { consumerTag: "zdr-test" };+ }),+ ack: vi.fn(),+ nack: vi.fn(),+ close: vi.fn(async () => {}),+ on: vi.fn(),+ };+ const connection: any = {+ createConfirmChannel: vi.fn(async () => publishChannel),+ createChannel: vi.fn(async () => consumeChannel),+ close: vi.fn(async () => {}),+ on: vi.fn(),+ };+ return {+ connect: vi.fn(async () => connection),+ connection,+ publishChannel,+ consumeChannel,+ consumerCallbacks,+ spans,+ withSpan,+ setSpanAttributes,+ recordSpanException: vi.fn(),+ publishEvents,+ };+});++vi.mock("amqplib", () => ({ default: { connect } }));+vi.mock("../config", () => ({+ config: { NUQ_RABBITMQ_URL: "amqp://rabbitmq" },+}));+vi.mock("./logger", () => {+ const logger: any = {+ child: vi.fn(() => logger),+ info: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ };+ return { logger };+});+vi.mock("./otel-tracer", () => ({+ withSpan,+ setSpanAttributes,+ recordSpanException,+}));++function message(body: unknown) {+ return {+ content: Buffer.from(+ typeof body === "string" ? body : JSON.stringify(body),+ ),+ fields: {},+ properties: {},+ };+}++describe("ZDR cleanup queue", () => {+ beforeEach(() => {+ vi.resetModules();+ vi.clearAllMocks();+ consumerCallbacks.length = 0;+ publishEvents.clear();+ spans.length = 0;+ });++ it("publishes persistent jobs only after a broker confirm", async () => {+ const { enqueueZdrCleanupJob, shutdownZdrQueue } = await import(+ "./zdr-queue.js"+ );++ await enqueueZdrCleanupJob("request-1");++ expect(connect).toHaveBeenCalledWith(+ "amqp://rabbitmq",+ expect.objectContaining({+ clientProperties: { connection_name: "zdr-cleanup" },+ }),+ );+ expect(publishChannel.sendToQueue).toHaveBeenCalledWith(+ "zdr.cleanup.delayed.24h",+ Buffer.from(JSON.stringify({ requestId: "request-1" })),+ expect.objectContaining({+ persistent: true,+ messageId: "request-1",+ correlationId: expect.any(String),+ timestamp: expect.any(Number),+ mandatory: true,+ }),+ expect.any(Function),+ );+ expect(spans).toContainEqual({+ name: "zdr.rabbitmq.publish",+ attributes: expect.objectContaining({+ "messaging.system": "rabbitmq",+ "messaging.destination.name": "zdr.cleanup.delayed.24h",+ "messaging.operation.name": "publish",+ "messaging.message.id": "request-1",+ "zdr.rabbitmq.outcome": "confirmed",+ }),+ });+ await shutdownZdrQueue();+ });++ it("retries a mandatory publish returned as unroutable", async () => {+ vi.useFakeTimers();+ try {+ publishChannel.sendToQueue.mockImplementationOnce(+ (+ _queue: string,+ _content: Buffer,+ options: { correlationId: string },+ callback: (error: Error | null) => void,+ ) => {+ publishEvents.get("return")?.({+ properties: { correlationId: options.correlationId },+ });+ callback(null);+ return true;+ },+ );+ const { enqueueZdrCleanupJob, shutdownZdrQueue } = await import(+ "./zdr-queue.js"+ );++ const enqueue = enqueueZdrCleanupJob("request-returned");+ await vi.runAllTimersAsync();+ await enqueue;++ expect(publishChannel.sendToQueue).toHaveBeenCalledTimes(2);+ await shutdownZdrQueue();+ } finally {+ vi.useRealTimers();+ }+ });++ it("uses replicated queues and at-least-once delayed delivery", async () => {+ const { enqueueZdrCleanupJob, shutdownZdrQueue } = await import(+ "./zdr-queue.js"+ );++ await enqueueZdrCleanupJob("request-2");++ const queueDeclarations = new Map(+ (publishChannel.assertQueue as Mock).mock.calls.map(([name, options]) => [+ name,+ options,+ ]),+ );+ expect(queueDeclarations.get("zdr.cleanup.delayed.24h")).toMatchObject({+ durable: true,+ arguments: {+ "x-queue-type": "quorum",+ "x-overflow": "reject-publish",+ "x-message-ttl": 86_400_000,+ "x-dead-letter-strategy": "at-least-once",+ "x-dead-letter-exchange": "zdr.cleanup",+ "x-dead-letter-routing-key": "ready",+ },+ });+ expect(queueDeclarations.get("zdr.cleanup.ready")).toMatchObject({+ durable: true,+ arguments: {+ "x-queue-type": "quorum",+ "x-overflow": "reject-publish",+ "x-delivery-limit": -1,+ },+ });+ await shutdownZdrQueue();+ });++ it("acks completed jobs and requeues failures without a delivery cap", async () => {+ const { consumeZdrCleanupJobs, shutdownZdrQueue } = await import(+ "./zdr-queue.js"+ );+ const handler = vi.fn(async () => {});+ await consumeZdrCleanupJobs(handler);+ const consume = consumerCallbacks[0];++ const completed = message({ requestId: "request-3" });+ await consume(completed);+ expect(handler).toHaveBeenCalledWith({ requestId: "request-3" });+ expect(consumeChannel.ack).toHaveBeenCalledWith(completed);+ expect(spans).toContainEqual({+ name: "zdr.rabbitmq.consume",+ attributes: expect.objectContaining({+ "messaging.system": "rabbitmq",+ "messaging.destination.name": "zdr.cleanup.ready",+ "messaging.operation.name": "process",+ "zdr.request_id": "request-3",+ "zdr.rabbitmq.outcome": "acknowledged",+ }),+ });++ handler.mockRejectedValueOnce(new Error("GCS unavailable"));+ const failed = message({ requestId: "request-4" });+ await consume(failed);+ expect(consumeChannel.nack).toHaveBeenCalledWith(failed, false, true);++ const malformed = message("not-json");+ await consume(malformed);+ expect(consumeChannel.nack).toHaveBeenCalledWith(malformed, false, false);+ expect(recordSpanException).toHaveBeenCalled();+ await shutdownZdrQueue();+ });+});diff --git a/apps/api/src/lib/zdr-queue.ts b/apps/api/src/lib/zdr-queue.tsnew file mode 100644index 000000000..b873ff2be--- /dev/null+++ b/apps/api/src/lib/zdr-queue.ts@@ -0,0 +1,414 @@+import amqp from "amqplib";+import { randomUUID } from "node:crypto";+import { config } from "../config";+import { logger as _logger } from "./logger";+import {+ recordSpanException,+ setSpanAttributes,+ withSpan,+} from "./otel-tracer";++const DELAY_MS = 24 * 60 * 60 * 1000;+const EXCHANGE = "zdr.cleanup";+const DELAY_QUEUE = "zdr.cleanup.delayed.24h";+const READY_QUEUE = "zdr.cleanup.ready";+const DLQ = "zdr.cleanup.dlq";+const READY_ROUTING_KEY = "ready";+const DLQ_ROUTING_KEY = "dead";+const RECONNECT_BASE_DELAY_MS = 1_000;+const RECONNECT_MAX_DELAY_MS = 30_000;++const logger = _logger.child({ module: "zdr-queue" });++type ZdrCleanupJob = {+ requestId: string;+};++type CleanupHandler = (job: ZdrCleanupJob) => Promise<void>;++let connection: amqp.ChannelModel | null = null;+let connectionPromise: Promise<amqp.ChannelModel> | null = null;+let publishChannel: amqp.ConfirmChannel | null = null;+let publishChannelPromise: Promise<amqp.ConfirmChannel> | null = null;+let consumeChannel: amqp.Channel | null = null;+let consumeChannelPromise: Promise<amqp.Channel> | null = null;+let registeredHandler: CleanupHandler | null = null;+let subscribed = false;+let reconnectTimer: NodeJS.Timeout | null = null;+let reconnectDelayMs = RECONNECT_BASE_DELAY_MS;+let closed = false;+const pendingPublishes = new WeakMap<+ amqp.ConfirmChannel,+ Map<string, { returned: boolean }>+>();++async function assertTopology(ch: amqp.Channel): Promise<void> {+ await ch.assertExchange(EXCHANGE, "direct", { durable: true });++ await ch.assertQueue(DLQ, {+ durable: true,+ arguments: {+ "x-queue-type": "quorum",+ "x-overflow": "reject-publish",+ },+ });+ await ch.bindQueue(DLQ, EXCHANGE, DLQ_ROUTING_KEY);++ await ch.assertQueue(READY_QUEUE, {+ durable: true,+ arguments: {+ "x-queue-type": "quorum",+ "x-overflow": "reject-publish",+ "x-delivery-limit": -1,+ "x-dead-letter-strategy": "at-least-once",+ "x-dead-letter-exchange": EXCHANGE,+ "x-dead-letter-routing-key": DLQ_ROUTING_KEY,+ },+ });+ await ch.bindQueue(READY_QUEUE, EXCHANGE, READY_ROUTING_KEY);++ await ch.assertQueue(DELAY_QUEUE, {+ durable: true,+ arguments: {+ "x-queue-type": "quorum",+ "x-overflow": "reject-publish",+ "x-message-ttl": DELAY_MS,+ "x-dead-letter-strategy": "at-least-once",+ "x-dead-letter-exchange": EXCHANGE,+ "x-dead-letter-routing-key": READY_ROUTING_KEY,+ },+ });+}++function resetCachedState(): void {+ connection = null;+ connectionPromise = null;+ publishChannel = null;+ publishChannelPromise = null;+ consumeChannel = null;+ consumeChannelPromise = null;+ subscribed = false;+}++function handleConnectionDrop(reason: string, error?: unknown): void {+ if (connection || publishChannel || consumeChannel) {+ logger.warn("ZDR queue connection dropped", { reason, error });+ }+ resetCachedState();+ if (registeredHandler && !closed) scheduleReconnect();+}++function scheduleReconnect(): void {+ if (reconnectTimer || closed) return;++ const delayMs = reconnectDelayMs;+ reconnectTimer = setTimeout(async () => {+ reconnectTimer = null;+ try {+ if (registeredHandler) {+ const ch = await getConsumeChannel();+ if (closed) {+ await ch.close().catch(() => {});+ await connection?.close().catch(() => {});+ resetCachedState();+ return;+ }+ await subscribe(ch, registeredHandler);+ }+ reconnectDelayMs = RECONNECT_BASE_DELAY_MS;+ logger.info("ZDR queue reconnected; consumer re-subscribed");+ } catch (error) {+ logger.error("ZDR queue reconnect failed", { error, delayMs });+ resetCachedState();+ reconnectDelayMs = Math.min(delayMs * 2, RECONNECT_MAX_DELAY_MS);+ scheduleReconnect();+ }+ }, delayMs);+ reconnectTimer.unref?.();+}++async function getConnection(): Promise<amqp.ChannelModel> {+ if (connection) return connection;+ if (!connectionPromise) {+ connectionPromise = amqp+ .connect(config.NUQ_RABBITMQ_URL!, {+ clientProperties: { connection_name: "zdr-cleanup" },+ })+ .then(conn => {+ if (closed) {+ void conn.close().catch(() => {});+ throw new Error("ZDR queue closed while connecting");+ }+ connection = conn;+ conn.on("close", () => handleConnectionDrop("connection closed"));+ conn.on("error", error =>+ logger.error("ZDR queue connection error", { error }),+ );+ return conn;+ })+ .catch(error => {+ connectionPromise = null;+ throw error;+ });+ }+ return connectionPromise;+}++async function createPublishChannel(): Promise<amqp.ConfirmChannel> {+ const conn = await getConnection();+ const ch = await conn.createConfirmChannel();+ await assertTopology(ch);+ ch.on("close", () => {+ if (publishChannel === ch) {+ publishChannel = null;+ publishChannelPromise = null;+ }+ });+ ch.on("error", error => logger.error("ZDR publish channel error", { error }));+ const publishes = new Map<string, { returned: boolean }>();+ pendingPublishes.set(ch, publishes);+ ch.on("return", msg => {+ const correlationId = msg.properties.correlationId;+ if (correlationId) {+ const publish = publishes.get(correlationId);+ if (publish) publish.returned = true;+ }+ });+ return ch;+}++async function createConsumeChannel(): Promise<amqp.Channel> {+ const conn = await getConnection();+ const ch = await conn.createChannel();+ await assertTopology(ch);+ ch.on("close", () => {+ if (consumeChannel === ch) {+ consumeChannel = null;+ consumeChannelPromise = null;+ subscribed = false;+ if (!closed) scheduleReconnect();+ }+ });+ ch.on("error", error => logger.error("ZDR consume channel error", { error }));+ return ch;+}++async function getPublishChannel(): Promise<amqp.ConfirmChannel> {+ if (publishChannel) return publishChannel;+ if (!publishChannelPromise) {+ publishChannelPromise = createPublishChannel()+ .then(ch => {+ publishChannel = ch;+ return ch;+ })+ .catch(error => {+ publishChannelPromise = null;+ throw error;+ });+ }+ return publishChannelPromise;+}++async function getConsumeChannel(): Promise<amqp.Channel> {+ if (consumeChannel) return consumeChannel;+ if (!consumeChannelPromise) {+ consumeChannelPromise = createConsumeChannel()+ .then(ch => {+ consumeChannel = ch;+ return ch;+ })+ .catch(error => {+ consumeChannelPromise = null;+ throw error;+ });+ }+ return consumeChannelPromise;+}++function publishConfirmed(+ ch: amqp.ConfirmChannel,+ job: ZdrCleanupJob,+): Promise<void> {+ const correlationId = randomUUID();+ const publish = { returned: false };+ const publishes = pendingPublishes.get(ch);+ publishes?.set(correlationId, publish);++ return new Promise((resolve, reject) => {+ ch.sendToQueue(+ DELAY_QUEUE,+ Buffer.from(JSON.stringify(job)),+ {+ persistent: true,+ contentType: "application/json",+ messageId: job.requestId,+ correlationId,+ timestamp: Math.floor(Date.now() / 1000),+ mandatory: true,+ },+ error => {+ setImmediate(() => {+ publishes?.delete(correlationId);+ if (error) reject(error);+ else if (publish.returned) {+ reject(new Error("RabbitMQ returned unroutable ZDR cleanup job"));+ } else resolve();+ });+ },+ );+ });+}++export async function enqueueZdrCleanupJob(requestId: string): Promise<void> {+ if (!config.NUQ_RABBITMQ_URL) {+ throw new Error("NUQ_RABBITMQ_URL is not configured");+ }++ let delayMs = RECONNECT_BASE_DELAY_MS;+ while (!closed) {+ try {+ await withSpan("zdr.rabbitmq.publish", async span => {+ setSpanAttributes(span, {+ "messaging.system": "rabbitmq",+ "messaging.destination.name": DELAY_QUEUE,+ "messaging.operation.name": "publish",+ "messaging.message.id": requestId,+ "zdr.rabbitmq.retry_delay_ms": delayMs,+ });+ const ch = await getPublishChannel();+ await publishConfirmed(ch, { requestId });+ setSpanAttributes(span, {+ "zdr.rabbitmq.outcome": "confirmed",+ });+ });+ return;+ } catch (error) {+ logger.error("Failed to durably enqueue ZDR cleanup job; retrying", {+ error,+ requestId,+ delayMs,+ });+ await publishChannel?.close().catch(() => {});+ publishChannel = null;+ publishChannelPromise = null;+ await new Promise(resolve => setTimeout(resolve, delayMs));+ delayMs = Math.min(delayMs * 2, RECONNECT_MAX_DELAY_MS);+ }+ }++ throw new Error("ZDR queue closed before the cleanup job was confirmed");+}++function isZdrCleanupJob(value: unknown): value is ZdrCleanupJob {+ return (+ typeof value === "object" &&+ value !== null &&+ typeof (value as Partial<ZdrCleanupJob>).requestId === "string" &&+ (value as ZdrCleanupJob).requestId.length > 0+ );+}++async function subscribe(+ ch: amqp.Channel,+ handler: CleanupHandler,+): Promise<void> {+ if (subscribed) return;+ await ch.prefetch(10);+ await ch.consume(+ READY_QUEUE,+ async msg => {+ if (!msg) {+ subscribed = false;+ scheduleReconnect();+ return;+ }+ try {+ await withSpan("zdr.rabbitmq.consume", async span => {+ setSpanAttributes(span, {+ "messaging.system": "rabbitmq",+ "messaging.destination.name": READY_QUEUE,+ "messaging.operation.name": "process",+ "messaging.message.id": msg.properties.messageId,+ "messaging.rabbitmq.delivery_tag": msg.fields.deliveryTag,+ "messaging.rabbitmq.redelivered": msg.fields.redelivered,+ });++ let job: unknown;+ try {+ job = JSON.parse(msg.content.toString());+ } catch (error) {+ recordSpanException(span, error);+ setSpanAttributes(span, {+ "zdr.rabbitmq.outcome": "dead_lettered",+ });+ logger.error("Dead-lettering malformed ZDR cleanup job", { error });+ ch.nack(msg, false, false);+ return;+ }++ if (!isZdrCleanupJob(job)) {+ const error = new Error("Invalid ZDR cleanup job");+ recordSpanException(span, error);+ setSpanAttributes(span, {+ "zdr.rabbitmq.outcome": "dead_lettered",+ });+ logger.error("Dead-lettering invalid ZDR cleanup job", { job });+ ch.nack(msg, false, false);+ return;+ }++ setSpanAttributes(span, {+ "zdr.request_id": job.requestId,+ });+ await handler(job);+ ch.ack(msg);+ setSpanAttributes(span, {+ "zdr.rabbitmq.outcome": "acknowledged",+ });+ });+ } catch (error) {+ logger.error("ZDR cleanup job failed; requeueing", {+ error,+ messageId: msg.properties.messageId,+ });+ ch.nack(msg, false, true);+ }+ },+ { noAck: false },+ );+ subscribed = true;+ logger.info("Started consuming ZDR cleanup jobs");+}++export async function consumeZdrCleanupJobs(+ handler: CleanupHandler,+): Promise<void> {+ registeredHandler = handler;+ closed = false;+ try {+ const ch = await getConsumeChannel();+ if (closed) {+ await ch.close().catch(() => {});+ await connection?.close().catch(() => {});+ resetCachedState();+ return;+ }+ await subscribe(ch, handler);+ } catch (error) {+ if (closed) return;+ logger.error("Failed to start ZDR queue consumer; retrying", { error });+ resetCachedState();+ scheduleReconnect();+ }+}++export async function shutdownZdrQueue(): Promise<void> {+ closed = true;+ registeredHandler = null;+ if (reconnectTimer) clearTimeout(reconnectTimer);+ reconnectTimer = null;+ await publishChannel?.close().catch(() => {});+ await consumeChannel?.close().catch(() => {});+ await connection?.close().catch(() => {});+ resetCachedState();+}diff --git a/apps/api/src/lib/zdrcleaner.test.ts b/apps/api/src/lib/zdrcleaner.test.tsnew file mode 100644index 000000000..396bae881--- /dev/null+++ b/apps/api/src/lib/zdrcleaner.test.ts@@ -0,0 +1,151 @@+import { vi } from "vitest";++const {+ tables,+ select,+ update,+ set,+ where,+ getZdrCleanupBatch,+ removeJobFromGCS,+ spans,+ withSpan,+ setSpanAttributes,+} = vi.hoisted(() => {+ const makeTable = (name: string) => ({+ name,+ id: `${name}.id`,+ request_id: `${name}.request_id`,+ });+ const tables = {+ requests: makeTable("requests"),+ scrapes: makeTable("scrapes"),+ searches: makeTable("searches"),+ extracts: makeTable("extracts"),+ maps: makeTable("maps"),+ llmstxts: makeTable("llmstxts"),+ deep_researches: makeTable("deep_researches"),+ };+ const where = vi.fn(async () => {});+ const set = vi.fn(() => ({ where }));+ const update = vi.fn(() => ({ set }));+ const select = vi.fn(() => ({+ from: (table: { name: string }) => ({+ where: vi.fn(async () => [{ id: `${table.name}-blob` }]),+ }),+ }));+ const spans: Array<{ name: string; attributes: Record<string, unknown> }> =+ [];+ const withSpan = vi.fn(async (name: string, fn: (span: any) => any) => {+ const span = { attributes: {} };+ spans.push({ name, attributes: span.attributes });+ return fn(span);+ });+ const setSpanAttributes = vi.fn(+ (span: { attributes: Record<string, unknown> }, attributes: object) => {+ Object.assign(span.attributes, attributes);+ },+ );+ return {+ tables,+ select,+ update,+ set,+ where,+ getZdrCleanupBatch: vi.fn<+ (limit: number) => Promise<Array<{ request_id: string; ids: string[] }>>+ >(async () => []),+ removeJobFromGCS: vi.fn<(id: string) => Promise<void>>(async () => {}),+ spans,+ withSpan,+ setSpanAttributes,+ };+});++vi.mock("drizzle-orm", () => ({+ eq: vi.fn(() => "eq"),+ inArray: vi.fn(() => "inArray"),+}));+vi.mock("../db/connection", () => ({ db: { select, update } }));+vi.mock("../db/schema", () => tables);+vi.mock("../db/rpc", () => ({ getZdrCleanupBatch }));+vi.mock("./gcs-jobs", () => ({ removeJobFromGCS }));+vi.mock("../config", () => ({ config: {} }));+vi.mock("./logger", () => {+ const logger: any = {+ child: vi.fn(() => logger),+ debug: vi.fn(),+ warn: vi.fn(),+ error: vi.fn(),+ };+ return { logger };+});+vi.mock("./otel-tracer", () => ({ withSpan, setSpanAttributes }));++import { cleanZdrRequest, zdrcleaner } from "./zdrcleaner";++describe("ZDR cleaner", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ getZdrCleanupBatch.mockResolvedValue([]);+ removeJobFromGCS.mockResolvedValue(undefined);+ spans.length = 0;+ });++ it("loads and removes every blob type for a queued request", async () => {+ await cleanZdrRequest("request-1");++ expect(select).toHaveBeenCalledTimes(6);+ expect(removeJobFromGCS.mock.calls.map(([id]) => id)).toEqual([+ "scrapes-blob",+ "searches-blob",+ "extracts-blob",+ "maps-blob",+ "llmstxts-blob",+ "deep_researches-blob",+ ]);+ expect(spans).toEqual(+ expect.arrayContaining([+ {+ name: "zdr.cleanup.request",+ attributes: expect.objectContaining({+ "zdr.request_id": "request-1",+ "zdr.blob_count": 6,+ "zdr.cleanup.outcome": "completed",+ }),+ },+ {+ name: "zdr.postgres.read_blobs",+ attributes: expect.objectContaining({+ "db.system": "postgresql",+ "db.operation.name": "select",+ "db.response.returned_rows": 6,+ }),+ },+ ]),+ );+ });++ it("fails a queued request if any blob could not be removed", async () => {+ removeJobFromGCS.mockRejectedValueOnce(new Error("GCS unavailable"));++ await expect(cleanZdrRequest("request-2")).rejects.toThrow(+ "Failed to remove 1 blobs for ZDR request request-2",+ );+ });++ it("keeps draining and clearing legacy PostgreSQL schedules", async () => {+ getZdrCleanupBatch.mockResolvedValueOnce([+ { request_id: "legacy-1", ids: ["blob-1"] },+ { request_id: "legacy-2", ids: ["blob-2"] },+ ]);+ removeJobFromGCS.mockImplementation(async id => {+ if (id === "blob-2") throw new Error("GCS unavailable");+ });++ await zdrcleaner();++ expect(set).toHaveBeenCalledWith({ dr_clean_by: null });+ expect(where).toHaveBeenCalledOnce();+ });+});diff --git a/apps/api/src/lib/zdrcleaner.ts b/apps/api/src/lib/zdrcleaner.tsindex 8ab72ecf4..029ac9377 100644--- a/apps/api/src/lib/zdrcleaner.ts+++ b/apps/api/src/lib/zdrcleaner.ts@@ -1,11 +1,12 @@import "dotenv/config";-import { inArray } from "drizzle-orm";+import { eq, inArray } from "drizzle-orm";import { db } from "../db/connection";import * as schema from "../db/schema";import { getZdrCleanupBatch } from "../db/rpc";import { removeJobFromGCS } from "./gcs-jobs";import { logger as _logger } from "./logger";import { config } from "../config";+import { setSpanAttributes, withSpan } from "./otel-tracer";async function sendHeartbeat() {if (config.ZDRCLEANER_HEARTBEAT_URL) {@@ -13,6 +14,86 @@ async function sendHeartbeat() {}}+async function removeBlobs(blobIds: string[]): Promise<unknown[]> {+ const logger = _logger.child({+ module: "zdrcleaner",+ method: "removeBlobs",+ zeroDataRetention: true,+ });+ const results = await Promise.allSettled(+ blobIds.map(blobId => removeJobFromGCS(blobId, logger)),+ );+ return results.flatMap(result =>+ result.status === "rejected" ? [result.reason] : [],+ );+}++async function getRequestBlobIds(requestId: string): Promise<string[]> {+ return withSpan("zdr.postgres.read_blobs", async span => {+ setSpanAttributes(span, {+ "db.system": "postgresql",+ "db.operation.name": "select",+ "zdr.request_id": requestId,+ });+ const [scrapes, searches, extracts, maps, llmstxts, deepResearches] =+ await Promise.all([+ db+ .select({ id: schema.scrapes.id })+ .from(schema.scrapes)+ .where(eq(schema.scrapes.request_id, requestId)),+ db+ .select({ id: schema.searches.id })+ .from(schema.searches)+ .where(eq(schema.searches.request_id, requestId)),+ db+ .select({ id: schema.extracts.id })+ .from(schema.extracts)+ .where(eq(schema.extracts.request_id, requestId)),+ db+ .select({ id: schema.maps.id })+ .from(schema.maps)+ .where(eq(schema.maps.request_id, requestId)),+ db+ .select({ id: schema.llmstxts.id })+ .from(schema.llmstxts)+ .where(eq(schema.llmstxts.request_id, requestId)),+ db+ .select({ id: schema.deep_researches.id })+ .from(schema.deep_researches)+ .where(eq(schema.deep_researches.request_id, requestId)),+ ]);++ const blobIds = [+ ...scrapes,+ ...searches,+ ...extracts,+ ...maps,+ ...llmstxts,+ ...deepResearches,+ ].map(row => row.id);+ setSpanAttributes(span, {+ "db.response.returned_rows": blobIds.length,+ });+ return blobIds;+ });+}++export async function cleanZdrRequest(requestId: string): Promise<void> {+ await withSpan("zdr.cleanup.request", async span => {+ setSpanAttributes(span, { "zdr.request_id": requestId });+ const blobIds = await getRequestBlobIds(requestId);+ setSpanAttributes(span, { "zdr.blob_count": blobIds.length });+ const errors = await removeBlobs(blobIds);+ if (errors.length > 0) {+ throw new AggregateError(+ errors,+ `Failed to remove ${errors.length} blobs for ZDR request ${requestId}`,+ );+ }+ setSpanAttributes(span, { "zdr.cleanup.outcome": "completed" });+ });+}+export async function zdrcleaner() {const logger = _logger.child({module: "zdrcleaner",@@ -38,43 +119,17 @@ export async function zdrcleaner() {return;}- const requestBlobs = new Map<string, Set<string>>();- const successfulBlobs = new Set<string>();-- // Track all blobs for each request before processing- for (const row of rows) {- requestBlobs.set(row.request_id, new Set(row.ids));- }-- // Process in batches of 50 for GCS cleanupconst deleteErrors: any[] = [];+ const fullyCleanedRequests: string[] = [];for (let j = 0; j < Math.ceil(rows.length / 50); j++) {const batch = rows.slice(j * 50, (j + 1) * 50);- await Promise.allSettled(- batch- .flatMap(x => x.ids)- .map(async blob_id => {- try {- await removeJobFromGCS(- blob_id,- logger.child({ zeroDataRetention: true }),- );- successfulBlobs.add(blob_id);- } catch (error) {- deleteErrors.push(error);- }- }),- );- }-- const fullyCleanedRequests: string[] = [];- for (const [requestId, blobIds] of requestBlobs) {- const allBlobsCleaned = [...blobIds].every(blobId =>- successfulBlobs.has(blobId),+ await Promise.all(+ batch.map(async row => {+ const errors = await removeBlobs(row.ids);+ if (errors.length === 0) fullyCleanedRequests.push(row.request_id);+ else deleteErrors.push(...errors);+ }),);- if (allBlobsCleaned) {- fullyCleanedRequests.push(requestId);- }}const updateErrors: any[] = [];diff --git a/apps/api/src/services/logging/log_job.test.ts b/apps/api/src/services/logging/log_job.test.tsindex 131e2d952..b5f6fe84b 100644--- a/apps/api/src/services/logging/log_job.test.ts+++ b/apps/api/src/services/logging/log_job.test.ts@@ -18,6 +18,7 @@ const {withSpan,setSpanAttributes,spans,+ enqueueZdrCleanupJob,} = vi.hoisted(() => {const logger: any = {info: vi.fn(),@@ -63,6 +64,7 @@ const {withSpan,setSpanAttributes: vi.fn(),spans,+ enqueueZdrCleanupJob: vi.fn(async () => {}),};});@@ -121,6 +123,10 @@ vi.mock("../../lib/gcs-jobs", () => ({saveSearchToGCS: vi.fn(),}));+vi.mock("../../lib/zdr-queue", () => ({+ enqueueZdrCleanupJob,+}));+vi.mock("../../lib/extract/extract-redis", () => ({saveExtractResult: vi.fn(),}));@@ -335,6 +341,32 @@ describe("logRequest", () => {);});+ it("durably queues ZDR cleanup before writing the request", async () => {+ await logRequest({+ ...makeRequest("op_integration_42"),+ zeroDataRetention: true,+ });++ expect(enqueueZdrCleanupJob).toHaveBeenCalledWith(+ "019e6f45-7778-727d-adf0-0abe9d5062b6",+ );+ expect(enqueueZdrCleanupJob.mock.invocationCallOrder[0]).toBeLessThan(+ values.mock.invocationCallOrder[0],+ );+ expect(values.mock.calls[0][0]).not.toHaveProperty("dr_clean_by");+ });++ it("does not write a ZDR request unless its cleanup job was confirmed", async () => {+ enqueueZdrCleanupJob.mockRejectedValueOnce(+ new Error("RabbitMQ unavailable"),+ );++ await expect(+ logRequest({ ...makeRequest(null), zeroDataRetention: true }),+ ).rejects.toThrow("RabbitMQ unavailable");+ expect(values).not.toHaveBeenCalled();+ });+it("keeps the database write when Pub/Sub fails", async () => {publishMessage.mockRejectedValueOnce(new Error("Pub/Sub unavailable"));diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex 6b1e7e853..86cf0fea7 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -2,6 +2,7 @@ import { db } from "../../db/connection";import * as schema from "../../db/schema";import { changeTrackingInsertScrape } from "../../lib/change-tracking-store";import { config } from "../../config";+import { enqueueZdrCleanupJob } from "../../lib/zdr-queue";import "dotenv/config";import { logger as _logger } from "../../lib/logger";import { EXTERNAL_REQUEST_ID_MAX_BYTES } from "../../lib/external-request-id";@@ -572,6 +573,10 @@ async function logRequestInternal(request: LoggedRequest) {}}+ if (request.zeroDataRetention && config.USE_DB_AUTHENTICATION === true) {+ await enqueueZdrCleanupJob(request.id);+ }+await robustInsert("requests",{@@ -582,13 +587,10 @@ async function logRequestInternal(request: LoggedRequest) {origin: sanitizedOrigin,integration: sanitizedIntegration,target_hint: sanitizedTargetHint,- dr_clean_by: request.zeroDataRetention- ? new Date(Date.now() + 24 * 60 * 60 * 1000)- : null,api_key_id: request.api_key_id ?? null,// Not redacted under zero data retention: it is the caller's own// operation id (attribution it asked for), not customer content — and- // the row is cleaned at dr_clean_by regardless.+ // the row's blobs are cleaned by the ZDR queue regardless.external_request_id: boundedExternalRequestId(sanitizeString(request.external_request_id ?? null),logger,diff --git a/apps/api/src/services/zdr-worker.ts b/apps/api/src/services/zdr-worker.tsindex 1ed439d7d..7702f9f7d 100644--- a/apps/api/src/services/zdr-worker.ts+++ b/apps/api/src/services/zdr-worker.ts@@ -1,7 +1,8 @@import "dotenv/config";import { shutdownTracing } from "../otel";import { logger } from "../lib/logger";-import { zdrcleaner } from "../lib/zdrcleaner";+import { cleanZdrRequest, zdrcleaner } from "../lib/zdrcleaner";+import { consumeZdrCleanupJobs, shutdownZdrQueue } from "../lib/zdr-queue";let isShuttingDown = false;@@ -16,10 +17,13 @@ process.on("SIGTERM", () => {});(async () => {+ await consumeZdrCleanupJobs(job => cleanZdrRequest(job.requestId));+while (!isShuttingDown) {await zdrcleaner();}+ await shutdownZdrQueue();await shutdownTracing();logger.info("zdr-worker exiting");process.exit(0);