Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): bound ZDR cleanup publish retries and drain them on shutdown (#5643)

Gergő Móriczf330df845016
diff --git a/apps/api/src/index.ts b/apps/api/src/index.ts
index fa18cf619..87dfa9a31 100644
--- a/apps/api/src/index.ts
+++ b/apps/api/src/index.ts
@@ -43,7 +43,10 @@ import responseTime from "response-time";
import { shutdownWebhookQueue } from "./services/webhook";
import { shutdownIndexerQueue } from "./services/indexing/indexer-queue";
import { isKeylessConfigured } from "./lib/keyless";
-import { shutdownPubSubLogging } from "./services/logging/log_job";
+import {
+ drainZdrRequestLogs,
+ shutdownPubSubLogging,
+} from "./services/logging/log_job";
import { notFoundHandler } from "./lib/not-found";
const { createBullBoard } = require("@bull-board/api");
@@ -242,6 +245,7 @@ async function startServer(port = DEFAULT_PORT) {
server.close(async () => {
await adminInternalClosed;
logger.info("Server closed.");
+ await drainZdrRequestLogs();
nuqShutdown().finally(() => {
shutdownWebhookQueue().finally(() => {
shutdownIndexerQueue().finally(async () => {
diff --git a/apps/api/src/lib/zdr-queue.ts b/apps/api/src/lib/zdr-queue.ts
index b873ff2be..189a9d978 100644
--- a/apps/api/src/lib/zdr-queue.ts
+++ b/apps/api/src/lib/zdr-queue.ts
@@ -17,6 +17,10 @@ const READY_ROUTING_KEY = "ready";
const DLQ_ROUTING_KEY = "dead";
const RECONNECT_BASE_DELAY_MS = 1_000;
const RECONNECT_MAX_DELAY_MS = 30_000;
+// 5 attempts of at most 5s each, with 1+2+4+8s between them: at most 40s,
+// well inside the API pod's shutdown grace.
+const PUBLISH_MAX_ATTEMPTS = 5;
+const PUBLISH_ATTEMPT_TIMEOUT_MS = 5_000;
const logger = _logger.child({ module: "zdr-queue" });
@@ -265,7 +269,8 @@ export async function enqueueZdrCleanupJob(requestId: string): Promise<void> {
}
let delayMs = RECONNECT_BASE_DELAY_MS;
- while (!closed) {
+ for (let attempt = 1; !closed; attempt++) {
+ let attemptTimer: NodeJS.Timeout | undefined;
try {
await withSpan("zdr.rabbitmq.publish", async span => {
setSpanAttributes(span, {
@@ -275,14 +280,31 @@ export async function enqueueZdrCleanupJob(requestId: string): Promise<void> {
"messaging.message.id": requestId,
"zdr.rabbitmq.retry_delay_ms": delayMs,
});
- const ch = await getPublishChannel();
- await publishConfirmed(ch, { requestId });
+ await Promise.race([
+ getPublishChannel().then(ch => publishConfirmed(ch, { requestId })),
+ new Promise<never>((_, reject) => {
+ attemptTimer = setTimeout(
+ () => reject(new Error("ZDR publish attempt timed out")),
+ PUBLISH_ATTEMPT_TIMEOUT_MS,
+ );
+ }),
+ ]);
setSpanAttributes(span, {
"zdr.rabbitmq.outcome": "confirmed",
});
});
return;
} catch (error) {
+ if (attempt >= PUBLISH_MAX_ATTEMPTS) {
+ logger.error("Failed to enqueue ZDR cleanup job; giving up", {
+ error,
+ requestId,
+ attempts: attempt,
+ });
+ throw new Error(
+ `ZDR cleanup job not confirmed after ${attempt} attempts`,
+ );
+ }
logger.error("Failed to durably enqueue ZDR cleanup job; retrying", {
error,
requestId,
@@ -293,6 +315,8 @@ export async function enqueueZdrCleanupJob(requestId: string): Promise<void> {
publishChannelPromise = null;
await new Promise(resolve => setTimeout(resolve, delayMs));
delayMs = Math.min(delayMs * 2, RECONNECT_MAX_DELAY_MS);
+ } finally {
+ clearTimeout(attemptTimer);
}
}
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index 636392a4d..c5532183d 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -476,8 +476,10 @@ function boundedExternalRequestId(
return null;
}
+const inFlightZdrRequestLogs = new Set<Promise<void>>();
+
export async function logRequest(request: LoggedRequest) {
- return withLogSpan(
+ const log = withLogSpan(
{
operation: "request",
table: "requests",
@@ -486,6 +488,18 @@ export async function logRequest(request: LoggedRequest) {
},
() => logRequestInternal(request),
);
+ if (!request.zeroDataRetention) return log;
+ inFlightZdrRequestLogs.add(log);
+ try {
+ return await log;
+ } finally {
+ inFlightZdrRequestLogs.delete(log);
+ }
+}
+
+/** Waits for in-flight ZDR request logs, which enqueue their cleanup job. */
+export async function drainZdrRequestLogs(): Promise<void> {
+ await Promise.allSettled([...inFlightZdrRequestLogs]);
}
async function logRequestInternal(request: LoggedRequest) {