Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): bound ZDR cleanup publish retries and drain them on shutdown (#5643)
Gergő Móriczf330df845016
diff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex fa18cf619..87dfa9a31 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -43,7 +43,10 @@ import responseTime from "response-time";import { shutdownWebhookQueue } from "./services/webhook";import { shutdownIndexerQueue } from "./services/indexing/indexer-queue";import { isKeylessConfigured } from "./lib/keyless";-import { shutdownPubSubLogging } from "./services/logging/log_job";+import {+ drainZdrRequestLogs,+ shutdownPubSubLogging,+} from "./services/logging/log_job";import { notFoundHandler } from "./lib/not-found";const { createBullBoard } = require("@bull-board/api");@@ -242,6 +245,7 @@ async function startServer(port = DEFAULT_PORT) {server.close(async () => {await adminInternalClosed;logger.info("Server closed.");+ await drainZdrRequestLogs();nuqShutdown().finally(() => {shutdownWebhookQueue().finally(() => {shutdownIndexerQueue().finally(async () => {diff --git a/apps/api/src/lib/zdr-queue.ts b/apps/api/src/lib/zdr-queue.tsindex b873ff2be..189a9d978 100644--- a/apps/api/src/lib/zdr-queue.ts+++ b/apps/api/src/lib/zdr-queue.ts@@ -17,6 +17,10 @@ const READY_ROUTING_KEY = "ready";const DLQ_ROUTING_KEY = "dead";const RECONNECT_BASE_DELAY_MS = 1_000;const RECONNECT_MAX_DELAY_MS = 30_000;+// 5 attempts of at most 5s each, with 1+2+4+8s between them: at most 40s,+// well inside the API pod's shutdown grace.+const PUBLISH_MAX_ATTEMPTS = 5;+const PUBLISH_ATTEMPT_TIMEOUT_MS = 5_000;const logger = _logger.child({ module: "zdr-queue" });@@ -265,7 +269,8 @@ export async function enqueueZdrCleanupJob(requestId: string): Promise<void> {}let delayMs = RECONNECT_BASE_DELAY_MS;- while (!closed) {+ for (let attempt = 1; !closed; attempt++) {+ let attemptTimer: NodeJS.Timeout | undefined;try {await withSpan("zdr.rabbitmq.publish", async span => {setSpanAttributes(span, {@@ -275,14 +280,31 @@ export async function enqueueZdrCleanupJob(requestId: string): Promise<void> {"messaging.message.id": requestId,"zdr.rabbitmq.retry_delay_ms": delayMs,});- const ch = await getPublishChannel();- await publishConfirmed(ch, { requestId });+ await Promise.race([+ getPublishChannel().then(ch => publishConfirmed(ch, { requestId })),+ new Promise<never>((_, reject) => {+ attemptTimer = setTimeout(+ () => reject(new Error("ZDR publish attempt timed out")),+ PUBLISH_ATTEMPT_TIMEOUT_MS,+ );+ }),+ ]);setSpanAttributes(span, {"zdr.rabbitmq.outcome": "confirmed",});});return;} catch (error) {+ if (attempt >= PUBLISH_MAX_ATTEMPTS) {+ logger.error("Failed to enqueue ZDR cleanup job; giving up", {+ error,+ requestId,+ attempts: attempt,+ });+ throw new Error(+ `ZDR cleanup job not confirmed after ${attempt} attempts`,+ );+ }logger.error("Failed to durably enqueue ZDR cleanup job; retrying", {error,requestId,@@ -293,6 +315,8 @@ export async function enqueueZdrCleanupJob(requestId: string): Promise<void> {publishChannelPromise = null;await new Promise(resolve => setTimeout(resolve, delayMs));delayMs = Math.min(delayMs * 2, RECONNECT_MAX_DELAY_MS);+ } finally {+ clearTimeout(attemptTimer);}}diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex 636392a4d..c5532183d 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -476,8 +476,10 @@ function boundedExternalRequestId(return null;}+const inFlightZdrRequestLogs = new Set<Promise<void>>();+export async function logRequest(request: LoggedRequest) {- return withLogSpan(+ const log = withLogSpan({operation: "request",table: "requests",@@ -486,6 +488,18 @@ export async function logRequest(request: LoggedRequest) {},() => logRequestInternal(request),);+ if (!request.zeroDataRetention) return log;+ inFlightZdrRequestLogs.add(log);+ try {+ return await log;+ } finally {+ inFlightZdrRequestLogs.delete(log);+ }+}++/** Waits for in-flight ZDR request logs, which enqueue their cleanup job. */+export async function drainZdrRequestLogs(): Promise<void> {+ await Promise.allSettled([...inFlightZdrRequestLogs]);}async function logRequestInternal(request: LoggedRequest) {