Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): drain pending log publications on shutdown (#4558)
rahul-firecrawl1db782f51e5d
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.tsindex d60d14de2..d52fba2b8 100644--- a/apps/api/src/controllers/v1/map.ts+++ b/apps/api/src/controllers/v1/map.ts@@ -545,6 +545,8 @@ export async function mapController(results: result.links,credits_cost: creditsToBill,zeroDataRetention: false, // not supported+ }).catch(error => {+ logger.error("Failed to log map", { error, mapId });});// Log final timing informationdiff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.tsindex 1b839873e..e5e641473 100644--- a/apps/api/src/controllers/v1/search.ts+++ b/apps/api/src/controllers/v1/search.ts@@ -328,7 +328,9 @@ export async function searchController(zeroDataRetention,},false,- );+ ).catch(error => {+ logger.error("Failed to log search", { error, jobId });+ });const totalRequestTime = new Date().getTime() - middlewareStartTime;const controllerTime = new Date().getTime() - controllerStartTime;diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 8bb8074a3..4d773290a 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -297,10 +297,7 @@ export async function searchController(const endTime = new Date().getTime();const timeTakenInSeconds = (endTime - middlewareStartTime) / 1000;- // Ensure the parent `requests` row is committed before the child- // `searches` insert, to avoid a request_id FK violation. The insert has- // been in flight since the top of the controller, so this is ~free in- // practice; robustInsert never rejects, so this await cannot throw.+ // Wait for the parent log before inserting the child search log.const logStart = Date.now();await logRequestPromise;const waited = Date.now() - logStart;@@ -327,7 +324,9 @@ export async function searchController(zeroDataRetention,},false,- );+ ).catch(error => {+ logger.error("Failed to log search", { error, jobId });+ });if (wantsDeveloperCategory(req.body.categories as CategoryOption[])) {logResearchEndpoint({diff --git a/apps/api/src/lib/pubsub-log-metrics.ts b/apps/api/src/lib/pubsub-log-metrics.tsindex 38313b9b8..0a8f3c9b0 100644--- a/apps/api/src/lib/pubsub-log-metrics.ts+++ b/apps/api/src/lib/pubsub-log-metrics.ts@@ -4,9 +4,8 @@ const NAME = "pubsub_log_publish_total";/*** Log rows handed to the Pub/Sub publisher, by table and outcome:- * `published`, `failed` (every retry exhausted), or `dropped` (backlog cap).- * A channel stall shows up here as `failed` and `dropped` rising together,- * without waiting for the daily reconciliation against PostgreSQL.+ * `published`, `failed` (publication or shutdown error), or `dropped` (backlog cap).+ * Use failed and dropped counts to detect losses between reconciliation checks.** Looked up before creation so a re-evaluated module (test isolation) does* not register the same series twice in the shared default registry.diff --git a/apps/api/src/services/logging/log_job.test.ts b/apps/api/src/services/logging/log_job.test.tsindex e69b0e11e..d4f39b59b 100644--- a/apps/api/src/services/logging/log_job.test.ts+++ b/apps/api/src/services/logging/log_job.test.ts@@ -82,6 +82,14 @@ vi.mock("../../db/connection", () => ({db: { insert },}));+vi.mock("../../lib/change-tracking-store", () => ({+ changeTrackingInsertScrape: vi.fn(),+}));++vi.mock("../../lib/keyless", () => ({+ keylessTeamUuid: vi.fn(() => null),+}));+vi.mock("../../lib/gcs-jobs", () => ({saveDeepResearchToGCS: vi.fn(),saveExtractToGCS: vi.fn(),@@ -112,6 +120,16 @@ import {import * as schema from "../../db/schema";import { config } from "../../config";+function deferred<T>() {+ let resolve!: (value: T | PromiseLike<T>) => void;+ let reject!: (reason: unknown) => void;+ const promise = new Promise<T>((resolvePromise, rejectPromise) => {+ resolve = resolvePromise;+ reject = rejectPromise;+ });+ return { promise, resolve, reject };+}+function makeSearch(overrides: Partial<LoggedSearch> = {}): LoggedSearch {return {id: "019e6f45-7778-727d-adf0-0abe9d5062b6",@@ -180,6 +198,23 @@ describe("logSearch", () => {expect(context.extra.data).not.toContain("\\u0000");expect(context.extra.data).toContain("badquery");});++ it("reports serialization failures without losing the PostgreSQL attempt", async () => {+ const search = makeSearch({ options: { unsupported: 1n } });++ await expect(logSearch(search)).resolves.toBeUndefined();++ expect(values).toHaveBeenCalledOnce();+ expect(publishMessage).not.toHaveBeenCalled();+ expect(metricInc).toHaveBeenCalledWith({+ table: "searches",+ outcome: "failed",+ });+ expect(logger.error).toHaveBeenCalledWith(+ "Failed to publish log to Pub/Sub",+ expect.objectContaining({ logId: search.id, error: expect.any(Error) }),+ );+ });});describe("logRequest", () => {@@ -255,10 +290,36 @@ describe("logRequest", () => {});it("does not hold the caller on a slow publish", async () => {- publishMessage.mockReturnValueOnce(new Promise(() => {}));+ const publication = deferred<string>();+ publishMessage.mockReturnValueOnce(publication.promise);await expect(logRequest(makeRequest(null))).resolves.toBeUndefined();expect(values).toHaveBeenCalled();+ expect(metricInc).not.toHaveBeenCalled();++ publication.resolve("message-id");+ await publication.promise;+ expect(metricInc).toHaveBeenCalledWith({+ table: "requests",+ outcome: "published",+ });+ });++ it("waits for PostgreSQL when publication finishes first", async () => {+ const insertion = deferred<void>();+ values.mockReturnValueOnce(insertion.promise);+ let finished = false;+ const logging = logRequest(makeRequest(null)).then(() => {+ finished = true;+ });++ await new Promise(resolve => setImmediate(resolve));+ expect(publishMessage).toHaveBeenCalledOnce();+ expect(finished).toBe(false);++ insertion.resolve();+ await logging;+ expect(finished).toBe(true);});it("stores null, not a truncation, when the id exceeds the byte cap", async () => {@@ -310,12 +371,10 @@ describe("logRequest", () => {});it("drops publishes beyond the outstanding cap instead of queueing them", async () => {- // A fresh module instance: the outstanding counters are process-wide and- // an earlier test leaves a publish that never settles.vi.resetModules();const fresh = await import("./log_job.js");- // Hung channel: publishes never settle, so each one stays outstanding.- publishMessage.mockReturnValue(new Promise(() => {}));+ const publication = deferred<string>();+ publishMessage.mockImplementation(async () => publication.promise);config.PUBSUB_MAX_OUTSTANDING_MESSAGES = 2;try {await fresh.logRequest(makeRequest(null));@@ -323,6 +382,8 @@ describe("logRequest", () => {await fresh.logRequest(makeRequest(null));} finally {config.PUBSUB_MAX_OUTSTANDING_MESSAGES = 10_000;+ publication.resolve("message-id");+ await fresh.shutdownPubSubLogging();}// The database write is never held back by the publisher.@@ -334,7 +395,12 @@ describe("logRequest", () => {});expect(logger.warn).toHaveBeenCalledWith("Dropping Pub/Sub log: publisher backlog is full",- expect.objectContaining({ outstandingMessages: 2, droppedTotal: 1 }),+ expect.objectContaining({+ table: "requests",+ logId: makeRequest(null).id,+ outstandingMessages: 2,+ droppedTotal: 1,+ }),);});@@ -355,6 +421,23 @@ describe("logRequest", () => {});});+ it("releases the backlog capacity after a failed publication", async () => {+ const originalCap = config.PUBSUB_MAX_OUTSTANDING_MESSAGES;+ config.PUBSUB_MAX_OUTSTANDING_MESSAGES = 1;+ try {+ publishMessage.mockRejectedValueOnce(new Error("Pub/Sub unavailable"));+ await logRequest(makeRequest(null));+ await logRequest(makeRequest(null));+ expect(publishMessage).toHaveBeenCalledTimes(2);+ expect(metricInc).not.toHaveBeenCalledWith({+ table: "requests",+ outcome: "dropped",+ });+ } finally {+ config.PUBSUB_MAX_OUTSTANDING_MESSAGES = originalCap;+ }+ });+it("flushes Pub/Sub messages during shutdown", async () => {await logRequest(makeRequest(null));@@ -377,6 +460,107 @@ describe("shutdownPubSubLogging deadline", () => {vi.useRealTimers();});+ it("waits for pending publications even when flush returns early", async () => {+ vi.resetModules();+ const fresh = await import("./log_job.js");+ const publication = deferred<string>();+ publishMessage.mockReturnValueOnce(publication.promise);+ await fresh.logSearch(makeSearch());++ const shutdown = fresh.shutdownPubSubLogging();+ await new Promise(resolve => setImmediate(resolve));+ expect(flush).toHaveBeenCalled();+ expect(close).not.toHaveBeenCalled();++ publication.resolve("message-id");+ await shutdown;+ expect(close).toHaveBeenCalledOnce();+ });++ it("closes at the deadline when publication stays pending after flush", async () => {+ vi.resetModules();+ const fresh = await import("./log_job.js");+ const publication = deferred<string>();+ publishMessage.mockReturnValueOnce(publication.promise);+ await fresh.logSearch(makeSearch());++ vi.useFakeTimers();+ const shutdown = fresh.shutdownPubSubLogging();+ await vi.advanceTimersByTimeAsync(39_999);+ expect(close).not.toHaveBeenCalled();+ await vi.advanceTimersByTimeAsync(1);+ await shutdown;+ expect(close).toHaveBeenCalledOnce();+ expect(logger.warn).toHaveBeenCalledWith(+ "Pub/Sub log flush did not finish before the shutdown deadline; closing anyway",+ expect.objectContaining({+ outstandingMessages: 1,+ pendingLogSample: [+ expect.objectContaining({+ table: "searches",+ logId: makeSearch().id,+ }),+ ],+ pendingLogSampleTruncated: false,+ }),+ );++ publication.resolve("message-id");+ await publication.promise;+ });++ it("reports publication failures during shutdown and still closes", async () => {+ vi.resetModules();+ const fresh = await import("./log_job.js");+ const publication = deferred<string>();+ publishMessage.mockReturnValueOnce(publication.promise);+ await fresh.logSearch(makeSearch());+ const shutdown = fresh.shutdownPubSubLogging();++ publication.reject(new Error("Pub/Sub unavailable"));+ await shutdown;+ expect(close).toHaveBeenCalledOnce();+ expect(metricInc).toHaveBeenCalledWith({+ table: "searches",+ outcome: "failed",+ });+ });++ it("bounds client close after draining so shutdown can finish", async () => {+ vi.resetModules();+ const fresh = await import("./log_job.js");+ await fresh.logSearch(makeSearch());+ close.mockReturnValueOnce(new Promise(() => {}));+ vi.useFakeTimers();+ const shutdown = fresh.shutdownPubSubLogging();+ await vi.advanceTimersByTimeAsync(5_000);+ await shutdown;+ expect(logger.error).toHaveBeenCalledWith(+ "Failed to close Pub/Sub log publisher",+ expect.objectContaining({ error: expect.any(Error) }),+ );+ });++ it("reports late logs instead of publishing after shutdown starts", async () => {+ vi.resetModules();+ const fresh = await import("./log_job.js");+ await fresh.logSearch(makeSearch());+ const flushing = deferred<void>();+ flush.mockReturnValueOnce(flushing.promise);+ const shutdown = fresh.shutdownPubSubLogging();++ await fresh.logSearch(makeSearch());+ expect(publishMessage).toHaveBeenCalledOnce();+ expect(values).toHaveBeenCalledTimes(2);+ expect(metricInc).toHaveBeenCalledWith({+ table: "searches",+ outcome: "failed",+ });++ flushing.resolve();+ await shutdown;+ });+it("closes the client when a flush outlives the shutdown deadline", async () => {// A fresh module instance: shutdown is memoized per process.vi.resetModules();diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex 38236e438..7e3ada468 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -70,8 +70,8 @@ let pubSubShutdown: Promise<void> | undefined;// whole retry budget to that one value (CallSettings.merge), so a stalled RPC// used to be a single 60 s attempt and then a lost row. An explicit `retry`// is applied after that override and replaces the backoff settings wholesale.-// Short attempts detect a hung channel quickly; the total budget covers every-// stall measured in production so far, all of which cleared within 140 s.+// Short attempts detect a stalled RPC quickly. The total budget allows+// retries across a longer connection disruption.// Retry codes stay the client's defaults for Publish (DEADLINE_EXCEEDED,// UNAVAILABLE, INTERNAL, UNKNOWN, ABORTED, CANCELLED, RESOURCE_EXHAUSTED).// A retry can deliver a batch twice when the first attempt was persisted but@@ -94,14 +94,15 @@ const PUBSUB_PUBLISH_OPTIONS: PublishOptions = {};// Shutdown waits this long for in-flight publishes before closing the client.-// It must fit inside the pods' termination grace period (60 s for nuq-// workers); a stall that outlives it loses what is still in flight, which is-// the same outcome as today and what a durable relay is for.+// The drain shares the existing pod grace period with active work and exit.+// Memory kills and work that exceeds the pod grace period can still lose logs.const PUBSUB_SHUTDOWN_FLUSH_TIMEOUT_MS = 40_000;-// Rows waiting on Pub/Sub across all topics. Publishing is fire-and-forget,-// so during a stall this is what grows; see PUBSUB_MAX_OUTSTANDING_* in config.-let outstandingMessages = 0;+// Track publication promises because topic.flush() can finish before an active RPC.+const pendingPublications = new Map<+ Promise<string>,+ { table: string; logId: string; startedAt: number }+>();let outstandingBytes = 0;let droppedTotal = 0;let lastDropWarningAt = 0;@@ -139,42 +140,67 @@ function getTopic(client: PubSub, table: string): Topic {}async function publishLog(table: string, data: any, logger: Logger) {- const client = getPubSubClient(logger);- if (!client) return;+ const startedAt = Date.now();+ try {+ if (pubSubShutdown) {+ throw new Error("Pub/Sub log publisher is shutting down");+ }+ const client = getPubSubClient(logger);+ if (!client) {+ if (config.PUBSUB_CREDENTIALS) {+ throw new Error("Pub/Sub log publisher initialization failed");+ }+ return;+ }- const payload = Buffer.from(JSON.stringify(data));- if (- outstandingMessages >= config.PUBSUB_MAX_OUTSTANDING_MESSAGES ||- outstandingBytes + payload.length > config.PUBSUB_MAX_OUTSTANDING_BYTES- ) {- droppedTotal++;- pubsubLogPublishTotal.inc({ table, outcome: "dropped" });- const now = Date.now();- if (now - lastDropWarningAt >= 60_000) {- lastDropWarningAt = now;- logger.warn("Dropping Pub/Sub log: publisher backlog is full", {- outstandingMessages,- outstandingBytes,- droppedTotal,- });+ const payload = Buffer.from(JSON.stringify(data));+ if (+ pendingPublications.size >= config.PUBSUB_MAX_OUTSTANDING_MESSAGES ||+ outstandingBytes + payload.length > config.PUBSUB_MAX_OUTSTANDING_BYTES+ ) {+ droppedTotal++;+ pubsubLogPublishTotal.inc({ table, outcome: "dropped" });+ const now = Date.now();+ if (now - lastDropWarningAt >= 60_000) {+ lastDropWarningAt = now;+ logger.warn("Dropping Pub/Sub log: publisher backlog is full", {+ table,+ logId: data.id,+ payloadBytes: payload.length,+ outstandingMessages: pendingPublications.size,+ outstandingBytes,+ droppedTotal,+ });+ }+ return;}- return;- }- outstandingMessages++;- outstandingBytes += payload.length;- try {- await getTopic(client, table).publishMessage({ data: payload });- pubsubLogPublishTotal.inc({ table, outcome: "published" });+ const publication = getTopic(client, table).publishMessage({+ data: payload,+ });+ pendingPublications.set(publication, { table, logId: data.id, startedAt });+ outstandingBytes += payload.length;++ try {+ await publication;+ pubsubLogPublishTotal.inc({ table, outcome: "published" });+ } finally {+ pendingPublications.delete(publication);+ outstandingBytes -= payload.length;+ }} catch (error) {pubsubLogPublishTotal.inc({ table, outcome: "failed" });- logger.error("Failed to publish log to Pub/Sub", { error });++ logger.error("Failed to publish log to Pub/Sub", {+ error,+ table,+ logId: data.id,+ durationMs: Date.now() - startedAt,+ });Sentry.captureException(error, {tags: { table, operation: "publishPubSubLog" },+ extra: { logId: data.id },});- } finally {- outstandingMessages--;- outstandingBytes -= payload.length;}}@@ -193,9 +219,18 @@ async function shutdownPubSubLoggingOnce(): Promise<void> {module: "log_job",method: "shutdownPubSubLogging",});- const flushed = Promise.allSettled(- [...pubSubTopics.values()].map(topic => topic.flush()),- );+ const startedAt = Date.now();+ logger.info("Draining Pub/Sub log publisher", {+ outstandingMessages: pendingPublications.size,+ outstandingBytes,+ timeoutMs: PUBSUB_SHUTDOWN_FLUSH_TIMEOUT_MS,+ });+ // Callers stop accepting work before shutdown. Reject new publications so+ // this snapshot includes every publication that can still use the client.+ const flushed = Promise.allSettled([+ ...[...pubSubTopics.values()].map(async topic => topic.flush()),+ ...pendingPublications.keys(),+ ]);let deadline: NodeJS.Timeout | undefined;const timedOut = new Promise<"timeout">(resolve => {deadline = setTimeout(@@ -211,8 +246,10 @@ async function shutdownPubSubLoggingOnce(): Promise<void> {"Pub/Sub log flush did not finish before the shutdown deadline; closing anyway",{timeoutMs: PUBSUB_SHUTDOWN_FLUSH_TIMEOUT_MS,- outstandingMessages,+ outstandingMessages: pendingPublications.size,outstandingBytes,+ pendingLogSample: [...pendingPublications.values()].slice(0, 50),+ pendingLogSampleTruncated: pendingPublications.size > 50,},);} else {@@ -221,21 +258,36 @@ async function shutdownPubSubLoggingOnce(): Promise<void> {);if (errors.length > 0) {- logger.error("Failed to flush Pub/Sub log publisher", { errors });+ logger.error("Failed to drain Pub/Sub log publisher", { errors });Sentry.captureException(errors[0], {tags: { operation: "flushPubSubLogPublisher" },extra: { failures: errors.length },});+ } else {+ logger.info("Pub/Sub log publisher drained", {+ durationMs: Date.now() - startedAt,+ });}}+ let closeDeadline: NodeJS.Timeout | undefined;try {- await client.close();+ await Promise.race([+ client.close(),+ new Promise<never>((_, reject) => {+ closeDeadline = setTimeout(+ () => reject(new Error("Pub/Sub client close exceeded 5 seconds")),+ 5_000,+ );+ }),+ ]);} catch (error) {logger.error("Failed to close Pub/Sub log publisher", { error });Sentry.captureException(error, {tags: { operation: "closePubSubLogPublisher" },});+ } finally {+ clearTimeout(closeDeadline);}}@@ -266,6 +318,7 @@ async function robustInsert(...data,created_at: data.created_at ?? new Date(),});+ // Publish in the background. Customer responses must not wait for Pub/Sub.void publishLog(table, data, logger);const attempts: { error: any; timeMs: number; backoffMs: number }[] = [];diff --git a/apps/api/src/services/worker/nuq-worker-runner.ts b/apps/api/src/services/worker/nuq-worker-runner.tsindex bbffc593f..3af1a4145 100644--- a/apps/api/src/services/worker/nuq-worker-runner.ts+++ b/apps/api/src/services/worker/nuq-worker-runner.ts@@ -69,6 +69,7 @@ export async function runNuqWorker(options: {}let isShuttingDown = false;+ let shutdownStartedAt: number | undefined;const app = Express();@@ -110,7 +111,12 @@ export async function runNuqWorker(options: {});function shutdown() {+ if (isShuttingDown) return;isShuttingDown = true;+ shutdownStartedAt = Date.now();+ _logger.info("NuQ worker stopping after active work", {+ module: options.serviceName,+ });}process.on("SIGINT", shutdown);@@ -200,7 +206,10 @@ export async function runNuqWorker(options: {}}- _logger.info("NuQ worker shutting down", { module: options.serviceName });+ _logger.info("NuQ worker shutting down", {+ module: options.serviceName,+ workDrainMs: Date.now() - shutdownStartedAt!,+ });server.close(async () => {await options.beforeShutdown?.();