Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

fix(api): drain pending log publications on shutdown (#4558)

rahul-firecrawl1db782f51e5d
diff --git a/apps/api/src/controllers/v1/map.ts b/apps/api/src/controllers/v1/map.ts
index d60d14de2..d52fba2b8 100644
--- a/apps/api/src/controllers/v1/map.ts
+++ b/apps/api/src/controllers/v1/map.ts
@@ -545,6 +545,8 @@ export async function mapController(
results: result.links,
credits_cost: creditsToBill,
zeroDataRetention: false, // not supported
+ }).catch(error => {
+ logger.error("Failed to log map", { error, mapId });
});
// Log final timing information
diff --git a/apps/api/src/controllers/v1/search.ts b/apps/api/src/controllers/v1/search.ts
index 1b839873e..e5e641473 100644
--- a/apps/api/src/controllers/v1/search.ts
+++ b/apps/api/src/controllers/v1/search.ts
@@ -328,7 +328,9 @@ export async function searchController(
zeroDataRetention,
},
false,
- );
+ ).catch(error => {
+ logger.error("Failed to log search", { error, jobId });
+ });
const totalRequestTime = new Date().getTime() - middlewareStartTime;
const controllerTime = new Date().getTime() - controllerStartTime;
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index 8bb8074a3..4d773290a 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -297,10 +297,7 @@ export async function searchController(
const endTime = new Date().getTime();
const timeTakenInSeconds = (endTime - middlewareStartTime) / 1000;
- // Ensure the parent `requests` row is committed before the child
- // `searches` insert, to avoid a request_id FK violation. The insert has
- // been in flight since the top of the controller, so this is ~free in
- // practice; robustInsert never rejects, so this await cannot throw.
+ // Wait for the parent log before inserting the child search log.
const logStart = Date.now();
await logRequestPromise;
const waited = Date.now() - logStart;
@@ -327,7 +324,9 @@ export async function searchController(
zeroDataRetention,
},
false,
- );
+ ).catch(error => {
+ logger.error("Failed to log search", { error, jobId });
+ });
if (wantsDeveloperCategory(req.body.categories as CategoryOption[])) {
logResearchEndpoint({
diff --git a/apps/api/src/lib/pubsub-log-metrics.ts b/apps/api/src/lib/pubsub-log-metrics.ts
index 38313b9b8..0a8f3c9b0 100644
--- a/apps/api/src/lib/pubsub-log-metrics.ts
+++ b/apps/api/src/lib/pubsub-log-metrics.ts
@@ -4,9 +4,8 @@ const NAME = "pubsub_log_publish_total";
/**
* Log rows handed to the Pub/Sub publisher, by table and outcome:
- * `published`, `failed` (every retry exhausted), or `dropped` (backlog cap).
- * A channel stall shows up here as `failed` and `dropped` rising together,
- * without waiting for the daily reconciliation against PostgreSQL.
+ * `published`, `failed` (publication or shutdown error), or `dropped` (backlog cap).
+ * Use failed and dropped counts to detect losses between reconciliation checks.
*
* Looked up before creation so a re-evaluated module (test isolation) does
* not register the same series twice in the shared default registry.
diff --git a/apps/api/src/services/logging/log_job.test.ts b/apps/api/src/services/logging/log_job.test.ts
index e69b0e11e..d4f39b59b 100644
--- a/apps/api/src/services/logging/log_job.test.ts
+++ b/apps/api/src/services/logging/log_job.test.ts
@@ -82,6 +82,14 @@ vi.mock("../../db/connection", () => ({
db: { insert },
}));
+vi.mock("../../lib/change-tracking-store", () => ({
+ changeTrackingInsertScrape: vi.fn(),
+}));
+
+vi.mock("../../lib/keyless", () => ({
+ keylessTeamUuid: vi.fn(() => null),
+}));
+
vi.mock("../../lib/gcs-jobs", () => ({
saveDeepResearchToGCS: vi.fn(),
saveExtractToGCS: vi.fn(),
@@ -112,6 +120,16 @@ import {
import * as schema from "../../db/schema";
import { config } from "../../config";
+function deferred<T>() {
+ let resolve!: (value: T | PromiseLike<T>) => void;
+ let reject!: (reason: unknown) => void;
+ const promise = new Promise<T>((resolvePromise, rejectPromise) => {
+ resolve = resolvePromise;
+ reject = rejectPromise;
+ });
+ return { promise, resolve, reject };
+}
+
function makeSearch(overrides: Partial<LoggedSearch> = {}): LoggedSearch {
return {
id: "019e6f45-7778-727d-adf0-0abe9d5062b6",
@@ -180,6 +198,23 @@ describe("logSearch", () => {
expect(context.extra.data).not.toContain("\\u0000");
expect(context.extra.data).toContain("badquery");
});
+
+ it("reports serialization failures without losing the PostgreSQL attempt", async () => {
+ const search = makeSearch({ options: { unsupported: 1n } });
+
+ await expect(logSearch(search)).resolves.toBeUndefined();
+
+ expect(values).toHaveBeenCalledOnce();
+ expect(publishMessage).not.toHaveBeenCalled();
+ expect(metricInc).toHaveBeenCalledWith({
+ table: "searches",
+ outcome: "failed",
+ });
+ expect(logger.error).toHaveBeenCalledWith(
+ "Failed to publish log to Pub/Sub",
+ expect.objectContaining({ logId: search.id, error: expect.any(Error) }),
+ );
+ });
});
describe("logRequest", () => {
@@ -255,10 +290,36 @@ describe("logRequest", () => {
});
it("does not hold the caller on a slow publish", async () => {
- publishMessage.mockReturnValueOnce(new Promise(() => {}));
+ const publication = deferred<string>();
+ publishMessage.mockReturnValueOnce(publication.promise);
await expect(logRequest(makeRequest(null))).resolves.toBeUndefined();
expect(values).toHaveBeenCalled();
+ expect(metricInc).not.toHaveBeenCalled();
+
+ publication.resolve("message-id");
+ await publication.promise;
+ expect(metricInc).toHaveBeenCalledWith({
+ table: "requests",
+ outcome: "published",
+ });
+ });
+
+ it("waits for PostgreSQL when publication finishes first", async () => {
+ const insertion = deferred<void>();
+ values.mockReturnValueOnce(insertion.promise);
+ let finished = false;
+ const logging = logRequest(makeRequest(null)).then(() => {
+ finished = true;
+ });
+
+ await new Promise(resolve => setImmediate(resolve));
+ expect(publishMessage).toHaveBeenCalledOnce();
+ expect(finished).toBe(false);
+
+ insertion.resolve();
+ await logging;
+ expect(finished).toBe(true);
});
it("stores null, not a truncation, when the id exceeds the byte cap", async () => {
@@ -310,12 +371,10 @@ describe("logRequest", () => {
});
it("drops publishes beyond the outstanding cap instead of queueing them", async () => {
- // A fresh module instance: the outstanding counters are process-wide and
- // an earlier test leaves a publish that never settles.
vi.resetModules();
const fresh = await import("./log_job.js");
- // Hung channel: publishes never settle, so each one stays outstanding.
- publishMessage.mockReturnValue(new Promise(() => {}));
+ const publication = deferred<string>();
+ publishMessage.mockImplementation(async () => publication.promise);
config.PUBSUB_MAX_OUTSTANDING_MESSAGES = 2;
try {
await fresh.logRequest(makeRequest(null));
@@ -323,6 +382,8 @@ describe("logRequest", () => {
await fresh.logRequest(makeRequest(null));
} finally {
config.PUBSUB_MAX_OUTSTANDING_MESSAGES = 10_000;
+ publication.resolve("message-id");
+ await fresh.shutdownPubSubLogging();
}
// The database write is never held back by the publisher.
@@ -334,7 +395,12 @@ describe("logRequest", () => {
});
expect(logger.warn).toHaveBeenCalledWith(
"Dropping Pub/Sub log: publisher backlog is full",
- expect.objectContaining({ outstandingMessages: 2, droppedTotal: 1 }),
+ expect.objectContaining({
+ table: "requests",
+ logId: makeRequest(null).id,
+ outstandingMessages: 2,
+ droppedTotal: 1,
+ }),
);
});
@@ -355,6 +421,23 @@ describe("logRequest", () => {
});
});
+ it("releases the backlog capacity after a failed publication", async () => {
+ const originalCap = config.PUBSUB_MAX_OUTSTANDING_MESSAGES;
+ config.PUBSUB_MAX_OUTSTANDING_MESSAGES = 1;
+ try {
+ publishMessage.mockRejectedValueOnce(new Error("Pub/Sub unavailable"));
+ await logRequest(makeRequest(null));
+ await logRequest(makeRequest(null));
+ expect(publishMessage).toHaveBeenCalledTimes(2);
+ expect(metricInc).not.toHaveBeenCalledWith({
+ table: "requests",
+ outcome: "dropped",
+ });
+ } finally {
+ config.PUBSUB_MAX_OUTSTANDING_MESSAGES = originalCap;
+ }
+ });
+
it("flushes Pub/Sub messages during shutdown", async () => {
await logRequest(makeRequest(null));
@@ -377,6 +460,107 @@ describe("shutdownPubSubLogging deadline", () => {
vi.useRealTimers();
});
+ it("waits for pending publications even when flush returns early", async () => {
+ vi.resetModules();
+ const fresh = await import("./log_job.js");
+ const publication = deferred<string>();
+ publishMessage.mockReturnValueOnce(publication.promise);
+ await fresh.logSearch(makeSearch());
+
+ const shutdown = fresh.shutdownPubSubLogging();
+ await new Promise(resolve => setImmediate(resolve));
+ expect(flush).toHaveBeenCalled();
+ expect(close).not.toHaveBeenCalled();
+
+ publication.resolve("message-id");
+ await shutdown;
+ expect(close).toHaveBeenCalledOnce();
+ });
+
+ it("closes at the deadline when publication stays pending after flush", async () => {
+ vi.resetModules();
+ const fresh = await import("./log_job.js");
+ const publication = deferred<string>();
+ publishMessage.mockReturnValueOnce(publication.promise);
+ await fresh.logSearch(makeSearch());
+
+ vi.useFakeTimers();
+ const shutdown = fresh.shutdownPubSubLogging();
+ await vi.advanceTimersByTimeAsync(39_999);
+ expect(close).not.toHaveBeenCalled();
+ await vi.advanceTimersByTimeAsync(1);
+ await shutdown;
+ expect(close).toHaveBeenCalledOnce();
+ expect(logger.warn).toHaveBeenCalledWith(
+ "Pub/Sub log flush did not finish before the shutdown deadline; closing anyway",
+ expect.objectContaining({
+ outstandingMessages: 1,
+ pendingLogSample: [
+ expect.objectContaining({
+ table: "searches",
+ logId: makeSearch().id,
+ }),
+ ],
+ pendingLogSampleTruncated: false,
+ }),
+ );
+
+ publication.resolve("message-id");
+ await publication.promise;
+ });
+
+ it("reports publication failures during shutdown and still closes", async () => {
+ vi.resetModules();
+ const fresh = await import("./log_job.js");
+ const publication = deferred<string>();
+ publishMessage.mockReturnValueOnce(publication.promise);
+ await fresh.logSearch(makeSearch());
+ const shutdown = fresh.shutdownPubSubLogging();
+
+ publication.reject(new Error("Pub/Sub unavailable"));
+ await shutdown;
+ expect(close).toHaveBeenCalledOnce();
+ expect(metricInc).toHaveBeenCalledWith({
+ table: "searches",
+ outcome: "failed",
+ });
+ });
+
+ it("bounds client close after draining so shutdown can finish", async () => {
+ vi.resetModules();
+ const fresh = await import("./log_job.js");
+ await fresh.logSearch(makeSearch());
+ close.mockReturnValueOnce(new Promise(() => {}));
+ vi.useFakeTimers();
+ const shutdown = fresh.shutdownPubSubLogging();
+ await vi.advanceTimersByTimeAsync(5_000);
+ await shutdown;
+ expect(logger.error).toHaveBeenCalledWith(
+ "Failed to close Pub/Sub log publisher",
+ expect.objectContaining({ error: expect.any(Error) }),
+ );
+ });
+
+ it("reports late logs instead of publishing after shutdown starts", async () => {
+ vi.resetModules();
+ const fresh = await import("./log_job.js");
+ await fresh.logSearch(makeSearch());
+ const flushing = deferred<void>();
+ flush.mockReturnValueOnce(flushing.promise);
+ const shutdown = fresh.shutdownPubSubLogging();
+
+ await fresh.logSearch(makeSearch());
+ expect(publishMessage).toHaveBeenCalledOnce();
+ expect(values).toHaveBeenCalledTimes(2);
+ expect(metricInc).toHaveBeenCalledWith({
+ table: "searches",
+ outcome: "failed",
+ });
+
+ flushing.resolve();
+ await shutdown;
+ });
+
it("closes the client when a flush outlives the shutdown deadline", async () => {
// A fresh module instance: shutdown is memoized per process.
vi.resetModules();
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index 38236e438..7e3ada468 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -70,8 +70,8 @@ let pubSubShutdown: Promise<void> | undefined;
// whole retry budget to that one value (CallSettings.merge), so a stalled RPC
// used to be a single 60 s attempt and then a lost row. An explicit `retry`
// is applied after that override and replaces the backoff settings wholesale.
-// Short attempts detect a hung channel quickly; the total budget covers every
-// stall measured in production so far, all of which cleared within 140 s.
+// Short attempts detect a stalled RPC quickly. The total budget allows
+// retries across a longer connection disruption.
// Retry codes stay the client's defaults for Publish (DEADLINE_EXCEEDED,
// UNAVAILABLE, INTERNAL, UNKNOWN, ABORTED, CANCELLED, RESOURCE_EXHAUSTED).
// A retry can deliver a batch twice when the first attempt was persisted but
@@ -94,14 +94,15 @@ const PUBSUB_PUBLISH_OPTIONS: PublishOptions = {
};
// Shutdown waits this long for in-flight publishes before closing the client.
-// It must fit inside the pods' termination grace period (60 s for nuq
-// workers); a stall that outlives it loses what is still in flight, which is
-// the same outcome as today and what a durable relay is for.
+// The drain shares the existing pod grace period with active work and exit.
+// Memory kills and work that exceeds the pod grace period can still lose logs.
const PUBSUB_SHUTDOWN_FLUSH_TIMEOUT_MS = 40_000;
-// Rows waiting on Pub/Sub across all topics. Publishing is fire-and-forget,
-// so during a stall this is what grows; see PUBSUB_MAX_OUTSTANDING_* in config.
-let outstandingMessages = 0;
+// Track publication promises because topic.flush() can finish before an active RPC.
+const pendingPublications = new Map<
+ Promise<string>,
+ { table: string; logId: string; startedAt: number }
+>();
let outstandingBytes = 0;
let droppedTotal = 0;
let lastDropWarningAt = 0;
@@ -139,42 +140,67 @@ function getTopic(client: PubSub, table: string): Topic {
}
async function publishLog(table: string, data: any, logger: Logger) {
- const client = getPubSubClient(logger);
- if (!client) return;
+ const startedAt = Date.now();
+ try {
+ if (pubSubShutdown) {
+ throw new Error("Pub/Sub log publisher is shutting down");
+ }
+ const client = getPubSubClient(logger);
+ if (!client) {
+ if (config.PUBSUB_CREDENTIALS) {
+ throw new Error("Pub/Sub log publisher initialization failed");
+ }
+ return;
+ }
- const payload = Buffer.from(JSON.stringify(data));
- if (
- outstandingMessages >= config.PUBSUB_MAX_OUTSTANDING_MESSAGES ||
- outstandingBytes + payload.length > config.PUBSUB_MAX_OUTSTANDING_BYTES
- ) {
- droppedTotal++;
- pubsubLogPublishTotal.inc({ table, outcome: "dropped" });
- const now = Date.now();
- if (now - lastDropWarningAt >= 60_000) {
- lastDropWarningAt = now;
- logger.warn("Dropping Pub/Sub log: publisher backlog is full", {
- outstandingMessages,
- outstandingBytes,
- droppedTotal,
- });
+ const payload = Buffer.from(JSON.stringify(data));
+ if (
+ pendingPublications.size >= config.PUBSUB_MAX_OUTSTANDING_MESSAGES ||
+ outstandingBytes + payload.length > config.PUBSUB_MAX_OUTSTANDING_BYTES
+ ) {
+ droppedTotal++;
+ pubsubLogPublishTotal.inc({ table, outcome: "dropped" });
+ const now = Date.now();
+ if (now - lastDropWarningAt >= 60_000) {
+ lastDropWarningAt = now;
+ logger.warn("Dropping Pub/Sub log: publisher backlog is full", {
+ table,
+ logId: data.id,
+ payloadBytes: payload.length,
+ outstandingMessages: pendingPublications.size,
+ outstandingBytes,
+ droppedTotal,
+ });
+ }
+ return;
}
- return;
- }
- outstandingMessages++;
- outstandingBytes += payload.length;
- try {
- await getTopic(client, table).publishMessage({ data: payload });
- pubsubLogPublishTotal.inc({ table, outcome: "published" });
+ const publication = getTopic(client, table).publishMessage({
+ data: payload,
+ });
+ pendingPublications.set(publication, { table, logId: data.id, startedAt });
+ outstandingBytes += payload.length;
+
+ try {
+ await publication;
+ pubsubLogPublishTotal.inc({ table, outcome: "published" });
+ } finally {
+ pendingPublications.delete(publication);
+ outstandingBytes -= payload.length;
+ }
} catch (error) {
pubsubLogPublishTotal.inc({ table, outcome: "failed" });
- logger.error("Failed to publish log to Pub/Sub", { error });
+
+ logger.error("Failed to publish log to Pub/Sub", {
+ error,
+ table,
+ logId: data.id,
+ durationMs: Date.now() - startedAt,
+ });
Sentry.captureException(error, {
tags: { table, operation: "publishPubSubLog" },
+ extra: { logId: data.id },
});
- } finally {
- outstandingMessages--;
- outstandingBytes -= payload.length;
}
}
@@ -193,9 +219,18 @@ async function shutdownPubSubLoggingOnce(): Promise<void> {
module: "log_job",
method: "shutdownPubSubLogging",
});
- const flushed = Promise.allSettled(
- [...pubSubTopics.values()].map(topic => topic.flush()),
- );
+ const startedAt = Date.now();
+ logger.info("Draining Pub/Sub log publisher", {
+ outstandingMessages: pendingPublications.size,
+ outstandingBytes,
+ timeoutMs: PUBSUB_SHUTDOWN_FLUSH_TIMEOUT_MS,
+ });
+ // Callers stop accepting work before shutdown. Reject new publications so
+ // this snapshot includes every publication that can still use the client.
+ const flushed = Promise.allSettled([
+ ...[...pubSubTopics.values()].map(async topic => topic.flush()),
+ ...pendingPublications.keys(),
+ ]);
let deadline: NodeJS.Timeout | undefined;
const timedOut = new Promise<"timeout">(resolve => {
deadline = setTimeout(
@@ -211,8 +246,10 @@ async function shutdownPubSubLoggingOnce(): Promise<void> {
"Pub/Sub log flush did not finish before the shutdown deadline; closing anyway",
{
timeoutMs: PUBSUB_SHUTDOWN_FLUSH_TIMEOUT_MS,
- outstandingMessages,
+ outstandingMessages: pendingPublications.size,
outstandingBytes,
+ pendingLogSample: [...pendingPublications.values()].slice(0, 50),
+ pendingLogSampleTruncated: pendingPublications.size > 50,
},
);
} else {
@@ -221,21 +258,36 @@ async function shutdownPubSubLoggingOnce(): Promise<void> {
);
if (errors.length > 0) {
- logger.error("Failed to flush Pub/Sub log publisher", { errors });
+ logger.error("Failed to drain Pub/Sub log publisher", { errors });
Sentry.captureException(errors[0], {
tags: { operation: "flushPubSubLogPublisher" },
extra: { failures: errors.length },
});
+ } else {
+ logger.info("Pub/Sub log publisher drained", {
+ durationMs: Date.now() - startedAt,
+ });
}
}
+ let closeDeadline: NodeJS.Timeout | undefined;
try {
- await client.close();
+ await Promise.race([
+ client.close(),
+ new Promise<never>((_, reject) => {
+ closeDeadline = setTimeout(
+ () => reject(new Error("Pub/Sub client close exceeded 5 seconds")),
+ 5_000,
+ );
+ }),
+ ]);
} catch (error) {
logger.error("Failed to close Pub/Sub log publisher", { error });
Sentry.captureException(error, {
tags: { operation: "closePubSubLogPublisher" },
});
+ } finally {
+ clearTimeout(closeDeadline);
}
}
@@ -266,6 +318,7 @@ async function robustInsert(
...data,
created_at: data.created_at ?? new Date(),
});
+ // Publish in the background. Customer responses must not wait for Pub/Sub.
void publishLog(table, data, logger);
const attempts: { error: any; timeMs: number; backoffMs: number }[] = [];
diff --git a/apps/api/src/services/worker/nuq-worker-runner.ts b/apps/api/src/services/worker/nuq-worker-runner.ts
index bbffc593f..3af1a4145 100644
--- a/apps/api/src/services/worker/nuq-worker-runner.ts
+++ b/apps/api/src/services/worker/nuq-worker-runner.ts
@@ -69,6 +69,7 @@ export async function runNuqWorker(options: {
}
let isShuttingDown = false;
+ let shutdownStartedAt: number | undefined;
const app = Express();
@@ -110,7 +111,12 @@ export async function runNuqWorker(options: {
});
function shutdown() {
+ if (isShuttingDown) return;
isShuttingDown = true;
+ shutdownStartedAt = Date.now();
+ _logger.info("NuQ worker stopping after active work", {
+ module: options.serviceName,
+ });
}
process.on("SIGINT", shutdown);
@@ -200,7 +206,10 @@ export async function runNuqWorker(options: {
}
}
- _logger.info("NuQ worker shutting down", { module: options.serviceName });
+ _logger.info("NuQ worker shutting down", {
+ module: options.serviceName,
+ workDrainMs: Date.now() - shutdownStartedAt!,
+ });
server.close(async () => {
await options.beforeShutdown?.();