Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): publish a job-log row to Pub/Sub first, and fail the log call when it cannot (#4699)

Gergő Móricz79eee0d567d7
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.ts
index 982b2107e..54bb66e5c 100644
--- a/apps/api/src/controllers/v2/search.ts
+++ b/apps/api/src/controllers/v2/search.ts
@@ -344,6 +344,9 @@ async function searchControllerInner(
zeroDataRetention,
api_key_id: req.acuc?.api_key_id ?? null,
});
+ // The rejection is surfaced where the promise is awaited below; this
+ // only stops it counting as unhandled until then.
+ logRequestPromise.catch(() => {});
}
const toolsOnly = isToolsOnlySearch(req.body.sources, req.body.categories);
@@ -359,6 +362,17 @@ async function searchControllerInner(
zeroDataRetention,
)
: 0;
+ // The request must be on record before anything is reserved, runs, or
+ // bills. A failed log fails the request here, ahead of the keyless
+ // reservation and the search, so there is nothing to refund or unbill.
+ const logStart = Date.now();
+ await logRequestPromise;
+ const waited = Date.now() - logStart;
+ if (waited >= 5)
+ logger.warn("Had to wait for log request promise to complete", {
+ timeMs: waited,
+ });
+
if (projectedKeylessCredits > 0) {
const reservation = await reserveKeylessCredits(
req.auth.team_id,
@@ -446,15 +460,6 @@ async function searchControllerInner(
const endTime = new Date().getTime();
const timeTakenInSeconds = (endTime - middlewareStartTime) / 1000;
- // Wait for the parent log before inserting the child search log.
- const logStart = Date.now();
- await logRequestPromise;
- const waited = Date.now() - logStart;
- if (waited >= 5)
- logger.warn("Had to wait for log request promise to complete", {
- timeMs: waited,
- });
-
logSearch(
{
id: jobId,
diff --git a/apps/api/src/services/logging/log_job.test.ts b/apps/api/src/services/logging/log_job.test.ts
index 91e6516ec..315687dfa 100644
--- a/apps/api/src/services/logging/log_job.test.ts
+++ b/apps/api/src/services/logging/log_job.test.ts
@@ -233,12 +233,12 @@ describe("logSearch", () => {
);
});
- it("reports serialization failures without losing the PostgreSQL attempt", async () => {
+ it("fails the log call on a serialization failure before touching PostgreSQL", async () => {
const search = makeSearch({ options: { unsupported: 1n } });
- await expect(logSearch(search)).resolves.toBeUndefined();
+ await expect(logSearch(search)).rejects.toThrow();
- expect(values).toHaveBeenCalledOnce();
+ expect(values).not.toHaveBeenCalled();
expect(publishMessage).not.toHaveBeenCalled();
expect(metricInc).toHaveBeenCalledWith({
table: "searches",
@@ -578,10 +578,11 @@ describe("logRequest", () => {
const gaxOpts = publishes[0].options.gaxOpts;
// A bare `timeout` would collapse the retry budget to one attempt.
expect(gaxOpts.timeout).toBeUndefined();
+ // The caller waits on the publish, so the whole budget is 30 s.
expect(gaxOpts.retry.backoffSettings).toMatchObject({
- initialRpcTimeoutMillis: 15_000,
- maxRpcTimeoutMillis: 15_000,
- totalTimeoutMillis: 300_000,
+ initialRpcTimeoutMillis: 10_000,
+ maxRpcTimeoutMillis: 10_000,
+ totalTimeoutMillis: 30_000,
});
expect(gaxOpts.retry.retryCodes).toBeUndefined();
@@ -633,12 +634,14 @@ describe("logRequest", () => {
expect(values).not.toHaveBeenCalled();
});
- it("keeps the database write when Pub/Sub fails", async () => {
+ it("fails the log call and skips PostgreSQL when Pub/Sub fails", async () => {
publishMessage.mockRejectedValueOnce(new Error("Pub/Sub unavailable"));
- await expect(logRequest(makeRequest(null))).resolves.toBeUndefined();
+ await expect(logRequest(makeRequest(null))).rejects.toThrow(
+ "Pub/Sub unavailable",
+ );
- expect(values).toHaveBeenCalled();
+ expect(values).not.toHaveBeenCalled();
expect(logger.error).toHaveBeenCalledWith(
"Failed to publish log to Pub/Sub",
expect.objectContaining({ error: expect.any(Error) }),
@@ -678,16 +681,22 @@ describe("logRequest", () => {
);
});
- it("does not hold the caller on a slow publish", async () => {
+ it("holds the caller until the publish is acknowledged", async () => {
const publication = deferred<string>();
publishMessage.mockReturnValueOnce(publication.promise);
+ let finished = false;
+ const logging = logRequest(makeRequest(null)).then(() => {
+ finished = true;
+ });
- await expect(logRequest(makeRequest(null))).resolves.toBeUndefined();
- expect(values).toHaveBeenCalled();
- expect(metricInc).not.toHaveBeenCalled();
+ await new Promise(resolve => setImmediate(resolve));
+ expect(publishMessage).toHaveBeenCalledOnce();
+ expect(values).not.toHaveBeenCalled();
+ expect(finished).toBe(false);
publication.resolve("message-id");
- await publication.promise;
+ await logging;
+ expect(values).toHaveBeenCalled();
expect(metricInc).toHaveBeenCalledWith({
table: "requests",
outcome: "published",
@@ -759,31 +768,42 @@ describe("logRequest", () => {
expect(published.origin).toBe("api");
});
- it("drops publishes beyond the outstanding cap instead of queueing them", async () => {
+ it("refuses a publish beyond the outstanding cap and fails that log call", async () => {
vi.resetModules();
const fresh = await import("./log_job.js");
const publication = deferred<string>();
publishMessage.mockImplementation(async () => publication.promise);
config.PUBSUB_MAX_OUTSTANDING_MESSAGES = 2;
try {
- await fresh.logRequest(makeRequest(null));
- await fresh.logRequest(makeRequest(null));
- await fresh.logRequest(makeRequest(null));
+ const first = fresh.logRequest(makeRequest(null));
+ const second = fresh.logRequest(makeRequest(null));
+ await new Promise(resolve => setImmediate(resolve));
+ await expect(fresh.logRequest(makeRequest(null))).rejects.toThrow(
+ "backlog is full",
+ );
+ publication.resolve("message-id");
+ await Promise.all([first, second]);
} finally {
config.PUBSUB_MAX_OUTSTANDING_MESSAGES = 10_000;
publication.resolve("message-id");
await fresh.shutdownPubSubLogging();
}
- // The database write is never held back by the publisher.
- expect(values).toHaveBeenCalledTimes(3);
+ // The refused row never reaches PostgreSQL either.
+ expect(values).toHaveBeenCalledTimes(2);
expect(publishMessage).toHaveBeenCalledTimes(2);
expect(metricInc).toHaveBeenCalledWith({
table: "requests",
outcome: "dropped",
});
+ // A refusal is the rate-limited warning, not a per-row error.
+ expect(logger.error).not.toHaveBeenCalled();
+ expect(metricInc).not.toHaveBeenCalledWith({
+ table: "requests",
+ outcome: "failed",
+ });
expect(logger.warn).toHaveBeenCalledWith(
- "Dropping Pub/Sub log: publisher backlog is full",
+ "Refusing Pub/Sub log: publisher backlog is full",
expect.objectContaining({
table: "requests",
logId: makeRequest(null).id,
@@ -802,8 +822,9 @@ describe("logRequest", () => {
});
publishMessage.mockRejectedValueOnce(new Error("Pub/Sub unavailable"));
- await logRequest(makeRequest(null));
- await new Promise(resolve => setImmediate(resolve));
+ await expect(logRequest(makeRequest(null))).rejects.toThrow(
+ "Pub/Sub unavailable",
+ );
expect(metricInc).toHaveBeenCalledWith({
table: "requests",
outcome: "failed",
@@ -815,7 +836,9 @@ describe("logRequest", () => {
config.PUBSUB_MAX_OUTSTANDING_MESSAGES = 1;
try {
publishMessage.mockRejectedValueOnce(new Error("Pub/Sub unavailable"));
- await logRequest(makeRequest(null));
+ await expect(logRequest(makeRequest(null))).rejects.toThrow(
+ "Pub/Sub unavailable",
+ );
await logRequest(makeRequest(null));
expect(publishMessage).toHaveBeenCalledTimes(2);
expect(metricInc).not.toHaveBeenCalledWith({
@@ -854,7 +877,8 @@ describe("shutdownPubSubLogging deadline", () => {
const fresh = await import("./log_job.js");
const publication = deferred<string>();
publishMessage.mockReturnValueOnce(publication.promise);
- await fresh.logSearch(makeSearch());
+ const logging = fresh.logSearch(makeSearch());
+ await new Promise(resolve => setImmediate(resolve));
const shutdown = fresh.shutdownPubSubLogging();
await new Promise(resolve => setImmediate(resolve));
@@ -863,6 +887,7 @@ describe("shutdownPubSubLogging deadline", () => {
publication.resolve("message-id");
await shutdown;
+ await logging;
expect(close).toHaveBeenCalledOnce();
});
@@ -871,7 +896,8 @@ describe("shutdownPubSubLogging deadline", () => {
const fresh = await import("./log_job.js");
const publication = deferred<string>();
publishMessage.mockReturnValueOnce(publication.promise);
- await fresh.logSearch(makeSearch());
+ const logging = fresh.logSearch(makeSearch());
+ await new Promise(resolve => setImmediate(resolve));
vi.useFakeTimers();
const shutdown = fresh.shutdownPubSubLogging();
@@ -895,7 +921,7 @@ describe("shutdownPubSubLogging deadline", () => {
);
publication.resolve("message-id");
- await publication.promise;
+ await logging;
});
it("reports publication failures during shutdown and still closes", async () => {
@@ -903,11 +929,13 @@ describe("shutdownPubSubLogging deadline", () => {
const fresh = await import("./log_job.js");
const publication = deferred<string>();
publishMessage.mockReturnValueOnce(publication.promise);
- await fresh.logSearch(makeSearch());
+ const logging = fresh.logSearch(makeSearch());
+ await new Promise(resolve => setImmediate(resolve));
const shutdown = fresh.shutdownPubSubLogging();
publication.reject(new Error("Pub/Sub unavailable"));
await shutdown;
+ await expect(logging).rejects.toThrow("Pub/Sub unavailable");
expect(close).toHaveBeenCalledOnce();
expect(metricInc).toHaveBeenCalledWith({
table: "searches",
@@ -938,9 +966,11 @@ describe("shutdownPubSubLogging deadline", () => {
flush.mockReturnValueOnce(flushing.promise);
const shutdown = fresh.shutdownPubSubLogging();
- await fresh.logSearch(makeSearch());
+ await expect(fresh.logSearch(makeSearch())).rejects.toThrow(
+ "shutting down",
+ );
expect(publishMessage).toHaveBeenCalledOnce();
- expect(values).toHaveBeenCalledTimes(2);
+ expect(values).toHaveBeenCalledTimes(1);
expect(metricInc).toHaveBeenCalledWith({
table: "searches",
outcome: "failed",
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index d4c57370b..4666044c6 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -151,17 +151,20 @@ let pubSubShutdown: Promise<void> | undefined;
// A retry can deliver a batch twice when the first attempt was persisted but
// its response was lost; the ClickHouse tables dedupe on row id, not message
// id, so those copies collapse.
+//
+// The log call waits for the publish, so the budget is what a caller can be
+// held for while Pub/Sub is unreachable before the log call fails.
const PUBSUB_PUBLISH_OPTIONS: PublishOptions = {
gaxOpts: {
retry: {
backoffSettings: {
initialRetryDelayMillis: 250,
retryDelayMultiplier: 2,
- maxRetryDelayMillis: 15_000,
- initialRpcTimeoutMillis: 15_000,
+ maxRetryDelayMillis: 5_000,
+ initialRpcTimeoutMillis: 10_000,
rpcTimeoutMultiplier: 1,
- maxRpcTimeoutMillis: 15_000,
- totalTimeoutMillis: 300_000,
+ maxRpcTimeoutMillis: 10_000,
+ totalTimeoutMillis: 30_000,
},
},
},
@@ -212,6 +215,20 @@ function getTopic(client: PubSub, table: string): Topic {
return topic;
}
+/** The publisher already holds as many unacknowledged rows as it may. */
+class PubSubBacklogFullError extends Error {
+ constructor() {
+ super("Pub/Sub log publisher backlog is full");
+ this.name = "PubSubBacklogFullError";
+ }
+}
+
+/**
+ * Publishes one job-log row to its Pub/Sub topic and waits for the
+ * acknowledgement. Pub/Sub is the log's store of record, so a publish that
+ * fails, is refused by a full backlog, or arrives during shutdown throws to
+ * the caller; nothing else is written for that row.
+ */
async function publishLog(table: string, data: any, logger: Logger) {
const startedAt = Date.now();
try {
@@ -252,7 +269,7 @@ async function publishLog(table: string, data: any, logger: Logger) {
const now = Date.now();
if (now - lastDropWarningAt >= 60_000) {
lastDropWarningAt = now;
- logger.warn("Dropping Pub/Sub log: publisher backlog is full", {
+ logger.warn("Refusing Pub/Sub log: publisher backlog is full", {
table,
logId: data.id,
payloadBytes: payload.length,
@@ -261,7 +278,7 @@ async function publishLog(table: string, data: any, logger: Logger) {
droppedTotal,
});
}
- return;
+ throw new PubSubBacklogFullError();
}
const publication = getTopic(client, table).publishMessage({
@@ -284,14 +301,18 @@ async function publishLog(table: string, data: any, logger: Logger) {
}
});
} catch (error) {
- pubsubLogPublishTotal.inc({ table, outcome: "failed" });
-
- logger.error("Failed to publish log to Pub/Sub", {
- error,
- table,
- logId: data.id,
- durationMs: Date.now() - startedAt,
- });
+ // A backlog refusal counted itself as "dropped" and warned, rate-limited,
+ // above; only a genuine publish failure is an error here.
+ if (!(error instanceof PubSubBacklogFullError)) {
+ pubsubLogPublishTotal.inc({ table, outcome: "failed" });
+ logger.error("Failed to publish log to Pub/Sub", {
+ error,
+ table,
+ logId: data.id,
+ durationMs: Date.now() - startedAt,
+ });
+ }
+ throw error;
}
}
@@ -388,37 +409,36 @@ async function robustInsert(
canonicalLog: "log_job/robustInsert",
});
+ if (config.USE_DB_AUTHENTICATION !== true) {
+ logger.info(
+ "Skipping database insertion due to USE_DB_AUTHENTICATION being off",
+ );
+ return;
+ }
+
+ // The single point where a row leaves for both stores: clean it once so
+ // PostgreSQL and ClickHouse receive identical, accepted values.
+ data = sanitizeLogData({
+ ...data,
+ created_at: data.created_at ?? new Date(),
+ });
+
+ // Pub/Sub is the store of record: publish first and wait for it. A failed
+ // publish fails the log call, and the PostgreSQL copy is not attempted.
+ await publishLog(table, data, logger);
+
+ // The PostgreSQL copy is on its way out; its failure is logged, not thrown.
const attempts: { error: any; timeMs: number; backoffMs: number }[] = [];
try {
- const inserted = await withSpan("log_job.postgres.insert", async span => {
+ await withSpan("log_job.postgres.insert", async span => {
setSpanAttributes(span, {
"db.system": "postgresql",
"log_job.table": table,
"log_job.id": data.id,
"log_job.force": force,
+ "log_job.postgres.enabled": true,
});
-
- if (config.USE_DB_AUTHENTICATION !== true) {
- logger.info(
- "Skipping database insertion due to USE_DB_AUTHENTICATION being off",
- );
- setSpanAttributes(span, {
- "log_job.postgres.enabled": false,
- "log_job.postgres.outcome": "skipped",
- });
- return false;
- }
-
- setSpanAttributes(span, { "log_job.postgres.enabled": true });
const target = tableMap[table];
- // The single point where a row leaves for both stores: clean it once so
- // PostgreSQL and ClickHouse receive identical, accepted values.
- data = sanitizeLogData({
- ...data,
- created_at: data.created_at ?? new Date(),
- });
- // Publish in the background. Customer responses must not wait for Pub/Sub.
- void publishLog(table, data, logger);
const maxAttempts = force ? 10 : 1;
for (let i = 0; i < maxAttempts; i++) {
@@ -456,10 +476,8 @@ async function robustInsert(
lastAttempt?.error ?? new Error("Database insert was not attempted")
);
}
- return true;
});
- if (!inserted) return;
if (attempts.length === 1) {
logger.debug("Inserted into database successfully", { attempts });
} else {