Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): publish a job-log row to Pub/Sub first, and fail the log call when it cannot (#4699)
Gergő Móricz79eee0d567d7
diff --git a/apps/api/src/controllers/v2/search.ts b/apps/api/src/controllers/v2/search.tsindex 982b2107e..54bb66e5c 100644--- a/apps/api/src/controllers/v2/search.ts+++ b/apps/api/src/controllers/v2/search.ts@@ -344,6 +344,9 @@ async function searchControllerInner(zeroDataRetention,api_key_id: req.acuc?.api_key_id ?? null,});+ // The rejection is surfaced where the promise is awaited below; this+ // only stops it counting as unhandled until then.+ logRequestPromise.catch(() => {});}const toolsOnly = isToolsOnlySearch(req.body.sources, req.body.categories);@@ -359,6 +362,17 @@ async function searchControllerInner(zeroDataRetention,): 0;+ // The request must be on record before anything is reserved, runs, or+ // bills. A failed log fails the request here, ahead of the keyless+ // reservation and the search, so there is nothing to refund or unbill.+ const logStart = Date.now();+ await logRequestPromise;+ const waited = Date.now() - logStart;+ if (waited >= 5)+ logger.warn("Had to wait for log request promise to complete", {+ timeMs: waited,+ });+if (projectedKeylessCredits > 0) {const reservation = await reserveKeylessCredits(req.auth.team_id,@@ -446,15 +460,6 @@ async function searchControllerInner(const endTime = new Date().getTime();const timeTakenInSeconds = (endTime - middlewareStartTime) / 1000;- // Wait for the parent log before inserting the child search log.- const logStart = Date.now();- await logRequestPromise;- const waited = Date.now() - logStart;- if (waited >= 5)- logger.warn("Had to wait for log request promise to complete", {- timeMs: waited,- });-logSearch({id: jobId,diff --git a/apps/api/src/services/logging/log_job.test.ts b/apps/api/src/services/logging/log_job.test.tsindex 91e6516ec..315687dfa 100644--- a/apps/api/src/services/logging/log_job.test.ts+++ b/apps/api/src/services/logging/log_job.test.ts@@ -233,12 +233,12 @@ describe("logSearch", () => {);});- it("reports serialization failures without losing the PostgreSQL attempt", async () => {+ it("fails the log call on a serialization failure before touching PostgreSQL", async () => {const search = makeSearch({ options: { unsupported: 1n } });- await expect(logSearch(search)).resolves.toBeUndefined();+ await expect(logSearch(search)).rejects.toThrow();- expect(values).toHaveBeenCalledOnce();+ expect(values).not.toHaveBeenCalled();expect(publishMessage).not.toHaveBeenCalled();expect(metricInc).toHaveBeenCalledWith({table: "searches",@@ -578,10 +578,11 @@ describe("logRequest", () => {const gaxOpts = publishes[0].options.gaxOpts;// A bare `timeout` would collapse the retry budget to one attempt.expect(gaxOpts.timeout).toBeUndefined();+ // The caller waits on the publish, so the whole budget is 30 s.expect(gaxOpts.retry.backoffSettings).toMatchObject({- initialRpcTimeoutMillis: 15_000,- maxRpcTimeoutMillis: 15_000,- totalTimeoutMillis: 300_000,+ initialRpcTimeoutMillis: 10_000,+ maxRpcTimeoutMillis: 10_000,+ totalTimeoutMillis: 30_000,});expect(gaxOpts.retry.retryCodes).toBeUndefined();@@ -633,12 +634,14 @@ describe("logRequest", () => {expect(values).not.toHaveBeenCalled();});- it("keeps the database write when Pub/Sub fails", async () => {+ it("fails the log call and skips PostgreSQL when Pub/Sub fails", async () => {publishMessage.mockRejectedValueOnce(new Error("Pub/Sub unavailable"));- await expect(logRequest(makeRequest(null))).resolves.toBeUndefined();+ await expect(logRequest(makeRequest(null))).rejects.toThrow(+ "Pub/Sub unavailable",+ );- expect(values).toHaveBeenCalled();+ expect(values).not.toHaveBeenCalled();expect(logger.error).toHaveBeenCalledWith("Failed to publish log to Pub/Sub",expect.objectContaining({ error: expect.any(Error) }),@@ -678,16 +681,22 @@ describe("logRequest", () => {);});- it("does not hold the caller on a slow publish", async () => {+ it("holds the caller until the publish is acknowledged", async () => {const publication = deferred<string>();publishMessage.mockReturnValueOnce(publication.promise);+ let finished = false;+ const logging = logRequest(makeRequest(null)).then(() => {+ finished = true;+ });- await expect(logRequest(makeRequest(null))).resolves.toBeUndefined();- expect(values).toHaveBeenCalled();- expect(metricInc).not.toHaveBeenCalled();+ await new Promise(resolve => setImmediate(resolve));+ expect(publishMessage).toHaveBeenCalledOnce();+ expect(values).not.toHaveBeenCalled();+ expect(finished).toBe(false);publication.resolve("message-id");- await publication.promise;+ await logging;+ expect(values).toHaveBeenCalled();expect(metricInc).toHaveBeenCalledWith({table: "requests",outcome: "published",@@ -759,31 +768,42 @@ describe("logRequest", () => {expect(published.origin).toBe("api");});- it("drops publishes beyond the outstanding cap instead of queueing them", async () => {+ it("refuses a publish beyond the outstanding cap and fails that log call", async () => {vi.resetModules();const fresh = await import("./log_job.js");const publication = deferred<string>();publishMessage.mockImplementation(async () => publication.promise);config.PUBSUB_MAX_OUTSTANDING_MESSAGES = 2;try {- await fresh.logRequest(makeRequest(null));- await fresh.logRequest(makeRequest(null));- await fresh.logRequest(makeRequest(null));+ const first = fresh.logRequest(makeRequest(null));+ const second = fresh.logRequest(makeRequest(null));+ await new Promise(resolve => setImmediate(resolve));+ await expect(fresh.logRequest(makeRequest(null))).rejects.toThrow(+ "backlog is full",+ );+ publication.resolve("message-id");+ await Promise.all([first, second]);} finally {config.PUBSUB_MAX_OUTSTANDING_MESSAGES = 10_000;publication.resolve("message-id");await fresh.shutdownPubSubLogging();}- // The database write is never held back by the publisher.- expect(values).toHaveBeenCalledTimes(3);+ // The refused row never reaches PostgreSQL either.+ expect(values).toHaveBeenCalledTimes(2);expect(publishMessage).toHaveBeenCalledTimes(2);expect(metricInc).toHaveBeenCalledWith({table: "requests",outcome: "dropped",});+ // A refusal is the rate-limited warning, not a per-row error.+ expect(logger.error).not.toHaveBeenCalled();+ expect(metricInc).not.toHaveBeenCalledWith({+ table: "requests",+ outcome: "failed",+ });expect(logger.warn).toHaveBeenCalledWith(- "Dropping Pub/Sub log: publisher backlog is full",+ "Refusing Pub/Sub log: publisher backlog is full",expect.objectContaining({table: "requests",logId: makeRequest(null).id,@@ -802,8 +822,9 @@ describe("logRequest", () => {});publishMessage.mockRejectedValueOnce(new Error("Pub/Sub unavailable"));- await logRequest(makeRequest(null));- await new Promise(resolve => setImmediate(resolve));+ await expect(logRequest(makeRequest(null))).rejects.toThrow(+ "Pub/Sub unavailable",+ );expect(metricInc).toHaveBeenCalledWith({table: "requests",outcome: "failed",@@ -815,7 +836,9 @@ describe("logRequest", () => {config.PUBSUB_MAX_OUTSTANDING_MESSAGES = 1;try {publishMessage.mockRejectedValueOnce(new Error("Pub/Sub unavailable"));- await logRequest(makeRequest(null));+ await expect(logRequest(makeRequest(null))).rejects.toThrow(+ "Pub/Sub unavailable",+ );await logRequest(makeRequest(null));expect(publishMessage).toHaveBeenCalledTimes(2);expect(metricInc).not.toHaveBeenCalledWith({@@ -854,7 +877,8 @@ describe("shutdownPubSubLogging deadline", () => {const fresh = await import("./log_job.js");const publication = deferred<string>();publishMessage.mockReturnValueOnce(publication.promise);- await fresh.logSearch(makeSearch());+ const logging = fresh.logSearch(makeSearch());+ await new Promise(resolve => setImmediate(resolve));const shutdown = fresh.shutdownPubSubLogging();await new Promise(resolve => setImmediate(resolve));@@ -863,6 +887,7 @@ describe("shutdownPubSubLogging deadline", () => {publication.resolve("message-id");await shutdown;+ await logging;expect(close).toHaveBeenCalledOnce();});@@ -871,7 +896,8 @@ describe("shutdownPubSubLogging deadline", () => {const fresh = await import("./log_job.js");const publication = deferred<string>();publishMessage.mockReturnValueOnce(publication.promise);- await fresh.logSearch(makeSearch());+ const logging = fresh.logSearch(makeSearch());+ await new Promise(resolve => setImmediate(resolve));vi.useFakeTimers();const shutdown = fresh.shutdownPubSubLogging();@@ -895,7 +921,7 @@ describe("shutdownPubSubLogging deadline", () => {);publication.resolve("message-id");- await publication.promise;+ await logging;});it("reports publication failures during shutdown and still closes", async () => {@@ -903,11 +929,13 @@ describe("shutdownPubSubLogging deadline", () => {const fresh = await import("./log_job.js");const publication = deferred<string>();publishMessage.mockReturnValueOnce(publication.promise);- await fresh.logSearch(makeSearch());+ const logging = fresh.logSearch(makeSearch());+ await new Promise(resolve => setImmediate(resolve));const shutdown = fresh.shutdownPubSubLogging();publication.reject(new Error("Pub/Sub unavailable"));await shutdown;+ await expect(logging).rejects.toThrow("Pub/Sub unavailable");expect(close).toHaveBeenCalledOnce();expect(metricInc).toHaveBeenCalledWith({table: "searches",@@ -938,9 +966,11 @@ describe("shutdownPubSubLogging deadline", () => {flush.mockReturnValueOnce(flushing.promise);const shutdown = fresh.shutdownPubSubLogging();- await fresh.logSearch(makeSearch());+ await expect(fresh.logSearch(makeSearch())).rejects.toThrow(+ "shutting down",+ );expect(publishMessage).toHaveBeenCalledOnce();- expect(values).toHaveBeenCalledTimes(2);+ expect(values).toHaveBeenCalledTimes(1);expect(metricInc).toHaveBeenCalledWith({table: "searches",outcome: "failed",diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex d4c57370b..4666044c6 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -151,17 +151,20 @@ let pubSubShutdown: Promise<void> | undefined;// A retry can deliver a batch twice when the first attempt was persisted but// its response was lost; the ClickHouse tables dedupe on row id, not message// id, so those copies collapse.+//+// The log call waits for the publish, so the budget is what a caller can be+// held for while Pub/Sub is unreachable before the log call fails.const PUBSUB_PUBLISH_OPTIONS: PublishOptions = {gaxOpts: {retry: {backoffSettings: {initialRetryDelayMillis: 250,retryDelayMultiplier: 2,- maxRetryDelayMillis: 15_000,- initialRpcTimeoutMillis: 15_000,+ maxRetryDelayMillis: 5_000,+ initialRpcTimeoutMillis: 10_000,rpcTimeoutMultiplier: 1,- maxRpcTimeoutMillis: 15_000,- totalTimeoutMillis: 300_000,+ maxRpcTimeoutMillis: 10_000,+ totalTimeoutMillis: 30_000,},},},@@ -212,6 +215,20 @@ function getTopic(client: PubSub, table: string): Topic {return topic;}+/** The publisher already holds as many unacknowledged rows as it may. */+class PubSubBacklogFullError extends Error {+ constructor() {+ super("Pub/Sub log publisher backlog is full");+ this.name = "PubSubBacklogFullError";+ }+}++/**+ * Publishes one job-log row to its Pub/Sub topic and waits for the+ * acknowledgement. Pub/Sub is the log's store of record, so a publish that+ * fails, is refused by a full backlog, or arrives during shutdown throws to+ * the caller; nothing else is written for that row.+ */async function publishLog(table: string, data: any, logger: Logger) {const startedAt = Date.now();try {@@ -252,7 +269,7 @@ async function publishLog(table: string, data: any, logger: Logger) {const now = Date.now();if (now - lastDropWarningAt >= 60_000) {lastDropWarningAt = now;- logger.warn("Dropping Pub/Sub log: publisher backlog is full", {+ logger.warn("Refusing Pub/Sub log: publisher backlog is full", {table,logId: data.id,payloadBytes: payload.length,@@ -261,7 +278,7 @@ async function publishLog(table: string, data: any, logger: Logger) {droppedTotal,});}- return;+ throw new PubSubBacklogFullError();}const publication = getTopic(client, table).publishMessage({@@ -284,14 +301,18 @@ async function publishLog(table: string, data: any, logger: Logger) {}});} catch (error) {- pubsubLogPublishTotal.inc({ table, outcome: "failed" });-- logger.error("Failed to publish log to Pub/Sub", {- error,- table,- logId: data.id,- durationMs: Date.now() - startedAt,- });+ // A backlog refusal counted itself as "dropped" and warned, rate-limited,+ // above; only a genuine publish failure is an error here.+ if (!(error instanceof PubSubBacklogFullError)) {+ pubsubLogPublishTotal.inc({ table, outcome: "failed" });+ logger.error("Failed to publish log to Pub/Sub", {+ error,+ table,+ logId: data.id,+ durationMs: Date.now() - startedAt,+ });+ }+ throw error;}}@@ -388,37 +409,36 @@ async function robustInsert(canonicalLog: "log_job/robustInsert",});+ if (config.USE_DB_AUTHENTICATION !== true) {+ logger.info(+ "Skipping database insertion due to USE_DB_AUTHENTICATION being off",+ );+ return;+ }++ // The single point where a row leaves for both stores: clean it once so+ // PostgreSQL and ClickHouse receive identical, accepted values.+ data = sanitizeLogData({+ ...data,+ created_at: data.created_at ?? new Date(),+ });++ // Pub/Sub is the store of record: publish first and wait for it. A failed+ // publish fails the log call, and the PostgreSQL copy is not attempted.+ await publishLog(table, data, logger);++ // The PostgreSQL copy is on its way out; its failure is logged, not thrown.const attempts: { error: any; timeMs: number; backoffMs: number }[] = [];try {- const inserted = await withSpan("log_job.postgres.insert", async span => {+ await withSpan("log_job.postgres.insert", async span => {setSpanAttributes(span, {"db.system": "postgresql","log_job.table": table,"log_job.id": data.id,"log_job.force": force,+ "log_job.postgres.enabled": true,});-- if (config.USE_DB_AUTHENTICATION !== true) {- logger.info(- "Skipping database insertion due to USE_DB_AUTHENTICATION being off",- );- setSpanAttributes(span, {- "log_job.postgres.enabled": false,- "log_job.postgres.outcome": "skipped",- });- return false;- }-- setSpanAttributes(span, { "log_job.postgres.enabled": true });const target = tableMap[table];- // The single point where a row leaves for both stores: clean it once so- // PostgreSQL and ClickHouse receive identical, accepted values.- data = sanitizeLogData({- ...data,- created_at: data.created_at ?? new Date(),- });- // Publish in the background. Customer responses must not wait for Pub/Sub.- void publishLog(table, data, logger);const maxAttempts = force ? 10 : 1;for (let i = 0; i < maxAttempts; i++) {@@ -456,10 +476,8 @@ async function robustInsert(lastAttempt?.error ?? new Error("Database insert was not attempted"));}- return true;});- if (!inserted) return;if (attempts.length === 1) {logger.debug("Inserted into database successfully", { attempts });} else {