Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): add PUBSUB_TOPIC_PREFIX for environment-scoped log topics (#4684)
Gergő Móricz2a71f0190a02
diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 44b9f4304..8d3d9436a 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -251,6 +251,11 @@ const configSchema = z.object({// Google Cloud Pub/SubPUBSUB_CREDENTIALS: z.string().optional(),+ // Prepended to every log topic name. Production leaves it unset and+ // publishes to `<table>`; staging sets `staging-` so its rows land in the+ // `staging-<table>` topics and the staging ClickHouse database instead of+ // the production tables.+ PUBSUB_TOPIC_PREFIX: z.string().default(""),// Publisher backlog cap, per process. Log publishing is fire-and-forget and// retries for up to five minutes, so during a stall the backlog is what// grows; rows beyond the cap are dropped and counted rather than letting adiff --git a/apps/api/src/services/logging/log_job.test.ts b/apps/api/src/services/logging/log_job.test.tsindex 61db30cbb..66e98631e 100644--- a/apps/api/src/services/logging/log_job.test.ts+++ b/apps/api/src/services/logging/log_job.test.ts@@ -90,6 +90,7 @@ vi.mock("../../config", () => ({USE_DB_AUTHENTICATION: true,PUBSUB_MAX_OUTSTANDING_MESSAGES: 10_000,PUBSUB_MAX_OUTSTANDING_BYTES: 64 * 1024 * 1024,+ PUBSUB_TOPIC_PREFIX: "",},}));@@ -401,6 +402,27 @@ describe("logRequest", () => {);});+ it("prefixes the Pub/Sub topic name with PUBSUB_TOPIC_PREFIX", async () => {+ config.PUBSUB_TOPIC_PREFIX = "staging-";+ try {+ await logRequest(makeRequest("op_integration_42"));+ } finally {+ config.PUBSUB_TOPIC_PREFIX = "";+ }++ expect(publishes[0].name).toBe("staging-requests");+ // The metric and span keep the bare table name.+ await new Promise(resolve => setImmediate(resolve));+ expect(metricInc).toHaveBeenCalledWith({+ table: "requests",+ outcome: "published",+ });+ expect(setSpanAttributes).toHaveBeenCalledWith(+ expect.anything(),+ expect.objectContaining({ "log_job.table": "requests" }),+ );+ });+it("writes the request to the database and its Pub/Sub topic", async () => {await logRequest(makeRequest("op_integration_42"));diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex adcd5718a..15806a383 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -200,12 +200,14 @@ function getPubSubClient(logger: Logger): PubSub | null {}}-// One Topic per table so publishes share a batch.+// One Topic per table so publishes share a batch. The topic is named after+// the table, behind the optional environment prefix (see PUBSUB_TOPIC_PREFIX).function getTopic(client: PubSub, table: string): Topic {- let topic = pubSubTopics.get(table);+ const name = `${config.PUBSUB_TOPIC_PREFIX}${table}`;+ let topic = pubSubTopics.get(name);if (!topic) {- topic = client.topic(table, PUBSUB_PUBLISH_OPTIONS);- pubSubTopics.set(table, topic);+ topic = client.topic(name, PUBSUB_PUBLISH_OPTIONS);+ pubSubTopics.set(name, topic);}return topic;}