Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): add Pub/Sub dual-write logging (#4519)
rahul-firecrawle4353e4f4e53
diff --git a/apps/api/package.json b/apps/api/package.jsonindex b580e71c4..a0240f4da 100644--- a/apps/api/package.json+++ b/apps/api/package.json@@ -87,6 +87,7 @@"@clickhouse/client": "^1.8.1","@dqbd/tiktoken": "^1.0.22","@google-cloud/bigtable": "^7.2.0",+ "@google-cloud/pubsub": "^6.0.1","@google-cloud/storage": "^7.21.0","@mendable/firecrawl-rs": "workspace:*","@openrouter/ai-sdk-provider": "^2.9.1",diff --git a/apps/api/pnpm-lock.yaml b/apps/api/pnpm-lock.yamlindex 4e4d23815..4afe58716 100644--- a/apps/api/pnpm-lock.yaml+++ b/apps/api/pnpm-lock.yaml@@ -93,6 +93,9 @@ importers:'@google-cloud/bigtable':specifier: ^7.2.0version: 7.2.0(@opentelemetry/[email protected](@opentelemetry/[email protected]))+ '@google-cloud/pubsub':+ specifier: ^6.0.1+ version: 6.0.1'@google-cloud/storage':specifier: ^7.21.0version: 7.21.0@@ -834,6 +837,10 @@ packages:resolution: {integrity: sha512-DJS3s0OVH4zFDB1PzjxAsHqJT6sKVbRwwML0ZBP9PbU7Yebtu/7SWMRzvO2J3nUi9pRNITCfu4LJeooM2w4pjg==}engines: {node: '>=14.0.0'}+ '@google-cloud/[email protected]':+ resolution: {integrity: sha512-k32cWlHAF8yTgg8rciLI8mPMI6UzuJdKp53YRxISRwMFxUl2FYplvs+Mr2UHxKn0W7rXsqZnUZy73AOJFDP8iA==}+ engines: {node: '>=22'}+'@google-cloud/[email protected]':resolution: {integrity: sha512-1TUx3KdaU3cN7nfCdNf+UVqA/PSX29Cjcox3fZZBtINlRrXVTmUkQnCKv2MbBUbCopbK4olAT1IHl76uZyCiVA==}engines: {node: '>=14.0.0'}@@ -858,6 +865,14 @@ packages:resolution: {integrity: sha512-lpN2AtoQ/iimp7jjm5zJFWbpW7OJc5qWmQdt59CI4ENeoIRIx+yf2ShSR2kanQfjFOshA74eSbimXXuRaSCUVg==}engines: {node: '>=22'}+ '@google-cloud/[email protected]':+ resolution: {integrity: sha512-x1L+lrIHgaMwN5fAhnNVS7rh0BZn9eKmqOBXnOUM3jkilF4GE4ooETS9SYnCtPCVHwQtByDTh5b7KIMUW7XAHQ==}+ engines: {node: '>=22'}++ '@google-cloud/[email protected]':+ resolution: {integrity: sha512-Ms/cE5wyxWoHHj6IUWZO67j7nDhAKa7eAUkUYBJjRie0Yr1V82NclaaIntdqAmmpOgXbIzdDlrIyWXLVMeGSDg==}+ engines: {node: '>=22'}+'@google-cloud/[email protected]':resolution: {integrity: sha512-l+IFTkd+6Y5LoAuXyYCKNAKtw/Ci+rAMqgdTB1jv4iZiLhw0rtq+0qjIRbBizXkNzEFmXiXUW0H7sZQQvk1ffA==}engines: {node: '>=14'}@@ -1543,6 +1558,10 @@ packages:peerDependencies:'@opentelemetry/api': '>=1.3.0 <1.10.0'+ '@opentelemetry/[email protected]':+ resolution: {integrity: sha512-R5R9tb2AXs2IRLNKLBJDynhkfmx7mX0vi8NkhZb3gUkPWHn6HXk5J8iQ/dql0U3ApfWym4kXXmBDRGO+oeOfjg==}+ engines: {node: '>=14'}+'@opentelemetry/[email protected]':resolution: {integrity: sha512-eSYWTm620tTk45EKSedaUL8MFYI8hW164hIXsgIHyxu3VobUB3fFCu5t0hQby6OoWRPsG1KkKUG2M5UadiLiVg==}engines: {node: '>=14'}@@ -3200,6 +3219,10 @@ packages:resolution: {integrity: sha512-F/1DnUGPopORZi0ni+CvrCgHQ5FyEAHRLSApuYWMmrbSwoN2Mn/7k+Gl38gJnR7yyDZk6WLXwiGod1JOWNDKGw==}hasBin: true+ resolution: {integrity: sha512-EQfezRg0NCZGNlhlDR3Evrw1FVL2G3LhU7EgPoxufQKruNBSYA8MiRPHeWbU+36o+Fhel0wMwM+sLEiBAlNLJA==}+ engines: {node: '>=10.0.0'}+resolution: {integrity: sha512-CV9TW3Y3f8/wT0BRFc1/KAVQ3TUHiXmaAb6VW9vtiMFf7SLoMd1PdAc4W3KFOFETBJUb90KatHqlsZMWV+R9Gg==}engines: {node: ^20.19.0 || ^22.12.0 || >=24.0.0}@@ -3327,6 +3350,9 @@ packages:resolution: {integrity: sha512-hvpoI6korhJMnej285dSg6nu1+e6uxs7zG3BYAm5byqDsgJNWwxzM6z6iZiAgQR4TJ30JmBTOwqZUw3WlyH3AQ==}+ resolution: {integrity: sha512-xj0XPvmr7bQFTvirqnFr50o0hQIh6ZItDqloxt5aJrR4NQsYeSsyFQERYGCAzfindAcnKjINnwEEgLx4IqVzQw==}+resolution: {integrity: sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg==}engines: {node: '>=8'}@@ -3837,6 +3863,10 @@ packages:oxlint-tsgolint:optional: true+ resolution: {integrity: sha512-ugZxsxmtTln604yeYd29EGrNhazN2lywetzpKhfmQjW/VJmhpDmWbiX+h0zL8V91R0UXkhb3KtPmyq9PZw3aYw==}+ engines: {node: '>=8'}+resolution: {integrity: sha512-LICb2p9CB7FS+0eR1oqWnHhp0FljGLZCWBE9aix0Uye9W8LTQPwMTYVGWQWIw9RdQiDg4+epXQODwIYJtSJaow==}engines: {node: '>=4'}@@ -5253,6 +5283,10 @@ snapshots:arrify: 2.0.1extend: 3.0.2+ '@google-cloud/[email protected]':+ dependencies:+ extend: 3.0.2+'@google-cloud/[email protected]': {}'@google-cloud/[email protected]': {}@@ -5265,6 +5299,35 @@ snapshots:'@google-cloud/[email protected]': {}+ '@google-cloud/[email protected]':+ dependencies:+ google-gax: 6.1.0+ transitivePeerDependencies:+ - supports-color++ '@google-cloud/[email protected]':+ dependencies:+ '@google-cloud/paginator': 7.0.1+ '@google-cloud/precise-date': 6.0.1+ '@google-cloud/projectify': 6.0.1+ '@google-cloud/promisify': 6.0.1+ '@google-cloud/pubsub-api': 0.3.0+ '@opentelemetry/api': 1.9.1+ '@opentelemetry/core': 2.9.0(@opentelemetry/[email protected])+ '@opentelemetry/semantic-conventions': 1.39.0+ arrify: 2.0.1+ extend: 3.0.2+ google-auth-library: 11.0.2+ google-gax: 6.1.0+ google-logging-utils: 2.0.1+ heap-js: 2.7.1+ is-stream-ended: 0.1.4+ lodash.snakecase: 4.1.1+ long: 5.3.2+ p-defer: 3.0.0+ transitivePeerDependencies:+ - supports-color+'@google-cloud/[email protected]':dependencies:'@google-cloud/paginator': 5.0.2@@ -5867,6 +5930,8 @@ snapshots:'@opentelemetry/resources': 2.9.0(@opentelemetry/[email protected])'@opentelemetry/semantic-conventions': 1.43.0+ '@opentelemetry/[email protected]': {}+'@opentelemetry/[email protected]': {}'@oxc-project/[email protected]': {}@@ -7361,6 +7426,8 @@ snapshots:+ [email protected]: {}+[email protected](@noble/[email protected]):dependencies:'@exodus/bytes': 1.15.1(@noble/[email protected])@@ -7510,6 +7577,8 @@ snapshots:+ [email protected]: {}+@@ -7977,6 +8046,8 @@ snapshots:'@oxlint/win32-arm64': 1.14.0'@oxlint/win32-x64': 1.14.0+ [email protected]: {}+diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 72cb20029..a42870fb8 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -239,6 +239,9 @@ const configSchema = z.object({PARSE_UPLOAD_REF_SECRET: emptyStringAsUndefined(z.string().trim().min(1)),PARSE_UPLOAD_PUBLIC_BASE_URL: z.string().url().optional(),+ // Google Cloud Pub/Sub+ PUBSUB_CREDENTIALS: z.string().optional(),+// Cloud Bigtable (change tracking bookkeeping store). The client// auto-detects BIGTABLE_EMULATOR_HOST, so local dev only needs the// emulator plus these vars. BIGTABLE_CREDENTIALS mirrorsdiff --git a/apps/api/src/index.ts b/apps/api/src/index.tsindex cdc08bea3..4c8af690c 100644--- a/apps/api/src/index.ts+++ b/apps/api/src/index.ts@@ -44,6 +44,7 @@ import responseTime from "response-time";import { shutdownWebhookQueue } from "./services/webhook";import { shutdownIndexerQueue } from "./services/indexing/indexer-queue";import { isKeylessConfigured } from "./lib/keyless";+import { shutdownPubSubLogging } from "./services/logging/log_job";const { createBullBoard } = require("@bull-board/api");const { BullMQAdapter } = require("@bull-board/api/bullMQAdapter");@@ -183,7 +184,8 @@ async function startServer(port = DEFAULT_PORT) {logger.info("Server closed.");nuqShutdown().finally(() => {shutdownWebhookQueue().finally(() => {- shutdownIndexerQueue().finally(() => {+ shutdownIndexerQueue().finally(async () => {+ await shutdownPubSubLogging();logger.info("NUQ shutdown complete");process.exit(0);});diff --git a/apps/api/src/services/extract-worker.ts b/apps/api/src/services/extract-worker.tsindex 037eb19a0..15e07efb4 100644--- a/apps/api/src/services/extract-worker.ts+++ b/apps/api/src/services/extract-worker.ts@@ -20,7 +20,7 @@ import {shutdownExtractQueue,ExtractJobData,} from "./extract-queue";-import { logExtract } from "./logging/log_job";+import { logExtract, shutdownPubSubLogging } from "./logging/log_job";import { jobDurationSeconds } from "../lib/job-metrics";import { register } from "prom-client";@@ -215,6 +215,7 @@ app.listen(workerPort, (error?: Error) => {async function shutdown() {_logger.info("Shutting down extract worker...");await shutdownExtractQueue();+ await shutdownPubSubLogging();_logger.info("Extract worker shut down");process.exit(0);}diff --git a/apps/api/src/services/indexing/index-worker.ts b/apps/api/src/services/indexing/index-worker.tsindex 5e74be8be..2de767688 100644--- a/apps/api/src/services/indexing/index-worker.ts+++ b/apps/api/src/services/indexing/index-worker.ts@@ -41,7 +41,7 @@ import { withSpan, setSpanAttributes } from "../../lib/otel-tracer";import { crawlGroup, resolveNewGroupBackend } from "../worker/nuq-router";import { getACUCTeam } from "../../controllers/auth";import { processEngpickerJob } from "../../lib/engpicker";-import { logRequest } from "../logging/log_job";+import { logRequest, shutdownPubSubLogging } from "../logging/log_job";import { startSiemLoggingConsumer } from "../siem-logging/worker";import { closeSiemLoggingTransport } from "../../lib/siem-logging/transport";@@ -686,6 +686,7 @@ const workerFun = async (await new Promise(resolve => setTimeout(resolve, 500));}logger.info("All jobs finished. Worker exiting!");+ await shutdownPubSubLogging();process.exit(0);};diff --git a/apps/api/src/services/logging/log_job.test.ts b/apps/api/src/services/logging/log_job.test.tsindex cb70ecdfd..d553ccd2a 100644--- a/apps/api/src/services/logging/log_job.test.ts+++ b/apps/api/src/services/logging/log_job.test.ts@@ -3,7 +3,17 @@ import { vi } from "vitest";// vi.mock is hoisted; anything its factories reference must be created in// vi.hoisted() (also hoisted). Under Jest these worked because importing `jest`// from @jest/globals disables jest.mock hoisting.-const { captureException, logger, values, insert } = vi.hoisted(() => {+const {+ captureException,+ logger,+ values,+ insert,+ topic,+ publishes,+ publishMessage,+ flush,+ close,+} = vi.hoisted(() => {const logger: any = {info: vi.fn(),warn: vi.fn(),@@ -13,9 +23,39 @@ const { captureException, logger, values, insert } = vi.hoisted(() => {};const values = vi.fn<(data: any) => Promise<void>>();const insert = vi.fn(() => ({ values }));- return { captureException: vi.fn(), logger, values, insert };+ const publishMessage = vi.fn(async (_message: any) => "message-id");+ const flush = vi.fn(async () => {});+ const close = vi.fn(async () => {});+ const publishes: { name: string; options: any }[] = [];+ const topic = vi.fn((name: string, options: any) => {+ return {+ publishMessage: (message: any) => {+ publishes.push({ name, options });+ return publishMessage(message);+ },+ flush,+ };+ });+ return {+ captureException: vi.fn(),+ logger,+ values,+ insert,+ topic,+ publishes,+ publishMessage,+ flush,+ close,+ };});+vi.mock("@google-cloud/pubsub", () => ({+ PubSub: class {+ topic = topic;+ close = close;+ },+}));+vi.mock("@sentry/node", () => ({captureException,}));@@ -23,6 +63,9 @@ vi.mock("@sentry/node", () => ({vi.mock("../../config", () => ({config: {GCS_BUCKET_NAME: undefined,+ PUBSUB_CREDENTIALS: Buffer.from(+ JSON.stringify({ project_id: "firecrawl" }),+ ).toString("base64"),USE_DB_AUTHENTICATION: true,},}));@@ -52,7 +95,12 @@ vi.mock("../posthog", () => ({trackFirstSurfaceUse: vi.fn(),}));-import { logRequest, logSearch, type LoggedSearch } from "./log_job";+import {+ logRequest,+ logSearch,+ shutdownPubSubLogging,+ type LoggedSearch,+} from "./log_job";import * as schema from "../../db/schema";function makeSearch(overrides: Partial<LoggedSearch> = {}): LoggedSearch {@@ -79,6 +127,8 @@ describe("logSearch", () => {beforeEach(() => {vi.clearAllMocks();values.mockResolvedValue(undefined);+ publishMessage.mockResolvedValue("message-id");+ publishes.length = 0;});it("removes null bytes from search query log fields", async () => {@@ -127,6 +177,8 @@ describe("logRequest", () => {beforeEach(() => {vi.clearAllMocks();values.mockResolvedValue(undefined);+ publishMessage.mockResolvedValue("message-id");+ publishes.length = 0;});function makeRequest(externalRequestId: string | null) {@@ -147,7 +199,51 @@ describe("logRequest", () => {await logRequest(makeRequest("op_integration_42"));expect(insert).toHaveBeenCalledWith(schema.requests);- expect(values.mock.calls[0][0].external_request_id).toBe("op_integration_42");+ expect(values.mock.calls[0][0].external_request_id).toBe(+ "op_integration_42",+ );+ });++ it("writes the request to the database and its Pub/Sub topic", async () => {+ await logRequest(makeRequest("op_integration_42"));++ expect(insert).toHaveBeenCalledWith(schema.requests);+ expect(publishes).toContainEqual({+ name: "requests",+ options: { gaxOpts: { timeout: 60_000 } },+ });++ const published = JSON.parse(+ publishMessage.mock.calls[0][0].data.toString("utf8"),+ );+ expect(published.id).toBe("019e6f45-7778-727d-adf0-0abe9d5062b6");+ expect(published.external_request_id).toBe("op_integration_42");+ expect(new Date(published.created_at).toISOString()).toBe(+ published.created_at,+ );+ expect(values.mock.calls[0][0].created_at.toISOString()).toBe(+ published.created_at,+ );+ });++ it("keeps the database write when Pub/Sub fails", async () => {+ publishMessage.mockRejectedValueOnce(new Error("Pub/Sub unavailable"));++ await expect(logRequest(makeRequest(null))).resolves.toBeUndefined();++ expect(values).toHaveBeenCalled();+ expect(logger.error).toHaveBeenCalledWith(+ "Failed to publish log to Pub/Sub",+ expect.objectContaining({ error: expect.any(Error) }),+ );+ expect(captureException).toHaveBeenCalled();+ });++ it("does not hold the caller on a slow publish", async () => {+ publishMessage.mockReturnValueOnce(new Promise(() => {}));++ await expect(logRequest(makeRequest(null))).resolves.toBeUndefined();+ expect(values).toHaveBeenCalled();});it("stores null, not a truncation, when the id exceeds the byte cap", async () => {@@ -173,4 +269,13 @@ describe("logRequest", () => {await logRequest(makeRequest("é".repeat(1024)));expect(values.mock.calls[1][0].external_request_id).toBe("é".repeat(1024));});++ it("flushes Pub/Sub messages during shutdown", async () => {+ await logRequest(makeRequest(null));++ await Promise.all([shutdownPubSubLogging(), shutdownPubSubLogging()]);++ expect(flush).toHaveBeenCalled();+ expect(close).toHaveBeenCalledOnce();+ });});diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex f81f9011a..8d3169e2f 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -23,6 +23,7 @@ import type { CostTracking } from "../../lib/cost-tracking";import type { Logger } from "winston";import { saveExtractResult } from "../../lib/extract/extract-redis";import { trackFirstSurfaceUse } from "../posthog";+import { PubSub, type Topic } from "@google-cloud/pubsub";configDotenv();const previewTeamId = "3adefd26-77ec-5968-8dcf-c94b5630d1de";@@ -58,6 +59,101 @@ const tableMap: Record<string, PgTable> = {deep_researches: schema.deep_researches,};+let pubSubClient: PubSub | null | undefined;+const pubSubTopics = new Map<string, Topic>();+let pubSubShutdown: Promise<void> | undefined;+const PUBSUB_PUBLISH_TIMEOUT_MS = 60_000;++function getPubSubClient(logger: Logger): PubSub | null {+ if (pubSubClient !== undefined) return pubSubClient;+ if (!config.PUBSUB_CREDENTIALS) return (pubSubClient = null);++ try {+ const credentials = JSON.parse(+ Buffer.from(config.PUBSUB_CREDENTIALS, "base64").toString("utf8"),+ );+ return (pubSubClient = new PubSub({+ projectId: credentials.project_id,+ credentials,+ }));+ } catch (error) {+ pubSubClient = null;+ logger.error("Failed to initialize Pub/Sub log publisher", { error });+ Sentry.captureException(error, {+ tags: { operation: "initializePubSubLogPublisher" },+ });+ return null;+ }+}++// One Topic per table so publishes share a batch.+function getTopic(client: PubSub, table: string): Topic {+ let topic = pubSubTopics.get(table);+ if (!topic) {+ topic = client.topic(table, {+ gaxOpts: { timeout: PUBSUB_PUBLISH_TIMEOUT_MS },+ });+ pubSubTopics.set(table, topic);+ }+ return topic;+}++async function publishLog(table: string, data: any, logger: Logger) {+ const client = getPubSubClient(logger);+ if (!client) return;++ try {+ await getTopic(client, table).publishMessage({+ data: Buffer.from(JSON.stringify(data)),+ });+ } catch (error) {+ logger.error("Failed to publish log to Pub/Sub", { error });+ Sentry.captureException(error, {+ tags: { table, operation: "publishPubSubLog" },+ });+ }+}++export function shutdownPubSubLogging(): Promise<void> {+ if (pubSubShutdown) return pubSubShutdown;++ pubSubShutdown = shutdownPubSubLoggingOnce();+ return pubSubShutdown;+}++async function shutdownPubSubLoggingOnce(): Promise<void> {+ const client = pubSubClient;+ if (!client) return;++ const logger = _logger.child({+ module: "log_job",+ method: "shutdownPubSubLogging",+ });+ const results = await Promise.allSettled(+ [...pubSubTopics.values()].map(topic => topic.flush()),+ );+ const errors = results.flatMap(result =>+ result.status === "rejected" ? [result.reason] : [],+ );++ if (errors.length > 0) {+ logger.error("Failed to flush Pub/Sub log publisher", { errors });+ Sentry.captureException(errors[0], {+ tags: { operation: "flushPubSubLogPublisher" },+ extra: { failures: errors.length },+ });+ }++ try {+ await client.close();+ } catch (error) {+ logger.error("Failed to close Pub/Sub log publisher", { error });+ Sentry.captureException(error, {+ tags: { operation: "closePubSubLogPublisher" },+ });+ }+}+async function robustInsert(table: string,data: any,@@ -79,6 +175,8 @@ async function robustInsert(}const target = tableMap[table];+ data = { ...data, created_at: data.created_at ?? new Date() };+ void publishLog(table, data, logger);const attempts: { error: any; timeMs: number; backoffMs: number }[] = [];diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.tsindex feeb46d93..ffe3085c0 100644--- a/apps/api/src/services/queue-worker.ts+++ b/apps/api/src/services/queue-worker.ts@@ -34,6 +34,7 @@ import {consumeMonitorCheckJobs,consumeMonitorSearchCheckJobs,} from "./monitoring/queue";+import { shutdownPubSubLogging } from "./logging/log_job";configDotenv();@@ -512,5 +513,6 @@ app.listen(workerPort, (error?: Error) => {}_logger.info("All jobs finished. Shutting down...");+ await shutdownPubSubLogging();process.exit(0);})();diff --git a/apps/api/src/services/worker/nuq-worker-runner.ts b/apps/api/src/services/worker/nuq-worker-runner.tsindex 63dfd6a91..bbffc593f 100644--- a/apps/api/src/services/worker/nuq-worker-runner.ts+++ b/apps/api/src/services/worker/nuq-worker-runner.ts@@ -7,6 +7,7 @@ import { register } from "prom-client";import Express from "express";import { initializeBlocklist } from "../../scraper/WebScraper/utils/blocklist";import { initializeEngineForcing } from "../../scraper/WebScraper/utils/engine-forcing";+import { shutdownPubSubLogging } from "../logging/log_job";export type WorkerQueue = {getJobToProcess(logger?: any): Promise<NuQJob<any, any> | null>;@@ -204,6 +205,7 @@ export async function runNuqWorker(options: {server.close(async () => {await options.beforeShutdown?.();await options.shutdown?.();+ await shutdownPubSubLogging();_logger.info("NuQ worker shut down", { module: options.serviceName });process.exit(0);});