Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): count and log PostgreSQL fallback hits on job-store reads (#4685)
Gergő Móriczfd8f1dda3e62
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex 3d9ff5b19..88662a5b1 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -30,6 +30,7 @@ import {import { ScrapeJobSingleUrls } from "../../types";import { readScrapeJobState } from "../../lib/job-state-store";import { readRequestCredits } from "../../lib/request-credits-store";+import { recordJobStorePostgresFallback } from "../../lib/job-store-fallback";configDotenv();export type PseudoJob<T> = {@@ -54,6 +55,7 @@ type DBScrape = {};export async function getJob(id: string): Promise<PseudoJob<any> | null> {+ let scrapeStateFailed = false;const [nuqJob, scrapeState, dbScrape, gcsJob] = await Promise.all([scrapeQueue.getJob(id) as Promise<NuQJob<ScrapeJobSingleUrls> | null>,readScrapeJobState(id).catch(error => {@@ -61,6 +63,7 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {error,scrapeId: id,});+ scrapeStateFailed = true;return null;}),(config.USE_DB_AUTHENTICATION@@ -70,6 +73,9 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {]);if (!nuqJob && !scrapeState && !dbScrape) return null;+ if (!scrapeState && !scrapeStateFailed && dbScrape) {+ recordJobStorePostgresFallback("scrape_state", id);+ }if (nuqJob && nuqJob.data.mode !== "single_urls") {return null;@@ -194,13 +200,18 @@ export async function crawlStatusController(logger.child({ zeroDataRetention }),);- let creditsBilled = await readRequestCredits(req.params.jobId).catch(- () => null,- );+ let creditsReadFailed = false;+ let creditsBilled = await readRequestCredits(req.params.jobId).catch(() => {+ creditsReadFailed = true;+ return null;+ });if (creditsBilled === null && config.USE_DB_AUTHENTICATION) {creditsBilled = await creditsBilledByCrawlId(dbRr, req.params.jobId).then(rows => rows[0]?.credits_billed ?? null).catch(() => null);+ if (creditsBilled !== null && !creditsReadFailed) {+ recordJobStorePostgresFallback("request_credits", req.params.jobId);+ }}// check if the crawl failed during kickoff (e.g. queue full)diff --git a/apps/api/src/controllers/v1/extract-status.ts b/apps/api/src/controllers/v1/extract-status.tsindex e19f68e52..bf5bbf6ca 100644--- a/apps/api/src/controllers/v1/extract-status.ts+++ b/apps/api/src/controllers/v1/extract-status.ts@@ -12,6 +12,7 @@ import { getJobFromGCS } from "../../lib/gcs-jobs";import { getExtractJobAccess } from "../../lib/operational-job-access";import { readExtractJobState } from "../../lib/job-state-store";import { normalizeJobAccessTeamId } from "../../lib/job-access-store";+import { recordJobStorePostgresFallback } from "../../lib/job-store-fallback";async function getExtractData(id: string): Promise<any> {// Try GCS first if configured@@ -69,10 +70,12 @@ export async function extractStatusController(// If not in Redis, check the database for completed jobsif (!extract) {if (config.USE_DB_AUTHENTICATION) {+ let stateReadFailed = false;const state = await readExtractJobState(req.params.jobId).catch(error => {logger.warn("Bigtable extract state read failed; using legacy lookup", {error,});+ stateReadFailed = true;return null;});if (state) {@@ -97,6 +100,9 @@ export async function extractStatusController(error: "Extract job not found",});}+ if (!stateReadFailed) {+ recordJobStorePostgresFallback("extract_state", req.params.jobId);+ }if (dbExtract.team_id !== req.auth.team_id) {return res.status(404).json({diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.tsindex e6eb12053..4aceae22d 100644--- a/apps/api/src/controllers/v2/crawl-status.ts+++ b/apps/api/src/controllers/v2/crawl-status.ts@@ -33,6 +33,7 @@ import { redisEvictConnection } from "../../../src/services/redis";import { isBaseDomain, extractBaseDomain } from "../../lib/url-utils";import { readScrapeJobState } from "../../lib/job-state-store";import { readRequestCredits } from "../../lib/request-credits-store";+import { recordJobStorePostgresFallback } from "../../lib/job-store-fallback";configDotenv();export type PseudoJob<T> = {@@ -60,6 +61,7 @@ export async function getJob(id: string,_logger = logger,): Promise<PseudoJob<any> | null> {+ let scrapeStateFailed = false;const [nuqJob, scrapeState, dbScrape, gcsJob] = await Promise.all([scrapeQueue.getJob(id,@@ -70,6 +72,7 @@ export async function getJob(error,scrapeId: id,});+ scrapeStateFailed = true;return null;}),(config.USE_DB_AUTHENTICATION@@ -79,6 +82,9 @@ export async function getJob(]);if (!nuqJob && !scrapeState && !dbScrape) return null;+ if (!scrapeState && !scrapeStateFailed && dbScrape) {+ recordJobStorePostgresFallback("scrape_state", id);+ }if (nuqJob && nuqJob.data.mode !== "single_urls") {return null;@@ -215,13 +221,20 @@ export async function crawlStatusController(logger.child({ zeroDataRetention }),);+ let creditsReadFailed = false;let creditsBilled = await readRequestCredits(sc?.requestId ?? req.params.jobId,- ).catch(() => null);+ ).catch(() => {+ creditsReadFailed = true;+ return null;+ });if (creditsBilled === null && config.USE_DB_AUTHENTICATION) {creditsBilled = await creditsBilledByCrawlId(dbRr, req.params.jobId).then(rows => rows[0]?.credits_billed ?? null).catch(() => null);+ if (creditsBilled !== null && !creditsReadFailed) {+ recordJobStorePostgresFallback("request_credits", req.params.jobId);+ }}// check if the crawl failed during kickoff (e.g. queue full)diff --git a/apps/api/src/controllers/v2/extract-status.ts b/apps/api/src/controllers/v2/extract-status.tsindex 89ed86563..3405f638d 100644--- a/apps/api/src/controllers/v2/extract-status.ts+++ b/apps/api/src/controllers/v2/extract-status.ts@@ -13,6 +13,7 @@ import { getExtractJobAccess } from "../../lib/operational-job-access";import { readExtractJobState } from "../../lib/job-state-store";import { normalizeJobAccessTeamId } from "../../lib/job-access-store";import { getExtractV3AgentStatus } from "../../lib/extract-v3-status";+import { recordJobStorePostgresFallback } from "../../lib/job-store-fallback";async function getExtractData(id: string): Promise<any> {// Try GCS first if configured@@ -69,11 +70,13 @@ export async function extractStatusController(// If not in Redis, check the database for completed jobsif (!redisExtract) {if (config.USE_DB_AUTHENTICATION) {+ let stateReadFailed = false;const state = await readExtractJobState(req.params.jobId).catch(error => {_logger.warn("Bigtable extract state read failed; using legacy lookup",{ error, extractId: req.params.jobId },);+ stateReadFailed = true;return null;});if (state) {@@ -92,6 +95,9 @@ export async function extractStatusController(const dbExtract = await supabaseGetExtractByIdDirect(req.params.jobId);if (dbExtract) {+ if (!stateReadFailed) {+ recordJobStorePostgresFallback("extract_state", req.params.jobId);+ }// Get result datalet data: any = [];if (dbExtract.is_successful) {diff --git a/apps/api/src/controllers/v2/feedback/feedback-store.ts b/apps/api/src/controllers/v2/feedback/feedback-store.tsindex 5f113ddff..357400eb7 100644--- a/apps/api/src/controllers/v2/feedback/feedback-store.ts+++ b/apps/api/src/controllers/v2/feedback/feedback-store.ts@@ -13,6 +13,7 @@ import {FeedbackRecordOptions,RefundPolicySnapshot,} from "./internal-types";+import { recordJobStorePostgresFallback } from "../../../lib/job-store-fallback";type DbError = { code?: string } & Record<string, unknown>;@@ -45,6 +46,7 @@ export async function lookupFeedbackJob(jobId: string,dbTeamId: string,): Promise<FeedbackJobRow | null> {+ let bigtableFailed = false;try {const job = await readFeedbackJob(jobId);if (job) {@@ -72,6 +74,7 @@ export async function lookupFeedbackJob(};}} catch (error) {+ bigtableFailed = true;logger.warn("Bigtable feedback job read failed; falling back to PostgreSQL",{@@ -98,6 +101,9 @@ export async function lookupFeedbackJob(.limit(1);if (!row) return null;+ if (!bigtableFailed) {+ recordJobStorePostgresFallback("feedback_job", jobId, { endpoint });+ }return {endpoint,diff --git a/apps/api/src/controllers/v2/scrape-browser.ts b/apps/api/src/controllers/v2/scrape-browser.tsindex 5eb7a496d..5096c4087 100644--- a/apps/api/src/controllers/v2/scrape-browser.ts+++ b/apps/api/src/controllers/v2/scrape-browser.ts@@ -72,6 +72,7 @@ import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";import { getScrapeJobAccess } from "../../lib/operational-job-access";import { readScrapeJobState } from "../../lib/job-state-store";import { scrapeQueue } from "../../services/worker/nuq-router";+import { recordJobStorePostgresFallback } from "../../lib/job-store-fallback";// ---------------------------------------------------------------------------// Schemas@@ -170,12 +171,14 @@ export async function scrapeInteractController(});}+ let stateReadFailed = false;const [nuqJob, state] = await Promise.all([scrapeQueue.getJob(scrapeId, logger),readScrapeJobState(scrapeId).catch(error => {logger.warn("Bigtable scrape state read failed; using legacy lookup", {error,});+ stateReadFailed = true;return null;}),]);@@ -217,6 +220,11 @@ export async function scrapeInteractController(legacyScrape = (await supabaseGetScrapeByIdDirect(scrapeId,)) as ScrapeContextRow | null;+ if (legacyScrape && !stateReadFailed) {+ recordJobStorePostgresFallback("scrape_state", scrapeId, {+ reason: "replay_context",+ });+ }const replay = legacyScrape? buildReplayContextFromScrape(legacyScrape): { error: "Replay context is unavailable for this scrape job." };diff --git a/apps/api/src/lib/change-tracking-store.ts b/apps/api/src/lib/change-tracking-store.tsindex f4b770a73..a46138b20 100644--- a/apps/api/src/lib/change-tracking-store.ts+++ b/apps/api/src/lib/change-tracking-store.ts@@ -3,6 +3,7 @@ import { config } from "../config";import { diffGetLastScrape } from "../db/rpc";import { getBigtableTable } from "./bigtable-client";import { setSpanAttributes, withSpan } from "./otel-tracer";+import { recordJobStorePostgresFallback } from "./job-store-fallback";// Change tracking bookkeeping. Final architecture (#4484 was the// transition): Bigtable is the store. Writes go to Bigtable only --@@ -158,6 +159,9 @@ export async function changeTrackingGetLastScrape(params: {);const pgRow = pgRows[0];if (!pgRow) return null;+ recordJobStorePostgresFallback("change_tracking", pgRow.o_job_id, {+ teamId: params.team_id,+ });return {job_id: pgRow.o_job_id,date_added: new Date(pgRow.o_date_added).toISOString(),diff --git a/apps/api/src/lib/job-store-fallback.test.ts b/apps/api/src/lib/job-store-fallback.test.tsnew file mode 100644index 000000000..0cc7fe750--- /dev/null+++ b/apps/api/src/lib/job-store-fallback.test.ts@@ -0,0 +1,80 @@+import { beforeEach, describe, expect, it, vi } from "vitest";++const { info, mutableConfig } = vi.hoisted(() => ({+ info: vi.fn(),+ mutableConfig: {+ BIGTABLE_JOB_ACCESS_TABLE: "api_job_access" as string | undefined,+ BIGTABLE_SCRAPE_STATE_TABLE: "scrape_state" as string | undefined,+ BIGTABLE_EXTRACT_STATE_TABLE: "extract_state" as string | undefined,+ BIGTABLE_FEEDBACK_JOBS_TABLE: "feedback_jobs" as string | undefined,+ BIGTABLE_REQUEST_CREDITS_TABLE: "request_credits" as string | undefined,+ BIGTABLE_CHANGE_TRACKING_TABLE: "change_tracking" as string | undefined,+ },+}));+vi.mock("./logger", () => ({ logger: { info } }));+vi.mock("../config", () => ({ config: mutableConfig }));++import {+ jobStorePostgresFallbackTotal,+ recordJobStorePostgresFallback,+} from "./job-store-fallback";++async function countsByStore() {+ const metric = await jobStorePostgresFallbackTotal.get();+ return Object.fromEntries(metric.values.map(v => [v.labels.store, v.value]));+}++describe("job store fallback accounting", () => {+ beforeEach(() => {+ jobStorePostgresFallbackTotal.reset();+ info.mockClear();+ mutableConfig.BIGTABLE_SCRAPE_STATE_TABLE = "scrape_state";+ });++ it("counts and logs one hit per store", async () => {+ recordJobStorePostgresFallback("scrape_state", "job-1");+ recordJobStorePostgresFallback("scrape_state", "job-2");+ recordJobStorePostgresFallback("job_access", "job-3", { kind: "crawl" });++ expect(await countsByStore()).toMatchObject({+ scrape_state: 2,+ job_access: 1,+ });++ expect(info).toHaveBeenCalledTimes(3);+ expect(info).toHaveBeenLastCalledWith(+ "PostgreSQL fallback served a job-store read",+ expect.objectContaining({+ store: "job_access",+ id: "job-3",+ kind: "crawl",+ }),+ );+ });++ it("lets the fixed log fields win over extra context", () => {+ recordJobStorePostgresFallback("scrape_state", "job-1", {+ id: "spoofed",+ store: "spoofed",+ module: "spoofed",+ });++ expect(info).toHaveBeenCalledWith(+ "PostgreSQL fallback served a job-store read",+ expect.objectContaining({+ module: "job-store-fallback",+ store: "scrape_state",+ id: "job-1",+ }),+ );+ });++ it("records nothing when the store is not configured", async () => {+ mutableConfig.BIGTABLE_SCRAPE_STATE_TABLE = undefined;++ recordJobStorePostgresFallback("scrape_state", "job-1");++ expect(await countsByStore()).toEqual({});+ expect(info).not.toHaveBeenCalled();+ });+});diff --git a/apps/api/src/lib/job-store-fallback.ts b/apps/api/src/lib/job-store-fallback.tsnew file mode 100644index 000000000..b7cf09c89--- /dev/null+++ b/apps/api/src/lib/job-store-fallback.ts@@ -0,0 +1,65 @@+import { Counter, register } from "prom-client";+import { config } from "../config";+import { logger } from "./logger";++const NAME = "job_store_postgres_fallback_total";++/**+ * Reads the new job stores (Bigtable, NuQ) could not answer and PostgreSQL+ * did. Each label is one fallback path; a store is ready to lose its+ * fallback once its series has stayed flat for the agreed window.+ *+ * Looked up before creation so a re-evaluated module (test isolation) does+ * not register the same series twice in the shared default registry.+ */+export const jobStorePostgresFallbackTotal =+ (register.getSingleMetric(NAME) as Counter<"store"> | undefined) ??+ new Counter({+ name: NAME,+ help: "Job-store reads answered by the PostgreSQL fallback, by store",+ labelNames: ["store"] as const,+ });++type JobStoreFallback =+ | "job_access"+ | "scrape_state"+ | "extract_state"+ | "feedback_job"+ | "request_credits"+ | "change_tracking";++/**+ * The Bigtable table behind each store. When it is not configured the+ * primary store never answers, PostgreSQL is the only store, and a hit is+ * not a fallback worth counting.+ */+const STORE_TABLE: Record<JobStoreFallback, () => string | undefined> = {+ job_access: () => config.BIGTABLE_JOB_ACCESS_TABLE,+ scrape_state: () => config.BIGTABLE_SCRAPE_STATE_TABLE,+ extract_state: () => config.BIGTABLE_EXTRACT_STATE_TABLE,+ feedback_job: () => config.BIGTABLE_FEEDBACK_JOBS_TABLE,+ request_credits: () => config.BIGTABLE_REQUEST_CREDITS_TABLE,+ change_tracking: () => config.BIGTABLE_CHANGE_TRACKING_TABLE,+};++/**+ * Record that PostgreSQL served a row the primary store did not have. Call+ * it only on a genuine miss: a miss on both sides is a normal "not found",+ * and a primary-store *error* is already reported where it is caught and+ * must not keep this series hot. Deployments without the store configured+ * record nothing.+ */+export function recordJobStorePostgresFallback(+ store: JobStoreFallback,+ id: string,+ extra: Record<string, unknown> = {},+): void {+ if (!STORE_TABLE[store]()) return;+ jobStorePostgresFallbackTotal.inc({ store });+ logger.info("PostgreSQL fallback served a job-store read", {+ ...extra,+ module: "job-store-fallback",+ store,+ id,+ });+}diff --git a/apps/api/src/lib/operational-job-access.ts b/apps/api/src/lib/operational-job-access.tsindex 4377477c2..32fc4f39b 100644--- a/apps/api/src/lib/operational-job-access.ts+++ b/apps/api/src/lib/operational-job-access.ts@@ -10,6 +10,7 @@ import {supabaseGetExtractRequestByIdDirect,supabaseGetScrapeById,} from "./supabase-jobs";+import { recordJobStorePostgresFallback } from "./job-store-fallback";const DEFAULT_TTL_MS = 24 * 60 * 60 * 1000;@@ -26,9 +27,11 @@ async function resolveOperationalJobAccess(params: {fallback: () => Promise<OperationalJobAccess | null>;}): Promise<OperationalJobAccess | null> {let access: ApiJobAccess | null = null;+ let readFailed = false;try {access = await readApiJobAccess(params.id);} catch (error) {+ readFailed = true;logger.warn("Bigtable job access read failed; using legacy lookup", {error,jobId: params.id,@@ -40,7 +43,13 @@ async function resolveOperationalJobAccess(params: {}const fallback = await params.fallback();- return fallback && Number.isFinite(fallback.expiresAtMs) ? fallback : null;+ if (!fallback || !Number.isFinite(fallback.expiresAtMs)) return null;+ if (!readFailed) {+ recordJobStorePostgresFallback("job_access", params.id, {+ kind: fallback.kind,+ });+ }+ return fallback;}export function getScrapeJobAccess(diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.tsindex af7fc46e6..e4ae1e5fb 100644--- a/apps/api/src/services/worker/crawl-logic.ts+++ b/apps/api/src/services/worker/crawl-logic.ts@@ -13,6 +13,7 @@ import { getJobs } from "../../controllers/v1/crawl-status";import { logCrawl, logBatchScrape } from "../logging/log_job";import { createWebhookSender, WebhookEvent } from "../webhook/index";import type { NuQJob } from "./nuq";+import { recordJobStorePostgresFallback } from "../../lib/job-store-fallback";export async function finishCrawlSuper(job: NuQJob<any>) {const crawlId = job.groupId;@@ -143,10 +144,12 @@ export async function finishCrawlSuper(job: NuQJob<any>) {const num_docs = await getDoneJobsOrderedLength(crawlId);let credits_billed: number | null = null;+ let creditsReadFailed = false;try {credits_billed = await readRequestCredits(requestId);} catch (error) {+ creditsReadFailed = true;logger.warn("Bigtable request credits read failed", { error });}@@ -154,6 +157,9 @@ export async function finishCrawlSuper(job: NuQJob<any>) {try {const creditsRows = await creditsBilledByCrawlId(db, crawlId);credits_billed = creditsRows?.[0]?.credits_billed ?? null;+ if (credits_billed !== null && !creditsReadFailed) {+ recordJobStorePostgresFallback("request_credits", requestId);+ }} catch (error) {logger.warn("Credits billed is null", { error });}