Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): count and log PostgreSQL fallback hits on job-store reads (#4685)

Gergő Móriczfd8f1dda3e62
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts
index 3d9ff5b19..88662a5b1 100644
--- a/apps/api/src/controllers/v1/crawl-status.ts
+++ b/apps/api/src/controllers/v1/crawl-status.ts
@@ -30,6 +30,7 @@ import {
import { ScrapeJobSingleUrls } from "../../types";
import { readScrapeJobState } from "../../lib/job-state-store";
import { readRequestCredits } from "../../lib/request-credits-store";
+import { recordJobStorePostgresFallback } from "../../lib/job-store-fallback";
configDotenv();
export type PseudoJob<T> = {
@@ -54,6 +55,7 @@ type DBScrape = {
};
export async function getJob(id: string): Promise<PseudoJob<any> | null> {
+ let scrapeStateFailed = false;
const [nuqJob, scrapeState, dbScrape, gcsJob] = await Promise.all([
scrapeQueue.getJob(id) as Promise<NuQJob<ScrapeJobSingleUrls> | null>,
readScrapeJobState(id).catch(error => {
@@ -61,6 +63,7 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {
error,
scrapeId: id,
});
+ scrapeStateFailed = true;
return null;
}),
(config.USE_DB_AUTHENTICATION
@@ -70,6 +73,9 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {
]);
if (!nuqJob && !scrapeState && !dbScrape) return null;
+ if (!scrapeState && !scrapeStateFailed && dbScrape) {
+ recordJobStorePostgresFallback("scrape_state", id);
+ }
if (nuqJob && nuqJob.data.mode !== "single_urls") {
return null;
@@ -194,13 +200,18 @@ export async function crawlStatusController(
logger.child({ zeroDataRetention }),
);
- let creditsBilled = await readRequestCredits(req.params.jobId).catch(
- () => null,
- );
+ let creditsReadFailed = false;
+ let creditsBilled = await readRequestCredits(req.params.jobId).catch(() => {
+ creditsReadFailed = true;
+ return null;
+ });
if (creditsBilled === null && config.USE_DB_AUTHENTICATION) {
creditsBilled = await creditsBilledByCrawlId(dbRr, req.params.jobId)
.then(rows => rows[0]?.credits_billed ?? null)
.catch(() => null);
+ if (creditsBilled !== null && !creditsReadFailed) {
+ recordJobStorePostgresFallback("request_credits", req.params.jobId);
+ }
}
// check if the crawl failed during kickoff (e.g. queue full)
diff --git a/apps/api/src/controllers/v1/extract-status.ts b/apps/api/src/controllers/v1/extract-status.ts
index e19f68e52..bf5bbf6ca 100644
--- a/apps/api/src/controllers/v1/extract-status.ts
+++ b/apps/api/src/controllers/v1/extract-status.ts
@@ -12,6 +12,7 @@ import { getJobFromGCS } from "../../lib/gcs-jobs";
import { getExtractJobAccess } from "../../lib/operational-job-access";
import { readExtractJobState } from "../../lib/job-state-store";
import { normalizeJobAccessTeamId } from "../../lib/job-access-store";
+import { recordJobStorePostgresFallback } from "../../lib/job-store-fallback";
async function getExtractData(id: string): Promise<any> {
// Try GCS first if configured
@@ -69,10 +70,12 @@ export async function extractStatusController(
// If not in Redis, check the database for completed jobs
if (!extract) {
if (config.USE_DB_AUTHENTICATION) {
+ let stateReadFailed = false;
const state = await readExtractJobState(req.params.jobId).catch(error => {
logger.warn("Bigtable extract state read failed; using legacy lookup", {
error,
});
+ stateReadFailed = true;
return null;
});
if (state) {
@@ -97,6 +100,9 @@ export async function extractStatusController(
error: "Extract job not found",
});
}
+ if (!stateReadFailed) {
+ recordJobStorePostgresFallback("extract_state", req.params.jobId);
+ }
if (dbExtract.team_id !== req.auth.team_id) {
return res.status(404).json({
diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.ts
index e6eb12053..4aceae22d 100644
--- a/apps/api/src/controllers/v2/crawl-status.ts
+++ b/apps/api/src/controllers/v2/crawl-status.ts
@@ -33,6 +33,7 @@ import { redisEvictConnection } from "../../../src/services/redis";
import { isBaseDomain, extractBaseDomain } from "../../lib/url-utils";
import { readScrapeJobState } from "../../lib/job-state-store";
import { readRequestCredits } from "../../lib/request-credits-store";
+import { recordJobStorePostgresFallback } from "../../lib/job-store-fallback";
configDotenv();
export type PseudoJob<T> = {
@@ -60,6 +61,7 @@ export async function getJob(
id: string,
_logger = logger,
): Promise<PseudoJob<any> | null> {
+ let scrapeStateFailed = false;
const [nuqJob, scrapeState, dbScrape, gcsJob] = await Promise.all([
scrapeQueue.getJob(
id,
@@ -70,6 +72,7 @@ export async function getJob(
error,
scrapeId: id,
});
+ scrapeStateFailed = true;
return null;
}),
(config.USE_DB_AUTHENTICATION
@@ -79,6 +82,9 @@ export async function getJob(
]);
if (!nuqJob && !scrapeState && !dbScrape) return null;
+ if (!scrapeState && !scrapeStateFailed && dbScrape) {
+ recordJobStorePostgresFallback("scrape_state", id);
+ }
if (nuqJob && nuqJob.data.mode !== "single_urls") {
return null;
@@ -215,13 +221,20 @@ export async function crawlStatusController(
logger.child({ zeroDataRetention }),
);
+ let creditsReadFailed = false;
let creditsBilled = await readRequestCredits(
sc?.requestId ?? req.params.jobId,
- ).catch(() => null);
+ ).catch(() => {
+ creditsReadFailed = true;
+ return null;
+ });
if (creditsBilled === null && config.USE_DB_AUTHENTICATION) {
creditsBilled = await creditsBilledByCrawlId(dbRr, req.params.jobId)
.then(rows => rows[0]?.credits_billed ?? null)
.catch(() => null);
+ if (creditsBilled !== null && !creditsReadFailed) {
+ recordJobStorePostgresFallback("request_credits", req.params.jobId);
+ }
}
// check if the crawl failed during kickoff (e.g. queue full)
diff --git a/apps/api/src/controllers/v2/extract-status.ts b/apps/api/src/controllers/v2/extract-status.ts
index 89ed86563..3405f638d 100644
--- a/apps/api/src/controllers/v2/extract-status.ts
+++ b/apps/api/src/controllers/v2/extract-status.ts
@@ -13,6 +13,7 @@ import { getExtractJobAccess } from "../../lib/operational-job-access";
import { readExtractJobState } from "../../lib/job-state-store";
import { normalizeJobAccessTeamId } from "../../lib/job-access-store";
import { getExtractV3AgentStatus } from "../../lib/extract-v3-status";
+import { recordJobStorePostgresFallback } from "../../lib/job-store-fallback";
async function getExtractData(id: string): Promise<any> {
// Try GCS first if configured
@@ -69,11 +70,13 @@ export async function extractStatusController(
// If not in Redis, check the database for completed jobs
if (!redisExtract) {
if (config.USE_DB_AUTHENTICATION) {
+ let stateReadFailed = false;
const state = await readExtractJobState(req.params.jobId).catch(error => {
_logger.warn(
"Bigtable extract state read failed; using legacy lookup",
{ error, extractId: req.params.jobId },
);
+ stateReadFailed = true;
return null;
});
if (state) {
@@ -92,6 +95,9 @@ export async function extractStatusController(
const dbExtract = await supabaseGetExtractByIdDirect(req.params.jobId);
if (dbExtract) {
+ if (!stateReadFailed) {
+ recordJobStorePostgresFallback("extract_state", req.params.jobId);
+ }
// Get result data
let data: any = [];
if (dbExtract.is_successful) {
diff --git a/apps/api/src/controllers/v2/feedback/feedback-store.ts b/apps/api/src/controllers/v2/feedback/feedback-store.ts
index 5f113ddff..357400eb7 100644
--- a/apps/api/src/controllers/v2/feedback/feedback-store.ts
+++ b/apps/api/src/controllers/v2/feedback/feedback-store.ts
@@ -13,6 +13,7 @@ import {
FeedbackRecordOptions,
RefundPolicySnapshot,
} from "./internal-types";
+import { recordJobStorePostgresFallback } from "../../../lib/job-store-fallback";
type DbError = { code?: string } & Record<string, unknown>;
@@ -45,6 +46,7 @@ export async function lookupFeedbackJob(
jobId: string,
dbTeamId: string,
): Promise<FeedbackJobRow | null> {
+ let bigtableFailed = false;
try {
const job = await readFeedbackJob(jobId);
if (job) {
@@ -72,6 +74,7 @@ export async function lookupFeedbackJob(
};
}
} catch (error) {
+ bigtableFailed = true;
logger.warn(
"Bigtable feedback job read failed; falling back to PostgreSQL",
{
@@ -98,6 +101,9 @@ export async function lookupFeedbackJob(
.limit(1);
if (!row) return null;
+ if (!bigtableFailed) {
+ recordJobStorePostgresFallback("feedback_job", jobId, { endpoint });
+ }
return {
endpoint,
diff --git a/apps/api/src/controllers/v2/scrape-browser.ts b/apps/api/src/controllers/v2/scrape-browser.ts
index 5eb7a496d..5096c4087 100644
--- a/apps/api/src/controllers/v2/scrape-browser.ts
+++ b/apps/api/src/controllers/v2/scrape-browser.ts
@@ -72,6 +72,7 @@ import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
import { getScrapeJobAccess } from "../../lib/operational-job-access";
import { readScrapeJobState } from "../../lib/job-state-store";
import { scrapeQueue } from "../../services/worker/nuq-router";
+import { recordJobStorePostgresFallback } from "../../lib/job-store-fallback";
// ---------------------------------------------------------------------------
// Schemas
@@ -170,12 +171,14 @@ export async function scrapeInteractController(
});
}
+ let stateReadFailed = false;
const [nuqJob, state] = await Promise.all([
scrapeQueue.getJob(scrapeId, logger),
readScrapeJobState(scrapeId).catch(error => {
logger.warn("Bigtable scrape state read failed; using legacy lookup", {
error,
});
+ stateReadFailed = true;
return null;
}),
]);
@@ -217,6 +220,11 @@ export async function scrapeInteractController(
legacyScrape = (await supabaseGetScrapeByIdDirect(
scrapeId,
)) as ScrapeContextRow | null;
+ if (legacyScrape && !stateReadFailed) {
+ recordJobStorePostgresFallback("scrape_state", scrapeId, {
+ reason: "replay_context",
+ });
+ }
const replay = legacyScrape
? buildReplayContextFromScrape(legacyScrape)
: { error: "Replay context is unavailable for this scrape job." };
diff --git a/apps/api/src/lib/change-tracking-store.ts b/apps/api/src/lib/change-tracking-store.ts
index f4b770a73..a46138b20 100644
--- a/apps/api/src/lib/change-tracking-store.ts
+++ b/apps/api/src/lib/change-tracking-store.ts
@@ -3,6 +3,7 @@ import { config } from "../config";
import { diffGetLastScrape } from "../db/rpc";
import { getBigtableTable } from "./bigtable-client";
import { setSpanAttributes, withSpan } from "./otel-tracer";
+import { recordJobStorePostgresFallback } from "./job-store-fallback";
// Change tracking bookkeeping. Final architecture (#4484 was the
// transition): Bigtable is the store. Writes go to Bigtable only --
@@ -158,6 +159,9 @@ export async function changeTrackingGetLastScrape(params: {
);
const pgRow = pgRows[0];
if (!pgRow) return null;
+ recordJobStorePostgresFallback("change_tracking", pgRow.o_job_id, {
+ teamId: params.team_id,
+ });
return {
job_id: pgRow.o_job_id,
date_added: new Date(pgRow.o_date_added).toISOString(),
diff --git a/apps/api/src/lib/job-store-fallback.test.ts b/apps/api/src/lib/job-store-fallback.test.ts
new file mode 100644
index 000000000..0cc7fe750
--- /dev/null
+++ b/apps/api/src/lib/job-store-fallback.test.ts
@@ -0,0 +1,80 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const { info, mutableConfig } = vi.hoisted(() => ({
+ info: vi.fn(),
+ mutableConfig: {
+ BIGTABLE_JOB_ACCESS_TABLE: "api_job_access" as string | undefined,
+ BIGTABLE_SCRAPE_STATE_TABLE: "scrape_state" as string | undefined,
+ BIGTABLE_EXTRACT_STATE_TABLE: "extract_state" as string | undefined,
+ BIGTABLE_FEEDBACK_JOBS_TABLE: "feedback_jobs" as string | undefined,
+ BIGTABLE_REQUEST_CREDITS_TABLE: "request_credits" as string | undefined,
+ BIGTABLE_CHANGE_TRACKING_TABLE: "change_tracking" as string | undefined,
+ },
+}));
+vi.mock("./logger", () => ({ logger: { info } }));
+vi.mock("../config", () => ({ config: mutableConfig }));
+
+import {
+ jobStorePostgresFallbackTotal,
+ recordJobStorePostgresFallback,
+} from "./job-store-fallback";
+
+async function countsByStore() {
+ const metric = await jobStorePostgresFallbackTotal.get();
+ return Object.fromEntries(metric.values.map(v => [v.labels.store, v.value]));
+}
+
+describe("job store fallback accounting", () => {
+ beforeEach(() => {
+ jobStorePostgresFallbackTotal.reset();
+ info.mockClear();
+ mutableConfig.BIGTABLE_SCRAPE_STATE_TABLE = "scrape_state";
+ });
+
+ it("counts and logs one hit per store", async () => {
+ recordJobStorePostgresFallback("scrape_state", "job-1");
+ recordJobStorePostgresFallback("scrape_state", "job-2");
+ recordJobStorePostgresFallback("job_access", "job-3", { kind: "crawl" });
+
+ expect(await countsByStore()).toMatchObject({
+ scrape_state: 2,
+ job_access: 1,
+ });
+
+ expect(info).toHaveBeenCalledTimes(3);
+ expect(info).toHaveBeenLastCalledWith(
+ "PostgreSQL fallback served a job-store read",
+ expect.objectContaining({
+ store: "job_access",
+ id: "job-3",
+ kind: "crawl",
+ }),
+ );
+ });
+
+ it("lets the fixed log fields win over extra context", () => {
+ recordJobStorePostgresFallback("scrape_state", "job-1", {
+ id: "spoofed",
+ store: "spoofed",
+ module: "spoofed",
+ });
+
+ expect(info).toHaveBeenCalledWith(
+ "PostgreSQL fallback served a job-store read",
+ expect.objectContaining({
+ module: "job-store-fallback",
+ store: "scrape_state",
+ id: "job-1",
+ }),
+ );
+ });
+
+ it("records nothing when the store is not configured", async () => {
+ mutableConfig.BIGTABLE_SCRAPE_STATE_TABLE = undefined;
+
+ recordJobStorePostgresFallback("scrape_state", "job-1");
+
+ expect(await countsByStore()).toEqual({});
+ expect(info).not.toHaveBeenCalled();
+ });
+});
diff --git a/apps/api/src/lib/job-store-fallback.ts b/apps/api/src/lib/job-store-fallback.ts
new file mode 100644
index 000000000..b7cf09c89
--- /dev/null
+++ b/apps/api/src/lib/job-store-fallback.ts
@@ -0,0 +1,65 @@
+import { Counter, register } from "prom-client";
+import { config } from "../config";
+import { logger } from "./logger";
+
+const NAME = "job_store_postgres_fallback_total";
+
+/**
+ * Reads the new job stores (Bigtable, NuQ) could not answer and PostgreSQL
+ * did. Each label is one fallback path; a store is ready to lose its
+ * fallback once its series has stayed flat for the agreed window.
+ *
+ * Looked up before creation so a re-evaluated module (test isolation) does
+ * not register the same series twice in the shared default registry.
+ */
+export const jobStorePostgresFallbackTotal =
+ (register.getSingleMetric(NAME) as Counter<"store"> | undefined) ??
+ new Counter({
+ name: NAME,
+ help: "Job-store reads answered by the PostgreSQL fallback, by store",
+ labelNames: ["store"] as const,
+ });
+
+type JobStoreFallback =
+ | "job_access"
+ | "scrape_state"
+ | "extract_state"
+ | "feedback_job"
+ | "request_credits"
+ | "change_tracking";
+
+/**
+ * The Bigtable table behind each store. When it is not configured the
+ * primary store never answers, PostgreSQL is the only store, and a hit is
+ * not a fallback worth counting.
+ */
+const STORE_TABLE: Record<JobStoreFallback, () => string | undefined> = {
+ job_access: () => config.BIGTABLE_JOB_ACCESS_TABLE,
+ scrape_state: () => config.BIGTABLE_SCRAPE_STATE_TABLE,
+ extract_state: () => config.BIGTABLE_EXTRACT_STATE_TABLE,
+ feedback_job: () => config.BIGTABLE_FEEDBACK_JOBS_TABLE,
+ request_credits: () => config.BIGTABLE_REQUEST_CREDITS_TABLE,
+ change_tracking: () => config.BIGTABLE_CHANGE_TRACKING_TABLE,
+};
+
+/**
+ * Record that PostgreSQL served a row the primary store did not have. Call
+ * it only on a genuine miss: a miss on both sides is a normal "not found",
+ * and a primary-store *error* is already reported where it is caught and
+ * must not keep this series hot. Deployments without the store configured
+ * record nothing.
+ */
+export function recordJobStorePostgresFallback(
+ store: JobStoreFallback,
+ id: string,
+ extra: Record<string, unknown> = {},
+): void {
+ if (!STORE_TABLE[store]()) return;
+ jobStorePostgresFallbackTotal.inc({ store });
+ logger.info("PostgreSQL fallback served a job-store read", {
+ ...extra,
+ module: "job-store-fallback",
+ store,
+ id,
+ });
+}
diff --git a/apps/api/src/lib/operational-job-access.ts b/apps/api/src/lib/operational-job-access.ts
index 4377477c2..32fc4f39b 100644
--- a/apps/api/src/lib/operational-job-access.ts
+++ b/apps/api/src/lib/operational-job-access.ts
@@ -10,6 +10,7 @@ import {
supabaseGetExtractRequestByIdDirect,
supabaseGetScrapeById,
} from "./supabase-jobs";
+import { recordJobStorePostgresFallback } from "./job-store-fallback";
const DEFAULT_TTL_MS = 24 * 60 * 60 * 1000;
@@ -26,9 +27,11 @@ async function resolveOperationalJobAccess(params: {
fallback: () => Promise<OperationalJobAccess | null>;
}): Promise<OperationalJobAccess | null> {
let access: ApiJobAccess | null = null;
+ let readFailed = false;
try {
access = await readApiJobAccess(params.id);
} catch (error) {
+ readFailed = true;
logger.warn("Bigtable job access read failed; using legacy lookup", {
error,
jobId: params.id,
@@ -40,7 +43,13 @@ async function resolveOperationalJobAccess(params: {
}
const fallback = await params.fallback();
- return fallback && Number.isFinite(fallback.expiresAtMs) ? fallback : null;
+ if (!fallback || !Number.isFinite(fallback.expiresAtMs)) return null;
+ if (!readFailed) {
+ recordJobStorePostgresFallback("job_access", params.id, {
+ kind: fallback.kind,
+ });
+ }
+ return fallback;
}
export function getScrapeJobAccess(
diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.ts
index af7fc46e6..e4ae1e5fb 100644
--- a/apps/api/src/services/worker/crawl-logic.ts
+++ b/apps/api/src/services/worker/crawl-logic.ts
@@ -13,6 +13,7 @@ import { getJobs } from "../../controllers/v1/crawl-status";
import { logCrawl, logBatchScrape } from "../logging/log_job";
import { createWebhookSender, WebhookEvent } from "../webhook/index";
import type { NuQJob } from "./nuq";
+import { recordJobStorePostgresFallback } from "../../lib/job-store-fallback";
export async function finishCrawlSuper(job: NuQJob<any>) {
const crawlId = job.groupId;
@@ -143,10 +144,12 @@ export async function finishCrawlSuper(job: NuQJob<any>) {
const num_docs = await getDoneJobsOrderedLength(crawlId);
let credits_billed: number | null = null;
+ let creditsReadFailed = false;
try {
credits_billed = await readRequestCredits(requestId);
} catch (error) {
+ creditsReadFailed = true;
logger.warn("Bigtable request credits read failed", { error });
}
@@ -154,6 +157,9 @@ export async function finishCrawlSuper(job: NuQJob<any>) {
try {
const creditsRows = await creditsBilledByCrawlId(db, crawlId);
credits_billed = creditsRows?.[0]?.credits_billed ?? null;
+ if (credits_billed !== null && !creditsReadFailed) {
+ recordJobStorePostgresFallback("request_credits", requestId);
+ }
} catch (error) {
logger.warn("Credits billed is null", { error });
}