Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): stop reading the PostgreSQL job tables (#4694)
Gergő Móricz85288f4941c9
diff --git a/apps/api/src/__tests__/routes/interact-agent-concurrency.routes.test.ts b/apps/api/src/__tests__/routes/interact-agent-concurrency.routes.test.tsindex 6812e6744..eb9e2642c 100644--- a/apps/api/src/__tests__/routes/interact-agent-concurrency.routes.test.ts+++ b/apps/api/src/__tests__/routes/interact-agent-concurrency.routes.test.ts@@ -85,18 +85,14 @@ vi.mock("../../lib/operational-job-access", () => ({expiresAtMs: Date.now() + 60_000,})),}));+// The scrape's replay context lives in its Bigtable terminal state.vi.mock("../../lib/job-state-store", () => ({- readScrapeJobState: vi.fn(async () => null),-}));-vi.mock("../../lib/job-store-fallback", () => ({- recordJobStorePostgresFallback: vi.fn(),-}));-vi.mock("../../lib/supabase-jobs", () => ({- supabaseGetScrapeByIdDirect: vi.fn(async () => ({- id: "scrape-1",- team_id: "team-free",- url: "https://example.com",- options: {},+ readScrapeJobState: vi.fn(async () => ({+ status: "completed",+ requestId: "scrape-1",+ completedAtMs: Date.now(),+ creditsBilled: 1,+ replay: { targetUrl: "https://example.com", waitForMs: 0, actions: [] },})),}));vi.mock("../../lib/browser-sessions", () => ({diff --git a/apps/api/src/controllers/v1/crawl-errors.ts b/apps/api/src/controllers/v1/crawl-errors.tsindex 808f7646a..9e387c2ed 100644--- a/apps/api/src/controllers/v1/crawl-errors.ts+++ b/apps/api/src/controllers/v1/crawl-errors.ts@@ -30,7 +30,7 @@ export async function crawlErrorsController(const crawlTtlHours = req.acuc?.flags?.crawlTtlHours ?? 24;let access;try {- access = await getCrawlJobAccess(req.params.jobId, crawlTtlHours);+ access = await getCrawlJobAccess(req.params.jobId);} catch (requestError) {_logger.error("Error getting request", { error: requestError });throw requestError;diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex 5499ea57a..c2bd0d652 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -15,11 +15,8 @@ import {getCrawlQualifiedJobCount,getDoneJobsOrderedUntil,} from "../../lib/crawl-redis";-import { supabaseGetScrapeById } from "../../lib/supabase-jobs";import { configDotenv } from "dotenv";import { logger } from "../../lib/logger";-import { creditsBilledByCrawlId } from "../../db/rpc";-import { dbRr } from "../../db/connection";import { getJobFromGCS } from "../../lib/gcs-jobs";import {scrapeQueue,@@ -30,7 +27,6 @@ import {import { ScrapeJobSingleUrls } from "../../types";import { readScrapeJobState } from "../../lib/job-state-store";import { readRequestCredits } from "../../lib/request-credits-store";-import { recordJobStorePostgresFallback } from "../../lib/job-store-fallback";import { readRequestCreditsFromAnalytics } from "../../lib/request-credits-analytics";configDotenv();@@ -46,36 +42,27 @@ export type PseudoJob<T> = {failedReason?: string;};-type DBScrape = {- id: string;- success: boolean;- options: any;- created_at: any;- error: string | null;- team_id: string;-};-export async function getJob(id: string): Promise<PseudoJob<any> | null> {- let scrapeStateFailed = false;- const [nuqJob, scrapeState, dbScrape, gcsJob] = await Promise.all([+ let stateReadError: unknown = null;+ const [nuqJob, scrapeState, gcsJob] = await Promise.all([scrapeQueue.getJob(id) as Promise<NuQJob<ScrapeJobSingleUrls> | null>,readScrapeJobState(id).catch(error => {- logger.warn("Bigtable scrape state read failed; using legacy lookup", {+ logger.warn("Bigtable scrape state read failed", {error,scrapeId: id,});- scrapeStateFailed = true;+ stateReadError = error;return null;}),- (config.USE_DB_AUTHENTICATION- ? supabaseGetScrapeById(id)- : null) as Promise<DBScrape | null>,(config.GCS_BUCKET_NAME ? getJobFromGCS(id) : null) as Promise<any | null>,]);- if (!nuqJob && !scrapeState && !dbScrape) return null;- if (!nuqJob && !scrapeState && !scrapeStateFailed && dbScrape) {- recordJobStorePostgresFallback("scrape_state", id);+ if (!nuqJob && !scrapeState) {+ // With no NuQ job, Bigtable is the only place a finished job's state+ // lives. A failed read is an outage, not a missing job: surface it rather+ // than answer 404 for a job that exists.+ if (stateReadError) throw stateReadError;+ return null;}if (nuqJob && nuqJob.data.mode !== "single_urls") {@@ -91,24 +78,13 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {const job: PseudoJob<any> = {id,- status:- scrapeState?.status ??- (dbScrape ? (dbScrape.success ? "completed" : "failed") : nuqJob!.status),+ status: scrapeState?.status ?? nuqJob!.status,returnvalue: Array.isArray(data) ? data[0] : data,data: {- scrapeOptions: nuqJob- ? nuqJob.data.scrapeOptions- : (dbScrape?.options ?? null),+ scrapeOptions: nuqJob ? nuqJob.data.scrapeOptions : null,},- timestamp:- scrapeState?.completedAtMs ??- (nuqJob- ? nuqJob.createdAt.valueOf()- : new Date(dbScrape!.created_at).valueOf()),- failedReason:- (scrapeState?.error ??- (nuqJob ? nuqJob.failedReason : dbScrape?.error)) ||- undefined,+ timestamp: scrapeState?.completedAtMs ?? nuqJob!.createdAt.valueOf(),+ failedReason: (scrapeState?.error ?? nuqJob?.failedReason) || undefined,};return job;@@ -201,29 +177,19 @@ export async function crawlStatusController(logger.child({ zeroDataRetention }),);- let creditsReadFailed = false;- let creditsBilled = await readRequestCredits(req.params.jobId).catch(() => {- creditsReadFailed = true;- return null;- });+ // A failed Bigtable read propagates: during an outage the analytics sum+ // could be behind by a second of ClickPipes lag and under-report credits.+ let creditsBilled = await readRequestCredits(req.params.jobId);if (creditsBilled === null) {// Requests from before the Bigtable credit rows existed: sum the scrape// job log instead.- creditsBilled = await readRequestCreditsFromAnalytics(- req.params.jobId,- ).catch(error => {+ creditsBilled = await readRequestCreditsFromAnalytics(req.params.jobId, {+ emptyAsZero: true,+ }).catch(error => {logger.warn("Analytics request credits read failed", { error });return null;});}- if (creditsBilled === null && config.USE_DB_AUTHENTICATION) {- creditsBilled = await creditsBilledByCrawlId(dbRr, req.params.jobId)- .then(rows => rows[0]?.credits_billed ?? null)- .catch(() => null);- if (creditsBilled !== null && !creditsReadFailed) {- recordJobStorePostgresFallback("request_credits", req.params.jobId);- }- }// check if the crawl failed during kickoff (e.g. queue full)const crawlError = await getCrawlError(req.params.jobId);diff --git a/apps/api/src/controllers/v1/extract-status.ts b/apps/api/src/controllers/v1/extract-status.tsindex bf5bbf6ca..b654df34d 100644--- a/apps/api/src/controllers/v1/extract-status.ts+++ b/apps/api/src/controllers/v1/extract-status.ts@@ -1,4 +1,5 @@import { Response } from "express";+import { JOB_ACCESS_TTL_MS } from "../../lib/job-access-store";import { config } from "../../config";import { RequestWithAuth } from "./types";import {@@ -6,13 +7,11 @@ import {getExtractExpiry,getExtractResult,} from "../../lib/extract/extract-redis";-import { supabaseGetExtractByIdDirect } from "../../lib/supabase-jobs";import { logger as _logger } from "../../lib/logger";import { getJobFromGCS } from "../../lib/gcs-jobs";import { getExtractJobAccess } from "../../lib/operational-job-access";import { readExtractJobState } from "../../lib/job-state-store";import { normalizeJobAccessTeamId } from "../../lib/job-access-store";-import { recordJobStorePostgresFallback } from "../../lib/job-store-fallback";async function getExtractData(id: string): Promise<any> {// Try GCS first if configured@@ -67,64 +66,25 @@ export async function extractStatusController(});}- // If not in Redis, check the database for completed jobs+ // Not in Redis: the job finished (or never existed). Bigtable holds the+ // terminal state for 24 hours; after that the job has expired.if (!extract) {- if (config.USE_DB_AUTHENTICATION) {- let stateReadFailed = false;- const state = await readExtractJobState(req.params.jobId).catch(error => {- logger.warn("Bigtable extract state read failed; using legacy lookup", {- error,- });- stateReadFailed = true;- return null;- });- if (state) {- return res.status(200).json({- success: state.status === "completed",- data:- state.status === "completed"- ? await getExtractData(req.params.jobId)- : [],- status: state.status,- error: state.error,- expiresAt: new Date(access!.expiresAtMs).toISOString(),- creditsUsed: state.creditsBilled,- });- }-- const dbExtract = await supabaseGetExtractByIdDirect(req.params.jobId);- if (!dbExtract) {- logger.warn("Extract job was not found");- return res.status(404).json({- success: false,- error: "Extract job not found",- });- }- if (!stateReadFailed) {- recordJobStorePostgresFallback("extract_state", req.params.jobId);- }-- if (dbExtract.team_id !== req.auth.team_id) {- return res.status(404).json({- success: false,- error: "Extract job not found",- });- }-- let data: any = [];- if (dbExtract.is_successful) {- data = await getExtractData(req.params.jobId);- }-- // Return DB-based status+ // A failed Bigtable read is an outage, not a missing job: it propagates+ // to the error handler rather than answering 404.+ const state = await readExtractJobState(req.params.jobId);+ if (state) {return res.status(200).json({- success: dbExtract.is_successful,- data,- status: dbExtract.is_successful ? "completed" : "failed",- error: dbExtract.error || undefined,+ success: state.status === "completed",+ data:+ state.status === "completed"+ ? await getExtractData(req.params.jobId)+ : [],+ status: state.status,+ error: state.error,expiresAt: new Date(- new Date(dbExtract.created_at).getTime() + 1000 * 60 * 60 * 24,+ access?.expiresAtMs ?? state.completedAtMs + JOB_ACCESS_TTL_MS,).toISOString(),+ creditsUsed: state.creditsBilled,});}diff --git a/apps/api/src/controllers/v2/__tests__/scrape-browser.test.ts b/apps/api/src/controllers/v2/__tests__/scrape-browser.test.tsindex ab6830fca..5aeadb758 100644--- a/apps/api/src/controllers/v2/__tests__/scrape-browser.test.ts+++ b/apps/api/src/controllers/v2/__tests__/scrape-browser.test.ts@@ -1,7 +1,7 @@import type { Response } from "express";+import { readScrapeJobState } from "../../../lib/job-state-store";import { vi } from "vitest";import { config } from "../../../config";-import { supabaseGetScrapeByIdDirect } from "../../../lib/supabase-jobs";import {insertBrowserSession,getBrowserSession,@@ -56,9 +56,6 @@ vi.mock("../../../lib/operational-job-access", () => ({vi.mock("../../../lib/job-state-store", () => ({readScrapeJobState: vi.fn(async () => null),}));-vi.mock("../../../lib/job-store-fallback", () => ({- recordJobStorePostgresFallback: vi.fn(),-}));vi.mock("../../auth", () => ({getACUCTeam: vi.fn(async () => ({ org_id: null })),}));@@ -74,10 +71,6 @@ vi.mock("../../../lib/scrape-interact/langsmith", () => ({sanitizeUrlForTrace: (url: string) => url,}));-vi.mock("../../../lib/supabase-jobs", () => ({- supabaseGetScrapeByIdDirect: vi.fn(),-}));-vi.mock("../../../lib/browser-sessions", () => ({insertBrowserSession: vi.fn(),completeBrowserSessionSettlement: vi.fn(async () => {}),@@ -157,7 +150,7 @@ describe("scrapeInteractController", () => {config.USE_DB_AUTHENTICATION = previousUseDbAuthentication;});- it("rejects self-hosted scrape interact before querying Supabase", async () => {+ it("rejects scrape interact when database authentication is disabled", async () => {config.USE_DB_AUTHENTICATION = false;const req = {@@ -170,7 +163,6 @@ describe("scrapeInteractController", () => {await scrapeInteractController(req, res);- expect(supabaseGetScrapeByIdDirect).not.toHaveBeenCalled();expect(res.status).toHaveBeenCalledWith(501);expect(res.json).toHaveBeenCalledWith({success: false,@@ -193,11 +185,13 @@ describe("scrapeInteractController", () => {control_url: "https://hangar.example/live#control",recording: true,};- vi.mocked(supabaseGetScrapeByIdDirect).mockResolvedValue({- id: "scrape-123",- team_id: "team-123",- url: "https://example.com",- options: {},+ // The replay context comes from the scrape's Bigtable terminal state.+ vi.mocked(readScrapeJobState).mockResolvedValueOnce({+ status: "completed",+ requestId: "scrape-123",+ completedAtMs: Date.now(),+ creditsBilled: 1,+ replay: { targetUrl: "https://example.com", waitForMs: 0, actions: [] },} as any);const executed = {stdout: "https://example.com",diff --git a/apps/api/src/controllers/v2/crawl-errors.ts b/apps/api/src/controllers/v2/crawl-errors.tsindex 808f7646a..9e387c2ed 100644--- a/apps/api/src/controllers/v2/crawl-errors.ts+++ b/apps/api/src/controllers/v2/crawl-errors.ts@@ -30,7 +30,7 @@ export async function crawlErrorsController(const crawlTtlHours = req.acuc?.flags?.crawlTtlHours ?? 24;let access;try {- access = await getCrawlJobAccess(req.params.jobId, crawlTtlHours);+ access = await getCrawlJobAccess(req.params.jobId);} catch (requestError) {_logger.error("Error getting request", { error: requestError });throw requestError;diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.tsindex 89ab75914..ae456e731 100644--- a/apps/api/src/controllers/v2/crawl-status.ts+++ b/apps/api/src/controllers/v2/crawl-status.ts@@ -16,11 +16,8 @@ import {getLastDoneJobTimestamp,isCrawlKickoffFinished,} from "../../lib/crawl-redis";-import { supabaseGetScrapeById } from "../../lib/supabase-jobs";import { configDotenv } from "dotenv";import { logger } from "../../lib/logger";-import { creditsBilledByCrawlId } from "../../db/rpc";-import { dbRr } from "../../db/connection";import { getJobFromGCS } from "../../lib/gcs-jobs";import {scrapeQueue,@@ -33,7 +30,6 @@ import { redisEvictConnection } from "../../../src/services/redis";import { isBaseDomain, extractBaseDomain } from "../../lib/url-utils";import { readScrapeJobState } from "../../lib/job-state-store";import { readRequestCredits } from "../../lib/request-credits-store";-import { recordJobStorePostgresFallback } from "../../lib/job-store-fallback";import { readRequestCreditsFromAnalytics } from "../../lib/request-credits-analytics";configDotenv();@@ -49,42 +45,33 @@ export type PseudoJob<T> = {failedReason?: string;};-type DBScrape = {- id: string;- success: boolean;- options: any;- created_at: any;- error: string | null;- team_id: string;-};-export async function getJob(id: string,_logger = logger,): Promise<PseudoJob<any> | null> {- let scrapeStateFailed = false;- const [nuqJob, scrapeState, dbScrape, gcsJob] = await Promise.all([+ let stateReadError: unknown = null;+ const [nuqJob, scrapeState, gcsJob] = await Promise.all([scrapeQueue.getJob(id,_logger,) as Promise<NuQJob<ScrapeJobSingleUrls> | null>,readScrapeJobState(id).catch(error => {- _logger.warn("Bigtable scrape state read failed; using legacy lookup", {+ _logger.warn("Bigtable scrape state read failed", {error,scrapeId: id,});- scrapeStateFailed = true;+ stateReadError = error;return null;}),- (config.USE_DB_AUTHENTICATION- ? supabaseGetScrapeById(id)- : null) as Promise<DBScrape | null>,(config.GCS_BUCKET_NAME ? getJobFromGCS(id) : null) as Promise<any | null>,]);- if (!nuqJob && !scrapeState && !dbScrape) return null;- if (!nuqJob && !scrapeState && !scrapeStateFailed && dbScrape) {- recordJobStorePostgresFallback("scrape_state", id);+ if (!nuqJob && !scrapeState) {+ // With no NuQ job, Bigtable is the only place a finished job's state+ // lives. A failed read is an outage, not a missing job: surface it rather+ // than answer 404 for a job that exists.+ if (stateReadError) throw stateReadError;+ return null;}if (nuqJob && nuqJob.data.mode !== "single_urls") {@@ -100,24 +87,13 @@ export async function getJob(const job: PseudoJob<any> = {id,- status:- scrapeState?.status ??- (dbScrape ? (dbScrape.success ? "completed" : "failed") : nuqJob!.status),+ status: scrapeState?.status ?? nuqJob!.status,returnvalue: Array.isArray(data) ? data[0] : data,data: {- scrapeOptions: nuqJob- ? nuqJob.data.scrapeOptions- : (dbScrape?.options ?? null),+ scrapeOptions: nuqJob ? nuqJob.data.scrapeOptions : null,},- timestamp:- scrapeState?.completedAtMs ??- (nuqJob- ? nuqJob.createdAt.valueOf()- : new Date(dbScrape!.created_at).valueOf()),- failedReason:- (scrapeState?.error ??- (nuqJob ? nuqJob.failedReason : dbScrape?.error)) ||- undefined,+ timestamp: scrapeState?.completedAtMs ?? nuqJob!.createdAt.valueOf(),+ failedReason: (scrapeState?.error ?? nuqJob?.failedReason) || undefined,};return job;@@ -222,31 +198,22 @@ export async function crawlStatusController(logger.child({ zeroDataRetention }),);- let creditsReadFailed = false;+ // A failed Bigtable read propagates: during an outage the analytics sum+ // could be behind by a second of ClickPipes lag and under-report credits.let creditsBilled = await readRequestCredits(sc?.requestId ?? req.params.jobId,- ).catch(() => {- creditsReadFailed = true;- return null;- });+ );if (creditsBilled === null) {// Requests from before the Bigtable credit rows existed: sum the scrape// job log instead.creditsBilled = await readRequestCreditsFromAnalytics(- req.params.jobId,+ sc?.requestId ?? req.params.jobId,+ { emptyAsZero: true },).catch(error => {logger.warn("Analytics request credits read failed", { error });return null;});}- if (creditsBilled === null && config.USE_DB_AUTHENTICATION) {- creditsBilled = await creditsBilledByCrawlId(dbRr, req.params.jobId)- .then(rows => rows[0]?.credits_billed ?? null)- .catch(() => null);- if (creditsBilled !== null && !creditsReadFailed) {- recordJobStorePostgresFallback("request_credits", req.params.jobId);- }- }// check if the crawl failed during kickoff (e.g. queue full)const crawlError = await getCrawlError(req.params.jobId);diff --git a/apps/api/src/controllers/v2/extract-status.ts b/apps/api/src/controllers/v2/extract-status.tsindex 3405f638d..2f50ebce9 100644--- a/apps/api/src/controllers/v2/extract-status.ts+++ b/apps/api/src/controllers/v2/extract-status.ts@@ -1,4 +1,5 @@import { Response } from "express";+import { JOB_ACCESS_TTL_MS } from "../../lib/job-access-store";import { config } from "../../config";import { RequestWithAuth } from "./types";import {@@ -6,14 +7,12 @@ import {getExtractExpiry,getExtractResult,} from "../../lib/extract/extract-redis";-import { supabaseGetExtractByIdDirect } from "../../lib/supabase-jobs";import { logger as _logger } from "../../lib/logger";import { getJobFromGCS } from "../../lib/gcs-jobs";import { getExtractJobAccess } from "../../lib/operational-job-access";import { readExtractJobState } from "../../lib/job-state-store";import { normalizeJobAccessTeamId } from "../../lib/job-access-store";import { getExtractV3AgentStatus } from "../../lib/extract-v3-status";-import { recordJobStorePostgresFallback } from "../../lib/job-store-fallback";async function getExtractData(id: string): Promise<any> {// Try GCS first if configured@@ -67,51 +66,26 @@ export async function extractStatusController(// Get extract status from Redis (for in-progress jobs)const redisExtract = await getExtract(req.params.jobId);- // If not in Redis, check the database for completed jobs+ // Not in Redis: finished, or still being set up. Bigtable holds the+ // terminal state for 24 hours; a request row without either is in flight.if (!redisExtract) {- if (config.USE_DB_AUTHENTICATION) {- let stateReadFailed = false;- const state = await readExtractJobState(req.params.jobId).catch(error => {- _logger.warn(- "Bigtable extract state read failed; using legacy lookup",- { error, extractId: req.params.jobId },- );- stateReadFailed = true;- return null;+ // A failed Bigtable read is an outage, not a job in flight: it propagates+ // to the error handler rather than answering "processing".+ const state = await readExtractJobState(req.params.jobId);+ if (state) {+ return res.status(200).json({+ success: state.status === "completed",+ data:+ state.status === "completed"+ ? await getExtractData(req.params.jobId)+ : [],+ status: state.status,+ error: state.error,+ expiresAt: new Date(+ access?.expiresAtMs ?? state.completedAtMs + JOB_ACCESS_TTL_MS,+ ).toISOString(),+ creditsUsed: state.creditsBilled,});- if (state) {- return res.status(200).json({- success: state.status === "completed",- data:- state.status === "completed"- ? await getExtractData(req.params.jobId)- : [],- status: state.status,- error: state.error,- expiresAt: new Date(access!.expiresAtMs).toISOString(),- creditsUsed: state.creditsBilled,- });- }-- const dbExtract = await supabaseGetExtractByIdDirect(req.params.jobId);- if (dbExtract) {- if (!stateReadFailed) {- recordJobStorePostgresFallback("extract_state", req.params.jobId);- }- // Get result data- let data: any = [];- if (dbExtract.is_successful) {- data = await getExtractData(req.params.jobId);- }-- return res.status(200).json({- success: dbExtract.is_successful,- data,- status: dbExtract.is_successful ? "completed" : "failed",- error: dbExtract.error || undefined,- expiresAt: new Date(access!.expiresAtMs).toISOString(),- });- }}// Fall back to extractRequest info@@ -119,7 +93,9 @@ export async function extractStatusController(success: true,data: [],status: "processing",- expiresAt: new Date(access!.expiresAtMs).toISOString(),+ expiresAt: new Date(+ access?.expiresAtMs ?? Date.now() + JOB_ACCESS_TTL_MS,+ ).toISOString(),});}diff --git a/apps/api/src/controllers/v2/feedback/feedback-store.ts b/apps/api/src/controllers/v2/feedback/feedback-store.tsindex 357400eb7..d0f84580e 100644--- a/apps/api/src/controllers/v2/feedback/feedback-store.ts+++ b/apps/api/src/controllers/v2/feedback/feedback-store.ts@@ -13,7 +13,6 @@ import {FeedbackRecordOptions,RefundPolicySnapshot,} from "./internal-types";-import { recordJobStorePostgresFallback } from "../../../lib/job-store-fallback";type DbError = { code?: string } & Record<string, unknown>;@@ -22,13 +21,6 @@ type ExistingFeedback = {credits_refunded: number | null;};-const JOB_TABLES = {- search: schema.searches,- scrape: schema.scrapes,- parse: schema.parses,- map: schema.maps,-} as const;-function feedbackMetadata(options: FeedbackRecordOptions,): Record<string, unknown> {@@ -46,74 +38,40 @@ export async function lookupFeedbackJob(jobId: string,dbTeamId: string,): Promise<FeedbackJobRow | null> {- let bigtableFailed = false;+ let job;try {- const job = await readFeedbackJob(jobId);- if (job) {- const storedEndpoint = endpointForRefundClass(job.refundClass);- if (job.teamId !== dbTeamId || storedEndpoint !== endpoint) return null;-- const feedbackWindowSec =- endpoint === "search"- ? config.SEARCH_FEEDBACK_MAX_AGE_SEC- : config.FEEDBACK_MAX_AGE_SEC;- return {- endpoint,- id: jobId,- request_id: job.requestId,- team_id: job.teamId,- credits_cost: job.creditsBilled,- created_at: new Date(- job.feedbackDeadlineMs - feedbackWindowSec * 1000,- ).toISOString(),- is_successful: job.succeeded,- options: null,- feedback_deadline_ms: job.feedbackDeadlineMs,- refund_class: job.refundClass,- zero_data_retention: job.zeroDataRetention,- };- }+ job = await readFeedbackJob(jobId);} catch (error) {- bigtableFailed = true;- logger.warn(- "Bigtable feedback job read failed; falling back to PostgreSQL",- {- error,- jobId,- endpoint,- },- );+ logger.warn("Bigtable feedback job read failed", {+ error,+ jobId,+ endpoint,+ });+ return null;}+ if (!job) return null;- const table = JOB_TABLES[endpoint] as any;- const [row] = await dbRr- .select({- id: table.id,- request_id: table.request_id,- team_id: table.team_id,- credits_cost: table.credits_cost,- created_at: table.created_at,- options: table.options,- ...(endpoint === "map" ? {} : { is_successful: table.is_successful }),- })- .from(table)- .where(and(eq(table.id, jobId), eq(table.team_id, dbTeamId)))- .limit(1);-- if (!row) return null;- if (!bigtableFailed) {- recordJobStorePostgresFallback("feedback_job", jobId, { endpoint });- }+ const storedEndpoint = endpointForRefundClass(job.refundClass);+ if (job.teamId !== dbTeamId || storedEndpoint !== endpoint) return null;+ const feedbackWindowSec =+ endpoint === "search"+ ? config.SEARCH_FEEDBACK_MAX_AGE_SEC+ : config.FEEDBACK_MAX_AGE_SEC;return {endpoint,- id: row.id,- request_id: row.request_id ?? null,- team_id: row.team_id,- credits_cost: row.credits_cost ?? 0,- created_at: row.created_at,- is_successful: endpoint === "map" ? true : (row.is_successful ?? null),- options: row.options ?? null,+ id: jobId,+ request_id: job.requestId,+ team_id: job.teamId,+ credits_cost: job.creditsBilled,+ created_at: new Date(+ job.feedbackDeadlineMs - feedbackWindowSec * 1000,+ ).toISOString(),+ is_successful: job.succeeded,+ options: null,+ feedback_deadline_ms: job.feedbackDeadlineMs,+ refund_class: job.refundClass,+ zero_data_retention: job.zeroDataRetention,};}diff --git a/apps/api/src/controllers/v2/scrape-browser.ts b/apps/api/src/controllers/v2/scrape-browser.tsindex be49a1e34..4abb2e1ec 100644--- a/apps/api/src/controllers/v2/scrape-browser.ts+++ b/apps/api/src/controllers/v2/scrape-browser.ts@@ -22,7 +22,6 @@ import {} from "../../lib/browser-lifecycle";import { browserCreateRequestSchema, browserError } from "./browser";import {- ScrapeContextRow,buildReplayContextFromScrape,estimateReplayTimeoutSeconds,buildReplayScript,@@ -47,12 +46,10 @@ import {} from "../../lib/keyless";import { enqueueBrowserSessionActivity } from "../../lib/browser-session-activity";import { integrationSchema } from "../../utils/integration";-import { supabaseGetScrapeByIdDirect } from "../../lib/supabase-jobs";import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";import { getScrapeJobAccess } from "../../lib/operational-job-access";import { readScrapeJobState } from "../../lib/job-state-store";import { scrapeQueue } from "../../services/worker/nuq-router";-import { recordJobStorePostgresFallback } from "../../lib/job-store-fallback";const browserExecuteRequestSchema = z.object({@@ -145,14 +142,10 @@ export async function scrapeInteractController(});}- let stateReadFailed = false;const [nuqJob, state] = await Promise.all([scrapeQueue.getJob(scrapeId, logger),readScrapeJobState(scrapeId).catch(error => {- logger.warn("Bigtable scrape state read failed; using legacy lookup", {- error,- });- stateReadFailed = true;+ logger.warn("Bigtable scrape state read failed", { error });return null;}),]);@@ -179,7 +172,6 @@ export async function scrapeInteractController(let replayContext = state?.replay;let replayError: string | undefined;- let legacyScrape: ScrapeContextRow | null = null;if (!replayContext && nuqJob?.data.mode === "single_urls") {const replay = buildReplayContextFromScrape({id: scrapeId,@@ -190,21 +182,6 @@ export async function scrapeInteractController(replayContext = replay.context;replayError = replay.error;}- if (!replayContext) {- legacyScrape = (await supabaseGetScrapeByIdDirect(- scrapeId,- )) as ScrapeContextRow | null;- if (legacyScrape && !stateReadFailed) {- recordJobStorePostgresFallback("scrape_state", scrapeId, {- reason: "replay_context",- });- }- const replay = legacyScrape- ? buildReplayContextFromScrape(legacyScrape)- : { error: "Replay context is unavailable for this scrape job." };- replayContext = replay.context;- replayError = replay.error;- }if (!replayContext) {return res.status(409).json({success: false,@@ -246,10 +223,7 @@ export async function scrapeInteractController(state?.profile ??(nuqJob?.data.mode === "single_urls"? nuqJob.data.scrapeOptions.profile- : undefined) ??- ((legacyScrape?.options as ScrapeOptions | undefined)?.profile as- | { name: string; saveChanges: boolean }- | undefined);+ : undefined);const created = await createSessionForScrape(req,scrapeId,@@ -300,20 +274,14 @@ export async function scrapeInteractController(// every run carries the URL / wait / actions / origin that set the stage// for what the agent does on top of it. URLs are stripped of query// strings to avoid leaking PII into LangSmith.- const scrapeOptions = (legacyScrape?.options ?? {}) as {- origin?: string;- };const traceScrapeContext = {- scrapeUrl: sanitizeUrlForTrace(- legacyScrape?.url ?? replayContext.targetUrl,- ),+ scrapeUrl: sanitizeUrlForTrace(replayContext.targetUrl),targetUrl: sanitizeUrlForTrace(replayContext.targetUrl),scrapeWaitForMs: replayContext.waitForMs,scrapeActions: replayContext.actions.length,scrapeOrigin:state?.origin ??- (nuqJob?.data.mode === "single_urls" ? nuqJob.data.origin : undefined) ??- scrapeOptions.origin,+ (nuqJob?.data.mode === "single_urls" ? nuqJob.data.origin : undefined),};// Identity fields below team_id — optional, normalized from null → undefineddiff --git a/apps/api/src/db/rpc.ts b/apps/api/src/db/rpc.tsindex eae478508..c196b12c7 100644--- a/apps/api/src/db/rpc.ts+++ b/apps/api/src/db/rpc.ts@@ -88,20 +88,6 @@ export function billTeam7(params: {);}-// `database` is a parameter because callers split between the primary and the-// read replica: status controllers (informational `creditsUsed`) read from the-// replica, while crawl finalization (crawl-logic) reads its own recent billing-// writes and must stay on the primary.-export function creditsBilledByCrawlId(- database: DB,- i_crawl_id: string,-): Promise<{ credits_billed: number }[]> {- return execRows(- database,- sql`select * from credits_billed_by_crawl_id_2(i_crawl_id => ${i_crawl_id})`,- );-}-export function diffGetLastScrape(i_team_id: string,i_url: string,diff --git a/apps/api/src/lib/feedback-job-store.ts b/apps/api/src/lib/feedback-job-store.tsindex 768bec2db..06b542b09 100644--- a/apps/api/src/lib/feedback-job-store.ts+++ b/apps/api/src/lib/feedback-job-store.ts@@ -6,6 +6,9 @@ import { saltedUuidV7RowKey } from "./bigtable-row-key";import { setSpanAttributes, withSpan } from "./otel-tracer";const FAMILY = "f";+const FEEDBACK_ROW_RETENTION_MS = 24 * 60 * 60 * 1000;+/** Headroom past a configured feedback window longer than the default retention. */+const FEEDBACK_ROW_RETENTION_MARGIN_MS = 60 * 60 * 1000;const QUALIFIER = "v";type FeedbackEndpoint = "search" | "scrape" | "parse" | "map";@@ -160,6 +163,17 @@ export async function writeFeedbackJob(const feedbackDeadline = new Date(completedAt.getTime() + feedbackWindowSec * 1000,);+ // The row outlives the feedback window so late feedback can be told+ // "window expired" rather than "job not found"; the deadline inside the+ // value is what enforces the window. A window configured longer than+ // the default retention still gets a row that outlives it.+ const retainUntil = new Date(+ completedAt.getTime() ++ Math.max(+ FEEDBACK_ROW_RETENTION_MS,+ feedbackWindowSec * 1000 + FEEDBACK_ROW_RETENTION_MARGIN_MS,+ ),+ );const value = Buffer.from(JSON.stringify({version: 1,@@ -184,7 +198,7 @@ export async function writeFeedbackJob([FAMILY]: {[QUALIFIER]: {value,- timestamp: feedbackDeadline,+ timestamp: retainUntil,},},},diff --git a/apps/api/src/lib/job-access-store.ts b/apps/api/src/lib/job-access-store.tsindex 479290dc6..f5c033cfc 100644--- a/apps/api/src/lib/job-access-store.ts+++ b/apps/api/src/lib/job-access-store.ts@@ -4,6 +4,14 @@ import { saltedUuidV7RowKey } from "./bigtable-row-key";import { keylessTeamUuid } from "./keyless";import { setSpanAttributes, withSpan } from "./otel-tracer";+/**+ * Default lifetime of a job's access row, used by writes that do not pass+ * their own `expiresAt` and as the fallback expiry where no row exists. It is+ * not every row's actual expiry: deep research writes a shorter TTL, and+ * crawl and batch scrape write a per-key `crawlTtlHours`.+ */+export const JOB_ACCESS_TTL_MS = 24 * 60 * 60 * 1000;+const FAMILY = "j";const QUALIFIER = "v";const PREVIEW_TEAM_ID = "3adefd26-77ec-5968-8dcf-c94b5630d1de";diff --git a/apps/api/src/lib/job-operational-stores.test.ts b/apps/api/src/lib/job-operational-stores.test.tsindex 51d670ff5..96b28d3ad 100644--- a/apps/api/src/lib/job-operational-stores.test.ts+++ b/apps/api/src/lib/job-operational-stores.test.ts@@ -257,7 +257,7 @@ describe("operational Bigtable stores", () => {zeroDataRetention: false,});expect(mutate.mock.calls[0][0][0].data.f.v.timestamp).toEqual(- new Date(completedAt.getTime() + 180_000),+ new Date(completedAt.getTime() + 24 * 60 * 60 * 1000),);expect(spans).toContainEqual({name: "bigtable.feedback_job.write",@@ -265,6 +265,35 @@ describe("operational Bigtable stores", () => {});});+ it("keeps the row past a feedback window longer than the default retention", async () => {+ const completedAt = new Date("2026-09-15T12:00:00.000Z");+ const original = mutableConfig.FEEDBACK_MAX_AGE_SEC;+ mutableConfig.FEEDBACK_MAX_AGE_SEC = 30 * 60 * 60; // 30 h, past the 24 h default+ try {+ await writeFeedbackJob({+ jobId: JOB_ID,+ requestId: REQUEST_ID,+ teamId: "team-id",+ endpoint: "scrape",+ scrapeOptions: scrapeOptions.parse({}),+ succeeded: true,+ creditsBilled: 1,+ zeroDataRetention: false,+ completedAt,+ });+ } finally {+ mutableConfig.FEEDBACK_MAX_AGE_SEC = original;+ }++ expect(writtenValue()).toMatchObject({+ feedbackDeadlineMs: completedAt.getTime() + 30 * 60 * 60 * 1000,+ });+ // window + one hour of margin, not the flat 24 h+ expect(mutate.mock.calls[0][0][0].data.f.v.timestamp).toEqual(+ new Date(completedAt.getTime() + 31 * 60 * 60 * 1000),+ );+ });+it("uses the search-specific feedback window", async () => {const completedAt = new Date("2026-09-15T12:00:00.000Z");await writeFeedbackJob({diff --git a/apps/api/src/lib/job-store-fallback.test.ts b/apps/api/src/lib/job-store-fallback.test.tsdeleted file mode 100644index 0cc7fe750..000000000--- a/apps/api/src/lib/job-store-fallback.test.ts+++ /dev/null@@ -1,80 +0,0 @@-import { beforeEach, describe, expect, it, vi } from "vitest";--const { info, mutableConfig } = vi.hoisted(() => ({- info: vi.fn(),- mutableConfig: {- BIGTABLE_JOB_ACCESS_TABLE: "api_job_access" as string | undefined,- BIGTABLE_SCRAPE_STATE_TABLE: "scrape_state" as string | undefined,- BIGTABLE_EXTRACT_STATE_TABLE: "extract_state" as string | undefined,- BIGTABLE_FEEDBACK_JOBS_TABLE: "feedback_jobs" as string | undefined,- BIGTABLE_REQUEST_CREDITS_TABLE: "request_credits" as string | undefined,- BIGTABLE_CHANGE_TRACKING_TABLE: "change_tracking" as string | undefined,- },-}));-vi.mock("./logger", () => ({ logger: { info } }));-vi.mock("../config", () => ({ config: mutableConfig }));--import {- jobStorePostgresFallbackTotal,- recordJobStorePostgresFallback,-} from "./job-store-fallback";--async function countsByStore() {- const metric = await jobStorePostgresFallbackTotal.get();- return Object.fromEntries(metric.values.map(v => [v.labels.store, v.value]));-}--describe("job store fallback accounting", () => {- beforeEach(() => {- jobStorePostgresFallbackTotal.reset();- info.mockClear();- mutableConfig.BIGTABLE_SCRAPE_STATE_TABLE = "scrape_state";- });-- it("counts and logs one hit per store", async () => {- recordJobStorePostgresFallback("scrape_state", "job-1");- recordJobStorePostgresFallback("scrape_state", "job-2");- recordJobStorePostgresFallback("job_access", "job-3", { kind: "crawl" });-- expect(await countsByStore()).toMatchObject({- scrape_state: 2,- job_access: 1,- });-- expect(info).toHaveBeenCalledTimes(3);- expect(info).toHaveBeenLastCalledWith(- "PostgreSQL fallback served a job-store read",- expect.objectContaining({- store: "job_access",- id: "job-3",- kind: "crawl",- }),- );- });-- it("lets the fixed log fields win over extra context", () => {- recordJobStorePostgresFallback("scrape_state", "job-1", {- id: "spoofed",- store: "spoofed",- module: "spoofed",- });-- expect(info).toHaveBeenCalledWith(- "PostgreSQL fallback served a job-store read",- expect.objectContaining({- module: "job-store-fallback",- store: "scrape_state",- id: "job-1",- }),- );- });-- it("records nothing when the store is not configured", async () => {- mutableConfig.BIGTABLE_SCRAPE_STATE_TABLE = undefined;-- recordJobStorePostgresFallback("scrape_state", "job-1");-- expect(await countsByStore()).toEqual({});- expect(info).not.toHaveBeenCalled();- });-});diff --git a/apps/api/src/lib/job-store-fallback.ts b/apps/api/src/lib/job-store-fallback.tsdeleted file mode 100644index 0d8451e85..000000000--- a/apps/api/src/lib/job-store-fallback.ts+++ /dev/null@@ -1,63 +0,0 @@-import { Counter, register } from "prom-client";-import { config } from "../config";-import { logger } from "./logger";--const NAME = "job_store_postgres_fallback_total";--/**- * Reads the new job stores (Bigtable, NuQ) could not answer and PostgreSQL- * did. Each label is one fallback path; a store is ready to lose its- * fallback once its series has stayed flat for the agreed window.- *- * Looked up before creation so a re-evaluated module (test isolation) does- * not register the same series twice in the shared default registry.- */-export const jobStorePostgresFallbackTotal =- (register.getSingleMetric(NAME) as Counter<"store"> | undefined) ??- new Counter({- name: NAME,- help: "Job-store reads answered by the PostgreSQL fallback, by store",- labelNames: ["store"] as const,- });--type JobStoreFallback =- | "job_access"- | "scrape_state"- | "extract_state"- | "feedback_job"- | "request_credits";--/**- * The Bigtable table behind each store. When it is not configured the- * primary store never answers, PostgreSQL is the only store, and a hit is- * not a fallback worth counting.- */-const STORE_TABLE: Record<JobStoreFallback, () => string | undefined> = {- job_access: () => config.BIGTABLE_JOB_ACCESS_TABLE,- scrape_state: () => config.BIGTABLE_SCRAPE_STATE_TABLE,- extract_state: () => config.BIGTABLE_EXTRACT_STATE_TABLE,- feedback_job: () => config.BIGTABLE_FEEDBACK_JOBS_TABLE,- request_credits: () => config.BIGTABLE_REQUEST_CREDITS_TABLE,-};--/**- * Record that PostgreSQL served a row the primary store did not have. Call- * it only on a genuine miss: a miss on both sides is a normal "not found",- * and a primary-store *error* is already reported where it is caught and- * must not keep this series hot. Deployments without the store configured- * record nothing.- */-export function recordJobStorePostgresFallback(- store: JobStoreFallback,- id: string,- extra: Record<string, unknown> = {},-): void {- if (!STORE_TABLE[store]()) return;- jobStorePostgresFallbackTotal.inc({ store });- logger.info("PostgreSQL fallback served a job-store read", {- ...extra,- module: "job-store-fallback",- store,- id,- });-}diff --git a/apps/api/src/lib/operational-job-access.test.ts b/apps/api/src/lib/operational-job-access.test.tsindex 588a7e21a..9a074393c 100644--- a/apps/api/src/lib/operational-job-access.test.ts+++ b/apps/api/src/lib/operational-job-access.test.ts@@ -1,33 +1,15 @@-const {- readApiJobAccess,- recordJobStorePostgresFallback,- supabaseGetAgentRequestByIdDirect,- supabaseGetCrawlRequestById,- supabaseGetExtractRequestByIdDirect,- supabaseGetScrapeById,-} = vi.hoisted(() => ({+const { readApiJobAccess } = vi.hoisted(() => ({readApiJobAccess: vi.fn(),- recordJobStorePostgresFallback: vi.fn(),- supabaseGetAgentRequestByIdDirect: vi.fn(),- supabaseGetCrawlRequestById: vi.fn(),- supabaseGetExtractRequestByIdDirect: vi.fn(),- supabaseGetScrapeById: vi.fn(),}));vi.mock("./job-access-store", () => ({ readApiJobAccess }));-vi.mock("./job-store-fallback", () => ({ recordJobStorePostgresFallback }));vi.mock("./logger", () => ({logger: { warn: vi.fn(), info: vi.fn(), error: vi.fn() },}));-vi.mock("./supabase-jobs", () => ({- supabaseGetAgentRequestByIdDirect,- supabaseGetCrawlRequestById,- supabaseGetExtractRequestByIdDirect,- supabaseGetScrapeById,-}));import {getAgentJobAccess,+ getCrawlJobAccess,getExtractJobAccess,getScrapeJobAccess,} from "./operational-job-access";@@ -37,7 +19,7 @@ const JOB_ID = "019e6f45-7778-727d-adf0-0abe9d5062b6";describe("operational job access", () => {beforeEach(() => vi.clearAllMocks());- it("returns the typed Bigtable record without querying PostgreSQL", async () => {+ it("returns the typed Bigtable record", async () => {const access = {teamId: "team-id",kind: "agent",@@ -47,63 +29,23 @@ describe("operational job access", () => {readApiJobAccess.mockResolvedValue(access);await expect(getAgentJobAccess(JOB_ID)).resolves.toBe(access);- expect(supabaseGetAgentRequestByIdDirect).not.toHaveBeenCalled();});- it("maps a live PostgreSQL fallback into the operational access type and counts it", async () => {- const createdAt = new Date(Date.now() - 60_000);- readApiJobAccess.mockResolvedValue(null);- supabaseGetExtractRequestByIdDirect.mockResolvedValue({- id: JOB_ID,- team_id: "team-id",+ it("extract access accepts extract and agent records and rejects a scrape", async () => {+ const base = { teamId: "team-id", expiresAtMs: Date.now() + 60_000 };+ readApiJobAccess.mockResolvedValueOnce({ ...base, kind: "extract" });+ await expect(getExtractJobAccess(JOB_ID)).resolves.toMatchObject({kind: "extract",- origin: "api",- created_at: createdAt.toISOString(),- unrelated_column: "not returned",- });-- await expect(getExtractJobAccess(JOB_ID)).resolves.toEqual({- teamId: "team-id",- kind: "extract",- clientOrigin: "api",- expiresAtMs: createdAt.getTime() + 24 * 60 * 60 * 1000,- });- expect(recordJobStorePostgresFallback).toHaveBeenCalledWith(- "job_access",- JOB_ID,- { kind: "extract" },- );- });-- it("does not count an expired PostgreSQL fallback (the caller 404s either way)", async () => {- readApiJobAccess.mockResolvedValue(null);- supabaseGetScrapeById.mockResolvedValue({- id: JOB_ID,- team_id: "team-id",- created_at: new Date(Date.now() - 25 * 60 * 60 * 1000).toISOString(),});-- const access = await getScrapeJobAccess(JOB_ID);- expect(access?.expiresAtMs).toBeLessThan(Date.now());- expect(recordJobStorePostgresFallback).not.toHaveBeenCalled();- });-- it("does not count a fallback taken because the Bigtable read failed", async () => {- readApiJobAccess.mockRejectedValue(new Error("Bigtable unavailable"));- supabaseGetScrapeById.mockResolvedValue({- id: JOB_ID,- team_id: "team-id",- created_at: new Date().toISOString(),- });-- await expect(getScrapeJobAccess(JOB_ID)).resolves.toMatchObject({- teamId: "team-id",- kind: "scrape",+ readApiJobAccess.mockResolvedValueOnce({ ...base, kind: "agent" });+ await expect(getExtractJobAccess(JOB_ID)).resolves.toMatchObject({+ kind: "agent",});- expect(recordJobStorePostgresFallback).not.toHaveBeenCalled();+ readApiJobAccess.mockResolvedValueOnce({ ...base, kind: "scrape" });+ await expect(getExtractJobAccess(JOB_ID)).resolves.toBeNull();});- it("does not fall back when Bigtable has an expired record", async () => {+ it("returns an expired record so the caller can answer 404 with its expiry", async () => {const access = {teamId: "team-id",kind: "scrape",@@ -112,10 +54,9 @@ describe("operational job access", () => {readApiJobAccess.mockResolvedValue(access);await expect(getScrapeJobAccess(JOB_ID)).resolves.toBe(access);- expect(supabaseGetScrapeById).not.toHaveBeenCalled();});- it("does not fall back when Bigtable has a different job kind", async () => {+ it("returns null for a different job kind", async () => {readApiJobAccess.mockResolvedValue({teamId: "team-id",kind: "crawl",@@ -123,6 +64,34 @@ describe("operational job access", () => {});await expect(getScrapeJobAccess(JOB_ID)).resolves.toBeNull();- expect(supabaseGetScrapeById).not.toHaveBeenCalled();+ await expect(getCrawlJobAccess(JOB_ID)).resolves.toMatchObject({+ kind: "crawl",+ });+ });++ it("crawl access accepts crawl and batch_scrape records and rejects a scrape", async () => {+ const base = { teamId: "team-id", expiresAtMs: Date.now() + 60_000 };+ readApiJobAccess.mockResolvedValueOnce({ ...base, kind: "crawl" });+ await expect(getCrawlJobAccess(JOB_ID)).resolves.toMatchObject({+ kind: "crawl",+ });+ readApiJobAccess.mockResolvedValueOnce({ ...base, kind: "batch_scrape" });+ await expect(getCrawlJobAccess(JOB_ID)).resolves.toMatchObject({+ kind: "batch_scrape",+ });+ readApiJobAccess.mockResolvedValueOnce({ ...base, kind: "scrape" });+ await expect(getCrawlJobAccess(JOB_ID)).resolves.toBeNull();+ });++ it("returns null when Bigtable has no row", async () => {+ readApiJobAccess.mockResolvedValue(null);+ await expect(getScrapeJobAccess(JOB_ID)).resolves.toBeNull();+ });++ it("rethrows a failed Bigtable read instead of reporting a missing job", async () => {+ readApiJobAccess.mockRejectedValue(new Error("Bigtable unavailable"));+ await expect(getScrapeJobAccess(JOB_ID)).rejects.toThrow(+ "Bigtable unavailable",+ );});});diff --git a/apps/api/src/lib/operational-job-access.ts b/apps/api/src/lib/operational-job-access.tsindex f36bd7f98..353321b77 100644--- a/apps/api/src/lib/operational-job-access.ts+++ b/apps/api/src/lib/operational-job-access.ts@@ -4,15 +4,6 @@ import {type ApiJobAccess,type ApiJobKind,} from "./job-access-store";-import {- supabaseGetAgentRequestByIdDirect,- supabaseGetCrawlRequestById,- supabaseGetExtractRequestByIdDirect,- supabaseGetScrapeById,-} from "./supabase-jobs";-import { recordJobStorePostgresFallback } from "./job-store-fallback";--const DEFAULT_TTL_MS = 24 * 60 * 60 * 1000;type OperationalJobAccess = {teamId: string;@@ -21,114 +12,52 @@ type OperationalJobAccess = {expiresAtMs: number;};+/**+ * Who owns a job and until when it may be fetched, from the Bigtable job+ * access row that logRequest (and, for crawl and batch children, logScrape)+ * writes. A missing or expired row is the documented end of the job's+ * fetchable life; there is no other store to ask. A failed read is an+ * outage, not a missing job: it is rethrown so the caller answers an error+ * rather than a 404 for a job that exists.+ */async function resolveOperationalJobAccess(params: {id: string;kinds: readonly ApiJobKind[];- fallback: () => Promise<OperationalJobAccess | null>;}): Promise<OperationalJobAccess | null> {- let access: ApiJobAccess | null = null;- let readFailed = false;+ let access: ApiJobAccess | null;try {access = await readApiJobAccess(params.id);} catch (error) {- readFailed = true;- logger.warn("Bigtable job access read failed; using legacy lookup", {+ logger.error("Bigtable job access read failed", {error,jobId: params.id,});+ throw error;}-- if (access) {- return params.kinds.includes(access.kind) ? access : null;- }-- const fallback = await params.fallback();- if (!fallback || !Number.isFinite(fallback.expiresAtMs)) return null;- // An expired row makes the caller answer 404 exactly as a miss would, so- // only a live row counts as PostgreSQL having been needed.- if (!readFailed && fallback.expiresAtMs > Date.now()) {- recordJobStorePostgresFallback("job_access", params.id, {- kind: fallback.kind,- });- }- return fallback;+ if (!access) return null;+ return params.kinds.includes(access.kind) ? access : null;}export function getScrapeJobAccess(id: string,): Promise<OperationalJobAccess | null> {- return resolveOperationalJobAccess({- id,- kinds: ["scrape"],- fallback: async () => {- const row = await supabaseGetScrapeById(id);- return row- ? {- teamId: row.team_id,- kind: "scrape",- expiresAtMs: new Date(row.created_at).getTime() + DEFAULT_TTL_MS,- }- : null;- },- });+ return resolveOperationalJobAccess({ id, kinds: ["scrape"] });}export function getExtractJobAccess(id: string,): Promise<OperationalJobAccess | null> {- return resolveOperationalJobAccess({- id,- kinds: ["extract", "agent"],- fallback: async () => {- const row = await supabaseGetExtractRequestByIdDirect(id);- if (!row || (row.kind !== "extract" && row.kind !== "agent")) return null;- return {- teamId: row.team_id,- kind: row.kind,- clientOrigin: row.origin ?? undefined,- expiresAtMs: new Date(row.created_at).getTime() + DEFAULT_TTL_MS,- };- },- });+ return resolveOperationalJobAccess({ id, kinds: ["extract", "agent"] });}export function getAgentJobAccess(id: string,): Promise<OperationalJobAccess | null> {- return resolveOperationalJobAccess({- id,- kinds: ["agent"],- fallback: async () => {- const row = await supabaseGetAgentRequestByIdDirect(id);- if (!row) return null;- return {- teamId: row.team_id,- kind: "agent",- clientOrigin: row.origin ?? undefined,- expiresAtMs: new Date(row.created_at).getTime() + DEFAULT_TTL_MS,- };- },- });+ return resolveOperationalJobAccess({ id, kinds: ["agent"] });}export function getCrawlJobAccess(id: string,- ttlHours: number,): Promise<OperationalJobAccess | null> {- return resolveOperationalJobAccess({- id,- kinds: ["crawl", "batch_scrape"],- fallback: async () => {- const row = await supabaseGetCrawlRequestById(id);- if (!row || (row.kind !== "crawl" && row.kind !== "batch_scrape")) {- return null;- }- return {- teamId: row.team_id,- kind: row.kind,- expiresAtMs:- new Date(row.created_at!).getTime() + ttlHours * 60 * 60 * 1000,- };- },- });+ return resolveOperationalJobAccess({ id, kinds: ["crawl", "batch_scrape"] });}diff --git a/apps/api/src/lib/request-credits-analytics.test.ts b/apps/api/src/lib/request-credits-analytics.test.tsindex 9c5b18208..2c72bc124 100644--- a/apps/api/src/lib/request-credits-analytics.test.ts+++ b/apps/api/src/lib/request-credits-analytics.test.ts@@ -33,9 +33,9 @@ describe("readRequestCreditsFromAnalytics", () => {it("sums the request's scrape rows", async () => {rows([{ credits: "17", jobs: "4" }]);- await expect(readRequestCreditsFromAnalytics("request-1")).resolves.toBe(- 17,- );+ await expect(+ readRequestCreditsFromAnalytics("request-1", { emptyAsZero: true }),+ ).resolves.toBe(17);expect(query.mock.calls[0][0]).toMatchObject({query: expect.stringContaining("FROM scrapes_by_request FINAL WHERE request_id = {requestId: UUID}",@@ -44,25 +44,32 @@ describe("readRequestCreditsFromAnalytics", () => {});});- it("is null when the request has no scrape rows", async () => {+ it("reads no scrape rows as zero for a status answer", async () => {+ rows([{ credits: 0, jobs: 0 }]);+ await expect(+ readRequestCreditsFromAnalytics("request-1", { emptyAsZero: true }),+ ).resolves.toBe(0);+ });++ it("reads no scrape rows as unknown for finalization", async () => {rows([{ credits: 0, jobs: 0 }]);await expect(- readRequestCreditsFromAnalytics("request-1"),+ readRequestCreditsFromAnalytics("request-1", { emptyAsZero: false }),).resolves.toBeNull();});it("is null when ClickHouse is not configured", async () => {client.current = null;await expect(- readRequestCreditsFromAnalytics("request-1"),+ readRequestCreditsFromAnalytics("request-1", { emptyAsZero: true }),).resolves.toBeNull();expect(query).not.toHaveBeenCalled();});it("propagates ClickHouse errors", async () => {query.mockRejectedValueOnce(new Error("ClickHouse unavailable"));- await expect(readRequestCreditsFromAnalytics("request-1")).rejects.toThrow(- "ClickHouse unavailable",- );+ await expect(+ readRequestCreditsFromAnalytics("request-1", { emptyAsZero: true }),+ ).rejects.toThrow("ClickHouse unavailable");});});diff --git a/apps/api/src/lib/request-credits-analytics.ts b/apps/api/src/lib/request-credits-analytics.tsindex 192f9fdbe..fd673a0a3 100644--- a/apps/api/src/lib/request-credits-analytics.ts+++ b/apps/api/src/lib/request-credits-analytics.ts@@ -12,11 +12,20 @@ import { setSpanAttributes, withSpan } from "./otel-tracer";* `FINAL` collapses any redelivered copy, so the sum matches what the job log* recorded.*- * Returns null when ClickHouse is not configured or the request has no scrape- * rows at all, so callers can tell "nothing billed yet" from "unknown".+ * ClickPipes lands rows about a second after the worker logs them, so a+ * request whose scrapes are still in flight can read as having none. What+ * that means depends on the caller:+ *+ * - a status endpoint answering a customer treats it as 0 (nothing billed+ * yet), the same answer the PostgreSQL sum gave (COALESCE(SUM, 0));+ * - crawl finalization, which records `credits_cost` for good, treats it as+ * unknown (null) rather than writing a zero that a second's lag produced.+ *+ * An unconfigured ClickHouse client always yields null.*/export async function readRequestCreditsFromAnalytics(requestId: string,+ options: { emptyAsZero: boolean },): Promise<number | null> {const client = clickhouseClient;if (client === null) return null;@@ -38,9 +47,12 @@ export async function readRequestCreditsFromAnalytics(jobs: number | string;}>();const jobs = Number(row?.jobs ?? 0);- if (!Number.isFinite(jobs) || jobs === 0) {- setSpanAttributes(span, { "request_credits.outcome": "not_found" });- return null;+ if (!Number.isSafeInteger(jobs)) {+ throw new Error(`Invalid analytics job count: ${row?.jobs}`);+ }+ if (jobs === 0) {+ setSpanAttributes(span, { "request_credits.outcome": "no_jobs" });+ return options.emptyAsZero ? 0 : null;}const credits = Number(row.credits);if (!Number.isSafeInteger(credits)) {diff --git a/apps/api/src/lib/scrape-interact/scrape-replay.ts b/apps/api/src/lib/scrape-interact/scrape-replay.tsindex 6f7fa4b3c..6b138d4b2 100644--- a/apps/api/src/lib/scrape-interact/scrape-replay.ts+++ b/apps/api/src/lib/scrape-interact/scrape-replay.ts@@ -4,7 +4,7 @@ import { rewriteUrl } from "../../scraper/scrapeURL/lib/rewriteUrl";// Types// ----------------------------------------------------------------------------export interface ScrapeContextRow {+interface ScrapeContextRow {id: string;team_id: string;url: string | null;diff --git a/apps/api/src/lib/supabase-jobs.test.ts b/apps/api/src/lib/supabase-jobs.test.tsdeleted file mode 100644index a0842f12d..000000000--- a/apps/api/src/lib/supabase-jobs.test.ts+++ /dev/null@@ -1,63 +0,0 @@-const { primarySelect, replicaSelect } = vi.hoisted(() => ({- primarySelect: vi.fn(),- replicaSelect: vi.fn(),-}));--vi.mock("../db/connection", () => ({- db: { select: primarySelect },- dbRr: { select: replicaSelect },-}));--import { supabaseGetScrapeByIdDirect } from "./supabase-jobs";-import { logger } from "./logger";--describe("supabaseGetScrapeByIdDirect", () => {- beforeEach(() => {- vi.clearAllMocks();- });-- afterEach(() => {- vi.restoreAllMocks();- });-- it("reads newly-created scrapes from the primary database", async () => {- const scrape = { id: "scrape-123" };- const query = {- from: vi.fn(),- where: vi.fn(),- limit: vi.fn().mockResolvedValue([scrape]),- };- query.from.mockReturnValue(query);- query.where.mockReturnValue(query);- primarySelect.mockReturnValue(query);-- await expect(supabaseGetScrapeByIdDirect(scrape.id)).resolves.toBe(scrape);-- expect(primarySelect).toHaveBeenCalledOnce();- expect(replicaSelect).not.toHaveBeenCalled();- });-- it("surfaces primary database errors instead of returning a false miss", async () => {- const error = new Error("primary unavailable");- const query = {- from: vi.fn(),- where: vi.fn(),- limit: vi.fn().mockRejectedValue(error),- };- query.from.mockReturnValue(query);- query.where.mockReturnValue(query);- primarySelect.mockReturnValue(query);- const logError = vi.spyOn(logger, "error").mockImplementation(() => logger);-- await expect(supabaseGetScrapeByIdDirect("scrape-123")).rejects.toBe(error);-- expect(logError).toHaveBeenCalledWith(- "Error in supabaseGetScrapeByIdDirect",- {- error,- scrapeId: "scrape-123",- },- );- expect(replicaSelect).not.toHaveBeenCalled();- });-});diff --git a/apps/api/src/lib/supabase-jobs.ts b/apps/api/src/lib/supabase-jobs.tsdeleted file mode 100644index e75bdc9e5..000000000--- a/apps/api/src/lib/supabase-jobs.ts+++ /dev/null@@ -1,115 +0,0 @@-import { eq, inArray, and } from "drizzle-orm";-import { db, dbRr } from "../db/connection";-import * as schema from "../db/schema";-import { logger } from "./logger";--/**- * Get a single scrape by ID from the scrapes table- * @param scrapeId ID of Scrape- * @returns Scrape data or null- */-export const supabaseGetScrapeById = async (scrapeId: string): Promise<any> => {- try {- const [data] = await dbRr- .select()- .from(schema.scrapes)- .where(eq(schema.scrapes.id, scrapeId))- .limit(1);- return data ?? null;- } catch (error) {- return null;- }-};--/**- * Get a single scrape by ID from the primary database.- * Use this when the scrape may have been created immediately before the read.- */-export const supabaseGetScrapeByIdDirect = async (- scrapeId: string,-): Promise<any> => {- try {- const [data] = await db- .select()- .from(schema.scrapes)- .where(eq(schema.scrapes.id, scrapeId))- .limit(1);- return data ?? null;- } catch (error) {- logger.error("Error in supabaseGetScrapeByIdDirect", {- error,- scrapeId,- });- throw error;- }-};--export const supabaseGetExtractByIdDirect = async (- extractId: string,-): Promise<any> => {- try {- const [data] = await db- .select()- .from(schema.extracts)- .where(eq(schema.extracts.id, extractId))- .limit(1);- return data ?? null;- } catch (error) {- return null;- }-};--export const supabaseGetExtractRequestByIdDirect = async (- extractId: string,-): Promise<typeof schema.requests.$inferSelect | null> => {- try {- const [data] = await db- .select()- .from(schema.requests)- .where(- and(- eq(schema.requests.id, extractId),- inArray(schema.requests.kind, ["extract", "agent"]),- ),- )- .limit(1);- return data ?? null;- } catch (error) {- return null;- }-};--export const supabaseGetAgentRequestByIdDirect = async (- agentId: string,-): Promise<typeof schema.requests.$inferSelect | null> => {- try {- const [data] = await db- .select()- .from(schema.requests)- .where(- and(eq(schema.requests.id, agentId), eq(schema.requests.kind, "agent")),- )- .limit(1);- return data ?? null;- } catch (error) {- return null;- }-};--export const supabaseGetCrawlRequestById = async (requestId: string) => {- const [data] = await dbRr- .select({- team_id: schema.requests.team_id,- kind: schema.requests.kind,- created_at: schema.requests.created_at,- })- .from(schema.requests)- .where(- and(- eq(schema.requests.id, requestId),- inArray(schema.requests.kind, ["crawl", "batch_scrape"]),- ),- )- .limit(1);- return data ?? null;-};diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex 4666044c6..5d15290c1 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -27,6 +27,7 @@ import { PubSub, type PublishOptions, type Topic } from "@google-cloud/pubsub";import { pubsubLogPublishTotal } from "../../lib/pubsub-log-metrics";import { sanitizeLogData, sanitizeText } from "./sanitize";import {+ JOB_ACCESS_TTL_MS,isApiJobKind,normalizeJobAccessTeamId,writeApiJobAccess,@@ -45,7 +46,7 @@ import {configDotenv();const previewTeamId = "3adefd26-77ec-5968-8dcf-c94b5630d1de";-const DEFAULT_JOB_ACCESS_TTL_MS = 24 * 60 * 60 * 1000;+const DEFAULT_JOB_ACCESS_TTL_MS = JOB_ACCESS_TTL_MS;async function withLogSpan<T>(params: {diff --git a/apps/api/src/services/worker/crawl-logic.test.ts b/apps/api/src/services/worker/crawl-logic.test.tsindex e9ab49319..510ec6413 100644--- a/apps/api/src/services/worker/crawl-logic.test.ts+++ b/apps/api/src/services/worker/crawl-logic.test.ts@@ -12,10 +12,6 @@ vi.mock("../../lib/crawl-redis", () => ({getDoneJobsOrderedLength: vi.fn(async () => 2),}));-vi.mock("../../db/rpc", () => ({- creditsBilledByCrawlId: vi.fn(async () => [{ credits_billed: 0 }]),-}));-vi.mock("../../controllers/v1/crawl-status", () => ({getJobs: vi.fn(async () => []),}));diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.tsindex 204fcbf0b..132b72a5a 100644--- a/apps/api/src/services/worker/crawl-logic.ts+++ b/apps/api/src/services/worker/crawl-logic.ts@@ -1,21 +1,26 @@import { logger as _logger } from "../../lib/logger";-import { config } from "../../config";import {finishCrawl,getCrawlJobs,getDoneJobsOrderedLength,} from "../../lib/crawl-redis";import { getCrawl } from "../../lib/crawl-redis";-import { creditsBilledByCrawlId } from "../../db/rpc";-import { db } from "../../db/connection";import { readRequestCredits } from "../../lib/request-credits-store";import { getJobs } from "../../controllers/v1/crawl-status";import { logCrawl, logBatchScrape } from "../logging/log_job";import { createWebhookSender, WebhookEvent } from "../webhook/index";import type { NuQJob } from "./nuq";-import { recordJobStorePostgresFallback } from "../../lib/job-store-fallback";import { readRequestCreditsFromAnalytics } from "../../lib/request-credits-analytics";+/**+ * How often, and how long apart, finalization re-reads the ClickHouse credits+ * sum before believing an empty result. ClickPipes lands a child's row one to+ * two seconds after the worker logs it; five reads two seconds apart outwait+ * that with room to spare without holding the finalizer for long.+ */+const FINALIZE_CREDITS_ATTEMPTS = 5;+const FINALIZE_CREDITS_RETRY_MS = 2_000;+export async function finishCrawlSuper(job: NuQJob<any>) {const crawlId = job.groupId;@@ -145,37 +150,45 @@ export async function finishCrawlSuper(job: NuQJob<any>) {const num_docs = await getDoneJobsOrderedLength(crawlId);let credits_billed: number | null = null;- let creditsReadFailed = false;try {credits_billed = await readRequestCredits(requestId);} catch (error) {- creditsReadFailed = true;logger.warn("Bigtable request credits read failed", { error });}if (credits_billed === null) {- try {- credits_billed = await readRequestCreditsFromAnalytics(crawlId);- } catch (error) {- logger.warn("Analytics request credits read failed", { error });- }- }-- if (credits_billed === null && config.USE_DB_AUTHENTICATION) {- try {- const creditsRows = await creditsBilledByCrawlId(db, crawlId);- credits_billed = creditsRows?.[0]?.credits_billed ?? null;- if (credits_billed !== null && !creditsReadFailed) {- recordJobStorePostgresFallback("request_credits", requestId);+ // Requests from before the Bigtable credit rows existed: sum the scrape+ // job log. Finalization records credits_cost for good, and ClickPipes+ // lands the last children a second or two after they are logged, so an+ // empty sum is retried before it is believed.+ for (let attempt = 1; attempt <= FINALIZE_CREDITS_ATTEMPTS; attempt++) {+ try {+ credits_billed = await readRequestCreditsFromAnalytics(requestId, {+ emptyAsZero: false,+ });+ } catch (error) {+ logger.warn("Analytics request credits read failed", {+ error,+ attempt,+ });+ }+ if (credits_billed !== null) break;+ if (attempt < FINALIZE_CREDITS_ATTEMPTS) {+ await new Promise(resolve =>+ setTimeout(resolve, FINALIZE_CREDITS_RETRY_MS),+ );}- } catch (error) {- logger.warn("Credits billed is null", { error });}+ }- if (credits_billed === null) {- logger.warn("Credits billed is null", {});- }+ if (credits_billed === null) {+ // The row's credits_cost is NOT NULL, so the record has to carry a+ // number; 0 is written and the gap is loud rather than silent.+ logger.error(+ "Credits billed unknown at crawl finalization; recording 0",+ { requestId, attempts: FINALIZE_CREDITS_ATTEMPTS },+ );}if (sc.crawlerOptions !== null) {