Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): stop reading the PostgreSQL job tables (#4694)

Gergő Móricz85288f4941c9
diff --git a/apps/api/src/__tests__/routes/interact-agent-concurrency.routes.test.ts b/apps/api/src/__tests__/routes/interact-agent-concurrency.routes.test.ts
index 6812e6744..eb9e2642c 100644
--- a/apps/api/src/__tests__/routes/interact-agent-concurrency.routes.test.ts
+++ b/apps/api/src/__tests__/routes/interact-agent-concurrency.routes.test.ts
@@ -85,18 +85,14 @@ vi.mock("../../lib/operational-job-access", () => ({
expiresAtMs: Date.now() + 60_000,
})),
}));
+// The scrape's replay context lives in its Bigtable terminal state.
vi.mock("../../lib/job-state-store", () => ({
- readScrapeJobState: vi.fn(async () => null),
-}));
-vi.mock("../../lib/job-store-fallback", () => ({
- recordJobStorePostgresFallback: vi.fn(),
-}));
-vi.mock("../../lib/supabase-jobs", () => ({
- supabaseGetScrapeByIdDirect: vi.fn(async () => ({
- id: "scrape-1",
- team_id: "team-free",
- url: "https://example.com",
- options: {},
+ readScrapeJobState: vi.fn(async () => ({
+ status: "completed",
+ requestId: "scrape-1",
+ completedAtMs: Date.now(),
+ creditsBilled: 1,
+ replay: { targetUrl: "https://example.com", waitForMs: 0, actions: [] },
})),
}));
vi.mock("../../lib/browser-sessions", () => ({
diff --git a/apps/api/src/controllers/v1/crawl-errors.ts b/apps/api/src/controllers/v1/crawl-errors.ts
index 808f7646a..9e387c2ed 100644
--- a/apps/api/src/controllers/v1/crawl-errors.ts
+++ b/apps/api/src/controllers/v1/crawl-errors.ts
@@ -30,7 +30,7 @@ export async function crawlErrorsController(
const crawlTtlHours = req.acuc?.flags?.crawlTtlHours ?? 24;
let access;
try {
- access = await getCrawlJobAccess(req.params.jobId, crawlTtlHours);
+ access = await getCrawlJobAccess(req.params.jobId);
} catch (requestError) {
_logger.error("Error getting request", { error: requestError });
throw requestError;
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts
index 5499ea57a..c2bd0d652 100644
--- a/apps/api/src/controllers/v1/crawl-status.ts
+++ b/apps/api/src/controllers/v1/crawl-status.ts
@@ -15,11 +15,8 @@ import {
getCrawlQualifiedJobCount,
getDoneJobsOrderedUntil,
} from "../../lib/crawl-redis";
-import { supabaseGetScrapeById } from "../../lib/supabase-jobs";
import { configDotenv } from "dotenv";
import { logger } from "../../lib/logger";
-import { creditsBilledByCrawlId } from "../../db/rpc";
-import { dbRr } from "../../db/connection";
import { getJobFromGCS } from "../../lib/gcs-jobs";
import {
scrapeQueue,
@@ -30,7 +27,6 @@ import {
import { ScrapeJobSingleUrls } from "../../types";
import { readScrapeJobState } from "../../lib/job-state-store";
import { readRequestCredits } from "../../lib/request-credits-store";
-import { recordJobStorePostgresFallback } from "../../lib/job-store-fallback";
import { readRequestCreditsFromAnalytics } from "../../lib/request-credits-analytics";
configDotenv();
@@ -46,36 +42,27 @@ export type PseudoJob<T> = {
failedReason?: string;
};
-type DBScrape = {
- id: string;
- success: boolean;
- options: any;
- created_at: any;
- error: string | null;
- team_id: string;
-};
-
export async function getJob(id: string): Promise<PseudoJob<any> | null> {
- let scrapeStateFailed = false;
- const [nuqJob, scrapeState, dbScrape, gcsJob] = await Promise.all([
+ let stateReadError: unknown = null;
+ const [nuqJob, scrapeState, gcsJob] = await Promise.all([
scrapeQueue.getJob(id) as Promise<NuQJob<ScrapeJobSingleUrls> | null>,
readScrapeJobState(id).catch(error => {
- logger.warn("Bigtable scrape state read failed; using legacy lookup", {
+ logger.warn("Bigtable scrape state read failed", {
error,
scrapeId: id,
});
- scrapeStateFailed = true;
+ stateReadError = error;
return null;
}),
- (config.USE_DB_AUTHENTICATION
- ? supabaseGetScrapeById(id)
- : null) as Promise<DBScrape | null>,
(config.GCS_BUCKET_NAME ? getJobFromGCS(id) : null) as Promise<any | null>,
]);
- if (!nuqJob && !scrapeState && !dbScrape) return null;
- if (!nuqJob && !scrapeState && !scrapeStateFailed && dbScrape) {
- recordJobStorePostgresFallback("scrape_state", id);
+ if (!nuqJob && !scrapeState) {
+ // With no NuQ job, Bigtable is the only place a finished job's state
+ // lives. A failed read is an outage, not a missing job: surface it rather
+ // than answer 404 for a job that exists.
+ if (stateReadError) throw stateReadError;
+ return null;
}
if (nuqJob && nuqJob.data.mode !== "single_urls") {
@@ -91,24 +78,13 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {
const job: PseudoJob<any> = {
id,
- status:
- scrapeState?.status ??
- (dbScrape ? (dbScrape.success ? "completed" : "failed") : nuqJob!.status),
+ status: scrapeState?.status ?? nuqJob!.status,
returnvalue: Array.isArray(data) ? data[0] : data,
data: {
- scrapeOptions: nuqJob
- ? nuqJob.data.scrapeOptions
- : (dbScrape?.options ?? null),
+ scrapeOptions: nuqJob ? nuqJob.data.scrapeOptions : null,
},
- timestamp:
- scrapeState?.completedAtMs ??
- (nuqJob
- ? nuqJob.createdAt.valueOf()
- : new Date(dbScrape!.created_at).valueOf()),
- failedReason:
- (scrapeState?.error ??
- (nuqJob ? nuqJob.failedReason : dbScrape?.error)) ||
- undefined,
+ timestamp: scrapeState?.completedAtMs ?? nuqJob!.createdAt.valueOf(),
+ failedReason: (scrapeState?.error ?? nuqJob?.failedReason) || undefined,
};
return job;
@@ -201,29 +177,19 @@ export async function crawlStatusController(
logger.child({ zeroDataRetention }),
);
- let creditsReadFailed = false;
- let creditsBilled = await readRequestCredits(req.params.jobId).catch(() => {
- creditsReadFailed = true;
- return null;
- });
+ // A failed Bigtable read propagates: during an outage the analytics sum
+ // could be behind by a second of ClickPipes lag and under-report credits.
+ let creditsBilled = await readRequestCredits(req.params.jobId);
if (creditsBilled === null) {
// Requests from before the Bigtable credit rows existed: sum the scrape
// job log instead.
- creditsBilled = await readRequestCreditsFromAnalytics(
- req.params.jobId,
- ).catch(error => {
+ creditsBilled = await readRequestCreditsFromAnalytics(req.params.jobId, {
+ emptyAsZero: true,
+ }).catch(error => {
logger.warn("Analytics request credits read failed", { error });
return null;
});
}
- if (creditsBilled === null && config.USE_DB_AUTHENTICATION) {
- creditsBilled = await creditsBilledByCrawlId(dbRr, req.params.jobId)
- .then(rows => rows[0]?.credits_billed ?? null)
- .catch(() => null);
- if (creditsBilled !== null && !creditsReadFailed) {
- recordJobStorePostgresFallback("request_credits", req.params.jobId);
- }
- }
// check if the crawl failed during kickoff (e.g. queue full)
const crawlError = await getCrawlError(req.params.jobId);
diff --git a/apps/api/src/controllers/v1/extract-status.ts b/apps/api/src/controllers/v1/extract-status.ts
index bf5bbf6ca..b654df34d 100644
--- a/apps/api/src/controllers/v1/extract-status.ts
+++ b/apps/api/src/controllers/v1/extract-status.ts
@@ -1,4 +1,5 @@
import { Response } from "express";
+import { JOB_ACCESS_TTL_MS } from "../../lib/job-access-store";
import { config } from "../../config";
import { RequestWithAuth } from "./types";
import {
@@ -6,13 +7,11 @@ import {
getExtractExpiry,
getExtractResult,
} from "../../lib/extract/extract-redis";
-import { supabaseGetExtractByIdDirect } from "../../lib/supabase-jobs";
import { logger as _logger } from "../../lib/logger";
import { getJobFromGCS } from "../../lib/gcs-jobs";
import { getExtractJobAccess } from "../../lib/operational-job-access";
import { readExtractJobState } from "../../lib/job-state-store";
import { normalizeJobAccessTeamId } from "../../lib/job-access-store";
-import { recordJobStorePostgresFallback } from "../../lib/job-store-fallback";
async function getExtractData(id: string): Promise<any> {
// Try GCS first if configured
@@ -67,64 +66,25 @@ export async function extractStatusController(
});
}
- // If not in Redis, check the database for completed jobs
+ // Not in Redis: the job finished (or never existed). Bigtable holds the
+ // terminal state for 24 hours; after that the job has expired.
if (!extract) {
- if (config.USE_DB_AUTHENTICATION) {
- let stateReadFailed = false;
- const state = await readExtractJobState(req.params.jobId).catch(error => {
- logger.warn("Bigtable extract state read failed; using legacy lookup", {
- error,
- });
- stateReadFailed = true;
- return null;
- });
- if (state) {
- return res.status(200).json({
- success: state.status === "completed",
- data:
- state.status === "completed"
- ? await getExtractData(req.params.jobId)
- : [],
- status: state.status,
- error: state.error,
- expiresAt: new Date(access!.expiresAtMs).toISOString(),
- creditsUsed: state.creditsBilled,
- });
- }
-
- const dbExtract = await supabaseGetExtractByIdDirect(req.params.jobId);
- if (!dbExtract) {
- logger.warn("Extract job was not found");
- return res.status(404).json({
- success: false,
- error: "Extract job not found",
- });
- }
- if (!stateReadFailed) {
- recordJobStorePostgresFallback("extract_state", req.params.jobId);
- }
-
- if (dbExtract.team_id !== req.auth.team_id) {
- return res.status(404).json({
- success: false,
- error: "Extract job not found",
- });
- }
-
- let data: any = [];
- if (dbExtract.is_successful) {
- data = await getExtractData(req.params.jobId);
- }
-
- // Return DB-based status
+ // A failed Bigtable read is an outage, not a missing job: it propagates
+ // to the error handler rather than answering 404.
+ const state = await readExtractJobState(req.params.jobId);
+ if (state) {
return res.status(200).json({
- success: dbExtract.is_successful,
- data,
- status: dbExtract.is_successful ? "completed" : "failed",
- error: dbExtract.error || undefined,
+ success: state.status === "completed",
+ data:
+ state.status === "completed"
+ ? await getExtractData(req.params.jobId)
+ : [],
+ status: state.status,
+ error: state.error,
expiresAt: new Date(
- new Date(dbExtract.created_at).getTime() + 1000 * 60 * 60 * 24,
+ access?.expiresAtMs ?? state.completedAtMs + JOB_ACCESS_TTL_MS,
).toISOString(),
+ creditsUsed: state.creditsBilled,
});
}
diff --git a/apps/api/src/controllers/v2/__tests__/scrape-browser.test.ts b/apps/api/src/controllers/v2/__tests__/scrape-browser.test.ts
index ab6830fca..5aeadb758 100644
--- a/apps/api/src/controllers/v2/__tests__/scrape-browser.test.ts
+++ b/apps/api/src/controllers/v2/__tests__/scrape-browser.test.ts
@@ -1,7 +1,7 @@
import type { Response } from "express";
+import { readScrapeJobState } from "../../../lib/job-state-store";
import { vi } from "vitest";
import { config } from "../../../config";
-import { supabaseGetScrapeByIdDirect } from "../../../lib/supabase-jobs";
import {
insertBrowserSession,
getBrowserSession,
@@ -56,9 +56,6 @@ vi.mock("../../../lib/operational-job-access", () => ({
vi.mock("../../../lib/job-state-store", () => ({
readScrapeJobState: vi.fn(async () => null),
}));
-vi.mock("../../../lib/job-store-fallback", () => ({
- recordJobStorePostgresFallback: vi.fn(),
-}));
vi.mock("../../auth", () => ({
getACUCTeam: vi.fn(async () => ({ org_id: null })),
}));
@@ -74,10 +71,6 @@ vi.mock("../../../lib/scrape-interact/langsmith", () => ({
sanitizeUrlForTrace: (url: string) => url,
}));
-vi.mock("../../../lib/supabase-jobs", () => ({
- supabaseGetScrapeByIdDirect: vi.fn(),
-}));
-
vi.mock("../../../lib/browser-sessions", () => ({
insertBrowserSession: vi.fn(),
completeBrowserSessionSettlement: vi.fn(async () => {}),
@@ -157,7 +150,7 @@ describe("scrapeInteractController", () => {
config.USE_DB_AUTHENTICATION = previousUseDbAuthentication;
});
- it("rejects self-hosted scrape interact before querying Supabase", async () => {
+ it("rejects scrape interact when database authentication is disabled", async () => {
config.USE_DB_AUTHENTICATION = false;
const req = {
@@ -170,7 +163,6 @@ describe("scrapeInteractController", () => {
await scrapeInteractController(req, res);
- expect(supabaseGetScrapeByIdDirect).not.toHaveBeenCalled();
expect(res.status).toHaveBeenCalledWith(501);
expect(res.json).toHaveBeenCalledWith({
success: false,
@@ -193,11 +185,13 @@ describe("scrapeInteractController", () => {
control_url: "https://hangar.example/live#control",
recording: true,
};
- vi.mocked(supabaseGetScrapeByIdDirect).mockResolvedValue({
- id: "scrape-123",
- team_id: "team-123",
- url: "https://example.com",
- options: {},
+ // The replay context comes from the scrape's Bigtable terminal state.
+ vi.mocked(readScrapeJobState).mockResolvedValueOnce({
+ status: "completed",
+ requestId: "scrape-123",
+ completedAtMs: Date.now(),
+ creditsBilled: 1,
+ replay: { targetUrl: "https://example.com", waitForMs: 0, actions: [] },
} as any);
const executed = {
stdout: "https://example.com",
diff --git a/apps/api/src/controllers/v2/crawl-errors.ts b/apps/api/src/controllers/v2/crawl-errors.ts
index 808f7646a..9e387c2ed 100644
--- a/apps/api/src/controllers/v2/crawl-errors.ts
+++ b/apps/api/src/controllers/v2/crawl-errors.ts
@@ -30,7 +30,7 @@ export async function crawlErrorsController(
const crawlTtlHours = req.acuc?.flags?.crawlTtlHours ?? 24;
let access;
try {
- access = await getCrawlJobAccess(req.params.jobId, crawlTtlHours);
+ access = await getCrawlJobAccess(req.params.jobId);
} catch (requestError) {
_logger.error("Error getting request", { error: requestError });
throw requestError;
diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.ts
index 89ab75914..ae456e731 100644
--- a/apps/api/src/controllers/v2/crawl-status.ts
+++ b/apps/api/src/controllers/v2/crawl-status.ts
@@ -16,11 +16,8 @@ import {
getLastDoneJobTimestamp,
isCrawlKickoffFinished,
} from "../../lib/crawl-redis";
-import { supabaseGetScrapeById } from "../../lib/supabase-jobs";
import { configDotenv } from "dotenv";
import { logger } from "../../lib/logger";
-import { creditsBilledByCrawlId } from "../../db/rpc";
-import { dbRr } from "../../db/connection";
import { getJobFromGCS } from "../../lib/gcs-jobs";
import {
scrapeQueue,
@@ -33,7 +30,6 @@ import { redisEvictConnection } from "../../../src/services/redis";
import { isBaseDomain, extractBaseDomain } from "../../lib/url-utils";
import { readScrapeJobState } from "../../lib/job-state-store";
import { readRequestCredits } from "../../lib/request-credits-store";
-import { recordJobStorePostgresFallback } from "../../lib/job-store-fallback";
import { readRequestCreditsFromAnalytics } from "../../lib/request-credits-analytics";
configDotenv();
@@ -49,42 +45,33 @@ export type PseudoJob<T> = {
failedReason?: string;
};
-type DBScrape = {
- id: string;
- success: boolean;
- options: any;
- created_at: any;
- error: string | null;
- team_id: string;
-};
-
export async function getJob(
id: string,
_logger = logger,
): Promise<PseudoJob<any> | null> {
- let scrapeStateFailed = false;
- const [nuqJob, scrapeState, dbScrape, gcsJob] = await Promise.all([
+ let stateReadError: unknown = null;
+ const [nuqJob, scrapeState, gcsJob] = await Promise.all([
scrapeQueue.getJob(
id,
_logger,
) as Promise<NuQJob<ScrapeJobSingleUrls> | null>,
readScrapeJobState(id).catch(error => {
- _logger.warn("Bigtable scrape state read failed; using legacy lookup", {
+ _logger.warn("Bigtable scrape state read failed", {
error,
scrapeId: id,
});
- scrapeStateFailed = true;
+ stateReadError = error;
return null;
}),
- (config.USE_DB_AUTHENTICATION
- ? supabaseGetScrapeById(id)
- : null) as Promise<DBScrape | null>,
(config.GCS_BUCKET_NAME ? getJobFromGCS(id) : null) as Promise<any | null>,
]);
- if (!nuqJob && !scrapeState && !dbScrape) return null;
- if (!nuqJob && !scrapeState && !scrapeStateFailed && dbScrape) {
- recordJobStorePostgresFallback("scrape_state", id);
+ if (!nuqJob && !scrapeState) {
+ // With no NuQ job, Bigtable is the only place a finished job's state
+ // lives. A failed read is an outage, not a missing job: surface it rather
+ // than answer 404 for a job that exists.
+ if (stateReadError) throw stateReadError;
+ return null;
}
if (nuqJob && nuqJob.data.mode !== "single_urls") {
@@ -100,24 +87,13 @@ export async function getJob(
const job: PseudoJob<any> = {
id,
- status:
- scrapeState?.status ??
- (dbScrape ? (dbScrape.success ? "completed" : "failed") : nuqJob!.status),
+ status: scrapeState?.status ?? nuqJob!.status,
returnvalue: Array.isArray(data) ? data[0] : data,
data: {
- scrapeOptions: nuqJob
- ? nuqJob.data.scrapeOptions
- : (dbScrape?.options ?? null),
+ scrapeOptions: nuqJob ? nuqJob.data.scrapeOptions : null,
},
- timestamp:
- scrapeState?.completedAtMs ??
- (nuqJob
- ? nuqJob.createdAt.valueOf()
- : new Date(dbScrape!.created_at).valueOf()),
- failedReason:
- (scrapeState?.error ??
- (nuqJob ? nuqJob.failedReason : dbScrape?.error)) ||
- undefined,
+ timestamp: scrapeState?.completedAtMs ?? nuqJob!.createdAt.valueOf(),
+ failedReason: (scrapeState?.error ?? nuqJob?.failedReason) || undefined,
};
return job;
@@ -222,31 +198,22 @@ export async function crawlStatusController(
logger.child({ zeroDataRetention }),
);
- let creditsReadFailed = false;
+ // A failed Bigtable read propagates: during an outage the analytics sum
+ // could be behind by a second of ClickPipes lag and under-report credits.
let creditsBilled = await readRequestCredits(
sc?.requestId ?? req.params.jobId,
- ).catch(() => {
- creditsReadFailed = true;
- return null;
- });
+ );
if (creditsBilled === null) {
// Requests from before the Bigtable credit rows existed: sum the scrape
// job log instead.
creditsBilled = await readRequestCreditsFromAnalytics(
- req.params.jobId,
+ sc?.requestId ?? req.params.jobId,
+ { emptyAsZero: true },
).catch(error => {
logger.warn("Analytics request credits read failed", { error });
return null;
});
}
- if (creditsBilled === null && config.USE_DB_AUTHENTICATION) {
- creditsBilled = await creditsBilledByCrawlId(dbRr, req.params.jobId)
- .then(rows => rows[0]?.credits_billed ?? null)
- .catch(() => null);
- if (creditsBilled !== null && !creditsReadFailed) {
- recordJobStorePostgresFallback("request_credits", req.params.jobId);
- }
- }
// check if the crawl failed during kickoff (e.g. queue full)
const crawlError = await getCrawlError(req.params.jobId);
diff --git a/apps/api/src/controllers/v2/extract-status.ts b/apps/api/src/controllers/v2/extract-status.ts
index 3405f638d..2f50ebce9 100644
--- a/apps/api/src/controllers/v2/extract-status.ts
+++ b/apps/api/src/controllers/v2/extract-status.ts
@@ -1,4 +1,5 @@
import { Response } from "express";
+import { JOB_ACCESS_TTL_MS } from "../../lib/job-access-store";
import { config } from "../../config";
import { RequestWithAuth } from "./types";
import {
@@ -6,14 +7,12 @@ import {
getExtractExpiry,
getExtractResult,
} from "../../lib/extract/extract-redis";
-import { supabaseGetExtractByIdDirect } from "../../lib/supabase-jobs";
import { logger as _logger } from "../../lib/logger";
import { getJobFromGCS } from "../../lib/gcs-jobs";
import { getExtractJobAccess } from "../../lib/operational-job-access";
import { readExtractJobState } from "../../lib/job-state-store";
import { normalizeJobAccessTeamId } from "../../lib/job-access-store";
import { getExtractV3AgentStatus } from "../../lib/extract-v3-status";
-import { recordJobStorePostgresFallback } from "../../lib/job-store-fallback";
async function getExtractData(id: string): Promise<any> {
// Try GCS first if configured
@@ -67,51 +66,26 @@ export async function extractStatusController(
// Get extract status from Redis (for in-progress jobs)
const redisExtract = await getExtract(req.params.jobId);
- // If not in Redis, check the database for completed jobs
+ // Not in Redis: finished, or still being set up. Bigtable holds the
+ // terminal state for 24 hours; a request row without either is in flight.
if (!redisExtract) {
- if (config.USE_DB_AUTHENTICATION) {
- let stateReadFailed = false;
- const state = await readExtractJobState(req.params.jobId).catch(error => {
- _logger.warn(
- "Bigtable extract state read failed; using legacy lookup",
- { error, extractId: req.params.jobId },
- );
- stateReadFailed = true;
- return null;
+ // A failed Bigtable read is an outage, not a job in flight: it propagates
+ // to the error handler rather than answering "processing".
+ const state = await readExtractJobState(req.params.jobId);
+ if (state) {
+ return res.status(200).json({
+ success: state.status === "completed",
+ data:
+ state.status === "completed"
+ ? await getExtractData(req.params.jobId)
+ : [],
+ status: state.status,
+ error: state.error,
+ expiresAt: new Date(
+ access?.expiresAtMs ?? state.completedAtMs + JOB_ACCESS_TTL_MS,
+ ).toISOString(),
+ creditsUsed: state.creditsBilled,
});
- if (state) {
- return res.status(200).json({
- success: state.status === "completed",
- data:
- state.status === "completed"
- ? await getExtractData(req.params.jobId)
- : [],
- status: state.status,
- error: state.error,
- expiresAt: new Date(access!.expiresAtMs).toISOString(),
- creditsUsed: state.creditsBilled,
- });
- }
-
- const dbExtract = await supabaseGetExtractByIdDirect(req.params.jobId);
- if (dbExtract) {
- if (!stateReadFailed) {
- recordJobStorePostgresFallback("extract_state", req.params.jobId);
- }
- // Get result data
- let data: any = [];
- if (dbExtract.is_successful) {
- data = await getExtractData(req.params.jobId);
- }
-
- return res.status(200).json({
- success: dbExtract.is_successful,
- data,
- status: dbExtract.is_successful ? "completed" : "failed",
- error: dbExtract.error || undefined,
- expiresAt: new Date(access!.expiresAtMs).toISOString(),
- });
- }
}
// Fall back to extractRequest info
@@ -119,7 +93,9 @@ export async function extractStatusController(
success: true,
data: [],
status: "processing",
- expiresAt: new Date(access!.expiresAtMs).toISOString(),
+ expiresAt: new Date(
+ access?.expiresAtMs ?? Date.now() + JOB_ACCESS_TTL_MS,
+ ).toISOString(),
});
}
diff --git a/apps/api/src/controllers/v2/feedback/feedback-store.ts b/apps/api/src/controllers/v2/feedback/feedback-store.ts
index 357400eb7..d0f84580e 100644
--- a/apps/api/src/controllers/v2/feedback/feedback-store.ts
+++ b/apps/api/src/controllers/v2/feedback/feedback-store.ts
@@ -13,7 +13,6 @@ import {
FeedbackRecordOptions,
RefundPolicySnapshot,
} from "./internal-types";
-import { recordJobStorePostgresFallback } from "../../../lib/job-store-fallback";
type DbError = { code?: string } & Record<string, unknown>;
@@ -22,13 +21,6 @@ type ExistingFeedback = {
credits_refunded: number | null;
};
-const JOB_TABLES = {
- search: schema.searches,
- scrape: schema.scrapes,
- parse: schema.parses,
- map: schema.maps,
-} as const;
-
function feedbackMetadata(
options: FeedbackRecordOptions,
): Record<string, unknown> {
@@ -46,74 +38,40 @@ export async function lookupFeedbackJob(
jobId: string,
dbTeamId: string,
): Promise<FeedbackJobRow | null> {
- let bigtableFailed = false;
+ let job;
try {
- const job = await readFeedbackJob(jobId);
- if (job) {
- const storedEndpoint = endpointForRefundClass(job.refundClass);
- if (job.teamId !== dbTeamId || storedEndpoint !== endpoint) return null;
-
- const feedbackWindowSec =
- endpoint === "search"
- ? config.SEARCH_FEEDBACK_MAX_AGE_SEC
- : config.FEEDBACK_MAX_AGE_SEC;
- return {
- endpoint,
- id: jobId,
- request_id: job.requestId,
- team_id: job.teamId,
- credits_cost: job.creditsBilled,
- created_at: new Date(
- job.feedbackDeadlineMs - feedbackWindowSec * 1000,
- ).toISOString(),
- is_successful: job.succeeded,
- options: null,
- feedback_deadline_ms: job.feedbackDeadlineMs,
- refund_class: job.refundClass,
- zero_data_retention: job.zeroDataRetention,
- };
- }
+ job = await readFeedbackJob(jobId);
} catch (error) {
- bigtableFailed = true;
- logger.warn(
- "Bigtable feedback job read failed; falling back to PostgreSQL",
- {
- error,
- jobId,
- endpoint,
- },
- );
+ logger.warn("Bigtable feedback job read failed", {
+ error,
+ jobId,
+ endpoint,
+ });
+ return null;
}
+ if (!job) return null;
- const table = JOB_TABLES[endpoint] as any;
- const [row] = await dbRr
- .select({
- id: table.id,
- request_id: table.request_id,
- team_id: table.team_id,
- credits_cost: table.credits_cost,
- created_at: table.created_at,
- options: table.options,
- ...(endpoint === "map" ? {} : { is_successful: table.is_successful }),
- })
- .from(table)
- .where(and(eq(table.id, jobId), eq(table.team_id, dbTeamId)))
- .limit(1);
-
- if (!row) return null;
- if (!bigtableFailed) {
- recordJobStorePostgresFallback("feedback_job", jobId, { endpoint });
- }
+ const storedEndpoint = endpointForRefundClass(job.refundClass);
+ if (job.teamId !== dbTeamId || storedEndpoint !== endpoint) return null;
+ const feedbackWindowSec =
+ endpoint === "search"
+ ? config.SEARCH_FEEDBACK_MAX_AGE_SEC
+ : config.FEEDBACK_MAX_AGE_SEC;
return {
endpoint,
- id: row.id,
- request_id: row.request_id ?? null,
- team_id: row.team_id,
- credits_cost: row.credits_cost ?? 0,
- created_at: row.created_at,
- is_successful: endpoint === "map" ? true : (row.is_successful ?? null),
- options: row.options ?? null,
+ id: jobId,
+ request_id: job.requestId,
+ team_id: job.teamId,
+ credits_cost: job.creditsBilled,
+ created_at: new Date(
+ job.feedbackDeadlineMs - feedbackWindowSec * 1000,
+ ).toISOString(),
+ is_successful: job.succeeded,
+ options: null,
+ feedback_deadline_ms: job.feedbackDeadlineMs,
+ refund_class: job.refundClass,
+ zero_data_retention: job.zeroDataRetention,
};
}
diff --git a/apps/api/src/controllers/v2/scrape-browser.ts b/apps/api/src/controllers/v2/scrape-browser.ts
index be49a1e34..4abb2e1ec 100644
--- a/apps/api/src/controllers/v2/scrape-browser.ts
+++ b/apps/api/src/controllers/v2/scrape-browser.ts
@@ -22,7 +22,6 @@ import {
} from "../../lib/browser-lifecycle";
import { browserCreateRequestSchema, browserError } from "./browser";
import {
- ScrapeContextRow,
buildReplayContextFromScrape,
estimateReplayTimeoutSeconds,
buildReplayScript,
@@ -47,12 +46,10 @@ import {
} from "../../lib/keyless";
import { enqueueBrowserSessionActivity } from "../../lib/browser-session-activity";
import { integrationSchema } from "../../utils/integration";
-import { supabaseGetScrapeByIdDirect } from "../../lib/supabase-jobs";
import { applyAgentAuthDiscoveryHeader } from "../../lib/agent-auth-discovery";
import { getScrapeJobAccess } from "../../lib/operational-job-access";
import { readScrapeJobState } from "../../lib/job-state-store";
import { scrapeQueue } from "../../services/worker/nuq-router";
-import { recordJobStorePostgresFallback } from "../../lib/job-store-fallback";
const browserExecuteRequestSchema = z
.object({
@@ -145,14 +142,10 @@ export async function scrapeInteractController(
});
}
- let stateReadFailed = false;
const [nuqJob, state] = await Promise.all([
scrapeQueue.getJob(scrapeId, logger),
readScrapeJobState(scrapeId).catch(error => {
- logger.warn("Bigtable scrape state read failed; using legacy lookup", {
- error,
- });
- stateReadFailed = true;
+ logger.warn("Bigtable scrape state read failed", { error });
return null;
}),
]);
@@ -179,7 +172,6 @@ export async function scrapeInteractController(
let replayContext = state?.replay;
let replayError: string | undefined;
- let legacyScrape: ScrapeContextRow | null = null;
if (!replayContext && nuqJob?.data.mode === "single_urls") {
const replay = buildReplayContextFromScrape({
id: scrapeId,
@@ -190,21 +182,6 @@ export async function scrapeInteractController(
replayContext = replay.context;
replayError = replay.error;
}
- if (!replayContext) {
- legacyScrape = (await supabaseGetScrapeByIdDirect(
- scrapeId,
- )) as ScrapeContextRow | null;
- if (legacyScrape && !stateReadFailed) {
- recordJobStorePostgresFallback("scrape_state", scrapeId, {
- reason: "replay_context",
- });
- }
- const replay = legacyScrape
- ? buildReplayContextFromScrape(legacyScrape)
- : { error: "Replay context is unavailable for this scrape job." };
- replayContext = replay.context;
- replayError = replay.error;
- }
if (!replayContext) {
return res.status(409).json({
success: false,
@@ -246,10 +223,7 @@ export async function scrapeInteractController(
state?.profile ??
(nuqJob?.data.mode === "single_urls"
? nuqJob.data.scrapeOptions.profile
- : undefined) ??
- ((legacyScrape?.options as ScrapeOptions | undefined)?.profile as
- | { name: string; saveChanges: boolean }
- | undefined);
+ : undefined);
const created = await createSessionForScrape(
req,
scrapeId,
@@ -300,20 +274,14 @@ export async function scrapeInteractController(
// every run carries the URL / wait / actions / origin that set the stage
// for what the agent does on top of it. URLs are stripped of query
// strings to avoid leaking PII into LangSmith.
- const scrapeOptions = (legacyScrape?.options ?? {}) as {
- origin?: string;
- };
const traceScrapeContext = {
- scrapeUrl: sanitizeUrlForTrace(
- legacyScrape?.url ?? replayContext.targetUrl,
- ),
+ scrapeUrl: sanitizeUrlForTrace(replayContext.targetUrl),
targetUrl: sanitizeUrlForTrace(replayContext.targetUrl),
scrapeWaitForMs: replayContext.waitForMs,
scrapeActions: replayContext.actions.length,
scrapeOrigin:
state?.origin ??
- (nuqJob?.data.mode === "single_urls" ? nuqJob.data.origin : undefined) ??
- scrapeOptions.origin,
+ (nuqJob?.data.mode === "single_urls" ? nuqJob.data.origin : undefined),
};
// Identity fields below team_id — optional, normalized from null → undefined
diff --git a/apps/api/src/db/rpc.ts b/apps/api/src/db/rpc.ts
index eae478508..c196b12c7 100644
--- a/apps/api/src/db/rpc.ts
+++ b/apps/api/src/db/rpc.ts
@@ -88,20 +88,6 @@ export function billTeam7(params: {
);
}
-// `database` is a parameter because callers split between the primary and the
-// read replica: status controllers (informational `creditsUsed`) read from the
-// replica, while crawl finalization (crawl-logic) reads its own recent billing
-// writes and must stay on the primary.
-export function creditsBilledByCrawlId(
- database: DB,
- i_crawl_id: string,
-): Promise<{ credits_billed: number }[]> {
- return execRows(
- database,
- sql`select * from credits_billed_by_crawl_id_2(i_crawl_id => ${i_crawl_id})`,
- );
-}
-
export function diffGetLastScrape(
i_team_id: string,
i_url: string,
diff --git a/apps/api/src/lib/feedback-job-store.ts b/apps/api/src/lib/feedback-job-store.ts
index 768bec2db..06b542b09 100644
--- a/apps/api/src/lib/feedback-job-store.ts
+++ b/apps/api/src/lib/feedback-job-store.ts
@@ -6,6 +6,9 @@ import { saltedUuidV7RowKey } from "./bigtable-row-key";
import { setSpanAttributes, withSpan } from "./otel-tracer";
const FAMILY = "f";
+const FEEDBACK_ROW_RETENTION_MS = 24 * 60 * 60 * 1000;
+/** Headroom past a configured feedback window longer than the default retention. */
+const FEEDBACK_ROW_RETENTION_MARGIN_MS = 60 * 60 * 1000;
const QUALIFIER = "v";
type FeedbackEndpoint = "search" | "scrape" | "parse" | "map";
@@ -160,6 +163,17 @@ export async function writeFeedbackJob(
const feedbackDeadline = new Date(
completedAt.getTime() + feedbackWindowSec * 1000,
);
+ // The row outlives the feedback window so late feedback can be told
+ // "window expired" rather than "job not found"; the deadline inside the
+ // value is what enforces the window. A window configured longer than
+ // the default retention still gets a row that outlives it.
+ const retainUntil = new Date(
+ completedAt.getTime() +
+ Math.max(
+ FEEDBACK_ROW_RETENTION_MS,
+ feedbackWindowSec * 1000 + FEEDBACK_ROW_RETENTION_MARGIN_MS,
+ ),
+ );
const value = Buffer.from(
JSON.stringify({
version: 1,
@@ -184,7 +198,7 @@ export async function writeFeedbackJob(
[FAMILY]: {
[QUALIFIER]: {
value,
- timestamp: feedbackDeadline,
+ timestamp: retainUntil,
},
},
},
diff --git a/apps/api/src/lib/job-access-store.ts b/apps/api/src/lib/job-access-store.ts
index 479290dc6..f5c033cfc 100644
--- a/apps/api/src/lib/job-access-store.ts
+++ b/apps/api/src/lib/job-access-store.ts
@@ -4,6 +4,14 @@ import { saltedUuidV7RowKey } from "./bigtable-row-key";
import { keylessTeamUuid } from "./keyless";
import { setSpanAttributes, withSpan } from "./otel-tracer";
+/**
+ * Default lifetime of a job's access row, used by writes that do not pass
+ * their own `expiresAt` and as the fallback expiry where no row exists. It is
+ * not every row's actual expiry: deep research writes a shorter TTL, and
+ * crawl and batch scrape write a per-key `crawlTtlHours`.
+ */
+export const JOB_ACCESS_TTL_MS = 24 * 60 * 60 * 1000;
+
const FAMILY = "j";
const QUALIFIER = "v";
const PREVIEW_TEAM_ID = "3adefd26-77ec-5968-8dcf-c94b5630d1de";
diff --git a/apps/api/src/lib/job-operational-stores.test.ts b/apps/api/src/lib/job-operational-stores.test.ts
index 51d670ff5..96b28d3ad 100644
--- a/apps/api/src/lib/job-operational-stores.test.ts
+++ b/apps/api/src/lib/job-operational-stores.test.ts
@@ -257,7 +257,7 @@ describe("operational Bigtable stores", () => {
zeroDataRetention: false,
});
expect(mutate.mock.calls[0][0][0].data.f.v.timestamp).toEqual(
- new Date(completedAt.getTime() + 180_000),
+ new Date(completedAt.getTime() + 24 * 60 * 60 * 1000),
);
expect(spans).toContainEqual({
name: "bigtable.feedback_job.write",
@@ -265,6 +265,35 @@ describe("operational Bigtable stores", () => {
});
});
+ it("keeps the row past a feedback window longer than the default retention", async () => {
+ const completedAt = new Date("2026-09-15T12:00:00.000Z");
+ const original = mutableConfig.FEEDBACK_MAX_AGE_SEC;
+ mutableConfig.FEEDBACK_MAX_AGE_SEC = 30 * 60 * 60; // 30 h, past the 24 h default
+ try {
+ await writeFeedbackJob({
+ jobId: JOB_ID,
+ requestId: REQUEST_ID,
+ teamId: "team-id",
+ endpoint: "scrape",
+ scrapeOptions: scrapeOptions.parse({}),
+ succeeded: true,
+ creditsBilled: 1,
+ zeroDataRetention: false,
+ completedAt,
+ });
+ } finally {
+ mutableConfig.FEEDBACK_MAX_AGE_SEC = original;
+ }
+
+ expect(writtenValue()).toMatchObject({
+ feedbackDeadlineMs: completedAt.getTime() + 30 * 60 * 60 * 1000,
+ });
+ // window + one hour of margin, not the flat 24 h
+ expect(mutate.mock.calls[0][0][0].data.f.v.timestamp).toEqual(
+ new Date(completedAt.getTime() + 31 * 60 * 60 * 1000),
+ );
+ });
+
it("uses the search-specific feedback window", async () => {
const completedAt = new Date("2026-09-15T12:00:00.000Z");
await writeFeedbackJob({
diff --git a/apps/api/src/lib/job-store-fallback.test.ts b/apps/api/src/lib/job-store-fallback.test.ts
deleted file mode 100644
index 0cc7fe750..000000000
--- a/apps/api/src/lib/job-store-fallback.test.ts
+++ /dev/null
@@ -1,80 +0,0 @@
-import { beforeEach, describe, expect, it, vi } from "vitest";
-
-const { info, mutableConfig } = vi.hoisted(() => ({
- info: vi.fn(),
- mutableConfig: {
- BIGTABLE_JOB_ACCESS_TABLE: "api_job_access" as string | undefined,
- BIGTABLE_SCRAPE_STATE_TABLE: "scrape_state" as string | undefined,
- BIGTABLE_EXTRACT_STATE_TABLE: "extract_state" as string | undefined,
- BIGTABLE_FEEDBACK_JOBS_TABLE: "feedback_jobs" as string | undefined,
- BIGTABLE_REQUEST_CREDITS_TABLE: "request_credits" as string | undefined,
- BIGTABLE_CHANGE_TRACKING_TABLE: "change_tracking" as string | undefined,
- },
-}));
-vi.mock("./logger", () => ({ logger: { info } }));
-vi.mock("../config", () => ({ config: mutableConfig }));
-
-import {
- jobStorePostgresFallbackTotal,
- recordJobStorePostgresFallback,
-} from "./job-store-fallback";
-
-async function countsByStore() {
- const metric = await jobStorePostgresFallbackTotal.get();
- return Object.fromEntries(metric.values.map(v => [v.labels.store, v.value]));
-}
-
-describe("job store fallback accounting", () => {
- beforeEach(() => {
- jobStorePostgresFallbackTotal.reset();
- info.mockClear();
- mutableConfig.BIGTABLE_SCRAPE_STATE_TABLE = "scrape_state";
- });
-
- it("counts and logs one hit per store", async () => {
- recordJobStorePostgresFallback("scrape_state", "job-1");
- recordJobStorePostgresFallback("scrape_state", "job-2");
- recordJobStorePostgresFallback("job_access", "job-3", { kind: "crawl" });
-
- expect(await countsByStore()).toMatchObject({
- scrape_state: 2,
- job_access: 1,
- });
-
- expect(info).toHaveBeenCalledTimes(3);
- expect(info).toHaveBeenLastCalledWith(
- "PostgreSQL fallback served a job-store read",
- expect.objectContaining({
- store: "job_access",
- id: "job-3",
- kind: "crawl",
- }),
- );
- });
-
- it("lets the fixed log fields win over extra context", () => {
- recordJobStorePostgresFallback("scrape_state", "job-1", {
- id: "spoofed",
- store: "spoofed",
- module: "spoofed",
- });
-
- expect(info).toHaveBeenCalledWith(
- "PostgreSQL fallback served a job-store read",
- expect.objectContaining({
- module: "job-store-fallback",
- store: "scrape_state",
- id: "job-1",
- }),
- );
- });
-
- it("records nothing when the store is not configured", async () => {
- mutableConfig.BIGTABLE_SCRAPE_STATE_TABLE = undefined;
-
- recordJobStorePostgresFallback("scrape_state", "job-1");
-
- expect(await countsByStore()).toEqual({});
- expect(info).not.toHaveBeenCalled();
- });
-});
diff --git a/apps/api/src/lib/job-store-fallback.ts b/apps/api/src/lib/job-store-fallback.ts
deleted file mode 100644
index 0d8451e85..000000000
--- a/apps/api/src/lib/job-store-fallback.ts
+++ /dev/null
@@ -1,63 +0,0 @@
-import { Counter, register } from "prom-client";
-import { config } from "../config";
-import { logger } from "./logger";
-
-const NAME = "job_store_postgres_fallback_total";
-
-/**
- * Reads the new job stores (Bigtable, NuQ) could not answer and PostgreSQL
- * did. Each label is one fallback path; a store is ready to lose its
- * fallback once its series has stayed flat for the agreed window.
- *
- * Looked up before creation so a re-evaluated module (test isolation) does
- * not register the same series twice in the shared default registry.
- */
-export const jobStorePostgresFallbackTotal =
- (register.getSingleMetric(NAME) as Counter<"store"> | undefined) ??
- new Counter({
- name: NAME,
- help: "Job-store reads answered by the PostgreSQL fallback, by store",
- labelNames: ["store"] as const,
- });
-
-type JobStoreFallback =
- | "job_access"
- | "scrape_state"
- | "extract_state"
- | "feedback_job"
- | "request_credits";
-
-/**
- * The Bigtable table behind each store. When it is not configured the
- * primary store never answers, PostgreSQL is the only store, and a hit is
- * not a fallback worth counting.
- */
-const STORE_TABLE: Record<JobStoreFallback, () => string | undefined> = {
- job_access: () => config.BIGTABLE_JOB_ACCESS_TABLE,
- scrape_state: () => config.BIGTABLE_SCRAPE_STATE_TABLE,
- extract_state: () => config.BIGTABLE_EXTRACT_STATE_TABLE,
- feedback_job: () => config.BIGTABLE_FEEDBACK_JOBS_TABLE,
- request_credits: () => config.BIGTABLE_REQUEST_CREDITS_TABLE,
-};
-
-/**
- * Record that PostgreSQL served a row the primary store did not have. Call
- * it only on a genuine miss: a miss on both sides is a normal "not found",
- * and a primary-store *error* is already reported where it is caught and
- * must not keep this series hot. Deployments without the store configured
- * record nothing.
- */
-export function recordJobStorePostgresFallback(
- store: JobStoreFallback,
- id: string,
- extra: Record<string, unknown> = {},
-): void {
- if (!STORE_TABLE[store]()) return;
- jobStorePostgresFallbackTotal.inc({ store });
- logger.info("PostgreSQL fallback served a job-store read", {
- ...extra,
- module: "job-store-fallback",
- store,
- id,
- });
-}
diff --git a/apps/api/src/lib/operational-job-access.test.ts b/apps/api/src/lib/operational-job-access.test.ts
index 588a7e21a..9a074393c 100644
--- a/apps/api/src/lib/operational-job-access.test.ts
+++ b/apps/api/src/lib/operational-job-access.test.ts
@@ -1,33 +1,15 @@
-const {
- readApiJobAccess,
- recordJobStorePostgresFallback,
- supabaseGetAgentRequestByIdDirect,
- supabaseGetCrawlRequestById,
- supabaseGetExtractRequestByIdDirect,
- supabaseGetScrapeById,
-} = vi.hoisted(() => ({
+const { readApiJobAccess } = vi.hoisted(() => ({
readApiJobAccess: vi.fn(),
- recordJobStorePostgresFallback: vi.fn(),
- supabaseGetAgentRequestByIdDirect: vi.fn(),
- supabaseGetCrawlRequestById: vi.fn(),
- supabaseGetExtractRequestByIdDirect: vi.fn(),
- supabaseGetScrapeById: vi.fn(),
}));
vi.mock("./job-access-store", () => ({ readApiJobAccess }));
-vi.mock("./job-store-fallback", () => ({ recordJobStorePostgresFallback }));
vi.mock("./logger", () => ({
logger: { warn: vi.fn(), info: vi.fn(), error: vi.fn() },
}));
-vi.mock("./supabase-jobs", () => ({
- supabaseGetAgentRequestByIdDirect,
- supabaseGetCrawlRequestById,
- supabaseGetExtractRequestByIdDirect,
- supabaseGetScrapeById,
-}));
import {
getAgentJobAccess,
+ getCrawlJobAccess,
getExtractJobAccess,
getScrapeJobAccess,
} from "./operational-job-access";
@@ -37,7 +19,7 @@ const JOB_ID = "019e6f45-7778-727d-adf0-0abe9d5062b6";
describe("operational job access", () => {
beforeEach(() => vi.clearAllMocks());
- it("returns the typed Bigtable record without querying PostgreSQL", async () => {
+ it("returns the typed Bigtable record", async () => {
const access = {
teamId: "team-id",
kind: "agent",
@@ -47,63 +29,23 @@ describe("operational job access", () => {
readApiJobAccess.mockResolvedValue(access);
await expect(getAgentJobAccess(JOB_ID)).resolves.toBe(access);
- expect(supabaseGetAgentRequestByIdDirect).not.toHaveBeenCalled();
});
- it("maps a live PostgreSQL fallback into the operational access type and counts it", async () => {
- const createdAt = new Date(Date.now() - 60_000);
- readApiJobAccess.mockResolvedValue(null);
- supabaseGetExtractRequestByIdDirect.mockResolvedValue({
- id: JOB_ID,
- team_id: "team-id",
+ it("extract access accepts extract and agent records and rejects a scrape", async () => {
+ const base = { teamId: "team-id", expiresAtMs: Date.now() + 60_000 };
+ readApiJobAccess.mockResolvedValueOnce({ ...base, kind: "extract" });
+ await expect(getExtractJobAccess(JOB_ID)).resolves.toMatchObject({
kind: "extract",
- origin: "api",
- created_at: createdAt.toISOString(),
- unrelated_column: "not returned",
- });
-
- await expect(getExtractJobAccess(JOB_ID)).resolves.toEqual({
- teamId: "team-id",
- kind: "extract",
- clientOrigin: "api",
- expiresAtMs: createdAt.getTime() + 24 * 60 * 60 * 1000,
- });
- expect(recordJobStorePostgresFallback).toHaveBeenCalledWith(
- "job_access",
- JOB_ID,
- { kind: "extract" },
- );
- });
-
- it("does not count an expired PostgreSQL fallback (the caller 404s either way)", async () => {
- readApiJobAccess.mockResolvedValue(null);
- supabaseGetScrapeById.mockResolvedValue({
- id: JOB_ID,
- team_id: "team-id",
- created_at: new Date(Date.now() - 25 * 60 * 60 * 1000).toISOString(),
});
-
- const access = await getScrapeJobAccess(JOB_ID);
- expect(access?.expiresAtMs).toBeLessThan(Date.now());
- expect(recordJobStorePostgresFallback).not.toHaveBeenCalled();
- });
-
- it("does not count a fallback taken because the Bigtable read failed", async () => {
- readApiJobAccess.mockRejectedValue(new Error("Bigtable unavailable"));
- supabaseGetScrapeById.mockResolvedValue({
- id: JOB_ID,
- team_id: "team-id",
- created_at: new Date().toISOString(),
- });
-
- await expect(getScrapeJobAccess(JOB_ID)).resolves.toMatchObject({
- teamId: "team-id",
- kind: "scrape",
+ readApiJobAccess.mockResolvedValueOnce({ ...base, kind: "agent" });
+ await expect(getExtractJobAccess(JOB_ID)).resolves.toMatchObject({
+ kind: "agent",
});
- expect(recordJobStorePostgresFallback).not.toHaveBeenCalled();
+ readApiJobAccess.mockResolvedValueOnce({ ...base, kind: "scrape" });
+ await expect(getExtractJobAccess(JOB_ID)).resolves.toBeNull();
});
- it("does not fall back when Bigtable has an expired record", async () => {
+ it("returns an expired record so the caller can answer 404 with its expiry", async () => {
const access = {
teamId: "team-id",
kind: "scrape",
@@ -112,10 +54,9 @@ describe("operational job access", () => {
readApiJobAccess.mockResolvedValue(access);
await expect(getScrapeJobAccess(JOB_ID)).resolves.toBe(access);
- expect(supabaseGetScrapeById).not.toHaveBeenCalled();
});
- it("does not fall back when Bigtable has a different job kind", async () => {
+ it("returns null for a different job kind", async () => {
readApiJobAccess.mockResolvedValue({
teamId: "team-id",
kind: "crawl",
@@ -123,6 +64,34 @@ describe("operational job access", () => {
});
await expect(getScrapeJobAccess(JOB_ID)).resolves.toBeNull();
- expect(supabaseGetScrapeById).not.toHaveBeenCalled();
+ await expect(getCrawlJobAccess(JOB_ID)).resolves.toMatchObject({
+ kind: "crawl",
+ });
+ });
+
+ it("crawl access accepts crawl and batch_scrape records and rejects a scrape", async () => {
+ const base = { teamId: "team-id", expiresAtMs: Date.now() + 60_000 };
+ readApiJobAccess.mockResolvedValueOnce({ ...base, kind: "crawl" });
+ await expect(getCrawlJobAccess(JOB_ID)).resolves.toMatchObject({
+ kind: "crawl",
+ });
+ readApiJobAccess.mockResolvedValueOnce({ ...base, kind: "batch_scrape" });
+ await expect(getCrawlJobAccess(JOB_ID)).resolves.toMatchObject({
+ kind: "batch_scrape",
+ });
+ readApiJobAccess.mockResolvedValueOnce({ ...base, kind: "scrape" });
+ await expect(getCrawlJobAccess(JOB_ID)).resolves.toBeNull();
+ });
+
+ it("returns null when Bigtable has no row", async () => {
+ readApiJobAccess.mockResolvedValue(null);
+ await expect(getScrapeJobAccess(JOB_ID)).resolves.toBeNull();
+ });
+
+ it("rethrows a failed Bigtable read instead of reporting a missing job", async () => {
+ readApiJobAccess.mockRejectedValue(new Error("Bigtable unavailable"));
+ await expect(getScrapeJobAccess(JOB_ID)).rejects.toThrow(
+ "Bigtable unavailable",
+ );
});
});
diff --git a/apps/api/src/lib/operational-job-access.ts b/apps/api/src/lib/operational-job-access.ts
index f36bd7f98..353321b77 100644
--- a/apps/api/src/lib/operational-job-access.ts
+++ b/apps/api/src/lib/operational-job-access.ts
@@ -4,15 +4,6 @@ import {
type ApiJobAccess,
type ApiJobKind,
} from "./job-access-store";
-import {
- supabaseGetAgentRequestByIdDirect,
- supabaseGetCrawlRequestById,
- supabaseGetExtractRequestByIdDirect,
- supabaseGetScrapeById,
-} from "./supabase-jobs";
-import { recordJobStorePostgresFallback } from "./job-store-fallback";
-
-const DEFAULT_TTL_MS = 24 * 60 * 60 * 1000;
type OperationalJobAccess = {
teamId: string;
@@ -21,114 +12,52 @@ type OperationalJobAccess = {
expiresAtMs: number;
};
+/**
+ * Who owns a job and until when it may be fetched, from the Bigtable job
+ * access row that logRequest (and, for crawl and batch children, logScrape)
+ * writes. A missing or expired row is the documented end of the job's
+ * fetchable life; there is no other store to ask. A failed read is an
+ * outage, not a missing job: it is rethrown so the caller answers an error
+ * rather than a 404 for a job that exists.
+ */
async function resolveOperationalJobAccess(params: {
id: string;
kinds: readonly ApiJobKind[];
- fallback: () => Promise<OperationalJobAccess | null>;
}): Promise<OperationalJobAccess | null> {
- let access: ApiJobAccess | null = null;
- let readFailed = false;
+ let access: ApiJobAccess | null;
try {
access = await readApiJobAccess(params.id);
} catch (error) {
- readFailed = true;
- logger.warn("Bigtable job access read failed; using legacy lookup", {
+ logger.error("Bigtable job access read failed", {
error,
jobId: params.id,
});
+ throw error;
}
-
- if (access) {
- return params.kinds.includes(access.kind) ? access : null;
- }
-
- const fallback = await params.fallback();
- if (!fallback || !Number.isFinite(fallback.expiresAtMs)) return null;
- // An expired row makes the caller answer 404 exactly as a miss would, so
- // only a live row counts as PostgreSQL having been needed.
- if (!readFailed && fallback.expiresAtMs > Date.now()) {
- recordJobStorePostgresFallback("job_access", params.id, {
- kind: fallback.kind,
- });
- }
- return fallback;
+ if (!access) return null;
+ return params.kinds.includes(access.kind) ? access : null;
}
export function getScrapeJobAccess(
id: string,
): Promise<OperationalJobAccess | null> {
- return resolveOperationalJobAccess({
- id,
- kinds: ["scrape"],
- fallback: async () => {
- const row = await supabaseGetScrapeById(id);
- return row
- ? {
- teamId: row.team_id,
- kind: "scrape",
- expiresAtMs: new Date(row.created_at).getTime() + DEFAULT_TTL_MS,
- }
- : null;
- },
- });
+ return resolveOperationalJobAccess({ id, kinds: ["scrape"] });
}
export function getExtractJobAccess(
id: string,
): Promise<OperationalJobAccess | null> {
- return resolveOperationalJobAccess({
- id,
- kinds: ["extract", "agent"],
- fallback: async () => {
- const row = await supabaseGetExtractRequestByIdDirect(id);
- if (!row || (row.kind !== "extract" && row.kind !== "agent")) return null;
- return {
- teamId: row.team_id,
- kind: row.kind,
- clientOrigin: row.origin ?? undefined,
- expiresAtMs: new Date(row.created_at).getTime() + DEFAULT_TTL_MS,
- };
- },
- });
+ return resolveOperationalJobAccess({ id, kinds: ["extract", "agent"] });
}
export function getAgentJobAccess(
id: string,
): Promise<OperationalJobAccess | null> {
- return resolveOperationalJobAccess({
- id,
- kinds: ["agent"],
- fallback: async () => {
- const row = await supabaseGetAgentRequestByIdDirect(id);
- if (!row) return null;
- return {
- teamId: row.team_id,
- kind: "agent",
- clientOrigin: row.origin ?? undefined,
- expiresAtMs: new Date(row.created_at).getTime() + DEFAULT_TTL_MS,
- };
- },
- });
+ return resolveOperationalJobAccess({ id, kinds: ["agent"] });
}
export function getCrawlJobAccess(
id: string,
- ttlHours: number,
): Promise<OperationalJobAccess | null> {
- return resolveOperationalJobAccess({
- id,
- kinds: ["crawl", "batch_scrape"],
- fallback: async () => {
- const row = await supabaseGetCrawlRequestById(id);
- if (!row || (row.kind !== "crawl" && row.kind !== "batch_scrape")) {
- return null;
- }
- return {
- teamId: row.team_id,
- kind: row.kind,
- expiresAtMs:
- new Date(row.created_at!).getTime() + ttlHours * 60 * 60 * 1000,
- };
- },
- });
+ return resolveOperationalJobAccess({ id, kinds: ["crawl", "batch_scrape"] });
}
diff --git a/apps/api/src/lib/request-credits-analytics.test.ts b/apps/api/src/lib/request-credits-analytics.test.ts
index 9c5b18208..2c72bc124 100644
--- a/apps/api/src/lib/request-credits-analytics.test.ts
+++ b/apps/api/src/lib/request-credits-analytics.test.ts
@@ -33,9 +33,9 @@ describe("readRequestCreditsFromAnalytics", () => {
it("sums the request's scrape rows", async () => {
rows([{ credits: "17", jobs: "4" }]);
- await expect(readRequestCreditsFromAnalytics("request-1")).resolves.toBe(
- 17,
- );
+ await expect(
+ readRequestCreditsFromAnalytics("request-1", { emptyAsZero: true }),
+ ).resolves.toBe(17);
expect(query.mock.calls[0][0]).toMatchObject({
query: expect.stringContaining(
"FROM scrapes_by_request FINAL WHERE request_id = {requestId: UUID}",
@@ -44,25 +44,32 @@ describe("readRequestCreditsFromAnalytics", () => {
});
});
- it("is null when the request has no scrape rows", async () => {
+ it("reads no scrape rows as zero for a status answer", async () => {
+ rows([{ credits: 0, jobs: 0 }]);
+ await expect(
+ readRequestCreditsFromAnalytics("request-1", { emptyAsZero: true }),
+ ).resolves.toBe(0);
+ });
+
+ it("reads no scrape rows as unknown for finalization", async () => {
rows([{ credits: 0, jobs: 0 }]);
await expect(
- readRequestCreditsFromAnalytics("request-1"),
+ readRequestCreditsFromAnalytics("request-1", { emptyAsZero: false }),
).resolves.toBeNull();
});
it("is null when ClickHouse is not configured", async () => {
client.current = null;
await expect(
- readRequestCreditsFromAnalytics("request-1"),
+ readRequestCreditsFromAnalytics("request-1", { emptyAsZero: true }),
).resolves.toBeNull();
expect(query).not.toHaveBeenCalled();
});
it("propagates ClickHouse errors", async () => {
query.mockRejectedValueOnce(new Error("ClickHouse unavailable"));
- await expect(readRequestCreditsFromAnalytics("request-1")).rejects.toThrow(
- "ClickHouse unavailable",
- );
+ await expect(
+ readRequestCreditsFromAnalytics("request-1", { emptyAsZero: true }),
+ ).rejects.toThrow("ClickHouse unavailable");
});
});
diff --git a/apps/api/src/lib/request-credits-analytics.ts b/apps/api/src/lib/request-credits-analytics.ts
index 192f9fdbe..fd673a0a3 100644
--- a/apps/api/src/lib/request-credits-analytics.ts
+++ b/apps/api/src/lib/request-credits-analytics.ts
@@ -12,11 +12,20 @@ import { setSpanAttributes, withSpan } from "./otel-tracer";
* `FINAL` collapses any redelivered copy, so the sum matches what the job log
* recorded.
*
- * Returns null when ClickHouse is not configured or the request has no scrape
- * rows at all, so callers can tell "nothing billed yet" from "unknown".
+ * ClickPipes lands rows about a second after the worker logs them, so a
+ * request whose scrapes are still in flight can read as having none. What
+ * that means depends on the caller:
+ *
+ * - a status endpoint answering a customer treats it as 0 (nothing billed
+ * yet), the same answer the PostgreSQL sum gave (COALESCE(SUM, 0));
+ * - crawl finalization, which records `credits_cost` for good, treats it as
+ * unknown (null) rather than writing a zero that a second's lag produced.
+ *
+ * An unconfigured ClickHouse client always yields null.
*/
export async function readRequestCreditsFromAnalytics(
requestId: string,
+ options: { emptyAsZero: boolean },
): Promise<number | null> {
const client = clickhouseClient;
if (client === null) return null;
@@ -38,9 +47,12 @@ export async function readRequestCreditsFromAnalytics(
jobs: number | string;
}>();
const jobs = Number(row?.jobs ?? 0);
- if (!Number.isFinite(jobs) || jobs === 0) {
- setSpanAttributes(span, { "request_credits.outcome": "not_found" });
- return null;
+ if (!Number.isSafeInteger(jobs)) {
+ throw new Error(`Invalid analytics job count: ${row?.jobs}`);
+ }
+ if (jobs === 0) {
+ setSpanAttributes(span, { "request_credits.outcome": "no_jobs" });
+ return options.emptyAsZero ? 0 : null;
}
const credits = Number(row.credits);
if (!Number.isSafeInteger(credits)) {
diff --git a/apps/api/src/lib/scrape-interact/scrape-replay.ts b/apps/api/src/lib/scrape-interact/scrape-replay.ts
index 6f7fa4b3c..6b138d4b2 100644
--- a/apps/api/src/lib/scrape-interact/scrape-replay.ts
+++ b/apps/api/src/lib/scrape-interact/scrape-replay.ts
@@ -4,7 +4,7 @@ import { rewriteUrl } from "../../scraper/scrapeURL/lib/rewriteUrl";
// Types
// ---------------------------------------------------------------------------
-export interface ScrapeContextRow {
+interface ScrapeContextRow {
id: string;
team_id: string;
url: string | null;
diff --git a/apps/api/src/lib/supabase-jobs.test.ts b/apps/api/src/lib/supabase-jobs.test.ts
deleted file mode 100644
index a0842f12d..000000000
--- a/apps/api/src/lib/supabase-jobs.test.ts
+++ /dev/null
@@ -1,63 +0,0 @@
-const { primarySelect, replicaSelect } = vi.hoisted(() => ({
- primarySelect: vi.fn(),
- replicaSelect: vi.fn(),
-}));
-
-vi.mock("../db/connection", () => ({
- db: { select: primarySelect },
- dbRr: { select: replicaSelect },
-}));
-
-import { supabaseGetScrapeByIdDirect } from "./supabase-jobs";
-import { logger } from "./logger";
-
-describe("supabaseGetScrapeByIdDirect", () => {
- beforeEach(() => {
- vi.clearAllMocks();
- });
-
- afterEach(() => {
- vi.restoreAllMocks();
- });
-
- it("reads newly-created scrapes from the primary database", async () => {
- const scrape = { id: "scrape-123" };
- const query = {
- from: vi.fn(),
- where: vi.fn(),
- limit: vi.fn().mockResolvedValue([scrape]),
- };
- query.from.mockReturnValue(query);
- query.where.mockReturnValue(query);
- primarySelect.mockReturnValue(query);
-
- await expect(supabaseGetScrapeByIdDirect(scrape.id)).resolves.toBe(scrape);
-
- expect(primarySelect).toHaveBeenCalledOnce();
- expect(replicaSelect).not.toHaveBeenCalled();
- });
-
- it("surfaces primary database errors instead of returning a false miss", async () => {
- const error = new Error("primary unavailable");
- const query = {
- from: vi.fn(),
- where: vi.fn(),
- limit: vi.fn().mockRejectedValue(error),
- };
- query.from.mockReturnValue(query);
- query.where.mockReturnValue(query);
- primarySelect.mockReturnValue(query);
- const logError = vi.spyOn(logger, "error").mockImplementation(() => logger);
-
- await expect(supabaseGetScrapeByIdDirect("scrape-123")).rejects.toBe(error);
-
- expect(logError).toHaveBeenCalledWith(
- "Error in supabaseGetScrapeByIdDirect",
- {
- error,
- scrapeId: "scrape-123",
- },
- );
- expect(replicaSelect).not.toHaveBeenCalled();
- });
-});
diff --git a/apps/api/src/lib/supabase-jobs.ts b/apps/api/src/lib/supabase-jobs.ts
deleted file mode 100644
index e75bdc9e5..000000000
--- a/apps/api/src/lib/supabase-jobs.ts
+++ /dev/null
@@ -1,115 +0,0 @@
-import { eq, inArray, and } from "drizzle-orm";
-import { db, dbRr } from "../db/connection";
-import * as schema from "../db/schema";
-import { logger } from "./logger";
-
-/**
- * Get a single scrape by ID from the scrapes table
- * @param scrapeId ID of Scrape
- * @returns Scrape data or null
- */
-export const supabaseGetScrapeById = async (scrapeId: string): Promise<any> => {
- try {
- const [data] = await dbRr
- .select()
- .from(schema.scrapes)
- .where(eq(schema.scrapes.id, scrapeId))
- .limit(1);
- return data ?? null;
- } catch (error) {
- return null;
- }
-};
-
-/**
- * Get a single scrape by ID from the primary database.
- * Use this when the scrape may have been created immediately before the read.
- */
-export const supabaseGetScrapeByIdDirect = async (
- scrapeId: string,
-): Promise<any> => {
- try {
- const [data] = await db
- .select()
- .from(schema.scrapes)
- .where(eq(schema.scrapes.id, scrapeId))
- .limit(1);
- return data ?? null;
- } catch (error) {
- logger.error("Error in supabaseGetScrapeByIdDirect", {
- error,
- scrapeId,
- });
- throw error;
- }
-};
-
-export const supabaseGetExtractByIdDirect = async (
- extractId: string,
-): Promise<any> => {
- try {
- const [data] = await db
- .select()
- .from(schema.extracts)
- .where(eq(schema.extracts.id, extractId))
- .limit(1);
- return data ?? null;
- } catch (error) {
- return null;
- }
-};
-
-export const supabaseGetExtractRequestByIdDirect = async (
- extractId: string,
-): Promise<typeof schema.requests.$inferSelect | null> => {
- try {
- const [data] = await db
- .select()
- .from(schema.requests)
- .where(
- and(
- eq(schema.requests.id, extractId),
- inArray(schema.requests.kind, ["extract", "agent"]),
- ),
- )
- .limit(1);
- return data ?? null;
- } catch (error) {
- return null;
- }
-};
-
-export const supabaseGetAgentRequestByIdDirect = async (
- agentId: string,
-): Promise<typeof schema.requests.$inferSelect | null> => {
- try {
- const [data] = await db
- .select()
- .from(schema.requests)
- .where(
- and(eq(schema.requests.id, agentId), eq(schema.requests.kind, "agent")),
- )
- .limit(1);
- return data ?? null;
- } catch (error) {
- return null;
- }
-};
-
-export const supabaseGetCrawlRequestById = async (requestId: string) => {
- const [data] = await dbRr
- .select({
- team_id: schema.requests.team_id,
- kind: schema.requests.kind,
- created_at: schema.requests.created_at,
- })
- .from(schema.requests)
- .where(
- and(
- eq(schema.requests.id, requestId),
- inArray(schema.requests.kind, ["crawl", "batch_scrape"]),
- ),
- )
- .limit(1);
- return data ?? null;
-};
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index 4666044c6..5d15290c1 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -27,6 +27,7 @@ import { PubSub, type PublishOptions, type Topic } from "@google-cloud/pubsub";
import { pubsubLogPublishTotal } from "../../lib/pubsub-log-metrics";
import { sanitizeLogData, sanitizeText } from "./sanitize";
import {
+ JOB_ACCESS_TTL_MS,
isApiJobKind,
normalizeJobAccessTeamId,
writeApiJobAccess,
@@ -45,7 +46,7 @@ import {
configDotenv();
const previewTeamId = "3adefd26-77ec-5968-8dcf-c94b5630d1de";
-const DEFAULT_JOB_ACCESS_TTL_MS = 24 * 60 * 60 * 1000;
+const DEFAULT_JOB_ACCESS_TTL_MS = JOB_ACCESS_TTL_MS;
async function withLogSpan<T>(
params: {
diff --git a/apps/api/src/services/worker/crawl-logic.test.ts b/apps/api/src/services/worker/crawl-logic.test.ts
index e9ab49319..510ec6413 100644
--- a/apps/api/src/services/worker/crawl-logic.test.ts
+++ b/apps/api/src/services/worker/crawl-logic.test.ts
@@ -12,10 +12,6 @@ vi.mock("../../lib/crawl-redis", () => ({
getDoneJobsOrderedLength: vi.fn(async () => 2),
}));
-vi.mock("../../db/rpc", () => ({
- creditsBilledByCrawlId: vi.fn(async () => [{ credits_billed: 0 }]),
-}));
-
vi.mock("../../controllers/v1/crawl-status", () => ({
getJobs: vi.fn(async () => []),
}));
diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.ts
index 204fcbf0b..132b72a5a 100644
--- a/apps/api/src/services/worker/crawl-logic.ts
+++ b/apps/api/src/services/worker/crawl-logic.ts
@@ -1,21 +1,26 @@
import { logger as _logger } from "../../lib/logger";
-import { config } from "../../config";
import {
finishCrawl,
getCrawlJobs,
getDoneJobsOrderedLength,
} from "../../lib/crawl-redis";
import { getCrawl } from "../../lib/crawl-redis";
-import { creditsBilledByCrawlId } from "../../db/rpc";
-import { db } from "../../db/connection";
import { readRequestCredits } from "../../lib/request-credits-store";
import { getJobs } from "../../controllers/v1/crawl-status";
import { logCrawl, logBatchScrape } from "../logging/log_job";
import { createWebhookSender, WebhookEvent } from "../webhook/index";
import type { NuQJob } from "./nuq";
-import { recordJobStorePostgresFallback } from "../../lib/job-store-fallback";
import { readRequestCreditsFromAnalytics } from "../../lib/request-credits-analytics";
+/**
+ * How often, and how long apart, finalization re-reads the ClickHouse credits
+ * sum before believing an empty result. ClickPipes lands a child's row one to
+ * two seconds after the worker logs it; five reads two seconds apart outwait
+ * that with room to spare without holding the finalizer for long.
+ */
+const FINALIZE_CREDITS_ATTEMPTS = 5;
+const FINALIZE_CREDITS_RETRY_MS = 2_000;
+
export async function finishCrawlSuper(job: NuQJob<any>) {
const crawlId = job.groupId;
@@ -145,37 +150,45 @@ export async function finishCrawlSuper(job: NuQJob<any>) {
const num_docs = await getDoneJobsOrderedLength(crawlId);
let credits_billed: number | null = null;
- let creditsReadFailed = false;
try {
credits_billed = await readRequestCredits(requestId);
} catch (error) {
- creditsReadFailed = true;
logger.warn("Bigtable request credits read failed", { error });
}
if (credits_billed === null) {
- try {
- credits_billed = await readRequestCreditsFromAnalytics(crawlId);
- } catch (error) {
- logger.warn("Analytics request credits read failed", { error });
- }
- }
-
- if (credits_billed === null && config.USE_DB_AUTHENTICATION) {
- try {
- const creditsRows = await creditsBilledByCrawlId(db, crawlId);
- credits_billed = creditsRows?.[0]?.credits_billed ?? null;
- if (credits_billed !== null && !creditsReadFailed) {
- recordJobStorePostgresFallback("request_credits", requestId);
+ // Requests from before the Bigtable credit rows existed: sum the scrape
+ // job log. Finalization records credits_cost for good, and ClickPipes
+ // lands the last children a second or two after they are logged, so an
+ // empty sum is retried before it is believed.
+ for (let attempt = 1; attempt <= FINALIZE_CREDITS_ATTEMPTS; attempt++) {
+ try {
+ credits_billed = await readRequestCreditsFromAnalytics(requestId, {
+ emptyAsZero: false,
+ });
+ } catch (error) {
+ logger.warn("Analytics request credits read failed", {
+ error,
+ attempt,
+ });
+ }
+ if (credits_billed !== null) break;
+ if (attempt < FINALIZE_CREDITS_ATTEMPTS) {
+ await new Promise(resolve =>
+ setTimeout(resolve, FINALIZE_CREDITS_RETRY_MS),
+ );
}
- } catch (error) {
- logger.warn("Credits billed is null", { error });
}
+ }
- if (credits_billed === null) {
- logger.warn("Credits billed is null", {});
- }
+ if (credits_billed === null) {
+ // The row's credits_cost is NOT NULL, so the record has to carry a
+ // number; 0 is written and the gap is loud rather than silent.
+ logger.error(
+ "Credits billed unknown at crawl finalization; recording 0",
+ { requestId, attempts: FINALIZE_CREDITS_ATTEMPTS },
+ );
}
if (sc.crawlerOptions !== null) {