Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): give crawl children Bigtable rows; sum old crawl credits from ClickHouse (#4693)

Gergő Móricz94654e6bae77
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts
index 88662a5b1..8a66ebf25 100644
--- a/apps/api/src/controllers/v1/crawl-status.ts
+++ b/apps/api/src/controllers/v1/crawl-status.ts
@@ -31,6 +31,7 @@ import { ScrapeJobSingleUrls } from "../../types";
import { readScrapeJobState } from "../../lib/job-state-store";
import { readRequestCredits } from "../../lib/request-credits-store";
import { recordJobStorePostgresFallback } from "../../lib/job-store-fallback";
+import { readRequestCreditsFromAnalytics } from "../../lib/request-credits-analytics";
configDotenv();
export type PseudoJob<T> = {
@@ -73,7 +74,7 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {
]);
if (!nuqJob && !scrapeState && !dbScrape) return null;
- if (!scrapeState && !scrapeStateFailed && dbScrape) {
+ if (!nuqJob && !scrapeState && !scrapeStateFailed && dbScrape) {
recordJobStorePostgresFallback("scrape_state", id);
}
@@ -205,6 +206,16 @@ export async function crawlStatusController(
creditsReadFailed = true;
return null;
});
+ if (creditsBilled === null) {
+ // Requests from before the Bigtable credit rows existed: sum the scrape
+ // job log instead.
+ creditsBilled = await readRequestCreditsFromAnalytics(
+ req.params.jobId,
+ ).catch(error => {
+ logger.warn("Analytics request credits read failed", { error });
+ return null;
+ });
+ }
if (creditsBilled === null && config.USE_DB_AUTHENTICATION) {
creditsBilled = await creditsBilledByCrawlId(dbRr, req.params.jobId)
.then(rows => rows[0]?.credits_billed ?? null)
diff --git a/apps/api/src/controllers/v1/scrape-status.ts b/apps/api/src/controllers/v1/scrape-status.ts
index b547b80ba..a6936c076 100644
--- a/apps/api/src/controllers/v1/scrape-status.ts
+++ b/apps/api/src/controllers/v1/scrape-status.ts
@@ -3,6 +3,7 @@ import { getJob } from "./crawl-status";
import { logger as _logger } from "../../lib/logger";
import { getScrapeZDR } from "../../lib/zdr-helpers";
import { getScrapeJobAccess } from "../../lib/operational-job-access";
+import { normalizeJobAccessTeamId } from "../../lib/job-access-store";
export async function scrapeStatusController(req: any, res: any) {
const logger = _logger.child({
@@ -31,7 +32,9 @@ export async function scrapeStatusController(req: any, res: any) {
});
}
- if (access.teamId !== req.auth.team_id) {
+ // Access rows store the normalized team (preview and keyless callers map to
+ // a placeholder UUID), so compare against the same normalization.
+ if (access.teamId !== normalizeJobAccessTeamId(req.auth.team_id)) {
return res.status(403).json({
success: false,
error: "You are not allowed to access this resource.",
diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.ts
index 4aceae22d..81678c947 100644
--- a/apps/api/src/controllers/v2/crawl-status.ts
+++ b/apps/api/src/controllers/v2/crawl-status.ts
@@ -34,6 +34,7 @@ import { isBaseDomain, extractBaseDomain } from "../../lib/url-utils";
import { readScrapeJobState } from "../../lib/job-state-store";
import { readRequestCredits } from "../../lib/request-credits-store";
import { recordJobStorePostgresFallback } from "../../lib/job-store-fallback";
+import { readRequestCreditsFromAnalytics } from "../../lib/request-credits-analytics";
configDotenv();
export type PseudoJob<T> = {
@@ -82,7 +83,7 @@ export async function getJob(
]);
if (!nuqJob && !scrapeState && !dbScrape) return null;
- if (!scrapeState && !scrapeStateFailed && dbScrape) {
+ if (!nuqJob && !scrapeState && !scrapeStateFailed && dbScrape) {
recordJobStorePostgresFallback("scrape_state", id);
}
@@ -228,6 +229,16 @@ export async function crawlStatusController(
creditsReadFailed = true;
return null;
});
+ if (creditsBilled === null) {
+ // Requests from before the Bigtable credit rows existed: sum the scrape
+ // job log instead.
+ creditsBilled = await readRequestCreditsFromAnalytics(
+ req.params.jobId,
+ ).catch(error => {
+ logger.warn("Analytics request credits read failed", { error });
+ return null;
+ });
+ }
if (creditsBilled === null && config.USE_DB_AUTHENTICATION) {
creditsBilled = await creditsBilledByCrawlId(dbRr, req.params.jobId)
.then(rows => rows[0]?.credits_billed ?? null)
diff --git a/apps/api/src/controllers/v2/scrape-status.ts b/apps/api/src/controllers/v2/scrape-status.ts
index 09f66647f..f3d9ab3c8 100644
--- a/apps/api/src/controllers/v2/scrape-status.ts
+++ b/apps/api/src/controllers/v2/scrape-status.ts
@@ -2,6 +2,7 @@ import { getJob } from "./crawl-status";
import { logger as _logger } from "../../lib/logger";
import { getScrapeZDR } from "../../lib/zdr-helpers";
import { getScrapeJobAccess } from "../../lib/operational-job-access";
+import { normalizeJobAccessTeamId } from "../../lib/job-access-store";
export async function scrapeStatusController(req: any, res: any) {
const uuidReg =
@@ -39,7 +40,9 @@ export async function scrapeStatusController(req: any, res: any) {
});
}
- if (access.teamId !== req.auth.team_id) {
+ // Access rows store the normalized team (preview and keyless callers map to
+ // a placeholder UUID), so compare against the same normalization.
+ if (access.teamId !== normalizeJobAccessTeamId(req.auth.team_id)) {
return res.status(403).json({
success: false,
error: "You are not allowed to access this resource.",
diff --git a/apps/api/src/lib/change-tracking-store.ts b/apps/api/src/lib/change-tracking-store.ts
index a46138b20..f4b770a73 100644
--- a/apps/api/src/lib/change-tracking-store.ts
+++ b/apps/api/src/lib/change-tracking-store.ts
@@ -3,7 +3,6 @@ import { config } from "../config";
import { diffGetLastScrape } from "../db/rpc";
import { getBigtableTable } from "./bigtable-client";
import { setSpanAttributes, withSpan } from "./otel-tracer";
-import { recordJobStorePostgresFallback } from "./job-store-fallback";
// Change tracking bookkeeping. Final architecture (#4484 was the
// transition): Bigtable is the store. Writes go to Bigtable only --
@@ -159,9 +158,6 @@ export async function changeTrackingGetLastScrape(params: {
);
const pgRow = pgRows[0];
if (!pgRow) return null;
- recordJobStorePostgresFallback("change_tracking", pgRow.o_job_id, {
- teamId: params.team_id,
- });
return {
job_id: pgRow.o_job_id,
date_added: new Date(pgRow.o_date_added).toISOString(),
diff --git a/apps/api/src/lib/job-store-fallback.ts b/apps/api/src/lib/job-store-fallback.ts
index b7cf09c89..0d8451e85 100644
--- a/apps/api/src/lib/job-store-fallback.ts
+++ b/apps/api/src/lib/job-store-fallback.ts
@@ -25,8 +25,7 @@ type JobStoreFallback =
| "scrape_state"
| "extract_state"
| "feedback_job"
- | "request_credits"
- | "change_tracking";
+ | "request_credits";
/**
* The Bigtable table behind each store. When it is not configured the
@@ -39,7 +38,6 @@ const STORE_TABLE: Record<JobStoreFallback, () => string | undefined> = {
extract_state: () => config.BIGTABLE_EXTRACT_STATE_TABLE,
feedback_job: () => config.BIGTABLE_FEEDBACK_JOBS_TABLE,
request_credits: () => config.BIGTABLE_REQUEST_CREDITS_TABLE,
- change_tracking: () => config.BIGTABLE_CHANGE_TRACKING_TABLE,
};
/**
diff --git a/apps/api/src/lib/operational-job-access.test.ts b/apps/api/src/lib/operational-job-access.test.ts
index 92d9ce575..588a7e21a 100644
--- a/apps/api/src/lib/operational-job-access.test.ts
+++ b/apps/api/src/lib/operational-job-access.test.ts
@@ -1,11 +1,13 @@
const {
readApiJobAccess,
+ recordJobStorePostgresFallback,
supabaseGetAgentRequestByIdDirect,
supabaseGetCrawlRequestById,
supabaseGetExtractRequestByIdDirect,
supabaseGetScrapeById,
} = vi.hoisted(() => ({
readApiJobAccess: vi.fn(),
+ recordJobStorePostgresFallback: vi.fn(),
supabaseGetAgentRequestByIdDirect: vi.fn(),
supabaseGetCrawlRequestById: vi.fn(),
supabaseGetExtractRequestByIdDirect: vi.fn(),
@@ -13,6 +15,10 @@ const {
}));
vi.mock("./job-access-store", () => ({ readApiJobAccess }));
+vi.mock("./job-store-fallback", () => ({ recordJobStorePostgresFallback }));
+vi.mock("./logger", () => ({
+ logger: { warn: vi.fn(), info: vi.fn(), error: vi.fn() },
+}));
vi.mock("./supabase-jobs", () => ({
supabaseGetAgentRequestByIdDirect,
supabaseGetCrawlRequestById,
@@ -44,14 +50,15 @@ describe("operational job access", () => {
expect(supabaseGetAgentRequestByIdDirect).not.toHaveBeenCalled();
});
- it("maps a PostgreSQL fallback into the operational access type", async () => {
+ it("maps a live PostgreSQL fallback into the operational access type and counts it", async () => {
+ const createdAt = new Date(Date.now() - 60_000);
readApiJobAccess.mockResolvedValue(null);
supabaseGetExtractRequestByIdDirect.mockResolvedValue({
id: JOB_ID,
team_id: "team-id",
kind: "extract",
origin: "api",
- created_at: "2026-09-16T12:00:00.000Z",
+ created_at: createdAt.toISOString(),
unrelated_column: "not returned",
});
@@ -59,8 +66,41 @@ describe("operational job access", () => {
teamId: "team-id",
kind: "extract",
clientOrigin: "api",
- expiresAtMs: new Date("2026-09-17T12:00:00.000Z").getTime(),
+ expiresAtMs: createdAt.getTime() + 24 * 60 * 60 * 1000,
+ });
+ expect(recordJobStorePostgresFallback).toHaveBeenCalledWith(
+ "job_access",
+ JOB_ID,
+ { kind: "extract" },
+ );
+ });
+
+ it("does not count an expired PostgreSQL fallback (the caller 404s either way)", async () => {
+ readApiJobAccess.mockResolvedValue(null);
+ supabaseGetScrapeById.mockResolvedValue({
+ id: JOB_ID,
+ team_id: "team-id",
+ created_at: new Date(Date.now() - 25 * 60 * 60 * 1000).toISOString(),
+ });
+
+ const access = await getScrapeJobAccess(JOB_ID);
+ expect(access?.expiresAtMs).toBeLessThan(Date.now());
+ expect(recordJobStorePostgresFallback).not.toHaveBeenCalled();
+ });
+
+ it("does not count a fallback taken because the Bigtable read failed", async () => {
+ readApiJobAccess.mockRejectedValue(new Error("Bigtable unavailable"));
+ supabaseGetScrapeById.mockResolvedValue({
+ id: JOB_ID,
+ team_id: "team-id",
+ created_at: new Date().toISOString(),
+ });
+
+ await expect(getScrapeJobAccess(JOB_ID)).resolves.toMatchObject({
+ teamId: "team-id",
+ kind: "scrape",
});
+ expect(recordJobStorePostgresFallback).not.toHaveBeenCalled();
});
it("does not fall back when Bigtable has an expired record", async () => {
diff --git a/apps/api/src/lib/operational-job-access.ts b/apps/api/src/lib/operational-job-access.ts
index 32fc4f39b..f36bd7f98 100644
--- a/apps/api/src/lib/operational-job-access.ts
+++ b/apps/api/src/lib/operational-job-access.ts
@@ -44,7 +44,9 @@ async function resolveOperationalJobAccess(params: {
const fallback = await params.fallback();
if (!fallback || !Number.isFinite(fallback.expiresAtMs)) return null;
- if (!readFailed) {
+ // An expired row makes the caller answer 404 exactly as a miss would, so
+ // only a live row counts as PostgreSQL having been needed.
+ if (!readFailed && fallback.expiresAtMs > Date.now()) {
recordJobStorePostgresFallback("job_access", params.id, {
kind: fallback.kind,
});
diff --git a/apps/api/src/lib/request-credits-analytics.test.ts b/apps/api/src/lib/request-credits-analytics.test.ts
new file mode 100644
index 000000000..9c5b18208
--- /dev/null
+++ b/apps/api/src/lib/request-credits-analytics.test.ts
@@ -0,0 +1,68 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const { query, client } = vi.hoisted(() => {
+ const query = vi.fn();
+ return {
+ query,
+ client: { current: { query } as { query: typeof query } | null },
+ };
+});
+vi.mock("./clickhouse-client", () => ({
+ get clickhouseClient() {
+ return client.current;
+ },
+}));
+vi.mock("./otel-tracer", () => ({
+ withSpan: vi.fn(async (_name: string, fn: (span: object) => unknown) =>
+ fn({}),
+ ),
+ setSpanAttributes: vi.fn(),
+}));
+
+import { readRequestCreditsFromAnalytics } from "./request-credits-analytics";
+
+function rows(list: Record<string, unknown>[]) {
+ query.mockResolvedValueOnce({ json: vi.fn(async () => list) });
+}
+
+describe("readRequestCreditsFromAnalytics", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ client.current = { query };
+ });
+
+ it("sums the request's scrape rows", async () => {
+ rows([{ credits: "17", jobs: "4" }]);
+ await expect(readRequestCreditsFromAnalytics("request-1")).resolves.toBe(
+ 17,
+ );
+ expect(query.mock.calls[0][0]).toMatchObject({
+ query: expect.stringContaining(
+ "FROM scrapes_by_request FINAL WHERE request_id = {requestId: UUID}",
+ ),
+ query_params: { requestId: "request-1" },
+ });
+ });
+
+ it("is null when the request has no scrape rows", async () => {
+ rows([{ credits: 0, jobs: 0 }]);
+ await expect(
+ readRequestCreditsFromAnalytics("request-1"),
+ ).resolves.toBeNull();
+ });
+
+ it("is null when ClickHouse is not configured", async () => {
+ client.current = null;
+ await expect(
+ readRequestCreditsFromAnalytics("request-1"),
+ ).resolves.toBeNull();
+ expect(query).not.toHaveBeenCalled();
+ });
+
+ it("propagates ClickHouse errors", async () => {
+ query.mockRejectedValueOnce(new Error("ClickHouse unavailable"));
+ await expect(readRequestCreditsFromAnalytics("request-1")).rejects.toThrow(
+ "ClickHouse unavailable",
+ );
+ });
+});
diff --git a/apps/api/src/lib/request-credits-analytics.ts b/apps/api/src/lib/request-credits-analytics.ts
new file mode 100644
index 000000000..192f9fdbe
--- /dev/null
+++ b/apps/api/src/lib/request-credits-analytics.ts
@@ -0,0 +1,56 @@
+import { clickhouseClient } from "./clickhouse-client";
+import { setSpanAttributes, withSpan } from "./otel-tracer";
+
+/**
+ * Credits billed under a request, summed from the `scrapes_by_request` copy
+ * of the scrape job log in the analytics ClickHouse service.
+ *
+ * The Bigtable request-credits row is the primary source. It only exists for
+ * requests started after that store went live, so crawls and batch scrapes
+ * from before then have nothing there and this sum is what answers for them.
+ * ClickPipes lands a scrape row about a second after the worker logs it, and
+ * `FINAL` collapses any redelivered copy, so the sum matches what the job log
+ * recorded.
+ *
+ * Returns null when ClickHouse is not configured or the request has no scrape
+ * rows at all, so callers can tell "nothing billed yet" from "unknown".
+ */
+export async function readRequestCreditsFromAnalytics(
+ requestId: string,
+): Promise<number | null> {
+ const client = clickhouseClient;
+ if (client === null) return null;
+
+ return withSpan("clickhouse.request_credits.read", async span => {
+ setSpanAttributes(span, {
+ "db.system": "clickhouse",
+ "db.collection.name": "scrapes_by_request",
+ "request_credits.request_id": requestId,
+ });
+ const result = await client.query({
+ query:
+ "SELECT sum(credits_cost) AS credits, count() AS jobs FROM scrapes_by_request FINAL WHERE request_id = {requestId: UUID}",
+ query_params: { requestId },
+ format: "JSONEachRow",
+ });
+ const [row] = await result.json<{
+ credits: number | string;
+ jobs: number | string;
+ }>();
+ const jobs = Number(row?.jobs ?? 0);
+ if (!Number.isFinite(jobs) || jobs === 0) {
+ setSpanAttributes(span, { "request_credits.outcome": "not_found" });
+ return null;
+ }
+ const credits = Number(row.credits);
+ if (!Number.isSafeInteger(credits)) {
+ throw new Error(`Invalid analytics credits total: ${row.credits}`);
+ }
+ setSpanAttributes(span, {
+ "request_credits.outcome": "found",
+ "request_credits.jobs": jobs,
+ "request_credits.total": credits,
+ });
+ return credits;
+ });
+}
diff --git a/apps/api/src/services/logging/log_job.test.ts b/apps/api/src/services/logging/log_job.test.ts
index 66e98631e..dcff85ef3 100644
--- a/apps/api/src/services/logging/log_job.test.ts
+++ b/apps/api/src/services/logging/log_job.test.ts
@@ -319,6 +319,37 @@ describe("operational job state logging", () => {
);
});
+ it("writes job access and terminal state for a crawl child", async () => {
+ const id = "019e6f45-7778-727d-adf0-0abe9d5062b7";
+ const crawlId = "019e6f45-7778-727d-adf0-0abe9d5062b6";
+ await logScrape({
+ id,
+ request_id: crawlId,
+ url: "https://example.com/page",
+ is_successful: false,
+ error: "boom",
+ time_taken: 1,
+ team_id: "team-id",
+ options: { formats: ["markdown"] } as any,
+ credits_cost: 1,
+ skipNuq: false,
+ zeroDataRetention: false,
+ });
+
+ expect(writeApiJobAccess).toHaveBeenCalledWith(
+ expect.objectContaining({ id, teamId: "team-id", kind: "scrape" }),
+ );
+ expect(writeScrapeJobState).toHaveBeenCalledWith(
+ id,
+ expect.objectContaining({
+ status: "failed",
+ requestId: crawlId,
+ creditsBilled: 1,
+ error: "boom",
+ }),
+ );
+ });
+
it("writes terminal extract state", async () => {
const id = "019e6f45-7778-727d-adf0-0abe9d5062b6";
await logExtract({
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index 15806a383..c7f4ec227 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -663,6 +663,8 @@ export type LoggedScrape = {
is_parse?: boolean;
monitor_id?: string | null;
monitor_check_id?: string | null;
+ /** False for jobs that must not be fetchable by id (defaults to true). */
+ jobAccess?: boolean;
};
export async function logScrape(scrape: LoggedScrape, force: boolean = false) {
@@ -752,9 +754,30 @@ async function logScrapeInternal(scrape: LoggedScrape, force: boolean = false) {
},
logger,
);
+
+ // A crawl or batch child is fetched by its own id (`GET /scrape/{id}`,
+ // typically off the crawl `page` webhook), so it needs its own access row.
+ // Standalone scrapes get theirs from logRequest.
+ if (!scrape.is_parse && scrape.jobAccess !== false) {
+ try {
+ await writeApiJobAccess({
+ id: scrape.id,
+ teamId: normalizeJobAccessTeamId(scrape.team_id),
+ kind: "scrape",
+ expiresAt: new Date(Date.now() + DEFAULT_JOB_ACCESS_TTL_MS),
+ zeroDataRetention: scrape.zeroDataRetention,
+ });
+ } catch (error) {
+ logger.error("Failed to write child scrape job access to Bigtable", {
+ error,
+ });
+ }
+ }
}
- if (!scrape.is_parse && scrape.id === scrape.request_id) {
+ // Terminal state for every scrape job, standalone or child, so status reads
+ // never need the PostgreSQL row.
+ if (!scrape.is_parse) {
try {
const replay = scrape.zeroDataRetention
? undefined
diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.ts
index e4ae1e5fb..204fcbf0b 100644
--- a/apps/api/src/services/worker/crawl-logic.ts
+++ b/apps/api/src/services/worker/crawl-logic.ts
@@ -14,6 +14,7 @@ import { logCrawl, logBatchScrape } from "../logging/log_job";
import { createWebhookSender, WebhookEvent } from "../webhook/index";
import type { NuQJob } from "./nuq";
import { recordJobStorePostgresFallback } from "../../lib/job-store-fallback";
+import { readRequestCreditsFromAnalytics } from "../../lib/request-credits-analytics";
export async function finishCrawlSuper(job: NuQJob<any>) {
const crawlId = job.groupId;
@@ -153,6 +154,14 @@ export async function finishCrawlSuper(job: NuQJob<any>) {
logger.warn("Bigtable request credits read failed", { error });
}
+ if (credits_billed === null) {
+ try {
+ credits_billed = await readRequestCreditsFromAnalytics(crawlId);
+ } catch (error) {
+ logger.warn("Analytics request credits read failed", { error });
+ }
+ }
+
if (credits_billed === null && config.USE_DB_AUTHENTICATION) {
try {
const creditsRows = await creditsBilledByCrawlId(db, crawlId);