Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): give crawl children Bigtable rows; sum old crawl credits from ClickHouse (#4693)
Gergő Móricz94654e6bae77
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex 88662a5b1..8a66ebf25 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -31,6 +31,7 @@ import { ScrapeJobSingleUrls } from "../../types";import { readScrapeJobState } from "../../lib/job-state-store";import { readRequestCredits } from "../../lib/request-credits-store";import { recordJobStorePostgresFallback } from "../../lib/job-store-fallback";+import { readRequestCreditsFromAnalytics } from "../../lib/request-credits-analytics";configDotenv();export type PseudoJob<T> = {@@ -73,7 +74,7 @@ export async function getJob(id: string): Promise<PseudoJob<any> | null> {]);if (!nuqJob && !scrapeState && !dbScrape) return null;- if (!scrapeState && !scrapeStateFailed && dbScrape) {+ if (!nuqJob && !scrapeState && !scrapeStateFailed && dbScrape) {recordJobStorePostgresFallback("scrape_state", id);}@@ -205,6 +206,16 @@ export async function crawlStatusController(creditsReadFailed = true;return null;});+ if (creditsBilled === null) {+ // Requests from before the Bigtable credit rows existed: sum the scrape+ // job log instead.+ creditsBilled = await readRequestCreditsFromAnalytics(+ req.params.jobId,+ ).catch(error => {+ logger.warn("Analytics request credits read failed", { error });+ return null;+ });+ }if (creditsBilled === null && config.USE_DB_AUTHENTICATION) {creditsBilled = await creditsBilledByCrawlId(dbRr, req.params.jobId).then(rows => rows[0]?.credits_billed ?? null)diff --git a/apps/api/src/controllers/v1/scrape-status.ts b/apps/api/src/controllers/v1/scrape-status.tsindex b547b80ba..a6936c076 100644--- a/apps/api/src/controllers/v1/scrape-status.ts+++ b/apps/api/src/controllers/v1/scrape-status.ts@@ -3,6 +3,7 @@ import { getJob } from "./crawl-status";import { logger as _logger } from "../../lib/logger";import { getScrapeZDR } from "../../lib/zdr-helpers";import { getScrapeJobAccess } from "../../lib/operational-job-access";+import { normalizeJobAccessTeamId } from "../../lib/job-access-store";export async function scrapeStatusController(req: any, res: any) {const logger = _logger.child({@@ -31,7 +32,9 @@ export async function scrapeStatusController(req: any, res: any) {});}- if (access.teamId !== req.auth.team_id) {+ // Access rows store the normalized team (preview and keyless callers map to+ // a placeholder UUID), so compare against the same normalization.+ if (access.teamId !== normalizeJobAccessTeamId(req.auth.team_id)) {return res.status(403).json({success: false,error: "You are not allowed to access this resource.",diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.tsindex 4aceae22d..81678c947 100644--- a/apps/api/src/controllers/v2/crawl-status.ts+++ b/apps/api/src/controllers/v2/crawl-status.ts@@ -34,6 +34,7 @@ import { isBaseDomain, extractBaseDomain } from "../../lib/url-utils";import { readScrapeJobState } from "../../lib/job-state-store";import { readRequestCredits } from "../../lib/request-credits-store";import { recordJobStorePostgresFallback } from "../../lib/job-store-fallback";+import { readRequestCreditsFromAnalytics } from "../../lib/request-credits-analytics";configDotenv();export type PseudoJob<T> = {@@ -82,7 +83,7 @@ export async function getJob(]);if (!nuqJob && !scrapeState && !dbScrape) return null;- if (!scrapeState && !scrapeStateFailed && dbScrape) {+ if (!nuqJob && !scrapeState && !scrapeStateFailed && dbScrape) {recordJobStorePostgresFallback("scrape_state", id);}@@ -228,6 +229,16 @@ export async function crawlStatusController(creditsReadFailed = true;return null;});+ if (creditsBilled === null) {+ // Requests from before the Bigtable credit rows existed: sum the scrape+ // job log instead.+ creditsBilled = await readRequestCreditsFromAnalytics(+ req.params.jobId,+ ).catch(error => {+ logger.warn("Analytics request credits read failed", { error });+ return null;+ });+ }if (creditsBilled === null && config.USE_DB_AUTHENTICATION) {creditsBilled = await creditsBilledByCrawlId(dbRr, req.params.jobId).then(rows => rows[0]?.credits_billed ?? null)diff --git a/apps/api/src/controllers/v2/scrape-status.ts b/apps/api/src/controllers/v2/scrape-status.tsindex 09f66647f..f3d9ab3c8 100644--- a/apps/api/src/controllers/v2/scrape-status.ts+++ b/apps/api/src/controllers/v2/scrape-status.ts@@ -2,6 +2,7 @@ import { getJob } from "./crawl-status";import { logger as _logger } from "../../lib/logger";import { getScrapeZDR } from "../../lib/zdr-helpers";import { getScrapeJobAccess } from "../../lib/operational-job-access";+import { normalizeJobAccessTeamId } from "../../lib/job-access-store";export async function scrapeStatusController(req: any, res: any) {const uuidReg =@@ -39,7 +40,9 @@ export async function scrapeStatusController(req: any, res: any) {});}- if (access.teamId !== req.auth.team_id) {+ // Access rows store the normalized team (preview and keyless callers map to+ // a placeholder UUID), so compare against the same normalization.+ if (access.teamId !== normalizeJobAccessTeamId(req.auth.team_id)) {return res.status(403).json({success: false,error: "You are not allowed to access this resource.",diff --git a/apps/api/src/lib/change-tracking-store.ts b/apps/api/src/lib/change-tracking-store.tsindex a46138b20..f4b770a73 100644--- a/apps/api/src/lib/change-tracking-store.ts+++ b/apps/api/src/lib/change-tracking-store.ts@@ -3,7 +3,6 @@ import { config } from "../config";import { diffGetLastScrape } from "../db/rpc";import { getBigtableTable } from "./bigtable-client";import { setSpanAttributes, withSpan } from "./otel-tracer";-import { recordJobStorePostgresFallback } from "./job-store-fallback";// Change tracking bookkeeping. Final architecture (#4484 was the// transition): Bigtable is the store. Writes go to Bigtable only --@@ -159,9 +158,6 @@ export async function changeTrackingGetLastScrape(params: {);const pgRow = pgRows[0];if (!pgRow) return null;- recordJobStorePostgresFallback("change_tracking", pgRow.o_job_id, {- teamId: params.team_id,- });return {job_id: pgRow.o_job_id,date_added: new Date(pgRow.o_date_added).toISOString(),diff --git a/apps/api/src/lib/job-store-fallback.ts b/apps/api/src/lib/job-store-fallback.tsindex b7cf09c89..0d8451e85 100644--- a/apps/api/src/lib/job-store-fallback.ts+++ b/apps/api/src/lib/job-store-fallback.ts@@ -25,8 +25,7 @@ type JobStoreFallback =| "scrape_state"| "extract_state"| "feedback_job"- | "request_credits"- | "change_tracking";+ | "request_credits";/*** The Bigtable table behind each store. When it is not configured the@@ -39,7 +38,6 @@ const STORE_TABLE: Record<JobStoreFallback, () => string | undefined> = {extract_state: () => config.BIGTABLE_EXTRACT_STATE_TABLE,feedback_job: () => config.BIGTABLE_FEEDBACK_JOBS_TABLE,request_credits: () => config.BIGTABLE_REQUEST_CREDITS_TABLE,- change_tracking: () => config.BIGTABLE_CHANGE_TRACKING_TABLE,};/**diff --git a/apps/api/src/lib/operational-job-access.test.ts b/apps/api/src/lib/operational-job-access.test.tsindex 92d9ce575..588a7e21a 100644--- a/apps/api/src/lib/operational-job-access.test.ts+++ b/apps/api/src/lib/operational-job-access.test.ts@@ -1,11 +1,13 @@const {readApiJobAccess,+ recordJobStorePostgresFallback,supabaseGetAgentRequestByIdDirect,supabaseGetCrawlRequestById,supabaseGetExtractRequestByIdDirect,supabaseGetScrapeById,} = vi.hoisted(() => ({readApiJobAccess: vi.fn(),+ recordJobStorePostgresFallback: vi.fn(),supabaseGetAgentRequestByIdDirect: vi.fn(),supabaseGetCrawlRequestById: vi.fn(),supabaseGetExtractRequestByIdDirect: vi.fn(),@@ -13,6 +15,10 @@ const {}));vi.mock("./job-access-store", () => ({ readApiJobAccess }));+vi.mock("./job-store-fallback", () => ({ recordJobStorePostgresFallback }));+vi.mock("./logger", () => ({+ logger: { warn: vi.fn(), info: vi.fn(), error: vi.fn() },+}));vi.mock("./supabase-jobs", () => ({supabaseGetAgentRequestByIdDirect,supabaseGetCrawlRequestById,@@ -44,14 +50,15 @@ describe("operational job access", () => {expect(supabaseGetAgentRequestByIdDirect).not.toHaveBeenCalled();});- it("maps a PostgreSQL fallback into the operational access type", async () => {+ it("maps a live PostgreSQL fallback into the operational access type and counts it", async () => {+ const createdAt = new Date(Date.now() - 60_000);readApiJobAccess.mockResolvedValue(null);supabaseGetExtractRequestByIdDirect.mockResolvedValue({id: JOB_ID,team_id: "team-id",kind: "extract",origin: "api",- created_at: "2026-09-16T12:00:00.000Z",+ created_at: createdAt.toISOString(),unrelated_column: "not returned",});@@ -59,8 +66,41 @@ describe("operational job access", () => {teamId: "team-id",kind: "extract",clientOrigin: "api",- expiresAtMs: new Date("2026-09-17T12:00:00.000Z").getTime(),+ expiresAtMs: createdAt.getTime() + 24 * 60 * 60 * 1000,+ });+ expect(recordJobStorePostgresFallback).toHaveBeenCalledWith(+ "job_access",+ JOB_ID,+ { kind: "extract" },+ );+ });++ it("does not count an expired PostgreSQL fallback (the caller 404s either way)", async () => {+ readApiJobAccess.mockResolvedValue(null);+ supabaseGetScrapeById.mockResolvedValue({+ id: JOB_ID,+ team_id: "team-id",+ created_at: new Date(Date.now() - 25 * 60 * 60 * 1000).toISOString(),+ });++ const access = await getScrapeJobAccess(JOB_ID);+ expect(access?.expiresAtMs).toBeLessThan(Date.now());+ expect(recordJobStorePostgresFallback).not.toHaveBeenCalled();+ });++ it("does not count a fallback taken because the Bigtable read failed", async () => {+ readApiJobAccess.mockRejectedValue(new Error("Bigtable unavailable"));+ supabaseGetScrapeById.mockResolvedValue({+ id: JOB_ID,+ team_id: "team-id",+ created_at: new Date().toISOString(),+ });++ await expect(getScrapeJobAccess(JOB_ID)).resolves.toMatchObject({+ teamId: "team-id",+ kind: "scrape",});+ expect(recordJobStorePostgresFallback).not.toHaveBeenCalled();});it("does not fall back when Bigtable has an expired record", async () => {diff --git a/apps/api/src/lib/operational-job-access.ts b/apps/api/src/lib/operational-job-access.tsindex 32fc4f39b..f36bd7f98 100644--- a/apps/api/src/lib/operational-job-access.ts+++ b/apps/api/src/lib/operational-job-access.ts@@ -44,7 +44,9 @@ async function resolveOperationalJobAccess(params: {const fallback = await params.fallback();if (!fallback || !Number.isFinite(fallback.expiresAtMs)) return null;- if (!readFailed) {+ // An expired row makes the caller answer 404 exactly as a miss would, so+ // only a live row counts as PostgreSQL having been needed.+ if (!readFailed && fallback.expiresAtMs > Date.now()) {recordJobStorePostgresFallback("job_access", params.id, {kind: fallback.kind,});diff --git a/apps/api/src/lib/request-credits-analytics.test.ts b/apps/api/src/lib/request-credits-analytics.test.tsnew file mode 100644index 000000000..9c5b18208--- /dev/null+++ b/apps/api/src/lib/request-credits-analytics.test.ts@@ -0,0 +1,68 @@+import { beforeEach, describe, expect, it, vi } from "vitest";++const { query, client } = vi.hoisted(() => {+ const query = vi.fn();+ return {+ query,+ client: { current: { query } as { query: typeof query } | null },+ };+});+vi.mock("./clickhouse-client", () => ({+ get clickhouseClient() {+ return client.current;+ },+}));+vi.mock("./otel-tracer", () => ({+ withSpan: vi.fn(async (_name: string, fn: (span: object) => unknown) =>+ fn({}),+ ),+ setSpanAttributes: vi.fn(),+}));++import { readRequestCreditsFromAnalytics } from "./request-credits-analytics";++function rows(list: Record<string, unknown>[]) {+ query.mockResolvedValueOnce({ json: vi.fn(async () => list) });+}++describe("readRequestCreditsFromAnalytics", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ client.current = { query };+ });++ it("sums the request's scrape rows", async () => {+ rows([{ credits: "17", jobs: "4" }]);+ await expect(readRequestCreditsFromAnalytics("request-1")).resolves.toBe(+ 17,+ );+ expect(query.mock.calls[0][0]).toMatchObject({+ query: expect.stringContaining(+ "FROM scrapes_by_request FINAL WHERE request_id = {requestId: UUID}",+ ),+ query_params: { requestId: "request-1" },+ });+ });++ it("is null when the request has no scrape rows", async () => {+ rows([{ credits: 0, jobs: 0 }]);+ await expect(+ readRequestCreditsFromAnalytics("request-1"),+ ).resolves.toBeNull();+ });++ it("is null when ClickHouse is not configured", async () => {+ client.current = null;+ await expect(+ readRequestCreditsFromAnalytics("request-1"),+ ).resolves.toBeNull();+ expect(query).not.toHaveBeenCalled();+ });++ it("propagates ClickHouse errors", async () => {+ query.mockRejectedValueOnce(new Error("ClickHouse unavailable"));+ await expect(readRequestCreditsFromAnalytics("request-1")).rejects.toThrow(+ "ClickHouse unavailable",+ );+ });+});diff --git a/apps/api/src/lib/request-credits-analytics.ts b/apps/api/src/lib/request-credits-analytics.tsnew file mode 100644index 000000000..192f9fdbe--- /dev/null+++ b/apps/api/src/lib/request-credits-analytics.ts@@ -0,0 +1,56 @@+import { clickhouseClient } from "./clickhouse-client";+import { setSpanAttributes, withSpan } from "./otel-tracer";++/**+ * Credits billed under a request, summed from the `scrapes_by_request` copy+ * of the scrape job log in the analytics ClickHouse service.+ *+ * The Bigtable request-credits row is the primary source. It only exists for+ * requests started after that store went live, so crawls and batch scrapes+ * from before then have nothing there and this sum is what answers for them.+ * ClickPipes lands a scrape row about a second after the worker logs it, and+ * `FINAL` collapses any redelivered copy, so the sum matches what the job log+ * recorded.+ *+ * Returns null when ClickHouse is not configured or the request has no scrape+ * rows at all, so callers can tell "nothing billed yet" from "unknown".+ */+export async function readRequestCreditsFromAnalytics(+ requestId: string,+): Promise<number | null> {+ const client = clickhouseClient;+ if (client === null) return null;++ return withSpan("clickhouse.request_credits.read", async span => {+ setSpanAttributes(span, {+ "db.system": "clickhouse",+ "db.collection.name": "scrapes_by_request",+ "request_credits.request_id": requestId,+ });+ const result = await client.query({+ query:+ "SELECT sum(credits_cost) AS credits, count() AS jobs FROM scrapes_by_request FINAL WHERE request_id = {requestId: UUID}",+ query_params: { requestId },+ format: "JSONEachRow",+ });+ const [row] = await result.json<{+ credits: number | string;+ jobs: number | string;+ }>();+ const jobs = Number(row?.jobs ?? 0);+ if (!Number.isFinite(jobs) || jobs === 0) {+ setSpanAttributes(span, { "request_credits.outcome": "not_found" });+ return null;+ }+ const credits = Number(row.credits);+ if (!Number.isSafeInteger(credits)) {+ throw new Error(`Invalid analytics credits total: ${row.credits}`);+ }+ setSpanAttributes(span, {+ "request_credits.outcome": "found",+ "request_credits.jobs": jobs,+ "request_credits.total": credits,+ });+ return credits;+ });+}diff --git a/apps/api/src/services/logging/log_job.test.ts b/apps/api/src/services/logging/log_job.test.tsindex 66e98631e..dcff85ef3 100644--- a/apps/api/src/services/logging/log_job.test.ts+++ b/apps/api/src/services/logging/log_job.test.ts@@ -319,6 +319,37 @@ describe("operational job state logging", () => {);});+ it("writes job access and terminal state for a crawl child", async () => {+ const id = "019e6f45-7778-727d-adf0-0abe9d5062b7";+ const crawlId = "019e6f45-7778-727d-adf0-0abe9d5062b6";+ await logScrape({+ id,+ request_id: crawlId,+ url: "https://example.com/page",+ is_successful: false,+ error: "boom",+ time_taken: 1,+ team_id: "team-id",+ options: { formats: ["markdown"] } as any,+ credits_cost: 1,+ skipNuq: false,+ zeroDataRetention: false,+ });++ expect(writeApiJobAccess).toHaveBeenCalledWith(+ expect.objectContaining({ id, teamId: "team-id", kind: "scrape" }),+ );+ expect(writeScrapeJobState).toHaveBeenCalledWith(+ id,+ expect.objectContaining({+ status: "failed",+ requestId: crawlId,+ creditsBilled: 1,+ error: "boom",+ }),+ );+ });+it("writes terminal extract state", async () => {const id = "019e6f45-7778-727d-adf0-0abe9d5062b6";await logExtract({diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex 15806a383..c7f4ec227 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -663,6 +663,8 @@ export type LoggedScrape = {is_parse?: boolean;monitor_id?: string | null;monitor_check_id?: string | null;+ /** False for jobs that must not be fetchable by id (defaults to true). */+ jobAccess?: boolean;};export async function logScrape(scrape: LoggedScrape, force: boolean = false) {@@ -752,9 +754,30 @@ async function logScrapeInternal(scrape: LoggedScrape, force: boolean = false) {},logger,);++ // A crawl or batch child is fetched by its own id (`GET /scrape/{id}`,+ // typically off the crawl `page` webhook), so it needs its own access row.+ // Standalone scrapes get theirs from logRequest.+ if (!scrape.is_parse && scrape.jobAccess !== false) {+ try {+ await writeApiJobAccess({+ id: scrape.id,+ teamId: normalizeJobAccessTeamId(scrape.team_id),+ kind: "scrape",+ expiresAt: new Date(Date.now() + DEFAULT_JOB_ACCESS_TTL_MS),+ zeroDataRetention: scrape.zeroDataRetention,+ });+ } catch (error) {+ logger.error("Failed to write child scrape job access to Bigtable", {+ error,+ });+ }+ }}- if (!scrape.is_parse && scrape.id === scrape.request_id) {+ // Terminal state for every scrape job, standalone or child, so status reads+ // never need the PostgreSQL row.+ if (!scrape.is_parse) {try {const replay = scrape.zeroDataRetention? undefineddiff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.tsindex e4ae1e5fb..204fcbf0b 100644--- a/apps/api/src/services/worker/crawl-logic.ts+++ b/apps/api/src/services/worker/crawl-logic.ts@@ -14,6 +14,7 @@ import { logCrawl, logBatchScrape } from "../logging/log_job";import { createWebhookSender, WebhookEvent } from "../webhook/index";import type { NuQJob } from "./nuq";import { recordJobStorePostgresFallback } from "../../lib/job-store-fallback";+import { readRequestCreditsFromAnalytics } from "../../lib/request-credits-analytics";export async function finishCrawlSuper(job: NuQJob<any>) {const crawlId = job.groupId;@@ -153,6 +154,14 @@ export async function finishCrawlSuper(job: NuQJob<any>) {logger.warn("Bigtable request credits read failed", { error });}+ if (credits_billed === null) {+ try {+ credits_billed = await readRequestCreditsFromAnalytics(crawlId);+ } catch (error) {+ logger.warn("Analytics request credits read failed", { error });+ }+ }+if (credits_billed === null && config.USE_DB_AUTHENTICATION) {try {const creditsRows = await creditsBilledByCrawlId(db, crawlId);