Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): aggregate request credits in Bigtable (#4669)
Gergő Móriczc19e58fc29c9
diff --git a/apps/api/src/config.ts b/apps/api/src/config.tsindex 8904fcdc3..3feeacffc 100644--- a/apps/api/src/config.ts+++ b/apps/api/src/config.ts@@ -279,6 +279,7 @@ const configSchema = z.object({BIGTABLE_FEEDBACK_JOBS_TABLE: z.string().optional(),BIGTABLE_SCRAPE_STATE_TABLE: z.string().optional(),BIGTABLE_EXTRACT_STATE_TABLE: z.string().optional(),+ BIGTABLE_REQUEST_CREDITS_TABLE: z.string().optional(),BIGTABLE_CREDENTIALS: z.string().optional(),// ClickHouse (Search Analytics)diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.tsindex 2a65685dc..8911b5d79 100644--- a/apps/api/src/controllers/v1/batch-scrape.ts+++ b/apps/api/src/controllers/v1/batch-scrape.ts@@ -48,6 +48,7 @@ import { externalRequestId } from "../../lib/external-request-id";import { getScrapeZDR } from "../../lib/zdr-helpers";import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";import { UnsupportedSiteError } from "../../lib/error";+import { requestCreditsShards } from "../../lib/request-credits-store";export async function batchScrapeController(req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,@@ -319,6 +320,7 @@ export async function batchScrapeController(jobAccessExpiresAt: new Date(Date.now() + (req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,),+ creditsShards: requestCreditsShards(urls.length),});}diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.tsindex 6dec6f3cd..3d9ff5b19 100644--- a/apps/api/src/controllers/v1/crawl-status.ts+++ b/apps/api/src/controllers/v1/crawl-status.ts@@ -29,6 +29,7 @@ import {} from "../../services/worker/nuq-router";import { ScrapeJobSingleUrls } from "../../types";import { readScrapeJobState } from "../../lib/job-state-store";+import { readRequestCredits } from "../../lib/request-credits-store";configDotenv();export type PseudoJob<T> = {@@ -193,9 +194,14 @@ export async function crawlStatusController(logger.child({ zeroDataRetention }),);- const creditsBilled = config.USE_DB_AUTHENTICATION- ? await creditsBilledByCrawlId(dbRr, req.params.jobId).catch(() => null)- : null;+ let creditsBilled = await readRequestCredits(req.params.jobId).catch(+ () => null,+ );+ if (creditsBilled === null && config.USE_DB_AUTHENTICATION) {+ creditsBilled = await creditsBilledByCrawlId(dbRr, req.params.jobId)+ .then(rows => rows[0]?.credits_billed ?? null)+ .catch(() => null);+ }// check if the crawl failed during kickoff (e.g. queue full)const crawlError = await getCrawlError(req.params.jobId);@@ -217,7 +223,7 @@ export async function crawlStatusController((numericStats.active ?? 0) +(numericStats.queued ?? 0) +(numericStats.backlog ?? 0),- creditsUsed: creditsBilled?.[0]?.credits_billed ?? -1,+ creditsUsed: creditsBilled ?? -1,};// if the crawl has a stored error and no jobs were ever created, mark as faileddiff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.tsindex 7432f50b3..b109a1fb7 100644--- a/apps/api/src/controllers/v1/crawl.ts+++ b/apps/api/src/controllers/v1/crawl.ts@@ -38,6 +38,7 @@ import { calculateThreatScanCredits } from "../../lib/scrape-billing";import { billTeam } from "../../services/billing/credit_billing";import { getEffectiveConcurrencyLimit } from "../../lib/concurrency-limit";import { emitRejectedScrapeActivityEvent } from "../../lib/siem-logging";+import { requestCreditsShards } from "../../lib/request-credits-store";export async function crawlController(req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,@@ -195,6 +196,7 @@ export async function crawlController(jobAccessExpiresAt: new Date(Date.now() + (req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,),+ creditsShards: requestCreditsShards(req.body.limit ?? 10_000),});// checkCreditsMiddleware (always runs before this controller) is the sourcediff --git a/apps/api/src/controllers/v2/agent.ts b/apps/api/src/controllers/v2/agent.tsindex b609fca35..0fe2e836b 100644--- a/apps/api/src/controllers/v2/agent.ts+++ b/apps/api/src/controllers/v2/agent.ts@@ -219,6 +219,7 @@ export async function agentController(target_hint: req.body.urls?.[0] ?? req.body.prompt ?? "",zeroDataRetention: false, // not supported for agentapi_key_id: req.acuc?.api_key_id ?? null,+ creditsShards: 8,});const passthrough = await fetch(diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.tsindex 0668c19cd..a44e5ab4f 100644--- a/apps/api/src/controllers/v2/batch-scrape.ts+++ b/apps/api/src/controllers/v2/batch-scrape.ts@@ -50,6 +50,7 @@ import { calculateThreatScanCredits } from "../../lib/scrape-billing";import { billTeam } from "../../services/billing/credit_billing";import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";import { UnsupportedSiteError } from "../../lib/error";+import { requestCreditsShards } from "../../lib/request-credits-store";export async function batchScrapeController(req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,@@ -348,6 +349,7 @@ export async function batchScrapeController(jobAccessExpiresAt: new Date(Date.now() + (req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,),+ creditsShards: requestCreditsShards(urls.length),});}diff --git a/apps/api/src/controllers/v2/browser.ts b/apps/api/src/controllers/v2/browser.tsindex ae4e7ccfe..de8542735 100644--- a/apps/api/src/controllers/v2/browser.ts+++ b/apps/api/src/controllers/v2/browser.ts@@ -37,6 +37,7 @@ import {import { autumnService } from "../../services/autumn/autumn.service";import { orgIdForTeam } from "../../lib/team-org";import { isAgentInteropSecretValid } from "../../lib/agent-interop";+import { recordRequestCredits } from "../../lib/request-credits-store";import {getSafeMode,SAFE_MODE_BROWSER_UNSUPPORTED_MESSAGE,@@ -686,11 +687,11 @@ export async function browserDeleteController(await updateBrowserSessionCreditsUsed(session.id, creditsBilled);+ const agentRequestId =+ session.request_id && session.request_id !== session.id+ ? session.request_id+ : null;if (session.should_bill) {- const agentRequestId =- session.request_id && session.request_id !== session.id- ? session.request_id- : null;billTeam(req.auth.team_id,req.acuc?.org_id ?? null,@@ -719,6 +720,21 @@ export async function browserDeleteController(});}+ if (agentRequestId) {+ await recordRequestCredits({+ requestId: agentRequestId,+ jobId: session.id,+ credits: creditsBilled,+ }).catch(error => {+ logger.error("Failed to record browser request credits in Bigtable", {+ error,+ requestId: agentRequestId,+ sessionId: session.id,+ creditsBilled,+ });+ });+ }+logger.info("Browser session destroyed", {sessionDurationMs: durationMs,creditsBilled,@@ -854,11 +870,11 @@ export async function browserWebhookDestroyedController(await updateBrowserSessionCreditsUsed(session.id, creditsBilled);+ const agentRequestId =+ session.request_id && session.request_id !== session.id+ ? session.request_id+ : null;if (session.should_bill) {- const agentRequestId =- session.request_id && session.request_id !== session.id- ? session.request_id- : null;// The webhook carries no request context, so the team's ACUC answers for// the org — the same lookup the biller used to make for itself.billTeam(@@ -889,6 +905,21 @@ export async function browserWebhookDestroyedController(});}+ if (agentRequestId) {+ await recordRequestCredits({+ requestId: agentRequestId,+ jobId: session.id,+ credits: creditsBilled,+ }).catch(error => {+ logger.error("Failed to record browser request credits in Bigtable", {+ error,+ requestId: agentRequestId,+ sessionId: session.id,+ creditsBilled,+ });+ });+ }+logger.info("Session marked as destroyed via webhook", {sessionId: session.id,browserId,diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.tsindex 5252473bf..e6eb12053 100644--- a/apps/api/src/controllers/v2/crawl-status.ts+++ b/apps/api/src/controllers/v2/crawl-status.ts@@ -32,6 +32,7 @@ import { ScrapeJobSingleUrls } from "../../types";import { redisEvictConnection } from "../../../src/services/redis";import { isBaseDomain, extractBaseDomain } from "../../lib/url-utils";import { readScrapeJobState } from "../../lib/job-state-store";+import { readRequestCredits } from "../../lib/request-credits-store";configDotenv();export type PseudoJob<T> = {@@ -214,9 +215,14 @@ export async function crawlStatusController(logger.child({ zeroDataRetention }),);- const creditsBilled = config.USE_DB_AUTHENTICATION- ? await creditsBilledByCrawlId(dbRr, req.params.jobId).catch(() => null)- : null;+ let creditsBilled = await readRequestCredits(+ sc?.requestId ?? req.params.jobId,+ ).catch(() => null);+ if (creditsBilled === null && config.USE_DB_AUTHENTICATION) {+ creditsBilled = await creditsBilledByCrawlId(dbRr, req.params.jobId)+ .then(rows => rows[0]?.credits_billed ?? null)+ .catch(() => null);+ }// check if the crawl failed during kickoff (e.g. queue full)const crawlError = await getCrawlError(req.params.jobId);@@ -238,7 +244,7 @@ export async function crawlStatusController((numericStats.active ?? 0) +(numericStats.queued ?? 0) +(numericStats.backlog ?? 0),- creditsUsed: creditsBilled?.[0]?.credits_billed ?? -1,+ creditsUsed: creditsBilled ?? -1,};// if the crawl has a stored error and no jobs were ever created, mark as faileddiff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.tsindex 6d2fcb227..3aef7f543 100644--- a/apps/api/src/controllers/v2/crawl.ts+++ b/apps/api/src/controllers/v2/crawl.ts@@ -41,6 +41,10 @@ import { calculateThreatScanCredits } from "../../lib/scrape-billing";import { billTeam } from "../../services/billing/credit_billing";import { getEffectiveConcurrencyLimit } from "../../lib/concurrency-limit";import { emitRejectedScrapeActivityEvent } from "../../lib/siem-logging";+import {+ initializeRequestCredits,+ requestCreditsShards,+} from "../../lib/request-credits-store";export async function crawlController(req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,@@ -305,6 +309,14 @@ export async function crawlController(originalBodyLimit: preNormalizedBody.limit,});+ const creditsShards = requestCreditsShards(finalCrawlerOptions.limit);+ await initializeRequestCredits(id, creditsShards).catch(error => {+ logger.warn("Failed to initialize Bigtable request credits", {+ error,+ shards: creditsShards,+ });+ });+const effectiveConcurrency = await getEffectiveConcurrencyLimit(req.auth.team_id,req.acuc?.org_id ?? null,diff --git a/apps/api/src/lib/request-credits-store.test.ts b/apps/api/src/lib/request-credits-store.test.tsnew file mode 100644index 000000000..858ce2359--- /dev/null+++ b/apps/api/src/lib/request-credits-store.test.ts@@ -0,0 +1,198 @@+import { beforeEach, describe, expect, it, vi } from "vitest";++const { request, getRows, getBigtableTable, mutableConfig, withSpan } =+ vi.hoisted(() => ({+ request: vi.fn(),+ getRows: vi.fn(),+ getBigtableTable: vi.fn(),+ mutableConfig: {+ BIGTABLE_REQUEST_CREDITS_TABLE: "request-credits" as string | undefined,+ },+ withSpan: vi.fn(async (_name: string, fn: (span: object) => unknown) =>+ fn({ setAttributes: vi.fn() }),+ ),+ }));++vi.mock("../config", () => ({ config: mutableConfig }));+vi.mock("./bigtable-client", () => ({ getBigtableTable }));+vi.mock("./otel-tracer", () => ({+ withSpan,+ setSpanAttributes: vi.fn(),+}));++import {+ clearRequestCreditsShardCacheForTest,+ initializeRequestCredits,+ readRequestCredits,+ recordRequestCredits,+ requestCreditsRowKey,+ requestCreditsShardForJob,+ requestCreditsShards,+} from "./request-credits-store";++function row(data: Record<string, unknown>) {+ return { data };+}++function int64(value: bigint): Buffer {+ const buffer = Buffer.alloc(8);+ buffer.writeBigInt64BE(value);+ return buffer;+}++describe("request credits store", () => {+ beforeEach(() => {+ vi.clearAllMocks();+ clearRequestCreditsShardCacheForTest();+ mutableConfig.BIGTABLE_REQUEST_CREDITS_TABLE = "request-credits";+ getBigtableTable.mockResolvedValue({+ name: "projects/p/instances/i/tables/request-credits",+ bigtable: { appProfileId: "single-cluster", request },+ getRows,+ });+ request.mockImplementation((_options, callback) =>+ callback(null, { predicateMatched: false }),+ );+ });++ it("uses stable salted request and job shards", () => {+ expect(requestCreditsRowKey("not-a-uuid", 3)).toBe(+ requestCreditsRowKey("not-a-uuid", 3),+ );+ expect(requestCreditsRowKey("not-a-uuid", 3)).not.toBe(+ requestCreditsRowKey("not-a-uuid", 4),+ );+ expect(requestCreditsShardForJob("job-1", 16)).toBeLessThan(16);+ expect(requestCreditsShardForJob("job-1", 16)).toBe(+ requestCreditsShardForJob("job-1", 16),+ );+ });++ it("caps shard selection at 512", () => {+ expect(requestCreditsShards(1_000)).toBe(8);+ expect(requestCreditsShards(10_000)).toBe(16);+ expect(requestCreditsShards(100_000)).toBe(64);+ expect(requestCreditsShards(1_000_000)).toBe(512);+ expect(requestCreditsShards(10_000_000)).toBe(512);+ });++ it("keeps the stored shard count on repeated initialization", async () => {+ await expect(initializeRequestCredits("request-1", 16)).resolves.toBe(true);++ const call = request.mock.calls[0][0];+ expect(call).toMatchObject({+ client: "BigtableClient",+ method: "checkAndMutateRow",+ reqOpts: {+ tableName: "projects/p/instances/i/tables/request-credits",+ appProfileId: "single-cluster",+ falseMutations: [+ {+ setCell: {+ familyName: "jobs",+ columnQualifier: Buffer.from("\x00shards"),+ timestampMicros: 0,+ value: Buffer.from("16"),+ },+ },+ ],+ },+ });++ request.mockImplementationOnce((_options, callback) =>+ callback(null, { predicateMatched: true }),+ );+ getRows.mockResolvedValueOnce([+ [row({ jobs: { "\x00shards": [{ value: Buffer.from("16") }] } })],+ ]);++ await expect(initializeRequestCredits("request-1", 16)).resolves.toBe(true);+ expect(request).toHaveBeenCalledTimes(2);+ expect(getRows).toHaveBeenCalledTimes(1);+ });++ it("atomically records one job marker and aggregate addition", async () => {+ await initializeRequestCredits("request-1", 8);+ request.mockClear();++ await expect(+ recordRequestCredits({+ requestId: "request-1",+ jobId: "job-1",+ credits: 7,+ }),+ ).resolves.toBe(true);++ const mutations = request.mock.calls[0][0].reqOpts.falseMutations;+ expect(mutations).toEqual([+ {+ setCell: {+ familyName: "jobs",+ columnQualifier: Buffer.from("job-1"),+ timestampMicros: 0,+ value: Buffer.from("7"),+ },+ },+ {+ addToCell: {+ familyName: "agg",+ columnQualifier: { rawValue: Buffer.from("total") },+ timestamp: { rawTimestampMicros: 0 },+ input: { intValue: 7 },+ },+ },+ ]);+ });++ it("reports a duplicate without another logical addition", async () => {+ await initializeRequestCredits("request-1", 8);+ request.mockImplementation((_options, callback) =>+ callback(null, { predicateMatched: true }),+ );+ getRows.mockResolvedValueOnce([+ [row({ jobs: { "job-1": [{ value: Buffer.from("7") }] } })],+ ]);++ await expect(+ recordRequestCredits({+ requestId: "request-1",+ jobId: "job-1",+ credits: 7,+ }),+ ).resolves.toBe(false);+ });++ it("reads and sums all aggregate shard cells", async () => {+ getRows+ .mockResolvedValueOnce([+ [row({ jobs: { "\x00shards": [{ value: Buffer.from("2") }] } })],+ ])+ .mockResolvedValueOnce([+ [+ row({ agg: { total: [{ value: int64(7n) }] } }),+ row({ agg: { total: [{ value: int64(5n) }] } }),+ ],+ ]);++ await expect(readRequestCredits("request-1")).resolves.toBe(12);+ expect(getRows.mock.calls[1][0].keys).toEqual([+ requestCreditsRowKey("request-1", 0),+ requestCreditsRowKey("request-1", 1),+ ]);+ });++ it("is disabled when the table is not configured", async () => {+ mutableConfig.BIGTABLE_REQUEST_CREDITS_TABLE = undefined;++ await expect(initializeRequestCredits("request-1", 8)).resolves.toBe(false);+ await expect(+ recordRequestCredits({+ requestId: "request-1",+ jobId: "job-1",+ credits: 1,+ }),+ ).resolves.toBe(false);+ await expect(readRequestCredits("request-1")).resolves.toBeNull();+ expect(getBigtableTable).not.toHaveBeenCalled();+ });+});diff --git a/apps/api/src/lib/request-credits-store.ts b/apps/api/src/lib/request-credits-store.tsnew file mode 100644index 000000000..b0411b189--- /dev/null+++ b/apps/api/src/lib/request-credits-store.ts@@ -0,0 +1,336 @@+import crypto from "node:crypto";+import type { Table } from "@google-cloud/bigtable";+import { config } from "../config";+import { getBigtableTable } from "./bigtable-client";+import { setSpanAttributes, withSpan } from "./otel-tracer";++const JOBS_FAMILY = "jobs";+const AGG_FAMILY = "agg";+const TOTAL_QUALIFIER = "total";+const SHARDS_QUALIFIER = "\x00shards";+const FIXED_TIMESTAMP_MICROS = 0;+const MAX_CACHED_SHARD_COUNTS = 10_000;+const shardCounts = new Map<string, number>();++function cacheShardCount(requestId: string, shards: number): void {+ shardCounts.delete(requestId);+ shardCounts.set(requestId, shards);+ if (shardCounts.size > MAX_CACHED_SHARD_COUNTS) {+ const oldest = shardCounts.keys().next().value;+ if (oldest !== undefined) shardCounts.delete(oldest);+ }+}++function requestHash(requestId: string): Buffer {+ return crypto.createHash("sha256").update(requestId, "utf8").digest();+}++export function requestCreditsRowKey(requestId: string, shard: number): string {+ if (!Number.isInteger(shard) || shard < 0 || shard > 0xffff) {+ throw new Error(`Invalid request credits shard: ${shard}`);+ }++ const hash = requestHash(requestId);+ const shardBytes = Buffer.allocUnsafe(2);+ shardBytes.writeUInt16BE(shard);+ const salt = crypto+ .createHash("sha256")+ .update(hash)+ .update(shardBytes)+ .digest()+ .subarray(0, 2)+ .toString("hex");+ return `${salt}#${hash.toString("hex")}#${shard}`;+}++export function requestCreditsShardForJob(+ jobId: string,+ shards: number,+): number {+ if (!Number.isInteger(shards) || shards <= 0 || shards > 512) {+ throw new Error(`Invalid request credits shard count: ${shards}`);+ }+ return (+ crypto.createHash("sha256").update(jobId, "utf8").digest().readUInt16BE(0) %+ shards+ );+}++export function requestCreditsShards(maxJobs: number): number {+ if (!Number.isFinite(maxJobs) || maxJobs <= 0) return 8;+ if (maxJobs <= 1_000) return 8;+ if (maxJobs <= 10_000) return 16;+ if (maxJobs <= 100_000) return 64;+ return 512;+}++function exactQualifierFilter(qualifier: string) {+ return {+ chain: {+ filters: [+ { familyNameRegexFilter: `^${JOBS_FAMILY}$` },+ {+ columnQualifierRegexFilter: Buffer.concat([+ Buffer.from("^"),+ Buffer.from(qualifier),+ Buffer.from("$"),+ ]),+ },+ ],+ },+ };+}++async function checkAndMutateRowOnce(+ table: Table,+ request: Record<string, unknown>,+): Promise<{ predicateMatched?: boolean | null }> {+ return new Promise((resolve, reject) => {+ table.bigtable.request(+ {+ client: "BigtableClient",+ method: "checkAndMutateRow",+ reqOpts: {+ tableName: table.name,+ appProfileId: table.bigtable.appProfileId,+ ...request,+ },+ },+ (+ error: Error | null,+ response?: { predicateMatched?: boolean | null },+ ) => {+ if (error) reject(error);+ else resolve(response ?? {});+ },+ );+ });+}++async function checkAndMutateRow(+ table: Table,+ request: Record<string, unknown>,+): Promise<{ predicateMatched?: boolean | null }> {+ let lastError: unknown;+ for (let attempt = 0; attempt < 3; attempt++) {+ try {+ return await checkAndMutateRowOnce(table, request);+ } catch (error) {+ lastError = error;+ const code =+ error && typeof error === "object"+ ? (error as { code?: unknown }).code+ : undefined;+ if (![1, 2, 4, 8, 10, 13, 14].includes(Number(code))) throw error;+ if (attempt < 2) {+ await new Promise(resolve => setTimeout(resolve, 100 * (attempt + 1)));+ }+ }+ }+ throw lastError;+}++function setCellMutation(qualifier: string, value: Buffer) {+ return {+ setCell: {+ familyName: JOBS_FAMILY,+ columnQualifier: Buffer.from(qualifier),+ timestampMicros: FIXED_TIMESTAMP_MICROS,+ value,+ },+ };+}++async function readShardCount(+ table: Table,+ requestId: string,+): Promise<number | null> {+ const [rows] = await table.getRows({+ keys: [requestCreditsRowKey(requestId, 0)],+ decode: false,+ filter: [+ { family: JOBS_FAMILY },+ { column: SHARDS_QUALIFIER, cellLimit: 1 },+ ],+ });+ const value = rows[0]?.data?.[JOBS_FAMILY]?.[SHARDS_QUALIFIER]?.[0]?.value;+ if (value == null) return null;+ const shards = Number(Buffer.from(value).toString("utf8"));+ if (!Number.isInteger(shards) || shards <= 0 || shards > 512) {+ throw new Error(`Invalid stored request credits shard count: ${shards}`);+ }+ cacheShardCount(requestId, shards);+ return shards;+}++async function getShardCount(+ table: Table,+ requestId: string,+): Promise<number | null> {+ return shardCounts.get(requestId) ?? readShardCount(table, requestId);+}++async function readJobCredits(+ table: Table,+ rowKey: string,+ jobId: string,+): Promise<number | null> {+ const [rows] = await table.getRows({+ keys: [rowKey],+ decode: false,+ filter: [{ family: JOBS_FAMILY }, { column: jobId, cellLimit: 1 }],+ });+ const value = rows[0]?.data?.[JOBS_FAMILY]?.[jobId]?.[0]?.value;+ if (value == null) return null;+ const credits = Number(Buffer.from(value).toString("utf8"));+ if (!Number.isSafeInteger(credits)) {+ throw new Error(`Invalid stored request credits value: ${credits}`);+ }+ return credits;+}++export async function initializeRequestCredits(+ requestId: string,+ shards: number,+): Promise<boolean> {+ const tableId = config.BIGTABLE_REQUEST_CREDITS_TABLE;+ if (!tableId) return false;+ if (!Number.isInteger(shards) || shards <= 0 || shards > 512) {+ throw new Error(`Invalid request credits shard count: ${shards}`);+ }++ return withSpan("bigtable.request_credits.initialize", async span => {+ setSpanAttributes(span, {+ "db.system": "bigtable",+ "bigtable.table": tableId,+ "bigtable.operation": "checkAndMutateRow",+ "request_credits.shards": shards,+ });+ const table = await getBigtableTable(tableId);+ const response = await checkAndMutateRow(table, {+ rowKey: Buffer.from(requestCreditsRowKey(requestId, 0)),+ predicateFilter: exactQualifierFilter(SHARDS_QUALIFIER),+ falseMutations: [+ setCellMutation(SHARDS_QUALIFIER, Buffer.from(String(shards))),+ ],+ });+ const stored = response.predicateMatched+ ? await readShardCount(table, requestId)+ : shards;+ if (stored !== shards) {+ throw new Error(+ `Request credits shard count mismatch for ${requestId}: ${stored} != ${shards}`,+ );+ }+ cacheShardCount(requestId, shards);+ return true;+ });+}++export async function recordRequestCredits(params: {+ requestId: string;+ jobId: string;+ credits: number;+}): Promise<boolean> {+ const tableId = config.BIGTABLE_REQUEST_CREDITS_TABLE;+ if (!tableId) return false;+ if (!Number.isSafeInteger(params.credits)) {+ throw new Error(`Invalid request credits value: ${params.credits}`);+ }++ return withSpan("bigtable.request_credits.record", async span => {+ setSpanAttributes(span, {+ "db.system": "bigtable",+ "bigtable.table": tableId,+ "bigtable.operation": "checkAndMutateRow",+ "request_credits.value": params.credits,+ });+ const table = await getBigtableTable(tableId);+ const shards = await getShardCount(table, params.requestId);+ if (shards === null) return false;+ const shard = requestCreditsShardForJob(params.jobId, shards);+ const rowKey = requestCreditsRowKey(params.requestId, shard);+ const response = await checkAndMutateRow(table, {+ rowKey: Buffer.from(rowKey),+ predicateFilter: exactQualifierFilter(params.jobId),+ falseMutations: [+ setCellMutation(params.jobId, Buffer.from(String(params.credits))),+ {+ addToCell: {+ familyName: AGG_FAMILY,+ columnQualifier: { rawValue: Buffer.from(TOTAL_QUALIFIER) },+ timestamp: { rawTimestampMicros: FIXED_TIMESTAMP_MICROS },+ input: { intValue: params.credits },+ },+ },+ ],+ });+ setSpanAttributes(span, {+ "request_credits.shard": shard,+ "request_credits.duplicate": response.predicateMatched === true,+ });+ if (response.predicateMatched !== true) return true;++ const storedCredits = await readJobCredits(table, rowKey, params.jobId);+ if (storedCredits !== params.credits) {+ throw new Error(+ `Request credits mismatch for ${params.jobId}: ${storedCredits} != ${params.credits}`,+ );+ }+ return false;+ });+}++export async function readRequestCredits(+ requestId: string,+): Promise<number | null> {+ const tableId = config.BIGTABLE_REQUEST_CREDITS_TABLE;+ if (!tableId) return null;++ return withSpan("bigtable.request_credits.read", async span => {+ setSpanAttributes(span, {+ "db.system": "bigtable",+ "bigtable.table": tableId,+ "bigtable.operation": "getRows",+ });+ const table = await getBigtableTable(tableId);+ const shards = await getShardCount(table, requestId);+ if (shards === null) return null;+ const [rows] = await table.getRows({+ keys: Array.from({ length: shards }, (_, shard) =>+ requestCreditsRowKey(requestId, shard),+ ),+ decode: false,+ filter: [+ { family: AGG_FAMILY },+ { column: TOTAL_QUALIFIER, cellLimit: 1 },+ ],+ });+ let total = 0n;+ for (const row of rows) {+ const value = row.data?.[AGG_FAMILY]?.[TOTAL_QUALIFIER]?.[0]?.value;+ if (value == null) continue;+ const encoded = Buffer.from(value);+ if (encoded.length !== 8) {+ throw new Error(+ `Invalid request credits aggregate length: ${encoded.length}`,+ );+ }+ total += encoded.readBigInt64BE();+ }+ const credits = Number(total);+ if (!Number.isSafeInteger(credits)) {+ throw new Error(`Request credits total exceeds safe integer: ${total}`);+ }+ setSpanAttributes(span, {+ "request_credits.shards": shards,+ "request_credits.returned_rows": rows.length,+ "request_credits.total": credits,+ });+ return credits;+ });+}++export function clearRequestCreditsShardCacheForTest(): void {+ shardCounts.clear();+}diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.tsindex 91e6fc869..adcd5718a 100644--- a/apps/api/src/services/logging/log_job.ts+++ b/apps/api/src/services/logging/log_job.ts@@ -38,6 +38,10 @@ import {writeScrapeJobState,} from "../../lib/job-state-store";import { buildReplayContextFromScrape } from "../../lib/scrape-interact/scrape-replay";+import {+ initializeRequestCredits,+ recordRequestCredits,+} from "../../lib/request-credits-store";configDotenv();const previewTeamId = "3adefd26-77ec-5968-8dcf-c94b5630d1de";@@ -84,6 +88,22 @@ async function writeFeedbackJobSafely(}}+async function recordRequestCreditsSafely(+ params: Parameters<typeof recordRequestCredits>[0],+ logger: Logger,+): Promise<void> {+ try {+ await recordRequestCredits(params);+ } catch (error) {+ logger.error("Failed to record request credits in Bigtable", {+ error,+ requestId: params.requestId,+ jobId: params.jobId,+ credits: params.credits,+ });+ }+}+/*** Null-aware wrapper around the shared text sanitizer, kept where a cleaned* value feeds a later decision (the external_request_id byte cap). Every row@@ -486,6 +506,7 @@ type LoggedRequest = {external_request_id?: string | null;jobAccess?: boolean;jobAccessExpiresAt?: Date;+ creditsShards?: number;};/**@@ -582,6 +603,17 @@ async function logRequestInternal(request: LoggedRequest) {}}+ if (request.creditsShards !== undefined) {+ try {+ await initializeRequestCredits(request.id, request.creditsShards);+ } catch (error) {+ logger.error("Failed to initialize request credits in Bigtable", {+ error,+ shards: request.creditsShards,+ });+ }+ }+if (request.zeroDataRetention && config.USE_DB_AUTHENTICATION === true) {await enqueueZdrCleanupJob(request.id);}@@ -709,6 +741,17 @@ async function logScrapeInternal(scrape: LoggedScrape, force: boolean = false) {logger,);+ if (scrape.id !== scrape.request_id) {+ await recordRequestCreditsSafely(+ {+ requestId: scrape.request_id,+ jobId: scrape.id,+ credits: scrape.credits_cost,+ },+ logger,+ );+ }+if (!scrape.is_parse && scrape.id === scrape.request_id) {try {const replay = scrape.zeroDataRetention@@ -989,6 +1032,17 @@ async function logSearchInternal(search: LoggedSearch, force: boolean = false) {logger,);+ if (search.id !== search.request_id) {+ await recordRequestCreditsSafely(+ {+ requestId: search.request_id,+ jobId: search.id,+ credits: search.credits_cost,+ },+ logger,+ );+ }+if (search.results && !search.zeroDataRetention) {await saveSearchToGCS(search, logger);}diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.tsindex 0e5120eb9..af7fc46e6 100644--- a/apps/api/src/services/worker/crawl-logic.ts+++ b/apps/api/src/services/worker/crawl-logic.ts@@ -8,6 +8,7 @@ import {import { getCrawl } from "../../lib/crawl-redis";import { creditsBilledByCrawlId } from "../../db/rpc";import { db } from "../../db/connection";+import { readRequestCredits } from "../../lib/request-credits-store";import { getJobs } from "../../controllers/v1/crawl-status";import { logCrawl, logBatchScrape } from "../logging/log_job";import { createWebhookSender, WebhookEvent } from "../webhook/index";@@ -143,7 +144,13 @@ export async function finishCrawlSuper(job: NuQJob<any>) {let credits_billed: number | null = null;- if (config.USE_DB_AUTHENTICATION) {+ try {+ credits_billed = await readRequestCredits(requestId);+ } catch (error) {+ logger.warn("Bigtable request credits read failed", { error });+ }++ if (credits_billed === null && config.USE_DB_AUTHENTICATION) {try {const creditsRows = await creditsBilledByCrawlId(db, crawlId);credits_billed = creditsRows?.[0]?.credits_billed ?? null;