Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): aggregate request credits in Bigtable (#4669)

Gergő Móriczc19e58fc29c9
diff --git a/apps/api/src/config.ts b/apps/api/src/config.ts
index 8904fcdc3..3feeacffc 100644
--- a/apps/api/src/config.ts
+++ b/apps/api/src/config.ts
@@ -279,6 +279,7 @@ const configSchema = z.object({
BIGTABLE_FEEDBACK_JOBS_TABLE: z.string().optional(),
BIGTABLE_SCRAPE_STATE_TABLE: z.string().optional(),
BIGTABLE_EXTRACT_STATE_TABLE: z.string().optional(),
+ BIGTABLE_REQUEST_CREDITS_TABLE: z.string().optional(),
BIGTABLE_CREDENTIALS: z.string().optional(),
// ClickHouse (Search Analytics)
diff --git a/apps/api/src/controllers/v1/batch-scrape.ts b/apps/api/src/controllers/v1/batch-scrape.ts
index 2a65685dc..8911b5d79 100644
--- a/apps/api/src/controllers/v1/batch-scrape.ts
+++ b/apps/api/src/controllers/v1/batch-scrape.ts
@@ -48,6 +48,7 @@ import { externalRequestId } from "../../lib/external-request-id";
import { getScrapeZDR } from "../../lib/zdr-helpers";
import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";
import { UnsupportedSiteError } from "../../lib/error";
+import { requestCreditsShards } from "../../lib/request-credits-store";
export async function batchScrapeController(
req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,
@@ -319,6 +320,7 @@ export async function batchScrapeController(
jobAccessExpiresAt: new Date(
Date.now() + (req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,
),
+ creditsShards: requestCreditsShards(urls.length),
});
}
diff --git a/apps/api/src/controllers/v1/crawl-status.ts b/apps/api/src/controllers/v1/crawl-status.ts
index 6dec6f3cd..3d9ff5b19 100644
--- a/apps/api/src/controllers/v1/crawl-status.ts
+++ b/apps/api/src/controllers/v1/crawl-status.ts
@@ -29,6 +29,7 @@ import {
} from "../../services/worker/nuq-router";
import { ScrapeJobSingleUrls } from "../../types";
import { readScrapeJobState } from "../../lib/job-state-store";
+import { readRequestCredits } from "../../lib/request-credits-store";
configDotenv();
export type PseudoJob<T> = {
@@ -193,9 +194,14 @@ export async function crawlStatusController(
logger.child({ zeroDataRetention }),
);
- const creditsBilled = config.USE_DB_AUTHENTICATION
- ? await creditsBilledByCrawlId(dbRr, req.params.jobId).catch(() => null)
- : null;
+ let creditsBilled = await readRequestCredits(req.params.jobId).catch(
+ () => null,
+ );
+ if (creditsBilled === null && config.USE_DB_AUTHENTICATION) {
+ creditsBilled = await creditsBilledByCrawlId(dbRr, req.params.jobId)
+ .then(rows => rows[0]?.credits_billed ?? null)
+ .catch(() => null);
+ }
// check if the crawl failed during kickoff (e.g. queue full)
const crawlError = await getCrawlError(req.params.jobId);
@@ -217,7 +223,7 @@ export async function crawlStatusController(
(numericStats.active ?? 0) +
(numericStats.queued ?? 0) +
(numericStats.backlog ?? 0),
- creditsUsed: creditsBilled?.[0]?.credits_billed ?? -1,
+ creditsUsed: creditsBilled ?? -1,
};
// if the crawl has a stored error and no jobs were ever created, mark as failed
diff --git a/apps/api/src/controllers/v1/crawl.ts b/apps/api/src/controllers/v1/crawl.ts
index 7432f50b3..b109a1fb7 100644
--- a/apps/api/src/controllers/v1/crawl.ts
+++ b/apps/api/src/controllers/v1/crawl.ts
@@ -38,6 +38,7 @@ import { calculateThreatScanCredits } from "../../lib/scrape-billing";
import { billTeam } from "../../services/billing/credit_billing";
import { getEffectiveConcurrencyLimit } from "../../lib/concurrency-limit";
import { emitRejectedScrapeActivityEvent } from "../../lib/siem-logging";
+import { requestCreditsShards } from "../../lib/request-credits-store";
export async function crawlController(
req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,
@@ -195,6 +196,7 @@ export async function crawlController(
jobAccessExpiresAt: new Date(
Date.now() + (req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,
),
+ creditsShards: requestCreditsShards(req.body.limit ?? 10_000),
});
// checkCreditsMiddleware (always runs before this controller) is the source
diff --git a/apps/api/src/controllers/v2/agent.ts b/apps/api/src/controllers/v2/agent.ts
index b609fca35..0fe2e836b 100644
--- a/apps/api/src/controllers/v2/agent.ts
+++ b/apps/api/src/controllers/v2/agent.ts
@@ -219,6 +219,7 @@ export async function agentController(
target_hint: req.body.urls?.[0] ?? req.body.prompt ?? "",
zeroDataRetention: false, // not supported for agent
api_key_id: req.acuc?.api_key_id ?? null,
+ creditsShards: 8,
});
const passthrough = await fetch(
diff --git a/apps/api/src/controllers/v2/batch-scrape.ts b/apps/api/src/controllers/v2/batch-scrape.ts
index 0668c19cd..a44e5ab4f 100644
--- a/apps/api/src/controllers/v2/batch-scrape.ts
+++ b/apps/api/src/controllers/v2/batch-scrape.ts
@@ -50,6 +50,7 @@ import { calculateThreatScanCredits } from "../../lib/scrape-billing";
import { billTeam } from "../../services/billing/credit_billing";
import { emitRejectedScrapeActivityEvents } from "../../lib/siem-logging";
import { UnsupportedSiteError } from "../../lib/error";
+import { requestCreditsShards } from "../../lib/request-credits-store";
export async function batchScrapeController(
req: RequestWithAuth<{}, BatchScrapeResponse, BatchScrapeRequest>,
@@ -348,6 +349,7 @@ export async function batchScrapeController(
jobAccessExpiresAt: new Date(
Date.now() + (req.acuc?.flags?.crawlTtlHours ?? 24) * 60 * 60 * 1000,
),
+ creditsShards: requestCreditsShards(urls.length),
});
}
diff --git a/apps/api/src/controllers/v2/browser.ts b/apps/api/src/controllers/v2/browser.ts
index ae4e7ccfe..de8542735 100644
--- a/apps/api/src/controllers/v2/browser.ts
+++ b/apps/api/src/controllers/v2/browser.ts
@@ -37,6 +37,7 @@ import {
import { autumnService } from "../../services/autumn/autumn.service";
import { orgIdForTeam } from "../../lib/team-org";
import { isAgentInteropSecretValid } from "../../lib/agent-interop";
+import { recordRequestCredits } from "../../lib/request-credits-store";
import {
getSafeMode,
SAFE_MODE_BROWSER_UNSUPPORTED_MESSAGE,
@@ -686,11 +687,11 @@ export async function browserDeleteController(
await updateBrowserSessionCreditsUsed(session.id, creditsBilled);
+ const agentRequestId =
+ session.request_id && session.request_id !== session.id
+ ? session.request_id
+ : null;
if (session.should_bill) {
- const agentRequestId =
- session.request_id && session.request_id !== session.id
- ? session.request_id
- : null;
billTeam(
req.auth.team_id,
req.acuc?.org_id ?? null,
@@ -719,6 +720,21 @@ export async function browserDeleteController(
});
}
+ if (agentRequestId) {
+ await recordRequestCredits({
+ requestId: agentRequestId,
+ jobId: session.id,
+ credits: creditsBilled,
+ }).catch(error => {
+ logger.error("Failed to record browser request credits in Bigtable", {
+ error,
+ requestId: agentRequestId,
+ sessionId: session.id,
+ creditsBilled,
+ });
+ });
+ }
+
logger.info("Browser session destroyed", {
sessionDurationMs: durationMs,
creditsBilled,
@@ -854,11 +870,11 @@ export async function browserWebhookDestroyedController(
await updateBrowserSessionCreditsUsed(session.id, creditsBilled);
+ const agentRequestId =
+ session.request_id && session.request_id !== session.id
+ ? session.request_id
+ : null;
if (session.should_bill) {
- const agentRequestId =
- session.request_id && session.request_id !== session.id
- ? session.request_id
- : null;
// The webhook carries no request context, so the team's ACUC answers for
// the org — the same lookup the biller used to make for itself.
billTeam(
@@ -889,6 +905,21 @@ export async function browserWebhookDestroyedController(
});
}
+ if (agentRequestId) {
+ await recordRequestCredits({
+ requestId: agentRequestId,
+ jobId: session.id,
+ credits: creditsBilled,
+ }).catch(error => {
+ logger.error("Failed to record browser request credits in Bigtable", {
+ error,
+ requestId: agentRequestId,
+ sessionId: session.id,
+ creditsBilled,
+ });
+ });
+ }
+
logger.info("Session marked as destroyed via webhook", {
sessionId: session.id,
browserId,
diff --git a/apps/api/src/controllers/v2/crawl-status.ts b/apps/api/src/controllers/v2/crawl-status.ts
index 5252473bf..e6eb12053 100644
--- a/apps/api/src/controllers/v2/crawl-status.ts
+++ b/apps/api/src/controllers/v2/crawl-status.ts
@@ -32,6 +32,7 @@ import { ScrapeJobSingleUrls } from "../../types";
import { redisEvictConnection } from "../../../src/services/redis";
import { isBaseDomain, extractBaseDomain } from "../../lib/url-utils";
import { readScrapeJobState } from "../../lib/job-state-store";
+import { readRequestCredits } from "../../lib/request-credits-store";
configDotenv();
export type PseudoJob<T> = {
@@ -214,9 +215,14 @@ export async function crawlStatusController(
logger.child({ zeroDataRetention }),
);
- const creditsBilled = config.USE_DB_AUTHENTICATION
- ? await creditsBilledByCrawlId(dbRr, req.params.jobId).catch(() => null)
- : null;
+ let creditsBilled = await readRequestCredits(
+ sc?.requestId ?? req.params.jobId,
+ ).catch(() => null);
+ if (creditsBilled === null && config.USE_DB_AUTHENTICATION) {
+ creditsBilled = await creditsBilledByCrawlId(dbRr, req.params.jobId)
+ .then(rows => rows[0]?.credits_billed ?? null)
+ .catch(() => null);
+ }
// check if the crawl failed during kickoff (e.g. queue full)
const crawlError = await getCrawlError(req.params.jobId);
@@ -238,7 +244,7 @@ export async function crawlStatusController(
(numericStats.active ?? 0) +
(numericStats.queued ?? 0) +
(numericStats.backlog ?? 0),
- creditsUsed: creditsBilled?.[0]?.credits_billed ?? -1,
+ creditsUsed: creditsBilled ?? -1,
};
// if the crawl has a stored error and no jobs were ever created, mark as failed
diff --git a/apps/api/src/controllers/v2/crawl.ts b/apps/api/src/controllers/v2/crawl.ts
index 6d2fcb227..3aef7f543 100644
--- a/apps/api/src/controllers/v2/crawl.ts
+++ b/apps/api/src/controllers/v2/crawl.ts
@@ -41,6 +41,10 @@ import { calculateThreatScanCredits } from "../../lib/scrape-billing";
import { billTeam } from "../../services/billing/credit_billing";
import { getEffectiveConcurrencyLimit } from "../../lib/concurrency-limit";
import { emitRejectedScrapeActivityEvent } from "../../lib/siem-logging";
+import {
+ initializeRequestCredits,
+ requestCreditsShards,
+} from "../../lib/request-credits-store";
export async function crawlController(
req: RequestWithAuth<{}, CrawlResponse, CrawlRequest>,
@@ -305,6 +309,14 @@ export async function crawlController(
originalBodyLimit: preNormalizedBody.limit,
});
+ const creditsShards = requestCreditsShards(finalCrawlerOptions.limit);
+ await initializeRequestCredits(id, creditsShards).catch(error => {
+ logger.warn("Failed to initialize Bigtable request credits", {
+ error,
+ shards: creditsShards,
+ });
+ });
+
const effectiveConcurrency = await getEffectiveConcurrencyLimit(
req.auth.team_id,
req.acuc?.org_id ?? null,
diff --git a/apps/api/src/lib/request-credits-store.test.ts b/apps/api/src/lib/request-credits-store.test.ts
new file mode 100644
index 000000000..858ce2359
--- /dev/null
+++ b/apps/api/src/lib/request-credits-store.test.ts
@@ -0,0 +1,198 @@
+import { beforeEach, describe, expect, it, vi } from "vitest";
+
+const { request, getRows, getBigtableTable, mutableConfig, withSpan } =
+ vi.hoisted(() => ({
+ request: vi.fn(),
+ getRows: vi.fn(),
+ getBigtableTable: vi.fn(),
+ mutableConfig: {
+ BIGTABLE_REQUEST_CREDITS_TABLE: "request-credits" as string | undefined,
+ },
+ withSpan: vi.fn(async (_name: string, fn: (span: object) => unknown) =>
+ fn({ setAttributes: vi.fn() }),
+ ),
+ }));
+
+vi.mock("../config", () => ({ config: mutableConfig }));
+vi.mock("./bigtable-client", () => ({ getBigtableTable }));
+vi.mock("./otel-tracer", () => ({
+ withSpan,
+ setSpanAttributes: vi.fn(),
+}));
+
+import {
+ clearRequestCreditsShardCacheForTest,
+ initializeRequestCredits,
+ readRequestCredits,
+ recordRequestCredits,
+ requestCreditsRowKey,
+ requestCreditsShardForJob,
+ requestCreditsShards,
+} from "./request-credits-store";
+
+function row(data: Record<string, unknown>) {
+ return { data };
+}
+
+function int64(value: bigint): Buffer {
+ const buffer = Buffer.alloc(8);
+ buffer.writeBigInt64BE(value);
+ return buffer;
+}
+
+describe("request credits store", () => {
+ beforeEach(() => {
+ vi.clearAllMocks();
+ clearRequestCreditsShardCacheForTest();
+ mutableConfig.BIGTABLE_REQUEST_CREDITS_TABLE = "request-credits";
+ getBigtableTable.mockResolvedValue({
+ name: "projects/p/instances/i/tables/request-credits",
+ bigtable: { appProfileId: "single-cluster", request },
+ getRows,
+ });
+ request.mockImplementation((_options, callback) =>
+ callback(null, { predicateMatched: false }),
+ );
+ });
+
+ it("uses stable salted request and job shards", () => {
+ expect(requestCreditsRowKey("not-a-uuid", 3)).toBe(
+ requestCreditsRowKey("not-a-uuid", 3),
+ );
+ expect(requestCreditsRowKey("not-a-uuid", 3)).not.toBe(
+ requestCreditsRowKey("not-a-uuid", 4),
+ );
+ expect(requestCreditsShardForJob("job-1", 16)).toBeLessThan(16);
+ expect(requestCreditsShardForJob("job-1", 16)).toBe(
+ requestCreditsShardForJob("job-1", 16),
+ );
+ });
+
+ it("caps shard selection at 512", () => {
+ expect(requestCreditsShards(1_000)).toBe(8);
+ expect(requestCreditsShards(10_000)).toBe(16);
+ expect(requestCreditsShards(100_000)).toBe(64);
+ expect(requestCreditsShards(1_000_000)).toBe(512);
+ expect(requestCreditsShards(10_000_000)).toBe(512);
+ });
+
+ it("keeps the stored shard count on repeated initialization", async () => {
+ await expect(initializeRequestCredits("request-1", 16)).resolves.toBe(true);
+
+ const call = request.mock.calls[0][0];
+ expect(call).toMatchObject({
+ client: "BigtableClient",
+ method: "checkAndMutateRow",
+ reqOpts: {
+ tableName: "projects/p/instances/i/tables/request-credits",
+ appProfileId: "single-cluster",
+ falseMutations: [
+ {
+ setCell: {
+ familyName: "jobs",
+ columnQualifier: Buffer.from("\x00shards"),
+ timestampMicros: 0,
+ value: Buffer.from("16"),
+ },
+ },
+ ],
+ },
+ });
+
+ request.mockImplementationOnce((_options, callback) =>
+ callback(null, { predicateMatched: true }),
+ );
+ getRows.mockResolvedValueOnce([
+ [row({ jobs: { "\x00shards": [{ value: Buffer.from("16") }] } })],
+ ]);
+
+ await expect(initializeRequestCredits("request-1", 16)).resolves.toBe(true);
+ expect(request).toHaveBeenCalledTimes(2);
+ expect(getRows).toHaveBeenCalledTimes(1);
+ });
+
+ it("atomically records one job marker and aggregate addition", async () => {
+ await initializeRequestCredits("request-1", 8);
+ request.mockClear();
+
+ await expect(
+ recordRequestCredits({
+ requestId: "request-1",
+ jobId: "job-1",
+ credits: 7,
+ }),
+ ).resolves.toBe(true);
+
+ const mutations = request.mock.calls[0][0].reqOpts.falseMutations;
+ expect(mutations).toEqual([
+ {
+ setCell: {
+ familyName: "jobs",
+ columnQualifier: Buffer.from("job-1"),
+ timestampMicros: 0,
+ value: Buffer.from("7"),
+ },
+ },
+ {
+ addToCell: {
+ familyName: "agg",
+ columnQualifier: { rawValue: Buffer.from("total") },
+ timestamp: { rawTimestampMicros: 0 },
+ input: { intValue: 7 },
+ },
+ },
+ ]);
+ });
+
+ it("reports a duplicate without another logical addition", async () => {
+ await initializeRequestCredits("request-1", 8);
+ request.mockImplementation((_options, callback) =>
+ callback(null, { predicateMatched: true }),
+ );
+ getRows.mockResolvedValueOnce([
+ [row({ jobs: { "job-1": [{ value: Buffer.from("7") }] } })],
+ ]);
+
+ await expect(
+ recordRequestCredits({
+ requestId: "request-1",
+ jobId: "job-1",
+ credits: 7,
+ }),
+ ).resolves.toBe(false);
+ });
+
+ it("reads and sums all aggregate shard cells", async () => {
+ getRows
+ .mockResolvedValueOnce([
+ [row({ jobs: { "\x00shards": [{ value: Buffer.from("2") }] } })],
+ ])
+ .mockResolvedValueOnce([
+ [
+ row({ agg: { total: [{ value: int64(7n) }] } }),
+ row({ agg: { total: [{ value: int64(5n) }] } }),
+ ],
+ ]);
+
+ await expect(readRequestCredits("request-1")).resolves.toBe(12);
+ expect(getRows.mock.calls[1][0].keys).toEqual([
+ requestCreditsRowKey("request-1", 0),
+ requestCreditsRowKey("request-1", 1),
+ ]);
+ });
+
+ it("is disabled when the table is not configured", async () => {
+ mutableConfig.BIGTABLE_REQUEST_CREDITS_TABLE = undefined;
+
+ await expect(initializeRequestCredits("request-1", 8)).resolves.toBe(false);
+ await expect(
+ recordRequestCredits({
+ requestId: "request-1",
+ jobId: "job-1",
+ credits: 1,
+ }),
+ ).resolves.toBe(false);
+ await expect(readRequestCredits("request-1")).resolves.toBeNull();
+ expect(getBigtableTable).not.toHaveBeenCalled();
+ });
+});
diff --git a/apps/api/src/lib/request-credits-store.ts b/apps/api/src/lib/request-credits-store.ts
new file mode 100644
index 000000000..b0411b189
--- /dev/null
+++ b/apps/api/src/lib/request-credits-store.ts
@@ -0,0 +1,336 @@
+import crypto from "node:crypto";
+import type { Table } from "@google-cloud/bigtable";
+import { config } from "../config";
+import { getBigtableTable } from "./bigtable-client";
+import { setSpanAttributes, withSpan } from "./otel-tracer";
+
+const JOBS_FAMILY = "jobs";
+const AGG_FAMILY = "agg";
+const TOTAL_QUALIFIER = "total";
+const SHARDS_QUALIFIER = "\x00shards";
+const FIXED_TIMESTAMP_MICROS = 0;
+const MAX_CACHED_SHARD_COUNTS = 10_000;
+const shardCounts = new Map<string, number>();
+
+function cacheShardCount(requestId: string, shards: number): void {
+ shardCounts.delete(requestId);
+ shardCounts.set(requestId, shards);
+ if (shardCounts.size > MAX_CACHED_SHARD_COUNTS) {
+ const oldest = shardCounts.keys().next().value;
+ if (oldest !== undefined) shardCounts.delete(oldest);
+ }
+}
+
+function requestHash(requestId: string): Buffer {
+ return crypto.createHash("sha256").update(requestId, "utf8").digest();
+}
+
+export function requestCreditsRowKey(requestId: string, shard: number): string {
+ if (!Number.isInteger(shard) || shard < 0 || shard > 0xffff) {
+ throw new Error(`Invalid request credits shard: ${shard}`);
+ }
+
+ const hash = requestHash(requestId);
+ const shardBytes = Buffer.allocUnsafe(2);
+ shardBytes.writeUInt16BE(shard);
+ const salt = crypto
+ .createHash("sha256")
+ .update(hash)
+ .update(shardBytes)
+ .digest()
+ .subarray(0, 2)
+ .toString("hex");
+ return `${salt}#${hash.toString("hex")}#${shard}`;
+}
+
+export function requestCreditsShardForJob(
+ jobId: string,
+ shards: number,
+): number {
+ if (!Number.isInteger(shards) || shards <= 0 || shards > 512) {
+ throw new Error(`Invalid request credits shard count: ${shards}`);
+ }
+ return (
+ crypto.createHash("sha256").update(jobId, "utf8").digest().readUInt16BE(0) %
+ shards
+ );
+}
+
+export function requestCreditsShards(maxJobs: number): number {
+ if (!Number.isFinite(maxJobs) || maxJobs <= 0) return 8;
+ if (maxJobs <= 1_000) return 8;
+ if (maxJobs <= 10_000) return 16;
+ if (maxJobs <= 100_000) return 64;
+ return 512;
+}
+
+function exactQualifierFilter(qualifier: string) {
+ return {
+ chain: {
+ filters: [
+ { familyNameRegexFilter: `^${JOBS_FAMILY}$` },
+ {
+ columnQualifierRegexFilter: Buffer.concat([
+ Buffer.from("^"),
+ Buffer.from(qualifier),
+ Buffer.from("$"),
+ ]),
+ },
+ ],
+ },
+ };
+}
+
+async function checkAndMutateRowOnce(
+ table: Table,
+ request: Record<string, unknown>,
+): Promise<{ predicateMatched?: boolean | null }> {
+ return new Promise((resolve, reject) => {
+ table.bigtable.request(
+ {
+ client: "BigtableClient",
+ method: "checkAndMutateRow",
+ reqOpts: {
+ tableName: table.name,
+ appProfileId: table.bigtable.appProfileId,
+ ...request,
+ },
+ },
+ (
+ error: Error | null,
+ response?: { predicateMatched?: boolean | null },
+ ) => {
+ if (error) reject(error);
+ else resolve(response ?? {});
+ },
+ );
+ });
+}
+
+async function checkAndMutateRow(
+ table: Table,
+ request: Record<string, unknown>,
+): Promise<{ predicateMatched?: boolean | null }> {
+ let lastError: unknown;
+ for (let attempt = 0; attempt < 3; attempt++) {
+ try {
+ return await checkAndMutateRowOnce(table, request);
+ } catch (error) {
+ lastError = error;
+ const code =
+ error && typeof error === "object"
+ ? (error as { code?: unknown }).code
+ : undefined;
+ if (![1, 2, 4, 8, 10, 13, 14].includes(Number(code))) throw error;
+ if (attempt < 2) {
+ await new Promise(resolve => setTimeout(resolve, 100 * (attempt + 1)));
+ }
+ }
+ }
+ throw lastError;
+}
+
+function setCellMutation(qualifier: string, value: Buffer) {
+ return {
+ setCell: {
+ familyName: JOBS_FAMILY,
+ columnQualifier: Buffer.from(qualifier),
+ timestampMicros: FIXED_TIMESTAMP_MICROS,
+ value,
+ },
+ };
+}
+
+async function readShardCount(
+ table: Table,
+ requestId: string,
+): Promise<number | null> {
+ const [rows] = await table.getRows({
+ keys: [requestCreditsRowKey(requestId, 0)],
+ decode: false,
+ filter: [
+ { family: JOBS_FAMILY },
+ { column: SHARDS_QUALIFIER, cellLimit: 1 },
+ ],
+ });
+ const value = rows[0]?.data?.[JOBS_FAMILY]?.[SHARDS_QUALIFIER]?.[0]?.value;
+ if (value == null) return null;
+ const shards = Number(Buffer.from(value).toString("utf8"));
+ if (!Number.isInteger(shards) || shards <= 0 || shards > 512) {
+ throw new Error(`Invalid stored request credits shard count: ${shards}`);
+ }
+ cacheShardCount(requestId, shards);
+ return shards;
+}
+
+async function getShardCount(
+ table: Table,
+ requestId: string,
+): Promise<number | null> {
+ return shardCounts.get(requestId) ?? readShardCount(table, requestId);
+}
+
+async function readJobCredits(
+ table: Table,
+ rowKey: string,
+ jobId: string,
+): Promise<number | null> {
+ const [rows] = await table.getRows({
+ keys: [rowKey],
+ decode: false,
+ filter: [{ family: JOBS_FAMILY }, { column: jobId, cellLimit: 1 }],
+ });
+ const value = rows[0]?.data?.[JOBS_FAMILY]?.[jobId]?.[0]?.value;
+ if (value == null) return null;
+ const credits = Number(Buffer.from(value).toString("utf8"));
+ if (!Number.isSafeInteger(credits)) {
+ throw new Error(`Invalid stored request credits value: ${credits}`);
+ }
+ return credits;
+}
+
+export async function initializeRequestCredits(
+ requestId: string,
+ shards: number,
+): Promise<boolean> {
+ const tableId = config.BIGTABLE_REQUEST_CREDITS_TABLE;
+ if (!tableId) return false;
+ if (!Number.isInteger(shards) || shards <= 0 || shards > 512) {
+ throw new Error(`Invalid request credits shard count: ${shards}`);
+ }
+
+ return withSpan("bigtable.request_credits.initialize", async span => {
+ setSpanAttributes(span, {
+ "db.system": "bigtable",
+ "bigtable.table": tableId,
+ "bigtable.operation": "checkAndMutateRow",
+ "request_credits.shards": shards,
+ });
+ const table = await getBigtableTable(tableId);
+ const response = await checkAndMutateRow(table, {
+ rowKey: Buffer.from(requestCreditsRowKey(requestId, 0)),
+ predicateFilter: exactQualifierFilter(SHARDS_QUALIFIER),
+ falseMutations: [
+ setCellMutation(SHARDS_QUALIFIER, Buffer.from(String(shards))),
+ ],
+ });
+ const stored = response.predicateMatched
+ ? await readShardCount(table, requestId)
+ : shards;
+ if (stored !== shards) {
+ throw new Error(
+ `Request credits shard count mismatch for ${requestId}: ${stored} != ${shards}`,
+ );
+ }
+ cacheShardCount(requestId, shards);
+ return true;
+ });
+}
+
+export async function recordRequestCredits(params: {
+ requestId: string;
+ jobId: string;
+ credits: number;
+}): Promise<boolean> {
+ const tableId = config.BIGTABLE_REQUEST_CREDITS_TABLE;
+ if (!tableId) return false;
+ if (!Number.isSafeInteger(params.credits)) {
+ throw new Error(`Invalid request credits value: ${params.credits}`);
+ }
+
+ return withSpan("bigtable.request_credits.record", async span => {
+ setSpanAttributes(span, {
+ "db.system": "bigtable",
+ "bigtable.table": tableId,
+ "bigtable.operation": "checkAndMutateRow",
+ "request_credits.value": params.credits,
+ });
+ const table = await getBigtableTable(tableId);
+ const shards = await getShardCount(table, params.requestId);
+ if (shards === null) return false;
+ const shard = requestCreditsShardForJob(params.jobId, shards);
+ const rowKey = requestCreditsRowKey(params.requestId, shard);
+ const response = await checkAndMutateRow(table, {
+ rowKey: Buffer.from(rowKey),
+ predicateFilter: exactQualifierFilter(params.jobId),
+ falseMutations: [
+ setCellMutation(params.jobId, Buffer.from(String(params.credits))),
+ {
+ addToCell: {
+ familyName: AGG_FAMILY,
+ columnQualifier: { rawValue: Buffer.from(TOTAL_QUALIFIER) },
+ timestamp: { rawTimestampMicros: FIXED_TIMESTAMP_MICROS },
+ input: { intValue: params.credits },
+ },
+ },
+ ],
+ });
+ setSpanAttributes(span, {
+ "request_credits.shard": shard,
+ "request_credits.duplicate": response.predicateMatched === true,
+ });
+ if (response.predicateMatched !== true) return true;
+
+ const storedCredits = await readJobCredits(table, rowKey, params.jobId);
+ if (storedCredits !== params.credits) {
+ throw new Error(
+ `Request credits mismatch for ${params.jobId}: ${storedCredits} != ${params.credits}`,
+ );
+ }
+ return false;
+ });
+}
+
+export async function readRequestCredits(
+ requestId: string,
+): Promise<number | null> {
+ const tableId = config.BIGTABLE_REQUEST_CREDITS_TABLE;
+ if (!tableId) return null;
+
+ return withSpan("bigtable.request_credits.read", async span => {
+ setSpanAttributes(span, {
+ "db.system": "bigtable",
+ "bigtable.table": tableId,
+ "bigtable.operation": "getRows",
+ });
+ const table = await getBigtableTable(tableId);
+ const shards = await getShardCount(table, requestId);
+ if (shards === null) return null;
+ const [rows] = await table.getRows({
+ keys: Array.from({ length: shards }, (_, shard) =>
+ requestCreditsRowKey(requestId, shard),
+ ),
+ decode: false,
+ filter: [
+ { family: AGG_FAMILY },
+ { column: TOTAL_QUALIFIER, cellLimit: 1 },
+ ],
+ });
+ let total = 0n;
+ for (const row of rows) {
+ const value = row.data?.[AGG_FAMILY]?.[TOTAL_QUALIFIER]?.[0]?.value;
+ if (value == null) continue;
+ const encoded = Buffer.from(value);
+ if (encoded.length !== 8) {
+ throw new Error(
+ `Invalid request credits aggregate length: ${encoded.length}`,
+ );
+ }
+ total += encoded.readBigInt64BE();
+ }
+ const credits = Number(total);
+ if (!Number.isSafeInteger(credits)) {
+ throw new Error(`Request credits total exceeds safe integer: ${total}`);
+ }
+ setSpanAttributes(span, {
+ "request_credits.shards": shards,
+ "request_credits.returned_rows": rows.length,
+ "request_credits.total": credits,
+ });
+ return credits;
+ });
+}
+
+export function clearRequestCreditsShardCacheForTest(): void {
+ shardCounts.clear();
+}
diff --git a/apps/api/src/services/logging/log_job.ts b/apps/api/src/services/logging/log_job.ts
index 91e6fc869..adcd5718a 100644
--- a/apps/api/src/services/logging/log_job.ts
+++ b/apps/api/src/services/logging/log_job.ts
@@ -38,6 +38,10 @@ import {
writeScrapeJobState,
} from "../../lib/job-state-store";
import { buildReplayContextFromScrape } from "../../lib/scrape-interact/scrape-replay";
+import {
+ initializeRequestCredits,
+ recordRequestCredits,
+} from "../../lib/request-credits-store";
configDotenv();
const previewTeamId = "3adefd26-77ec-5968-8dcf-c94b5630d1de";
@@ -84,6 +88,22 @@ async function writeFeedbackJobSafely(
}
}
+async function recordRequestCreditsSafely(
+ params: Parameters<typeof recordRequestCredits>[0],
+ logger: Logger,
+): Promise<void> {
+ try {
+ await recordRequestCredits(params);
+ } catch (error) {
+ logger.error("Failed to record request credits in Bigtable", {
+ error,
+ requestId: params.requestId,
+ jobId: params.jobId,
+ credits: params.credits,
+ });
+ }
+}
+
/**
* Null-aware wrapper around the shared text sanitizer, kept where a cleaned
* value feeds a later decision (the external_request_id byte cap). Every row
@@ -486,6 +506,7 @@ type LoggedRequest = {
external_request_id?: string | null;
jobAccess?: boolean;
jobAccessExpiresAt?: Date;
+ creditsShards?: number;
};
/**
@@ -582,6 +603,17 @@ async function logRequestInternal(request: LoggedRequest) {
}
}
+ if (request.creditsShards !== undefined) {
+ try {
+ await initializeRequestCredits(request.id, request.creditsShards);
+ } catch (error) {
+ logger.error("Failed to initialize request credits in Bigtable", {
+ error,
+ shards: request.creditsShards,
+ });
+ }
+ }
+
if (request.zeroDataRetention && config.USE_DB_AUTHENTICATION === true) {
await enqueueZdrCleanupJob(request.id);
}
@@ -709,6 +741,17 @@ async function logScrapeInternal(scrape: LoggedScrape, force: boolean = false) {
logger,
);
+ if (scrape.id !== scrape.request_id) {
+ await recordRequestCreditsSafely(
+ {
+ requestId: scrape.request_id,
+ jobId: scrape.id,
+ credits: scrape.credits_cost,
+ },
+ logger,
+ );
+ }
+
if (!scrape.is_parse && scrape.id === scrape.request_id) {
try {
const replay = scrape.zeroDataRetention
@@ -989,6 +1032,17 @@ async function logSearchInternal(search: LoggedSearch, force: boolean = false) {
logger,
);
+ if (search.id !== search.request_id) {
+ await recordRequestCreditsSafely(
+ {
+ requestId: search.request_id,
+ jobId: search.id,
+ credits: search.credits_cost,
+ },
+ logger,
+ );
+ }
+
if (search.results && !search.zeroDataRetention) {
await saveSearchToGCS(search, logger);
}
diff --git a/apps/api/src/services/worker/crawl-logic.ts b/apps/api/src/services/worker/crawl-logic.ts
index 0e5120eb9..af7fc46e6 100644
--- a/apps/api/src/services/worker/crawl-logic.ts
+++ b/apps/api/src/services/worker/crawl-logic.ts
@@ -8,6 +8,7 @@ import {
import { getCrawl } from "../../lib/crawl-redis";
import { creditsBilledByCrawlId } from "../../db/rpc";
import { db } from "../../db/connection";
+import { readRequestCredits } from "../../lib/request-credits-store";
import { getJobs } from "../../controllers/v1/crawl-status";
import { logCrawl, logBatchScrape } from "../logging/log_job";
import { createWebhookSender, WebhookEvent } from "../webhook/index";
@@ -143,7 +144,13 @@ export async function finishCrawlSuper(job: NuQJob<any>) {
let credits_billed: number | null = null;
- if (config.USE_DB_AUTHENTICATION) {
+ try {
+ credits_billed = await readRequestCredits(requestId);
+ } catch (error) {
+ logger.warn("Bigtable request credits read failed", { error });
+ }
+
+ if (credits_billed === null && config.USE_DB_AUTHENTICATION) {
try {
const creditsRows = await creditsBilledByCrawlId(db, crawlId);
credits_billed = creditsRows?.[0]?.credits_billed ?? null;