Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
fix(api): read Bigtable credit cells the client pre-decodes to numbers (#4680)
Gergő Móricz865cb3e3e2cd
diff --git a/apps/api/src/lib/request-credits-store.test.ts b/apps/api/src/lib/request-credits-store.test.tsindex 858ce2359..3eecce79e 100644--- a/apps/api/src/lib/request-credits-store.test.ts+++ b/apps/api/src/lib/request-credits-store.test.ts@@ -23,6 +23,7 @@ vi.mock("./otel-tracer", () => ({import {clearRequestCreditsShardCacheForTest,initializeRequestCredits,+ cellBytes,readRequestCredits,recordRequestCredits,requestCreditsRowKey,@@ -181,6 +182,33 @@ describe("request credits store", () => {]);});+ it("restores the bytes of cells the client pre-decoded to numbers", () => {+ // @google-cloud/bigtable converts any 8-byte value that fits a safe+ // integer into a number before honouring `decode: false`.+ expect(cellBytes(7)).toEqual(int64(7n));+ expect(cellBytes(-4)).toEqual(int64(-4n));+ expect(cellBytes(0)).toEqual(int64(0n));+ expect(cellBytes(int64(9n))).toEqual(int64(9n));+ expect(cellBytes("16")).toEqual(Buffer.from("16"));+ expect(() => cellBytes(1.5)).toThrow(/non-integer/);+ });++ it("sums aggregate cells that arrive as numbers instead of bytes", async () => {+ getRows+ .mockResolvedValueOnce([+ [row({ jobs: { "\x00shards": [{ value: Buffer.from("3") }] } })],+ ])+ .mockResolvedValueOnce([+ [+ row({ agg: { total: [{ value: 1 }] } }),+ row({ agg: { total: [{ value: int64(2n) }] } }),+ row({ agg: { total: [{ value: 0 }] } }),+ ],+ ]);++ await expect(readRequestCredits("request-1")).resolves.toBe(3);+ });+it("is disabled when the table is not configured", async () => {mutableConfig.BIGTABLE_REQUEST_CREDITS_TABLE = undefined;diff --git a/apps/api/src/lib/request-credits-store.ts b/apps/api/src/lib/request-credits-store.tsindex b0411b189..f1e4e87a6 100644--- a/apps/api/src/lib/request-credits-store.ts+++ b/apps/api/src/lib/request-credits-store.ts@@ -21,6 +21,31 @@ function cacheShardCount(requestId: string, shards: number): void {}}+/**+ * Restore the raw bytes of a cell value read with `decode: false`.+ *+ * The Bigtable client converts any 8-byte value that fits a safe integer into+ * a JavaScript number *before* it honours `decode: false`, so the `agg:total`+ * Int64 aggregate (and, in principle, an 8-character text cell) arrives as a+ * number. The conversion is `Long.fromBytes(buf).toNumber()`, which is exact+ * for safe integers, so writing the number back as a big-endian Int64 yields+ * the original bytes.+ */+export function cellBytes(+ value: Buffer | Uint8Array | string | number,+): Buffer {+ if (Buffer.isBuffer(value)) return value;+ if (typeof value === "number") {+ if (!Number.isSafeInteger(value)) {+ throw new Error(`Unexpected non-integer Bigtable cell value: ${value}`);+ }+ const buffer = Buffer.alloc(8);+ buffer.writeBigInt64BE(BigInt(value));+ return buffer;+ }+ return Buffer.from(value as any);+}+function requestHash(requestId: string): Buffer {return crypto.createHash("sha256").update(requestId, "utf8").digest();}@@ -155,7 +180,7 @@ async function readShardCount(});const value = rows[0]?.data?.[JOBS_FAMILY]?.[SHARDS_QUALIFIER]?.[0]?.value;if (value == null) return null;- const shards = Number(Buffer.from(value).toString("utf8"));+ const shards = Number(cellBytes(value).toString("utf8"));if (!Number.isInteger(shards) || shards <= 0 || shards > 512) {throw new Error(`Invalid stored request credits shard count: ${shards}`);}@@ -182,7 +207,7 @@ async function readJobCredits(});const value = rows[0]?.data?.[JOBS_FAMILY]?.[jobId]?.[0]?.value;if (value == null) return null;- const credits = Number(Buffer.from(value).toString("utf8"));+ const credits = Number(cellBytes(value).toString("utf8"));if (!Number.isSafeInteger(credits)) {throw new Error(`Invalid stored request credits value: ${credits}`);}@@ -310,7 +335,7 @@ export async function readRequestCredits(for (const row of rows) {const value = row.data?.[AGG_FAMILY]?.[TOTAL_QUALIFIER]?.[0]?.value;if (value == null) continue;- const encoded = Buffer.from(value);+ const encoded = cellBytes(value);if (encoded.length !== 8) {throw new Error(`Invalid request credits aggregate length: ${encoded.length}`,