Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥
TypeScript aiai-agentsai-crawlerai-scrapingai-searchcrawlerdata-extractionhtml-to-markdownllmmarkdownscraperscrapingweb-crawlerweb-dataweb-data-extractionweb-scraperweb-scrapingweb-searchwebscraping
feat(api): cut change tracking over to Bigtable; keep Postgres as read-only fallback (#4488)
Gergő Móricza82d24dafd9d
diff --git a/apps/api/src/db/rpc.ts b/apps/api/src/db/rpc.tsindex 817fa88bc..86d7510e0 100644--- a/apps/api/src/db/rpc.ts+++ b/apps/api/src/db/rpc.ts@@ -88,18 +88,6 @@ export function billTeam7(params: {);}-export async function changeTrackingInsertScrape(params: {- team_id: string;- url: string;- job_id: string;- change_tracking_tag: string | null;- date_added: string;-}): Promise<void> {- await db.execute(- sql`select change_tracking_insert_scrape(p_team_id => ${params.team_id}, p_url => ${params.url}, p_job_id => ${params.job_id}, p_change_tracking_tag => ${params.change_tracking_tag}, p_date_added => ${params.date_added}::timestamptz)`,- );-}-// `database` is a parameter because callers split between the primary and the// read replica: status controllers (informational `creditsUsed`) read from the// replica, while crawl finalization (crawl-logic) reads its own recent billingdiff --git a/apps/api/src/lib/change-tracking-store.ts b/apps/api/src/lib/change-tracking-store.tsindex 2a02afc69..a011e1206 100644--- a/apps/api/src/lib/change-tracking-store.ts+++ b/apps/api/src/lib/change-tracking-store.ts@@ -1,20 +1,23 @@import type { Bigtable, Table } from "@google-cloud/bigtable";import crypto from "crypto";import { config } from "../config";-import { logger } from "./logger";-import {- changeTrackingInsertScrape as changeTrackingInsertScrapeRpc,- diffGetLastScrape,-} from "../db/rpc";+import { diffGetLastScrape } from "../db/rpc";-// Change tracking bookkeeping. Migration step 1 (#4475 is the intended-// endpoint): Postgres stays the primary store -- writes go to-// change_tracking_scrapes, reads come from it -- and, when Bigtable is-// configured, every write is double-written to the Bigtable layout the-// endpoint uses: one row per (team_id, url, tag) holding the latest-// scrape's job id, cell timestamp = date_added, family GC max_versions=1.-// The Bigtable side is never read yet; its writes are detached and-// best-effort.+// Change tracking bookkeeping. Final architecture (#4484 was the+// transition): Bigtable is the store. Writes go to Bigtable only --+// no Postgres insert. Reads are Bigtable-primary with a Postgres+// fallback on miss: every Bigtable row was written after the cutover,+// so it is at least as recent as any Postgres row for the same key,+// and a miss means the latest scrape predates the cutover, which the+// frozen Postgres archive still holds. Drop the archive once the+// fallback read rate decays to ~0. Bigtable errors do not fall back --+// they surface through the existing deriveDiff warning.+//+// One row per (team_id, url, tag) holding the latest scrape's job id;+// content itself lives in GCS. Cell timestamp = date_added, family GC+// rule max_versions=1, so reads return the highest date_added and+// regressed (out-of-order) writes are shadowed then collected at+// compaction.//// Row key: team_id || sha256(url) || sha256(tag)// - team_id first: team-scoped prefix operations (bulk delete on cleanup)@@ -55,16 +58,11 @@ function changeTrackingRowKey(]);}-function bigtableConfigured(): boolean {- return !!config.BIGTABLE_INSTANCE_ID;-}-let bigtableClient: Bigtable | null = null;let table: Table | null = null;-// The Bigtable package is loaded lazily: log_job imports this module on-// the primary scrape-logging path, so a broken or slow package load must-// never take that path down -- failures surface only in the double-write.+// The Bigtable package is loaded lazily so a broken or slow package load+// surfaces as a caught error in the callers, not an import-time crash.async function getChangeTrackingTable(): Promise<Table> {if (!bigtableClient) {const { Bigtable } = await import("@google-cloud/bigtable");@@ -74,9 +72,9 @@ async function getChangeTrackingTable(): Promise<Table> {? { appProfileId: config.BIGTABLE_APP_PROFILE_ID }: {}),// Mirrors GCS_CREDENTIALS: base64-encoded service-account JSON.- // Parsed here (not at module load) so a malformed value can only- // fail the non-fatal double-write, never the primary path. Unset- // falls back to Application Default Credentials.+ // Parsed here (not at module load) so a malformed value fails as a+ // caught error, never an import-time crash. Unset falls back to+ // Application Default Credentials....(config.BIGTABLE_CREDENTIALS? {credentials: JSON.parse(atob(config.BIGTABLE_CREDENTIALS)),@@ -90,81 +88,18 @@ async function getChangeTrackingTable(): Promise<Table> {});}if (!table) {+ if (!config.BIGTABLE_INSTANCE_ID) {+ throw new Error(+ "BIGTABLE_INSTANCE_ID is not configured; change tracking requires the Bigtable store",+ );+ }table = bigtableClient- .instance(config.BIGTABLE_INSTANCE_ID!)+ .instance(config.BIGTABLE_INSTANCE_ID).table(TABLE_ID);}return table;}-// The double-write is best-effort: it runs detached with a bounded-// timeout so a slow or hung Bigtable can never add failure latency to-// the already-successful Postgres write.-const BIGTABLE_DOUBLE_WRITE_TIMEOUT_MS = 10_000;--function doubleWriteToBigtable(params: {- team_id: string;- url: string;- job_id: string;- tag: string | null;- date_added: Date;-}): void {- void (async () => {- let timer: NodeJS.Timeout | undefined;- try {- const write = getChangeTrackingTable().then(table =>- // Runtime accepts Buffer keys (converted verbatim); the .d.ts- // only declares string.- table.mutate([- {- key: changeTrackingRowKey(- params.team_id,- params.url,- params.tag,- ) as unknown as string,- // Required by Mutation.parse -- without it the entry- // carries no setCell mutations.- method: "insert" as const,- data: {- [FAMILY]: {- [QUALIFIER]: {- value: params.job_id,- timestamp: params.date_added,- },- },- },- },- ]),- );- // Keep the write handled in case the race below settles on the- // timeout first and it rejects later.- write.catch(() => {});- await Promise.race([- write,- new Promise<never>((_, reject) => {- timer = setTimeout(- () =>- reject(- new Error(- `Bigtable change tracking double-write timed out after ${BIGTABLE_DOUBLE_WRITE_TIMEOUT_MS}ms`,- ),- ),- BIGTABLE_DOUBLE_WRITE_TIMEOUT_MS,- );- timer.unref();- }),- ]);- } catch (error) {- logger.warn("Error inserting change tracking record into Bigtable", {- error,- teamId: params.team_id,- });- } finally {- if (timer) clearTimeout(timer);- }- })();-}-export async function changeTrackingInsertScrape(params: {team_id: string;url: string;@@ -173,33 +108,81 @@ export async function changeTrackingInsertScrape(params: {/** When the scrape was logged; becomes the cell timestamp. */date_added: Date;}): Promise<void> {- await changeTrackingInsertScrapeRpc({- team_id: params.team_id,- url: params.url,- job_id: params.job_id,- change_tracking_tag: params.tag,- date_added: params.date_added.toISOString(),- });-- if (bigtableConfigured()) {- doubleWriteToBigtable(params);- }+ const table = await getChangeTrackingTable();+ // Runtime accepts Buffer keys (converted verbatim); the .d.ts only+ // declares string.+ await table.mutate([+ {+ key: changeTrackingRowKey(+ params.team_id,+ params.url,+ params.tag,+ ) as unknown as string,+ // Required by Mutation.parse -- without it the entry carries no+ // setCell mutations.+ method: "insert" as const,+ data: {+ [FAMILY]: {+ [QUALIFIER]: {+ value: params.job_id,+ timestamp: params.date_added,+ },+ },+ },+ },+ ]);}/**- * Point lookup of the latest scrape for (team_id, url, tag). Returns null- * when the team never scraped this url+tag combination.+ * Point lookup of the latest scrape for (team_id, url, tag). Bigtable is+ * primary; a miss falls back to the frozen Postgres archive, which holds+ * pre-cutover history. Returns null when the team never scraped this+ * url+tag combination.*/export async function changeTrackingGetLastScrape(params: {team_id: string;url: string;tag: string | null;}): Promise<{ job_id: string; date_added: string } | null> {- const rows = await diffGetLastScrape(params.team_id, params.url, params.tag);+ const table = await getChangeTrackingTable();+ const key = changeTrackingRowKey(params.team_id, params.url, params.tag);+ const [rows] = await table.getRows({+ keys: [key as unknown as string],+ // Qualifier regex + latest-version-only, expressed via the column+ // filter's cellLimit (there is no standalone `versions` filter key).+ filter: [{ column: { name: QUALIFIER, cellLimit: 1 } }],+ });const row = rows[0];- if (!row) return null;+ if (row) {+ const cells = row.data?.[FAMILY]?.[QUALIFIER];+ const cell = Array.isArray(cells) ? cells[0] : undefined;+ if (cell && cell.value != null) {+ // Cell timestamps are microseconds, delivered as string or number+ // (protos render >32-bit longs as strings). Both are safe+ // integers in JS until ~year 2255.+ const timestampMicros = Number(cell.timestamp);+ const dateAdded = new Date(+ Number.isFinite(timestampMicros)+ ? Math.floor(timestampMicros / 1000)+ : Date.now(),+ );+ return {+ job_id: String(cell.value),+ date_added: dateAdded.toISOString(),+ };+ }+ }++ // Bigtable miss: the latest scrape for this key predates the cutover.+ const pgRows = await diffGetLastScrape(+ params.team_id,+ params.url,+ params.tag,+ );+ const pgRow = pgRows[0];+ if (!pgRow) return null;return {- job_id: row.o_job_id,- date_added: new Date(row.o_date_added).toISOString(),+ job_id: pgRow.o_job_id,+ date_added: new Date(pgRow.o_date_added).toISOString(),};}