Skip to content

Supercharge your AI agents with data from the web and beyond. Building the library for superintelligence. 🔥

feat(api): cut change tracking over to Bigtable; keep Postgres as read-only fallback (#4488)

Gergő Móricza82d24dafd9d
diff --git a/apps/api/src/db/rpc.ts b/apps/api/src/db/rpc.ts
index 817fa88bc..86d7510e0 100644
--- a/apps/api/src/db/rpc.ts
+++ b/apps/api/src/db/rpc.ts
@@ -88,18 +88,6 @@ export function billTeam7(params: {
);
}
-export async function changeTrackingInsertScrape(params: {
- team_id: string;
- url: string;
- job_id: string;
- change_tracking_tag: string | null;
- date_added: string;
-}): Promise<void> {
- await db.execute(
- sql`select change_tracking_insert_scrape(p_team_id => ${params.team_id}, p_url => ${params.url}, p_job_id => ${params.job_id}, p_change_tracking_tag => ${params.change_tracking_tag}, p_date_added => ${params.date_added}::timestamptz)`,
- );
-}
-
// `database` is a parameter because callers split between the primary and the
// read replica: status controllers (informational `creditsUsed`) read from the
// replica, while crawl finalization (crawl-logic) reads its own recent billing
diff --git a/apps/api/src/lib/change-tracking-store.ts b/apps/api/src/lib/change-tracking-store.ts
index 2a02afc69..a011e1206 100644
--- a/apps/api/src/lib/change-tracking-store.ts
+++ b/apps/api/src/lib/change-tracking-store.ts
@@ -1,20 +1,23 @@
import type { Bigtable, Table } from "@google-cloud/bigtable";
import crypto from "crypto";
import { config } from "../config";
-import { logger } from "./logger";
-import {
- changeTrackingInsertScrape as changeTrackingInsertScrapeRpc,
- diffGetLastScrape,
-} from "../db/rpc";
+import { diffGetLastScrape } from "../db/rpc";
-// Change tracking bookkeeping. Migration step 1 (#4475 is the intended
-// endpoint): Postgres stays the primary store -- writes go to
-// change_tracking_scrapes, reads come from it -- and, when Bigtable is
-// configured, every write is double-written to the Bigtable layout the
-// endpoint uses: one row per (team_id, url, tag) holding the latest
-// scrape's job id, cell timestamp = date_added, family GC max_versions=1.
-// The Bigtable side is never read yet; its writes are detached and
-// best-effort.
+// Change tracking bookkeeping. Final architecture (#4484 was the
+// transition): Bigtable is the store. Writes go to Bigtable only --
+// no Postgres insert. Reads are Bigtable-primary with a Postgres
+// fallback on miss: every Bigtable row was written after the cutover,
+// so it is at least as recent as any Postgres row for the same key,
+// and a miss means the latest scrape predates the cutover, which the
+// frozen Postgres archive still holds. Drop the archive once the
+// fallback read rate decays to ~0. Bigtable errors do not fall back --
+// they surface through the existing deriveDiff warning.
+//
+// One row per (team_id, url, tag) holding the latest scrape's job id;
+// content itself lives in GCS. Cell timestamp = date_added, family GC
+// rule max_versions=1, so reads return the highest date_added and
+// regressed (out-of-order) writes are shadowed then collected at
+// compaction.
//
// Row key: team_id || sha256(url) || sha256(tag)
// - team_id first: team-scoped prefix operations (bulk delete on cleanup)
@@ -55,16 +58,11 @@ function changeTrackingRowKey(
]);
}
-function bigtableConfigured(): boolean {
- return !!config.BIGTABLE_INSTANCE_ID;
-}
-
let bigtableClient: Bigtable | null = null;
let table: Table | null = null;
-// The Bigtable package is loaded lazily: log_job imports this module on
-// the primary scrape-logging path, so a broken or slow package load must
-// never take that path down -- failures surface only in the double-write.
+// The Bigtable package is loaded lazily so a broken or slow package load
+// surfaces as a caught error in the callers, not an import-time crash.
async function getChangeTrackingTable(): Promise<Table> {
if (!bigtableClient) {
const { Bigtable } = await import("@google-cloud/bigtable");
@@ -74,9 +72,9 @@ async function getChangeTrackingTable(): Promise<Table> {
? { appProfileId: config.BIGTABLE_APP_PROFILE_ID }
: {}),
// Mirrors GCS_CREDENTIALS: base64-encoded service-account JSON.
- // Parsed here (not at module load) so a malformed value can only
- // fail the non-fatal double-write, never the primary path. Unset
- // falls back to Application Default Credentials.
+ // Parsed here (not at module load) so a malformed value fails as a
+ // caught error, never an import-time crash. Unset falls back to
+ // Application Default Credentials.
...(config.BIGTABLE_CREDENTIALS
? {
credentials: JSON.parse(atob(config.BIGTABLE_CREDENTIALS)),
@@ -90,81 +88,18 @@ async function getChangeTrackingTable(): Promise<Table> {
});
}
if (!table) {
+ if (!config.BIGTABLE_INSTANCE_ID) {
+ throw new Error(
+ "BIGTABLE_INSTANCE_ID is not configured; change tracking requires the Bigtable store",
+ );
+ }
table = bigtableClient
- .instance(config.BIGTABLE_INSTANCE_ID!)
+ .instance(config.BIGTABLE_INSTANCE_ID)
.table(TABLE_ID);
}
return table;
}
-// The double-write is best-effort: it runs detached with a bounded
-// timeout so a slow or hung Bigtable can never add failure latency to
-// the already-successful Postgres write.
-const BIGTABLE_DOUBLE_WRITE_TIMEOUT_MS = 10_000;
-
-function doubleWriteToBigtable(params: {
- team_id: string;
- url: string;
- job_id: string;
- tag: string | null;
- date_added: Date;
-}): void {
- void (async () => {
- let timer: NodeJS.Timeout | undefined;
- try {
- const write = getChangeTrackingTable().then(table =>
- // Runtime accepts Buffer keys (converted verbatim); the .d.ts
- // only declares string.
- table.mutate([
- {
- key: changeTrackingRowKey(
- params.team_id,
- params.url,
- params.tag,
- ) as unknown as string,
- // Required by Mutation.parse -- without it the entry
- // carries no setCell mutations.
- method: "insert" as const,
- data: {
- [FAMILY]: {
- [QUALIFIER]: {
- value: params.job_id,
- timestamp: params.date_added,
- },
- },
- },
- },
- ]),
- );
- // Keep the write handled in case the race below settles on the
- // timeout first and it rejects later.
- write.catch(() => {});
- await Promise.race([
- write,
- new Promise<never>((_, reject) => {
- timer = setTimeout(
- () =>
- reject(
- new Error(
- `Bigtable change tracking double-write timed out after ${BIGTABLE_DOUBLE_WRITE_TIMEOUT_MS}ms`,
- ),
- ),
- BIGTABLE_DOUBLE_WRITE_TIMEOUT_MS,
- );
- timer.unref();
- }),
- ]);
- } catch (error) {
- logger.warn("Error inserting change tracking record into Bigtable", {
- error,
- teamId: params.team_id,
- });
- } finally {
- if (timer) clearTimeout(timer);
- }
- })();
-}
-
export async function changeTrackingInsertScrape(params: {
team_id: string;
url: string;
@@ -173,33 +108,81 @@ export async function changeTrackingInsertScrape(params: {
/** When the scrape was logged; becomes the cell timestamp. */
date_added: Date;
}): Promise<void> {
- await changeTrackingInsertScrapeRpc({
- team_id: params.team_id,
- url: params.url,
- job_id: params.job_id,
- change_tracking_tag: params.tag,
- date_added: params.date_added.toISOString(),
- });
-
- if (bigtableConfigured()) {
- doubleWriteToBigtable(params);
- }
+ const table = await getChangeTrackingTable();
+ // Runtime accepts Buffer keys (converted verbatim); the .d.ts only
+ // declares string.
+ await table.mutate([
+ {
+ key: changeTrackingRowKey(
+ params.team_id,
+ params.url,
+ params.tag,
+ ) as unknown as string,
+ // Required by Mutation.parse -- without it the entry carries no
+ // setCell mutations.
+ method: "insert" as const,
+ data: {
+ [FAMILY]: {
+ [QUALIFIER]: {
+ value: params.job_id,
+ timestamp: params.date_added,
+ },
+ },
+ },
+ },
+ ]);
}
/**
- * Point lookup of the latest scrape for (team_id, url, tag). Returns null
- * when the team never scraped this url+tag combination.
+ * Point lookup of the latest scrape for (team_id, url, tag). Bigtable is
+ * primary; a miss falls back to the frozen Postgres archive, which holds
+ * pre-cutover history. Returns null when the team never scraped this
+ * url+tag combination.
*/
export async function changeTrackingGetLastScrape(params: {
team_id: string;
url: string;
tag: string | null;
}): Promise<{ job_id: string; date_added: string } | null> {
- const rows = await diffGetLastScrape(params.team_id, params.url, params.tag);
+ const table = await getChangeTrackingTable();
+ const key = changeTrackingRowKey(params.team_id, params.url, params.tag);
+ const [rows] = await table.getRows({
+ keys: [key as unknown as string],
+ // Qualifier regex + latest-version-only, expressed via the column
+ // filter's cellLimit (there is no standalone `versions` filter key).
+ filter: [{ column: { name: QUALIFIER, cellLimit: 1 } }],
+ });
const row = rows[0];
- if (!row) return null;
+ if (row) {
+ const cells = row.data?.[FAMILY]?.[QUALIFIER];
+ const cell = Array.isArray(cells) ? cells[0] : undefined;
+ if (cell && cell.value != null) {
+ // Cell timestamps are microseconds, delivered as string or number
+ // (protos render >32-bit longs as strings). Both are safe
+ // integers in JS until ~year 2255.
+ const timestampMicros = Number(cell.timestamp);
+ const dateAdded = new Date(
+ Number.isFinite(timestampMicros)
+ ? Math.floor(timestampMicros / 1000)
+ : Date.now(),
+ );
+ return {
+ job_id: String(cell.value),
+ date_added: dateAdded.toISOString(),
+ };
+ }
+ }
+
+ // Bigtable miss: the latest scrape for this key predates the cutover.
+ const pgRows = await diffGetLastScrape(
+ params.team_id,
+ params.url,
+ params.tag,
+ );
+ const pgRow = pgRows[0];
+ if (!pgRow) return null;
return {
- job_id: row.o_job_id,
- date_added: new Date(row.o_date_added).toISOString(),
+ job_id: pgRow.o_job_id,
+ date_added: new Date(pgRow.o_date_added).toISOString(),
};
}