Rallly is an open-source scheduling and collaboration tool designed to make organizing events and meetings easier.
TypeScript i18nextnext-authnextjspostgresqlprismareact-emailt3-stacktailwindcsstrpcturborepotypescriptzod
♻️ Snapshot guest ids before purging PostHog profiles
Luke Vella92d9397f98fa
diff --git a/packages/posthog/src/scripts/purge-guest-profiles.ts b/packages/posthog/src/scripts/purge-guest-profiles.tsindex bc5168d24..5725bfa32 100644--- a/packages/posthog/src/scripts/purge-guest-profiles.ts+++ b/packages/posthog/src/scripts/purge-guest-profiles.ts@@ -1,11 +1,19 @@/*** Deletes guest person profiles (no email property) from PostHog.*- * Guests are matched in two passes: email property not set, then email set to- * an empty string. Selection uses the persons REST endpoint rather than HogQL- * so the ids we fetch are exactly what bulk_delete expects. Each cycle- * re-fetches the first page (no offset pagination — offsets drift as rows are- * deleted) until the filter returns nothing.+ * Two-phase design so the delete loop never paginates a table it is mutating:+ *+ * 1. SNAPSHOT — page through HogQL once, up front, collecting every guest+ * person id into memory (keyset pagination on `id`; OFFSET is rejected for+ * personal API keys). HogQL `persons.id` is the person UUID, which is+ * exactly what bulk_delete expects.+ * 2. DRAIN — delete the snapshot ids in fixed batches. Deleting an id that is+ * already gone is a no-op, so the eventually-consistent persons replica+ * lagging behind bulk_delete no longer matters: we drain a frozen list, we+ * never re-read the mutating table for the next cursor. This removes the+ * empty-page / read-replica-lag failure mode of the old first-page-refetch+ * loop, which stalled at ~1.2M rows because the list replica fell more than+ * a minute behind under sustained delete load.** Events are preserved (delete_events: false) — only the person profiles are* removed; guest page views and events remain as historical data.@@ -31,43 +39,35 @@ const API_HOST = process.env.POSTHOG_API_HOST ?? "https://us.posthog.com";const PROJECT_ID = process.env.POSTHOG_PROJECT_ID;const API_KEY = process.env.POSTHOG_PERSONAL_API_KEY;-const PAGE_SIZE = 1000;const SAMPLE_SIZE = 50;+// HogQL rows fetched per keyset page while building the id snapshot.+const SNAPSHOT_PAGE_SIZE = 10000;+// bulk_delete accepts at most 1000 ids per call (PostHog validates this).+const DELETE_BATCH_SIZE = 1000;// CRUD endpoints are rate limited to 480/min org-wideconst DELETE_INTERVAL_MS = 800;const MAX_RATE_LIMIT_RETRIES = 5;-// The persons list endpoint (ClickHouse) lags behind bulk_delete. When it hands-// back an empty page but the count still reports matches, back off and retry-// before treating the filter as exhausted.-const EMPTY_PAGE_BACKOFF_MS = 5000;-const MAX_EMPTY_PAGE_RETRIES = 12;-const GUEST_FILTERS = [- {- label: "email is not set",- countQuery: "SELECT count() FROM persons WHERE isNull(properties.email)",- properties: [- {- key: "email",- operator: "is_not_set",- value: "is_not_set",- type: "person",- },- ],- },- {- label: "email is empty string",- countQuery: "SELECT count() FROM persons WHERE properties.email = ''",- properties: [- { key: "email", operator: "exact", value: "", type: "person" },- ],- },+// Guests are persons with no usable email. `email = ''` matched 0 rows in+// production, but keep it in the predicate for correctness.+const GUEST_PREDICATE = "isNull(properties.email) OR properties.email = ''";+const GUEST_COUNT_QUERY = `SELECT count() FROM persons WHERE ${GUEST_PREDICATE}`;++// Persons REST filter for the sample/canary reads (that endpoint uses property+// filters, not HogQL). Matches the "email not set" pass of the predicate.+const GUEST_REST_PROPERTIES = [+ { key: "email", operator: "is_not_set", value: "is_not_set", type: "person" },];function sleep(ms: number) {return new Promise((resolve) => setTimeout(resolve, ms));}+// Thousands separators for log readability (avoids Intl date-format lint rule).+function fmt(n: number): string {+ return String(n).replace(/\B(?=(\d{3})+(?!\d))/g, ",");+}+async function apiRequest({path,method = "GET",@@ -105,13 +105,57 @@ async function apiRequest({}}-async function hogqlCount(query: string): Promise<number> {+async function hogqlQuery(query: string): Promise<unknown[][]> {const data = await apiRequest({path: `/api/projects/${PROJECT_ID}/query/`,method: "POST",body: { query: { kind: "HogQLQuery", query } },});- return Number(data.results?.[0]?.[0] ?? 0);+ return data.results ?? [];+}++async function hogqlCount(query: string): Promise<number> {+ const results = await hogqlQuery(query);+ return Number(results?.[0]?.[0] ?? 0);+}++// Escape a UUID for safe interpolation into a HogQL string literal. The values+// come from PostHog itself and are always well-formed UUIDs, but keep the escape+// so a malformed value can't break out of the toUUID('…') literal.+function quoteUuid(id: string): string {+ return id.replaceAll("\\", "\\\\").replaceAll("'", "\\'");+}++// Page through the guest filter once, up front, collecting every id into memory.+// Keyset pagination on `id` (OFFSET is rejected for personal API keys). Returns+// the full frozen list the drain loop consumes.+async function snapshotGuestIds(): Promise<string[]> {+ console.info(+ `\n📸 Snapshotting guest person ids (HogQL, ${fmt(SNAPSHOT_PAGE_SIZE)}/page)…`,+ );+ const ids: string[] = [];+ let cursor: string | undefined;++ for (;;) {+ const where = cursor+ ? `(${GUEST_PREDICATE}) AND id > toUUID('${quoteUuid(cursor)}')`+ : GUEST_PREDICATE;+ const query = `SELECT id FROM persons WHERE ${where} ORDER BY id LIMIT ${SNAPSHOT_PAGE_SIZE}`;++ const rows = await hogqlQuery(query);+ if (rows.length === 0) break;++ for (const row of rows) {+ ids.push(String(row[0]));+ }+ cursor = String(rows[rows.length - 1][0]);+ console.info(`• Snapshotted ${fmt(ids.length)} id(s) so far…`);++ if (rows.length < SNAPSHOT_PAGE_SIZE) break;+ }++ console.info(`📸 Snapshot complete — ${fmt(ids.length)} guest id(s).`);+ return ids;}async function fetchGuestPage({@@ -164,7 +208,7 @@ async function preflight() {async function printSample() {const sample = await fetchGuestPage({- properties: GUEST_FILTERS[0].properties,+ properties: GUEST_REST_PROPERTIES,limit: SAMPLE_SIZE,});@@ -213,7 +257,7 @@ async function verifyCanary() {);const sample = await fetchGuestPage({- properties: GUEST_FILTERS[0].properties,+ properties: GUEST_REST_PROPERTIES,limit: SAMPLE_SIZE,});@@ -287,76 +331,30 @@ async function verifyCanary() {);}-async function purge({- properties,- countQuery,- label,-}: {- properties: unknown[];- countQuery: string;- label: string;-}) {- console.info(`\n🗑️ Purging profiles where ${label}…`);+// Drains the frozen id snapshot in fixed batches. Deleting an already-deleted id+// is a no-op, so replica lag never stalls the loop and offsets never drift.+async function purge(ids: string[]) {+ console.info(+ `\n🗑️ Deleting ${fmt(ids.length)} guest profile(s) in batches of ${DELETE_BATCH_SIZE}…`,+ );let deleted = 0;- let previousFirstId: string | number | undefined;- let stuckCycles = 0;- // The persons list endpoint reads from ClickHouse and is eventually- // consistent: right after a bulk_delete it can briefly return a partial or- // empty page while the deletes propagate. A single empty page is therefore- // NOT proof the work is done — confirm against the authoritative count and- // retry a few times before concluding the filter is exhausted.- let emptyPages = 0;- for (;;) {- const page = await fetchGuestPage({ properties, limit: PAGE_SIZE });-- if (page.length === 0) {- const remaining = await hogqlCount(countQuery);- if (remaining === 0) {- break;- }- emptyPages++;- if (emptyPages >= MAX_EMPTY_PAGE_RETRIES) {- throw new Error(- `Persons list returned empty ${emptyPages} times but the count still reports ${remaining} matching profile(s). Read replica may be lagging — aborting so the run isn't silently left incomplete.`,- );- }- console.info(- `• Empty page but ${remaining} still match — waiting for read replica (retry ${emptyPages}/${MAX_EMPTY_PAGE_RETRIES})`,- );- await sleep(EMPTY_PAGE_BACKOFF_MS);- continue;- }-- emptyPages = 0;-- if (page[0].id === previousFirstId) {- stuckCycles++;- if (stuckCycles >= 3) {- throw new Error(- `Deletes are not taking effect — the same page keeps coming back (first id ${page[0].id}). Aborting.`,- );- }- } else {- stuckCycles = 0;- }- previousFirstId = page[0].id;+ for (let i = 0; i < ids.length; i += DELETE_BATCH_SIZE) {+ const batch = ids.slice(i, i + DELETE_BATCH_SIZE);await apiRequest({path: `/api/projects/${PROJECT_ID}/persons/bulk_delete/`,method: "POST",- body: { ids: page.map((person) => person.id), delete_events: false },+ body: { ids: batch, delete_events: false },});- deleted += page.length;- console.info(`• Deleted ${page.length} (running total: ${deleted})`);+ deleted += batch.length;+ console.info(+ `• Deleted ${batch.length} (running total: ${fmt(deleted)}/${fmt(ids.length)})`,+ );await sleep(DELETE_INTERVAL_MS);}- const remaining = await hogqlCount(countQuery);- console.info(- `✅ Done — ${deleted} profile(s) deleted where ${label}. Count now reports ${remaining} matching.`,- );return deleted;}@@ -396,14 +394,18 @@ async function purge({return;}- let total = 0;- for (const filter of GUEST_FILTERS) {- total += await purge(filter);- }+ const ids = await snapshotGuestIds();+ const deleted = await purge(ids);+ const remaining = await hogqlCount(GUEST_COUNT_QUERY);console.info(- `\n📊 Done: ${total} guest profile(s) deleted. Their events were preserved.`,+ `\n📊 Done: ${fmt(deleted)} guest profile(s) deleted. Their events were preserved. Count now reports ${fmt(remaining)} matching.`,);+ if (remaining > 0) {+ console.info(+ "ℹ️ A non-zero remaining count right after the run is usually replica lag or profiles created since the snapshot — re-run to sweep any stragglers.",+ );+ }})().catch((error) => {console.error(`❌ ${error instanceof Error ? error.message : error}`);process.exit(1);