OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.
perf(ch): data-skipping index on events.profile_id (migration 18) (#445)
Ritik Jain4d98a598e09a
diff --git a/packages/db/code-migrations/18-events-profile-id-index.ts b/packages/db/code-migrations/18-events-profile-id-index.tsnew file mode 100644index 000000000..ae311076c--- /dev/null+++ b/packages/db/code-migrations/18-events-profile-id-index.ts@@ -0,0 +1,138 @@+import {+ chMigrationClient,+ runClickhouseMigrationCommands,+} from '../src/clickhouse/migration';+import { getIsCluster } from './helpers';++/**+ * Data-skipping index on events.profile_id.+ *+ * The events table sorts by (project_id, toDate(created_at), created_at,+ * name) and has no index on profile_id, so every profile-scoped query — the+ * profile panel, per-profile event lists and counts, profile-filtered+ * charts — scans the project's entire events slice to find one profile's+ * rows. A bloom_filter index lets both `profile_id = X` and+ * `profile_id IN (...)` prune to the few granules that can contain that+ * profile; on our deployment (~1.5B events) profile pages went from+ * tens-of-seconds full scans to sub-second reads.+ *+ * ADD INDEX is metadata-only and covers newly written parts; existing parts+ * are backfilled by MATERIALIZE INDEX, which this migration submits by+ * default. The mutation is asynchronous (the migration doesn't block on it)+ * and idempotent, reads only the profile_id column, and writes small .idx+ * files — unlike a projection it does not rewrite table data. Progress:+ *+ * SELECT * FROM system.mutations WHERE command LIKE '%idx_profile_id%';+ *+ * Deployments with very large events tables that want to control WHEN the+ * backfill runs can apply both statements manually before upgrading+ * (off-peak), targeting the same table this migration alters:+ *+ * Non-clustered:+ * ALTER TABLE events ADD INDEX IF NOT EXISTS idx_profile_id profile_id TYPE bloom_filter(0.01) GRANULARITY 1;+ * ALTER TABLE events MATERIALIZE INDEX idx_profile_id;+ *+ * Clustered:+ * ... events_replicated ON CLUSTER '{cluster}' ...+ *+ * The migration skips re-materializing only when BOTH hold: the index+ * already exists AND system.mutations records a completed MATERIALIZE for+ * it. Index presence alone doesn't prove the backfill happened (a crashed+ * earlier attempt may have run ADD without MATERIALIZE), and if mutation+ * state can't be read the migration materializes anyway — the mutation is+ * idempotent, so the failure direction is redundant work, never silently+ * missing index files. Fresh installs have no existing parts; no-op.+ */++const INDEX_NAME = 'idx_profile_id';++// Token-boundary match so a similarly named index (idx_profile_id_v2) can+// never satisfy the check — LIKE's % and _ wildcards would.+const NAME_BOUNDARY = '($|[^A-Za-z0-9_])';++async function hasIndex(table: string): Promise<boolean> {+ const res = await chMigrationClient.query({+ query: `SHOW CREATE TABLE ${table}`,+ format: 'JSONEachRow',+ });+ const [row] = await res.json<{ statement: string }>();+ return new RegExp(`INDEX ${INDEX_NAME}([^A-Za-z0-9_]|$)`).test(+ row?.statement ?? '',+ );+}++async function hasCompletedMaterialization(+ table: string,+ isClustered: boolean,+): Promise<boolean> {+ const matchExpr = `match(command, 'MATERIALIZE INDEX ${INDEX_NAME}${NAME_BOUNDARY}') AND is_done = 1`;+ try {+ if (!isClustered) {+ const res = await chMigrationClient.query({+ query: `SELECT count() AS c FROM system.mutations+ WHERE database = currentDatabase() AND table = '${table}'+ AND ${matchExpr}`,+ format: 'JSONEachRow',+ });+ const [row] = await res.json<{ c: string | number }>();+ return Number(row?.c ?? 0) > 0;+ }++ // Clustered: a completed mutation on the connected host doesn't prove+ // the other hosts finished (or even received) theirs — require every+ // host in the cluster to report one.+ // (hostName(), tcpPort()) rather than hostName() alone: multiple+ // replicas on one machine share a hostname, and collapsing them could+ // count an incomplete replica as done.+ const res = await chMigrationClient.query({+ query: `SELECT+ (SELECT countDistinct((hostName(), tcpPort())) FROM clusterAllReplicas('{cluster}', system.one)) AS total,+ countDistinct((hostName(), tcpPort())) AS done+ FROM clusterAllReplicas('{cluster}', system.mutations)+ WHERE database = currentDatabase() AND table = '${table}'+ AND ${matchExpr}`,+ format: 'JSONEachRow',+ });+ const [row] = await res.json<{ total: string | number; done: string | number }>();+ return Number(row?.total ?? 0) > 0 && Number(row?.done ?? 0) >= Number(row?.total ?? 0);+ } catch {+ // Can't verify (e.g. no grant on system.mutations / cluster functions) —+ // materialize; the mutation is idempotent and redundant work beats a+ // silent skip.+ return false;+ }+}++export async function up() {+ const isClustered = getIsCluster();+ const table = isClustered ? 'events_replicated' : 'events';+ const onCluster = isClustered ? " ON CLUSTER '{cluster}'" : '';++ // Skip the backfill only when the index exists AND a completed+ // MATERIALIZE mutation is on record (a manual pre-apply, see header).+ // Index presence alone could be a crashed earlier attempt's ADD.+ const backfilled =+ (await hasIndex(table)) &&+ (await hasCompletedMaterialization(table, isClustered));++ const sqls = [+ `ALTER TABLE ${table}${onCluster} ADD INDEX IF NOT EXISTS ${INDEX_NAME} profile_id TYPE bloom_filter(0.01) GRANULARITY 1`,+ ];+ if (!backfilled) {+ sqls.push(+ `ALTER TABLE ${table}${onCluster} MATERIALIZE INDEX ${INDEX_NAME}`,+ );+ }++ await runClickhouseMigrationCommands(sqls);+}++export async function down() {+ const isClustered = getIsCluster();+ const table = isClustered ? 'events_replicated' : 'events';+ const onCluster = isClustered ? " ON CLUSTER '{cluster}'" : '';++ await runClickhouseMigrationCommands([+ `ALTER TABLE ${table}${onCluster} DROP INDEX IF EXISTS ${INDEX_NAME}`,+ ]);+}