Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

perf(ch): data-skipping index on events.profile_id (migration 18) (#445)

Ritik Jain4d98a598e09a
diff --git a/packages/db/code-migrations/18-events-profile-id-index.ts b/packages/db/code-migrations/18-events-profile-id-index.ts
new file mode 100644
index 000000000..ae311076c
--- /dev/null
+++ b/packages/db/code-migrations/18-events-profile-id-index.ts
@@ -0,0 +1,138 @@
+import {
+ chMigrationClient,
+ runClickhouseMigrationCommands,
+} from '../src/clickhouse/migration';
+import { getIsCluster } from './helpers';
+
+/**
+ * Data-skipping index on events.profile_id.
+ *
+ * The events table sorts by (project_id, toDate(created_at), created_at,
+ * name) and has no index on profile_id, so every profile-scoped query — the
+ * profile panel, per-profile event lists and counts, profile-filtered
+ * charts — scans the project's entire events slice to find one profile's
+ * rows. A bloom_filter index lets both `profile_id = X` and
+ * `profile_id IN (...)` prune to the few granules that can contain that
+ * profile; on our deployment (~1.5B events) profile pages went from
+ * tens-of-seconds full scans to sub-second reads.
+ *
+ * ADD INDEX is metadata-only and covers newly written parts; existing parts
+ * are backfilled by MATERIALIZE INDEX, which this migration submits by
+ * default. The mutation is asynchronous (the migration doesn't block on it)
+ * and idempotent, reads only the profile_id column, and writes small .idx
+ * files — unlike a projection it does not rewrite table data. Progress:
+ *
+ * SELECT * FROM system.mutations WHERE command LIKE '%idx_profile_id%';
+ *
+ * Deployments with very large events tables that want to control WHEN the
+ * backfill runs can apply both statements manually before upgrading
+ * (off-peak), targeting the same table this migration alters:
+ *
+ * Non-clustered:
+ * ALTER TABLE events ADD INDEX IF NOT EXISTS idx_profile_id profile_id TYPE bloom_filter(0.01) GRANULARITY 1;
+ * ALTER TABLE events MATERIALIZE INDEX idx_profile_id;
+ *
+ * Clustered:
+ * ... events_replicated ON CLUSTER '{cluster}' ...
+ *
+ * The migration skips re-materializing only when BOTH hold: the index
+ * already exists AND system.mutations records a completed MATERIALIZE for
+ * it. Index presence alone doesn't prove the backfill happened (a crashed
+ * earlier attempt may have run ADD without MATERIALIZE), and if mutation
+ * state can't be read the migration materializes anyway — the mutation is
+ * idempotent, so the failure direction is redundant work, never silently
+ * missing index files. Fresh installs have no existing parts; no-op.
+ */
+
+const INDEX_NAME = 'idx_profile_id';
+
+// Token-boundary match so a similarly named index (idx_profile_id_v2) can
+// never satisfy the check — LIKE's % and _ wildcards would.
+const NAME_BOUNDARY = '($|[^A-Za-z0-9_])';
+
+async function hasIndex(table: string): Promise<boolean> {
+ const res = await chMigrationClient.query({
+ query: `SHOW CREATE TABLE ${table}`,
+ format: 'JSONEachRow',
+ });
+ const [row] = await res.json<{ statement: string }>();
+ return new RegExp(`INDEX ${INDEX_NAME}([^A-Za-z0-9_]|$)`).test(
+ row?.statement ?? '',
+ );
+}
+
+async function hasCompletedMaterialization(
+ table: string,
+ isClustered: boolean,
+): Promise<boolean> {
+ const matchExpr = `match(command, 'MATERIALIZE INDEX ${INDEX_NAME}${NAME_BOUNDARY}') AND is_done = 1`;
+ try {
+ if (!isClustered) {
+ const res = await chMigrationClient.query({
+ query: `SELECT count() AS c FROM system.mutations
+ WHERE database = currentDatabase() AND table = '${table}'
+ AND ${matchExpr}`,
+ format: 'JSONEachRow',
+ });
+ const [row] = await res.json<{ c: string | number }>();
+ return Number(row?.c ?? 0) > 0;
+ }
+
+ // Clustered: a completed mutation on the connected host doesn't prove
+ // the other hosts finished (or even received) theirs — require every
+ // host in the cluster to report one.
+ // (hostName(), tcpPort()) rather than hostName() alone: multiple
+ // replicas on one machine share a hostname, and collapsing them could
+ // count an incomplete replica as done.
+ const res = await chMigrationClient.query({
+ query: `SELECT
+ (SELECT countDistinct((hostName(), tcpPort())) FROM clusterAllReplicas('{cluster}', system.one)) AS total,
+ countDistinct((hostName(), tcpPort())) AS done
+ FROM clusterAllReplicas('{cluster}', system.mutations)
+ WHERE database = currentDatabase() AND table = '${table}'
+ AND ${matchExpr}`,
+ format: 'JSONEachRow',
+ });
+ const [row] = await res.json<{ total: string | number; done: string | number }>();
+ return Number(row?.total ?? 0) > 0 && Number(row?.done ?? 0) >= Number(row?.total ?? 0);
+ } catch {
+ // Can't verify (e.g. no grant on system.mutations / cluster functions) —
+ // materialize; the mutation is idempotent and redundant work beats a
+ // silent skip.
+ return false;
+ }
+}
+
+export async function up() {
+ const isClustered = getIsCluster();
+ const table = isClustered ? 'events_replicated' : 'events';
+ const onCluster = isClustered ? " ON CLUSTER '{cluster}'" : '';
+
+ // Skip the backfill only when the index exists AND a completed
+ // MATERIALIZE mutation is on record (a manual pre-apply, see header).
+ // Index presence alone could be a crashed earlier attempt's ADD.
+ const backfilled =
+ (await hasIndex(table)) &&
+ (await hasCompletedMaterialization(table, isClustered));
+
+ const sqls = [
+ `ALTER TABLE ${table}${onCluster} ADD INDEX IF NOT EXISTS ${INDEX_NAME} profile_id TYPE bloom_filter(0.01) GRANULARITY 1`,
+ ];
+ if (!backfilled) {
+ sqls.push(
+ `ALTER TABLE ${table}${onCluster} MATERIALIZE INDEX ${INDEX_NAME}`,
+ );
+ }
+
+ await runClickhouseMigrationCommands(sqls);
+}
+
+export async function down() {
+ const isClustered = getIsCluster();
+ const table = isClustered ? 'events_replicated' : 'events';
+ const onCluster = isClustered ? " ON CLUSTER '{cluster}'" : '';
+
+ await runClickhouseMigrationCommands([
+ `ALTER TABLE ${table}${onCluster} DROP INDEX IF EXISTS ${INDEX_NAME}`,
+ ]);
+}