Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

perf(cohorts): re-key the cohort summary MVs for the criteria that read them (#458)

Ritik Jain247744a835fc
diff --git a/packages/db/code-migrations/20-cohort-summary-mv-sort-key.ts b/packages/db/code-migrations/20-cohort-summary-mv-sort-key.ts
new file mode 100644
index 000000000..786c6ffb4
--- /dev/null
+++ b/packages/db/code-migrations/20-cohort-summary-mv-sort-key.ts
@@ -0,0 +1,143 @@
+import fs from 'node:fs';
+import path from 'node:path';
+import { TABLE_NAMES } from '../src/clickhouse/client';
+import {
+ createMaterializedView,
+ getExistingTables,
+ runClickhouseMigrationCommands,
+} from '../src/clickhouse/migration';
+import { getIsCluster } from './helpers';
+
+/**
+ * Re-key the cohort summary MVs for the queries that actually read them.
+ *
+ * profile_event_summary_mv is ordered (project_id, profile_id, name,
+ * event_date), and the property variant similarly with property_key after
+ * name. Cohort criteria filter on name, a date range and optionally a
+ * property, then GROUP BY profile_id — they never filter profile_id. With
+ * profile_id second, the usable key prefix ends at project_id, so every
+ * criterion reads the project's entire slice of the MV however narrow the
+ * criterion is. Cohorts run one such query per criterion, on a schedule.
+ *
+ * Measured on a ~100M-row summary MV: a criterion that can only prune on
+ * project_id reads 103,985,861 rows (10.9s); the same criterion with name
+ * and event_date in the key prefix reads 115,141 rows (0.11s).
+ *
+ * A sort key cannot be altered in place, so this creates replacement MVs
+ * keyed for the consumer:
+ *
+ * event_profile_summary_mv
+ * (project_id, name, event_date, profile_id)
+ * event_property_profile_summary_mv
+ * (project_id, name, property_key, property_value, event_date, profile_id)
+ *
+ * The SELECT bodies, the identity filter and the aggregate columns are
+ * unchanged, so the new tables hold exactly the same rows as the old ones.
+ * Only the physical order differs.
+ *
+ * populate: false — these index events inserted after CREATE. History is
+ * filled by migration 21, which rebuilds them month by month from events.
+ *
+ * The old MVs are left in place and keep receiving inserts. Once the new
+ * ones are verified, dropping them is a one-line follow-up migration.
+ */
+export async function up() {
+ const replicatedVersion = '1';
+ const existingTables = await getExistingTables();
+ const isClustered = getIsCluster();
+ const sqls: string[] = [];
+
+ if (
+ !existingTables.includes(
+ `${TABLE_NAMES.event_profile_summary_mv}_distributed`,
+ ) &&
+ !existingTables.includes(TABLE_NAMES.event_profile_summary_mv)
+ ) {
+ sqls.push(
+ ...createMaterializedView({
+ name: TABLE_NAMES.event_profile_summary_mv,
+ tableName: 'events',
+ engine: 'AggregatingMergeTree()',
+ orderBy: ['project_id', 'name', 'event_date', 'profile_id'],
+ partitionBy: 'toYYYYMM(event_date)',
+ query: `SELECT
+ project_id,
+ profile_id,
+ name,
+ toStartOfDay(created_at) AS event_date,
+ countState() AS event_count,
+ minState(created_at) AS first_event_time,
+ maxState(created_at) AS last_event_time,
+ sumState(duration) AS total_duration
+ FROM {events}
+ WHERE profile_id != device_id
+ GROUP BY project_id, profile_id, name, event_date`,
+ distributionHash: 'cityHash64(project_id, profile_id)',
+ replicatedVersion,
+ isClustered,
+ populate: false,
+ }),
+ );
+ }
+
+ if (
+ !existingTables.includes(
+ `${TABLE_NAMES.event_property_profile_summary_mv}_distributed`,
+ ) &&
+ !existingTables.includes(TABLE_NAMES.event_property_profile_summary_mv)
+ ) {
+ sqls.push(
+ ...createMaterializedView({
+ name: TABLE_NAMES.event_property_profile_summary_mv,
+ tableName: 'events',
+ engine: 'AggregatingMergeTree()',
+ orderBy: [
+ 'project_id',
+ 'name',
+ 'property_key',
+ 'property_value',
+ 'event_date',
+ 'profile_id',
+ ],
+ partitionBy: 'toYYYYMM(event_date)',
+ query: `SELECT
+ project_id,
+ profile_id,
+ name,
+ property_key,
+ property_value,
+ toStartOfDay(created_at) AS event_date,
+ countState() AS event_count,
+ minState(created_at) AS first_event_time,
+ maxState(created_at) AS last_event_time
+ FROM {events}
+ ARRAY JOIN mapKeys(properties) AS property_key, mapValues(properties) AS property_value
+ WHERE profile_id != device_id
+ AND property_key != ''
+ AND property_value != ''
+ GROUP BY project_id, profile_id, name, property_key, property_value, event_date`,
+ distributionHash: 'cityHash64(project_id, profile_id)',
+ replicatedVersion,
+ isClustered,
+ populate: false,
+ }),
+ );
+ }
+
+ fs.writeFileSync(
+ path.join(import.meta.filename.replace('.ts', '.sql')),
+ sqls
+ .map((sql) => sql.trim().replace(/;$/, '').replace(/\n{2,}/g, '\n').concat(';'))
+ .join('\n\n---\n\n'),
+ );
+
+ if (process.argv.includes('--dry')) {
+ console.log('🔍 DRY RUN — CREATE statements:');
+ for (const sql of sqls) {
+ console.log(`\n${sql}\n`);
+ }
+ return;
+ }
+
+ await runClickhouseMigrationCommands(sqls);
+}
diff --git a/packages/db/code-migrations/21-backfill-cohort-summary-mvs.ts b/packages/db/code-migrations/21-backfill-cohort-summary-mvs.ts
new file mode 100644
index 000000000..0206fe8c7
--- /dev/null
+++ b/packages/db/code-migrations/21-backfill-cohort-summary-mvs.ts
@@ -0,0 +1,377 @@
+import { TABLE_NAMES } from '../src/clickhouse/client';
+import {
+ chMigrationClient,
+ runClickhouseMigrationCommands,
+} from '../src/clickhouse/migration';
+import { getIsCluster } from './helpers';
+
+/**
+ * Fill the re-keyed cohort summary MVs from migration 20 with history.
+ *
+ * - event_profile_summary_mv
+ * - event_property_profile_summary_mv
+ *
+ * Both are created with populate: false, so the live trigger only indexes
+ * events inserted after CREATE. Everything older has to be aggregated from
+ * the events table, which is what this does.
+ *
+ * REBUILD, NOT APPEND
+ * Each month's partition is dropped and rebuilt from events rather than
+ * appended to. These are AggregatingMergeTree tables, so an append that
+ * overlaps rows the trigger already wrote would double count them; a
+ * rebuild cannot, because the trigger's copies for that month are dropped
+ * first. That makes the work idempotent: an interrupted run is resumed by
+ * running it again, and a month can be redone at any time.
+ *
+ * It also removes the need to know when the MVs were created. Each month is
+ * bounded by a now64(3) read immediately after that month's DROP: rows older
+ * than the bound had their trigger-written copies dropped and are rebuilt
+ * here, rows newer are left to the trigger. Each event is counted once.
+ *
+ * The bound has to be per month, not per run. A run can take hours, and the
+ * current month is dropped at the end of it, so a bound captured at the
+ * start would delete every row the trigger wrote while the run was going and
+ * then decline to rebuild them. Taking it after each drop narrows the
+ * exposure to the round trip between the drop and the read, in which an
+ * arriving event can be counted twice. Pass --until to cut somewhere fixed
+ * instead, which is what a supervised re-run of a closed month wants.
+ *
+ * The tradeoff a rebuild makes is visibility: while a month is being
+ * rebuilt its partition is incomplete, so a cohort computed in that window
+ * under-counts that month. Nothing needs doing about it, the cohortRefresh
+ * cron recomputes every non-static cohort every 30 minutes. Months are
+ * rebuilt newest first so the ones cohorts actually read, which mostly use
+ * relative timeframes, are correct soonest.
+ *
+ * AUTOMATIC BY DEFAULT
+ * This runs as a normal migration so a new install ends up with working
+ * cohorts without anyone reading this file. Because the rebuild is
+ * idempotent, an evicted migration pod is safe: the migration is not
+ * recorded, and the next boot redoes it.
+ *
+ * MANUAL USE
+ * The same file is the supervised tool. Run it directly to control when
+ * the work happens, or to redo part of it:
+ *
+ * CLICKHOUSE_URL=... jiti packages/db/code-migrations/21-backfill-cohort-summary-mvs.ts
+ *
+ * Flags:
+ * --dry Print the plan and the first batch; run nothing.
+ * --from=YYYYMM First month (default: month of min(created_at)).
+ * --to=YYYYMM Last month (default: current month).
+ * --until=DATETIME Fixed upper bound on created_at for every month
+ * (default: now64(3) taken after each month's drop).
+ * --batch-days=N Days per INSERT within a month (default 2).
+ * --parallel=N Concurrent batches (default 2).
+ * --only=summary|property Rebuild just one of the two tables.
+ *
+ * CLUSTERS
+ * Reads and writes go through the Distributed tables, so one run from one
+ * node covers every shard. The one statement that cannot be distributed is
+ * DROP PARTITION, which the Distributed engine rejects outright, so that
+ * goes ON CLUSTER against the local table instead. Still a single run.
+ */
+
+const DEFAULT_BATCH_DAYS = 2;
+const DEFAULT_PARALLEL = 2;
+
+// Spill the per-batch GROUP BY rather than OOM on the ARRAY JOIN fan-out;
+// max_insert_threads parallelises the part-building stage, which is
+// otherwise single-threaded and leaves cores idle during a rebuild.
+// distributed_foreground_insert so a finished batch means the rows have
+// actually landed on their shards, not that they were queued for delivery.
+const INSERT_SETTINGS = `SETTINGS
+ max_bytes_before_external_group_by = 4294967296,
+ max_insert_threads = 8,
+ distributed_foreground_insert = 1`;
+
+type Batch = { label: string; sql: string };
+
+function getArg(name: string): string | undefined {
+ const prefix = `--${name}=`;
+ return process.argv.find((a) => a.startsWith(prefix))?.slice(prefix.length);
+}
+
+function getPositiveInt(value: string | undefined, fallback: number): number {
+ if (value === undefined) {
+ return fallback;
+ }
+ if (!/^\d+$/.test(value)) {
+ throw new Error(`expected a positive integer, got "${value}"`);
+ }
+ const parsed = Number.parseInt(value, 10);
+ if (!Number.isSafeInteger(parsed) || parsed < 1) {
+ throw new Error(`expected a positive integer, got "${value}"`);
+ }
+ return parsed;
+}
+
+// DROP PARTITION is the one statement that cannot go through the Distributed
+// table: the engine rejects partitioning outright. ON CLUSTER against the
+// local table reaches every shard in a single statement instead.
+function dropPartition(
+ table: string,
+ month: number,
+ isClustered: boolean,
+): string {
+ return isClustered
+ ? `ALTER TABLE ${table}_replicated ON CLUSTER '{cluster}' DROP PARTITION '${month}'`
+ : `ALTER TABLE ${table} DROP PARTITION '${month}'`;
+}
+
+function monthStart(yyyymm: number): string {
+ const y = Math.floor(yyyymm / 100);
+ const m = yyyymm % 100;
+ return `${y}-${String(m).padStart(2, '0')}-01 00:00:00`;
+}
+
+function nextMonth(yyyymm: number): number {
+ const y = Math.floor(yyyymm / 100);
+ const m = yyyymm % 100;
+ return m === 12 ? (y + 1) * 100 + 1 : yyyymm + 1;
+}
+
+// Column projections and the identity filter are byte-identical to the MV
+// definitions in migration 20; only the time bounds differ.
+function summarySelect(start: string, end: string): string {
+ return `SELECT
+ project_id,
+ profile_id,
+ name,
+ toStartOfDay(created_at) AS event_date,
+ countState() AS event_count,
+ minState(created_at) AS first_event_time,
+ maxState(created_at) AS last_event_time,
+ sumState(duration) AS total_duration
+FROM ${TABLE_NAMES.events}
+WHERE created_at >= toDateTime64('${start}', 3)
+ AND created_at < toDateTime64('${end}', 3)
+ AND profile_id != device_id
+GROUP BY project_id, profile_id, name, event_date`;
+}
+
+function propertySelect(start: string, end: string): string {
+ return `SELECT
+ project_id,
+ profile_id,
+ name,
+ property_key,
+ property_value,
+ toStartOfDay(created_at) AS event_date,
+ countState() AS event_count,
+ minState(created_at) AS first_event_time,
+ maxState(created_at) AS last_event_time
+FROM ${TABLE_NAMES.events}
+ARRAY JOIN mapKeys(properties) AS property_key, mapValues(properties) AS property_value
+WHERE created_at >= toDateTime64('${start}', 3)
+ AND created_at < toDateTime64('${end}', 3)
+ AND profile_id != device_id
+ AND property_key != ''
+ AND property_value != ''
+GROUP BY project_id, profile_id, name, property_key, property_value, event_date`;
+}
+
+function monthBatches(
+ month: number,
+ until: string,
+ batchDays: number,
+ target: string,
+ select: (start: string, end: string) => string,
+): Batch[] {
+ const batches: Batch[] = [];
+ const start = new Date(`${monthStart(month).replace(' ', 'T')}Z`);
+ const monthEnd = new Date(
+ `${monthStart(nextMonth(month)).replace(' ', 'T')}Z`,
+ );
+ const untilDate = new Date(`${until.replace(' ', 'T')}Z`);
+ const end = monthEnd < untilDate ? monthEnd : untilDate;
+
+ let cursor = new Date(start);
+ while (cursor < end) {
+ const next = new Date(cursor);
+ next.setUTCDate(next.getUTCDate() + batchDays);
+ const batchEnd = next > end ? end : next;
+ // Keep milliseconds: created_at is DateTime64(3) and the final batch end
+ // is the month's bound, so truncating to whole seconds would drop every
+ // event in the boundary second after its trigger row had been dropped.
+ const s = cursor.toISOString().slice(0, 23).replace('T', ' ');
+ const e = batchEnd.toISOString().slice(0, 23).replace('T', ' ');
+ batches.push({
+ label: `${s} -> ${e}`,
+ sql: `INSERT INTO ${target}\n${select(s, e)}\n${INSERT_SETTINGS}`,
+ });
+ cursor = batchEnd;
+ }
+ return batches;
+}
+
+// N workers drain the batch list in order. Batches are independent time
+// ranges within an already-dropped month, so ordering does not matter.
+async function runPool(
+ batches: Batch[],
+ parallel: number,
+ onDone: (batch: Batch, seconds: number) => void,
+): Promise<void> {
+ let next = 0;
+ await Promise.all(
+ Array.from(
+ { length: Math.max(1, Math.min(parallel, batches.length)) },
+ async () => {
+ while (next < batches.length) {
+ const batch = batches[next++]!;
+ const startedAt = Date.now();
+ await chMigrationClient.command({ query: batch.sql });
+ onDone(batch, Math.round((Date.now() - startedAt) / 1000));
+ }
+ },
+ ),
+ );
+}
+
+async function scalar(query: string): Promise<string | undefined> {
+ const res = await chMigrationClient.query({ query, format: 'JSONEachRow' });
+ const rows = await res.json<Record<string, string>>();
+ return rows[0] ? Object.values(rows[0])[0] : undefined;
+}
+
+export async function up() {
+ const isClustered = getIsCluster();
+ const isDry = process.argv.includes('--dry');
+ const only = getArg('only');
+ const batchDays = getPositiveInt(getArg('batch-days'), DEFAULT_BATCH_DAYS);
+ const parallel = getPositiveInt(getArg('parallel'), DEFAULT_PARALLEL);
+ const totalEvents = Number(
+ (await scalar(`SELECT count() AS c FROM ${TABLE_NAMES.events}`)) ?? 0,
+ );
+ if (totalEvents === 0) {
+ console.log('📦 Cohort summary MVs: no events to aggregate, nothing to do');
+ return;
+ }
+ // A fixed --until applies to every month. Without one, each month takes its
+ // own bound straight after its drop (see the header) so a long run cannot
+ // lose the rows the trigger wrote while it was running. Read from
+ // ClickHouse rather than the local clock to stay in created_at's clock
+ // domain.
+ const fixedUntil = getArg('until');
+ const planningUntil =
+ fixedUntil ?? (await scalar('SELECT toString(now64(3)) AS t'));
+ if (!planningUntil) {
+ throw new Error('could not resolve the rebuild upper bound');
+ }
+
+ const currentMonth = Number(
+ await scalar(
+ `SELECT toString(toYYYYMM(toDateTime64('${planningUntil}', 3))) AS m`,
+ ),
+ );
+ const fromMonth = getPositiveInt(
+ getArg('from') ??
+ (await scalar(
+ `SELECT toString(toYYYYMM(min(created_at))) AS m FROM ${TABLE_NAMES.events}`,
+ )),
+ currentMonth,
+ );
+ const toMonth = getPositiveInt(getArg('to'), currentMonth);
+
+ const targets: Array<{
+ label: string;
+ table: string;
+ select: (start: string, end: string) => string;
+ }> = [];
+ if (only !== 'property') {
+ targets.push({
+ label: TABLE_NAMES.event_profile_summary_mv,
+ table: TABLE_NAMES.event_profile_summary_mv,
+ select: summarySelect,
+ });
+ }
+ if (only !== 'summary') {
+ targets.push({
+ label: TABLE_NAMES.event_property_profile_summary_mv,
+ table: TABLE_NAMES.event_property_profile_summary_mv,
+ select: propertySelect,
+ });
+ }
+
+ // Newest first: cohort criteria mostly use relative timeframes, so the
+ // recent months are the ones that make cohorts correct again. It also means
+ // an interrupted run leaves the useful end done.
+ const months: number[] = [];
+ for (let m = fromMonth; m <= toMonth; m = nextMonth(m)) {
+ months.push(m);
+ }
+ months.reverse();
+
+ console.log('');
+ console.log('📦 Cohort summary MV rebuild');
+ console.log(` Events: ${totalEvents.toLocaleString()}`);
+ console.log(` Months: ${fromMonth} -> ${toMonth} (${months.length})`);
+ console.log(` Until: ${fixedUntil ?? 'per month, after each drop'}`);
+ console.log(` Batch days: ${batchDays} Parallel: ${parallel}`);
+ console.log(` Targets: ${targets.map((t) => t.label).join(', ')}`);
+ console.log(` Clustered: ${isClustered}`);
+ console.log(` Mode: ${isDry ? 'DRY RUN' : 'EXECUTE'}`);
+
+ if (isDry) {
+ for (const target of targets) {
+ const sample = monthBatches(
+ months[0]!,
+ planningUntil,
+ batchDays,
+ target.table,
+ target.select,
+ );
+ console.log(
+ `\n-- ${target.label}: ${sample.length} batches for ${months[0]} --`,
+ );
+ console.log(dropPartition(target.table, months[0]!, isClustered));
+ console.log(sample[0]?.sql);
+ }
+ return;
+ }
+
+ const startedAt = Date.now();
+ for (const target of targets) {
+ console.log(`\n🚀 ${target.label}`);
+ for (const month of months) {
+ // Drop first, then take the bound, so anything the trigger writes from
+ // here on is kept by the trigger and excluded from the rebuild.
+ await runClickhouseMigrationCommands([
+ dropPartition(target.table, month, isClustered),
+ ]);
+ const until =
+ fixedUntil ?? (await scalar('SELECT toString(now64(3)) AS t'));
+ if (!until) {
+ throw new Error(`${month}: could not resolve the rebuild upper bound`);
+ }
+ const batches = monthBatches(
+ month,
+ until,
+ batchDays,
+ target.table,
+ target.select,
+ );
+ if (batches.length === 0) {
+ continue;
+ }
+ const monthStartedAt = Date.now();
+ await runPool(batches, parallel, (batch, seconds) => {
+ console.log(` · ${batch.label} (${seconds}s)`);
+ });
+ console.log(
+ ` ✅ ${month} in ${Math.round((Date.now() - monthStartedAt) / 1000)}s (${batches.length} batches, elapsed=${Math.round((Date.now() - startedAt) / 1000)}s)`,
+ );
+ }
+ }
+ console.log('\n✅ Rebuild complete.');
+}
+
+// Allow direct execution for the supervised path.
+if (import.meta.url === `file://${process.argv[1]}`) {
+ up()
+ .then(() => process.exit(0))
+ .catch((err) => {
+ console.error(err);
+ process.exit(1);
+ });
+}
diff --git a/packages/db/src/clickhouse/client.ts b/packages/db/src/clickhouse/client.ts
index fc4941af8..d78a5220c 100644
--- a/packages/db/src/clickhouse/client.ts
+++ b/packages/db/src/clickhouse/client.ts
@@ -67,6 +67,11 @@ export const TABLE_NAMES = {
cohort_members: 'cohort_members',
cohort_metadata: 'cohort_metadata',
profile_event_summary_mv: 'profile_event_summary_mv',
+ // Same content as the two MVs above, keyed for the cohort criteria that
+ // read them (event + window first, profile last) rather than by profile.
+ // See migration 20.
+ event_profile_summary_mv: 'event_profile_summary_mv',
+ event_property_profile_summary_mv: 'event_property_profile_summary_mv',
profile_event_property_summary_mv: 'profile_event_property_summary_mv',
};
diff --git a/packages/db/src/services/cohort.service.ts b/packages/db/src/services/cohort.service.ts
index 8a20fe06d..dacf352ad 100644
--- a/packages/db/src/services/cohort.service.ts
+++ b/packages/db/src/services/cohort.service.ts
@@ -192,7 +192,7 @@ export function buildEventCriteriaQuery(
const frequencyOp = getFrequencyOperator(frequency);
return `
SELECT profile_id
- FROM ${TABLE_NAMES.profile_event_property_summary_mv}
+ FROM ${TABLE_NAMES.event_property_profile_summary_mv}
WHERE project_id = ${sqlstring.escape(projectId)}
AND name = ${sqlstring.escape(name)}
AND ${timeConstraint.replace('created_at', 'event_date')}
@@ -204,7 +204,7 @@ export function buildEventCriteriaQuery(
return `
SELECT DISTINCT profile_id
- FROM ${TABLE_NAMES.profile_event_property_summary_mv}
+ FROM ${TABLE_NAMES.event_property_profile_summary_mv}
WHERE project_id = ${sqlstring.escape(projectId)}
AND name = ${sqlstring.escape(name)}
AND ${timeConstraint.replace('created_at', 'event_date')}
@@ -216,7 +216,7 @@ export function buildEventCriteriaQuery(
const frequencyOp = getFrequencyOperator(frequency);
return `
SELECT profile_id
- FROM ${TABLE_NAMES.profile_event_summary_mv}
+ FROM ${TABLE_NAMES.event_profile_summary_mv}
WHERE project_id = ${sqlstring.escape(projectId)}
AND name = ${sqlstring.escape(name)}
AND ${timeConstraint.replace('created_at', 'event_date')}
@@ -227,7 +227,7 @@ export function buildEventCriteriaQuery(
return `
SELECT DISTINCT profile_id
- FROM ${TABLE_NAMES.profile_event_summary_mv}
+ FROM ${TABLE_NAMES.event_profile_summary_mv}
WHERE project_id = ${sqlstring.escape(projectId)}
AND name = ${sqlstring.escape(name)}
AND ${timeConstraint.replace('created_at', 'event_date')}
diff --git a/packages/db/src/services/delete.service.ts b/packages/db/src/services/delete.service.ts
index b8466f333..7dace2640 100644
--- a/packages/db/src/services/delete.service.ts
+++ b/packages/db/src/services/delete.service.ts
@@ -51,6 +51,8 @@ export async function deleteFromClickhouse(projectIds: string[]) {
TABLE_NAMES.cohort_metadata,
TABLE_NAMES.profile_event_summary_mv,
TABLE_NAMES.profile_event_property_summary_mv,
+ TABLE_NAMES.event_profile_summary_mv,
+ TABLE_NAMES.event_property_profile_summary_mv,
];
for (const table of tables) {