Skip to content

OpenPanel is an open-source web and product analytics platform, an open-source alternative to Mixpanel with optional self-hosting.

fix(ingest): same-millisecond sessions no longer corrupt each other (#573)

Carl-Gerhard Lindesvärd8d1f83c3cb8e
diff --git a/AGENTS.md b/AGENTS.md
index cd4c39a0e..1cd4ed807 100644
--- a/AGENTS.md
+++ b/AGENTS.md
@@ -177,7 +177,7 @@ ones. So write every change in two parts:
background, and it must:
- stay correct while ingestion and the dashboard read and write the table. If the
write path reads the table back (the profile buffer does), copy before you swap;
- otherwise swap first and copy after;
+ otherwise swap first and copy after (`28-sessions-sort-key-id.ts`);
- be safe to rerun from any point, since a process can die mid-way and another one
resumes it;
- never be something a blocking migration depends on.
diff --git a/packages/db/src/clickhouse/migration.ts b/packages/db/src/clickhouse/migration.ts
index 7fdf8929f..e8544cf65 100644
--- a/packages/db/src/clickhouse/migration.ts
+++ b/packages/db/src/clickhouse/migration.ts
@@ -194,6 +194,35 @@ export async function getExistingTables() {
}
}
+export interface TableKeys {
+ sortingKey: string[];
+ partitionKey: string;
+}
+
+/** Sort and partition key of every table in the migration database, by name. */
+export async function getTableKeys() {
+ const response = await chMigrationClient.query({
+ query:
+ 'SELECT name, sorting_key, partition_key FROM system.tables WHERE database = {database:String}',
+ query_params: { database: migrationDatabase() },
+ format: 'JSONEachRow',
+ });
+ const rows = await response.json<{
+ name: string;
+ sorting_key: string;
+ partition_key: string;
+ }>();
+ return new Map<string, TableKeys>(
+ rows.map((row) => [
+ row.name,
+ {
+ sortingKey: row.sorting_key.split(', '),
+ partitionKey: row.partition_key,
+ },
+ ])
+ );
+}
+
export function renameTable({
from,
to,
diff --git a/packages/db/src/code-migrations/28-sessions-sort-key-id.ts b/packages/db/src/code-migrations/28-sessions-sort-key-id.ts
new file mode 100644
index 000000000..58accd0b3
--- /dev/null
+++ b/packages/db/src/code-migrations/28-sessions-sort-key-id.ts
@@ -0,0 +1,259 @@
+import {
+ chMigrationClient,
+ createTable,
+ getTableKeys,
+ runClickhouseMigrationCommands,
+} from '../clickhouse/migration';
+import { type CodeMigrationEnv, getIsCluster, writeSqlDump } from './helpers';
+
+/**
+ * Rebuilds `sessions` with `id` at the end of the sort key.
+ *
+ * VersionedCollapsingMergeTree cancels a +1 against a −1 that shares the sort
+ * key and version. Without `id` in the key, two sessions of one project that
+ * start in the same millisecond are indistinguishable to a merge, so one
+ * session's −1 can cancel the other's +1 and both end up with wrong rows.
+ * ClickHouse cannot add an existing column to a sort key in place, hence the
+ * rebuild.
+ *
+ * Order of operations:
+ * 1. Create the new table.
+ * 2. Swap it in with EXCHANGE TABLES, which is atomic, so ingestion writes
+ * to it from then on and nothing written during the copy is lost, even
+ * with workers running. When clustered only the `_replicated` tables are
+ * swapped: the `sessions` Distributed table points at
+ * `sessions_replicated` by name.
+ * 3. Copy the old table in, newest day first, so recent dashboards fill in
+ * first. A −1 written after the swap still collapses against the copied
+ * +1 it cancels: collapsing does not depend on insert order.
+ *
+ * The copy keeps one row per session, the +1 at its highest version. That
+ * repairs sessions the old sort key already corrupted, and sessions that got
+ * a duplicated +1 from a retried insert. A session that lost its latest +1 to
+ * a collision comes back at its last surviving version. A session that lost
+ * every row stays missing; its events are untouched.
+ *
+ * Each day is one INSERT. A session's rows share `created_at`, so they never
+ * straddle a batch, and a day bounds the GROUP BY memory on the initiator.
+ *
+ * The old table stays as `sessions_<SUFFIX>`. Drop it once the copy is
+ * verified (Σ sign per session = 1):
+ * Non-clustered: DROP TABLE sessions_<SUFFIX>
+ * Clustered: DROP TABLE sessions_<SUFFIX> ON CLUSTER '{cluster}';
+ * DROP TABLE sessions_<SUFFIX>_replicated ON CLUSTER '{cluster}'
+ *
+ * Safe to rerun after a failure at any step: the swap steps are skipped once
+ * the live table has the new sort key, and a day's copy only inserts sessions
+ * whose Σ sign in the new table is still below 1. That sum survives merges, so
+ * a session already copied (1) is skipped, while one that is missing or only
+ * has a −1/+1 pair written after the swap (0) is copied.
+ *
+ * Swapping first is safe for sessions because nothing on the write path reads
+ * them back from ClickHouse; the session buffer keeps its state in Redis.
+ */
+
+const SUFFIX = '20261019';
+const NEW_TABLE = `sessions_new_${SUFFIX}`;
+const OLD_TABLE = `sessions_${SUFFIX}`;
+const DISTRIBUTION_HASH =
+ 'cityHash64(project_id, toString(toStartOfHour(created_at)))';
+
+const COLUMNS = [
+ '`id` String',
+ '`project_id` String CODEC(ZSTD(3))',
+ '`profile_id` String CODEC(ZSTD(3))',
+ '`device_id` String CODEC(ZSTD(3))',
+ '`groups` Array(String) DEFAULT [] CODEC(ZSTD(3))',
+ '`created_at` DateTime64(3) CODEC(DoubleDelta, ZSTD(3))',
+ '`ended_at` DateTime64(3) CODEC(DoubleDelta, ZSTD(3))',
+ '`is_bounce` Bool',
+ '`entry_origin` LowCardinality(String)',
+ '`entry_path` String CODEC(ZSTD(3))',
+ '`exit_origin` LowCardinality(String)',
+ '`exit_path` String CODEC(ZSTD(3))',
+ '`screen_view_count` Int32',
+ '`revenue` Float64',
+ '`event_count` Int32',
+ '`duration` UInt32',
+ '`country` LowCardinality(FixedString(2))',
+ '`region` LowCardinality(String)',
+ '`city` String',
+ '`longitude` Nullable(Float32) CODEC(Gorilla(4), LZ4)',
+ '`latitude` Nullable(Float32) CODEC(Gorilla(4), LZ4)',
+ '`device` LowCardinality(String)',
+ '`brand` LowCardinality(String)',
+ '`model` LowCardinality(String)',
+ '`browser` LowCardinality(String)',
+ '`browser_version` LowCardinality(String)',
+ '`os` LowCardinality(String)',
+ '`os_version` LowCardinality(String)',
+ '`utm_medium` String CODEC(ZSTD(3))',
+ '`utm_source` String CODEC(ZSTD(3))',
+ '`utm_campaign` String CODEC(ZSTD(3))',
+ '`utm_content` String CODEC(ZSTD(3))',
+ '`utm_term` String CODEC(ZSTD(3))',
+ '`referrer` String CODEC(ZSTD(3))',
+ '`referrer_name` String CODEC(ZSTD(3))',
+ '`referrer_type` LowCardinality(String)',
+ '`sign` Int8',
+ '`version` UInt64',
+];
+
+const COLUMN_NAMES = COLUMNS.map((column) => column.split('`')[1] ?? '');
+const GROUP_KEY = new Set(['project_id', 'id']);
+
+/**
+ * Aliases are prefixed so they do not shadow the source columns inside
+ * argMax(); INSERT … (columns) SELECT maps by position, not by name.
+ */
+function latestVersionSelect(column: string) {
+ if (GROUP_KEY.has(column)) {
+ return column;
+ }
+ if (column === 'sign') {
+ return '1 AS v_sign';
+ }
+ if (column === 'version') {
+ return 'max(version) AS v_version';
+ }
+ return `argMax(${column}, version) AS v_${column}`;
+}
+
+function dayRange(day: string) {
+ return `created_at >= toDateTime64('${day} 00:00:00', 3)
+ AND created_at < toDateTime64('${day} 00:00:00', 3) + INTERVAL 1 DAY`;
+}
+
+/**
+ * parallel_distributed_insert_select would run this whole statement on each
+ * shard, where the GLOBAL subquery's unqualified table does not exist; it has
+ * to run once, on the initiator, against the Distributed tables.
+ */
+function copyDaySql(source: string, day: string) {
+ return `INSERT INTO sessions (${COLUMN_NAMES.join(', ')})
+SELECT ${COLUMN_NAMES.map(latestVersionSelect).join(', ')}
+FROM ${source}
+WHERE sign = 1
+ AND ${dayRange(day)}
+ AND (project_id, id) GLOBAL NOT IN (
+ SELECT project_id, id FROM sessions
+ WHERE ${dayRange(day)}
+ GROUP BY project_id, id
+ HAVING sum(sign) >= 1
+ )
+GROUP BY project_id, id
+SETTINGS insert_distributed_sync = 1, parallel_distributed_insert_select = 0`;
+}
+
+const local = (table: string, isClustered: boolean) =>
+ isClustered ? `${table}_replicated` : table;
+
+const onCluster = (isClustered: boolean) =>
+ isClustered ? " ON CLUSTER '{cluster}'" : '';
+
+/**
+ * The statements still needed to put the new table in place, given what an
+ * earlier, interrupted run may already have done.
+ */
+async function swapSqls(isClustered: boolean) {
+ const tables = await getTableKeys();
+ const live = tables.get(local('sessions', isClustered));
+ const isSwapped = live?.sortingKey.includes('id') ?? false;
+ const isRenamePending =
+ isSwapped && tables.has(local(NEW_TABLE, isClustered));
+ // The old table is only dropped by hand once a finished copy is verified.
+ const isOldTableDropped =
+ isSwapped && !isRenamePending && !tables.has(local(OLD_TABLE, isClustered));
+ if (isOldTableDropped) {
+ return { sqls: [], isSwapped, isCopyDone: true };
+ }
+ const cluster = onCluster(isClustered);
+ const sqls: string[] = [];
+
+ if (!isSwapped) {
+ // Clustered, only the `_replicated` table is created: the existing
+ // `sessions` Distributed table routes to whatever is named
+ // sessions_replicated.
+ sqls.push(
+ ...createTable({
+ name: NEW_TABLE,
+ engine: 'VersionedCollapsingMergeTree(sign, version)',
+ columns: COLUMNS,
+ indices: ['INDEX idx_id id TYPE bloom_filter(0.01) GRANULARITY 1'],
+ orderBy: ['project_id', 'toDate(created_at)', 'created_at', 'id'],
+ partitionBy: 'toYYYYMM(created_at)',
+ settings: { index_granularity: 8192 },
+ distributionHash: DISTRIBUTION_HASH,
+ replicatedVersion: '1',
+ isClustered,
+ }).slice(0, 1),
+ `EXCHANGE TABLES ${local(NEW_TABLE, isClustered)} AND ${local('sessions', isClustered)}${cluster}`
+ );
+ }
+ if (!isSwapped || isRenamePending) {
+ sqls.push(
+ `RENAME TABLE ${local(NEW_TABLE, isClustered)} TO ${local(OLD_TABLE, isClustered)}${cluster}`
+ );
+ }
+ if (isClustered) {
+ sqls.push(`CREATE TABLE IF NOT EXISTS ${OLD_TABLE}${cluster} AS ${OLD_TABLE}_replicated
+ENGINE = Distributed('{cluster}', currentDatabase(), ${OLD_TABLE}_replicated, ${DISTRIBUTION_HASH})`);
+ }
+ return { sqls, isSwapped, isCopyDone: false };
+}
+
+/**
+ * The days that hold sessions, newest first. Taken from the data rather than
+ * a calendar range, so a stray old or future-dated `created_at` neither adds
+ * empty days nor falls outside the copy. The old table no longer changes
+ * after the swap, so this is exactly the set to copy.
+ */
+async function getDaysToCopy(source: string) {
+ const response = await chMigrationClient.query({
+ query: `SELECT DISTINCT toString(toDate(created_at)) AS day FROM ${source} WHERE created_at > '1970-01-01' ORDER BY day DESC`,
+ format: 'JSONEachRow',
+ });
+ const rows = await response.json<{ day: string }>();
+ return rows.map((row) => row.day);
+}
+
+/** Runs in the background once the api is up; see the rerun notes above. */
+export const online = true;
+
+export async function up(env: CodeMigrationEnv) {
+ const isClustered = getIsCluster(env);
+ const isDry = process.argv.includes('--dry');
+ const swap = await swapSqls(isClustered);
+ if (swap.isCopyDone) {
+ return;
+ }
+
+ if (!isDry) {
+ await runClickhouseMigrationCommands(swap.sqls);
+ }
+
+ // A dry run has not swapped anything, so the old rows are still in
+ // `sessions` unless an earlier run already moved them.
+ const source = isDry && !swap.isSwapped ? 'sessions' : OLD_TABLE;
+ const copySqls = (await getDaysToCopy(source)).map((day) =>
+ copyDaySql(OLD_TABLE, day)
+ );
+
+ // The swap already ran, so a failed dump must not stop the copy; in a dry
+ // run the dump is the whole output.
+ try {
+ writeSqlDump(import.meta.url, [...swap.sqls, ...copySqls]);
+ } catch (error) {
+ if (isDry) {
+ throw error;
+ }
+ console.warn(
+ 'Could not write the SQL dump; continuing with the copy',
+ error
+ );
+ }
+
+ if (!isDry) {
+ await runClickhouseMigrationCommands(copySqls);
+ }
+}
diff --git a/test/clickhouse-schema.sql b/test/clickhouse-schema.sql
index e3238c4f8..ecc48bc13 100644
--- a/test/clickhouse-schema.sql
+++ b/test/clickhouse-schema.sql
@@ -319,11 +319,12 @@ CREATE TABLE IF NOT EXISTS sessions
`referrer_name` String CODEC(ZSTD(3)),
`referrer_type` LowCardinality(String),
`sign` Int8,
- `version` UInt64
+ `version` UInt64,
+ INDEX idx_id id TYPE bloom_filter(0.01) GRANULARITY 1
)
ENGINE = VersionedCollapsingMergeTree(sign, version)
PARTITION BY toYYYYMM(created_at)
-ORDER BY (project_id, toDate(created_at), created_at)
+ORDER BY (project_id, toDate(created_at), created_at, id)
SETTINGS index_granularity = 8192
;