Skip to content

πŸ“ˆ Open source, privacy-first web analytics β€” a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

Clickhouse - idemponent queries

Blue Mouse25d8c40a86c6
diff --git a/backend/migrations/clickhouse/2026_08_01_session_id.js b/backend/migrations/clickhouse/2026_08_01_session_id.js
index 52385a288..5d9b91645 100644
--- a/backend/migrations/clickhouse/2026_08_01_session_id.js
+++ b/backend/migrations/clickhouse/2026_08_01_session_id.js
@@ -1,10 +1,15 @@
-const { queriesRunner, dbName } = require('./setup')
+const { clickhouse, queriesRunner, dbName } = require('./setup')
-const queries = [
+const OLD_SESSIONS_TABLE = 'sessions_by_visitor_day'
+const STATE_TABLE = 'sessions_sid_migration_state'
+
+const ADD_SID_COLUMN_QUERIES = [
`ALTER TABLE ${dbName}.events ADD COLUMN IF NOT EXISTS sid Nullable(UInt64) AFTER psid`,
`ALTER TABLE ${dbName}.customEV ADD COLUMN IF NOT EXISTS sid Nullable(UInt64) AFTER psid`,
`ALTER TABLE ${dbName}.errors ADD COLUMN IF NOT EXISTS sid Nullable(UInt64) AFTER psid`,
+]
+const BACKFILL_QUERIES = [
`CREATE TABLE IF NOT EXISTS ${dbName}.sessions_v2
(
sid UInt64 DEFAULT psid,
@@ -24,20 +29,140 @@ const queries = [
WHERE firstSeen <= lastSeen
AND lastSeen > toDateTime(0)`,
- `RENAME TABLE ${dbName}.sessions TO ${dbName}.sessions_by_visitor_day, ${dbName}.sessions_v2 TO ${dbName}.sessions`,
+ `RENAME TABLE ${dbName}.sessions TO ${dbName}.${OLD_SESSIONS_TABLE}, ${dbName}.sessions_v2 TO ${dbName}.sessions`,
+]
+
+const tableExists = async (table) => {
+ const resultSet = await clickhouse.query({
+ query: `SELECT count() AS total
+ FROM system.tables
+ WHERE database = {database:String} AND name = {table:String}`,
+ query_params: { database: dbName, table },
+ format: 'JSONEachRow',
+ })
+ const [row] = await resultSet.json()
+
+ return Number(row.total) > 0
+}
+
+const readMigrationStart = async () => {
+ if (!(await tableExists(STATE_TABLE))) {
+ return null
+ }
+
+ const resultSet = await clickhouse.query({
+ // The aliases must not reuse the startedAt column name β€” ClickHouse would
+ // resolve the later references to the alias instead of the column.
+ query: `SELECT
+ count() AS total,
+ toUnixTimestamp(min(startedAt)) AS startedAtTs,
+ toString(min(startedAt)) AS startedAtText
+ FROM ${dbName}.${STATE_TABLE}`,
+ format: 'JSONEachRow',
+ })
+ const [row] = await resultSet.json()
+
+ return Number(row.total) > 0 ? row : null
+}
+
+// The replay at the end has to cover every row written to the old table from
+// the moment the backfill started, so the watermark is taken before it β€” and
+// persisted, because a retry that resumes after the rename must reuse the
+// original timestamp instead of a fresh now().
+const captureMigrationStart = async () => {
+ // An attempt that died before the rename already recorded one β€” keep it, the
+ // backfill it started may still have missed rows written after that point.
+ const existing = await readMigrationStart()
+
+ if (existing) {
+ return existing
+ }
+
+ await queriesRunner([
+ `CREATE TABLE IF NOT EXISTS ${dbName}.${STATE_TABLE}
+ (
+ startedAt DateTime('UTC')
+ )
+ ENGINE = MergeTree()
+ ORDER BY startedAt;`,
+
+ `INSERT INTO ${dbName}.${STATE_TABLE} (startedAt) SELECT now()`,
+ ])
+
+ return readMigrationStart()
+}
- // The backfill above runs while the old code is still writing to the old
- // table, so anything recorded between the INSERT and the RENAME would be
- // lost. Replay the tail of the old table into the new one β€” ReplacingMergeTree
- // dedupes on (pid, sid) and keeps the row with the greatest lastSeen. Only the
- // last day is replayed: sessions expire after 30 minutes of inactivity, so
- // nothing older than that can have changed during the migration.
- `INSERT INTO ${dbName}.sessions (sid, psid, pid, profileId, firstSeen, lastSeen)
+// ReplacingMergeTree dedupes on (pid, sid) and keeps the row with the greatest
+// lastSeen, so replaying rows the backfill already copied is harmless β€” the
+// watermark only exists to keep the amount of replayed data sane.
+const buildReplayQuery = (migrationStart) => {
+ const conditions = [
+ // lastSeen is stamped on the API host and inserted asynchronously, so a row
+ // can land after the timestamp it carries. The extra hour absorbs that lag
+ // and any clock skew between the API hosts and ClickHouse.
+ migrationStart &&
+ `lastSeen >= toDateTime({startedAt:UInt32}) - INTERVAL 1 HOUR`,
+ 'firstSeen <= lastSeen',
+ 'lastSeen > toDateTime(0)',
+ ].filter(Boolean)
+
+ return `INSERT INTO ${dbName}.sessions (sid, psid, pid, profileId, firstSeen, lastSeen)
SELECT psid AS sid, psid, pid, profileId, firstSeen, lastSeen
- FROM ${dbName}.sessions_by_visitor_day
- WHERE lastSeen >= now() - INTERVAL 1 DAY
- AND firstSeen <= lastSeen
- AND lastSeen > toDateTime(0)`,
-]
+ FROM ${dbName}.${OLD_SESSIONS_TABLE}
+ WHERE ${conditions.join('\n AND ')}`
+}
+
+const run = async () => {
+ // The rename is the point of no return: once it lands, sessions is the new
+ // table and the old one is frozen. Re-running the backfill from there would
+ // fail on the existing rename target and copy the new table back over itself,
+ // so a resumed attempt only replays the tail.
+ const renamed = await tableExists(OLD_SESSIONS_TABLE)
+
+ await queriesRunner(ADD_SID_COLUMN_QUERIES)
+
+ let migrationStart
+
+ if (renamed) {
+ if (!(await tableExists('sessions'))) {
+ throw new Error(
+ `${dbName}.${OLD_SESSIONS_TABLE} exists but ${dbName}.sessions does not β€” the rename left the database half-migrated. Restore ${dbName}.sessions manually before rerunning.`,
+ )
+ }
+
+ if (await tableExists('sessions_v2')) {
+ console.warn(
+ `${dbName}.sessions_v2 is left over from an earlier attempt and is not used any more β€” drop it once this migration is verified.`,
+ )
+ }
+
+ migrationStart = await readMigrationStart()
+
+ if (migrationStart) {
+ console.log(
+ `Resuming after the rename, replaying ${dbName}.${OLD_SESSIONS_TABLE} from ${migrationStart.startedAtText} UTC.`,
+ )
+ } else {
+ console.warn(
+ `Resuming after the rename, but no ${dbName}.${STATE_TABLE} watermark was recorded. Replaying ${dbName}.${OLD_SESSIONS_TABLE} in full β€” slower, but the overlap is deduped.`,
+ )
+ }
+ } else {
+ migrationStart = await captureMigrationStart()
+ await queriesRunner(BACKFILL_QUERIES)
+ }
+
+ // The backfill runs while the old code is still writing to the old table, so
+ // anything recorded between the INSERT and the RENAME would be lost. Replay
+ // everything the old table received from the watermark onwards.
+ await queriesRunner(
+ [buildReplayQuery(migrationStart)],
+ true,
+ migrationStart
+ ? { query_params: { startedAt: migrationStart.startedAtTs } }
+ : {},
+ )
+}
-queriesRunner(queries)
+// Drop ${dbName}.sessions_sid_migration_state once the migration is verified.
+run()