π Open source, privacy-first web analytics β a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
Clickhouse - idemponent queries
Blue Mouse25d8c40a86c6
diff --git a/backend/migrations/clickhouse/2026_08_01_session_id.js b/backend/migrations/clickhouse/2026_08_01_session_id.jsindex 52385a288..5d9b91645 100644--- a/backend/migrations/clickhouse/2026_08_01_session_id.js+++ b/backend/migrations/clickhouse/2026_08_01_session_id.js@@ -1,10 +1,15 @@-const { queriesRunner, dbName } = require('./setup')+const { clickhouse, queriesRunner, dbName } = require('./setup')-const queries = [+const OLD_SESSIONS_TABLE = 'sessions_by_visitor_day'+const STATE_TABLE = 'sessions_sid_migration_state'++const ADD_SID_COLUMN_QUERIES = [`ALTER TABLE ${dbName}.events ADD COLUMN IF NOT EXISTS sid Nullable(UInt64) AFTER psid`,`ALTER TABLE ${dbName}.customEV ADD COLUMN IF NOT EXISTS sid Nullable(UInt64) AFTER psid`,`ALTER TABLE ${dbName}.errors ADD COLUMN IF NOT EXISTS sid Nullable(UInt64) AFTER psid`,+]+const BACKFILL_QUERIES = [`CREATE TABLE IF NOT EXISTS ${dbName}.sessions_v2(sid UInt64 DEFAULT psid,@@ -24,20 +29,140 @@ const queries = [WHERE firstSeen <= lastSeenAND lastSeen > toDateTime(0)`,- `RENAME TABLE ${dbName}.sessions TO ${dbName}.sessions_by_visitor_day, ${dbName}.sessions_v2 TO ${dbName}.sessions`,+ `RENAME TABLE ${dbName}.sessions TO ${dbName}.${OLD_SESSIONS_TABLE}, ${dbName}.sessions_v2 TO ${dbName}.sessions`,+]++const tableExists = async (table) => {+ const resultSet = await clickhouse.query({+ query: `SELECT count() AS total+ FROM system.tables+ WHERE database = {database:String} AND name = {table:String}`,+ query_params: { database: dbName, table },+ format: 'JSONEachRow',+ })+ const [row] = await resultSet.json()++ return Number(row.total) > 0+}++const readMigrationStart = async () => {+ if (!(await tableExists(STATE_TABLE))) {+ return null+ }++ const resultSet = await clickhouse.query({+ // The aliases must not reuse the startedAt column name β ClickHouse would+ // resolve the later references to the alias instead of the column.+ query: `SELECT+ count() AS total,+ toUnixTimestamp(min(startedAt)) AS startedAtTs,+ toString(min(startedAt)) AS startedAtText+ FROM ${dbName}.${STATE_TABLE}`,+ format: 'JSONEachRow',+ })+ const [row] = await resultSet.json()++ return Number(row.total) > 0 ? row : null+}++// The replay at the end has to cover every row written to the old table from+// the moment the backfill started, so the watermark is taken before it β and+// persisted, because a retry that resumes after the rename must reuse the+// original timestamp instead of a fresh now().+const captureMigrationStart = async () => {+ // An attempt that died before the rename already recorded one β keep it, the+ // backfill it started may still have missed rows written after that point.+ const existing = await readMigrationStart()++ if (existing) {+ return existing+ }++ await queriesRunner([+ `CREATE TABLE IF NOT EXISTS ${dbName}.${STATE_TABLE}+ (+ startedAt DateTime('UTC')+ )+ ENGINE = MergeTree()+ ORDER BY startedAt;`,++ `INSERT INTO ${dbName}.${STATE_TABLE} (startedAt) SELECT now()`,+ ])++ return readMigrationStart()+}- // The backfill above runs while the old code is still writing to the old- // table, so anything recorded between the INSERT and the RENAME would be- // lost. Replay the tail of the old table into the new one β ReplacingMergeTree- // dedupes on (pid, sid) and keeps the row with the greatest lastSeen. Only the- // last day is replayed: sessions expire after 30 minutes of inactivity, so- // nothing older than that can have changed during the migration.- `INSERT INTO ${dbName}.sessions (sid, psid, pid, profileId, firstSeen, lastSeen)+// ReplacingMergeTree dedupes on (pid, sid) and keeps the row with the greatest+// lastSeen, so replaying rows the backfill already copied is harmless β the+// watermark only exists to keep the amount of replayed data sane.+const buildReplayQuery = (migrationStart) => {+ const conditions = [+ // lastSeen is stamped on the API host and inserted asynchronously, so a row+ // can land after the timestamp it carries. The extra hour absorbs that lag+ // and any clock skew between the API hosts and ClickHouse.+ migrationStart &&+ `lastSeen >= toDateTime({startedAt:UInt32}) - INTERVAL 1 HOUR`,+ 'firstSeen <= lastSeen',+ 'lastSeen > toDateTime(0)',+ ].filter(Boolean)++ return `INSERT INTO ${dbName}.sessions (sid, psid, pid, profileId, firstSeen, lastSeen)SELECT psid AS sid, psid, pid, profileId, firstSeen, lastSeen- FROM ${dbName}.sessions_by_visitor_day- WHERE lastSeen >= now() - INTERVAL 1 DAY- AND firstSeen <= lastSeen- AND lastSeen > toDateTime(0)`,-]+ FROM ${dbName}.${OLD_SESSIONS_TABLE}+ WHERE ${conditions.join('\n AND ')}`+}++const run = async () => {+ // The rename is the point of no return: once it lands, sessions is the new+ // table and the old one is frozen. Re-running the backfill from there would+ // fail on the existing rename target and copy the new table back over itself,+ // so a resumed attempt only replays the tail.+ const renamed = await tableExists(OLD_SESSIONS_TABLE)++ await queriesRunner(ADD_SID_COLUMN_QUERIES)++ let migrationStart++ if (renamed) {+ if (!(await tableExists('sessions'))) {+ throw new Error(+ `${dbName}.${OLD_SESSIONS_TABLE} exists but ${dbName}.sessions does not β the rename left the database half-migrated. Restore ${dbName}.sessions manually before rerunning.`,+ )+ }++ if (await tableExists('sessions_v2')) {+ console.warn(+ `${dbName}.sessions_v2 is left over from an earlier attempt and is not used any more β drop it once this migration is verified.`,+ )+ }++ migrationStart = await readMigrationStart()++ if (migrationStart) {+ console.log(+ `Resuming after the rename, replaying ${dbName}.${OLD_SESSIONS_TABLE} from ${migrationStart.startedAtText} UTC.`,+ )+ } else {+ console.warn(+ `Resuming after the rename, but no ${dbName}.${STATE_TABLE} watermark was recorded. Replaying ${dbName}.${OLD_SESSIONS_TABLE} in full β slower, but the overlap is deduped.`,+ )+ }+ } else {+ migrationStart = await captureMigrationStart()+ await queriesRunner(BACKFILL_QUERIES)+ }++ // The backfill runs while the old code is still writing to the old table, so+ // anything recorded between the INSERT and the RENAME would be lost. Replay+ // everything the old table received from the watermark onwards.+ await queriesRunner(+ [buildReplayQuery(migrationStart)],+ true,+ migrationStart+ ? { query_params: { startedAt: migrationStart.startedAtTs } }+ : {},+ )+}-queriesRunner(queries)+// Drop ${dbName}.sessions_sid_migration_state once the migration is verified.+run()