π Open source, privacy-first web analytics β a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
A few more fixes
Blue Mouseaa57bc300dde
diff --git a/backend/apps/cloud/src/analytics/analytics.controller.ts b/backend/apps/cloud/src/analytics/analytics.controller.tsindex 4bcda7b4c..868c87b34 100644--- a/backend/apps/cloud/src/analytics/analytics.controller.ts+++ b/backend/apps/cloud/src/analytics/analytics.controller.ts@@ -1610,11 +1610,12 @@ export class AnalyticsController {const project = await this.analyticsService.validate(errorDTO, origin, ip)- const [, psid, sid] = await this.analyticsService.generateAndStoreSessionId(- errorDTO.pid,- userAgent,- ip,- )+ const [isNewSession, psid, sid] =+ await this.analyticsService.generateAndStoreSessionId(+ errorDTO.pid,+ userAgent,+ ip,+ )const profileId = await this.analyticsService.generateProfileId(errorDTO.pid,@@ -1628,6 +1629,7 @@ export class AnalyticsController {psid,errorDTO.pid,profileId,+ isNewSession,)const {@@ -1789,11 +1791,12 @@ export class AnalyticsController {const { deviceType, browserName, browserVersion, osName, osVersion } =await this.analyticsService.getRequestInformation(headers)- const [, psid, sid] = await this.analyticsService.generateAndStoreSessionId(- eventsDTO.pid,- userAgent,- ip,- )+ const [isNewSession, psid, sid] =+ await this.analyticsService.generateAndStoreSessionId(+ eventsDTO.pid,+ userAgent,+ ip,+ )const profileId = await this.analyticsService.generateProfileId(eventsDTO.pid,@@ -1807,6 +1810,7 @@ export class AnalyticsController {psid,eventsDTO.pid,profileId,+ isNewSession,)enrichTrafficSource(eventsDTO)@@ -2041,6 +2045,7 @@ export class AnalyticsController {psid,logDTO.pid,profileId,+ unique,)if (!unique && logDTO.unique) {@@ -2208,11 +2213,12 @@ export class AnalyticsController {const project = await this.analyticsService.validate(logDTO, origin, ip)- const [, psid, sid] = await this.analyticsService.generateAndStoreSessionId(- logDTO.pid,- userAgent,- ip,- )+ const [isNewSession, psid, sid] =+ await this.analyticsService.generateAndStoreSessionId(+ logDTO.pid,+ userAgent,+ ip,+ )const profileId = await this.analyticsService.generateProfileId(logDTO.pid,@@ -2225,6 +2231,7 @@ export class AnalyticsController {psid,logDTO.pid,profileId,+ isNewSession,)const {diff --git a/backend/apps/cloud/src/analytics/analytics.service.ts b/backend/apps/cloud/src/analytics/analytics.service.tsindex 0631c73eb..5cb6fbe70 100644--- a/backend/apps/cloud/src/analytics/analytics.service.ts+++ b/backend/apps/cloud/src/analytics/analytics.service.ts@@ -2841,11 +2841,37 @@ export class AnalyticsService {return `ses:fs:${pid}:${sid}`}+ // Recovers the start of an already-running session after its Redis key was+ // lost. sid leads the sessions ORDER BY, so this is a primary key lookup.+ private async getSessionFirstSeenFromClickHouse(+ sid: string,+ pid: string,+ ): Promise<string | null> {+ const query = `+ SELECT minOrNull(firstSeen) AS firstSeen+ FROM sessions+ WHERE sid = {sid:UInt64}+ AND pid = {pid:FixedString(12)}+ `++ const { data } = await clickhouse+ .query({+ query,+ query_params: { sid, pid },+ })+ .then((resultSet) => resultSet.json<{ firstSeen: string | null }>())++ const firstSeen = data[0]?.firstSeen++ return firstSeen ? dayjs.utc(firstSeen).format('YYYY-MM-DD HH:mm:ss') : null+ }+async recordSessionActivity(sid: string,psid: string,pid: string,profileId: string,+ isNewSession = false,): Promise<void> {const now = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')const cacheKey = this.getSessionFirstSeenKey(pid, sid)@@ -2864,6 +2890,19 @@ export class AnalyticsService {if (stored !== 'OK') {firstSeen = (await redis.get(cacheKey)) || nowawait redis.expire(cacheKey, UNIQUE_SESSION_LIFE_TIME)+ } else if (!isNewSession) {+ // We created the key, yet the session predates this request β Redis+ // dropped it (restart, eviction). Leaving firstSeen at `now` would be+ // silently destructive: sessions is a ReplacingMergeTree(lastSeen), so+ // this row wins the merge and the real start is lost for good.+ // Unreachable while Redis holds the key, so it costs nothing in the+ // normal path.+ const recovered = await this.getSessionFirstSeenFromClickHouse(sid, pid)++ if (recovered) {+ firstSeen = recovered+ await redis.set(cacheKey, firstSeen, 'EX', UNIQUE_SESSION_LIFE_TIME)+ }}await clickhouse.insert({@@ -2886,39 +2925,6 @@ export class AnalyticsService {}}- async checkSessionExistsInClickHouse(- sid: string,- pid: string,- ): Promise<boolean> {- const cutoff = dayjs- .utc()- .subtract(UNIQUE_SESSION_LIFE_TIME, 'second')- .format('YYYY-MM-DD HH:mm:ss')-- try {- const query = `- SELECT 1- FROM sessions FINAL- WHERE sid = {sid:UInt64}- AND pid = {pid:FixedString(12)}- AND lastSeen >= {cutoff:DateTime}- LIMIT 1- `-- const { data } = await clickhouse- .query({- query,- query_params: { sid, pid, cutoff },- })- .then((resultSet) => resultSet.json())-- return data.length > 0- } catch (error) {- console.error('Failed to check session existence:', error)- return false- }- }-async getSessionDurationFromClickHouse(psid: string,pid: string,@@ -2982,13 +2988,13 @@ export class AnalyticsService {pid: string,userAgent: string,ip: string,- ): Promise<{ psid: string; sid: string }> {- const [, psid, sid] = await this.generateAndStoreSessionId(+ ): Promise<{ psid: string; sid: string; isNewSession: boolean }> {+ const [isNewSession, psid, sid] = await this.generateAndStoreSessionId(pid,userAgent,ip,)- return { psid, sid }+ return { psid, sid, isNewSession }}async startSessionReplay(@@ -3006,7 +3012,11 @@ export class AnalyticsService {)}- const { psid, sid } = await this.resolveReplaySession(pid, userAgent, ip)+ const { psid, sid, isNewSession } = await this.resolveReplaySession(+ pid,+ userAgent,+ ip,+ )const resolvedProfileId = await this.generateProfileId(pid,userAgent,@@ -3014,7 +3024,13 @@ export class AnalyticsService {profileId,)- await this.recordSessionActivity(sid, psid, pid, resolvedProfileId)+ await this.recordSessionActivity(+ sid,+ psid,+ pid,+ resolvedProfileId,+ isNewSession,+ )const retention = this.getSessionReplayRetention(project)const replayStart = await this.reserveActiveReplayStart(pid, psid, replayId)@@ -4932,7 +4948,7 @@ export class AnalyticsService {bounce_counts AS (SELECT count() AS bouncesFROM (- SELECT psid+ SELECT coalesce(sid, psid) AS sessionIdFROM eventsWHERE pid = {pid:FixedString(12)}AND type = 'pageview'@@ -4947,7 +4963,7 @@ export class AnalyticsService {AND psid != 0${filtersQuery})- GROUP BY psid+ GROUP BY sessionIdHAVING count() = 1))@@ -5094,7 +5110,7 @@ export class AnalyticsService {bounce_counts AS (SELECT count() AS bouncesFROM (- SELECT psid+ SELECT coalesce(sid, psid) AS sessionIdFROM eventsWHERE pid = {pid:FixedString(12)}AND type = 'pageview'@@ -5111,7 +5127,7 @@ export class AnalyticsService {AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String}${currentFiltersQuery})- GROUP BY psid+ GROUP BY sessionIdHAVING count() = 1))@@ -5159,7 +5175,7 @@ export class AnalyticsService {bounce_counts AS (SELECT count() AS bouncesFROM (- SELECT psid+ SELECT coalesce(sid, psid) AS sessionIdFROM eventsWHERE pid = {pid:FixedString(12)}AND type = 'pageview'@@ -5176,7 +5192,7 @@ export class AnalyticsService {AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String}${previousFiltersQuery})- GROUP BY psid+ GROUP BY sessionIdHAVING count() = 1))@@ -6041,7 +6057,7 @@ export class AnalyticsService {${selector},avgOrNull(sessions_data.duration) as sdur,count() as pageviews,- count(DISTINCT subquery.psid) as uniques,+ count(DISTINCT subquery.sessionId) as uniques,countIf(session_counts.pageviews = 1) as bouncesFROM (SELECT@@ -6203,7 +6219,7 @@ export class AnalyticsService {SELECT${selector},avgOrNull(sessions_data.duration) as sdur,- count(DISTINCT psid) as uniques,+ count(DISTINCT subquery.sessionId) as uniques,count() as countFROM (SELECT@@ -7886,18 +7902,22 @@ export class AnalyticsService {LIMIT 1;`+ // A psid covers a whole day, so it can span several sessions β sum their+ // individual durations instead of measuring first-to-last, which would+ // count the idle gaps between them. Matches the sessions list's sdur.const querySessionDuration = `SELECT- dateDiff('second', firstSeen, lastSeen) as duration,- lastSeen+ sumOrNull(duration) as duration,+ maxOrNull(sessionEnd) as lastSeenFROM (SELECT- minOrNull(firstSeen) AS firstSeen,- maxOrNull(lastSeen) AS lastSeen+ dateDiff('second', min(firstSeen), max(lastSeen)) as duration,+ max(lastSeen) AS sessionEndFROM sessionsWHEREpid = {pid:FixedString(12)}AND psid = toUInt64OrNull({psid:String})+ GROUP BY sid)`diff --git a/backend/apps/community/src/analytics/analytics.controller.ts b/backend/apps/community/src/analytics/analytics.controller.tsindex aeadcf4d8..6c6400821 100644--- a/backend/apps/community/src/analytics/analytics.controller.ts+++ b/backend/apps/community/src/analytics/analytics.controller.ts@@ -1339,11 +1339,12 @@ export class AnalyticsController {const { deviceType, browserName, browserVersion, osName, osVersion } =await this.analyticsService.getRequestInformation(headers)- const [, psid, sid] = await this.analyticsService.generateAndStoreSessionId(- eventsDTO.pid,- userAgent,- ip,- )+ const [isNewSession, psid, sid] =+ await this.analyticsService.generateAndStoreSessionId(+ eventsDTO.pid,+ userAgent,+ ip,+ )const profileId = await this.analyticsService.generateProfileId(eventsDTO.pid,@@ -1357,6 +1358,7 @@ export class AnalyticsController {psid,eventsDTO.pid,profileId,+ isNewSession,)enrichTrafficSource(eventsDTO)@@ -1586,6 +1588,7 @@ export class AnalyticsController {psid,logDTO.pid,profileId,+ unique,)if (!unique && logDTO.unique) {@@ -1753,11 +1756,12 @@ export class AnalyticsController {const project = await this.analyticsService.validate(logDTO, origin, ip)- const [, psid, sid] = await this.analyticsService.generateAndStoreSessionId(- logDTO.pid,- userAgent,- ip,- )+ const [isNewSession, psid, sid] =+ await this.analyticsService.generateAndStoreSessionId(+ logDTO.pid,+ userAgent,+ ip,+ )// For noscript requests, we generate an anonymous profile ID since// user-supplied profileId is not available without JavaScript@@ -1772,6 +1776,7 @@ export class AnalyticsController {psid,logDTO.pid,profileId,+ isNewSession,)const {@@ -2145,11 +2150,12 @@ export class AnalyticsController {const project = await this.analyticsService.validate(errorDTO, origin, ip)- const [, psid, sid] = await this.analyticsService.generateAndStoreSessionId(- errorDTO.pid,- userAgent,- ip,- )+ const [isNewSession, psid, sid] =+ await this.analyticsService.generateAndStoreSessionId(+ errorDTO.pid,+ userAgent,+ ip,+ )const profileId = await this.analyticsService.generateProfileId(errorDTO.pid,@@ -2163,6 +2169,7 @@ export class AnalyticsController {psid,errorDTO.pid,profileId,+ isNewSession,)const {diff --git a/backend/apps/community/src/analytics/analytics.service.ts b/backend/apps/community/src/analytics/analytics.service.tsindex 2674fe8a6..7cd8e618d 100644--- a/backend/apps/community/src/analytics/analytics.service.ts+++ b/backend/apps/community/src/analytics/analytics.service.ts@@ -2482,11 +2482,37 @@ export class AnalyticsService {return `ses:fs:${pid}:${sid}`}+ // Recovers the start of an already-running session after its Redis key was+ // lost. sid leads the sessions ORDER BY, so this is a primary key lookup.+ private async getSessionFirstSeenFromClickHouse(+ sid: string,+ pid: string,+ ): Promise<string | null> {+ const query = `+ SELECT minOrNull(firstSeen) AS firstSeen+ FROM sessions+ WHERE sid = {sid:UInt64}+ AND pid = {pid:FixedString(12)}+ `++ const { data } = await clickhouse+ .query({+ query,+ query_params: { sid, pid },+ })+ .then((resultSet) => resultSet.json<{ firstSeen: string | null }>())++ const firstSeen = data[0]?.firstSeen++ return firstSeen ? dayjs.utc(firstSeen).format('YYYY-MM-DD HH:mm:ss') : null+ }+async recordSessionActivity(sid: string,psid: string,pid: string,profileId: string,+ isNewSession = false,): Promise<void> {const now = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')const cacheKey = this.getSessionFirstSeenKey(pid, sid)@@ -2505,6 +2531,19 @@ export class AnalyticsService {if (stored !== 'OK') {firstSeen = (await redis.get(cacheKey)) || nowawait redis.expire(cacheKey, UNIQUE_SESSION_LIFE_TIME)+ } else if (!isNewSession) {+ // We created the key, yet the session predates this request β Redis+ // dropped it (restart, eviction). Leaving firstSeen at `now` would be+ // silently destructive: sessions is a ReplacingMergeTree(lastSeen), so+ // this row wins the merge and the real start is lost for good.+ // Unreachable while Redis holds the key, so it costs nothing in the+ // normal path.+ const recovered = await this.getSessionFirstSeenFromClickHouse(sid, pid)++ if (recovered) {+ firstSeen = recovered+ await redis.set(cacheKey, firstSeen, 'EX', UNIQUE_SESSION_LIFE_TIME)+ }}await clickhouse.insert({@@ -3512,7 +3551,7 @@ export class AnalyticsService {bounce_counts AS (SELECT count() AS bouncesFROM (- SELECT psid+ SELECT coalesce(sid, psid) AS sessionIdFROM eventsWHERE pid = {pid:FixedString(12)}AND type = 'pageview'@@ -3527,7 +3566,7 @@ export class AnalyticsService {AND psid != 0${filtersQuery})- GROUP BY psid+ GROUP BY sessionIdHAVING count() = 1))@@ -3670,7 +3709,7 @@ export class AnalyticsService {bounce_counts AS (SELECT count() AS bouncesFROM (- SELECT psid+ SELECT coalesce(sid, psid) AS sessionIdFROM eventsWHERE pid = {pid:FixedString(12)}AND type = 'pageview'@@ -3687,7 +3726,7 @@ export class AnalyticsService {AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String}${currentFiltersQuery})- GROUP BY psid+ GROUP BY sessionIdHAVING count() = 1))@@ -3735,7 +3774,7 @@ export class AnalyticsService {bounce_counts AS (SELECT count() AS bouncesFROM (- SELECT psid+ SELECT coalesce(sid, psid) AS sessionIdFROM eventsWHERE pid = {pid:FixedString(12)}AND type = 'pageview'@@ -3752,7 +3791,7 @@ export class AnalyticsService {AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String}${previousFiltersQuery})- GROUP BY psid+ GROUP BY sessionIdHAVING count() = 1))@@ -4593,7 +4632,7 @@ export class AnalyticsService {${selector},avgOrNull(sessions_data.duration) as sdur,count() as pageviews,- count(DISTINCT subquery.psid) as uniques,+ count(DISTINCT subquery.sessionId) as uniques,countIf(session_counts.pageviews = 1) as bouncesFROM (SELECT@@ -4755,7 +4794,7 @@ export class AnalyticsService {SELECT${selector},count() as count,- countDistinct(subquery.psid) as uniques,+ countDistinct(subquery.sessionId) as uniques,avgOrNull(sessions_data.duration) as sdurFROM (SELECT@@ -6361,18 +6400,22 @@ export class AnalyticsService {LIMIT 1;`+ // A psid covers a whole day, so it can span several sessions β sum their+ // individual durations instead of measuring first-to-last, which would+ // count the idle gaps between them. Matches the sessions list's sdur.const querySessionDuration = `SELECT- dateDiff('second', firstSeen, lastSeen) as duration,- lastSeen+ sumOrNull(duration) as duration,+ maxOrNull(sessionEnd) as lastSeenFROM (SELECT- minOrNull(firstSeen) AS firstSeen,- maxOrNull(lastSeen) AS lastSeen+ dateDiff('second', min(firstSeen), max(lastSeen)) as duration,+ max(lastSeen) AS sessionEndFROM sessionsWHEREpid = {pid:FixedString(12)}AND psid = toUInt64OrNull({psid:String})+ GROUP BY sid)`diff --git a/backend/migrations/clickhouse/2026_08_01_session_id.js b/backend/migrations/clickhouse/2026_08_01_session_id.jsindex 5d9b91645..68b9b0efb 100644--- a/backend/migrations/clickhouse/2026_08_01_session_id.js+++ b/backend/migrations/clickhouse/2026_08_01_session_id.js@@ -5,8 +5,6 @@ const STATE_TABLE = 'sessions_sid_migration_state'const ADD_SID_COLUMN_QUERIES = [`ALTER TABLE ${dbName}.events ADD COLUMN IF NOT EXISTS sid Nullable(UInt64) AFTER psid`,- `ALTER TABLE ${dbName}.customEV ADD COLUMN IF NOT EXISTS sid Nullable(UInt64) AFTER psid`,- `ALTER TABLE ${dbName}.errors ADD COLUMN IF NOT EXISTS sid Nullable(UInt64) AFTER psid`,]const BACKFILL_QUERIES = [diff --git a/backend/migrations/clickhouse/initialise_database.js b/backend/migrations/clickhouse/initialise_database.jsindex d60828387..aabea4ccc 100644--- a/backend/migrations/clickhouse/initialise_database.js+++ b/backend/migrations/clickhouse/initialise_database.js@@ -10,6 +10,7 @@ const CLICKHOUSE_INIT_QUERIES = [type LowCardinality(String),pid FixedString(12),psid Nullable(UInt64),+ sid Nullable(UInt64),profileId Nullable(String) CODEC(ZSTD(3)),host Nullable(String) CODEC(ZSTD(3)),pg Nullable(String) CODEC(ZSTD(3)),@@ -68,19 +69,21 @@ const CLICKHOUSE_INIT_QUERIES = [ENGINE = ReplacingMergeTree()PRIMARY KEY (eid, pid);`,- // Sessions table with ReplacingMergeTree for tracking session state+ // Sessions table with ReplacingMergeTree for tracking session state.+ // Keyed on sid (one row per browsing session, 30 min of inactivity ends it),+ // not psid (one value per visitor per day) β see the 2026_08_01_session_id+ // migration, which this definition has to stay in sync with.`CREATE TABLE IF NOT EXISTS ${dbName}.sessions(+ sid UInt64 DEFAULT psid,psid UInt64,pid FixedString(12),profileId Nullable(String) CODEC(ZSTD(3)),firstSeen DateTime('UTC') CODEC(Delta(4), LZ4),- lastSeen DateTime('UTC') CODEC(Delta(4), LZ4),- pageviews UInt32 DEFAULT 1,- events UInt32 DEFAULT 0+ lastSeen DateTime('UTC') CODEC(Delta(4), LZ4))ENGINE = ReplacingMergeTree(lastSeen)- ORDER BY (pid, psid)+ ORDER BY (pid, sid)PARTITION BY toYYYYMM(firstSeen);`,// Profile aliases table: maps anonymous (anon_) profile IDs to identified (usr_)