Skip to content

πŸ“ˆ Open source, privacy-first web analytics β€” a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

A few more fixes

Blue Mouseaa57bc300dde
diff --git a/backend/apps/cloud/src/analytics/analytics.controller.ts b/backend/apps/cloud/src/analytics/analytics.controller.ts
index 4bcda7b4c..868c87b34 100644
--- a/backend/apps/cloud/src/analytics/analytics.controller.ts
+++ b/backend/apps/cloud/src/analytics/analytics.controller.ts
@@ -1610,11 +1610,12 @@ export class AnalyticsController {
const project = await this.analyticsService.validate(errorDTO, origin, ip)
- const [, psid, sid] = await this.analyticsService.generateAndStoreSessionId(
- errorDTO.pid,
- userAgent,
- ip,
- )
+ const [isNewSession, psid, sid] =
+ await this.analyticsService.generateAndStoreSessionId(
+ errorDTO.pid,
+ userAgent,
+ ip,
+ )
const profileId = await this.analyticsService.generateProfileId(
errorDTO.pid,
@@ -1628,6 +1629,7 @@ export class AnalyticsController {
psid,
errorDTO.pid,
profileId,
+ isNewSession,
)
const {
@@ -1789,11 +1791,12 @@ export class AnalyticsController {
const { deviceType, browserName, browserVersion, osName, osVersion } =
await this.analyticsService.getRequestInformation(headers)
- const [, psid, sid] = await this.analyticsService.generateAndStoreSessionId(
- eventsDTO.pid,
- userAgent,
- ip,
- )
+ const [isNewSession, psid, sid] =
+ await this.analyticsService.generateAndStoreSessionId(
+ eventsDTO.pid,
+ userAgent,
+ ip,
+ )
const profileId = await this.analyticsService.generateProfileId(
eventsDTO.pid,
@@ -1807,6 +1810,7 @@ export class AnalyticsController {
psid,
eventsDTO.pid,
profileId,
+ isNewSession,
)
enrichTrafficSource(eventsDTO)
@@ -2041,6 +2045,7 @@ export class AnalyticsController {
psid,
logDTO.pid,
profileId,
+ unique,
)
if (!unique && logDTO.unique) {
@@ -2208,11 +2213,12 @@ export class AnalyticsController {
const project = await this.analyticsService.validate(logDTO, origin, ip)
- const [, psid, sid] = await this.analyticsService.generateAndStoreSessionId(
- logDTO.pid,
- userAgent,
- ip,
- )
+ const [isNewSession, psid, sid] =
+ await this.analyticsService.generateAndStoreSessionId(
+ logDTO.pid,
+ userAgent,
+ ip,
+ )
const profileId = await this.analyticsService.generateProfileId(
logDTO.pid,
@@ -2225,6 +2231,7 @@ export class AnalyticsController {
psid,
logDTO.pid,
profileId,
+ isNewSession,
)
const {
diff --git a/backend/apps/cloud/src/analytics/analytics.service.ts b/backend/apps/cloud/src/analytics/analytics.service.ts
index 0631c73eb..5cb6fbe70 100644
--- a/backend/apps/cloud/src/analytics/analytics.service.ts
+++ b/backend/apps/cloud/src/analytics/analytics.service.ts
@@ -2841,11 +2841,37 @@ export class AnalyticsService {
return `ses:fs:${pid}:${sid}`
}
+ // Recovers the start of an already-running session after its Redis key was
+ // lost. sid leads the sessions ORDER BY, so this is a primary key lookup.
+ private async getSessionFirstSeenFromClickHouse(
+ sid: string,
+ pid: string,
+ ): Promise<string | null> {
+ const query = `
+ SELECT minOrNull(firstSeen) AS firstSeen
+ FROM sessions
+ WHERE sid = {sid:UInt64}
+ AND pid = {pid:FixedString(12)}
+ `
+
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: { sid, pid },
+ })
+ .then((resultSet) => resultSet.json<{ firstSeen: string | null }>())
+
+ const firstSeen = data[0]?.firstSeen
+
+ return firstSeen ? dayjs.utc(firstSeen).format('YYYY-MM-DD HH:mm:ss') : null
+ }
+
async recordSessionActivity(
sid: string,
psid: string,
pid: string,
profileId: string,
+ isNewSession = false,
): Promise<void> {
const now = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')
const cacheKey = this.getSessionFirstSeenKey(pid, sid)
@@ -2864,6 +2890,19 @@ export class AnalyticsService {
if (stored !== 'OK') {
firstSeen = (await redis.get(cacheKey)) || now
await redis.expire(cacheKey, UNIQUE_SESSION_LIFE_TIME)
+ } else if (!isNewSession) {
+ // We created the key, yet the session predates this request β€” Redis
+ // dropped it (restart, eviction). Leaving firstSeen at `now` would be
+ // silently destructive: sessions is a ReplacingMergeTree(lastSeen), so
+ // this row wins the merge and the real start is lost for good.
+ // Unreachable while Redis holds the key, so it costs nothing in the
+ // normal path.
+ const recovered = await this.getSessionFirstSeenFromClickHouse(sid, pid)
+
+ if (recovered) {
+ firstSeen = recovered
+ await redis.set(cacheKey, firstSeen, 'EX', UNIQUE_SESSION_LIFE_TIME)
+ }
}
await clickhouse.insert({
@@ -2886,39 +2925,6 @@ export class AnalyticsService {
}
}
- async checkSessionExistsInClickHouse(
- sid: string,
- pid: string,
- ): Promise<boolean> {
- const cutoff = dayjs
- .utc()
- .subtract(UNIQUE_SESSION_LIFE_TIME, 'second')
- .format('YYYY-MM-DD HH:mm:ss')
-
- try {
- const query = `
- SELECT 1
- FROM sessions FINAL
- WHERE sid = {sid:UInt64}
- AND pid = {pid:FixedString(12)}
- AND lastSeen >= {cutoff:DateTime}
- LIMIT 1
- `
-
- const { data } = await clickhouse
- .query({
- query,
- query_params: { sid, pid, cutoff },
- })
- .then((resultSet) => resultSet.json())
-
- return data.length > 0
- } catch (error) {
- console.error('Failed to check session existence:', error)
- return false
- }
- }
-
async getSessionDurationFromClickHouse(
psid: string,
pid: string,
@@ -2982,13 +2988,13 @@ export class AnalyticsService {
pid: string,
userAgent: string,
ip: string,
- ): Promise<{ psid: string; sid: string }> {
- const [, psid, sid] = await this.generateAndStoreSessionId(
+ ): Promise<{ psid: string; sid: string; isNewSession: boolean }> {
+ const [isNewSession, psid, sid] = await this.generateAndStoreSessionId(
pid,
userAgent,
ip,
)
- return { psid, sid }
+ return { psid, sid, isNewSession }
}
async startSessionReplay(
@@ -3006,7 +3012,11 @@ export class AnalyticsService {
)
}
- const { psid, sid } = await this.resolveReplaySession(pid, userAgent, ip)
+ const { psid, sid, isNewSession } = await this.resolveReplaySession(
+ pid,
+ userAgent,
+ ip,
+ )
const resolvedProfileId = await this.generateProfileId(
pid,
userAgent,
@@ -3014,7 +3024,13 @@ export class AnalyticsService {
profileId,
)
- await this.recordSessionActivity(sid, psid, pid, resolvedProfileId)
+ await this.recordSessionActivity(
+ sid,
+ psid,
+ pid,
+ resolvedProfileId,
+ isNewSession,
+ )
const retention = this.getSessionReplayRetention(project)
const replayStart = await this.reserveActiveReplayStart(pid, psid, replayId)
@@ -4932,7 +4948,7 @@ export class AnalyticsService {
bounce_counts AS (
SELECT count() AS bounces
FROM (
- SELECT psid
+ SELECT coalesce(sid, psid) AS sessionId
FROM events
WHERE pid = {pid:FixedString(12)}
AND type = 'pageview'
@@ -4947,7 +4963,7 @@ export class AnalyticsService {
AND psid != 0
${filtersQuery}
)
- GROUP BY psid
+ GROUP BY sessionId
HAVING count() = 1
)
)
@@ -5094,7 +5110,7 @@ export class AnalyticsService {
bounce_counts AS (
SELECT count() AS bounces
FROM (
- SELECT psid
+ SELECT coalesce(sid, psid) AS sessionId
FROM events
WHERE pid = {pid:FixedString(12)}
AND type = 'pageview'
@@ -5111,7 +5127,7 @@ export class AnalyticsService {
AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String}
${currentFiltersQuery}
)
- GROUP BY psid
+ GROUP BY sessionId
HAVING count() = 1
)
)
@@ -5159,7 +5175,7 @@ export class AnalyticsService {
bounce_counts AS (
SELECT count() AS bounces
FROM (
- SELECT psid
+ SELECT coalesce(sid, psid) AS sessionId
FROM events
WHERE pid = {pid:FixedString(12)}
AND type = 'pageview'
@@ -5176,7 +5192,7 @@ export class AnalyticsService {
AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String}
${previousFiltersQuery}
)
- GROUP BY psid
+ GROUP BY sessionId
HAVING count() = 1
)
)
@@ -6041,7 +6057,7 @@ export class AnalyticsService {
${selector},
avgOrNull(sessions_data.duration) as sdur,
count() as pageviews,
- count(DISTINCT subquery.psid) as uniques,
+ count(DISTINCT subquery.sessionId) as uniques,
countIf(session_counts.pageviews = 1) as bounces
FROM (
SELECT
@@ -6203,7 +6219,7 @@ export class AnalyticsService {
SELECT
${selector},
avgOrNull(sessions_data.duration) as sdur,
- count(DISTINCT psid) as uniques,
+ count(DISTINCT subquery.sessionId) as uniques,
count() as count
FROM (
SELECT
@@ -7886,18 +7902,22 @@ export class AnalyticsService {
LIMIT 1;
`
+ // A psid covers a whole day, so it can span several sessions β€” sum their
+ // individual durations instead of measuring first-to-last, which would
+ // count the idle gaps between them. Matches the sessions list's sdur.
const querySessionDuration = `
SELECT
- dateDiff('second', firstSeen, lastSeen) as duration,
- lastSeen
+ sumOrNull(duration) as duration,
+ maxOrNull(sessionEnd) as lastSeen
FROM (
SELECT
- minOrNull(firstSeen) AS firstSeen,
- maxOrNull(lastSeen) AS lastSeen
+ dateDiff('second', min(firstSeen), max(lastSeen)) as duration,
+ max(lastSeen) AS sessionEnd
FROM sessions
WHERE
pid = {pid:FixedString(12)}
AND psid = toUInt64OrNull({psid:String})
+ GROUP BY sid
)
`
diff --git a/backend/apps/community/src/analytics/analytics.controller.ts b/backend/apps/community/src/analytics/analytics.controller.ts
index aeadcf4d8..6c6400821 100644
--- a/backend/apps/community/src/analytics/analytics.controller.ts
+++ b/backend/apps/community/src/analytics/analytics.controller.ts
@@ -1339,11 +1339,12 @@ export class AnalyticsController {
const { deviceType, browserName, browserVersion, osName, osVersion } =
await this.analyticsService.getRequestInformation(headers)
- const [, psid, sid] = await this.analyticsService.generateAndStoreSessionId(
- eventsDTO.pid,
- userAgent,
- ip,
- )
+ const [isNewSession, psid, sid] =
+ await this.analyticsService.generateAndStoreSessionId(
+ eventsDTO.pid,
+ userAgent,
+ ip,
+ )
const profileId = await this.analyticsService.generateProfileId(
eventsDTO.pid,
@@ -1357,6 +1358,7 @@ export class AnalyticsController {
psid,
eventsDTO.pid,
profileId,
+ isNewSession,
)
enrichTrafficSource(eventsDTO)
@@ -1586,6 +1588,7 @@ export class AnalyticsController {
psid,
logDTO.pid,
profileId,
+ unique,
)
if (!unique && logDTO.unique) {
@@ -1753,11 +1756,12 @@ export class AnalyticsController {
const project = await this.analyticsService.validate(logDTO, origin, ip)
- const [, psid, sid] = await this.analyticsService.generateAndStoreSessionId(
- logDTO.pid,
- userAgent,
- ip,
- )
+ const [isNewSession, psid, sid] =
+ await this.analyticsService.generateAndStoreSessionId(
+ logDTO.pid,
+ userAgent,
+ ip,
+ )
// For noscript requests, we generate an anonymous profile ID since
// user-supplied profileId is not available without JavaScript
@@ -1772,6 +1776,7 @@ export class AnalyticsController {
psid,
logDTO.pid,
profileId,
+ isNewSession,
)
const {
@@ -2145,11 +2150,12 @@ export class AnalyticsController {
const project = await this.analyticsService.validate(errorDTO, origin, ip)
- const [, psid, sid] = await this.analyticsService.generateAndStoreSessionId(
- errorDTO.pid,
- userAgent,
- ip,
- )
+ const [isNewSession, psid, sid] =
+ await this.analyticsService.generateAndStoreSessionId(
+ errorDTO.pid,
+ userAgent,
+ ip,
+ )
const profileId = await this.analyticsService.generateProfileId(
errorDTO.pid,
@@ -2163,6 +2169,7 @@ export class AnalyticsController {
psid,
errorDTO.pid,
profileId,
+ isNewSession,
)
const {
diff --git a/backend/apps/community/src/analytics/analytics.service.ts b/backend/apps/community/src/analytics/analytics.service.ts
index 2674fe8a6..7cd8e618d 100644
--- a/backend/apps/community/src/analytics/analytics.service.ts
+++ b/backend/apps/community/src/analytics/analytics.service.ts
@@ -2482,11 +2482,37 @@ export class AnalyticsService {
return `ses:fs:${pid}:${sid}`
}
+ // Recovers the start of an already-running session after its Redis key was
+ // lost. sid leads the sessions ORDER BY, so this is a primary key lookup.
+ private async getSessionFirstSeenFromClickHouse(
+ sid: string,
+ pid: string,
+ ): Promise<string | null> {
+ const query = `
+ SELECT minOrNull(firstSeen) AS firstSeen
+ FROM sessions
+ WHERE sid = {sid:UInt64}
+ AND pid = {pid:FixedString(12)}
+ `
+
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: { sid, pid },
+ })
+ .then((resultSet) => resultSet.json<{ firstSeen: string | null }>())
+
+ const firstSeen = data[0]?.firstSeen
+
+ return firstSeen ? dayjs.utc(firstSeen).format('YYYY-MM-DD HH:mm:ss') : null
+ }
+
async recordSessionActivity(
sid: string,
psid: string,
pid: string,
profileId: string,
+ isNewSession = false,
): Promise<void> {
const now = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')
const cacheKey = this.getSessionFirstSeenKey(pid, sid)
@@ -2505,6 +2531,19 @@ export class AnalyticsService {
if (stored !== 'OK') {
firstSeen = (await redis.get(cacheKey)) || now
await redis.expire(cacheKey, UNIQUE_SESSION_LIFE_TIME)
+ } else if (!isNewSession) {
+ // We created the key, yet the session predates this request β€” Redis
+ // dropped it (restart, eviction). Leaving firstSeen at `now` would be
+ // silently destructive: sessions is a ReplacingMergeTree(lastSeen), so
+ // this row wins the merge and the real start is lost for good.
+ // Unreachable while Redis holds the key, so it costs nothing in the
+ // normal path.
+ const recovered = await this.getSessionFirstSeenFromClickHouse(sid, pid)
+
+ if (recovered) {
+ firstSeen = recovered
+ await redis.set(cacheKey, firstSeen, 'EX', UNIQUE_SESSION_LIFE_TIME)
+ }
}
await clickhouse.insert({
@@ -3512,7 +3551,7 @@ export class AnalyticsService {
bounce_counts AS (
SELECT count() AS bounces
FROM (
- SELECT psid
+ SELECT coalesce(sid, psid) AS sessionId
FROM events
WHERE pid = {pid:FixedString(12)}
AND type = 'pageview'
@@ -3527,7 +3566,7 @@ export class AnalyticsService {
AND psid != 0
${filtersQuery}
)
- GROUP BY psid
+ GROUP BY sessionId
HAVING count() = 1
)
)
@@ -3670,7 +3709,7 @@ export class AnalyticsService {
bounce_counts AS (
SELECT count() AS bounces
FROM (
- SELECT psid
+ SELECT coalesce(sid, psid) AS sessionId
FROM events
WHERE pid = {pid:FixedString(12)}
AND type = 'pageview'
@@ -3687,7 +3726,7 @@ export class AnalyticsService {
AND created BETWEEN {groupFromUTC:String} AND {groupToUTC:String}
${currentFiltersQuery}
)
- GROUP BY psid
+ GROUP BY sessionId
HAVING count() = 1
)
)
@@ -3735,7 +3774,7 @@ export class AnalyticsService {
bounce_counts AS (
SELECT count() AS bounces
FROM (
- SELECT psid
+ SELECT coalesce(sid, psid) AS sessionId
FROM events
WHERE pid = {pid:FixedString(12)}
AND type = 'pageview'
@@ -3752,7 +3791,7 @@ export class AnalyticsService {
AND created BETWEEN {periodSubtracted:String} AND {groupFromUTC:String}
${previousFiltersQuery}
)
- GROUP BY psid
+ GROUP BY sessionId
HAVING count() = 1
)
)
@@ -4593,7 +4632,7 @@ export class AnalyticsService {
${selector},
avgOrNull(sessions_data.duration) as sdur,
count() as pageviews,
- count(DISTINCT subquery.psid) as uniques,
+ count(DISTINCT subquery.sessionId) as uniques,
countIf(session_counts.pageviews = 1) as bounces
FROM (
SELECT
@@ -4755,7 +4794,7 @@ export class AnalyticsService {
SELECT
${selector},
count() as count,
- countDistinct(subquery.psid) as uniques,
+ countDistinct(subquery.sessionId) as uniques,
avgOrNull(sessions_data.duration) as sdur
FROM (
SELECT
@@ -6361,18 +6400,22 @@ export class AnalyticsService {
LIMIT 1;
`
+ // A psid covers a whole day, so it can span several sessions β€” sum their
+ // individual durations instead of measuring first-to-last, which would
+ // count the idle gaps between them. Matches the sessions list's sdur.
const querySessionDuration = `
SELECT
- dateDiff('second', firstSeen, lastSeen) as duration,
- lastSeen
+ sumOrNull(duration) as duration,
+ maxOrNull(sessionEnd) as lastSeen
FROM (
SELECT
- minOrNull(firstSeen) AS firstSeen,
- maxOrNull(lastSeen) AS lastSeen
+ dateDiff('second', min(firstSeen), max(lastSeen)) as duration,
+ max(lastSeen) AS sessionEnd
FROM sessions
WHERE
pid = {pid:FixedString(12)}
AND psid = toUInt64OrNull({psid:String})
+ GROUP BY sid
)
`
diff --git a/backend/migrations/clickhouse/2026_08_01_session_id.js b/backend/migrations/clickhouse/2026_08_01_session_id.js
index 5d9b91645..68b9b0efb 100644
--- a/backend/migrations/clickhouse/2026_08_01_session_id.js
+++ b/backend/migrations/clickhouse/2026_08_01_session_id.js
@@ -5,8 +5,6 @@ const STATE_TABLE = 'sessions_sid_migration_state'
const ADD_SID_COLUMN_QUERIES = [
`ALTER TABLE ${dbName}.events ADD COLUMN IF NOT EXISTS sid Nullable(UInt64) AFTER psid`,
- `ALTER TABLE ${dbName}.customEV ADD COLUMN IF NOT EXISTS sid Nullable(UInt64) AFTER psid`,
- `ALTER TABLE ${dbName}.errors ADD COLUMN IF NOT EXISTS sid Nullable(UInt64) AFTER psid`,
]
const BACKFILL_QUERIES = [
diff --git a/backend/migrations/clickhouse/initialise_database.js b/backend/migrations/clickhouse/initialise_database.js
index d60828387..aabea4ccc 100644
--- a/backend/migrations/clickhouse/initialise_database.js
+++ b/backend/migrations/clickhouse/initialise_database.js
@@ -10,6 +10,7 @@ const CLICKHOUSE_INIT_QUERIES = [
type LowCardinality(String),
pid FixedString(12),
psid Nullable(UInt64),
+ sid Nullable(UInt64),
profileId Nullable(String) CODEC(ZSTD(3)),
host Nullable(String) CODEC(ZSTD(3)),
pg Nullable(String) CODEC(ZSTD(3)),
@@ -68,19 +69,21 @@ const CLICKHOUSE_INIT_QUERIES = [
ENGINE = ReplacingMergeTree()
PRIMARY KEY (eid, pid);`,
- // Sessions table with ReplacingMergeTree for tracking session state
+ // Sessions table with ReplacingMergeTree for tracking session state.
+ // Keyed on sid (one row per browsing session, 30 min of inactivity ends it),
+ // not psid (one value per visitor per day) β€” see the 2026_08_01_session_id
+ // migration, which this definition has to stay in sync with.
`CREATE TABLE IF NOT EXISTS ${dbName}.sessions
(
+ sid UInt64 DEFAULT psid,
psid UInt64,
pid FixedString(12),
profileId Nullable(String) CODEC(ZSTD(3)),
firstSeen DateTime('UTC') CODEC(Delta(4), LZ4),
- lastSeen DateTime('UTC') CODEC(Delta(4), LZ4),
- pageviews UInt32 DEFAULT 1,
- events UInt32 DEFAULT 0
+ lastSeen DateTime('UTC') CODEC(Delta(4), LZ4)
)
ENGINE = ReplacingMergeTree(lastSeen)
- ORDER BY (pid, psid)
+ ORDER BY (pid, sid)
PARTITION BY toYYYYMM(firstSeen);`,
// Profile aliases table: maps anonymous (anon_) profile IDs to identified (usr_)