Skip to content

πŸ“ˆ Open source, privacy-first web analytics β€” a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

fix: Hard cap session at 30 minutes after last event

Blue Mouse66407f0e8a9f
diff --git a/backend/apps/cloud/src/analytics/analytics.controller.ts b/backend/apps/cloud/src/analytics/analytics.controller.ts
index e622c8893..4bcda7b4c 100644
--- a/backend/apps/cloud/src/analytics/analytics.controller.ts
+++ b/backend/apps/cloud/src/analytics/analytics.controller.ts
@@ -1610,7 +1610,7 @@ export class AnalyticsController {
const project = await this.analyticsService.validate(errorDTO, origin, ip)
- const [, psid] = await this.analyticsService.generateAndStoreSessionId(
+ const [, psid, sid] = await this.analyticsService.generateAndStoreSessionId(
errorDTO.pid,
userAgent,
ip,
@@ -1624,6 +1624,7 @@ export class AnalyticsController {
)
await this.analyticsService.recordSessionActivity(
+ sid,
psid,
errorDTO.pid,
profileId,
@@ -1656,6 +1657,7 @@ export class AnalyticsController {
const transformed = eventTransformer({
type: 'error',
psid,
+ sid,
profileId,
eid: this.analyticsService.getErrorID(errorDTO),
pid: errorDTO.pid,
@@ -1787,7 +1789,7 @@ export class AnalyticsController {
const { deviceType, browserName, browserVersion, osName, osVersion } =
await this.analyticsService.getRequestInformation(headers)
- const [, psid] = await this.analyticsService.generateAndStoreSessionId(
+ const [, psid, sid] = await this.analyticsService.generateAndStoreSessionId(
eventsDTO.pid,
userAgent,
ip,
@@ -1801,6 +1803,7 @@ export class AnalyticsController {
)
await this.analyticsService.recordSessionActivity(
+ sid,
psid,
eventsDTO.pid,
profileId,
@@ -1811,6 +1814,7 @@ export class AnalyticsController {
const transformed = eventTransformer({
type: 'custom_event',
psid,
+ sid,
profileId,
pid: eventsDTO.pid,
host: this.analyticsService.getHostFromOrigin(headers.origin),
@@ -1967,13 +1971,13 @@ export class AnalyticsController {
await this.analyticsService.validateHeartbeat(logDTO, origin, ip)
- const { exists, psid } = await this.analyticsService.getSessionId(
+ const { exists, psid, sid } = await this.analyticsService.getSessionId(
pid,
userAgent,
ip,
)
- if (!exists) {
+ if (!exists || !sid) {
throw new ForbiddenException(
'The heartbeat was not saved because there is no session for this request. Please, send a pageview or custom event request first to initialise the session.',
)
@@ -1987,7 +1991,7 @@ export class AnalyticsController {
)
await this.analyticsService.extendSessionTTL(psid)
- await this.analyticsService.recordSessionActivity(psid, pid, profileId)
+ await this.analyticsService.recordSessionActivity(sid, psid, pid, profileId)
this.logger.log(`pid: ${pid}, psid: ${psid}`, 'POST /analytics/hb')
@@ -2018,7 +2022,7 @@ export class AnalyticsController {
const project = await this.analyticsService.validate(logDTO, origin, ip)
- const [unique, psid] =
+ const [unique, psid, sid] =
await this.analyticsService.generateAndStoreSessionId(
logDTO.pid,
userAgent,
@@ -2033,6 +2037,7 @@ export class AnalyticsController {
)
await this.analyticsService.recordSessionActivity(
+ sid,
psid,
logDTO.pid,
profileId,
@@ -2067,6 +2072,7 @@ export class AnalyticsController {
const transformed = eventTransformer({
type: 'pageview',
psid,
+ sid,
profileId,
pid: logDTO.pid,
host: this.analyticsService.getHostFromOrigin(headers.origin),
@@ -2111,6 +2117,7 @@ export class AnalyticsController {
perfTransformed = eventTransformer({
type: 'performance',
psid,
+ sid,
profileId,
pid: logDTO.pid,
host: this.analyticsService.getHostFromOrigin(headers.origin),
@@ -2201,7 +2208,7 @@ export class AnalyticsController {
const project = await this.analyticsService.validate(logDTO, origin, ip)
- const [, psid] = await this.analyticsService.generateAndStoreSessionId(
+ const [, psid, sid] = await this.analyticsService.generateAndStoreSessionId(
logDTO.pid,
userAgent,
ip,
@@ -2214,6 +2221,7 @@ export class AnalyticsController {
)
await this.analyticsService.recordSessionActivity(
+ sid,
psid,
logDTO.pid,
profileId,
@@ -2240,6 +2248,7 @@ export class AnalyticsController {
const transformed = eventTransformer({
type: 'pageview',
psid,
+ sid,
profileId,
pid: logDTO.pid,
host: this.analyticsService.getHostFromOrigin(headers.origin),
@@ -3372,14 +3381,15 @@ export class AnalyticsController {
// identified profile (e.g. a second account on a shared device) - the
// session stays with the current identity until new events re-stamp it.
if (linked) {
- const { exists, psid } = await this.analyticsService.getSessionId(
+ const { exists, psid, sid } = await this.analyticsService.getSessionId(
dto.pid,
userAgent,
ip,
)
- if (exists) {
+ if (exists && sid) {
await this.analyticsService.recordSessionActivity(
+ sid,
psid,
dto.pid,
userProfileId,
diff --git a/backend/apps/cloud/src/analytics/analytics.service.ts b/backend/apps/cloud/src/analytics/analytics.service.ts
index 3cf5ccd5d..8d8a54970 100644
--- a/backend/apps/cloud/src/analytics/analytics.service.ts
+++ b/backend/apps/cloud/src/analytics/analytics.service.ts
@@ -142,6 +142,60 @@ const isValidLocale = (lc: string): boolean => {
// 2 minutes
const hasTimeComponent = (date: string): boolean => /\d{2}:\d{2}/.test(date)
+const LEGACY_SESSION_FLAG = '1'
+
+redis.defineCommand('swetrixResolveSession', {
+ numberOfKeys: 1,
+ lua: `
+ local existing = redis.call('GET', KEYS[1])
+ if existing and existing ~= ARGV[3] then
+ redis.call('EXPIRE', KEYS[1], ARGV[2])
+ return {existing, '0'}
+ end
+ redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[2])
+ if existing == ARGV[3] then
+ return {ARGV[1], '0'}
+ end
+ return {ARGV[1], '1'}
+ `,
+})
+
+redis.defineCommand('swetrixPeekSession', {
+ numberOfKeys: 1,
+ lua: `
+ local existing = redis.call('GET', KEYS[1])
+ if not existing then
+ return ''
+ end
+ if existing == ARGV[3] then
+ local ttl = redis.call('TTL', KEYS[1])
+ if ttl < 0 then
+ ttl = tonumber(ARGV[2])
+ end
+ redis.call('SET', KEYS[1], ARGV[1], 'EX', ttl)
+ return ARGV[1]
+ end
+ return existing
+ `,
+})
+
+interface RedisWithSessionScripts {
+ swetrixResolveSession(
+ key: string,
+ candidateSid: string,
+ ttlSeconds: number,
+ legacyFlag: string,
+ ): Promise<[string, string]>
+ swetrixPeekSession(
+ key: string,
+ candidateSid: string,
+ ttlSeconds: number,
+ legacyFlag: string,
+ ): Promise<string>
+}
+
+const sessionScripts = redis as unknown as RedisWithSessionScripts
+
const LIVE_SESSION_THRESHOLD_SECONDS = 120
const MAX_FILTERS = 100
const MAX_FILTER_VALUES = 100
@@ -2338,55 +2392,57 @@ export class AnalyticsService {
return `ses:${psid}`
}
+ private generateSid(psid: string): string {
+ return crypto
+ .createHash('sha256')
+ .update(`${psid}:${Date.now()}:${crypto.randomBytes(16).toString('hex')}`)
+ .digest()
+ .readBigUInt64BE(0)
+ .toString()
+ }
+
async getSessionId(
pid: string,
userAgent: string,
ip: string,
- ): Promise<{ exists: boolean; psid: string }> {
+ ): Promise<{ exists: boolean; psid: string; sid: string | null }> {
const salt = await this.saltService.getSaltForSession()
const psid = this.derivePsidFromInputs(pid, userAgent, ip, salt)
const sessionKey = this.getSessionKey(psid)
- const exists = Boolean(await redis.exists(sessionKey))
+ const sid = (await sessionScripts.swetrixPeekSession(
+ sessionKey,
+ this.generateSid(psid),
+ UNIQUE_SESSION_LIFE_TIME,
+ LEGACY_SESSION_FLAG,
+ )) as string
- return { exists, psid }
+ return { exists: Boolean(sid), psid, sid: sid || null }
}
async generateAndStoreSessionId(
pid: string,
userAgent: string,
ip: string,
- ): Promise<[boolean, string]> {
+ ): Promise<[boolean, string, string]> {
const salt = await this.saltService.getSaltForSession()
const psid = this.derivePsidFromInputs(pid, userAgent, ip, salt)
const sessionKey = this.getSessionKey(psid)
- // Use SET with NX (only set if not exists) for atomic "new session" detection
- // This prevents race conditions where multiple workers could both see the key
- // doesn't exist and both think they're creating a "new session"
- const result = await redis.set(
+ const [sid, isNewFlag] = (await sessionScripts.swetrixResolveSession(
sessionKey,
- '1',
- 'EX',
+ this.generateSid(psid),
UNIQUE_SESSION_LIFE_TIME,
- 'NX',
- )
-
- // If SET NX succeeded (returned 'OK'), this is a new session
- // If it returned null, the session already existed
- const isNew = result === 'OK'
-
- if (!isNew) {
- // Session already exists, extend TTL
- await this.extendSessionTTL(psid)
- }
+ LEGACY_SESSION_FLAG,
+ )) as [string, string]
- return [isNew, psid]
+ return [isNewFlag === '1', psid, sid]
}
+ // EXPIRE, not SET: the value is the sid and must survive a heartbeat.
async extendSessionTTL(psid: string): Promise<void> {
const sessionKey = this.getSessionKey(psid)
- await redis.set(sessionKey, '1', 'EX', UNIQUE_SESSION_LIFE_TIME)
+ await redis.expire(sessionKey, UNIQUE_SESSION_LIFE_TIME)
}
static readonly PROFILE_PREFIX_ANON = 'anon_'
@@ -2781,46 +2837,33 @@ export class AnalyticsService {
)`
}
- private getSessionFirstSeenKey(pid: string, psid: string): string {
- return `ses:fs:${pid}:${psid}`
+ private getSessionFirstSeenKey(pid: string, sid: string): string {
+ return `ses:fs:${pid}:${sid}`
}
async recordSessionActivity(
+ sid: string,
psid: string,
pid: string,
profileId: string,
): Promise<void> {
const now = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')
- const cacheKey = this.getSessionFirstSeenKey(pid, psid)
+ const cacheKey = this.getSessionFirstSeenKey(pid, sid)
try {
- let firstSeen = await redis.get(cacheKey)
-
- if (!firstSeen) {
- const query = `
- SELECT minOrNull(firstSeen) AS firstSeen
- FROM sessions
- WHERE psid = {psid:UInt64}
- AND pid = {pid:FixedString(12)}
- `
-
- const { data } = await clickhouse
- .query({
- query,
- query_params: { psid, pid },
- })
- .then((resultSet) =>
- resultSet.json<{
- firstSeen: string | null
- }>(),
- )
+ const stored = await redis.set(
+ cacheKey,
+ now,
+ 'EX',
+ UNIQUE_SESSION_LIFE_TIME,
+ 'NX',
+ )
- const existingSession = data[0]
- firstSeen = existingSession?.firstSeen
- ? dayjs.utc(existingSession.firstSeen).format('YYYY-MM-DD HH:mm:ss')
- : now
+ let firstSeen = now
- await redis.set(cacheKey, firstSeen, 'EX', UNIQUE_SESSION_LIFE_TIME)
+ if (stored !== 'OK') {
+ firstSeen = (await redis.get(cacheKey)) || now
+ await redis.expire(cacheKey, UNIQUE_SESSION_LIFE_TIME)
}
await clickhouse.insert({
@@ -2828,6 +2871,7 @@ export class AnalyticsService {
format: 'JSONEachRow',
values: [
{
+ sid,
psid,
pid,
profileId,
@@ -2843,7 +2887,7 @@ export class AnalyticsService {
}
async checkSessionExistsInClickHouse(
- psid: string,
+ sid: string,
pid: string,
): Promise<boolean> {
const cutoff = dayjs
@@ -2855,7 +2899,7 @@ export class AnalyticsService {
const query = `
SELECT 1
FROM sessions FINAL
- WHERE psid = {psid:UInt64}
+ WHERE sid = {sid:UInt64}
AND pid = {pid:FixedString(12)}
AND lastSeen >= {cutoff:DateTime}
LIMIT 1
@@ -2864,7 +2908,7 @@ export class AnalyticsService {
const { data } = await clickhouse
.query({
query,
- query_params: { psid, pid, cutoff },
+ query_params: { sid, pid, cutoff },
})
.then((resultSet) => resultSet.json())
@@ -2881,10 +2925,14 @@ export class AnalyticsService {
): Promise<number | null> {
try {
const query = `
- SELECT dateDiff('second', min(firstSeen), max(lastSeen)) as duration
- FROM sessions
- WHERE psid = {psid:UInt64}
- AND pid = {pid:FixedString(12)}
+ SELECT max(duration) as duration
+ FROM (
+ SELECT dateDiff('second', min(firstSeen), max(lastSeen)) as duration
+ FROM sessions
+ WHERE psid = {psid:UInt64}
+ AND pid = {pid:FixedString(12)}
+ GROUP BY sid
+ )
`
const { data } = await clickhouse
@@ -2934,9 +2982,13 @@ export class AnalyticsService {
pid: string,
userAgent: string,
ip: string,
- ): Promise<string> {
- const [, psid] = await this.generateAndStoreSessionId(pid, userAgent, ip)
- return psid
+ ): Promise<{ psid: string; sid: string }> {
+ const [, psid, sid] = await this.generateAndStoreSessionId(
+ pid,
+ userAgent,
+ ip,
+ )
+ return { psid, sid }
}
async startSessionReplay(
@@ -2954,7 +3006,7 @@ export class AnalyticsService {
)
}
- const psid = await this.resolveReplaySession(pid, userAgent, ip)
+ const { psid, sid } = await this.resolveReplaySession(pid, userAgent, ip)
const resolvedProfileId = await this.generateProfileId(
pid,
userAgent,
@@ -2962,7 +3014,7 @@ export class AnalyticsService {
profileId,
)
- await this.recordSessionActivity(psid, pid, resolvedProfileId)
+ await this.recordSessionActivity(sid, psid, pid, resolvedProfileId)
const retention = this.getSessionReplayRetention(project)
const replayStart = await this.reserveActiveReplayStart(pid, psid, replayId)
@@ -3326,7 +3378,7 @@ export class AnalyticsService {
this.validateSessionReplayEventSizes(events)
- const psid = await this.resolveReplaySession(pid, userAgent, ip)
+ const { psid } = await this.resolveReplaySession(pid, userAgent, ip)
const retention = this.getSessionReplayRetention(project)
const payload = JSON.stringify({ events })
const uncompressedBytes = Buffer.byteLength(payload)
@@ -4096,14 +4148,24 @@ export class AnalyticsService {
),
session_duration_agg AS (
SELECT
- CAST(psid, 'String') AS psidCasted,
+ psidCasted,
pid,
- dateDiff('second', min(firstSeen), max(lastSeen)) as avg_duration,
- argMax(coalesce(nullIf(pam.userProfileId, ''), s.profileId), lastSeen) as profileId
- FROM sessions AS s
- LEFT JOIN profile_alias_map pam ON s.profileId = pam.anonProfileId
- WHERE pid = {pid:FixedString(12)}
- AND psid IN (SELECT psid FROM funnel_qualified)
+ sum(session_duration) as avg_duration,
+ argMax(sessionProfileId, lastActivity) as profileId
+ FROM (
+ SELECT
+ CAST(psid, 'String') AS psidCasted,
+ pid,
+ sid,
+ dateDiff('second', min(firstSeen), max(lastSeen)) as session_duration,
+ max(lastSeen) as lastActivity,
+ argMax(coalesce(nullIf(pam.userProfileId, ''), s.profileId), lastSeen) as sessionProfileId
+ FROM sessions AS s
+ LEFT JOIN profile_alias_map pam ON s.profileId = pam.anonProfileId
+ WHERE pid = {pid:FixedString(12)}
+ AND psid IN (SELECT psid FROM funnel_qualified)
+ GROUP BY psidCasted, pid, sid
+ )
GROUP BY psidCasted, pid
),
first_session_per_profile AS (
@@ -4254,14 +4316,24 @@ export class AnalyticsService {
),
session_duration_agg AS (
SELECT
- CAST(psid, 'String') AS psidCasted,
+ psidCasted,
pid,
- dateDiff('second', min(firstSeen), max(lastSeen)) as avg_duration,
- argMax(coalesce(nullIf(pam.userProfileId, ''), s.profileId), lastSeen) as profileId
- FROM sessions AS s
- LEFT JOIN profile_alias_map pam ON s.profileId = pam.anonProfileId
- WHERE pid = {pid:FixedString(12)}
- AND psid IN (SELECT psid FROM journey_qualified)
+ sum(session_duration) as avg_duration,
+ argMax(sessionProfileId, lastActivity) as profileId
+ FROM (
+ SELECT
+ CAST(psid, 'String') AS psidCasted,
+ pid,
+ sid,
+ dateDiff('second', min(firstSeen), max(lastSeen)) as session_duration,
+ max(lastSeen) as lastActivity,
+ argMax(coalesce(nullIf(pam.userProfileId, ''), s.profileId), lastSeen) as sessionProfileId
+ FROM sessions AS s
+ LEFT JOIN profile_alias_map pam ON s.profileId = pam.anonProfileId
+ WHERE pid = {pid:FixedString(12)}
+ AND psid IN (SELECT psid FROM journey_qualified)
+ GROUP BY psidCasted, pid, sid
+ )
GROUP BY psidCasted, pid
),
first_session_per_profile AS (
@@ -4745,7 +4817,7 @@ export class AnalyticsService {
const query = `
SELECT
pid,
- uniqExact(psid) as totalSessions
+ uniqExact(coalesce(sid, psid)) as totalSessions
FROM events
WHERE
pid IN {pids:Array(FixedString(12))}
@@ -4830,7 +4902,7 @@ export class AnalyticsService {
WITH analytics_counts AS (
SELECT
count(*) AS all,
- count(DISTINCT psid) AS unique,
+ count(DISTINCT coalesce(sid, psid)) AS unique,
count(DISTINCT profileId) AS users
FROM events
WHERE
@@ -4990,7 +5062,7 @@ export class AnalyticsService {
SELECT
1 AS sortOrder,
count(*) AS all,
- count(DISTINCT psid) AS unique,
+ count(DISTINCT coalesce(sid, psid)) AS unique,
count(DISTINCT profileId) AS users
FROM events
WHERE
@@ -5003,7 +5075,7 @@ export class AnalyticsService {
SELECT avgOrNull(duration) as sdur
FROM (
SELECT
- psid,
+ sid,
dateDiff('second', min(firstSeen), max(lastSeen)) as duration
FROM sessions
WHERE pid = {pid:FixedString(12)}
@@ -5055,7 +5127,7 @@ export class AnalyticsService {
SELECT
2 AS sortOrder,
count(*) AS all,
- count(DISTINCT psid) AS unique,
+ count(DISTINCT coalesce(sid, psid)) AS unique,
count(DISTINCT profileId) AS users
FROM events
WHERE
@@ -5068,7 +5140,7 @@ export class AnalyticsService {
SELECT avgOrNull(duration) as sdur
FROM (
SELECT
- psid,
+ sid,
dateDiff('second', min(firstSeen), max(lastSeen)) as duration
FROM sessions
WHERE pid = {pid:FixedString(12)}
@@ -5975,6 +6047,7 @@ export class AnalyticsService {
SELECT
pid,
psid,
+ coalesce(sid, psid) as sessionId,
${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created
FROM events
PREWHERE pid = {pid:FixedString(12)} AND type = 'pageview'
@@ -5984,37 +6057,40 @@ export class AnalyticsService {
LEFT JOIN (
SELECT
pid,
- psid,
+ sid,
dateDiff('second', min(firstSeen), max(lastSeen)) as duration
FROM sessions
WHERE pid = {pid:FixedString(12)}
- GROUP BY pid, psid
+ GROUP BY pid, sid
) as sessions_data
ON subquery.pid = sessions_data.pid
- AND subquery.psid = sessions_data.psid
+ AND subquery.sessionId = sessions_data.sid
LEFT JOIN (
SELECT
pid,
- psid,
+ sessionId,
count() as pageviews
- FROM events
- PREWHERE pid = {pid:FixedString(12)} AND type = 'pageview'
- WHERE created BETWEEN ${tzFromDate} AND ${tzToDate}
- AND psid IS NOT NULL
- AND psid != 0
- AND psid IN (
- SELECT DISTINCT psid
- FROM events
- PREWHERE pid = {pid:FixedString(12)} AND type = 'pageview'
- WHERE created BETWEEN ${tzFromDate} AND ${tzToDate}
- AND psid IS NOT NULL
- AND psid != 0
- ${filtersQuery}
- )
- GROUP BY pid, psid
+ FROM (
+ SELECT pid, coalesce(sid, psid) as sessionId, psid
+ FROM events
+ PREWHERE pid = {pid:FixedString(12)} AND type = 'pageview'
+ WHERE created BETWEEN ${tzFromDate} AND ${tzToDate}
+ AND psid IS NOT NULL
+ AND psid != 0
+ AND psid IN (
+ SELECT DISTINCT psid
+ FROM events
+ PREWHERE pid = {pid:FixedString(12)} AND type = 'pageview'
+ WHERE created BETWEEN ${tzFromDate} AND ${tzToDate}
+ AND psid IS NOT NULL
+ AND psid != 0
+ ${filtersQuery}
+ )
+ )
+ GROUP BY pid, sessionId
) as session_counts
ON subquery.pid = session_counts.pid
- AND subquery.psid = session_counts.psid
+ AND subquery.sessionId = session_counts.sessionId
GROUP BY ${groupBy}
ORDER BY ${groupBy}
`
@@ -6133,6 +6209,7 @@ export class AnalyticsService {
SELECT
pid,
psid,
+ coalesce(sid, psid) as sessionId,
${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created
FROM events
WHERE pid = {pid:FixedString(12)}
@@ -6143,14 +6220,14 @@ export class AnalyticsService {
LEFT JOIN (
SELECT
pid,
- psid,
+ sid,
dateDiff('second', min(firstSeen), max(lastSeen)) as duration
FROM sessions
WHERE pid = {pid:FixedString(12)}
- GROUP BY pid, psid
+ GROUP BY pid, sid
) as sessions_data
ON subquery.pid = sessions_data.pid
- AND subquery.psid = sessions_data.psid
+ AND subquery.sessionId = sessions_data.sid
GROUP BY ${groupBy}
ORDER BY ${groupBy}
`
@@ -6472,19 +6549,6 @@ export class AnalyticsService {
}
}
- // Reconstructs the number of concurrent (live) visitors for each display bucket.
- // A visitor is live at a given minute if they fired a pageview or custom event
- // within the preceding ONLINE_VISITORS_WINDOW_MINUTES - the exact rule the
- // real-time live visitors counter applies. Every active minute is stretched
- // across that window, visitors are deduplicated per minute, and each bucket
- // reports its peak.
- //
- // This deliberately reads from `events` rather than from the `sessions` table:
- // a psid is derived from a salt that only rotates daily, so a single psid
- // covers every visit a person makes in a day. Treating its
- // [firstSeen, lastSeen] range as one continuous presence counted returning
- // visitors as live for the hours they were away, and `sessions` additionally
- // records activity for requests that never become a countable event.
generateConcurrencyAggregationQuery(timeBucket: TimeBucketType): string {
const timeBucketFunc = timeBucketConversion[timeBucket]
const [selector, groupBy] = this.getGroupSubquery(timeBucket)
@@ -6566,10 +6630,6 @@ export class AnalyticsService {
customEVFilterApplied,
)
- // Concurrency is only computed when explicitly requested (the live visitors
- // metric is off by default). It is also not wired up to the dashboard
- // filters yet, so return zeros rather than misleading unfiltered data
- // whenever a filter is active
const shouldComputeConcurrency =
includeConcurrency && !customEVFilterApplied && !filtersQuery
@@ -8133,13 +8193,23 @@ export class AnalyticsService {
),
session_duration_agg AS (
SELECT
- toString(psid) AS psidCasted,
+ psidCasted,
pid,
- dateDiff('second', min(firstSeen), max(lastSeen)) as avg_duration,
- argMax(coalesce(nullIf(pam.userProfileId, ''), s.profileId), lastSeen) as profileId
- FROM sessions AS s
- LEFT JOIN profile_alias_map pam ON s.profileId = pam.anonProfileId
- WHERE pid = {pid:FixedString(12)}
+ sum(session_duration) as avg_duration,
+ argMax(sessionProfileId, lastActivity) as profileId
+ FROM (
+ SELECT
+ toString(psid) AS psidCasted,
+ pid,
+ sid,
+ dateDiff('second', min(firstSeen), max(lastSeen)) as session_duration,
+ max(lastSeen) as lastActivity,
+ argMax(coalesce(nullIf(pam.userProfileId, ''), s.profileId), lastSeen) as sessionProfileId
+ FROM sessions AS s
+ LEFT JOIN profile_alias_map pam ON s.profileId = pam.anonProfileId
+ WHERE pid = {pid:FixedString(12)}
+ GROUP BY psidCasted, pid, sid
+ )
GROUP BY psidCasted, pid
),
replay_summary AS (
@@ -8404,13 +8474,23 @@ export class AnalyticsService {
),
session_duration_agg AS (
SELECT
- toString(psid) AS psidCasted,
+ psidCasted,
pid,
- dateDiff('second', min(firstSeen), max(lastSeen)) as avg_duration,
- argMax(coalesce(nullIf(pam.userProfileId, ''), s.profileId), lastSeen) as profileId
- FROM sessions AS s
- LEFT JOIN profile_alias_map pam ON s.profileId = pam.anonProfileId
- WHERE pid = {pid:FixedString(12)}
+ sum(session_duration) as avg_duration,
+ argMax(sessionProfileId, lastActivity) as profileId
+ FROM (
+ SELECT
+ toString(psid) AS psidCasted,
+ pid,
+ sid,
+ dateDiff('second', min(firstSeen), max(lastSeen)) as session_duration,
+ max(lastSeen) as lastActivity,
+ argMax(coalesce(nullIf(pam.userProfileId, ''), s.profileId), lastSeen) as sessionProfileId
+ FROM sessions AS s
+ LEFT JOIN profile_alias_map pam ON s.profileId = pam.anonProfileId
+ WHERE pid = {pid:FixedString(12)}
+ GROUP BY psidCasted, pid, sid
+ )
GROUP BY psidCasted, pid
),
replays_enriched AS (
@@ -9320,12 +9400,20 @@ export class AnalyticsService {
),
session_duration_agg AS (
SELECT
- CAST(psid, 'String') AS psidCasted,
+ psidCasted,
pid,
- dateDiff('second', min(firstSeen), max(lastSeen)) as avg_duration
- FROM sessions
- WHERE pid = {pid:FixedString(12)}
- AND profileId IN {profileIds:Array(String)}
+ sum(session_duration) as avg_duration
+ FROM (
+ SELECT
+ CAST(psid, 'String') AS psidCasted,
+ pid,
+ sid,
+ dateDiff('second', min(firstSeen), max(lastSeen)) as session_duration
+ FROM sessions
+ WHERE pid = {pid:FixedString(12)}
+ AND profileId IN {profileIds:Array(String)}
+ GROUP BY psidCasted, pid, sid
+ )
GROUP BY psidCasted, pid
)
SELECT
@@ -9408,7 +9496,7 @@ export class AnalyticsService {
count(*) as count,
max(created) as last_seen,
count(DISTINCT profileId) as users,
- count(DISTINCT psid) as sessions,
+ count(DISTINCT coalesce(sid, psid)) as sessions,
status.status
FROM (
SELECT
@@ -9615,7 +9703,7 @@ export class AnalyticsService {
// Get total sessions from all matching events for the time range
const queryTotalSessions = `
- SELECT count(DISTINCT psid) as totalSessions
+ SELECT count(DISTINCT coalesce(sid, psid)) as totalSessions
FROM events
WHERE pid = {pid:FixedString(12)}
AND type IN ('pageview', 'custom_event', 'error')
diff --git a/backend/apps/cloud/src/analytics/heartbeat.gateway.ts b/backend/apps/cloud/src/analytics/heartbeat.gateway.ts
index 97674a250..a43864cb9 100644
--- a/backend/apps/cloud/src/analytics/heartbeat.gateway.ts
+++ b/backend/apps/cloud/src/analytics/heartbeat.gateway.ts
@@ -17,6 +17,7 @@ const HEARTBEAT_INTERVAL_MS = 60_000
interface WsSession {
pid: string
psid: string
+ sid: string
profileId: string
intervalRef: ReturnType<typeof setInterval>
}
@@ -81,13 +82,13 @@ export class HeartbeatGateway
}
// Get or validate existing session
- const { exists, psid } = await this.analyticsService.getSessionId(
+ const { exists, psid, sid } = await this.analyticsService.getSessionId(
pid,
userAgent,
ip,
)
- if (!exists) {
+ if (!exists || !sid) {
this.logger.warn(
`[WS HB] Connection rejected: no session exists for pid ${pid}`,
'HeartbeatGateway',
@@ -110,13 +111,19 @@ export class HeartbeatGateway
// Initial heartbeat
await this.analyticsService.extendSessionTTL(psid)
- await this.analyticsService.recordSessionActivity(psid, pid, profileId)
+ await this.analyticsService.recordSessionActivity(
+ sid,
+ psid,
+ pid,
+ profileId,
+ )
// Set up interval for periodic heartbeat
const intervalRef = setInterval(async () => {
try {
await this.analyticsService.extendSessionTTL(psid)
await this.analyticsService.recordSessionActivity(
+ sid,
psid,
pid,
profileId,
@@ -134,6 +141,7 @@ export class HeartbeatGateway
this.sessions.set(client.id, {
pid,
psid,
+ sid,
profileId,
intervalRef,
})
@@ -159,6 +167,7 @@ export class HeartbeatGateway
// Final session activity update
try {
await this.analyticsService.recordSessionActivity(
+ session.sid,
session.psid,
session.pid,
session.profileId,
diff --git a/backend/apps/cloud/src/analytics/utils/transformers.ts b/backend/apps/cloud/src/analytics/utils/transformers.ts
index 78649a6a8..e192424b0 100644
--- a/backend/apps/cloud/src/analytics/utils/transformers.ts
+++ b/backend/apps/cloud/src/analytics/utils/transformers.ts
@@ -26,6 +26,7 @@ const processMetaKV = (
interface CommonOptions {
pid: string
psid?: string | null
+ sid?: string | null
profileId?: string | null
host?: string | null
pg?: string | null
@@ -108,6 +109,7 @@ type EventTransformerOptions =
const buildCommon = (opts: CommonOptions) => ({
pid: opts.pid,
psid: opts.psid ?? null,
+ sid: opts.sid ?? null,
profileId: opts.profileId ?? null,
host: opts.host || null,
pg: opts.pg || null,
@@ -178,6 +180,7 @@ export const eventTransformer = (opts: EventTransformerOptions) => {
type: 'performance' as const,
pid: opts.pid,
psid: opts.psid ?? null,
+ sid: opts.sid ?? null,
profileId: opts.profileId ?? null,
host: opts.host || null,
pg: opts.pg || null,
diff --git a/backend/apps/cloud/src/analytics/v2/registry/metrics.ts b/backend/apps/cloud/src/analytics/v2/registry/metrics.ts
index 84b715f91..964eb8dfe 100644
--- a/backend/apps/cloud/src/analytics/v2/registry/metrics.ts
+++ b/backend/apps/cloud/src/analytics/v2/registry/metrics.ts
@@ -24,7 +24,18 @@ export const V2_METRICS: V2MetricDef[] = [
customEVFilterApplied ? 'count(*)' : 'count(DISTINCT psid)',
isDefault: true,
format: 'integer',
- description: 'Unique sessions',
+ description: 'Unique visitors (a visitor is counted once per day)',
+ },
+ {
+ api: 'sessions',
+ types: ['traffic'],
+ sqlExpr: ({ customEVFilterApplied }) =>
+ customEVFilterApplied
+ ? 'count(*)'
+ : 'count(DISTINCT coalesce(sid, psid))',
+ format: 'integer',
+ description:
+ 'Session count. A session ends after 30 minutes of inactivity, so one visitor can have several in a day',
},
{
api: 'pageviews',
diff --git a/backend/apps/cloud/src/common/constants.ts b/backend/apps/cloud/src/common/constants.ts
index 3ffb08933..8e075989e 100644
--- a/backend/apps/cloud/src/common/constants.ts
+++ b/backend/apps/cloud/src/common/constants.ts
@@ -61,9 +61,6 @@ const redisUserUsageinfoCacheTimeout = 300
// 30 minues -> the amount of time analytics requests within one session are counted as non-unique
const UNIQUE_SESSION_LIFE_TIME = 1800
-// A visitor counts as live for this long after their last pageview or custom
-// event. Shared by the real-time live visitors counter and the historical live
-// visitors chart series so the two can never disagree.
const ONLINE_VISITORS_WINDOW_MINUTES = 5
// send email warning when 85% of events in tier are used
diff --git a/backend/apps/community/src/analytics/analytics.controller.ts b/backend/apps/community/src/analytics/analytics.controller.ts
index d49b8423e..5ca02528f 100644
--- a/backend/apps/community/src/analytics/analytics.controller.ts
+++ b/backend/apps/community/src/analytics/analytics.controller.ts
@@ -1339,7 +1339,7 @@ export class AnalyticsController {
const { deviceType, browserName, browserVersion, osName, osVersion } =
await this.analyticsService.getRequestInformation(headers)
- const [, psid] = await this.analyticsService.generateAndStoreSessionId(
+ const [, psid, sid] = await this.analyticsService.generateAndStoreSessionId(
eventsDTO.pid,
userAgent,
ip,
@@ -1353,6 +1353,7 @@ export class AnalyticsController {
)
await this.analyticsService.recordSessionActivity(
+ sid,
psid,
eventsDTO.pid,
profileId,
@@ -1363,6 +1364,7 @@ export class AnalyticsController {
const transformed = eventTransformer({
type: 'custom_event',
psid,
+ sid,
profileId,
pid: eventsDTO.pid,
host: this.analyticsService.getHostFromOrigin(headers.origin),
@@ -1515,13 +1517,13 @@ export class AnalyticsController {
await this.analyticsService.validateHeartbeat(logDTO, origin, ip)
- const { exists, psid } = await this.analyticsService.getSessionId(
+ const { exists, psid, sid } = await this.analyticsService.getSessionId(
pid,
userAgent,
ip,
)
- if (!exists) {
+ if (!exists || !sid) {
throw new ForbiddenException(
'The heartbeat was not saved because there is no session for this request. Please, send a pageview or custom event request first to initialise the session.',
)
@@ -1535,7 +1537,7 @@ export class AnalyticsController {
)
await this.analyticsService.extendSessionTTL(psid)
- await this.analyticsService.recordSessionActivity(psid, pid, profileId)
+ await this.analyticsService.recordSessionActivity(sid, psid, pid, profileId)
this.logger.log(`pid: ${pid}, psid: ${psid}`, 'POST /analytics/hb')
@@ -1565,7 +1567,7 @@ export class AnalyticsController {
const project = await this.analyticsService.validate(logDTO, origin, ip)
- const [unique, psid] =
+ const [unique, psid, sid] =
await this.analyticsService.generateAndStoreSessionId(
logDTO.pid,
userAgent,
@@ -1580,6 +1582,7 @@ export class AnalyticsController {
)
await this.analyticsService.recordSessionActivity(
+ sid,
psid,
logDTO.pid,
profileId,
@@ -1614,6 +1617,7 @@ export class AnalyticsController {
const transformed = eventTransformer({
type: 'pageview',
psid,
+ sid,
profileId,
pid: logDTO.pid,
host: this.analyticsService.getHostFromOrigin(headers.origin),
@@ -1746,7 +1750,7 @@ export class AnalyticsController {
const project = await this.analyticsService.validate(logDTO, origin, ip)
- const [, psid] = await this.analyticsService.generateAndStoreSessionId(
+ const [, psid, sid] = await this.analyticsService.generateAndStoreSessionId(
logDTO.pid,
userAgent,
ip,
@@ -1761,6 +1765,7 @@ export class AnalyticsController {
)
await this.analyticsService.recordSessionActivity(
+ sid,
psid,
logDTO.pid,
profileId,
@@ -1787,6 +1792,7 @@ export class AnalyticsController {
const transformed = eventTransformer({
type: 'pageview',
psid,
+ sid,
profileId,
pid: logDTO.pid,
host: this.analyticsService.getHostFromOrigin(headers.origin),
@@ -2136,7 +2142,7 @@ export class AnalyticsController {
const project = await this.analyticsService.validate(errorDTO, origin, ip)
- const [, psid] = await this.analyticsService.generateAndStoreSessionId(
+ const [, psid, sid] = await this.analyticsService.generateAndStoreSessionId(
errorDTO.pid,
userAgent,
ip,
@@ -2150,6 +2156,7 @@ export class AnalyticsController {
)
await this.analyticsService.recordSessionActivity(
+ sid,
psid,
errorDTO.pid,
profileId,
@@ -2182,6 +2189,7 @@ export class AnalyticsController {
const transformed = eventTransformer({
type: 'error',
psid,
+ sid,
profileId,
eid: this.analyticsService.getErrorID(errorDTO),
pid: errorDTO.pid,
@@ -2863,14 +2871,15 @@ export class AnalyticsController {
// identified profile (e.g. a second account on a shared device) - the
// session stays with the current identity until new events re-stamp it.
if (linked) {
- const { exists, psid } = await this.analyticsService.getSessionId(
+ const { exists, psid, sid } = await this.analyticsService.getSessionId(
dto.pid,
userAgent,
ip,
)
- if (exists) {
+ if (exists && sid) {
await this.analyticsService.recordSessionActivity(
+ sid,
psid,
dto.pid,
userProfileId,
diff --git a/backend/apps/community/src/analytics/analytics.service.ts b/backend/apps/community/src/analytics/analytics.service.ts
index d6ef2b686..f0121df9e 100644
--- a/backend/apps/community/src/analytics/analytics.service.ts
+++ b/backend/apps/community/src/analytics/analytics.service.ts
@@ -125,6 +125,60 @@ const isValidLocale = (lc: string): boolean => {
// 2 minutes
const hasTimeComponent = (date: string): boolean => /\d{2}:\d{2}/.test(date)
+const LEGACY_SESSION_FLAG = '1'
+
+redis.defineCommand('swetrixResolveSession', {
+ numberOfKeys: 1,
+ lua: `
+ local existing = redis.call('GET', KEYS[1])
+ if existing and existing ~= ARGV[3] then
+ redis.call('EXPIRE', KEYS[1], ARGV[2])
+ return {existing, '0'}
+ end
+ redis.call('SET', KEYS[1], ARGV[1], 'EX', ARGV[2])
+ if existing == ARGV[3] then
+ return {ARGV[1], '0'}
+ end
+ return {ARGV[1], '1'}
+ `,
+})
+
+redis.defineCommand('swetrixPeekSession', {
+ numberOfKeys: 1,
+ lua: `
+ local existing = redis.call('GET', KEYS[1])
+ if not existing then
+ return ''
+ end
+ if existing == ARGV[3] then
+ local ttl = redis.call('TTL', KEYS[1])
+ if ttl < 0 then
+ ttl = tonumber(ARGV[2])
+ end
+ redis.call('SET', KEYS[1], ARGV[1], 'EX', ttl)
+ return ARGV[1]
+ end
+ return existing
+ `,
+})
+
+interface RedisWithSessionScripts {
+ swetrixResolveSession(
+ key: string,
+ candidateSid: string,
+ ttlSeconds: number,
+ legacyFlag: string,
+ ): Promise<[string, string]>
+ swetrixPeekSession(
+ key: string,
+ candidateSid: string,
+ ttlSeconds: number,
+ legacyFlag: string,
+ ): Promise<string>
+}
+
+const sessionScripts = redis as unknown as RedisWithSessionScripts
+
const LIVE_SESSION_THRESHOLD_SECONDS = 120
const MAX_FILTERS = 100
const MAX_FILTER_VALUES = 100
@@ -1982,55 +2036,57 @@ export class AnalyticsService {
return `ses:${psid}`
}
+ private generateSid(psid: string): string {
+ return crypto
+ .createHash('sha256')
+ .update(`${psid}:${Date.now()}:${crypto.randomBytes(16).toString('hex')}`)
+ .digest()
+ .readBigUInt64BE(0)
+ .toString()
+ }
+
async getSessionId(
pid: string,
userAgent: string,
ip: string,
- ): Promise<{ exists: boolean; psid: string }> {
- const salt = await this.saltService.getSaltForProject(pid)
+ ): Promise<{ exists: boolean; psid: string; sid: string | null }> {
+ const salt = await this.saltService.getSaltForSession()
const psid = this.derivePsidFromInputs(pid, userAgent, ip, salt)
const sessionKey = this.getSessionKey(psid)
- const exists = Boolean(await redis.exists(sessionKey))
+ const sid = (await sessionScripts.swetrixPeekSession(
+ sessionKey,
+ this.generateSid(psid),
+ UNIQUE_SESSION_LIFE_TIME,
+ LEGACY_SESSION_FLAG,
+ )) as string
- return { exists, psid }
+ return { exists: Boolean(sid), psid, sid: sid || null }
}
async generateAndStoreSessionId(
pid: string,
userAgent: string,
ip: string,
- ): Promise<[boolean, string]> {
- const salt = await this.saltService.getSaltForProject(pid)
+ ): Promise<[boolean, string, string]> {
+ const salt = await this.saltService.getSaltForSession()
const psid = this.derivePsidFromInputs(pid, userAgent, ip, salt)
const sessionKey = this.getSessionKey(psid)
- // Use SET with NX (only set if not exists) for atomic "new session" detection
- // This prevents race conditions where multiple workers could both see the key
- // doesn't exist and both think they're creating a "new session"
- const result = await redis.set(
+ const [sid, isNewFlag] = (await sessionScripts.swetrixResolveSession(
sessionKey,
- '1',
- 'EX',
+ this.generateSid(psid),
UNIQUE_SESSION_LIFE_TIME,
- 'NX',
- )
-
- // If SET NX succeeded (returned 'OK'), this is a new session
- // If it returned null, the session already existed
- const isNew = result === 'OK'
-
- if (!isNew) {
- // Session already exists, extend TTL
- await this.extendSessionTTL(psid)
- }
+ LEGACY_SESSION_FLAG,
+ )) as [string, string]
- return [isNew, psid]
+ return [isNewFlag === '1', psid, sid]
}
+ // EXPIRE, not SET: the value is the sid and must survive a heartbeat.
async extendSessionTTL(psid: string): Promise<void> {
const sessionKey = this.getSessionKey(psid)
- await redis.set(sessionKey, '1', 'EX', UNIQUE_SESSION_LIFE_TIME)
+ await redis.expire(sessionKey, UNIQUE_SESSION_LIFE_TIME)
}
private hashToNumericString(value: string): string {
@@ -2422,46 +2478,33 @@ export class AnalyticsService {
)`
}
- private getSessionFirstSeenKey(pid: string, psid: string): string {
- return `ses:fs:${pid}:${psid}`
+ private getSessionFirstSeenKey(pid: string, sid: string): string {
+ return `ses:fs:${pid}:${sid}`
}
async recordSessionActivity(
+ sid: string,
psid: string,
pid: string,
profileId: string,
): Promise<void> {
const now = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')
- const cacheKey = this.getSessionFirstSeenKey(pid, psid)
+ const cacheKey = this.getSessionFirstSeenKey(pid, sid)
try {
- let firstSeen = await redis.get(cacheKey)
-
- if (!firstSeen) {
- const query = `
- SELECT minOrNull(firstSeen) AS firstSeen
- FROM sessions
- WHERE psid = {psid:UInt64}
- AND pid = {pid:FixedString(12)}
- `
-
- const { data } = await clickhouse
- .query({
- query,
- query_params: { psid, pid },
- })
- .then((resultSet) =>
- resultSet.json<{
- firstSeen: string | null
- }>(),
- )
+ const stored = await redis.set(
+ cacheKey,
+ now,
+ 'EX',
+ UNIQUE_SESSION_LIFE_TIME,
+ 'NX',
+ )
- const existingSession = data[0]
- firstSeen = existingSession?.firstSeen
- ? dayjs.utc(existingSession.firstSeen).format('YYYY-MM-DD HH:mm:ss')
- : now
+ let firstSeen = now
- await redis.set(cacheKey, firstSeen, 'EX', UNIQUE_SESSION_LIFE_TIME)
+ if (stored !== 'OK') {
+ firstSeen = (await redis.get(cacheKey)) || now
+ await redis.expire(cacheKey, UNIQUE_SESSION_LIFE_TIME)
}
await clickhouse.insert({
@@ -2469,6 +2512,7 @@ export class AnalyticsService {
format: 'JSONEachRow',
values: [
{
+ sid,
psid,
pid,
profileId,
@@ -2479,8 +2523,7 @@ export class AnalyticsService {
clickhouse_settings: { async_insert: 1 },
})
} catch (error) {
- this.logger.error('Failed to record session:', error)
- throw error
+ console.error('Failed to record session:', error)
}
}
@@ -2915,14 +2958,24 @@ export class AnalyticsService {
),
session_duration_agg AS (
SELECT
- CAST(psid, 'String') AS psidCasted,
+ psidCasted,
pid,
- dateDiff('second', min(firstSeen), max(lastSeen)) as avg_duration,
- argMax(coalesce(nullIf(pam.userProfileId, ''), s.profileId), lastSeen) as profileId
- FROM sessions AS s
- LEFT JOIN profile_alias_map pam ON s.profileId = pam.anonProfileId
- WHERE pid = {pid:FixedString(12)}
- AND psid IN (SELECT psid FROM funnel_qualified)
+ sum(session_duration) as avg_duration,
+ argMax(sessionProfileId, lastActivity) as profileId
+ FROM (
+ SELECT
+ CAST(psid, 'String') AS psidCasted,
+ pid,
+ sid,
+ dateDiff('second', min(firstSeen), max(lastSeen)) as session_duration,
+ max(lastSeen) as lastActivity,
+ argMax(coalesce(nullIf(pam.userProfileId, ''), s.profileId), lastSeen) as sessionProfileId
+ FROM sessions AS s
+ LEFT JOIN profile_alias_map pam ON s.profileId = pam.anonProfileId
+ WHERE pid = {pid:FixedString(12)}
+ AND psid IN (SELECT psid FROM funnel_qualified)
+ GROUP BY psidCasted, pid, sid
+ )
GROUP BY psidCasted, pid
),
first_session_per_profile AS (
@@ -3073,14 +3126,24 @@ export class AnalyticsService {
),
session_duration_agg AS (
SELECT
- CAST(psid, 'String') AS psidCasted,
+ psidCasted,
pid,
- dateDiff('second', min(firstSeen), max(lastSeen)) as avg_duration,
- argMax(coalesce(nullIf(pam.userProfileId, ''), s.profileId), lastSeen) as profileId
- FROM sessions AS s
- LEFT JOIN profile_alias_map pam ON s.profileId = pam.anonProfileId
- WHERE pid = {pid:FixedString(12)}
- AND psid IN (SELECT psid FROM journey_qualified)
+ sum(session_duration) as avg_duration,
+ argMax(sessionProfileId, lastActivity) as profileId
+ FROM (
+ SELECT
+ CAST(psid, 'String') AS psidCasted,
+ pid,
+ sid,
+ dateDiff('second', min(firstSeen), max(lastSeen)) as session_duration,
+ max(lastSeen) as lastActivity,
+ argMax(coalesce(nullIf(pam.userProfileId, ''), s.profileId), lastSeen) as sessionProfileId
+ FROM sessions AS s
+ LEFT JOIN profile_alias_map pam ON s.profileId = pam.anonProfileId
+ WHERE pid = {pid:FixedString(12)}
+ AND psid IN (SELECT psid FROM journey_qualified)
+ GROUP BY psidCasted, pid, sid
+ )
GROUP BY psidCasted, pid
),
first_session_per_profile AS (
@@ -3419,7 +3482,7 @@ export class AnalyticsService {
WITH analytics_counts AS (
SELECT
count(*) AS all,
- count(DISTINCT psid) AS unique,
+ count(DISTINCT coalesce(sid, psid)) AS unique,
count(DISTINCT profileId) AS users
FROM events
WHERE
@@ -3575,7 +3638,7 @@ export class AnalyticsService {
SELECT
1 AS sortOrder,
count(*) AS all,
- count(DISTINCT psid) AS unique,
+ count(DISTINCT coalesce(sid, psid)) AS unique,
count(DISTINCT profileId) AS users
FROM events
WHERE
@@ -3588,7 +3651,7 @@ export class AnalyticsService {
SELECT avgOrNull(duration) as sdur
FROM (
SELECT
- psid,
+ sid,
dateDiff('second', min(firstSeen), max(lastSeen)) as duration
FROM sessions
WHERE pid = {pid:FixedString(12)}
@@ -3640,7 +3703,7 @@ export class AnalyticsService {
SELECT
2 AS sortOrder,
count(*) AS all,
- count(DISTINCT psid) AS unique,
+ count(DISTINCT coalesce(sid, psid)) AS unique,
count(DISTINCT profileId) AS users
FROM events
WHERE
@@ -3653,7 +3716,7 @@ export class AnalyticsService {
SELECT avgOrNull(duration) as sdur
FROM (
SELECT
- psid,
+ sid,
dateDiff('second', min(firstSeen), max(lastSeen)) as duration
FROM sessions
WHERE pid = {pid:FixedString(12)}
@@ -4536,6 +4599,7 @@ export class AnalyticsService {
SELECT
pid,
psid,
+ coalesce(sid, psid) as sessionId,
${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created
FROM events
PREWHERE pid = {pid:FixedString(12)} AND type = 'pageview'
@@ -4545,37 +4609,40 @@ export class AnalyticsService {
LEFT JOIN (
SELECT
pid,
- psid,
+ sid,
dateDiff('second', min(firstSeen), max(lastSeen)) as duration
FROM sessions
WHERE pid = {pid:FixedString(12)}
- GROUP BY pid, psid
+ GROUP BY pid, sid
) as sessions_data
ON subquery.pid = sessions_data.pid
- AND subquery.psid = sessions_data.psid
+ AND subquery.sessionId = sessions_data.sid
LEFT JOIN (
SELECT
pid,
- psid,
+ sessionId,
count() as pageviews
- FROM events
- PREWHERE pid = {pid:FixedString(12)} AND type = 'pageview'
- WHERE created BETWEEN ${tzFromDate} AND ${tzToDate}
- AND psid IS NOT NULL
- AND psid != 0
- AND psid IN (
- SELECT DISTINCT psid
- FROM events
- PREWHERE pid = {pid:FixedString(12)} AND type = 'pageview'
- WHERE created BETWEEN ${tzFromDate} AND ${tzToDate}
- AND psid IS NOT NULL
- AND psid != 0
- ${filtersQuery}
- )
- GROUP BY pid, psid
+ FROM (
+ SELECT pid, coalesce(sid, psid) as sessionId, psid
+ FROM events
+ PREWHERE pid = {pid:FixedString(12)} AND type = 'pageview'
+ WHERE created BETWEEN ${tzFromDate} AND ${tzToDate}
+ AND psid IS NOT NULL
+ AND psid != 0
+ AND psid IN (
+ SELECT DISTINCT psid
+ FROM events
+ PREWHERE pid = {pid:FixedString(12)} AND type = 'pageview'
+ WHERE created BETWEEN ${tzFromDate} AND ${tzToDate}
+ AND psid IS NOT NULL
+ AND psid != 0
+ ${filtersQuery}
+ )
+ )
+ GROUP BY pid, sessionId
) as session_counts
ON subquery.pid = session_counts.pid
- AND subquery.psid = session_counts.psid
+ AND subquery.sessionId = session_counts.sessionId
GROUP BY ${groupBy}
ORDER BY ${groupBy}
`
@@ -4694,6 +4761,7 @@ export class AnalyticsService {
SELECT
pid,
psid,
+ coalesce(sid, psid) as sessionId,
${timeBucketFunc}(toTimeZone(created, {timezone:String})) as tz_created
FROM events
WHERE pid = {pid:FixedString(12)}
@@ -4704,14 +4772,14 @@ export class AnalyticsService {
LEFT JOIN (
SELECT
pid,
- psid,
+ sid,
dateDiff('second', min(firstSeen), max(lastSeen)) as duration
FROM sessions
WHERE pid = {pid:FixedString(12)}
- GROUP BY pid, psid
+ GROUP BY pid, sid
) as sessions_data
ON subquery.pid = sessions_data.pid
- AND subquery.psid = sessions_data.psid
+ AND subquery.sessionId = sessions_data.sid
GROUP BY ${groupBy}
ORDER BY ${groupBy}
`
@@ -5018,19 +5086,6 @@ export class AnalyticsService {
}
}
- // Reconstructs the number of concurrent (live) visitors for each display bucket.
- // A visitor is live at a given minute if they fired a pageview or custom event
- // within the preceding ONLINE_VISITORS_WINDOW_MINUTES - the exact rule the
- // real-time live visitors counter applies. Every active minute is stretched
- // across that window, visitors are deduplicated per minute, and each bucket
- // reports its peak.
- //
- // This deliberately reads from `events` rather than from the `sessions` table:
- // a psid is derived from a salt that only rotates daily, so a single psid
- // covers every visit a person makes in a day. Treating its
- // [firstSeen, lastSeen] range as one continuous presence counted returning
- // visitors as live for the hours they were away, and `sessions` additionally
- // records activity for requests that never become a countable event.
generateConcurrencyAggregationQuery(timeBucket: TimeBucketType): string {
const timeBucketFunc = timeBucketConversion[timeBucket]
const [selector, groupBy] = this.getGroupSubquery(timeBucket)
@@ -5112,10 +5167,6 @@ export class AnalyticsService {
customEVFilterApplied,
)
- // Concurrency is only computed when explicitly requested (the live visitors
- // metric is off by default). It is also not wired up to the dashboard
- // filters yet, so return zeros rather than misleading unfiltered data
- // whenever a filter is active
const shouldComputeConcurrency =
includeConcurrency && !customEVFilterApplied && !filtersQuery
@@ -6545,13 +6596,23 @@ export class AnalyticsService {
),
session_duration_agg AS (
SELECT
- CAST(psid, 'String') AS psidCasted,
+ psidCasted,
pid,
- dateDiff('second', min(firstSeen), max(lastSeen)) as avg_duration,
- argMax(coalesce(nullIf(pam.userProfileId, ''), s.profileId), lastSeen) as profileId
- FROM sessions AS s
- LEFT JOIN profile_alias_map pam ON s.profileId = pam.anonProfileId
- WHERE pid = {pid:FixedString(12)}
+ sum(session_duration) as avg_duration,
+ argMax(sessionProfileId, lastActivity) as profileId
+ FROM (
+ SELECT
+ CAST(psid, 'String') AS psidCasted,
+ pid,
+ sid,
+ dateDiff('second', min(firstSeen), max(lastSeen)) as session_duration,
+ max(lastSeen) as lastActivity,
+ argMax(coalesce(nullIf(pam.userProfileId, ''), s.profileId), lastSeen) as sessionProfileId
+ FROM sessions AS s
+ LEFT JOIN profile_alias_map pam ON s.profileId = pam.anonProfileId
+ WHERE pid = {pid:FixedString(12)}
+ GROUP BY psidCasted, pid, sid
+ )
GROUP BY psidCasted, pid
),
first_session_per_profile AS (
@@ -6749,7 +6810,7 @@ export class AnalyticsService {
count(*) as count,
max(created) as last_seen,
count(DISTINCT profileId) as users,
- count(DISTINCT psid) as sessions,
+ count(DISTINCT coalesce(sid, psid)) as sessions,
status.status
FROM (
SELECT
@@ -6955,7 +7016,7 @@ export class AnalyticsService {
// Get total sessions from all matching events for the time range
const queryTotalSessions = `
- SELECT count(DISTINCT psid) as totalSessions
+ SELECT count(DISTINCT coalesce(sid, psid)) as totalSessions
FROM events
WHERE pid = {pid:FixedString(12)}
AND type IN ('pageview', 'custom_event', 'error')
@@ -8075,12 +8136,20 @@ export class AnalyticsService {
),
session_duration_agg AS (
SELECT
- CAST(psid, 'String') AS psidCasted,
+ psidCasted,
pid,
- dateDiff('second', min(firstSeen), max(lastSeen)) as avg_duration
- FROM sessions
- WHERE pid = {pid:FixedString(12)}
- AND profileId IN {profileIds:Array(String)}
+ sum(session_duration) as avg_duration
+ FROM (
+ SELECT
+ CAST(psid, 'String') AS psidCasted,
+ pid,
+ sid,
+ dateDiff('second', min(firstSeen), max(lastSeen)) as session_duration
+ FROM sessions
+ WHERE pid = {pid:FixedString(12)}
+ AND profileId IN {profileIds:Array(String)}
+ GROUP BY psidCasted, pid, sid
+ )
GROUP BY psidCasted, pid
)
SELECT
diff --git a/backend/apps/community/src/analytics/heartbeat.gateway.ts b/backend/apps/community/src/analytics/heartbeat.gateway.ts
index a39d2a9b5..66ae5df8f 100644
--- a/backend/apps/community/src/analytics/heartbeat.gateway.ts
+++ b/backend/apps/community/src/analytics/heartbeat.gateway.ts
@@ -17,6 +17,7 @@ const HEARTBEAT_INTERVAL_MS = 60_000
interface WsSession {
pid: string
psid: string
+ sid: string
profileId: string
intervalRef: ReturnType<typeof setInterval>
}
@@ -81,13 +82,13 @@ export class HeartbeatGateway
}
// Get or validate existing session
- const { exists, psid } = await this.analyticsService.getSessionId(
+ const { exists, psid, sid } = await this.analyticsService.getSessionId(
pid,
userAgent,
ip,
)
- if (!exists) {
+ if (!exists || !sid) {
this.logger.warn(
`[WS HB] Connection rejected: no session exists for pid ${pid}`,
'HeartbeatGateway',
@@ -110,13 +111,19 @@ export class HeartbeatGateway
// Initial heartbeat
await this.analyticsService.extendSessionTTL(psid)
- await this.analyticsService.recordSessionActivity(psid, pid, profileId)
+ await this.analyticsService.recordSessionActivity(
+ sid,
+ psid,
+ pid,
+ profileId,
+ )
// Set up interval for periodic heartbeat
const intervalRef = setInterval(async () => {
try {
await this.analyticsService.extendSessionTTL(psid)
await this.analyticsService.recordSessionActivity(
+ sid,
psid,
pid,
profileId,
@@ -134,6 +141,7 @@ export class HeartbeatGateway
this.sessions.set(client.id, {
pid,
psid,
+ sid,
profileId,
intervalRef,
})
@@ -159,6 +167,7 @@ export class HeartbeatGateway
// Final session activity update
try {
await this.analyticsService.recordSessionActivity(
+ session.sid,
session.psid,
session.pid,
session.profileId,
diff --git a/backend/apps/community/src/analytics/utils/transformers.ts b/backend/apps/community/src/analytics/utils/transformers.ts
index eae2b2cf8..90e570788 100644
--- a/backend/apps/community/src/analytics/utils/transformers.ts
+++ b/backend/apps/community/src/analytics/utils/transformers.ts
@@ -26,6 +26,7 @@ const processMetaKV = (
interface CommonOptions {
pid: string
psid?: string | null
+ sid?: string | null
profileId?: string | null
host?: string | null
pg?: string | null
@@ -119,6 +120,7 @@ type EventTransformerOptions =
const buildCommon = (opts: CommonOptions) => ({
pid: opts.pid,
psid: opts.psid ?? null,
+ sid: opts.sid ?? null,
profileId: opts.profileId ?? null,
host: opts.host || null,
pg: opts.pg || null,
diff --git a/backend/apps/community/src/analytics/v2/registry/metrics.ts b/backend/apps/community/src/analytics/v2/registry/metrics.ts
index 84b715f91..964eb8dfe 100644
--- a/backend/apps/community/src/analytics/v2/registry/metrics.ts
+++ b/backend/apps/community/src/analytics/v2/registry/metrics.ts
@@ -24,7 +24,18 @@ export const V2_METRICS: V2MetricDef[] = [
customEVFilterApplied ? 'count(*)' : 'count(DISTINCT psid)',
isDefault: true,
format: 'integer',
- description: 'Unique sessions',
+ description: 'Unique visitors (a visitor is counted once per day)',
+ },
+ {
+ api: 'sessions',
+ types: ['traffic'],
+ sqlExpr: ({ customEVFilterApplied }) =>
+ customEVFilterApplied
+ ? 'count(*)'
+ : 'count(DISTINCT coalesce(sid, psid))',
+ format: 'integer',
+ description:
+ 'Session count. A session ends after 30 minutes of inactivity, so one visitor can have several in a day',
},
{
api: 'pageviews',
diff --git a/backend/apps/community/src/common/constants.ts b/backend/apps/community/src/common/constants.ts
index a2a2aa21b..c53bcce91 100644
--- a/backend/apps/community/src/common/constants.ts
+++ b/backend/apps/community/src/common/constants.ts
@@ -83,9 +83,6 @@ const redisProjectCacheTimeout = 3600
// 30 minues -> the amount of time analytics requests within one session are counted as non-unique
const UNIQUE_SESSION_LIFE_TIME = 1800
-// A visitor counts as live for this long after their last pageview or custom
-// event. Shared by the real-time live visitors counter and the historical live
-// visitors chart series so the two can never disagree.
const ONLINE_VISITORS_WINDOW_MINUTES = 5
// Funnels
diff --git a/backend/migrations/clickhouse/2026_08_01_session_id.js b/backend/migrations/clickhouse/2026_08_01_session_id.js
new file mode 100644
index 000000000..ca43fa77f
--- /dev/null
+++ b/backend/migrations/clickhouse/2026_08_01_session_id.js
@@ -0,0 +1,29 @@
+const { queriesRunner, dbName } = require('./setup')
+
+const queries = [
+ `ALTER TABLE ${dbName}.events ADD COLUMN IF NOT EXISTS sid Nullable(UInt64) AFTER psid`,
+ `ALTER TABLE ${dbName}.customEV ADD COLUMN IF NOT EXISTS sid Nullable(UInt64) AFTER psid`,
+ `ALTER TABLE ${dbName}.errors ADD COLUMN IF NOT EXISTS sid Nullable(UInt64) AFTER psid`,
+
+ `CREATE TABLE IF NOT EXISTS ${dbName}.sessions_v2
+ (
+ sid UInt64 DEFAULT psid,
+ psid UInt64,
+ pid FixedString(12),
+ profileId Nullable(String) CODEC(ZSTD(3)),
+ firstSeen DateTime('UTC') CODEC(Delta(4), LZ4),
+ lastSeen DateTime('UTC') CODEC(Delta(4), LZ4)
+ )
+ ENGINE = ReplacingMergeTree(lastSeen)
+ ORDER BY (pid, sid)
+ PARTITION BY toYYYYMM(firstSeen);`,
+
+ `INSERT INTO ${dbName}.sessions_v2 (sid, psid, pid, profileId, firstSeen, lastSeen)
+ SELECT psid AS sid, psid, pid, profileId, firstSeen, lastSeen
+ FROM ${dbName}.sessions
+ WHERE firstSeen <= lastSeen`,
+
+ `RENAME TABLE ${dbName}.sessions TO ${dbName}.sessions_by_visitor_day, ${dbName}.sessions_v2 TO ${dbName}.sessions`,
+]
+
+queriesRunner(queries)
diff --git a/docs/content/docs/api/stats-v2.mdx b/docs/content/docs/api/stats-v2.mdx
index 46240fd58..395de4a24 100644
--- a/docs/content/docs/api/stats-v2.mdx
+++ b/docs/content/docs/api/stats-v2.mdx
@@ -59,7 +59,7 @@ Every endpoint returns the same envelope:
| Data type | Metrics |
| ----------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
-| traffic | `visitors` (unique sessions), `pageviews`, `users` (unique profiles; breakdown/summary only), `session_duration` (timeseries/summary only), `bounce_rate` (timeseries/summary only), `concurrency` (live visitors over time; timeseries only, zero-filled when filters are applied) |
+| traffic | `visitors` (unique visitors, counted once per day), `sessions` (a session ends after 30 minutes of inactivity, so one visitor can have several per day), `pageviews`, `users` (unique profiles; breakdown/summary only), `session_duration` (timeseries/summary only), `bounce_rate` (timeseries/summary only), `concurrency` (live visitors over time; timeseries only, zero-filled when filters are applied) |
| performance | `load_time`, `ttfb`, `dns`, `tls`, `connection`, `response`, `render`, `dom_load` β€” all in seconds, aggregated by `measure` |
| errors | `occurrences`, `affected_users` |
| captcha | `events` (breakdown), `generated`, `passed`, `failed`, `validation_failed`, `replayed` (timeseries/summary) |
@@ -204,7 +204,7 @@ curl 'https://api.swetrix.com/v2/projects/YOUR_PROJECT_ID/traffic/timeseries?per
}
```
-The `concurrency` metric returns the number of concurrent live visitors over time (the "live visitors" series in the dashboard's traffic chart), reconstructed from session activity. It is only computed when explicitly requested. Because it is derived from session intervals β€” which carry no dimension data β€” it cannot be segmented: when `filters` are applied, or the requested range exceeds one year, the series is zero-filled. For the `minute` time bucket it is the exact number of visitors active during that minute; for larger buckets it is the peak concurrency within the bucket.
+The `concurrency` metric returns the number of concurrent live visitors over time (the "live visitors" series in the dashboard's traffic chart). A visitor counts as live for 5 minutes after their last pageview or custom event, which is the same rule the real-time live visitor counter uses. It is only computed when explicitly requested, and it cannot be segmented: when `filters` are applied the series is zero-filled. For the `minute` time bucket it is the exact number of visitors active during that minute; for larger buckets it is the peak within the bucket.
### GET /v2/projects/:pid/traffic/breakdown
diff --git a/docs/content/docs/visitor-identification.mdx b/docs/content/docs/visitor-identification.mdx
index 3eb974afc..d6f963f70 100644
--- a/docs/content/docs/visitor-identification.mdx
+++ b/docs/content/docs/visitor-identification.mdx
@@ -40,8 +40,7 @@ person.
We use **two** salts that rotate on different schedules:
- **Session salt** β€” rotates **every 24 hours** at UTC midnight. Used to
- detect that two pageviews 5 minutes apart belong to the same browsing
- session.
+ recognise the same visitor across the pageviews they make during a day.
- **Profile salt** β€” rotates **every month**. Used to detect returning
visitors over a longer period (DAU / WAU / MAU charts).
@@ -49,6 +48,14 @@ When a salt rotates, the previous fingerprints become permanently
unlinkable to the new ones. This design is what allows Swetrix to
operate without persistent identifiers on the visitor's device.
+A **session** is a separate thing from either salt: it starts on a
+visitor's first pageview and ends after **30 minutes of inactivity**, the
+same rule Plausible, Umami, Matomo and Google Analytics use. Someone who
+reads a page in the morning and comes back in the evening counts as one
+visitor and **two sessions**. Because a session is bounded by inactivity
+rather than by the salt, a session that is still running at UTC midnight
+is split when the session salt rotates.
+
</Callout>
### What we don't store