Skip to content

πŸ“ˆ Open source, privacy-first web analytics β€” a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

fix: Inflated live-visitors chart

Blue Mouse22614c3e426f
diff --git a/backend/apps/cloud/src/analytics/analytics.controller.ts b/backend/apps/cloud/src/analytics/analytics.controller.ts
index dd95b567e..e622c8893 100644
--- a/backend/apps/cloud/src/analytics/analytics.controller.ts
+++ b/backend/apps/cloud/src/analytics/analytics.controller.ts
@@ -66,6 +66,7 @@ import {
REDIS_TRIALS_COUNT_KEY,
REDIS_PROJECTS_COUNT_KEY,
REDIS_EVENTS_COUNT_KEY,
+ ONLINE_VISITORS_WINDOW_MINUTES,
} from '../common/constants'
import { clickhouse } from '../common/integrations/clickhouse'
import { checkRateLimit, getIPDetails, getIPFromHeaders } from '../common/utils'
@@ -128,8 +129,6 @@ const DEFAULT_MEASURE = 'median'
// https://github.com/Swetrix/swetrix/issues/371
const BOT_RESPONSE = { message: 'Bot traffic detected, request is ignored' }
-const ONLINE_VISITORS_WINDOW_MINUTES = 5 // minutes
-
// Performance object validator: none of the values cannot be bigger than 1000 * 60 * 5 (5 minutes) and are >= 0
const MAX_PERFORMANCE_VALUE = 1000 * 60 * 5
const isPerformanceValid = (perf: any) => {
diff --git a/backend/apps/cloud/src/analytics/analytics.service.ts b/backend/apps/cloud/src/analytics/analytics.service.ts
index 5bd9c0726..3cf5ccd5d 100644
--- a/backend/apps/cloud/src/analytics/analytics.service.ts
+++ b/backend/apps/cloud/src/analytics/analytics.service.ts
@@ -47,6 +47,7 @@ import {
import {
redis,
UNIQUE_SESSION_LIFE_TIME,
+ ONLINE_VISITORS_WINDOW_MINUTES,
TRAFFIC_COLUMNS,
ALL_COLUMNS,
CAPTCHA_COLUMNS,
@@ -142,9 +143,6 @@ const isValidLocale = (lc: string): boolean => {
const hasTimeComponent = (date: string): boolean => /\d{2}:\d{2}/.test(date)
const LIVE_SESSION_THRESHOLD_SECONDS = 120
-// The concurrency sweep materializes one row per minute of the requested window,
-// so cap how far back it can be reconstructed to keep the query bounded
-const MAX_CONCURRENCY_WINDOW_MINUTES = 366 * 24 * 60
const MAX_FILTERS = 100
const MAX_FILTER_VALUES = 100
// Journeys keep only the first {steps} distinct pages of a session, but the
@@ -6474,12 +6472,19 @@ export class AnalyticsService {
}
}
- // Reconstructs the number of concurrent (live) visitors for each display bucket
- // from the sessions table intervals [firstSeen, lastSeen] using a sweep-line:
- // +1 at each session's start minute, -1 right after its end minute, a zero-delta
- // "spine" row for every minute in the requested window, then a running sum gives
- // exact per-minute concurrency. Buckets larger than a minute report the peak
- // concurrency within the bucket.
+ // Reconstructs the number of concurrent (live) visitors for each display bucket.
+ // A visitor is live at a given minute if they fired a pageview or custom event
+ // within the preceding ONLINE_VISITORS_WINDOW_MINUTES - the exact rule the
+ // real-time live visitors counter applies. Every active minute is stretched
+ // across that window, visitors are deduplicated per minute, and each bucket
+ // reports its peak.
+ //
+ // This deliberately reads from `events` rather than from the `sessions` table:
+ // a psid is derived from a salt that only rotates daily, so a single psid
+ // covers every visit a person makes in a day. Treating its
+ // [firstSeen, lastSeen] range as one continuous presence counted returning
+ // visitors as live for the hours they were away, and `sessions` additionally
+ // records activity for requests that never become a countable event.
generateConcurrencyAggregationQuery(timeBucket: TimeBucketType): string {
const timeBucketFunc = timeBucketConversion[timeBucket]
const [selector, groupBy] = this.getGroupSubquery(timeBucket)
@@ -6489,45 +6494,34 @@ export class AnalyticsService {
parseDateTimeBestEffort({groupFrom:String}) AS fromTs,
parseDateTimeBestEffort({groupTo:String}) AS toTs,
toStartOfMinute(fromTs) AS fromMinute,
- toStartOfMinute(toTs) AS toMinute,
- session_intervals AS (
- SELECT
- greatest(min(firstSeen), fromTs) AS sessionStart,
- least(max(lastSeen), toTs) AS sessionEnd
- FROM sessions
- WHERE
- pid = {pid:FixedString(12)}
- AND firstSeen <= toTs
- AND lastSeen >= fromTs
- GROUP BY psid
- )
+ toStartOfMinute(toTs) AS toMinute
SELECT
${selector},
- toInt32(max(concurrency)) AS concurrency
+ toInt32(max(live)) AS concurrency
FROM (
SELECT
${timeBucketFunc}(toTimeZone(m, {timezone:String})) AS tz_created,
- concurrency
+ uniqExact(psid) AS live
FROM (
SELECT
- m,
- sum(minute_delta) OVER (ORDER BY m ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS concurrency
+ psid,
+ activeMinute + toIntervalMinute(arrayJoin(range(0, ${ONLINE_VISITORS_WINDOW_MINUTES}))) AS m
FROM (
- SELECT m, sum(delta) AS minute_delta
- FROM (
- SELECT toStartOfMinute(sessionStart) AS m, toInt32(1) AS delta
- FROM session_intervals
- UNION ALL
- SELECT toStartOfMinute(sessionEnd) + INTERVAL 1 MINUTE AS m, toInt32(-1) AS delta
- FROM session_intervals
- UNION ALL
- SELECT fromMinute + toIntervalMinute(number) AS m, toInt32(0) AS delta
- FROM numbers(least(toUInt64(dateDiff('minute', fromMinute, toMinute)) + 1, ${MAX_CONCURRENCY_WINDOW_MINUTES}))
- )
- GROUP BY m
+ SELECT DISTINCT
+ psid,
+ toStartOfMinute(created) AS activeMinute
+ FROM events
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND type IN ('pageview', 'custom_event')
+ AND psid IS NOT NULL
+ AND psid != 0
+ AND created > fromTs - toIntervalMinute(${ONLINE_VISITORS_WINDOW_MINUTES})
+ AND created <= toTs
)
)
WHERE m BETWEEN fromMinute AND toMinute
+ GROUP BY tz_created, m
)
GROUP BY ${groupBy}
ORDER BY ${groupBy}
@@ -6573,16 +6567,11 @@ export class AnalyticsService {
)
// Concurrency is only computed when explicitly requested (the live visitors
- // metric is off by default) and within a bounded window β€” the sweep-line
- // query materializes a row per minute. The sessions table also has no
- // dimension columns, so concurrency cannot respect dashboard filters β€”
- // return zeros instead of misleading unfiltered data
+ // metric is off by default). It is also not wired up to the dashboard
+ // filters yet, so return zeros rather than misleading unfiltered data
+ // whenever a filter is active
const shouldComputeConcurrency =
- includeConcurrency &&
- !customEVFilterApplied &&
- !filtersQuery &&
- dayjs.utc(to).diff(dayjs.utc(from), 'minute') <=
- MAX_CONCURRENCY_WINDOW_MINUTES
+ includeConcurrency && !customEVFilterApplied && !filtersQuery
const [{ data }, concurrency] = await Promise.all([
clickhouse
@@ -7543,8 +7532,6 @@ export class AnalyticsService {
}
async getOnlineUserCount(pid: string): Promise<number> {
- const ONLINE_VISITORS_WINDOW_MINUTES = 5
-
const since = dayjs
.utc()
.subtract(ONLINE_VISITORS_WINDOW_MINUTES, 'minute')
diff --git a/backend/apps/cloud/src/analytics/v2/analytics-v2.service.ts b/backend/apps/cloud/src/analytics/v2/analytics-v2.service.ts
index 7da1e041e..abd452366 100644
--- a/backend/apps/cloud/src/analytics/v2/analytics-v2.service.ts
+++ b/backend/apps/cloud/src/analytics/v2/analytics-v2.service.ts
@@ -8,6 +8,7 @@ import dayjs from 'dayjs'
import utc from 'dayjs/plugin/utc'
import { clickhouse } from '../../common/integrations/clickhouse'
+import { ONLINE_VISITORS_WINDOW_MINUTES } from '../../common/constants'
import { AppLoggerService } from '../../logger/logger.service'
import { MAX_METRICS_IN_VIEW } from '../../project/dto/create-project-view.dto'
import { IFunnel, PerfMeasure } from '../interfaces'
@@ -96,8 +97,6 @@ const CAPTCHA_TIMESERIES_METRICS: Record<string, string> = {
replayed: 'replayed',
}
-const ONLINE_VISITORS_WINDOW_MINUTES = 5
-
export interface ResolvedTimeframe {
period: string | null
timeBucket: TimeBucketType | null
diff --git a/backend/apps/cloud/src/common/constants.ts b/backend/apps/cloud/src/common/constants.ts
index 6d77523fe..3ffb08933 100644
--- a/backend/apps/cloud/src/common/constants.ts
+++ b/backend/apps/cloud/src/common/constants.ts
@@ -61,6 +61,11 @@ const redisUserUsageinfoCacheTimeout = 300
// 30 minues -> the amount of time analytics requests within one session are counted as non-unique
const UNIQUE_SESSION_LIFE_TIME = 1800
+// A visitor counts as live for this long after their last pageview or custom
+// event. Shared by the real-time live visitors counter and the historical live
+// visitors chart series so the two can never disagree.
+const ONLINE_VISITORS_WINDOW_MINUTES = 5
+
// send email warning when 85% of events in tier are used
const SEND_WARNING_AT_PERC = 85
@@ -238,6 +243,7 @@ export {
getRedisProjectKey,
redisProjectCacheTimeout,
UNIQUE_SESSION_LIFE_TIME,
+ ONLINE_VISITORS_WINDOW_MINUTES,
getRedisUserCountKey,
redisProjectCountCacheTimeout,
REDIS_USERS_COUNT_KEY,
diff --git a/backend/apps/community/src/analytics/analytics.controller.ts b/backend/apps/community/src/analytics/analytics.controller.ts
index 60cb697bd..d49b8423e 100644
--- a/backend/apps/community/src/analytics/analytics.controller.ts
+++ b/backend/apps/community/src/analytics/analytics.controller.ts
@@ -39,6 +39,7 @@ import { normalizeFiltersToV1Json } from './v2/query/filters.translator'
import { VALID_PERIODS } from './decorators/validate-period.decorator'
import { CurrentUserId } from '../auth/decorators/current-user-id.decorator'
import { DEFAULT_TIMEZONE } from '../user/entities/user.entity'
+import { ONLINE_VISITORS_WINDOW_MINUTES } from '../common/constants'
import { AuthenticationGuard } from '../auth/guards/authentication.guard'
import { PageviewsDto } from './dto/pageviews.dto'
import { EventsDto } from './dto/events.dto'
@@ -93,8 +94,6 @@ const DEFAULT_MEASURE = 'median'
// https://github.com/Swetrix/swetrix/issues/371
const BOT_RESPONSE = { message: 'Bot traffic detected, request is ignored' }
-const ONLINE_VISITORS_WINDOW_MINUTES = 5 // minutes
-
// Performance object validator: none of the values cannot be bigger than 1000 * 60 * 5 (5 minutes) and are >= 0
const MAX_PERFORMANCE_VALUE = 1000 * 60 * 5
const isPerformanceValid = (perf: any) => {
diff --git a/backend/apps/community/src/analytics/analytics.service.ts b/backend/apps/community/src/analytics/analytics.service.ts
index b60aaab50..d6ef2b686 100644
--- a/backend/apps/community/src/analytics/analytics.service.ts
+++ b/backend/apps/community/src/analytics/analytics.service.ts
@@ -40,6 +40,7 @@ import { DEFAULT_TIMEZONE } from '../user/entities/user.entity'
import {
redis,
UNIQUE_SESSION_LIFE_TIME,
+ ONLINE_VISITORS_WINDOW_MINUTES,
TRAFFIC_COLUMNS,
PERFORMANCE_COLUMNS,
ERROR_COLUMNS,
@@ -125,9 +126,6 @@ const isValidLocale = (lc: string): boolean => {
const hasTimeComponent = (date: string): boolean => /\d{2}:\d{2}/.test(date)
const LIVE_SESSION_THRESHOLD_SECONDS = 120
-// The concurrency sweep materializes one row per minute of the requested window,
-// so cap how far back it can be reconstructed to keep the query bounded
-const MAX_CONCURRENCY_WINDOW_MINUTES = 366 * 24 * 60
const MAX_FILTERS = 100
const MAX_FILTER_VALUES = 100
// Journeys keep only the first {steps} distinct pages of a session, but the
@@ -5020,12 +5018,19 @@ export class AnalyticsService {
}
}
- // Reconstructs the number of concurrent (live) visitors for each display bucket
- // from the sessions table intervals [firstSeen, lastSeen] using a sweep-line:
- // +1 at each session's start minute, -1 right after its end minute, a zero-delta
- // "spine" row for every minute in the requested window, then a running sum gives
- // exact per-minute concurrency. Buckets larger than a minute report the peak
- // concurrency within the bucket.
+ // Reconstructs the number of concurrent (live) visitors for each display bucket.
+ // A visitor is live at a given minute if they fired a pageview or custom event
+ // within the preceding ONLINE_VISITORS_WINDOW_MINUTES - the exact rule the
+ // real-time live visitors counter applies. Every active minute is stretched
+ // across that window, visitors are deduplicated per minute, and each bucket
+ // reports its peak.
+ //
+ // This deliberately reads from `events` rather than from the `sessions` table:
+ // a psid is derived from a salt that only rotates daily, so a single psid
+ // covers every visit a person makes in a day. Treating its
+ // [firstSeen, lastSeen] range as one continuous presence counted returning
+ // visitors as live for the hours they were away, and `sessions` additionally
+ // records activity for requests that never become a countable event.
generateConcurrencyAggregationQuery(timeBucket: TimeBucketType): string {
const timeBucketFunc = timeBucketConversion[timeBucket]
const [selector, groupBy] = this.getGroupSubquery(timeBucket)
@@ -5035,45 +5040,34 @@ export class AnalyticsService {
parseDateTimeBestEffort({groupFrom:String}) AS fromTs,
parseDateTimeBestEffort({groupTo:String}) AS toTs,
toStartOfMinute(fromTs) AS fromMinute,
- toStartOfMinute(toTs) AS toMinute,
- session_intervals AS (
- SELECT
- greatest(min(firstSeen), fromTs) AS sessionStart,
- least(max(lastSeen), toTs) AS sessionEnd
- FROM sessions
- WHERE
- pid = {pid:FixedString(12)}
- AND firstSeen <= toTs
- AND lastSeen >= fromTs
- GROUP BY psid
- )
+ toStartOfMinute(toTs) AS toMinute
SELECT
${selector},
- toInt32(max(concurrency)) AS concurrency
+ toInt32(max(live)) AS concurrency
FROM (
SELECT
${timeBucketFunc}(toTimeZone(m, {timezone:String})) AS tz_created,
- concurrency
+ uniqExact(psid) AS live
FROM (
SELECT
- m,
- sum(minute_delta) OVER (ORDER BY m ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS concurrency
+ psid,
+ activeMinute + toIntervalMinute(arrayJoin(range(0, ${ONLINE_VISITORS_WINDOW_MINUTES}))) AS m
FROM (
- SELECT m, sum(delta) AS minute_delta
- FROM (
- SELECT toStartOfMinute(sessionStart) AS m, toInt32(1) AS delta
- FROM session_intervals
- UNION ALL
- SELECT toStartOfMinute(sessionEnd) + INTERVAL 1 MINUTE AS m, toInt32(-1) AS delta
- FROM session_intervals
- UNION ALL
- SELECT fromMinute + toIntervalMinute(number) AS m, toInt32(0) AS delta
- FROM numbers(least(toUInt64(dateDiff('minute', fromMinute, toMinute)) + 1, ${MAX_CONCURRENCY_WINDOW_MINUTES}))
- )
- GROUP BY m
+ SELECT DISTINCT
+ psid,
+ toStartOfMinute(created) AS activeMinute
+ FROM events
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND type IN ('pageview', 'custom_event')
+ AND psid IS NOT NULL
+ AND psid != 0
+ AND created > fromTs - toIntervalMinute(${ONLINE_VISITORS_WINDOW_MINUTES})
+ AND created <= toTs
)
)
WHERE m BETWEEN fromMinute AND toMinute
+ GROUP BY tz_created, m
)
GROUP BY ${groupBy}
ORDER BY ${groupBy}
@@ -5119,16 +5113,11 @@ export class AnalyticsService {
)
// Concurrency is only computed when explicitly requested (the live visitors
- // metric is off by default) and within a bounded window β€” the sweep-line
- // query materializes a row per minute. The sessions table also has no
- // dimension columns, so concurrency cannot respect dashboard filters β€”
- // return zeros instead of misleading unfiltered data
+ // metric is off by default). It is also not wired up to the dashboard
+ // filters yet, so return zeros rather than misleading unfiltered data
+ // whenever a filter is active
const shouldComputeConcurrency =
- includeConcurrency &&
- !customEVFilterApplied &&
- !filtersQuery &&
- dayjs.utc(to).diff(dayjs.utc(from), 'minute') <=
- MAX_CONCURRENCY_WINDOW_MINUTES
+ includeConcurrency && !customEVFilterApplied && !filtersQuery
const [{ data }, concurrency] = await Promise.all([
clickhouse
@@ -6106,8 +6095,6 @@ export class AnalyticsService {
}
async getOnlineUserCount(pid: string): Promise<number> {
- const ONLINE_VISITORS_WINDOW_MINUTES = 5
-
const since = dayjs
.utc()
.subtract(ONLINE_VISITORS_WINDOW_MINUTES, 'minute')
diff --git a/backend/apps/community/src/analytics/v2/analytics-v2.service.ts b/backend/apps/community/src/analytics/v2/analytics-v2.service.ts
index 4e71a3b24..8f1670bbf 100644
--- a/backend/apps/community/src/analytics/v2/analytics-v2.service.ts
+++ b/backend/apps/community/src/analytics/v2/analytics-v2.service.ts
@@ -8,6 +8,7 @@ import dayjs from 'dayjs'
import utc from 'dayjs/plugin/utc'
import { clickhouse } from '../../common/integrations/clickhouse'
+import { ONLINE_VISITORS_WINDOW_MINUTES } from '../../common/constants'
import { AppLoggerService } from '../../logger/logger.service'
import { MAX_METRICS_IN_VIEW } from '../../project/dto/create-project-view.dto'
import { IFunnel, PerfMeasure } from '../interfaces'
@@ -96,8 +97,6 @@ const CAPTCHA_TIMESERIES_METRICS: Record<string, string> = {
replayed: 'replayed',
}
-const ONLINE_VISITORS_WINDOW_MINUTES = 5
-
export interface ResolvedTimeframe {
period: string | null
timeBucket: TimeBucketType | null
diff --git a/backend/apps/community/src/common/constants.ts b/backend/apps/community/src/common/constants.ts
index b2d7632bf..a2a2aa21b 100644
--- a/backend/apps/community/src/common/constants.ts
+++ b/backend/apps/community/src/common/constants.ts
@@ -83,6 +83,11 @@ const redisProjectCacheTimeout = 3600
// 30 minues -> the amount of time analytics requests within one session are counted as non-unique
const UNIQUE_SESSION_LIFE_TIME = 1800
+// A visitor counts as live for this long after their last pageview or custom
+// event. Shared by the real-time live visitors counter and the historical live
+// visitors chart series so the two can never disagree.
+const ONLINE_VISITORS_WINDOW_MINUTES = 5
+
// Funnels
const MIN_PAGES_IN_FUNNEL = 2
const MAX_PAGES_IN_FUNNEL = 10
@@ -190,6 +195,7 @@ export {
getRedisProjectKey,
redisProjectCacheTimeout,
UNIQUE_SESSION_LIFE_TIME,
+ ONLINE_VISITORS_WINDOW_MINUTES,
TRAFFIC_METAKEY_COLUMNS,
IP_REGEX,
ORIGINS_REGEX,