π Open source, privacy-first web analytics β a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
fix: Inflated live-visitors chart
Blue Mouse22614c3e426f
diff --git a/backend/apps/cloud/src/analytics/analytics.controller.ts b/backend/apps/cloud/src/analytics/analytics.controller.tsindex dd95b567e..e622c8893 100644--- a/backend/apps/cloud/src/analytics/analytics.controller.ts+++ b/backend/apps/cloud/src/analytics/analytics.controller.ts@@ -66,6 +66,7 @@ import {REDIS_TRIALS_COUNT_KEY,REDIS_PROJECTS_COUNT_KEY,REDIS_EVENTS_COUNT_KEY,+ ONLINE_VISITORS_WINDOW_MINUTES,} from '../common/constants'import { clickhouse } from '../common/integrations/clickhouse'import { checkRateLimit, getIPDetails, getIPFromHeaders } from '../common/utils'@@ -128,8 +129,6 @@ const DEFAULT_MEASURE = 'median'// https://github.com/Swetrix/swetrix/issues/371const BOT_RESPONSE = { message: 'Bot traffic detected, request is ignored' }-const ONLINE_VISITORS_WINDOW_MINUTES = 5 // minutes-// Performance object validator: none of the values cannot be bigger than 1000 * 60 * 5 (5 minutes) and are >= 0const MAX_PERFORMANCE_VALUE = 1000 * 60 * 5const isPerformanceValid = (perf: any) => {diff --git a/backend/apps/cloud/src/analytics/analytics.service.ts b/backend/apps/cloud/src/analytics/analytics.service.tsindex 5bd9c0726..3cf5ccd5d 100644--- a/backend/apps/cloud/src/analytics/analytics.service.ts+++ b/backend/apps/cloud/src/analytics/analytics.service.ts@@ -47,6 +47,7 @@ import {import {redis,UNIQUE_SESSION_LIFE_TIME,+ ONLINE_VISITORS_WINDOW_MINUTES,TRAFFIC_COLUMNS,ALL_COLUMNS,CAPTCHA_COLUMNS,@@ -142,9 +143,6 @@ const isValidLocale = (lc: string): boolean => {const hasTimeComponent = (date: string): boolean => /\d{2}:\d{2}/.test(date)const LIVE_SESSION_THRESHOLD_SECONDS = 120-// The concurrency sweep materializes one row per minute of the requested window,-// so cap how far back it can be reconstructed to keep the query bounded-const MAX_CONCURRENCY_WINDOW_MINUTES = 366 * 24 * 60const MAX_FILTERS = 100const MAX_FILTER_VALUES = 100// Journeys keep only the first {steps} distinct pages of a session, but the@@ -6474,12 +6472,19 @@ export class AnalyticsService {}}- // Reconstructs the number of concurrent (live) visitors for each display bucket- // from the sessions table intervals [firstSeen, lastSeen] using a sweep-line:- // +1 at each session's start minute, -1 right after its end minute, a zero-delta- // "spine" row for every minute in the requested window, then a running sum gives- // exact per-minute concurrency. Buckets larger than a minute report the peak- // concurrency within the bucket.+ // Reconstructs the number of concurrent (live) visitors for each display bucket.+ // A visitor is live at a given minute if they fired a pageview or custom event+ // within the preceding ONLINE_VISITORS_WINDOW_MINUTES - the exact rule the+ // real-time live visitors counter applies. Every active minute is stretched+ // across that window, visitors are deduplicated per minute, and each bucket+ // reports its peak.+ //+ // This deliberately reads from `events` rather than from the `sessions` table:+ // a psid is derived from a salt that only rotates daily, so a single psid+ // covers every visit a person makes in a day. Treating its+ // [firstSeen, lastSeen] range as one continuous presence counted returning+ // visitors as live for the hours they were away, and `sessions` additionally+ // records activity for requests that never become a countable event.generateConcurrencyAggregationQuery(timeBucket: TimeBucketType): string {const timeBucketFunc = timeBucketConversion[timeBucket]const [selector, groupBy] = this.getGroupSubquery(timeBucket)@@ -6489,45 +6494,34 @@ export class AnalyticsService {parseDateTimeBestEffort({groupFrom:String}) AS fromTs,parseDateTimeBestEffort({groupTo:String}) AS toTs,toStartOfMinute(fromTs) AS fromMinute,- toStartOfMinute(toTs) AS toMinute,- session_intervals AS (- SELECT- greatest(min(firstSeen), fromTs) AS sessionStart,- least(max(lastSeen), toTs) AS sessionEnd- FROM sessions- WHERE- pid = {pid:FixedString(12)}- AND firstSeen <= toTs- AND lastSeen >= fromTs- GROUP BY psid- )+ toStartOfMinute(toTs) AS toMinuteSELECT${selector},- toInt32(max(concurrency)) AS concurrency+ toInt32(max(live)) AS concurrencyFROM (SELECT${timeBucketFunc}(toTimeZone(m, {timezone:String})) AS tz_created,- concurrency+ uniqExact(psid) AS liveFROM (SELECT- m,- sum(minute_delta) OVER (ORDER BY m ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS concurrency+ psid,+ activeMinute + toIntervalMinute(arrayJoin(range(0, ${ONLINE_VISITORS_WINDOW_MINUTES}))) AS mFROM (- SELECT m, sum(delta) AS minute_delta- FROM (- SELECT toStartOfMinute(sessionStart) AS m, toInt32(1) AS delta- FROM session_intervals- UNION ALL- SELECT toStartOfMinute(sessionEnd) + INTERVAL 1 MINUTE AS m, toInt32(-1) AS delta- FROM session_intervals- UNION ALL- SELECT fromMinute + toIntervalMinute(number) AS m, toInt32(0) AS delta- FROM numbers(least(toUInt64(dateDiff('minute', fromMinute, toMinute)) + 1, ${MAX_CONCURRENCY_WINDOW_MINUTES}))- )- GROUP BY m+ SELECT DISTINCT+ psid,+ toStartOfMinute(created) AS activeMinute+ FROM events+ WHERE+ pid = {pid:FixedString(12)}+ AND type IN ('pageview', 'custom_event')+ AND psid IS NOT NULL+ AND psid != 0+ AND created > fromTs - toIntervalMinute(${ONLINE_VISITORS_WINDOW_MINUTES})+ AND created <= toTs))WHERE m BETWEEN fromMinute AND toMinute+ GROUP BY tz_created, m)GROUP BY ${groupBy}ORDER BY ${groupBy}@@ -6573,16 +6567,11 @@ export class AnalyticsService {)// Concurrency is only computed when explicitly requested (the live visitors- // metric is off by default) and within a bounded window β the sweep-line- // query materializes a row per minute. The sessions table also has no- // dimension columns, so concurrency cannot respect dashboard filters β- // return zeros instead of misleading unfiltered data+ // metric is off by default). It is also not wired up to the dashboard+ // filters yet, so return zeros rather than misleading unfiltered data+ // whenever a filter is activeconst shouldComputeConcurrency =- includeConcurrency &&- !customEVFilterApplied &&- !filtersQuery &&- dayjs.utc(to).diff(dayjs.utc(from), 'minute') <=- MAX_CONCURRENCY_WINDOW_MINUTES+ includeConcurrency && !customEVFilterApplied && !filtersQueryconst [{ data }, concurrency] = await Promise.all([clickhouse@@ -7543,8 +7532,6 @@ export class AnalyticsService {}async getOnlineUserCount(pid: string): Promise<number> {- const ONLINE_VISITORS_WINDOW_MINUTES = 5-const since = dayjs.utc().subtract(ONLINE_VISITORS_WINDOW_MINUTES, 'minute')diff --git a/backend/apps/cloud/src/analytics/v2/analytics-v2.service.ts b/backend/apps/cloud/src/analytics/v2/analytics-v2.service.tsindex 7da1e041e..abd452366 100644--- a/backend/apps/cloud/src/analytics/v2/analytics-v2.service.ts+++ b/backend/apps/cloud/src/analytics/v2/analytics-v2.service.ts@@ -8,6 +8,7 @@ import dayjs from 'dayjs'import utc from 'dayjs/plugin/utc'import { clickhouse } from '../../common/integrations/clickhouse'+import { ONLINE_VISITORS_WINDOW_MINUTES } from '../../common/constants'import { AppLoggerService } from '../../logger/logger.service'import { MAX_METRICS_IN_VIEW } from '../../project/dto/create-project-view.dto'import { IFunnel, PerfMeasure } from '../interfaces'@@ -96,8 +97,6 @@ const CAPTCHA_TIMESERIES_METRICS: Record<string, string> = {replayed: 'replayed',}-const ONLINE_VISITORS_WINDOW_MINUTES = 5-export interface ResolvedTimeframe {period: string | nulltimeBucket: TimeBucketType | nulldiff --git a/backend/apps/cloud/src/common/constants.ts b/backend/apps/cloud/src/common/constants.tsindex 6d77523fe..3ffb08933 100644--- a/backend/apps/cloud/src/common/constants.ts+++ b/backend/apps/cloud/src/common/constants.ts@@ -61,6 +61,11 @@ const redisUserUsageinfoCacheTimeout = 300// 30 minues -> the amount of time analytics requests within one session are counted as non-uniqueconst UNIQUE_SESSION_LIFE_TIME = 1800+// A visitor counts as live for this long after their last pageview or custom+// event. Shared by the real-time live visitors counter and the historical live+// visitors chart series so the two can never disagree.+const ONLINE_VISITORS_WINDOW_MINUTES = 5+// send email warning when 85% of events in tier are usedconst SEND_WARNING_AT_PERC = 85@@ -238,6 +243,7 @@ export {getRedisProjectKey,redisProjectCacheTimeout,UNIQUE_SESSION_LIFE_TIME,+ ONLINE_VISITORS_WINDOW_MINUTES,getRedisUserCountKey,redisProjectCountCacheTimeout,REDIS_USERS_COUNT_KEY,diff --git a/backend/apps/community/src/analytics/analytics.controller.ts b/backend/apps/community/src/analytics/analytics.controller.tsindex 60cb697bd..d49b8423e 100644--- a/backend/apps/community/src/analytics/analytics.controller.ts+++ b/backend/apps/community/src/analytics/analytics.controller.ts@@ -39,6 +39,7 @@ import { normalizeFiltersToV1Json } from './v2/query/filters.translator'import { VALID_PERIODS } from './decorators/validate-period.decorator'import { CurrentUserId } from '../auth/decorators/current-user-id.decorator'import { DEFAULT_TIMEZONE } from '../user/entities/user.entity'+import { ONLINE_VISITORS_WINDOW_MINUTES } from '../common/constants'import { AuthenticationGuard } from '../auth/guards/authentication.guard'import { PageviewsDto } from './dto/pageviews.dto'import { EventsDto } from './dto/events.dto'@@ -93,8 +94,6 @@ const DEFAULT_MEASURE = 'median'// https://github.com/Swetrix/swetrix/issues/371const BOT_RESPONSE = { message: 'Bot traffic detected, request is ignored' }-const ONLINE_VISITORS_WINDOW_MINUTES = 5 // minutes-// Performance object validator: none of the values cannot be bigger than 1000 * 60 * 5 (5 minutes) and are >= 0const MAX_PERFORMANCE_VALUE = 1000 * 60 * 5const isPerformanceValid = (perf: any) => {diff --git a/backend/apps/community/src/analytics/analytics.service.ts b/backend/apps/community/src/analytics/analytics.service.tsindex b60aaab50..d6ef2b686 100644--- a/backend/apps/community/src/analytics/analytics.service.ts+++ b/backend/apps/community/src/analytics/analytics.service.ts@@ -40,6 +40,7 @@ import { DEFAULT_TIMEZONE } from '../user/entities/user.entity'import {redis,UNIQUE_SESSION_LIFE_TIME,+ ONLINE_VISITORS_WINDOW_MINUTES,TRAFFIC_COLUMNS,PERFORMANCE_COLUMNS,ERROR_COLUMNS,@@ -125,9 +126,6 @@ const isValidLocale = (lc: string): boolean => {const hasTimeComponent = (date: string): boolean => /\d{2}:\d{2}/.test(date)const LIVE_SESSION_THRESHOLD_SECONDS = 120-// The concurrency sweep materializes one row per minute of the requested window,-// so cap how far back it can be reconstructed to keep the query bounded-const MAX_CONCURRENCY_WINDOW_MINUTES = 366 * 24 * 60const MAX_FILTERS = 100const MAX_FILTER_VALUES = 100// Journeys keep only the first {steps} distinct pages of a session, but the@@ -5020,12 +5018,19 @@ export class AnalyticsService {}}- // Reconstructs the number of concurrent (live) visitors for each display bucket- // from the sessions table intervals [firstSeen, lastSeen] using a sweep-line:- // +1 at each session's start minute, -1 right after its end minute, a zero-delta- // "spine" row for every minute in the requested window, then a running sum gives- // exact per-minute concurrency. Buckets larger than a minute report the peak- // concurrency within the bucket.+ // Reconstructs the number of concurrent (live) visitors for each display bucket.+ // A visitor is live at a given minute if they fired a pageview or custom event+ // within the preceding ONLINE_VISITORS_WINDOW_MINUTES - the exact rule the+ // real-time live visitors counter applies. Every active minute is stretched+ // across that window, visitors are deduplicated per minute, and each bucket+ // reports its peak.+ //+ // This deliberately reads from `events` rather than from the `sessions` table:+ // a psid is derived from a salt that only rotates daily, so a single psid+ // covers every visit a person makes in a day. Treating its+ // [firstSeen, lastSeen] range as one continuous presence counted returning+ // visitors as live for the hours they were away, and `sessions` additionally+ // records activity for requests that never become a countable event.generateConcurrencyAggregationQuery(timeBucket: TimeBucketType): string {const timeBucketFunc = timeBucketConversion[timeBucket]const [selector, groupBy] = this.getGroupSubquery(timeBucket)@@ -5035,45 +5040,34 @@ export class AnalyticsService {parseDateTimeBestEffort({groupFrom:String}) AS fromTs,parseDateTimeBestEffort({groupTo:String}) AS toTs,toStartOfMinute(fromTs) AS fromMinute,- toStartOfMinute(toTs) AS toMinute,- session_intervals AS (- SELECT- greatest(min(firstSeen), fromTs) AS sessionStart,- least(max(lastSeen), toTs) AS sessionEnd- FROM sessions- WHERE- pid = {pid:FixedString(12)}- AND firstSeen <= toTs- AND lastSeen >= fromTs- GROUP BY psid- )+ toStartOfMinute(toTs) AS toMinuteSELECT${selector},- toInt32(max(concurrency)) AS concurrency+ toInt32(max(live)) AS concurrencyFROM (SELECT${timeBucketFunc}(toTimeZone(m, {timezone:String})) AS tz_created,- concurrency+ uniqExact(psid) AS liveFROM (SELECT- m,- sum(minute_delta) OVER (ORDER BY m ROWS BETWEEN UNBOUNDED PRECEDING AND CURRENT ROW) AS concurrency+ psid,+ activeMinute + toIntervalMinute(arrayJoin(range(0, ${ONLINE_VISITORS_WINDOW_MINUTES}))) AS mFROM (- SELECT m, sum(delta) AS minute_delta- FROM (- SELECT toStartOfMinute(sessionStart) AS m, toInt32(1) AS delta- FROM session_intervals- UNION ALL- SELECT toStartOfMinute(sessionEnd) + INTERVAL 1 MINUTE AS m, toInt32(-1) AS delta- FROM session_intervals- UNION ALL- SELECT fromMinute + toIntervalMinute(number) AS m, toInt32(0) AS delta- FROM numbers(least(toUInt64(dateDiff('minute', fromMinute, toMinute)) + 1, ${MAX_CONCURRENCY_WINDOW_MINUTES}))- )- GROUP BY m+ SELECT DISTINCT+ psid,+ toStartOfMinute(created) AS activeMinute+ FROM events+ WHERE+ pid = {pid:FixedString(12)}+ AND type IN ('pageview', 'custom_event')+ AND psid IS NOT NULL+ AND psid != 0+ AND created > fromTs - toIntervalMinute(${ONLINE_VISITORS_WINDOW_MINUTES})+ AND created <= toTs))WHERE m BETWEEN fromMinute AND toMinute+ GROUP BY tz_created, m)GROUP BY ${groupBy}ORDER BY ${groupBy}@@ -5119,16 +5113,11 @@ export class AnalyticsService {)// Concurrency is only computed when explicitly requested (the live visitors- // metric is off by default) and within a bounded window β the sweep-line- // query materializes a row per minute. The sessions table also has no- // dimension columns, so concurrency cannot respect dashboard filters β- // return zeros instead of misleading unfiltered data+ // metric is off by default). It is also not wired up to the dashboard+ // filters yet, so return zeros rather than misleading unfiltered data+ // whenever a filter is activeconst shouldComputeConcurrency =- includeConcurrency &&- !customEVFilterApplied &&- !filtersQuery &&- dayjs.utc(to).diff(dayjs.utc(from), 'minute') <=- MAX_CONCURRENCY_WINDOW_MINUTES+ includeConcurrency && !customEVFilterApplied && !filtersQueryconst [{ data }, concurrency] = await Promise.all([clickhouse@@ -6106,8 +6095,6 @@ export class AnalyticsService {}async getOnlineUserCount(pid: string): Promise<number> {- const ONLINE_VISITORS_WINDOW_MINUTES = 5-const since = dayjs.utc().subtract(ONLINE_VISITORS_WINDOW_MINUTES, 'minute')diff --git a/backend/apps/community/src/analytics/v2/analytics-v2.service.ts b/backend/apps/community/src/analytics/v2/analytics-v2.service.tsindex 4e71a3b24..8f1670bbf 100644--- a/backend/apps/community/src/analytics/v2/analytics-v2.service.ts+++ b/backend/apps/community/src/analytics/v2/analytics-v2.service.ts@@ -8,6 +8,7 @@ import dayjs from 'dayjs'import utc from 'dayjs/plugin/utc'import { clickhouse } from '../../common/integrations/clickhouse'+import { ONLINE_VISITORS_WINDOW_MINUTES } from '../../common/constants'import { AppLoggerService } from '../../logger/logger.service'import { MAX_METRICS_IN_VIEW } from '../../project/dto/create-project-view.dto'import { IFunnel, PerfMeasure } from '../interfaces'@@ -96,8 +97,6 @@ const CAPTCHA_TIMESERIES_METRICS: Record<string, string> = {replayed: 'replayed',}-const ONLINE_VISITORS_WINDOW_MINUTES = 5-export interface ResolvedTimeframe {period: string | nulltimeBucket: TimeBucketType | nulldiff --git a/backend/apps/community/src/common/constants.ts b/backend/apps/community/src/common/constants.tsindex b2d7632bf..a2a2aa21b 100644--- a/backend/apps/community/src/common/constants.ts+++ b/backend/apps/community/src/common/constants.ts@@ -83,6 +83,11 @@ const redisProjectCacheTimeout = 3600// 30 minues -> the amount of time analytics requests within one session are counted as non-uniqueconst UNIQUE_SESSION_LIFE_TIME = 1800+// A visitor counts as live for this long after their last pageview or custom+// event. Shared by the real-time live visitors counter and the historical live+// visitors chart series so the two can never disagree.+const ONLINE_VISITORS_WINDOW_MINUTES = 5+// Funnelsconst MIN_PAGES_IN_FUNNEL = 2const MAX_PAGES_IN_FUNNEL = 10@@ -190,6 +195,7 @@ export {getRedisProjectKey,redisProjectCacheTimeout,UNIQUE_SESSION_LIFE_TIME,+ ONLINE_VISITORS_WINDOW_MINUTES,TRAFFIC_METAKEY_COLUMNS,IP_REGEX,ORIGINS_REGEX,