📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
Selfhosted support
Blue Mouse0c3059aab6cc
diff --git a/backend/apps/production/src/analytics/analytics.controller.ts b/backend/apps/production/src/analytics/analytics.controller.tsindex 52c276e1f..7b749fafb 100644--- a/backend/apps/production/src/analytics/analytics.controller.ts+++ b/backend/apps/production/src/analytics/analytics.controller.ts@@ -1073,6 +1073,7 @@ export class AnalyticsController {@Headers() headers: { 'x-password'?: string },): Promise<object> {const { pid } = queryParams+this.analyticsService.validatePID(pid)await this.analyticsService.checkProjectAccess(pid,diff --git a/backend/apps/selfhosted/src/analytics/analytics.controller.ts b/backend/apps/selfhosted/src/analytics/analytics.controller.tsindex 8aa9acbe7..ae5ccc1bd 100644--- a/backend/apps/selfhosted/src/analytics/analytics.controller.ts+++ b/backend/apps/selfhosted/src/analytics/analytics.controller.ts@@ -1,8 +1,6 @@import _isEmpty from 'lodash/isEmpty'import _isArray from 'lodash/isArray'-import _pick from 'lodash/pick'import _map from 'lodash/map'-import _uniqBy from 'lodash/uniqBy'import _includes from 'lodash/includes'import _size from 'lodash/size'import dayjs from 'dayjs'@@ -32,7 +30,6 @@ import { OptionalJwtAccessTokenGuard } from '../auth/guards'import { Auth, Public } from '../auth/decorators'import {AnalyticsService,- getSessionKey,getHeartbeatKey,validPeriods,DataType,@@ -50,17 +47,12 @@ import { GetPagePropertyMetaDTO } from './dto/get-page-property-meta.dto'import { GetUserFlowDTO } from './dto/getUserFlow.dto'import { GetFunnelsDTO } from './dto/getFunnels.dto'import { AppLoggerService } from '../logger/logger.service'-import {- redis,- HEARTBEAT_SID_LIFE_TIME,- REDIS_SESSION_SALT_KEY,-} from '../common/constants'+import { redis, HEARTBEAT_SID_LIFE_TIME } from '../common/constants'import { clickhouse } from '../common/integrations/clickhouse'import {checkRateLimit,getGeoDetails,getIPFromHeaders,- hash,} from '../common/utils'import { BotDetection } from '../common/decorators/bot-detection.decorator'import { BotDetectionGuard } from '../common/guards/bot-detection.guard'@@ -91,14 +83,6 @@ dayjs.extend(dayjsTimezone)export const DEFAULT_MEASURE = 'median'-const getSessionKeyCustom = (- ip: string,- ua: string,- pid: string,- ev: string,- salt = '',-) => `cses_${hash(`${ua}${ip}${pid}${ev}${salt}`)}`-// Performance object validator: none of the values cannot be bigger than 1000 * 60 * 5 (5 minutes) and are >= 0const MAX_PERFORMANCE_VALUE = 1000 * 60 * 5const isPerformanceValid = (perf: any) => {@@ -897,26 +881,42 @@ export class AnalyticsController {)const keys = await redis.keys(`sd:*:${pid}`)-if (_isEmpty(keys)) {return []}- const sids = _map(keys, key => key.split(':')[1])+ const psids = _map(keys, key => key.split(':')[1])++ const query = `+ SELECT DISTINCT ON (psid)+ any(dv) AS dv,+ any(br) AS br,+ any(os) AS os,+ any(cc) AS cc,+ toString(psid) AS psid+ FROM analytics+ WHERE+ psid IN ({ psids: Array(String) })+ AND created > ({ createdAfter: String })+ AND pid = ({ pid: FixedString(12) })+ GROUP BY psid+ `- const query = `SELECT sid, dv, br, os, cc FROM analytics WHERE sid IN (${sids- .map(el => `'${el}'`)- .join(',')})`const { data } = await clickhouse.query({query,+ query_params: {+ psids,+ createdAfter: dayjs+ .utc()+ .subtract(3, 'hours')+ .format('YYYY-MM-DDTHH:mm:ss'),+ pid,+ },}).then(resultSet => resultSet.json())- const processed = _map(_uniqBy(data, 'sid'), el =>- _pick(el, ['dv', 'br', 'os', 'cc']),- )- return processed+ return data}// Log custom event@@ -936,17 +936,12 @@ export class AnalyticsController {this.analyticsService.validateCustomEVMeta(eventsDTO.meta)await this.analyticsService.validate(eventsDTO, origin, 'custom', ip)- const salt = await redis.get(REDIS_SESSION_SALT_KEY)-if (eventsDTO.unique) {- const sessionHash = getSessionKeyCustom(- ip,+ const [unique] = await this.analyticsService.isUnique(+ `${eventsDTO.pid}-${eventsDTO.ev}`,userAgent,- eventsDTO.pid,- eventsDTO.ev,- salt,+ ip,)- const [unique] = await this.analyticsService.isUnique(sessionHash)if (!unique) {throw new ForbiddenException(@@ -962,8 +957,11 @@ export class AnalyticsController {const { city, region, country } = getGeoDetails(ip, eventsDTO.tz, headers)- const sessionHash = getSessionKey(ip, userAgent, eventsDTO.pid, salt)- const [, psid] = await this.analyticsService.isUnique(sessionHash)+ const [, psid] = await this.analyticsService.isUnique(+ eventsDTO.pid,+ userAgent,+ ip,+ )const transformed = customEventTransformer(psid,@@ -1012,11 +1010,13 @@ export class AnalyticsController {@Headers() headers,@Ip() reqIP,): Promise<any> {- const { 'user-agent': userAgent } = headers+ const { 'user-agent': userAgent, origin } = headersconst { pid } = logDTOconst ip = getIPFromHeaders(headers) || reqIP || ''- const sessionID = await this.analyticsService.getSessionHash(+ await this.analyticsService.validateHeartbeat(logDTO, origin, ip)++ const [, sessionID] = await this.analyticsService.isUnique(pid,userAgent,ip,@@ -1047,11 +1047,13 @@ export class AnalyticsController {await this.analyticsService.validate(logDTO, origin, 'log', ip)- const salt = await redis.get(REDIS_SESSION_SALT_KEY)- const sessionHash = getSessionKey(ip, userAgent, logDTO.pid, salt)- const [unique, psid] = await this.analyticsService.isUnique(sessionHash)+ const [unique, psid] = await this.analyticsService.isUnique(+ logDTO.pid,+ userAgent,+ ip,+ )- await this.analyticsService.processInteractionSD(sessionHash, logDTO.pid)+ await this.analyticsService.processInteractionSD(psid, logDTO.pid)if (unique && logDTO.unique) {throw new ForbiddenException(@@ -1068,7 +1070,6 @@ export class AnalyticsController {const transformed = trafficTransformer(psid,- sessionHash,logDTO.pid,logDTO.pg,logDTO.prev,@@ -1177,11 +1178,13 @@ export class AnalyticsController {await this.analyticsService.validate(logDTO, origin)const ip = getIPFromHeaders(headers) || reqIP || ''- const salt = await redis.get(REDIS_SESSION_SALT_KEY)- const sessionHash = getSessionKey(ip, userAgent, logDTO.pid, salt)- const [unique, psid] = await this.analyticsService.isUnique(sessionHash)+ const [unique, psid] = await this.analyticsService.isUnique(+ logDTO.pid,+ userAgent,+ ip,+ )- await this.analyticsService.processInteractionSD(sessionHash, logDTO.pid)+ await this.analyticsService.processInteractionSD(psid, logDTO.pid)const ua = UAParser(userAgent)const dv = ua.device.type || 'desktop'@@ -1192,7 +1195,6 @@ export class AnalyticsController {const transformed = trafficTransformer(psid,- sessionHash,logDTO.pid,null,null,diff --git a/backend/apps/selfhosted/src/analytics/analytics.service.ts b/backend/apps/selfhosted/src/analytics/analytics.service.tsindex b3df566d6..afe2db70c 100644--- a/backend/apps/selfhosted/src/analytics/analytics.service.ts+++ b/backend/apps/selfhosted/src/analytics/analytics.service.ts@@ -98,14 +98,10 @@ dayjs.extend(utc)dayjs.extend(dayjsTimezone)dayjs.extend(isSameOrBefore)-export const getSessionKey = (ip: string, ua: string, pid: string, salt = '') =>- `ses_${hash(`${ua}${ip}${pid}${salt}`)}`-export const getHeartbeatKey = (pid: string, sessionID: string) =>`hb:${pid}:${sessionID}`-const getSessionDurationKey = (sessionHash: string, pid: string) =>- `sd:${sessionHash}:${pid}`+const getSessionDurationKey = (psid: string, pid: string) => `sd:${psid}:${pid}`const GMT_0_TIMEZONES = ['Atlantic/Azores',@@ -464,6 +460,33 @@ export class AnalyticsService {return null}+ async validateHeartbeat(+ logDTO: PageviewsDTO,+ origin: string,+ ip?: string,+ ): Promise<string | null> {+ if (_isEmpty(logDTO)) {+ throw new BadRequestException('The request cannot be empty')+ }++ const { pid } = logDTO+ this.validatePID(pid)++ const project = await this.projectService.getRedisProject(pid)++ this.checkIpBlacklist(project, ip)++ if (!project.active) {+ throw new BadRequestException(+ 'Incoming analytics is disabled for this project',+ )+ }++ this.checkOrigin(project, origin)++ return null+ }+async getErrorsFilters(pid: string, type: string): Promise<Array<string>> {if (!_includes(ERROR_COLUMNS, type)) {throw new UnprocessableEntityException(@@ -708,11 +731,9 @@ export class AnalyticsService {userAgent: string,ip: string,): Promise<string> {- this.validatePID(pid)-const salt = await redis.get(REDIS_SESSION_SALT_KEY)- return getSessionKey(ip, userAgent, pid, salt)+ return `ses_${hash(`${userAgent}${ip}${pid}${salt || ''}`)}`}async isSessionDurationOpen(sdKey: string): Promise<[string, boolean]> {@@ -721,8 +742,8 @@ export class AnalyticsService {}// Processes interaction for session duration- async processInteractionSD(sessionHash: string, pid: string): Promise<void> {- const sdKey = getSessionDurationKey(sessionHash, pid)+ async processInteractionSD(psid: string, pid: string): Promise<void> {+ const sdKey = getSessionDurationKey(psid, pid)const [sd, isOpened] = await this.isSessionDurationOpen(sdKey)const now = _now()@@ -1067,7 +1088,13 @@ export class AnalyticsService {* @param sessionHash* @returns [isUnique, psid]*/- async isUnique(sessionHash: string): Promise<[boolean, string]> {+ async isUnique(+ pid: string,+ userAgent: string,+ ip: string,+ ): Promise<[boolean, string]> {+ const sessionHash = await this.getSessionHash(pid, userAgent, ip)+let psid = await redis.get(sessionHash)const exists = Boolean(psid)@@ -3142,7 +3169,6 @@ export class AnalyticsService {): Promise<object | void> {const analyticsSubquery = `SELECT- isNotNull(sid) AS active,CAST(psid, 'String') AS psidCasted,cc,os,@@ -3158,7 +3184,6 @@ export class AnalyticsService {const customEVSubquery = `SELECT- 0 AS active,CAST(psid, 'String') AS psidCasted,cc,os,@@ -3175,7 +3200,6 @@ export class AnalyticsService {const query = `SELECTpsidCasted AS psid,- any(active) AS active,any(cc) AS cc,any(os) AS os,any(br) AS br,diff --git a/backend/apps/selfhosted/src/analytics/utils/transformers.ts b/backend/apps/selfhosted/src/analytics/utils/transformers.tsindex 37af1b160..95787a1fa 100644--- a/backend/apps/selfhosted/src/analytics/utils/transformers.ts+++ b/backend/apps/selfhosted/src/analytics/utils/transformers.ts@@ -25,7 +25,6 @@ const processMetaKV = (export const trafficTransformer = (psid: string,- sid: string,pid: string,pg: string | null,prev: string | null,@@ -48,7 +47,6 @@ export const trafficTransformer = () => {return {psid,- sid,pid,pg: pg || null,prev: prev || null,diff --git a/backend/apps/selfhosted/src/task-manager/task-manager.service.ts b/backend/apps/selfhosted/src/task-manager/task-manager.service.tsindex 4701dddff..74e40346d 100644--- a/backend/apps/selfhosted/src/task-manager/task-manager.service.ts+++ b/backend/apps/selfhosted/src/task-manager/task-manager.service.ts@@ -23,18 +23,6 @@ export class TaskManagerService {await redis.set(REDIS_SESSION_SALT_KEY, salt, 'EX', 87000)}- @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)- async cleanUpSessions(): Promise<void> {- const delSidQuery = `ALTER TABLE analytics UPDATE sid = NULL WHERE created < '${dayjs- .utc()- .subtract(20, 'm')- .format('YYYY-MM-DD HH:mm:ss')}'`-- await clickhouse.query({- query: delSidQuery,- })- }-@Cron(CronExpression.EVERY_MINUTE)async processSessionDuration(): Promise<void> {const keys = await redis.keys('sd:*')@@ -60,17 +48,23 @@ export class TaskManagerService {if (_size(toSave) > 0) {await redis.del(..._map(toSave, ([key]) => key))- const setSdurQuery = `ALTER TABLE analytics UPDATE sdur = sdur + CASE ${_map(- toSave,- ([key, duration]) =>- `WHEN sid = '${key.split(':')[1]}' THEN ${duration / 1000}`, // converting to seconds- ).join(' ')} END WHERE sid IN (${_map(- toSave,- ([key]) => `'${key.split(':')[1]}'`,- ).join(',')})`+ const psids = _map(toSave, ([key]) => key.split(':')[1])++ const setSdurQuery = `+ ALTER TABLE+ analytics+ UPDATE sdur = sdur + CASE ${_map(+ toSave,+ ([key, duration]) =>+ `WHEN psid = ${key.split(':')[1]} THEN ${duration / 1000}`, // converting to seconds+ ).join(' ')} END WHERE psid IN ({ psids: Array(String) })+ `await clickhouse.query({query: setSdurQuery,+ query_params: {+ psids,+ },})}}