Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

Selfhosted support

Blue Mouse0c3059aab6cc
diff --git a/backend/apps/production/src/analytics/analytics.controller.ts b/backend/apps/production/src/analytics/analytics.controller.ts
index 52c276e1f..7b749fafb 100644
--- a/backend/apps/production/src/analytics/analytics.controller.ts
+++ b/backend/apps/production/src/analytics/analytics.controller.ts
@@ -1073,6 +1073,7 @@ export class AnalyticsController {
@Headers() headers: { 'x-password'?: string },
): Promise<object> {
const { pid } = queryParams
+
this.analyticsService.validatePID(pid)
await this.analyticsService.checkProjectAccess(
pid,
diff --git a/backend/apps/selfhosted/src/analytics/analytics.controller.ts b/backend/apps/selfhosted/src/analytics/analytics.controller.ts
index 8aa9acbe7..ae5ccc1bd 100644
--- a/backend/apps/selfhosted/src/analytics/analytics.controller.ts
+++ b/backend/apps/selfhosted/src/analytics/analytics.controller.ts
@@ -1,8 +1,6 @@
import _isEmpty from 'lodash/isEmpty'
import _isArray from 'lodash/isArray'
-import _pick from 'lodash/pick'
import _map from 'lodash/map'
-import _uniqBy from 'lodash/uniqBy'
import _includes from 'lodash/includes'
import _size from 'lodash/size'
import dayjs from 'dayjs'
@@ -32,7 +30,6 @@ import { OptionalJwtAccessTokenGuard } from '../auth/guards'
import { Auth, Public } from '../auth/decorators'
import {
AnalyticsService,
- getSessionKey,
getHeartbeatKey,
validPeriods,
DataType,
@@ -50,17 +47,12 @@ import { GetPagePropertyMetaDTO } from './dto/get-page-property-meta.dto'
import { GetUserFlowDTO } from './dto/getUserFlow.dto'
import { GetFunnelsDTO } from './dto/getFunnels.dto'
import { AppLoggerService } from '../logger/logger.service'
-import {
- redis,
- HEARTBEAT_SID_LIFE_TIME,
- REDIS_SESSION_SALT_KEY,
-} from '../common/constants'
+import { redis, HEARTBEAT_SID_LIFE_TIME } from '../common/constants'
import { clickhouse } from '../common/integrations/clickhouse'
import {
checkRateLimit,
getGeoDetails,
getIPFromHeaders,
- hash,
} from '../common/utils'
import { BotDetection } from '../common/decorators/bot-detection.decorator'
import { BotDetectionGuard } from '../common/guards/bot-detection.guard'
@@ -91,14 +83,6 @@ dayjs.extend(dayjsTimezone)
export const DEFAULT_MEASURE = 'median'
-const getSessionKeyCustom = (
- ip: string,
- ua: string,
- pid: string,
- ev: string,
- salt = '',
-) => `cses_${hash(`${ua}${ip}${pid}${ev}${salt}`)}`
-
// Performance object validator: none of the values cannot be bigger than 1000 * 60 * 5 (5 minutes) and are >= 0
const MAX_PERFORMANCE_VALUE = 1000 * 60 * 5
const isPerformanceValid = (perf: any) => {
@@ -897,26 +881,42 @@ export class AnalyticsController {
)
const keys = await redis.keys(`sd:*:${pid}`)
-
if (_isEmpty(keys)) {
return []
}
- const sids = _map(keys, key => key.split(':')[1])
+ const psids = _map(keys, key => key.split(':')[1])
+
+ const query = `
+ SELECT DISTINCT ON (psid)
+ any(dv) AS dv,
+ any(br) AS br,
+ any(os) AS os,
+ any(cc) AS cc,
+ toString(psid) AS psid
+ FROM analytics
+ WHERE
+ psid IN ({ psids: Array(String) })
+ AND created > ({ createdAfter: String })
+ AND pid = ({ pid: FixedString(12) })
+ GROUP BY psid
+ `
- const query = `SELECT sid, dv, br, os, cc FROM analytics WHERE sid IN (${sids
- .map(el => `'${el}'`)
- .join(',')})`
const { data } = await clickhouse
.query({
query,
+ query_params: {
+ psids,
+ createdAfter: dayjs
+ .utc()
+ .subtract(3, 'hours')
+ .format('YYYY-MM-DDTHH:mm:ss'),
+ pid,
+ },
})
.then(resultSet => resultSet.json())
- const processed = _map(_uniqBy(data, 'sid'), el =>
- _pick(el, ['dv', 'br', 'os', 'cc']),
- )
- return processed
+ return data
}
// Log custom event
@@ -936,17 +936,12 @@ export class AnalyticsController {
this.analyticsService.validateCustomEVMeta(eventsDTO.meta)
await this.analyticsService.validate(eventsDTO, origin, 'custom', ip)
- const salt = await redis.get(REDIS_SESSION_SALT_KEY)
-
if (eventsDTO.unique) {
- const sessionHash = getSessionKeyCustom(
- ip,
+ const [unique] = await this.analyticsService.isUnique(
+ `${eventsDTO.pid}-${eventsDTO.ev}`,
userAgent,
- eventsDTO.pid,
- eventsDTO.ev,
- salt,
+ ip,
)
- const [unique] = await this.analyticsService.isUnique(sessionHash)
if (!unique) {
throw new ForbiddenException(
@@ -962,8 +957,11 @@ export class AnalyticsController {
const { city, region, country } = getGeoDetails(ip, eventsDTO.tz, headers)
- const sessionHash = getSessionKey(ip, userAgent, eventsDTO.pid, salt)
- const [, psid] = await this.analyticsService.isUnique(sessionHash)
+ const [, psid] = await this.analyticsService.isUnique(
+ eventsDTO.pid,
+ userAgent,
+ ip,
+ )
const transformed = customEventTransformer(
psid,
@@ -1012,11 +1010,13 @@ export class AnalyticsController {
@Headers() headers,
@Ip() reqIP,
): Promise<any> {
- const { 'user-agent': userAgent } = headers
+ const { 'user-agent': userAgent, origin } = headers
const { pid } = logDTO
const ip = getIPFromHeaders(headers) || reqIP || ''
- const sessionID = await this.analyticsService.getSessionHash(
+ await this.analyticsService.validateHeartbeat(logDTO, origin, ip)
+
+ const [, sessionID] = await this.analyticsService.isUnique(
pid,
userAgent,
ip,
@@ -1047,11 +1047,13 @@ export class AnalyticsController {
await this.analyticsService.validate(logDTO, origin, 'log', ip)
- const salt = await redis.get(REDIS_SESSION_SALT_KEY)
- const sessionHash = getSessionKey(ip, userAgent, logDTO.pid, salt)
- const [unique, psid] = await this.analyticsService.isUnique(sessionHash)
+ const [unique, psid] = await this.analyticsService.isUnique(
+ logDTO.pid,
+ userAgent,
+ ip,
+ )
- await this.analyticsService.processInteractionSD(sessionHash, logDTO.pid)
+ await this.analyticsService.processInteractionSD(psid, logDTO.pid)
if (unique && logDTO.unique) {
throw new ForbiddenException(
@@ -1068,7 +1070,6 @@ export class AnalyticsController {
const transformed = trafficTransformer(
psid,
- sessionHash,
logDTO.pid,
logDTO.pg,
logDTO.prev,
@@ -1177,11 +1178,13 @@ export class AnalyticsController {
await this.analyticsService.validate(logDTO, origin)
const ip = getIPFromHeaders(headers) || reqIP || ''
- const salt = await redis.get(REDIS_SESSION_SALT_KEY)
- const sessionHash = getSessionKey(ip, userAgent, logDTO.pid, salt)
- const [unique, psid] = await this.analyticsService.isUnique(sessionHash)
+ const [unique, psid] = await this.analyticsService.isUnique(
+ logDTO.pid,
+ userAgent,
+ ip,
+ )
- await this.analyticsService.processInteractionSD(sessionHash, logDTO.pid)
+ await this.analyticsService.processInteractionSD(psid, logDTO.pid)
const ua = UAParser(userAgent)
const dv = ua.device.type || 'desktop'
@@ -1192,7 +1195,6 @@ export class AnalyticsController {
const transformed = trafficTransformer(
psid,
- sessionHash,
logDTO.pid,
null,
null,
diff --git a/backend/apps/selfhosted/src/analytics/analytics.service.ts b/backend/apps/selfhosted/src/analytics/analytics.service.ts
index b3df566d6..afe2db70c 100644
--- a/backend/apps/selfhosted/src/analytics/analytics.service.ts
+++ b/backend/apps/selfhosted/src/analytics/analytics.service.ts
@@ -98,14 +98,10 @@ dayjs.extend(utc)
dayjs.extend(dayjsTimezone)
dayjs.extend(isSameOrBefore)
-export const getSessionKey = (ip: string, ua: string, pid: string, salt = '') =>
- `ses_${hash(`${ua}${ip}${pid}${salt}`)}`
-
export const getHeartbeatKey = (pid: string, sessionID: string) =>
`hb:${pid}:${sessionID}`
-const getSessionDurationKey = (sessionHash: string, pid: string) =>
- `sd:${sessionHash}:${pid}`
+const getSessionDurationKey = (psid: string, pid: string) => `sd:${psid}:${pid}`
const GMT_0_TIMEZONES = [
'Atlantic/Azores',
@@ -464,6 +460,33 @@ export class AnalyticsService {
return null
}
+ async validateHeartbeat(
+ logDTO: PageviewsDTO,
+ origin: string,
+ ip?: string,
+ ): Promise<string | null> {
+ if (_isEmpty(logDTO)) {
+ throw new BadRequestException('The request cannot be empty')
+ }
+
+ const { pid } = logDTO
+ this.validatePID(pid)
+
+ const project = await this.projectService.getRedisProject(pid)
+
+ this.checkIpBlacklist(project, ip)
+
+ if (!project.active) {
+ throw new BadRequestException(
+ 'Incoming analytics is disabled for this project',
+ )
+ }
+
+ this.checkOrigin(project, origin)
+
+ return null
+ }
+
async getErrorsFilters(pid: string, type: string): Promise<Array<string>> {
if (!_includes(ERROR_COLUMNS, type)) {
throw new UnprocessableEntityException(
@@ -708,11 +731,9 @@ export class AnalyticsService {
userAgent: string,
ip: string,
): Promise<string> {
- this.validatePID(pid)
-
const salt = await redis.get(REDIS_SESSION_SALT_KEY)
- return getSessionKey(ip, userAgent, pid, salt)
+ return `ses_${hash(`${userAgent}${ip}${pid}${salt || ''}`)}`
}
async isSessionDurationOpen(sdKey: string): Promise<[string, boolean]> {
@@ -721,8 +742,8 @@ export class AnalyticsService {
}
// Processes interaction for session duration
- async processInteractionSD(sessionHash: string, pid: string): Promise<void> {
- const sdKey = getSessionDurationKey(sessionHash, pid)
+ async processInteractionSD(psid: string, pid: string): Promise<void> {
+ const sdKey = getSessionDurationKey(psid, pid)
const [sd, isOpened] = await this.isSessionDurationOpen(sdKey)
const now = _now()
@@ -1067,7 +1088,13 @@ export class AnalyticsService {
* @param sessionHash
* @returns [isUnique, psid]
*/
- async isUnique(sessionHash: string): Promise<[boolean, string]> {
+ async isUnique(
+ pid: string,
+ userAgent: string,
+ ip: string,
+ ): Promise<[boolean, string]> {
+ const sessionHash = await this.getSessionHash(pid, userAgent, ip)
+
let psid = await redis.get(sessionHash)
const exists = Boolean(psid)
@@ -3142,7 +3169,6 @@ export class AnalyticsService {
): Promise<object | void> {
const analyticsSubquery = `
SELECT
- isNotNull(sid) AS active,
CAST(psid, 'String') AS psidCasted,
cc,
os,
@@ -3158,7 +3184,6 @@ export class AnalyticsService {
const customEVSubquery = `
SELECT
- 0 AS active,
CAST(psid, 'String') AS psidCasted,
cc,
os,
@@ -3175,7 +3200,6 @@ export class AnalyticsService {
const query = `
SELECT
psidCasted AS psid,
- any(active) AS active,
any(cc) AS cc,
any(os) AS os,
any(br) AS br,
diff --git a/backend/apps/selfhosted/src/analytics/utils/transformers.ts b/backend/apps/selfhosted/src/analytics/utils/transformers.ts
index 37af1b160..95787a1fa 100644
--- a/backend/apps/selfhosted/src/analytics/utils/transformers.ts
+++ b/backend/apps/selfhosted/src/analytics/utils/transformers.ts
@@ -25,7 +25,6 @@ const processMetaKV = (
export const trafficTransformer = (
psid: string,
- sid: string,
pid: string,
pg: string | null,
prev: string | null,
@@ -48,7 +47,6 @@ export const trafficTransformer = (
) => {
return {
psid,
- sid,
pid,
pg: pg || null,
prev: prev || null,
diff --git a/backend/apps/selfhosted/src/task-manager/task-manager.service.ts b/backend/apps/selfhosted/src/task-manager/task-manager.service.ts
index 4701dddff..74e40346d 100644
--- a/backend/apps/selfhosted/src/task-manager/task-manager.service.ts
+++ b/backend/apps/selfhosted/src/task-manager/task-manager.service.ts
@@ -23,18 +23,6 @@ export class TaskManagerService {
await redis.set(REDIS_SESSION_SALT_KEY, salt, 'EX', 87000)
}
- @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)
- async cleanUpSessions(): Promise<void> {
- const delSidQuery = `ALTER TABLE analytics UPDATE sid = NULL WHERE created < '${dayjs
- .utc()
- .subtract(20, 'm')
- .format('YYYY-MM-DD HH:mm:ss')}'`
-
- await clickhouse.query({
- query: delSidQuery,
- })
- }
-
@Cron(CronExpression.EVERY_MINUTE)
async processSessionDuration(): Promise<void> {
const keys = await redis.keys('sd:*')
@@ -60,17 +48,23 @@ export class TaskManagerService {
if (_size(toSave) > 0) {
await redis.del(..._map(toSave, ([key]) => key))
- const setSdurQuery = `ALTER TABLE analytics UPDATE sdur = sdur + CASE ${_map(
- toSave,
- ([key, duration]) =>
- `WHEN sid = '${key.split(':')[1]}' THEN ${duration / 1000}`, // converting to seconds
- ).join(' ')} END WHERE sid IN (${_map(
- toSave,
- ([key]) => `'${key.split(':')[1]}'`,
- ).join(',')})`
+ const psids = _map(toSave, ([key]) => key.split(':')[1])
+
+ const setSdurQuery = `
+ ALTER TABLE
+ analytics
+ UPDATE sdur = sdur + CASE ${_map(
+ toSave,
+ ([key, duration]) =>
+ `WHEN psid = ${key.split(':')[1]} THEN ${duration / 1000}`, // converting to seconds
+ ).join(' ')} END WHERE psid IN ({ psids: Array(String) })
+ `
await clickhouse.query({
query: setSdurQuery,
+ query_params: {
+ psids,
+ },
})
}
}