Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

CE support

Blue Mouse7757f9a87117
diff --git a/backend/apps/community/src/analytics/analytics.controller.ts b/backend/apps/community/src/analytics/analytics.controller.ts
index 3cd89ed28..d2405a84a 100644
--- a/backend/apps/community/src/analytics/analytics.controller.ts
+++ b/backend/apps/community/src/analytics/analytics.controller.ts
@@ -914,6 +914,8 @@ export class AnalyticsController {
ip,
)
+ await this.analyticsService.recordSession(psid, eventsDTO.pid, false)
+
const transformed = customEventTransformer(
psid,
eventsDTO.pid,
@@ -988,8 +990,7 @@ export class AnalyticsController {
}
await this.analyticsService.extendSessionTTL(psid)
-
- await this.analyticsService.processInteractionSD(psid, pid)
+ await this.analyticsService.updateSessionActivity(psid, pid)
return {}
}
@@ -1016,7 +1017,7 @@ export class AnalyticsController {
ip,
)
- await this.analyticsService.processInteractionSD(psid, logDTO.pid)
+ await this.analyticsService.recordSession(psid, logDTO.pid, true)
if (unique && logDTO.unique) {
throw new ForbiddenException(
@@ -1147,7 +1148,7 @@ export class AnalyticsController {
ip,
)
- await this.analyticsService.processInteractionSD(psid, logDTO.pid)
+ await this.analyticsService.recordSession(psid, logDTO.pid, true)
const { deviceType, browserName, browserVersion, osName, osVersion } =
await this.analyticsService.getRequestInformation(headers)
@@ -1440,7 +1441,7 @@ export class AnalyticsController {
ip,
)
- await this.analyticsService.processInteractionSD(psid, errorDTO.pid)
+ await this.analyticsService.recordSession(psid, errorDTO.pid, false)
const { city, region, regionCode, country } = getGeoDetails(ip, errorDTO.tz)
diff --git a/backend/apps/community/src/analytics/analytics.module.ts b/backend/apps/community/src/analytics/analytics.module.ts
index aa35b8dd4..f0dffb899 100644
--- a/backend/apps/community/src/analytics/analytics.module.ts
+++ b/backend/apps/community/src/analytics/analytics.module.ts
@@ -1,13 +1,15 @@
import { Module } from '@nestjs/common'
+import { TypeOrmModule } from '@nestjs/typeorm'
import { AnalyticsService } from './analytics.service'
import { AnalyticsController } from './analytics.controller'
import { SaltService } from './salt.service'
+import { Salt } from './entities/salt.entity'
import { AppLoggerModule } from '../logger/logger.module'
import { ProjectModule } from '../project/project.module'
@Module({
- imports: [AppLoggerModule, ProjectModule],
+ imports: [TypeOrmModule.forFeature([Salt]), AppLoggerModule, ProjectModule],
providers: [AnalyticsService, SaltService],
exports: [AnalyticsService, SaltService],
controllers: [AnalyticsController],
diff --git a/backend/apps/community/src/analytics/analytics.service.ts b/backend/apps/community/src/analytics/analytics.service.ts
index 9ce9d0e3e..020abf276 100644
--- a/backend/apps/community/src/analytics/analytics.service.ts
+++ b/backend/apps/community/src/analytics/analytics.service.ts
@@ -17,7 +17,6 @@ import _isArray from 'lodash/isArray'
import _startsWith from 'lodash/startsWith'
import _reduce from 'lodash/reduce'
import _keys from 'lodash/keys'
-import _now from 'lodash/now'
import _values from 'lodash/values'
import _round from 'lodash/round'
import _filter from 'lodash/filter'
@@ -104,8 +103,6 @@ dayjs.extend(isSameOrBefore)
// 2 minutes
const LIVE_SESSION_THRESHOLD_SECONDS = 120
-const getSessionDurationKey = (psid: string, pid: string) => `sd:${psid}:${pid}`
-
const SOFTWARE_WITH_PATCH_VERSION = ['GameVault']
const GMT_0_TIMEZONES = [
@@ -735,26 +732,6 @@ export class AnalyticsService {
}
}
- async isSessionDurationOpen(sdKey: string): Promise<[string, boolean]> {
- const sd = await redis.get(sdKey)
- return [sd, Boolean(sd)]
- }
-
- // Processes interaction for session duration
- async processInteractionSD(psid: string, pid: string): Promise<void> {
- const sdKey = getSessionDurationKey(psid, pid)
- const [sd, isOpened] = await this.isSessionDurationOpen(sdKey)
- const now = _now()
-
- // the value is START_UNIX_TIMESTAMP:LAST_INTERACTION_UNIX_TIMESTAMP
- if (isOpened) {
- const [start] = _split(sd, ':')
- await redis.set(sdKey, `${start}:${now}`, 'EX', UNIQUE_SESSION_LIFE_TIME)
- } else {
- await redis.set(sdKey, `${now}:${now}`, 'EX', UNIQUE_SESSION_LIFE_TIME)
- }
- }
-
async getPagesArray(
rawPages?: string,
funnelId?: string,
@@ -1174,6 +1151,60 @@ export class AnalyticsService {
await redis.set(sessionKey, '1', 'EX', UNIQUE_SESSION_LIFE_TIME)
}
+ async recordSession(
+ psid: string,
+ pid: string,
+ isPageview: boolean = true,
+ ): Promise<void> {
+ const now = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')
+
+ try {
+ await clickhouse.insert({
+ table: 'sessions',
+ format: 'JSONEachRow',
+ values: [
+ {
+ psid,
+ pid,
+ profileId: '',
+ firstSeen: now,
+ lastSeen: now,
+ pageviews: isPageview ? 1 : 0,
+ events: isPageview ? 0 : 1,
+ },
+ ],
+ clickhouse_settings: { async_insert: 1 },
+ })
+ } catch (error) {
+ console.error('Failed to record session:', error)
+ }
+ }
+
+ async updateSessionActivity(psid: string, pid: string): Promise<void> {
+ const now = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')
+
+ try {
+ await clickhouse.insert({
+ table: 'sessions',
+ format: 'JSONEachRow',
+ values: [
+ {
+ psid,
+ pid,
+ profileId: '',
+ firstSeen: now,
+ lastSeen: now,
+ pageviews: 0,
+ events: 0,
+ },
+ ],
+ clickhouse_settings: { async_insert: 1 },
+ })
+ } catch (error) {
+ console.error('Failed to update session activity:', error)
+ }
+ }
+
formatFunnel(data: IFunnelCHResponse[], pages: string[]): IFunnel[] {
const funnel = _map(data, (row, index) => {
const value = pages[index]
diff --git a/backend/apps/community/src/analytics/entities/salt.entity.ts b/backend/apps/community/src/analytics/entities/salt.entity.ts
new file mode 100644
index 000000000..4ba00304c
--- /dev/null
+++ b/backend/apps/community/src/analytics/entities/salt.entity.ts
@@ -0,0 +1,18 @@
+import { Column, Entity, PrimaryColumn } from 'typeorm'
+
+import { SaltRotationType } from '../salt.service'
+
+@Entity()
+export class Salt {
+ @PrimaryColumn('varchar', { length: 10 })
+ rotation: SaltRotationType
+
+ @Column('text')
+ salt: string
+
+ @Column({ type: 'timestamp' })
+ expiresAt: Date
+
+ @Column({ type: 'timestamp', default: () => 'CURRENT_TIMESTAMP' })
+ created: Date
+}
diff --git a/backend/apps/community/src/analytics/salt.service.ts b/backend/apps/community/src/analytics/salt.service.ts
index e1aeb7397..ea72486fb 100644
--- a/backend/apps/community/src/analytics/salt.service.ts
+++ b/backend/apps/community/src/analytics/salt.service.ts
@@ -1,31 +1,31 @@
import { Injectable, Logger } from '@nestjs/common'
+import { InjectRepository } from '@nestjs/typeorm'
+import { Repository } from 'typeorm'
import * as bcrypt from 'bcrypt'
import dayjs from 'dayjs'
import utc from 'dayjs/plugin/utc'
import { redis } from '../common/constants'
+import { Salt } from './entities/salt.entity'
dayjs.extend(utc)
const REDIS_SALT_DAILY = 'salt:daily'
const REDIS_SALT_MONTHLY = 'salt:monthly'
+const REDIS_CACHE_TTL = 600 // 10 minutes in seconds
export type SaltRotationType = 'daily' | 'monthly'
-const getTTLUntilEndOfPeriod = (rotation: SaltRotationType): number => {
+const getEndOfPeriod = (rotation: SaltRotationType): dayjs.Dayjs => {
const now = dayjs.utc()
- let endOfPeriod: dayjs.Dayjs
switch (rotation) {
case 'monthly':
- endOfPeriod = now.endOf('month')
- break
+ return now.endOf('month')
case 'daily':
default:
- endOfPeriod = now.endOf('day')
+ return now.endOf('day')
}
-
- return Math.max(1, endOfPeriod.diff(now, 'second'))
}
const getRedisKeyForRotation = (rotation: SaltRotationType): string => {
@@ -36,6 +36,11 @@ const getRedisKeyForRotation = (rotation: SaltRotationType): string => {
export class SaltService {
private readonly logger = new Logger(SaltService.name)
+ constructor(
+ @InjectRepository(Salt)
+ private saltRepository: Repository<Salt>,
+ ) {}
+
private async generateSalt(): Promise<string> {
return bcrypt.genSalt(10)
}
@@ -56,32 +61,71 @@ export class SaltService {
async getGlobalSalt(rotation: SaltRotationType = 'daily'): Promise<string> {
const redisKey = getRedisKeyForRotation(rotation)
- const existingSalt = await redis.get(redisKey)
- if (existingSalt) {
- return existingSalt
+ // Check Redis cache first
+ const cachedSalt = await redis.get(redisKey)
+ if (cachedSalt) {
+ return cachedSalt
+ }
+
+ // Check MySQL
+ const now = dayjs.utc().toDate()
+ const existingSalt = await this.saltRepository.findOne({
+ where: { rotation },
+ })
+
+ if (existingSalt && existingSalt.expiresAt > now) {
+ // Cache in Redis with 10-minute TTL
+ await redis.set(redisKey, existingSalt.salt, 'EX', REDIS_CACHE_TTL)
+ return existingSalt.salt
}
+ // Generate new salt
const salt = await this.generateSalt()
- const ttl = getTTLUntilEndOfPeriod(rotation)
- await redis.set(redisKey, salt, 'EX', ttl)
+ const expiresAt = getEndOfPeriod(rotation).toDate()
- this.logger.log(`Generated new ${rotation} salt with TTL ${ttl}s`)
+ // Save to MySQL (upsert)
+ await this.saltRepository.save({
+ rotation,
+ salt,
+ expiresAt,
+ })
+
+ // Cache in Redis with 10-minute TTL
+ await redis.set(redisKey, salt, 'EX', REDIS_CACHE_TTL)
+
+ this.logger.log(
+ `Generated new ${rotation} salt, expires at ${expiresAt.toISOString()}`,
+ )
return salt
}
async regenerateExpiredSalts(): Promise<void> {
const rotations: SaltRotationType[] = ['daily', 'monthly']
+ const now = dayjs.utc().toDate()
for (const rotation of rotations) {
- const redisKey = getRedisKeyForRotation(rotation)
- const exists = await redis.exists(redisKey)
+ const existingSalt = await this.saltRepository.findOne({
+ where: { rotation },
+ })
- if (!exists) {
+ if (!existingSalt || existingSalt.expiresAt <= now) {
const salt = await this.generateSalt()
- const ttl = getTTLUntilEndOfPeriod(rotation)
- await redis.set(redisKey, salt, 'EX', ttl)
- this.logger.log(`Regenerated ${rotation} salt with TTL ${ttl}s`)
+ const expiresAt = getEndOfPeriod(rotation).toDate()
+
+ await this.saltRepository.save({
+ rotation,
+ salt,
+ expiresAt,
+ })
+
+ // Cache in Redis with 10-minute TTL
+ const redisKey = getRedisKeyForRotation(rotation)
+ await redis.set(redisKey, salt, 'EX', REDIS_CACHE_TTL)
+
+ this.logger.log(
+ `Regenerated ${rotation} salt, expires at ${expiresAt.toISOString()}`,
+ )
}
}
}