📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
implemented the session duration handling
Blue Mouse20e16463defa
diff --git a/migrations/clickhouse/2022_12_24.js b/migrations/clickhouse/2022_12_24.jsindex 66b91b906..a8b014420 100644--- a/migrations/clickhouse/2022_12_24.js+++ b/migrations/clickhouse/2022_12_24.js@@ -4,7 +4,7 @@ const queries = [`DROP TABLE IF EXISTS ${dbName}.analytics_temp`,`CREATE TABLE IF NOT EXISTS ${dbName}.analytics_temp(- sId Nullable(String),+ sid Nullable(String),pid FixedString(12),pg Nullable(String),dv Nullable(String),@@ -25,7 +25,7 @@ const queries = [PARTITION BY toYYYYMM(created)ORDER BY (pid, created);`,- `INSERT INTO ${dbName}.analytics_temp (sId, pid, pg, dv, br, os, lc, ref, so, me, ca, lt, cc, sdur, unique, created)+ `INSERT INTO ${dbName}.analytics_temp (sid, pid, pg, dv, br, os, lc, ref, so, me, ca, lt, cc, sdur, unique, created)SELECT NULL, pid, pg, dv, br, os, lc, ref, so, me, ca, lt, cc, NULL, unique, created FROM ${dbName}.analytics`,`DROP TABLE ${dbName}.analytics`,diff --git a/src/analytics/analytics.controller.ts b/src/analytics/analytics.controller.tsindex dd903204d..dde6198fe 100644--- a/src/analytics/analytics.controller.ts+++ b/src/analytics/analytics.controller.ts@@ -83,6 +83,7 @@ const analyticsDTO = (ca: string,lt: number | string,cc: string,+ sdur: number | string,unique: number,): Array<string | number> => {return [@@ -99,7 +100,7 @@ const analyticsDTO = (ca,lt,cc,- 'NULL',+ sdur,unique,dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),]@@ -499,6 +500,7 @@ export class AnalyticsController {logDTO.ca,'NULL' /* logDTO.lt */,cc,+ 0,1,)} else if (!logDTO.unique) {@@ -516,6 +518,7 @@ export class AnalyticsController {'NULL','NULL','NULL',+ 'NULL',0,)} else {@@ -593,6 +596,7 @@ export class AnalyticsController {'NULL','NULL',cc,+ 0,1,)} else {@@ -610,6 +614,7 @@ export class AnalyticsController {'NULL','NULL','NULL',+ 'NULL',0,)}diff --git a/src/analytics/analytics.service.ts b/src/analytics/analytics.service.tsindex 6c5898583..5527c4c9f 100644--- a/src/analytics/analytics.service.ts+++ b/src/analytics/analytics.service.ts@@ -34,7 +34,6 @@ import {clickhouse,isSelfhosted,REDIS_SESSION_SALT_KEY,- HEARTBEAT_SID_LIFE_TIME,} from '../common/constants'import {getProjectsClickhouse,@@ -363,13 +362,14 @@ export class AnalyticsService {async processInteractionSD(sessionHash: string): Promise<void> {const sdKey = getSessionDurationKey(sessionHash)const [sd, isOpened] = await this.isSessionDurationOpen(sdKey)+ const now = _now()- // the value is START_UNIX_TIMESTAMP:END_UNIX_TIMESTAMP+ // the value is START_UNIX_TIMESTAMP:LAST_INTERACTION_UNIX_TIMESTAMPif (isOpened) {const [start] = _split(sd, ':')- await redis.set(sdKey, `${start}:${_now()}`, 'EX', HEARTBEAT_SID_LIFE_TIME)+ await redis.set(sdKey, `${start}:${now}`, 'EX', UNIQUE_SESSION_LIFE_TIME)} else {- await redis.set(sdKey, `${_now()}:`, 'EX', HEARTBEAT_SID_LIFE_TIME)+ await redis.set(sdKey, `${now}:${now}`, 'EX', UNIQUE_SESSION_LIFE_TIME)}}diff --git a/src/common/constants.ts b/src/common/constants.tsindex aeffa9d46..69dcb7a7d 100644--- a/src/common/constants.ts+++ b/src/common/constants.ts@@ -45,7 +45,7 @@ const CLICKHOUSE_INIT_QUERIES = ['CREATE DATABASE IF NOT EXISTS analytics',`CREATE TABLE IF NOT EXISTS analytics.analytics(- sId Nullable(String),+ sid Nullable(String),pid FixedString(12),pg Nullable(String),dv Nullable(String),diff --git a/src/task-manager/task-manager.service.ts b/src/task-manager/task-manager.service.tsindex a72fee555..d59135867 100644--- a/src/task-manager/task-manager.service.ts+++ b/src/task-manager/task-manager.service.ts@@ -9,6 +9,7 @@ import * as _isNull from 'lodash/isNull'import * as _join from 'lodash/join'import * as _size from 'lodash/size'import * as _map from 'lodash/map'+import * as _now from 'lodash/now'import { MailerService } from '../mailer/mailer.service'import { UserService } from '../user/user.service'@@ -141,6 +142,16 @@ export class TaskManagerService {await redis.set(REDIS_SESSION_SALT_KEY, salt, 'EX', 87000)}+ @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)+ async cleanUpSessions(): Promise<void> {+ const delSidQuery = `ALTER TABLE analytics UPDATE sid = NULL WHERE created < '${dayjs+ .utc()+ .subtract(20, 'm')+ .format('YYYY-MM-DD HH:mm:ss')}'`++ await clickhouse.query(delSidQuery).toPromise()+ }+// EVERY SUNDAY AT 2:30 AM@Cron('30 02 * * 0')async weeklyReportsHandler(): Promise<void> {@@ -257,4 +268,36 @@ export class TaskManagerService {pageviews,}}++ @Cron(CronExpression.EVERY_MINUTE)+ async processSessionDuration(): Promise<void> {+ const keys = await redis.keys('sd:*')+ const toSave = []+ const now = _now()+++ for (let i = 0; i < _size(keys); ++i) {+ const [start, last] = (await redis.get(keys[i])).split(':')+ const duration = now - Number(last)++ // storing to the DB if last interaction was more than 1 minute ago+ if (duration > 60000) {+ toSave.push([keys[i], Number(last) - Number(start)])+ }+ }++ if (_size(toSave) > 0) {+ await redis.del(..._map(toSave, ([key]) => key))++ const setSdurQuery = `ALTER TABLE analytics UPDATE sdur = sdur + CASE ${_map(+ toSave,+ ([key, duration]) => `WHEN sid = '${key.split(':')[1]}' THEN ${duration / 1000}`, // converting to seconds+ ).join(' ')} END WHERE sid IN (${_map(+ toSave,+ ([key]) => `'${key.split(':')[1]}'`,+ ).join(',')})`++ await clickhouse.query(setSdurQuery).toPromise()+ }+ }}