Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

implemented the session duration handling

Blue Mouse20e16463defa
diff --git a/migrations/clickhouse/2022_12_24.js b/migrations/clickhouse/2022_12_24.js
index 66b91b906..a8b014420 100644
--- a/migrations/clickhouse/2022_12_24.js
+++ b/migrations/clickhouse/2022_12_24.js
@@ -4,7 +4,7 @@ const queries = [
`DROP TABLE IF EXISTS ${dbName}.analytics_temp`,
`CREATE TABLE IF NOT EXISTS ${dbName}.analytics_temp
(
- sId Nullable(String),
+ sid Nullable(String),
pid FixedString(12),
pg Nullable(String),
dv Nullable(String),
@@ -25,7 +25,7 @@ const queries = [
PARTITION BY toYYYYMM(created)
ORDER BY (pid, created);`,
- `INSERT INTO ${dbName}.analytics_temp (sId, pid, pg, dv, br, os, lc, ref, so, me, ca, lt, cc, sdur, unique, created)
+ `INSERT INTO ${dbName}.analytics_temp (sid, pid, pg, dv, br, os, lc, ref, so, me, ca, lt, cc, sdur, unique, created)
SELECT NULL, pid, pg, dv, br, os, lc, ref, so, me, ca, lt, cc, NULL, unique, created FROM ${dbName}.analytics`,
`DROP TABLE ${dbName}.analytics`,
diff --git a/src/analytics/analytics.controller.ts b/src/analytics/analytics.controller.ts
index dd903204d..dde6198fe 100644
--- a/src/analytics/analytics.controller.ts
+++ b/src/analytics/analytics.controller.ts
@@ -83,6 +83,7 @@ const analyticsDTO = (
ca: string,
lt: number | string,
cc: string,
+ sdur: number | string,
unique: number,
): Array<string | number> => {
return [
@@ -99,7 +100,7 @@ const analyticsDTO = (
ca,
lt,
cc,
- 'NULL',
+ sdur,
unique,
dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
]
@@ -499,6 +500,7 @@ export class AnalyticsController {
logDTO.ca,
'NULL' /* logDTO.lt */,
cc,
+ 0,
1,
)
} else if (!logDTO.unique) {
@@ -516,6 +518,7 @@ export class AnalyticsController {
'NULL',
'NULL',
'NULL',
+ 'NULL',
0,
)
} else {
@@ -593,6 +596,7 @@ export class AnalyticsController {
'NULL',
'NULL',
cc,
+ 0,
1,
)
} else {
@@ -610,6 +614,7 @@ export class AnalyticsController {
'NULL',
'NULL',
'NULL',
+ 'NULL',
0,
)
}
diff --git a/src/analytics/analytics.service.ts b/src/analytics/analytics.service.ts
index 6c5898583..5527c4c9f 100644
--- a/src/analytics/analytics.service.ts
+++ b/src/analytics/analytics.service.ts
@@ -34,7 +34,6 @@ import {
clickhouse,
isSelfhosted,
REDIS_SESSION_SALT_KEY,
- HEARTBEAT_SID_LIFE_TIME,
} from '../common/constants'
import {
getProjectsClickhouse,
@@ -363,13 +362,14 @@ export class AnalyticsService {
async processInteractionSD(sessionHash: string): Promise<void> {
const sdKey = getSessionDurationKey(sessionHash)
const [sd, isOpened] = await this.isSessionDurationOpen(sdKey)
+ const now = _now()
- // the value is START_UNIX_TIMESTAMP:END_UNIX_TIMESTAMP
+ // the value is START_UNIX_TIMESTAMP:LAST_INTERACTION_UNIX_TIMESTAMP
if (isOpened) {
const [start] = _split(sd, ':')
- await redis.set(sdKey, `${start}:${_now()}`, 'EX', HEARTBEAT_SID_LIFE_TIME)
+ await redis.set(sdKey, `${start}:${now}`, 'EX', UNIQUE_SESSION_LIFE_TIME)
} else {
- await redis.set(sdKey, `${_now()}:`, 'EX', HEARTBEAT_SID_LIFE_TIME)
+ await redis.set(sdKey, `${now}:${now}`, 'EX', UNIQUE_SESSION_LIFE_TIME)
}
}
diff --git a/src/common/constants.ts b/src/common/constants.ts
index aeffa9d46..69dcb7a7d 100644
--- a/src/common/constants.ts
+++ b/src/common/constants.ts
@@ -45,7 +45,7 @@ const CLICKHOUSE_INIT_QUERIES = [
'CREATE DATABASE IF NOT EXISTS analytics',
`CREATE TABLE IF NOT EXISTS analytics.analytics
(
- sId Nullable(String),
+ sid Nullable(String),
pid FixedString(12),
pg Nullable(String),
dv Nullable(String),
diff --git a/src/task-manager/task-manager.service.ts b/src/task-manager/task-manager.service.ts
index a72fee555..d59135867 100644
--- a/src/task-manager/task-manager.service.ts
+++ b/src/task-manager/task-manager.service.ts
@@ -9,6 +9,7 @@ import * as _isNull from 'lodash/isNull'
import * as _join from 'lodash/join'
import * as _size from 'lodash/size'
import * as _map from 'lodash/map'
+import * as _now from 'lodash/now'
import { MailerService } from '../mailer/mailer.service'
import { UserService } from '../user/user.service'
@@ -141,6 +142,16 @@ export class TaskManagerService {
await redis.set(REDIS_SESSION_SALT_KEY, salt, 'EX', 87000)
}
+ @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)
+ async cleanUpSessions(): Promise<void> {
+ const delSidQuery = `ALTER TABLE analytics UPDATE sid = NULL WHERE created < '${dayjs
+ .utc()
+ .subtract(20, 'm')
+ .format('YYYY-MM-DD HH:mm:ss')}'`
+
+ await clickhouse.query(delSidQuery).toPromise()
+ }
+
// EVERY SUNDAY AT 2:30 AM
@Cron('30 02 * * 0')
async weeklyReportsHandler(): Promise<void> {
@@ -257,4 +268,36 @@ export class TaskManagerService {
pageviews,
}
}
+
+ @Cron(CronExpression.EVERY_MINUTE)
+ async processSessionDuration(): Promise<void> {
+ const keys = await redis.keys('sd:*')
+ const toSave = []
+ const now = _now()
+
+
+ for (let i = 0; i < _size(keys); ++i) {
+ const [start, last] = (await redis.get(keys[i])).split(':')
+ const duration = now - Number(last)
+
+ // storing to the DB if last interaction was more than 1 minute ago
+ if (duration > 60000) {
+ toSave.push([keys[i], Number(last) - Number(start)])
+ }
+ }
+
+ if (_size(toSave) > 0) {
+ await redis.del(..._map(toSave, ([key]) => key))
+
+ const setSdurQuery = `ALTER TABLE analytics UPDATE sdur = sdur + CASE ${_map(
+ toSave,
+ ([key, duration]) => `WHEN sid = '${key.split(':')[1]}' THEN ${duration / 1000}`, // converting to seconds
+ ).join(' ')} END WHERE sid IN (${_map(
+ toSave,
+ ([key]) => `'${key.split(':')[1]}'`,
+ ).join(',')})`
+
+ await clickhouse.query(setSdurQuery).toPromise()
+ }
+ }
}