Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

Selfhosted: Lower load on Clickhouse database for session duration calculations

Blue Mouse5cc23e5cf5e2
diff --git a/backend/apps/production/src/analytics/analytics.service.ts b/backend/apps/production/src/analytics/analytics.service.ts
index 7dd41adad..2135216a8 100644
--- a/backend/apps/production/src/analytics/analytics.service.ts
+++ b/backend/apps/production/src/analytics/analytics.service.ts
@@ -2034,7 +2034,7 @@ export class AnalyticsService {
}
} catch (reason) {
console.error(
- `[ERROR] (getAnalyticsSummary) Error occurred for PID ${pid}`,
+ `[ERROR] (getPerformanceSummary) Error occurred for PID ${pid}`,
)
console.error(reason)
throw new InternalServerErrorException(
diff --git a/backend/apps/production/src/task-manager/task-manager.service.ts b/backend/apps/production/src/task-manager/task-manager.service.ts
index 516c5f304..e3f47d947 100644
--- a/backend/apps/production/src/task-manager/task-manager.service.ts
+++ b/backend/apps/production/src/task-manager/task-manager.service.ts
@@ -761,7 +761,7 @@ export class TaskManagerService {
if (duration > 60000) {
const [, psid, pid] = key.split(':')
toSave.push({
- psid: _toNumber(psid),
+ psid,
pid,
duration: Math.floor((Number(last) - Number(start)) / 1000), // convert to seconds
})
diff --git a/backend/apps/selfhosted/src/analytics/analytics.controller.ts b/backend/apps/selfhosted/src/analytics/analytics.controller.ts
index 9fff8531f..1cab1a1be 100644
--- a/backend/apps/selfhosted/src/analytics/analytics.controller.ts
+++ b/backend/apps/selfhosted/src/analytics/analytics.controller.ts
@@ -1042,7 +1042,6 @@ export class AnalyticsController {
region,
city,
logDTO.meta,
- 0,
Number(unique),
)
@@ -1176,7 +1175,6 @@ export class AnalyticsController {
region,
city,
null,
- 0,
Number(unique),
)
diff --git a/backend/apps/selfhosted/src/analytics/analytics.service.ts b/backend/apps/selfhosted/src/analytics/analytics.service.ts
index 680f8ce7f..64ca4b515 100644
--- a/backend/apps/selfhosted/src/analytics/analytics.service.ts
+++ b/backend/apps/selfhosted/src/analytics/analytics.service.ts
@@ -1255,10 +1255,36 @@ export class AnalyticsService {
const promises = pids.map(async pid => {
try {
if (period === 'all') {
- let queryAll = `SELECT count(*) AS all, countIf(unique=1) AS unique, avgIf(sdur, sdur IS NOT NULL AND analytics.unique=1) AS sdur FROM analytics WHERE pid = {pid:FixedString(12)} ${filtersQuery}`
+ let queryAll = `
+ WITH analytics_counts AS (
+ SELECT
+ count(*) AS all,
+ countIf(unique=1) AS unique
+ FROM analytics
+ WHERE
+ pid = {pid:FixedString(12)}
+ ${filtersQuery}
+ ),
+ duration_avg AS (
+ SELECT avg(duration) as sdur
+ FROM session_durations
+ WHERE pid = {pid:FixedString(12)}
+ )
+ SELECT
+ analytics_counts.*,
+ duration_avg.sdur
+ FROM analytics_counts, duration_avg
+ `
if (customEVFilterApplied) {
- queryAll = `SELECT count(*) AS all FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery}`
+ queryAll = `
+ SELECT
+ count(*) AS all
+ FROM customEV
+ WHERE
+ pid = {pid:FixedString(12)}
+ ${filtersQuery}
+ `
}
const { data } = await clickhouse
@@ -1324,12 +1350,85 @@ export class AnalyticsService {
.format('YYYY-MM-DD HH:mm:ss')
}
- let queryCurrent = `SELECT 1 AS sortOrder, count(*) AS all, countIf(unique=1) AS unique, avgIf(sdur, sdur IS NOT NULL AND analytics.unique=1) AS sdur FROM analytics WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodFormatted:String} AND {now:String} ${filtersQuery}`
- let queryPrevious = `SELECT 2 AS sortOrder, count(*) AS all, countIf(unique=1) AS unique, avgIf(sdur, sdur IS NOT NULL AND analytics.unique=1) AS sdur FROM analytics WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String} ${filtersQuery}`
+ let queryCurrent = `
+ WITH analytics_counts AS (
+ SELECT
+ 1 AS sortOrder,
+ count(*) AS all,
+ countIf(unique=1) AS unique
+ FROM analytics
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND created BETWEEN {periodFormatted:String} AND {now:String}
+ ${filtersQuery}
+ ),
+ duration_avg AS (
+ SELECT avg(duration) as sdur
+ FROM session_durations
+ WHERE pid = {pid:FixedString(12)}
+ AND psid IN (
+ SELECT psid
+ FROM analytics
+ WHERE pid = {pid:FixedString(12)}
+ AND created BETWEEN {periodFormatted:String} AND {now:String}
+ AND unique = 1
+ ${filtersQuery}
+ )
+ )
+ SELECT
+ analytics_counts.*,
+ duration_avg.sdur
+ FROM analytics_counts, duration_avg
+ `
+
+ let queryPrevious = `
+ WITH analytics_counts AS (
+ SELECT
+ 2 AS sortOrder,
+ count(*) AS all,
+ countIf(unique=1) AS unique
+ FROM analytics
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String}
+ ${filtersQuery}
+ ),
+ duration_avg AS (
+ SELECT avg(duration) as sdur
+ FROM session_durations
+ WHERE pid = {pid:FixedString(12)}
+ AND psid IN (
+ SELECT psid
+ FROM analytics
+ WHERE pid = {pid:FixedString(12)}
+ AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String}
+ AND unique = 1
+ ${filtersQuery}
+ )
+ )
+ SELECT
+ analytics_counts.*,
+ duration_avg.sdur
+ FROM analytics_counts, duration_avg
+ `
if (customEVFilterApplied) {
- queryCurrent = `SELECT 1 AS sortOrder, count(*) AS all FROM customEV WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodFormatted:String} AND {now:String} ${filtersQuery}`
- queryPrevious = `SELECT 2 AS sortOrder, count(*) AS all FROM customEV WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String} ${filtersQuery}`
+ queryCurrent = `
+ SELECT 1 AS sortOrder, count(*) AS all
+ FROM customEV
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND created BETWEEN {periodFormatted:String} AND {now:String}
+ ${filtersQuery}
+ `
+ queryPrevious = `
+ SELECT 2 AS sortOrder, count(*) AS all
+ FROM customEV
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String}
+ ${filtersQuery}
+ `
}
const query = `${queryCurrent} UNION ALL ${queryPrevious}`
@@ -1580,7 +1679,7 @@ export class AnalyticsService {
}
} catch (reason) {
console.error(
- `[ERROR] (getAnalyticsSummary) Error occurred for PID ${pid}`,
+ `[ERROR] (getPerformanceSummary) Error occurred for PID ${pid}`,
)
console.error(reason)
throw new InternalServerErrorException(
@@ -1879,9 +1978,9 @@ export class AnalyticsService {
const baseQuery = `
SELECT
${selector},
- avg(sdur) as sdur,
+ avg(session_durations.duration) as sdur,
count() as pageviews,
- sumIf(1, unique = 1) as uniques
+ countIf(unique=1) as uniques
FROM (
SELECT *,
${timeBucketFunc}(toTimeZone(created, '${safeTimezone}')) as tz_created
@@ -1891,18 +1990,21 @@ export class AnalyticsService {
AND created BETWEEN ${tzFromDate} AND ${tzToDate}
${filtersQuery}
) as subquery
+ LEFT JOIN session_durations
+ ON subquery.pid = session_durations.pid
+ AND subquery.psid = session_durations.psid
GROUP BY ${groupBy}
ORDER BY ${groupBy}
- `
+ `
if (mode === ChartRenderMode.CUMULATIVE) {
return `
- SELECT
- *,
- sum(pageviews) OVER (ORDER BY ${groupBy}) as pageviews,
- sum(uniques) OVER (ORDER BY ${groupBy}) as uniques
- FROM (${baseQuery})
- `
+ SELECT
+ *,
+ sum(pageviews) OVER (ORDER BY ${groupBy}) as pageviews,
+ sum(uniques) OVER (ORDER BY ${groupBy}) as uniques
+ FROM (${baseQuery})
+ `
}
return baseQuery
@@ -1921,7 +2023,7 @@ export class AnalyticsService {
const baseQuery = `
SELECT
${selector},
- avg(sdur) as sdur,
+ avg(session_durations.duration) as sdur,
count() as pageviews,
countIf(unique=1) as uniques
FROM (
@@ -1933,6 +2035,9 @@ export class AnalyticsService {
AND created BETWEEN ${tzFromDate} AND ${tzToDate}
${filtersQuery}
) as subquery
+ LEFT JOIN session_durations
+ ON subquery.pid = session_durations.pid
+ AND subquery.psid = session_durations.psid
GROUP BY ${groupBy}
ORDER BY ${groupBy}
`
@@ -2991,7 +3096,7 @@ export class AnalyticsService {
const querySessionDetails = `
SELECT
- dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, ct, sdur
+ dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, ct
FROM analytics
WHERE
pid = {pid:FixedString(12)}
@@ -3000,6 +3105,15 @@ export class AnalyticsService {
LIMIT 1;
`
+ const querySessionDuration = `
+ SELECT
+ avg(duration) as duration
+ FROM session_durations
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND psid = {psid:String}
+ `
+
const paramsData = {
params: {
pid,
@@ -3022,12 +3136,24 @@ export class AnalyticsService {
})
.then(resultSet => resultSet.json())
.then(({ data }) => data)
- )[0]
+ )[0] as any
+
+ const { duration = 0 } =
+ (
+ await clickhouse
+ .query({
+ query: querySessionDuration,
+ query_params: paramsData.params,
+ })
+ .then(resultSet =>
+ resultSet.json<{ duration: number }>().then(({ data }) => data),
+ )
+ )[0] || {}
if (!details) {
const querySessionDetailsBackup = `
SELECT
- dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, ct, sdur
+ dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, ct
FROM analytics
WHERE
pid = {pid:FixedString(12)}
@@ -3089,7 +3215,10 @@ export class AnalyticsService {
return {
pages: this.processPageflow(pages),
- details,
+ details: {
+ ...(details || {}),
+ sdur: duration,
+ },
psid,
chart: chartData,
timeBucket,
diff --git a/backend/apps/selfhosted/src/analytics/utils/transformers.ts b/backend/apps/selfhosted/src/analytics/utils/transformers.ts
index 2d213eb4a..6a1bb27e0 100644
--- a/backend/apps/selfhosted/src/analytics/utils/transformers.ts
+++ b/backend/apps/selfhosted/src/analytics/utils/transformers.ts
@@ -44,7 +44,6 @@ export const trafficTransformer = (
rg: string | null,
ct: string | null,
meta: Record<string, string> | null,
- sdur: number,
unique: number,
) => {
return {
@@ -68,7 +67,6 @@ export const trafficTransformer = (
rg: rg || null,
ct: ct || null,
...processMetaKV(meta),
- sdur,
unique,
created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
}
diff --git a/backend/apps/selfhosted/src/common/utils.ts b/backend/apps/selfhosted/src/common/utils.ts
index 87230c565..e89a9a947 100644
--- a/backend/apps/selfhosted/src/common/utils.ts
+++ b/backend/apps/selfhosted/src/common/utils.ts
@@ -153,7 +153,7 @@ const updateFunnelClickhouse = async (funnel: any) => {
}
const deleteFunnelClickhouse = async (id: string) => {
- const query = `ALTER table funnel DELETE WHERE id = {id:String}`
+ const query = `DELETE FROM funnel WHERE id = {id:String}`
await clickhouse.command({
query,
@@ -281,7 +281,7 @@ const calculateRelativePercentage = (
}
const deleteProjectClickhouse = async (id: string) => {
- const query = `ALTER table project DELETE WHERE id = {id:FixedString(12)}`
+ const query = `DELETE FROM project WHERE id = {id:FixedString(12)}`
await clickhouse.command({
query,
@@ -354,7 +354,7 @@ const deleteRefreshTokenClickhouse = async (
userId: string,
refreshToken: string,
) => {
- const query = `ALTER table refresh_token DELETE WHERE userId = {userId:String} AND refreshToken = {refreshToken:String}`
+ const query = `DELETE FROM refresh_token WHERE userId = {userId:String} AND refreshToken = {refreshToken:String}`
await clickhouse.command({
query,
@@ -595,7 +595,7 @@ const findProjectViewsClickhouse = async (projectId: string) => {
}
const deleteProjectViewClickhouse = async (id: string) => {
- const query = `ALTER TABLE project_views DELETE WHERE id = {id:FixedString(36)}`
+ const query = `DELETE FROM project_views WHERE id = {id:FixedString(36)}`
await clickhouse.command({
query,
@@ -606,7 +606,7 @@ const deleteProjectViewClickhouse = async (id: string) => {
}
const deleteCustomMetricsClickhouse = async (viewId: string) => {
- const query = `ALTER TABLE projects_views_custom_events DELETE WHERE viewId = {viewId:FixedString(36)}`
+ const query = `DELETE FROM projects_views_custom_events WHERE viewId = {viewId:FixedString(36)}`
await clickhouse.command({
query,
diff --git a/backend/apps/selfhosted/src/task-manager/task-manager.service.ts b/backend/apps/selfhosted/src/task-manager/task-manager.service.ts
index 8d6ac965e..60d1de678 100644
--- a/backend/apps/selfhosted/src/task-manager/task-manager.service.ts
+++ b/backend/apps/selfhosted/src/task-manager/task-manager.service.ts
@@ -26,6 +26,7 @@ export class TaskManagerService {
@Cron(CronExpression.EVERY_MINUTE)
async processSessionDuration(): Promise<void> {
const keys = await redis.keys('sd:*')
+ const keysToDelete = []
const toSave = []
const now = _now()
@@ -35,7 +36,13 @@ export class TaskManagerService {
// storing to the DB if last interaction was more than 1 minute ago
if (duration > 60000) {
- toSave.push([key, Number(last) - Number(start)])
+ const [, psid, pid] = key.split(':')
+ toSave.push({
+ psid,
+ pid,
+ duration: Math.floor((Number(last) - Number(start)) / 1000), // convert to seconds
+ })
+ keysToDelete.push(key)
}
})
@@ -46,25 +53,12 @@ export class TaskManagerService {
})
if (_size(toSave) > 0) {
- await redis.del(..._map(toSave, ([key]) => key))
+ await redis.del(...keysToDelete)
- const psids = _map(toSave, ([key]) => key.split(':')[1])
-
- const setSdurQuery = `
- ALTER TABLE
- analytics
- UPDATE sdur = sdur + CASE ${_map(
- toSave,
- ([key, duration]) =>
- `WHEN psid = ${key.split(':')[1]} THEN ${duration / 1000}`, // converting to seconds
- ).join(' ')} END WHERE psid IN ({ psids: Array(String) })
- `
-
- await clickhouse.query({
- query: setSdurQuery,
- query_params: {
- psids,
- },
+ await clickhouse.insert({
+ table: 'session_durations',
+ values: toSave,
+ format: 'JSONEachRow',
})
}
}