📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
Selfhosted: Lower load on Clickhouse database for session duration calculations
Blue Mouse5cc23e5cf5e2
diff --git a/backend/apps/production/src/analytics/analytics.service.ts b/backend/apps/production/src/analytics/analytics.service.tsindex 7dd41adad..2135216a8 100644--- a/backend/apps/production/src/analytics/analytics.service.ts+++ b/backend/apps/production/src/analytics/analytics.service.ts@@ -2034,7 +2034,7 @@ export class AnalyticsService {}} catch (reason) {console.error(- `[ERROR] (getAnalyticsSummary) Error occurred for PID ${pid}`,+ `[ERROR] (getPerformanceSummary) Error occurred for PID ${pid}`,)console.error(reason)throw new InternalServerErrorException(diff --git a/backend/apps/production/src/task-manager/task-manager.service.ts b/backend/apps/production/src/task-manager/task-manager.service.tsindex 516c5f304..e3f47d947 100644--- a/backend/apps/production/src/task-manager/task-manager.service.ts+++ b/backend/apps/production/src/task-manager/task-manager.service.ts@@ -761,7 +761,7 @@ export class TaskManagerService {if (duration > 60000) {const [, psid, pid] = key.split(':')toSave.push({- psid: _toNumber(psid),+ psid,pid,duration: Math.floor((Number(last) - Number(start)) / 1000), // convert to seconds})diff --git a/backend/apps/selfhosted/src/analytics/analytics.controller.ts b/backend/apps/selfhosted/src/analytics/analytics.controller.tsindex 9fff8531f..1cab1a1be 100644--- a/backend/apps/selfhosted/src/analytics/analytics.controller.ts+++ b/backend/apps/selfhosted/src/analytics/analytics.controller.ts@@ -1042,7 +1042,6 @@ export class AnalyticsController {region,city,logDTO.meta,- 0,Number(unique),)@@ -1176,7 +1175,6 @@ export class AnalyticsController {region,city,null,- 0,Number(unique),)diff --git a/backend/apps/selfhosted/src/analytics/analytics.service.ts b/backend/apps/selfhosted/src/analytics/analytics.service.tsindex 680f8ce7f..64ca4b515 100644--- a/backend/apps/selfhosted/src/analytics/analytics.service.ts+++ b/backend/apps/selfhosted/src/analytics/analytics.service.ts@@ -1255,10 +1255,36 @@ export class AnalyticsService {const promises = pids.map(async pid => {try {if (period === 'all') {- let queryAll = `SELECT count(*) AS all, countIf(unique=1) AS unique, avgIf(sdur, sdur IS NOT NULL AND analytics.unique=1) AS sdur FROM analytics WHERE pid = {pid:FixedString(12)} ${filtersQuery}`+ let queryAll = `+ WITH analytics_counts AS (+ SELECT+ count(*) AS all,+ countIf(unique=1) AS unique+ FROM analytics+ WHERE+ pid = {pid:FixedString(12)}+ ${filtersQuery}+ ),+ duration_avg AS (+ SELECT avg(duration) as sdur+ FROM session_durations+ WHERE pid = {pid:FixedString(12)}+ )+ SELECT+ analytics_counts.*,+ duration_avg.sdur+ FROM analytics_counts, duration_avg+ `if (customEVFilterApplied) {- queryAll = `SELECT count(*) AS all FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery}`+ queryAll = `+ SELECT+ count(*) AS all+ FROM customEV+ WHERE+ pid = {pid:FixedString(12)}+ ${filtersQuery}+ `}const { data } = await clickhouse@@ -1324,12 +1350,85 @@ export class AnalyticsService {.format('YYYY-MM-DD HH:mm:ss')}- let queryCurrent = `SELECT 1 AS sortOrder, count(*) AS all, countIf(unique=1) AS unique, avgIf(sdur, sdur IS NOT NULL AND analytics.unique=1) AS sdur FROM analytics WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodFormatted:String} AND {now:String} ${filtersQuery}`- let queryPrevious = `SELECT 2 AS sortOrder, count(*) AS all, countIf(unique=1) AS unique, avgIf(sdur, sdur IS NOT NULL AND analytics.unique=1) AS sdur FROM analytics WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String} ${filtersQuery}`+ let queryCurrent = `+ WITH analytics_counts AS (+ SELECT+ 1 AS sortOrder,+ count(*) AS all,+ countIf(unique=1) AS unique+ FROM analytics+ WHERE+ pid = {pid:FixedString(12)}+ AND created BETWEEN {periodFormatted:String} AND {now:String}+ ${filtersQuery}+ ),+ duration_avg AS (+ SELECT avg(duration) as sdur+ FROM session_durations+ WHERE pid = {pid:FixedString(12)}+ AND psid IN (+ SELECT psid+ FROM analytics+ WHERE pid = {pid:FixedString(12)}+ AND created BETWEEN {periodFormatted:String} AND {now:String}+ AND unique = 1+ ${filtersQuery}+ )+ )+ SELECT+ analytics_counts.*,+ duration_avg.sdur+ FROM analytics_counts, duration_avg+ `++ let queryPrevious = `+ WITH analytics_counts AS (+ SELECT+ 2 AS sortOrder,+ count(*) AS all,+ countIf(unique=1) AS unique+ FROM analytics+ WHERE+ pid = {pid:FixedString(12)}+ AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String}+ ${filtersQuery}+ ),+ duration_avg AS (+ SELECT avg(duration) as sdur+ FROM session_durations+ WHERE pid = {pid:FixedString(12)}+ AND psid IN (+ SELECT psid+ FROM analytics+ WHERE pid = {pid:FixedString(12)}+ AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String}+ AND unique = 1+ ${filtersQuery}+ )+ )+ SELECT+ analytics_counts.*,+ duration_avg.sdur+ FROM analytics_counts, duration_avg+ `if (customEVFilterApplied) {- queryCurrent = `SELECT 1 AS sortOrder, count(*) AS all FROM customEV WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodFormatted:String} AND {now:String} ${filtersQuery}`- queryPrevious = `SELECT 2 AS sortOrder, count(*) AS all FROM customEV WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String} ${filtersQuery}`+ queryCurrent = `+ SELECT 1 AS sortOrder, count(*) AS all+ FROM customEV+ WHERE+ pid = {pid:FixedString(12)}+ AND created BETWEEN {periodFormatted:String} AND {now:String}+ ${filtersQuery}+ `+ queryPrevious = `+ SELECT 2 AS sortOrder, count(*) AS all+ FROM customEV+ WHERE+ pid = {pid:FixedString(12)}+ AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String}+ ${filtersQuery}+ `}const query = `${queryCurrent} UNION ALL ${queryPrevious}`@@ -1580,7 +1679,7 @@ export class AnalyticsService {}} catch (reason) {console.error(- `[ERROR] (getAnalyticsSummary) Error occurred for PID ${pid}`,+ `[ERROR] (getPerformanceSummary) Error occurred for PID ${pid}`,)console.error(reason)throw new InternalServerErrorException(@@ -1879,9 +1978,9 @@ export class AnalyticsService {const baseQuery = `SELECT${selector},- avg(sdur) as sdur,+ avg(session_durations.duration) as sdur,count() as pageviews,- sumIf(1, unique = 1) as uniques+ countIf(unique=1) as uniquesFROM (SELECT *,${timeBucketFunc}(toTimeZone(created, '${safeTimezone}')) as tz_created@@ -1891,18 +1990,21 @@ export class AnalyticsService {AND created BETWEEN ${tzFromDate} AND ${tzToDate}${filtersQuery}) as subquery+ LEFT JOIN session_durations+ ON subquery.pid = session_durations.pid+ AND subquery.psid = session_durations.psidGROUP BY ${groupBy}ORDER BY ${groupBy}- `+ `if (mode === ChartRenderMode.CUMULATIVE) {return `- SELECT- *,- sum(pageviews) OVER (ORDER BY ${groupBy}) as pageviews,- sum(uniques) OVER (ORDER BY ${groupBy}) as uniques- FROM (${baseQuery})- `+ SELECT+ *,+ sum(pageviews) OVER (ORDER BY ${groupBy}) as pageviews,+ sum(uniques) OVER (ORDER BY ${groupBy}) as uniques+ FROM (${baseQuery})+ `}return baseQuery@@ -1921,7 +2023,7 @@ export class AnalyticsService {const baseQuery = `SELECT${selector},- avg(sdur) as sdur,+ avg(session_durations.duration) as sdur,count() as pageviews,countIf(unique=1) as uniquesFROM (@@ -1933,6 +2035,9 @@ export class AnalyticsService {AND created BETWEEN ${tzFromDate} AND ${tzToDate}${filtersQuery}) as subquery+ LEFT JOIN session_durations+ ON subquery.pid = session_durations.pid+ AND subquery.psid = session_durations.psidGROUP BY ${groupBy}ORDER BY ${groupBy}`@@ -2991,7 +3096,7 @@ export class AnalyticsService {const querySessionDetails = `SELECT- dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, ct, sdur+ dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, ctFROM analyticsWHEREpid = {pid:FixedString(12)}@@ -3000,6 +3105,15 @@ export class AnalyticsService {LIMIT 1;`+ const querySessionDuration = `+ SELECT+ avg(duration) as duration+ FROM session_durations+ WHERE+ pid = {pid:FixedString(12)}+ AND psid = {psid:String}+ `+const paramsData = {params: {pid,@@ -3022,12 +3136,24 @@ export class AnalyticsService {}).then(resultSet => resultSet.json()).then(({ data }) => data)- )[0]+ )[0] as any++ const { duration = 0 } =+ (+ await clickhouse+ .query({+ query: querySessionDuration,+ query_params: paramsData.params,+ })+ .then(resultSet =>+ resultSet.json<{ duration: number }>().then(({ data }) => data),+ )+ )[0] || {}if (!details) {const querySessionDetailsBackup = `SELECT- dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, ct, sdur+ dv, br, brv, os, osv, lc, ref, so, me, ca, te, co, cc, rg, ctFROM analyticsWHEREpid = {pid:FixedString(12)}@@ -3089,7 +3215,10 @@ export class AnalyticsService {return {pages: this.processPageflow(pages),- details,+ details: {+ ...(details || {}),+ sdur: duration,+ },psid,chart: chartData,timeBucket,diff --git a/backend/apps/selfhosted/src/analytics/utils/transformers.ts b/backend/apps/selfhosted/src/analytics/utils/transformers.tsindex 2d213eb4a..6a1bb27e0 100644--- a/backend/apps/selfhosted/src/analytics/utils/transformers.ts+++ b/backend/apps/selfhosted/src/analytics/utils/transformers.ts@@ -44,7 +44,6 @@ export const trafficTransformer = (rg: string | null,ct: string | null,meta: Record<string, string> | null,- sdur: number,unique: number,) => {return {@@ -68,7 +67,6 @@ export const trafficTransformer = (rg: rg || null,ct: ct || null,...processMetaKV(meta),- sdur,unique,created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),}diff --git a/backend/apps/selfhosted/src/common/utils.ts b/backend/apps/selfhosted/src/common/utils.tsindex 87230c565..e89a9a947 100644--- a/backend/apps/selfhosted/src/common/utils.ts+++ b/backend/apps/selfhosted/src/common/utils.ts@@ -153,7 +153,7 @@ const updateFunnelClickhouse = async (funnel: any) => {}const deleteFunnelClickhouse = async (id: string) => {- const query = `ALTER table funnel DELETE WHERE id = {id:String}`+ const query = `DELETE FROM funnel WHERE id = {id:String}`await clickhouse.command({query,@@ -281,7 +281,7 @@ const calculateRelativePercentage = (}const deleteProjectClickhouse = async (id: string) => {- const query = `ALTER table project DELETE WHERE id = {id:FixedString(12)}`+ const query = `DELETE FROM project WHERE id = {id:FixedString(12)}`await clickhouse.command({query,@@ -354,7 +354,7 @@ const deleteRefreshTokenClickhouse = async (userId: string,refreshToken: string,) => {- const query = `ALTER table refresh_token DELETE WHERE userId = {userId:String} AND refreshToken = {refreshToken:String}`+ const query = `DELETE FROM refresh_token WHERE userId = {userId:String} AND refreshToken = {refreshToken:String}`await clickhouse.command({query,@@ -595,7 +595,7 @@ const findProjectViewsClickhouse = async (projectId: string) => {}const deleteProjectViewClickhouse = async (id: string) => {- const query = `ALTER TABLE project_views DELETE WHERE id = {id:FixedString(36)}`+ const query = `DELETE FROM project_views WHERE id = {id:FixedString(36)}`await clickhouse.command({query,@@ -606,7 +606,7 @@ const deleteProjectViewClickhouse = async (id: string) => {}const deleteCustomMetricsClickhouse = async (viewId: string) => {- const query = `ALTER TABLE projects_views_custom_events DELETE WHERE viewId = {viewId:FixedString(36)}`+ const query = `DELETE FROM projects_views_custom_events WHERE viewId = {viewId:FixedString(36)}`await clickhouse.command({query,diff --git a/backend/apps/selfhosted/src/task-manager/task-manager.service.ts b/backend/apps/selfhosted/src/task-manager/task-manager.service.tsindex 8d6ac965e..60d1de678 100644--- a/backend/apps/selfhosted/src/task-manager/task-manager.service.ts+++ b/backend/apps/selfhosted/src/task-manager/task-manager.service.ts@@ -26,6 +26,7 @@ export class TaskManagerService {@Cron(CronExpression.EVERY_MINUTE)async processSessionDuration(): Promise<void> {const keys = await redis.keys('sd:*')+ const keysToDelete = []const toSave = []const now = _now()@@ -35,7 +36,13 @@ export class TaskManagerService {// storing to the DB if last interaction was more than 1 minute agoif (duration > 60000) {- toSave.push([key, Number(last) - Number(start)])+ const [, psid, pid] = key.split(':')+ toSave.push({+ psid,+ pid,+ duration: Math.floor((Number(last) - Number(start)) / 1000), // convert to seconds+ })+ keysToDelete.push(key)}})@@ -46,25 +53,12 @@ export class TaskManagerService {})if (_size(toSave) > 0) {- await redis.del(..._map(toSave, ([key]) => key))+ await redis.del(...keysToDelete)- const psids = _map(toSave, ([key]) => key.split(':')[1])-- const setSdurQuery = `- ALTER TABLE- analytics- UPDATE sdur = sdur + CASE ${_map(- toSave,- ([key, duration]) =>- `WHEN psid = ${key.split(':')[1]} THEN ${duration / 1000}`, // converting to seconds- ).join(' ')} END WHERE psid IN ({ psids: Array(String) })- `-- await clickhouse.query({- query: setSdurQuery,- query_params: {- psids,- },+ await clickhouse.insert({+ table: 'session_durations',+ values: toSave,+ format: 'JSONEachRow',})}}