Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

(improvement) Performance metrics (#222)

Andrii Romasiunf0d1a736bca8
diff --git a/apps/production/src/analytics/analytics.controller.ts b/apps/production/src/analytics/analytics.controller.ts
index a9892cc5e..6c7bf652c 100644
--- a/apps/production/src/analytics/analytics.controller.ts
+++ b/apps/production/src/analytics/analytics.controller.ts
@@ -75,6 +75,7 @@ import {
IFunnel,
IGetFunnel,
IUserFlow,
+ PerfMeasure,
} from './interfaces'
import { GetSessionsDto } from './dto/get-sessions.dto'
import { GetSessionDto } from './dto/get-session.dto'
@@ -85,6 +86,8 @@ const mysql = require('mysql2')
dayjs.extend(utc)
dayjs.extend(dayjsTimezone)
+export const DEFAULT_MEASURE = 'median'
+
const getSessionKeyCustom = (
ip: string,
ua: string,
@@ -626,7 +629,7 @@ export class AnalyticsController {
@Get('performance')
@Auth([], true, true)
async getPerfData(
- @Query() data: AnalyticsGET_DTO,
+ @Query() data: AnalyticsGET_DTO & { measure: PerfMeasure },
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
): Promise<any> {
@@ -638,9 +641,12 @@ export class AnalyticsController {
to,
filters,
timezone = DEFAULT_TIMEZONE,
+ measure = DEFAULT_MEASURE,
} = data
this.analyticsService.validatePID(pid)
+ this.analyticsService.checkIfPerfMeasureIsValid(measure)
+
if (!_isEmpty(period)) {
this.analyticsService.validatePeriod(period)
}
@@ -705,6 +711,7 @@ export class AnalyticsController {
filtersQuery,
paramsData,
safeTimezone,
+ measure,
)
return {
@@ -717,7 +724,7 @@ export class AnalyticsController {
@Get('performance/chart')
@Auth([], true, true)
async getPerfChartData(
- @Query() data: AnalyticsGET_DTO,
+ @Query() data: AnalyticsGET_DTO & { measure: PerfMeasure },
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
): Promise<any> {
@@ -729,9 +736,12 @@ export class AnalyticsController {
to,
filters,
timezone = DEFAULT_TIMEZONE,
+ measure = DEFAULT_MEASURE,
} = data
this.analyticsService.validatePID(pid)
+ this.analyticsService.checkIfPerfMeasureIsValid(measure)
+
if (!_isEmpty(period)) {
this.analyticsService.validatePeriod(period)
}
@@ -772,6 +782,7 @@ export class AnalyticsController {
filtersQuery,
paramsData,
safeTimezone,
+ measure,
)
return {
diff --git a/apps/production/src/analytics/analytics.service.ts b/apps/production/src/analytics/analytics.service.ts
index 9dcf5584d..c68b25ccc 100644
--- a/apps/production/src/analytics/analytics.service.ts
+++ b/apps/production/src/analytics/analytics.service.ts
@@ -55,6 +55,7 @@ import {
import {
calculateRelativePercentage,
millisecondsToSeconds,
+ sumArrays,
} from '../common/utils'
import { PageviewsDTO } from './dto/pageviews.dto'
import { EventsDTO } from './dto/events.dto'
@@ -84,6 +85,7 @@ import {
IOverallCaptcha,
IOverallPerformance,
IPageflow,
+ PerfMeasure,
} from './interfaces'
dayjs.extend(utc)
@@ -2049,23 +2051,30 @@ export class AnalyticsService {
generatePerformanceAggregationQuery(
timeBucket: TimeBucketType,
filtersQuery: string,
- safeTimezone,
+ safeTimezone: string,
+ measure: PerfMeasure,
): string {
const timeBucketFunc = timeBucketConversion[timeBucket]
const [selector, groupBy] = this.getGroupSubquery(timeBucket)
const tzFromDate = `toTimeZone(parseDateTimeBestEffort({groupFrom:String}), '${safeTimezone}')`
const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), '${safeTimezone}')`
+ const cols = ['dns', 'tls', 'conn', 'response', 'render', 'domLoad', 'ttfb']
+
+ const measures = {
+ average: 'avg',
+ median: 'median',
+ p95: 'quantileExact(0.95)',
+ }
+
+ const columnSelectors = cols
+ .map(col => `${measures[measure]}(${col}) as ${col}`)
+ .join(', ')
+
return `
SELECT
${selector},
- avg(dns) as dns,
- avg(tls) as tls,
- avg(conn) as conn,
- avg(response) as response,
- avg(render) as render,
- avg(domLoad) as domLoad,
- avg(ttfb) as ttfb
+ ${columnSelectors}
FROM (
SELECT *,
${timeBucketFunc}(toTimeZone(created, '${safeTimezone}')) as tz_created
@@ -2077,7 +2086,41 @@ export class AnalyticsService {
) as subquery
GROUP BY ${groupBy}
ORDER BY ${groupBy}
- `
+ `
+ }
+
+ generatePerformanceQuantilesQuery(
+ timeBucket: TimeBucketType,
+ filtersQuery: string,
+ safeTimezone: string,
+ ): string {
+ const timeBucketFunc = timeBucketConversion[timeBucket]
+ const [selector, groupBy] = this.getGroupSubquery(timeBucket)
+ const tzFromDate = `toTimeZone(parseDateTimeBestEffort({groupFrom:String}), '${safeTimezone}')`
+ const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), '${safeTimezone}')`
+
+ const cols = ['dns', 'tls', 'conn', 'response', 'render', 'domLoad', 'ttfb']
+
+ const columnSelectors = cols
+ .map(col => `quantilesExactInclusive(0.5, 0.75, 0.95)(${col}) as ${col}`)
+ .join(', ')
+
+ return `
+ SELECT
+ ${selector},
+ ${columnSelectors}
+ FROM (
+ SELECT *,
+ ${timeBucketFunc}(toTimeZone(created, '${safeTimezone}')) as tz_created
+ FROM performance
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND created BETWEEN ${tzFromDate} AND ${tzToDate}
+ ${filtersQuery}
+ ) as subquery
+ GROUP BY ${groupBy}
+ ORDER BY ${groupBy}
+ `
}
generateCaptchaAggregationQuery(
@@ -2380,6 +2423,39 @@ export class AnalyticsService {
}
}
+ extractPerformanceQuantilesData(result, x: string[]): any {
+ const cols = ['dns', 'tls', 'conn', 'response', 'render', 'domLoad', 'ttfb']
+
+ const p50 = Array(x.length).fill(0)
+ const p75 = Array(x.length).fill(0)
+ const p95 = Array(x.length).fill(0)
+
+ for (let row = 0; row < _size(result); ++row) {
+ const dateString = this.generateDateString(result[row])
+
+ const index = x.indexOf(dateString)
+
+ if (index !== -1) {
+ let quantiles = [0, 0, 0]
+
+ for (let i = 0; i < cols.length; ++i) {
+ const col = cols[i]
+ quantiles = sumArrays(quantiles, result[row][col] || [])
+ }
+
+ p50[index] = _round(millisecondsToSeconds(quantiles[0]), 2)
+ p75[index] = _round(millisecondsToSeconds(quantiles[1]), 2)
+ p95[index] = _round(millisecondsToSeconds(quantiles[2]), 2)
+ }
+ }
+
+ return {
+ p50,
+ p75,
+ p95,
+ }
+ }
+
async getPerfChartData(
timeBucket: TimeBucketType,
from: string,
@@ -2387,13 +2463,30 @@ export class AnalyticsService {
filtersQuery: string,
paramsData: object,
safeTimezone: string,
+ measure: PerfMeasure,
) {
const { xShifted } = this.generateXAxis(timeBucket, from, to, safeTimezone)
+ if (measure === 'quantiles') {
+ const query = this.generatePerformanceQuantilesQuery(
+ timeBucket,
+ filtersQuery,
+ safeTimezone,
+ )
+
+ const result = await clickhouse.query(query, paramsData).toPromise()
+
+ return {
+ x: xShifted,
+ ...this.extractPerformanceQuantilesData(result, xShifted),
+ }
+ }
+
const query = this.generatePerformanceAggregationQuery(
timeBucket,
filtersQuery,
safeTimezone,
+ measure,
)
const result = <Array<PerformanceCHResponse>>(
@@ -2414,6 +2507,7 @@ export class AnalyticsService {
filtersQuery: string,
paramsData: object,
safeTimezone: string,
+ measure: PerfMeasure,
): Promise<object | void> {
let params: unknown = {}
let chart: unknown = {}
@@ -2446,6 +2540,7 @@ export class AnalyticsService {
filtersQuery,
paramsData,
safeTimezone,
+ measure,
)
})(),
]
@@ -2810,4 +2905,14 @@ export class AnalyticsService {
// @ts-ignore
return redis.countKeysByPattern(`hb:${projectId}:*`)
}
+
+ checkIfPerfMeasureIsValid(measure: PerfMeasure) {
+ const validMeasures = ['average', 'median', 'p95', 'quantiles']
+
+ if (!_includes(validMeasures, measure)) {
+ throw new UnprocessableEntityException(
+ `Please provide a valid "measure" parameter, it must be one of ${validMeasures}`,
+ )
+ }
+ }
}
diff --git a/apps/production/src/analytics/interfaces/index.ts b/apps/production/src/analytics/interfaces/index.ts
index b543f220d..bee862b0a 100644
--- a/apps/production/src/analytics/interfaces/index.ts
+++ b/apps/production/src/analytics/interfaces/index.ts
@@ -184,3 +184,5 @@ export interface IPageflow {
value: string
created: string
}
+
+export type PerfMeasure = 'average' | 'median' | 'p95' | 'quantiles'
diff --git a/apps/production/src/common/utils.ts b/apps/production/src/common/utils.ts
index 5d24cc708..f6a570876 100644
--- a/apps/production/src/common/utils.ts
+++ b/apps/production/src/common/utils.ts
@@ -9,6 +9,7 @@ import * as _sample from 'lodash/sample'
import * as _toNumber from 'lodash/toNumber'
import * as _replace from 'lodash/replace'
import * as _find from 'lodash/find'
+import * as _size from 'lodash/size'
import * as _round from 'lodash/round'
import * as _split from 'lodash/split'
@@ -241,6 +242,17 @@ const getIPFromHeaders = (headers: any, tryXClientIPAddress?: boolean) => {
return _split(ip, ',')[0]
}
+const sumArrays = (source: number[], target: number[]) => {
+ const result = []
+ const size = _size(source)
+
+ for (let i = 0; i < size; ++i) {
+ result.push(source[i] + target[i])
+ }
+
+ return result
+}
+
export {
getRandomTip,
checkRateLimit,
@@ -253,4 +265,5 @@ export {
getGeoDetails,
getIPFromHeaders,
generateRefCode,
+ sumArrays,
}
diff --git a/apps/selfhosted/src/analytics/analytics.controller.ts b/apps/selfhosted/src/analytics/analytics.controller.ts
index 499762a58..637988e2d 100644
--- a/apps/selfhosted/src/analytics/analytics.controller.ts
+++ b/apps/selfhosted/src/analytics/analytics.controller.ts
@@ -74,6 +74,7 @@ import {
IFunnel,
IGetFunnel,
IUserFlow,
+ PerfMeasure,
} from './interfaces'
import { GetSessionsDto } from './dto/get-sessions.dto'
import { GetSessionDto } from './dto/get-session.dto'
@@ -84,6 +85,8 @@ const mysql = require('mysql2')
dayjs.extend(utc)
dayjs.extend(dayjsTimezone)
+export const DEFAULT_MEASURE = 'median'
+
const getSessionKeyCustom = (
ip: string,
ua: string,
@@ -586,7 +589,7 @@ export class AnalyticsController {
@Get('performance')
@Auth([], true, true)
async getPerfData(
- @Query() data: AnalyticsGET_DTO,
+ @Query() data: AnalyticsGET_DTO & { measure: PerfMeasure },
@CurrentUserId() uid: string,
): Promise<any> {
const {
@@ -597,9 +600,12 @@ export class AnalyticsController {
to,
filters,
timezone = DEFAULT_TIMEZONE,
+ measure = DEFAULT_MEASURE,
} = data
this.analyticsService.validatePID(pid)
+ this.analyticsService.checkIfPerfMeasureIsValid(measure)
+
if (!_isEmpty(period)) {
this.analyticsService.validatePeriod(period)
}
@@ -658,6 +664,7 @@ export class AnalyticsController {
filtersQuery,
paramsData,
safeTimezone,
+ measure,
)
return {
@@ -670,7 +677,7 @@ export class AnalyticsController {
@Get('performance/chart')
@Auth([], true, true)
async getPerfChartData(
- @Query() data: AnalyticsGET_DTO,
+ @Query() data: AnalyticsGET_DTO & { measure: PerfMeasure },
@CurrentUserId() uid: string,
): Promise<any> {
const {
@@ -681,9 +688,12 @@ export class AnalyticsController {
to,
filters,
timezone = DEFAULT_TIMEZONE,
+ measure = DEFAULT_MEASURE,
} = data
this.analyticsService.validatePID(pid)
+ this.analyticsService.checkIfPerfMeasureIsValid(measure)
+
if (!_isEmpty(period)) {
this.analyticsService.validatePeriod(period)
}
@@ -718,6 +728,7 @@ export class AnalyticsController {
filtersQuery,
paramsData,
safeTimezone,
+ measure,
)
return {
diff --git a/apps/selfhosted/src/analytics/analytics.service.ts b/apps/selfhosted/src/analytics/analytics.service.ts
index af3f45fbd..282de7c13 100644
--- a/apps/selfhosted/src/analytics/analytics.service.ts
+++ b/apps/selfhosted/src/analytics/analytics.service.ts
@@ -46,7 +46,7 @@ import {
TRAFFIC_COLUMNS,
PERFORMANCE_COLUMNS,
} from '../common/constants'
-import { millisecondsToSeconds } from '../common/utils'
+import { millisecondsToSeconds, sumArrays } from '../common/utils'
import { PageviewsDTO } from './dto/pageviews.dto'
import { EventsDTO } from './dto/events.dto'
import { ProjectService } from '../project/project.service'
@@ -74,6 +74,7 @@ import {
IOverall,
IOverallPerformance,
IPageflow,
+ PerfMeasure,
} from './interfaces'
dayjs.extend(utc)
@@ -1789,23 +1790,30 @@ export class AnalyticsService {
generatePerformanceAggregationQuery(
timeBucket: TimeBucketType,
filtersQuery: string,
- safeTimezone,
+ safeTimezone: string,
+ measure: PerfMeasure,
): string {
const timeBucketFunc = timeBucketConversion[timeBucket]
const [selector, groupBy] = this.getGroupSubquery(timeBucket)
const tzFromDate = `toTimeZone(parseDateTimeBestEffort({groupFrom:String}), '${safeTimezone}')`
const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), '${safeTimezone}')`
+ const cols = ['dns', 'tls', 'conn', 'response', 'render', 'domLoad', 'ttfb']
+
+ const measures = {
+ average: 'avg',
+ median: 'median',
+ p95: 'quantileExact(0.95)',
+ }
+
+ const columnSelectors = cols
+ .map(col => `${measures[measure]}(${col}) as ${col}`)
+ .join(', ')
+
return `
SELECT
${selector},
- avg(dns) as dns,
- avg(tls) as tls,
- avg(conn) as conn,
- avg(response) as response,
- avg(render) as render,
- avg(domLoad) as domLoad,
- avg(ttfb) as ttfb
+ ${columnSelectors}
FROM (
SELECT *,
${timeBucketFunc}(toTimeZone(created, '${safeTimezone}')) as tz_created
@@ -1817,7 +1825,74 @@ export class AnalyticsService {
) as subquery
GROUP BY ${groupBy}
ORDER BY ${groupBy}
- `
+ `
+ }
+
+ generatePerformanceQuantilesQuery(
+ timeBucket: TimeBucketType,
+ filtersQuery: string,
+ safeTimezone: string,
+ ): string {
+ const timeBucketFunc = timeBucketConversion[timeBucket]
+ const [selector, groupBy] = this.getGroupSubquery(timeBucket)
+ const tzFromDate = `toTimeZone(parseDateTimeBestEffort({groupFrom:String}), '${safeTimezone}')`
+ const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), '${safeTimezone}')`
+
+ const cols = ['dns', 'tls', 'conn', 'response', 'render', 'domLoad', 'ttfb']
+
+ const columnSelectors = cols
+ .map(col => `quantilesExactInclusive(0.5, 0.75, 0.95)(${col}) as ${col}`)
+ .join(', ')
+
+ return `
+ SELECT
+ ${selector},
+ ${columnSelectors}
+ FROM (
+ SELECT *,
+ ${timeBucketFunc}(toTimeZone(created, '${safeTimezone}')) as tz_created
+ FROM performance
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND created BETWEEN ${tzFromDate} AND ${tzToDate}
+ ${filtersQuery}
+ ) as subquery
+ GROUP BY ${groupBy}
+ ORDER BY ${groupBy}
+ `
+ }
+
+ extractPerformanceQuantilesData(result, x: string[]): any {
+ const cols = ['dns', 'tls', 'conn', 'response', 'render', 'domLoad', 'ttfb']
+
+ const p50 = Array(x.length).fill(0)
+ const p75 = Array(x.length).fill(0)
+ const p95 = Array(x.length).fill(0)
+
+ for (let row = 0; row < _size(result); ++row) {
+ const dateString = this.generateDateString(result[row])
+
+ const index = x.indexOf(dateString)
+
+ if (index !== -1) {
+ let quantiles = [0, 0, 0]
+
+ for (let i = 0; i < cols.length; ++i) {
+ const col = cols[i]
+ quantiles = sumArrays(quantiles, result[row][col] || [])
+ }
+
+ p50[index] = _round(millisecondsToSeconds(quantiles[0]), 2)
+ p75[index] = _round(millisecondsToSeconds(quantiles[1]), 2)
+ p95[index] = _round(millisecondsToSeconds(quantiles[2]), 2)
+ }
+ }
+
+ return {
+ p50,
+ p75,
+ p95,
+ }
}
async groupByTimeBucket(
@@ -1999,13 +2074,30 @@ export class AnalyticsService {
filtersQuery: string,
paramsData: object,
safeTimezone: string,
+ measure: PerfMeasure,
) {
const { xShifted } = this.generateXAxis(timeBucket, from, to, safeTimezone)
+ if (measure === 'quantiles') {
+ const query = this.generatePerformanceQuantilesQuery(
+ timeBucket,
+ filtersQuery,
+ safeTimezone,
+ )
+
+ const result = await clickhouse.query(query, paramsData).toPromise()
+
+ return {
+ x: xShifted,
+ ...this.extractPerformanceQuantilesData(result, xShifted),
+ }
+ }
+
const query = this.generatePerformanceAggregationQuery(
timeBucket,
filtersQuery,
safeTimezone,
+ measure,
)
const result = <Array<PerformanceCHResponse>>(
@@ -2026,6 +2118,7 @@ export class AnalyticsService {
filtersQuery: string,
paramsData: object,
safeTimezone: string,
+ measure: PerfMeasure,
): Promise<object | void> {
let params: unknown = {}
let chart: unknown = {}
@@ -2057,6 +2150,7 @@ export class AnalyticsService {
filtersQuery,
paramsData,
safeTimezone,
+ measure,
)
})(),
]
@@ -2321,7 +2415,10 @@ export class AnalyticsService {
.format('YYYY-MM-DD HH:mm:ss')
// eslint-disable-next-line
- timeBucket = dayjs(to).diff(dayjs(from), 'hour') > 1 ? TimeBucketType.HOUR : TimeBucketType.MINUTE
+ timeBucket =
+ dayjs(to).diff(dayjs(from), 'hour') > 1
+ ? TimeBucketType.HOUR
+ : TimeBucketType.MINUTE
const groupedChart = await this.groupChartByTimeBucket(
timeBucket,
@@ -2418,4 +2515,14 @@ export class AnalyticsService {
// @ts-ignore
return redis.countKeysByPattern(`hb:${projectId}:*`)
}
+
+ checkIfPerfMeasureIsValid(measure: PerfMeasure) {
+ const validMeasures = ['average', 'median', 'p95', 'quantiles']
+
+ if (!_includes(validMeasures, measure)) {
+ throw new UnprocessableEntityException(
+ `Please provide a valid "measure" parameter, it must be one of ${validMeasures}`,
+ )
+ }
+ }
}
diff --git a/apps/selfhosted/src/analytics/interfaces/index.ts b/apps/selfhosted/src/analytics/interfaces/index.ts
index d487b75e7..88de841f8 100644
--- a/apps/selfhosted/src/analytics/interfaces/index.ts
+++ b/apps/selfhosted/src/analytics/interfaces/index.ts
@@ -170,3 +170,5 @@ export interface IPageflow {
value: string
created: string
}
+
+export type PerfMeasure = 'average' | 'median' | 'p95' | 'quantiles'
diff --git a/apps/selfhosted/src/common/utils.ts b/apps/selfhosted/src/common/utils.ts
index 90fd4d3e6..20cf585de 100644
--- a/apps/selfhosted/src/common/utils.ts
+++ b/apps/selfhosted/src/common/utils.ts
@@ -12,6 +12,7 @@ import * as _split from 'lodash/split'
import * as _isEmpty from 'lodash/isEmpty'
import * as _head from 'lodash/head'
import * as _round from 'lodash/round'
+import * as _size from 'lodash/size'
import * as dayjs from 'dayjs'
import * as utc from 'dayjs/plugin/utc'
import * as _map from 'lodash/map'
@@ -355,6 +356,17 @@ const getIPFromHeaders = (headers: any) => {
return _split(ip, ',')[0]
}
+const sumArrays = (source: number[], target: number[]) => {
+ const result = []
+ const size = _size(source)
+
+ for (let i = 0; i < size; ++i) {
+ result.push(source[i] + target[i])
+ }
+
+ return result
+}
+
export {
checkRateLimit,
createProjectClickhouse,
@@ -372,4 +384,5 @@ export {
createUserClickhouse,
getGeoDetails,
getIPFromHeaders,
+ sumArrays,
}