📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
(improvement) Performance metrics (#222)
Andrii Romasiunf0d1a736bca8
diff --git a/apps/production/src/analytics/analytics.controller.ts b/apps/production/src/analytics/analytics.controller.tsindex a9892cc5e..6c7bf652c 100644--- a/apps/production/src/analytics/analytics.controller.ts+++ b/apps/production/src/analytics/analytics.controller.ts@@ -75,6 +75,7 @@ import {IFunnel,IGetFunnel,IUserFlow,+ PerfMeasure,} from './interfaces'import { GetSessionsDto } from './dto/get-sessions.dto'import { GetSessionDto } from './dto/get-session.dto'@@ -85,6 +86,8 @@ const mysql = require('mysql2')dayjs.extend(utc)dayjs.extend(dayjsTimezone)+export const DEFAULT_MEASURE = 'median'+const getSessionKeyCustom = (ip: string,ua: string,@@ -626,7 +629,7 @@ export class AnalyticsController {@Get('performance')@Auth([], true, true)async getPerfData(- @Query() data: AnalyticsGET_DTO,+ @Query() data: AnalyticsGET_DTO & { measure: PerfMeasure },@CurrentUserId() uid: string,@Headers() headers: { 'x-password'?: string },): Promise<any> {@@ -638,9 +641,12 @@ export class AnalyticsController {to,filters,timezone = DEFAULT_TIMEZONE,+ measure = DEFAULT_MEASURE,} = datathis.analyticsService.validatePID(pid)+ this.analyticsService.checkIfPerfMeasureIsValid(measure)+if (!_isEmpty(period)) {this.analyticsService.validatePeriod(period)}@@ -705,6 +711,7 @@ export class AnalyticsController {filtersQuery,paramsData,safeTimezone,+ measure,)return {@@ -717,7 +724,7 @@ export class AnalyticsController {@Get('performance/chart')@Auth([], true, true)async getPerfChartData(- @Query() data: AnalyticsGET_DTO,+ @Query() data: AnalyticsGET_DTO & { measure: PerfMeasure },@CurrentUserId() uid: string,@Headers() headers: { 'x-password'?: string },): Promise<any> {@@ -729,9 +736,12 @@ export class AnalyticsController {to,filters,timezone = DEFAULT_TIMEZONE,+ measure = DEFAULT_MEASURE,} = datathis.analyticsService.validatePID(pid)+ this.analyticsService.checkIfPerfMeasureIsValid(measure)+if (!_isEmpty(period)) {this.analyticsService.validatePeriod(period)}@@ -772,6 +782,7 @@ export class AnalyticsController {filtersQuery,paramsData,safeTimezone,+ measure,)return {diff --git a/apps/production/src/analytics/analytics.service.ts b/apps/production/src/analytics/analytics.service.tsindex 9dcf5584d..c68b25ccc 100644--- a/apps/production/src/analytics/analytics.service.ts+++ b/apps/production/src/analytics/analytics.service.ts@@ -55,6 +55,7 @@ import {import {calculateRelativePercentage,millisecondsToSeconds,+ sumArrays,} from '../common/utils'import { PageviewsDTO } from './dto/pageviews.dto'import { EventsDTO } from './dto/events.dto'@@ -84,6 +85,7 @@ import {IOverallCaptcha,IOverallPerformance,IPageflow,+ PerfMeasure,} from './interfaces'dayjs.extend(utc)@@ -2049,23 +2051,30 @@ export class AnalyticsService {generatePerformanceAggregationQuery(timeBucket: TimeBucketType,filtersQuery: string,- safeTimezone,+ safeTimezone: string,+ measure: PerfMeasure,): string {const timeBucketFunc = timeBucketConversion[timeBucket]const [selector, groupBy] = this.getGroupSubquery(timeBucket)const tzFromDate = `toTimeZone(parseDateTimeBestEffort({groupFrom:String}), '${safeTimezone}')`const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), '${safeTimezone}')`+ const cols = ['dns', 'tls', 'conn', 'response', 'render', 'domLoad', 'ttfb']++ const measures = {+ average: 'avg',+ median: 'median',+ p95: 'quantileExact(0.95)',+ }++ const columnSelectors = cols+ .map(col => `${measures[measure]}(${col}) as ${col}`)+ .join(', ')+return `SELECT${selector},- avg(dns) as dns,- avg(tls) as tls,- avg(conn) as conn,- avg(response) as response,- avg(render) as render,- avg(domLoad) as domLoad,- avg(ttfb) as ttfb+ ${columnSelectors}FROM (SELECT *,${timeBucketFunc}(toTimeZone(created, '${safeTimezone}')) as tz_created@@ -2077,7 +2086,41 @@ export class AnalyticsService {) as subqueryGROUP BY ${groupBy}ORDER BY ${groupBy}- `+ `+ }++ generatePerformanceQuantilesQuery(+ timeBucket: TimeBucketType,+ filtersQuery: string,+ safeTimezone: string,+ ): string {+ const timeBucketFunc = timeBucketConversion[timeBucket]+ const [selector, groupBy] = this.getGroupSubquery(timeBucket)+ const tzFromDate = `toTimeZone(parseDateTimeBestEffort({groupFrom:String}), '${safeTimezone}')`+ const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), '${safeTimezone}')`++ const cols = ['dns', 'tls', 'conn', 'response', 'render', 'domLoad', 'ttfb']++ const columnSelectors = cols+ .map(col => `quantilesExactInclusive(0.5, 0.75, 0.95)(${col}) as ${col}`)+ .join(', ')++ return `+ SELECT+ ${selector},+ ${columnSelectors}+ FROM (+ SELECT *,+ ${timeBucketFunc}(toTimeZone(created, '${safeTimezone}')) as tz_created+ FROM performance+ WHERE+ pid = {pid:FixedString(12)}+ AND created BETWEEN ${tzFromDate} AND ${tzToDate}+ ${filtersQuery}+ ) as subquery+ GROUP BY ${groupBy}+ ORDER BY ${groupBy}+ `}generateCaptchaAggregationQuery(@@ -2380,6 +2423,39 @@ export class AnalyticsService {}}+ extractPerformanceQuantilesData(result, x: string[]): any {+ const cols = ['dns', 'tls', 'conn', 'response', 'render', 'domLoad', 'ttfb']++ const p50 = Array(x.length).fill(0)+ const p75 = Array(x.length).fill(0)+ const p95 = Array(x.length).fill(0)++ for (let row = 0; row < _size(result); ++row) {+ const dateString = this.generateDateString(result[row])++ const index = x.indexOf(dateString)++ if (index !== -1) {+ let quantiles = [0, 0, 0]++ for (let i = 0; i < cols.length; ++i) {+ const col = cols[i]+ quantiles = sumArrays(quantiles, result[row][col] || [])+ }++ p50[index] = _round(millisecondsToSeconds(quantiles[0]), 2)+ p75[index] = _round(millisecondsToSeconds(quantiles[1]), 2)+ p95[index] = _round(millisecondsToSeconds(quantiles[2]), 2)+ }+ }++ return {+ p50,+ p75,+ p95,+ }+ }+async getPerfChartData(timeBucket: TimeBucketType,from: string,@@ -2387,13 +2463,30 @@ export class AnalyticsService {filtersQuery: string,paramsData: object,safeTimezone: string,+ measure: PerfMeasure,) {const { xShifted } = this.generateXAxis(timeBucket, from, to, safeTimezone)+ if (measure === 'quantiles') {+ const query = this.generatePerformanceQuantilesQuery(+ timeBucket,+ filtersQuery,+ safeTimezone,+ )++ const result = await clickhouse.query(query, paramsData).toPromise()++ return {+ x: xShifted,+ ...this.extractPerformanceQuantilesData(result, xShifted),+ }+ }+const query = this.generatePerformanceAggregationQuery(timeBucket,filtersQuery,safeTimezone,+ measure,)const result = <Array<PerformanceCHResponse>>(@@ -2414,6 +2507,7 @@ export class AnalyticsService {filtersQuery: string,paramsData: object,safeTimezone: string,+ measure: PerfMeasure,): Promise<object | void> {let params: unknown = {}let chart: unknown = {}@@ -2446,6 +2540,7 @@ export class AnalyticsService {filtersQuery,paramsData,safeTimezone,+ measure,)})(),]@@ -2810,4 +2905,14 @@ export class AnalyticsService {// @ts-ignorereturn redis.countKeysByPattern(`hb:${projectId}:*`)}++ checkIfPerfMeasureIsValid(measure: PerfMeasure) {+ const validMeasures = ['average', 'median', 'p95', 'quantiles']++ if (!_includes(validMeasures, measure)) {+ throw new UnprocessableEntityException(+ `Please provide a valid "measure" parameter, it must be one of ${validMeasures}`,+ )+ }+ }}diff --git a/apps/production/src/analytics/interfaces/index.ts b/apps/production/src/analytics/interfaces/index.tsindex b543f220d..bee862b0a 100644--- a/apps/production/src/analytics/interfaces/index.ts+++ b/apps/production/src/analytics/interfaces/index.ts@@ -184,3 +184,5 @@ export interface IPageflow {value: stringcreated: string}++export type PerfMeasure = 'average' | 'median' | 'p95' | 'quantiles'diff --git a/apps/production/src/common/utils.ts b/apps/production/src/common/utils.tsindex 5d24cc708..f6a570876 100644--- a/apps/production/src/common/utils.ts+++ b/apps/production/src/common/utils.ts@@ -9,6 +9,7 @@ import * as _sample from 'lodash/sample'import * as _toNumber from 'lodash/toNumber'import * as _replace from 'lodash/replace'import * as _find from 'lodash/find'+import * as _size from 'lodash/size'import * as _round from 'lodash/round'import * as _split from 'lodash/split'@@ -241,6 +242,17 @@ const getIPFromHeaders = (headers: any, tryXClientIPAddress?: boolean) => {return _split(ip, ',')[0]}+const sumArrays = (source: number[], target: number[]) => {+ const result = []+ const size = _size(source)++ for (let i = 0; i < size; ++i) {+ result.push(source[i] + target[i])+ }++ return result+}+export {getRandomTip,checkRateLimit,@@ -253,4 +265,5 @@ export {getGeoDetails,getIPFromHeaders,generateRefCode,+ sumArrays,}diff --git a/apps/selfhosted/src/analytics/analytics.controller.ts b/apps/selfhosted/src/analytics/analytics.controller.tsindex 499762a58..637988e2d 100644--- a/apps/selfhosted/src/analytics/analytics.controller.ts+++ b/apps/selfhosted/src/analytics/analytics.controller.ts@@ -74,6 +74,7 @@ import {IFunnel,IGetFunnel,IUserFlow,+ PerfMeasure,} from './interfaces'import { GetSessionsDto } from './dto/get-sessions.dto'import { GetSessionDto } from './dto/get-session.dto'@@ -84,6 +85,8 @@ const mysql = require('mysql2')dayjs.extend(utc)dayjs.extend(dayjsTimezone)+export const DEFAULT_MEASURE = 'median'+const getSessionKeyCustom = (ip: string,ua: string,@@ -586,7 +589,7 @@ export class AnalyticsController {@Get('performance')@Auth([], true, true)async getPerfData(- @Query() data: AnalyticsGET_DTO,+ @Query() data: AnalyticsGET_DTO & { measure: PerfMeasure },@CurrentUserId() uid: string,): Promise<any> {const {@@ -597,9 +600,12 @@ export class AnalyticsController {to,filters,timezone = DEFAULT_TIMEZONE,+ measure = DEFAULT_MEASURE,} = datathis.analyticsService.validatePID(pid)+ this.analyticsService.checkIfPerfMeasureIsValid(measure)+if (!_isEmpty(period)) {this.analyticsService.validatePeriod(period)}@@ -658,6 +664,7 @@ export class AnalyticsController {filtersQuery,paramsData,safeTimezone,+ measure,)return {@@ -670,7 +677,7 @@ export class AnalyticsController {@Get('performance/chart')@Auth([], true, true)async getPerfChartData(- @Query() data: AnalyticsGET_DTO,+ @Query() data: AnalyticsGET_DTO & { measure: PerfMeasure },@CurrentUserId() uid: string,): Promise<any> {const {@@ -681,9 +688,12 @@ export class AnalyticsController {to,filters,timezone = DEFAULT_TIMEZONE,+ measure = DEFAULT_MEASURE,} = datathis.analyticsService.validatePID(pid)+ this.analyticsService.checkIfPerfMeasureIsValid(measure)+if (!_isEmpty(period)) {this.analyticsService.validatePeriod(period)}@@ -718,6 +728,7 @@ export class AnalyticsController {filtersQuery,paramsData,safeTimezone,+ measure,)return {diff --git a/apps/selfhosted/src/analytics/analytics.service.ts b/apps/selfhosted/src/analytics/analytics.service.tsindex af3f45fbd..282de7c13 100644--- a/apps/selfhosted/src/analytics/analytics.service.ts+++ b/apps/selfhosted/src/analytics/analytics.service.ts@@ -46,7 +46,7 @@ import {TRAFFIC_COLUMNS,PERFORMANCE_COLUMNS,} from '../common/constants'-import { millisecondsToSeconds } from '../common/utils'+import { millisecondsToSeconds, sumArrays } from '../common/utils'import { PageviewsDTO } from './dto/pageviews.dto'import { EventsDTO } from './dto/events.dto'import { ProjectService } from '../project/project.service'@@ -74,6 +74,7 @@ import {IOverall,IOverallPerformance,IPageflow,+ PerfMeasure,} from './interfaces'dayjs.extend(utc)@@ -1789,23 +1790,30 @@ export class AnalyticsService {generatePerformanceAggregationQuery(timeBucket: TimeBucketType,filtersQuery: string,- safeTimezone,+ safeTimezone: string,+ measure: PerfMeasure,): string {const timeBucketFunc = timeBucketConversion[timeBucket]const [selector, groupBy] = this.getGroupSubquery(timeBucket)const tzFromDate = `toTimeZone(parseDateTimeBestEffort({groupFrom:String}), '${safeTimezone}')`const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), '${safeTimezone}')`+ const cols = ['dns', 'tls', 'conn', 'response', 'render', 'domLoad', 'ttfb']++ const measures = {+ average: 'avg',+ median: 'median',+ p95: 'quantileExact(0.95)',+ }++ const columnSelectors = cols+ .map(col => `${measures[measure]}(${col}) as ${col}`)+ .join(', ')+return `SELECT${selector},- avg(dns) as dns,- avg(tls) as tls,- avg(conn) as conn,- avg(response) as response,- avg(render) as render,- avg(domLoad) as domLoad,- avg(ttfb) as ttfb+ ${columnSelectors}FROM (SELECT *,${timeBucketFunc}(toTimeZone(created, '${safeTimezone}')) as tz_created@@ -1817,7 +1825,74 @@ export class AnalyticsService {) as subqueryGROUP BY ${groupBy}ORDER BY ${groupBy}- `+ `+ }++ generatePerformanceQuantilesQuery(+ timeBucket: TimeBucketType,+ filtersQuery: string,+ safeTimezone: string,+ ): string {+ const timeBucketFunc = timeBucketConversion[timeBucket]+ const [selector, groupBy] = this.getGroupSubquery(timeBucket)+ const tzFromDate = `toTimeZone(parseDateTimeBestEffort({groupFrom:String}), '${safeTimezone}')`+ const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), '${safeTimezone}')`++ const cols = ['dns', 'tls', 'conn', 'response', 'render', 'domLoad', 'ttfb']++ const columnSelectors = cols+ .map(col => `quantilesExactInclusive(0.5, 0.75, 0.95)(${col}) as ${col}`)+ .join(', ')++ return `+ SELECT+ ${selector},+ ${columnSelectors}+ FROM (+ SELECT *,+ ${timeBucketFunc}(toTimeZone(created, '${safeTimezone}')) as tz_created+ FROM performance+ WHERE+ pid = {pid:FixedString(12)}+ AND created BETWEEN ${tzFromDate} AND ${tzToDate}+ ${filtersQuery}+ ) as subquery+ GROUP BY ${groupBy}+ ORDER BY ${groupBy}+ `+ }++ extractPerformanceQuantilesData(result, x: string[]): any {+ const cols = ['dns', 'tls', 'conn', 'response', 'render', 'domLoad', 'ttfb']++ const p50 = Array(x.length).fill(0)+ const p75 = Array(x.length).fill(0)+ const p95 = Array(x.length).fill(0)++ for (let row = 0; row < _size(result); ++row) {+ const dateString = this.generateDateString(result[row])++ const index = x.indexOf(dateString)++ if (index !== -1) {+ let quantiles = [0, 0, 0]++ for (let i = 0; i < cols.length; ++i) {+ const col = cols[i]+ quantiles = sumArrays(quantiles, result[row][col] || [])+ }++ p50[index] = _round(millisecondsToSeconds(quantiles[0]), 2)+ p75[index] = _round(millisecondsToSeconds(quantiles[1]), 2)+ p95[index] = _round(millisecondsToSeconds(quantiles[2]), 2)+ }+ }++ return {+ p50,+ p75,+ p95,+ }}async groupByTimeBucket(@@ -1999,13 +2074,30 @@ export class AnalyticsService {filtersQuery: string,paramsData: object,safeTimezone: string,+ measure: PerfMeasure,) {const { xShifted } = this.generateXAxis(timeBucket, from, to, safeTimezone)+ if (measure === 'quantiles') {+ const query = this.generatePerformanceQuantilesQuery(+ timeBucket,+ filtersQuery,+ safeTimezone,+ )++ const result = await clickhouse.query(query, paramsData).toPromise()++ return {+ x: xShifted,+ ...this.extractPerformanceQuantilesData(result, xShifted),+ }+ }+const query = this.generatePerformanceAggregationQuery(timeBucket,filtersQuery,safeTimezone,+ measure,)const result = <Array<PerformanceCHResponse>>(@@ -2026,6 +2118,7 @@ export class AnalyticsService {filtersQuery: string,paramsData: object,safeTimezone: string,+ measure: PerfMeasure,): Promise<object | void> {let params: unknown = {}let chart: unknown = {}@@ -2057,6 +2150,7 @@ export class AnalyticsService {filtersQuery,paramsData,safeTimezone,+ measure,)})(),]@@ -2321,7 +2415,10 @@ export class AnalyticsService {.format('YYYY-MM-DD HH:mm:ss')// eslint-disable-next-line- timeBucket = dayjs(to).diff(dayjs(from), 'hour') > 1 ? TimeBucketType.HOUR : TimeBucketType.MINUTE+ timeBucket =+ dayjs(to).diff(dayjs(from), 'hour') > 1+ ? TimeBucketType.HOUR+ : TimeBucketType.MINUTEconst groupedChart = await this.groupChartByTimeBucket(timeBucket,@@ -2418,4 +2515,14 @@ export class AnalyticsService {// @ts-ignorereturn redis.countKeysByPattern(`hb:${projectId}:*`)}++ checkIfPerfMeasureIsValid(measure: PerfMeasure) {+ const validMeasures = ['average', 'median', 'p95', 'quantiles']++ if (!_includes(validMeasures, measure)) {+ throw new UnprocessableEntityException(+ `Please provide a valid "measure" parameter, it must be one of ${validMeasures}`,+ )+ }+ }}diff --git a/apps/selfhosted/src/analytics/interfaces/index.ts b/apps/selfhosted/src/analytics/interfaces/index.tsindex d487b75e7..88de841f8 100644--- a/apps/selfhosted/src/analytics/interfaces/index.ts+++ b/apps/selfhosted/src/analytics/interfaces/index.ts@@ -170,3 +170,5 @@ export interface IPageflow {value: stringcreated: string}++export type PerfMeasure = 'average' | 'median' | 'p95' | 'quantiles'diff --git a/apps/selfhosted/src/common/utils.ts b/apps/selfhosted/src/common/utils.tsindex 90fd4d3e6..20cf585de 100644--- a/apps/selfhosted/src/common/utils.ts+++ b/apps/selfhosted/src/common/utils.ts@@ -12,6 +12,7 @@ import * as _split from 'lodash/split'import * as _isEmpty from 'lodash/isEmpty'import * as _head from 'lodash/head'import * as _round from 'lodash/round'+import * as _size from 'lodash/size'import * as dayjs from 'dayjs'import * as utc from 'dayjs/plugin/utc'import * as _map from 'lodash/map'@@ -355,6 +356,17 @@ const getIPFromHeaders = (headers: any) => {return _split(ip, ',')[0]}+const sumArrays = (source: number[], target: number[]) => {+ const result = []+ const size = _size(source)++ for (let i = 0; i < size; ++i) {+ result.push(source[i] + target[i])+ }++ return result+}+export {checkRateLimit,createProjectClickhouse,@@ -372,4 +384,5 @@ export {createUserClickhouse,getGeoDetails,getIPFromHeaders,+ sumArrays,}