📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
(feature) Custom events metadata (#196)
Andrii Romasiuna69455358997
diff --git a/apps/production/src/analytics/analytics.controller.ts b/apps/production/src/analytics/analytics.controller.tsindex 061e2083a..3df90089f 100644--- a/apps/production/src/analytics/analytics.controller.ts+++ b/apps/production/src/analytics/analytics.controller.ts@@ -3,6 +3,8 @@ import * as _isArray from 'lodash/isArray'import * as _toNumber from 'lodash/toNumber'import * as _pick from 'lodash/pick'import * as _includes from 'lodash/includes'+import * as _keys from 'lodash/keys'+import * as _values from 'lodash/values'import * as _map from 'lodash/map'import * as _uniqBy from 'lodash/uniqBy'import * as _round from 'lodash/round'@@ -45,6 +47,7 @@ import { RolesGuard } from '../auth/guards/roles.guard'import { PageviewsDTO } from './dto/pageviews.dto'import { EventsDTO } from './dto/events.dto'import { AnalyticsGET_DTO } from './dto/getData.dto'+import { GetCustomEventMetadata } from './dto/get-custom-event-meta.dto'import { GetUserFlowDTO } from './dto/getUserFlow.dto'import { AppLoggerService } from '../logger/logger.service'import {@@ -64,7 +67,7 @@ import { BotDetection } from '../common/decorators/bot-detection.decorator'import { BotDetectionGuard } from '../common/guards/bot-detection.guard'import { GetCustomEventsDto } from './dto/get-custom-events.dto'import { GetFiltersDto } from './dto/get-filters.dto'-import { IUserFlow } from './interfaces'+import { IAggregatedMetadata, IUserFlow } from './interfaces'// eslint-disable-next-line @typescript-eslint/no-var-requiresconst mysql = require('mysql2')@@ -173,7 +176,9 @@ const customLogDTO = (cc: string,rg: string,ct: string,-): Array<string | number> => {+ keys: string[],+ values: string[],+): Array<string | number | string[]> => {return [pid,ev,@@ -189,11 +194,17 @@ const customLogDTO = (cc,rg,ct,+ keys,+ values,dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),]}export const getElValue = el => {+ if (_isArray(el)) {+ return `[${_map(el, getElValue).join(',')}]`+ }+if (el === undefined || el === null || el === 'NULL') return 'NULL'return mysql.escape(el)}@@ -403,6 +414,29 @@ export class AnalyticsController {}}+ @Get('meta')+ @Auth([], true, true)+ async getCustomEventMetadata(+ @Query() data: GetCustomEventMetadata,+ @CurrentUserId() uid: string,+ @Headers() headers: { 'x-password'?: string },+ ): Promise<IAggregatedMetadata[]> {+ const { pid, period } = data+ this.analyticsService.validatePID(pid)++ if (!_isEmpty(period)) {+ this.analyticsService.validatePeriod(period)+ }++ await this.analyticsService.checkProjectAccess(+ pid,+ uid,+ headers['x-password'],+ )++ return this.analyticsService.getCustomEventMetadata(data)+ }+@Get('filters')@Auth([], true, true)async getFilters(@@ -876,6 +910,7 @@ export class AnalyticsController {const ip = getIPFromHeaders(headers, true) || reqIP || ''+ this.analyticsService.validateCustomEVMeta(eventsDTO.meta)await this.analyticsService.validate(eventsDTO, origin, 'custom', ip)if (eventsDTO.unique) {@@ -922,6 +957,8 @@ export class AnalyticsController {country,region,city,+ _keys(eventsDTO.meta),+ _values(eventsDTO.meta),)try {diff --git a/apps/production/src/analytics/analytics.service.ts b/apps/production/src/analytics/analytics.service.tsindex d6802b529..912835a57 100644--- a/apps/production/src/analytics/analytics.service.ts+++ b/apps/production/src/analytics/analytics.service.ts@@ -12,6 +12,7 @@ import * as _last from 'lodash/last'import * as _some from 'lodash/some'import * as _find from 'lodash/find'import * as _now from 'lodash/now'+import * as _isString from 'lodash/isString'import * as _values from 'lodash/values'import * as _round from 'lodash/round'import * as _filter from 'lodash/filter'@@ -56,6 +57,7 @@ import { EventsDTO } from './dto/events.dto'import { ProjectService } from '../project/project.service'import { Project } from '../project/entity/project.entity'import { TimeBucketType } from './dto/getData.dto'+import { GetCustomEventMetadata } from './dto/get-custom-event-meta.dto'import {PerformanceCHResponse,CustomsCHResponse,@@ -70,6 +72,7 @@ import {IBuildUserFlow,IExtractChartData,IGenerateXAxis,+ IAggregatedMetadata,} from './interfaces'dayjs.extend(utc)@@ -1696,6 +1699,88 @@ export class AnalyticsService {return result}+ validateCustomEVMeta(meta: any) {+ if (typeof meta === 'undefined') {+ return+ }++ if (_some(_values(meta), val => !_isString(val))) {+ throw new UnprocessableEntityException(+ 'The provided custom event metadata is incorrect (some values are not strings)',+ )+ }+ }++ async getCustomEventMetadata(+ data: GetCustomEventMetadata,+ ): Promise<IAggregatedMetadata[]> {+ const {+ pid,+ period,+ timeBucket,+ from,+ to,+ timezone = DEFAULT_TIMEZONE,+ event,+ } = data++ let newTimebucket = timeBucket++ let diff++ if (period === 'all') {+ const res = await this.getTimeBucketForAllTime(pid, period, timezone)++ diff = res.diff+ // eslint-disable-next-line prefer-destructuring+ newTimebucket = _includes(res.timeBucket, timeBucket)+ ? timeBucket+ : res.timeBucket[0]+ }++ this.validateTimebucket(newTimebucket)++ const safeTimezone = this.getSafeTimezone(timezone)+ const { groupFromUTC, groupToUTC } = this.getGroupFromTo(+ from,+ to,+ newTimebucket,+ period,+ safeTimezone,+ diff,+ )++ const query = `SELECT+ meta.key AS key,+ meta.value AS value,+ count() AS count+ FROM customEV+ ARRAY JOIN meta.key, meta.value+ WHERE pid = {pid:FixedString(12)}+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}+ AND ev = {event:String}+ GROUP BY key, value`++ const paramsData = {+ params: {+ pid,+ groupFrom: groupFromUTC,+ groupTo: groupToUTC,+ event,+ },+ }++ try {+ const result = await clickhouse.query(query, paramsData).toPromise()+ return result as IAggregatedMetadata[]+ } catch (reason) {+ console.error(`[ERROR](getCustomEventMetadata): ${reason}`)+ throw new InternalServerErrorException(+ 'Something went wrong. Please, try again later.',+ )+ }+ }+async getOnlineUserCount(pid: string): Promise<number> {// @ts-ignorereturn redis.countKeysByPattern(`hb:${pid}:*`)diff --git a/apps/production/src/analytics/dto/events.dto.ts b/apps/production/src/analytics/dto/events.dto.tsindex bceec4535..2ebc02d89 100644--- a/apps/production/src/analytics/dto/events.dto.ts+++ b/apps/production/src/analytics/dto/events.dto.ts@@ -1,5 +1,41 @@+import * as _keys from 'lodash/keys'+import * as _values from 'lodash/values'import { ApiProperty } from '@nestjs/swagger'-import { IsNotEmpty } from 'class-validator'+import {+ IsNotEmpty,+ IsOptional,+ IsObject,+ ValidatorConstraint,+ ValidatorConstraintInterface,+ Validate,+} from 'class-validator'++const MAX_METADATA_KEYS = 20+const MAX_METADATA_VALUE_LENGTH = 1000++@ValidatorConstraint()+class MetadataSizeLimit implements ValidatorConstraintInterface {+ validate(metadata: Record<string, string>) {+ const values = _values(metadata)+ let totalSize = 0++ for (const value of values) {+ totalSize += value.length+ if (totalSize > MAX_METADATA_VALUE_LENGTH) {+ return false+ }+ }++ return true+ }+}++@ValidatorConstraint()+class MetadataKeysQuantity implements ValidatorConstraintInterface {+ validate(metadata: Record<string, string>) {+ return _keys(metadata).length <= MAX_METADATA_KEYS+ }+}export class EventsDTO {@ApiProperty({@@ -66,4 +102,21 @@ export class EventsDTO {description: 'utm_campaign URL parameter',})ca?: string++ @ApiProperty({+ example: {+ affiliate: 'Yes',+ protocol: 'HTTPS',+ },+ description: 'Event-related metadata object with string values',+ })+ @IsOptional()+ @IsObject()+ @Validate(MetadataKeysQuantity, {+ message: `Metadata object can't have more than ${MAX_METADATA_KEYS} keys`,+ })+ @Validate(MetadataSizeLimit, {+ message: `Metadata object can't have values with total length more than ${MAX_METADATA_VALUE_LENGTH} characters`,+ })+ meta?: Record<string, string>}diff --git a/apps/production/src/analytics/dto/get-custom-event-meta.dto.ts b/apps/production/src/analytics/dto/get-custom-event-meta.dto.tsnew file mode 100644index 000000000..d91e1cba3--- /dev/null+++ b/apps/production/src/analytics/dto/get-custom-event-meta.dto.ts@@ -0,0 +1,14 @@+import { PickType } from '@nestjs/swagger'+import { AnalyticsGET_DTO } from './getData.dto'++export class GetCustomEventMetadata extends PickType(AnalyticsGET_DTO, [+ 'pid',+ 'period',+ 'timeBucket',+ 'from',+ 'to',+ 'filters',+ 'timezone',+] as const) {+ event: string+}diff --git a/apps/production/src/analytics/interfaces/index.ts b/apps/production/src/analytics/interfaces/index.tsindex fda538f0c..0b1e533b9 100644--- a/apps/production/src/analytics/interfaces/index.ts+++ b/apps/production/src/analytics/interfaces/index.ts@@ -93,3 +93,9 @@ export interface IExtractChartData {uniques: number[]sdur: number[]}++export interface IAggregatedMetadata {+ key: string+ value: string+ count: number+}diff --git a/apps/selfhosted/src/analytics/analytics.controller.ts b/apps/selfhosted/src/analytics/analytics.controller.tsindex 9e3669eb9..82e577ae1 100644--- a/apps/selfhosted/src/analytics/analytics.controller.ts+++ b/apps/selfhosted/src/analytics/analytics.controller.ts@@ -5,6 +5,8 @@ import * as _map from 'lodash/map'import * as _uniqBy from 'lodash/uniqBy'import * as _round from 'lodash/round'import * as _includes from 'lodash/includes'+import * as _keys from 'lodash/keys'+import * as _values from 'lodash/values'import * as dayjs from 'dayjs'import * as utc from 'dayjs/plugin/utc'import * as dayjsTimezone from 'dayjs/plugin/timezone'@@ -44,6 +46,7 @@ import { PageviewsDTO } from './dto/pageviews.dto'import { EventsDTO } from './dto/events.dto'import { AnalyticsGET_DTO } from './dto/getData.dto'import { GetFiltersDto } from './dto/get-filters.dto'+import { GetCustomEventMetadata } from './dto/get-custom-event-meta.dto'import { GetUserFlowDTO } from './dto/getUserFlow.dto'import { AppLoggerService } from '../logger/logger.service'import {@@ -59,7 +62,7 @@ import { getGeoDetails, getIPFromHeaders } from '../common/utils'import { BotDetection } from '../common/decorators/bot-detection.decorator'import { BotDetectionGuard } from '../common/guards/bot-detection.guard'import { GetCustomEventsDto } from './dto/get-custom-events.dto'-import { IUserFlow } from './interfaces'+import { IAggregatedMetadata, IUserFlow } from './interfaces'// eslint-disable-next-line @typescript-eslint/no-var-requiresconst mysql = require('mysql2')@@ -168,7 +171,9 @@ const customLogDTO = (cc: string,rg: string,ct: string,-): Array<string | number> => {+ keys: string[],+ values: string[],+): Array<string | number | string[]> => {return [pid,ev,@@ -184,11 +189,17 @@ const customLogDTO = (cc,rg,ct,+ keys,+ values,dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),]}const getElValue = el => {+ if (_isArray(el)) {+ return `[${_map(el, getElValue).join(',')}]`+ }+if (el === undefined || el === null || el === 'NULL') return 'NULL'return mysql.escape(el)}@@ -364,6 +375,24 @@ export class AnalyticsController {}}+ @Get('meta')+ @Auth([], true, true)+ async getCustomEventMetadata(+ @Query() data: GetCustomEventMetadata,+ @CurrentUserId() uid: string,+ ): Promise<IAggregatedMetadata[]> {+ const { pid, period } = data+ this.analyticsService.validatePID(pid)++ if (!_isEmpty(period)) {+ this.analyticsService.validatePeriod(period)+ }++ await this.analyticsService.checkProjectAccess(pid, uid)++ return this.analyticsService.getCustomEventMetadata(data)+ }+@Get('filters')@Auth([], true, true)async getFilters(@@ -739,6 +768,7 @@ export class AnalyticsController {const ip = getIPFromHeaders(headers) || reqIP || ''+ this.analyticsService.validateCustomEVMeta(eventsDTO.meta)await this.analyticsService.validate(eventsDTO, origin, 'custom', ip)if (eventsDTO.unique) {@@ -785,6 +815,8 @@ export class AnalyticsController {country,region,city,+ _keys(eventsDTO.meta),+ _values(eventsDTO.meta),)try {diff --git a/apps/selfhosted/src/analytics/analytics.service.ts b/apps/selfhosted/src/analytics/analytics.service.tsindex 2c2400764..a67bb4cc3 100644--- a/apps/selfhosted/src/analytics/analytics.service.ts+++ b/apps/selfhosted/src/analytics/analytics.service.ts@@ -15,6 +15,7 @@ import * as _now from 'lodash/now'import * as _values from 'lodash/values'import * as _round from 'lodash/round'import * as _filter from 'lodash/filter'+import * as _isString from 'lodash/isString'import * as dayjs from 'dayjs'import * as utc from 'dayjs/plugin/utc'import * as dayjsTimezone from 'dayjs/plugin/timezone'@@ -48,6 +49,7 @@ import { PageviewsDTO } from './dto/pageviews.dto'import { EventsDTO } from './dto/events.dto'import { ProjectService } from '../project/project.service'import { Project } from '../project/entity/project.entity'+import { GetCustomEventMetadata } from './dto/get-custom-event-meta.dto'import { TimeBucketType } from './dto/getData.dto'import {PerformanceCHResponse,@@ -63,6 +65,7 @@ import {IBuildUserFlow,IExtractChartData,IGenerateXAxis,+ IAggregatedMetadata,} from './interfaces'dayjs.extend(utc)@@ -1523,6 +1526,88 @@ export class AnalyticsService {return result}+ validateCustomEVMeta(meta: any) {+ if (typeof meta === 'undefined') {+ return+ }++ if (_some(_values(meta), val => !_isString(val))) {+ throw new UnprocessableEntityException(+ 'The provided custom event metadata is incorrect (some values are not strings)',+ )+ }+ }++ async getCustomEventMetadata(+ data: GetCustomEventMetadata,+ ): Promise<IAggregatedMetadata[]> {+ const {+ pid,+ period,+ timeBucket,+ from,+ to,+ timezone = DEFAULT_TIMEZONE,+ event,+ } = data++ let newTimebucket = timeBucket++ let diff++ if (period === 'all') {+ const res = await this.getTimeBucketForAllTime(pid, period, timezone)++ diff = res.diff+ // eslint-disable-next-line prefer-destructuring+ newTimebucket = _includes(res.timeBucket, timeBucket)+ ? timeBucket+ : res.timeBucket[0]+ }++ this.validateTimebucket(newTimebucket)++ const safeTimezone = this.getSafeTimezone(timezone)+ const { groupFromUTC, groupToUTC } = this.getGroupFromTo(+ from,+ to,+ newTimebucket,+ period,+ safeTimezone,+ diff,+ )++ const query = `SELECT+ meta.key AS key,+ meta.value AS value,+ count() AS count+ FROM customEV+ ARRAY JOIN meta.key, meta.value+ WHERE pid = {pid:FixedString(12)}+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}+ AND ev = {event:String}+ GROUP BY key, value`++ const paramsData = {+ params: {+ pid,+ groupFrom: groupFromUTC,+ groupTo: groupToUTC,+ event,+ },+ }++ try {+ const result = await clickhouse.query(query, paramsData).toPromise()+ return result as IAggregatedMetadata[]+ } catch (reason) {+ console.error(`[ERROR](getCustomEventMetadata): ${reason}`)+ throw new InternalServerErrorException(+ 'Something went wrong. Please, try again later.',+ )+ }+ }+async getOnlineUserCount(pid: string): Promise<number> {// @ts-ignorereturn redis.countKeysByPattern(`hb:${pid}:*`)diff --git a/apps/selfhosted/src/analytics/dto/events.dto.ts b/apps/selfhosted/src/analytics/dto/events.dto.tsindex 7cec51ad4..e16cbc86d 100644--- a/apps/selfhosted/src/analytics/dto/events.dto.ts+++ b/apps/selfhosted/src/analytics/dto/events.dto.ts@@ -1,5 +1,41 @@+import * as _keys from 'lodash/keys'+import * as _values from 'lodash/values'import { ApiProperty } from '@nestjs/swagger'-import { IsNotEmpty } from 'class-validator'+import {+ IsNotEmpty,+ IsOptional,+ IsObject,+ ValidatorConstraint,+ ValidatorConstraintInterface,+ Validate,+} from 'class-validator'++const MAX_METADATA_KEYS = 20+const MAX_METADATA_VALUE_LENGTH = 1000++@ValidatorConstraint()+class MetadataSizeLimit implements ValidatorConstraintInterface {+ validate(metadata: Record<string, string>) {+ const values = _values(metadata)+ let totalSize = 0++ for (const value of values) {+ totalSize += value.length+ if (totalSize > MAX_METADATA_VALUE_LENGTH) {+ return false+ }+ }++ return true+ }+}++@ValidatorConstraint()+class MetadataKeysQuantity implements ValidatorConstraintInterface {+ validate(metadata: Record<string, string>) {+ return _keys(metadata).length <= MAX_METADATA_KEYS+ }+}export class EventsDTO {@ApiProperty({@@ -66,4 +102,21 @@ export class EventsDTO {description: 'utm_campaign URL parameter',})ca?: string++ @ApiProperty({+ example: {+ affiliate: 'Yes',+ protocol: 'HTTPS',+ },+ description: 'Event-related metadata object with string values',+ })+ @IsOptional()+ @IsObject()+ @Validate(MetadataKeysQuantity, {+ message: `Metadata object can't have more than ${MAX_METADATA_KEYS} keys`,+ })+ @Validate(MetadataSizeLimit, {+ message: `Metadata object can't have values with total length more than ${MAX_METADATA_VALUE_LENGTH} characters`,+ })+ meta?: Record<string, string>}diff --git a/apps/selfhosted/src/analytics/dto/get-custom-event-meta.dto.ts b/apps/selfhosted/src/analytics/dto/get-custom-event-meta.dto.tsnew file mode 100644index 000000000..d91e1cba3--- /dev/null+++ b/apps/selfhosted/src/analytics/dto/get-custom-event-meta.dto.ts@@ -0,0 +1,14 @@+import { PickType } from '@nestjs/swagger'+import { AnalyticsGET_DTO } from './getData.dto'++export class GetCustomEventMetadata extends PickType(AnalyticsGET_DTO, [+ 'pid',+ 'period',+ 'timeBucket',+ 'from',+ 'to',+ 'filters',+ 'timezone',+] as const) {+ event: string+}diff --git a/apps/selfhosted/src/analytics/interfaces/index.ts b/apps/selfhosted/src/analytics/interfaces/index.tsindex fda538f0c..0b1e533b9 100644--- a/apps/selfhosted/src/analytics/interfaces/index.ts+++ b/apps/selfhosted/src/analytics/interfaces/index.ts@@ -93,3 +93,9 @@ export interface IExtractChartData {uniques: number[]sdur: number[]}++export interface IAggregatedMetadata {+ key: string+ value: string+ count: number+}diff --git a/migrations/clickhouse/2023_10_13.js b/migrations/clickhouse/2023_10_13.jsnew file mode 100644index 000000000..6ad27bcb5--- /dev/null+++ b/migrations/clickhouse/2023_10_13.js@@ -0,0 +1,40 @@+const { queriesRunner, dbName } = require('./setup')++const queries = [+ // Custom events table: added 'meta.key' and 'meta.value' columns+ `DROP TABLE IF EXISTS ${dbName}.customEV_temp`,+ `CREATE TABLE IF NOT EXISTS ${dbName}.customEV_temp+ (+ pid FixedString(12),+ ev String,+ pg Nullable(String),+ dv LowCardinality(Nullable(String)),+ br LowCardinality(Nullable(String)),+ os LowCardinality(Nullable(String)),+ lc LowCardinality(Nullable(String)),+ ref Nullable(String),+ so Nullable(String),+ me Nullable(String),+ ca Nullable(String),+ cc Nullable(FixedString(2)),+ rg LowCardinality(Nullable(String)),+ ct Nullable(String),+ meta Nested+ (+ key String,+ value String+ ),+ created DateTime('UTC')+ )+ ENGINE = MergeTree()+ PARTITION BY toYYYYMM(created)+ ORDER BY (pid, created);`,++ `INSERT INTO ${dbName}.customEV_temp (pid, ev, pg, dv, br, os, lc, ref, so, me, ca, cc, rg, ct, meta.key, meta.value, created)+ SELECT pid, ev, pg, dv, br, os, lc, ref, so, me, ca, cc, rg, ct, [], [], created FROM ${dbName}.customEV`,++ `DROP TABLE ${dbName}.customEV`,+ `RENAME TABLE ${dbName}.customEV_temp TO ${dbName}.customEV`,+]++queriesRunner(queries)diff --git a/migrations/clickhouse/initialise_database.js b/migrations/clickhouse/initialise_database.jsindex fde352fce..6088f0a72 100644--- a/migrations/clickhouse/initialise_database.js+++ b/migrations/clickhouse/initialise_database.js@@ -49,6 +49,11 @@ const CLICKHOUSE_INIT_QUERIES = [cc Nullable(FixedString(2)),rg LowCardinality(Nullable(String)),ct Nullable(String),+ meta Nested+ (+ key String,+ value String+ ),created DateTime('UTC'))ENGINE = MergeTree()