Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

(feature) Custom events metadata (#196)

Andrii Romasiuna69455358997
diff --git a/apps/production/src/analytics/analytics.controller.ts b/apps/production/src/analytics/analytics.controller.ts
index 061e2083a..3df90089f 100644
--- a/apps/production/src/analytics/analytics.controller.ts
+++ b/apps/production/src/analytics/analytics.controller.ts
@@ -3,6 +3,8 @@ import * as _isArray from 'lodash/isArray'
import * as _toNumber from 'lodash/toNumber'
import * as _pick from 'lodash/pick'
import * as _includes from 'lodash/includes'
+import * as _keys from 'lodash/keys'
+import * as _values from 'lodash/values'
import * as _map from 'lodash/map'
import * as _uniqBy from 'lodash/uniqBy'
import * as _round from 'lodash/round'
@@ -45,6 +47,7 @@ import { RolesGuard } from '../auth/guards/roles.guard'
import { PageviewsDTO } from './dto/pageviews.dto'
import { EventsDTO } from './dto/events.dto'
import { AnalyticsGET_DTO } from './dto/getData.dto'
+import { GetCustomEventMetadata } from './dto/get-custom-event-meta.dto'
import { GetUserFlowDTO } from './dto/getUserFlow.dto'
import { AppLoggerService } from '../logger/logger.service'
import {
@@ -64,7 +67,7 @@ import { BotDetection } from '../common/decorators/bot-detection.decorator'
import { BotDetectionGuard } from '../common/guards/bot-detection.guard'
import { GetCustomEventsDto } from './dto/get-custom-events.dto'
import { GetFiltersDto } from './dto/get-filters.dto'
-import { IUserFlow } from './interfaces'
+import { IAggregatedMetadata, IUserFlow } from './interfaces'
// eslint-disable-next-line @typescript-eslint/no-var-requires
const mysql = require('mysql2')
@@ -173,7 +176,9 @@ const customLogDTO = (
cc: string,
rg: string,
ct: string,
-): Array<string | number> => {
+ keys: string[],
+ values: string[],
+): Array<string | number | string[]> => {
return [
pid,
ev,
@@ -189,11 +194,17 @@ const customLogDTO = (
cc,
rg,
ct,
+ keys,
+ values,
dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
]
}
export const getElValue = el => {
+ if (_isArray(el)) {
+ return `[${_map(el, getElValue).join(',')}]`
+ }
+
if (el === undefined || el === null || el === 'NULL') return 'NULL'
return mysql.escape(el)
}
@@ -403,6 +414,29 @@ export class AnalyticsController {
}
}
+ @Get('meta')
+ @Auth([], true, true)
+ async getCustomEventMetadata(
+ @Query() data: GetCustomEventMetadata,
+ @CurrentUserId() uid: string,
+ @Headers() headers: { 'x-password'?: string },
+ ): Promise<IAggregatedMetadata[]> {
+ const { pid, period } = data
+ this.analyticsService.validatePID(pid)
+
+ if (!_isEmpty(period)) {
+ this.analyticsService.validatePeriod(period)
+ }
+
+ await this.analyticsService.checkProjectAccess(
+ pid,
+ uid,
+ headers['x-password'],
+ )
+
+ return this.analyticsService.getCustomEventMetadata(data)
+ }
+
@Get('filters')
@Auth([], true, true)
async getFilters(
@@ -876,6 +910,7 @@ export class AnalyticsController {
const ip = getIPFromHeaders(headers, true) || reqIP || ''
+ this.analyticsService.validateCustomEVMeta(eventsDTO.meta)
await this.analyticsService.validate(eventsDTO, origin, 'custom', ip)
if (eventsDTO.unique) {
@@ -922,6 +957,8 @@ export class AnalyticsController {
country,
region,
city,
+ _keys(eventsDTO.meta),
+ _values(eventsDTO.meta),
)
try {
diff --git a/apps/production/src/analytics/analytics.service.ts b/apps/production/src/analytics/analytics.service.ts
index d6802b529..912835a57 100644
--- a/apps/production/src/analytics/analytics.service.ts
+++ b/apps/production/src/analytics/analytics.service.ts
@@ -12,6 +12,7 @@ import * as _last from 'lodash/last'
import * as _some from 'lodash/some'
import * as _find from 'lodash/find'
import * as _now from 'lodash/now'
+import * as _isString from 'lodash/isString'
import * as _values from 'lodash/values'
import * as _round from 'lodash/round'
import * as _filter from 'lodash/filter'
@@ -56,6 +57,7 @@ import { EventsDTO } from './dto/events.dto'
import { ProjectService } from '../project/project.service'
import { Project } from '../project/entity/project.entity'
import { TimeBucketType } from './dto/getData.dto'
+import { GetCustomEventMetadata } from './dto/get-custom-event-meta.dto'
import {
PerformanceCHResponse,
CustomsCHResponse,
@@ -70,6 +72,7 @@ import {
IBuildUserFlow,
IExtractChartData,
IGenerateXAxis,
+ IAggregatedMetadata,
} from './interfaces'
dayjs.extend(utc)
@@ -1696,6 +1699,88 @@ export class AnalyticsService {
return result
}
+ validateCustomEVMeta(meta: any) {
+ if (typeof meta === 'undefined') {
+ return
+ }
+
+ if (_some(_values(meta), val => !_isString(val))) {
+ throw new UnprocessableEntityException(
+ 'The provided custom event metadata is incorrect (some values are not strings)',
+ )
+ }
+ }
+
+ async getCustomEventMetadata(
+ data: GetCustomEventMetadata,
+ ): Promise<IAggregatedMetadata[]> {
+ const {
+ pid,
+ period,
+ timeBucket,
+ from,
+ to,
+ timezone = DEFAULT_TIMEZONE,
+ event,
+ } = data
+
+ let newTimebucket = timeBucket
+
+ let diff
+
+ if (period === 'all') {
+ const res = await this.getTimeBucketForAllTime(pid, period, timezone)
+
+ diff = res.diff
+ // eslint-disable-next-line prefer-destructuring
+ newTimebucket = _includes(res.timeBucket, timeBucket)
+ ? timeBucket
+ : res.timeBucket[0]
+ }
+
+ this.validateTimebucket(newTimebucket)
+
+ const safeTimezone = this.getSafeTimezone(timezone)
+ const { groupFromUTC, groupToUTC } = this.getGroupFromTo(
+ from,
+ to,
+ newTimebucket,
+ period,
+ safeTimezone,
+ diff,
+ )
+
+ const query = `SELECT
+ meta.key AS key,
+ meta.value AS value,
+ count() AS count
+ FROM customEV
+ ARRAY JOIN meta.key, meta.value
+ WHERE pid = {pid:FixedString(12)}
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ AND ev = {event:String}
+ GROUP BY key, value`
+
+ const paramsData = {
+ params: {
+ pid,
+ groupFrom: groupFromUTC,
+ groupTo: groupToUTC,
+ event,
+ },
+ }
+
+ try {
+ const result = await clickhouse.query(query, paramsData).toPromise()
+ return result as IAggregatedMetadata[]
+ } catch (reason) {
+ console.error(`[ERROR](getCustomEventMetadata): ${reason}`)
+ throw new InternalServerErrorException(
+ 'Something went wrong. Please, try again later.',
+ )
+ }
+ }
+
async getOnlineUserCount(pid: string): Promise<number> {
// @ts-ignore
return redis.countKeysByPattern(`hb:${pid}:*`)
diff --git a/apps/production/src/analytics/dto/events.dto.ts b/apps/production/src/analytics/dto/events.dto.ts
index bceec4535..2ebc02d89 100644
--- a/apps/production/src/analytics/dto/events.dto.ts
+++ b/apps/production/src/analytics/dto/events.dto.ts
@@ -1,5 +1,41 @@
+import * as _keys from 'lodash/keys'
+import * as _values from 'lodash/values'
import { ApiProperty } from '@nestjs/swagger'
-import { IsNotEmpty } from 'class-validator'
+import {
+ IsNotEmpty,
+ IsOptional,
+ IsObject,
+ ValidatorConstraint,
+ ValidatorConstraintInterface,
+ Validate,
+} from 'class-validator'
+
+const MAX_METADATA_KEYS = 20
+const MAX_METADATA_VALUE_LENGTH = 1000
+
+@ValidatorConstraint()
+class MetadataSizeLimit implements ValidatorConstraintInterface {
+ validate(metadata: Record<string, string>) {
+ const values = _values(metadata)
+ let totalSize = 0
+
+ for (const value of values) {
+ totalSize += value.length
+ if (totalSize > MAX_METADATA_VALUE_LENGTH) {
+ return false
+ }
+ }
+
+ return true
+ }
+}
+
+@ValidatorConstraint()
+class MetadataKeysQuantity implements ValidatorConstraintInterface {
+ validate(metadata: Record<string, string>) {
+ return _keys(metadata).length <= MAX_METADATA_KEYS
+ }
+}
export class EventsDTO {
@ApiProperty({
@@ -66,4 +102,21 @@ export class EventsDTO {
description: 'utm_campaign URL parameter',
})
ca?: string
+
+ @ApiProperty({
+ example: {
+ affiliate: 'Yes',
+ protocol: 'HTTPS',
+ },
+ description: 'Event-related metadata object with string values',
+ })
+ @IsOptional()
+ @IsObject()
+ @Validate(MetadataKeysQuantity, {
+ message: `Metadata object can't have more than ${MAX_METADATA_KEYS} keys`,
+ })
+ @Validate(MetadataSizeLimit, {
+ message: `Metadata object can't have values with total length more than ${MAX_METADATA_VALUE_LENGTH} characters`,
+ })
+ meta?: Record<string, string>
}
diff --git a/apps/production/src/analytics/dto/get-custom-event-meta.dto.ts b/apps/production/src/analytics/dto/get-custom-event-meta.dto.ts
new file mode 100644
index 000000000..d91e1cba3
--- /dev/null
+++ b/apps/production/src/analytics/dto/get-custom-event-meta.dto.ts
@@ -0,0 +1,14 @@
+import { PickType } from '@nestjs/swagger'
+import { AnalyticsGET_DTO } from './getData.dto'
+
+export class GetCustomEventMetadata extends PickType(AnalyticsGET_DTO, [
+ 'pid',
+ 'period',
+ 'timeBucket',
+ 'from',
+ 'to',
+ 'filters',
+ 'timezone',
+] as const) {
+ event: string
+}
diff --git a/apps/production/src/analytics/interfaces/index.ts b/apps/production/src/analytics/interfaces/index.ts
index fda538f0c..0b1e533b9 100644
--- a/apps/production/src/analytics/interfaces/index.ts
+++ b/apps/production/src/analytics/interfaces/index.ts
@@ -93,3 +93,9 @@ export interface IExtractChartData {
uniques: number[]
sdur: number[]
}
+
+export interface IAggregatedMetadata {
+ key: string
+ value: string
+ count: number
+}
diff --git a/apps/selfhosted/src/analytics/analytics.controller.ts b/apps/selfhosted/src/analytics/analytics.controller.ts
index 9e3669eb9..82e577ae1 100644
--- a/apps/selfhosted/src/analytics/analytics.controller.ts
+++ b/apps/selfhosted/src/analytics/analytics.controller.ts
@@ -5,6 +5,8 @@ import * as _map from 'lodash/map'
import * as _uniqBy from 'lodash/uniqBy'
import * as _round from 'lodash/round'
import * as _includes from 'lodash/includes'
+import * as _keys from 'lodash/keys'
+import * as _values from 'lodash/values'
import * as dayjs from 'dayjs'
import * as utc from 'dayjs/plugin/utc'
import * as dayjsTimezone from 'dayjs/plugin/timezone'
@@ -44,6 +46,7 @@ import { PageviewsDTO } from './dto/pageviews.dto'
import { EventsDTO } from './dto/events.dto'
import { AnalyticsGET_DTO } from './dto/getData.dto'
import { GetFiltersDto } from './dto/get-filters.dto'
+import { GetCustomEventMetadata } from './dto/get-custom-event-meta.dto'
import { GetUserFlowDTO } from './dto/getUserFlow.dto'
import { AppLoggerService } from '../logger/logger.service'
import {
@@ -59,7 +62,7 @@ import { getGeoDetails, getIPFromHeaders } from '../common/utils'
import { BotDetection } from '../common/decorators/bot-detection.decorator'
import { BotDetectionGuard } from '../common/guards/bot-detection.guard'
import { GetCustomEventsDto } from './dto/get-custom-events.dto'
-import { IUserFlow } from './interfaces'
+import { IAggregatedMetadata, IUserFlow } from './interfaces'
// eslint-disable-next-line @typescript-eslint/no-var-requires
const mysql = require('mysql2')
@@ -168,7 +171,9 @@ const customLogDTO = (
cc: string,
rg: string,
ct: string,
-): Array<string | number> => {
+ keys: string[],
+ values: string[],
+): Array<string | number | string[]> => {
return [
pid,
ev,
@@ -184,11 +189,17 @@ const customLogDTO = (
cc,
rg,
ct,
+ keys,
+ values,
dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
]
}
const getElValue = el => {
+ if (_isArray(el)) {
+ return `[${_map(el, getElValue).join(',')}]`
+ }
+
if (el === undefined || el === null || el === 'NULL') return 'NULL'
return mysql.escape(el)
}
@@ -364,6 +375,24 @@ export class AnalyticsController {
}
}
+ @Get('meta')
+ @Auth([], true, true)
+ async getCustomEventMetadata(
+ @Query() data: GetCustomEventMetadata,
+ @CurrentUserId() uid: string,
+ ): Promise<IAggregatedMetadata[]> {
+ const { pid, period } = data
+ this.analyticsService.validatePID(pid)
+
+ if (!_isEmpty(period)) {
+ this.analyticsService.validatePeriod(period)
+ }
+
+ await this.analyticsService.checkProjectAccess(pid, uid)
+
+ return this.analyticsService.getCustomEventMetadata(data)
+ }
+
@Get('filters')
@Auth([], true, true)
async getFilters(
@@ -739,6 +768,7 @@ export class AnalyticsController {
const ip = getIPFromHeaders(headers) || reqIP || ''
+ this.analyticsService.validateCustomEVMeta(eventsDTO.meta)
await this.analyticsService.validate(eventsDTO, origin, 'custom', ip)
if (eventsDTO.unique) {
@@ -785,6 +815,8 @@ export class AnalyticsController {
country,
region,
city,
+ _keys(eventsDTO.meta),
+ _values(eventsDTO.meta),
)
try {
diff --git a/apps/selfhosted/src/analytics/analytics.service.ts b/apps/selfhosted/src/analytics/analytics.service.ts
index 2c2400764..a67bb4cc3 100644
--- a/apps/selfhosted/src/analytics/analytics.service.ts
+++ b/apps/selfhosted/src/analytics/analytics.service.ts
@@ -15,6 +15,7 @@ import * as _now from 'lodash/now'
import * as _values from 'lodash/values'
import * as _round from 'lodash/round'
import * as _filter from 'lodash/filter'
+import * as _isString from 'lodash/isString'
import * as dayjs from 'dayjs'
import * as utc from 'dayjs/plugin/utc'
import * as dayjsTimezone from 'dayjs/plugin/timezone'
@@ -48,6 +49,7 @@ import { PageviewsDTO } from './dto/pageviews.dto'
import { EventsDTO } from './dto/events.dto'
import { ProjectService } from '../project/project.service'
import { Project } from '../project/entity/project.entity'
+import { GetCustomEventMetadata } from './dto/get-custom-event-meta.dto'
import { TimeBucketType } from './dto/getData.dto'
import {
PerformanceCHResponse,
@@ -63,6 +65,7 @@ import {
IBuildUserFlow,
IExtractChartData,
IGenerateXAxis,
+ IAggregatedMetadata,
} from './interfaces'
dayjs.extend(utc)
@@ -1523,6 +1526,88 @@ export class AnalyticsService {
return result
}
+ validateCustomEVMeta(meta: any) {
+ if (typeof meta === 'undefined') {
+ return
+ }
+
+ if (_some(_values(meta), val => !_isString(val))) {
+ throw new UnprocessableEntityException(
+ 'The provided custom event metadata is incorrect (some values are not strings)',
+ )
+ }
+ }
+
+ async getCustomEventMetadata(
+ data: GetCustomEventMetadata,
+ ): Promise<IAggregatedMetadata[]> {
+ const {
+ pid,
+ period,
+ timeBucket,
+ from,
+ to,
+ timezone = DEFAULT_TIMEZONE,
+ event,
+ } = data
+
+ let newTimebucket = timeBucket
+
+ let diff
+
+ if (period === 'all') {
+ const res = await this.getTimeBucketForAllTime(pid, period, timezone)
+
+ diff = res.diff
+ // eslint-disable-next-line prefer-destructuring
+ newTimebucket = _includes(res.timeBucket, timeBucket)
+ ? timeBucket
+ : res.timeBucket[0]
+ }
+
+ this.validateTimebucket(newTimebucket)
+
+ const safeTimezone = this.getSafeTimezone(timezone)
+ const { groupFromUTC, groupToUTC } = this.getGroupFromTo(
+ from,
+ to,
+ newTimebucket,
+ period,
+ safeTimezone,
+ diff,
+ )
+
+ const query = `SELECT
+ meta.key AS key,
+ meta.value AS value,
+ count() AS count
+ FROM customEV
+ ARRAY JOIN meta.key, meta.value
+ WHERE pid = {pid:FixedString(12)}
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ AND ev = {event:String}
+ GROUP BY key, value`
+
+ const paramsData = {
+ params: {
+ pid,
+ groupFrom: groupFromUTC,
+ groupTo: groupToUTC,
+ event,
+ },
+ }
+
+ try {
+ const result = await clickhouse.query(query, paramsData).toPromise()
+ return result as IAggregatedMetadata[]
+ } catch (reason) {
+ console.error(`[ERROR](getCustomEventMetadata): ${reason}`)
+ throw new InternalServerErrorException(
+ 'Something went wrong. Please, try again later.',
+ )
+ }
+ }
+
async getOnlineUserCount(pid: string): Promise<number> {
// @ts-ignore
return redis.countKeysByPattern(`hb:${pid}:*`)
diff --git a/apps/selfhosted/src/analytics/dto/events.dto.ts b/apps/selfhosted/src/analytics/dto/events.dto.ts
index 7cec51ad4..e16cbc86d 100644
--- a/apps/selfhosted/src/analytics/dto/events.dto.ts
+++ b/apps/selfhosted/src/analytics/dto/events.dto.ts
@@ -1,5 +1,41 @@
+import * as _keys from 'lodash/keys'
+import * as _values from 'lodash/values'
import { ApiProperty } from '@nestjs/swagger'
-import { IsNotEmpty } from 'class-validator'
+import {
+ IsNotEmpty,
+ IsOptional,
+ IsObject,
+ ValidatorConstraint,
+ ValidatorConstraintInterface,
+ Validate,
+} from 'class-validator'
+
+const MAX_METADATA_KEYS = 20
+const MAX_METADATA_VALUE_LENGTH = 1000
+
+@ValidatorConstraint()
+class MetadataSizeLimit implements ValidatorConstraintInterface {
+ validate(metadata: Record<string, string>) {
+ const values = _values(metadata)
+ let totalSize = 0
+
+ for (const value of values) {
+ totalSize += value.length
+ if (totalSize > MAX_METADATA_VALUE_LENGTH) {
+ return false
+ }
+ }
+
+ return true
+ }
+}
+
+@ValidatorConstraint()
+class MetadataKeysQuantity implements ValidatorConstraintInterface {
+ validate(metadata: Record<string, string>) {
+ return _keys(metadata).length <= MAX_METADATA_KEYS
+ }
+}
export class EventsDTO {
@ApiProperty({
@@ -66,4 +102,21 @@ export class EventsDTO {
description: 'utm_campaign URL parameter',
})
ca?: string
+
+ @ApiProperty({
+ example: {
+ affiliate: 'Yes',
+ protocol: 'HTTPS',
+ },
+ description: 'Event-related metadata object with string values',
+ })
+ @IsOptional()
+ @IsObject()
+ @Validate(MetadataKeysQuantity, {
+ message: `Metadata object can't have more than ${MAX_METADATA_KEYS} keys`,
+ })
+ @Validate(MetadataSizeLimit, {
+ message: `Metadata object can't have values with total length more than ${MAX_METADATA_VALUE_LENGTH} characters`,
+ })
+ meta?: Record<string, string>
}
diff --git a/apps/selfhosted/src/analytics/dto/get-custom-event-meta.dto.ts b/apps/selfhosted/src/analytics/dto/get-custom-event-meta.dto.ts
new file mode 100644
index 000000000..d91e1cba3
--- /dev/null
+++ b/apps/selfhosted/src/analytics/dto/get-custom-event-meta.dto.ts
@@ -0,0 +1,14 @@
+import { PickType } from '@nestjs/swagger'
+import { AnalyticsGET_DTO } from './getData.dto'
+
+export class GetCustomEventMetadata extends PickType(AnalyticsGET_DTO, [
+ 'pid',
+ 'period',
+ 'timeBucket',
+ 'from',
+ 'to',
+ 'filters',
+ 'timezone',
+] as const) {
+ event: string
+}
diff --git a/apps/selfhosted/src/analytics/interfaces/index.ts b/apps/selfhosted/src/analytics/interfaces/index.ts
index fda538f0c..0b1e533b9 100644
--- a/apps/selfhosted/src/analytics/interfaces/index.ts
+++ b/apps/selfhosted/src/analytics/interfaces/index.ts
@@ -93,3 +93,9 @@ export interface IExtractChartData {
uniques: number[]
sdur: number[]
}
+
+export interface IAggregatedMetadata {
+ key: string
+ value: string
+ count: number
+}
diff --git a/migrations/clickhouse/2023_10_13.js b/migrations/clickhouse/2023_10_13.js
new file mode 100644
index 000000000..6ad27bcb5
--- /dev/null
+++ b/migrations/clickhouse/2023_10_13.js
@@ -0,0 +1,40 @@
+const { queriesRunner, dbName } = require('./setup')
+
+const queries = [
+ // Custom events table: added 'meta.key' and 'meta.value' columns
+ `DROP TABLE IF EXISTS ${dbName}.customEV_temp`,
+ `CREATE TABLE IF NOT EXISTS ${dbName}.customEV_temp
+ (
+ pid FixedString(12),
+ ev String,
+ pg Nullable(String),
+ dv LowCardinality(Nullable(String)),
+ br LowCardinality(Nullable(String)),
+ os LowCardinality(Nullable(String)),
+ lc LowCardinality(Nullable(String)),
+ ref Nullable(String),
+ so Nullable(String),
+ me Nullable(String),
+ ca Nullable(String),
+ cc Nullable(FixedString(2)),
+ rg LowCardinality(Nullable(String)),
+ ct Nullable(String),
+ meta Nested
+ (
+ key String,
+ value String
+ ),
+ created DateTime('UTC')
+ )
+ ENGINE = MergeTree()
+ PARTITION BY toYYYYMM(created)
+ ORDER BY (pid, created);`,
+
+ `INSERT INTO ${dbName}.customEV_temp (pid, ev, pg, dv, br, os, lc, ref, so, me, ca, cc, rg, ct, meta.key, meta.value, created)
+ SELECT pid, ev, pg, dv, br, os, lc, ref, so, me, ca, cc, rg, ct, [], [], created FROM ${dbName}.customEV`,
+
+ `DROP TABLE ${dbName}.customEV`,
+ `RENAME TABLE ${dbName}.customEV_temp TO ${dbName}.customEV`,
+]
+
+queriesRunner(queries)
diff --git a/migrations/clickhouse/initialise_database.js b/migrations/clickhouse/initialise_database.js
index fde352fce..6088f0a72 100644
--- a/migrations/clickhouse/initialise_database.js
+++ b/migrations/clickhouse/initialise_database.js
@@ -49,6 +49,11 @@ const CLICKHOUSE_INIT_QUERIES = [
cc Nullable(FixedString(2)),
rg LowCardinality(Nullable(String)),
ct Nullable(String),
+ meta Nested
+ (
+ key String,
+ value String
+ ),
created DateTime('UTC')
)
ENGINE = MergeTree()