📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
(feature) Pageview tags (#239)
Andrii Romasiunf4b35b20be17
diff --git a/apps/production/src/analytics/analytics.controller.ts b/apps/production/src/analytics/analytics.controller.tsindex 094e317ab..3e0713726 100644--- a/apps/production/src/analytics/analytics.controller.ts+++ b/apps/production/src/analytics/analytics.controller.ts@@ -50,6 +50,7 @@ import { PageviewsDTO } from './dto/pageviews.dto'import { EventsDTO } from './dto/events.dto'import { AnalyticsGET_DTO, ChartRenderMode } from './dto/getData.dto'import { GetCustomEventMetadata } from './dto/get-custom-event-meta.dto'+import { GetPagePropertyMetaDTO } from './dto/get-page-property-meta.dto'import { GetUserFlowDTO } from './dto/getUserFlow.dto'import { GetFunnelsDTO } from './dto/getFunnels.dto'import { AppLoggerService } from '../logger/logger.service'@@ -76,9 +77,9 @@ import { BotDetectionGuard } from '../common/guards/bot-detection.guard'import { GetCustomEventsDto } from './dto/get-custom-events.dto'import { GetFiltersDto } from './dto/get-filters.dto'import {- IAggregatedMetadata,IFunnel,IGetFunnel,+ IPageProperty,IUserFlow,PerfMeasure,} from './interfaces'@@ -122,9 +123,11 @@ const analyticsDTO = (cc: string,rg: string,ct: string,+ keys: string[],+ values: string[],sdur: number | string,unique: number,-): Array<string | number> => {+): Array<string | number | string[]> => {return [psid,sid,@@ -142,6 +145,8 @@ const analyticsDTO = (cc,rg,ct,+ keys,+ values,sdur,unique,dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),@@ -433,14 +438,11 @@ export class AnalyticsController {diff,)- let queryCustoms = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`let subQuery = `FROM ${isCaptcha ? 'captcha' : 'analytics'} WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`if (customEVFilterApplied && !isCaptcha) {- queryCustoms = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`-subQuery = `FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`}@@ -489,14 +491,24 @@ export class AnalyticsController {}}- const customs = await this.analyticsService.processCustomEV(- queryCustoms,+ const customs = await this.analyticsService.getCustomEvents(+ filtersQuery,paramsData,)+ let properties: IPageProperty = {}++ if (!customEVFilterApplied) {+ properties = await this.analyticsService.getPageProperties(+ filtersQuery,+ paramsData,+ )+ }+return {...result,customs,+ properties,appliedFilters,timeBucket: allowedTumebucketForPeriodAll,}@@ -583,7 +595,7 @@ export class AnalyticsController {@Query() data: GetCustomEventMetadata,@CurrentUserId() uid: string,@Headers() headers: { 'x-password'?: string },- ): Promise<IAggregatedMetadata[]> {+ ): Promise<ReturnType<typeof this.analyticsService.getCustomEventMetadata>> {const { pid, period } = datathis.analyticsService.validatePID(pid)@@ -602,6 +614,31 @@ export class AnalyticsController {return this.analyticsService.getCustomEventMetadata(data)}+ @Get('property')+ @Auth([], true, true)+ async getPagePropertyMetadata(+ @Query() data: GetPagePropertyMetaDTO,+ @CurrentUserId() uid: string,+ @Headers() headers: { 'x-password'?: string },+ ): Promise<ReturnType<typeof this.analyticsService.getPagePropertyMeta>> {+ const { pid, period } = data+ this.analyticsService.validatePID(pid)++ if (!_isEmpty(period)) {+ this.analyticsService.validatePeriod(period)+ }++ await this.analyticsService.checkProjectAccess(+ pid,+ uid,+ headers['x-password'],+ )++ await this.analyticsService.checkBillingAccess(pid)++ return this.analyticsService.getPagePropertyMeta(data)+ }+@Get('filters')@Auth([], true, true)async getFilters(@@ -1424,6 +1461,8 @@ export class AnalyticsController {country,region,city,+ _keys(logDTO.meta),+ _values(logDTO.meta),0,Number(unique),)@@ -1538,6 +1577,8 @@ export class AnalyticsController {country,region,city,+ [],+ [],0,Number(unique),)diff --git a/apps/production/src/analytics/analytics.service.ts b/apps/production/src/analytics/analytics.service.tsindex 6671c37cf..ab6dda25f 100644--- a/apps/production/src/analytics/analytics.service.ts+++ b/apps/production/src/analytics/analytics.service.ts@@ -9,6 +9,7 @@ import * as _toUpper from 'lodash/toUpper'import * as _head from 'lodash/head'import * as _join from 'lodash/join'import * as _isArray from 'lodash/isArray'+import * as _startsWith from 'lodash/startsWith'import * as _sortBy from 'lodash/sortBy'import * as _reduce from 'lodash/reduce'import * as _keys from 'lodash/keys'@@ -55,6 +56,7 @@ import {REDIS_USERS_COUNT_KEY,REDIS_PROJECTS_COUNT_KEY,REDIS_EVENTS_COUNT_KEY,+ TRAFFIC_METAKEY_COLUMNS,} from '../common/constants'import {calculateRelativePercentage,@@ -91,8 +93,12 @@ import {IOverallPerformance,IPageflow,PerfMeasure,+ PropertiesCHResponse,+ IPageProperty,+ ICustomEvent,} from './interfaces'import { ErrorDTO } from './dto/error.dto'+import { GetPagePropertyMetaDTO } from './dto/get-page-property-meta.dto'dayjs.extend(utc)dayjs.extend(dayjsTimezone)@@ -910,9 +916,19 @@ export class AnalyticsService {return prev}- if (column === 'ev') {+ // ev:key -> custom event metadata+ // ev:key: -> custom event metadata, but we want to check if some meta.key's meta.value equals to a specific value+ if (+ column === 'ev' ||+ column === 'ev:key' ||+ _startsWith(column, 'ev:key:')+ ) {customEVFilterApplied = true- } else if (!_includes(SUPPORTED_COLUMNS, column)) {+ } else if (+ !_includes(SUPPORTED_COLUMNS, column) &&+ !_includes(TRAFFIC_METAKEY_COLUMNS, column) &&+ !_startsWith(column, 'tag:key:')+ ) {throw new UnprocessableEntityException(`The provided filter (${column}) is not supported`,)@@ -958,7 +974,7 @@ export class AnalyticsService {const columns = _keys(converted)for (let col = 0; col < _size(columns); ++col) {- const column = columns[col]+ const column: string = columns[col]query += ' AND ('for (let f = 0; f < _size(converted[column]); ++f) {@@ -967,18 +983,46 @@ export class AnalyticsService {}const { filter, isExclusive } = converted[column][f]+ let sqlColumn = column+ let isArrayDataset = falseconst param = `qf_${col}_${f}`+ params[param] = filter- if (filter === null) {- query += `${column} IS ${isExclusive ? 'NOT' : ''} NULL`- params[param] = filter+ // when we want to filter meta.value for a specific meta.key+ if (_startsWith(column, 'ev:key:') || _startsWith(column, 'tag:key:')) {+ const key = column.replace(/^ev:key:/, '').replace(/^tag:key:/, '')+ const keyParam = `qfk_${col}_${f}`+ params[keyParam] = key++ query += `indexOf(meta.key, {${keyParam}:String}) > 0 AND meta.value[indexOf(meta.key, {${keyParam}:String})] ${isExclusive ? '!= ' : '='} {${param}:String}`continue}- query += `${isExclusive ? 'NOT ' : ''}${column} = {${param}:String}`+ // meta.key filters for page properties and custom event metadata+ // e.g. article "author" (property)+ if (column === 'ev:key' || column === 'tag:key') {+ sqlColumn = 'meta.key'+ isArrayDataset = true+ // meta.value filters for page properties and custom event metadata+ // e.g. "Andrii" ("author" value)+ } else if (column === 'ev:value' || column === 'tag:value') {+ sqlColumn = 'meta.value'+ isArrayDataset = true+ }++ // TODO: In future I will add contains / not contains filters as well via ILIKE operator- params[param] = filter+ if (filter === null) {+ query += isArrayDataset+ ? ''+ : `${sqlColumn} IS ${isExclusive ? 'NOT' : ''} NULL`+ continue+ }++ query += isArrayDataset+ ? `indexOf(${sqlColumn}, {${param}:String}) ${isExclusive ? '=' : '>'} 0`+ : `${isExclusive ? 'NOT ' : ''}${sqlColumn} = {${param}:String}`}query += ')'@@ -2844,7 +2888,11 @@ export class AnalyticsService {})}- async processCustomEV(query: string, params: object): Promise<object> {+ async getCustomEvents(+ filtersQuery: string,+ params: object,+ ): Promise<ICustomEvent> {+ const query = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`const result = {}const rawCustoms = <Array<CustomsCHResponse>>(@@ -2860,6 +2908,40 @@ export class AnalyticsService {return result}+ async getPageProperties(+ filtersQuery: string,+ params: object,+ ): Promise<IPageProperty> {+ const query = `+ SELECT+ meta.key AS property,+ count()+ FROM analytics+ WHERE pid = {pid:FixedString(12)}+ ${filtersQuery}+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}+ GROUP BY property`+ const result = {}++ const rawProperties = <Array<PropertiesCHResponse>>(+ await clickhouse.query(query, params).toPromise()+ )+ const size = _size(rawProperties)++ for (let i = 0; i < size; ++i) {+ const { property: propertyArr, 'count()': c } = rawProperties[i]+ const [property] = propertyArr++ if (!property) {+ continue+ }++ result[property] = c+ }++ return result+ }+validateCustomEVMeta(meta: any) {if (typeof meta === 'undefined') {return@@ -2872,15 +2954,17 @@ export class AnalyticsService {}}- async getCustomEventMetadata(- data: GetCustomEventMetadata,- ): Promise<IAggregatedMetadata[]> {+ async getCustomEventMetadata(data: GetCustomEventMetadata): Promise<{+ result: IAggregatedMetadata[]+ appliedFilters: GetFiltersQuery[2]+ }> {const {pid,period,timeBucket,from,to,+ filters,timezone = DEFAULT_TIMEZONE,event,} = data@@ -2889,6 +2973,11 @@ export class AnalyticsService {let diff+ const [filtersQuery, filtersParams, appliedFilters] = this.getFiltersQuery(+ filters,+ DataType.ANALYTICS,+ )+if (period === 'all') {const res = await this.getTimeBucketForAllTime(pid, period, timezone)@@ -2911,16 +3000,26 @@ export class AnalyticsService {diff,)- const query = `SELECT- meta.key AS key,- meta.value AS value,- count() AS count- FROM customEV- ARRAY JOIN meta.key, meta.value- WHERE pid = {pid:FixedString(12)}- AND created BETWEEN {groupFrom:String} AND {groupTo:String}- AND ev = {event:String}- GROUP BY key, value`+ const query = `+ SELECT+ meta.key AS key,+ meta.value AS value,+ count() AS count+ FROM (+ SELECT+ meta.key,+ meta.value+ FROM+ customEV+ WHERE+ pid = {pid:FixedString(12)}+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}+ AND ev = {event:String}+ ${filtersQuery}+ )+ ARRAY JOIN meta.key, meta.value+ GROUP BY key, value+ `const paramsData = {params: {@@ -2928,12 +3027,19 @@ export class AnalyticsService {groupFrom: groupFromUTC,groupTo: groupToUTC,event,+ ...filtersParams,},}try {- const result = await clickhouse.query(query, paramsData).toPromise()- return result as IAggregatedMetadata[]+ const result = (await clickhouse+ .query(query, paramsData)+ .toPromise()) as IAggregatedMetadata[]++ return {+ result,+ appliedFilters,+ }} catch (reason) {console.error(`[ERROR](getCustomEventMetadata): ${reason}`)throw new InternalServerErrorException(@@ -2942,6 +3048,107 @@ export class AnalyticsService {}}+ async getPagePropertyMeta(data: GetPagePropertyMetaDTO): Promise<{+ result: IAggregatedMetadata[]+ appliedFilters: GetFiltersQuery[2]+ }> {+ const {+ pid,+ period,+ timeBucket,+ from,+ to,+ timezone = DEFAULT_TIMEZONE,+ property,+ filters,+ } = data++ let newTimebucket = timeBucket++ let diff++ const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] =+ this.getFiltersQuery(filters, DataType.ANALYTICS)++ // We cannot make a query to customEV table using analytics table properties+ if (customEVFilterApplied) {+ return {+ result: [],+ appliedFilters,+ }+ }++ if (period === 'all') {+ const res = await this.getTimeBucketForAllTime(pid, period, timezone)++ diff = res.diff+ // eslint-disable-next-line prefer-destructuring+ newTimebucket = _includes(res.timeBucket, timeBucket)+ ? timeBucket+ : res.timeBucket[0]+ }++ this.validateTimebucket(newTimebucket)++ const safeTimezone = this.getSafeTimezone(timezone)+ const { groupFromUTC, groupToUTC } = this.getGroupFromTo(+ from,+ to,+ newTimebucket,+ period,+ safeTimezone,+ diff,+ )++ const query = `+ SELECT+ meta.key AS key,+ meta.value AS value,+ count() AS count+ FROM (+ SELECT+ meta.key,+ meta.value+ FROM+ analytics+ WHERE+ pid = {pid:FixedString(12)}+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}+ AND indexOf(meta.key, {property:String}) > 0+ ${filtersQuery}+ )+ ARRAY JOIN meta.key, meta.value+ WHERE meta.key = {property:String}+ GROUP BY key, value+ `++ const paramsData = {+ params: {+ pid,+ groupFrom: groupFromUTC,+ groupTo: groupToUTC,+ property,+ ...filtersParams,+ },+ }++ try {+ const result = (await clickhouse+ .query(query, paramsData)+ .toPromise()) as IAggregatedMetadata[]++ return {+ result,+ appliedFilters,+ }+ } catch (reason) {+ console.error(`[ERROR](getPagePropertyMeta): ${reason}`)+ throw new InternalServerErrorException(+ 'Something went wrong. Please, try again later.',+ )+ }+ }+async getOnlineUserCount(pid: string): Promise<number> {// @ts-ignorereturn redis.countKeysByPattern(`hb:${pid}:*`)diff --git a/apps/production/src/analytics/dto/events.dto.ts b/apps/production/src/analytics/dto/events.dto.tsindex 2ebc02d89..b2655e857 100644--- a/apps/production/src/analytics/dto/events.dto.ts+++ b/apps/production/src/analytics/dto/events.dto.ts@@ -10,11 +10,11 @@ import {Validate,} from 'class-validator'-const MAX_METADATA_KEYS = 20-const MAX_METADATA_VALUE_LENGTH = 1000+export const MAX_METADATA_KEYS = 20+export const MAX_METADATA_VALUE_LENGTH = 1000@ValidatorConstraint()-class MetadataSizeLimit implements ValidatorConstraintInterface {+export class MetadataSizeLimit implements ValidatorConstraintInterface {validate(metadata: Record<string, string>) {const values = _values(metadata)let totalSize = 0@@ -31,7 +31,7 @@ class MetadataSizeLimit implements ValidatorConstraintInterface {}@ValidatorConstraint()-class MetadataKeysQuantity implements ValidatorConstraintInterface {+export class MetadataKeysQuantity implements ValidatorConstraintInterface {validate(metadata: Record<string, string>) {return _keys(metadata).length <= MAX_METADATA_KEYS}diff --git a/apps/production/src/analytics/dto/get-custom-event-meta.dto.ts b/apps/production/src/analytics/dto/get-custom-event-meta.dto.tsindex d91e1cba3..8f84e6b9f 100644--- a/apps/production/src/analytics/dto/get-custom-event-meta.dto.ts+++ b/apps/production/src/analytics/dto/get-custom-event-meta.dto.ts@@ -1,4 +1,5 @@-import { PickType } from '@nestjs/swagger'+import { ApiProperty, PickType } from '@nestjs/swagger'+import { IsNotEmpty, IsString } from 'class-validator'import { AnalyticsGET_DTO } from './getData.dto'export class GetCustomEventMetadata extends PickType(AnalyticsGET_DTO, [@@ -9,6 +10,10 @@ export class GetCustomEventMetadata extends PickType(AnalyticsGET_DTO, ['to','filters','timezone',+ 'filters',] as const) {+ @ApiProperty()+ @IsNotEmpty()+ @IsString()event: string}diff --git a/apps/production/src/analytics/dto/get-page-property-meta.dto.ts b/apps/production/src/analytics/dto/get-page-property-meta.dto.tsnew file mode 100644index 000000000..ccd75a113--- /dev/null+++ b/apps/production/src/analytics/dto/get-page-property-meta.dto.ts@@ -0,0 +1,19 @@+import { ApiProperty, PickType } from '@nestjs/swagger'+import { IsNotEmpty, IsString } from 'class-validator'+import { AnalyticsGET_DTO } from './getData.dto'++export class GetPagePropertyMetaDTO extends PickType(AnalyticsGET_DTO, [+ 'pid',+ 'period',+ 'timeBucket',+ 'from',+ 'to',+ 'filters',+ 'timezone',+ 'filters',+] as const) {+ @ApiProperty()+ @IsNotEmpty()+ @IsString()+ property: string+}diff --git a/apps/production/src/analytics/dto/pageviews.dto.ts b/apps/production/src/analytics/dto/pageviews.dto.tsindex a8ea10615..deb809921 100644--- a/apps/production/src/analytics/dto/pageviews.dto.ts+++ b/apps/production/src/analytics/dto/pageviews.dto.ts@@ -1,5 +1,12 @@import { ApiProperty } from '@nestjs/swagger'-import { IsNotEmpty } from 'class-validator'+import { IsNotEmpty, IsObject, IsOptional, Validate } from 'class-validator'++import {+ MAX_METADATA_KEYS,+ MAX_METADATA_VALUE_LENGTH,+ MetadataKeysQuantity,+ MetadataSizeLimit,+} from './events.dto'export class PageviewsDTO {@ApiProperty({@@ -88,4 +95,21 @@ export class PageviewsDTO {page_load: numberttfb: number}++ @ApiProperty({+ example: {+ affiliate: 'Yes',+ protocol: 'HTTPS',+ },+ description: 'Event-related metadata object with string values',+ })+ @IsOptional()+ @IsObject()+ @Validate(MetadataKeysQuantity, {+ message: `Metadata object can't have more than ${MAX_METADATA_KEYS} keys`,+ })+ @Validate(MetadataSizeLimit, {+ message: `Metadata object can't have values with total length more than ${MAX_METADATA_VALUE_LENGTH} characters`,+ })+ meta?: Record<string, string>}diff --git a/apps/production/src/analytics/interfaces/index.ts b/apps/production/src/analytics/interfaces/index.tsindex bee862b0a..813d0722a 100644--- a/apps/production/src/analytics/interfaces/index.ts+++ b/apps/production/src/analytics/interfaces/index.ts@@ -42,6 +42,12 @@ export interface CustomsCHResponse {index: number}+export interface PropertiesCHResponse {+ property: string+ 'count()': number+ index: number+}+export interface CustomsCHAggregatedResponse {year: numbermonth: number@@ -185,4 +191,12 @@ export interface IPageflow {created: string}+export interface IPageProperty {+ [key: string]: number+}++export interface ICustomEvent {+ [key: string]: number+}+export type PerfMeasure = 'average' | 'median' | 'p95' | 'quantiles'diff --git a/apps/production/src/common/constants.ts b/apps/production/src/common/constants.tsindex 9453d51d4..d66e035f4 100644--- a/apps/production/src/common/constants.ts+++ b/apps/production/src/common/constants.ts@@ -144,6 +144,8 @@ const TRAFFIC_COLUMNS = ['ca',]+const TRAFFIC_METAKEY_COLUMNS = ['tag:key', 'tag:value']+const ERROR_COLUMNS = ['pg', 'dv', 'br', 'os', 'lc', 'cc', 'rg', 'ct']const ALL_COLUMNS = [...TRAFFIC_COLUMNS, 'ev']@@ -228,6 +230,7 @@ export {getRedisUserUsageInfoKey,redisUserUsageinfoCacheTimeout,TRAFFIC_COLUMNS,+ TRAFFIC_METAKEY_COLUMNS,CAPTCHA_COLUMNS,ERROR_COLUMNS,PERFORMANCE_COLUMNS,diff --git a/apps/selfhosted/src/analytics/analytics.controller.ts b/apps/selfhosted/src/analytics/analytics.controller.tsindex 3af2f00a4..aed291262 100644--- a/apps/selfhosted/src/analytics/analytics.controller.ts+++ b/apps/selfhosted/src/analytics/analytics.controller.ts@@ -49,6 +49,7 @@ import { EventsDTO } from './dto/events.dto'import { AnalyticsGET_DTO, ChartRenderMode } from './dto/getData.dto'import { GetFiltersDto } from './dto/get-filters.dto'import { GetCustomEventMetadata } from './dto/get-custom-event-meta.dto'+import { GetPagePropertyMetaDTO } from './dto/get-page-property-meta.dto'import { GetUserFlowDTO } from './dto/getUserFlow.dto'import { GetFunnelsDTO } from './dto/getFunnels.dto'import { AppLoggerService } from '../logger/logger.service'@@ -71,9 +72,9 @@ import { BotDetection } from '../common/decorators/bot-detection.decorator'import { BotDetectionGuard } from '../common/guards/bot-detection.guard'import { GetCustomEventsDto } from './dto/get-custom-events.dto'import {- IAggregatedMetadata,IFunnel,IGetFunnel,+ IPageProperty,IUserFlow,PerfMeasure,} from './interfaces'@@ -117,9 +118,11 @@ const analyticsDTO = (cc: string,rg: string,ct: string,+ keys: string[],+ values: string[],sdur: number | string,unique: number,-): Array<string | number> => {+): Array<string | number | string[]> => {return [psid,sid,@@ -137,6 +140,8 @@ const analyticsDTO = (cc,rg,ct,+ keys,+ values,sdur,unique,dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),@@ -422,12 +427,9 @@ export class AnalyticsController {diff,)- let queryCustoms = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`let subQuery = `FROM analytics WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`if (customEVFilterApplied) {- queryCustoms = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`-subQuery = `FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`}@@ -453,14 +455,24 @@ export class AnalyticsController {mode,)- const customs = await this.analyticsService.processCustomEV(- queryCustoms,+ const customs = await this.analyticsService.getCustomEvents(+ filtersQuery,paramsData,)+ let properties: IPageProperty = {}++ if (!customEVFilterApplied) {+ properties = await this.analyticsService.getPageProperties(+ filtersQuery,+ paramsData,+ )+ }+return {...result,customs,+ properties,appliedFilters: parsedFilters,timeBucket: allowedTumebucketForPeriodAll,}@@ -545,7 +557,7 @@ export class AnalyticsController {@Query() data: GetCustomEventMetadata,@CurrentUserId() uid: string,@Headers() headers: { 'x-password'?: string },- ): Promise<IAggregatedMetadata[]> {+ ): Promise<ReturnType<typeof this.analyticsService.getCustomEventMetadata>> {const { pid, period } = datathis.analyticsService.validatePID(pid)@@ -562,6 +574,29 @@ export class AnalyticsController {return this.analyticsService.getCustomEventMetadata(data)}+ @Get('property')+ @Auth([], true, true)+ async getPagePropertyMetadata(+ @Query() data: GetPagePropertyMetaDTO,+ @CurrentUserId() uid: string,+ @Headers() headers: { 'x-password'?: string },+ ): Promise<ReturnType<typeof this.analyticsService.getPagePropertyMeta>> {+ const { pid, period } = data+ this.analyticsService.validatePID(pid)++ if (!_isEmpty(period)) {+ this.analyticsService.validatePeriod(period)+ }++ await this.analyticsService.checkProjectAccess(+ pid,+ uid,+ headers['x-password'],+ )++ return this.analyticsService.getPagePropertyMeta(data)+ }+@Get('filters')@Auth([], true, true)async getFilters(@@ -1193,6 +1228,8 @@ export class AnalyticsController {country,region,city,+ _keys(logDTO.meta),+ _values(logDTO.meta),0,Number(unique),)@@ -1306,6 +1343,8 @@ export class AnalyticsController {city,region,country,+ [],+ [],0,Number(unique),)diff --git a/apps/selfhosted/src/analytics/analytics.service.ts b/apps/selfhosted/src/analytics/analytics.service.tsindex a3c649bcf..e0c1d5c93 100644--- a/apps/selfhosted/src/analytics/analytics.service.ts+++ b/apps/selfhosted/src/analytics/analytics.service.ts@@ -14,6 +14,7 @@ import * as _last from 'lodash/last'import * as _some from 'lodash/some'import * as _find from 'lodash/find'import * as _isArray from 'lodash/isArray'+import * as _startsWith from 'lodash/startsWith'import * as _reduce from 'lodash/reduce'import * as _keys from 'lodash/keys'import * as _now from 'lodash/now'@@ -49,6 +50,7 @@ import {MIN_PAGES_IN_FUNNEL,MAX_PAGES_IN_FUNNEL,ALL_COLUMNS,+ TRAFFIC_METAKEY_COLUMNS,} from '../common/constants'import {getFunnelsClickhouse,@@ -83,8 +85,12 @@ import {IOverallPerformance,IPageflow,PerfMeasure,+ PropertiesCHResponse,+ IPageProperty,+ ICustomEvent,} from './interfaces'import { ErrorDTO } from './dto/error.dto'+import { GetPagePropertyMetaDTO } from './dto/get-page-property-meta.dto'dayjs.extend(utc)dayjs.extend(dayjsTimezone)@@ -884,9 +890,19 @@ export class AnalyticsService {return prev}- if (column === 'ev') {+ // ev:key -> custom event metadata+ // ev:key: -> custom event metadata, but we want to check if some meta.key's meta.value equals to a specific value+ if (+ column === 'ev' ||+ column === 'ev:key' ||+ _startsWith(column, 'ev:key:')+ ) {customEVFilterApplied = true- } else if (!_includes(SUPPORTED_COLUMNS, column)) {+ } else if (+ !_includes(SUPPORTED_COLUMNS, column) &&+ !_includes(TRAFFIC_METAKEY_COLUMNS, column) &&+ !_startsWith(column, 'tag:key:')+ ) {throw new UnprocessableEntityException(`The provided filter (${column}) is not supported`,)@@ -894,24 +910,28 @@ export class AnalyticsService {const res = []+ // commented encodeURIComponent lines of code to support filtering for pages like /product/[id]+ // until I find a more suitable solution for that laterif (_isArray(filter)) {for (const f of filter) {- let encoded = f+ // let encoded = f- if (column === 'pg' && f !== null) {- encoded = _replace(encodeURIComponent(f), /%2F/g, '/')- }+ // if (column === 'pg' && f !== null) {+ // encoded = _replace(encodeURIComponent(f), /%2F/g, '/')+ // }- res.push({ filter: encoded, isExclusive })+ // res.push({ filter: encoded, isExclusive })+ res.push({ filter: f, isExclusive })}} else {- let encoded = filter+ // let encoded = filter- if (column === 'pg' && filter !== null) {- encoded = _replace(encodeURIComponent(filter), /%2F/g, '/')- }+ // if (column === 'pg' && filter !== null) {+ // encoded = _replace(encodeURIComponent(filter), /%2F/g, '/')+ // }- res.push({ filter: encoded, isExclusive })+ // res.push({ filter: encoded, isExclusive })+ res.push({ filter, isExclusive })}if (prev[column]) {@@ -928,7 +948,7 @@ export class AnalyticsService {const columns = _keys(converted)for (let col = 0; col < _size(columns); ++col) {- const column = columns[col]+ const column: string = columns[col]query += ' AND ('for (let f = 0; f < _size(converted[column]); ++f) {@@ -937,18 +957,46 @@ export class AnalyticsService {}const { filter, isExclusive } = converted[column][f]+ let sqlColumn = column+ let isArrayDataset = falseconst param = `qf_${col}_${f}`+ params[param] = filter- if (filter === null) {- query += `${column} IS ${isExclusive ? 'NOT' : ''} NULL`- params[param] = filter+ // when we want to filter meta.value for a specific meta.key+ if (_startsWith(column, 'ev:key:') || _startsWith(column, 'tag:key:')) {+ const key = column.replace(/^ev:key:/, '').replace(/^tag:key:/, '')+ const keyParam = `qfk_${col}_${f}`+ params[keyParam] = key++ query += `indexOf(meta.key, {${keyParam}:String}) > 0 AND meta.value[indexOf(meta.key, {${keyParam}:String})] ${isExclusive ? '!= ' : '='} {${param}:String}`continue}- query += `${isExclusive ? 'NOT ' : ''}${column} = {${param}:String}`+ // meta.key filters for page properties and custom event metadata+ // e.g. article "author" (property)+ if (column === 'ev:key' || column === 'tag:key') {+ sqlColumn = 'meta.key'+ isArrayDataset = true+ // meta.value filters for page properties and custom event metadata+ // e.g. "Andrii" ("author" value)+ } else if (column === 'ev:value' || column === 'tag:value') {+ sqlColumn = 'meta.value'+ isArrayDataset = true+ }- params[param] = filter+ // TODO: In future I will add contains / not contains filters as well via ILIKE operator++ if (filter === null) {+ query += isArrayDataset+ ? ''+ : `${sqlColumn} IS ${isExclusive ? 'NOT' : ''} NULL`+ continue+ }++ query += isArrayDataset+ ? `indexOf(${sqlColumn}, {${param}:String}) ${isExclusive ? '=' : '>'} 0`+ : `${isExclusive ? 'NOT ' : ''}${sqlColumn} = {${param}:String}`}query += ')'@@ -2536,7 +2584,11 @@ export class AnalyticsService {})}- async processCustomEV(query: string, params: object): Promise<object> {+ async getCustomEvents(+ filtersQuery: string,+ params: object,+ ): Promise<ICustomEvent> {+ const query = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`const result = {}const rawCustoms = <Array<CustomsCHResponse>>(@@ -2564,15 +2616,51 @@ export class AnalyticsService {}}- async getCustomEventMetadata(- data: GetCustomEventMetadata,- ): Promise<IAggregatedMetadata[]> {+ async getPageProperties(+ filtersQuery: string,+ params: object,+ ): Promise<IPageProperty> {+ const query = `+ SELECT+ meta.key AS property,+ count()+ FROM analytics+ WHERE pid = {pid:FixedString(12)}+ ${filtersQuery}+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}+ GROUP BY property`+ const result = {}++ const rawProperties = <Array<PropertiesCHResponse>>(+ await clickhouse.query(query, params).toPromise()+ )+ const size = _size(rawProperties)++ for (let i = 0; i < size; ++i) {+ const { property: propertyArr, 'count()': c } = rawProperties[i]+ const [property] = propertyArr++ if (!property) {+ continue+ }++ result[property] = c+ }++ return result+ }++ async getCustomEventMetadata(data: GetCustomEventMetadata): Promise<{+ result: IAggregatedMetadata[]+ appliedFilters: GetFiltersQuery[2]+ }> {const {pid,period,timeBucket,from,to,+ filters,timezone = DEFAULT_TIMEZONE,event,} = data@@ -2581,6 +2669,11 @@ export class AnalyticsService {let diff+ const [filtersQuery, filtersParams, appliedFilters] = this.getFiltersQuery(+ filters,+ DataType.ANALYTICS,+ )+if (period === 'all') {const res = await this.getTimeBucketForAllTime(pid, period, timezone)@@ -2603,16 +2696,26 @@ export class AnalyticsService {diff,)- const query = `SELECT- meta.key AS key,- meta.value AS value,- count() AS count- FROM customEV- ARRAY JOIN meta.key, meta.value- WHERE pid = {pid:FixedString(12)}- AND created BETWEEN {groupFrom:String} AND {groupTo:String}- AND ev = {event:String}- GROUP BY key, value`+ const query = `+ SELECT+ meta.key AS key,+ meta.value AS value,+ count() AS count+ FROM (+ SELECT+ meta.key,+ meta.value+ FROM+ customEV+ WHERE+ pid = {pid:FixedString(12)}+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}+ AND ev = {event:String}+ ${filtersQuery}+ )+ ARRAY JOIN meta.key, meta.value+ GROUP BY key, value+ `const paramsData = {params: {@@ -2620,12 +2723,19 @@ export class AnalyticsService {groupFrom: groupFromUTC,groupTo: groupToUTC,event,+ ...filtersParams,},}try {- const result = await clickhouse.query(query, paramsData).toPromise()- return result as IAggregatedMetadata[]+ const result = (await clickhouse+ .query(query, paramsData)+ .toPromise()) as IAggregatedMetadata[]++ return {+ result,+ appliedFilters,+ }} catch (reason) {console.error(`[ERROR](getCustomEventMetadata): ${reason}`)throw new InternalServerErrorException(@@ -2634,6 +2744,107 @@ export class AnalyticsService {}}+ async getPagePropertyMeta(data: GetPagePropertyMetaDTO): Promise<{+ result: IAggregatedMetadata[]+ appliedFilters: GetFiltersQuery[2]+ }> {+ const {+ pid,+ period,+ timeBucket,+ from,+ to,+ timezone = DEFAULT_TIMEZONE,+ property,+ filters,+ } = data++ let newTimebucket = timeBucket++ let diff++ const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] =+ this.getFiltersQuery(filters, DataType.ANALYTICS)++ // We cannot make a query to customEV table using analytics table properties+ if (customEVFilterApplied) {+ return {+ result: [],+ appliedFilters,+ }+ }++ if (period === 'all') {+ const res = await this.getTimeBucketForAllTime(pid, period, timezone)++ diff = res.diff+ // eslint-disable-next-line prefer-destructuring+ newTimebucket = _includes(res.timeBucket, timeBucket)+ ? timeBucket+ : res.timeBucket[0]+ }++ this.validateTimebucket(newTimebucket)++ const safeTimezone = this.getSafeTimezone(timezone)+ const { groupFromUTC, groupToUTC } = this.getGroupFromTo(+ from,+ to,+ newTimebucket,+ period,+ safeTimezone,+ diff,+ )++ const query = `+ SELECT+ meta.key AS key,+ meta.value AS value,+ count() AS count+ FROM (+ SELECT+ meta.key,+ meta.value+ FROM+ analytics+ WHERE+ pid = {pid:FixedString(12)}+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}+ AND indexOf(meta.key, {property:String}) > 0+ ${filtersQuery}+ )+ ARRAY JOIN meta.key, meta.value+ WHERE meta.key = {property:String}+ GROUP BY key, value+ `++ const paramsData = {+ params: {+ pid,+ groupFrom: groupFromUTC,+ groupTo: groupToUTC,+ property,+ ...filtersParams,+ },+ }++ try {+ const result = (await clickhouse+ .query(query, paramsData)+ .toPromise()) as IAggregatedMetadata[]++ return {+ result,+ appliedFilters,+ }+ } catch (reason) {+ console.error(`[ERROR](getPagePropertyMeta): ${reason}`)+ throw new InternalServerErrorException(+ 'Something went wrong. Please, try again later.',+ )+ }+ }+async getOnlineUserCount(pid: string): Promise<number> {// @ts-ignorereturn redis.countKeysByPattern(`hb:${pid}:*`)diff --git a/apps/selfhosted/src/analytics/dto/events.dto.ts b/apps/selfhosted/src/analytics/dto/events.dto.tsindex e16cbc86d..16ad3c809 100644--- a/apps/selfhosted/src/analytics/dto/events.dto.ts+++ b/apps/selfhosted/src/analytics/dto/events.dto.ts@@ -10,11 +10,11 @@ import {Validate,} from 'class-validator'-const MAX_METADATA_KEYS = 20-const MAX_METADATA_VALUE_LENGTH = 1000+export const MAX_METADATA_KEYS = 20+export const MAX_METADATA_VALUE_LENGTH = 1000@ValidatorConstraint()-class MetadataSizeLimit implements ValidatorConstraintInterface {+export class MetadataSizeLimit implements ValidatorConstraintInterface {validate(metadata: Record<string, string>) {const values = _values(metadata)let totalSize = 0@@ -31,7 +31,7 @@ class MetadataSizeLimit implements ValidatorConstraintInterface {}@ValidatorConstraint()-class MetadataKeysQuantity implements ValidatorConstraintInterface {+export class MetadataKeysQuantity implements ValidatorConstraintInterface {validate(metadata: Record<string, string>) {return _keys(metadata).length <= MAX_METADATA_KEYS}diff --git a/apps/selfhosted/src/analytics/dto/get-custom-event-meta.dto.ts b/apps/selfhosted/src/analytics/dto/get-custom-event-meta.dto.tsindex d91e1cba3..8f84e6b9f 100644--- a/apps/selfhosted/src/analytics/dto/get-custom-event-meta.dto.ts+++ b/apps/selfhosted/src/analytics/dto/get-custom-event-meta.dto.ts@@ -1,4 +1,5 @@-import { PickType } from '@nestjs/swagger'+import { ApiProperty, PickType } from '@nestjs/swagger'+import { IsNotEmpty, IsString } from 'class-validator'import { AnalyticsGET_DTO } from './getData.dto'export class GetCustomEventMetadata extends PickType(AnalyticsGET_DTO, [@@ -9,6 +10,10 @@ export class GetCustomEventMetadata extends PickType(AnalyticsGET_DTO, ['to','filters','timezone',+ 'filters',] as const) {+ @ApiProperty()+ @IsNotEmpty()+ @IsString()event: string}diff --git a/apps/selfhosted/src/analytics/dto/get-page-property-meta.dto.ts b/apps/selfhosted/src/analytics/dto/get-page-property-meta.dto.tsnew file mode 100644index 000000000..ccd75a113--- /dev/null+++ b/apps/selfhosted/src/analytics/dto/get-page-property-meta.dto.ts@@ -0,0 +1,19 @@+import { ApiProperty, PickType } from '@nestjs/swagger'+import { IsNotEmpty, IsString } from 'class-validator'+import { AnalyticsGET_DTO } from './getData.dto'++export class GetPagePropertyMetaDTO extends PickType(AnalyticsGET_DTO, [+ 'pid',+ 'period',+ 'timeBucket',+ 'from',+ 'to',+ 'filters',+ 'timezone',+ 'filters',+] as const) {+ @ApiProperty()+ @IsNotEmpty()+ @IsString()+ property: string+}diff --git a/apps/selfhosted/src/analytics/dto/pageviews.dto.ts b/apps/selfhosted/src/analytics/dto/pageviews.dto.tsindex a8ea10615..deb809921 100644--- a/apps/selfhosted/src/analytics/dto/pageviews.dto.ts+++ b/apps/selfhosted/src/analytics/dto/pageviews.dto.ts@@ -1,5 +1,12 @@import { ApiProperty } from '@nestjs/swagger'-import { IsNotEmpty } from 'class-validator'+import { IsNotEmpty, IsObject, IsOptional, Validate } from 'class-validator'++import {+ MAX_METADATA_KEYS,+ MAX_METADATA_VALUE_LENGTH,+ MetadataKeysQuantity,+ MetadataSizeLimit,+} from './events.dto'export class PageviewsDTO {@ApiProperty({@@ -88,4 +95,21 @@ export class PageviewsDTO {page_load: numberttfb: number}++ @ApiProperty({+ example: {+ affiliate: 'Yes',+ protocol: 'HTTPS',+ },+ description: 'Event-related metadata object with string values',+ })+ @IsOptional()+ @IsObject()+ @Validate(MetadataKeysQuantity, {+ message: `Metadata object can't have more than ${MAX_METADATA_KEYS} keys`,+ })+ @Validate(MetadataSizeLimit, {+ message: `Metadata object can't have values with total length more than ${MAX_METADATA_VALUE_LENGTH} characters`,+ })+ meta?: Record<string, string>}diff --git a/apps/selfhosted/src/analytics/interfaces/index.ts b/apps/selfhosted/src/analytics/interfaces/index.tsindex 770ada1ad..971910973 100644--- a/apps/selfhosted/src/analytics/interfaces/index.ts+++ b/apps/selfhosted/src/analytics/interfaces/index.ts@@ -36,6 +36,12 @@ export interface CustomsCHResponse {index: number}+export interface PropertiesCHResponse {+ property: string+ 'count()': number+ index: number+}+export interface CustomsCHAggregatedResponse {year: numbermonth: number@@ -171,4 +177,12 @@ export interface IPageflow {created: string}+export interface IPageProperty {+ [key: string]: number+}++export interface ICustomEvent {+ [key: string]: number+}+export type PerfMeasure = 'average' | 'median' | 'p95' | 'quantiles'diff --git a/apps/selfhosted/src/common/constants.ts b/apps/selfhosted/src/common/constants.tsindex ae2f97e59..5077a9a61 100644--- a/apps/selfhosted/src/common/constants.ts+++ b/apps/selfhosted/src/common/constants.ts@@ -112,6 +112,8 @@ const TRAFFIC_COLUMNS = [const ALL_COLUMNS = [...TRAFFIC_COLUMNS, 'ev']+const TRAFFIC_METAKEY_COLUMNS = ['tag:key', 'tag:value']+const PERFORMANCE_COLUMNS = ['cc', 'rg', 'ct', 'pg', 'dv', 'br']const ERROR_COLUMNS = ['pg', 'dv', 'br', 'os', 'lc', 'cc', 'rg', 'ct']@@ -128,6 +130,7 @@ export {REDIS_LOG_DATA_CACHE_KEY,REDIS_LOG_CAPTCHA_CACHE_KEY,REDIS_LOG_CUSTOM_CACHE_KEY,+ TRAFFIC_METAKEY_COLUMNS,REDIS_SESSION_SALT_KEY,HEARTBEAT_SID_LIFE_TIME,UUIDV5_NAMESPACE,diff --git a/migrations/clickhouse/2024_06_22.js b/migrations/clickhouse/2024_06_22.jsnew file mode 100644index 000000000..b3a25caed--- /dev/null+++ b/migrations/clickhouse/2024_06_22.js@@ -0,0 +1,44 @@+const { queriesRunner, dbName } = require('./setup')++const queries = [+ // Added 'meta' column+ `DROP TABLE IF EXISTS ${dbName}.analytics_temp`,+ `CREATE TABLE IF NOT EXISTS ${dbName}.analytics_temp+ (+ psid Nullable(UInt64),+ sid Nullable(String),+ pid FixedString(12),+ pg Nullable(String),+ prev Nullable(String),+ dv LowCardinality(Nullable(String)),+ br LowCardinality(Nullable(String)),+ os LowCardinality(Nullable(String)),+ lc LowCardinality(Nullable(String)),+ ref Nullable(String),+ so Nullable(String),+ me Nullable(String),+ ca Nullable(String),+ cc Nullable(FixedString(2)),+ rg LowCardinality(Nullable(String)),+ ct Nullable(String),+ meta Nested+ (+ key String,+ value String+ ),+ sdur Nullable(UInt32),+ unique UInt8,+ created DateTime('UTC')+ )+ ENGINE = MergeTree()+ PARTITION BY toYYYYMM(created)+ ORDER BY (pid, created);`,++ `INSERT INTO ${dbName}.analytics_temp (psid, sid, pid, pg, prev, dv, br, os, lc, ref, so, me, ca, cc, rg, ct, meta.key, meta.value, sdur, unique, created)+ SELECT psid, sid, pid, pg, prev, dv, br, os, lc, ref, so, me, ca, cc, rg, ct, [], [], sdur, unique, created FROM ${dbName}.analytics`,++ `DROP TABLE ${dbName}.analytics`,+ `RENAME TABLE ${dbName}.analytics_temp TO ${dbName}.analytics`,+]++queriesRunner(queries)diff --git a/migrations/clickhouse/initialise_database.js b/migrations/clickhouse/initialise_database.jsindex c41928733..a48c80f22 100644--- a/migrations/clickhouse/initialise_database.js+++ b/migrations/clickhouse/initialise_database.js@@ -23,6 +23,11 @@ const CLICKHOUSE_INIT_QUERIES = [cc Nullable(FixedString(2)),rg LowCardinality(Nullable(String)),ct Nullable(String),+ meta Nested+ (+ key String,+ value String+ ),sdur Nullable(UInt32),unique UInt8,created DateTime('UTC')