Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

(feature) Pageview tags (#239)

Andrii Romasiunf4b35b20be17
diff --git a/apps/production/src/analytics/analytics.controller.ts b/apps/production/src/analytics/analytics.controller.ts
index 094e317ab..3e0713726 100644
--- a/apps/production/src/analytics/analytics.controller.ts
+++ b/apps/production/src/analytics/analytics.controller.ts
@@ -50,6 +50,7 @@ import { PageviewsDTO } from './dto/pageviews.dto'
import { EventsDTO } from './dto/events.dto'
import { AnalyticsGET_DTO, ChartRenderMode } from './dto/getData.dto'
import { GetCustomEventMetadata } from './dto/get-custom-event-meta.dto'
+import { GetPagePropertyMetaDTO } from './dto/get-page-property-meta.dto'
import { GetUserFlowDTO } from './dto/getUserFlow.dto'
import { GetFunnelsDTO } from './dto/getFunnels.dto'
import { AppLoggerService } from '../logger/logger.service'
@@ -76,9 +77,9 @@ import { BotDetectionGuard } from '../common/guards/bot-detection.guard'
import { GetCustomEventsDto } from './dto/get-custom-events.dto'
import { GetFiltersDto } from './dto/get-filters.dto'
import {
- IAggregatedMetadata,
IFunnel,
IGetFunnel,
+ IPageProperty,
IUserFlow,
PerfMeasure,
} from './interfaces'
@@ -122,9 +123,11 @@ const analyticsDTO = (
cc: string,
rg: string,
ct: string,
+ keys: string[],
+ values: string[],
sdur: number | string,
unique: number,
-): Array<string | number> => {
+): Array<string | number | string[]> => {
return [
psid,
sid,
@@ -142,6 +145,8 @@ const analyticsDTO = (
cc,
rg,
ct,
+ keys,
+ values,
sdur,
unique,
dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
@@ -433,14 +438,11 @@ export class AnalyticsController {
diff,
)
- let queryCustoms = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`
let subQuery = `FROM ${
isCaptcha ? 'captcha' : 'analytics'
} WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
if (customEVFilterApplied && !isCaptcha) {
- queryCustoms = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`
-
subQuery = `FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
}
@@ -489,14 +491,24 @@ export class AnalyticsController {
}
}
- const customs = await this.analyticsService.processCustomEV(
- queryCustoms,
+ const customs = await this.analyticsService.getCustomEvents(
+ filtersQuery,
paramsData,
)
+ let properties: IPageProperty = {}
+
+ if (!customEVFilterApplied) {
+ properties = await this.analyticsService.getPageProperties(
+ filtersQuery,
+ paramsData,
+ )
+ }
+
return {
...result,
customs,
+ properties,
appliedFilters,
timeBucket: allowedTumebucketForPeriodAll,
}
@@ -583,7 +595,7 @@ export class AnalyticsController {
@Query() data: GetCustomEventMetadata,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
- ): Promise<IAggregatedMetadata[]> {
+ ): Promise<ReturnType<typeof this.analyticsService.getCustomEventMetadata>> {
const { pid, period } = data
this.analyticsService.validatePID(pid)
@@ -602,6 +614,31 @@ export class AnalyticsController {
return this.analyticsService.getCustomEventMetadata(data)
}
+ @Get('property')
+ @Auth([], true, true)
+ async getPagePropertyMetadata(
+ @Query() data: GetPagePropertyMetaDTO,
+ @CurrentUserId() uid: string,
+ @Headers() headers: { 'x-password'?: string },
+ ): Promise<ReturnType<typeof this.analyticsService.getPagePropertyMeta>> {
+ const { pid, period } = data
+ this.analyticsService.validatePID(pid)
+
+ if (!_isEmpty(period)) {
+ this.analyticsService.validatePeriod(period)
+ }
+
+ await this.analyticsService.checkProjectAccess(
+ pid,
+ uid,
+ headers['x-password'],
+ )
+
+ await this.analyticsService.checkBillingAccess(pid)
+
+ return this.analyticsService.getPagePropertyMeta(data)
+ }
+
@Get('filters')
@Auth([], true, true)
async getFilters(
@@ -1424,6 +1461,8 @@ export class AnalyticsController {
country,
region,
city,
+ _keys(logDTO.meta),
+ _values(logDTO.meta),
0,
Number(unique),
)
@@ -1538,6 +1577,8 @@ export class AnalyticsController {
country,
region,
city,
+ [],
+ [],
0,
Number(unique),
)
diff --git a/apps/production/src/analytics/analytics.service.ts b/apps/production/src/analytics/analytics.service.ts
index 6671c37cf..ab6dda25f 100644
--- a/apps/production/src/analytics/analytics.service.ts
+++ b/apps/production/src/analytics/analytics.service.ts
@@ -9,6 +9,7 @@ import * as _toUpper from 'lodash/toUpper'
import * as _head from 'lodash/head'
import * as _join from 'lodash/join'
import * as _isArray from 'lodash/isArray'
+import * as _startsWith from 'lodash/startsWith'
import * as _sortBy from 'lodash/sortBy'
import * as _reduce from 'lodash/reduce'
import * as _keys from 'lodash/keys'
@@ -55,6 +56,7 @@ import {
REDIS_USERS_COUNT_KEY,
REDIS_PROJECTS_COUNT_KEY,
REDIS_EVENTS_COUNT_KEY,
+ TRAFFIC_METAKEY_COLUMNS,
} from '../common/constants'
import {
calculateRelativePercentage,
@@ -91,8 +93,12 @@ import {
IOverallPerformance,
IPageflow,
PerfMeasure,
+ PropertiesCHResponse,
+ IPageProperty,
+ ICustomEvent,
} from './interfaces'
import { ErrorDTO } from './dto/error.dto'
+import { GetPagePropertyMetaDTO } from './dto/get-page-property-meta.dto'
dayjs.extend(utc)
dayjs.extend(dayjsTimezone)
@@ -910,9 +916,19 @@ export class AnalyticsService {
return prev
}
- if (column === 'ev') {
+ // ev:key -> custom event metadata
+ // ev:key: -> custom event metadata, but we want to check if some meta.key's meta.value equals to a specific value
+ if (
+ column === 'ev' ||
+ column === 'ev:key' ||
+ _startsWith(column, 'ev:key:')
+ ) {
customEVFilterApplied = true
- } else if (!_includes(SUPPORTED_COLUMNS, column)) {
+ } else if (
+ !_includes(SUPPORTED_COLUMNS, column) &&
+ !_includes(TRAFFIC_METAKEY_COLUMNS, column) &&
+ !_startsWith(column, 'tag:key:')
+ ) {
throw new UnprocessableEntityException(
`The provided filter (${column}) is not supported`,
)
@@ -958,7 +974,7 @@ export class AnalyticsService {
const columns = _keys(converted)
for (let col = 0; col < _size(columns); ++col) {
- const column = columns[col]
+ const column: string = columns[col]
query += ' AND ('
for (let f = 0; f < _size(converted[column]); ++f) {
@@ -967,18 +983,46 @@ export class AnalyticsService {
}
const { filter, isExclusive } = converted[column][f]
+ let sqlColumn = column
+ let isArrayDataset = false
const param = `qf_${col}_${f}`
+ params[param] = filter
- if (filter === null) {
- query += `${column} IS ${isExclusive ? 'NOT' : ''} NULL`
- params[param] = filter
+ // when we want to filter meta.value for a specific meta.key
+ if (_startsWith(column, 'ev:key:') || _startsWith(column, 'tag:key:')) {
+ const key = column.replace(/^ev:key:/, '').replace(/^tag:key:/, '')
+ const keyParam = `qfk_${col}_${f}`
+ params[keyParam] = key
+
+ query += `indexOf(meta.key, {${keyParam}:String}) > 0 AND meta.value[indexOf(meta.key, {${keyParam}:String})] ${isExclusive ? '!= ' : '='} {${param}:String}`
continue
}
- query += `${isExclusive ? 'NOT ' : ''}${column} = {${param}:String}`
+ // meta.key filters for page properties and custom event metadata
+ // e.g. article "author" (property)
+ if (column === 'ev:key' || column === 'tag:key') {
+ sqlColumn = 'meta.key'
+ isArrayDataset = true
+ // meta.value filters for page properties and custom event metadata
+ // e.g. "Andrii" ("author" value)
+ } else if (column === 'ev:value' || column === 'tag:value') {
+ sqlColumn = 'meta.value'
+ isArrayDataset = true
+ }
+
+ // TODO: In future I will add contains / not contains filters as well via ILIKE operator
- params[param] = filter
+ if (filter === null) {
+ query += isArrayDataset
+ ? ''
+ : `${sqlColumn} IS ${isExclusive ? 'NOT' : ''} NULL`
+ continue
+ }
+
+ query += isArrayDataset
+ ? `indexOf(${sqlColumn}, {${param}:String}) ${isExclusive ? '=' : '>'} 0`
+ : `${isExclusive ? 'NOT ' : ''}${sqlColumn} = {${param}:String}`
}
query += ')'
@@ -2844,7 +2888,11 @@ export class AnalyticsService {
})
}
- async processCustomEV(query: string, params: object): Promise<object> {
+ async getCustomEvents(
+ filtersQuery: string,
+ params: object,
+ ): Promise<ICustomEvent> {
+ const query = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`
const result = {}
const rawCustoms = <Array<CustomsCHResponse>>(
@@ -2860,6 +2908,40 @@ export class AnalyticsService {
return result
}
+ async getPageProperties(
+ filtersQuery: string,
+ params: object,
+ ): Promise<IPageProperty> {
+ const query = `
+ SELECT
+ meta.key AS property,
+ count()
+ FROM analytics
+ WHERE pid = {pid:FixedString(12)}
+ ${filtersQuery}
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ GROUP BY property`
+ const result = {}
+
+ const rawProperties = <Array<PropertiesCHResponse>>(
+ await clickhouse.query(query, params).toPromise()
+ )
+ const size = _size(rawProperties)
+
+ for (let i = 0; i < size; ++i) {
+ const { property: propertyArr, 'count()': c } = rawProperties[i]
+ const [property] = propertyArr
+
+ if (!property) {
+ continue
+ }
+
+ result[property] = c
+ }
+
+ return result
+ }
+
validateCustomEVMeta(meta: any) {
if (typeof meta === 'undefined') {
return
@@ -2872,15 +2954,17 @@ export class AnalyticsService {
}
}
- async getCustomEventMetadata(
- data: GetCustomEventMetadata,
- ): Promise<IAggregatedMetadata[]> {
+ async getCustomEventMetadata(data: GetCustomEventMetadata): Promise<{
+ result: IAggregatedMetadata[]
+ appliedFilters: GetFiltersQuery[2]
+ }> {
const {
pid,
period,
timeBucket,
from,
to,
+ filters,
timezone = DEFAULT_TIMEZONE,
event,
} = data
@@ -2889,6 +2973,11 @@ export class AnalyticsService {
let diff
+ const [filtersQuery, filtersParams, appliedFilters] = this.getFiltersQuery(
+ filters,
+ DataType.ANALYTICS,
+ )
+
if (period === 'all') {
const res = await this.getTimeBucketForAllTime(pid, period, timezone)
@@ -2911,16 +3000,26 @@ export class AnalyticsService {
diff,
)
- const query = `SELECT
- meta.key AS key,
- meta.value AS value,
- count() AS count
- FROM customEV
- ARRAY JOIN meta.key, meta.value
- WHERE pid = {pid:FixedString(12)}
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- AND ev = {event:String}
- GROUP BY key, value`
+ const query = `
+ SELECT
+ meta.key AS key,
+ meta.value AS value,
+ count() AS count
+ FROM (
+ SELECT
+ meta.key,
+ meta.value
+ FROM
+ customEV
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ AND ev = {event:String}
+ ${filtersQuery}
+ )
+ ARRAY JOIN meta.key, meta.value
+ GROUP BY key, value
+ `
const paramsData = {
params: {
@@ -2928,12 +3027,19 @@ export class AnalyticsService {
groupFrom: groupFromUTC,
groupTo: groupToUTC,
event,
+ ...filtersParams,
},
}
try {
- const result = await clickhouse.query(query, paramsData).toPromise()
- return result as IAggregatedMetadata[]
+ const result = (await clickhouse
+ .query(query, paramsData)
+ .toPromise()) as IAggregatedMetadata[]
+
+ return {
+ result,
+ appliedFilters,
+ }
} catch (reason) {
console.error(`[ERROR](getCustomEventMetadata): ${reason}`)
throw new InternalServerErrorException(
@@ -2942,6 +3048,107 @@ export class AnalyticsService {
}
}
+ async getPagePropertyMeta(data: GetPagePropertyMetaDTO): Promise<{
+ result: IAggregatedMetadata[]
+ appliedFilters: GetFiltersQuery[2]
+ }> {
+ const {
+ pid,
+ period,
+ timeBucket,
+ from,
+ to,
+ timezone = DEFAULT_TIMEZONE,
+ property,
+ filters,
+ } = data
+
+ let newTimebucket = timeBucket
+
+ let diff
+
+ const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] =
+ this.getFiltersQuery(filters, DataType.ANALYTICS)
+
+ // We cannot make a query to customEV table using analytics table properties
+ if (customEVFilterApplied) {
+ return {
+ result: [],
+ appliedFilters,
+ }
+ }
+
+ if (period === 'all') {
+ const res = await this.getTimeBucketForAllTime(pid, period, timezone)
+
+ diff = res.diff
+ // eslint-disable-next-line prefer-destructuring
+ newTimebucket = _includes(res.timeBucket, timeBucket)
+ ? timeBucket
+ : res.timeBucket[0]
+ }
+
+ this.validateTimebucket(newTimebucket)
+
+ const safeTimezone = this.getSafeTimezone(timezone)
+ const { groupFromUTC, groupToUTC } = this.getGroupFromTo(
+ from,
+ to,
+ newTimebucket,
+ period,
+ safeTimezone,
+ diff,
+ )
+
+ const query = `
+ SELECT
+ meta.key AS key,
+ meta.value AS value,
+ count() AS count
+ FROM (
+ SELECT
+ meta.key,
+ meta.value
+ FROM
+ analytics
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ AND indexOf(meta.key, {property:String}) > 0
+ ${filtersQuery}
+ )
+ ARRAY JOIN meta.key, meta.value
+ WHERE meta.key = {property:String}
+ GROUP BY key, value
+ `
+
+ const paramsData = {
+ params: {
+ pid,
+ groupFrom: groupFromUTC,
+ groupTo: groupToUTC,
+ property,
+ ...filtersParams,
+ },
+ }
+
+ try {
+ const result = (await clickhouse
+ .query(query, paramsData)
+ .toPromise()) as IAggregatedMetadata[]
+
+ return {
+ result,
+ appliedFilters,
+ }
+ } catch (reason) {
+ console.error(`[ERROR](getPagePropertyMeta): ${reason}`)
+ throw new InternalServerErrorException(
+ 'Something went wrong. Please, try again later.',
+ )
+ }
+ }
+
async getOnlineUserCount(pid: string): Promise<number> {
// @ts-ignore
return redis.countKeysByPattern(`hb:${pid}:*`)
diff --git a/apps/production/src/analytics/dto/events.dto.ts b/apps/production/src/analytics/dto/events.dto.ts
index 2ebc02d89..b2655e857 100644
--- a/apps/production/src/analytics/dto/events.dto.ts
+++ b/apps/production/src/analytics/dto/events.dto.ts
@@ -10,11 +10,11 @@ import {
Validate,
} from 'class-validator'
-const MAX_METADATA_KEYS = 20
-const MAX_METADATA_VALUE_LENGTH = 1000
+export const MAX_METADATA_KEYS = 20
+export const MAX_METADATA_VALUE_LENGTH = 1000
@ValidatorConstraint()
-class MetadataSizeLimit implements ValidatorConstraintInterface {
+export class MetadataSizeLimit implements ValidatorConstraintInterface {
validate(metadata: Record<string, string>) {
const values = _values(metadata)
let totalSize = 0
@@ -31,7 +31,7 @@ class MetadataSizeLimit implements ValidatorConstraintInterface {
}
@ValidatorConstraint()
-class MetadataKeysQuantity implements ValidatorConstraintInterface {
+export class MetadataKeysQuantity implements ValidatorConstraintInterface {
validate(metadata: Record<string, string>) {
return _keys(metadata).length <= MAX_METADATA_KEYS
}
diff --git a/apps/production/src/analytics/dto/get-custom-event-meta.dto.ts b/apps/production/src/analytics/dto/get-custom-event-meta.dto.ts
index d91e1cba3..8f84e6b9f 100644
--- a/apps/production/src/analytics/dto/get-custom-event-meta.dto.ts
+++ b/apps/production/src/analytics/dto/get-custom-event-meta.dto.ts
@@ -1,4 +1,5 @@
-import { PickType } from '@nestjs/swagger'
+import { ApiProperty, PickType } from '@nestjs/swagger'
+import { IsNotEmpty, IsString } from 'class-validator'
import { AnalyticsGET_DTO } from './getData.dto'
export class GetCustomEventMetadata extends PickType(AnalyticsGET_DTO, [
@@ -9,6 +10,10 @@ export class GetCustomEventMetadata extends PickType(AnalyticsGET_DTO, [
'to',
'filters',
'timezone',
+ 'filters',
] as const) {
+ @ApiProperty()
+ @IsNotEmpty()
+ @IsString()
event: string
}
diff --git a/apps/production/src/analytics/dto/get-page-property-meta.dto.ts b/apps/production/src/analytics/dto/get-page-property-meta.dto.ts
new file mode 100644
index 000000000..ccd75a113
--- /dev/null
+++ b/apps/production/src/analytics/dto/get-page-property-meta.dto.ts
@@ -0,0 +1,19 @@
+import { ApiProperty, PickType } from '@nestjs/swagger'
+import { IsNotEmpty, IsString } from 'class-validator'
+import { AnalyticsGET_DTO } from './getData.dto'
+
+export class GetPagePropertyMetaDTO extends PickType(AnalyticsGET_DTO, [
+ 'pid',
+ 'period',
+ 'timeBucket',
+ 'from',
+ 'to',
+ 'filters',
+ 'timezone',
+ 'filters',
+] as const) {
+ @ApiProperty()
+ @IsNotEmpty()
+ @IsString()
+ property: string
+}
diff --git a/apps/production/src/analytics/dto/pageviews.dto.ts b/apps/production/src/analytics/dto/pageviews.dto.ts
index a8ea10615..deb809921 100644
--- a/apps/production/src/analytics/dto/pageviews.dto.ts
+++ b/apps/production/src/analytics/dto/pageviews.dto.ts
@@ -1,5 +1,12 @@
import { ApiProperty } from '@nestjs/swagger'
-import { IsNotEmpty } from 'class-validator'
+import { IsNotEmpty, IsObject, IsOptional, Validate } from 'class-validator'
+
+import {
+ MAX_METADATA_KEYS,
+ MAX_METADATA_VALUE_LENGTH,
+ MetadataKeysQuantity,
+ MetadataSizeLimit,
+} from './events.dto'
export class PageviewsDTO {
@ApiProperty({
@@ -88,4 +95,21 @@ export class PageviewsDTO {
page_load: number
ttfb: number
}
+
+ @ApiProperty({
+ example: {
+ affiliate: 'Yes',
+ protocol: 'HTTPS',
+ },
+ description: 'Event-related metadata object with string values',
+ })
+ @IsOptional()
+ @IsObject()
+ @Validate(MetadataKeysQuantity, {
+ message: `Metadata object can't have more than ${MAX_METADATA_KEYS} keys`,
+ })
+ @Validate(MetadataSizeLimit, {
+ message: `Metadata object can't have values with total length more than ${MAX_METADATA_VALUE_LENGTH} characters`,
+ })
+ meta?: Record<string, string>
}
diff --git a/apps/production/src/analytics/interfaces/index.ts b/apps/production/src/analytics/interfaces/index.ts
index bee862b0a..813d0722a 100644
--- a/apps/production/src/analytics/interfaces/index.ts
+++ b/apps/production/src/analytics/interfaces/index.ts
@@ -42,6 +42,12 @@ export interface CustomsCHResponse {
index: number
}
+export interface PropertiesCHResponse {
+ property: string
+ 'count()': number
+ index: number
+}
+
export interface CustomsCHAggregatedResponse {
year: number
month: number
@@ -185,4 +191,12 @@ export interface IPageflow {
created: string
}
+export interface IPageProperty {
+ [key: string]: number
+}
+
+export interface ICustomEvent {
+ [key: string]: number
+}
+
export type PerfMeasure = 'average' | 'median' | 'p95' | 'quantiles'
diff --git a/apps/production/src/common/constants.ts b/apps/production/src/common/constants.ts
index 9453d51d4..d66e035f4 100644
--- a/apps/production/src/common/constants.ts
+++ b/apps/production/src/common/constants.ts
@@ -144,6 +144,8 @@ const TRAFFIC_COLUMNS = [
'ca',
]
+const TRAFFIC_METAKEY_COLUMNS = ['tag:key', 'tag:value']
+
const ERROR_COLUMNS = ['pg', 'dv', 'br', 'os', 'lc', 'cc', 'rg', 'ct']
const ALL_COLUMNS = [...TRAFFIC_COLUMNS, 'ev']
@@ -228,6 +230,7 @@ export {
getRedisUserUsageInfoKey,
redisUserUsageinfoCacheTimeout,
TRAFFIC_COLUMNS,
+ TRAFFIC_METAKEY_COLUMNS,
CAPTCHA_COLUMNS,
ERROR_COLUMNS,
PERFORMANCE_COLUMNS,
diff --git a/apps/selfhosted/src/analytics/analytics.controller.ts b/apps/selfhosted/src/analytics/analytics.controller.ts
index 3af2f00a4..aed291262 100644
--- a/apps/selfhosted/src/analytics/analytics.controller.ts
+++ b/apps/selfhosted/src/analytics/analytics.controller.ts
@@ -49,6 +49,7 @@ import { EventsDTO } from './dto/events.dto'
import { AnalyticsGET_DTO, ChartRenderMode } from './dto/getData.dto'
import { GetFiltersDto } from './dto/get-filters.dto'
import { GetCustomEventMetadata } from './dto/get-custom-event-meta.dto'
+import { GetPagePropertyMetaDTO } from './dto/get-page-property-meta.dto'
import { GetUserFlowDTO } from './dto/getUserFlow.dto'
import { GetFunnelsDTO } from './dto/getFunnels.dto'
import { AppLoggerService } from '../logger/logger.service'
@@ -71,9 +72,9 @@ import { BotDetection } from '../common/decorators/bot-detection.decorator'
import { BotDetectionGuard } from '../common/guards/bot-detection.guard'
import { GetCustomEventsDto } from './dto/get-custom-events.dto'
import {
- IAggregatedMetadata,
IFunnel,
IGetFunnel,
+ IPageProperty,
IUserFlow,
PerfMeasure,
} from './interfaces'
@@ -117,9 +118,11 @@ const analyticsDTO = (
cc: string,
rg: string,
ct: string,
+ keys: string[],
+ values: string[],
sdur: number | string,
unique: number,
-): Array<string | number> => {
+): Array<string | number | string[]> => {
return [
psid,
sid,
@@ -137,6 +140,8 @@ const analyticsDTO = (
cc,
rg,
ct,
+ keys,
+ values,
sdur,
unique,
dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
@@ -422,12 +427,9 @@ export class AnalyticsController {
diff,
)
- let queryCustoms = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`
let subQuery = `FROM analytics WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
if (customEVFilterApplied) {
- queryCustoms = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`
-
subQuery = `FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
}
@@ -453,14 +455,24 @@ export class AnalyticsController {
mode,
)
- const customs = await this.analyticsService.processCustomEV(
- queryCustoms,
+ const customs = await this.analyticsService.getCustomEvents(
+ filtersQuery,
paramsData,
)
+ let properties: IPageProperty = {}
+
+ if (!customEVFilterApplied) {
+ properties = await this.analyticsService.getPageProperties(
+ filtersQuery,
+ paramsData,
+ )
+ }
+
return {
...result,
customs,
+ properties,
appliedFilters: parsedFilters,
timeBucket: allowedTumebucketForPeriodAll,
}
@@ -545,7 +557,7 @@ export class AnalyticsController {
@Query() data: GetCustomEventMetadata,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
- ): Promise<IAggregatedMetadata[]> {
+ ): Promise<ReturnType<typeof this.analyticsService.getCustomEventMetadata>> {
const { pid, period } = data
this.analyticsService.validatePID(pid)
@@ -562,6 +574,29 @@ export class AnalyticsController {
return this.analyticsService.getCustomEventMetadata(data)
}
+ @Get('property')
+ @Auth([], true, true)
+ async getPagePropertyMetadata(
+ @Query() data: GetPagePropertyMetaDTO,
+ @CurrentUserId() uid: string,
+ @Headers() headers: { 'x-password'?: string },
+ ): Promise<ReturnType<typeof this.analyticsService.getPagePropertyMeta>> {
+ const { pid, period } = data
+ this.analyticsService.validatePID(pid)
+
+ if (!_isEmpty(period)) {
+ this.analyticsService.validatePeriod(period)
+ }
+
+ await this.analyticsService.checkProjectAccess(
+ pid,
+ uid,
+ headers['x-password'],
+ )
+
+ return this.analyticsService.getPagePropertyMeta(data)
+ }
+
@Get('filters')
@Auth([], true, true)
async getFilters(
@@ -1193,6 +1228,8 @@ export class AnalyticsController {
country,
region,
city,
+ _keys(logDTO.meta),
+ _values(logDTO.meta),
0,
Number(unique),
)
@@ -1306,6 +1343,8 @@ export class AnalyticsController {
city,
region,
country,
+ [],
+ [],
0,
Number(unique),
)
diff --git a/apps/selfhosted/src/analytics/analytics.service.ts b/apps/selfhosted/src/analytics/analytics.service.ts
index a3c649bcf..e0c1d5c93 100644
--- a/apps/selfhosted/src/analytics/analytics.service.ts
+++ b/apps/selfhosted/src/analytics/analytics.service.ts
@@ -14,6 +14,7 @@ import * as _last from 'lodash/last'
import * as _some from 'lodash/some'
import * as _find from 'lodash/find'
import * as _isArray from 'lodash/isArray'
+import * as _startsWith from 'lodash/startsWith'
import * as _reduce from 'lodash/reduce'
import * as _keys from 'lodash/keys'
import * as _now from 'lodash/now'
@@ -49,6 +50,7 @@ import {
MIN_PAGES_IN_FUNNEL,
MAX_PAGES_IN_FUNNEL,
ALL_COLUMNS,
+ TRAFFIC_METAKEY_COLUMNS,
} from '../common/constants'
import {
getFunnelsClickhouse,
@@ -83,8 +85,12 @@ import {
IOverallPerformance,
IPageflow,
PerfMeasure,
+ PropertiesCHResponse,
+ IPageProperty,
+ ICustomEvent,
} from './interfaces'
import { ErrorDTO } from './dto/error.dto'
+import { GetPagePropertyMetaDTO } from './dto/get-page-property-meta.dto'
dayjs.extend(utc)
dayjs.extend(dayjsTimezone)
@@ -884,9 +890,19 @@ export class AnalyticsService {
return prev
}
- if (column === 'ev') {
+ // ev:key -> custom event metadata
+ // ev:key: -> custom event metadata, but we want to check if some meta.key's meta.value equals to a specific value
+ if (
+ column === 'ev' ||
+ column === 'ev:key' ||
+ _startsWith(column, 'ev:key:')
+ ) {
customEVFilterApplied = true
- } else if (!_includes(SUPPORTED_COLUMNS, column)) {
+ } else if (
+ !_includes(SUPPORTED_COLUMNS, column) &&
+ !_includes(TRAFFIC_METAKEY_COLUMNS, column) &&
+ !_startsWith(column, 'tag:key:')
+ ) {
throw new UnprocessableEntityException(
`The provided filter (${column}) is not supported`,
)
@@ -894,24 +910,28 @@ export class AnalyticsService {
const res = []
+ // commented encodeURIComponent lines of code to support filtering for pages like /product/[id]
+ // until I find a more suitable solution for that later
if (_isArray(filter)) {
for (const f of filter) {
- let encoded = f
+ // let encoded = f
- if (column === 'pg' && f !== null) {
- encoded = _replace(encodeURIComponent(f), /%2F/g, '/')
- }
+ // if (column === 'pg' && f !== null) {
+ // encoded = _replace(encodeURIComponent(f), /%2F/g, '/')
+ // }
- res.push({ filter: encoded, isExclusive })
+ // res.push({ filter: encoded, isExclusive })
+ res.push({ filter: f, isExclusive })
}
} else {
- let encoded = filter
+ // let encoded = filter
- if (column === 'pg' && filter !== null) {
- encoded = _replace(encodeURIComponent(filter), /%2F/g, '/')
- }
+ // if (column === 'pg' && filter !== null) {
+ // encoded = _replace(encodeURIComponent(filter), /%2F/g, '/')
+ // }
- res.push({ filter: encoded, isExclusive })
+ // res.push({ filter: encoded, isExclusive })
+ res.push({ filter, isExclusive })
}
if (prev[column]) {
@@ -928,7 +948,7 @@ export class AnalyticsService {
const columns = _keys(converted)
for (let col = 0; col < _size(columns); ++col) {
- const column = columns[col]
+ const column: string = columns[col]
query += ' AND ('
for (let f = 0; f < _size(converted[column]); ++f) {
@@ -937,18 +957,46 @@ export class AnalyticsService {
}
const { filter, isExclusive } = converted[column][f]
+ let sqlColumn = column
+ let isArrayDataset = false
const param = `qf_${col}_${f}`
+ params[param] = filter
- if (filter === null) {
- query += `${column} IS ${isExclusive ? 'NOT' : ''} NULL`
- params[param] = filter
+ // when we want to filter meta.value for a specific meta.key
+ if (_startsWith(column, 'ev:key:') || _startsWith(column, 'tag:key:')) {
+ const key = column.replace(/^ev:key:/, '').replace(/^tag:key:/, '')
+ const keyParam = `qfk_${col}_${f}`
+ params[keyParam] = key
+
+ query += `indexOf(meta.key, {${keyParam}:String}) > 0 AND meta.value[indexOf(meta.key, {${keyParam}:String})] ${isExclusive ? '!= ' : '='} {${param}:String}`
continue
}
- query += `${isExclusive ? 'NOT ' : ''}${column} = {${param}:String}`
+ // meta.key filters for page properties and custom event metadata
+ // e.g. article "author" (property)
+ if (column === 'ev:key' || column === 'tag:key') {
+ sqlColumn = 'meta.key'
+ isArrayDataset = true
+ // meta.value filters for page properties and custom event metadata
+ // e.g. "Andrii" ("author" value)
+ } else if (column === 'ev:value' || column === 'tag:value') {
+ sqlColumn = 'meta.value'
+ isArrayDataset = true
+ }
- params[param] = filter
+ // TODO: In future I will add contains / not contains filters as well via ILIKE operator
+
+ if (filter === null) {
+ query += isArrayDataset
+ ? ''
+ : `${sqlColumn} IS ${isExclusive ? 'NOT' : ''} NULL`
+ continue
+ }
+
+ query += isArrayDataset
+ ? `indexOf(${sqlColumn}, {${param}:String}) ${isExclusive ? '=' : '>'} 0`
+ : `${isExclusive ? 'NOT ' : ''}${sqlColumn} = {${param}:String}`
}
query += ')'
@@ -2536,7 +2584,11 @@ export class AnalyticsService {
})
}
- async processCustomEV(query: string, params: object): Promise<object> {
+ async getCustomEvents(
+ filtersQuery: string,
+ params: object,
+ ): Promise<ICustomEvent> {
+ const query = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`
const result = {}
const rawCustoms = <Array<CustomsCHResponse>>(
@@ -2564,15 +2616,51 @@ export class AnalyticsService {
}
}
- async getCustomEventMetadata(
- data: GetCustomEventMetadata,
- ): Promise<IAggregatedMetadata[]> {
+ async getPageProperties(
+ filtersQuery: string,
+ params: object,
+ ): Promise<IPageProperty> {
+ const query = `
+ SELECT
+ meta.key AS property,
+ count()
+ FROM analytics
+ WHERE pid = {pid:FixedString(12)}
+ ${filtersQuery}
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ GROUP BY property`
+ const result = {}
+
+ const rawProperties = <Array<PropertiesCHResponse>>(
+ await clickhouse.query(query, params).toPromise()
+ )
+ const size = _size(rawProperties)
+
+ for (let i = 0; i < size; ++i) {
+ const { property: propertyArr, 'count()': c } = rawProperties[i]
+ const [property] = propertyArr
+
+ if (!property) {
+ continue
+ }
+
+ result[property] = c
+ }
+
+ return result
+ }
+
+ async getCustomEventMetadata(data: GetCustomEventMetadata): Promise<{
+ result: IAggregatedMetadata[]
+ appliedFilters: GetFiltersQuery[2]
+ }> {
const {
pid,
period,
timeBucket,
from,
to,
+ filters,
timezone = DEFAULT_TIMEZONE,
event,
} = data
@@ -2581,6 +2669,11 @@ export class AnalyticsService {
let diff
+ const [filtersQuery, filtersParams, appliedFilters] = this.getFiltersQuery(
+ filters,
+ DataType.ANALYTICS,
+ )
+
if (period === 'all') {
const res = await this.getTimeBucketForAllTime(pid, period, timezone)
@@ -2603,16 +2696,26 @@ export class AnalyticsService {
diff,
)
- const query = `SELECT
- meta.key AS key,
- meta.value AS value,
- count() AS count
- FROM customEV
- ARRAY JOIN meta.key, meta.value
- WHERE pid = {pid:FixedString(12)}
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
- AND ev = {event:String}
- GROUP BY key, value`
+ const query = `
+ SELECT
+ meta.key AS key,
+ meta.value AS value,
+ count() AS count
+ FROM (
+ SELECT
+ meta.key,
+ meta.value
+ FROM
+ customEV
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ AND ev = {event:String}
+ ${filtersQuery}
+ )
+ ARRAY JOIN meta.key, meta.value
+ GROUP BY key, value
+ `
const paramsData = {
params: {
@@ -2620,12 +2723,19 @@ export class AnalyticsService {
groupFrom: groupFromUTC,
groupTo: groupToUTC,
event,
+ ...filtersParams,
},
}
try {
- const result = await clickhouse.query(query, paramsData).toPromise()
- return result as IAggregatedMetadata[]
+ const result = (await clickhouse
+ .query(query, paramsData)
+ .toPromise()) as IAggregatedMetadata[]
+
+ return {
+ result,
+ appliedFilters,
+ }
} catch (reason) {
console.error(`[ERROR](getCustomEventMetadata): ${reason}`)
throw new InternalServerErrorException(
@@ -2634,6 +2744,107 @@ export class AnalyticsService {
}
}
+ async getPagePropertyMeta(data: GetPagePropertyMetaDTO): Promise<{
+ result: IAggregatedMetadata[]
+ appliedFilters: GetFiltersQuery[2]
+ }> {
+ const {
+ pid,
+ period,
+ timeBucket,
+ from,
+ to,
+ timezone = DEFAULT_TIMEZONE,
+ property,
+ filters,
+ } = data
+
+ let newTimebucket = timeBucket
+
+ let diff
+
+ const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] =
+ this.getFiltersQuery(filters, DataType.ANALYTICS)
+
+ // We cannot make a query to customEV table using analytics table properties
+ if (customEVFilterApplied) {
+ return {
+ result: [],
+ appliedFilters,
+ }
+ }
+
+ if (period === 'all') {
+ const res = await this.getTimeBucketForAllTime(pid, period, timezone)
+
+ diff = res.diff
+ // eslint-disable-next-line prefer-destructuring
+ newTimebucket = _includes(res.timeBucket, timeBucket)
+ ? timeBucket
+ : res.timeBucket[0]
+ }
+
+ this.validateTimebucket(newTimebucket)
+
+ const safeTimezone = this.getSafeTimezone(timezone)
+ const { groupFromUTC, groupToUTC } = this.getGroupFromTo(
+ from,
+ to,
+ newTimebucket,
+ period,
+ safeTimezone,
+ diff,
+ )
+
+ const query = `
+ SELECT
+ meta.key AS key,
+ meta.value AS value,
+ count() AS count
+ FROM (
+ SELECT
+ meta.key,
+ meta.value
+ FROM
+ analytics
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ AND indexOf(meta.key, {property:String}) > 0
+ ${filtersQuery}
+ )
+ ARRAY JOIN meta.key, meta.value
+ WHERE meta.key = {property:String}
+ GROUP BY key, value
+ `
+
+ const paramsData = {
+ params: {
+ pid,
+ groupFrom: groupFromUTC,
+ groupTo: groupToUTC,
+ property,
+ ...filtersParams,
+ },
+ }
+
+ try {
+ const result = (await clickhouse
+ .query(query, paramsData)
+ .toPromise()) as IAggregatedMetadata[]
+
+ return {
+ result,
+ appliedFilters,
+ }
+ } catch (reason) {
+ console.error(`[ERROR](getPagePropertyMeta): ${reason}`)
+ throw new InternalServerErrorException(
+ 'Something went wrong. Please, try again later.',
+ )
+ }
+ }
+
async getOnlineUserCount(pid: string): Promise<number> {
// @ts-ignore
return redis.countKeysByPattern(`hb:${pid}:*`)
diff --git a/apps/selfhosted/src/analytics/dto/events.dto.ts b/apps/selfhosted/src/analytics/dto/events.dto.ts
index e16cbc86d..16ad3c809 100644
--- a/apps/selfhosted/src/analytics/dto/events.dto.ts
+++ b/apps/selfhosted/src/analytics/dto/events.dto.ts
@@ -10,11 +10,11 @@ import {
Validate,
} from 'class-validator'
-const MAX_METADATA_KEYS = 20
-const MAX_METADATA_VALUE_LENGTH = 1000
+export const MAX_METADATA_KEYS = 20
+export const MAX_METADATA_VALUE_LENGTH = 1000
@ValidatorConstraint()
-class MetadataSizeLimit implements ValidatorConstraintInterface {
+export class MetadataSizeLimit implements ValidatorConstraintInterface {
validate(metadata: Record<string, string>) {
const values = _values(metadata)
let totalSize = 0
@@ -31,7 +31,7 @@ class MetadataSizeLimit implements ValidatorConstraintInterface {
}
@ValidatorConstraint()
-class MetadataKeysQuantity implements ValidatorConstraintInterface {
+export class MetadataKeysQuantity implements ValidatorConstraintInterface {
validate(metadata: Record<string, string>) {
return _keys(metadata).length <= MAX_METADATA_KEYS
}
diff --git a/apps/selfhosted/src/analytics/dto/get-custom-event-meta.dto.ts b/apps/selfhosted/src/analytics/dto/get-custom-event-meta.dto.ts
index d91e1cba3..8f84e6b9f 100644
--- a/apps/selfhosted/src/analytics/dto/get-custom-event-meta.dto.ts
+++ b/apps/selfhosted/src/analytics/dto/get-custom-event-meta.dto.ts
@@ -1,4 +1,5 @@
-import { PickType } from '@nestjs/swagger'
+import { ApiProperty, PickType } from '@nestjs/swagger'
+import { IsNotEmpty, IsString } from 'class-validator'
import { AnalyticsGET_DTO } from './getData.dto'
export class GetCustomEventMetadata extends PickType(AnalyticsGET_DTO, [
@@ -9,6 +10,10 @@ export class GetCustomEventMetadata extends PickType(AnalyticsGET_DTO, [
'to',
'filters',
'timezone',
+ 'filters',
] as const) {
+ @ApiProperty()
+ @IsNotEmpty()
+ @IsString()
event: string
}
diff --git a/apps/selfhosted/src/analytics/dto/get-page-property-meta.dto.ts b/apps/selfhosted/src/analytics/dto/get-page-property-meta.dto.ts
new file mode 100644
index 000000000..ccd75a113
--- /dev/null
+++ b/apps/selfhosted/src/analytics/dto/get-page-property-meta.dto.ts
@@ -0,0 +1,19 @@
+import { ApiProperty, PickType } from '@nestjs/swagger'
+import { IsNotEmpty, IsString } from 'class-validator'
+import { AnalyticsGET_DTO } from './getData.dto'
+
+export class GetPagePropertyMetaDTO extends PickType(AnalyticsGET_DTO, [
+ 'pid',
+ 'period',
+ 'timeBucket',
+ 'from',
+ 'to',
+ 'filters',
+ 'timezone',
+ 'filters',
+] as const) {
+ @ApiProperty()
+ @IsNotEmpty()
+ @IsString()
+ property: string
+}
diff --git a/apps/selfhosted/src/analytics/dto/pageviews.dto.ts b/apps/selfhosted/src/analytics/dto/pageviews.dto.ts
index a8ea10615..deb809921 100644
--- a/apps/selfhosted/src/analytics/dto/pageviews.dto.ts
+++ b/apps/selfhosted/src/analytics/dto/pageviews.dto.ts
@@ -1,5 +1,12 @@
import { ApiProperty } from '@nestjs/swagger'
-import { IsNotEmpty } from 'class-validator'
+import { IsNotEmpty, IsObject, IsOptional, Validate } from 'class-validator'
+
+import {
+ MAX_METADATA_KEYS,
+ MAX_METADATA_VALUE_LENGTH,
+ MetadataKeysQuantity,
+ MetadataSizeLimit,
+} from './events.dto'
export class PageviewsDTO {
@ApiProperty({
@@ -88,4 +95,21 @@ export class PageviewsDTO {
page_load: number
ttfb: number
}
+
+ @ApiProperty({
+ example: {
+ affiliate: 'Yes',
+ protocol: 'HTTPS',
+ },
+ description: 'Event-related metadata object with string values',
+ })
+ @IsOptional()
+ @IsObject()
+ @Validate(MetadataKeysQuantity, {
+ message: `Metadata object can't have more than ${MAX_METADATA_KEYS} keys`,
+ })
+ @Validate(MetadataSizeLimit, {
+ message: `Metadata object can't have values with total length more than ${MAX_METADATA_VALUE_LENGTH} characters`,
+ })
+ meta?: Record<string, string>
}
diff --git a/apps/selfhosted/src/analytics/interfaces/index.ts b/apps/selfhosted/src/analytics/interfaces/index.ts
index 770ada1ad..971910973 100644
--- a/apps/selfhosted/src/analytics/interfaces/index.ts
+++ b/apps/selfhosted/src/analytics/interfaces/index.ts
@@ -36,6 +36,12 @@ export interface CustomsCHResponse {
index: number
}
+export interface PropertiesCHResponse {
+ property: string
+ 'count()': number
+ index: number
+}
+
export interface CustomsCHAggregatedResponse {
year: number
month: number
@@ -171,4 +177,12 @@ export interface IPageflow {
created: string
}
+export interface IPageProperty {
+ [key: string]: number
+}
+
+export interface ICustomEvent {
+ [key: string]: number
+}
+
export type PerfMeasure = 'average' | 'median' | 'p95' | 'quantiles'
diff --git a/apps/selfhosted/src/common/constants.ts b/apps/selfhosted/src/common/constants.ts
index ae2f97e59..5077a9a61 100644
--- a/apps/selfhosted/src/common/constants.ts
+++ b/apps/selfhosted/src/common/constants.ts
@@ -112,6 +112,8 @@ const TRAFFIC_COLUMNS = [
const ALL_COLUMNS = [...TRAFFIC_COLUMNS, 'ev']
+const TRAFFIC_METAKEY_COLUMNS = ['tag:key', 'tag:value']
+
const PERFORMANCE_COLUMNS = ['cc', 'rg', 'ct', 'pg', 'dv', 'br']
const ERROR_COLUMNS = ['pg', 'dv', 'br', 'os', 'lc', 'cc', 'rg', 'ct']
@@ -128,6 +130,7 @@ export {
REDIS_LOG_DATA_CACHE_KEY,
REDIS_LOG_CAPTCHA_CACHE_KEY,
REDIS_LOG_CUSTOM_CACHE_KEY,
+ TRAFFIC_METAKEY_COLUMNS,
REDIS_SESSION_SALT_KEY,
HEARTBEAT_SID_LIFE_TIME,
UUIDV5_NAMESPACE,
diff --git a/migrations/clickhouse/2024_06_22.js b/migrations/clickhouse/2024_06_22.js
new file mode 100644
index 000000000..b3a25caed
--- /dev/null
+++ b/migrations/clickhouse/2024_06_22.js
@@ -0,0 +1,44 @@
+const { queriesRunner, dbName } = require('./setup')
+
+const queries = [
+ // Added 'meta' column
+ `DROP TABLE IF EXISTS ${dbName}.analytics_temp`,
+ `CREATE TABLE IF NOT EXISTS ${dbName}.analytics_temp
+ (
+ psid Nullable(UInt64),
+ sid Nullable(String),
+ pid FixedString(12),
+ pg Nullable(String),
+ prev Nullable(String),
+ dv LowCardinality(Nullable(String)),
+ br LowCardinality(Nullable(String)),
+ os LowCardinality(Nullable(String)),
+ lc LowCardinality(Nullable(String)),
+ ref Nullable(String),
+ so Nullable(String),
+ me Nullable(String),
+ ca Nullable(String),
+ cc Nullable(FixedString(2)),
+ rg LowCardinality(Nullable(String)),
+ ct Nullable(String),
+ meta Nested
+ (
+ key String,
+ value String
+ ),
+ sdur Nullable(UInt32),
+ unique UInt8,
+ created DateTime('UTC')
+ )
+ ENGINE = MergeTree()
+ PARTITION BY toYYYYMM(created)
+ ORDER BY (pid, created);`,
+
+ `INSERT INTO ${dbName}.analytics_temp (psid, sid, pid, pg, prev, dv, br, os, lc, ref, so, me, ca, cc, rg, ct, meta.key, meta.value, sdur, unique, created)
+ SELECT psid, sid, pid, pg, prev, dv, br, os, lc, ref, so, me, ca, cc, rg, ct, [], [], sdur, unique, created FROM ${dbName}.analytics`,
+
+ `DROP TABLE ${dbName}.analytics`,
+ `RENAME TABLE ${dbName}.analytics_temp TO ${dbName}.analytics`,
+]
+
+queriesRunner(queries)
diff --git a/migrations/clickhouse/initialise_database.js b/migrations/clickhouse/initialise_database.js
index c41928733..a48c80f22 100644
--- a/migrations/clickhouse/initialise_database.js
+++ b/migrations/clickhouse/initialise_database.js
@@ -23,6 +23,11 @@ const CLICKHOUSE_INIT_QUERIES = [
cc Nullable(FixedString(2)),
rg LowCardinality(Nullable(String)),
ct Nullable(String),
+ meta Nested
+ (
+ key String,
+ value String
+ ),
sdur Nullable(UInt32),
unique UInt8,
created DateTime('UTC')