📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
implemented custom events filtering
Blue Mouse202d4c73b96f
diff --git a/migrations/clickhouse/2023_02_09.js b/migrations/clickhouse/2023_02_09.jsnew file mode 100644index 000000000..bda1a4009--- /dev/null+++ b/migrations/clickhouse/2023_02_09.js@@ -0,0 +1,34 @@+const { queriesRunner, dbName } = require('./setup')++const queries = [+ `DROP TABLE IF EXISTS ${dbName}.customEV_temp`,++ `CREATE TABLE IF NOT EXISTS ${dbName}.customEV_temp+ (+ pid FixedString(12),+ ev String,+ pg Nullable(String),+ dv Nullable(String),+ br Nullable(String),+ os Nullable(String),+ lc Nullable(String),+ ref Nullable(String),+ so Nullable(String),+ me Nullable(String),+ ca Nullable(String),+ cc Nullable(FixedString(2)),+ created DateTime+ )+ ENGINE = MergeTree()+ PARTITION BY toYYYYMM(created)+ ORDER BY (pid, created);`,++ `INSERT INTO ${dbName}.customEV_temp (pid, ev, pg, dv, br, os, lc, ref, so, me, ca, cc, created)+ SELECT pid, ev, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, created FROM ${dbName}.customEV`,++ `DROP TABLE ${dbName}.customEV`,++ `RENAME TABLE ${dbName}.customEV_temp TO ${dbName}.customEV`,+]++queriesRunner(queries)diff --git a/src/analytics/analytics.controller.ts b/src/analytics/analytics.controller.tsindex 5e8c31ce2..a84b11a87 100644--- a/src/analytics/analytics.controller.ts+++ b/src/analytics/analytics.controller.ts@@ -147,15 +147,35 @@ const performanceDTO = (]}-const customLogDTO = (pid: string, ev: string): string => {- const dto = {- id: uuidv4(),+const customLogDTO = (+ pid: string,+ ev: string,+ pg: string,+ dv: string,+ br: string,+ os: string,+ lc: string,+ ref: string,+ so: string,+ me: string,+ ca: string,+ cc: string,+): Array<string | number> => {+ return [pid,ev,- created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),- }-- return JSON.stringify(dto)+ pg,+ dv,+ br,+ os,+ lc,+ ref,+ so,+ me,+ ca,+ cc,+ dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),+ ]}const getElValue = el => {@@ -240,9 +260,20 @@ export class AnalyticsController {let groupFrom = from,groupTo = to- const queryCustoms =+ let queryCustoms ='SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev'- const subQuery = `FROM analytics WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`+ let subQuery = `FROM analytics WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`+ let customEVFilterApplied = false++ // @ts-ignore+ if (filtersParams?.ev) {+ customEVFilterApplied = true+ // @ts-ignore+ queryCustoms = `SELECT ev, count() FROM customEV WHERE ${filtersParams.ev_exclusive ? 'NOT' : ''} ev = {ev:String} AND pid = {pid:FixedString(12)} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`++ // @ts-ignore+ subQuery = `FROM customEV WHERE ${filtersParams.ev_exclusive ? 'NOT' : ''} ev = {ev:String} AND pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`+ }const paramsData = {params: {@@ -347,6 +378,7 @@ export class AnalyticsController {filtersQuery,paramsData,timezone,+ customEVFilterApplied,)const customs = await this.analyticsService.processCustomEV(@@ -651,10 +683,32 @@ export class AnalyticsController {}}- const dto = customLogDTO(eventsDTO.pid, eventsDTO.ev)+ const ua = UAParser(userAgent)+ const dv = ua.device.type || 'desktop'+ const br = ua.browser.name+ const os = ua.os.name+ // trying to get country from timezome, otherwise using CloudFlare's IP based country code as a fallback+ const cc = ct.getCountryForTimezone(eventsDTO.tz)?.id || (headers['cf-ipcountry'] === 'XX' ? 'NULL' : headers['cf-ipcountry'])++ const dto = customLogDTO(+ eventsDTO.pid,+ eventsDTO.ev,+ eventsDTO.pg,+ dv,+ br,+ os,+ eventsDTO.lc,+ eventsDTO.ref,+ eventsDTO.so,+ eventsDTO.me,+ eventsDTO.ca,+ cc,+ )try {- await redis.rpush(REDIS_LOG_CUSTOM_CACHE_KEY, dto)+ // todo: fix: may be vulnerable to sql injection attack+ const values = `(${dto.map(getElValue).join(',')})`+ await redis.rpush(REDIS_LOG_CUSTOM_CACHE_KEY, values)return} catch (e) {this.logger.error(e)diff --git a/src/analytics/analytics.service.ts b/src/analytics/analytics.service.tsindex caa59366b..0c7675a75 100644--- a/src/analytics/analytics.service.ts+++ b/src/analytics/analytics.service.ts@@ -165,7 +165,7 @@ export const checkIfTBAllowed = (@Injectable()export class AnalyticsService {- constructor(private readonly projectService: ProjectService) {}+ constructor(private readonly projectService: ProjectService) { }async getRedisProject(pid: string): Promise<Project | null> {const pidKey = getRedisProjectKey(pid)@@ -288,28 +288,27 @@ export class AnalyticsService {'An incorrect event name (ev) is provided',)}- } else {- // the type is 'log'- // 'tz' does not need validation as it's based on getCountryForTimezone detection- // @ts-ignore- const { lc } = logDTO-- // validate locale ('lc' param)- if (!_isEmpty(lc)) {- if (validator.isLocale(lc)) {- // uppercase the locale after '-' char, so for example both 'en-gb' and 'en-GB' in result will be 'en-GB'- const lcParted = _split(lc, '-')-- if (_size(lcParted) > 1) {- lcParted[1] = _toUpper(lcParted[1])- }-- // @ts-ignore- logDTO.lc = _join(lcParted, '-')- } else {- // @ts-ignore- logDTO.lc = 'NULL'+ }++ // 'tz' does not need validation as it's based on getCountryForTimezone detection+ // @ts-ignore+ const { lc } = logDTO++ // validate locale ('lc' param)+ if (!_isEmpty(lc)) {+ if (validator.isLocale(lc)) {+ // uppercase the locale after '-' char, so for example both 'en-gb' and 'en-GB' in result will be 'en-GB'+ const lcParted = _split(lc, '-')++ if (_size(lcParted) > 1) {+ lcParted[1] = _toUpper(lcParted[1])}++ // @ts-ignore+ logDTO.lc = _join(lcParted, '-')+ } else {+ // @ts-ignore+ logDTO.lc = 'NULL'}}@@ -410,6 +409,16 @@ export class AnalyticsService {for (let i = 0; i < _size(parsed); ++i) {const { column, filter, isExclusive } = parsed[i]+ if (column === 'ev') {+ params = {+ ...params,+ [column]: filter,+ [`${column}_exclusive`]: isExclusive,+ }++ continue+ }+if (!_includes(cols, column)) {throw new UnprocessableEntityException(`The provided filter (${column}) is not supported`,@@ -520,6 +529,7 @@ export class AnalyticsService {filtersQuery: string,paramsData: object,timezone: string,+ customEVFilterApplied: boolean,): Promise<object | void> {const params = {}@@ -542,9 +552,12 @@ export class AnalyticsService {}// Average session duration calculation- const avgSdurQuery = `SELECT avg(sdur) ${subQuery} AND sdur IS NOT NULL`- let avgSdur = await clickhouse.query(avgSdurQuery, paramsData).toPromise()- avgSdur = _round(avgSdur[0]['avg(sdur)'])+ let avgSdur = 0+ if (!customEVFilterApplied) {+ const avgSdurQuery = `SELECT avg(sdur) ${subQuery} AND sdur IS NOT NULL`+ let avgSdur = await clickhouse.query(avgSdurQuery, paramsData).toPromise()+ avgSdur = _round(avgSdur[0]['avg(sdur)'])+ }let groupDateIteratorconst now = dayjs.utc().endOf(timeBucket)@@ -577,14 +590,57 @@ export class AnalyticsService {const xM = [...x, groupDateIterator.format('YYYY-MM-DD HH:mm:ss')]let query = ''+ if (customEVFilterApplied) {+ for (let i = 0; i < _size(x); ++i) {+ if (i > 0) {+ query += ' UNION ALL '+ }++ // @ts-ignore+ query += `select count() from customEV where ${paramsData.params.ev_exclusive ? 'NOT' : ''} ev = {ev:String} AND pid = {pid:FixedString(12)} and created between '${xM[i]+ }' and '${xM[1 + i]}' ${filtersQuery}`+ }++ // @ts-ignore+ const result: Array<chartCHResponse> = (+ await clickhouse.query(query, paramsData).toPromise()+ )+ // @ts-ignore+ .sort((a, b) => a.index - b.index)++ const uniques = []+ let sdur = []++ for (let i = 0; i < _size(x); ++i) {+ uniques[i] = result[i]['count()']+ sdur[i] = 0+ }++ if (timezone !== DEFAULT_TIMEZONE && isValidTimezone(timezone)) {+ x = _map(x, el =>+ dayjs.utc(el).tz(timezone).format('YYYY-MM-DD HH:mm:ss'),+ )+ }++ return Promise.resolve({+ params,+ chart: {+ x,+ visits: uniques,+ uniques,+ sdur,+ },+ avgSdur,+ })+ }+for (let i = 0; i < _size(x); ++i) {if (i > 0) {query += ' UNION ALL '}- query += `select ${i} index, unique, count(), avg(sdur) from analytics where pid = {pid:FixedString(12)} and created between '${- xM[i]- }' and '${xM[1 + i]}' ${filtersQuery} group by unique`+ query += `select ${i} index, unique, count(), avg(sdur) from analytics where pid = {pid:FixedString(12)} and created between '${xM[i]+ }' and '${xM[1 + i]}' ${filtersQuery} group by unique`}// @ts-ignore@@ -724,9 +780,8 @@ export class AnalyticsService {query += ' UNION ALL '}- query += `select ${i} index, avg(dns), avg(tls), avg(conn), avg(response), avg(render), avg(domLoad), avg(ttfb) from performance where pid = {pid:FixedString(12)} and created between '${- xM[i]- }' and '${xM[1 + i]}' ${filtersQuery} group by pid`+ query += `select ${i} index, avg(dns), avg(tls), avg(conn), avg(response), avg(render), avg(domLoad), avg(ttfb) from performance where pid = {pid:FixedString(12)} and created between '${xM[i]+ }' and '${xM[1 + i]}' ${filtersQuery} group by pid`}// @ts-ignorediff --git a/src/analytics/dto/events.dto.ts b/src/analytics/dto/events.dto.tsindex 3c3fcdf50..7cec51ad4 100644--- a/src/analytics/dto/events.dto.ts+++ b/src/analytics/dto/events.dto.ts@@ -23,4 +23,47 @@ export class EventsDTO {'If true, only 1 event with the same ID will be saved per user session',})unique: object++ // Tracking metrics+ @ApiProperty({+ example: 'Europe/Kiev',+ description: "User's timezone",+ })+ tz?: string++ @ApiProperty({+ example: '/articles/my-awesome-article-1',+ description: 'A page that user sent data from',+ })+ pg?: string++ @ApiProperty({+ example: 'en-GB',+ description: "User's locale",+ })+ lc?: string++ @ApiProperty({+ example: 'https://example.com',+ description: 'The referrer',+ })+ ref?: string++ @ApiProperty({+ example: 'duckduckgo',+ description: 'utm_source URL parameter',+ })+ so?: string++ @ApiProperty({+ example: 'cpc',+ description: 'utm_medium URL parameter',+ })+ me?: string++ @ApiProperty({+ example: 'spring_sale',+ description: 'utm_campaign URL parameter',+ })+ ca?: string}diff --git a/src/common/constants.ts b/src/common/constants.tsindex 892f8dba4..c67444c89 100644--- a/src/common/constants.ts+++ b/src/common/constants.ts@@ -43,6 +43,8 @@ const isNewRelicEnabled = Boolean(process.env.USE_NEW_RELIC)const CLICKHOUSE_INIT_QUERIES = ['CREATE DATABASE IF NOT EXISTS analytics',++ // The traffic data table`CREATE TABLE IF NOT EXISTS analytics.analytics(sid Nullable(String),@@ -64,16 +66,29 @@ const CLICKHOUSE_INIT_QUERIES = [ENGINE = MergeTree()PARTITION BY toYYYYMM(created)ORDER BY (pid, created);`,++ // Custom events table`CREATE TABLE IF NOT EXISTS analytics.customEV(- id UUID,pid FixedString(12),ev String,+ pg Nullable(String),+ dv Nullable(String),+ br Nullable(String),+ os Nullable(String),+ lc Nullable(String),+ ref Nullable(String),+ so Nullable(String),+ me Nullable(String),+ ca Nullable(String),+ cc Nullable(FixedString(2)),created DateTime)ENGINE = MergeTree()PARTITION BY toYYYYMM(created)- ORDER BY (id, created, pid);`,+ ORDER BY (pid, created);`,++ // The performance data table`CREATE TABLE IF NOT EXISTS analytics.performance(pid FixedString(12),@@ -94,6 +109,8 @@ const CLICKHOUSE_INIT_QUERIES = [ENGINE = MergeTree()PARTITION BY toYYYYMM(created)ORDER BY (pid, created);`,++ // Project data table (used for self-hosted only)isSelfhosted &&`CREATE TABLE IF NOT EXISTS analytics.project(@@ -149,7 +166,7 @@ const getRedisUserCountKey = (uid: string) => `user_c_${uid}`const REDIS_LOG_DATA_CACHE_KEY = 'log_cache'const REDIS_LOG_PERF_CACHE_KEY = 'perf_cache'-const REDIS_LOG_CUSTOM_CACHE_KEY = 'log_custom_cache'+const REDIS_LOG_CUSTOM_CACHE_KEY = 'log_custom_cache_v2'const REDIS_SESSION_SALT_KEY = 'log_salt' // is updated every 24 hoursconst REDIS_USERS_COUNT_KEY = 'stats:users_count'const REDIS_PROJECTS_COUNT_KEY = 'stats:projects_count'diff --git a/src/task-manager/task-manager.service.ts b/src/task-manager/task-manager.service.tsindex c79d83523..fb01b3416 100644--- a/src/task-manager/task-manager.service.ts+++ b/src/task-manager/task-manager.service.ts@@ -106,11 +106,10 @@ export class TaskManagerService {if (!_isEmpty(customData)) {await redis.del(REDIS_LOG_CUSTOM_CACHE_KEY)+ const query = `INSERT INTO customEV (*) VALUES ${_join(customData, ',')}`try {- const parsed = _map(customData, JSON.parse)- const query = `INSERT INTO customEV (id, pid, ev, created)`- await clickhouse.query(query, parsed).toPromise()+ await clickhouse.query(query).toPromise()} catch (e) {console.error(`[CRON WORKER] Error whilst saving custom events data: ${e}`,