Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

implemented custom events filtering

Blue Mouse202d4c73b96f
diff --git a/migrations/clickhouse/2023_02_09.js b/migrations/clickhouse/2023_02_09.js
new file mode 100644
index 000000000..bda1a4009
--- /dev/null
+++ b/migrations/clickhouse/2023_02_09.js
@@ -0,0 +1,34 @@
+const { queriesRunner, dbName } = require('./setup')
+
+const queries = [
+ `DROP TABLE IF EXISTS ${dbName}.customEV_temp`,
+
+ `CREATE TABLE IF NOT EXISTS ${dbName}.customEV_temp
+ (
+ pid FixedString(12),
+ ev String,
+ pg Nullable(String),
+ dv Nullable(String),
+ br Nullable(String),
+ os Nullable(String),
+ lc Nullable(String),
+ ref Nullable(String),
+ so Nullable(String),
+ me Nullable(String),
+ ca Nullable(String),
+ cc Nullable(FixedString(2)),
+ created DateTime
+ )
+ ENGINE = MergeTree()
+ PARTITION BY toYYYYMM(created)
+ ORDER BY (pid, created);`,
+
+ `INSERT INTO ${dbName}.customEV_temp (pid, ev, pg, dv, br, os, lc, ref, so, me, ca, cc, created)
+ SELECT pid, ev, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, created FROM ${dbName}.customEV`,
+
+ `DROP TABLE ${dbName}.customEV`,
+
+ `RENAME TABLE ${dbName}.customEV_temp TO ${dbName}.customEV`,
+]
+
+queriesRunner(queries)
diff --git a/src/analytics/analytics.controller.ts b/src/analytics/analytics.controller.ts
index 5e8c31ce2..a84b11a87 100644
--- a/src/analytics/analytics.controller.ts
+++ b/src/analytics/analytics.controller.ts
@@ -147,15 +147,35 @@ const performanceDTO = (
]
}
-const customLogDTO = (pid: string, ev: string): string => {
- const dto = {
- id: uuidv4(),
+const customLogDTO = (
+ pid: string,
+ ev: string,
+ pg: string,
+ dv: string,
+ br: string,
+ os: string,
+ lc: string,
+ ref: string,
+ so: string,
+ me: string,
+ ca: string,
+ cc: string,
+): Array<string | number> => {
+ return [
pid,
ev,
- created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
- }
-
- return JSON.stringify(dto)
+ pg,
+ dv,
+ br,
+ os,
+ lc,
+ ref,
+ so,
+ me,
+ ca,
+ cc,
+ dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
+ ]
}
const getElValue = el => {
@@ -240,9 +260,20 @@ export class AnalyticsController {
let groupFrom = from,
groupTo = to
- const queryCustoms =
+ let queryCustoms =
'SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev'
- const subQuery = `FROM analytics WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
+ let subQuery = `FROM analytics WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
+ let customEVFilterApplied = false
+
+ // @ts-ignore
+ if (filtersParams?.ev) {
+ customEVFilterApplied = true
+ // @ts-ignore
+ queryCustoms = `SELECT ev, count() FROM customEV WHERE ${filtersParams.ev_exclusive ? 'NOT' : ''} ev = {ev:String} AND pid = {pid:FixedString(12)} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`
+
+ // @ts-ignore
+ subQuery = `FROM customEV WHERE ${filtersParams.ev_exclusive ? 'NOT' : ''} ev = {ev:String} AND pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
+ }
const paramsData = {
params: {
@@ -347,6 +378,7 @@ export class AnalyticsController {
filtersQuery,
paramsData,
timezone,
+ customEVFilterApplied,
)
const customs = await this.analyticsService.processCustomEV(
@@ -651,10 +683,32 @@ export class AnalyticsController {
}
}
- const dto = customLogDTO(eventsDTO.pid, eventsDTO.ev)
+ const ua = UAParser(userAgent)
+ const dv = ua.device.type || 'desktop'
+ const br = ua.browser.name
+ const os = ua.os.name
+ // trying to get country from timezome, otherwise using CloudFlare's IP based country code as a fallback
+ const cc = ct.getCountryForTimezone(eventsDTO.tz)?.id || (headers['cf-ipcountry'] === 'XX' ? 'NULL' : headers['cf-ipcountry'])
+
+ const dto = customLogDTO(
+ eventsDTO.pid,
+ eventsDTO.ev,
+ eventsDTO.pg,
+ dv,
+ br,
+ os,
+ eventsDTO.lc,
+ eventsDTO.ref,
+ eventsDTO.so,
+ eventsDTO.me,
+ eventsDTO.ca,
+ cc,
+ )
try {
- await redis.rpush(REDIS_LOG_CUSTOM_CACHE_KEY, dto)
+ // todo: fix: may be vulnerable to sql injection attack
+ const values = `(${dto.map(getElValue).join(',')})`
+ await redis.rpush(REDIS_LOG_CUSTOM_CACHE_KEY, values)
return
} catch (e) {
this.logger.error(e)
diff --git a/src/analytics/analytics.service.ts b/src/analytics/analytics.service.ts
index caa59366b..0c7675a75 100644
--- a/src/analytics/analytics.service.ts
+++ b/src/analytics/analytics.service.ts
@@ -165,7 +165,7 @@ export const checkIfTBAllowed = (
@Injectable()
export class AnalyticsService {
- constructor(private readonly projectService: ProjectService) {}
+ constructor(private readonly projectService: ProjectService) { }
async getRedisProject(pid: string): Promise<Project | null> {
const pidKey = getRedisProjectKey(pid)
@@ -288,28 +288,27 @@ export class AnalyticsService {
'An incorrect event name (ev) is provided',
)
}
- } else {
- // the type is 'log'
- // 'tz' does not need validation as it's based on getCountryForTimezone detection
- // @ts-ignore
- const { lc } = logDTO
-
- // validate locale ('lc' param)
- if (!_isEmpty(lc)) {
- if (validator.isLocale(lc)) {
- // uppercase the locale after '-' char, so for example both 'en-gb' and 'en-GB' in result will be 'en-GB'
- const lcParted = _split(lc, '-')
-
- if (_size(lcParted) > 1) {
- lcParted[1] = _toUpper(lcParted[1])
- }
-
- // @ts-ignore
- logDTO.lc = _join(lcParted, '-')
- } else {
- // @ts-ignore
- logDTO.lc = 'NULL'
+ }
+
+ // 'tz' does not need validation as it's based on getCountryForTimezone detection
+ // @ts-ignore
+ const { lc } = logDTO
+
+ // validate locale ('lc' param)
+ if (!_isEmpty(lc)) {
+ if (validator.isLocale(lc)) {
+ // uppercase the locale after '-' char, so for example both 'en-gb' and 'en-GB' in result will be 'en-GB'
+ const lcParted = _split(lc, '-')
+
+ if (_size(lcParted) > 1) {
+ lcParted[1] = _toUpper(lcParted[1])
}
+
+ // @ts-ignore
+ logDTO.lc = _join(lcParted, '-')
+ } else {
+ // @ts-ignore
+ logDTO.lc = 'NULL'
}
}
@@ -410,6 +409,16 @@ export class AnalyticsService {
for (let i = 0; i < _size(parsed); ++i) {
const { column, filter, isExclusive } = parsed[i]
+ if (column === 'ev') {
+ params = {
+ ...params,
+ [column]: filter,
+ [`${column}_exclusive`]: isExclusive,
+ }
+
+ continue
+ }
+
if (!_includes(cols, column)) {
throw new UnprocessableEntityException(
`The provided filter (${column}) is not supported`,
@@ -520,6 +529,7 @@ export class AnalyticsService {
filtersQuery: string,
paramsData: object,
timezone: string,
+ customEVFilterApplied: boolean,
): Promise<object | void> {
const params = {}
@@ -542,9 +552,12 @@ export class AnalyticsService {
}
// Average session duration calculation
- const avgSdurQuery = `SELECT avg(sdur) ${subQuery} AND sdur IS NOT NULL`
- let avgSdur = await clickhouse.query(avgSdurQuery, paramsData).toPromise()
- avgSdur = _round(avgSdur[0]['avg(sdur)'])
+ let avgSdur = 0
+ if (!customEVFilterApplied) {
+ const avgSdurQuery = `SELECT avg(sdur) ${subQuery} AND sdur IS NOT NULL`
+ let avgSdur = await clickhouse.query(avgSdurQuery, paramsData).toPromise()
+ avgSdur = _round(avgSdur[0]['avg(sdur)'])
+ }
let groupDateIterator
const now = dayjs.utc().endOf(timeBucket)
@@ -577,14 +590,57 @@ export class AnalyticsService {
const xM = [...x, groupDateIterator.format('YYYY-MM-DD HH:mm:ss')]
let query = ''
+ if (customEVFilterApplied) {
+ for (let i = 0; i < _size(x); ++i) {
+ if (i > 0) {
+ query += ' UNION ALL '
+ }
+
+ // @ts-ignore
+ query += `select count() from customEV where ${paramsData.params.ev_exclusive ? 'NOT' : ''} ev = {ev:String} AND pid = {pid:FixedString(12)} and created between '${xM[i]
+ }' and '${xM[1 + i]}' ${filtersQuery}`
+ }
+
+ // @ts-ignore
+ const result: Array<chartCHResponse> = (
+ await clickhouse.query(query, paramsData).toPromise()
+ )
+ // @ts-ignore
+ .sort((a, b) => a.index - b.index)
+
+ const uniques = []
+ let sdur = []
+
+ for (let i = 0; i < _size(x); ++i) {
+ uniques[i] = result[i]['count()']
+ sdur[i] = 0
+ }
+
+ if (timezone !== DEFAULT_TIMEZONE && isValidTimezone(timezone)) {
+ x = _map(x, el =>
+ dayjs.utc(el).tz(timezone).format('YYYY-MM-DD HH:mm:ss'),
+ )
+ }
+
+ return Promise.resolve({
+ params,
+ chart: {
+ x,
+ visits: uniques,
+ uniques,
+ sdur,
+ },
+ avgSdur,
+ })
+ }
+
for (let i = 0; i < _size(x); ++i) {
if (i > 0) {
query += ' UNION ALL '
}
- query += `select ${i} index, unique, count(), avg(sdur) from analytics where pid = {pid:FixedString(12)} and created between '${
- xM[i]
- }' and '${xM[1 + i]}' ${filtersQuery} group by unique`
+ query += `select ${i} index, unique, count(), avg(sdur) from analytics where pid = {pid:FixedString(12)} and created between '${xM[i]
+ }' and '${xM[1 + i]}' ${filtersQuery} group by unique`
}
// @ts-ignore
@@ -724,9 +780,8 @@ export class AnalyticsService {
query += ' UNION ALL '
}
- query += `select ${i} index, avg(dns), avg(tls), avg(conn), avg(response), avg(render), avg(domLoad), avg(ttfb) from performance where pid = {pid:FixedString(12)} and created between '${
- xM[i]
- }' and '${xM[1 + i]}' ${filtersQuery} group by pid`
+ query += `select ${i} index, avg(dns), avg(tls), avg(conn), avg(response), avg(render), avg(domLoad), avg(ttfb) from performance where pid = {pid:FixedString(12)} and created between '${xM[i]
+ }' and '${xM[1 + i]}' ${filtersQuery} group by pid`
}
// @ts-ignore
diff --git a/src/analytics/dto/events.dto.ts b/src/analytics/dto/events.dto.ts
index 3c3fcdf50..7cec51ad4 100644
--- a/src/analytics/dto/events.dto.ts
+++ b/src/analytics/dto/events.dto.ts
@@ -23,4 +23,47 @@ export class EventsDTO {
'If true, only 1 event with the same ID will be saved per user session',
})
unique: object
+
+ // Tracking metrics
+ @ApiProperty({
+ example: 'Europe/Kiev',
+ description: "User's timezone",
+ })
+ tz?: string
+
+ @ApiProperty({
+ example: '/articles/my-awesome-article-1',
+ description: 'A page that user sent data from',
+ })
+ pg?: string
+
+ @ApiProperty({
+ example: 'en-GB',
+ description: "User's locale",
+ })
+ lc?: string
+
+ @ApiProperty({
+ example: 'https://example.com',
+ description: 'The referrer',
+ })
+ ref?: string
+
+ @ApiProperty({
+ example: 'duckduckgo',
+ description: 'utm_source URL parameter',
+ })
+ so?: string
+
+ @ApiProperty({
+ example: 'cpc',
+ description: 'utm_medium URL parameter',
+ })
+ me?: string
+
+ @ApiProperty({
+ example: 'spring_sale',
+ description: 'utm_campaign URL parameter',
+ })
+ ca?: string
}
diff --git a/src/common/constants.ts b/src/common/constants.ts
index 892f8dba4..c67444c89 100644
--- a/src/common/constants.ts
+++ b/src/common/constants.ts
@@ -43,6 +43,8 @@ const isNewRelicEnabled = Boolean(process.env.USE_NEW_RELIC)
const CLICKHOUSE_INIT_QUERIES = [
'CREATE DATABASE IF NOT EXISTS analytics',
+
+ // The traffic data table
`CREATE TABLE IF NOT EXISTS analytics.analytics
(
sid Nullable(String),
@@ -64,16 +66,29 @@ const CLICKHOUSE_INIT_QUERIES = [
ENGINE = MergeTree()
PARTITION BY toYYYYMM(created)
ORDER BY (pid, created);`,
+
+ // Custom events table
`CREATE TABLE IF NOT EXISTS analytics.customEV
(
- id UUID,
pid FixedString(12),
ev String,
+ pg Nullable(String),
+ dv Nullable(String),
+ br Nullable(String),
+ os Nullable(String),
+ lc Nullable(String),
+ ref Nullable(String),
+ so Nullable(String),
+ me Nullable(String),
+ ca Nullable(String),
+ cc Nullable(FixedString(2)),
created DateTime
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(created)
- ORDER BY (id, created, pid);`,
+ ORDER BY (pid, created);`,
+
+ // The performance data table
`CREATE TABLE IF NOT EXISTS analytics.performance
(
pid FixedString(12),
@@ -94,6 +109,8 @@ const CLICKHOUSE_INIT_QUERIES = [
ENGINE = MergeTree()
PARTITION BY toYYYYMM(created)
ORDER BY (pid, created);`,
+
+ // Project data table (used for self-hosted only)
isSelfhosted &&
`CREATE TABLE IF NOT EXISTS analytics.project
(
@@ -149,7 +166,7 @@ const getRedisUserCountKey = (uid: string) => `user_c_${uid}`
const REDIS_LOG_DATA_CACHE_KEY = 'log_cache'
const REDIS_LOG_PERF_CACHE_KEY = 'perf_cache'
-const REDIS_LOG_CUSTOM_CACHE_KEY = 'log_custom_cache'
+const REDIS_LOG_CUSTOM_CACHE_KEY = 'log_custom_cache_v2'
const REDIS_SESSION_SALT_KEY = 'log_salt' // is updated every 24 hours
const REDIS_USERS_COUNT_KEY = 'stats:users_count'
const REDIS_PROJECTS_COUNT_KEY = 'stats:projects_count'
diff --git a/src/task-manager/task-manager.service.ts b/src/task-manager/task-manager.service.ts
index c79d83523..fb01b3416 100644
--- a/src/task-manager/task-manager.service.ts
+++ b/src/task-manager/task-manager.service.ts
@@ -106,11 +106,10 @@ export class TaskManagerService {
if (!_isEmpty(customData)) {
await redis.del(REDIS_LOG_CUSTOM_CACHE_KEY)
+ const query = `INSERT INTO customEV (*) VALUES ${_join(customData, ',')}`
try {
- const parsed = _map(customData, JSON.parse)
- const query = `INSERT INTO customEV (id, pid, ev, created)`
- await clickhouse.query(query, parsed).toPromise()
+ await clickhouse.query(query).toPromise()
} catch (e) {
console.error(
`[CRON WORKER] Error whilst saving custom events data: ${e}`,