📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
dropped redis for analytics data queue and instead relying on clickhouse async inserts
Blue Mouse956d3ecb7a1c
diff --git a/apps/production/src/analytics/analytics.controller.ts b/apps/production/src/analytics/analytics.controller.tsindex 9bc6072d6..b3afbc5b4 100644--- a/apps/production/src/analytics/analytics.controller.ts+++ b/apps/production/src/analytics/analytics.controller.ts@@ -3,11 +3,8 @@ import * as _isArray from 'lodash/isArray'import * as _toNumber from 'lodash/toNumber'import * as _pick from 'lodash/pick'import * as _includes from 'lodash/includes'-import * as _keys from 'lodash/keys'-import * as _values from 'lodash/values'import * as _map from 'lodash/map'import * as _uniqBy from 'lodash/uniqBy'-import * as _round from 'lodash/round'import * as dayjs from 'dayjs'import * as utc from 'dayjs/plugin/utc'import * as dayjsTimezone from 'dayjs/plugin/timezone'@@ -57,16 +54,12 @@ import { GetUserFlowDTO } from './dto/getUserFlow.dto'import { GetFunnelsDTO } from './dto/getFunnels.dto'import { AppLoggerService } from '../logger/logger.service'import {- REDIS_LOG_DATA_CACHE_KEY,- REDIS_LOG_PERF_CACHE_KEY,redis,- REDIS_LOG_CUSTOM_CACHE_KEY,HEARTBEAT_SID_LIFE_TIME,REDIS_USERS_COUNT_KEY,REDIS_PROJECTS_COUNT_KEY,REDIS_EVENTS_COUNT_KEY,REDIS_SESSION_SALT_KEY,- REDIS_LOG_ERROR_CACHE_KEY,} from '../common/constants'import { clickhouse } from '../common/integrations/clickhouse'import {@@ -91,11 +84,13 @@ import { ErrorDTO } from './dto/error.dto'import { GetErrorsDto } from './dto/get-errors.dto'import { GetErrorDTO } from './dto/get-error.dto'import { PatchStatusDTO } from './dto/patch-status.dto'-import { ProjectService } from '../project/project.service'import { ProjectsViewsRepository } from '../project/repositories/projects-views.repository'--// eslint-disable-next-line @typescript-eslint/no-var-requires-const mysql = require('mysql2')+import {+ customEventTransformer,+ errorEventTransformer,+ performanceTransformer,+ trafficTransformer,+} from './utils/transformers'dayjs.extend(utc)dayjs.extend(dayjsTimezone)@@ -110,177 +105,6 @@ const getSessionKeyCustom = (salt = '',) => `cses_${hash(`${ua}${ip}${pid}${ev}${salt}`).toString('hex')}`-const analyticsDTO = (- psid: string,- sid: string,- pid: string,- pg: string,- prev: string,- dv: string,- br: string,- os: string,- lc: string,- ref: string,- so: string,- me: string,- ca: string,- cc: string,- rg: string,- ct: string,- keys: string[],- values: string[],- sdur: number | string,- unique: number,-): Array<string | number | string[]> => {- return [- psid,- sid,- pid,- pg,- prev,- dv,- br,- os,- lc,- ref,- so,- me,- ca,- cc,- rg,- ct,- keys,- values,- sdur,- unique,- dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),- ]-}--const performanceDTO = (- pid: string,- pg: string,- dv: string,- br: string,- cc: string,- rg: string,- ct: string,- dns: number,- tls: number,- conn: number,- response: number,- render: number,- domLoad: number,- pageLoad: number,- ttfb: number,-): Array<string | number> => {- return [- pid,- pg,- dv,- br,- cc,- rg,- ct,- _round(dns),- _round(tls),- _round(conn),- _round(response),- _round(render),- _round(domLoad),- _round(pageLoad),- _round(ttfb),- dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),- ]-}--const customLogDTO = (- psid: string,- pid: string,- ev: string,- pg: string,- dv: string,- br: string,- os: string,- lc: string,- ref: string,- so: string,- me: string,- ca: string,- cc: string,- rg: string,- ct: string,- keys: string[],- values: string[],-): Array<string | number | string[]> => {- return [- psid,- pid,- ev,- pg,- dv,- br,- os,- lc,- ref,- so,- me,- ca,- cc,- rg,- ct,- keys,- values,- dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),- ]-}--const errorLogDTO = (- eid: string,- pid: string,- pg: string,- dv: string,- br: string,- os: string,- lc: string,- cc: string,- rg: string,- ct: string,- name: string,- message: string,- lineno: number,- colno: number,- filename: string,-): Array<string | number | string[]> => {- return [- eid,- pid,- pg,- dv,- br,- os,- lc,- cc,- rg,- ct,- name,- message,- lineno,- colno,- filename,- dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),- ]-}--export const getElValue = el => {- if (_isArray(el)) {- return `[${_map(el, getElValue).join(',')}]`- }-- if (el === undefined || el === null || el === 'NULL') return 'NULL'- return mysql.escape(el)-}-// Performance object validator: none of the values cannot be bigger than 1000 * 60 * 5 (5 minutes) and are >= 0const MAX_PERFORMANCE_VALUE = 1000 * 60 * 5const isPerformanceValid = (perf: any) => {@@ -370,7 +194,6 @@ export class AnalyticsController {constructor(private readonly analyticsService: AnalyticsService,private readonly logger: AppLoggerService,- private readonly projectService: ProjectService,private readonly projectsViewsRepository: ProjectsViewsRepository,) {}@@ -1285,7 +1108,7 @@ export class AnalyticsController {const { name, message, lineno, colno, filename } = errorDTO- const dto = errorLogDTO(+ const transformed = errorEventTransformer(this.analyticsService.getErrorID(errorDTO),errorDTO.pid,errorDTO.pg,@@ -1304,8 +1127,14 @@ export class AnalyticsController {)try {- const values = `(${dto.map(getElValue).join(',')})`- await redis.rpush(REDIS_LOG_ERROR_CACHE_KEY, values)+ await clickhouse.insert({+ table: 'errors',+ format: 'JSONEachRow',+ values: [transformed],+ clickhouse_settings: {+ async_insert: 1,+ },+ })} catch (e) {this.logger.error(e)throw new InternalServerErrorException(@@ -1389,7 +1218,7 @@ export class AnalyticsController {const sessionHash = getSessionKey(ip, userAgent, eventsDTO.pid, salt)const [, psid] = await this.analyticsService.isUnique(sessionHash)- const dto = customLogDTO(+ const transformed = customEventTransformer(psid,eventsDTO.pid,eventsDTO.ev,@@ -1405,13 +1234,18 @@ export class AnalyticsController {country,region,city,- _keys(eventsDTO.meta),- _values(eventsDTO.meta),+ eventsDTO.meta,)try {- const values = `(${dto.map(getElValue).join(',')})`- await redis.rpush(REDIS_LOG_CUSTOM_CACHE_KEY, values)+ await clickhouse.insert({+ table: 'customEV',+ format: 'JSONEachRow',+ values: [transformed],+ clickhouse_settings: {+ async_insert: 1,+ },+ })} catch (e) {this.logger.error(e)throw new InternalServerErrorException(@@ -1476,17 +1310,13 @@ export class AnalyticsController {)}- const {- city = 'NULL',- region = 'NULL',- country = 'NULL',- } = getGeoDetails(ip, logDTO.tz)+ const { city, region, country } = getGeoDetails(ip, logDTO.tz)const ua = UAParser(userAgent)const dv = ua.device.type || 'desktop'const br = ua.browser.nameconst os = ua.os.name- const dto = analyticsDTO(+ const transformed = trafficTransformer(psid,sessionHash,logDTO.pid,@@ -1503,13 +1333,12 @@ export class AnalyticsController {country,region,city,- _keys(logDTO.meta),- _values(logDTO.meta),+ logDTO.meta,0,Number(unique),)- let perfDTO: Array<string | number> = []+ let perfTransformed = nullif (!_isEmpty(logDTO.perf) && isPerformanceValid(logDTO.perf)) {const {@@ -1523,7 +1352,7 @@ export class AnalyticsController {ttfb,} = logDTO.perf- perfDTO = performanceDTO(+ perfTransformed = performanceTransformer(logDTO.pid,logDTO.pg,dv,@@ -1542,14 +1371,25 @@ export class AnalyticsController {)}- const values = `(${dto.map(getElValue).join(',')})`-try {- await redis.rpush(REDIS_LOG_DATA_CACHE_KEY, values)-- if (!_isEmpty(perfDTO)) {- const perfValues = `(${perfDTO.map(getElValue).join(',')})`- await redis.rpush(REDIS_LOG_PERF_CACHE_KEY, perfValues)+ await clickhouse.insert({+ table: 'analytics',+ format: 'JSONEachRow',+ values: [transformed],+ clickhouse_settings: {+ async_insert: 1,+ },+ })++ if (!_isEmpty(perfTransformed)) {+ await clickhouse.insert({+ table: 'performance',+ format: 'JSONEachRow',+ values: [perfTransformed],+ clickhouse_settings: {+ async_insert: 1,+ },+ })}} catch (e) {this.logger.error(e)@@ -1602,32 +1442,37 @@ export class AnalyticsController {const br = ua.browser.nameconst os = ua.os.name- const dto = analyticsDTO(+ const transformed = trafficTransformer(psid,sessionHash,logDTO.pid,- 'NULL',- 'NULL',+ null,+ null,dv,br,os,- 'NULL',- 'NULL',- 'NULL',- 'NULL',- 'NULL',+ null,+ null,+ null,+ null,+ null,country,region,city,- [],- [],+ null,0,Number(unique),)- const values = `(${dto.map(getElValue).join(',')})`try {- await redis.rpush(REDIS_LOG_DATA_CACHE_KEY, values)+ await clickhouse.insert({+ table: 'analytics',+ format: 'JSONEachRow',+ values: [transformed],+ clickhouse_settings: {+ async_insert: 1,+ },+ })} catch (e) {this.logger.error(e)}diff --git a/apps/production/src/analytics/utils/transformers.ts b/apps/production/src/analytics/utils/transformers.tsnew file mode 100644index 000000000..1ab11b888--- /dev/null+++ b/apps/production/src/analytics/utils/transformers.ts@@ -0,0 +1,182 @@+import * as _round from 'lodash/round'+import * as _isEmpty from 'lodash/isEmpty'+import * as _keys from 'lodash/keys'+import * as _values from 'lodash/values'+import * as dayjs from 'dayjs'+import * as utc from 'dayjs/plugin/utc'++dayjs.extend(utc)++const processMetaKV = (+ rawMeta: Record<string, string>,+): { 'meta.key': string | null; 'meta.value': string | null } => {+ if (!rawMeta || _isEmpty(rawMeta)) {+ return {+ 'meta.key': null,+ 'meta.value': null,+ }+ }++ return {+ 'meta.key': _keys(rawMeta),+ 'meta.value': _values(rawMeta),+ }+}++export const trafficTransformer = (+ psid: string,+ sid: string,+ pid: string,+ pg: string | null,+ prev: string | null,+ dv: string | null,+ br: string | null,+ os: string | null,+ lc: string | null,+ ref: string | null,+ so: string | null,+ me: string | null,+ ca: string | null,+ cc: string | null,+ rg: string | null,+ ct: string | null,+ meta: Record<string, string> | null,+ sdur: number,+ unique: number,+) => {+ return {+ psid,+ sid,+ pid,+ pg: pg || null,+ prev: prev || null,+ dv: dv || null,+ br: br || null,+ os: os || null,+ lc: lc || null,+ ref: ref || null,+ so: so || null,+ me: me || null,+ ca: ca || null,+ cc: cc || null,+ rg: rg || null,+ ct: ct || null,+ ...processMetaKV(meta),+ sdur,+ unique,+ created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),+ }+}++export const customEventTransformer = (+ psid: string,+ pid: string,+ ev: string,+ pg: string | null,+ dv: string | null,+ br: string | null,+ os: string | null,+ lc: string | null,+ ref: string | null,+ so: string | null,+ me: string | null,+ ca: string | null,+ cc: string | null,+ rg: string | null,+ ct: string | null,+ meta: Record<string, string> | null,+) => {+ return {+ psid,+ pid,+ ev,+ pg: pg || null,+ dv: dv || null,+ br: br || null,+ os: os || null,+ lc: lc || null,+ ref: ref || null,+ so: so || null,+ me: me || null,+ ca: ca || null,+ cc: cc || null,+ rg: rg || null,+ ct: ct || null,+ ...processMetaKV(meta),+ created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),+ }+}++export const errorEventTransformer = (+ eid: string,+ pid: string,+ pg: string | null,+ dv: string | null,+ br: string | null,+ os: string | null,+ lc: string | null,+ cc: string | null,+ rg: string | null,+ ct: string | null,+ name: string | null,+ message: string | null,+ lineno: number | null,+ colno: number | null,+ filename: string | null,+) => {+ return {+ eid,+ pid,+ pg: pg || null,+ dv: dv || null,+ br: br || null,+ os: os || null,+ lc: lc || null,+ cc: cc || null,+ rg: rg || null,+ ct: ct || null,+ name: name || null,+ message: message || null,+ lineno: lineno || null,+ colno: colno || null,+ filename: filename || null,+ created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),+ }+}++export const performanceTransformer = (+ pid: string,+ pg: string | null,+ dv: string | null,+ br: string | null,+ cc: string | null,+ rg: string | null,+ ct: string | null,+ dns: number,+ tls: number,+ conn: number,+ response: number,+ render: number,+ domLoad: number,+ pageLoad: number,+ ttfb: number,+) => {+ return {+ pid,+ pg: pg || null,+ dv: dv || null,+ br: br || null,+ cc: cc || null,+ rg: rg || null,+ ct: ct || null,+ dns: _round(dns),+ tls: _round(tls),+ conn: _round(conn),+ response: _round(response),+ render: _round(render),+ domLoad: _round(domLoad),+ pageLoad: _round(pageLoad),+ ttfb: _round(ttfb),+ created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),+ }+}diff --git a/apps/production/src/captcha/captcha.controller.ts b/apps/production/src/captcha/captcha.controller.tsindex 1788d15b6..e93460360 100644--- a/apps/production/src/captcha/captcha.controller.ts+++ b/apps/production/src/captcha/captcha.controller.ts@@ -89,13 +89,7 @@ export class CaptchaController {const ip = getIPFromHeaders(headers) || reqIP || ''- await this.captchaService.logCaptchaPass(- pid,- userAgent,- timestamp,- true,- ip,- )+ await this.captchaService.logCaptchaPass(pid, userAgent, timestamp, ip)return {success: true,diff --git a/apps/production/src/captcha/captcha.service.ts b/apps/production/src/captcha/captcha.service.tsindex b781c6a5d..8b156cce0 100644--- a/apps/production/src/captcha/captcha.service.ts+++ b/apps/production/src/captcha/captcha.service.ts@@ -4,7 +4,6 @@ import * as svgCaptcha from 'svg-captcha'import * as CryptoJS from 'crypto-js'import * as _toLower from 'lodash/toLower'import * as UAParser from 'ua-parser-js'-import * as _map from 'lodash/map'import * as _values from 'lodash/values'import * as _includes from 'lodash/includes'import * as dayjs from 'dayjs'@@ -14,15 +13,15 @@ import { ProjectService } from '../project/project.service'import { AppLoggerService } from '../logger/logger.service'import {redis,- REDIS_LOG_CAPTCHA_CACHE_KEY,isValidPID,getRedisCaptchaKey,CAPTCHA_SALT,CAPTCHA_TOKEN_LIFETIME,} from '../common/constants'import { getGeoDetails } from '../common/utils'-import { getElValue } from '../analytics/analytics.controller'import { GeneratedCaptcha } from './interfaces/generated-captcha'+import { captchaTransformer } from './utils/transformers'+import { clickhouse } from '../common/integrations/clickhouse'dayjs.extend(utc)@@ -65,26 +64,6 @@ const isTokenAlreadyUsed = async (token: string): Promise<boolean> => {return false}-const captchaDTO = (- pid: string,- dv: string,- br: string,- os: string,- cc: string,- isManual: boolean,- timestamp: number,-): Array<string | number> => {- return [- pid,- dv,- br,- os,- cc,- isManual ? 1 : 0,- dayjs.unix(timestamp).format('YYYY-MM-DD HH:mm:ss'),- ]-}-@Injectable()export class CaptchaService {constructor(@@ -124,7 +103,6 @@ export class CaptchaService {pid: string,userAgent: string,timestamp: number,- isManual: boolean,ip: string,): Promise<void> {const ua = UAParser(userAgent)@@ -132,12 +110,18 @@ export class CaptchaService {const br = ua.browser.nameconst os = ua.os.name- const { country = 'NULL' } = getGeoDetails(ip)- const dto = captchaDTO(pid, dv, br, os, country, isManual, timestamp)- const values = `(${_map(dto, getElValue).join(',')})`+ const { country } = getGeoDetails(ip)+ const transformed = captchaTransformer(pid, dv, br, os, country, timestamp)try {- await redis.rpush(REDIS_LOG_CAPTCHA_CACHE_KEY, values)+ await clickhouse.insert({+ table: 'errors',+ format: 'JSONEachRow',+ values: [transformed],+ clickhouse_settings: {+ async_insert: 1,+ },+ })} catch (e) {this.logger.error(`[CaptchaService -> logCaptchaPass] ${e}`)}diff --git a/apps/production/src/captcha/utils/transformers.ts b/apps/production/src/captcha/utils/transformers.tsnew file mode 100644index 000000000..5d7eeebad--- /dev/null+++ b/apps/production/src/captcha/utils/transformers.ts@@ -0,0 +1,23 @@+import * as dayjs from 'dayjs'+import * as utc from 'dayjs/plugin/utc'++dayjs.extend(utc)++export const captchaTransformer = (+ pid: string,+ dv: string | null,+ br: string | null,+ os: string | null,+ cc: string | null,+ timestamp: number,+) => {+ return {+ pid,+ dv: dv || null,+ br: br || null,+ os: os || null,+ cc: cc || null,+ manuallyPassed: 1,+ created: dayjs.utc(timestamp).format('YYYY-MM-DD HH:mm:ss'),+ }+}diff --git a/apps/production/src/common/constants.ts b/apps/production/src/common/constants.tsindex 27089bd35..d95ecd9e0 100644--- a/apps/production/src/common/constants.ts+++ b/apps/production/src/common/constants.ts@@ -50,11 +50,6 @@ const getRedisUserCountKey = (uid: string) => `user_c_${uid}`const getRedisUserUsageInfoKey = (uid: string) => `user_ui_${uid}`const getRedisCaptchaKey = (token: string) => `captcha_${hash(token)}`-const REDIS_LOG_DATA_CACHE_KEY = 'log_cache'-const REDIS_LOG_CAPTCHA_CACHE_KEY = 'log:captcha'-const REDIS_LOG_PERF_CACHE_KEY = 'perf_cache'-const REDIS_LOG_CUSTOM_CACHE_KEY = 'log_custom_cache_v3'-const REDIS_LOG_ERROR_CACHE_KEY = 'log_error_cache'const REDIS_SESSION_SALT_KEY = 'log_salt' // is updated every 24 hoursconst REDIS_USERS_COUNT_KEY = 'stats:users_count'const REDIS_PROJECTS_COUNT_KEY = 'stats:projects_count'@@ -173,12 +168,9 @@ export {getRedisProjectKey,redisProjectCacheTimeout,UNIQUE_SESSION_LIFE_TIME,- REDIS_LOG_DATA_CACHE_KEY,- REDIS_LOG_CAPTCHA_CACHE_KEY,GDPR_EXPORT_TIMEFRAME,getRedisUserCountKey,redisProjectCountCacheTimeout,- REDIS_LOG_CUSTOM_CACHE_KEY,REDIS_SESSION_SALT_KEY,HEARTBEAT_SID_LIFE_TIME,REDIS_USERS_COUNT_KEY,@@ -189,7 +181,6 @@ export {TWO_FACTOR_AUTHENTICATION_APP_NAME,IP_REGEX,ORIGINS_REGEX,- REDIS_LOG_PERF_CACHE_KEY,CAPTCHA_SALT,EMAIL_ACTION_ENCRYPTION_KEY,isDevelopment,@@ -220,6 +211,5 @@ export {BLOG_POSTS_ROOT,TRAFFIC_SPIKE_ALLOWED_PERCENTAGE,AFFILIATE_CUT,- REDIS_LOG_ERROR_CACHE_KEY,PID_REGEX,}diff --git a/apps/production/src/common/integrations/clickhouse.ts b/apps/production/src/common/integrations/clickhouse.tsindex 1074adb2a..338a9ff0e 100644--- a/apps/production/src/common/integrations/clickhouse.ts+++ b/apps/production/src/common/integrations/clickhouse.ts@@ -69,6 +69,11 @@ const clickhouse = createClient({output_format_json_quote_64bit_integers: 0,enable_http_compression: 0,log_queries: 0,++ // Used for analytics & captcha stuff.+ // https://clickhouse.com/docs/en/optimize/asynchronous-inserts+ wait_for_async_insert: 0, // Return ACK (await) when row was added to the buffer, not flushed to the database+ async_insert_busy_timeout_ms: 15000, // 15 seconds; this is how long the buffer will be kept before writing stuff into the database},log: {LoggerClass: CHLogger,diff --git a/apps/production/src/task-manager/task-manager.service.ts b/apps/production/src/task-manager/task-manager.service.tsindex dc43b7bd6..1fb15e0ce 100644--- a/apps/production/src/task-manager/task-manager.service.ts+++ b/apps/production/src/task-manager/task-manager.service.ts@@ -8,7 +8,6 @@ import * as dayjs from 'dayjs'import * as utc from 'dayjs/plugin/utc'import * as _isEmpty from 'lodash/isEmpty'import * as _isNull from 'lodash/isNull'-import * as _join from 'lodash/join'import * as _size from 'lodash/size'import * as _map from 'lodash/map'import * as _now from 'lodash/now'@@ -44,19 +43,14 @@ import {} from '../user/entities/user.entity'import {redis,- REDIS_LOG_DATA_CACHE_KEY,- REDIS_LOG_CUSTOM_CACHE_KEY,REDIS_SESSION_SALT_KEY,- REDIS_LOG_PERF_CACHE_KEY,SEND_WARNING_AT_PERC,PROJECT_INVITE_EXPIRE,- REDIS_LOG_CAPTCHA_CACHE_KEY,JWT_REFRESH_TOKEN_LIFETIME,PAYPAL_CLIENT_ID,PAYPAL_CLIENT_SECRET,TRAFFIC_SPIKE_ALLOWED_PERCENTAGE,isDevelopment,- REDIS_LOG_ERROR_CACHE_KEY,} from '../common/constants'import { clickhouse } from '../common/integrations/clickhouse'import { CHPlanUsage } from './interfaces'@@ -408,87 +402,6 @@ export class TaskManagerService {})}- @Cron(CronExpression.EVERY_MINUTE)- async saveLogData(): Promise<void> {- const data = await redis.lrange(REDIS_LOG_DATA_CACHE_KEY, 0, -1)- const customData = await redis.lrange(REDIS_LOG_CUSTOM_CACHE_KEY, 0, -1)- const perfData = await redis.lrange(REDIS_LOG_PERF_CACHE_KEY, 0, -1)- const captchaData = await redis.lrange(REDIS_LOG_CAPTCHA_CACHE_KEY, 0, -1)- const errorData = await redis.lrange(REDIS_LOG_ERROR_CACHE_KEY, 0, -1)-- if (!_isEmpty(data)) {- await redis.del(REDIS_LOG_DATA_CACHE_KEY)- const query = `INSERT INTO analytics (*) VALUES ${_join(data, ',')}`- try {- await clickhouse.query({- query,- })- } catch (e) {- console.error(`[CRON WORKER] Error whilst saving log data: ${e}`)- }- }-- if (!_isEmpty(captchaData)) {- await redis.del(REDIS_LOG_CAPTCHA_CACHE_KEY)- const query = `INSERT INTO captcha (*) VALUES ${_join(captchaData, ',')}`- try {- await clickhouse.query({- query,- })- } catch (e) {- console.error(- `[CRON WORKER] Error whilst saving CAPTCHA log data: ${e}`,- )- }- }-- if (!_isEmpty(customData)) {- await redis.del(REDIS_LOG_CUSTOM_CACHE_KEY)- const query = `INSERT INTO customEV (*) VALUES ${_join(customData, ',')}`-- try {- await clickhouse.query({- query,- })- } catch (e) {- console.error(- `[CRON WORKER] Error whilst saving custom events data: ${e}`,- )- }- }-- if (!_isEmpty(errorData)) {- await redis.del(REDIS_LOG_ERROR_CACHE_KEY)- const query = `INSERT INTO errors (*) VALUES ${_join(errorData, ',')}`-- try {- await clickhouse.query({- query,- })- } catch (e) {- console.error(- `[CRON WORKER] Error whilst saving error events data: ${e}`,- )- }- }-- if (!_isEmpty(perfData)) {- await redis.del(REDIS_LOG_PERF_CACHE_KEY)-- const query = `INSERT INTO performance (*) VALUES ${_join(perfData, ',')}`-- try {- await clickhouse.query({- query,- })- } catch (e) {- console.error(- `[CRON WORKER] Error whilst saving performance data: ${e}`,- )- }- }- }-@Cron(CronExpression.EVERY_DAY_AT_5PM)async lockDashboards(): Promise<void> {const sevenDaysAgo = dayjsdiff --git a/apps/selfhosted/src/analytics/analytics.controller.ts b/apps/selfhosted/src/analytics/analytics.controller.tsindex 1b912908d..6ea96aa6c 100644--- a/apps/selfhosted/src/analytics/analytics.controller.ts+++ b/apps/selfhosted/src/analytics/analytics.controller.ts@@ -3,10 +3,7 @@ import * as _isArray from 'lodash/isArray'import * as _pick from 'lodash/pick'import * as _map from 'lodash/map'import * as _uniqBy from 'lodash/uniqBy'-import * as _round from 'lodash/round'import * as _includes from 'lodash/includes'-import * as _keys from 'lodash/keys'-import * as _values from 'lodash/values'import * as dayjs from 'dayjs'import * as utc from 'dayjs/plugin/utc'import * as dayjsTimezone from 'dayjs/plugin/timezone'@@ -54,13 +51,9 @@ import { GetUserFlowDTO } from './dto/getUserFlow.dto'import { GetFunnelsDTO } from './dto/getFunnels.dto'import { AppLoggerService } from '../logger/logger.service'import {- REDIS_LOG_DATA_CACHE_KEY,- REDIS_LOG_PERF_CACHE_KEY,redis,- REDIS_LOG_CUSTOM_CACHE_KEY,HEARTBEAT_SID_LIFE_TIME,REDIS_SESSION_SALT_KEY,- REDIS_LOG_ERROR_CACHE_KEY,} from '../common/constants'import { clickhouse } from '../common/integrations/clickhouse'import {@@ -84,9 +77,12 @@ import { ErrorDTO } from './dto/error.dto'import { GetErrorsDto } from './dto/get-errors.dto'import { GetErrorDTO } from './dto/get-error.dto'import { PatchStatusDTO } from './dto/patch-status.dto'--// eslint-disable-next-line @typescript-eslint/no-var-requires-const mysql = require('mysql2')+import {+ customEventTransformer,+ errorEventTransformer,+ performanceTransformer,+ trafficTransformer,+} from './utils/transformers'dayjs.extend(utc)dayjs.extend(dayjsTimezone)@@ -101,177 +97,6 @@ const getSessionKeyCustom = (salt = '',) => `cses_${hash(`${ua}${ip}${pid}${ev}${salt}`).toString('hex')}`-const analyticsDTO = (- psid: string,- sid: string,- pid: string,- pg: string,- prev: string,- dv: string,- br: string,- os: string,- lc: string,- ref: string,- so: string,- me: string,- ca: string,- cc: string,- rg: string,- ct: string,- keys: string[],- values: string[],- sdur: number | string,- unique: number,-): Array<string | number | string[]> => {- return [- psid,- sid,- pid,- pg,- prev,- dv,- br,- os,- lc,- ref,- so,- me,- ca,- cc,- rg,- ct,- keys,- values,- sdur,- unique,- dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),- ]-}--const performanceDTO = (- pid: string,- pg: string,- dv: string,- br: string,- cc: string,- rg: string,- ct: string,- dns: number,- tls: number,- conn: number,- response: number,- render: number,- domLoad: number,- pageLoad: number,- ttfb: number,-): Array<string | number> => {- return [- pid,- pg,- dv,- br,- cc,- rg,- ct,- _round(dns),- _round(tls),- _round(conn),- _round(response),- _round(render),- _round(domLoad),- _round(pageLoad),- _round(ttfb),- dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),- ]-}--const customLogDTO = (- psid: string,- pid: string,- ev: string,- pg: string,- dv: string,- br: string,- os: string,- lc: string,- ref: string,- so: string,- me: string,- ca: string,- cc: string,- rg: string,- ct: string,- keys: string[],- values: string[],-): Array<string | number | string[]> => {- return [- psid,- pid,- ev,- pg,- dv,- br,- os,- lc,- ref,- so,- me,- ca,- cc,- rg,- ct,- keys,- values,- dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),- ]-}--const errorLogDTO = (- eid: string,- pid: string,- pg: string,- dv: string,- br: string,- os: string,- lc: string,- cc: string,- rg: string,- ct: string,- name: string,- message: string,- lineno: number,- colno: number,- filename: string,-): Array<string | number | string[]> => {- return [- eid,- pid,- pg,- dv,- br,- os,- lc,- cc,- rg,- ct,- name,- message,- lineno,- colno,- filename,- dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),- ]-}--const getElValue = el => {- if (_isArray(el)) {- return `[${_map(el, getElValue).join(',')}]`- }-- if (el === undefined || el === null || el === 'NULL') return 'NULL'- return mysql.escape(el)-}-// Performance object validator: none of the values cannot be bigger than 1000 * 60 * 5 (5 minutes) and are >= 0const MAX_PERFORMANCE_VALUE = 1000 * 60 * 5const isPerformanceValid = (perf: any) => {@@ -1117,7 +942,7 @@ export class AnalyticsController {const sessionHash = getSessionKey(ip, userAgent, eventsDTO.pid, salt)const [, psid] = await this.analyticsService.isUnique(sessionHash)- const dto = customLogDTO(+ const transformed = customEventTransformer(psid,eventsDTO.pid,eventsDTO.ev,@@ -1133,13 +958,18 @@ export class AnalyticsController {country,region,city,- _keys(eventsDTO.meta),- _values(eventsDTO.meta),+ eventsDTO.meta,)try {- const values = `(${dto.map(getElValue).join(',')})`- await redis.rpush(REDIS_LOG_CUSTOM_CACHE_KEY, values)+ await clickhouse.insert({+ table: 'customEV',+ format: 'JSONEachRow',+ values: [transformed],+ clickhouse_settings: {+ async_insert: 1,+ },+ })} catch (e) {this.logger.error(e)throw new InternalServerErrorException(@@ -1215,7 +1045,7 @@ export class AnalyticsController {const br = ua.browser.nameconst os = ua.os.name- const dto = analyticsDTO(+ const transformed = trafficTransformer(psid,sessionHash,logDTO.pid,@@ -1232,13 +1062,12 @@ export class AnalyticsController {country,region,city,- _keys(logDTO.meta),- _values(logDTO.meta),+ logDTO.meta,0,Number(unique),)- let perfDTO: Array<string | number> = []+ let perfTransformed = nullif (!_isEmpty(logDTO.perf) && isPerformanceValid(logDTO.perf)) {const {@@ -1252,7 +1081,7 @@ export class AnalyticsController {ttfb,} = logDTO.perf- perfDTO = performanceDTO(+ perfTransformed = performanceTransformer(logDTO.pid,logDTO.pg,dv,@@ -1271,13 +1100,25 @@ export class AnalyticsController {)}- const values = `(${dto.map(getElValue).join(',')})`try {- await redis.rpush(REDIS_LOG_DATA_CACHE_KEY, values)+ await clickhouse.insert({+ table: 'analytics',+ format: 'JSONEachRow',+ values: [transformed],+ clickhouse_settings: {+ async_insert: 1,+ },+ })- if (!_isEmpty(perfDTO)) {- const perfValues = `(${perfDTO.map(getElValue).join(',')})`- await redis.rpush(REDIS_LOG_PERF_CACHE_KEY, perfValues)+ if (!_isEmpty(perfTransformed)) {+ await clickhouse.insert({+ table: 'performance',+ format: 'JSONEachRow',+ values: [perfTransformed],+ clickhouse_settings: {+ async_insert: 1,+ },+ })}} catch (e) {this.logger.error(e)@@ -1330,32 +1171,37 @@ export class AnalyticsController {country = 'NULL',} = getGeoDetails(ip, null, headers)- const dto = analyticsDTO(+ const transformed = trafficTransformer(psid,sessionHash,logDTO.pid,- 'NULL',- 'NULL',+ null,+ null,dv,br,os,- 'NULL',- 'NULL',- 'NULL',- 'NULL',- 'NULL',- city,- region,+ null,+ null,+ null,+ null,+ null,country,- [],- [],+ region,+ city,+ null,0,Number(unique),)- const values = `(${dto.map(getElValue).join(',')})`try {- await redis.rpush(REDIS_LOG_DATA_CACHE_KEY, values)+ await clickhouse.insert({+ table: 'analytics',+ format: 'JSONEachRow',+ values: [transformed],+ clickhouse_settings: {+ async_insert: 1,+ },+ })} catch (e) {this.logger.error(e)}@@ -1635,7 +1481,7 @@ export class AnalyticsController {const { name, message, lineno, colno, filename } = errorDTO- const dto = errorLogDTO(+ const transformed = errorEventTransformer(this.analyticsService.getErrorID(errorDTO),errorDTO.pid,errorDTO.pg,@@ -1654,8 +1500,14 @@ export class AnalyticsController {)try {- const values = `(${dto.map(getElValue).join(',')})`- await redis.rpush(REDIS_LOG_ERROR_CACHE_KEY, values)+ await clickhouse.insert({+ table: 'errors',+ format: 'JSONEachRow',+ values: [transformed],+ clickhouse_settings: {+ async_insert: 1,+ },+ })} catch (e) {this.logger.error(e)throw new InternalServerErrorException(diff --git a/apps/selfhosted/src/analytics/utils/transformers.ts b/apps/selfhosted/src/analytics/utils/transformers.tsnew file mode 100644index 000000000..1ab11b888--- /dev/null+++ b/apps/selfhosted/src/analytics/utils/transformers.ts@@ -0,0 +1,182 @@+import * as _round from 'lodash/round'+import * as _isEmpty from 'lodash/isEmpty'+import * as _keys from 'lodash/keys'+import * as _values from 'lodash/values'+import * as dayjs from 'dayjs'+import * as utc from 'dayjs/plugin/utc'++dayjs.extend(utc)++const processMetaKV = (+ rawMeta: Record<string, string>,+): { 'meta.key': string | null; 'meta.value': string | null } => {+ if (!rawMeta || _isEmpty(rawMeta)) {+ return {+ 'meta.key': null,+ 'meta.value': null,+ }+ }++ return {+ 'meta.key': _keys(rawMeta),+ 'meta.value': _values(rawMeta),+ }+}++export const trafficTransformer = (+ psid: string,+ sid: string,+ pid: string,+ pg: string | null,+ prev: string | null,+ dv: string | null,+ br: string | null,+ os: string | null,+ lc: string | null,+ ref: string | null,+ so: string | null,+ me: string | null,+ ca: string | null,+ cc: string | null,+ rg: string | null,+ ct: string | null,+ meta: Record<string, string> | null,+ sdur: number,+ unique: number,+) => {+ return {+ psid,+ sid,+ pid,+ pg: pg || null,+ prev: prev || null,+ dv: dv || null,+ br: br || null,+ os: os || null,+ lc: lc || null,+ ref: ref || null,+ so: so || null,+ me: me || null,+ ca: ca || null,+ cc: cc || null,+ rg: rg || null,+ ct: ct || null,+ ...processMetaKV(meta),+ sdur,+ unique,+ created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),+ }+}++export const customEventTransformer = (+ psid: string,+ pid: string,+ ev: string,+ pg: string | null,+ dv: string | null,+ br: string | null,+ os: string | null,+ lc: string | null,+ ref: string | null,+ so: string | null,+ me: string | null,+ ca: string | null,+ cc: string | null,+ rg: string | null,+ ct: string | null,+ meta: Record<string, string> | null,+) => {+ return {+ psid,+ pid,+ ev,+ pg: pg || null,+ dv: dv || null,+ br: br || null,+ os: os || null,+ lc: lc || null,+ ref: ref || null,+ so: so || null,+ me: me || null,+ ca: ca || null,+ cc: cc || null,+ rg: rg || null,+ ct: ct || null,+ ...processMetaKV(meta),+ created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),+ }+}++export const errorEventTransformer = (+ eid: string,+ pid: string,+ pg: string | null,+ dv: string | null,+ br: string | null,+ os: string | null,+ lc: string | null,+ cc: string | null,+ rg: string | null,+ ct: string | null,+ name: string | null,+ message: string | null,+ lineno: number | null,+ colno: number | null,+ filename: string | null,+) => {+ return {+ eid,+ pid,+ pg: pg || null,+ dv: dv || null,+ br: br || null,+ os: os || null,+ lc: lc || null,+ cc: cc || null,+ rg: rg || null,+ ct: ct || null,+ name: name || null,+ message: message || null,+ lineno: lineno || null,+ colno: colno || null,+ filename: filename || null,+ created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),+ }+}++export const performanceTransformer = (+ pid: string,+ pg: string | null,+ dv: string | null,+ br: string | null,+ cc: string | null,+ rg: string | null,+ ct: string | null,+ dns: number,+ tls: number,+ conn: number,+ response: number,+ render: number,+ domLoad: number,+ pageLoad: number,+ ttfb: number,+) => {+ return {+ pid,+ pg: pg || null,+ dv: dv || null,+ br: br || null,+ cc: cc || null,+ rg: rg || null,+ ct: ct || null,+ dns: _round(dns),+ tls: _round(tls),+ conn: _round(conn),+ response: _round(response),+ render: _round(render),+ domLoad: _round(domLoad),+ pageLoad: _round(pageLoad),+ ttfb: _round(ttfb),+ created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),+ }+}diff --git a/apps/selfhosted/src/common/constants.ts b/apps/selfhosted/src/common/constants.tsindex 7ac72181c..af6c249b3 100644--- a/apps/selfhosted/src/common/constants.ts+++ b/apps/selfhosted/src/common/constants.ts@@ -50,14 +50,9 @@ const isValidPID = (pid: string) => PID_REGEX.test(pid)// redis keysconst getRedisProjectKey = (pid: string) => `pid_${pid}`-const REDIS_LOG_DATA_CACHE_KEY = 'log_cache_v2'-const REDIS_LOG_CAPTCHA_CACHE_KEY = 'log:captcha'-const REDIS_LOG_PERF_CACHE_KEY = 'perf_cache'-const REDIS_LOG_CUSTOM_CACHE_KEY = 'log_custom_cache_v3'const REDIS_SESSION_SALT_KEY = 'log_salt' // is updated every 24 hoursconst REDIS_USERS_COUNT_KEY = 'stats:users_count'const REDIS_PROJECTS_COUNT_KEY = 'stats:projects_count'-const REDIS_LOG_ERROR_CACHE_KEY = 'log_error_cache'// 3600 sec -> 1 hourconst redisProjectCacheTimeout = 3600@@ -103,9 +98,6 @@ export {getRedisProjectKey,redisProjectCacheTimeout,UNIQUE_SESSION_LIFE_TIME,- REDIS_LOG_DATA_CACHE_KEY,- REDIS_LOG_CAPTCHA_CACHE_KEY,- REDIS_LOG_CUSTOM_CACHE_KEY,TRAFFIC_METAKEY_COLUMNS,REDIS_SESSION_SALT_KEY,HEARTBEAT_SID_LIFE_TIME,@@ -119,7 +111,6 @@ export {REDIS_PROJECTS_COUNT_KEY,IP_REGEX,ORIGINS_REGEX,- REDIS_LOG_PERF_CACHE_KEY,isDevelopment,DEFAULT_SELFHOSTED_UUID,JWT_ACCESS_TOKEN_SECRET,@@ -131,7 +122,6 @@ export {MIN_PAGES_IN_FUNNEL,MAX_PAGES_IN_FUNNEL,ERROR_COLUMNS,- REDIS_LOG_ERROR_CACHE_KEY,PID_REGEX,ALL_COLUMNS,}diff --git a/apps/selfhosted/src/task-manager/task-manager.service.ts b/apps/selfhosted/src/task-manager/task-manager.service.tsindex 99c661726..feea927c7 100644--- a/apps/selfhosted/src/task-manager/task-manager.service.ts+++ b/apps/selfhosted/src/task-manager/task-manager.service.ts@@ -3,21 +3,11 @@ import { Cron, CronExpression } from '@nestjs/schedule'import * as bcrypt from 'bcrypt'import * as dayjs from 'dayjs'import * as utc from 'dayjs/plugin/utc'-import * as _isEmpty from 'lodash/isEmpty'-import * as _join from 'lodash/join'import * as _size from 'lodash/size'import * as _map from 'lodash/map'import * as _now from 'lodash/now'-import {- redis,- REDIS_LOG_DATA_CACHE_KEY,- REDIS_LOG_CUSTOM_CACHE_KEY,- REDIS_SESSION_SALT_KEY,- REDIS_LOG_PERF_CACHE_KEY,- REDIS_LOG_CAPTCHA_CACHE_KEY,- REDIS_LOG_ERROR_CACHE_KEY,-} from '../common/constants'+import { redis, REDIS_SESSION_SALT_KEY } from '../common/constants'import { clickhouse } from '../common/integrations/clickhouse'import { AppLoggerService } from '../logger/logger.service'@@ -27,87 +17,6 @@ dayjs.extend(utc)export class TaskManagerService {constructor(private readonly logger: AppLoggerService) {}- @Cron(CronExpression.EVERY_MINUTE)- async saveLogData(): Promise<void> {- const data = await redis.lrange(REDIS_LOG_DATA_CACHE_KEY, 0, -1)- const customData = await redis.lrange(REDIS_LOG_CUSTOM_CACHE_KEY, 0, -1)- const perfData = await redis.lrange(REDIS_LOG_PERF_CACHE_KEY, 0, -1)- const captchaData = await redis.lrange(REDIS_LOG_CAPTCHA_CACHE_KEY, 0, -1)- const errorData = await redis.lrange(REDIS_LOG_ERROR_CACHE_KEY, 0, -1)-- if (!_isEmpty(data)) {- await redis.del(REDIS_LOG_DATA_CACHE_KEY)- const query = `INSERT INTO analytics (*) VALUES ${_join(data, ',')}`- try {- await clickhouse.query({- query,- })- } catch (e) {- console.error(`[CRON WORKER] Error whilst saving log data: ${e}`)- }- }-- if (!_isEmpty(captchaData)) {- await redis.del(REDIS_LOG_CAPTCHA_CACHE_KEY)- const query = `INSERT INTO captcha (*) VALUES ${_join(captchaData, ',')}`- try {- await clickhouse.query({- query,- })- } catch (e) {- console.error(- `[CRON WORKER] Error whilst saving CAPTCHA log data: ${e}`,- )- }- }-- if (!_isEmpty(customData)) {- await redis.del(REDIS_LOG_CUSTOM_CACHE_KEY)- const query = `INSERT INTO customEV (*) VALUES ${_join(customData, ',')}`-- try {- await clickhouse.query({- query,- })- } catch (e) {- console.error(- `[CRON WORKER] Error whilst saving custom events data: ${e}`,- )- }- }-- if (!_isEmpty(errorData)) {- await redis.del(REDIS_LOG_ERROR_CACHE_KEY)- const query = `INSERT INTO errors (*) VALUES ${_join(errorData, ',')}`-- try {- await clickhouse.query({- query,- })- } catch (e) {- console.error(- `[CRON WORKER] Error whilst saving error events data: ${e}`,- )- }- }-- if (!_isEmpty(perfData)) {- await redis.del(REDIS_LOG_PERF_CACHE_KEY)-- const query = `INSERT INTO performance (*) VALUES ${_join(perfData, ',')}`-- try {- await clickhouse.query({- query,- })- } catch (e) {- console.error(- `[CRON WORKER] Error whilst saving performance data: ${e}`,- )- }- }- }-@Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)async generateSessionSalt(): Promise<void> {const salt = await bcrypt.genSalt(10)diff --git a/migrations/clickhouse/setup.js b/migrations/clickhouse/setup.jsindex 0c3765bec..7d259754d 100644--- a/migrations/clickhouse/setup.js+++ b/migrations/clickhouse/setup.js@@ -49,7 +49,7 @@ const databaselessQueriesRunner = async queries => {if (query) {try {// eslint-disable-next-line no-await-in-loop- await clickhouseNoDatabase.query({+ await clickhouseNoDatabase.command({query,})console.log(chalk.green('Query OK: '), query)@@ -73,7 +73,7 @@ const queriesRunner = async (queries, log = true) => {if (query) {try {// eslint-disable-next-line no-await-in-loop- await clickhouse.query({+ await clickhouse.command({query,})diff --git a/migrations/clickhouse/test_migration.js b/migrations/clickhouse/test_migration.jsindex 35d8d3fdd..e20dec9c5 100644--- a/migrations/clickhouse/test_migration.js+++ b/migrations/clickhouse/test_migration.js@@ -1,7 +1,5 @@const { queriesRunner, dbName } = require('./setup')-const queries = [- `SELECT * FROM ${dbName}.analytics LIMIT 10`,-]+const queries = [`SELECT * FROM ${dbName}.analytics LIMIT 10`]queriesRunner(queries)