Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

dropped redis for analytics data queue and instead relying on clickhouse async inserts

Blue Mouse956d3ecb7a1c
diff --git a/apps/production/src/analytics/analytics.controller.ts b/apps/production/src/analytics/analytics.controller.ts
index 9bc6072d6..b3afbc5b4 100644
--- a/apps/production/src/analytics/analytics.controller.ts
+++ b/apps/production/src/analytics/analytics.controller.ts
@@ -3,11 +3,8 @@ import * as _isArray from 'lodash/isArray'
import * as _toNumber from 'lodash/toNumber'
import * as _pick from 'lodash/pick'
import * as _includes from 'lodash/includes'
-import * as _keys from 'lodash/keys'
-import * as _values from 'lodash/values'
import * as _map from 'lodash/map'
import * as _uniqBy from 'lodash/uniqBy'
-import * as _round from 'lodash/round'
import * as dayjs from 'dayjs'
import * as utc from 'dayjs/plugin/utc'
import * as dayjsTimezone from 'dayjs/plugin/timezone'
@@ -57,16 +54,12 @@ import { GetUserFlowDTO } from './dto/getUserFlow.dto'
import { GetFunnelsDTO } from './dto/getFunnels.dto'
import { AppLoggerService } from '../logger/logger.service'
import {
- REDIS_LOG_DATA_CACHE_KEY,
- REDIS_LOG_PERF_CACHE_KEY,
redis,
- REDIS_LOG_CUSTOM_CACHE_KEY,
HEARTBEAT_SID_LIFE_TIME,
REDIS_USERS_COUNT_KEY,
REDIS_PROJECTS_COUNT_KEY,
REDIS_EVENTS_COUNT_KEY,
REDIS_SESSION_SALT_KEY,
- REDIS_LOG_ERROR_CACHE_KEY,
} from '../common/constants'
import { clickhouse } from '../common/integrations/clickhouse'
import {
@@ -91,11 +84,13 @@ import { ErrorDTO } from './dto/error.dto'
import { GetErrorsDto } from './dto/get-errors.dto'
import { GetErrorDTO } from './dto/get-error.dto'
import { PatchStatusDTO } from './dto/patch-status.dto'
-import { ProjectService } from '../project/project.service'
import { ProjectsViewsRepository } from '../project/repositories/projects-views.repository'
-
-// eslint-disable-next-line @typescript-eslint/no-var-requires
-const mysql = require('mysql2')
+import {
+ customEventTransformer,
+ errorEventTransformer,
+ performanceTransformer,
+ trafficTransformer,
+} from './utils/transformers'
dayjs.extend(utc)
dayjs.extend(dayjsTimezone)
@@ -110,177 +105,6 @@ const getSessionKeyCustom = (
salt = '',
) => `cses_${hash(`${ua}${ip}${pid}${ev}${salt}`).toString('hex')}`
-const analyticsDTO = (
- psid: string,
- sid: string,
- pid: string,
- pg: string,
- prev: string,
- dv: string,
- br: string,
- os: string,
- lc: string,
- ref: string,
- so: string,
- me: string,
- ca: string,
- cc: string,
- rg: string,
- ct: string,
- keys: string[],
- values: string[],
- sdur: number | string,
- unique: number,
-): Array<string | number | string[]> => {
- return [
- psid,
- sid,
- pid,
- pg,
- prev,
- dv,
- br,
- os,
- lc,
- ref,
- so,
- me,
- ca,
- cc,
- rg,
- ct,
- keys,
- values,
- sdur,
- unique,
- dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
- ]
-}
-
-const performanceDTO = (
- pid: string,
- pg: string,
- dv: string,
- br: string,
- cc: string,
- rg: string,
- ct: string,
- dns: number,
- tls: number,
- conn: number,
- response: number,
- render: number,
- domLoad: number,
- pageLoad: number,
- ttfb: number,
-): Array<string | number> => {
- return [
- pid,
- pg,
- dv,
- br,
- cc,
- rg,
- ct,
- _round(dns),
- _round(tls),
- _round(conn),
- _round(response),
- _round(render),
- _round(domLoad),
- _round(pageLoad),
- _round(ttfb),
- dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
- ]
-}
-
-const customLogDTO = (
- psid: string,
- pid: string,
- ev: string,
- pg: string,
- dv: string,
- br: string,
- os: string,
- lc: string,
- ref: string,
- so: string,
- me: string,
- ca: string,
- cc: string,
- rg: string,
- ct: string,
- keys: string[],
- values: string[],
-): Array<string | number | string[]> => {
- return [
- psid,
- pid,
- ev,
- pg,
- dv,
- br,
- os,
- lc,
- ref,
- so,
- me,
- ca,
- cc,
- rg,
- ct,
- keys,
- values,
- dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
- ]
-}
-
-const errorLogDTO = (
- eid: string,
- pid: string,
- pg: string,
- dv: string,
- br: string,
- os: string,
- lc: string,
- cc: string,
- rg: string,
- ct: string,
- name: string,
- message: string,
- lineno: number,
- colno: number,
- filename: string,
-): Array<string | number | string[]> => {
- return [
- eid,
- pid,
- pg,
- dv,
- br,
- os,
- lc,
- cc,
- rg,
- ct,
- name,
- message,
- lineno,
- colno,
- filename,
- dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
- ]
-}
-
-export const getElValue = el => {
- if (_isArray(el)) {
- return `[${_map(el, getElValue).join(',')}]`
- }
-
- if (el === undefined || el === null || el === 'NULL') return 'NULL'
- return mysql.escape(el)
-}
-
// Performance object validator: none of the values cannot be bigger than 1000 * 60 * 5 (5 minutes) and are >= 0
const MAX_PERFORMANCE_VALUE = 1000 * 60 * 5
const isPerformanceValid = (perf: any) => {
@@ -370,7 +194,6 @@ export class AnalyticsController {
constructor(
private readonly analyticsService: AnalyticsService,
private readonly logger: AppLoggerService,
- private readonly projectService: ProjectService,
private readonly projectsViewsRepository: ProjectsViewsRepository,
) {}
@@ -1285,7 +1108,7 @@ export class AnalyticsController {
const { name, message, lineno, colno, filename } = errorDTO
- const dto = errorLogDTO(
+ const transformed = errorEventTransformer(
this.analyticsService.getErrorID(errorDTO),
errorDTO.pid,
errorDTO.pg,
@@ -1304,8 +1127,14 @@ export class AnalyticsController {
)
try {
- const values = `(${dto.map(getElValue).join(',')})`
- await redis.rpush(REDIS_LOG_ERROR_CACHE_KEY, values)
+ await clickhouse.insert({
+ table: 'errors',
+ format: 'JSONEachRow',
+ values: [transformed],
+ clickhouse_settings: {
+ async_insert: 1,
+ },
+ })
} catch (e) {
this.logger.error(e)
throw new InternalServerErrorException(
@@ -1389,7 +1218,7 @@ export class AnalyticsController {
const sessionHash = getSessionKey(ip, userAgent, eventsDTO.pid, salt)
const [, psid] = await this.analyticsService.isUnique(sessionHash)
- const dto = customLogDTO(
+ const transformed = customEventTransformer(
psid,
eventsDTO.pid,
eventsDTO.ev,
@@ -1405,13 +1234,18 @@ export class AnalyticsController {
country,
region,
city,
- _keys(eventsDTO.meta),
- _values(eventsDTO.meta),
+ eventsDTO.meta,
)
try {
- const values = `(${dto.map(getElValue).join(',')})`
- await redis.rpush(REDIS_LOG_CUSTOM_CACHE_KEY, values)
+ await clickhouse.insert({
+ table: 'customEV',
+ format: 'JSONEachRow',
+ values: [transformed],
+ clickhouse_settings: {
+ async_insert: 1,
+ },
+ })
} catch (e) {
this.logger.error(e)
throw new InternalServerErrorException(
@@ -1476,17 +1310,13 @@ export class AnalyticsController {
)
}
- const {
- city = 'NULL',
- region = 'NULL',
- country = 'NULL',
- } = getGeoDetails(ip, logDTO.tz)
+ const { city, region, country } = getGeoDetails(ip, logDTO.tz)
const ua = UAParser(userAgent)
const dv = ua.device.type || 'desktop'
const br = ua.browser.name
const os = ua.os.name
- const dto = analyticsDTO(
+ const transformed = trafficTransformer(
psid,
sessionHash,
logDTO.pid,
@@ -1503,13 +1333,12 @@ export class AnalyticsController {
country,
region,
city,
- _keys(logDTO.meta),
- _values(logDTO.meta),
+ logDTO.meta,
0,
Number(unique),
)
- let perfDTO: Array<string | number> = []
+ let perfTransformed = null
if (!_isEmpty(logDTO.perf) && isPerformanceValid(logDTO.perf)) {
const {
@@ -1523,7 +1352,7 @@ export class AnalyticsController {
ttfb,
} = logDTO.perf
- perfDTO = performanceDTO(
+ perfTransformed = performanceTransformer(
logDTO.pid,
logDTO.pg,
dv,
@@ -1542,14 +1371,25 @@ export class AnalyticsController {
)
}
- const values = `(${dto.map(getElValue).join(',')})`
-
try {
- await redis.rpush(REDIS_LOG_DATA_CACHE_KEY, values)
-
- if (!_isEmpty(perfDTO)) {
- const perfValues = `(${perfDTO.map(getElValue).join(',')})`
- await redis.rpush(REDIS_LOG_PERF_CACHE_KEY, perfValues)
+ await clickhouse.insert({
+ table: 'analytics',
+ format: 'JSONEachRow',
+ values: [transformed],
+ clickhouse_settings: {
+ async_insert: 1,
+ },
+ })
+
+ if (!_isEmpty(perfTransformed)) {
+ await clickhouse.insert({
+ table: 'performance',
+ format: 'JSONEachRow',
+ values: [perfTransformed],
+ clickhouse_settings: {
+ async_insert: 1,
+ },
+ })
}
} catch (e) {
this.logger.error(e)
@@ -1602,32 +1442,37 @@ export class AnalyticsController {
const br = ua.browser.name
const os = ua.os.name
- const dto = analyticsDTO(
+ const transformed = trafficTransformer(
psid,
sessionHash,
logDTO.pid,
- 'NULL',
- 'NULL',
+ null,
+ null,
dv,
br,
os,
- 'NULL',
- 'NULL',
- 'NULL',
- 'NULL',
- 'NULL',
+ null,
+ null,
+ null,
+ null,
+ null,
country,
region,
city,
- [],
- [],
+ null,
0,
Number(unique),
)
- const values = `(${dto.map(getElValue).join(',')})`
try {
- await redis.rpush(REDIS_LOG_DATA_CACHE_KEY, values)
+ await clickhouse.insert({
+ table: 'analytics',
+ format: 'JSONEachRow',
+ values: [transformed],
+ clickhouse_settings: {
+ async_insert: 1,
+ },
+ })
} catch (e) {
this.logger.error(e)
}
diff --git a/apps/production/src/analytics/utils/transformers.ts b/apps/production/src/analytics/utils/transformers.ts
new file mode 100644
index 000000000..1ab11b888
--- /dev/null
+++ b/apps/production/src/analytics/utils/transformers.ts
@@ -0,0 +1,182 @@
+import * as _round from 'lodash/round'
+import * as _isEmpty from 'lodash/isEmpty'
+import * as _keys from 'lodash/keys'
+import * as _values from 'lodash/values'
+import * as dayjs from 'dayjs'
+import * as utc from 'dayjs/plugin/utc'
+
+dayjs.extend(utc)
+
+const processMetaKV = (
+ rawMeta: Record<string, string>,
+): { 'meta.key': string | null; 'meta.value': string | null } => {
+ if (!rawMeta || _isEmpty(rawMeta)) {
+ return {
+ 'meta.key': null,
+ 'meta.value': null,
+ }
+ }
+
+ return {
+ 'meta.key': _keys(rawMeta),
+ 'meta.value': _values(rawMeta),
+ }
+}
+
+export const trafficTransformer = (
+ psid: string,
+ sid: string,
+ pid: string,
+ pg: string | null,
+ prev: string | null,
+ dv: string | null,
+ br: string | null,
+ os: string | null,
+ lc: string | null,
+ ref: string | null,
+ so: string | null,
+ me: string | null,
+ ca: string | null,
+ cc: string | null,
+ rg: string | null,
+ ct: string | null,
+ meta: Record<string, string> | null,
+ sdur: number,
+ unique: number,
+) => {
+ return {
+ psid,
+ sid,
+ pid,
+ pg: pg || null,
+ prev: prev || null,
+ dv: dv || null,
+ br: br || null,
+ os: os || null,
+ lc: lc || null,
+ ref: ref || null,
+ so: so || null,
+ me: me || null,
+ ca: ca || null,
+ cc: cc || null,
+ rg: rg || null,
+ ct: ct || null,
+ ...processMetaKV(meta),
+ sdur,
+ unique,
+ created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
+ }
+}
+
+export const customEventTransformer = (
+ psid: string,
+ pid: string,
+ ev: string,
+ pg: string | null,
+ dv: string | null,
+ br: string | null,
+ os: string | null,
+ lc: string | null,
+ ref: string | null,
+ so: string | null,
+ me: string | null,
+ ca: string | null,
+ cc: string | null,
+ rg: string | null,
+ ct: string | null,
+ meta: Record<string, string> | null,
+) => {
+ return {
+ psid,
+ pid,
+ ev,
+ pg: pg || null,
+ dv: dv || null,
+ br: br || null,
+ os: os || null,
+ lc: lc || null,
+ ref: ref || null,
+ so: so || null,
+ me: me || null,
+ ca: ca || null,
+ cc: cc || null,
+ rg: rg || null,
+ ct: ct || null,
+ ...processMetaKV(meta),
+ created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
+ }
+}
+
+export const errorEventTransformer = (
+ eid: string,
+ pid: string,
+ pg: string | null,
+ dv: string | null,
+ br: string | null,
+ os: string | null,
+ lc: string | null,
+ cc: string | null,
+ rg: string | null,
+ ct: string | null,
+ name: string | null,
+ message: string | null,
+ lineno: number | null,
+ colno: number | null,
+ filename: string | null,
+) => {
+ return {
+ eid,
+ pid,
+ pg: pg || null,
+ dv: dv || null,
+ br: br || null,
+ os: os || null,
+ lc: lc || null,
+ cc: cc || null,
+ rg: rg || null,
+ ct: ct || null,
+ name: name || null,
+ message: message || null,
+ lineno: lineno || null,
+ colno: colno || null,
+ filename: filename || null,
+ created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
+ }
+}
+
+export const performanceTransformer = (
+ pid: string,
+ pg: string | null,
+ dv: string | null,
+ br: string | null,
+ cc: string | null,
+ rg: string | null,
+ ct: string | null,
+ dns: number,
+ tls: number,
+ conn: number,
+ response: number,
+ render: number,
+ domLoad: number,
+ pageLoad: number,
+ ttfb: number,
+) => {
+ return {
+ pid,
+ pg: pg || null,
+ dv: dv || null,
+ br: br || null,
+ cc: cc || null,
+ rg: rg || null,
+ ct: ct || null,
+ dns: _round(dns),
+ tls: _round(tls),
+ conn: _round(conn),
+ response: _round(response),
+ render: _round(render),
+ domLoad: _round(domLoad),
+ pageLoad: _round(pageLoad),
+ ttfb: _round(ttfb),
+ created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
+ }
+}
diff --git a/apps/production/src/captcha/captcha.controller.ts b/apps/production/src/captcha/captcha.controller.ts
index 1788d15b6..e93460360 100644
--- a/apps/production/src/captcha/captcha.controller.ts
+++ b/apps/production/src/captcha/captcha.controller.ts
@@ -89,13 +89,7 @@ export class CaptchaController {
const ip = getIPFromHeaders(headers) || reqIP || ''
- await this.captchaService.logCaptchaPass(
- pid,
- userAgent,
- timestamp,
- true,
- ip,
- )
+ await this.captchaService.logCaptchaPass(pid, userAgent, timestamp, ip)
return {
success: true,
diff --git a/apps/production/src/captcha/captcha.service.ts b/apps/production/src/captcha/captcha.service.ts
index b781c6a5d..8b156cce0 100644
--- a/apps/production/src/captcha/captcha.service.ts
+++ b/apps/production/src/captcha/captcha.service.ts
@@ -4,7 +4,6 @@ import * as svgCaptcha from 'svg-captcha'
import * as CryptoJS from 'crypto-js'
import * as _toLower from 'lodash/toLower'
import * as UAParser from 'ua-parser-js'
-import * as _map from 'lodash/map'
import * as _values from 'lodash/values'
import * as _includes from 'lodash/includes'
import * as dayjs from 'dayjs'
@@ -14,15 +13,15 @@ import { ProjectService } from '../project/project.service'
import { AppLoggerService } from '../logger/logger.service'
import {
redis,
- REDIS_LOG_CAPTCHA_CACHE_KEY,
isValidPID,
getRedisCaptchaKey,
CAPTCHA_SALT,
CAPTCHA_TOKEN_LIFETIME,
} from '../common/constants'
import { getGeoDetails } from '../common/utils'
-import { getElValue } from '../analytics/analytics.controller'
import { GeneratedCaptcha } from './interfaces/generated-captcha'
+import { captchaTransformer } from './utils/transformers'
+import { clickhouse } from '../common/integrations/clickhouse'
dayjs.extend(utc)
@@ -65,26 +64,6 @@ const isTokenAlreadyUsed = async (token: string): Promise<boolean> => {
return false
}
-const captchaDTO = (
- pid: string,
- dv: string,
- br: string,
- os: string,
- cc: string,
- isManual: boolean,
- timestamp: number,
-): Array<string | number> => {
- return [
- pid,
- dv,
- br,
- os,
- cc,
- isManual ? 1 : 0,
- dayjs.unix(timestamp).format('YYYY-MM-DD HH:mm:ss'),
- ]
-}
-
@Injectable()
export class CaptchaService {
constructor(
@@ -124,7 +103,6 @@ export class CaptchaService {
pid: string,
userAgent: string,
timestamp: number,
- isManual: boolean,
ip: string,
): Promise<void> {
const ua = UAParser(userAgent)
@@ -132,12 +110,18 @@ export class CaptchaService {
const br = ua.browser.name
const os = ua.os.name
- const { country = 'NULL' } = getGeoDetails(ip)
- const dto = captchaDTO(pid, dv, br, os, country, isManual, timestamp)
- const values = `(${_map(dto, getElValue).join(',')})`
+ const { country } = getGeoDetails(ip)
+ const transformed = captchaTransformer(pid, dv, br, os, country, timestamp)
try {
- await redis.rpush(REDIS_LOG_CAPTCHA_CACHE_KEY, values)
+ await clickhouse.insert({
+ table: 'errors',
+ format: 'JSONEachRow',
+ values: [transformed],
+ clickhouse_settings: {
+ async_insert: 1,
+ },
+ })
} catch (e) {
this.logger.error(`[CaptchaService -> logCaptchaPass] ${e}`)
}
diff --git a/apps/production/src/captcha/utils/transformers.ts b/apps/production/src/captcha/utils/transformers.ts
new file mode 100644
index 000000000..5d7eeebad
--- /dev/null
+++ b/apps/production/src/captcha/utils/transformers.ts
@@ -0,0 +1,23 @@
+import * as dayjs from 'dayjs'
+import * as utc from 'dayjs/plugin/utc'
+
+dayjs.extend(utc)
+
+export const captchaTransformer = (
+ pid: string,
+ dv: string | null,
+ br: string | null,
+ os: string | null,
+ cc: string | null,
+ timestamp: number,
+) => {
+ return {
+ pid,
+ dv: dv || null,
+ br: br || null,
+ os: os || null,
+ cc: cc || null,
+ manuallyPassed: 1,
+ created: dayjs.utc(timestamp).format('YYYY-MM-DD HH:mm:ss'),
+ }
+}
diff --git a/apps/production/src/common/constants.ts b/apps/production/src/common/constants.ts
index 27089bd35..d95ecd9e0 100644
--- a/apps/production/src/common/constants.ts
+++ b/apps/production/src/common/constants.ts
@@ -50,11 +50,6 @@ const getRedisUserCountKey = (uid: string) => `user_c_${uid}`
const getRedisUserUsageInfoKey = (uid: string) => `user_ui_${uid}`
const getRedisCaptchaKey = (token: string) => `captcha_${hash(token)}`
-const REDIS_LOG_DATA_CACHE_KEY = 'log_cache'
-const REDIS_LOG_CAPTCHA_CACHE_KEY = 'log:captcha'
-const REDIS_LOG_PERF_CACHE_KEY = 'perf_cache'
-const REDIS_LOG_CUSTOM_CACHE_KEY = 'log_custom_cache_v3'
-const REDIS_LOG_ERROR_CACHE_KEY = 'log_error_cache'
const REDIS_SESSION_SALT_KEY = 'log_salt' // is updated every 24 hours
const REDIS_USERS_COUNT_KEY = 'stats:users_count'
const REDIS_PROJECTS_COUNT_KEY = 'stats:projects_count'
@@ -173,12 +168,9 @@ export {
getRedisProjectKey,
redisProjectCacheTimeout,
UNIQUE_SESSION_LIFE_TIME,
- REDIS_LOG_DATA_CACHE_KEY,
- REDIS_LOG_CAPTCHA_CACHE_KEY,
GDPR_EXPORT_TIMEFRAME,
getRedisUserCountKey,
redisProjectCountCacheTimeout,
- REDIS_LOG_CUSTOM_CACHE_KEY,
REDIS_SESSION_SALT_KEY,
HEARTBEAT_SID_LIFE_TIME,
REDIS_USERS_COUNT_KEY,
@@ -189,7 +181,6 @@ export {
TWO_FACTOR_AUTHENTICATION_APP_NAME,
IP_REGEX,
ORIGINS_REGEX,
- REDIS_LOG_PERF_CACHE_KEY,
CAPTCHA_SALT,
EMAIL_ACTION_ENCRYPTION_KEY,
isDevelopment,
@@ -220,6 +211,5 @@ export {
BLOG_POSTS_ROOT,
TRAFFIC_SPIKE_ALLOWED_PERCENTAGE,
AFFILIATE_CUT,
- REDIS_LOG_ERROR_CACHE_KEY,
PID_REGEX,
}
diff --git a/apps/production/src/common/integrations/clickhouse.ts b/apps/production/src/common/integrations/clickhouse.ts
index 1074adb2a..338a9ff0e 100644
--- a/apps/production/src/common/integrations/clickhouse.ts
+++ b/apps/production/src/common/integrations/clickhouse.ts
@@ -69,6 +69,11 @@ const clickhouse = createClient({
output_format_json_quote_64bit_integers: 0,
enable_http_compression: 0,
log_queries: 0,
+
+ // Used for analytics & captcha stuff.
+ // https://clickhouse.com/docs/en/optimize/asynchronous-inserts
+ wait_for_async_insert: 0, // Return ACK (await) when row was added to the buffer, not flushed to the database
+ async_insert_busy_timeout_ms: 15000, // 15 seconds; this is how long the buffer will be kept before writing stuff into the database
},
log: {
LoggerClass: CHLogger,
diff --git a/apps/production/src/task-manager/task-manager.service.ts b/apps/production/src/task-manager/task-manager.service.ts
index dc43b7bd6..1fb15e0ce 100644
--- a/apps/production/src/task-manager/task-manager.service.ts
+++ b/apps/production/src/task-manager/task-manager.service.ts
@@ -8,7 +8,6 @@ import * as dayjs from 'dayjs'
import * as utc from 'dayjs/plugin/utc'
import * as _isEmpty from 'lodash/isEmpty'
import * as _isNull from 'lodash/isNull'
-import * as _join from 'lodash/join'
import * as _size from 'lodash/size'
import * as _map from 'lodash/map'
import * as _now from 'lodash/now'
@@ -44,19 +43,14 @@ import {
} from '../user/entities/user.entity'
import {
redis,
- REDIS_LOG_DATA_CACHE_KEY,
- REDIS_LOG_CUSTOM_CACHE_KEY,
REDIS_SESSION_SALT_KEY,
- REDIS_LOG_PERF_CACHE_KEY,
SEND_WARNING_AT_PERC,
PROJECT_INVITE_EXPIRE,
- REDIS_LOG_CAPTCHA_CACHE_KEY,
JWT_REFRESH_TOKEN_LIFETIME,
PAYPAL_CLIENT_ID,
PAYPAL_CLIENT_SECRET,
TRAFFIC_SPIKE_ALLOWED_PERCENTAGE,
isDevelopment,
- REDIS_LOG_ERROR_CACHE_KEY,
} from '../common/constants'
import { clickhouse } from '../common/integrations/clickhouse'
import { CHPlanUsage } from './interfaces'
@@ -408,87 +402,6 @@ export class TaskManagerService {
})
}
- @Cron(CronExpression.EVERY_MINUTE)
- async saveLogData(): Promise<void> {
- const data = await redis.lrange(REDIS_LOG_DATA_CACHE_KEY, 0, -1)
- const customData = await redis.lrange(REDIS_LOG_CUSTOM_CACHE_KEY, 0, -1)
- const perfData = await redis.lrange(REDIS_LOG_PERF_CACHE_KEY, 0, -1)
- const captchaData = await redis.lrange(REDIS_LOG_CAPTCHA_CACHE_KEY, 0, -1)
- const errorData = await redis.lrange(REDIS_LOG_ERROR_CACHE_KEY, 0, -1)
-
- if (!_isEmpty(data)) {
- await redis.del(REDIS_LOG_DATA_CACHE_KEY)
- const query = `INSERT INTO analytics (*) VALUES ${_join(data, ',')}`
- try {
- await clickhouse.query({
- query,
- })
- } catch (e) {
- console.error(`[CRON WORKER] Error whilst saving log data: ${e}`)
- }
- }
-
- if (!_isEmpty(captchaData)) {
- await redis.del(REDIS_LOG_CAPTCHA_CACHE_KEY)
- const query = `INSERT INTO captcha (*) VALUES ${_join(captchaData, ',')}`
- try {
- await clickhouse.query({
- query,
- })
- } catch (e) {
- console.error(
- `[CRON WORKER] Error whilst saving CAPTCHA log data: ${e}`,
- )
- }
- }
-
- if (!_isEmpty(customData)) {
- await redis.del(REDIS_LOG_CUSTOM_CACHE_KEY)
- const query = `INSERT INTO customEV (*) VALUES ${_join(customData, ',')}`
-
- try {
- await clickhouse.query({
- query,
- })
- } catch (e) {
- console.error(
- `[CRON WORKER] Error whilst saving custom events data: ${e}`,
- )
- }
- }
-
- if (!_isEmpty(errorData)) {
- await redis.del(REDIS_LOG_ERROR_CACHE_KEY)
- const query = `INSERT INTO errors (*) VALUES ${_join(errorData, ',')}`
-
- try {
- await clickhouse.query({
- query,
- })
- } catch (e) {
- console.error(
- `[CRON WORKER] Error whilst saving error events data: ${e}`,
- )
- }
- }
-
- if (!_isEmpty(perfData)) {
- await redis.del(REDIS_LOG_PERF_CACHE_KEY)
-
- const query = `INSERT INTO performance (*) VALUES ${_join(perfData, ',')}`
-
- try {
- await clickhouse.query({
- query,
- })
- } catch (e) {
- console.error(
- `[CRON WORKER] Error whilst saving performance data: ${e}`,
- )
- }
- }
- }
-
@Cron(CronExpression.EVERY_DAY_AT_5PM)
async lockDashboards(): Promise<void> {
const sevenDaysAgo = dayjs
diff --git a/apps/selfhosted/src/analytics/analytics.controller.ts b/apps/selfhosted/src/analytics/analytics.controller.ts
index 1b912908d..6ea96aa6c 100644
--- a/apps/selfhosted/src/analytics/analytics.controller.ts
+++ b/apps/selfhosted/src/analytics/analytics.controller.ts
@@ -3,10 +3,7 @@ import * as _isArray from 'lodash/isArray'
import * as _pick from 'lodash/pick'
import * as _map from 'lodash/map'
import * as _uniqBy from 'lodash/uniqBy'
-import * as _round from 'lodash/round'
import * as _includes from 'lodash/includes'
-import * as _keys from 'lodash/keys'
-import * as _values from 'lodash/values'
import * as dayjs from 'dayjs'
import * as utc from 'dayjs/plugin/utc'
import * as dayjsTimezone from 'dayjs/plugin/timezone'
@@ -54,13 +51,9 @@ import { GetUserFlowDTO } from './dto/getUserFlow.dto'
import { GetFunnelsDTO } from './dto/getFunnels.dto'
import { AppLoggerService } from '../logger/logger.service'
import {
- REDIS_LOG_DATA_CACHE_KEY,
- REDIS_LOG_PERF_CACHE_KEY,
redis,
- REDIS_LOG_CUSTOM_CACHE_KEY,
HEARTBEAT_SID_LIFE_TIME,
REDIS_SESSION_SALT_KEY,
- REDIS_LOG_ERROR_CACHE_KEY,
} from '../common/constants'
import { clickhouse } from '../common/integrations/clickhouse'
import {
@@ -84,9 +77,12 @@ import { ErrorDTO } from './dto/error.dto'
import { GetErrorsDto } from './dto/get-errors.dto'
import { GetErrorDTO } from './dto/get-error.dto'
import { PatchStatusDTO } from './dto/patch-status.dto'
-
-// eslint-disable-next-line @typescript-eslint/no-var-requires
-const mysql = require('mysql2')
+import {
+ customEventTransformer,
+ errorEventTransformer,
+ performanceTransformer,
+ trafficTransformer,
+} from './utils/transformers'
dayjs.extend(utc)
dayjs.extend(dayjsTimezone)
@@ -101,177 +97,6 @@ const getSessionKeyCustom = (
salt = '',
) => `cses_${hash(`${ua}${ip}${pid}${ev}${salt}`).toString('hex')}`
-const analyticsDTO = (
- psid: string,
- sid: string,
- pid: string,
- pg: string,
- prev: string,
- dv: string,
- br: string,
- os: string,
- lc: string,
- ref: string,
- so: string,
- me: string,
- ca: string,
- cc: string,
- rg: string,
- ct: string,
- keys: string[],
- values: string[],
- sdur: number | string,
- unique: number,
-): Array<string | number | string[]> => {
- return [
- psid,
- sid,
- pid,
- pg,
- prev,
- dv,
- br,
- os,
- lc,
- ref,
- so,
- me,
- ca,
- cc,
- rg,
- ct,
- keys,
- values,
- sdur,
- unique,
- dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
- ]
-}
-
-const performanceDTO = (
- pid: string,
- pg: string,
- dv: string,
- br: string,
- cc: string,
- rg: string,
- ct: string,
- dns: number,
- tls: number,
- conn: number,
- response: number,
- render: number,
- domLoad: number,
- pageLoad: number,
- ttfb: number,
-): Array<string | number> => {
- return [
- pid,
- pg,
- dv,
- br,
- cc,
- rg,
- ct,
- _round(dns),
- _round(tls),
- _round(conn),
- _round(response),
- _round(render),
- _round(domLoad),
- _round(pageLoad),
- _round(ttfb),
- dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
- ]
-}
-
-const customLogDTO = (
- psid: string,
- pid: string,
- ev: string,
- pg: string,
- dv: string,
- br: string,
- os: string,
- lc: string,
- ref: string,
- so: string,
- me: string,
- ca: string,
- cc: string,
- rg: string,
- ct: string,
- keys: string[],
- values: string[],
-): Array<string | number | string[]> => {
- return [
- psid,
- pid,
- ev,
- pg,
- dv,
- br,
- os,
- lc,
- ref,
- so,
- me,
- ca,
- cc,
- rg,
- ct,
- keys,
- values,
- dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
- ]
-}
-
-const errorLogDTO = (
- eid: string,
- pid: string,
- pg: string,
- dv: string,
- br: string,
- os: string,
- lc: string,
- cc: string,
- rg: string,
- ct: string,
- name: string,
- message: string,
- lineno: number,
- colno: number,
- filename: string,
-): Array<string | number | string[]> => {
- return [
- eid,
- pid,
- pg,
- dv,
- br,
- os,
- lc,
- cc,
- rg,
- ct,
- name,
- message,
- lineno,
- colno,
- filename,
- dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
- ]
-}
-
-const getElValue = el => {
- if (_isArray(el)) {
- return `[${_map(el, getElValue).join(',')}]`
- }
-
- if (el === undefined || el === null || el === 'NULL') return 'NULL'
- return mysql.escape(el)
-}
-
// Performance object validator: none of the values cannot be bigger than 1000 * 60 * 5 (5 minutes) and are >= 0
const MAX_PERFORMANCE_VALUE = 1000 * 60 * 5
const isPerformanceValid = (perf: any) => {
@@ -1117,7 +942,7 @@ export class AnalyticsController {
const sessionHash = getSessionKey(ip, userAgent, eventsDTO.pid, salt)
const [, psid] = await this.analyticsService.isUnique(sessionHash)
- const dto = customLogDTO(
+ const transformed = customEventTransformer(
psid,
eventsDTO.pid,
eventsDTO.ev,
@@ -1133,13 +958,18 @@ export class AnalyticsController {
country,
region,
city,
- _keys(eventsDTO.meta),
- _values(eventsDTO.meta),
+ eventsDTO.meta,
)
try {
- const values = `(${dto.map(getElValue).join(',')})`
- await redis.rpush(REDIS_LOG_CUSTOM_CACHE_KEY, values)
+ await clickhouse.insert({
+ table: 'customEV',
+ format: 'JSONEachRow',
+ values: [transformed],
+ clickhouse_settings: {
+ async_insert: 1,
+ },
+ })
} catch (e) {
this.logger.error(e)
throw new InternalServerErrorException(
@@ -1215,7 +1045,7 @@ export class AnalyticsController {
const br = ua.browser.name
const os = ua.os.name
- const dto = analyticsDTO(
+ const transformed = trafficTransformer(
psid,
sessionHash,
logDTO.pid,
@@ -1232,13 +1062,12 @@ export class AnalyticsController {
country,
region,
city,
- _keys(logDTO.meta),
- _values(logDTO.meta),
+ logDTO.meta,
0,
Number(unique),
)
- let perfDTO: Array<string | number> = []
+ let perfTransformed = null
if (!_isEmpty(logDTO.perf) && isPerformanceValid(logDTO.perf)) {
const {
@@ -1252,7 +1081,7 @@ export class AnalyticsController {
ttfb,
} = logDTO.perf
- perfDTO = performanceDTO(
+ perfTransformed = performanceTransformer(
logDTO.pid,
logDTO.pg,
dv,
@@ -1271,13 +1100,25 @@ export class AnalyticsController {
)
}
- const values = `(${dto.map(getElValue).join(',')})`
try {
- await redis.rpush(REDIS_LOG_DATA_CACHE_KEY, values)
+ await clickhouse.insert({
+ table: 'analytics',
+ format: 'JSONEachRow',
+ values: [transformed],
+ clickhouse_settings: {
+ async_insert: 1,
+ },
+ })
- if (!_isEmpty(perfDTO)) {
- const perfValues = `(${perfDTO.map(getElValue).join(',')})`
- await redis.rpush(REDIS_LOG_PERF_CACHE_KEY, perfValues)
+ if (!_isEmpty(perfTransformed)) {
+ await clickhouse.insert({
+ table: 'performance',
+ format: 'JSONEachRow',
+ values: [perfTransformed],
+ clickhouse_settings: {
+ async_insert: 1,
+ },
+ })
}
} catch (e) {
this.logger.error(e)
@@ -1330,32 +1171,37 @@ export class AnalyticsController {
country = 'NULL',
} = getGeoDetails(ip, null, headers)
- const dto = analyticsDTO(
+ const transformed = trafficTransformer(
psid,
sessionHash,
logDTO.pid,
- 'NULL',
- 'NULL',
+ null,
+ null,
dv,
br,
os,
- 'NULL',
- 'NULL',
- 'NULL',
- 'NULL',
- 'NULL',
- city,
- region,
+ null,
+ null,
+ null,
+ null,
+ null,
country,
- [],
- [],
+ region,
+ city,
+ null,
0,
Number(unique),
)
- const values = `(${dto.map(getElValue).join(',')})`
try {
- await redis.rpush(REDIS_LOG_DATA_CACHE_KEY, values)
+ await clickhouse.insert({
+ table: 'analytics',
+ format: 'JSONEachRow',
+ values: [transformed],
+ clickhouse_settings: {
+ async_insert: 1,
+ },
+ })
} catch (e) {
this.logger.error(e)
}
@@ -1635,7 +1481,7 @@ export class AnalyticsController {
const { name, message, lineno, colno, filename } = errorDTO
- const dto = errorLogDTO(
+ const transformed = errorEventTransformer(
this.analyticsService.getErrorID(errorDTO),
errorDTO.pid,
errorDTO.pg,
@@ -1654,8 +1500,14 @@ export class AnalyticsController {
)
try {
- const values = `(${dto.map(getElValue).join(',')})`
- await redis.rpush(REDIS_LOG_ERROR_CACHE_KEY, values)
+ await clickhouse.insert({
+ table: 'errors',
+ format: 'JSONEachRow',
+ values: [transformed],
+ clickhouse_settings: {
+ async_insert: 1,
+ },
+ })
} catch (e) {
this.logger.error(e)
throw new InternalServerErrorException(
diff --git a/apps/selfhosted/src/analytics/utils/transformers.ts b/apps/selfhosted/src/analytics/utils/transformers.ts
new file mode 100644
index 000000000..1ab11b888
--- /dev/null
+++ b/apps/selfhosted/src/analytics/utils/transformers.ts
@@ -0,0 +1,182 @@
+import * as _round from 'lodash/round'
+import * as _isEmpty from 'lodash/isEmpty'
+import * as _keys from 'lodash/keys'
+import * as _values from 'lodash/values'
+import * as dayjs from 'dayjs'
+import * as utc from 'dayjs/plugin/utc'
+
+dayjs.extend(utc)
+
+const processMetaKV = (
+ rawMeta: Record<string, string>,
+): { 'meta.key': string | null; 'meta.value': string | null } => {
+ if (!rawMeta || _isEmpty(rawMeta)) {
+ return {
+ 'meta.key': null,
+ 'meta.value': null,
+ }
+ }
+
+ return {
+ 'meta.key': _keys(rawMeta),
+ 'meta.value': _values(rawMeta),
+ }
+}
+
+export const trafficTransformer = (
+ psid: string,
+ sid: string,
+ pid: string,
+ pg: string | null,
+ prev: string | null,
+ dv: string | null,
+ br: string | null,
+ os: string | null,
+ lc: string | null,
+ ref: string | null,
+ so: string | null,
+ me: string | null,
+ ca: string | null,
+ cc: string | null,
+ rg: string | null,
+ ct: string | null,
+ meta: Record<string, string> | null,
+ sdur: number,
+ unique: number,
+) => {
+ return {
+ psid,
+ sid,
+ pid,
+ pg: pg || null,
+ prev: prev || null,
+ dv: dv || null,
+ br: br || null,
+ os: os || null,
+ lc: lc || null,
+ ref: ref || null,
+ so: so || null,
+ me: me || null,
+ ca: ca || null,
+ cc: cc || null,
+ rg: rg || null,
+ ct: ct || null,
+ ...processMetaKV(meta),
+ sdur,
+ unique,
+ created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
+ }
+}
+
+export const customEventTransformer = (
+ psid: string,
+ pid: string,
+ ev: string,
+ pg: string | null,
+ dv: string | null,
+ br: string | null,
+ os: string | null,
+ lc: string | null,
+ ref: string | null,
+ so: string | null,
+ me: string | null,
+ ca: string | null,
+ cc: string | null,
+ rg: string | null,
+ ct: string | null,
+ meta: Record<string, string> | null,
+) => {
+ return {
+ psid,
+ pid,
+ ev,
+ pg: pg || null,
+ dv: dv || null,
+ br: br || null,
+ os: os || null,
+ lc: lc || null,
+ ref: ref || null,
+ so: so || null,
+ me: me || null,
+ ca: ca || null,
+ cc: cc || null,
+ rg: rg || null,
+ ct: ct || null,
+ ...processMetaKV(meta),
+ created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
+ }
+}
+
+export const errorEventTransformer = (
+ eid: string,
+ pid: string,
+ pg: string | null,
+ dv: string | null,
+ br: string | null,
+ os: string | null,
+ lc: string | null,
+ cc: string | null,
+ rg: string | null,
+ ct: string | null,
+ name: string | null,
+ message: string | null,
+ lineno: number | null,
+ colno: number | null,
+ filename: string | null,
+) => {
+ return {
+ eid,
+ pid,
+ pg: pg || null,
+ dv: dv || null,
+ br: br || null,
+ os: os || null,
+ lc: lc || null,
+ cc: cc || null,
+ rg: rg || null,
+ ct: ct || null,
+ name: name || null,
+ message: message || null,
+ lineno: lineno || null,
+ colno: colno || null,
+ filename: filename || null,
+ created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
+ }
+}
+
+export const performanceTransformer = (
+ pid: string,
+ pg: string | null,
+ dv: string | null,
+ br: string | null,
+ cc: string | null,
+ rg: string | null,
+ ct: string | null,
+ dns: number,
+ tls: number,
+ conn: number,
+ response: number,
+ render: number,
+ domLoad: number,
+ pageLoad: number,
+ ttfb: number,
+) => {
+ return {
+ pid,
+ pg: pg || null,
+ dv: dv || null,
+ br: br || null,
+ cc: cc || null,
+ rg: rg || null,
+ ct: ct || null,
+ dns: _round(dns),
+ tls: _round(tls),
+ conn: _round(conn),
+ response: _round(response),
+ render: _round(render),
+ domLoad: _round(domLoad),
+ pageLoad: _round(pageLoad),
+ ttfb: _round(ttfb),
+ created: dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
+ }
+}
diff --git a/apps/selfhosted/src/common/constants.ts b/apps/selfhosted/src/common/constants.ts
index 7ac72181c..af6c249b3 100644
--- a/apps/selfhosted/src/common/constants.ts
+++ b/apps/selfhosted/src/common/constants.ts
@@ -50,14 +50,9 @@ const isValidPID = (pid: string) => PID_REGEX.test(pid)
// redis keys
const getRedisProjectKey = (pid: string) => `pid_${pid}`
-const REDIS_LOG_DATA_CACHE_KEY = 'log_cache_v2'
-const REDIS_LOG_CAPTCHA_CACHE_KEY = 'log:captcha'
-const REDIS_LOG_PERF_CACHE_KEY = 'perf_cache'
-const REDIS_LOG_CUSTOM_CACHE_KEY = 'log_custom_cache_v3'
const REDIS_SESSION_SALT_KEY = 'log_salt' // is updated every 24 hours
const REDIS_USERS_COUNT_KEY = 'stats:users_count'
const REDIS_PROJECTS_COUNT_KEY = 'stats:projects_count'
-const REDIS_LOG_ERROR_CACHE_KEY = 'log_error_cache'
// 3600 sec -> 1 hour
const redisProjectCacheTimeout = 3600
@@ -103,9 +98,6 @@ export {
getRedisProjectKey,
redisProjectCacheTimeout,
UNIQUE_SESSION_LIFE_TIME,
- REDIS_LOG_DATA_CACHE_KEY,
- REDIS_LOG_CAPTCHA_CACHE_KEY,
- REDIS_LOG_CUSTOM_CACHE_KEY,
TRAFFIC_METAKEY_COLUMNS,
REDIS_SESSION_SALT_KEY,
HEARTBEAT_SID_LIFE_TIME,
@@ -119,7 +111,6 @@ export {
REDIS_PROJECTS_COUNT_KEY,
IP_REGEX,
ORIGINS_REGEX,
- REDIS_LOG_PERF_CACHE_KEY,
isDevelopment,
DEFAULT_SELFHOSTED_UUID,
JWT_ACCESS_TOKEN_SECRET,
@@ -131,7 +122,6 @@ export {
MIN_PAGES_IN_FUNNEL,
MAX_PAGES_IN_FUNNEL,
ERROR_COLUMNS,
- REDIS_LOG_ERROR_CACHE_KEY,
PID_REGEX,
ALL_COLUMNS,
}
diff --git a/apps/selfhosted/src/task-manager/task-manager.service.ts b/apps/selfhosted/src/task-manager/task-manager.service.ts
index 99c661726..feea927c7 100644
--- a/apps/selfhosted/src/task-manager/task-manager.service.ts
+++ b/apps/selfhosted/src/task-manager/task-manager.service.ts
@@ -3,21 +3,11 @@ import { Cron, CronExpression } from '@nestjs/schedule'
import * as bcrypt from 'bcrypt'
import * as dayjs from 'dayjs'
import * as utc from 'dayjs/plugin/utc'
-import * as _isEmpty from 'lodash/isEmpty'
-import * as _join from 'lodash/join'
import * as _size from 'lodash/size'
import * as _map from 'lodash/map'
import * as _now from 'lodash/now'
-import {
- redis,
- REDIS_LOG_DATA_CACHE_KEY,
- REDIS_LOG_CUSTOM_CACHE_KEY,
- REDIS_SESSION_SALT_KEY,
- REDIS_LOG_PERF_CACHE_KEY,
- REDIS_LOG_CAPTCHA_CACHE_KEY,
- REDIS_LOG_ERROR_CACHE_KEY,
-} from '../common/constants'
+import { redis, REDIS_SESSION_SALT_KEY } from '../common/constants'
import { clickhouse } from '../common/integrations/clickhouse'
import { AppLoggerService } from '../logger/logger.service'
@@ -27,87 +17,6 @@ dayjs.extend(utc)
export class TaskManagerService {
constructor(private readonly logger: AppLoggerService) {}
- @Cron(CronExpression.EVERY_MINUTE)
- async saveLogData(): Promise<void> {
- const data = await redis.lrange(REDIS_LOG_DATA_CACHE_KEY, 0, -1)
- const customData = await redis.lrange(REDIS_LOG_CUSTOM_CACHE_KEY, 0, -1)
- const perfData = await redis.lrange(REDIS_LOG_PERF_CACHE_KEY, 0, -1)
- const captchaData = await redis.lrange(REDIS_LOG_CAPTCHA_CACHE_KEY, 0, -1)
- const errorData = await redis.lrange(REDIS_LOG_ERROR_CACHE_KEY, 0, -1)
-
- if (!_isEmpty(data)) {
- await redis.del(REDIS_LOG_DATA_CACHE_KEY)
- const query = `INSERT INTO analytics (*) VALUES ${_join(data, ',')}`
- try {
- await clickhouse.query({
- query,
- })
- } catch (e) {
- console.error(`[CRON WORKER] Error whilst saving log data: ${e}`)
- }
- }
-
- if (!_isEmpty(captchaData)) {
- await redis.del(REDIS_LOG_CAPTCHA_CACHE_KEY)
- const query = `INSERT INTO captcha (*) VALUES ${_join(captchaData, ',')}`
- try {
- await clickhouse.query({
- query,
- })
- } catch (e) {
- console.error(
- `[CRON WORKER] Error whilst saving CAPTCHA log data: ${e}`,
- )
- }
- }
-
- if (!_isEmpty(customData)) {
- await redis.del(REDIS_LOG_CUSTOM_CACHE_KEY)
- const query = `INSERT INTO customEV (*) VALUES ${_join(customData, ',')}`
-
- try {
- await clickhouse.query({
- query,
- })
- } catch (e) {
- console.error(
- `[CRON WORKER] Error whilst saving custom events data: ${e}`,
- )
- }
- }
-
- if (!_isEmpty(errorData)) {
- await redis.del(REDIS_LOG_ERROR_CACHE_KEY)
- const query = `INSERT INTO errors (*) VALUES ${_join(errorData, ',')}`
-
- try {
- await clickhouse.query({
- query,
- })
- } catch (e) {
- console.error(
- `[CRON WORKER] Error whilst saving error events data: ${e}`,
- )
- }
- }
-
- if (!_isEmpty(perfData)) {
- await redis.del(REDIS_LOG_PERF_CACHE_KEY)
-
- const query = `INSERT INTO performance (*) VALUES ${_join(perfData, ',')}`
-
- try {
- await clickhouse.query({
- query,
- })
- } catch (e) {
- console.error(
- `[CRON WORKER] Error whilst saving performance data: ${e}`,
- )
- }
- }
- }
-
@Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)
async generateSessionSalt(): Promise<void> {
const salt = await bcrypt.genSalt(10)
diff --git a/migrations/clickhouse/setup.js b/migrations/clickhouse/setup.js
index 0c3765bec..7d259754d 100644
--- a/migrations/clickhouse/setup.js
+++ b/migrations/clickhouse/setup.js
@@ -49,7 +49,7 @@ const databaselessQueriesRunner = async queries => {
if (query) {
try {
// eslint-disable-next-line no-await-in-loop
- await clickhouseNoDatabase.query({
+ await clickhouseNoDatabase.command({
query,
})
console.log(chalk.green('Query OK: '), query)
@@ -73,7 +73,7 @@ const queriesRunner = async (queries, log = true) => {
if (query) {
try {
// eslint-disable-next-line no-await-in-loop
- await clickhouse.query({
+ await clickhouse.command({
query,
})
diff --git a/migrations/clickhouse/test_migration.js b/migrations/clickhouse/test_migration.js
index 35d8d3fdd..e20dec9c5 100644
--- a/migrations/clickhouse/test_migration.js
+++ b/migrations/clickhouse/test_migration.js
@@ -1,7 +1,5 @@
const { queriesRunner, dbName } = require('./setup')
-const queries = [
- `SELECT * FROM ${dbName}.analytics LIMIT 10`,
-]
+const queries = [`SELECT * FROM ${dbName}.analytics LIMIT 10`]
queriesRunner(queries)