📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
selfhosted: migrate to the official clickhouse/client library
Blue Mousee09d2e38a0b5
diff --git a/apps/selfhosted/src/analytics/analytics.controller.ts b/apps/selfhosted/src/analytics/analytics.controller.tsindex aed291262..1b912908d 100644--- a/apps/selfhosted/src/analytics/analytics.controller.ts+++ b/apps/selfhosted/src/analytics/analytics.controller.ts@@ -60,9 +60,9 @@ import {REDIS_LOG_CUSTOM_CACHE_KEY,HEARTBEAT_SID_LIFE_TIME,REDIS_SESSION_SALT_KEY,- clickhouse,REDIS_LOG_ERROR_CACHE_KEY,} from '../common/constants'+import { clickhouse } from '../common/integrations/clickhouse'import {checkRateLimit,getGeoDetails,@@ -1031,11 +1031,11 @@ export class AnalyticsController {@Get('liveVisitors')@Auth([], true, true)async getLiveVisitors(- @Query() data,+ @Query() queryParams,@CurrentUserId() uid: string,@Headers() headers: { 'x-password'?: string },): Promise<object> {- const { pid } = data+ const { pid } = queryParamsthis.analyticsService.validatePID(pid)await this.analyticsService.checkProjectAccess(@@ -1055,8 +1055,12 @@ export class AnalyticsController {const query = `SELECT sid, dv, br, os, cc FROM analytics WHERE sid IN (${sids.map(el => `'${el}'`).join(',')})`- const result = await clickhouse.query(query).toPromise()- const processed = _map(_uniqBy(result, 'sid'), el =>+ const { data } = await clickhouse+ .query({+ query,+ })+ .then(resultSet => resultSet.json())+ const processed = _map(_uniqBy(data, 'sid'), el =>_pick(el, ['dv', 'br', 'os', 'cc']),)diff --git a/apps/selfhosted/src/analytics/analytics.service.ts b/apps/selfhosted/src/analytics/analytics.service.tsindex e0c1d5c93..b2ee34ab5 100644--- a/apps/selfhosted/src/analytics/analytics.service.ts+++ b/apps/selfhosted/src/analytics/analytics.service.ts@@ -42,7 +42,6 @@ import {redis,isValidPID,UNIQUE_SESSION_LIFE_TIME,- clickhouse,REDIS_SESSION_SALT_KEY,TRAFFIC_COLUMNS,PERFORMANCE_COLUMNS,@@ -52,6 +51,7 @@ import {ALL_COLUMNS,TRAFFIC_METAKEY_COLUMNS,} from '../common/constants'+import { clickhouse } from '../common/integrations/clickhouse'import {getFunnelsClickhouse,millisecondsToSeconds,@@ -471,11 +471,16 @@ export class AnalyticsService {const query = `SELECT ${type} FROM errors WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}`- const results = await clickhouse- .query(query, { params: { pid } })- .toPromise()+ const { data } = await clickhouse+ .query({+ query,+ query_params: {+ pid,+ },+ })+ .then(resultSet => resultSet.json())- return _map(results, type)+ return _map(data, type)}getDataTypeColumns(dataType: DataType): string[] {@@ -633,7 +638,10 @@ export class AnalyticsService {return { nodes, links }}- async getUserFlow(params: unknown, filtersQuery: string): Promise<IUserFlow> {+ async getUserFlow(+ params: Record<string, unknown>,+ filtersQuery: string,+ ): Promise<IUserFlow> {const query = `SELECTpg AS source,@@ -650,11 +658,14 @@ export class AnalyticsService {prev`- const results = <IUserFlowLink[]>(- await clickhouse.query(query, { params }).toPromise()- )+ const { data } = await clickhouse+ .query({+ query,+ query_params: params,+ })+ .then(resultSet => resultSet.json<IUserFlowLink>())- if (_isEmpty(results)) {+ if (_isEmpty(data)) {const empty = { nodes: [], links: [] }return {ascending: empty,@@ -665,7 +676,7 @@ export class AnalyticsService {const ascendingLinks: IUserFlowLink[] = []const descendingLinks: IUserFlowLink[] = []- this.removeCyclicDependencies(results).forEach((row: any) => {+ this.removeCyclicDependencies(data).forEach((row: any) => {const link: IUserFlowLink = {source: row.source,target: row.target,@@ -790,12 +801,14 @@ export class AnalyticsService {return null}- const from: any = await clickhouse- .query(- 'SELECT created FROM analytics where pid = {pid:FixedString(12)} ORDER BY created ASC LIMIT 1',- { params: { pid } },- )- .toPromise()+ const { data: from } = await clickhouse+ .query({+ query:+ 'SELECT created FROM analytics WHERE pid = {pid:FixedString(12)} ORDER BY created ASC LIMIT 1',+ query_params: { pid },+ })+ .then(res => res.json<{ created?: string }>())+const to = _includes(GMT_0_TIMEZONES, safeTimezone)? dayjs.utc(): dayjs().tz(safeTimezone)@@ -1169,20 +1182,18 @@ export class AnalyticsService {ORDER BY level DESC;`- const result = <Array<IFunnelCHResponse>>(- await clickhouse- .query(query, { params: { ...params, ...pageParams } })- .toPromise()- )+ const { data } = await clickhouse+ .query({+ query,+ query_params: { ...params, ...pageParams },+ })+ .then(resultSet => resultSet.json<IFunnelCHResponse>())- if (_isEmpty(result)) {+ if (_isEmpty(data)) {return this.generateEmptyFunnel(pages)}- return this.formatFunnel(- _reverse(this.backfillFunnel(result, pages)),- pages,- )+ return this.formatFunnel(_reverse(this.backfillFunnel(data, pages)), pages)}async getTotalPageviews(@@ -1197,13 +1208,14 @@ export class AnalyticsService {WHERE pid = {pid:FixedString(12)}AND created BETWEEN {groupFrom:String} AND {groupTo:String}`- const result = <{ c: number }[]>(- await clickhouse- .query(query, { params: { pid, groupFrom, groupTo } })- .toPromise()- )+ const { data } = await clickhouse+ .query({+ query,+ query_params: { pid, groupFrom, groupTo },+ })+ .then(resultSet => resultSet.json<{ c: number }>())- return result[0]?.c || 0+ return data[0]?.c || 0}async getAnalyticsSummary(@@ -1256,27 +1268,28 @@ export class AnalyticsService {queryAll = `SELECT count(*) AS all FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery}`}- const rawResult = <Array<BirdseyeCHResponse>>await clickhouse- .query(queryAll, {- params: { pid, ...filtersParams },+ const { data } = await clickhouse+ .query({+ query: queryAll,+ query_params: {+ pid,+ ...filtersParams,+ },})- .toPromise()+ .then(resultSet => resultSet.json<BirdseyeCHResponse>())let bounceRate = 0- if (rawResult[0].all > 0 && !customEVFilterApplied) {- bounceRate = _round(- (rawResult[0].unique * 100) / rawResult[0].all,- 1,- )+ if (data[0].all > 0 && !customEVFilterApplied) {+ bounceRate = _round((data[0].unique * 100) / data[0].all, 1)}result[pid] = {current: {- all: rawResult[0].all,- unique: rawResult[0].unique,+ all: data[0].all,+ unique: data[0].unique,bounceRate,- sdur: rawResult[0].sdur,+ sdur: data[0].sdur,},previous: {all: 0,@@ -1284,10 +1297,10 @@ export class AnalyticsService {bounceRate: 0,sdur: 0,},- change: rawResult[0].all,- uniqueChange: rawResult[0].unique,+ change: data[0].all,+ uniqueChange: data[0].unique,bounceRateChange: bounceRate,- sdurChange: rawResult[0].sdur,+ sdurChange: data[0].sdur,customEVFilterApplied,}return@@ -1328,9 +1341,10 @@ export class AnalyticsService {const query = `${queryCurrent} UNION ALL ${queryPrevious}`- let rawResult = <Array<BirdseyeCHResponse>>await clickhouse- .query(query, {- params: {+ let { data } = await clickhouse+ .query({+ query,+ query_params: {pid,periodFormatted,periodSubtracted,@@ -1338,12 +1352,12 @@ export class AnalyticsService {...filtersParams,},})- .toPromise()+ .then(resultSet => resultSet.json<BirdseyeCHResponse>())- rawResult = _sortBy(rawResult, 'sortOrder')+ data = _sortBy(data, 'sortOrder')- const currentPeriod = rawResult[0]- const previousPeriod = rawResult[1]+ const currentPeriod = data[0]+ const previousPeriod = data[1]let bounceRate = 0let prevBounceRate = 0@@ -1452,26 +1466,23 @@ export class AnalyticsService {try {if (period === 'all') {const queryAll = `SELECT ${columnSelectors} FROM performance WHERE pid = {pid:FixedString(12)} ${filtersQuery}`- const rawResult = <Array<Partial<PerformanceCHResponse>>>(- await clickhouse- .query(queryAll, {- params: { pid, ...filtersParams },- })- .toPromise()- )+ const { data } = await clickhouse+ .query({+ query: queryAll,+ query_params: {+ pid,+ ...filtersParams,+ },+ })+ .then(resultSet => resultSet.json<Partial<PerformanceCHResponse>>())result[pid] = {current: {- frontend: millisecondsToSeconds(- rawResult[0].render + rawResult[0].domLoad,- ),+ frontend: millisecondsToSeconds(data[0].render + data[0].domLoad),network: millisecondsToSeconds(- rawResult[0].dns +- rawResult[0].tls +- rawResult[0].conn +- rawResult[0].response,+ data[0].dns + data[0].tls + data[0].conn + data[0].response,),- backend: millisecondsToSeconds(rawResult[0].ttfb),+ backend: millisecondsToSeconds(data[0].ttfb),},previous: {frontend: 0,@@ -1479,15 +1490,12 @@ export class AnalyticsService {backend: 0,},frontendChange: millisecondsToSeconds(- rawResult[0].render + rawResult[0].domLoad,+ data[0].render + data[0].domLoad,),networkChange: millisecondsToSeconds(- rawResult[0].dns +- rawResult[0].tls +- rawResult[0].conn +- rawResult[0].response,+ data[0].dns + data[0].tls + data[0].conn + data[0].response,),- backendChange: millisecondsToSeconds(rawResult[0].ttfb),+ backendChange: millisecondsToSeconds(data[0].ttfb),}return}@@ -1522,9 +1530,10 @@ export class AnalyticsService {const query = `${queryCurrent} UNION ALL ${queryPrevious}`- let rawResult = <Array<Partial<PerformanceCHResponse>>>await clickhouse- .query(query, {- params: {+ let { data } = await clickhouse+ .query({+ query,+ query_params: {pid,periodFormatted,periodSubtracted,@@ -1532,12 +1541,12 @@ export class AnalyticsService {...filtersParams,},})- .toPromise()+ .then(resultSet => resultSet.json<Partial<PerformanceCHResponse>>())- rawResult = _sortBy(rawResult, 'sortOrder')+ data = _sortBy(data, 'sortOrder')- const currentPeriod = rawResult[0]- const previousPeriod = rawResult[1]+ const currentPeriod = data[0]+ const previousPeriod = data[1]result[pid] = {current: {@@ -1614,11 +1623,16 @@ export class AnalyticsService {'SELECT ev FROM customEV WHERE pid={pid:FixedString(12)} AND ev IS NOT NULL GROUP BY ev'}- const results = await clickhouse- .query(query, { params: { pid } })- .toPromise()+ const { data } = await clickhouse+ .query({+ query,+ query_params: {+ pid,+ },+ })+ .then(resultSet => resultSet.json())- return _map(results, type)+ return _map(data, type)}async generateParams(@@ -1661,9 +1675,14 @@ export class AnalyticsService {type,measure,)- const res = await clickhouse.query(query, paramsData).toPromise()-- params[col] = res+ const { data } = await clickhouse+ .query({+ query,+ query_params: paramsData.params,+ })+ .then(resultSet => resultSet.json())++ params[col] = data})await Promise.all(paramsPromises)@@ -1671,18 +1690,6 @@ export class AnalyticsService {return params}- async calculateAverageSessionDuration(- subQuery: string,- paramsData: any,- ): Promise<number> {- const avgSdurQuery = `SELECT avg(sdur) ${subQuery} AND sdur IS NOT NULL AND unique='1'`- const avgSdurObject = await clickhouse- .query(avgSdurQuery, paramsData)- .toPromise()-- return _round(avgSdurObject[0]['avg(sdur)'])- }-generateXAxis(timeBucket: TimeBucketType,from: string, // timezone is already applied to the from and to parameters@@ -2217,13 +2224,15 @@ export class AnalyticsService {mode,)- const result = <Array<TrafficCEFilterCHResponse>>(- await clickhouse.query(query, paramsData).toPromise()- )+ const { data } = await clickhouse+ .query({+ query,+ query_params: paramsData.params,+ })+ .then(resultSet => resultSet.json<TrafficCEFilterCHResponse>())const uniques =- this.extractCustomEventsChartData(result, xShifted)?._unknown_event ||- []+ this.extractCustomEventsChartData(data, xShifted)?._unknown_event || []const sdur = Array(_size(xShifted)).fill(0)@@ -2244,11 +2253,14 @@ export class AnalyticsService {mode,)- const result = <Array<TrafficCHResponse>>(- await clickhouse.query(query, paramsData).toPromise()- )+ const { data } = await clickhouse+ .query({+ query,+ query_params: paramsData.params,+ })+ .then(resultSet => resultSet.json<TrafficCHResponse>())- const { visits, uniques, sdur } = this.extractChartData(result, xShifted)+ const { visits, uniques, sdur } = this.extractChartData(data, xShifted)return Promise.resolve({chart: {@@ -2352,9 +2364,12 @@ export class AnalyticsService {// ChartRenderMode.PERIODICAL,// )- // const result = <Array<TrafficCEFilterCHResponse>>(- // await clickhouse.query(query, paramsData).toPromise()- // )+ // const { data } = await clickhouse+ // .query({+ // query,+ // query_params: paramsData.params,+ // })+ // .then(resultSet => resultSet.json<TrafficCEFilterCHResponse>())// const uniques =// this.extractCustomEventsChartData(result, xShifted)?._unknown_event ||@@ -2374,11 +2389,14 @@ export class AnalyticsService {const query = this.generateSessionAggregationQuery(timeBucket, filtersQuery)- const result = <Array<TrafficCHResponse>>(- await clickhouse.query(query, paramsData).toPromise()- )+ const { data } = await clickhouse+ .query({+ query,+ query_params: paramsData.params,+ })+ .then(resultSet => resultSet.json<TrafficCHResponse>())- const { visits, uniques, sdur } = this.extractChartData(result, x)+ const { visits, uniques, sdur } = this.extractChartData(data, x)return Promise.resolve({chart: {@@ -2396,7 +2414,7 @@ export class AnalyticsService {to: string,subQuery: string,filtersQuery: string,- paramsData: object,+ paramsData: any,safeTimezone: string,mode: ChartRenderMode,): Promise<object | void> {@@ -2431,10 +2449,13 @@ export class AnalyticsService {mode,)- const result = <Array<TrafficCEFilterCHResponse>>(- await clickhouse.query(query, paramsData).toPromise()- )- const { count } = this.extractErrorsChartData(result, x)+ const { data } = await clickhouse+ .query({+ query,+ query_params: paramsData.params,+ })+ .then(resultSet => resultSet.json<TrafficCEFilterCHResponse>())+ const { count } = this.extractErrorsChartData(data, x)chart = {x: this.shiftToTimezone(x, safeTimezone, format),@@ -2492,7 +2513,7 @@ export class AnalyticsService {from: string,to: string,filtersQuery: string,- paramsData: object,+ paramsData: any,safeTimezone: string,measure: PerfMeasure,) {@@ -2505,11 +2526,16 @@ export class AnalyticsService {safeTimezone,)- const result = await clickhouse.query(query, paramsData).toPromise()+ const { data } = await clickhouse+ .query({+ query,+ query_params: paramsData.params,+ })+ .then(resultSet => resultSet.json())return {x: xShifted,- ...this.extractPerformanceQuantilesData(result, xShifted),+ ...this.extractPerformanceQuantilesData(data, xShifted),}}@@ -2520,13 +2546,16 @@ export class AnalyticsService {measure,)- const result = <Array<PerformanceCHResponse>>(- await clickhouse.query(query, paramsData).toPromise()- )+ const { data } = await clickhouse+ .query({+ query,+ query_params: paramsData.params,+ })+ .then(resultSet => resultSet.json<PerformanceCHResponse>())return {x: xShifted,- ...this.extractPerformanceChartData(result, xShifted),+ ...this.extractPerformanceChartData(data, xShifted),}}@@ -2536,7 +2565,7 @@ export class AnalyticsService {to: string,subQuery: string,filtersQuery: string,- paramsData: object,+ paramsData: any,safeTimezone: string,measure: PerfMeasure,): Promise<object | void> {@@ -2586,18 +2615,21 @@ export class AnalyticsService {async getCustomEvents(filtersQuery: string,- params: object,+ params: any,): Promise<ICustomEvent> {const query = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`const result = {}- const rawCustoms = <Array<CustomsCHResponse>>(- await clickhouse.query(query, params).toPromise()- )- const size = _size(rawCustoms)+ const { data } = await clickhouse+ .query({+ query,+ query_params: params.params,+ })+ .then(resultSet => resultSet.json<CustomsCHResponse>())+ const size = _size(data)for (let i = 0; i < size; ++i) {- const { ev, 'count()': c } = rawCustoms[i]+ const { ev, 'count()': c } = data[i]result[ev] = c}@@ -2618,7 +2650,7 @@ export class AnalyticsService {async getPageProperties(filtersQuery: string,- params: object,+ params: any,): Promise<IPageProperty> {const query = `SELECT@@ -2631,13 +2663,16 @@ export class AnalyticsService {GROUP BY property`const result = {}- const rawProperties = <Array<PropertiesCHResponse>>(- await clickhouse.query(query, params).toPromise()- )- const size = _size(rawProperties)+ const { data } = await clickhouse+ .query({+ query,+ query_params: params.params,+ })+ .then(resultSet => resultSet.json<PropertiesCHResponse>())+ const size = _size(data)for (let i = 0; i < size; ++i) {- const { property: propertyArr, 'count()': c } = rawProperties[i]+ const { property: propertyArr, 'count()': c } = data[i]const [property] = propertyArrif (!property) {@@ -2728,9 +2763,12 @@ export class AnalyticsService {}try {- const result = (await clickhouse- .query(query, paramsData)- .toPromise()) as IAggregatedMetadata[]+ const { data: result } = await clickhouse+ .query({+ query,+ query_params: paramsData.params,+ })+ .then(resultSet => resultSet.json<IAggregatedMetadata>())return {result,@@ -2829,9 +2867,12 @@ export class AnalyticsService {}try {- const result = (await clickhouse- .query(query, paramsData)- .toPromise()) as IAggregatedMetadata[]+ const { data: result } = await clickhouse+ .query({+ query,+ query_params: paramsData.params,+ })+ .then(resultSet => resultSet.json<IAggregatedMetadata>())return {result,@@ -2855,7 +2896,7 @@ export class AnalyticsService {from: string,to: string,filtersQuery: string,- paramsData: object,+ paramsData: any,safeTimezone: string,): Promise<object | void> {const { xShifted } = this.generateXAxis(timeBucket, from, to, safeTimezone)@@ -2883,11 +2924,14 @@ export class AnalyticsService {ORDER BY ${groupBy}`- const result = <Array<CustomsCHAggregatedResponse>>(- await clickhouse.query(query, paramsData).toPromise()- )+ const { data } = await clickhouse+ .query({+ query,+ query_params: paramsData.params,+ })+ .then(resultSet => resultSet.json<CustomsCHAggregatedResponse>())- const events = this.extractCustomEventsChartData(result, xShifted)+ const events = this.extractCustomEventsChartData(data, xShifted)return Promise.resolve({chart: {@@ -2961,11 +3005,21 @@ export class AnalyticsService {},}- const pages = <IPageflow[]>(- await clickhouse.query(queryPages, paramsData).toPromise()- )+ const { data: pages } = await clickhouse+ .query({+ query: queryPages,+ query_params: paramsData.params,+ })+ .then(resultSet => resultSet.json<IPageflow>())+let details = (- await clickhouse.query(querySessionDetails, paramsData).toPromise()+ await clickhouse+ .query({+ query: querySessionDetails,+ query_params: paramsData.params,+ })+ .then(resultSet => resultSet.json())+ .then(({ data }) => data))[0]if (!details) {@@ -2982,8 +3036,12 @@ export class AnalyticsService {// eslint-disable-next-line prefer-destructuringdetails = (await clickhouse- .query(querySessionDetailsBackup, paramsData)- .toPromise()+ .query({+ query: querySessionDetailsBackup,+ query_params: paramsData.params,+ })+ .then(resultSet => resultSet.json())+ .then(({ data }) => data))[0]}@@ -3032,7 +3090,7 @@ export class AnalyticsService {async getSessionsList(filtersQuery: string,- paramsData: object,+ paramsData: any,safeTimezone: string,take = 30,skip = 0,@@ -3089,15 +3147,20 @@ export class AnalyticsService {OFFSET ${skip}`- const result = await clickhouse.query(query, paramsData).toPromise()+ const { data } = await clickhouse+ .query({+ query,+ query_params: paramsData.params,+ })+ .then(resultSet => resultSet.json())- return result+ return data}async getErrorsList(options: string,filtersQuery: string,- paramsData: object,+ paramsData: any,safeTimezone: string,take = 30,skip = 0,@@ -3147,9 +3210,14 @@ export class AnalyticsService {OFFSET ${skip};`- const result = await clickhouse.query(query, paramsData).toPromise()+ const { data } = await clickhouse+ .query({+ query,+ query_params: paramsData.params,+ })+ .then(resultSet => resultSet.json())- return result+ return data}async getErrorDetails(@@ -3215,11 +3283,23 @@ export class AnalyticsService {}const details = (- await clickhouse.query(queryErrorDetails, paramsData).toPromise()+ await clickhouse+ .query({+ query: queryErrorDetails,+ query_params: paramsData.params,+ })+ .then(resultSet => resultSet.json<any>())+ .then(({ data }) => data))[0]const occurenceDetails = (- await clickhouse.query(queryFirstLastSeen, paramsData).toPromise()+ await clickhouse+ .query({+ query: queryFirstLastSeen,+ query_params: paramsData.params,+ })+ .then(resultSet => resultSet.json<any>())+ .then(({ data }) => data))[0]const groupedChart = await this.groupErrorsByTimeBucket(@@ -3269,9 +3349,16 @@ export class AnalyticsService {let resulttry {- ;[result] = await clickhouse- .query(query, { params: { ...params, pid } })- .toPromise()+ // eslint-disable-next-line prefer-destructuring+ result = (+ await clickhouse+ .query({+ query,+ query_params: { ...params, pid },+ })+ .then(resultSet => resultSet.json())+ .then(({ data }) => data)+ )[0]} catch (reason) {console.error('validateEIDs - clickhouse request error')console.error(reason)@@ -3308,7 +3395,13 @@ export class AnalyticsService {).join(', ')}`try {- await clickhouse.query(query, { params: { ...params, pid } }).toPromise()+ await clickhouse.query({+ query,+ query_params: {+ ...params,+ pid,+ },+ })} catch (reason) {console.error('Error at PATCH error-status:')console.error(reason)diff --git a/apps/selfhosted/src/common/constants.ts b/apps/selfhosted/src/common/constants.tsindex 5690541a7..7ac72181c 100644--- a/apps/selfhosted/src/common/constants.ts+++ b/apps/selfhosted/src/common/constants.ts@@ -1,4 +1,3 @@-import { ClickHouse } from 'clickhouse'import Redis from 'ioredis'import * as _toNumber from 'lodash/toNumber'@@ -7,8 +6,6 @@ import { getSelfhostedUUID } from './utils'// eslint-disable-next-line @typescript-eslint/no-var-requiresrequire('dotenv').config()-const { CLICKHOUSE_DATABASE } = process.env-const redis = new Redis(_toNumber(process.env.REDIS_PORT),process.env.REDIS_HOST,@@ -24,26 +21,6 @@ redis.defineCommand('countKeysByPattern', {lua: "return #redis.call('keys', ARGV[1])",})-const clickhouse = new ClickHouse({- url: process.env.CLICKHOUSE_HOST,- port: _toNumber(process.env.CLICKHOUSE_PORT),- debug: false,- basicAuth: {- username: process.env.CLICKHOUSE_USER,- password: process.env.CLICKHOUSE_PASSWORD,- },- isUseGzip: false,- format: 'json',- raw: false,- config: {- session_timeout: 60,- output_format_json_quote_64bit_integers: 0,- enable_http_compression: 0,- database: CLICKHOUSE_DATABASE,- log_queries: 0,- },-})-const {JWT_ACCESS_TOKEN_SECRET,// 30 days@@ -121,7 +98,6 @@ const NUMBER_JWT_REFRESH_TOKEN_LIFETIME = Number(JWT_REFRESH_TOKEN_LIFETIME)const NUMBER_JWT_ACCESS_TOKEN_LIFETIME = Number(JWT_ACCESS_TOKEN_LIFETIME)export {- clickhouse,redis,isValidPID,getRedisProjectKey,diff --git a/apps/selfhosted/src/common/integrations/clickhouse.ts b/apps/selfhosted/src/common/integrations/clickhouse.tsnew file mode 100644index 000000000..039dd8e04--- /dev/null+++ b/apps/selfhosted/src/common/integrations/clickhouse.ts@@ -0,0 +1,21 @@+import { createClient } from '@clickhouse/client'++const clickhouse = createClient({+ host: `${process.env.CLICKHOUSE_HOST}:${process.env.CLICKHOUSE_PORT}`,+ username: process.env.CLICKHOUSE_USER,+ password: process.env.CLICKHOUSE_PASSWORD,+ database: process.env.CLICKHOUSE_DATABASE,+ request_timeout: 60000, // 1 minute+ clickhouse_settings: {+ connect_timeout: 60000,+ date_time_output_format: 'iso',+ max_download_buffer_size: (10 * 1024 * 1024).toString(),+ max_download_threads: 32,+ max_execution_time: 60000,+ output_format_json_quote_64bit_integers: 0,+ enable_http_compression: 0,+ log_queries: 0,+ },+})++export { clickhouse }diff --git a/apps/selfhosted/src/common/utils.ts b/apps/selfhosted/src/common/utils.tsindex af030b49f..2d3c02deb 100644--- a/apps/selfhosted/src/common/utils.ts+++ b/apps/selfhosted/src/common/utils.ts@@ -19,7 +19,6 @@ import * as utc from 'dayjs/plugin/utc'import * as _map from 'lodash/map'import {- clickhouse,redis,DEFAULT_SELFHOSTED_UUID,SELFHOSTED_EMAIL,@@ -27,6 +26,7 @@ import {isDevelopment,isProxiedByCloudflare,} from './constants'+import { clickhouse } from './integrations/clickhouse'import { DEFAULT_TIMEZONE, TimeFormat } from '../user/entities/user.entity'import { Project } from '../project/entity/project.entity'import { Funnel } from '../project/entity/funnel.entity'@@ -72,17 +72,23 @@ const checkRateLimit = async (}const getFunnelsClickhouse = async (projectId: string, funnelId = null) => {- const paramsData = {- params: {- projectId,- funnelId,- },+ const queryParams = {+ projectId,+ funnelId,}if (!funnelId) {const query ='SELECT * FROM funnel WHERE projectId = {projectId:FixedString(12)}'- return clickhouse.query(query, paramsData).toPromise()++ const { data } = await clickhouse+ .query({+ query,+ query_params: queryParams,+ })+ .then(resultSet => resultSet.json())++ return data}const query = `@@ -93,15 +99,20 @@ const getFunnelsClickhouse = async (projectId: string, funnelId = null) => {projectId = {projectId:FixedString(12)}AND id = {funnelId:String}`- const funnel = await clickhouse.query(query, paramsData).toPromise()+ const { data } = await clickhouse+ .query({+ query,+ query_params: queryParams,+ })+ .then(resultSet => resultSet.json())- if (_isEmpty(funnel)) {+ if (_isEmpty(data)) {throw new NotFoundException(`Funnel ${funnelId} was not found in the database`,)}- return _head(funnel)+ return _head(data)}const updateFunnelClickhouse = async (funnel: any) => {@@ -120,43 +131,48 @@ const updateFunnelClickhouse = async (funnel: any) => {', ',)} WHERE id='${funnel.id}'`- return clickhouse.query(query).toPromise()+ const { data } = await clickhouse+ .query({+ query,+ })+ .then(resultSet => resultSet.json())++ return data}const deleteFunnelClickhouse = async (id: string) => {- const paramsData = {- params: {- id,- },- }-const query = `ALTER table funnel DELETE WHERE id = {id:String}`- return clickhouse.query(query, paramsData).toPromise()++ const { data } = await clickhouse+ .query({+ query,+ query_params: {+ id,+ },+ })+ .then(resultSet => resultSet.json())++ return data}const createFunnelClickhouse = async (funnel: Partial<Funnel>) => {- const paramsData = {- params: {- ...funnel,- },- }-const query = `INSERT INTO funnel (*) VALUES ({id:String},{name:String},{steps:String},{projectId:FixedString(12)},'${dayjs.utc().format('YYYY-MM-DD HH:mm:ss')}')`- return clickhouse.query(query, paramsData).toPromise()+ const { data } = await clickhouse+ .query({+ query,+ query_params: funnel,+ })+ .then(resultSet => resultSet.json())++ return data}const getProjectsClickhouse = async (id = null, search: string = null) => {if (!id) {if (search) {- const paramsData = {- params: {- search: `%${search}%`,- },- }-const query = `SELECT*@@ -164,29 +180,48 @@ const getProjectsClickhouse = async (id = null, search: string = null) => {WHEREname ILIKE {search:String} ORid ILIKE {search:String}- ORDER BY created ASC`-- return clickhouse.query(query, paramsData).toPromise()+ ORDER BY created ASC+ `++ const { data } = await clickhouse+ .query({+ query,+ query_params: {+ search: `%${search}%`,+ },+ })+ .then(resultSet => resultSet.json())++ return data}const query = 'SELECT * FROM project ORDER BY created ASC;'- return clickhouse.query(query).toPromise()- }- const paramsData = {- params: {- id,- },+ const { data } = await clickhouse+ .query({+ query,+ })+ .then(resultSet => resultSet.json())++ return data}const query = `SELECT * FROM project WHERE id = {id:FixedString(12)};`- const project = await clickhouse.query(query, paramsData).toPromise()- if (_isEmpty(project)) {+ const { data } = await clickhouse+ .query({+ query,+ query_params: {+ id,+ },+ })+ .then(resultSet => resultSet.json())++ if (_isEmpty(data)) {throw new NotFoundException(`Project ${id} was not found in the database`)}- return _head(project)+ return _head(data)}const updateProjectClickhouse = async (project: any) => {@@ -204,7 +239,14 @@ const updateProjectClickhouse = async (project: any) => {_map(columns, (col, id) => `${col}='${values[id]}'`),', ',)} WHERE id='${project.id}'`- return clickhouse.query(query).toPromise()++ const { data } = await clickhouse+ .query({+ query,+ })+ .then(resultSet => resultSet.json())++ return data}/**@@ -230,71 +272,91 @@ const calculateRelativePercentage = (}const deleteProjectClickhouse = async (id: string) => {- const paramsData = {- params: {- id,- },- }-const query = `ALTER table project DELETE WHERE id = {id:FixedString(12)}`- return clickhouse.query(query, paramsData).toPromise()++ const { data } = await clickhouse+ .query({+ query,+ query_params: {+ id,+ },+ })+ .then(resultSet => resultSet.json())++ return data}const createProjectClickhouse = async (project: Partial<Project>) => {- const paramsData = {- params: {- ...project,- },- }const created = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')const query = `INSERT INTO project (*) VALUES ({id:FixedString(12)},{name:String},'','',1,0,0,NULL,'${created}')`- return clickhouse.query(query, paramsData).toPromise()+ const { data } = await clickhouse+ .query({+ query,+ query_params: project,+ })+ .then(resultSet => resultSet.json())++ return data}const saveRefreshTokenClickhouse = async (userId: string,refreshToken: string,) => {- const paramsData = {- params: {- userId,- refreshToken,- },- }const query ='INSERT INTO refresh_token (*) VALUES ({userId:String},{refreshToken:String})'- return clickhouse.query(query, paramsData).toPromise()+ const { data } = await clickhouse+ .query({+ query,+ query_params: {+ userId,+ refreshToken,+ },+ })+ .then(resultSet => resultSet.json())++ return data}const findRefreshTokenClickhouse = async (userId: string,refreshToken: string,) => {- const paramsData = {- params: {- userId,- refreshToken,- },- }const query ='SELECT * FROM refresh_token WHERE userId = {userId:String} AND refreshToken = {refreshToken:String}'- return clickhouse.query(query, paramsData).toPromise()++ const { data } = await clickhouse+ .query({+ query,+ query_params: {+ userId,+ refreshToken,+ },+ })+ .then(resultSet => resultSet.json())++ return data}const deleteRefreshTokenClickhouse = async (userId: string,refreshToken: string,) => {- const paramsData = {- params: {- userId,- refreshToken,- },- }const query = `ALTER table refresh_token DELETE WHERE userId = {userId:String} AND refreshToken = {refreshToken:String}`- return clickhouse.query(query, paramsData).toPromise()++ const { data } = await clickhouse+ .query({+ query,+ query_params: {+ userId,+ refreshToken,+ },+ })+ .then(resultSet => resultSet.json())++ return data}interface IClickhouseUser {@@ -306,29 +368,35 @@ interface IClickhouseUser {const CLICKHOUSE_SETTINGS_ID = 'sfuser'const createUserClickhouse = async (user: IClickhouseUser) => {- const paramsData = {- params: {- ...user,- id: CLICKHOUSE_SETTINGS_ID,- },- }-const query = `INSERT INTO sfuser (*) VALUES ({id:String},{timezone:String},{timeFormat:String},{showLiveVisitorsInTitle:Int8})`- return clickhouse.query(query, paramsData).toPromise()+ const { data } = await clickhouse+ .query({+ query,+ query_params: {+ ...user,+ id: CLICKHOUSE_SETTINGS_ID,+ },+ })+ .then(resultSet => resultSet.json())++ return data}const getUserClickhouse = async () => {- const paramsData = {- params: {- id: CLICKHOUSE_SETTINGS_ID,- },- }-const query = `SELECT * FROM sfuser WHERE id = {id:String}`try {- return (await clickhouse.query(query, paramsData).toPromise())[0] || {}+ const { data } = await clickhouse+ .query({+ query,+ query_params: {+ id: CLICKHOUSE_SETTINGS_ID,+ },+ })+ .then(resultSet => resultSet.json())++ return data[0] || {}} catch {return {}}@@ -338,8 +406,13 @@ const userClickhouseExists = async () => {const query = `SELECT * FROM sfuser WHERE id = '${CLICKHOUSE_SETTINGS_ID}'`try {- const result = await clickhouse.query(query).toPromise()- return !_isEmpty(result)+ const { data } = await clickhouse+ .query({+ query,+ })+ .then(resultSet => resultSet.json())++ return !_isEmpty(data)} catch {return false}@@ -357,13 +430,6 @@ const updateUserClickhouse = async (user: IClickhouseUser) => {})}- const paramsData = {- params: {- ...user,- id: CLICKHOUSE_SETTINGS_ID,- },- }-let query = 'ALTER table sfuser UPDATE 'let separator = ''@@ -384,7 +450,17 @@ const updateUserClickhouse = async (user: IClickhouseUser) => {query += ` WHERE id={id:String}`- return clickhouse.query(query, paramsData).toPromise()+ const { data } = await clickhouse+ .query({+ query,+ query_params: {+ ...user,+ id: CLICKHOUSE_SETTINGS_ID,+ },+ })+ .then(resultSet => resultSet.json())++ return data}const millisecondsToSeconds = (milliseconds: number) => milliseconds / 1000diff --git a/apps/selfhosted/src/project/project.controller.ts b/apps/selfhosted/src/project/project.controller.tsindex 747251b3f..cfb531eec 100644--- a/apps/selfhosted/src/project/project.controller.ts+++ b/apps/selfhosted/src/project/project.controller.ts@@ -52,7 +52,8 @@ import {FunnelUpdateDTO,} from './dto'import { AppLoggerService } from '../logger/logger.service'-import { isValidPID, clickhouse } from '../common/constants'+import { isValidPID } from '../common/constants'+import { clickhouse } from '../common/integrations/clickhouse'import {getProjectsClickhouse,createProjectClickhouse,@@ -201,12 +202,13 @@ export class ProjectController {)}- const query1 = `ALTER table analytics DELETE WHERE pid='${id}'`- const query2 = `ALTER table customEV DELETE WHERE pid='${id}'`-try {- await clickhouse.query(query1).toPromise()- await clickhouse.query(query2).toPromise()+ await clickhouse.query({+ query: `ALTER table analytics DELETE WHERE pid='${id}'`,+ })+ await clickhouse.query({+ query: `ALTER table customEV DELETE WHERE pid='${id}'`,+ })return 'Project resetted successfully'} catch (e) {this.logger.error(e)@@ -433,12 +435,13 @@ export class ProjectController {await deleteProjectClickhouse(id)- const query1 = `ALTER table analytics DELETE WHERE pid='${id}'`- const query2 = `ALTER table customEV DELETE WHERE pid='${id}'`-try {- await clickhouse.query(query1).toPromise()- await clickhouse.query(query2).toPromise()+ await clickhouse.query({+ query: `ALTER table analytics DELETE WHERE pid='${id}'`,+ })+ await clickhouse.query({+ query: `ALTER table customEV DELETE WHERE pid='${id}'`,+ })return 'Project deleted successfully'} catch (e) {this.logger.error(e)diff --git a/apps/selfhosted/src/project/project.service.ts b/apps/selfhosted/src/project/project.service.tsindex 40ec72ed9..72d4b3bd7 100644--- a/apps/selfhosted/src/project/project.service.ts+++ b/apps/selfhosted/src/project/project.service.ts@@ -25,12 +25,12 @@ import { ProjectDTO } from './dto/project.dto'import {isValidPID,redis,- clickhouse,IP_REGEX,ORIGINS_REGEX,getRedisProjectKey,redisProjectCacheTimeout,} from '../common/constants'+import { clickhouse } from '../common/integrations/clickhouse'import { getProjectsClickhouse } from '../common/utils'import { MAX_PROJECT_PASSWORD_LENGTH, UpdateProjectDto } from './dto'import { Funnel } from './entity/funnel.entity'@@ -171,9 +171,14 @@ export class ProjectService {);`- const result: any = await clickhouse.query(query, { params }).toPromise()+ const { data } = await clickhouse+ .query({+ query,+ query_params: params,+ })+ .then(resultSet => resultSet.json())- return _map(result, ({ pid }) => pid)+ return _map(data, ({ pid }) => pid)}async getPIDsWhereErrorsDataExists(projectIds: string[]): Promise<string[]> {@@ -215,9 +220,14 @@ export class ProjectService {);`- const result: any = await clickhouse.query(query, { params }).toPromise()+ const { data } = await clickhouse+ .query({+ query,+ query_params: params,+ })+ .then(resultSet => resultSet.json())- return _map(result, ({ pid }) => pid)+ return _map(data, ({ pid }) => pid)}async removeDataFromClickhouse(@@ -239,11 +249,18 @@ export class ProjectService {},}- await Promise.all([- clickhouse.query(queryAnalytics, params).toPromise(),- clickhouse.query(queryCustomEvents, params).toPromise(),- clickhouse.query(queryPerformance, params).toPromise(),- ])+ await clickhouse.query({+ query: queryAnalytics,+ query_params: params,+ })+ await clickhouse.query({+ query: queryCustomEvents,+ query_params: params,+ })+ await clickhouse.query({+ query: queryPerformance,+ query_params: params,+ })}formatToClickhouse(project: any): object {diff --git a/apps/selfhosted/src/task-manager/task-manager.service.ts b/apps/selfhosted/src/task-manager/task-manager.service.tsindex 2a19640bc..99c661726 100644--- a/apps/selfhosted/src/task-manager/task-manager.service.ts+++ b/apps/selfhosted/src/task-manager/task-manager.service.ts@@ -10,7 +10,6 @@ import * as _map from 'lodash/map'import * as _now from 'lodash/now'import {- clickhouse,redis,REDIS_LOG_DATA_CACHE_KEY,REDIS_LOG_CUSTOM_CACHE_KEY,@@ -19,6 +18,7 @@ import {REDIS_LOG_CAPTCHA_CACHE_KEY,REDIS_LOG_ERROR_CACHE_KEY,} from '../common/constants'+import { clickhouse } from '../common/integrations/clickhouse'import { AppLoggerService } from '../logger/logger.service'dayjs.extend(utc)@@ -39,7 +39,9 @@ export class TaskManagerService {await redis.del(REDIS_LOG_DATA_CACHE_KEY)const query = `INSERT INTO analytics (*) VALUES ${_join(data, ',')}`try {- await clickhouse.query(query).toPromise()+ await clickhouse.query({+ query,+ })} catch (e) {console.error(`[CRON WORKER] Error whilst saving log data: ${e}`)}@@ -49,7 +51,9 @@ export class TaskManagerService {await redis.del(REDIS_LOG_CAPTCHA_CACHE_KEY)const query = `INSERT INTO captcha (*) VALUES ${_join(captchaData, ',')}`try {- await clickhouse.query(query).toPromise()+ await clickhouse.query({+ query,+ })} catch (e) {console.error(`[CRON WORKER] Error whilst saving CAPTCHA log data: ${e}`,@@ -62,7 +66,9 @@ export class TaskManagerService {const query = `INSERT INTO customEV (*) VALUES ${_join(customData, ',')}`try {- await clickhouse.query(query).toPromise()+ await clickhouse.query({+ query,+ })} catch (e) {console.error(`[CRON WORKER] Error whilst saving custom events data: ${e}`,@@ -75,7 +81,9 @@ export class TaskManagerService {const query = `INSERT INTO errors (*) VALUES ${_join(errorData, ',')}`try {- await clickhouse.query(query).toPromise()+ await clickhouse.query({+ query,+ })} catch (e) {console.error(`[CRON WORKER] Error whilst saving error events data: ${e}`,@@ -89,7 +97,9 @@ export class TaskManagerService {const query = `INSERT INTO performance (*) VALUES ${_join(perfData, ',')}`try {- await clickhouse.query(query).toPromise()+ await clickhouse.query({+ query,+ })} catch (e) {console.error(`[CRON WORKER] Error whilst saving performance data: ${e}`,@@ -111,7 +121,9 @@ export class TaskManagerService {.subtract(20, 'm').format('YYYY-MM-DD HH:mm:ss')}'`- await clickhouse.query(delSidQuery).toPromise()+ await clickhouse.query({+ query: delSidQuery,+ })}@Cron(CronExpression.EVERY_MINUTE)@@ -148,7 +160,9 @@ export class TaskManagerService {([key]) => `'${key.split(':')[1]}'`,).join(',')})`- await clickhouse.query(setSdurQuery).toPromise()+ await clickhouse.query({+ query: setSdurQuery,+ })}}@@ -163,7 +177,9 @@ export class TaskManagerService {]const promises = _map(queries, async query => {- await clickhouse.query(query).toPromise()+ await clickhouse.query({+ query,+ })})await Promise.allSettled(promises).catch(reason => {