Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

selfhosted: migrate to the official clickhouse/client library

Blue Mousee09d2e38a0b5
diff --git a/apps/selfhosted/src/analytics/analytics.controller.ts b/apps/selfhosted/src/analytics/analytics.controller.ts
index aed291262..1b912908d 100644
--- a/apps/selfhosted/src/analytics/analytics.controller.ts
+++ b/apps/selfhosted/src/analytics/analytics.controller.ts
@@ -60,9 +60,9 @@ import {
REDIS_LOG_CUSTOM_CACHE_KEY,
HEARTBEAT_SID_LIFE_TIME,
REDIS_SESSION_SALT_KEY,
- clickhouse,
REDIS_LOG_ERROR_CACHE_KEY,
} from '../common/constants'
+import { clickhouse } from '../common/integrations/clickhouse'
import {
checkRateLimit,
getGeoDetails,
@@ -1031,11 +1031,11 @@ export class AnalyticsController {
@Get('liveVisitors')
@Auth([], true, true)
async getLiveVisitors(
- @Query() data,
+ @Query() queryParams,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
): Promise<object> {
- const { pid } = data
+ const { pid } = queryParams
this.analyticsService.validatePID(pid)
await this.analyticsService.checkProjectAccess(
@@ -1055,8 +1055,12 @@ export class AnalyticsController {
const query = `SELECT sid, dv, br, os, cc FROM analytics WHERE sid IN (${sids
.map(el => `'${el}'`)
.join(',')})`
- const result = await clickhouse.query(query).toPromise()
- const processed = _map(_uniqBy(result, 'sid'), el =>
+ const { data } = await clickhouse
+ .query({
+ query,
+ })
+ .then(resultSet => resultSet.json())
+ const processed = _map(_uniqBy(data, 'sid'), el =>
_pick(el, ['dv', 'br', 'os', 'cc']),
)
diff --git a/apps/selfhosted/src/analytics/analytics.service.ts b/apps/selfhosted/src/analytics/analytics.service.ts
index e0c1d5c93..b2ee34ab5 100644
--- a/apps/selfhosted/src/analytics/analytics.service.ts
+++ b/apps/selfhosted/src/analytics/analytics.service.ts
@@ -42,7 +42,6 @@ import {
redis,
isValidPID,
UNIQUE_SESSION_LIFE_TIME,
- clickhouse,
REDIS_SESSION_SALT_KEY,
TRAFFIC_COLUMNS,
PERFORMANCE_COLUMNS,
@@ -52,6 +51,7 @@ import {
ALL_COLUMNS,
TRAFFIC_METAKEY_COLUMNS,
} from '../common/constants'
+import { clickhouse } from '../common/integrations/clickhouse'
import {
getFunnelsClickhouse,
millisecondsToSeconds,
@@ -471,11 +471,16 @@ export class AnalyticsService {
const query = `SELECT ${type} FROM errors WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}`
- const results = await clickhouse
- .query(query, { params: { pid } })
- .toPromise()
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: {
+ pid,
+ },
+ })
+ .then(resultSet => resultSet.json())
- return _map(results, type)
+ return _map(data, type)
}
getDataTypeColumns(dataType: DataType): string[] {
@@ -633,7 +638,10 @@ export class AnalyticsService {
return { nodes, links }
}
- async getUserFlow(params: unknown, filtersQuery: string): Promise<IUserFlow> {
+ async getUserFlow(
+ params: Record<string, unknown>,
+ filtersQuery: string,
+ ): Promise<IUserFlow> {
const query = `
SELECT
pg AS source,
@@ -650,11 +658,14 @@ export class AnalyticsService {
prev
`
- const results = <IUserFlowLink[]>(
- await clickhouse.query(query, { params }).toPromise()
- )
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: params,
+ })
+ .then(resultSet => resultSet.json<IUserFlowLink>())
- if (_isEmpty(results)) {
+ if (_isEmpty(data)) {
const empty = { nodes: [], links: [] }
return {
ascending: empty,
@@ -665,7 +676,7 @@ export class AnalyticsService {
const ascendingLinks: IUserFlowLink[] = []
const descendingLinks: IUserFlowLink[] = []
- this.removeCyclicDependencies(results).forEach((row: any) => {
+ this.removeCyclicDependencies(data).forEach((row: any) => {
const link: IUserFlowLink = {
source: row.source,
target: row.target,
@@ -790,12 +801,14 @@ export class AnalyticsService {
return null
}
- const from: any = await clickhouse
- .query(
- 'SELECT created FROM analytics where pid = {pid:FixedString(12)} ORDER BY created ASC LIMIT 1',
- { params: { pid } },
- )
- .toPromise()
+ const { data: from } = await clickhouse
+ .query({
+ query:
+ 'SELECT created FROM analytics WHERE pid = {pid:FixedString(12)} ORDER BY created ASC LIMIT 1',
+ query_params: { pid },
+ })
+ .then(res => res.json<{ created?: string }>())
+
const to = _includes(GMT_0_TIMEZONES, safeTimezone)
? dayjs.utc()
: dayjs().tz(safeTimezone)
@@ -1169,20 +1182,18 @@ export class AnalyticsService {
ORDER BY level DESC;
`
- const result = <Array<IFunnelCHResponse>>(
- await clickhouse
- .query(query, { params: { ...params, ...pageParams } })
- .toPromise()
- )
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: { ...params, ...pageParams },
+ })
+ .then(resultSet => resultSet.json<IFunnelCHResponse>())
- if (_isEmpty(result)) {
+ if (_isEmpty(data)) {
return this.generateEmptyFunnel(pages)
}
- return this.formatFunnel(
- _reverse(this.backfillFunnel(result, pages)),
- pages,
- )
+ return this.formatFunnel(_reverse(this.backfillFunnel(data, pages)), pages)
}
async getTotalPageviews(
@@ -1197,13 +1208,14 @@ export class AnalyticsService {
WHERE pid = {pid:FixedString(12)}
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
`
- const result = <{ c: number }[]>(
- await clickhouse
- .query(query, { params: { pid, groupFrom, groupTo } })
- .toPromise()
- )
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: { pid, groupFrom, groupTo },
+ })
+ .then(resultSet => resultSet.json<{ c: number }>())
- return result[0]?.c || 0
+ return data[0]?.c || 0
}
async getAnalyticsSummary(
@@ -1256,27 +1268,28 @@ export class AnalyticsService {
queryAll = `SELECT count(*) AS all FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery}`
}
- const rawResult = <Array<BirdseyeCHResponse>>await clickhouse
- .query(queryAll, {
- params: { pid, ...filtersParams },
+ const { data } = await clickhouse
+ .query({
+ query: queryAll,
+ query_params: {
+ pid,
+ ...filtersParams,
+ },
})
- .toPromise()
+ .then(resultSet => resultSet.json<BirdseyeCHResponse>())
let bounceRate = 0
- if (rawResult[0].all > 0 && !customEVFilterApplied) {
- bounceRate = _round(
- (rawResult[0].unique * 100) / rawResult[0].all,
- 1,
- )
+ if (data[0].all > 0 && !customEVFilterApplied) {
+ bounceRate = _round((data[0].unique * 100) / data[0].all, 1)
}
result[pid] = {
current: {
- all: rawResult[0].all,
- unique: rawResult[0].unique,
+ all: data[0].all,
+ unique: data[0].unique,
bounceRate,
- sdur: rawResult[0].sdur,
+ sdur: data[0].sdur,
},
previous: {
all: 0,
@@ -1284,10 +1297,10 @@ export class AnalyticsService {
bounceRate: 0,
sdur: 0,
},
- change: rawResult[0].all,
- uniqueChange: rawResult[0].unique,
+ change: data[0].all,
+ uniqueChange: data[0].unique,
bounceRateChange: bounceRate,
- sdurChange: rawResult[0].sdur,
+ sdurChange: data[0].sdur,
customEVFilterApplied,
}
return
@@ -1328,9 +1341,10 @@ export class AnalyticsService {
const query = `${queryCurrent} UNION ALL ${queryPrevious}`
- let rawResult = <Array<BirdseyeCHResponse>>await clickhouse
- .query(query, {
- params: {
+ let { data } = await clickhouse
+ .query({
+ query,
+ query_params: {
pid,
periodFormatted,
periodSubtracted,
@@ -1338,12 +1352,12 @@ export class AnalyticsService {
...filtersParams,
},
})
- .toPromise()
+ .then(resultSet => resultSet.json<BirdseyeCHResponse>())
- rawResult = _sortBy(rawResult, 'sortOrder')
+ data = _sortBy(data, 'sortOrder')
- const currentPeriod = rawResult[0]
- const previousPeriod = rawResult[1]
+ const currentPeriod = data[0]
+ const previousPeriod = data[1]
let bounceRate = 0
let prevBounceRate = 0
@@ -1452,26 +1466,23 @@ export class AnalyticsService {
try {
if (period === 'all') {
const queryAll = `SELECT ${columnSelectors} FROM performance WHERE pid = {pid:FixedString(12)} ${filtersQuery}`
- const rawResult = <Array<Partial<PerformanceCHResponse>>>(
- await clickhouse
- .query(queryAll, {
- params: { pid, ...filtersParams },
- })
- .toPromise()
- )
+ const { data } = await clickhouse
+ .query({
+ query: queryAll,
+ query_params: {
+ pid,
+ ...filtersParams,
+ },
+ })
+ .then(resultSet => resultSet.json<Partial<PerformanceCHResponse>>())
result[pid] = {
current: {
- frontend: millisecondsToSeconds(
- rawResult[0].render + rawResult[0].domLoad,
- ),
+ frontend: millisecondsToSeconds(data[0].render + data[0].domLoad),
network: millisecondsToSeconds(
- rawResult[0].dns +
- rawResult[0].tls +
- rawResult[0].conn +
- rawResult[0].response,
+ data[0].dns + data[0].tls + data[0].conn + data[0].response,
),
- backend: millisecondsToSeconds(rawResult[0].ttfb),
+ backend: millisecondsToSeconds(data[0].ttfb),
},
previous: {
frontend: 0,
@@ -1479,15 +1490,12 @@ export class AnalyticsService {
backend: 0,
},
frontendChange: millisecondsToSeconds(
- rawResult[0].render + rawResult[0].domLoad,
+ data[0].render + data[0].domLoad,
),
networkChange: millisecondsToSeconds(
- rawResult[0].dns +
- rawResult[0].tls +
- rawResult[0].conn +
- rawResult[0].response,
+ data[0].dns + data[0].tls + data[0].conn + data[0].response,
),
- backendChange: millisecondsToSeconds(rawResult[0].ttfb),
+ backendChange: millisecondsToSeconds(data[0].ttfb),
}
return
}
@@ -1522,9 +1530,10 @@ export class AnalyticsService {
const query = `${queryCurrent} UNION ALL ${queryPrevious}`
- let rawResult = <Array<Partial<PerformanceCHResponse>>>await clickhouse
- .query(query, {
- params: {
+ let { data } = await clickhouse
+ .query({
+ query,
+ query_params: {
pid,
periodFormatted,
periodSubtracted,
@@ -1532,12 +1541,12 @@ export class AnalyticsService {
...filtersParams,
},
})
- .toPromise()
+ .then(resultSet => resultSet.json<Partial<PerformanceCHResponse>>())
- rawResult = _sortBy(rawResult, 'sortOrder')
+ data = _sortBy(data, 'sortOrder')
- const currentPeriod = rawResult[0]
- const previousPeriod = rawResult[1]
+ const currentPeriod = data[0]
+ const previousPeriod = data[1]
result[pid] = {
current: {
@@ -1614,11 +1623,16 @@ export class AnalyticsService {
'SELECT ev FROM customEV WHERE pid={pid:FixedString(12)} AND ev IS NOT NULL GROUP BY ev'
}
- const results = await clickhouse
- .query(query, { params: { pid } })
- .toPromise()
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: {
+ pid,
+ },
+ })
+ .then(resultSet => resultSet.json())
- return _map(results, type)
+ return _map(data, type)
}
async generateParams(
@@ -1661,9 +1675,14 @@ export class AnalyticsService {
type,
measure,
)
- const res = await clickhouse.query(query, paramsData).toPromise()
-
- params[col] = res
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json())
+
+ params[col] = data
})
await Promise.all(paramsPromises)
@@ -1671,18 +1690,6 @@ export class AnalyticsService {
return params
}
- async calculateAverageSessionDuration(
- subQuery: string,
- paramsData: any,
- ): Promise<number> {
- const avgSdurQuery = `SELECT avg(sdur) ${subQuery} AND sdur IS NOT NULL AND unique='1'`
- const avgSdurObject = await clickhouse
- .query(avgSdurQuery, paramsData)
- .toPromise()
-
- return _round(avgSdurObject[0]['avg(sdur)'])
- }
-
generateXAxis(
timeBucket: TimeBucketType,
from: string, // timezone is already applied to the from and to parameters
@@ -2217,13 +2224,15 @@ export class AnalyticsService {
mode,
)
- const result = <Array<TrafficCEFilterCHResponse>>(
- await clickhouse.query(query, paramsData).toPromise()
- )
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json<TrafficCEFilterCHResponse>())
const uniques =
- this.extractCustomEventsChartData(result, xShifted)?._unknown_event ||
- []
+ this.extractCustomEventsChartData(data, xShifted)?._unknown_event || []
const sdur = Array(_size(xShifted)).fill(0)
@@ -2244,11 +2253,14 @@ export class AnalyticsService {
mode,
)
- const result = <Array<TrafficCHResponse>>(
- await clickhouse.query(query, paramsData).toPromise()
- )
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json<TrafficCHResponse>())
- const { visits, uniques, sdur } = this.extractChartData(result, xShifted)
+ const { visits, uniques, sdur } = this.extractChartData(data, xShifted)
return Promise.resolve({
chart: {
@@ -2352,9 +2364,12 @@ export class AnalyticsService {
// ChartRenderMode.PERIODICAL,
// )
- // const result = <Array<TrafficCEFilterCHResponse>>(
- // await clickhouse.query(query, paramsData).toPromise()
- // )
+ // const { data } = await clickhouse
+ // .query({
+ // query,
+ // query_params: paramsData.params,
+ // })
+ // .then(resultSet => resultSet.json<TrafficCEFilterCHResponse>())
// const uniques =
// this.extractCustomEventsChartData(result, xShifted)?._unknown_event ||
@@ -2374,11 +2389,14 @@ export class AnalyticsService {
const query = this.generateSessionAggregationQuery(timeBucket, filtersQuery)
- const result = <Array<TrafficCHResponse>>(
- await clickhouse.query(query, paramsData).toPromise()
- )
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json<TrafficCHResponse>())
- const { visits, uniques, sdur } = this.extractChartData(result, x)
+ const { visits, uniques, sdur } = this.extractChartData(data, x)
return Promise.resolve({
chart: {
@@ -2396,7 +2414,7 @@ export class AnalyticsService {
to: string,
subQuery: string,
filtersQuery: string,
- paramsData: object,
+ paramsData: any,
safeTimezone: string,
mode: ChartRenderMode,
): Promise<object | void> {
@@ -2431,10 +2449,13 @@ export class AnalyticsService {
mode,
)
- const result = <Array<TrafficCEFilterCHResponse>>(
- await clickhouse.query(query, paramsData).toPromise()
- )
- const { count } = this.extractErrorsChartData(result, x)
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json<TrafficCEFilterCHResponse>())
+ const { count } = this.extractErrorsChartData(data, x)
chart = {
x: this.shiftToTimezone(x, safeTimezone, format),
@@ -2492,7 +2513,7 @@ export class AnalyticsService {
from: string,
to: string,
filtersQuery: string,
- paramsData: object,
+ paramsData: any,
safeTimezone: string,
measure: PerfMeasure,
) {
@@ -2505,11 +2526,16 @@ export class AnalyticsService {
safeTimezone,
)
- const result = await clickhouse.query(query, paramsData).toPromise()
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json())
return {
x: xShifted,
- ...this.extractPerformanceQuantilesData(result, xShifted),
+ ...this.extractPerformanceQuantilesData(data, xShifted),
}
}
@@ -2520,13 +2546,16 @@ export class AnalyticsService {
measure,
)
- const result = <Array<PerformanceCHResponse>>(
- await clickhouse.query(query, paramsData).toPromise()
- )
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json<PerformanceCHResponse>())
return {
x: xShifted,
- ...this.extractPerformanceChartData(result, xShifted),
+ ...this.extractPerformanceChartData(data, xShifted),
}
}
@@ -2536,7 +2565,7 @@ export class AnalyticsService {
to: string,
subQuery: string,
filtersQuery: string,
- paramsData: object,
+ paramsData: any,
safeTimezone: string,
measure: PerfMeasure,
): Promise<object | void> {
@@ -2586,18 +2615,21 @@ export class AnalyticsService {
async getCustomEvents(
filtersQuery: string,
- params: object,
+ params: any,
): Promise<ICustomEvent> {
const query = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`
const result = {}
- const rawCustoms = <Array<CustomsCHResponse>>(
- await clickhouse.query(query, params).toPromise()
- )
- const size = _size(rawCustoms)
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: params.params,
+ })
+ .then(resultSet => resultSet.json<CustomsCHResponse>())
+ const size = _size(data)
for (let i = 0; i < size; ++i) {
- const { ev, 'count()': c } = rawCustoms[i]
+ const { ev, 'count()': c } = data[i]
result[ev] = c
}
@@ -2618,7 +2650,7 @@ export class AnalyticsService {
async getPageProperties(
filtersQuery: string,
- params: object,
+ params: any,
): Promise<IPageProperty> {
const query = `
SELECT
@@ -2631,13 +2663,16 @@ export class AnalyticsService {
GROUP BY property`
const result = {}
- const rawProperties = <Array<PropertiesCHResponse>>(
- await clickhouse.query(query, params).toPromise()
- )
- const size = _size(rawProperties)
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: params.params,
+ })
+ .then(resultSet => resultSet.json<PropertiesCHResponse>())
+ const size = _size(data)
for (let i = 0; i < size; ++i) {
- const { property: propertyArr, 'count()': c } = rawProperties[i]
+ const { property: propertyArr, 'count()': c } = data[i]
const [property] = propertyArr
if (!property) {
@@ -2728,9 +2763,12 @@ export class AnalyticsService {
}
try {
- const result = (await clickhouse
- .query(query, paramsData)
- .toPromise()) as IAggregatedMetadata[]
+ const { data: result } = await clickhouse
+ .query({
+ query,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json<IAggregatedMetadata>())
return {
result,
@@ -2829,9 +2867,12 @@ export class AnalyticsService {
}
try {
- const result = (await clickhouse
- .query(query, paramsData)
- .toPromise()) as IAggregatedMetadata[]
+ const { data: result } = await clickhouse
+ .query({
+ query,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json<IAggregatedMetadata>())
return {
result,
@@ -2855,7 +2896,7 @@ export class AnalyticsService {
from: string,
to: string,
filtersQuery: string,
- paramsData: object,
+ paramsData: any,
safeTimezone: string,
): Promise<object | void> {
const { xShifted } = this.generateXAxis(timeBucket, from, to, safeTimezone)
@@ -2883,11 +2924,14 @@ export class AnalyticsService {
ORDER BY ${groupBy}
`
- const result = <Array<CustomsCHAggregatedResponse>>(
- await clickhouse.query(query, paramsData).toPromise()
- )
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json<CustomsCHAggregatedResponse>())
- const events = this.extractCustomEventsChartData(result, xShifted)
+ const events = this.extractCustomEventsChartData(data, xShifted)
return Promise.resolve({
chart: {
@@ -2961,11 +3005,21 @@ export class AnalyticsService {
},
}
- const pages = <IPageflow[]>(
- await clickhouse.query(queryPages, paramsData).toPromise()
- )
+ const { data: pages } = await clickhouse
+ .query({
+ query: queryPages,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json<IPageflow>())
+
let details = (
- await clickhouse.query(querySessionDetails, paramsData).toPromise()
+ await clickhouse
+ .query({
+ query: querySessionDetails,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json())
+ .then(({ data }) => data)
)[0]
if (!details) {
@@ -2982,8 +3036,12 @@ export class AnalyticsService {
// eslint-disable-next-line prefer-destructuring
details = (
await clickhouse
- .query(querySessionDetailsBackup, paramsData)
- .toPromise()
+ .query({
+ query: querySessionDetailsBackup,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json())
+ .then(({ data }) => data)
)[0]
}
@@ -3032,7 +3090,7 @@ export class AnalyticsService {
async getSessionsList(
filtersQuery: string,
- paramsData: object,
+ paramsData: any,
safeTimezone: string,
take = 30,
skip = 0,
@@ -3089,15 +3147,20 @@ export class AnalyticsService {
OFFSET ${skip}
`
- const result = await clickhouse.query(query, paramsData).toPromise()
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json())
- return result
+ return data
}
async getErrorsList(
options: string,
filtersQuery: string,
- paramsData: object,
+ paramsData: any,
safeTimezone: string,
take = 30,
skip = 0,
@@ -3147,9 +3210,14 @@ export class AnalyticsService {
OFFSET ${skip};
`
- const result = await clickhouse.query(query, paramsData).toPromise()
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json())
- return result
+ return data
}
async getErrorDetails(
@@ -3215,11 +3283,23 @@ export class AnalyticsService {
}
const details = (
- await clickhouse.query(queryErrorDetails, paramsData).toPromise()
+ await clickhouse
+ .query({
+ query: queryErrorDetails,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json<any>())
+ .then(({ data }) => data)
)[0]
const occurenceDetails = (
- await clickhouse.query(queryFirstLastSeen, paramsData).toPromise()
+ await clickhouse
+ .query({
+ query: queryFirstLastSeen,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json<any>())
+ .then(({ data }) => data)
)[0]
const groupedChart = await this.groupErrorsByTimeBucket(
@@ -3269,9 +3349,16 @@ export class AnalyticsService {
let result
try {
- ;[result] = await clickhouse
- .query(query, { params: { ...params, pid } })
- .toPromise()
+ // eslint-disable-next-line prefer-destructuring
+ result = (
+ await clickhouse
+ .query({
+ query,
+ query_params: { ...params, pid },
+ })
+ .then(resultSet => resultSet.json())
+ .then(({ data }) => data)
+ )[0]
} catch (reason) {
console.error('validateEIDs - clickhouse request error')
console.error(reason)
@@ -3308,7 +3395,13 @@ export class AnalyticsService {
).join(', ')}`
try {
- await clickhouse.query(query, { params: { ...params, pid } }).toPromise()
+ await clickhouse.query({
+ query,
+ query_params: {
+ ...params,
+ pid,
+ },
+ })
} catch (reason) {
console.error('Error at PATCH error-status:')
console.error(reason)
diff --git a/apps/selfhosted/src/common/constants.ts b/apps/selfhosted/src/common/constants.ts
index 5690541a7..7ac72181c 100644
--- a/apps/selfhosted/src/common/constants.ts
+++ b/apps/selfhosted/src/common/constants.ts
@@ -1,4 +1,3 @@
-import { ClickHouse } from 'clickhouse'
import Redis from 'ioredis'
import * as _toNumber from 'lodash/toNumber'
@@ -7,8 +6,6 @@ import { getSelfhostedUUID } from './utils'
// eslint-disable-next-line @typescript-eslint/no-var-requires
require('dotenv').config()
-const { CLICKHOUSE_DATABASE } = process.env
-
const redis = new Redis(
_toNumber(process.env.REDIS_PORT),
process.env.REDIS_HOST,
@@ -24,26 +21,6 @@ redis.defineCommand('countKeysByPattern', {
lua: "return #redis.call('keys', ARGV[1])",
})
-const clickhouse = new ClickHouse({
- url: process.env.CLICKHOUSE_HOST,
- port: _toNumber(process.env.CLICKHOUSE_PORT),
- debug: false,
- basicAuth: {
- username: process.env.CLICKHOUSE_USER,
- password: process.env.CLICKHOUSE_PASSWORD,
- },
- isUseGzip: false,
- format: 'json',
- raw: false,
- config: {
- session_timeout: 60,
- output_format_json_quote_64bit_integers: 0,
- enable_http_compression: 0,
- database: CLICKHOUSE_DATABASE,
- log_queries: 0,
- },
-})
-
const {
JWT_ACCESS_TOKEN_SECRET,
// 30 days
@@ -121,7 +98,6 @@ const NUMBER_JWT_REFRESH_TOKEN_LIFETIME = Number(JWT_REFRESH_TOKEN_LIFETIME)
const NUMBER_JWT_ACCESS_TOKEN_LIFETIME = Number(JWT_ACCESS_TOKEN_LIFETIME)
export {
- clickhouse,
redis,
isValidPID,
getRedisProjectKey,
diff --git a/apps/selfhosted/src/common/integrations/clickhouse.ts b/apps/selfhosted/src/common/integrations/clickhouse.ts
new file mode 100644
index 000000000..039dd8e04
--- /dev/null
+++ b/apps/selfhosted/src/common/integrations/clickhouse.ts
@@ -0,0 +1,21 @@
+import { createClient } from '@clickhouse/client'
+
+const clickhouse = createClient({
+ host: `${process.env.CLICKHOUSE_HOST}:${process.env.CLICKHOUSE_PORT}`,
+ username: process.env.CLICKHOUSE_USER,
+ password: process.env.CLICKHOUSE_PASSWORD,
+ database: process.env.CLICKHOUSE_DATABASE,
+ request_timeout: 60000, // 1 minute
+ clickhouse_settings: {
+ connect_timeout: 60000,
+ date_time_output_format: 'iso',
+ max_download_buffer_size: (10 * 1024 * 1024).toString(),
+ max_download_threads: 32,
+ max_execution_time: 60000,
+ output_format_json_quote_64bit_integers: 0,
+ enable_http_compression: 0,
+ log_queries: 0,
+ },
+})
+
+export { clickhouse }
diff --git a/apps/selfhosted/src/common/utils.ts b/apps/selfhosted/src/common/utils.ts
index af030b49f..2d3c02deb 100644
--- a/apps/selfhosted/src/common/utils.ts
+++ b/apps/selfhosted/src/common/utils.ts
@@ -19,7 +19,6 @@ import * as utc from 'dayjs/plugin/utc'
import * as _map from 'lodash/map'
import {
- clickhouse,
redis,
DEFAULT_SELFHOSTED_UUID,
SELFHOSTED_EMAIL,
@@ -27,6 +26,7 @@ import {
isDevelopment,
isProxiedByCloudflare,
} from './constants'
+import { clickhouse } from './integrations/clickhouse'
import { DEFAULT_TIMEZONE, TimeFormat } from '../user/entities/user.entity'
import { Project } from '../project/entity/project.entity'
import { Funnel } from '../project/entity/funnel.entity'
@@ -72,17 +72,23 @@ const checkRateLimit = async (
}
const getFunnelsClickhouse = async (projectId: string, funnelId = null) => {
- const paramsData = {
- params: {
- projectId,
- funnelId,
- },
+ const queryParams = {
+ projectId,
+ funnelId,
}
if (!funnelId) {
const query =
'SELECT * FROM funnel WHERE projectId = {projectId:FixedString(12)}'
- return clickhouse.query(query, paramsData).toPromise()
+
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: queryParams,
+ })
+ .then(resultSet => resultSet.json())
+
+ return data
}
const query = `
@@ -93,15 +99,20 @@ const getFunnelsClickhouse = async (projectId: string, funnelId = null) => {
projectId = {projectId:FixedString(12)}
AND id = {funnelId:String}`
- const funnel = await clickhouse.query(query, paramsData).toPromise()
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: queryParams,
+ })
+ .then(resultSet => resultSet.json())
- if (_isEmpty(funnel)) {
+ if (_isEmpty(data)) {
throw new NotFoundException(
`Funnel ${funnelId} was not found in the database`,
)
}
- return _head(funnel)
+ return _head(data)
}
const updateFunnelClickhouse = async (funnel: any) => {
@@ -120,43 +131,48 @@ const updateFunnelClickhouse = async (funnel: any) => {
', ',
)} WHERE id='${funnel.id}'`
- return clickhouse.query(query).toPromise()
+ const { data } = await clickhouse
+ .query({
+ query,
+ })
+ .then(resultSet => resultSet.json())
+
+ return data
}
const deleteFunnelClickhouse = async (id: string) => {
- const paramsData = {
- params: {
- id,
- },
- }
-
const query = `ALTER table funnel DELETE WHERE id = {id:String}`
- return clickhouse.query(query, paramsData).toPromise()
+
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: {
+ id,
+ },
+ })
+ .then(resultSet => resultSet.json())
+
+ return data
}
const createFunnelClickhouse = async (funnel: Partial<Funnel>) => {
- const paramsData = {
- params: {
- ...funnel,
- },
- }
-
const query = `INSERT INTO funnel (*) VALUES ({id:String},{name:String},{steps:String},{projectId:FixedString(12)},'${dayjs
.utc()
.format('YYYY-MM-DD HH:mm:ss')}')`
- return clickhouse.query(query, paramsData).toPromise()
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: funnel,
+ })
+ .then(resultSet => resultSet.json())
+
+ return data
}
const getProjectsClickhouse = async (id = null, search: string = null) => {
if (!id) {
if (search) {
- const paramsData = {
- params: {
- search: `%${search}%`,
- },
- }
-
const query = `
SELECT
*
@@ -164,29 +180,48 @@ const getProjectsClickhouse = async (id = null, search: string = null) => {
WHERE
name ILIKE {search:String} OR
id ILIKE {search:String}
- ORDER BY created ASC`
-
- return clickhouse.query(query, paramsData).toPromise()
+ ORDER BY created ASC
+ `
+
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: {
+ search: `%${search}%`,
+ },
+ })
+ .then(resultSet => resultSet.json())
+
+ return data
}
const query = 'SELECT * FROM project ORDER BY created ASC;'
- return clickhouse.query(query).toPromise()
- }
- const paramsData = {
- params: {
- id,
- },
+ const { data } = await clickhouse
+ .query({
+ query,
+ })
+ .then(resultSet => resultSet.json())
+
+ return data
}
const query = `SELECT * FROM project WHERE id = {id:FixedString(12)};`
- const project = await clickhouse.query(query, paramsData).toPromise()
- if (_isEmpty(project)) {
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: {
+ id,
+ },
+ })
+ .then(resultSet => resultSet.json())
+
+ if (_isEmpty(data)) {
throw new NotFoundException(`Project ${id} was not found in the database`)
}
- return _head(project)
+ return _head(data)
}
const updateProjectClickhouse = async (project: any) => {
@@ -204,7 +239,14 @@ const updateProjectClickhouse = async (project: any) => {
_map(columns, (col, id) => `${col}='${values[id]}'`),
', ',
)} WHERE id='${project.id}'`
- return clickhouse.query(query).toPromise()
+
+ const { data } = await clickhouse
+ .query({
+ query,
+ })
+ .then(resultSet => resultSet.json())
+
+ return data
}
/**
@@ -230,71 +272,91 @@ const calculateRelativePercentage = (
}
const deleteProjectClickhouse = async (id: string) => {
- const paramsData = {
- params: {
- id,
- },
- }
-
const query = `ALTER table project DELETE WHERE id = {id:FixedString(12)}`
- return clickhouse.query(query, paramsData).toPromise()
+
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: {
+ id,
+ },
+ })
+ .then(resultSet => resultSet.json())
+
+ return data
}
const createProjectClickhouse = async (project: Partial<Project>) => {
- const paramsData = {
- params: {
- ...project,
- },
- }
const created = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')
const query = `INSERT INTO project (*) VALUES ({id:FixedString(12)},{name:String},'','',1,0,0,NULL,'${created}')`
- return clickhouse.query(query, paramsData).toPromise()
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: project,
+ })
+ .then(resultSet => resultSet.json())
+
+ return data
}
const saveRefreshTokenClickhouse = async (
userId: string,
refreshToken: string,
) => {
- const paramsData = {
- params: {
- userId,
- refreshToken,
- },
- }
const query =
'INSERT INTO refresh_token (*) VALUES ({userId:String},{refreshToken:String})'
- return clickhouse.query(query, paramsData).toPromise()
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: {
+ userId,
+ refreshToken,
+ },
+ })
+ .then(resultSet => resultSet.json())
+
+ return data
}
const findRefreshTokenClickhouse = async (
userId: string,
refreshToken: string,
) => {
- const paramsData = {
- params: {
- userId,
- refreshToken,
- },
- }
const query =
'SELECT * FROM refresh_token WHERE userId = {userId:String} AND refreshToken = {refreshToken:String}'
- return clickhouse.query(query, paramsData).toPromise()
+
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: {
+ userId,
+ refreshToken,
+ },
+ })
+ .then(resultSet => resultSet.json())
+
+ return data
}
const deleteRefreshTokenClickhouse = async (
userId: string,
refreshToken: string,
) => {
- const paramsData = {
- params: {
- userId,
- refreshToken,
- },
- }
const query = `ALTER table refresh_token DELETE WHERE userId = {userId:String} AND refreshToken = {refreshToken:String}`
- return clickhouse.query(query, paramsData).toPromise()
+
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: {
+ userId,
+ refreshToken,
+ },
+ })
+ .then(resultSet => resultSet.json())
+
+ return data
}
interface IClickhouseUser {
@@ -306,29 +368,35 @@ interface IClickhouseUser {
const CLICKHOUSE_SETTINGS_ID = 'sfuser'
const createUserClickhouse = async (user: IClickhouseUser) => {
- const paramsData = {
- params: {
- ...user,
- id: CLICKHOUSE_SETTINGS_ID,
- },
- }
-
const query = `INSERT INTO sfuser (*) VALUES ({id:String},{timezone:String},{timeFormat:String},{showLiveVisitorsInTitle:Int8})`
- return clickhouse.query(query, paramsData).toPromise()
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: {
+ ...user,
+ id: CLICKHOUSE_SETTINGS_ID,
+ },
+ })
+ .then(resultSet => resultSet.json())
+
+ return data
}
const getUserClickhouse = async () => {
- const paramsData = {
- params: {
- id: CLICKHOUSE_SETTINGS_ID,
- },
- }
-
const query = `SELECT * FROM sfuser WHERE id = {id:String}`
try {
- return (await clickhouse.query(query, paramsData).toPromise())[0] || {}
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: {
+ id: CLICKHOUSE_SETTINGS_ID,
+ },
+ })
+ .then(resultSet => resultSet.json())
+
+ return data[0] || {}
} catch {
return {}
}
@@ -338,8 +406,13 @@ const userClickhouseExists = async () => {
const query = `SELECT * FROM sfuser WHERE id = '${CLICKHOUSE_SETTINGS_ID}'`
try {
- const result = await clickhouse.query(query).toPromise()
- return !_isEmpty(result)
+ const { data } = await clickhouse
+ .query({
+ query,
+ })
+ .then(resultSet => resultSet.json())
+
+ return !_isEmpty(data)
} catch {
return false
}
@@ -357,13 +430,6 @@ const updateUserClickhouse = async (user: IClickhouseUser) => {
})
}
- const paramsData = {
- params: {
- ...user,
- id: CLICKHOUSE_SETTINGS_ID,
- },
- }
-
let query = 'ALTER table sfuser UPDATE '
let separator = ''
@@ -384,7 +450,17 @@ const updateUserClickhouse = async (user: IClickhouseUser) => {
query += ` WHERE id={id:String}`
- return clickhouse.query(query, paramsData).toPromise()
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: {
+ ...user,
+ id: CLICKHOUSE_SETTINGS_ID,
+ },
+ })
+ .then(resultSet => resultSet.json())
+
+ return data
}
const millisecondsToSeconds = (milliseconds: number) => milliseconds / 1000
diff --git a/apps/selfhosted/src/project/project.controller.ts b/apps/selfhosted/src/project/project.controller.ts
index 747251b3f..cfb531eec 100644
--- a/apps/selfhosted/src/project/project.controller.ts
+++ b/apps/selfhosted/src/project/project.controller.ts
@@ -52,7 +52,8 @@ import {
FunnelUpdateDTO,
} from './dto'
import { AppLoggerService } from '../logger/logger.service'
-import { isValidPID, clickhouse } from '../common/constants'
+import { isValidPID } from '../common/constants'
+import { clickhouse } from '../common/integrations/clickhouse'
import {
getProjectsClickhouse,
createProjectClickhouse,
@@ -201,12 +202,13 @@ export class ProjectController {
)
}
- const query1 = `ALTER table analytics DELETE WHERE pid='${id}'`
- const query2 = `ALTER table customEV DELETE WHERE pid='${id}'`
-
try {
- await clickhouse.query(query1).toPromise()
- await clickhouse.query(query2).toPromise()
+ await clickhouse.query({
+ query: `ALTER table analytics DELETE WHERE pid='${id}'`,
+ })
+ await clickhouse.query({
+ query: `ALTER table customEV DELETE WHERE pid='${id}'`,
+ })
return 'Project resetted successfully'
} catch (e) {
this.logger.error(e)
@@ -433,12 +435,13 @@ export class ProjectController {
await deleteProjectClickhouse(id)
- const query1 = `ALTER table analytics DELETE WHERE pid='${id}'`
- const query2 = `ALTER table customEV DELETE WHERE pid='${id}'`
-
try {
- await clickhouse.query(query1).toPromise()
- await clickhouse.query(query2).toPromise()
+ await clickhouse.query({
+ query: `ALTER table analytics DELETE WHERE pid='${id}'`,
+ })
+ await clickhouse.query({
+ query: `ALTER table customEV DELETE WHERE pid='${id}'`,
+ })
return 'Project deleted successfully'
} catch (e) {
this.logger.error(e)
diff --git a/apps/selfhosted/src/project/project.service.ts b/apps/selfhosted/src/project/project.service.ts
index 40ec72ed9..72d4b3bd7 100644
--- a/apps/selfhosted/src/project/project.service.ts
+++ b/apps/selfhosted/src/project/project.service.ts
@@ -25,12 +25,12 @@ import { ProjectDTO } from './dto/project.dto'
import {
isValidPID,
redis,
- clickhouse,
IP_REGEX,
ORIGINS_REGEX,
getRedisProjectKey,
redisProjectCacheTimeout,
} from '../common/constants'
+import { clickhouse } from '../common/integrations/clickhouse'
import { getProjectsClickhouse } from '../common/utils'
import { MAX_PROJECT_PASSWORD_LENGTH, UpdateProjectDto } from './dto'
import { Funnel } from './entity/funnel.entity'
@@ -171,9 +171,14 @@ export class ProjectService {
);
`
- const result: any = await clickhouse.query(query, { params }).toPromise()
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: params,
+ })
+ .then(resultSet => resultSet.json())
- return _map(result, ({ pid }) => pid)
+ return _map(data, ({ pid }) => pid)
}
async getPIDsWhereErrorsDataExists(projectIds: string[]): Promise<string[]> {
@@ -215,9 +220,14 @@ export class ProjectService {
);
`
- const result: any = await clickhouse.query(query, { params }).toPromise()
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: params,
+ })
+ .then(resultSet => resultSet.json())
- return _map(result, ({ pid }) => pid)
+ return _map(data, ({ pid }) => pid)
}
async removeDataFromClickhouse(
@@ -239,11 +249,18 @@ export class ProjectService {
},
}
- await Promise.all([
- clickhouse.query(queryAnalytics, params).toPromise(),
- clickhouse.query(queryCustomEvents, params).toPromise(),
- clickhouse.query(queryPerformance, params).toPromise(),
- ])
+ await clickhouse.query({
+ query: queryAnalytics,
+ query_params: params,
+ })
+ await clickhouse.query({
+ query: queryCustomEvents,
+ query_params: params,
+ })
+ await clickhouse.query({
+ query: queryPerformance,
+ query_params: params,
+ })
}
formatToClickhouse(project: any): object {
diff --git a/apps/selfhosted/src/task-manager/task-manager.service.ts b/apps/selfhosted/src/task-manager/task-manager.service.ts
index 2a19640bc..99c661726 100644
--- a/apps/selfhosted/src/task-manager/task-manager.service.ts
+++ b/apps/selfhosted/src/task-manager/task-manager.service.ts
@@ -10,7 +10,6 @@ import * as _map from 'lodash/map'
import * as _now from 'lodash/now'
import {
- clickhouse,
redis,
REDIS_LOG_DATA_CACHE_KEY,
REDIS_LOG_CUSTOM_CACHE_KEY,
@@ -19,6 +18,7 @@ import {
REDIS_LOG_CAPTCHA_CACHE_KEY,
REDIS_LOG_ERROR_CACHE_KEY,
} from '../common/constants'
+import { clickhouse } from '../common/integrations/clickhouse'
import { AppLoggerService } from '../logger/logger.service'
dayjs.extend(utc)
@@ -39,7 +39,9 @@ export class TaskManagerService {
await redis.del(REDIS_LOG_DATA_CACHE_KEY)
const query = `INSERT INTO analytics (*) VALUES ${_join(data, ',')}`
try {
- await clickhouse.query(query).toPromise()
+ await clickhouse.query({
+ query,
+ })
} catch (e) {
console.error(`[CRON WORKER] Error whilst saving log data: ${e}`)
}
@@ -49,7 +51,9 @@ export class TaskManagerService {
await redis.del(REDIS_LOG_CAPTCHA_CACHE_KEY)
const query = `INSERT INTO captcha (*) VALUES ${_join(captchaData, ',')}`
try {
- await clickhouse.query(query).toPromise()
+ await clickhouse.query({
+ query,
+ })
} catch (e) {
console.error(
`[CRON WORKER] Error whilst saving CAPTCHA log data: ${e}`,
@@ -62,7 +66,9 @@ export class TaskManagerService {
const query = `INSERT INTO customEV (*) VALUES ${_join(customData, ',')}`
try {
- await clickhouse.query(query).toPromise()
+ await clickhouse.query({
+ query,
+ })
} catch (e) {
console.error(
`[CRON WORKER] Error whilst saving custom events data: ${e}`,
@@ -75,7 +81,9 @@ export class TaskManagerService {
const query = `INSERT INTO errors (*) VALUES ${_join(errorData, ',')}`
try {
- await clickhouse.query(query).toPromise()
+ await clickhouse.query({
+ query,
+ })
} catch (e) {
console.error(
`[CRON WORKER] Error whilst saving error events data: ${e}`,
@@ -89,7 +97,9 @@ export class TaskManagerService {
const query = `INSERT INTO performance (*) VALUES ${_join(perfData, ',')}`
try {
- await clickhouse.query(query).toPromise()
+ await clickhouse.query({
+ query,
+ })
} catch (e) {
console.error(
`[CRON WORKER] Error whilst saving performance data: ${e}`,
@@ -111,7 +121,9 @@ export class TaskManagerService {
.subtract(20, 'm')
.format('YYYY-MM-DD HH:mm:ss')}'`
- await clickhouse.query(delSidQuery).toPromise()
+ await clickhouse.query({
+ query: delSidQuery,
+ })
}
@Cron(CronExpression.EVERY_MINUTE)
@@ -148,7 +160,9 @@ export class TaskManagerService {
([key]) => `'${key.split(':')[1]}'`,
).join(',')})`
- await clickhouse.query(setSdurQuery).toPromise()
+ await clickhouse.query({
+ query: setSdurQuery,
+ })
}
}
@@ -163,7 +177,9 @@ export class TaskManagerService {
]
const promises = _map(queries, async query => {
- await clickhouse.query(query).toPromise()
+ await clickhouse.query({
+ query,
+ })
})
await Promise.allSettled(promises).catch(reason => {