Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

migrate to the official clickhouse/client library

Blue Mouse901d48570262
diff --git a/apps/production/src/analytics/analytics.controller.ts b/apps/production/src/analytics/analytics.controller.ts
index 1e12dcf24..9bc6072d6 100644
--- a/apps/production/src/analytics/analytics.controller.ts
+++ b/apps/production/src/analytics/analytics.controller.ts
@@ -66,9 +66,9 @@ import {
REDIS_PROJECTS_COUNT_KEY,
REDIS_EVENTS_COUNT_KEY,
REDIS_SESSION_SALT_KEY,
- clickhouse,
REDIS_LOG_ERROR_CACHE_KEY,
} from '../common/constants'
+import { clickhouse } from '../common/integrations/clickhouse'
import {
checkRateLimit,
getGeoDetails,
@@ -1220,11 +1220,11 @@ export class AnalyticsController {
@Get('liveVisitors')
@Auth([], true, true)
async getLiveVisitors(
- @Query() data,
+ @Query() queryParams,
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
): Promise<object> {
- const { pid } = data
+ const { pid } = queryParams
this.analyticsService.validatePID(pid)
await this.analyticsService.checkProjectAccess(
@@ -1246,8 +1246,10 @@ export class AnalyticsController {
const query = `SELECT sid, dv, br, os, cc FROM analytics WHERE sid IN (${sids
.map(el => `'${el}'`)
.join(',')})`
- const result = await clickhouse.query(query).toPromise()
- const processed = _map(_uniqBy(result, 'sid'), el =>
+ const { data } = await clickhouse
+ .query({ query })
+ .then(resultSet => resultSet.json())
+ const processed = _map(_uniqBy(data, 'sid'), el =>
_pick(el, ['dv', 'br', 'os', 'cc']),
)
diff --git a/apps/production/src/analytics/analytics.service.ts b/apps/production/src/analytics/analytics.service.ts
index e45ecec49..a0c7c6d2d 100644
--- a/apps/production/src/analytics/analytics.service.ts
+++ b/apps/production/src/analytics/analytics.service.ts
@@ -44,7 +44,6 @@ import {
redis,
isValidPID,
UNIQUE_SESSION_LIFE_TIME,
- clickhouse,
REDIS_SESSION_SALT_KEY,
TRAFFIC_COLUMNS,
ALL_COLUMNS,
@@ -58,6 +57,7 @@ import {
REDIS_EVENTS_COUNT_KEY,
TRAFFIC_METAKEY_COLUMNS,
} from '../common/constants'
+import { clickhouse } from '../common/integrations/clickhouse'
import {
calculateRelativePercentage,
millisecondsToSeconds,
@@ -662,7 +662,10 @@ export class AnalyticsService {
return { nodes, links }
}
- async getUserFlow(params: unknown, filtersQuery: string): Promise<IUserFlow> {
+ async getUserFlow(
+ params: Record<string, unknown>,
+ filtersQuery: string,
+ ): Promise<IUserFlow> {
const query = `
SELECT
pg AS source,
@@ -679,11 +682,14 @@ export class AnalyticsService {
prev
`
- const results = <IUserFlowLink[]>(
- await clickhouse.query(query, { params }).toPromise()
- )
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: params,
+ })
+ .then(res => res.json<IUserFlowLink>())
- if (_isEmpty(results)) {
+ if (_isEmpty(data)) {
const empty = { nodes: [], links: [] }
return {
ascending: empty,
@@ -694,7 +700,7 @@ export class AnalyticsService {
const ascendingLinks: IUserFlowLink[] = []
const descendingLinks: IUserFlowLink[] = []
- this.removeCyclicDependencies(results).forEach((row: any) => {
+ this.removeCyclicDependencies(data).forEach((row: any) => {
const link: IUserFlowLink = {
source: row.source,
target: row.target,
@@ -817,12 +823,14 @@ export class AnalyticsService {
return null
}
- const from: any = await clickhouse
- .query(
- 'SELECT created FROM analytics where pid = {pid:FixedString(12)} ORDER BY created ASC LIMIT 1',
- { params: { pid } },
- )
- .toPromise()
+ const { data: from } = await clickhouse
+ .query({
+ query:
+ 'SELECT created FROM analytics where pid = {pid:FixedString(12)} ORDER BY created ASC LIMIT 1',
+ query_params: { pid },
+ })
+ .then(res => res.json<{ created?: string }>())
+
const to = _includes(GMT_0_TIMEZONES, safeTimezone)
? dayjs.utc()
: dayjs().tz(safeTimezone)
@@ -1196,20 +1204,18 @@ export class AnalyticsService {
ORDER BY level DESC;
`
- const result = <Array<IFunnelCHResponse>>(
- await clickhouse
- .query(query, { params: { ...params, ...pageParams } })
- .toPromise()
- )
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: { ...params, ...pageParams },
+ })
+ .then(resultSet => resultSet.json<IFunnelCHResponse>())
- if (_isEmpty(result)) {
+ if (_isEmpty(data)) {
return this.generateEmptyFunnel(pages)
}
- return this.formatFunnel(
- _reverse(this.backfillFunnel(result, pages)),
- pages,
- )
+ return this.formatFunnel(_reverse(this.backfillFunnel(data, pages)), pages)
}
async getTotalPageviews(
@@ -1224,13 +1230,15 @@ export class AnalyticsService {
WHERE pid = {pid:FixedString(12)}
AND created BETWEEN {groupFrom:String} AND {groupTo:String}
`
- const result = <{ c: number }[]>(
- await clickhouse
- .query(query, { params: { pid, groupFrom, groupTo } })
- .toPromise()
- )
- return result[0]?.c || 0
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: { pid, groupFrom, groupTo },
+ })
+ .then(resultSet => resultSet.json<{ c: number }>())
+
+ return data[0]?.c || 0
}
async getCaptchaSummary(
@@ -1281,20 +1289,21 @@ export class AnalyticsService {
try {
if (period === 'all') {
const queryAll = `SELECT count(*) AS all FROM captcha WHERE pid = {pid:FixedString(12)} ${filtersQuery}`
- const rawResult = <Array<Partial<BirdseyeCHResponse>>>await clickhouse
- .query(queryAll, {
- params: { pid, ...filtersParams },
+ const { data } = await clickhouse
+ .query({
+ query: queryAll,
+ query_params: { pid, ...filtersParams },
})
- .toPromise()
+ .then(resultSet => resultSet.json<Partial<BirdseyeCHResponse>>())
result[pid] = {
current: {
- all: rawResult[0].all,
+ all: data[0].all,
},
previous: {
all: 0,
},
- change: rawResult[0].all,
+ change: data[0].all,
}
return
}
@@ -1329,9 +1338,10 @@ export class AnalyticsService {
const query = `${queryCurrent} UNION ALL ${queryPrevious}`
- let rawResult = <Array<Partial<BirdseyeCHResponse>>>await clickhouse
- .query(query, {
- params: {
+ let { data } = await clickhouse
+ .query({
+ query,
+ query_params: {
pid,
periodFormatted,
periodSubtracted,
@@ -1339,12 +1349,12 @@ export class AnalyticsService {
...filtersParams,
},
})
- .toPromise()
+ .then(resultSet => resultSet.json<Partial<BirdseyeCHResponse>>())
- rawResult = _sortBy(rawResult, 'sortOrder')
+ data = _sortBy(data, 'sortOrder')
- const currentPeriod = rawResult[0]
- const previousPeriod = rawResult[1]
+ const currentPeriod = data[0]
+ const previousPeriod = data[1]
result[pid] = {
current: {
@@ -1439,27 +1449,28 @@ export class AnalyticsService {
queryAll = `SELECT count(*) AS all FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery}`
}
- const rawResult = <Array<BirdseyeCHResponse>>await clickhouse
- .query(queryAll, {
- params: { pid, ...filtersParams },
+ const { data } = await clickhouse
+ .query({
+ query: queryAll,
+ query_params: {
+ pid,
+ ...filtersParams,
+ },
})
- .toPromise()
+ .then(resultSet => resultSet.json<BirdseyeCHResponse>())
let bounceRate = 0
- if (rawResult[0].all > 0 && !customEVFilterApplied) {
- bounceRate = _round(
- (rawResult[0].unique * 100) / rawResult[0].all,
- 1,
- )
+ if (data[0].all > 0 && !customEVFilterApplied) {
+ bounceRate = _round((data[0].unique * 100) / data[0].all, 1)
}
result[pid] = {
current: {
- all: rawResult[0].all,
- unique: rawResult[0].unique,
+ all: data[0].all,
+ unique: data[0].unique,
bounceRate,
- sdur: rawResult[0].sdur,
+ sdur: data[0].sdur,
},
previous: {
all: 0,
@@ -1467,10 +1478,10 @@ export class AnalyticsService {
bounceRate: 0,
sdur: 0,
},
- change: rawResult[0].all,
- uniqueChange: rawResult[0].unique,
+ change: data[0].all,
+ uniqueChange: data[0].unique,
bounceRateChange: bounceRate,
- sdurChange: rawResult[0].sdur,
+ sdurChange: data[0].sdur,
customEVFilterApplied,
}
return
@@ -1511,9 +1522,10 @@ export class AnalyticsService {
const query = `${queryCurrent} UNION ALL ${queryPrevious}`
- let rawResult = <Array<BirdseyeCHResponse>>await clickhouse
- .query(query, {
- params: {
+ let { data } = await clickhouse
+ .query({
+ query,
+ query_params: {
pid,
periodFormatted,
periodSubtracted,
@@ -1521,12 +1533,12 @@ export class AnalyticsService {
...filtersParams,
},
})
- .toPromise()
+ .then(resultSet => resultSet.json<BirdseyeCHResponse>())
- rawResult = _sortBy(rawResult, 'sortOrder')
+ data = _sortBy(data, 'sortOrder')
- const currentPeriod = rawResult[0]
- const previousPeriod = rawResult[1]
+ const currentPeriod = data[0]
+ const previousPeriod = data[1]
let bounceRate = 0
let prevBounceRate = 0
@@ -1635,26 +1647,24 @@ export class AnalyticsService {
try {
if (period === 'all') {
const queryAll = `SELECT ${columnSelectors} FROM performance WHERE pid = {pid:FixedString(12)} ${filtersQuery}`
- const rawResult = <Array<Partial<PerformanceCHResponse>>>(
- await clickhouse
- .query(queryAll, {
- params: { pid, ...filtersParams },
- })
- .toPromise()
- )
+
+ const { data } = await clickhouse
+ .query({
+ query: queryAll,
+ query_params: {
+ pid,
+ ...filtersParams,
+ },
+ })
+ .then(resultSet => resultSet.json<Partial<PerformanceCHResponse>>())
result[pid] = {
current: {
- frontend: millisecondsToSeconds(
- rawResult[0].render + rawResult[0].domLoad,
- ),
+ frontend: millisecondsToSeconds(data[0].render + data[0].domLoad),
network: millisecondsToSeconds(
- rawResult[0].dns +
- rawResult[0].tls +
- rawResult[0].conn +
- rawResult[0].response,
+ data[0].dns + data[0].tls + data[0].conn + data[0].response,
),
- backend: millisecondsToSeconds(rawResult[0].ttfb),
+ backend: millisecondsToSeconds(data[0].ttfb),
},
previous: {
frontend: 0,
@@ -1662,15 +1672,12 @@ export class AnalyticsService {
backend: 0,
},
frontendChange: millisecondsToSeconds(
- rawResult[0].render + rawResult[0].domLoad,
+ data[0].render + data[0].domLoad,
),
networkChange: millisecondsToSeconds(
- rawResult[0].dns +
- rawResult[0].tls +
- rawResult[0].conn +
- rawResult[0].response,
+ data[0].dns + data[0].tls + data[0].conn + data[0].response,
),
- backendChange: millisecondsToSeconds(rawResult[0].ttfb),
+ backendChange: millisecondsToSeconds(data[0].ttfb),
}
return
}
@@ -1705,9 +1712,10 @@ export class AnalyticsService {
const query = `${queryCurrent} UNION ALL ${queryPrevious}`
- let rawResult = <Array<Partial<PerformanceCHResponse>>>await clickhouse
- .query(query, {
- params: {
+ let { data } = await clickhouse
+ .query({
+ query,
+ query_params: {
pid,
periodFormatted,
periodSubtracted,
@@ -1715,12 +1723,12 @@ export class AnalyticsService {
...filtersParams,
},
})
- .toPromise()
+ .then(resultSet => resultSet.json<Partial<PerformanceCHResponse>>())
- rawResult = _sortBy(rawResult, 'sortOrder')
+ data = _sortBy(data, 'sortOrder')
- const currentPeriod = rawResult[0]
- const previousPeriod = rawResult[1]
+ const currentPeriod = data[0]
+ const previousPeriod = data[1]
result[pid] = {
current: {
@@ -1796,11 +1804,16 @@ export class AnalyticsService {
query = `SELECT ${type} FROM customEV WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}`
}
- const results = await clickhouse
- .query(query, { params: { pid } })
- .toPromise()
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: {
+ pid,
+ },
+ })
+ .then(resultSet => resultSet.json())
- return _map(results, type)
+ return _map(data, type)
}
async getErrorsFilters(pid: string, type: string): Promise<Array<string>> {
@@ -1812,11 +1825,16 @@ export class AnalyticsService {
const query = `SELECT ${type} FROM errors WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}`
- const results = await clickhouse
- .query(query, { params: { pid } })
- .toPromise()
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: {
+ pid,
+ },
+ })
+ .then(resultSet => resultSet.json())
- return _map(results, type)
+ return _map(data, type)
}
async generateParams(
@@ -1862,9 +1880,15 @@ export class AnalyticsService {
type,
measure,
)
- const res = await clickhouse.query(query, paramsData).toPromise()
- params[col] = res
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json())
+
+ params[col] = data
})
await Promise.all(paramsPromises)
@@ -2422,13 +2446,15 @@ export class AnalyticsService {
mode,
)
- const result = <Array<TrafficCEFilterCHResponse>>(
- await clickhouse.query(query, paramsData).toPromise()
- )
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json<TrafficCEFilterCHResponse>())
const uniques =
- this.extractCustomEventsChartData(result, xShifted)?._unknown_event ||
- []
+ this.extractCustomEventsChartData(data, xShifted)?._unknown_event || []
const sdur = Array(_size(xShifted)).fill(0)
@@ -2449,11 +2475,14 @@ export class AnalyticsService {
mode,
)
- const result = <Array<TrafficCHResponse>>(
- await clickhouse.query(query, paramsData).toPromise()
- )
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json<TrafficCHResponse>())
- const { visits, uniques, sdur } = this.extractChartData(result, xShifted)
+ const { visits, uniques, sdur } = this.extractChartData(data, xShifted)
return Promise.resolve({
chart: {
@@ -2539,9 +2568,12 @@ export class AnalyticsService {
// ChartRenderMode.PERIODICAL,
// )
- // const result = <Array<TrafficCEFilterCHResponse>>(
- // await clickhouse.query(query, paramsData).toPromise()
- // )
+ // const { data } = await clickhouse
+ // .query({
+ // query,
+ // query_params: paramsData.params,
+ // })
+ // .then(resultSet => resultSet.json<TrafficCEFilterCHResponse>())
// const uniques =
// this.extractCustomEventsChartData(result, xShifted)?._unknown_event ||
@@ -2561,11 +2593,14 @@ export class AnalyticsService {
const query = this.generateSessionAggregationQuery(timeBucket, filtersQuery)
- const result = <Array<TrafficCHResponse>>(
- await clickhouse.query(query, paramsData).toPromise()
- )
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json<TrafficCHResponse>())
- const { visits, uniques, sdur } = this.extractChartData(result, x)
+ const { visits, uniques, sdur } = this.extractChartData(data, x)
return Promise.resolve({
chart: {
@@ -2601,7 +2636,7 @@ export class AnalyticsService {
to: string,
subQuery: string,
filtersQuery: string,
- paramsData: object,
+ paramsData: any,
safeTimezone: string,
mode: ChartRenderMode,
): Promise<object | void> {
@@ -2642,10 +2677,13 @@ export class AnalyticsService {
mode,
)
- const result = <Array<TrafficCEFilterCHResponse>>(
- await clickhouse.query(query, paramsData).toPromise()
- )
- const { count } = this.extractCaptchaChartData(result, xShifted)
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json<TrafficCEFilterCHResponse>())
+ const { count } = this.extractCaptchaChartData(data, xShifted)
chart = {
x: xShifted,
@@ -2668,7 +2706,7 @@ export class AnalyticsService {
to: string,
subQuery: string,
filtersQuery: string,
- paramsData: object,
+ paramsData: any,
safeTimezone: string,
mode: ChartRenderMode,
): Promise<object | void> {
@@ -2703,10 +2741,13 @@ export class AnalyticsService {
mode,
)
- const result = <Array<TrafficCEFilterCHResponse>>(
- await clickhouse.query(query, paramsData).toPromise()
- )
- const { count } = this.extractCaptchaChartData(result, x)
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json<TrafficCEFilterCHResponse>())
+ const { count } = this.extractCaptchaChartData(data, x)
chart = {
x: this.shiftToTimezone(x, safeTimezone, format),
@@ -2797,7 +2838,7 @@ export class AnalyticsService {
from: string,
to: string,
filtersQuery: string,
- paramsData: object,
+ paramsData: any,
safeTimezone: string,
measure: PerfMeasure,
) {
@@ -2810,11 +2851,16 @@ export class AnalyticsService {
safeTimezone,
)
- const result = await clickhouse.query(query, paramsData).toPromise()
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json())
return {
x: xShifted,
- ...this.extractPerformanceQuantilesData(result, xShifted),
+ ...this.extractPerformanceQuantilesData(data, xShifted),
}
}
@@ -2825,13 +2871,16 @@ export class AnalyticsService {
measure,
)
- const result = <Array<PerformanceCHResponse>>(
- await clickhouse.query(query, paramsData).toPromise()
- )
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json<PerformanceCHResponse>())
return {
x: xShifted,
- ...this.extractPerformanceChartData(result, xShifted),
+ ...this.extractPerformanceChartData(data, xShifted),
}
}
@@ -2841,7 +2890,7 @@ export class AnalyticsService {
to: string,
subQuery: string,
filtersQuery: string,
- paramsData: object,
+ paramsData: any,
safeTimezone: string,
measure: PerfMeasure,
): Promise<object | void> {
@@ -2891,18 +2940,21 @@ export class AnalyticsService {
async getCustomEvents(
filtersQuery: string,
- params: object,
+ params: any,
): Promise<ICustomEvent> {
const query = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`
const result = {}
- const rawCustoms = <Array<CustomsCHResponse>>(
- await clickhouse.query(query, params).toPromise()
- )
- const size = _size(rawCustoms)
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: params.params,
+ })
+ .then(resultSet => resultSet.json<CustomsCHResponse>())
+ const size = _size(data)
for (let i = 0; i < size; ++i) {
- const { ev, 'count()': c } = rawCustoms[i]
+ const { ev, 'count()': c } = data[i]
result[ev] = c
}
@@ -2911,7 +2963,7 @@ export class AnalyticsService {
async getPageProperties(
filtersQuery: string,
- params: object,
+ params: any,
): Promise<IPageProperty> {
const query = `
SELECT
@@ -2924,13 +2976,16 @@ export class AnalyticsService {
GROUP BY property`
const result = {}
- const rawProperties = <Array<PropertiesCHResponse>>(
- await clickhouse.query(query, params).toPromise()
- )
- const size = _size(rawProperties)
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: params.params,
+ })
+ .then(resultSet => resultSet.json<PropertiesCHResponse>())
+ const size = _size(data)
for (let i = 0; i < size; ++i) {
- const { property: propertyArr, 'count()': c } = rawProperties[i]
+ const { property: propertyArr, 'count()': c } = data[i]
const [property] = propertyArr
if (!property) {
@@ -3033,9 +3088,12 @@ export class AnalyticsService {
}
try {
- const result = (await clickhouse
- .query(query, paramsData)
- .toPromise()) as IAggregatedMetadata[]
+ const { data: result } = await clickhouse
+ .query({
+ query,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json<IAggregatedMetadata>())
return {
result,
@@ -3134,9 +3192,12 @@ export class AnalyticsService {
}
try {
- const result = (await clickhouse
- .query(query, paramsData)
- .toPromise()) as IAggregatedMetadata[]
+ const { data: result } = await clickhouse
+ .query({
+ query,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json<IAggregatedMetadata>())
return {
result,
@@ -3160,7 +3221,7 @@ export class AnalyticsService {
from: string,
to: string,
filtersQuery: string,
- paramsData: object,
+ paramsData: any,
safeTimezone: string,
): Promise<object | void> {
const { xShifted } = this.generateXAxis(timeBucket, from, to, safeTimezone)
@@ -3188,11 +3249,14 @@ export class AnalyticsService {
ORDER BY ${groupBy}
`
- const result = <Array<CustomsCHAggregatedResponse>>(
- await clickhouse.query(query, paramsData).toPromise()
- )
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json<CustomsCHAggregatedResponse>())
- const events = this.extractCustomEventsChartData(result, xShifted)
+ const events = this.extractCustomEventsChartData(data, xShifted)
return Promise.resolve({
chart: {
@@ -3266,11 +3330,20 @@ export class AnalyticsService {
},
}
- const pages = <IPageflow[]>(
- await clickhouse.query(queryPages, paramsData).toPromise()
- )
+ const { data: pages } = await clickhouse
+ .query({
+ query: queryPages,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json<IPageflow>())
+
let details = (
- await clickhouse.query(querySessionDetails, paramsData).toPromise()
+ await clickhouse
+ .query({
+ query: querySessionDetails,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json())
)[0]
if (!details) {
@@ -3287,8 +3360,11 @@ export class AnalyticsService {
// eslint-disable-next-line prefer-destructuring
details = (
await clickhouse
- .query(querySessionDetailsBackup, paramsData)
- .toPromise()
+ .query({
+ query: querySessionDetailsBackup,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json())
)[0]
}
@@ -3337,7 +3413,7 @@ export class AnalyticsService {
async getSessionsList(
filtersQuery: string,
- paramsData: object,
+ paramsData: any,
safeTimezone: string,
take = 30,
skip = 0,
@@ -3394,15 +3470,20 @@ export class AnalyticsService {
OFFSET ${skip}
`
- const result = await clickhouse.query(query, paramsData).toPromise()
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json())
- return result
+ return data
}
async getErrorsList(
options: string,
filtersQuery: string,
- paramsData: object,
+ paramsData: any,
safeTimezone: string,
take = 30,
skip = 0,
@@ -3452,9 +3533,14 @@ export class AnalyticsService {
OFFSET ${skip};
`
- const result = await clickhouse.query(query, paramsData).toPromise()
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json())
- return result
+ return data
}
async getErrorDetails(
@@ -3520,11 +3606,21 @@ export class AnalyticsService {
}
const details = (
- await clickhouse.query(queryErrorDetails, paramsData).toPromise()
+ await clickhouse
+ .query({
+ query: queryErrorDetails,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json())
)[0]
const occurenceDetails = (
- await clickhouse.query(queryFirstLastSeen, paramsData).toPromise()
+ await clickhouse
+ .query({
+ query: queryFirstLastSeen,
+ query_params: paramsData.params,
+ })
+ .then(resultSet => resultSet.json())
)[0]
const groupedChart = await this.groupErrorsByTimeBucket(
@@ -3574,9 +3670,15 @@ export class AnalyticsService {
let result
try {
- ;[result] = await clickhouse
- .query(query, { params: { ...params, pid } })
- .toPromise()
+ // eslint-disable-next-line prefer-destructuring
+ result = (
+ await clickhouse
+ .query({
+ query,
+ query_params: { ...params, pid },
+ })
+ .then(resultSet => resultSet.json())
+ )[0]
} catch (reason) {
console.error('validateEIDs - clickhouse request error')
console.error(reason)
@@ -3613,7 +3715,13 @@ export class AnalyticsService {
).join(', ')}`
try {
- await clickhouse.query(query, { params: { ...params, pid } }).toPromise()
+ await clickhouse.query({
+ query,
+ query_params: {
+ ...params,
+ pid,
+ },
+ })
} catch (reason) {
console.error('Error at PATCH error-status:')
console.error(reason)
@@ -3657,9 +3765,14 @@ export class AnalyticsService {
const users = await this.userService.count()
const projects = await this.projectService.count()
- const results = await clickhouse.query(query).toPromise()
+ const { data } = await clickhouse
+ .query({
+ query,
+ })
+ .then(resultSet => resultSet.json())
+
// @ts-expect-error
- const events = results.reduce((total, row) => total + row.count, 0)
+ const events = data.reduce((total, row) => total + row.count, 0) as number
await redis.set(REDIS_USERS_COUNT_KEY, users, 'EX', 630)
await redis.set(REDIS_PROJECTS_COUNT_KEY, projects, 'EX', 630)
@@ -3731,8 +3844,12 @@ export class AnalyticsService {
`
try {
- const result = await clickhouse.query(query, { params }).toPromise()
- return result
+ return await clickhouse
+ .query({
+ query,
+ query_params: params,
+ })
+ .then(resultSet => resultSet.json())
} catch (reason) {
console.error('[ERROR] (getMetaResult) - Clickhouse query error:')
console.error(reason)
diff --git a/apps/production/src/project/project.service.ts b/apps/production/src/project/project.service.ts
index 204457c55..659492be2 100644
--- a/apps/production/src/project/project.service.ts
+++ b/apps/production/src/project/project.service.ts
@@ -42,7 +42,6 @@ import {
redisProjectCountCacheTimeout,
getRedisUserCountKey,
redis,
- clickhouse,
IP_REGEX,
ORIGINS_REGEX,
getRedisProjectKey,
@@ -54,6 +53,7 @@ import {
TRAFFIC_COLUMNS,
EMAIL_ACTION_ENCRYPTION_KEY,
} from '../common/constants'
+import { clickhouse } from '../common/integrations/clickhouse'
import { IUsageInfoRedis } from '../user/interfaces'
import { ProjectSubscriber, Funnel } from './entity'
import { AddSubscriberType } from './types'
@@ -571,7 +571,10 @@ export class ProjectService {
query += ')'
- await clickhouse.query(query, { params }).toPromise()
+ await clickhouse.query({
+ query,
+ query_params: params,
+ })
}
async removeDataFromClickhouse(
@@ -594,9 +597,18 @@ export class ProjectService {
}
await Promise.all([
- clickhouse.query(queryAnalytics, params).toPromise(),
- clickhouse.query(queryCustomEvents, params).toPromise(),
- clickhouse.query(queryPerformance, params).toPromise(),
+ clickhouse.query({
+ query: queryAnalytics,
+ query_params: params,
+ }),
+ clickhouse.query({
+ query: queryCustomEvents,
+ query_params: params,
+ }),
+ clickhouse.query({
+ query: queryPerformance,
+ query_params: params,
+ }),
])
}
@@ -686,15 +698,26 @@ export class ProjectService {
const countCaptchaQuery = `SELECT count() FROM captcha ${selector}`
const countErrorsQuery = `SELECT count() FROM errors ${selector}`
- const promises = [
- clickhouse.query(countEVQuery).toPromise(),
- clickhouse.query(countCustomEVQuery).toPromise(),
- clickhouse.query(countCaptchaQuery).toPromise(),
- clickhouse.query(countErrorsQuery).toPromise(),
- ]
-
- const [pageviews, customEvents, captcha, errors] =
- await Promise.all(promises)
+ const { data: pageviews } = await clickhouse
+ .query({
+ query: countEVQuery,
+ })
+ .then(resultSet => resultSet.json())
+ const { data: customEvents } = await clickhouse
+ .query({
+ query: countCustomEVQuery,
+ })
+ .then(resultSet => resultSet.json())
+ const { data: captcha } = await clickhouse
+ .query({
+ query: countCaptchaQuery,
+ })
+ .then(resultSet => resultSet.json())
+ const { data: errors } = await clickhouse
+ .query({
+ query: countErrorsQuery,
+ })
+ .then(resultSet => resultSet.json())
count =
pageviews[0]['count()'] +
@@ -751,15 +774,26 @@ export class ProjectService {
const countCaptchaQuery = `SELECT count() FROM captcha ${selector}`
const countErrorsQuery = `SELECT count() FROM errors ${selector}`
- const promises = [
- clickhouse.query(countEVQuery).toPromise(),
- clickhouse.query(countCustomEVQuery).toPromise(),
- clickhouse.query(countCaptchaQuery).toPromise(),
- clickhouse.query(countErrorsQuery).toPromise(),
- ]
-
- const [rawTraffic, rawCustomEvents, rawCaptcha, rawErrors] =
- await Promise.all(promises)
+ const { data: rawTraffic } = await clickhouse
+ .query({
+ query: countEVQuery,
+ })
+ .then(resultSet => resultSet.json())
+ const { data: rawCustomEvents } = await clickhouse
+ .query({
+ query: countCustomEVQuery,
+ })
+ .then(resultSet => resultSet.json())
+ const { data: rawCaptcha } = await clickhouse
+ .query({
+ query: countCaptchaQuery,
+ })
+ .then(resultSet => resultSet.json())
+ const { data: rawErrors } = await clickhouse
+ .query({
+ query: countErrorsQuery,
+ })
+ .then(resultSet => resultSet.json())
const traffic = rawTraffic[0]['count()']
const customEvents = rawCustomEvents[0]['count()']
@@ -892,9 +926,14 @@ export class ProjectService {
);
`
- const result: any = await clickhouse.query(query, { params }).toPromise()
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: params,
+ })
+ .then(resultSet => resultSet.json())
- return _map(result, ({ pid }) => pid)
+ return _map(data, ({ pid }) => pid)
}
async getPIDsWhereErrorsDataExists(projectIds: string[]): Promise<string[]> {
@@ -936,9 +975,14 @@ export class ProjectService {
);
`
- const result: any = await clickhouse.query(query, { params }).toPromise()
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: params,
+ })
+ .then(resultSet => resultSet.json())
- return _map(result, ({ pid }) => pid)
+ return _map(data, ({ pid }) => pid)
}
async getSubscriberByEmail(projectId: string, email: string) {
@@ -1264,8 +1308,14 @@ export class ProjectService {
let count = 0
try {
- const result = await clickhouse.query(query, { params }).toPromise()
- count = result[0]['count()']
+ const { data } = await clickhouse
+ .query({
+ query,
+ query_params: params,
+ })
+ .then(resultSet => resultSet.json())
+
+ count = data[0]['count()']
} catch (reason) {
console.error(
'[ERROR](project service -> countVisitorsFromTo) Error while counting visitors',
diff --git a/apps/production/src/task-manager/task-manager.service.ts b/apps/production/src/task-manager/task-manager.service.ts
index 03c8eb0a0..dc43b7bd6 100644
--- a/apps/production/src/task-manager/task-manager.service.ts
+++ b/apps/production/src/task-manager/task-manager.service.ts
@@ -43,7 +43,6 @@ import {
DashboardBlockReason,
} from '../user/entities/user.entity'
import {
- clickhouse,
redis,
REDIS_LOG_DATA_CACHE_KEY,
REDIS_LOG_CUSTOM_CACHE_KEY,
@@ -59,6 +58,7 @@ import {
isDevelopment,
REDIS_LOG_ERROR_CACHE_KEY,
} from '../common/constants'
+import { clickhouse } from '../common/integrations/clickhouse'
import { CHPlanUsage } from './interfaces'
import { getRandomTip } from '../common/utils'
import { AppLoggerService } from '../logger/logger.service'
@@ -420,7 +420,9 @@ export class TaskManagerService {
await redis.del(REDIS_LOG_DATA_CACHE_KEY)
const query = `INSERT INTO analytics (*) VALUES ${_join(data, ',')}`
try {
- await clickhouse.query(query).toPromise()
+ await clickhouse.query({
+ query,
+ })
} catch (e) {
console.error(`[CRON WORKER] Error whilst saving log data: ${e}`)
}
@@ -430,7 +432,9 @@ export class TaskManagerService {
await redis.del(REDIS_LOG_CAPTCHA_CACHE_KEY)
const query = `INSERT INTO captcha (*) VALUES ${_join(captchaData, ',')}`
try {
- await clickhouse.query(query).toPromise()
+ await clickhouse.query({
+ query,
+ })
} catch (e) {
console.error(
`[CRON WORKER] Error whilst saving CAPTCHA log data: ${e}`,
@@ -443,7 +447,9 @@ export class TaskManagerService {
const query = `INSERT INTO customEV (*) VALUES ${_join(customData, ',')}`
try {
- await clickhouse.query(query).toPromise()
+ await clickhouse.query({
+ query,
+ })
} catch (e) {
console.error(
`[CRON WORKER] Error whilst saving custom events data: ${e}`,
@@ -456,7 +462,9 @@ export class TaskManagerService {
const query = `INSERT INTO errors (*) VALUES ${_join(errorData, ',')}`
try {
- await clickhouse.query(query).toPromise()
+ await clickhouse.query({
+ query,
+ })
} catch (e) {
console.error(
`[CRON WORKER] Error whilst saving error events data: ${e}`,
@@ -470,7 +478,9 @@ export class TaskManagerService {
const query = `INSERT INTO performance (*) VALUES ${_join(perfData, ',')}`
try {
- await clickhouse.query(query).toPromise()
+ await clickhouse.query({
+ query,
+ })
} catch (e) {
console.error(
`[CRON WORKER] Error whilst saving performance data: ${e}`,
@@ -509,9 +519,11 @@ export class TaskManagerService {
dayjs.utc(user.planExceedContactedAt).format('YYYY-MM-01'),
)
- const monthlyUsage = <CHPlanUsage[]>(
- await clickhouse.query(monthlyUsageQuery).toPromise()
- )
+ const { data: monthlyUsage } = await clickhouse
+ .query({
+ query: monthlyUsageQuery,
+ })
+ .then(resultSet => resultSet.json<CHPlanUsage>())
const exceedingUserIds = getUserIDsThatExceedPlanUsage(users, monthlyUsage)
@@ -571,9 +583,11 @@ export class TaskManagerService {
const thisMonthDate = dayjs.utc().format('YYYY-MM-01')
const thisMonthQuery = generatePlanUsageQuery(users, () => thisMonthDate)
- const thisMonthUsage = <CHPlanUsage[]>(
- await clickhouse.query(thisMonthQuery).toPromise()
- )
+ const { data: thisMonthUsage } = await clickhouse
+ .query({
+ query: thisMonthQuery,
+ })
+ .then(resultSet => resultSet.json<CHPlanUsage>())
const exceedingUsers = getUsersThatExceedPlanUsage(users, thisMonthUsage)
@@ -627,9 +641,11 @@ export class TaskManagerService {
() => lastMonthDate,
)
- const lastMonthUsage = <CHPlanUsage[]>(
- await clickhouse.query(lastMonthQuery).toPromise()
- )
+ const { data: lastMonthUsage } = await clickhouse
+ .query({
+ query: lastMonthQuery,
+ })
+ .then(resultSet => resultSet.json<CHPlanUsage>())
const continuousExceedingUsers = getUsersThatExceedContinuously(users, [
// the order should be kept like this
@@ -764,7 +780,9 @@ export class TaskManagerService {
.subtract(20, 'm')
.format('YYYY-MM-DD HH:mm:ss')}'`
- await clickhouse.query(delSidQuery).toPromise()
+ await clickhouse.query({
+ query: delSidQuery,
+ })
}
// EVERY SUNDAY AT 2:30 AM
@@ -842,7 +860,9 @@ export class TaskManagerService {
([key]) => `'${key.split(':')[1]}'`,
).join(',')})`
- await clickhouse.query(setSdurQuery).toPromise()
+ await clickhouse.query({
+ query: setSdurQuery,
+ })
}
}
@@ -1107,7 +1127,13 @@ export class TaskManagerService {
const params = {
ev: alert.queryCustomEvent,
}
- const queryResult = await clickhouse.query(query, { params }).toPromise()
+
+ const { data: queryResult } = await clickhouse
+ .query({
+ query,
+ query_params: params,
+ })
+ .then(resultSet => resultSet.json())
const count = Number(queryResult[0]['count()'])
@@ -1237,7 +1263,9 @@ export class TaskManagerService {
]
const promises = _map(queries, async query => {
- await clickhouse.query(query).toPromise()
+ await clickhouse.query({
+ query,
+ })
})
await Promise.allSettled(promises).catch(reason => {
@@ -1326,30 +1354,33 @@ export class TaskManagerService {
const queryCaptcha = `SELECT count() FROM captcha WHERE pid IN (${pidsStringified}) AND created BETWEEN '${nineWeeksAgo}' AND '${now}'`
const queryCustomEvents = `SELECT count() FROM customEV WHERE pid IN (${pidsStringified}) AND created BETWEEN '${nineWeeksAgo}' AND '${now}'`
- const hasAnalyticsActivity =
- Number(
- (await clickhouse.query(queryAnalytics).toPromise())[0]['count()'],
- ) > 0
+ const { data: analyticsResult } = await clickhouse
+ .query({
+ query: queryAnalytics,
+ })
+ .then(resultSet => resultSet.json<{ 'count()': number }>())
- if (hasAnalyticsActivity) {
+ if (analyticsResult[0]['count()'] > 0) {
return
}
- const hasCaptchaActivity =
- Number(
- (await clickhouse.query(queryCaptcha).toPromise())[0]['count()'],
- ) > 0
+ const { data: captchaResult } = await clickhouse
+ .query({
+ query: queryCaptcha,
+ })
+ .then(resultSet => resultSet.json<{ 'count()': number }>())
- if (hasCaptchaActivity) {
+ if (captchaResult[0]['count()'] > 0) {
return
}
- const hasCustomEventsActivity =
- Number(
- (await clickhouse.query(queryCustomEvents).toPromise())[0]['count()'],
- ) > 0
+ const { data: customEventsResult } = await clickhouse
+ .query({
+ query: queryCustomEvents,
+ })
+ .then(resultSet => resultSet.json<{ 'count()': number }>())
- if (hasCustomEventsActivity) {
+ if (customEventsResult[0]['count()'] > 0) {
return
}
diff --git a/apps/production/src/user/user.controller.ts b/apps/production/src/user/user.controller.ts
index 3f037ee5d..bbf3a8c30 100644
--- a/apps/production/src/user/user.controller.ts
+++ b/apps/production/src/user/user.controller.ts
@@ -49,10 +49,10 @@ import {
import { Pagination } from '../common/pagination/pagination'
import {
GDPR_EXPORT_TIMEFRAME,
- clickhouse,
isDevelopment,
PRODUCTION_ORIGIN,
} from '../common/constants'
+import { clickhouse } from '../common/integrations/clickhouse'
import { RolesGuard } from '../auth/guards/roles.guard'
import { UpdateUserProfileDTO } from './dto/update-user.dto'
import { IChangePlanDTO } from './dto/change-plan.dto'
@@ -318,20 +318,26 @@ export class UserController {
_map(user.projects, el => `'${el.id}'`),
',',
)
- const query1 = `ALTER table analytics DELETE WHERE pid IN (${pids})`
- const query2 = `ALTER table customEV DELETE WHERE pid IN (${pids})`
- const query3 = `ALTER table performance DELETE WHERE pid IN (${pids})`
- const query4 = `ALTER table errors DELETE WHERE pid IN (${pids})`
- const query5 = `ALTER table error_statuses DELETE WHERE pid IN (${pids})`
- const query6 = `ALTER table captcha DELETE WHERE pid IN (${pids})`
+
+ const queries = [
+ `ALTER table analytics DELETE WHERE pid IN (${pids})`,
+ `ALTER table customEV DELETE WHERE pid IN (${pids})`,
+ `ALTER table performance DELETE WHERE pid IN (${pids})`,
+ `ALTER table errors DELETE WHERE pid IN (${pids})`,
+ `ALTER table error_statuses DELETE WHERE pid IN (${pids})`,
+ `ALTER table captcha DELETE WHERE pid IN (${pids})`,
+ ]
+
await this.projectService.deleteMultiple(pids)
- await clickhouse.query(query1).toPromise()
- await clickhouse.query(query2).toPromise()
- await clickhouse.query(query3).toPromise()
- await clickhouse.query(query4).toPromise()
- await clickhouse.query(query5).toPromise()
- await clickhouse.query(query6).toPromise()
+
+ const promises = _map(queries, async query =>
+ clickhouse.query({
+ query,
+ }),
+ )
+ await Promise.all(promises)
}
+ await this.actionTokensService.deleteMultiple(`userId="${id}"`)
await this.userService.delete(id)
return 'accountDeleted'
@@ -367,19 +373,21 @@ export class UserController {
_map(user.projects, el => `'${el.id}'`),
',',
)
- const query1 = `ALTER table analytics DELETE WHERE pid IN (${pids})`
- const query2 = `ALTER table customEV DELETE WHERE pid IN (${pids})`
- const query3 = `ALTER table performance DELETE WHERE pid IN (${pids})`
- const query4 = `ALTER table errors DELETE WHERE pid IN (${pids})`
- const query5 = `ALTER table error_statuses DELETE WHERE pid IN (${pids})`
- const query6 = `ALTER table captcha DELETE WHERE pid IN (${pids})`
+ const queries = [
+ `ALTER table analytics DELETE WHERE pid IN (${pids})`,
+ `ALTER table customEV DELETE WHERE pid IN (${pids})`,
+ `ALTER table performance DELETE WHERE pid IN (${pids})`,
+ `ALTER table errors DELETE WHERE pid IN (${pids})`,
+ `ALTER table error_statuses DELETE WHERE pid IN (${pids})`,
+ `ALTER table captcha DELETE WHERE pid IN (${pids})`,
+ ]
await this.projectService.deleteMultiple(pids)
- await clickhouse.query(query1).toPromise()
- await clickhouse.query(query2).toPromise()
- await clickhouse.query(query3).toPromise()
- await clickhouse.query(query4).toPromise()
- await clickhouse.query(query5).toPromise()
- await clickhouse.query(query6).toPromise()
+ const promises = _map(queries, async query =>
+ clickhouse.query({
+ query,
+ }),
+ )
+ await Promise.all(promises)
}
await this.actionTokensService.deleteMultiple(`userId="${id}"`)
await this.userService.delete(id)