Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

(improvement) Enhanced `GET /log/birdseye` (#202)

Andrii Romasiun4f34ae71efc6
diff --git a/apps/production/src/analytics/analytics.controller.ts b/apps/production/src/analytics/analytics.controller.ts
index 59a227788..b3acb1299 100644
--- a/apps/production/src/analytics/analytics.controller.ts
+++ b/apps/production/src/analytics/analytics.controller.ts
@@ -583,12 +583,6 @@ export class AnalyticsController {
headers['x-password'],
)
- let subQuery = `FROM analytics WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
-
- if (customEVFilterApplied) {
- subQuery = `FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
- }
-
const paramsData = {
params: {
pid,
@@ -602,7 +596,6 @@ export class AnalyticsController {
timeBucket,
groupFrom,
groupTo,
- subQuery,
filtersQuery,
paramsData,
safeTimezone,
@@ -846,16 +839,19 @@ export class AnalyticsController {
@CurrentUserId() uid: string,
@Headers() headers: { 'x-password'?: string },
): Promise<any> {
- const { pids, pid } = data
+ const {
+ pids,
+ pid,
+ period,
+ from,
+ to,
+ timezone = DEFAULT_TIMEZONE,
+ filters,
+ } = data
const pidsArray = getPIDsArray(pids, pid)
const validationPromises = _map(pidsArray, async currentPID => {
this.analyticsService.validatePID(currentPID)
- // fix needed
- // fix needed
- // fix needed
- // fix needed
- // passwords and for projects protected []
await this.analyticsService.checkProjectAccess(
currentPID,
uid,
@@ -865,7 +861,14 @@ export class AnalyticsController {
await Promise.all(validationPromises)
- return this.analyticsService.getSummary(pidsArray, 'w')
+ return this.analyticsService.getAnalyticsSummary(
+ pidsArray,
+ period,
+ from,
+ to,
+ timezone,
+ filters,
+ )
}
@Get('captcha/birdseye')
@@ -875,7 +878,15 @@ export class AnalyticsController {
@Query() data,
@CurrentUserId() uid: string,
): Promise<any> {
- const { pids, pid } = data
+ const {
+ pids,
+ pid,
+ period,
+ from,
+ to,
+ timezone = DEFAULT_TIMEZONE,
+ filters,
+ } = data
const pidsArray = getPIDsArray(pids, pid)
const validationPromises = _map(pidsArray, async currentPID => {
@@ -885,7 +896,53 @@ export class AnalyticsController {
await Promise.all(validationPromises)
- return this.analyticsService.getCaptchaSummary(pidsArray, 'w')
+ return this.analyticsService.getCaptchaSummary(
+ pidsArray,
+ period,
+ from,
+ to,
+ timezone,
+ filters,
+ )
+ }
+
+ @Get('performance/birdseye')
+ @Auth([], true, true)
+ async getPerformanceOverallStats(
+ @Query() data,
+ @CurrentUserId() uid: string,
+ @Headers() headers: { 'x-password'?: string },
+ ): Promise<any> {
+ const {
+ pids,
+ pid,
+ period,
+ from,
+ to,
+ timezone = DEFAULT_TIMEZONE,
+ filters,
+ } = data
+ const pidsArray = getPIDsArray(pids, pid)
+
+ const validationPromises = _map(pidsArray, async currentPID => {
+ this.analyticsService.validatePID(currentPID)
+ await this.analyticsService.checkProjectAccess(
+ currentPID,
+ uid,
+ headers['x-password'],
+ )
+ })
+
+ await Promise.all(validationPromises)
+
+ return this.analyticsService.getPerformanceSummary(
+ pidsArray,
+ period,
+ from,
+ to,
+ timezone,
+ filters,
+ )
}
@Public()
diff --git a/apps/production/src/analytics/analytics.service.ts b/apps/production/src/analytics/analytics.service.ts
index 8757997e8..567fda164 100644
--- a/apps/production/src/analytics/analytics.service.ts
+++ b/apps/production/src/analytics/analytics.service.ts
@@ -8,6 +8,7 @@ import * as _map from 'lodash/map'
import * as _toUpper from 'lodash/toUpper'
import * as _join from 'lodash/join'
import * as _isArray from 'lodash/isArray'
+import * as _sortBy from 'lodash/sortBy'
import * as _reduce from 'lodash/reduce'
import * as _keys from 'lodash/keys'
import * as _last from 'lodash/last'
@@ -69,6 +70,7 @@ import {
CustomsCHResponse,
CustomsCHAggregatedResponse,
TrafficCEFilterCHResponse,
+ BirdseyeCHResponse,
TrafficCHResponse,
IGetGroupFromTo,
GetFiltersQuery,
@@ -81,6 +83,9 @@ import {
IAggregatedMetadata,
IFunnelCHResponse,
IFunnel,
+ IOverall,
+ IOverallCaptcha,
+ IOverallPerformance,
} from './interfaces'
dayjs.extend(utc)
@@ -790,7 +795,7 @@ export class AnalyticsService {
let query = ''
let customEVFilterApplied = false
- if (_isEmpty(filters)) {
+ if (filters === '""' || _isEmpty(filters)) {
return [query, params, parsed, customEVFilterApplied]
}
@@ -1095,26 +1100,198 @@ export class AnalyticsService {
return result[0]?.c || 0
}
- async getSummary(
+ async getCaptchaSummary(
pids: string[],
- period: 'w' | 'M' = 'w',
- amountToSubtract = 1,
- ) {
- return this.getSummaryStats(pids, 'analytics', period, amountToSubtract)
+ period?: string,
+ from?: string,
+ to?: string,
+ timezone?: string,
+ filters?: string,
+ ): Promise<IOverallCaptcha> {
+ // eslint-disable-next-line
+ let _from: string
+ // eslint-disable-next-line
+ let _to: string
+
+ if (_isEmpty(period) || period === 'custom') {
+ const safeTimezone = this.getSafeTimezone(timezone)
+
+ const { groupFrom, groupTo } = this.getGroupFromTo(
+ from,
+ to,
+ null,
+ period,
+ safeTimezone,
+ )
+
+ _from = groupFrom
+ _to = groupTo
+ } else {
+ this.validatePeriod(period)
+ }
+
+ const result = {}
+
+ const [filtersQuery, filtersParams] = this.getFiltersQuery(
+ filters,
+ DataType.CAPTCHA,
+ )
+
+ const promises = pids.map(async pid => {
+ if (!isValidPID(pid)) {
+ throw new BadRequestException(
+ `The provided Project ID (${pid}) is incorrect`,
+ )
+ }
+
+ try {
+ if (period === 'all') {
+ const queryAll = `SELECT count(*) AS all FROM captcha WHERE pid = {pid:FixedString(12)} ${filtersQuery}`
+ const rawResult = <Array<Partial<BirdseyeCHResponse>>>await clickhouse
+ .query(queryAll, {
+ params: { pid, ...filtersParams },
+ })
+ .toPromise()
+
+ result[pid] = {
+ current: {
+ all: rawResult[0].all,
+ },
+ previous: {
+ all: 0,
+ },
+ change: rawResult[0].all,
+ }
+ return
+ }
+
+ let now: string
+ let periodFormatted: string
+ let periodSubtracted: string
+
+ if (_from && _to) {
+ // diff may be 0 (when selecting data for 1 day), so let's make it 1 to grab some data for the prev day as well
+ const diff = dayjs(_to).diff(dayjs(_from), 'days') || 1
+
+ now = _to
+ periodFormatted = _from
+ periodSubtracted = dayjs(_from)
+ .subtract(diff, 'days')
+ .format('YYYY-MM-DD HH:mm:ss')
+ } else {
+ const amountToSubtract = parseInt(period, 10)
+ const unit = _replace(period, /[0-9]/g, '')
+
+ now = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')
+ const periodRaw = dayjs.utc().subtract(amountToSubtract, unit)
+ periodFormatted = periodRaw.format('YYYY-MM-DD HH:mm:ss')
+ periodSubtracted = periodRaw
+ .subtract(amountToSubtract, unit)
+ .format('YYYY-MM-DD HH:mm:ss')
+ }
+
+ const queryCurrent = `SELECT 1 AS sortOrder, count(*) AS all FROM captcha WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodFormatted:String} AND {now:String} ${filtersQuery}`
+ const queryPrevious = `SELECT 2 AS sortOrder, count(*) AS all FROM captcha WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String} ${filtersQuery}`
+
+ const query = `${queryCurrent} UNION ALL ${queryPrevious}`
+
+ let rawResult = <Array<Partial<BirdseyeCHResponse>>>await clickhouse
+ .query(query, {
+ params: {
+ pid,
+ periodFormatted,
+ periodSubtracted,
+ now,
+ ...filtersParams,
+ },
+ })
+ .toPromise()
+
+ rawResult = _sortBy(rawResult, 'sortOrder')
+
+ const currentPeriod = rawResult[0]
+ const previousPeriod = rawResult[1]
+
+ result[pid] = {
+ current: {
+ all: currentPeriod.all,
+ },
+ previous: {
+ all: previousPeriod.all,
+ },
+ change: currentPeriod.all - previousPeriod.all,
+ }
+ } catch {
+ throw new InternalServerErrorException(
+ "Can't process the provided PID. Please, try again later.",
+ )
+ }
+ })
+
+ await Promise.all(promises)
+
+ return result
}
- async getCaptchaSummary(pids: string[], period: 'w' | 'M' = 'w') {
- return this.getSummaryStats(pids, 'captcha', period)
+ convertSummaryToObsoleteFormat(summary: IOverall): any {
+ const result = {}
+
+ for (const pid of _keys(summary)) {
+ const { current, previous } = summary[pid]
+
+ result[pid] = {
+ thisWeek: current.all,
+ lastWeek: previous.all,
+ thisWeekUnique: current.unique,
+ lastWeekUnique: previous.unique,
+ percChange: calculateRelativePercentage(previous.all, current.all),
+ percChangeUnique: calculateRelativePercentage(
+ previous.unique,
+ current.unique,
+ ),
+ }
+ }
+
+ return result
}
- async getSummaryStats(
+ async getAnalyticsSummary(
pids: string[],
- tableName: 'analytics' | 'captcha',
- period: 'w' | 'M' = 'w',
- amountToSubtract = 1,
- ) {
+ period?: string,
+ from?: string,
+ to?: string,
+ timezone?: string,
+ filters?: string,
+ ): Promise<IOverall> {
+ // eslint-disable-next-line
+ let _from: string
+ // eslint-disable-next-line
+ let _to: string
+
+ if (_isEmpty(period) || period === 'custom') {
+ const safeTimezone = this.getSafeTimezone(timezone)
+
+ const { groupFrom, groupTo } = this.getGroupFromTo(
+ from,
+ to,
+ null,
+ period,
+ safeTimezone,
+ )
+
+ _from = groupFrom
+ _to = groupTo
+ } else {
+ this.validatePeriod(period)
+ }
+
const result = {}
+ const [filtersQuery, filtersParams] = this.getFiltersQuery(
+ filters,
+ DataType.ANALYTICS,
+ )
+
const promises = pids.map(async pid => {
if (!isValidPID(pid)) {
throw new BadRequestException(
@@ -1122,75 +1299,327 @@ export class AnalyticsService {
)
}
- const now = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')
- const periodRaw = dayjs.utc().subtract(amountToSubtract, period)
- const periodFormatted = periodRaw.format('YYYY-MM-DD HH:mm:ss')
- const periodSubtracted = periodRaw
- .subtract(amountToSubtract, period)
- .format('YYYY-MM-DD HH:mm:ss')
-
- const queryThisWeek = `SELECT ${
- tableName === 'analytics' ? 'unique,' : ''
- } count() FROM ${tableName} WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodFormatted:String} AND {now:String}${
- tableName === 'analytics' ? ' GROUP BY unique' : ''
- }`
- const queryLastWeek = `SELECT ${
- tableName === 'analytics' ? 'unique,' : ''
- } count() FROM ${tableName} WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String}${
- tableName === 'analytics' ? ' GROUP BY unique' : ''
- }`
-
- const paramsData = {
- params: {
- pid,
- periodFormatted,
- periodSubtracted,
- now,
- },
- }
-
try {
- const thisWeekResult = await clickhouse
- .query(queryThisWeek, paramsData)
- .toPromise()
- const lastWeekResult = await clickhouse
- .query(queryLastWeek, paramsData)
+ if (period === 'all') {
+ const queryAll = `SELECT count(*) AS all, countIf(unique=1) AS unique, avgIf(sdur, sdur IS NOT NULL AND analytics.unique=1) AS sdur FROM analytics WHERE pid = {pid:FixedString(12)} ${filtersQuery}`
+ const rawResult = <Array<BirdseyeCHResponse>>await clickhouse
+ .query(queryAll, {
+ params: { pid, ...filtersParams },
+ })
+ .toPromise()
+
+ let bounceRate = 0
+
+ if (rawResult[0].all > 0) {
+ bounceRate = _round(
+ (rawResult[0].unique * 100) / rawResult[0].all,
+ 1,
+ )
+ }
+
+ result[pid] = {
+ current: {
+ all: rawResult[0].all,
+ unique: rawResult[0].unique,
+ bounceRate,
+ sdur: rawResult[0].sdur,
+ },
+ previous: {
+ all: 0,
+ unique: 0,
+ bounceRate: 0,
+ sdur: 0,
+ },
+ change: rawResult[0].all,
+ uniqueChange: rawResult[0].unique,
+ bounceRateChange: bounceRate,
+ sdurChange: rawResult[0].sdur,
+ }
+ return
+ }
+
+ let now: string
+ let periodFormatted: string
+ let periodSubtracted: string
+
+ if (_from && _to) {
+ // diff may be 0 (when selecting data for 1 day), so let's make it 1 to grab some data for the prev day as well
+ const diff = dayjs(_to).diff(dayjs(_from), 'days') || 1
+
+ now = _to
+ periodFormatted = _from
+ periodSubtracted = dayjs(_from)
+ .subtract(diff, 'days')
+ .format('YYYY-MM-DD HH:mm:ss')
+ } else {
+ const amountToSubtract = parseInt(period, 10)
+ const unit = _replace(period, /[0-9]/g, '')
+
+ now = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')
+ const periodRaw = dayjs.utc().subtract(amountToSubtract, unit)
+ periodFormatted = periodRaw.format('YYYY-MM-DD HH:mm:ss')
+ periodSubtracted = periodRaw
+ .subtract(amountToSubtract, unit)
+ .format('YYYY-MM-DD HH:mm:ss')
+ }
+
+ const queryCurrent = `SELECT 1 AS sortOrder, count(*) AS all, countIf(unique=1) AS unique, avgIf(sdur, sdur IS NOT NULL AND analytics.unique=1) AS sdur FROM analytics WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodFormatted:String} AND {now:String} ${filtersQuery}`
+ const queryPrevious = `SELECT 2 AS sortOrder, count(*) AS all, countIf(unique=1) AS unique, avgIf(sdur, sdur IS NOT NULL AND analytics.unique=1) AS sdur FROM analytics WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String} ${filtersQuery}`
+
+ const query = `${queryCurrent} UNION ALL ${queryPrevious}`
+
+ let rawResult = <Array<BirdseyeCHResponse>>await clickhouse
+ .query(query, {
+ params: {
+ pid,
+ periodFormatted,
+ periodSubtracted,
+ now,
+ ...filtersParams,
+ },
+ })
.toPromise()
- if (tableName === 'analytics') {
- const thisWeekUnique =
- _find(thisWeekResult, ({ unique }) => unique)?.['count()'] || 0
- const thisWeekPV =
- (_find(thisWeekResult, ({ unique }) => !unique)?.['count()'] || 0) +
- thisWeekUnique
- const lastWeekUnique =
- _find(lastWeekResult, ({ unique }) => unique)?.['count()'] || 0
- const lastWeekPV =
- (_find(lastWeekResult, ({ unique }) => !unique)?.['count()'] || 0) +
- lastWeekUnique
+ rawResult = _sortBy(rawResult, 'sortOrder')
+
+ const currentPeriod = rawResult[0]
+ const previousPeriod = rawResult[1]
+
+ let bounceRate = 0
+ let prevBounceRate = 0
+
+ if (currentPeriod.all > 0) {
+ bounceRate = _round(
+ (currentPeriod.unique * 100) / currentPeriod.all,
+ 1,
+ )
+ }
+
+ if (previousPeriod.all > 0) {
+ prevBounceRate = _round(
+ (previousPeriod.unique * 100) / previousPeriod.all,
+ 1,
+ )
+ }
+
+ result[pid] = {
+ current: {
+ all: currentPeriod.all,
+ unique: currentPeriod.unique,
+ sdur: currentPeriod.sdur || 0,
+ bounceRate,
+ },
+ previous: {
+ all: previousPeriod.all,
+ unique: previousPeriod.unique,
+ sdur: previousPeriod.sdur || 0,
+ bounceRate: prevBounceRate,
+ },
+ change: currentPeriod.all - previousPeriod.all,
+ uniqueChange: currentPeriod.unique - previousPeriod.unique,
+ bounceRateChange: (bounceRate - prevBounceRate) * -1,
+ sdurChange: currentPeriod.sdur - previousPeriod.sdur,
+ }
+ } catch (reason) {
+ console.error(
+ `[ERROR] (getAnalyticsSummary) Error occurred for PID ${pid}`,
+ )
+ console.error(reason)
+ throw new InternalServerErrorException(
+ "Can't process the provided PID. Please, try again later.",
+ )
+ }
+ })
+
+ await Promise.all(promises)
+
+ return result
+ }
+
+ async getPerformanceSummary(
+ pids: string[],
+ period?: string,
+ from?: string,
+ to?: string,
+ timezone?: string,
+ filters?: string,
+ ): Promise<IOverallPerformance> {
+ // eslint-disable-next-line
+ let _from: string
+ // eslint-disable-next-line
+ let _to: string
+
+ if (_isEmpty(period) || period === 'custom') {
+ const safeTimezone = this.getSafeTimezone(timezone)
+
+ const { groupFrom, groupTo } = this.getGroupFromTo(
+ from,
+ to,
+ null,
+ period,
+ safeTimezone,
+ )
+
+ _from = groupFrom
+ _to = groupTo
+ } else {
+ this.validatePeriod(period)
+ }
+
+ const result = {}
+
+ const [filtersQuery, filtersParams] = this.getFiltersQuery(
+ filters,
+ DataType.PERFORMANCE,
+ )
+
+ const promises = pids.map(async pid => {
+ if (!isValidPID(pid)) {
+ throw new BadRequestException(
+ `The provided Project ID (${pid}) is incorrect`,
+ )
+ }
+
+ try {
+ if (period === 'all') {
+ const queryAll = `SELECT avg(dns) AS dns, avg(tls) as tls, avg(conn) as conn, avg(response) as response, avg(render) as render, avg(domLoad) as domLoad, avg(ttfb) as ttfb FROM performance WHERE pid = {pid:FixedString(12)} ${filtersQuery}`
+ const rawResult = <Array<Partial<PerformanceCHResponse>>>(
+ await clickhouse
+ .query(queryAll, {
+ params: { pid, ...filtersParams },
+ })
+ .toPromise()
+ )
result[pid] = {
- thisWeek: thisWeekPV,
- lastWeek: lastWeekPV,
- thisWeekUnique,
- lastWeekUnique,
- percChange: calculateRelativePercentage(lastWeekPV, thisWeekPV),
- percChangeUnique: calculateRelativePercentage(
- lastWeekUnique,
- thisWeekUnique,
+ current: {
+ frontend: millisecondsToSeconds(
+ rawResult[0].render + rawResult[0].domLoad,
+ ),
+ network: millisecondsToSeconds(
+ rawResult[0].dns +
+ rawResult[0].tls +
+ rawResult[0].conn +
+ rawResult[0].response,
+ ),
+ backend: millisecondsToSeconds(rawResult[0].ttfb),
+ },
+ previous: {
+ frontend: 0,
+ network: 0,
+ backend: 0,
+ },
+ frontendChange: millisecondsToSeconds(
+ rawResult[0].render + rawResult[0].domLoad,
),
+ networkChange: millisecondsToSeconds(
+ rawResult[0].dns +
+ rawResult[0].tls +
+ rawResult[0].conn +
+ rawResult[0].response,
+ ),
+ backendChange: millisecondsToSeconds(rawResult[0].ttfb),
}
+ return
+ }
+
+ let now: string
+ let periodFormatted: string
+ let periodSubtracted: string
+
+ if (_from && _to) {
+ // diff may be 0 (when selecting data for 1 day), so let's make it 1 to grab some data for the prev day as well
+ const diff = dayjs(_to).diff(dayjs(_from), 'days') || 1
+
+ now = _to
+ periodFormatted = _from
+ periodSubtracted = dayjs(_from)
+ .subtract(diff, 'days')
+ .format('YYYY-MM-DD HH:mm:ss')
} else {
- const thisWeek = thisWeekResult?.['count()'] || 0
- const lastWeek = lastWeekResult?.['count()'] || 0
+ const amountToSubtract = parseInt(period, 10)
+ const unit = _replace(period, /[0-9]/g, '')
+
+ now = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')
+ const periodRaw = dayjs.utc().subtract(amountToSubtract, unit)
+ periodFormatted = periodRaw.format('YYYY-MM-DD HH:mm:ss')
+ periodSubtracted = periodRaw
+ .subtract(amountToSubtract, unit)
+ .format('YYYY-MM-DD HH:mm:ss')
+ }
- result[pid] = {
- thisWeek,
- lastWeek,
- percChange: calculateRelativePercentage(lastWeek, thisWeek),
- }
+ const queryCurrent = `SELECT 1 AS sortOrder, avg(dns) AS dns, avg(tls) as tls, avg(conn) as conn, avg(response) as response, avg(render) as render, avg(domLoad) as domLoad, avg(ttfb) as ttfb FROM performance WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodFormatted:String} AND {now:String} ${filtersQuery}`
+ const queryPrevious = `SELECT 2 AS sortOrder, avg(dns) AS dns, avg(tls) as tls, avg(conn) as conn, avg(response) as response, avg(render) as render, avg(domLoad) as domLoad, avg(ttfb) as ttfb FROM performance WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String} ${filtersQuery}`
+
+ const query = `${queryCurrent} UNION ALL ${queryPrevious}`
+
+ let rawResult = <Array<Partial<PerformanceCHResponse>>>await clickhouse
+ .query(query, {
+ params: {
+ pid,
+ periodFormatted,
+ periodSubtracted,
+ now,
+ ...filtersParams,
+ },
+ })
+ .toPromise()
+
+ rawResult = _sortBy(rawResult, 'sortOrder')
+
+ const currentPeriod = rawResult[0]
+ const previousPeriod = rawResult[1]
+
+ result[pid] = {
+ current: {
+ frontend: millisecondsToSeconds(
+ currentPeriod.render + currentPeriod.domLoad,
+ ),
+ network: millisecondsToSeconds(
+ currentPeriod.dns +
+ currentPeriod.tls +
+ currentPeriod.conn +
+ currentPeriod.response,
+ ),
+ backend: millisecondsToSeconds(currentPeriod.ttfb),
+ },
+ previous: {
+ frontend: millisecondsToSeconds(
+ previousPeriod.render + previousPeriod.domLoad,
+ ),
+ network: millisecondsToSeconds(
+ previousPeriod.dns +
+ previousPeriod.tls +
+ previousPeriod.conn +
+ previousPeriod.response,
+ ),
+ backend: millisecondsToSeconds(previousPeriod.ttfb),
+ },
+ frontendChange: millisecondsToSeconds(
+ currentPeriod.render +
+ currentPeriod.domLoad -
+ previousPeriod.render -
+ previousPeriod.domLoad,
+ ),
+ networkChange: millisecondsToSeconds(
+ currentPeriod.dns +
+ currentPeriod.tls +
+ currentPeriod.conn +
+ currentPeriod.response -
+ previousPeriod.dns -
+ previousPeriod.tls -
+ previousPeriod.conn -
+ previousPeriod.response,
+ ),
+ backendChange: millisecondsToSeconds(
+ currentPeriod.ttfb - previousPeriod.ttfb,
+ ),
}
- } catch {
+ } catch (reason) {
+ console.error(
+ `[ERROR] (getAnalyticsSummary) Error occurred for PID ${pid}`,
+ )
+ console.error(reason)
throw new InternalServerErrorException(
"Can't process the provided PID. Please, try again later.",
)
@@ -1272,18 +1701,6 @@ export class AnalyticsService {
return params
}
- async calculateAverageSessionDuration(
- subQuery: string,
- paramsData: any,
- ): Promise<number> {
- const avgSdurQuery = `SELECT avg(sdur) ${subQuery} AND sdur IS NOT NULL AND unique='1'`
- const avgSdurObject = await clickhouse
- .query(avgSdurQuery, paramsData)
- .toPromise()
-
- return _round(avgSdurObject[0]['avg(sdur)'])
- }
-
generateXAxis(
timeBucket: TimeBucketType,
from: string, // timezone is already applied to the from and to parameters
@@ -1498,7 +1915,7 @@ export class AnalyticsService {
${selector},
avg(sdur) as sdur,
count() as pageviews,
- sumIf(1, unique = 1) as uniques
+ countIf(unique=1) as uniques
FROM (
SELECT *,
${timeBucketFunc}(toTimeZone(created, '${safeTimezone}')) as tz_created
@@ -1653,7 +2070,6 @@ export class AnalyticsService {
): Promise<object | void> {
let params: unknown = {}
let chart: unknown = {}
- let avgSdur = 0
const promises = [
// Getting params
@@ -1678,7 +2094,6 @@ export class AnalyticsService {
timeBucket,
from,
to,
- subQuery,
filtersQuery,
paramsData,
safeTimezone,
@@ -1688,9 +2103,6 @@ export class AnalyticsService {
// @ts-ignore
chart = groupedData.chart
-
- // @ts-ignore
- avgSdur = groupedData.avgSdur
})(),
]
@@ -1699,7 +2111,6 @@ export class AnalyticsService {
return {
params,
chart,
- avgSdur,
}
}
@@ -1723,17 +2134,12 @@ export class AnalyticsService {
timeBucket: TimeBucketType,
from: string,
to: string,
- subQuery: string,
filtersQuery: string,
paramsData: any,
safeTimezone: string,
customEVFilterApplied: boolean,
mode: ChartRenderMode,
): Promise<object | void> {
- const avgSdur = customEVFilterApplied
- ? 0
- : await this.calculateAverageSessionDuration(subQuery, paramsData)
-
const { xShifted } = this.generateXAxis(timeBucket, from, to, safeTimezone)
if (customEVFilterApplied) {
@@ -1761,7 +2167,6 @@ export class AnalyticsService {
uniques,
sdur,
},
- avgSdur,
})
}
@@ -1785,7 +2190,6 @@ export class AnalyticsService {
uniques,
sdur,
},
- avgSdur,
})
}
@@ -2143,8 +2547,4 @@ export class AnalyticsService {
// @ts-ignore
return redis.countKeysByPattern(`hb:${projectId}:*`)
}
-
- async getStatsByProjectId(projectId: string) {
- return this.getSummaryStats([projectId], 'analytics', 'w', 1)
- }
}
diff --git a/apps/production/src/analytics/interfaces/index.ts b/apps/production/src/analytics/interfaces/index.ts
index fd7732776..1db51a665 100644
--- a/apps/production/src/analytics/interfaces/index.ts
+++ b/apps/production/src/analytics/interfaces/index.ts
@@ -16,6 +16,12 @@ export interface TrafficCHResponse {
pageviews: number
}
+export interface BirdseyeCHResponse {
+ all: number
+ unique: number
+ sdur: number
+}
+
export interface PerformanceCHResponse {
year: number
month: number
@@ -118,3 +124,57 @@ export interface IGetFunnel {
funnel: IFunnel[]
totalPageviews: number
}
+
+interface IOverallPeriodStats {
+ all: number
+ unique: number
+ bounceRate: number
+ sdur: number
+}
+
+export interface ICaptchaObject {
+ current: {
+ all: number
+ }
+ previous: {
+ all: number
+ }
+ change: number
+}
+
+export interface IPerformanceObject {
+ current: {
+ frontend: number
+ backend: number
+ network: number
+ }
+ previous: {
+ frontend: number
+ backend: number
+ network: number
+ }
+ frontendChange: number
+ backendChange: number
+ networkChange: number
+}
+
+export interface IOverallObject {
+ current: IOverallPeriodStats
+ previous: IOverallPeriodStats
+ change: number
+ uniqueChange?: number
+ bounceRateChange?: number
+ sdurChange?: number
+}
+
+export interface IOverall {
+ [key: string]: IOverallObject
+}
+
+export interface IOverallCaptcha {
+ [key: string]: ICaptchaObject
+}
+
+export interface IOverallPerformance {
+ [key: string]: IPerformanceObject
+}
diff --git a/apps/production/src/integrations/telegram/scene/projects.scene.ts b/apps/production/src/integrations/telegram/scene/projects.scene.ts
index a76c8342d..f256dcc45 100644
--- a/apps/production/src/integrations/telegram/scene/projects.scene.ts
+++ b/apps/production/src/integrations/telegram/scene/projects.scene.ts
@@ -3,9 +3,9 @@ import { Markup } from 'telegraf'
import * as dayjs from 'dayjs'
import * as utc from 'dayjs/plugin/utc'
import * as timezone from 'dayjs/plugin/timezone'
-import { ProjectService } from 'src/project/project.service'
-import { UserService } from 'src/user/user.service'
-import { AnalyticsService } from 'src/analytics/analytics.service'
+import { ProjectService } from '../../../project/project.service'
+import { UserService } from '../../../user/user.service'
+import { AnalyticsService } from '../../../analytics/analytics.service'
import { Context } from '../interface/context.interface'
import { START_SCENE_ID } from './start.scene'
@@ -78,7 +78,10 @@ export class ProjectsScene {
const onlineCount = await this.analyticsService.getOnlineCountByProjectId(
project.id,
)
- const stats = await this.analyticsService.getStatsByProjectId(project.id)
+ const stats = await this.analyticsService.getAnalyticsSummary(
+ [project.id],
+ '7d',
+ )
const text =
`📊 *${project.name}*` +
@@ -100,9 +103,9 @@ export class ProjectsScene {
`\n` +
`Online users: \`${onlineCount}\`` +
`\n` +
- `Page views: \`${stats[project.id].thisWeek}\`` +
+ `Page views: \`${stats[project.id].current.all}\`` +
`\n` +
- `Unique page views: \`${stats[project.id].thisWeekUnique}\``
+ `Unique page views: \`${stats[project.id].current.unique}\``
await context.replyWithMarkdown(text)
}
diff --git a/apps/production/src/project/project.service.ts b/apps/production/src/project/project.service.ts
index 20a5eb0c5..a32cd51b0 100644
--- a/apps/production/src/project/project.service.ts
+++ b/apps/production/src/project/project.service.ts
@@ -468,6 +468,14 @@ export class ProjectService {
uid: string | null,
password?: string | null,
): void {
+ if (
+ project.public ||
+ uid === project.admin?.id ||
+ _findIndex(project.share, ({ user }) => user?.id === uid) !== -1
+ ) {
+ return null
+ }
+
if (project.isPasswordProtected && password) {
if (
_size(password) <= MAX_PROJECT_PASSWORD_LENGTH &&
@@ -483,14 +491,6 @@ export class ProjectService {
throw new ForbiddenException('This project is password protected')
}
- if (
- project.public ||
- uid === project.admin?.id ||
- _findIndex(project.share, ({ user }) => user?.id === uid) !== -1
- ) {
- return null
- }
-
throw new ForbiddenException('You are not allowed to view this project')
}
diff --git a/apps/production/src/task-manager/task-manager.service.ts b/apps/production/src/task-manager/task-manager.service.ts
index 58a568228..1453542a5 100644
--- a/apps/production/src/task-manager/task-manager.service.ts
+++ b/apps/production/src/task-manager/task-manager.service.ts
@@ -278,7 +278,9 @@ export class TaskManagerService {
}
const ids = _map(projects, p => p.id)
- const data = await this.analyticsService.getSummary(ids, 'w')
+ const data = this.analyticsService.convertSummaryToObsoleteFormat(
+ await this.analyticsService.getAnalyticsSummary(ids, '7d'),
+ )
const result = {
type: 'w', // week
@@ -328,7 +330,9 @@ export class TaskManagerService {
}
const ids = _map(projects, p => p.id)
- const data = await this.analyticsService.getSummary(ids, 'M')
+ const data = this.analyticsService.convertSummaryToObsoleteFormat(
+ await this.analyticsService.getAnalyticsSummary(ids, '4w'),
+ )
const result = {
type: 'M', // month
@@ -377,7 +381,9 @@ export class TaskManagerService {
}
const ids = _map(projects, p => p.id)
- const data = await this.analyticsService.getSummary(ids, 'M', 3)
+ const data = this.analyticsService.convertSummaryToObsoleteFormat(
+ await this.analyticsService.getAnalyticsSummary(ids, '3M'),
+ )
const result = {
type: 'Q', // quarterly
@@ -421,7 +427,9 @@ export class TaskManagerService {
const projects = await this.projectService.getSubscriberProjects(id)
const ids = projects.map(project => project.id)
- const data = await this.analyticsService.getSummary(ids, 'M', 3)
+ const data = this.analyticsService.convertSummaryToObsoleteFormat(
+ await this.analyticsService.getAnalyticsSummary(ids, '3M'),
+ )
const result = {
type: 'Q', // quarter
@@ -464,7 +472,9 @@ export class TaskManagerService {
const projects = await this.projectService.getSubscriberProjects(id)
const ids = projects.map(project => project.id)
- const data = await this.analyticsService.getSummary(ids, 'M')
+ const data = this.analyticsService.convertSummaryToObsoleteFormat(
+ await this.analyticsService.getAnalyticsSummary(ids, '4w'),
+ )
const result = {
type: 'M', // month
@@ -507,7 +517,9 @@ export class TaskManagerService {
const projects = await this.projectService.getSubscriberProjects(id)
const ids = projects.map(project => project.id)
- const data = await this.analyticsService.getSummary(ids, 'w')
+ const data = this.analyticsService.convertSummaryToObsoleteFormat(
+ await this.analyticsService.getAnalyticsSummary(ids, '7d'),
+ )
const result = {
type: 'w', // week
diff --git a/apps/selfhosted/src/analytics/analytics.controller.ts b/apps/selfhosted/src/analytics/analytics.controller.ts
index b9ca6c2e3..2f00f04e5 100644
--- a/apps/selfhosted/src/analytics/analytics.controller.ts
+++ b/apps/selfhosted/src/analytics/analytics.controller.ts
@@ -792,7 +792,49 @@ export class AnalyticsController {
@Query() data,
@CurrentUserId() uid: string,
): Promise<any> {
- const { pids, pid } = data
+ const {
+ pids,
+ pid,
+ period,
+ from,
+ to,
+ timezone = DEFAULT_TIMEZONE,
+ filters,
+ } = data
+ const pidsArray = getPIDsArray(pids, pid)
+
+ const validationPromises = _map(pidsArray, async currentPID => {
+ this.analyticsService.validatePID(currentPID)
+ await this.analyticsService.checkProjectAccess(currentPID, uid)
+ })
+
+ await Promise.all(validationPromises)
+
+ return this.analyticsService.getAnalyticsSummary(
+ pidsArray,
+ period,
+ from,
+ to,
+ timezone,
+ filters,
+ )
+ }
+
+ @Get('performance/birdseye')
+ @Auth([], true, true)
+ async getPerformanceOverallStats(
+ @Query() data,
+ @CurrentUserId() uid: string,
+ ): Promise<any> {
+ const {
+ pids,
+ pid,
+ period,
+ from,
+ to,
+ timezone = DEFAULT_TIMEZONE,
+ filters,
+ } = data
const pidsArray = getPIDsArray(pids, pid)
const validationPromises = _map(pidsArray, async currentPID => {
@@ -802,7 +844,14 @@ export class AnalyticsController {
await Promise.all(validationPromises)
- return this.analyticsService.getSummary(pidsArray, 'w')
+ return this.analyticsService.getPerformanceSummary(
+ pidsArray,
+ period,
+ from,
+ to,
+ timezone,
+ filters,
+ )
}
@Get('hb')
diff --git a/apps/selfhosted/src/analytics/analytics.service.ts b/apps/selfhosted/src/analytics/analytics.service.ts
index dbbf8b333..49a9dde52 100644
--- a/apps/selfhosted/src/analytics/analytics.service.ts
+++ b/apps/selfhosted/src/analytics/analytics.service.ts
@@ -6,6 +6,7 @@ import * as _size from 'lodash/size'
import * as _includes from 'lodash/includes'
import * as _map from 'lodash/map'
import * as _toUpper from 'lodash/toUpper'
+import * as _sortBy from 'lodash/sortBy'
import * as _join from 'lodash/join'
import * as _replace from 'lodash/replace'
import * as _last from 'lodash/last'
@@ -44,10 +45,7 @@ import {
TRAFFIC_COLUMNS,
PERFORMANCE_COLUMNS,
} from '../common/constants'
-import {
- calculateRelativePercentage,
- millisecondsToSeconds,
-} from '../common/utils'
+import { millisecondsToSeconds } from '../common/utils'
import { PageviewsDTO } from './dto/pageviews.dto'
import { EventsDTO } from './dto/events.dto'
import { ProjectService } from '../project/project.service'
@@ -59,6 +57,7 @@ import {
CustomsCHResponse,
CustomsCHAggregatedResponse,
TrafficCEFilterCHResponse,
+ BirdseyeCHResponse,
TrafficCHResponse,
IGetGroupFromTo,
GetFiltersQuery,
@@ -71,6 +70,8 @@ import {
IAggregatedMetadata,
IFunnelCHResponse,
IFunnel,
+ IOverall,
+ IOverallPerformance,
} from './interfaces'
dayjs.extend(utc)
@@ -999,22 +1000,43 @@ export class AnalyticsService {
return result[0]?.c || 0
}
- async getSummary(
+ async getAnalyticsSummary(
pids: string[],
- period: 'w' | 'M' = 'w',
- amountToSubtract = 1,
- ) {
- return this.getSummaryStats(pids, 'analytics', period, amountToSubtract)
- }
+ period?: string,
+ from?: string,
+ to?: string,
+ timezone?: string,
+ filters?: string,
+ ): Promise<IOverall> {
+ // eslint-disable-next-line
+ let _from: string
+ // eslint-disable-next-line
+ let _to: string
+
+ if (_isEmpty(period) || period === 'custom') {
+ const safeTimezone = this.getSafeTimezone(timezone)
+
+ const { groupFrom, groupTo } = this.getGroupFromTo(
+ from,
+ to,
+ null,
+ period,
+ safeTimezone,
+ )
+
+ _from = groupFrom
+ _to = groupTo
+ } else {
+ this.validatePeriod(period)
+ }
- async getSummaryStats(
- pids: string[],
- tableName: 'analytics',
- period: 'w' | 'M' = 'w',
- amountToSubtract = 1,
- ) {
const result = {}
+ const [filtersQuery, filtersParams] = this.getFiltersQuery(
+ filters,
+ DataType.ANALYTICS,
+ )
+
const promises = pids.map(async pid => {
if (!isValidPID(pid)) {
throw new BadRequestException(
@@ -1022,75 +1044,327 @@ export class AnalyticsService {
)
}
- const now = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')
- const periodRaw = dayjs.utc().subtract(amountToSubtract, period)
- const periodFormatted = periodRaw.format('YYYY-MM-DD HH:mm:ss')
- const periodSubtracted = periodRaw
- .subtract(amountToSubtract, period)
- .format('YYYY-MM-DD HH:mm:ss')
-
- const queryThisWeek = `SELECT ${
- tableName === 'analytics' ? 'unique,' : ''
- } count() FROM ${tableName} WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodFormatted:String} AND {now:String}${
- tableName === 'analytics' ? ' GROUP BY unique' : ''
- }`
- const queryLastWeek = `SELECT ${
- tableName === 'analytics' ? 'unique,' : ''
- } count() FROM ${tableName} WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String}${
- tableName === 'analytics' ? ' GROUP BY unique' : ''
- }`
-
- const paramsData = {
- params: {
- pid,
- periodFormatted,
- periodSubtracted,
- now,
- },
- }
-
try {
- const thisWeekResult = await clickhouse
- .query(queryThisWeek, paramsData)
- .toPromise()
- const lastWeekResult = await clickhouse
- .query(queryLastWeek, paramsData)
+ if (period === 'all') {
+ const queryAll = `SELECT count(*) AS all, countIf(unique=1) AS unique, avgIf(sdur, sdur IS NOT NULL AND analytics.unique=1) AS sdur FROM analytics WHERE pid = {pid:FixedString(12)} ${filtersQuery}`
+ const rawResult = <Array<BirdseyeCHResponse>>await clickhouse
+ .query(queryAll, {
+ params: { pid, ...filtersParams },
+ })
+ .toPromise()
+
+ let bounceRate = 0
+
+ if (rawResult[0].all > 0) {
+ bounceRate = _round(
+ (rawResult[0].unique * 100) / rawResult[0].all,
+ 1,
+ )
+ }
+
+ result[pid] = {
+ current: {
+ all: rawResult[0].all,
+ unique: rawResult[0].unique,
+ bounceRate,
+ sdur: rawResult[0].sdur,
+ },
+ previous: {
+ all: 0,
+ unique: 0,
+ bounceRate: 0,
+ sdur: 0,
+ },
+ change: rawResult[0].all,
+ uniqueChange: rawResult[0].unique,
+ bounceRateChange: bounceRate,
+ sdurChange: rawResult[0].sdur,
+ }
+ return
+ }
+
+ let now: string
+ let periodFormatted: string
+ let periodSubtracted: string
+
+ if (_from && _to) {
+ // diff may be 0 (when selecting data for 1 day), so let's make it 1 to grab some data for the prev day as well
+ const diff = dayjs(_to).diff(dayjs(_from), 'days') || 1
+
+ now = _to
+ periodFormatted = _from
+ periodSubtracted = dayjs(_from)
+ .subtract(diff, 'days')
+ .format('YYYY-MM-DD HH:mm:ss')
+ } else {
+ const amountToSubtract = parseInt(period, 10)
+ const unit = _replace(period, /[0-9]/g, '')
+
+ now = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')
+ const periodRaw = dayjs.utc().subtract(amountToSubtract, unit)
+ periodFormatted = periodRaw.format('YYYY-MM-DD HH:mm:ss')
+ periodSubtracted = periodRaw
+ .subtract(amountToSubtract, unit)
+ .format('YYYY-MM-DD HH:mm:ss')
+ }
+
+ const queryCurrent = `SELECT 1 AS sortOrder, count(*) AS all, countIf(unique=1) AS unique, avgIf(sdur, sdur IS NOT NULL AND analytics.unique=1) AS sdur FROM analytics WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodFormatted:String} AND {now:String} ${filtersQuery}`
+ const queryPrevious = `SELECT 2 AS sortOrder, count(*) AS all, countIf(unique=1) AS unique, avgIf(sdur, sdur IS NOT NULL AND analytics.unique=1) AS sdur FROM analytics WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String} ${filtersQuery}`
+
+ const query = `${queryCurrent} UNION ALL ${queryPrevious}`
+
+ let rawResult = <Array<BirdseyeCHResponse>>await clickhouse
+ .query(query, {
+ params: {
+ pid,
+ periodFormatted,
+ periodSubtracted,
+ now,
+ ...filtersParams,
+ },
+ })
.toPromise()
- if (tableName === 'analytics') {
- const thisWeekUnique =
- _find(thisWeekResult, ({ unique }) => unique)?.['count()'] || 0
- const thisWeekPV =
- (_find(thisWeekResult, ({ unique }) => !unique)?.['count()'] || 0) +
- thisWeekUnique
- const lastWeekUnique =
- _find(lastWeekResult, ({ unique }) => unique)?.['count()'] || 0
- const lastWeekPV =
- (_find(lastWeekResult, ({ unique }) => !unique)?.['count()'] || 0) +
- lastWeekUnique
+ rawResult = _sortBy(rawResult, 'sortOrder')
+
+ const currentPeriod = rawResult[0]
+ const previousPeriod = rawResult[1]
+
+ let bounceRate = 0
+ let prevBounceRate = 0
+
+ if (currentPeriod.all > 0) {
+ bounceRate = _round(
+ (currentPeriod.unique * 100) / currentPeriod.all,
+ 1,
+ )
+ }
+
+ if (previousPeriod.all > 0) {
+ prevBounceRate = _round(
+ (previousPeriod.unique * 100) / previousPeriod.all,
+ 1,
+ )
+ }
+
+ result[pid] = {
+ current: {
+ all: currentPeriod.all,
+ unique: currentPeriod.unique,
+ sdur: currentPeriod.sdur || 0,
+ bounceRate,
+ },
+ previous: {
+ all: previousPeriod.all,
+ unique: previousPeriod.unique,
+ sdur: previousPeriod.sdur || 0,
+ bounceRate: prevBounceRate,
+ },
+ change: currentPeriod.all - previousPeriod.all,
+ uniqueChange: currentPeriod.unique - previousPeriod.unique,
+ bounceRateChange: (bounceRate - prevBounceRate) * -1,
+ sdurChange: currentPeriod.sdur - previousPeriod.sdur,
+ }
+ } catch (reason) {
+ console.error(
+ `[ERROR] (getAnalyticsSummary) Error occurred for PID ${pid}`,
+ )
+ console.error(reason)
+ throw new InternalServerErrorException(
+ "Can't process the provided PID. Please, try again later.",
+ )
+ }
+ })
+
+ await Promise.all(promises)
+
+ return result
+ }
+
+ async getPerformanceSummary(
+ pids: string[],
+ period?: string,
+ from?: string,
+ to?: string,
+ timezone?: string,
+ filters?: string,
+ ): Promise<IOverallPerformance> {
+ // eslint-disable-next-line
+ let _from: string
+ // eslint-disable-next-line
+ let _to: string
+
+ if (_isEmpty(period) || period === 'custom') {
+ const safeTimezone = this.getSafeTimezone(timezone)
+
+ const { groupFrom, groupTo } = this.getGroupFromTo(
+ from,
+ to,
+ null,
+ period,
+ safeTimezone,
+ )
+
+ _from = groupFrom
+ _to = groupTo
+ } else {
+ this.validatePeriod(period)
+ }
+
+ const result = {}
+
+ const [filtersQuery, filtersParams] = this.getFiltersQuery(
+ filters,
+ DataType.PERFORMANCE,
+ )
+
+ const promises = pids.map(async pid => {
+ if (!isValidPID(pid)) {
+ throw new BadRequestException(
+ `The provided Project ID (${pid}) is incorrect`,
+ )
+ }
+
+ try {
+ if (period === 'all') {
+ const queryAll = `SELECT avg(dns) AS dns, avg(tls) as tls, avg(conn) as conn, avg(response) as response, avg(render) as render, avg(domLoad) as domLoad, avg(ttfb) as ttfb FROM performance WHERE pid = {pid:FixedString(12)} ${filtersQuery}`
+ const rawResult = <Array<Partial<PerformanceCHResponse>>>(
+ await clickhouse
+ .query(queryAll, {
+ params: { pid, ...filtersParams },
+ })
+ .toPromise()
+ )
result[pid] = {
- thisWeek: thisWeekPV,
- lastWeek: lastWeekPV,
- thisWeekUnique,
- lastWeekUnique,
- percChange: calculateRelativePercentage(lastWeekPV, thisWeekPV),
- percChangeUnique: calculateRelativePercentage(
- lastWeekUnique,
- thisWeekUnique,
+ current: {
+ frontend: millisecondsToSeconds(
+ rawResult[0].render + rawResult[0].domLoad,
+ ),
+ network: millisecondsToSeconds(
+ rawResult[0].dns +
+ rawResult[0].tls +
+ rawResult[0].conn +
+ rawResult[0].response,
+ ),
+ backend: millisecondsToSeconds(rawResult[0].ttfb),
+ },
+ previous: {
+ frontend: 0,
+ network: 0,
+ backend: 0,
+ },
+ frontendChange: millisecondsToSeconds(
+ rawResult[0].render + rawResult[0].domLoad,
),
+ networkChange: millisecondsToSeconds(
+ rawResult[0].dns +
+ rawResult[0].tls +
+ rawResult[0].conn +
+ rawResult[0].response,
+ ),
+ backendChange: millisecondsToSeconds(rawResult[0].ttfb),
}
+ return
+ }
+
+ let now: string
+ let periodFormatted: string
+ let periodSubtracted: string
+
+ if (_from && _to) {
+ // diff may be 0 (when selecting data for 1 day), so let's make it 1 to grab some data for the prev day as well
+ const diff = dayjs(_to).diff(dayjs(_from), 'days') || 1
+
+ now = _to
+ periodFormatted = _from
+ periodSubtracted = dayjs(_from)
+ .subtract(diff, 'days')
+ .format('YYYY-MM-DD HH:mm:ss')
} else {
- const thisWeek = thisWeekResult?.['count()'] || 0
- const lastWeek = lastWeekResult?.['count()'] || 0
+ const amountToSubtract = parseInt(period, 10)
+ const unit = _replace(period, /[0-9]/g, '')
+
+ now = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')
+ const periodRaw = dayjs.utc().subtract(amountToSubtract, unit)
+ periodFormatted = periodRaw.format('YYYY-MM-DD HH:mm:ss')
+ periodSubtracted = periodRaw
+ .subtract(amountToSubtract, unit)
+ .format('YYYY-MM-DD HH:mm:ss')
+ }
- result[pid] = {
- thisWeek,
- lastWeek,
- percChange: calculateRelativePercentage(lastWeek, thisWeek),
- }
+ const queryCurrent = `SELECT 1 AS sortOrder, avg(dns) AS dns, avg(tls) as tls, avg(conn) as conn, avg(response) as response, avg(render) as render, avg(domLoad) as domLoad, avg(ttfb) as ttfb FROM performance WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodFormatted:String} AND {now:String} ${filtersQuery}`
+ const queryPrevious = `SELECT 2 AS sortOrder, avg(dns) AS dns, avg(tls) as tls, avg(conn) as conn, avg(response) as response, avg(render) as render, avg(domLoad) as domLoad, avg(ttfb) as ttfb FROM performance WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String} ${filtersQuery}`
+
+ const query = `${queryCurrent} UNION ALL ${queryPrevious}`
+
+ let rawResult = <Array<Partial<PerformanceCHResponse>>>await clickhouse
+ .query(query, {
+ params: {
+ pid,
+ periodFormatted,
+ periodSubtracted,
+ now,
+ ...filtersParams,
+ },
+ })
+ .toPromise()
+
+ rawResult = _sortBy(rawResult, 'sortOrder')
+
+ const currentPeriod = rawResult[0]
+ const previousPeriod = rawResult[1]
+
+ result[pid] = {
+ current: {
+ frontend: millisecondsToSeconds(
+ currentPeriod.render + currentPeriod.domLoad,
+ ),
+ network: millisecondsToSeconds(
+ currentPeriod.dns +
+ currentPeriod.tls +
+ currentPeriod.conn +
+ currentPeriod.response,
+ ),
+ backend: millisecondsToSeconds(currentPeriod.ttfb),
+ },
+ previous: {
+ frontend: millisecondsToSeconds(
+ previousPeriod.render + previousPeriod.domLoad,
+ ),
+ network: millisecondsToSeconds(
+ previousPeriod.dns +
+ previousPeriod.tls +
+ previousPeriod.conn +
+ previousPeriod.response,
+ ),
+ backend: millisecondsToSeconds(previousPeriod.ttfb),
+ },
+ frontendChange: millisecondsToSeconds(
+ currentPeriod.render +
+ currentPeriod.domLoad -
+ previousPeriod.render -
+ previousPeriod.domLoad,
+ ),
+ networkChange: millisecondsToSeconds(
+ currentPeriod.dns +
+ currentPeriod.tls +
+ currentPeriod.conn +
+ currentPeriod.response -
+ previousPeriod.dns -
+ previousPeriod.tls -
+ previousPeriod.conn -
+ previousPeriod.response,
+ ),
+ backendChange: millisecondsToSeconds(
+ currentPeriod.ttfb - previousPeriod.ttfb,
+ ),
}
- } catch {
+ } catch (reason) {
+ console.error(
+ `[ERROR] (getAnalyticsSummary) Error occurred for PID ${pid}`,
+ )
+ console.error(reason)
throw new InternalServerErrorException(
"Can't process the provided PID. Please, try again later.",
)
@@ -1893,8 +2167,4 @@ export class AnalyticsService {
// @ts-ignore
return redis.countKeysByPattern(`hb:${projectId}:*`)
}
-
- async getStatsByProjectId(projectId: string) {
- return this.getSummaryStats([projectId], 'analytics', 'w', 1)
- }
}
diff --git a/apps/selfhosted/src/analytics/interfaces/index.ts b/apps/selfhosted/src/analytics/interfaces/index.ts
index 66084f88e..98ba0edec 100644
--- a/apps/selfhosted/src/analytics/interfaces/index.ts
+++ b/apps/selfhosted/src/analytics/interfaces/index.ts
@@ -118,3 +118,49 @@ export interface IGetFunnel {
funnel: IFunnel[]
totalPageviews: number
}
+
+export interface BirdseyeCHResponse {
+ all: number
+ unique: number
+ sdur: number
+}
+
+interface IOverallPeriodStats {
+ all: number
+ unique: number
+ bounceRate: number
+ sdur: number
+}
+
+export interface IPerformanceObject {
+ current: {
+ frontend: number
+ backend: number
+ network: number
+ }
+ previous: {
+ frontend: number
+ backend: number
+ network: number
+ }
+ frontendChange: number
+ backendChange: number
+ networkChange: number
+}
+
+export interface IOverallObject {
+ current: IOverallPeriodStats
+ previous: IOverallPeriodStats
+ change: number
+ uniqueChange?: number
+ bounceRateChange?: number
+ sdurChange?: number
+}
+
+export interface IOverall {
+ [key: string]: IOverallObject
+}
+
+export interface IOverallPerformance {
+ [key: string]: IPerformanceObject
+}