Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

fix performance tracking api

Blue Mouse744778237dc2
diff --git a/migrations/clickhouse/2022_12_29.js b/migrations/clickhouse/2022_12_29.js
index f48226e45..0bd56f625 100644
--- a/migrations/clickhouse/2022_12_29.js
+++ b/migrations/clickhouse/2022_12_29.js
@@ -8,6 +8,7 @@ const queries = [
pg Nullable(String),
dv Nullable(String),
br Nullable(String),
+ cc Nullable(FixedString(2)),
dns Nullable(UInt32),
tls Nullable(UInt32),
conn Nullable(UInt32),
diff --git a/src/analytics/analytics.controller.ts b/src/analytics/analytics.controller.ts
index 0f2ced3e9..74b7f8f44 100644
--- a/src/analytics/analytics.controller.ts
+++ b/src/analytics/analytics.controller.ts
@@ -116,6 +116,7 @@ const performanceDTO = (
pg: string,
dv: string,
br: string,
+ cc: string,
dns: number,
tls: number,
conn: number,
@@ -130,6 +131,7 @@ const performanceDTO = (
pg,
dv,
br,
+ cc,
_round(dns),
_round(tls),
_round(conn),
@@ -355,6 +357,150 @@ export class AnalyticsController {
}
}
+ @Get('/performance')
+ async getPerfData(
+ @Query() data: AnalyticsGET_DTO,
+ @CurrentUserId() uid: string,
+ ): Promise<any> {
+ const {
+ pid,
+ period,
+ timeBucket,
+ from,
+ to,
+ filters,
+ timezone = DEFAULT_TIMEZONE,
+ } = data
+ this.analyticsService.validatePID(pid)
+
+ if (!_isEmpty(period)) {
+ this.analyticsService.validatePeriod(period)
+ }
+
+ this.analyticsService.validateTimebucket(timeBucket)
+ // const [filtersQuery, filtersParams] =
+ // this.analyticsService.getFiltersQuery(filters)
+ await this.analyticsService.checkProjectAccess(pid, uid)
+
+ let groupFrom = from,
+ groupTo = to
+
+ // const subQuery = `FROM performance WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
+ const subQuery = `FROM performance WHERE pid = {pid:FixedString(12)} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
+
+ const paramsData = {
+ params: {
+ pid,
+ groupFrom: null,
+ groupTo: null,
+ // ...filtersParams,
+ },
+ }
+
+ if (!_isEmpty(from) && !_isEmpty(to)) {
+ if (!isValidDate(from)) {
+ throw new PreconditionFailedException(
+ "The timeframe 'from' parameter is invalid",
+ )
+ }
+
+ if (!isValidDate(to)) {
+ throw new PreconditionFailedException(
+ "The timeframe 'to' parameter is invalid",
+ )
+ }
+
+ if (dayjs.utc(from).isAfter(dayjs.utc(to), 'second')) {
+ throw new PreconditionFailedException(
+ "The timeframe 'from' parameter cannot be greater than 'to'",
+ )
+ }
+
+ checkIfTBAllowed(timeBucket, from, to)
+
+ groupFrom = dayjs.tz(from, timezone).utc().format('YYYY-MM-DD HH:mm:ss')
+
+ if (from === to) {
+ groupTo = dayjs
+ .tz(to, timezone)
+ .add(1, 'day')
+ .format('YYYY-MM-DD HH:mm:ss')
+ } else {
+ groupTo = dayjs.tz(to, timezone).format('YYYY-MM-DD HH:mm:ss')
+ }
+ } else if (!_isEmpty(period)) {
+ if (period === 'today') {
+ if (timezone !== DEFAULT_TIMEZONE && isValidTimezone(timezone)) {
+ groupFrom = dayjs()
+ .tz(timezone)
+ .startOf('d')
+ .utc()
+ .format('YYYY-MM-DD HH:mm:ss')
+ groupTo = dayjs().tz(timezone).utc().format('YYYY-MM-DD HH:mm:ss')
+ } else {
+ groupFrom = dayjs.utc().startOf('d').format('YYYY-MM-DD')
+ groupTo = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')
+ }
+ } else if (period === 'yesterday') {
+ if (timezone !== DEFAULT_TIMEZONE && isValidTimezone(timezone)) {
+ groupFrom = dayjs()
+ .tz(timezone)
+ .startOf('d')
+ .subtract(1, 'day')
+ .utc()
+ .format('YYYY-MM-DD HH:mm:ss')
+ groupTo = dayjs()
+ .tz(timezone)
+ .startOf('d')
+ .utc()
+ .format('YYYY-MM-DD HH:mm:ss')
+ } else {
+ groupFrom = dayjs
+ .utc()
+ .startOf('d')
+ .subtract(1, 'day')
+ .format('YYYY-MM-DD')
+ groupTo = dayjs.utc().startOf('d').format('YYYY-MM-DD HH:mm:ss')
+ }
+ } else {
+ groupFrom = dayjs
+ .utc()
+ .subtract(parseInt(period), _last(period))
+ .format('YYYY-MM-DD')
+ groupTo = dayjs.utc().format('YYYY-MM-DD 23:59:59')
+
+ checkIfTBAllowed(timeBucket, groupFrom, groupTo)
+ }
+ } else {
+ throw new BadRequestException(
+ 'The timeframe (either from/to pair or period) has to be provided',
+ )
+ }
+
+ paramsData.params = {
+ ...paramsData.params,
+ groupFrom,
+ groupTo,
+ }
+
+ const result = await this.analyticsService.groupPerfByTimeBucket(
+ timeBucket,
+ groupFrom,
+ groupTo,
+ subQuery,
+ '',
+ paramsData,
+ timezone,
+ )
+
+ console.log(result)
+
+ return {
+ ...result,
+ appliedFilters: filters,
+ }
+ }
+
@Get('/birdseye')
// returns overall short statistics per project
async getOverallStats(
@@ -539,6 +685,7 @@ export class AnalyticsController {
let dto: Array<string | number>
let perfDTO: Array<string | number> = []
let ua: UAParser.IResult
+ let cc
if (unique) {
ua = UAParser(userAgent)
@@ -546,7 +693,7 @@ export class AnalyticsController {
const br = ua.browser.name
const os = ua.os.name
// trying to get country from timezome, otherwise using CloudFlare's IP based country code as a fallback
- const cc =
+ cc =
ct.getCountryForTimezone(logDTO.tz)?.id ||
(headers['cf-ipcountry'] === 'XX' ? 'NULL' : headers['cf-ipcountry'])
dto = analyticsDTO(
@@ -592,6 +739,9 @@ export class AnalyticsController {
if (!ua) {
ua = UAParser(userAgent)
}
+ if (!cc) {
+ cc = ct.getCountryForTimezone(logDTO.tz)?.id || (headers['cf-ipcountry'] === 'XX' ? 'NULL' : headers['cf-ipcountry'])
+ }
const dv = ua.device.type || 'desktop'
const br = ua.browser.name
@@ -604,6 +754,7 @@ export class AnalyticsController {
logDTO.pg,
dv,
br,
+ cc,
dns,
tls,
conn,
diff --git a/src/analytics/analytics.service.ts b/src/analytics/analytics.service.ts
index a7e25da53..4d9547e4c 100644
--- a/src/analytics/analytics.service.ts
+++ b/src/analytics/analytics.service.ts
@@ -67,6 +67,13 @@ export const cols = [
'ca',
]
+export const perfCols = [
+ 'cc',
+ 'pg',
+ 'dv',
+ 'br',
+]
+
interface chartCHResponse {
index: number
unique: number
@@ -651,6 +658,152 @@ export class AnalyticsService {
})
}
+ async groupPerfByTimeBucket(
+ timeBucket: TimeBucketType,
+ from: string,
+ to: string,
+ subQuery: string,
+ filtersQuery: string,
+ paramsData: object,
+ timezone: string,
+ ): Promise<object | void> {
+ const params = {}
+
+ for (const i of perfCols) {
+ const query1 = `SELECT ${i}, avg(dns), avg(tls), avg(conn), avg(response), avg(render), avg(domLoad), avg(pageLoad), avg(ttfb) ${subQuery} AND ${i} IS NOT NULL GROUP BY ${i}`
+ const res = await clickhouse.query(query1, paramsData).toPromise()
+
+ params[i] = {}
+
+ const size = _size(res)
+ for (let j = 0; j < size; ++j) {
+ const key = res[j][i]
+ params[i][key] = {
+ dns: res[j]['avg(dns)'],
+ tls: res[j]['avg(tls)'],
+ conn: res[j]['avg(conn)'],
+ response: res[j]['avg(response)'],
+ render: res[j]['avg(render)'],
+ domLoad: res[j]['avg(domLoad)'],
+ pageLoad: res[j]['avg(pageLoad)'],
+ ttfb: res[j]['avg(ttfb)'],
+ }
+ }
+ }
+
+ if (!_some(_values(params), val => !_isEmpty(val))) {
+ return Promise.resolve()
+ }
+
+ let groupDateIterator
+ const now = dayjs.utc().endOf(timeBucket)
+ const djsTo = dayjs.utc(to).endOf(timeBucket)
+ const iterateTo = djsTo > now ? now : djsTo
+
+ switch (timeBucket) {
+ case TimeBucketType.HOUR:
+ groupDateIterator = dayjs.utc(from).startOf('hour')
+ break
+
+ case TimeBucketType.DAY:
+ case TimeBucketType.WEEK:
+ case TimeBucketType.MONTH:
+ groupDateIterator = dayjs.utc(from).startOf('day')
+ break
+
+ default:
+ return Promise.reject()
+ }
+
+ let x = []
+
+ while (groupDateIterator < iterateTo) {
+ const nextIteration = groupDateIterator.add(1, timeBucket)
+ x.push(groupDateIterator.format('YYYY-MM-DD HH:mm:ss'))
+ groupDateIterator = nextIteration
+ }
+
+ const xM = [...x, groupDateIterator.format('YYYY-MM-DD HH:mm:ss')]
+ let query = ''
+
+ for (let i = 0; i < _size(x); ++i) {
+ if (i > 0) {
+ query += ' UNION ALL '
+ }
+
+ query += `select ${i} index, avg(dns), avg(tls), avg(conn), avg(response), avg(render), avg(domLoad), avg(pageLoad), avg(ttfb) from performance where pid = {pid:FixedString(12)} and created between '${
+ xM[i]
+ }' and '${xM[1 + i]}' ${filtersQuery} group by pid`
+ }
+
+ // @ts-ignore
+ const result: Array<chartCHResponse> = (
+ await clickhouse.query(query, paramsData).toPromise()
+ )
+ // @ts-ignore
+ .sort((a, b) => a.index - b.index)
+
+ const dns = []
+ const tls = []
+ const conn = []
+ const response = []
+ const render = []
+ const domLoad = []
+ const pageLoad = []
+ const ttfb = []
+
+ let idx = 0
+ const resSize = _size(result)
+
+ while (idx < resSize) {
+ const res = result[idx]
+ const index = res.index
+
+ dns[index] = res['avg(dns)']
+ tls[index] = res['avg(tls)']
+ conn[index] = res['avg(conn)']
+ response[index] = res['avg(response)']
+ render[index] = res['avg(render)']
+ domLoad[index] = res['avg(domLoad)']
+ pageLoad[index] = res['avg(pageLoad)']
+ ttfb[index] = res['avg(ttfb)']
+
+ idx++
+ }
+
+ for (let i = 0; i < _size(x); ++i) {
+ if (!dns[i]) dns[i] = 0
+ if (!tls[i]) tls[i] = 0
+ if (!conn[i]) conn[i] = 0
+ if (!response[i]) response[i] = 0
+ if (!render[i]) render[i] = 0
+ if (!domLoad[i]) domLoad[i] = 0
+ if (!pageLoad[i]) pageLoad[i] = 0
+ if (!ttfb[i]) ttfb[i] = 0
+ }
+
+ if (timezone !== DEFAULT_TIMEZONE && isValidTimezone(timezone)) {
+ x = _map(x, el =>
+ dayjs.utc(el).tz(timezone).format('YYYY-MM-DD HH:mm:ss'),
+ )
+ }
+
+ return Promise.resolve({
+ params,
+ chart: {
+ x,
+ dns,
+ tls,
+ conn,
+ response,
+ render,
+ domLoad,
+ pageLoad,
+ ttfb,
+ },
+ })
+ }
+
async processCustomEV(query: string, params: object): Promise<object> {
const result = {}
diff --git a/src/common/constants.ts b/src/common/constants.ts
index 5bab71aed..01f23b616 100644
--- a/src/common/constants.ts
+++ b/src/common/constants.ts
@@ -80,6 +80,7 @@ const CLICKHOUSE_INIT_QUERIES = [
pg Nullable(String),
dv Nullable(String),
br Nullable(String),
+ cc Nullable(FixedString(2)),
dns Nullable(UInt32),
tls Nullable(UInt32),
conn Nullable(UInt32),