📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
fix performance tracking api
Blue Mouse744778237dc2
diff --git a/migrations/clickhouse/2022_12_29.js b/migrations/clickhouse/2022_12_29.jsindex f48226e45..0bd56f625 100644--- a/migrations/clickhouse/2022_12_29.js+++ b/migrations/clickhouse/2022_12_29.js@@ -8,6 +8,7 @@ const queries = [pg Nullable(String),dv Nullable(String),br Nullable(String),+ cc Nullable(FixedString(2)),dns Nullable(UInt32),tls Nullable(UInt32),conn Nullable(UInt32),diff --git a/src/analytics/analytics.controller.ts b/src/analytics/analytics.controller.tsindex 0f2ced3e9..74b7f8f44 100644--- a/src/analytics/analytics.controller.ts+++ b/src/analytics/analytics.controller.ts@@ -116,6 +116,7 @@ const performanceDTO = (pg: string,dv: string,br: string,+ cc: string,dns: number,tls: number,conn: number,@@ -130,6 +131,7 @@ const performanceDTO = (pg,dv,br,+ cc,_round(dns),_round(tls),_round(conn),@@ -355,6 +357,150 @@ export class AnalyticsController {}}+ @Get('/performance')+ async getPerfData(+ @Query() data: AnalyticsGET_DTO,+ @CurrentUserId() uid: string,+ ): Promise<any> {+ const {+ pid,+ period,+ timeBucket,+ from,+ to,+ filters,+ timezone = DEFAULT_TIMEZONE,+ } = data+ this.analyticsService.validatePID(pid)++ if (!_isEmpty(period)) {+ this.analyticsService.validatePeriod(period)+ }++ this.analyticsService.validateTimebucket(timeBucket)+ // const [filtersQuery, filtersParams] =+ // this.analyticsService.getFiltersQuery(filters)+ await this.analyticsService.checkProjectAccess(pid, uid)++ let groupFrom = from,+ groupTo = to++ // const subQuery = `FROM performance WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`+ const subQuery = `FROM performance WHERE pid = {pid:FixedString(12)} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`++ const paramsData = {+ params: {+ pid,+ groupFrom: null,+ groupTo: null,+ // ...filtersParams,+ },+ }++ if (!_isEmpty(from) && !_isEmpty(to)) {+ if (!isValidDate(from)) {+ throw new PreconditionFailedException(+ "The timeframe 'from' parameter is invalid",+ )+ }++ if (!isValidDate(to)) {+ throw new PreconditionFailedException(+ "The timeframe 'to' parameter is invalid",+ )+ }++ if (dayjs.utc(from).isAfter(dayjs.utc(to), 'second')) {+ throw new PreconditionFailedException(+ "The timeframe 'from' parameter cannot be greater than 'to'",+ )+ }++ checkIfTBAllowed(timeBucket, from, to)++ groupFrom = dayjs.tz(from, timezone).utc().format('YYYY-MM-DD HH:mm:ss')++ if (from === to) {+ groupTo = dayjs+ .tz(to, timezone)+ .add(1, 'day')+ .format('YYYY-MM-DD HH:mm:ss')+ } else {+ groupTo = dayjs.tz(to, timezone).format('YYYY-MM-DD HH:mm:ss')+ }+ } else if (!_isEmpty(period)) {+ if (period === 'today') {+ if (timezone !== DEFAULT_TIMEZONE && isValidTimezone(timezone)) {+ groupFrom = dayjs()+ .tz(timezone)+ .startOf('d')+ .utc()+ .format('YYYY-MM-DD HH:mm:ss')+ groupTo = dayjs().tz(timezone).utc().format('YYYY-MM-DD HH:mm:ss')+ } else {+ groupFrom = dayjs.utc().startOf('d').format('YYYY-MM-DD')+ groupTo = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')+ }+ } else if (period === 'yesterday') {+ if (timezone !== DEFAULT_TIMEZONE && isValidTimezone(timezone)) {+ groupFrom = dayjs()+ .tz(timezone)+ .startOf('d')+ .subtract(1, 'day')+ .utc()+ .format('YYYY-MM-DD HH:mm:ss')+ groupTo = dayjs()+ .tz(timezone)+ .startOf('d')+ .utc()+ .format('YYYY-MM-DD HH:mm:ss')+ } else {+ groupFrom = dayjs+ .utc()+ .startOf('d')+ .subtract(1, 'day')+ .format('YYYY-MM-DD')+ groupTo = dayjs.utc().startOf('d').format('YYYY-MM-DD HH:mm:ss')+ }+ } else {+ groupFrom = dayjs+ .utc()+ .subtract(parseInt(period), _last(period))+ .format('YYYY-MM-DD')+ groupTo = dayjs.utc().format('YYYY-MM-DD 23:59:59')++ checkIfTBAllowed(timeBucket, groupFrom, groupTo)+ }+ } else {+ throw new BadRequestException(+ 'The timeframe (either from/to pair or period) has to be provided',+ )+ }++ paramsData.params = {+ ...paramsData.params,+ groupFrom,+ groupTo,+ }++ const result = await this.analyticsService.groupPerfByTimeBucket(+ timeBucket,+ groupFrom,+ groupTo,+ subQuery,+ '',+ paramsData,+ timezone,+ )++ console.log(result)++ return {+ ...result,+ appliedFilters: filters,+ }+ }+@Get('/birdseye')// returns overall short statistics per projectasync getOverallStats(@@ -539,6 +685,7 @@ export class AnalyticsController {let dto: Array<string | number>let perfDTO: Array<string | number> = []let ua: UAParser.IResult+ let ccif (unique) {ua = UAParser(userAgent)@@ -546,7 +693,7 @@ export class AnalyticsController {const br = ua.browser.nameconst os = ua.os.name// trying to get country from timezome, otherwise using CloudFlare's IP based country code as a fallback- const cc =+ cc =ct.getCountryForTimezone(logDTO.tz)?.id ||(headers['cf-ipcountry'] === 'XX' ? 'NULL' : headers['cf-ipcountry'])dto = analyticsDTO(@@ -592,6 +739,9 @@ export class AnalyticsController {if (!ua) {ua = UAParser(userAgent)}+ if (!cc) {+ cc = ct.getCountryForTimezone(logDTO.tz)?.id || (headers['cf-ipcountry'] === 'XX' ? 'NULL' : headers['cf-ipcountry'])+ }const dv = ua.device.type || 'desktop'const br = ua.browser.name@@ -604,6 +754,7 @@ export class AnalyticsController {logDTO.pg,dv,br,+ cc,dns,tls,conn,diff --git a/src/analytics/analytics.service.ts b/src/analytics/analytics.service.tsindex a7e25da53..4d9547e4c 100644--- a/src/analytics/analytics.service.ts+++ b/src/analytics/analytics.service.ts@@ -67,6 +67,13 @@ export const cols = ['ca',]+export const perfCols = [+ 'cc',+ 'pg',+ 'dv',+ 'br',+]+interface chartCHResponse {index: numberunique: number@@ -651,6 +658,152 @@ export class AnalyticsService {})}+ async groupPerfByTimeBucket(+ timeBucket: TimeBucketType,+ from: string,+ to: string,+ subQuery: string,+ filtersQuery: string,+ paramsData: object,+ timezone: string,+ ): Promise<object | void> {+ const params = {}++ for (const i of perfCols) {+ const query1 = `SELECT ${i}, avg(dns), avg(tls), avg(conn), avg(response), avg(render), avg(domLoad), avg(pageLoad), avg(ttfb) ${subQuery} AND ${i} IS NOT NULL GROUP BY ${i}`+ const res = await clickhouse.query(query1, paramsData).toPromise()++ params[i] = {}++ const size = _size(res)+ for (let j = 0; j < size; ++j) {+ const key = res[j][i]+ params[i][key] = {+ dns: res[j]['avg(dns)'],+ tls: res[j]['avg(tls)'],+ conn: res[j]['avg(conn)'],+ response: res[j]['avg(response)'],+ render: res[j]['avg(render)'],+ domLoad: res[j]['avg(domLoad)'],+ pageLoad: res[j]['avg(pageLoad)'],+ ttfb: res[j]['avg(ttfb)'],+ }+ }+ }++ if (!_some(_values(params), val => !_isEmpty(val))) {+ return Promise.resolve()+ }++ let groupDateIterator+ const now = dayjs.utc().endOf(timeBucket)+ const djsTo = dayjs.utc(to).endOf(timeBucket)+ const iterateTo = djsTo > now ? now : djsTo++ switch (timeBucket) {+ case TimeBucketType.HOUR:+ groupDateIterator = dayjs.utc(from).startOf('hour')+ break++ case TimeBucketType.DAY:+ case TimeBucketType.WEEK:+ case TimeBucketType.MONTH:+ groupDateIterator = dayjs.utc(from).startOf('day')+ break++ default:+ return Promise.reject()+ }++ let x = []++ while (groupDateIterator < iterateTo) {+ const nextIteration = groupDateIterator.add(1, timeBucket)+ x.push(groupDateIterator.format('YYYY-MM-DD HH:mm:ss'))+ groupDateIterator = nextIteration+ }++ const xM = [...x, groupDateIterator.format('YYYY-MM-DD HH:mm:ss')]+ let query = ''++ for (let i = 0; i < _size(x); ++i) {+ if (i > 0) {+ query += ' UNION ALL '+ }++ query += `select ${i} index, avg(dns), avg(tls), avg(conn), avg(response), avg(render), avg(domLoad), avg(pageLoad), avg(ttfb) from performance where pid = {pid:FixedString(12)} and created between '${+ xM[i]+ }' and '${xM[1 + i]}' ${filtersQuery} group by pid`+ }++ // @ts-ignore+ const result: Array<chartCHResponse> = (+ await clickhouse.query(query, paramsData).toPromise()+ )+ // @ts-ignore+ .sort((a, b) => a.index - b.index)++ const dns = []+ const tls = []+ const conn = []+ const response = []+ const render = []+ const domLoad = []+ const pageLoad = []+ const ttfb = []++ let idx = 0+ const resSize = _size(result)++ while (idx < resSize) {+ const res = result[idx]+ const index = res.index++ dns[index] = res['avg(dns)']+ tls[index] = res['avg(tls)']+ conn[index] = res['avg(conn)']+ response[index] = res['avg(response)']+ render[index] = res['avg(render)']+ domLoad[index] = res['avg(domLoad)']+ pageLoad[index] = res['avg(pageLoad)']+ ttfb[index] = res['avg(ttfb)']++ idx+++ }++ for (let i = 0; i < _size(x); ++i) {+ if (!dns[i]) dns[i] = 0+ if (!tls[i]) tls[i] = 0+ if (!conn[i]) conn[i] = 0+ if (!response[i]) response[i] = 0+ if (!render[i]) render[i] = 0+ if (!domLoad[i]) domLoad[i] = 0+ if (!pageLoad[i]) pageLoad[i] = 0+ if (!ttfb[i]) ttfb[i] = 0+ }++ if (timezone !== DEFAULT_TIMEZONE && isValidTimezone(timezone)) {+ x = _map(x, el =>+ dayjs.utc(el).tz(timezone).format('YYYY-MM-DD HH:mm:ss'),+ )+ }++ return Promise.resolve({+ params,+ chart: {+ x,+ dns,+ tls,+ conn,+ response,+ render,+ domLoad,+ pageLoad,+ ttfb,+ },+ })+ }+async processCustomEV(query: string, params: object): Promise<object> {const result = {}diff --git a/src/common/constants.ts b/src/common/constants.tsindex 5bab71aed..01f23b616 100644--- a/src/common/constants.ts+++ b/src/common/constants.ts@@ -80,6 +80,7 @@ const CLICKHOUSE_INIT_QUERIES = [pg Nullable(String),dv Nullable(String),br Nullable(String),+ cc Nullable(FixedString(2)),dns Nullable(UInt32),tls Nullable(UInt32),conn Nullable(UInt32),