📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
(improvement) Timezone support for sessions (#233)
Andrii Romasiun8ecb8e8a5705
diff --git a/apps/production/src/analytics/analytics.controller.ts b/apps/production/src/analytics/analytics.controller.tsindex f26e8e506..5b2332a5f 100644--- a/apps/production/src/analytics/analytics.controller.ts+++ b/apps/production/src/analytics/analytics.controller.ts@@ -1606,7 +1606,7 @@ export class AnalyticsController {this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)const safeTimezone = this.analyticsService.getSafeTimezone(timezone)- const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(+ const { groupFromUTC, groupToUTC } = this.analyticsService.getGroupFromTo(from,to,timeBucket,@@ -1618,8 +1618,8 @@ export class AnalyticsController {const paramsData = {params: {pid,- groupFrom,- groupTo,+ groupFrom: groupFromUTC,+ groupTo: groupToUTC,...filtersParams,},}diff --git a/apps/production/src/analytics/analytics.service.ts b/apps/production/src/analytics/analytics.service.tsindex e15b152c8..e00c3f5ba 100644--- a/apps/production/src/analytics/analytics.service.ts+++ b/apps/production/src/analytics/analytics.service.ts@@ -2068,6 +2068,35 @@ export class AnalyticsService {return baseQuery}+ generateSessionAggregationQuery(+ timeBucket: TimeBucketType,+ filtersQuery: string,+ ): string {+ const timeBucketFunc = timeBucketConversion[timeBucket]+ const [selector, groupBy] = this.getGroupSubquery(timeBucket)++ const baseQuery = `+ SELECT+ ${selector},+ avg(sdur) as sdur,+ count() as pageviews,+ countIf(unique=1) as uniques+ FROM (+ SELECT *,+ ${timeBucketFunc}(created) as tz_created+ FROM analytics+ WHERE+ pid = {pid:FixedString(12)}+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}+ ${filtersQuery}+ ) as subquery+ GROUP BY ${groupBy}+ ORDER BY ${groupBy}+ `++ return baseQuery+ }+generateCustomEventsAggregationQuery(timeBucket: TimeBucketType,filtersQuery: string,@@ -2394,6 +2423,118 @@ export class AnalyticsService {})}+ generateUTCXAxis(+ timeBucket: TimeBucketType,+ from: string,+ to: string,+ ): {+ x: string[]+ format: string+ } {+ const iterateTo = dayjs.utc(to)+ let djsFrom = dayjs.utc(from)++ let format++ switch (timeBucket) {+ case TimeBucketType.MINUTE:+ case TimeBucketType.HOUR:+ format = 'YYYY-MM-DD HH:mm:ss'+ break++ case TimeBucketType.DAY:+ format = 'YYYY-MM-DD'+ break++ case TimeBucketType.MONTH:+ format = 'YYYY-MM'+ break++ case TimeBucketType.YEAR:+ format = 'YYYY'+ break++ default:+ throw new BadRequestException(+ `The provided time bucket (${timeBucket}) is incorrect`,+ )+ }++ const x = []++ while (djsFrom.isSameOrBefore(iterateTo, timeBucket)) {+ x.push(djsFrom.format(format))+ djsFrom = djsFrom.add(1, timeBucket)+ }++ return {+ x,+ format,+ }+ }++ shiftToTimezone(x: string[], timezone: string, format: string): string[] {+ return _map(x, (date: string) =>+ dayjs.utc(date).tz(timezone).format(format),+ )+ }++ async groupSessionChartByTimeBucket(+ timeBucket: TimeBucketType,+ from: string,+ to: string,+ filtersQuery: string,+ paramsData: any,+ safeTimezone: string,+ ): Promise<object | void> {+ const { x, format } = this.generateUTCXAxis(timeBucket, from, to)++ // if (false) {+ // const query = this.generateCustomEventsAggregationQuery(+ // timeBucket,+ // filtersQuery,+ // safeTimezone,+ // ChartRenderMode.PERIODICAL,+ // )++ // const result = <Array<TrafficCEFilterCHResponse>>(+ // await clickhouse.query(query, paramsData).toPromise()+ // )++ // const uniques =+ // this.extractCustomEventsChartData(result, xShifted)?._unknown_event ||+ // []++ // const sdur = Array(_size(xShifted)).fill(0)++ // return Promise.resolve({+ // chart: {+ // x: xShifted,+ // visits: uniques,+ // uniques,+ // sdur,+ // },+ // })+ // }++ const query = this.generateSessionAggregationQuery(timeBucket, filtersQuery)++ const result = <Array<TrafficCHResponse>>(+ await clickhouse.query(query, paramsData).toPromise()+ )++ const { visits, uniques, sdur } = this.extractChartData(result, x)++ return Promise.resolve({+ chart: {+ x: this.shiftToTimezone(x, safeTimezone, format),+ visits,+ uniques,+ sdur,+ },+ })+ }+extractCaptchaChartData(result, x: string[]): any {const count = Array(x.length).fill(0)@@ -2888,7 +3029,7 @@ export class AnalyticsService {SELECT'pageview' AS type,pg AS value,- created+ toTimeZone(analytics.created, '${safeTimezone}') AS createdFROM analyticsWHEREpid = {pid:FixedString(12)}@@ -2899,7 +3040,7 @@ export class AnalyticsService {SELECT'event' AS type,ev AS value,- created+ toTimeZone(customEV.created, '${safeTimezone}') AS createdFROM customEVWHEREpid = {pid:FixedString(12)}@@ -2956,10 +3097,14 @@ export class AnalyticsService {let timeBucket = nullif (!_isEmpty(pages)) {- const from = dayjs(pages[0].created)+ const from = dayjs+ .tz(pages[0].created, safeTimezone)+ .utc().startOf('minute').format('YYYY-MM-DD HH:mm:ss')- const to = dayjs(pages[_size(pages) - 1].created)+ const to = dayjs+ .tz(pages[_size(pages) - 1].created, safeTimezone)+ .utc().endOf('minute').format('YYYY-MM-DD HH:mm:ss')@@ -2969,7 +3114,7 @@ export class AnalyticsService {? TimeBucketType.HOUR: TimeBucketType.MINUTE- const groupedChart = await this.groupChartByTimeBucket(+ const groupedChart = await this.groupSessionChartByTimeBucket(timeBucket,from,to,@@ -2982,8 +3127,6 @@ export class AnalyticsService {},},safeTimezone,- false,- ChartRenderMode.PERIODICAL,)// @ts-ignore@@ -3001,9 +3144,6 @@ export class AnalyticsService {skip = 0,customEVFilterApplied = false,): Promise<object | void> {- const tzFromDate = `toTimeZone(parseDateTimeBestEffort({groupFrom:String}), '${safeTimezone}')`- const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), '${safeTimezone}')`-const analyticsSubquery = `SELECTisNotNull(sid) AS active,@@ -3011,12 +3151,12 @@ export class AnalyticsService {cc,os,br,- created+ toTimeZone(analytics.created, '${safeTimezone}') AS createdFROM analyticsWHEREpid = {pid:FixedString(12)}AND psid IS NOT NULL- AND created BETWEEN ${tzFromDate} AND ${tzToDate}+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}${filtersQuery}`@@ -3027,12 +3167,12 @@ export class AnalyticsService {cc,os,br,- created+ toTimeZone(customEV.created, '${safeTimezone}') AS createdFROM customEVWHEREpid = {pid:FixedString(12)}AND psid IS NOT NULL- AND created BETWEEN ${tzFromDate} AND ${tzToDate}+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}${filtersQuery}`diff --git a/apps/selfhosted/src/analytics/analytics.controller.ts b/apps/selfhosted/src/analytics/analytics.controller.tsindex a125666ba..5fd8abef7 100644--- a/apps/selfhosted/src/analytics/analytics.controller.ts+++ b/apps/selfhosted/src/analytics/analytics.controller.ts@@ -1338,7 +1338,7 @@ export class AnalyticsController {this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)const safeTimezone = this.analyticsService.getSafeTimezone(timezone)- const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(+ const { groupFromUTC, groupToUTC } = this.analyticsService.getGroupFromTo(from,to,timeBucket,@@ -1350,8 +1350,8 @@ export class AnalyticsController {const paramsData = {params: {pid,- groupFrom,- groupTo,+ groupFrom: groupFromUTC,+ groupTo: groupToUTC,...filtersParams,},}diff --git a/apps/selfhosted/src/analytics/analytics.service.ts b/apps/selfhosted/src/analytics/analytics.service.tsindex 073c162b7..0e6d1b655 100644--- a/apps/selfhosted/src/analytics/analytics.service.ts+++ b/apps/selfhosted/src/analytics/analytics.service.ts@@ -1795,6 +1795,35 @@ export class AnalyticsService {return baseQuery}+ generateSessionAggregationQuery(+ timeBucket: TimeBucketType,+ filtersQuery: string,+ ): string {+ const timeBucketFunc = timeBucketConversion[timeBucket]+ const [selector, groupBy] = this.getGroupSubquery(timeBucket)++ const baseQuery = `+ SELECT+ ${selector},+ avg(sdur) as sdur,+ count() as pageviews,+ countIf(unique=1) as uniques+ FROM (+ SELECT *,+ ${timeBucketFunc}(created) as tz_created+ FROM analytics+ WHERE+ pid = {pid:FixedString(12)}+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}+ ${filtersQuery}+ ) as subquery+ GROUP BY ${groupBy}+ ORDER BY ${groupBy}+ `++ return baseQuery+ }+generateCustomEventsAggregationQuery(timeBucket: TimeBucketType,filtersQuery: string,@@ -2131,6 +2160,118 @@ export class AnalyticsService {}}+ generateUTCXAxis(+ timeBucket: TimeBucketType,+ from: string,+ to: string,+ ): {+ x: string[]+ format: string+ } {+ const iterateTo = dayjs.utc(to)+ let djsFrom = dayjs.utc(from)++ let format++ switch (timeBucket) {+ case TimeBucketType.MINUTE:+ case TimeBucketType.HOUR:+ format = 'YYYY-MM-DD HH:mm:ss'+ break++ case TimeBucketType.DAY:+ format = 'YYYY-MM-DD'+ break++ case TimeBucketType.MONTH:+ format = 'YYYY-MM'+ break++ case TimeBucketType.YEAR:+ format = 'YYYY'+ break++ default:+ throw new BadRequestException(+ `The provided time bucket (${timeBucket}) is incorrect`,+ )+ }++ const x = []++ while (djsFrom.isSameOrBefore(iterateTo, timeBucket)) {+ x.push(djsFrom.format(format))+ djsFrom = djsFrom.add(1, timeBucket)+ }++ return {+ x,+ format,+ }+ }++ shiftToTimezone(x: string[], timezone: string, format: string): string[] {+ return _map(x, (date: string) =>+ dayjs.utc(date).tz(timezone).format(format),+ )+ }++ async groupSessionChartByTimeBucket(+ timeBucket: TimeBucketType,+ from: string,+ to: string,+ filtersQuery: string,+ paramsData: any,+ safeTimezone: string,+ ): Promise<object | void> {+ const { x, format } = this.generateUTCXAxis(timeBucket, from, to)++ // if (false) {+ // const query = this.generateCustomEventsAggregationQuery(+ // timeBucket,+ // filtersQuery,+ // safeTimezone,+ // ChartRenderMode.PERIODICAL,+ // )++ // const result = <Array<TrafficCEFilterCHResponse>>(+ // await clickhouse.query(query, paramsData).toPromise()+ // )++ // const uniques =+ // this.extractCustomEventsChartData(result, xShifted)?._unknown_event ||+ // []++ // const sdur = Array(_size(xShifted)).fill(0)++ // return Promise.resolve({+ // chart: {+ // x: xShifted,+ // visits: uniques,+ // uniques,+ // sdur,+ // },+ // })+ // }++ const query = this.generateSessionAggregationQuery(timeBucket, filtersQuery)++ const result = <Array<TrafficCHResponse>>(+ await clickhouse.query(query, paramsData).toPromise()+ )++ const { visits, uniques, sdur } = this.extractChartData(result, x)++ return Promise.resolve({+ chart: {+ x: this.shiftToTimezone(x, safeTimezone, format),+ visits,+ uniques,+ sdur,+ },+ })+ }+async groupErrorsByTimeBucket(timeBucket: TimeBucketType,from: string,@@ -2507,7 +2648,7 @@ export class AnalyticsService {SELECT'pageview' AS type,pg AS value,- created+ toTimeZone(analytics.created, '${safeTimezone}') AS createdFROM analyticsWHEREpid = {pid:FixedString(12)}@@ -2518,7 +2659,7 @@ export class AnalyticsService {SELECT'event' AS type,ev AS value,- created+ toTimeZone(customEV.created, '${safeTimezone}') AS createdFROM customEVWHEREpid = {pid:FixedString(12)}@@ -2575,10 +2716,14 @@ export class AnalyticsService {let timeBucket = nullif (!_isEmpty(pages)) {- const from = dayjs(pages[0].created)+ const from = dayjs+ .tz(pages[0].created, safeTimezone)+ .utc().startOf('minute').format('YYYY-MM-DD HH:mm:ss')- const to = dayjs(pages[_size(pages) - 1].created)+ const to = dayjs+ .tz(pages[_size(pages) - 1].created, safeTimezone)+ .utc().endOf('minute').format('YYYY-MM-DD HH:mm:ss')@@ -2588,7 +2733,7 @@ export class AnalyticsService {? TimeBucketType.HOUR: TimeBucketType.MINUTE- const groupedChart = await this.groupChartByTimeBucket(+ const groupedChart = await this.groupSessionChartByTimeBucket(timeBucket,from,to,@@ -2601,8 +2746,6 @@ export class AnalyticsService {},},safeTimezone,- false,- ChartRenderMode.PERIODICAL,)// @ts-ignore@@ -2620,9 +2763,6 @@ export class AnalyticsService {skip = 0,customEVFilterApplied = false,): Promise<object | void> {- const tzFromDate = `toTimeZone(parseDateTimeBestEffort({groupFrom:String}), '${safeTimezone}')`- const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), '${safeTimezone}')`-const analyticsSubquery = `SELECTisNotNull(sid) AS active,@@ -2630,12 +2770,12 @@ export class AnalyticsService {cc,os,br,- created+ toTimeZone(analytics.created, '${safeTimezone}') AS createdFROM analyticsWHEREpid = {pid:FixedString(12)}AND psid IS NOT NULL- AND created BETWEEN ${tzFromDate} AND ${tzToDate}+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}${filtersQuery}`@@ -2646,12 +2786,12 @@ export class AnalyticsService {cc,os,br,- created+ toTimeZone(customEV.created, '${safeTimezone}') AS createdFROM customEVWHEREpid = {pid:FixedString(12)}AND psid IS NOT NULL- AND created BETWEEN ${tzFromDate} AND ${tzToDate}+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}${filtersQuery}`