Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

(improvement) Timezone support for sessions (#233)

Andrii Romasiun8ecb8e8a5705
diff --git a/apps/production/src/analytics/analytics.controller.ts b/apps/production/src/analytics/analytics.controller.ts
index f26e8e506..5b2332a5f 100644
--- a/apps/production/src/analytics/analytics.controller.ts
+++ b/apps/production/src/analytics/analytics.controller.ts
@@ -1606,7 +1606,7 @@ export class AnalyticsController {
this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
- const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
+ const { groupFromUTC, groupToUTC } = this.analyticsService.getGroupFromTo(
from,
to,
timeBucket,
@@ -1618,8 +1618,8 @@ export class AnalyticsController {
const paramsData = {
params: {
pid,
- groupFrom,
- groupTo,
+ groupFrom: groupFromUTC,
+ groupTo: groupToUTC,
...filtersParams,
},
}
diff --git a/apps/production/src/analytics/analytics.service.ts b/apps/production/src/analytics/analytics.service.ts
index e15b152c8..e00c3f5ba 100644
--- a/apps/production/src/analytics/analytics.service.ts
+++ b/apps/production/src/analytics/analytics.service.ts
@@ -2068,6 +2068,35 @@ export class AnalyticsService {
return baseQuery
}
+ generateSessionAggregationQuery(
+ timeBucket: TimeBucketType,
+ filtersQuery: string,
+ ): string {
+ const timeBucketFunc = timeBucketConversion[timeBucket]
+ const [selector, groupBy] = this.getGroupSubquery(timeBucket)
+
+ const baseQuery = `
+ SELECT
+ ${selector},
+ avg(sdur) as sdur,
+ count() as pageviews,
+ countIf(unique=1) as uniques
+ FROM (
+ SELECT *,
+ ${timeBucketFunc}(created) as tz_created
+ FROM analytics
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ ${filtersQuery}
+ ) as subquery
+ GROUP BY ${groupBy}
+ ORDER BY ${groupBy}
+ `
+
+ return baseQuery
+ }
+
generateCustomEventsAggregationQuery(
timeBucket: TimeBucketType,
filtersQuery: string,
@@ -2394,6 +2423,118 @@ export class AnalyticsService {
})
}
+ generateUTCXAxis(
+ timeBucket: TimeBucketType,
+ from: string,
+ to: string,
+ ): {
+ x: string[]
+ format: string
+ } {
+ const iterateTo = dayjs.utc(to)
+ let djsFrom = dayjs.utc(from)
+
+ let format
+
+ switch (timeBucket) {
+ case TimeBucketType.MINUTE:
+ case TimeBucketType.HOUR:
+ format = 'YYYY-MM-DD HH:mm:ss'
+ break
+
+ case TimeBucketType.DAY:
+ format = 'YYYY-MM-DD'
+ break
+
+ case TimeBucketType.MONTH:
+ format = 'YYYY-MM'
+ break
+
+ case TimeBucketType.YEAR:
+ format = 'YYYY'
+ break
+
+ default:
+ throw new BadRequestException(
+ `The provided time bucket (${timeBucket}) is incorrect`,
+ )
+ }
+
+ const x = []
+
+ while (djsFrom.isSameOrBefore(iterateTo, timeBucket)) {
+ x.push(djsFrom.format(format))
+ djsFrom = djsFrom.add(1, timeBucket)
+ }
+
+ return {
+ x,
+ format,
+ }
+ }
+
+ shiftToTimezone(x: string[], timezone: string, format: string): string[] {
+ return _map(x, (date: string) =>
+ dayjs.utc(date).tz(timezone).format(format),
+ )
+ }
+
+ async groupSessionChartByTimeBucket(
+ timeBucket: TimeBucketType,
+ from: string,
+ to: string,
+ filtersQuery: string,
+ paramsData: any,
+ safeTimezone: string,
+ ): Promise<object | void> {
+ const { x, format } = this.generateUTCXAxis(timeBucket, from, to)
+
+ // if (false) {
+ // const query = this.generateCustomEventsAggregationQuery(
+ // timeBucket,
+ // filtersQuery,
+ // safeTimezone,
+ // ChartRenderMode.PERIODICAL,
+ // )
+
+ // const result = <Array<TrafficCEFilterCHResponse>>(
+ // await clickhouse.query(query, paramsData).toPromise()
+ // )
+
+ // const uniques =
+ // this.extractCustomEventsChartData(result, xShifted)?._unknown_event ||
+ // []
+
+ // const sdur = Array(_size(xShifted)).fill(0)
+
+ // return Promise.resolve({
+ // chart: {
+ // x: xShifted,
+ // visits: uniques,
+ // uniques,
+ // sdur,
+ // },
+ // })
+ // }
+
+ const query = this.generateSessionAggregationQuery(timeBucket, filtersQuery)
+
+ const result = <Array<TrafficCHResponse>>(
+ await clickhouse.query(query, paramsData).toPromise()
+ )
+
+ const { visits, uniques, sdur } = this.extractChartData(result, x)
+
+ return Promise.resolve({
+ chart: {
+ x: this.shiftToTimezone(x, safeTimezone, format),
+ visits,
+ uniques,
+ sdur,
+ },
+ })
+ }
+
extractCaptchaChartData(result, x: string[]): any {
const count = Array(x.length).fill(0)
@@ -2888,7 +3029,7 @@ export class AnalyticsService {
SELECT
'pageview' AS type,
pg AS value,
- created
+ toTimeZone(analytics.created, '${safeTimezone}') AS created
FROM analytics
WHERE
pid = {pid:FixedString(12)}
@@ -2899,7 +3040,7 @@ export class AnalyticsService {
SELECT
'event' AS type,
ev AS value,
- created
+ toTimeZone(customEV.created, '${safeTimezone}') AS created
FROM customEV
WHERE
pid = {pid:FixedString(12)}
@@ -2956,10 +3097,14 @@ export class AnalyticsService {
let timeBucket = null
if (!_isEmpty(pages)) {
- const from = dayjs(pages[0].created)
+ const from = dayjs
+ .tz(pages[0].created, safeTimezone)
+ .utc()
.startOf('minute')
.format('YYYY-MM-DD HH:mm:ss')
- const to = dayjs(pages[_size(pages) - 1].created)
+ const to = dayjs
+ .tz(pages[_size(pages) - 1].created, safeTimezone)
+ .utc()
.endOf('minute')
.format('YYYY-MM-DD HH:mm:ss')
@@ -2969,7 +3114,7 @@ export class AnalyticsService {
? TimeBucketType.HOUR
: TimeBucketType.MINUTE
- const groupedChart = await this.groupChartByTimeBucket(
+ const groupedChart = await this.groupSessionChartByTimeBucket(
timeBucket,
from,
to,
@@ -2982,8 +3127,6 @@ export class AnalyticsService {
},
},
safeTimezone,
- false,
- ChartRenderMode.PERIODICAL,
)
// @ts-ignore
@@ -3001,9 +3144,6 @@ export class AnalyticsService {
skip = 0,
customEVFilterApplied = false,
): Promise<object | void> {
- const tzFromDate = `toTimeZone(parseDateTimeBestEffort({groupFrom:String}), '${safeTimezone}')`
- const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), '${safeTimezone}')`
-
const analyticsSubquery = `
SELECT
isNotNull(sid) AS active,
@@ -3011,12 +3151,12 @@ export class AnalyticsService {
cc,
os,
br,
- created
+ toTimeZone(analytics.created, '${safeTimezone}') AS created
FROM analytics
WHERE
pid = {pid:FixedString(12)}
AND psid IS NOT NULL
- AND created BETWEEN ${tzFromDate} AND ${tzToDate}
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
${filtersQuery}
`
@@ -3027,12 +3167,12 @@ export class AnalyticsService {
cc,
os,
br,
- created
+ toTimeZone(customEV.created, '${safeTimezone}') AS created
FROM customEV
WHERE
pid = {pid:FixedString(12)}
AND psid IS NOT NULL
- AND created BETWEEN ${tzFromDate} AND ${tzToDate}
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
${filtersQuery}
`
diff --git a/apps/selfhosted/src/analytics/analytics.controller.ts b/apps/selfhosted/src/analytics/analytics.controller.ts
index a125666ba..5fd8abef7 100644
--- a/apps/selfhosted/src/analytics/analytics.controller.ts
+++ b/apps/selfhosted/src/analytics/analytics.controller.ts
@@ -1338,7 +1338,7 @@ export class AnalyticsController {
this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
- const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
+ const { groupFromUTC, groupToUTC } = this.analyticsService.getGroupFromTo(
from,
to,
timeBucket,
@@ -1350,8 +1350,8 @@ export class AnalyticsController {
const paramsData = {
params: {
pid,
- groupFrom,
- groupTo,
+ groupFrom: groupFromUTC,
+ groupTo: groupToUTC,
...filtersParams,
},
}
diff --git a/apps/selfhosted/src/analytics/analytics.service.ts b/apps/selfhosted/src/analytics/analytics.service.ts
index 073c162b7..0e6d1b655 100644
--- a/apps/selfhosted/src/analytics/analytics.service.ts
+++ b/apps/selfhosted/src/analytics/analytics.service.ts
@@ -1795,6 +1795,35 @@ export class AnalyticsService {
return baseQuery
}
+ generateSessionAggregationQuery(
+ timeBucket: TimeBucketType,
+ filtersQuery: string,
+ ): string {
+ const timeBucketFunc = timeBucketConversion[timeBucket]
+ const [selector, groupBy] = this.getGroupSubquery(timeBucket)
+
+ const baseQuery = `
+ SELECT
+ ${selector},
+ avg(sdur) as sdur,
+ count() as pageviews,
+ countIf(unique=1) as uniques
+ FROM (
+ SELECT *,
+ ${timeBucketFunc}(created) as tz_created
+ FROM analytics
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ ${filtersQuery}
+ ) as subquery
+ GROUP BY ${groupBy}
+ ORDER BY ${groupBy}
+ `
+
+ return baseQuery
+ }
+
generateCustomEventsAggregationQuery(
timeBucket: TimeBucketType,
filtersQuery: string,
@@ -2131,6 +2160,118 @@ export class AnalyticsService {
}
}
+ generateUTCXAxis(
+ timeBucket: TimeBucketType,
+ from: string,
+ to: string,
+ ): {
+ x: string[]
+ format: string
+ } {
+ const iterateTo = dayjs.utc(to)
+ let djsFrom = dayjs.utc(from)
+
+ let format
+
+ switch (timeBucket) {
+ case TimeBucketType.MINUTE:
+ case TimeBucketType.HOUR:
+ format = 'YYYY-MM-DD HH:mm:ss'
+ break
+
+ case TimeBucketType.DAY:
+ format = 'YYYY-MM-DD'
+ break
+
+ case TimeBucketType.MONTH:
+ format = 'YYYY-MM'
+ break
+
+ case TimeBucketType.YEAR:
+ format = 'YYYY'
+ break
+
+ default:
+ throw new BadRequestException(
+ `The provided time bucket (${timeBucket}) is incorrect`,
+ )
+ }
+
+ const x = []
+
+ while (djsFrom.isSameOrBefore(iterateTo, timeBucket)) {
+ x.push(djsFrom.format(format))
+ djsFrom = djsFrom.add(1, timeBucket)
+ }
+
+ return {
+ x,
+ format,
+ }
+ }
+
+ shiftToTimezone(x: string[], timezone: string, format: string): string[] {
+ return _map(x, (date: string) =>
+ dayjs.utc(date).tz(timezone).format(format),
+ )
+ }
+
+ async groupSessionChartByTimeBucket(
+ timeBucket: TimeBucketType,
+ from: string,
+ to: string,
+ filtersQuery: string,
+ paramsData: any,
+ safeTimezone: string,
+ ): Promise<object | void> {
+ const { x, format } = this.generateUTCXAxis(timeBucket, from, to)
+
+ // if (false) {
+ // const query = this.generateCustomEventsAggregationQuery(
+ // timeBucket,
+ // filtersQuery,
+ // safeTimezone,
+ // ChartRenderMode.PERIODICAL,
+ // )
+
+ // const result = <Array<TrafficCEFilterCHResponse>>(
+ // await clickhouse.query(query, paramsData).toPromise()
+ // )
+
+ // const uniques =
+ // this.extractCustomEventsChartData(result, xShifted)?._unknown_event ||
+ // []
+
+ // const sdur = Array(_size(xShifted)).fill(0)
+
+ // return Promise.resolve({
+ // chart: {
+ // x: xShifted,
+ // visits: uniques,
+ // uniques,
+ // sdur,
+ // },
+ // })
+ // }
+
+ const query = this.generateSessionAggregationQuery(timeBucket, filtersQuery)
+
+ const result = <Array<TrafficCHResponse>>(
+ await clickhouse.query(query, paramsData).toPromise()
+ )
+
+ const { visits, uniques, sdur } = this.extractChartData(result, x)
+
+ return Promise.resolve({
+ chart: {
+ x: this.shiftToTimezone(x, safeTimezone, format),
+ visits,
+ uniques,
+ sdur,
+ },
+ })
+ }
+
async groupErrorsByTimeBucket(
timeBucket: TimeBucketType,
from: string,
@@ -2507,7 +2648,7 @@ export class AnalyticsService {
SELECT
'pageview' AS type,
pg AS value,
- created
+ toTimeZone(analytics.created, '${safeTimezone}') AS created
FROM analytics
WHERE
pid = {pid:FixedString(12)}
@@ -2518,7 +2659,7 @@ export class AnalyticsService {
SELECT
'event' AS type,
ev AS value,
- created
+ toTimeZone(customEV.created, '${safeTimezone}') AS created
FROM customEV
WHERE
pid = {pid:FixedString(12)}
@@ -2575,10 +2716,14 @@ export class AnalyticsService {
let timeBucket = null
if (!_isEmpty(pages)) {
- const from = dayjs(pages[0].created)
+ const from = dayjs
+ .tz(pages[0].created, safeTimezone)
+ .utc()
.startOf('minute')
.format('YYYY-MM-DD HH:mm:ss')
- const to = dayjs(pages[_size(pages) - 1].created)
+ const to = dayjs
+ .tz(pages[_size(pages) - 1].created, safeTimezone)
+ .utc()
.endOf('minute')
.format('YYYY-MM-DD HH:mm:ss')
@@ -2588,7 +2733,7 @@ export class AnalyticsService {
? TimeBucketType.HOUR
: TimeBucketType.MINUTE
- const groupedChart = await this.groupChartByTimeBucket(
+ const groupedChart = await this.groupSessionChartByTimeBucket(
timeBucket,
from,
to,
@@ -2601,8 +2746,6 @@ export class AnalyticsService {
},
},
safeTimezone,
- false,
- ChartRenderMode.PERIODICAL,
)
// @ts-ignore
@@ -2620,9 +2763,6 @@ export class AnalyticsService {
skip = 0,
customEVFilterApplied = false,
): Promise<object | void> {
- const tzFromDate = `toTimeZone(parseDateTimeBestEffort({groupFrom:String}), '${safeTimezone}')`
- const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), '${safeTimezone}')`
-
const analyticsSubquery = `
SELECT
isNotNull(sid) AS active,
@@ -2630,12 +2770,12 @@ export class AnalyticsService {
cc,
os,
br,
- created
+ toTimeZone(analytics.created, '${safeTimezone}') AS created
FROM analytics
WHERE
pid = {pid:FixedString(12)}
AND psid IS NOT NULL
- AND created BETWEEN ${tzFromDate} AND ${tzToDate}
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
${filtersQuery}
`
@@ -2646,12 +2786,12 @@ export class AnalyticsService {
cc,
os,
br,
- created
+ toTimeZone(customEV.created, '${safeTimezone}') AS created
FROM customEV
WHERE
pid = {pid:FixedString(12)}
AND psid IS NOT NULL
- AND created BETWEEN ${tzFromDate} AND ${tzToDate}
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
${filtersQuery}
`