Skip to content

πŸ“ˆ Open source, privacy-first web analytics β€” a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

(feature) Session overview / Clickstream (#207)

Andrii Romasiun8510fb43cc6b
diff --git a/apps/production/src/analytics/analytics.controller.ts b/apps/production/src/analytics/analytics.controller.ts
index b3acb1299..b81f22d19 100644
--- a/apps/production/src/analytics/analytics.controller.ts
+++ b/apps/production/src/analytics/analytics.controller.ts
@@ -39,6 +39,7 @@ import {
getHeartbeatKey,
DataType,
validPeriods,
+ getLowestPossibleTimeBucket,
} from './analytics.service'
import { TaskManagerService } from '../task-manager/task-manager.service'
import { CurrentUserId } from '../auth/decorators/current-user-id.decorator'
@@ -74,6 +75,8 @@ import {
IGetFunnel,
IUserFlow,
} from './interfaces'
+import { GetSessionsDto } from './dto/get-sessions.dto'
+import { GetSessionDto } from './dto/get-session.dto'
// eslint-disable-next-line @typescript-eslint/no-var-requires
const mysql = require('mysql2')
@@ -1333,6 +1336,123 @@ export class AnalyticsController {
return res.end(TRANSPARENT_GIF_BUFFER, 'binary')
}
+ @Get('sessions')
+ @Auth([], true, true)
+ async getSessions(
+ @Query() data: GetSessionsDto,
+ @CurrentUserId() uid: string,
+ @Headers() headers: { 'x-password'?: string },
+ ): Promise<any> {
+ const { pid, period, from, to, filters, timezone = DEFAULT_TIMEZONE } = data
+ this.analyticsService.validatePID(pid)
+
+ if (!_isEmpty(period)) {
+ this.analyticsService.validatePeriod(period)
+ }
+
+ await this.analyticsService.checkProjectAccess(
+ pid,
+ uid,
+ headers['x-password'],
+ )
+
+ const take = this.analyticsService.getSafeNumber(data.take, 30)
+ const skip = this.analyticsService.getSafeNumber(data.skip, 0)
+
+ if (take > 150) {
+ throw new BadRequestException(
+ 'The maximum number of sessions to return is 150',
+ )
+ }
+
+ let timeBucket
+ let diff
+
+ if (period === 'all') {
+ const res = await this.analyticsService.getTimeBucketForAllTime(
+ pid,
+ period,
+ timezone,
+ )
+
+ // eslint-disable-next-line prefer-destructuring
+ timeBucket = res.timeBucket[0]
+ diff = res.diff
+ } else {
+ timeBucket = getLowestPossibleTimeBucket(period, from, to)
+ }
+
+ this.analyticsService.validateTimebucket(timeBucket)
+ const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] =
+ this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)
+
+ const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
+ const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
+ from,
+ to,
+ timeBucket,
+ period,
+ safeTimezone,
+ diff,
+ )
+
+ const paramsData = {
+ params: {
+ pid,
+ groupFrom,
+ groupTo,
+ ...filtersParams,
+ },
+ }
+
+ const sessions = await this.analyticsService.getSessionsList(
+ filtersQuery,
+ paramsData,
+ safeTimezone,
+ take,
+ skip,
+ customEVFilterApplied,
+ )
+
+ return {
+ sessions,
+ appliedFilters,
+ take,
+ skip,
+ }
+ }
+
+ @Get('session')
+ @Auth([], true, true)
+ async getSession(
+ @Query() data: GetSessionDto,
+ @CurrentUserId() uid: string,
+ @Headers() headers: { 'x-password'?: string },
+ ): Promise<any> {
+ const {
+ pid,
+ psid,
+ timezone = DEFAULT_TIMEZONE,
+ //
+ } = data
+ this.analyticsService.validatePID(pid)
+
+ await this.analyticsService.checkProjectAccess(
+ pid,
+ uid,
+ headers['x-password'],
+ )
+ const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
+
+ const result = await this.analyticsService.getSessionDetails(
+ pid,
+ psid,
+ safeTimezone,
+ )
+
+ return result
+ }
+
@Get('custom-events')
@Auth([], true, true)
async getCustomEvents(
diff --git a/apps/production/src/analytics/analytics.service.ts b/apps/production/src/analytics/analytics.service.ts
index de82e2c98..6a2e3f7ab 100644
--- a/apps/production/src/analytics/analytics.service.ts
+++ b/apps/production/src/analytics/analytics.service.ts
@@ -6,6 +6,7 @@ import * as _size from 'lodash/size'
import * as _includes from 'lodash/includes'
import * as _map from 'lodash/map'
import * as _toUpper from 'lodash/toUpper'
+import * as _head from 'lodash/head'
import * as _join from 'lodash/join'
import * as _isArray from 'lodash/isArray'
import * as _sortBy from 'lodash/sortBy'
@@ -86,6 +87,7 @@ import {
IOverall,
IOverallCaptcha,
IOverallPerformance,
+ IPageflow,
} from './interfaces'
dayjs.extend(utc)
@@ -206,6 +208,53 @@ const checkIfTBAllowed = (
}
}
+export const getLowestPossibleTimeBucket = (
+ period?: string,
+ from?: string,
+ to?: string,
+): TimeBucketType => {
+ if (!from && !to) {
+ if (!_includes(validPeriods, period)) {
+ throw new UnprocessableEntityException('The provided period is incorrect')
+ }
+
+ if (period === '1h') {
+ return TimeBucketType.MINUTE
+ }
+
+ if (
+ period === 'today' ||
+ period === 'yesterday' ||
+ period === '1d' ||
+ period === '7d'
+ ) {
+ return TimeBucketType.HOUR
+ }
+
+ if (period === '4w') {
+ return TimeBucketType.DAY
+ }
+
+ if (period === '3M' || period === '12M' || period === '24M') {
+ return TimeBucketType.MONTH
+ }
+
+ return TimeBucketType.YEAR
+ }
+
+ const diff = dayjs(to).diff(dayjs(from), 'days')
+
+ const tbMap = _find(timeBucketToDays, ({ lt }) => diff <= lt)
+
+ if (_isEmpty(tbMap)) {
+ throw new PreconditionFailedException(
+ "The difference between 'from' and 'to' is greater than allowed",
+ )
+ }
+
+ return _head(tbMap.tb)
+}
+
const generateParamsQuery = (
col: string,
subQuery: string,
@@ -736,7 +785,7 @@ export class AnalyticsService {
const from: any = await clickhouse
.query(
- 'SELECT created as from FROM analytics where pid = {pid:FixedString(12)} ORDER BY created ASC LIMIT 1',
+ 'SELECT created FROM analytics where pid = {pid:FixedString(12)} ORDER BY created ASC LIMIT 1',
{ params: { pid } },
)
.toPromise()
@@ -748,7 +797,7 @@ export class AnalyticsService {
let diff = null
if (from && to) {
- diff = dayjs(to).diff(dayjs(from[0].from), 'days')
+ diff = dayjs(to).diff(dayjs(from?.[0].created || to), 'days')
const tbMap = _find(timeBucketToDays, ({ lt }) => diff <= lt)
@@ -758,7 +807,7 @@ export class AnalyticsService {
)
}
- newTimeBucket = tbMap.tb
+ newTimeBucket = tbMap?.tb
}
return {
@@ -2563,6 +2612,201 @@ export class AnalyticsService {
})
}
+ getSafeNumber(value: any, defaultValue: number): number {
+ if (typeof value === 'undefined') {
+ return defaultValue
+ }
+
+ const parsed = parseInt(value, 10)
+
+ if (Number.isNaN(parsed)) {
+ return defaultValue
+ }
+
+ return parsed
+ }
+
+ async getSessionDetails(
+ pid: string,
+ psid: string,
+ safeTimezone: string,
+ ): Promise<any> {
+ const queryPages = `
+ SELECT
+ *
+ FROM (
+ SELECT
+ 'pageview' AS type,
+ pg AS value,
+ created
+ FROM analytics
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND psid = {psid:String}
+
+ UNION ALL
+
+ SELECT
+ 'event' AS type,
+ ev AS value,
+ created
+ FROM customEV
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND psid = {psid:String}
+ )
+ ORDER BY created ASC;
+ `
+
+ const querySessionDetails = `
+ SELECT
+ dv, br, os, lc, ref, so, me, ca, cc, rg, ct, sdur
+ FROM analytics
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND psid = {psid:String}
+ AND unique = 1
+ LIMIT 1;
+ `
+
+ const paramsData = {
+ params: {
+ pid,
+ psid,
+ },
+ }
+
+ const pages = <IPageflow[]>(
+ await clickhouse.query(queryPages, paramsData).toPromise()
+ )
+ let details = (
+ await clickhouse.query(querySessionDetails, paramsData).toPromise()
+ )[0]
+
+ if (!details) {
+ const querySessionDetailsBackup = `
+ SELECT
+ dv, br, os, lc, ref, so, me, ca, cc, rg, ct, sdur
+ FROM analytics
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND psid = {psid:String}
+ LIMIT 1;
+ `
+
+ // eslint-disable-next-line prefer-destructuring
+ details = (
+ await clickhouse
+ .query(querySessionDetailsBackup, paramsData)
+ .toPromise()
+ )[0]
+ }
+
+ let chartData = {}
+ let timeBucket = null
+
+ if (!_isEmpty(pages)) {
+ const from = dayjs(pages[0].created)
+ .startOf('minute')
+ .format('YYYY-MM-DD HH:mm:ss')
+ const to = dayjs(pages[_size(pages) - 1].created)
+ .endOf('minute')
+ .format('YYYY-MM-DD HH:mm:ss')
+
+ // eslint-disable-next-line
+ timeBucket = dayjs(to).diff(dayjs(from), 'hour') > 1 ? TimeBucketType.HOUR : TimeBucketType.MINUTE
+
+ const groupedChart = await this.groupChartByTimeBucket(
+ timeBucket,
+ from,
+ to,
+ 'AND psid = {psid:String}',
+ {
+ params: {
+ ...paramsData.params,
+ groupFrom: from,
+ groupTo: to,
+ },
+ },
+ safeTimezone,
+ false,
+ ChartRenderMode.PERIODICAL,
+ )
+
+ // @ts-ignore
+ chartData = groupedChart.chart
+ }
+
+ return { pages, details, psid, chart: chartData, timeBucket }
+ }
+
+ async getSessionsList(
+ filtersQuery: string,
+ paramsData: object,
+ safeTimezone: string,
+ take = 30,
+ skip = 0,
+ customEVFilterApplied = false,
+ ): Promise<object | void> {
+ const tzFromDate = `toTimeZone(parseDateTimeBestEffort({groupFrom:String}), '${safeTimezone}')`
+ const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), '${safeTimezone}')`
+
+ const analyticsSubquery = `
+ SELECT
+ isNotNull(sid) AS active,
+ CAST(psid, 'String') AS psidCasted,
+ cc,
+ os,
+ br,
+ created
+ FROM analytics
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND psid IS NOT NULL
+ AND created BETWEEN ${tzFromDate} AND ${tzToDate}
+ ${filtersQuery}
+ `
+
+ const customEVSubquery = `
+ SELECT
+ 0 AS active,
+ CAST(psid, 'String') AS psidCasted,
+ cc,
+ os,
+ br,
+ created
+ FROM customEV
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND psid IS NOT NULL
+ AND created BETWEEN ${tzFromDate} AND ${tzToDate}
+ ${filtersQuery}
+ `
+
+ const query = `
+ SELECT
+ psidCasted AS psid,
+ any(active) AS active,
+ any(cc) AS cc,
+ any(os) AS os,
+ any(br) AS br,
+ count() AS pageviews,
+ min(created) AS created
+ FROM
+ (
+ ${customEVFilterApplied ? customEVSubquery : analyticsSubquery}
+ )
+ GROUP BY psid
+ ORDER BY created DESC
+ LIMIT ${take}
+ OFFSET ${skip}
+ `
+
+ const result = await clickhouse.query(query, paramsData).toPromise()
+
+ return result
+ }
+
async getOnlineCountByProjectId(projectId: string) {
// @ts-ignore
return redis.countKeysByPattern(`hb:${projectId}:*`)
diff --git a/apps/production/src/analytics/dto/get-session.dto.ts b/apps/production/src/analytics/dto/get-session.dto.ts
new file mode 100644
index 000000000..9782d45fd
--- /dev/null
+++ b/apps/production/src/analytics/dto/get-session.dto.ts
@@ -0,0 +1,20 @@
+import { ApiProperty } from '@nestjs/swagger'
+import { IsNotEmpty } from 'class-validator'
+import { DEFAULT_TIMEZONE } from '../../user/entities/user.entity'
+
+// eslint-disable-next-line @typescript-eslint/naming-convention
+export class GetSessionDto {
+ @ApiProperty()
+ @IsNotEmpty()
+ pid: string
+
+ @ApiProperty()
+ @IsNotEmpty()
+ psid: string
+
+ @ApiProperty({
+ description: 'Timezone to display data in',
+ default: DEFAULT_TIMEZONE,
+ })
+ timezone: string
+}
diff --git a/apps/production/src/analytics/dto/get-sessions.dto.ts b/apps/production/src/analytics/dto/get-sessions.dto.ts
new file mode 100644
index 000000000..55c7358d7
--- /dev/null
+++ b/apps/production/src/analytics/dto/get-sessions.dto.ts
@@ -0,0 +1,15 @@
+import { PickType } from '@nestjs/swagger'
+import { AnalyticsGET_DTO } from './getData.dto'
+
+export class GetSessionsDto extends PickType(AnalyticsGET_DTO, [
+ 'pid',
+ 'period',
+ 'from',
+ 'to',
+ 'filters',
+ 'timezone',
+] as const) {
+ take: number
+
+ skip: number
+}
diff --git a/apps/production/src/analytics/interfaces/index.ts b/apps/production/src/analytics/interfaces/index.ts
index 1db51a665..b543f220d 100644
--- a/apps/production/src/analytics/interfaces/index.ts
+++ b/apps/production/src/analytics/interfaces/index.ts
@@ -178,3 +178,9 @@ export interface IOverallCaptcha {
export interface IOverallPerformance {
[key: string]: IPerformanceObject
}
+
+export interface IPageflow {
+ type: 'pageview' | 'event'
+ value: string
+ created: string
+}
diff --git a/apps/production/src/task-manager/task-manager.service.ts b/apps/production/src/task-manager/task-manager.service.ts
index 1453542a5..7304da4e4 100644
--- a/apps/production/src/task-manager/task-manager.service.ts
+++ b/apps/production/src/task-manager/task-manager.service.ts
@@ -874,8 +874,8 @@ export class TaskManagerService {
alert.queryMetric === QueryMetric.CUSTOM_EVENTS
? 'custom events'
: alert.queryMetric === QueryMetric.UNIQUE_PAGE_VIEWS
- ? 'unique page views'
- : 'page views'
+ ? 'unique page views'
+ : 'page views'
const text = `πŸ”” Alert *${alert.name}* got triggered!\nYour project *${
project.name
}* has had *${count}*${
diff --git a/apps/selfhosted/src/analytics/analytics.controller.ts b/apps/selfhosted/src/analytics/analytics.controller.ts
index 2f00f04e5..499762a58 100644
--- a/apps/selfhosted/src/analytics/analytics.controller.ts
+++ b/apps/selfhosted/src/analytics/analytics.controller.ts
@@ -41,6 +41,7 @@ import {
getHeartbeatKey,
validPeriods,
DataType,
+ getLowestPossibleTimeBucket,
} from './analytics.service'
import { CurrentUserId } from '../auth/decorators/current-user-id.decorator'
import { DEFAULT_TIMEZONE } from '../user/entities/user.entity'
@@ -74,6 +75,8 @@ import {
IGetFunnel,
IUserFlow,
} from './interfaces'
+import { GetSessionsDto } from './dto/get-sessions.dto'
+import { GetSessionDto } from './dto/get-session.dto'
// eslint-disable-next-line @typescript-eslint/no-var-requires
const mysql = require('mysql2')
@@ -554,12 +557,6 @@ export class AnalyticsController {
)
await this.analyticsService.checkProjectAccess(pid, uid)
- let subQuery = `FROM analytics WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
-
- if (customEVFilterApplied) {
- subQuery = `FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
- }
-
const paramsData = {
params: {
pid,
@@ -573,7 +570,6 @@ export class AnalyticsController {
timeBucket,
groupFrom,
groupTo,
- subQuery,
filtersQuery,
paramsData,
safeTimezone,
@@ -1209,6 +1205,113 @@ export class AnalyticsController {
return res.end(TRANSPARENT_GIF_BUFFER, 'binary')
}
+ @Get('sessions')
+ @Auth([], true, true)
+ async getSessions(
+ @Query() data: GetSessionsDto,
+ @CurrentUserId() uid: string,
+ ): Promise<any> {
+ const { pid, period, from, to, filters, timezone = DEFAULT_TIMEZONE } = data
+ this.analyticsService.validatePID(pid)
+
+ if (!_isEmpty(period)) {
+ this.analyticsService.validatePeriod(period)
+ }
+
+ await this.analyticsService.checkProjectAccess(pid, uid)
+
+ const take = this.analyticsService.getSafeNumber(data.take, 30)
+ const skip = this.analyticsService.getSafeNumber(data.skip, 0)
+
+ if (take > 150) {
+ throw new BadRequestException(
+ 'The maximum number of sessions to return is 150',
+ )
+ }
+
+ let timeBucket
+ let diff
+
+ if (period === 'all') {
+ const res = await this.analyticsService.getTimeBucketForAllTime(
+ pid,
+ period,
+ timezone,
+ )
+
+ // eslint-disable-next-line prefer-destructuring
+ timeBucket = res.timeBucket[0]
+ diff = res.diff
+ } else {
+ timeBucket = getLowestPossibleTimeBucket(period, from, to)
+ }
+
+ this.analyticsService.validateTimebucket(timeBucket)
+ const [filtersQuery, filtersParams, appliedFilters, customEVFilterApplied] =
+ this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)
+
+ const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
+ const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
+ from,
+ to,
+ timeBucket,
+ period,
+ safeTimezone,
+ diff,
+ )
+
+ const paramsData = {
+ params: {
+ pid,
+ groupFrom,
+ groupTo,
+ ...filtersParams,
+ },
+ }
+
+ const sessions = await this.analyticsService.getSessionsList(
+ filtersQuery,
+ paramsData,
+ safeTimezone,
+ take,
+ skip,
+ customEVFilterApplied,
+ )
+
+ return {
+ sessions,
+ appliedFilters,
+ take,
+ skip,
+ }
+ }
+
+ @Get('session')
+ @Auth([], true, true)
+ async getSession(
+ @Query() data: GetSessionDto,
+ @CurrentUserId() uid: string,
+ ): Promise<any> {
+ const {
+ pid,
+ psid,
+ timezone = DEFAULT_TIMEZONE,
+ //
+ } = data
+ this.analyticsService.validatePID(pid)
+
+ await this.analyticsService.checkProjectAccess(pid, uid)
+ const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
+
+ const result = await this.analyticsService.getSessionDetails(
+ pid,
+ psid,
+ safeTimezone,
+ )
+
+ return result
+ }
+
@Get('custom-events')
@Auth([], true, true)
async getCustomEvents(
diff --git a/apps/selfhosted/src/analytics/analytics.service.ts b/apps/selfhosted/src/analytics/analytics.service.ts
index 1c6d31f63..af3f45fbd 100644
--- a/apps/selfhosted/src/analytics/analytics.service.ts
+++ b/apps/selfhosted/src/analytics/analytics.service.ts
@@ -5,6 +5,7 @@ import * as _reverse from 'lodash/reverse'
import * as _size from 'lodash/size'
import * as _includes from 'lodash/includes'
import * as _map from 'lodash/map'
+import * as _head from 'lodash/head'
import * as _toUpper from 'lodash/toUpper'
import * as _sortBy from 'lodash/sortBy'
import * as _join from 'lodash/join'
@@ -72,6 +73,7 @@ import {
IFunnel,
IOverall,
IOverallPerformance,
+ IPageflow,
} from './interfaces'
dayjs.extend(utc)
@@ -168,6 +170,53 @@ export const isValidDate = (date: string, format = 'YYYY-MM-DD'): boolean => {
return dayjs(date, format).format(format) === date
}
+export const getLowestPossibleTimeBucket = (
+ period?: string,
+ from?: string,
+ to?: string,
+): TimeBucketType => {
+ if (!from && !to) {
+ if (!_includes(validPeriods, period)) {
+ throw new UnprocessableEntityException('The provided period is incorrect')
+ }
+
+ if (period === '1h') {
+ return TimeBucketType.MINUTE
+ }
+
+ if (
+ period === 'today' ||
+ period === 'yesterday' ||
+ period === '1d' ||
+ period === '7d'
+ ) {
+ return TimeBucketType.HOUR
+ }
+
+ if (period === '4w') {
+ return TimeBucketType.DAY
+ }
+
+ if (period === '3M' || period === '12M' || period === '24M') {
+ return TimeBucketType.MONTH
+ }
+
+ return TimeBucketType.YEAR
+ }
+
+ const diff = dayjs(to).diff(dayjs(from), 'days')
+
+ const tbMap = _find(timeBucketToDays, ({ lt }) => diff <= lt)
+
+ if (_isEmpty(tbMap)) {
+ throw new PreconditionFailedException(
+ "The difference between 'from' and 'to' is greater than allowed",
+ )
+ }
+
+ return _head(tbMap.tb)
+}
+
const checkIfTBAllowed = (
timeBucket: TimeBucketType,
from: string,
@@ -636,7 +685,7 @@ export class AnalyticsService {
const from: any = await clickhouse
.query(
- 'SELECT created as from FROM analytics where pid = {pid:FixedString(12)} ORDER BY created ASC LIMIT 1',
+ 'SELECT created FROM analytics where pid = {pid:FixedString(12)} ORDER BY created ASC LIMIT 1',
{ params: { pid } },
)
.toPromise()
@@ -648,7 +697,7 @@ export class AnalyticsService {
let diff = null
if (from && to) {
- diff = dayjs(to).diff(dayjs(from[0].from), 'days')
+ diff = dayjs(to).diff(dayjs(from?.[0].created || to), 'days')
const tbMap = _find(timeBucketToDays, ({ lt }) => diff <= lt)
@@ -658,7 +707,7 @@ export class AnalyticsService {
)
}
- newTimeBucket = tbMap.tb
+ newTimeBucket = tbMap?.tb
}
return {
@@ -1785,7 +1834,6 @@ export class AnalyticsService {
): Promise<object | void> {
let params: unknown = {}
let chart: unknown = {}
- let avgSdur = 0
const promises = [
// Getting params
@@ -1810,7 +1858,6 @@ export class AnalyticsService {
timeBucket,
from,
to,
- subQuery,
filtersQuery,
paramsData,
safeTimezone,
@@ -1820,9 +1867,6 @@ export class AnalyticsService {
// @ts-ignore
chart = groupedData.chart
-
- // @ts-ignore
- avgSdur = groupedData.avgSdur
})(),
]
@@ -1831,7 +1875,6 @@ export class AnalyticsService {
return {
params,
chart,
- avgSdur,
}
}
@@ -1854,17 +1897,12 @@ export class AnalyticsService {
timeBucket: TimeBucketType,
from: string,
to: string,
- subQuery: string,
filtersQuery: string,
paramsData: any,
safeTimezone: string,
customEVFilterApplied: boolean,
mode: ChartRenderMode,
): Promise<object | void> {
- const avgSdur = customEVFilterApplied
- ? 0
- : await this.calculateAverageSessionDuration(subQuery, paramsData)
-
const { xShifted } = this.generateXAxis(timeBucket, from, to, safeTimezone)
if (customEVFilterApplied) {
@@ -1892,9 +1930,9 @@ export class AnalyticsService {
uniques,
sdur,
},
- avgSdur,
})
}
+
const query = this.generateAnalyticsAggregationQuery(
timeBucket,
filtersQuery,
@@ -1915,7 +1953,6 @@ export class AnalyticsService {
uniques,
sdur,
},
- avgSdur,
})
}
@@ -2182,6 +2219,201 @@ export class AnalyticsService {
})
}
+ getSafeNumber(value: any, defaultValue: number): number {
+ if (typeof value === 'undefined') {
+ return defaultValue
+ }
+
+ const parsed = parseInt(value, 10)
+
+ if (Number.isNaN(parsed)) {
+ return defaultValue
+ }
+
+ return parsed
+ }
+
+ async getSessionDetails(
+ pid: string,
+ psid: string,
+ safeTimezone: string,
+ ): Promise<any> {
+ const queryPages = `
+ SELECT
+ *
+ FROM (
+ SELECT
+ 'pageview' AS type,
+ pg AS value,
+ created
+ FROM analytics
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND psid = {psid:String}
+
+ UNION ALL
+
+ SELECT
+ 'event' AS type,
+ ev AS value,
+ created
+ FROM customEV
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND psid = {psid:String}
+ )
+ ORDER BY created ASC;
+ `
+
+ const querySessionDetails = `
+ SELECT
+ dv, br, os, lc, ref, so, me, ca, cc, rg, ct, sdur
+ FROM analytics
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND psid = {psid:String}
+ AND unique = 1
+ LIMIT 1;
+ `
+
+ const paramsData = {
+ params: {
+ pid,
+ psid,
+ },
+ }
+
+ const pages = <IPageflow[]>(
+ await clickhouse.query(queryPages, paramsData).toPromise()
+ )
+ let details = (
+ await clickhouse.query(querySessionDetails, paramsData).toPromise()
+ )[0]
+
+ if (!details) {
+ const querySessionDetailsBackup = `
+ SELECT
+ dv, br, os, lc, ref, so, me, ca, cc, rg, ct, sdur
+ FROM analytics
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND psid = {psid:String}
+ LIMIT 1;
+ `
+
+ // eslint-disable-next-line prefer-destructuring
+ details = (
+ await clickhouse
+ .query(querySessionDetailsBackup, paramsData)
+ .toPromise()
+ )[0]
+ }
+
+ let chartData = {}
+ let timeBucket = null
+
+ if (!_isEmpty(pages)) {
+ const from = dayjs(pages[0].created)
+ .startOf('minute')
+ .format('YYYY-MM-DD HH:mm:ss')
+ const to = dayjs(pages[_size(pages) - 1].created)
+ .endOf('minute')
+ .format('YYYY-MM-DD HH:mm:ss')
+
+ // eslint-disable-next-line
+ timeBucket = dayjs(to).diff(dayjs(from), 'hour') > 1 ? TimeBucketType.HOUR : TimeBucketType.MINUTE
+
+ const groupedChart = await this.groupChartByTimeBucket(
+ timeBucket,
+ from,
+ to,
+ 'AND psid = {psid:String}',
+ {
+ params: {
+ ...paramsData.params,
+ groupFrom: from,
+ groupTo: to,
+ },
+ },
+ safeTimezone,
+ false,
+ ChartRenderMode.PERIODICAL,
+ )
+
+ // @ts-ignore
+ chartData = groupedChart.chart
+ }
+
+ return { pages, details, psid, chart: chartData, timeBucket }
+ }
+
+ async getSessionsList(
+ filtersQuery: string,
+ paramsData: object,
+ safeTimezone: string,
+ take = 30,
+ skip = 0,
+ customEVFilterApplied = false,
+ ): Promise<object | void> {
+ const tzFromDate = `toTimeZone(parseDateTimeBestEffort({groupFrom:String}), '${safeTimezone}')`
+ const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), '${safeTimezone}')`
+
+ const analyticsSubquery = `
+ SELECT
+ isNotNull(sid) AS active,
+ CAST(psid, 'String') AS psidCasted,
+ cc,
+ os,
+ br,
+ created
+ FROM analytics
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND psid IS NOT NULL
+ AND created BETWEEN ${tzFromDate} AND ${tzToDate}
+ ${filtersQuery}
+ `
+
+ const customEVSubquery = `
+ SELECT
+ 0 AS active,
+ CAST(psid, 'String') AS psidCasted,
+ cc,
+ os,
+ br,
+ created
+ FROM customEV
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND psid IS NOT NULL
+ AND created BETWEEN ${tzFromDate} AND ${tzToDate}
+ ${filtersQuery}
+ `
+
+ const query = `
+ SELECT
+ psidCasted AS psid,
+ any(active) AS active,
+ any(cc) AS cc,
+ any(os) AS os,
+ any(br) AS br,
+ count() AS pageviews,
+ min(created) AS created
+ FROM
+ (
+ ${customEVFilterApplied ? customEVSubquery : analyticsSubquery}
+ )
+ GROUP BY psid
+ ORDER BY created DESC
+ LIMIT ${take}
+ OFFSET ${skip}
+ `
+
+ const result = await clickhouse.query(query, paramsData).toPromise()
+
+ return result
+ }
+
async getOnlineCountByProjectId(projectId: string) {
// @ts-ignore
return redis.countKeysByPattern(`hb:${projectId}:*`)
diff --git a/apps/selfhosted/src/analytics/dto/get-session.dto.ts b/apps/selfhosted/src/analytics/dto/get-session.dto.ts
new file mode 100644
index 000000000..9782d45fd
--- /dev/null
+++ b/apps/selfhosted/src/analytics/dto/get-session.dto.ts
@@ -0,0 +1,20 @@
+import { ApiProperty } from '@nestjs/swagger'
+import { IsNotEmpty } from 'class-validator'
+import { DEFAULT_TIMEZONE } from '../../user/entities/user.entity'
+
+// eslint-disable-next-line @typescript-eslint/naming-convention
+export class GetSessionDto {
+ @ApiProperty()
+ @IsNotEmpty()
+ pid: string
+
+ @ApiProperty()
+ @IsNotEmpty()
+ psid: string
+
+ @ApiProperty({
+ description: 'Timezone to display data in',
+ default: DEFAULT_TIMEZONE,
+ })
+ timezone: string
+}
diff --git a/apps/selfhosted/src/analytics/dto/get-sessions.dto.ts b/apps/selfhosted/src/analytics/dto/get-sessions.dto.ts
new file mode 100644
index 000000000..55c7358d7
--- /dev/null
+++ b/apps/selfhosted/src/analytics/dto/get-sessions.dto.ts
@@ -0,0 +1,15 @@
+import { PickType } from '@nestjs/swagger'
+import { AnalyticsGET_DTO } from './getData.dto'
+
+export class GetSessionsDto extends PickType(AnalyticsGET_DTO, [
+ 'pid',
+ 'period',
+ 'from',
+ 'to',
+ 'filters',
+ 'timezone',
+] as const) {
+ take: number
+
+ skip: number
+}
diff --git a/apps/selfhosted/src/analytics/interfaces/index.ts b/apps/selfhosted/src/analytics/interfaces/index.ts
index 98ba0edec..d487b75e7 100644
--- a/apps/selfhosted/src/analytics/interfaces/index.ts
+++ b/apps/selfhosted/src/analytics/interfaces/index.ts
@@ -164,3 +164,9 @@ export interface IOverall {
export interface IOverallPerformance {
[key: string]: IPerformanceObject
}
+
+export interface IPageflow {
+ type: 'pageview' | 'event'
+ value: string
+ created: string
+}