Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

(feature) Funnels (#198)

Andrii Romasiun3e17222efc53
diff --git a/apps/production/src/analytics/analytics.controller.ts b/apps/production/src/analytics/analytics.controller.ts
index af7e2cd54..e1965fc2e 100644
--- a/apps/production/src/analytics/analytics.controller.ts
+++ b/apps/production/src/analytics/analytics.controller.ts
@@ -4,6 +4,9 @@ import * as _toNumber from 'lodash/toNumber'
import * as _pick from 'lodash/pick'
import * as _includes from 'lodash/includes'
import * as _keys from 'lodash/keys'
+import * as _size from 'lodash/size'
+import * as _some from 'lodash/some'
+import * as _isString from 'lodash/isString'
import * as _values from 'lodash/values'
import * as _map from 'lodash/map'
import * as _uniqBy from 'lodash/uniqBy'
@@ -49,6 +52,7 @@ import { EventsDTO } from './dto/events.dto'
import { AnalyticsGET_DTO, ChartRenderMode } from './dto/getData.dto'
import { GetCustomEventMetadata } from './dto/get-custom-event-meta.dto'
import { GetUserFlowDTO } from './dto/getUserFlow.dto'
+import { GetFunnelsDTO } from './dto/getFunnels.dto'
import { AppLoggerService } from '../logger/logger.service'
import {
REDIS_LOG_DATA_CACHE_KEY,
@@ -60,6 +64,8 @@ import {
REDIS_PROJECTS_COUNT_KEY,
REDIS_EVENTS_COUNT_KEY,
REDIS_SESSION_SALT_KEY,
+ MIN_PAGES_IN_FUNNEL,
+ MAX_PAGES_IN_FUNNEL,
clickhouse,
} from '../common/constants'
import { getGeoDetails, getIPFromHeaders } from '../common/utils'
@@ -67,7 +73,12 @@ import { BotDetection } from '../common/decorators/bot-detection.decorator'
import { BotDetectionGuard } from '../common/guards/bot-detection.guard'
import { GetCustomEventsDto } from './dto/get-custom-events.dto'
import { GetFiltersDto } from './dto/get-filters.dto'
-import { IAggregatedMetadata, IUserFlow } from './interfaces'
+import {
+ IAggregatedMetadata,
+ IFunnel,
+ IGetFunnel,
+ IUserFlow,
+} from './interfaces'
// eslint-disable-next-line @typescript-eslint/no-var-requires
const mysql = require('mysql2')
@@ -84,6 +95,7 @@ const getSessionKeyCustom = (
) => `cses_${hash(`${ua}${ip}${pid}${ev}${salt}`).toString('hex')}`
const analyticsDTO = (
+ psid: string,
sid: string,
pid: string,
pg: string,
@@ -103,6 +115,7 @@ const analyticsDTO = (
unique: number,
): Array<string | number> => {
return [
+ psid,
sid,
pid,
pg,
@@ -162,6 +175,7 @@ const performanceDTO = (
}
const customLogDTO = (
+ psid: string,
pid: string,
ev: string,
pg: string,
@@ -180,6 +194,7 @@ const customLogDTO = (
values: string[],
): Array<string | number | string[]> => {
return [
+ psid,
pid,
ev,
pg,
@@ -264,6 +279,44 @@ const getPIDsArray = (pids, pid) => {
return pids
}
+const getPagesArray = (rawPages: string): string[] => {
+ try {
+ const pages = JSON.parse(rawPages)
+
+ if (!_isArray(pages)) {
+ throw new UnprocessableEntityException(
+ 'An array of pages has to be provided as a pages param',
+ )
+ }
+
+ const size = _size(pages)
+
+ if (size < MIN_PAGES_IN_FUNNEL) {
+ throw new UnprocessableEntityException(
+ `A minimum of ${MIN_PAGES_IN_FUNNEL} pages or events has to be provided`,
+ )
+ }
+
+ if (size > MAX_PAGES_IN_FUNNEL) {
+ throw new UnprocessableEntityException(
+ `A maximum of ${MAX_PAGES_IN_FUNNEL} pages or events can be provided`,
+ )
+ }
+
+ if (_some(pages, page => !_isString(page))) {
+ throw new UnprocessableEntityException(
+ 'Pages array must contain string values only',
+ )
+ }
+
+ return pages
+ } catch (e) {
+ throw new UnprocessableEntityException(
+ 'Cannot process the provided array of pages',
+ )
+ }
+}
+
// needed for serving 1x1 px GIF
const TRANSPARENT_GIF_BUFFER = Buffer.from(
'R0lGODlhAQABAIAAAP///wAAACwAAAAAAQABAAACAkQBADs=',
@@ -417,6 +470,67 @@ export class AnalyticsController {
}
}
+ @Get('funnel')
+ @Auth([], true, true)
+ async getFunnel(
+ @Query() data: GetFunnelsDTO,
+ @CurrentUserId() uid: string,
+ @Headers() headers: { 'x-password'?: string },
+ ): Promise<IGetFunnel> {
+ const { pid, period, from, to, timezone = DEFAULT_TIMEZONE, pages } = data
+ this.analyticsService.validatePID(pid)
+
+ if (!_isEmpty(period)) {
+ this.analyticsService.validatePeriod(period)
+ }
+
+ const pagesArr = getPagesArray(pages)
+
+ await this.analyticsService.checkProjectAccess(
+ pid,
+ uid,
+ headers['x-password'],
+ )
+
+ const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
+ const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
+ from,
+ to,
+ null,
+ period,
+ safeTimezone,
+ )
+
+ const params = {
+ pid,
+ groupFrom,
+ groupTo,
+ }
+
+ let funnel: IFunnel[] = []
+ let totalPageviews: number = 0
+
+ const promises = [
+ (async () => {
+ funnel = await this.analyticsService.getFunnel(pagesArr, params)
+ })(),
+ (async () => {
+ totalPageviews = await this.analyticsService.getTotalPageviews(
+ pid,
+ groupFrom,
+ groupTo,
+ )
+ })(),
+ ]
+
+ await Promise.all(promises)
+
+ return {
+ funnel,
+ totalPageviews,
+ }
+ }
+
@Get('meta')
@Auth([], true, true)
async getCustomEventMetadata(
@@ -918,8 +1032,9 @@ export class AnalyticsController {
this.analyticsService.validateCustomEVMeta(eventsDTO.meta)
await this.analyticsService.validate(eventsDTO, origin, 'custom', ip)
+ const salt = await redis.get(REDIS_SESSION_SALT_KEY)
+
if (eventsDTO.unique) {
- const salt = await redis.get(REDIS_SESSION_SALT_KEY)
const sessionHash = getSessionKeyCustom(
ip,
userAgent,
@@ -927,7 +1042,7 @@ export class AnalyticsController {
eventsDTO.ev,
salt,
)
- const unique = await this.analyticsService.isUnique(sessionHash)
+ const [unique] = await this.analyticsService.isUnique(sessionHash)
if (!unique) {
throw new ForbiddenException(
@@ -947,7 +1062,11 @@ export class AnalyticsController {
const br = ua.browser.name
const os = ua.os.name
+ const sessionHash = getSessionKey(ip, userAgent, eventsDTO.pid, salt)
+ const [, psid] = await this.analyticsService.isUnique(sessionHash)
+
const dto = customLogDTO(
+ psid,
eventsDTO.pid,
eventsDTO.ev,
eventsDTO.pg,
@@ -1023,7 +1142,7 @@ export class AnalyticsController {
const salt = await redis.get(REDIS_SESSION_SALT_KEY)
const sessionHash = getSessionKey(ip, userAgent, logDTO.pid, salt)
- const unique = await this.analyticsService.isUnique(sessionHash)
+ const [unique, psid] = await this.analyticsService.isUnique(sessionHash)
await this.analyticsService.processInteractionSD(sessionHash, logDTO.pid)
@@ -1044,6 +1163,7 @@ export class AnalyticsController {
const os = ua.os.name
const dto = analyticsDTO(
+ psid,
sessionHash,
logDTO.pid,
logDTO.pg,
@@ -1097,11 +1217,12 @@ export class AnalyticsController {
}
const values = `(${dto.map(getElValue).join(',')})`
- const perfValues = `(${perfDTO.map(getElValue).join(',')})`
+
try {
await redis.rpush(REDIS_LOG_DATA_CACHE_KEY, values)
if (!_isEmpty(perfDTO)) {
+ const perfValues = `(${perfDTO.map(getElValue).join(',')})`
await redis.rpush(REDIS_LOG_PERF_CACHE_KEY, perfValues)
}
} catch (e) {
@@ -1140,7 +1261,7 @@ export class AnalyticsController {
const ip = getIPFromHeaders(headers) || reqIP || ''
const salt = await redis.get(REDIS_SESSION_SALT_KEY)
const sessionHash = getSessionKey(ip, userAgent, logDTO.pid, salt)
- const unique = await this.analyticsService.isUnique(sessionHash)
+ const [unique, psid] = await this.analyticsService.isUnique(sessionHash)
await this.analyticsService.processInteractionSD(sessionHash, logDTO.pid)
@@ -1155,6 +1276,7 @@ export class AnalyticsController {
const os = ua.os.name
const dto = analyticsDTO(
+ psid,
sessionHash,
logDTO.pid,
'NULL',
diff --git a/apps/production/src/analytics/analytics.service.ts b/apps/production/src/analytics/analytics.service.ts
index 4a49a134b..adc3f0e29 100644
--- a/apps/production/src/analytics/analytics.service.ts
+++ b/apps/production/src/analytics/analytics.service.ts
@@ -1,5 +1,7 @@
+import * as crypto from 'crypto'
import * as _isEmpty from 'lodash/isEmpty'
import * as _split from 'lodash/split'
+import * as _reverse from 'lodash/reverse'
import * as _size from 'lodash/size'
import * as _includes from 'lodash/includes'
import * as _map from 'lodash/map'
@@ -73,6 +75,8 @@ import {
IExtractChartData,
IGenerateXAxis,
IAggregatedMetadata,
+ IFunnelCHResponse,
+ IFunnel,
} from './interfaces'
dayjs.extend(utc)
@@ -833,10 +837,192 @@ export class AnalyticsService {
}
}
- async isUnique(sessionHash: string) {
- const session = await redis.get(sessionHash)
- await redis.set(sessionHash, 1, 'EX', UNIQUE_SESSION_LIFE_TIME)
- return !session
+ generateUInt64(): string {
+ return crypto.randomBytes(8).readBigUInt64BE(0).toString()
+ }
+
+ /**
+ * Checks if the session is unique and returns the session psid (or creates a new one)
+ * @param sessionHash
+ * @returns [isUnique, psid]
+ */
+ async isUnique(sessionHash: string): Promise<[boolean, string]> {
+ let psid = await redis.get(sessionHash)
+ const exists = Boolean(psid)
+
+ if (!exists) {
+ psid = this.generateUInt64()
+ }
+
+ await redis.set(sessionHash, psid, 'EX', UNIQUE_SESSION_LIFE_TIME)
+ return [!exists, psid]
+ }
+
+ formatFunnel(data: IFunnelCHResponse[], pages: string[]): IFunnel[] {
+ const funnel = _map(data, (row, index) => {
+ const value = pages[index]
+
+ let events = row.c
+ let dropoff = 0
+ let eventsPerc = 100
+ let eventsPercStep = 100
+ let dropoffPerc = 0
+
+ if (index > 0) {
+ const prev = data[index - 1]
+ events = row.c
+ dropoff = prev.c - row.c
+ eventsPerc = _round((row.c / data[0].c) * 100, 2)
+ eventsPercStep = _round((row.c / prev.c) * 100, 2)
+ dropoffPerc = _round((dropoff / prev.c) * 100, 2)
+ }
+
+ return {
+ value,
+ events,
+ eventsPerc,
+ eventsPercStep,
+ dropoff,
+ dropoffPerc,
+ }
+ })
+
+ return funnel
+ }
+
+ /*
+ ClickHouse's windowFunnel() function returns steps that have non-zero counts.
+ I.e. if we have data like [{level: 1, c: 100}, {level: 3, c: 50}],
+ it means that level 2 should be 50 as well (because 50 users got up to level 3).
+
+ This function backfills missing levels with previous count.
+ */
+ backfillFunnel(
+ data: IFunnelCHResponse[],
+ pages: string[],
+ ): IFunnelCHResponse[] {
+ // Get max level
+ const maxLevel = _size(pages)
+
+ // Build map of level -> count
+ const levelCounts = new Map()
+ for (const d of data) {
+ levelCounts.set(d.level, d.c)
+ }
+
+ let prevCount = 0
+
+ // Backfill missing levels with previous count
+ const filled = []
+ for (let level = maxLevel; level >= 1; level--) {
+ const count = levelCounts.get(level)
+ if (count !== undefined) {
+ prevCount += count
+ }
+
+ filled.push({
+ level,
+ c: prevCount,
+ })
+ }
+
+ return filled
+ }
+
+ generateEmptyFunnel(pages: string[]): IFunnel[] {
+ return _map(pages, value => ({
+ value,
+ events: 0,
+ eventsPerc: 0,
+ eventsPercStep: 0,
+ dropoff: 0,
+ dropoffPerc: 0,
+ }))
+ }
+
+ async getFunnel(pages: string[], params: any): Promise<IFunnel[]> {
+ const pageParams = {}
+
+ const pagesStr = _join(
+ _map(pages, (value, index) => {
+ pageParams[`v${index}`] = value
+
+ return `value={v${index}:String}`
+ }),
+ ',',
+ )
+
+ const query = `
+ SELECT
+ level,
+ count() as c
+ FROM (
+ SELECT
+ psid,
+ windowFunnel(86400)(created, ${pagesStr}) AS level
+ FROM (
+ SELECT
+ psid,
+ pg AS value,
+ created
+ FROM analytics
+ WHERE pid = {pid:FixedString(12)}
+ AND psid != 0
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+
+ UNION ALL
+
+ SELECT
+ psid,
+ ev AS value,
+ created
+ FROM customEV
+ WHERE pid = {pid:FixedString(12)}
+ AND psid != 0
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ )
+ GROUP BY psid
+ )
+ WHERE level != 0
+ GROUP BY level
+ ORDER BY level DESC;
+ `
+
+ const result = <Array<IFunnelCHResponse>>(
+ await clickhouse
+ .query(query, { params: { ...params, ...pageParams } })
+ .toPromise()
+ )
+
+ if (_isEmpty(result)) {
+ return this.generateEmptyFunnel(pages)
+ }
+
+ return this.formatFunnel(
+ _reverse(this.backfillFunnel(result, pages)),
+ pages,
+ )
+ }
+
+ async getTotalPageviews(
+ pid: string,
+ groupFrom: string,
+ groupTo: string,
+ ): Promise<number> {
+ const query = `
+ SELECT
+ count() as c
+ FROM analytics
+ WHERE pid = {pid:FixedString(12)}
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ `
+ const result = <{ c: number }[]>(
+ await clickhouse
+ .query(query, { params: { pid, groupFrom, groupTo } })
+ .toPromise()
+ )
+
+ return result[0]?.c || 0
}
async getSummary(
diff --git a/apps/production/src/analytics/dto/getFunnels.dto.ts b/apps/production/src/analytics/dto/getFunnels.dto.ts
new file mode 100644
index 000000000..afde79179
--- /dev/null
+++ b/apps/production/src/analytics/dto/getFunnels.dto.ts
@@ -0,0 +1,30 @@
+import { ApiProperty } from '@nestjs/swagger'
+import { IsNotEmpty } from 'class-validator'
+import { DEFAULT_TIMEZONE } from '../../user/entities/user.entity'
+
+// eslint-disable-next-line @typescript-eslint/naming-convention
+export class GetFunnelsDTO {
+ @ApiProperty()
+ @IsNotEmpty()
+ pid: string
+
+ @ApiProperty()
+ period: string
+
+ @ApiProperty()
+ from: string
+
+ @ApiProperty()
+ to: string
+
+ @ApiProperty({
+ description: 'Timezone to display data in',
+ default: DEFAULT_TIMEZONE,
+ })
+ timezone: string
+
+ @ApiProperty({
+ description: 'A stringified array of pages to generate funnel for',
+ })
+ pages: string
+}
diff --git a/apps/production/src/analytics/interfaces/index.ts b/apps/production/src/analytics/interfaces/index.ts
index 0b1e533b9..66084f88e 100644
--- a/apps/production/src/analytics/interfaces/index.ts
+++ b/apps/production/src/analytics/interfaces/index.ts
@@ -99,3 +99,22 @@ export interface IAggregatedMetadata {
value: string
count: number
}
+
+export interface IFunnelCHResponse {
+ level: number
+ c: number
+}
+
+export interface IFunnel {
+ value: string
+ events: number
+ eventsPerc: number
+ eventsPercStep: number
+ dropoff: number
+ dropoffPerc: number
+}
+
+export interface IGetFunnel {
+ funnel: IFunnel[]
+ totalPageviews: number
+}
diff --git a/apps/production/src/common/constants.ts b/apps/production/src/common/constants.ts
index 42a1c4771..a03777dbb 100644
--- a/apps/production/src/common/constants.ts
+++ b/apps/production/src/common/constants.ts
@@ -111,6 +111,10 @@ const CAPTCHA_COOKIE_KEY = 'swetrix-captcha-token'
const CAPTCHA_TOKEN_LIFETIME = 300 // seconds (5 minutes).
const CAPTCHA_SECRET_KEY_LENGTH = 50
+// Funnels
+const MIN_PAGES_IN_FUNNEL = 2
+const MAX_PAGES_IN_FUNNEL = 10
+
const BLOG_POSTS_PATH = isDevelopment
? path.join(__dirname, '../../../..', 'blog-posts', 'posts')
: path.join(__dirname, '../..', 'blog-posts', 'posts')
@@ -216,4 +220,6 @@ export {
PAYPAL_CLIENT_ID,
PAYPAL_CLIENT_SECRET,
BLOG_POSTS_PATH,
+ MIN_PAGES_IN_FUNNEL,
+ MAX_PAGES_IN_FUNNEL,
}
diff --git a/apps/selfhosted/src/analytics/analytics.controller.ts b/apps/selfhosted/src/analytics/analytics.controller.ts
index eeb873b48..b9ca6c2e3 100644
--- a/apps/selfhosted/src/analytics/analytics.controller.ts
+++ b/apps/selfhosted/src/analytics/analytics.controller.ts
@@ -6,6 +6,9 @@ import * as _uniqBy from 'lodash/uniqBy'
import * as _round from 'lodash/round'
import * as _includes from 'lodash/includes'
import * as _keys from 'lodash/keys'
+import * as _size from 'lodash/size'
+import * as _some from 'lodash/some'
+import * as _isString from 'lodash/isString'
import * as _values from 'lodash/values'
import * as dayjs from 'dayjs'
import * as utc from 'dayjs/plugin/utc'
@@ -48,6 +51,7 @@ import { AnalyticsGET_DTO, ChartRenderMode } from './dto/getData.dto'
import { GetFiltersDto } from './dto/get-filters.dto'
import { GetCustomEventMetadata } from './dto/get-custom-event-meta.dto'
import { GetUserFlowDTO } from './dto/getUserFlow.dto'
+import { GetFunnelsDTO } from './dto/getFunnels.dto'
import { AppLoggerService } from '../logger/logger.service'
import {
REDIS_LOG_DATA_CACHE_KEY,
@@ -56,13 +60,20 @@ import {
REDIS_LOG_CUSTOM_CACHE_KEY,
HEARTBEAT_SID_LIFE_TIME,
REDIS_SESSION_SALT_KEY,
+ MIN_PAGES_IN_FUNNEL,
+ MAX_PAGES_IN_FUNNEL,
clickhouse,
} from '../common/constants'
import { getGeoDetails, getIPFromHeaders } from '../common/utils'
import { BotDetection } from '../common/decorators/bot-detection.decorator'
import { BotDetectionGuard } from '../common/guards/bot-detection.guard'
import { GetCustomEventsDto } from './dto/get-custom-events.dto'
-import { IAggregatedMetadata, IUserFlow } from './interfaces'
+import {
+ IAggregatedMetadata,
+ IFunnel,
+ IGetFunnel,
+ IUserFlow,
+} from './interfaces'
// eslint-disable-next-line @typescript-eslint/no-var-requires
const mysql = require('mysql2')
@@ -79,6 +90,7 @@ const getSessionKeyCustom = (
) => `cses_${hash(`${ua}${ip}${pid}${ev}${salt}`).toString('hex')}`
const analyticsDTO = (
+ psid: string,
sid: string,
pid: string,
pg: string,
@@ -98,6 +110,7 @@ const analyticsDTO = (
unique: number,
): Array<string | number> => {
return [
+ psid,
sid,
pid,
pg,
@@ -157,6 +170,7 @@ const performanceDTO = (
}
const customLogDTO = (
+ psid: string,
pid: string,
ev: string,
pg: string,
@@ -175,6 +189,7 @@ const customLogDTO = (
values: string[],
): Array<string | number | string[]> => {
return [
+ psid,
pid,
ev,
pg,
@@ -259,6 +274,44 @@ const getPIDsArray = (pids, pid) => {
return pids
}
+const getPagesArray = (rawPages: string): string[] => {
+ try {
+ const pages = JSON.parse(rawPages)
+
+ if (!_isArray(pages)) {
+ throw new UnprocessableEntityException(
+ 'An array of pages has to be provided as a pages param',
+ )
+ }
+
+ const size = _size(pages)
+
+ if (size < MIN_PAGES_IN_FUNNEL) {
+ throw new UnprocessableEntityException(
+ `A minimum of ${MIN_PAGES_IN_FUNNEL} pages or events has to be provided`,
+ )
+ }
+
+ if (size > MAX_PAGES_IN_FUNNEL) {
+ throw new UnprocessableEntityException(
+ `A maximum of ${MAX_PAGES_IN_FUNNEL} pages or events can be provided`,
+ )
+ }
+
+ if (_some(pages, page => !_isString(page))) {
+ throw new UnprocessableEntityException(
+ 'Pages array must contain string values only',
+ )
+ }
+
+ return pages
+ } catch (e) {
+ throw new UnprocessableEntityException(
+ 'Cannot process the provided array of pages',
+ )
+ }
+}
+
// needed for serving 1x1 px GIF
const TRANSPARENT_GIF_BUFFER = Buffer.from(
'R0lGODlhAQABAIAAAP///wAAACwAAAAAAQABAAACAkQBADs=',
@@ -377,6 +430,62 @@ export class AnalyticsController {
}
}
+ @Get('funnel')
+ @Auth([], true, true)
+ async getFunnel(
+ @Query() data: GetFunnelsDTO,
+ @CurrentUserId() uid: string,
+ ): Promise<IGetFunnel> {
+ const { pid, period, from, to, timezone = DEFAULT_TIMEZONE, pages } = data
+ this.analyticsService.validatePID(pid)
+
+ if (!_isEmpty(period)) {
+ this.analyticsService.validatePeriod(period)
+ }
+
+ const pagesArr = getPagesArray(pages)
+
+ await this.analyticsService.checkProjectAccess(pid, uid)
+
+ const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
+ const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
+ from,
+ to,
+ null,
+ period,
+ safeTimezone,
+ )
+
+ const params = {
+ pid,
+ groupFrom,
+ groupTo,
+ }
+
+ let funnel: IFunnel[] = []
+ let totalPageviews: number = 0
+
+ const promises = [
+ (async () => {
+ funnel = await this.analyticsService.getFunnel(pagesArr, params)
+ })(),
+ (async () => {
+ totalPageviews = await this.analyticsService.getTotalPageviews(
+ pid,
+ groupFrom,
+ groupTo,
+ )
+ })(),
+ ]
+
+ await Promise.all(promises)
+
+ return {
+ funnel,
+ totalPageviews,
+ }
+ }
+
@Get('meta')
@Auth([], true, true)
async getCustomEventMetadata(
@@ -775,8 +884,9 @@ export class AnalyticsController {
this.analyticsService.validateCustomEVMeta(eventsDTO.meta)
await this.analyticsService.validate(eventsDTO, origin, 'custom', ip)
+ const salt = await redis.get(REDIS_SESSION_SALT_KEY)
+
if (eventsDTO.unique) {
- const salt = await redis.get(REDIS_SESSION_SALT_KEY)
const sessionHash = getSessionKeyCustom(
ip,
userAgent,
@@ -784,7 +894,7 @@ export class AnalyticsController {
eventsDTO.ev,
salt,
)
- const unique = await this.analyticsService.isUnique(sessionHash)
+ const [unique] = await this.analyticsService.isUnique(sessionHash)
if (!unique) {
throw new ForbiddenException(
@@ -804,7 +914,11 @@ export class AnalyticsController {
country = 'NULL',
} = getGeoDetails(ip, eventsDTO.tz, headers)
+ const sessionHash = getSessionKey(ip, userAgent, eventsDTO.pid, salt)
+ const [, psid] = await this.analyticsService.isUnique(sessionHash)
+
const dto = customLogDTO(
+ psid,
eventsDTO.pid,
eventsDTO.ev,
eventsDTO.pg,
@@ -880,7 +994,7 @@ export class AnalyticsController {
const salt = await redis.get(REDIS_SESSION_SALT_KEY)
const sessionHash = getSessionKey(ip, userAgent, logDTO.pid, salt)
- const unique = await this.analyticsService.isUnique(sessionHash)
+ const [unique, psid] = await this.analyticsService.isUnique(sessionHash)
await this.analyticsService.processInteractionSD(sessionHash, logDTO.pid)
@@ -902,6 +1016,7 @@ export class AnalyticsController {
const os = ua.os.name
const dto = analyticsDTO(
+ psid,
sessionHash,
logDTO.pid,
logDTO.pg,
@@ -955,11 +1070,11 @@ export class AnalyticsController {
}
const values = `(${dto.map(getElValue).join(',')})`
- const perfValues = `(${perfDTO.map(getElValue).join(',')})`
try {
await redis.rpush(REDIS_LOG_DATA_CACHE_KEY, values)
if (!_isEmpty(perfDTO)) {
+ const perfValues = `(${perfDTO.map(getElValue).join(',')})`
await redis.rpush(REDIS_LOG_PERF_CACHE_KEY, perfValues)
}
} catch (e) {
@@ -998,7 +1113,7 @@ export class AnalyticsController {
const ip = getIPFromHeaders(headers) || reqIP || ''
const salt = await redis.get(REDIS_SESSION_SALT_KEY)
const sessionHash = getSessionKey(ip, userAgent, logDTO.pid, salt)
- const unique = await this.analyticsService.isUnique(sessionHash)
+ const [unique, psid] = await this.analyticsService.isUnique(sessionHash)
await this.analyticsService.processInteractionSD(sessionHash, logDTO.pid)
@@ -1014,6 +1129,7 @@ export class AnalyticsController {
} = getGeoDetails(ip, null, headers)
const dto = analyticsDTO(
+ psid,
sessionHash,
logDTO.pid,
'NULL',
diff --git a/apps/selfhosted/src/analytics/analytics.service.ts b/apps/selfhosted/src/analytics/analytics.service.ts
index 19b49c7b3..ddb5dca79 100644
--- a/apps/selfhosted/src/analytics/analytics.service.ts
+++ b/apps/selfhosted/src/analytics/analytics.service.ts
@@ -1,5 +1,7 @@
+import * as crypto from 'crypto'
import * as _isEmpty from 'lodash/isEmpty'
import * as _split from 'lodash/split'
+import * as _reverse from 'lodash/reverse'
import * as _size from 'lodash/size'
import * as _includes from 'lodash/includes'
import * as _map from 'lodash/map'
@@ -66,6 +68,8 @@ import {
IExtractChartData,
IGenerateXAxis,
IAggregatedMetadata,
+ IFunnelCHResponse,
+ IFunnel,
} from './interfaces'
dayjs.extend(utc)
@@ -794,10 +798,192 @@ export class AnalyticsService {
}
}
- async isUnique(sessionHash: string) {
- const session = await redis.get(sessionHash)
- await redis.set(sessionHash, 1, 'EX', UNIQUE_SESSION_LIFE_TIME)
- return !session
+ generateUInt64(): string {
+ return crypto.randomBytes(8).readBigUInt64BE(0).toString()
+ }
+
+ /**
+ * Checks if the session is unique and returns the session psid (or creates a new one)
+ * @param sessionHash
+ * @returns [isUnique, psid]
+ */
+ async isUnique(sessionHash: string): Promise<[boolean, string]> {
+ let psid = await redis.get(sessionHash)
+ const exists = Boolean(psid)
+
+ if (!exists) {
+ psid = this.generateUInt64()
+ }
+
+ await redis.set(sessionHash, psid, 'EX', UNIQUE_SESSION_LIFE_TIME)
+ return [!exists, psid]
+ }
+
+ formatFunnel(data: IFunnelCHResponse[], pages: string[]): IFunnel[] {
+ const funnel = _map(data, (row, index) => {
+ const value = pages[index]
+
+ let events = row.c
+ let dropoff = 0
+ let eventsPerc = 100
+ let eventsPercStep = 100
+ let dropoffPerc = 0
+
+ if (index > 0) {
+ const prev = data[index - 1]
+ events = row.c
+ dropoff = prev.c - row.c
+ eventsPerc = _round((row.c / data[0].c) * 100, 2)
+ eventsPercStep = _round((row.c / prev.c) * 100, 2)
+ dropoffPerc = _round((dropoff / prev.c) * 100, 2)
+ }
+
+ return {
+ value,
+ events,
+ eventsPerc,
+ eventsPercStep,
+ dropoff,
+ dropoffPerc,
+ }
+ })
+
+ return funnel
+ }
+
+ /*
+ ClickHouse's windowFunnel() function returns steps that have non-zero counts.
+ I.e. if we have data like [{level: 1, c: 100}, {level: 3, c: 50}],
+ it means that level 2 should be 50 as well (because 50 users got up to level 3).
+
+ This function backfills missing levels with previous count.
+ */
+ backfillFunnel(
+ data: IFunnelCHResponse[],
+ pages: string[],
+ ): IFunnelCHResponse[] {
+ // Get max level
+ const maxLevel = _size(pages)
+
+ // Build map of level -> count
+ const levelCounts = new Map()
+ for (const d of data) {
+ levelCounts.set(d.level, d.c)
+ }
+
+ let prevCount = 0
+
+ // Backfill missing levels with previous count
+ const filled = []
+ for (let level = maxLevel; level >= 1; level--) {
+ const count = levelCounts.get(level)
+ if (count !== undefined) {
+ prevCount += count
+ }
+
+ filled.push({
+ level,
+ c: prevCount,
+ })
+ }
+
+ return filled
+ }
+
+ generateEmptyFunnel(pages: string[]): IFunnel[] {
+ return _map(pages, value => ({
+ value,
+ events: 0,
+ eventsPerc: 0,
+ eventsPercStep: 0,
+ dropoff: 0,
+ dropoffPerc: 0,
+ }))
+ }
+
+ async getFunnel(pages: string[], params: any): Promise<IFunnel[]> {
+ const pageParams = {}
+
+ const pagesStr = _join(
+ _map(pages, (value, index) => {
+ pageParams[`v${index}`] = value
+
+ return `value={v${index}:String}`
+ }),
+ ',',
+ )
+
+ const query = `
+ SELECT
+ level,
+ count() as c
+ FROM (
+ SELECT
+ psid,
+ windowFunnel(86400)(created, ${pagesStr}) AS level
+ FROM (
+ SELECT
+ psid,
+ pg AS value,
+ created
+ FROM analytics
+ WHERE pid = {pid:FixedString(12)}
+ AND psid != 0
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+
+ UNION ALL
+
+ SELECT
+ psid,
+ ev AS value,
+ created
+ FROM customEV
+ WHERE pid = {pid:FixedString(12)}
+ AND psid != 0
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ )
+ GROUP BY psid
+ )
+ WHERE level != 0
+ GROUP BY level
+ ORDER BY level DESC;
+ `
+
+ const result = <Array<IFunnelCHResponse>>(
+ await clickhouse
+ .query(query, { params: { ...params, ...pageParams } })
+ .toPromise()
+ )
+
+ if (_isEmpty(result)) {
+ return this.generateEmptyFunnel(pages)
+ }
+
+ return this.formatFunnel(
+ _reverse(this.backfillFunnel(result, pages)),
+ pages,
+ )
+ }
+
+ async getTotalPageviews(
+ pid: string,
+ groupFrom: string,
+ groupTo: string,
+ ): Promise<number> {
+ const query = `
+ SELECT
+ count() as c
+ FROM analytics
+ WHERE pid = {pid:FixedString(12)}
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ `
+ const result = <{ c: number }[]>(
+ await clickhouse
+ .query(query, { params: { pid, groupFrom, groupTo } })
+ .toPromise()
+ )
+
+ return result[0]?.c || 0
}
async getSummary(
diff --git a/apps/selfhosted/src/analytics/dto/getFunnels.dto.ts b/apps/selfhosted/src/analytics/dto/getFunnels.dto.ts
new file mode 100644
index 000000000..afde79179
--- /dev/null
+++ b/apps/selfhosted/src/analytics/dto/getFunnels.dto.ts
@@ -0,0 +1,30 @@
+import { ApiProperty } from '@nestjs/swagger'
+import { IsNotEmpty } from 'class-validator'
+import { DEFAULT_TIMEZONE } from '../../user/entities/user.entity'
+
+// eslint-disable-next-line @typescript-eslint/naming-convention
+export class GetFunnelsDTO {
+ @ApiProperty()
+ @IsNotEmpty()
+ pid: string
+
+ @ApiProperty()
+ period: string
+
+ @ApiProperty()
+ from: string
+
+ @ApiProperty()
+ to: string
+
+ @ApiProperty({
+ description: 'Timezone to display data in',
+ default: DEFAULT_TIMEZONE,
+ })
+ timezone: string
+
+ @ApiProperty({
+ description: 'A stringified array of pages to generate funnel for',
+ })
+ pages: string
+}
diff --git a/apps/selfhosted/src/analytics/interfaces/index.ts b/apps/selfhosted/src/analytics/interfaces/index.ts
index 0b1e533b9..66084f88e 100644
--- a/apps/selfhosted/src/analytics/interfaces/index.ts
+++ b/apps/selfhosted/src/analytics/interfaces/index.ts
@@ -99,3 +99,22 @@ export interface IAggregatedMetadata {
value: string
count: number
}
+
+export interface IFunnelCHResponse {
+ level: number
+ c: number
+}
+
+export interface IFunnel {
+ value: string
+ events: number
+ eventsPerc: number
+ eventsPercStep: number
+ dropoff: number
+ dropoffPerc: number
+}
+
+export interface IGetFunnel {
+ funnel: IFunnel[]
+ totalPageviews: number
+}
diff --git a/apps/selfhosted/src/common/constants.ts b/apps/selfhosted/src/common/constants.ts
index 72a942b6b..f04d39783 100644
--- a/apps/selfhosted/src/common/constants.ts
+++ b/apps/selfhosted/src/common/constants.ts
@@ -70,10 +70,10 @@ const isValidPID = (pid: string) => PID_REGEX.test(pid)
// redis keys
const getRedisProjectKey = (pid: string) => `pid_${pid}`
-const REDIS_LOG_DATA_CACHE_KEY = 'log_cache'
+const REDIS_LOG_DATA_CACHE_KEY = 'log_cache_v2'
const REDIS_LOG_CAPTCHA_CACHE_KEY = 'log:captcha'
const REDIS_LOG_PERF_CACHE_KEY = 'perf_cache'
-const REDIS_LOG_CUSTOM_CACHE_KEY = 'log_custom_cache_v2'
+const REDIS_LOG_CUSTOM_CACHE_KEY = 'log_custom_cache_v3'
const REDIS_SESSION_SALT_KEY = 'log_salt' // is updated every 24 hours
const REDIS_USERS_COUNT_KEY = 'stats:users_count'
const REDIS_PROJECTS_COUNT_KEY = 'stats:projects_count'
@@ -87,6 +87,10 @@ const UNIQUE_SESSION_LIFE_TIME = 1800
// 35 seconds
const HEARTBEAT_SID_LIFE_TIME = 35
+// Funnels
+const MIN_PAGES_IN_FUNNEL = 2
+const MAX_PAGES_IN_FUNNEL = 10
+
const TRAFFIC_COLUMNS = [
'cc',
'rg',
@@ -136,4 +140,6 @@ export {
TRAFFIC_COLUMNS,
PERFORMANCE_COLUMNS,
isProxiedByCloudflare,
+ MIN_PAGES_IN_FUNNEL,
+ MAX_PAGES_IN_FUNNEL,
}
diff --git a/migrations/clickhouse/2023_10_18.js b/migrations/clickhouse/2023_10_18.js
new file mode 100644
index 000000000..b075c648f
--- /dev/null
+++ b/migrations/clickhouse/2023_10_18.js
@@ -0,0 +1,75 @@
+const { queriesRunner, dbName } = require('./setup')
+
+const queries = [
+ // Analytics table: added psid column
+ `DROP TABLE IF EXISTS ${dbName}.analytics_temp`,
+ `CREATE TABLE IF NOT EXISTS ${dbName}.analytics_temp
+ (
+ psid Nullable(UInt64),
+ sid Nullable(String),
+ pid FixedString(12),
+ pg Nullable(String),
+ prev Nullable(String),
+ dv LowCardinality(Nullable(String)),
+ br LowCardinality(Nullable(String)),
+ os LowCardinality(Nullable(String)),
+ lc LowCardinality(Nullable(String)),
+ ref Nullable(String),
+ so Nullable(String),
+ me Nullable(String),
+ ca Nullable(String),
+ cc Nullable(FixedString(2)),
+ rg LowCardinality(Nullable(String)),
+ ct Nullable(String),
+ sdur Nullable(UInt32),
+ unique UInt8,
+ created DateTime('UTC')
+ )
+ ENGINE = MergeTree()
+ PARTITION BY toYYYYMM(created)
+ ORDER BY (pid, created);`,
+
+ `INSERT INTO ${dbName}.analytics_temp (psid, sid, pid, pg, prev, dv, br, os, lc, ref, so, me, ca, cc, rg, ct, sdur, unique, created)
+ SELECT NULL, sid, pid, pg, prev, dv, br, os, lc, ref, so, me, ca, cc, rg, ct, sdur, unique, created FROM ${dbName}.analytics`,
+
+ `DROP TABLE ${dbName}.analytics`,
+ `RENAME TABLE ${dbName}.analytics_temp TO ${dbName}.analytics`,
+
+ // Custom events table: added psid column
+ `DROP TABLE IF EXISTS ${dbName}.customEV_temp`,
+ `CREATE TABLE IF NOT EXISTS ${dbName}.customEV_temp
+ (
+ psid Nullable(UInt64),
+ pid FixedString(12),
+ ev String,
+ pg Nullable(String),
+ dv LowCardinality(Nullable(String)),
+ br LowCardinality(Nullable(String)),
+ os LowCardinality(Nullable(String)),
+ lc LowCardinality(Nullable(String)),
+ ref Nullable(String),
+ so Nullable(String),
+ me Nullable(String),
+ ca Nullable(String),
+ cc Nullable(FixedString(2)),
+ rg LowCardinality(Nullable(String)),
+ ct Nullable(String),
+ meta Nested
+ (
+ key String,
+ value String
+ ),
+ created DateTime('UTC')
+ )
+ ENGINE = MergeTree()
+ PARTITION BY toYYYYMM(created)
+ ORDER BY (pid, created);`,
+
+ `INSERT INTO ${dbName}.customEV_temp (psid, pid, ev, pg, dv, br, os, lc, ref, so, me, ca, cc, rg, ct, meta.key, meta.value, created)
+ SELECT NULL, pid, ev, pg, dv, br, os, lc, ref, so, me, ca, cc, rg, ct, meta.key, meta.value, created FROM ${dbName}.customEV`,
+
+ `DROP TABLE ${dbName}.customEV`,
+ `RENAME TABLE ${dbName}.customEV_temp TO ${dbName}.customEV`,
+]
+
+queriesRunner(queries)
diff --git a/migrations/clickhouse/initialise_database.js b/migrations/clickhouse/initialise_database.js
index 6088f0a72..4172594a1 100644
--- a/migrations/clickhouse/initialise_database.js
+++ b/migrations/clickhouse/initialise_database.js
@@ -9,6 +9,7 @@ const CLICKHOUSE_INIT_QUERIES = [
// The traffic data table
`CREATE TABLE IF NOT EXISTS ${dbName}.analytics
(
+ psid Nullable(UInt64),
sid Nullable(String),
pid FixedString(12),
pg Nullable(String),
@@ -35,6 +36,7 @@ const CLICKHOUSE_INIT_QUERIES = [
// Custom events table
`CREATE TABLE IF NOT EXISTS ${dbName}.customEV
(
+ psid Nullable(UInt64),
pid FixedString(12),
ev String,
pg Nullable(String),