📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
(feature) Funnels (#198)
Andrii Romasiun3e17222efc53
diff --git a/apps/production/src/analytics/analytics.controller.ts b/apps/production/src/analytics/analytics.controller.tsindex af7e2cd54..e1965fc2e 100644--- a/apps/production/src/analytics/analytics.controller.ts+++ b/apps/production/src/analytics/analytics.controller.ts@@ -4,6 +4,9 @@ import * as _toNumber from 'lodash/toNumber'import * as _pick from 'lodash/pick'import * as _includes from 'lodash/includes'import * as _keys from 'lodash/keys'+import * as _size from 'lodash/size'+import * as _some from 'lodash/some'+import * as _isString from 'lodash/isString'import * as _values from 'lodash/values'import * as _map from 'lodash/map'import * as _uniqBy from 'lodash/uniqBy'@@ -49,6 +52,7 @@ import { EventsDTO } from './dto/events.dto'import { AnalyticsGET_DTO, ChartRenderMode } from './dto/getData.dto'import { GetCustomEventMetadata } from './dto/get-custom-event-meta.dto'import { GetUserFlowDTO } from './dto/getUserFlow.dto'+import { GetFunnelsDTO } from './dto/getFunnels.dto'import { AppLoggerService } from '../logger/logger.service'import {REDIS_LOG_DATA_CACHE_KEY,@@ -60,6 +64,8 @@ import {REDIS_PROJECTS_COUNT_KEY,REDIS_EVENTS_COUNT_KEY,REDIS_SESSION_SALT_KEY,+ MIN_PAGES_IN_FUNNEL,+ MAX_PAGES_IN_FUNNEL,clickhouse,} from '../common/constants'import { getGeoDetails, getIPFromHeaders } from '../common/utils'@@ -67,7 +73,12 @@ import { BotDetection } from '../common/decorators/bot-detection.decorator'import { BotDetectionGuard } from '../common/guards/bot-detection.guard'import { GetCustomEventsDto } from './dto/get-custom-events.dto'import { GetFiltersDto } from './dto/get-filters.dto'-import { IAggregatedMetadata, IUserFlow } from './interfaces'+import {+ IAggregatedMetadata,+ IFunnel,+ IGetFunnel,+ IUserFlow,+} from './interfaces'// eslint-disable-next-line @typescript-eslint/no-var-requiresconst mysql = require('mysql2')@@ -84,6 +95,7 @@ const getSessionKeyCustom = () => `cses_${hash(`${ua}${ip}${pid}${ev}${salt}`).toString('hex')}`const analyticsDTO = (+ psid: string,sid: string,pid: string,pg: string,@@ -103,6 +115,7 @@ const analyticsDTO = (unique: number,): Array<string | number> => {return [+ psid,sid,pid,pg,@@ -162,6 +175,7 @@ const performanceDTO = (}const customLogDTO = (+ psid: string,pid: string,ev: string,pg: string,@@ -180,6 +194,7 @@ const customLogDTO = (values: string[],): Array<string | number | string[]> => {return [+ psid,pid,ev,pg,@@ -264,6 +279,44 @@ const getPIDsArray = (pids, pid) => {return pids}+const getPagesArray = (rawPages: string): string[] => {+ try {+ const pages = JSON.parse(rawPages)++ if (!_isArray(pages)) {+ throw new UnprocessableEntityException(+ 'An array of pages has to be provided as a pages param',+ )+ }++ const size = _size(pages)++ if (size < MIN_PAGES_IN_FUNNEL) {+ throw new UnprocessableEntityException(+ `A minimum of ${MIN_PAGES_IN_FUNNEL} pages or events has to be provided`,+ )+ }++ if (size > MAX_PAGES_IN_FUNNEL) {+ throw new UnprocessableEntityException(+ `A maximum of ${MAX_PAGES_IN_FUNNEL} pages or events can be provided`,+ )+ }++ if (_some(pages, page => !_isString(page))) {+ throw new UnprocessableEntityException(+ 'Pages array must contain string values only',+ )+ }++ return pages+ } catch (e) {+ throw new UnprocessableEntityException(+ 'Cannot process the provided array of pages',+ )+ }+}+// needed for serving 1x1 px GIFconst TRANSPARENT_GIF_BUFFER = Buffer.from('R0lGODlhAQABAIAAAP///wAAACwAAAAAAQABAAACAkQBADs=',@@ -417,6 +470,67 @@ export class AnalyticsController {}}+ @Get('funnel')+ @Auth([], true, true)+ async getFunnel(+ @Query() data: GetFunnelsDTO,+ @CurrentUserId() uid: string,+ @Headers() headers: { 'x-password'?: string },+ ): Promise<IGetFunnel> {+ const { pid, period, from, to, timezone = DEFAULT_TIMEZONE, pages } = data+ this.analyticsService.validatePID(pid)++ if (!_isEmpty(period)) {+ this.analyticsService.validatePeriod(period)+ }++ const pagesArr = getPagesArray(pages)++ await this.analyticsService.checkProjectAccess(+ pid,+ uid,+ headers['x-password'],+ )++ const safeTimezone = this.analyticsService.getSafeTimezone(timezone)+ const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(+ from,+ to,+ null,+ period,+ safeTimezone,+ )++ const params = {+ pid,+ groupFrom,+ groupTo,+ }++ let funnel: IFunnel[] = []+ let totalPageviews: number = 0++ const promises = [+ (async () => {+ funnel = await this.analyticsService.getFunnel(pagesArr, params)+ })(),+ (async () => {+ totalPageviews = await this.analyticsService.getTotalPageviews(+ pid,+ groupFrom,+ groupTo,+ )+ })(),+ ]++ await Promise.all(promises)++ return {+ funnel,+ totalPageviews,+ }+ }+@Get('meta')@Auth([], true, true)async getCustomEventMetadata(@@ -918,8 +1032,9 @@ export class AnalyticsController {this.analyticsService.validateCustomEVMeta(eventsDTO.meta)await this.analyticsService.validate(eventsDTO, origin, 'custom', ip)+ const salt = await redis.get(REDIS_SESSION_SALT_KEY)+if (eventsDTO.unique) {- const salt = await redis.get(REDIS_SESSION_SALT_KEY)const sessionHash = getSessionKeyCustom(ip,userAgent,@@ -927,7 +1042,7 @@ export class AnalyticsController {eventsDTO.ev,salt,)- const unique = await this.analyticsService.isUnique(sessionHash)+ const [unique] = await this.analyticsService.isUnique(sessionHash)if (!unique) {throw new ForbiddenException(@@ -947,7 +1062,11 @@ export class AnalyticsController {const br = ua.browser.nameconst os = ua.os.name+ const sessionHash = getSessionKey(ip, userAgent, eventsDTO.pid, salt)+ const [, psid] = await this.analyticsService.isUnique(sessionHash)+const dto = customLogDTO(+ psid,eventsDTO.pid,eventsDTO.ev,eventsDTO.pg,@@ -1023,7 +1142,7 @@ export class AnalyticsController {const salt = await redis.get(REDIS_SESSION_SALT_KEY)const sessionHash = getSessionKey(ip, userAgent, logDTO.pid, salt)- const unique = await this.analyticsService.isUnique(sessionHash)+ const [unique, psid] = await this.analyticsService.isUnique(sessionHash)await this.analyticsService.processInteractionSD(sessionHash, logDTO.pid)@@ -1044,6 +1163,7 @@ export class AnalyticsController {const os = ua.os.nameconst dto = analyticsDTO(+ psid,sessionHash,logDTO.pid,logDTO.pg,@@ -1097,11 +1217,12 @@ export class AnalyticsController {}const values = `(${dto.map(getElValue).join(',')})`- const perfValues = `(${perfDTO.map(getElValue).join(',')})`+try {await redis.rpush(REDIS_LOG_DATA_CACHE_KEY, values)if (!_isEmpty(perfDTO)) {+ const perfValues = `(${perfDTO.map(getElValue).join(',')})`await redis.rpush(REDIS_LOG_PERF_CACHE_KEY, perfValues)}} catch (e) {@@ -1140,7 +1261,7 @@ export class AnalyticsController {const ip = getIPFromHeaders(headers) || reqIP || ''const salt = await redis.get(REDIS_SESSION_SALT_KEY)const sessionHash = getSessionKey(ip, userAgent, logDTO.pid, salt)- const unique = await this.analyticsService.isUnique(sessionHash)+ const [unique, psid] = await this.analyticsService.isUnique(sessionHash)await this.analyticsService.processInteractionSD(sessionHash, logDTO.pid)@@ -1155,6 +1276,7 @@ export class AnalyticsController {const os = ua.os.nameconst dto = analyticsDTO(+ psid,sessionHash,logDTO.pid,'NULL',diff --git a/apps/production/src/analytics/analytics.service.ts b/apps/production/src/analytics/analytics.service.tsindex 4a49a134b..adc3f0e29 100644--- a/apps/production/src/analytics/analytics.service.ts+++ b/apps/production/src/analytics/analytics.service.ts@@ -1,5 +1,7 @@+import * as crypto from 'crypto'import * as _isEmpty from 'lodash/isEmpty'import * as _split from 'lodash/split'+import * as _reverse from 'lodash/reverse'import * as _size from 'lodash/size'import * as _includes from 'lodash/includes'import * as _map from 'lodash/map'@@ -73,6 +75,8 @@ import {IExtractChartData,IGenerateXAxis,IAggregatedMetadata,+ IFunnelCHResponse,+ IFunnel,} from './interfaces'dayjs.extend(utc)@@ -833,10 +837,192 @@ export class AnalyticsService {}}- async isUnique(sessionHash: string) {- const session = await redis.get(sessionHash)- await redis.set(sessionHash, 1, 'EX', UNIQUE_SESSION_LIFE_TIME)- return !session+ generateUInt64(): string {+ return crypto.randomBytes(8).readBigUInt64BE(0).toString()+ }++ /**+ * Checks if the session is unique and returns the session psid (or creates a new one)+ * @param sessionHash+ * @returns [isUnique, psid]+ */+ async isUnique(sessionHash: string): Promise<[boolean, string]> {+ let psid = await redis.get(sessionHash)+ const exists = Boolean(psid)++ if (!exists) {+ psid = this.generateUInt64()+ }++ await redis.set(sessionHash, psid, 'EX', UNIQUE_SESSION_LIFE_TIME)+ return [!exists, psid]+ }++ formatFunnel(data: IFunnelCHResponse[], pages: string[]): IFunnel[] {+ const funnel = _map(data, (row, index) => {+ const value = pages[index]++ let events = row.c+ let dropoff = 0+ let eventsPerc = 100+ let eventsPercStep = 100+ let dropoffPerc = 0++ if (index > 0) {+ const prev = data[index - 1]+ events = row.c+ dropoff = prev.c - row.c+ eventsPerc = _round((row.c / data[0].c) * 100, 2)+ eventsPercStep = _round((row.c / prev.c) * 100, 2)+ dropoffPerc = _round((dropoff / prev.c) * 100, 2)+ }++ return {+ value,+ events,+ eventsPerc,+ eventsPercStep,+ dropoff,+ dropoffPerc,+ }+ })++ return funnel+ }++ /*+ ClickHouse's windowFunnel() function returns steps that have non-zero counts.+ I.e. if we have data like [{level: 1, c: 100}, {level: 3, c: 50}],+ it means that level 2 should be 50 as well (because 50 users got up to level 3).++ This function backfills missing levels with previous count.+ */+ backfillFunnel(+ data: IFunnelCHResponse[],+ pages: string[],+ ): IFunnelCHResponse[] {+ // Get max level+ const maxLevel = _size(pages)++ // Build map of level -> count+ const levelCounts = new Map()+ for (const d of data) {+ levelCounts.set(d.level, d.c)+ }++ let prevCount = 0++ // Backfill missing levels with previous count+ const filled = []+ for (let level = maxLevel; level >= 1; level--) {+ const count = levelCounts.get(level)+ if (count !== undefined) {+ prevCount += count+ }++ filled.push({+ level,+ c: prevCount,+ })+ }++ return filled+ }++ generateEmptyFunnel(pages: string[]): IFunnel[] {+ return _map(pages, value => ({+ value,+ events: 0,+ eventsPerc: 0,+ eventsPercStep: 0,+ dropoff: 0,+ dropoffPerc: 0,+ }))+ }++ async getFunnel(pages: string[], params: any): Promise<IFunnel[]> {+ const pageParams = {}++ const pagesStr = _join(+ _map(pages, (value, index) => {+ pageParams[`v${index}`] = value++ return `value={v${index}:String}`+ }),+ ',',+ )++ const query = `+ SELECT+ level,+ count() as c+ FROM (+ SELECT+ psid,+ windowFunnel(86400)(created, ${pagesStr}) AS level+ FROM (+ SELECT+ psid,+ pg AS value,+ created+ FROM analytics+ WHERE pid = {pid:FixedString(12)}+ AND psid != 0+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}++ UNION ALL++ SELECT+ psid,+ ev AS value,+ created+ FROM customEV+ WHERE pid = {pid:FixedString(12)}+ AND psid != 0+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}+ )+ GROUP BY psid+ )+ WHERE level != 0+ GROUP BY level+ ORDER BY level DESC;+ `++ const result = <Array<IFunnelCHResponse>>(+ await clickhouse+ .query(query, { params: { ...params, ...pageParams } })+ .toPromise()+ )++ if (_isEmpty(result)) {+ return this.generateEmptyFunnel(pages)+ }++ return this.formatFunnel(+ _reverse(this.backfillFunnel(result, pages)),+ pages,+ )+ }++ async getTotalPageviews(+ pid: string,+ groupFrom: string,+ groupTo: string,+ ): Promise<number> {+ const query = `+ SELECT+ count() as c+ FROM analytics+ WHERE pid = {pid:FixedString(12)}+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}+ `+ const result = <{ c: number }[]>(+ await clickhouse+ .query(query, { params: { pid, groupFrom, groupTo } })+ .toPromise()+ )++ return result[0]?.c || 0}async getSummary(diff --git a/apps/production/src/analytics/dto/getFunnels.dto.ts b/apps/production/src/analytics/dto/getFunnels.dto.tsnew file mode 100644index 000000000..afde79179--- /dev/null+++ b/apps/production/src/analytics/dto/getFunnels.dto.ts@@ -0,0 +1,30 @@+import { ApiProperty } from '@nestjs/swagger'+import { IsNotEmpty } from 'class-validator'+import { DEFAULT_TIMEZONE } from '../../user/entities/user.entity'++// eslint-disable-next-line @typescript-eslint/naming-convention+export class GetFunnelsDTO {+ @ApiProperty()+ @IsNotEmpty()+ pid: string++ @ApiProperty()+ period: string++ @ApiProperty()+ from: string++ @ApiProperty()+ to: string++ @ApiProperty({+ description: 'Timezone to display data in',+ default: DEFAULT_TIMEZONE,+ })+ timezone: string++ @ApiProperty({+ description: 'A stringified array of pages to generate funnel for',+ })+ pages: string+}diff --git a/apps/production/src/analytics/interfaces/index.ts b/apps/production/src/analytics/interfaces/index.tsindex 0b1e533b9..66084f88e 100644--- a/apps/production/src/analytics/interfaces/index.ts+++ b/apps/production/src/analytics/interfaces/index.ts@@ -99,3 +99,22 @@ export interface IAggregatedMetadata {value: stringcount: number}++export interface IFunnelCHResponse {+ level: number+ c: number+}++export interface IFunnel {+ value: string+ events: number+ eventsPerc: number+ eventsPercStep: number+ dropoff: number+ dropoffPerc: number+}++export interface IGetFunnel {+ funnel: IFunnel[]+ totalPageviews: number+}diff --git a/apps/production/src/common/constants.ts b/apps/production/src/common/constants.tsindex 42a1c4771..a03777dbb 100644--- a/apps/production/src/common/constants.ts+++ b/apps/production/src/common/constants.ts@@ -111,6 +111,10 @@ const CAPTCHA_COOKIE_KEY = 'swetrix-captcha-token'const CAPTCHA_TOKEN_LIFETIME = 300 // seconds (5 minutes).const CAPTCHA_SECRET_KEY_LENGTH = 50+// Funnels+const MIN_PAGES_IN_FUNNEL = 2+const MAX_PAGES_IN_FUNNEL = 10+const BLOG_POSTS_PATH = isDevelopment? path.join(__dirname, '../../../..', 'blog-posts', 'posts'): path.join(__dirname, '../..', 'blog-posts', 'posts')@@ -216,4 +220,6 @@ export {PAYPAL_CLIENT_ID,PAYPAL_CLIENT_SECRET,BLOG_POSTS_PATH,+ MIN_PAGES_IN_FUNNEL,+ MAX_PAGES_IN_FUNNEL,}diff --git a/apps/selfhosted/src/analytics/analytics.controller.ts b/apps/selfhosted/src/analytics/analytics.controller.tsindex eeb873b48..b9ca6c2e3 100644--- a/apps/selfhosted/src/analytics/analytics.controller.ts+++ b/apps/selfhosted/src/analytics/analytics.controller.ts@@ -6,6 +6,9 @@ import * as _uniqBy from 'lodash/uniqBy'import * as _round from 'lodash/round'import * as _includes from 'lodash/includes'import * as _keys from 'lodash/keys'+import * as _size from 'lodash/size'+import * as _some from 'lodash/some'+import * as _isString from 'lodash/isString'import * as _values from 'lodash/values'import * as dayjs from 'dayjs'import * as utc from 'dayjs/plugin/utc'@@ -48,6 +51,7 @@ import { AnalyticsGET_DTO, ChartRenderMode } from './dto/getData.dto'import { GetFiltersDto } from './dto/get-filters.dto'import { GetCustomEventMetadata } from './dto/get-custom-event-meta.dto'import { GetUserFlowDTO } from './dto/getUserFlow.dto'+import { GetFunnelsDTO } from './dto/getFunnels.dto'import { AppLoggerService } from '../logger/logger.service'import {REDIS_LOG_DATA_CACHE_KEY,@@ -56,13 +60,20 @@ import {REDIS_LOG_CUSTOM_CACHE_KEY,HEARTBEAT_SID_LIFE_TIME,REDIS_SESSION_SALT_KEY,+ MIN_PAGES_IN_FUNNEL,+ MAX_PAGES_IN_FUNNEL,clickhouse,} from '../common/constants'import { getGeoDetails, getIPFromHeaders } from '../common/utils'import { BotDetection } from '../common/decorators/bot-detection.decorator'import { BotDetectionGuard } from '../common/guards/bot-detection.guard'import { GetCustomEventsDto } from './dto/get-custom-events.dto'-import { IAggregatedMetadata, IUserFlow } from './interfaces'+import {+ IAggregatedMetadata,+ IFunnel,+ IGetFunnel,+ IUserFlow,+} from './interfaces'// eslint-disable-next-line @typescript-eslint/no-var-requiresconst mysql = require('mysql2')@@ -79,6 +90,7 @@ const getSessionKeyCustom = () => `cses_${hash(`${ua}${ip}${pid}${ev}${salt}`).toString('hex')}`const analyticsDTO = (+ psid: string,sid: string,pid: string,pg: string,@@ -98,6 +110,7 @@ const analyticsDTO = (unique: number,): Array<string | number> => {return [+ psid,sid,pid,pg,@@ -157,6 +170,7 @@ const performanceDTO = (}const customLogDTO = (+ psid: string,pid: string,ev: string,pg: string,@@ -175,6 +189,7 @@ const customLogDTO = (values: string[],): Array<string | number | string[]> => {return [+ psid,pid,ev,pg,@@ -259,6 +274,44 @@ const getPIDsArray = (pids, pid) => {return pids}+const getPagesArray = (rawPages: string): string[] => {+ try {+ const pages = JSON.parse(rawPages)++ if (!_isArray(pages)) {+ throw new UnprocessableEntityException(+ 'An array of pages has to be provided as a pages param',+ )+ }++ const size = _size(pages)++ if (size < MIN_PAGES_IN_FUNNEL) {+ throw new UnprocessableEntityException(+ `A minimum of ${MIN_PAGES_IN_FUNNEL} pages or events has to be provided`,+ )+ }++ if (size > MAX_PAGES_IN_FUNNEL) {+ throw new UnprocessableEntityException(+ `A maximum of ${MAX_PAGES_IN_FUNNEL} pages or events can be provided`,+ )+ }++ if (_some(pages, page => !_isString(page))) {+ throw new UnprocessableEntityException(+ 'Pages array must contain string values only',+ )+ }++ return pages+ } catch (e) {+ throw new UnprocessableEntityException(+ 'Cannot process the provided array of pages',+ )+ }+}+// needed for serving 1x1 px GIFconst TRANSPARENT_GIF_BUFFER = Buffer.from('R0lGODlhAQABAIAAAP///wAAACwAAAAAAQABAAACAkQBADs=',@@ -377,6 +430,62 @@ export class AnalyticsController {}}+ @Get('funnel')+ @Auth([], true, true)+ async getFunnel(+ @Query() data: GetFunnelsDTO,+ @CurrentUserId() uid: string,+ ): Promise<IGetFunnel> {+ const { pid, period, from, to, timezone = DEFAULT_TIMEZONE, pages } = data+ this.analyticsService.validatePID(pid)++ if (!_isEmpty(period)) {+ this.analyticsService.validatePeriod(period)+ }++ const pagesArr = getPagesArray(pages)++ await this.analyticsService.checkProjectAccess(pid, uid)++ const safeTimezone = this.analyticsService.getSafeTimezone(timezone)+ const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(+ from,+ to,+ null,+ period,+ safeTimezone,+ )++ const params = {+ pid,+ groupFrom,+ groupTo,+ }++ let funnel: IFunnel[] = []+ let totalPageviews: number = 0++ const promises = [+ (async () => {+ funnel = await this.analyticsService.getFunnel(pagesArr, params)+ })(),+ (async () => {+ totalPageviews = await this.analyticsService.getTotalPageviews(+ pid,+ groupFrom,+ groupTo,+ )+ })(),+ ]++ await Promise.all(promises)++ return {+ funnel,+ totalPageviews,+ }+ }+@Get('meta')@Auth([], true, true)async getCustomEventMetadata(@@ -775,8 +884,9 @@ export class AnalyticsController {this.analyticsService.validateCustomEVMeta(eventsDTO.meta)await this.analyticsService.validate(eventsDTO, origin, 'custom', ip)+ const salt = await redis.get(REDIS_SESSION_SALT_KEY)+if (eventsDTO.unique) {- const salt = await redis.get(REDIS_SESSION_SALT_KEY)const sessionHash = getSessionKeyCustom(ip,userAgent,@@ -784,7 +894,7 @@ export class AnalyticsController {eventsDTO.ev,salt,)- const unique = await this.analyticsService.isUnique(sessionHash)+ const [unique] = await this.analyticsService.isUnique(sessionHash)if (!unique) {throw new ForbiddenException(@@ -804,7 +914,11 @@ export class AnalyticsController {country = 'NULL',} = getGeoDetails(ip, eventsDTO.tz, headers)+ const sessionHash = getSessionKey(ip, userAgent, eventsDTO.pid, salt)+ const [, psid] = await this.analyticsService.isUnique(sessionHash)+const dto = customLogDTO(+ psid,eventsDTO.pid,eventsDTO.ev,eventsDTO.pg,@@ -880,7 +994,7 @@ export class AnalyticsController {const salt = await redis.get(REDIS_SESSION_SALT_KEY)const sessionHash = getSessionKey(ip, userAgent, logDTO.pid, salt)- const unique = await this.analyticsService.isUnique(sessionHash)+ const [unique, psid] = await this.analyticsService.isUnique(sessionHash)await this.analyticsService.processInteractionSD(sessionHash, logDTO.pid)@@ -902,6 +1016,7 @@ export class AnalyticsController {const os = ua.os.nameconst dto = analyticsDTO(+ psid,sessionHash,logDTO.pid,logDTO.pg,@@ -955,11 +1070,11 @@ export class AnalyticsController {}const values = `(${dto.map(getElValue).join(',')})`- const perfValues = `(${perfDTO.map(getElValue).join(',')})`try {await redis.rpush(REDIS_LOG_DATA_CACHE_KEY, values)if (!_isEmpty(perfDTO)) {+ const perfValues = `(${perfDTO.map(getElValue).join(',')})`await redis.rpush(REDIS_LOG_PERF_CACHE_KEY, perfValues)}} catch (e) {@@ -998,7 +1113,7 @@ export class AnalyticsController {const ip = getIPFromHeaders(headers) || reqIP || ''const salt = await redis.get(REDIS_SESSION_SALT_KEY)const sessionHash = getSessionKey(ip, userAgent, logDTO.pid, salt)- const unique = await this.analyticsService.isUnique(sessionHash)+ const [unique, psid] = await this.analyticsService.isUnique(sessionHash)await this.analyticsService.processInteractionSD(sessionHash, logDTO.pid)@@ -1014,6 +1129,7 @@ export class AnalyticsController {} = getGeoDetails(ip, null, headers)const dto = analyticsDTO(+ psid,sessionHash,logDTO.pid,'NULL',diff --git a/apps/selfhosted/src/analytics/analytics.service.ts b/apps/selfhosted/src/analytics/analytics.service.tsindex 19b49c7b3..ddb5dca79 100644--- a/apps/selfhosted/src/analytics/analytics.service.ts+++ b/apps/selfhosted/src/analytics/analytics.service.ts@@ -1,5 +1,7 @@+import * as crypto from 'crypto'import * as _isEmpty from 'lodash/isEmpty'import * as _split from 'lodash/split'+import * as _reverse from 'lodash/reverse'import * as _size from 'lodash/size'import * as _includes from 'lodash/includes'import * as _map from 'lodash/map'@@ -66,6 +68,8 @@ import {IExtractChartData,IGenerateXAxis,IAggregatedMetadata,+ IFunnelCHResponse,+ IFunnel,} from './interfaces'dayjs.extend(utc)@@ -794,10 +798,192 @@ export class AnalyticsService {}}- async isUnique(sessionHash: string) {- const session = await redis.get(sessionHash)- await redis.set(sessionHash, 1, 'EX', UNIQUE_SESSION_LIFE_TIME)- return !session+ generateUInt64(): string {+ return crypto.randomBytes(8).readBigUInt64BE(0).toString()+ }++ /**+ * Checks if the session is unique and returns the session psid (or creates a new one)+ * @param sessionHash+ * @returns [isUnique, psid]+ */+ async isUnique(sessionHash: string): Promise<[boolean, string]> {+ let psid = await redis.get(sessionHash)+ const exists = Boolean(psid)++ if (!exists) {+ psid = this.generateUInt64()+ }++ await redis.set(sessionHash, psid, 'EX', UNIQUE_SESSION_LIFE_TIME)+ return [!exists, psid]+ }++ formatFunnel(data: IFunnelCHResponse[], pages: string[]): IFunnel[] {+ const funnel = _map(data, (row, index) => {+ const value = pages[index]++ let events = row.c+ let dropoff = 0+ let eventsPerc = 100+ let eventsPercStep = 100+ let dropoffPerc = 0++ if (index > 0) {+ const prev = data[index - 1]+ events = row.c+ dropoff = prev.c - row.c+ eventsPerc = _round((row.c / data[0].c) * 100, 2)+ eventsPercStep = _round((row.c / prev.c) * 100, 2)+ dropoffPerc = _round((dropoff / prev.c) * 100, 2)+ }++ return {+ value,+ events,+ eventsPerc,+ eventsPercStep,+ dropoff,+ dropoffPerc,+ }+ })++ return funnel+ }++ /*+ ClickHouse's windowFunnel() function returns steps that have non-zero counts.+ I.e. if we have data like [{level: 1, c: 100}, {level: 3, c: 50}],+ it means that level 2 should be 50 as well (because 50 users got up to level 3).++ This function backfills missing levels with previous count.+ */+ backfillFunnel(+ data: IFunnelCHResponse[],+ pages: string[],+ ): IFunnelCHResponse[] {+ // Get max level+ const maxLevel = _size(pages)++ // Build map of level -> count+ const levelCounts = new Map()+ for (const d of data) {+ levelCounts.set(d.level, d.c)+ }++ let prevCount = 0++ // Backfill missing levels with previous count+ const filled = []+ for (let level = maxLevel; level >= 1; level--) {+ const count = levelCounts.get(level)+ if (count !== undefined) {+ prevCount += count+ }++ filled.push({+ level,+ c: prevCount,+ })+ }++ return filled+ }++ generateEmptyFunnel(pages: string[]): IFunnel[] {+ return _map(pages, value => ({+ value,+ events: 0,+ eventsPerc: 0,+ eventsPercStep: 0,+ dropoff: 0,+ dropoffPerc: 0,+ }))+ }++ async getFunnel(pages: string[], params: any): Promise<IFunnel[]> {+ const pageParams = {}++ const pagesStr = _join(+ _map(pages, (value, index) => {+ pageParams[`v${index}`] = value++ return `value={v${index}:String}`+ }),+ ',',+ )++ const query = `+ SELECT+ level,+ count() as c+ FROM (+ SELECT+ psid,+ windowFunnel(86400)(created, ${pagesStr}) AS level+ FROM (+ SELECT+ psid,+ pg AS value,+ created+ FROM analytics+ WHERE pid = {pid:FixedString(12)}+ AND psid != 0+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}++ UNION ALL++ SELECT+ psid,+ ev AS value,+ created+ FROM customEV+ WHERE pid = {pid:FixedString(12)}+ AND psid != 0+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}+ )+ GROUP BY psid+ )+ WHERE level != 0+ GROUP BY level+ ORDER BY level DESC;+ `++ const result = <Array<IFunnelCHResponse>>(+ await clickhouse+ .query(query, { params: { ...params, ...pageParams } })+ .toPromise()+ )++ if (_isEmpty(result)) {+ return this.generateEmptyFunnel(pages)+ }++ return this.formatFunnel(+ _reverse(this.backfillFunnel(result, pages)),+ pages,+ )+ }++ async getTotalPageviews(+ pid: string,+ groupFrom: string,+ groupTo: string,+ ): Promise<number> {+ const query = `+ SELECT+ count() as c+ FROM analytics+ WHERE pid = {pid:FixedString(12)}+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}+ `+ const result = <{ c: number }[]>(+ await clickhouse+ .query(query, { params: { pid, groupFrom, groupTo } })+ .toPromise()+ )++ return result[0]?.c || 0}async getSummary(diff --git a/apps/selfhosted/src/analytics/dto/getFunnels.dto.ts b/apps/selfhosted/src/analytics/dto/getFunnels.dto.tsnew file mode 100644index 000000000..afde79179--- /dev/null+++ b/apps/selfhosted/src/analytics/dto/getFunnels.dto.ts@@ -0,0 +1,30 @@+import { ApiProperty } from '@nestjs/swagger'+import { IsNotEmpty } from 'class-validator'+import { DEFAULT_TIMEZONE } from '../../user/entities/user.entity'++// eslint-disable-next-line @typescript-eslint/naming-convention+export class GetFunnelsDTO {+ @ApiProperty()+ @IsNotEmpty()+ pid: string++ @ApiProperty()+ period: string++ @ApiProperty()+ from: string++ @ApiProperty()+ to: string++ @ApiProperty({+ description: 'Timezone to display data in',+ default: DEFAULT_TIMEZONE,+ })+ timezone: string++ @ApiProperty({+ description: 'A stringified array of pages to generate funnel for',+ })+ pages: string+}diff --git a/apps/selfhosted/src/analytics/interfaces/index.ts b/apps/selfhosted/src/analytics/interfaces/index.tsindex 0b1e533b9..66084f88e 100644--- a/apps/selfhosted/src/analytics/interfaces/index.ts+++ b/apps/selfhosted/src/analytics/interfaces/index.ts@@ -99,3 +99,22 @@ export interface IAggregatedMetadata {value: stringcount: number}++export interface IFunnelCHResponse {+ level: number+ c: number+}++export interface IFunnel {+ value: string+ events: number+ eventsPerc: number+ eventsPercStep: number+ dropoff: number+ dropoffPerc: number+}++export interface IGetFunnel {+ funnel: IFunnel[]+ totalPageviews: number+}diff --git a/apps/selfhosted/src/common/constants.ts b/apps/selfhosted/src/common/constants.tsindex 72a942b6b..f04d39783 100644--- a/apps/selfhosted/src/common/constants.ts+++ b/apps/selfhosted/src/common/constants.ts@@ -70,10 +70,10 @@ const isValidPID = (pid: string) => PID_REGEX.test(pid)// redis keysconst getRedisProjectKey = (pid: string) => `pid_${pid}`-const REDIS_LOG_DATA_CACHE_KEY = 'log_cache'+const REDIS_LOG_DATA_CACHE_KEY = 'log_cache_v2'const REDIS_LOG_CAPTCHA_CACHE_KEY = 'log:captcha'const REDIS_LOG_PERF_CACHE_KEY = 'perf_cache'-const REDIS_LOG_CUSTOM_CACHE_KEY = 'log_custom_cache_v2'+const REDIS_LOG_CUSTOM_CACHE_KEY = 'log_custom_cache_v3'const REDIS_SESSION_SALT_KEY = 'log_salt' // is updated every 24 hoursconst REDIS_USERS_COUNT_KEY = 'stats:users_count'const REDIS_PROJECTS_COUNT_KEY = 'stats:projects_count'@@ -87,6 +87,10 @@ const UNIQUE_SESSION_LIFE_TIME = 1800// 35 secondsconst HEARTBEAT_SID_LIFE_TIME = 35+// Funnels+const MIN_PAGES_IN_FUNNEL = 2+const MAX_PAGES_IN_FUNNEL = 10+const TRAFFIC_COLUMNS = ['cc','rg',@@ -136,4 +140,6 @@ export {TRAFFIC_COLUMNS,PERFORMANCE_COLUMNS,isProxiedByCloudflare,+ MIN_PAGES_IN_FUNNEL,+ MAX_PAGES_IN_FUNNEL,}diff --git a/migrations/clickhouse/2023_10_18.js b/migrations/clickhouse/2023_10_18.jsnew file mode 100644index 000000000..b075c648f--- /dev/null+++ b/migrations/clickhouse/2023_10_18.js@@ -0,0 +1,75 @@+const { queriesRunner, dbName } = require('./setup')++const queries = [+ // Analytics table: added psid column+ `DROP TABLE IF EXISTS ${dbName}.analytics_temp`,+ `CREATE TABLE IF NOT EXISTS ${dbName}.analytics_temp+ (+ psid Nullable(UInt64),+ sid Nullable(String),+ pid FixedString(12),+ pg Nullable(String),+ prev Nullable(String),+ dv LowCardinality(Nullable(String)),+ br LowCardinality(Nullable(String)),+ os LowCardinality(Nullable(String)),+ lc LowCardinality(Nullable(String)),+ ref Nullable(String),+ so Nullable(String),+ me Nullable(String),+ ca Nullable(String),+ cc Nullable(FixedString(2)),+ rg LowCardinality(Nullable(String)),+ ct Nullable(String),+ sdur Nullable(UInt32),+ unique UInt8,+ created DateTime('UTC')+ )+ ENGINE = MergeTree()+ PARTITION BY toYYYYMM(created)+ ORDER BY (pid, created);`,++ `INSERT INTO ${dbName}.analytics_temp (psid, sid, pid, pg, prev, dv, br, os, lc, ref, so, me, ca, cc, rg, ct, sdur, unique, created)+ SELECT NULL, sid, pid, pg, prev, dv, br, os, lc, ref, so, me, ca, cc, rg, ct, sdur, unique, created FROM ${dbName}.analytics`,++ `DROP TABLE ${dbName}.analytics`,+ `RENAME TABLE ${dbName}.analytics_temp TO ${dbName}.analytics`,++ // Custom events table: added psid column+ `DROP TABLE IF EXISTS ${dbName}.customEV_temp`,+ `CREATE TABLE IF NOT EXISTS ${dbName}.customEV_temp+ (+ psid Nullable(UInt64),+ pid FixedString(12),+ ev String,+ pg Nullable(String),+ dv LowCardinality(Nullable(String)),+ br LowCardinality(Nullable(String)),+ os LowCardinality(Nullable(String)),+ lc LowCardinality(Nullable(String)),+ ref Nullable(String),+ so Nullable(String),+ me Nullable(String),+ ca Nullable(String),+ cc Nullable(FixedString(2)),+ rg LowCardinality(Nullable(String)),+ ct Nullable(String),+ meta Nested+ (+ key String,+ value String+ ),+ created DateTime('UTC')+ )+ ENGINE = MergeTree()+ PARTITION BY toYYYYMM(created)+ ORDER BY (pid, created);`,++ `INSERT INTO ${dbName}.customEV_temp (psid, pid, ev, pg, dv, br, os, lc, ref, so, me, ca, cc, rg, ct, meta.key, meta.value, created)+ SELECT NULL, pid, ev, pg, dv, br, os, lc, ref, so, me, ca, cc, rg, ct, meta.key, meta.value, created FROM ${dbName}.customEV`,++ `DROP TABLE ${dbName}.customEV`,+ `RENAME TABLE ${dbName}.customEV_temp TO ${dbName}.customEV`,+]++queriesRunner(queries)diff --git a/migrations/clickhouse/initialise_database.js b/migrations/clickhouse/initialise_database.jsindex 6088f0a72..4172594a1 100644--- a/migrations/clickhouse/initialise_database.js+++ b/migrations/clickhouse/initialise_database.js@@ -9,6 +9,7 @@ const CLICKHOUSE_INIT_QUERIES = [// The traffic data table`CREATE TABLE IF NOT EXISTS ${dbName}.analytics(+ psid Nullable(UInt64),sid Nullable(String),pid FixedString(12),pg Nullable(String),@@ -35,6 +36,7 @@ const CLICKHOUSE_INIT_QUERIES = [// Custom events table`CREATE TABLE IF NOT EXISTS ${dbName}.customEV(+ psid Nullable(UInt64),pid FixedString(12),ev String,pg Nullable(String),