📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
(improvement) Preparing for the next selfhosted release (#182)
Andrii Romasiun135604c59a11
diff --git a/.env.example b/.env.exampleindex e39e8d74d..ba0bb89f4 100644--- a/.env.example+++ b/.env.example@@ -75,4 +75,7 @@ PADDLE_API_KEY=# SentrySENTRY_ENABLED=false-SENTRY_DSN=''\ No newline at end of file+SENTRY_DSN=''++# Cloudflare Proxy+CLOUDFLARE_PROXY_ENABLED=falsediff --git a/apps/production/src/analytics/analytics.controller.ts b/apps/production/src/analytics/analytics.controller.tsindex d66c0ddc5..07750370a 100644--- a/apps/production/src/analytics/analytics.controller.ts+++ b/apps/production/src/analytics/analytics.controller.ts@@ -292,6 +292,12 @@ export class AnalyticsController {this.analyticsService.validatePeriod(period)}+ await this.analyticsService.checkProjectAccess(+ pid,+ uid,+ headers['x-password'],+ )+let newTimebucket = timeBucketlet allowedTumebucketForPeriodAll@@ -329,11 +335,6 @@ export class AnalyticsController {safeTimezone,diff,)- await this.analyticsService.checkProjectAccess(- pid,- uid,- headers['x-password'],- )let queryCustoms = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`let subQuery = `FROM ${@@ -515,6 +516,12 @@ export class AnalyticsController {this.analyticsService.validatePeriod(period)}+ await this.analyticsService.checkProjectAccess(+ pid,+ uid,+ headers['x-password'],+ )+let newTimeBucket = timeBucketlet allowedTumebucketForPeriodAlllet diff@@ -547,11 +554,6 @@ export class AnalyticsController {safeTimezone,diff,)- await this.analyticsService.checkProjectAccess(- pid,- uid,- headers['x-password'],- )const subQuery = `FROM performance WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`@@ -668,6 +670,12 @@ export class AnalyticsController {this.analyticsService.validatePeriod(period)}+ await this.analyticsService.checkProjectAccess(+ pid,+ uid,+ headers['x-password'],+ )+let diffif (period === 'all') {@@ -680,12 +688,6 @@ export class AnalyticsController {diff = res.diff}- await this.analyticsService.checkProjectAccess(- pid,- uid,- headers['x-password'],- )-const safeTimezone = this.analyticsService.getSafeTimezone(timezone)const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(from,@@ -1165,6 +1167,12 @@ export class AnalyticsController {this.analyticsService.validatePeriod(period)}+ await this.analyticsService.checkProjectAccess(+ pid,+ uid,+ headers['x-password'],+ )+let newTimeBucket = timeBucketlet difflet timeBucketForAllTime@@ -1187,11 +1195,6 @@ export class AnalyticsController {this.analyticsService.validateTimebucket(newTimeBucket)const [filtersQuery, filtersParams, appliedFilters] =this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)- await this.analyticsService.checkProjectAccess(- pid,- uid,- headers['x-password'],- )const safeTimezone = this.analyticsService.getSafeTimezone(timezone)const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(diff --git a/apps/production/src/common/constants.ts b/apps/production/src/common/constants.tsindex a93e54a1c..d7b085967 100644--- a/apps/production/src/common/constants.ts+++ b/apps/production/src/common/constants.ts@@ -44,6 +44,7 @@ const clickhouse = new ClickHouse({const { JWT_ACCESS_TOKEN_SECRET } = process.envconst isDevelopment = process.env.NODE_ENV === 'development'+const isProxiedByCloudflare = process.env.CLOUDFLARE_PROXY_ENABLED === 'true'const PRODUCTION_ORIGIN = process.env.CLIENT_URL || 'https://swetrix.com'const { TWO_FACTOR_AUTHENTICATION_APP_NAME } = process.env@@ -193,4 +194,5 @@ export {CAPTCHA_COLUMNS,PERFORMANCE_COLUMNS,sentryIgnoreErrors,+ isProxiedByCloudflare,}diff --git a/apps/production/src/common/utils.ts b/apps/production/src/common/utils.tsindex caa3624c7..62ec58874 100644--- a/apps/production/src/common/utils.ts+++ b/apps/production/src/common/utils.ts@@ -12,7 +12,7 @@ import * as _find from 'lodash/find'import * as _round from 'lodash/round'import * as _split from 'lodash/split'-import { redis, isDevelopment } from './constants'+import { redis, isDevelopment, isProxiedByCloudflare } from './constants'const marketingTips = {en: [@@ -209,11 +209,9 @@ const getGeoDetails = (ip: string, tz?: string): IPGeoDetails => {}}-const CLOUDFLARE_ENABLED = false-const getIPFromHeaders = (headers: any) => {- if (CLOUDFLARE_ENABLED) {- return headers['cf-connecting-ip'] || null+ if (isProxiedByCloudflare && headers['cf-connecting-ip']) {+ return headers['cf-connecting-ip']}// Get IP based on the NGINX configurationdiff --git a/apps/selfhosted/src/analytics/analytics.controller.ts b/apps/selfhosted/src/analytics/analytics.controller.tsindex 1d2d8798f..34740dfba 100644--- a/apps/selfhosted/src/analytics/analytics.controller.ts+++ b/apps/selfhosted/src/analytics/analytics.controller.ts@@ -4,11 +4,11 @@ import * as _pick from 'lodash/pick'import * as _map from 'lodash/map'import * as _uniqBy from 'lodash/uniqBy'import * as _round from 'lodash/round'+import * as _includes from 'lodash/includes'import * as dayjs from 'dayjs'import * as utc from 'dayjs/plugin/utc'import * as dayjsTimezone from 'dayjs/plugin/timezone'import { hash } from 'blake3'-import ct from 'countries-and-timezones'import {Controller,Body,@@ -34,15 +34,16 @@ import {AnalyticsService,getSessionKey,getHeartbeatKey,+ validPeriods,DataType,} from './analytics.service'-import { TaskManagerService } from '../task-manager/task-manager.service'import { CurrentUserId } from '../auth/decorators/current-user-id.decorator'import { DEFAULT_TIMEZONE } from '../user/entities/user.entity'import { RolesGuard } from '../auth/guards/roles.guard'import { PageviewsDTO } from './dto/pageviews.dto'import { EventsDTO } from './dto/events.dto'import { AnalyticsGET_DTO } from './dto/getData.dto'+import { GetFiltersDto } from './dto/get-filters.dto'import { GetUserFlowDTO } from './dto/getUserFlow.dto'import { AppLoggerService } from '../logger/logger.service'import {@@ -54,6 +55,7 @@ import {REDIS_SESSION_SALT_KEY,clickhouse,} from '../common/constants'+import { getGeoDetails, getIPFromHeaders } from '../common/utils'import { BotDetection } from '../common/decorators/bot-detection.decorator'import { BotDetectionGuard } from '../common/guards/bot-detection.guard'import { GetCustomEventsDto } from './dto/get-custom-events.dto'@@ -87,6 +89,8 @@ const analyticsDTO = (me: string,ca: string,cc: string,+ rg: string,+ ct: string,sdur: number | string,unique: number,): Array<string | number> => {@@ -104,6 +108,8 @@ const analyticsDTO = (me,ca,cc,+ rg,+ ct,sdur,unique,dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),@@ -116,6 +122,8 @@ const performanceDTO = (dv: string,br: string,cc: string,+ rg: string,+ ct: string,dns: number,tls: number,conn: number,@@ -131,6 +139,8 @@ const performanceDTO = (dv,br,cc,+ rg,+ ct,_round(dns),_round(tls),_round(conn),@@ -156,6 +166,8 @@ const customLogDTO = (me: string,ca: string,cc: string,+ rg: string,+ ct: string,): Array<string | number> => {return [pid,@@ -170,6 +182,8 @@ const customLogDTO = (me,ca,cc,+ rg,+ ct,dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),]}@@ -247,7 +261,6 @@ export class AnalyticsController {constructor(private readonly analyticsService: AnalyticsService,private readonly logger: AppLoggerService,- private readonly taskManagerService: TaskManagerService,) {}@Get()@@ -271,7 +284,29 @@ export class AnalyticsController {this.analyticsService.validatePeriod(period)}- this.analyticsService.validateTimebucket(timeBucket)+ await this.analyticsService.checkProjectAccess(pid, uid)++ let newTimebucket = timeBucket+ let allowedTumebucketForPeriodAll++ let diff++ if (period === 'all') {+ const res = await this.analyticsService.getTimeBucketForAllTime(+ pid,+ period,+ timezone,+ )++ diff = res.diff+ // eslint-disable-next-line prefer-destructuring+ newTimebucket = _includes(res.timeBucket, timeBucket)+ ? timeBucket+ : res.timeBucket[0]+ allowedTumebucketForPeriodAll = res.timeBucket+ }++ this.analyticsService.validateTimebucket(newTimebucket)const [filtersQuery, filtersParams, parsedFilters, customEVFilterApplied] =this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)@@ -280,11 +315,11 @@ export class AnalyticsController {this.analyticsService.getGroupFromTo(from,to,- timeBucket,+ newTimebucket,period,safeTimezone,+ diff,)- await this.analyticsService.checkProjectAccess(pid, uid)let queryCustoms = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`let subQuery = `FROM analytics WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`@@ -305,7 +340,7 @@ export class AnalyticsController {}const result = await this.analyticsService.groupByTimeBucket(- timeBucket,+ newTimebucket,groupFrom,groupTo,subQuery,@@ -325,9 +360,24 @@ export class AnalyticsController {...result,customs,appliedFilters: parsedFilters,+ timeBucket: allowedTumebucketForPeriodAll,}}+ @Get('filters')+ @Auth([], true, true)+ async getFilters(+ @Query() data: GetFiltersDto,+ @CurrentUserId() uid: string,+ ): Promise<any> {+ const { pid, type } = data+ this.analyticsService.validatePID(pid)++ await this.analyticsService.checkProjectAccess(pid, uid)++ return this.analyticsService.getFilters(pid, type)+ }+@Get('chart')@Auth([], true, true)async getChartData(@@ -416,7 +466,28 @@ export class AnalyticsController {this.analyticsService.validatePeriod(period)}- this.analyticsService.validateTimebucket(timeBucket)+ await this.analyticsService.checkProjectAccess(pid, uid)++ let newTimeBucket = timeBucket+ let allowedTumebucketForPeriodAll+ let diff++ if (period === 'all') {+ const res = await this.analyticsService.getTimeBucketForAllTime(+ pid,+ period,+ timezone,+ )++ diff = res.diff+ // eslint-disable-next-line prefer-destructuring+ newTimeBucket = _includes(res.timeBucket, timeBucket)+ ? timeBucket+ : res.timeBucket[0]+ allowedTumebucketForPeriodAll = res.timeBucket+ }++ this.analyticsService.validateTimebucket(newTimeBucket)const [filtersQuery, filtersParams, parsedFilters] =this.analyticsService.getFiltersQuery(filters, DataType.PERFORMANCE)@@ -424,11 +495,11 @@ export class AnalyticsController {const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(from,to,- timeBucket,+ newTimeBucket,period,safeTimezone,+ diff,)- await this.analyticsService.checkProjectAccess(pid, uid)const subQuery = `FROM performance WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`@@ -442,7 +513,7 @@ export class AnalyticsController {}const result = await this.analyticsService.groupPerfByTimeBucket(- timeBucket,+ newTimeBucket,groupFrom,groupTo,subQuery,@@ -454,6 +525,7 @@ export class AnalyticsController {return {...result,appliedFilters: parsedFilters,+ timeBucket: allowedTumebucketForPeriodAll,}}@@ -531,6 +603,18 @@ export class AnalyticsController {await this.analyticsService.checkProjectAccess(pid, uid)+ let diff++ if (period === 'all') {+ const res = await this.analyticsService.getTimeBucketForAllTime(+ pid,+ period,+ timezone,+ )++ diff = res.diff+ }+const safeTimezone = this.analyticsService.getSafeTimezone(timezone)const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(from,@@ -538,6 +622,7 @@ export class AnalyticsController {null,period,safeTimezone,+ diff,)const [filtersQuery, filtersParams, parsedFilters] =@@ -653,8 +738,7 @@ export class AnalyticsController {): Promise<any> {const { 'user-agent': userAgent, origin } = headers- const ip =- headers['cf-connecting-ip'] || headers['x-forwarded-for'] || reqIP || ''+ const ip = getIPFromHeaders(headers) || reqIP || ''await this.analyticsService.validate(eventsDTO, origin, 'custom', ip)@@ -680,10 +764,12 @@ export class AnalyticsController {const dv = ua.device.type || 'desktop'const br = ua.browser.nameconst os = ua.os.name- // trying to get country from timezome, otherwise using CloudFlare's IP based country code as a fallback- const cc =- ct.getCountryForTimezone(eventsDTO.tz)?.id ||- (headers['cf-ipcountry'] === 'XX' ? 'NULL' : headers['cf-ipcountry'])++ const {+ city = 'NULL',+ region = 'NULL',+ country = 'NULL',+ } = getGeoDetails(ip, eventsDTO.tz, headers)const dto = customLogDTO(eventsDTO.pid,@@ -697,7 +783,9 @@ export class AnalyticsController {eventsDTO.so,eventsDTO.me,eventsDTO.ca,- cc,+ country,+ region,+ city,)try {@@ -722,8 +810,7 @@ export class AnalyticsController {): Promise<any> {const { 'user-agent': userAgent } = headersconst { pid } = logDTO- const ip =- headers['cf-connecting-ip'] || headers['x-forwarded-for'] || reqIP || ''+ const ip = getIPFromHeaders(headers) || reqIP || ''const sessionID = await this.analyticsService.getSessionHash(pid,@@ -752,8 +839,7 @@ export class AnalyticsController {): Promise<any> {const { 'user-agent': userAgent, origin } = headers- const ip =- headers['cf-connecting-ip'] || headers['x-forwarded-for'] || reqIP || ''+ const ip = getIPFromHeaders(headers) || reqIP || ''await this.analyticsService.validate(logDTO, origin, 'log', ip)@@ -769,10 +855,11 @@ export class AnalyticsController {)}- // trying to get country from timezome, otherwise using CloudFlare's IP based country code as a fallback- const cc =- ct.getCountryForTimezone(logDTO.tz)?.id ||- (headers['cf-ipcountry'] === 'XX' ? 'NULL' : headers['cf-ipcountry'])+ const {+ city = 'NULL',+ region = 'NULL',+ country = 'NULL',+ } = getGeoDetails(ip, logDTO.tz, headers)const ua = UAParser(userAgent)const dv = ua.device.type || 'desktop'@@ -792,7 +879,9 @@ export class AnalyticsController {logDTO.so,logDTO.me,logDTO.ca,- cc,+ country,+ region,+ city,0,Number(unique),)@@ -816,7 +905,9 @@ export class AnalyticsController {logDTO.pg,dv,br,- cc,+ country,+ region,+ city,dns,tls,conn,@@ -869,8 +960,7 @@ export class AnalyticsController {await this.analyticsService.validate(logDTO, origin)- const ip =- headers['cf-connecting-ip'] || headers['x-forwarded-for'] || reqIP || ''+ const ip = getIPFromHeaders(headers) || reqIP || ''const salt = await redis.get(REDIS_SESSION_SALT_KEY)const sessionHash = getSessionKey(ip, userAgent, logDTO.pid, salt)const unique = await this.analyticsService.isUnique(sessionHash)@@ -882,9 +972,12 @@ export class AnalyticsController {const br = ua.browser.nameconst os = ua.os.name- // using CloudFlare's IP based country code- const cc =- headers['cf-ipcountry'] === 'XX' ? 'NULL' : headers['cf-ipcountry']+ const {+ city = 'NULL',+ region = 'NULL',+ country = 'NULL',+ } = getGeoDetails(ip, null, headers)+const dto = analyticsDTO(sessionHash,logDTO.pid,@@ -898,7 +991,9 @@ export class AnalyticsController {'NULL','NULL','NULL',- cc,+ city,+ region,+ country,0,Number(unique),)@@ -936,18 +1031,39 @@ export class AnalyticsController {this.analyticsService.validatePeriod(period)}- this.analyticsService.validateTimebucket(timeBucket)+ await this.analyticsService.checkProjectAccess(pid, uid)++ let newTimeBucket = timeBucket+ let diff+ let timeBucketForAllTime++ if (period === validPeriods[validPeriods.length - 1]) {+ const res = await this.analyticsService.getTimeBucketForAllTime(+ pid,+ period,+ timezone,+ )++ // eslint-disable-next-line prefer-destructuring+ newTimeBucket = _includes(res.timeBucket, timeBucket)+ ? timeBucket+ : res.timeBucket[0]+ diff = res.diff+ timeBucketForAllTime = res.timeBucket+ }++ this.analyticsService.validateTimebucket(newTimeBucket)const [filtersQuery, filtersParams, parsedFilters] =this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)- await this.analyticsService.checkProjectAccess(pid, uid)const safeTimezone = this.analyticsService.getSafeTimezone(timezone)const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(from,to,- timeBucket,+ newTimeBucket,period,safeTimezone,+ diff,)const paramsData = {@@ -960,7 +1076,7 @@ export class AnalyticsController {}const result: any = await this.analyticsService.groupCustomEVByTimeBucket(- timeBucket,+ newTimeBucket,groupFrom,groupTo,filtersQuery,@@ -990,6 +1106,7 @@ export class AnalyticsController {return {...result,appliedFilters: parsedFilters,+ timeBucket: timeBucketForAllTime,}}}diff --git a/apps/selfhosted/src/analytics/analytics.service.ts b/apps/selfhosted/src/analytics/analytics.service.tsindex d5cc74225..7de478980 100644--- a/apps/selfhosted/src/analytics/analytics.service.ts+++ b/apps/selfhosted/src/analytics/analytics.service.ts@@ -37,6 +37,8 @@ import {UNIQUE_SESSION_LIFE_TIME,clickhouse,REDIS_SESSION_SALT_KEY,+ TRAFFIC_COLUMNS,+ PERFORMANCE_COLUMNS,} from '../common/constants'import {calculateRelativePercentage,@@ -82,11 +84,8 @@ const GMT_0_TIMEZONES = [// 'Africa/Casablanca',]-const cols = ['cc', 'pg', 'lc', 'br', 'os', 'dv', 'ref', 'so', 'me', 'ca']--const perfColumns = ['cc', 'pg', 'dv', 'br']--const validPeriods = [+export const validPeriods = [+ '1h','today','yesterday','1d',@@ -95,17 +94,24 @@ const validPeriods = ['3M','12M','24M',+ 'all',]const validTimebuckets = [+ TimeBucketType.MINUTE,TimeBucketType.HOUR,TimeBucketType.DAY,TimeBucketType.MONTH,+ TimeBucketType.YEAR,]// mapping of allowed timebuckets per difference between days// (e.g. if difference is lower than (lt) (including) -> then the specified timebuckets are allowed to be applied)const timeBucketToDays = [+ {+ lt: 0,+ tb: [TimeBucketType.MINUTE, TimeBucketType.HOUR],+ }, // < 1 day{lt: 7,tb: [TimeBucketType.HOUR, TimeBucketType.DAY, TimeBucketType.MONTH],@@ -116,6 +122,8 @@ const timeBucketToDays = [}, // 4 weeks{ lt: 366, tb: [TimeBucketType.MONTH] }, // 12 months{ lt: 732, tb: [TimeBucketType.MONTH] }, // 24 months+ { lt: 1464, tb: [TimeBucketType.MONTH, TimeBucketType.YEAR] }, // 48 months+ { lt: 99999, tb: [TimeBucketType.YEAR] },]// Smaller than 64 characters, must start with an English letter and contain only letters (a-z A-Z), numbers (0-9), underscores (_) and dots (.)@@ -123,9 +131,11 @@ const timeBucketToDays = [const customEVvalidate = /^[a-zA-Z](?:[\w\.]){0,62}$/const timeBucketConversion = {+ minute: 'toStartOfMinute',hour: 'toStartOfHour',day: 'toStartOfDay',month: 'toStartOfMonth',+ year: 'toStartOfYear',}const isValidTimezone = (timezone: string): boolean => {@@ -353,10 +363,10 @@ export class AnalyticsService {getDataTypeColumns(dataType: DataType): string[] {if (dataType === DataType.ANALYTICS) {- return cols+ return TRAFFIC_COLUMNS}- return perfColumns+ return PERFORMANCE_COLUMNS}getGroupFromTo(@@ -365,6 +375,7 @@ export class AnalyticsService {timeBucket: TimeBucketType | null,period: string,safeTimezone: string,+ diff?: number,): IGetGroupFromTo {let groupFrom: dayjs.Dayjslet groupTo: dayjs.Dayjs@@ -434,8 +445,14 @@ export class AnalyticsService {} else if (period === 'yesterday') {groupFrom = djsNow.subtract(1, 'day').startOf('d')groupTo = djsNow.subtract(1, 'day').endOf('d')+ } else if (period === 'all' && (diff === 0 || diff === 1)) {+ groupFrom = djsNow.subtract(1, 'day').startOf('d')+ groupTo = djsNow+ } else if (period === 'all') {+ groupFrom = djsNow.subtract(diff - 1, 'day').startOf(timeBucket)+ groupTo = djsNow} else {- if (period === '1d') {+ if (period === '1d' || period === '1h') {groupFrom = djsNow.subtract(parseInt(period, 10), _last(period))} else {groupFrom = djsNow.subtract(parseInt(period, 10) - 1, _last(period))@@ -588,6 +605,51 @@ export class AnalyticsService {}}+ async getTimeBucketForAllTime(+ pid: string,+ period: string,+ safeTimezone: string,+ ): Promise<{+ timeBucket: TimeBucketType[]+ diff: number+ }> {+ if (period !== 'all') {+ return null+ }++ const from: any = await clickhouse+ .query(+ 'SELECT created as from FROM analytics where pid = {pid:FixedString(12)} ORDER BY created ASC LIMIT 1',+ { params: { pid } },+ )+ .toPromise()+ const to = _includes(GMT_0_TIMEZONES, safeTimezone)+ ? dayjs.utc()+ : dayjs().tz(safeTimezone)++ let newTimeBucket = [TimeBucketType.MONTH]+ let diff = null++ if (from && to) {+ diff = dayjs(to).diff(dayjs(from[0].from), 'days')++ const tbMap = _find(timeBucketToDays, ({ lt }) => diff <= lt)++ if (_isEmpty(tbMap)) {+ throw new PreconditionFailedException(+ "The difference between 'from' and 'to' is greater than allowed",+ )+ }++ newTimeBucket = tbMap.tb+ }++ return {+ timeBucket: newTimeBucket,+ diff,+ }+ }+postProcessParsedFilters(parsedFilters: any[]): any[] {return _reduce(parsedFilters,@@ -824,6 +886,21 @@ export class AnalyticsService {return result}+ async getFilters(pid: string, type: string): Promise<Array<string>> {+ if (!_includes(TRAFFIC_COLUMNS, type)) {+ throw new UnprocessableEntityException(+ `The provided type (${type}) is incorrect`,+ )+ }++ const query = `SELECT ${type} FROM analytics WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}`+ const results = await clickhouse+ .query(query, { params: { pid } })+ .toPromise()++ return _map(results, type)+ }+async generateParams(parsedFilters: Array<{ [key: string]: string }>,subQuery: string,@@ -843,10 +920,10 @@ export class AnalyticsService {),)- let columns = cols+ let columns = TRAFFIC_COLUMNSif (isPerformance) {- columns = perfColumns+ columns = PERFORMANCE_COLUMNS}const paramsPromises = _map(columns, async col => {@@ -859,24 +936,7 @@ export class AnalyticsService {)const res = await clickhouse.query(query, paramsData).toPromise()- params[col] = {}-- const size = _size(res)-- if (isPerformance) {- for (let j = 0; j < size; ++j) {- const key = res[j][col]- params[col][key] = _round(- millisecondsToSeconds(res[j]['avg(pageLoad)']),- 2,- )- }- } else {- for (let j = 0; j < size; ++j) {- const key = res[j][col]- params[col][key] = res[j]['count()']- }- }+ params[col] = res})await Promise.all(paramsPromises)@@ -912,6 +972,7 @@ export class AnalyticsService {let formatswitch (timeBucket) {+ case TimeBucketType.MINUTE:case TimeBucketType.HOUR:format = 'YYYY-MM-DD HH:mm:ss'break@@ -924,6 +985,10 @@ export class AnalyticsService {format = 'YYYY-MM'break+ case TimeBucketType.YEAR:+ format = 'YYYY'+ break+default:throw new BadRequestException(`The provided time bucket (${timeBucket}) is incorrect`,@@ -956,7 +1021,7 @@ export class AnalyticsService {}generateDateString(row: { [key: string]: number }): string {- const { year, month, day, hour } = row+ const { year, month, day, hour, minute } = rowlet dateString = `${year}-${month < 10 ? `0${month}` : month}`@@ -969,10 +1034,17 @@ export class AnalyticsService {}if (typeof hour === 'number') {+ const strMinute =+ typeof minute === 'number'+ ? minute < 10+ ? `0${minute}`+ : minute+ : '00'+if (hour < 10) {- dateString += ` 0${hour}:00:00`+ dateString += ` 0${hour}:${strMinute}:00`} else {- dateString += ` ${hour}:00:00`+ dateString += ` ${hour}:${strMinute}:00`}}@@ -1033,6 +1105,17 @@ export class AnalyticsService {}getGroupSubquery(timeBucket: TimeBucketType): [string, string] {+ if (timeBucket === TimeBucketType.MINUTE) {+ return [+ `toYear(tz_created) as year,+ toMonth(tz_created) as month,+ toDayOfMonth(tz_created) as day,+ toHour(tz_created) as hour,+ toMinute(tz_created) as minute`,+ 'year, month, day, hour, minute',+ ]+ }+if (timeBucket === TimeBucketType.HOUR) {return [`toYear(tz_created) as year,@@ -1051,6 +1134,10 @@ export class AnalyticsService {]}+ if (timeBucket === TimeBucketType.YEAR) {+ return [`toYear(tz_created) as year`, 'year']+ }+return [`toYear(tz_created) as year,toMonth(tz_created) as month,@@ -1439,6 +1526,8 @@ export class AnalyticsService {const timeBucketFunc = timeBucketConversion[timeBucket]const [selector, groupBy] = this.getGroupSubquery(timeBucket)+ const tzFromDate = `toTimeZone(parseDateTimeBestEffort({groupFrom:String}), '${safeTimezone}')`+ const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), '${safeTimezone}')`const query = `SELECT@@ -1447,11 +1536,11 @@ export class AnalyticsService {count() as countFROM (SELECT *,- ${timeBucketFunc}(created) as tz_created+ ${timeBucketFunc}(toTimeZone(created, '${safeTimezone}')) as tz_createdFROM customEVWHEREpid = {pid:FixedString(12)}- AND created BETWEEN {groupFrom:String} AND {groupTo:String}+ AND created BETWEEN ${tzFromDate} AND ${tzToDate}${filtersQuery}) as subqueryGROUP BY ${groupBy}, evdiff --git a/apps/selfhosted/src/analytics/dto/get-filters.dto.ts b/apps/selfhosted/src/analytics/dto/get-filters.dto.tsnew file mode 100644index 000000000..408bf0fc5--- /dev/null+++ b/apps/selfhosted/src/analytics/dto/get-filters.dto.ts@@ -0,0 +1,20 @@+import { ApiProperty } from '@nestjs/swagger'+import { IsNotEmpty } from 'class-validator'++export class GetFiltersDto {+ @ApiProperty({+ example: 'aUn1quEid-3',+ required: true,+ description: 'The project ID',+ })+ @IsNotEmpty()+ pid: string++ @ApiProperty({+ example: 'br',+ required: true,+ description: 'Params type. Example: br, os, cc, etc.',+ })+ @IsNotEmpty()+ type: string+}diff --git a/apps/selfhosted/src/analytics/dto/getData.dto.ts b/apps/selfhosted/src/analytics/dto/getData.dto.tsindex 53a39d5bc..c51c3d227 100644--- a/apps/selfhosted/src/analytics/dto/getData.dto.ts+++ b/apps/selfhosted/src/analytics/dto/getData.dto.ts@@ -3,9 +3,11 @@ import { IsNotEmpty } from 'class-validator'import { DEFAULT_TIMEZONE } from '../../user/entities/user.entity'export enum TimeBucketType {+ MINUTE = 'minute',HOUR = 'hour',DAY = 'day',MONTH = 'month',+ YEAR = 'year',}// eslint-disable-next-line @typescript-eslint/naming-conventiondiff --git a/apps/selfhosted/src/auth/auth.controller.ts b/apps/selfhosted/src/auth/auth.controller.tsindex f335e8853..6d46eedbf 100644--- a/apps/selfhosted/src/auth/auth.controller.ts+++ b/apps/selfhosted/src/auth/auth.controller.ts@@ -16,7 +16,7 @@ import {import { ApiTags, ApiOperation, ApiOkResponse } from '@nestjs/swagger'import { I18nValidationExceptionFilter, I18n, I18nContext } from 'nestjs-i18n'-import { checkRateLimit } from '../common/utils'+import { checkRateLimit, getIPFromHeaders } from '../common/utils'import { generateSelfhostedUser } from '../user/entities/user.entity'import { AuthService } from './auth.service'import { Public, CurrentUserId, CurrentUser } from './decorators'@@ -50,10 +50,9 @@ export class AuthController {@Body() body: LoginRequestDto,@I18n() i18n: I18nContext,@Headers() headers: unknown,- @Ip() requestIp: string,+ @Ip() reqIP: string,): Promise<LoginResponseDto> {- const ip =- headers['x-forwarded-for'] || headers['cf-connecting-ip'] || requestIp+ const ip = getIPFromHeaders(headers) || reqIP || ''await checkRateLimit(ip, 'login', 10, 1800)diff --git a/apps/selfhosted/src/common/constants.ts b/apps/selfhosted/src/common/constants.tsindex 1285c4caf..f9cb8de4c 100644--- a/apps/selfhosted/src/common/constants.ts+++ b/apps/selfhosted/src/common/constants.ts@@ -44,6 +44,7 @@ const clickhouse = new ClickHouse({})const { JWT_ACCESS_TOKEN_SECRET } = process.env+const isProxiedByCloudflare = process.env.CLOUDFLARE_PROXY_ENABLED === 'true'const isDevelopment = process.env.NODE_ENV === 'development'const SELFHOSTED_EMAIL = process.env.EMAIL@@ -80,6 +81,23 @@ const UNIQUE_SESSION_LIFE_TIME = 1800// 35 secondsconst HEARTBEAT_SID_LIFE_TIME = 35+const TRAFFIC_COLUMNS = [+ 'cc',+ 'rg',+ 'ct',+ 'pg',+ 'lc',+ 'br',+ 'os',+ 'dv',+ 'ref',+ 'so',+ 'me',+ 'ca',+]++const PERFORMANCE_COLUMNS = ['cc', 'rg', 'ct', 'pg', 'dv', 'br']+export {clickhouse,redis,@@ -104,4 +122,7 @@ export {isDevelopment,DEFAULT_SELFHOSTED_UUID,JWT_ACCESS_TOKEN_SECRET,+ TRAFFIC_COLUMNS,+ PERFORMANCE_COLUMNS,+ isProxiedByCloudflare,}diff --git a/apps/selfhosted/src/common/utils.ts b/apps/selfhosted/src/common/utils.tsindex 2567d8209..33cd7ec16 100644--- a/apps/selfhosted/src/common/utils.ts+++ b/apps/selfhosted/src/common/utils.ts@@ -1,4 +1,5 @@import { NotFoundException, HttpException } from '@nestjs/common'+import timezones from 'countries-and-timezones'import { hash } from 'blake3'import { v5 as uuidv5 } from 'uuid'import * as _join from 'lodash/join'@@ -7,6 +8,7 @@ import * as _values from 'lodash/values'import * as _reduce from 'lodash/reduce'import * as _keys from 'lodash/keys'import * as _toNumber from 'lodash/toNumber'+import * as _split from 'lodash/split'import * as _isEmpty from 'lodash/isEmpty'import * as _head from 'lodash/head'import * as _round from 'lodash/round'@@ -21,6 +23,7 @@ import {SELFHOSTED_EMAIL,UUIDV5_NAMESPACE,isDevelopment,+ isProxiedByCloudflare,} from './constants'import { Project } from '../project/entity/project.entity'@@ -129,7 +132,7 @@ const deleteProjectClickhouse = async id => {return clickhouse.query(query).toPromise()}-const createProjectClickhouse = async (project: Project) => {+const createProjectClickhouse = async (project: Partial<Project>) => {const paramsData = {params: {...project,@@ -281,6 +284,71 @@ const getSelfhostedUUID = (): string => {}}+interface IPGeoDetails {+ country?: string+ region?: string+ city?: string+}++const getGeoDetails = (+ ip: string,+ tz?: string,+ headers?: unknown,+): IPGeoDetails => {+ // TODO: Add support for DBIP for self-hosted++ // Stage 1: Using IP address based geo lookup+ // const data = lookup.get(ip)++ // const country = data?.country?.iso_code+ // // TODO: Add city overrides, for example, Colinton -> Edinburgh, etc.+ // const city = data?.city?.names?.en+ // const region = data?.subdivisions?.[0]?.names?.en++ // if (country) {+ // return {+ // country,+ // city,+ // region,+ // }+ // }++ // Stage 2: If Cloudflare is enabled, use their headers+ if (isProxiedByCloudflare && headers?.['cf-ipcountry'] !== 'XX') {+ return headers['cf-ipcountry']+ }++ // Stage 3: Using timezone based geo lookup as a fallback+ const tzCountry = timezones.getCountryForTimezone(tz)?.id || null++ return {+ country: tzCountry,+ city: null,+ region: null,+ }+}++const getIPFromHeaders = (headers: any) => {+ if (isProxiedByCloudflare && headers['cf-connecting-ip']) {+ return headers['cf-connecting-ip']+ }++ // Get IP based on the NGINX configuration+ let ip = headers['x-real-ip']++ if (ip) {+ return ip+ }++ ip = headers['x-forwarded-for'] || null++ if (!ip) {+ return null+ }++ return _split(ip, ',')[0]+}+export {checkRateLimit,createProjectClickhouse,@@ -296,4 +364,6 @@ export {updateUserClickhouse,getUserClickhouse,createUserClickhouse,+ getGeoDetails,+ getIPFromHeaders,}diff --git a/apps/selfhosted/src/project/project.controller.ts b/apps/selfhosted/src/project/project.controller.tsindex c27d4678a..f474f065b 100644--- a/apps/selfhosted/src/project/project.controller.ts+++ b/apps/selfhosted/src/project/project.controller.ts@@ -110,7 +110,7 @@ export class ProjectController {const project = new Project()project.id = pid- projectDTO.name = _trim(projectDTO.name)+ project.name = _trim(projectDTO.name)await createProjectClickhouse(project)diff --git a/docker-compose.yml b/docker-compose.ymlindex 80c43e980..4fae851fe 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -11,7 +11,7 @@ services:environment:- API_URL=http://localhost:8080/- REACT_APP_AIAPI_URL=https://aiapi.swetrix.com/- - REACT_APP_SELFHOSTED=true+ - SELFHOSTED=true# This is the analytics APIswetrix-api:@@ -28,6 +28,7 @@ services:- API_ORIGINS=- PASSWORD=12345678+ - CLOUDFLARE_PROXY_ENABLED=falselinks:- 'redis'- 'clickhouse'