Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

(improvement) Preparing for the next selfhosted release (#182)

Andrii Romasiun135604c59a11
diff --git a/.env.example b/.env.example
index e39e8d74d..ba0bb89f4 100644
--- a/.env.example
+++ b/.env.example
@@ -75,4 +75,7 @@ PADDLE_API_KEY=
# Sentry
SENTRY_ENABLED=false
-SENTRY_DSN=''
\ No newline at end of file
+SENTRY_DSN=''
+
+# Cloudflare Proxy
+CLOUDFLARE_PROXY_ENABLED=false
diff --git a/apps/production/src/analytics/analytics.controller.ts b/apps/production/src/analytics/analytics.controller.ts
index d66c0ddc5..07750370a 100644
--- a/apps/production/src/analytics/analytics.controller.ts
+++ b/apps/production/src/analytics/analytics.controller.ts
@@ -292,6 +292,12 @@ export class AnalyticsController {
this.analyticsService.validatePeriod(period)
}
+ await this.analyticsService.checkProjectAccess(
+ pid,
+ uid,
+ headers['x-password'],
+ )
+
let newTimebucket = timeBucket
let allowedTumebucketForPeriodAll
@@ -329,11 +335,6 @@ export class AnalyticsController {
safeTimezone,
diff,
)
- await this.analyticsService.checkProjectAccess(
- pid,
- uid,
- headers['x-password'],
- )
let queryCustoms = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`
let subQuery = `FROM ${
@@ -515,6 +516,12 @@ export class AnalyticsController {
this.analyticsService.validatePeriod(period)
}
+ await this.analyticsService.checkProjectAccess(
+ pid,
+ uid,
+ headers['x-password'],
+ )
+
let newTimeBucket = timeBucket
let allowedTumebucketForPeriodAll
let diff
@@ -547,11 +554,6 @@ export class AnalyticsController {
safeTimezone,
diff,
)
- await this.analyticsService.checkProjectAccess(
- pid,
- uid,
- headers['x-password'],
- )
const subQuery = `FROM performance WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
@@ -668,6 +670,12 @@ export class AnalyticsController {
this.analyticsService.validatePeriod(period)
}
+ await this.analyticsService.checkProjectAccess(
+ pid,
+ uid,
+ headers['x-password'],
+ )
+
let diff
if (period === 'all') {
@@ -680,12 +688,6 @@ export class AnalyticsController {
diff = res.diff
}
- await this.analyticsService.checkProjectAccess(
- pid,
- uid,
- headers['x-password'],
- )
-
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
from,
@@ -1165,6 +1167,12 @@ export class AnalyticsController {
this.analyticsService.validatePeriod(period)
}
+ await this.analyticsService.checkProjectAccess(
+ pid,
+ uid,
+ headers['x-password'],
+ )
+
let newTimeBucket = timeBucket
let diff
let timeBucketForAllTime
@@ -1187,11 +1195,6 @@ export class AnalyticsController {
this.analyticsService.validateTimebucket(newTimeBucket)
const [filtersQuery, filtersParams, appliedFilters] =
this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)
- await this.analyticsService.checkProjectAccess(
- pid,
- uid,
- headers['x-password'],
- )
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
diff --git a/apps/production/src/common/constants.ts b/apps/production/src/common/constants.ts
index a93e54a1c..d7b085967 100644
--- a/apps/production/src/common/constants.ts
+++ b/apps/production/src/common/constants.ts
@@ -44,6 +44,7 @@ const clickhouse = new ClickHouse({
const { JWT_ACCESS_TOKEN_SECRET } = process.env
const isDevelopment = process.env.NODE_ENV === 'development'
+const isProxiedByCloudflare = process.env.CLOUDFLARE_PROXY_ENABLED === 'true'
const PRODUCTION_ORIGIN = process.env.CLIENT_URL || 'https://swetrix.com'
const { TWO_FACTOR_AUTHENTICATION_APP_NAME } = process.env
@@ -193,4 +194,5 @@ export {
CAPTCHA_COLUMNS,
PERFORMANCE_COLUMNS,
sentryIgnoreErrors,
+ isProxiedByCloudflare,
}
diff --git a/apps/production/src/common/utils.ts b/apps/production/src/common/utils.ts
index caa3624c7..62ec58874 100644
--- a/apps/production/src/common/utils.ts
+++ b/apps/production/src/common/utils.ts
@@ -12,7 +12,7 @@ import * as _find from 'lodash/find'
import * as _round from 'lodash/round'
import * as _split from 'lodash/split'
-import { redis, isDevelopment } from './constants'
+import { redis, isDevelopment, isProxiedByCloudflare } from './constants'
const marketingTips = {
en: [
@@ -209,11 +209,9 @@ const getGeoDetails = (ip: string, tz?: string): IPGeoDetails => {
}
}
-const CLOUDFLARE_ENABLED = false
-
const getIPFromHeaders = (headers: any) => {
- if (CLOUDFLARE_ENABLED) {
- return headers['cf-connecting-ip'] || null
+ if (isProxiedByCloudflare && headers['cf-connecting-ip']) {
+ return headers['cf-connecting-ip']
}
// Get IP based on the NGINX configuration
diff --git a/apps/selfhosted/src/analytics/analytics.controller.ts b/apps/selfhosted/src/analytics/analytics.controller.ts
index 1d2d8798f..34740dfba 100644
--- a/apps/selfhosted/src/analytics/analytics.controller.ts
+++ b/apps/selfhosted/src/analytics/analytics.controller.ts
@@ -4,11 +4,11 @@ import * as _pick from 'lodash/pick'
import * as _map from 'lodash/map'
import * as _uniqBy from 'lodash/uniqBy'
import * as _round from 'lodash/round'
+import * as _includes from 'lodash/includes'
import * as dayjs from 'dayjs'
import * as utc from 'dayjs/plugin/utc'
import * as dayjsTimezone from 'dayjs/plugin/timezone'
import { hash } from 'blake3'
-import ct from 'countries-and-timezones'
import {
Controller,
Body,
@@ -34,15 +34,16 @@ import {
AnalyticsService,
getSessionKey,
getHeartbeatKey,
+ validPeriods,
DataType,
} from './analytics.service'
-import { TaskManagerService } from '../task-manager/task-manager.service'
import { CurrentUserId } from '../auth/decorators/current-user-id.decorator'
import { DEFAULT_TIMEZONE } from '../user/entities/user.entity'
import { RolesGuard } from '../auth/guards/roles.guard'
import { PageviewsDTO } from './dto/pageviews.dto'
import { EventsDTO } from './dto/events.dto'
import { AnalyticsGET_DTO } from './dto/getData.dto'
+import { GetFiltersDto } from './dto/get-filters.dto'
import { GetUserFlowDTO } from './dto/getUserFlow.dto'
import { AppLoggerService } from '../logger/logger.service'
import {
@@ -54,6 +55,7 @@ import {
REDIS_SESSION_SALT_KEY,
clickhouse,
} from '../common/constants'
+import { getGeoDetails, getIPFromHeaders } from '../common/utils'
import { BotDetection } from '../common/decorators/bot-detection.decorator'
import { BotDetectionGuard } from '../common/guards/bot-detection.guard'
import { GetCustomEventsDto } from './dto/get-custom-events.dto'
@@ -87,6 +89,8 @@ const analyticsDTO = (
me: string,
ca: string,
cc: string,
+ rg: string,
+ ct: string,
sdur: number | string,
unique: number,
): Array<string | number> => {
@@ -104,6 +108,8 @@ const analyticsDTO = (
me,
ca,
cc,
+ rg,
+ ct,
sdur,
unique,
dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
@@ -116,6 +122,8 @@ const performanceDTO = (
dv: string,
br: string,
cc: string,
+ rg: string,
+ ct: string,
dns: number,
tls: number,
conn: number,
@@ -131,6 +139,8 @@ const performanceDTO = (
dv,
br,
cc,
+ rg,
+ ct,
_round(dns),
_round(tls),
_round(conn),
@@ -156,6 +166,8 @@ const customLogDTO = (
me: string,
ca: string,
cc: string,
+ rg: string,
+ ct: string,
): Array<string | number> => {
return [
pid,
@@ -170,6 +182,8 @@ const customLogDTO = (
me,
ca,
cc,
+ rg,
+ ct,
dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
]
}
@@ -247,7 +261,6 @@ export class AnalyticsController {
constructor(
private readonly analyticsService: AnalyticsService,
private readonly logger: AppLoggerService,
- private readonly taskManagerService: TaskManagerService,
) {}
@Get()
@@ -271,7 +284,29 @@ export class AnalyticsController {
this.analyticsService.validatePeriod(period)
}
- this.analyticsService.validateTimebucket(timeBucket)
+ await this.analyticsService.checkProjectAccess(pid, uid)
+
+ let newTimebucket = timeBucket
+ let allowedTumebucketForPeriodAll
+
+ let diff
+
+ if (period === 'all') {
+ const res = await this.analyticsService.getTimeBucketForAllTime(
+ pid,
+ period,
+ timezone,
+ )
+
+ diff = res.diff
+ // eslint-disable-next-line prefer-destructuring
+ newTimebucket = _includes(res.timeBucket, timeBucket)
+ ? timeBucket
+ : res.timeBucket[0]
+ allowedTumebucketForPeriodAll = res.timeBucket
+ }
+
+ this.analyticsService.validateTimebucket(newTimebucket)
const [filtersQuery, filtersParams, parsedFilters, customEVFilterApplied] =
this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)
@@ -280,11 +315,11 @@ export class AnalyticsController {
this.analyticsService.getGroupFromTo(
from,
to,
- timeBucket,
+ newTimebucket,
period,
safeTimezone,
+ diff,
)
- await this.analyticsService.checkProjectAccess(pid, uid)
let queryCustoms = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`
let subQuery = `FROM analytics WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
@@ -305,7 +340,7 @@ export class AnalyticsController {
}
const result = await this.analyticsService.groupByTimeBucket(
- timeBucket,
+ newTimebucket,
groupFrom,
groupTo,
subQuery,
@@ -325,9 +360,24 @@ export class AnalyticsController {
...result,
customs,
appliedFilters: parsedFilters,
+ timeBucket: allowedTumebucketForPeriodAll,
}
}
+ @Get('filters')
+ @Auth([], true, true)
+ async getFilters(
+ @Query() data: GetFiltersDto,
+ @CurrentUserId() uid: string,
+ ): Promise<any> {
+ const { pid, type } = data
+ this.analyticsService.validatePID(pid)
+
+ await this.analyticsService.checkProjectAccess(pid, uid)
+
+ return this.analyticsService.getFilters(pid, type)
+ }
+
@Get('chart')
@Auth([], true, true)
async getChartData(
@@ -416,7 +466,28 @@ export class AnalyticsController {
this.analyticsService.validatePeriod(period)
}
- this.analyticsService.validateTimebucket(timeBucket)
+ await this.analyticsService.checkProjectAccess(pid, uid)
+
+ let newTimeBucket = timeBucket
+ let allowedTumebucketForPeriodAll
+ let diff
+
+ if (period === 'all') {
+ const res = await this.analyticsService.getTimeBucketForAllTime(
+ pid,
+ period,
+ timezone,
+ )
+
+ diff = res.diff
+ // eslint-disable-next-line prefer-destructuring
+ newTimeBucket = _includes(res.timeBucket, timeBucket)
+ ? timeBucket
+ : res.timeBucket[0]
+ allowedTumebucketForPeriodAll = res.timeBucket
+ }
+
+ this.analyticsService.validateTimebucket(newTimeBucket)
const [filtersQuery, filtersParams, parsedFilters] =
this.analyticsService.getFiltersQuery(filters, DataType.PERFORMANCE)
@@ -424,11 +495,11 @@ export class AnalyticsController {
const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
from,
to,
- timeBucket,
+ newTimeBucket,
period,
safeTimezone,
+ diff,
)
- await this.analyticsService.checkProjectAccess(pid, uid)
const subQuery = `FROM performance WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
@@ -442,7 +513,7 @@ export class AnalyticsController {
}
const result = await this.analyticsService.groupPerfByTimeBucket(
- timeBucket,
+ newTimeBucket,
groupFrom,
groupTo,
subQuery,
@@ -454,6 +525,7 @@ export class AnalyticsController {
return {
...result,
appliedFilters: parsedFilters,
+ timeBucket: allowedTumebucketForPeriodAll,
}
}
@@ -531,6 +603,18 @@ export class AnalyticsController {
await this.analyticsService.checkProjectAccess(pid, uid)
+ let diff
+
+ if (period === 'all') {
+ const res = await this.analyticsService.getTimeBucketForAllTime(
+ pid,
+ period,
+ timezone,
+ )
+
+ diff = res.diff
+ }
+
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
from,
@@ -538,6 +622,7 @@ export class AnalyticsController {
null,
period,
safeTimezone,
+ diff,
)
const [filtersQuery, filtersParams, parsedFilters] =
@@ -653,8 +738,7 @@ export class AnalyticsController {
): Promise<any> {
const { 'user-agent': userAgent, origin } = headers
- const ip =
- headers['cf-connecting-ip'] || headers['x-forwarded-for'] || reqIP || ''
+ const ip = getIPFromHeaders(headers) || reqIP || ''
await this.analyticsService.validate(eventsDTO, origin, 'custom', ip)
@@ -680,10 +764,12 @@ export class AnalyticsController {
const dv = ua.device.type || 'desktop'
const br = ua.browser.name
const os = ua.os.name
- // trying to get country from timezome, otherwise using CloudFlare's IP based country code as a fallback
- const cc =
- ct.getCountryForTimezone(eventsDTO.tz)?.id ||
- (headers['cf-ipcountry'] === 'XX' ? 'NULL' : headers['cf-ipcountry'])
+
+ const {
+ city = 'NULL',
+ region = 'NULL',
+ country = 'NULL',
+ } = getGeoDetails(ip, eventsDTO.tz, headers)
const dto = customLogDTO(
eventsDTO.pid,
@@ -697,7 +783,9 @@ export class AnalyticsController {
eventsDTO.so,
eventsDTO.me,
eventsDTO.ca,
- cc,
+ country,
+ region,
+ city,
)
try {
@@ -722,8 +810,7 @@ export class AnalyticsController {
): Promise<any> {
const { 'user-agent': userAgent } = headers
const { pid } = logDTO
- const ip =
- headers['cf-connecting-ip'] || headers['x-forwarded-for'] || reqIP || ''
+ const ip = getIPFromHeaders(headers) || reqIP || ''
const sessionID = await this.analyticsService.getSessionHash(
pid,
@@ -752,8 +839,7 @@ export class AnalyticsController {
): Promise<any> {
const { 'user-agent': userAgent, origin } = headers
- const ip =
- headers['cf-connecting-ip'] || headers['x-forwarded-for'] || reqIP || ''
+ const ip = getIPFromHeaders(headers) || reqIP || ''
await this.analyticsService.validate(logDTO, origin, 'log', ip)
@@ -769,10 +855,11 @@ export class AnalyticsController {
)
}
- // trying to get country from timezome, otherwise using CloudFlare's IP based country code as a fallback
- const cc =
- ct.getCountryForTimezone(logDTO.tz)?.id ||
- (headers['cf-ipcountry'] === 'XX' ? 'NULL' : headers['cf-ipcountry'])
+ const {
+ city = 'NULL',
+ region = 'NULL',
+ country = 'NULL',
+ } = getGeoDetails(ip, logDTO.tz, headers)
const ua = UAParser(userAgent)
const dv = ua.device.type || 'desktop'
@@ -792,7 +879,9 @@ export class AnalyticsController {
logDTO.so,
logDTO.me,
logDTO.ca,
- cc,
+ country,
+ region,
+ city,
0,
Number(unique),
)
@@ -816,7 +905,9 @@ export class AnalyticsController {
logDTO.pg,
dv,
br,
- cc,
+ country,
+ region,
+ city,
dns,
tls,
conn,
@@ -869,8 +960,7 @@ export class AnalyticsController {
await this.analyticsService.validate(logDTO, origin)
- const ip =
- headers['cf-connecting-ip'] || headers['x-forwarded-for'] || reqIP || ''
+ const ip = getIPFromHeaders(headers) || reqIP || ''
const salt = await redis.get(REDIS_SESSION_SALT_KEY)
const sessionHash = getSessionKey(ip, userAgent, logDTO.pid, salt)
const unique = await this.analyticsService.isUnique(sessionHash)
@@ -882,9 +972,12 @@ export class AnalyticsController {
const br = ua.browser.name
const os = ua.os.name
- // using CloudFlare's IP based country code
- const cc =
- headers['cf-ipcountry'] === 'XX' ? 'NULL' : headers['cf-ipcountry']
+ const {
+ city = 'NULL',
+ region = 'NULL',
+ country = 'NULL',
+ } = getGeoDetails(ip, null, headers)
+
const dto = analyticsDTO(
sessionHash,
logDTO.pid,
@@ -898,7 +991,9 @@ export class AnalyticsController {
'NULL',
'NULL',
'NULL',
- cc,
+ city,
+ region,
+ country,
0,
Number(unique),
)
@@ -936,18 +1031,39 @@ export class AnalyticsController {
this.analyticsService.validatePeriod(period)
}
- this.analyticsService.validateTimebucket(timeBucket)
+ await this.analyticsService.checkProjectAccess(pid, uid)
+
+ let newTimeBucket = timeBucket
+ let diff
+ let timeBucketForAllTime
+
+ if (period === validPeriods[validPeriods.length - 1]) {
+ const res = await this.analyticsService.getTimeBucketForAllTime(
+ pid,
+ period,
+ timezone,
+ )
+
+ // eslint-disable-next-line prefer-destructuring
+ newTimeBucket = _includes(res.timeBucket, timeBucket)
+ ? timeBucket
+ : res.timeBucket[0]
+ diff = res.diff
+ timeBucketForAllTime = res.timeBucket
+ }
+
+ this.analyticsService.validateTimebucket(newTimeBucket)
const [filtersQuery, filtersParams, parsedFilters] =
this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)
- await this.analyticsService.checkProjectAccess(pid, uid)
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
from,
to,
- timeBucket,
+ newTimeBucket,
period,
safeTimezone,
+ diff,
)
const paramsData = {
@@ -960,7 +1076,7 @@ export class AnalyticsController {
}
const result: any = await this.analyticsService.groupCustomEVByTimeBucket(
- timeBucket,
+ newTimeBucket,
groupFrom,
groupTo,
filtersQuery,
@@ -990,6 +1106,7 @@ export class AnalyticsController {
return {
...result,
appliedFilters: parsedFilters,
+ timeBucket: timeBucketForAllTime,
}
}
}
diff --git a/apps/selfhosted/src/analytics/analytics.service.ts b/apps/selfhosted/src/analytics/analytics.service.ts
index d5cc74225..7de478980 100644
--- a/apps/selfhosted/src/analytics/analytics.service.ts
+++ b/apps/selfhosted/src/analytics/analytics.service.ts
@@ -37,6 +37,8 @@ import {
UNIQUE_SESSION_LIFE_TIME,
clickhouse,
REDIS_SESSION_SALT_KEY,
+ TRAFFIC_COLUMNS,
+ PERFORMANCE_COLUMNS,
} from '../common/constants'
import {
calculateRelativePercentage,
@@ -82,11 +84,8 @@ const GMT_0_TIMEZONES = [
// 'Africa/Casablanca',
]
-const cols = ['cc', 'pg', 'lc', 'br', 'os', 'dv', 'ref', 'so', 'me', 'ca']
-
-const perfColumns = ['cc', 'pg', 'dv', 'br']
-
-const validPeriods = [
+export const validPeriods = [
+ '1h',
'today',
'yesterday',
'1d',
@@ -95,17 +94,24 @@ const validPeriods = [
'3M',
'12M',
'24M',
+ 'all',
]
const validTimebuckets = [
+ TimeBucketType.MINUTE,
TimeBucketType.HOUR,
TimeBucketType.DAY,
TimeBucketType.MONTH,
+ TimeBucketType.YEAR,
]
// mapping of allowed timebuckets per difference between days
// (e.g. if difference is lower than (lt) (including) -> then the specified timebuckets are allowed to be applied)
const timeBucketToDays = [
+ {
+ lt: 0,
+ tb: [TimeBucketType.MINUTE, TimeBucketType.HOUR],
+ }, // < 1 day
{
lt: 7,
tb: [TimeBucketType.HOUR, TimeBucketType.DAY, TimeBucketType.MONTH],
@@ -116,6 +122,8 @@ const timeBucketToDays = [
}, // 4 weeks
{ lt: 366, tb: [TimeBucketType.MONTH] }, // 12 months
{ lt: 732, tb: [TimeBucketType.MONTH] }, // 24 months
+ { lt: 1464, tb: [TimeBucketType.MONTH, TimeBucketType.YEAR] }, // 48 months
+ { lt: 99999, tb: [TimeBucketType.YEAR] },
]
// Smaller than 64 characters, must start with an English letter and contain only letters (a-z A-Z), numbers (0-9), underscores (_) and dots (.)
@@ -123,9 +131,11 @@ const timeBucketToDays = [
const customEVvalidate = /^[a-zA-Z](?:[\w\.]){0,62}$/
const timeBucketConversion = {
+ minute: 'toStartOfMinute',
hour: 'toStartOfHour',
day: 'toStartOfDay',
month: 'toStartOfMonth',
+ year: 'toStartOfYear',
}
const isValidTimezone = (timezone: string): boolean => {
@@ -353,10 +363,10 @@ export class AnalyticsService {
getDataTypeColumns(dataType: DataType): string[] {
if (dataType === DataType.ANALYTICS) {
- return cols
+ return TRAFFIC_COLUMNS
}
- return perfColumns
+ return PERFORMANCE_COLUMNS
}
getGroupFromTo(
@@ -365,6 +375,7 @@ export class AnalyticsService {
timeBucket: TimeBucketType | null,
period: string,
safeTimezone: string,
+ diff?: number,
): IGetGroupFromTo {
let groupFrom: dayjs.Dayjs
let groupTo: dayjs.Dayjs
@@ -434,8 +445,14 @@ export class AnalyticsService {
} else if (period === 'yesterday') {
groupFrom = djsNow.subtract(1, 'day').startOf('d')
groupTo = djsNow.subtract(1, 'day').endOf('d')
+ } else if (period === 'all' && (diff === 0 || diff === 1)) {
+ groupFrom = djsNow.subtract(1, 'day').startOf('d')
+ groupTo = djsNow
+ } else if (period === 'all') {
+ groupFrom = djsNow.subtract(diff - 1, 'day').startOf(timeBucket)
+ groupTo = djsNow
} else {
- if (period === '1d') {
+ if (period === '1d' || period === '1h') {
groupFrom = djsNow.subtract(parseInt(period, 10), _last(period))
} else {
groupFrom = djsNow.subtract(parseInt(period, 10) - 1, _last(period))
@@ -588,6 +605,51 @@ export class AnalyticsService {
}
}
+ async getTimeBucketForAllTime(
+ pid: string,
+ period: string,
+ safeTimezone: string,
+ ): Promise<{
+ timeBucket: TimeBucketType[]
+ diff: number
+ }> {
+ if (period !== 'all') {
+ return null
+ }
+
+ const from: any = await clickhouse
+ .query(
+ 'SELECT created as from FROM analytics where pid = {pid:FixedString(12)} ORDER BY created ASC LIMIT 1',
+ { params: { pid } },
+ )
+ .toPromise()
+ const to = _includes(GMT_0_TIMEZONES, safeTimezone)
+ ? dayjs.utc()
+ : dayjs().tz(safeTimezone)
+
+ let newTimeBucket = [TimeBucketType.MONTH]
+ let diff = null
+
+ if (from && to) {
+ diff = dayjs(to).diff(dayjs(from[0].from), 'days')
+
+ const tbMap = _find(timeBucketToDays, ({ lt }) => diff <= lt)
+
+ if (_isEmpty(tbMap)) {
+ throw new PreconditionFailedException(
+ "The difference between 'from' and 'to' is greater than allowed",
+ )
+ }
+
+ newTimeBucket = tbMap.tb
+ }
+
+ return {
+ timeBucket: newTimeBucket,
+ diff,
+ }
+ }
+
postProcessParsedFilters(parsedFilters: any[]): any[] {
return _reduce(
parsedFilters,
@@ -824,6 +886,21 @@ export class AnalyticsService {
return result
}
+ async getFilters(pid: string, type: string): Promise<Array<string>> {
+ if (!_includes(TRAFFIC_COLUMNS, type)) {
+ throw new UnprocessableEntityException(
+ `The provided type (${type}) is incorrect`,
+ )
+ }
+
+ const query = `SELECT ${type} FROM analytics WHERE pid={pid:FixedString(12)} AND ${type} IS NOT NULL GROUP BY ${type}`
+ const results = await clickhouse
+ .query(query, { params: { pid } })
+ .toPromise()
+
+ return _map(results, type)
+ }
+
async generateParams(
parsedFilters: Array<{ [key: string]: string }>,
subQuery: string,
@@ -843,10 +920,10 @@ export class AnalyticsService {
),
)
- let columns = cols
+ let columns = TRAFFIC_COLUMNS
if (isPerformance) {
- columns = perfColumns
+ columns = PERFORMANCE_COLUMNS
}
const paramsPromises = _map(columns, async col => {
@@ -859,24 +936,7 @@ export class AnalyticsService {
)
const res = await clickhouse.query(query, paramsData).toPromise()
- params[col] = {}
-
- const size = _size(res)
-
- if (isPerformance) {
- for (let j = 0; j < size; ++j) {
- const key = res[j][col]
- params[col][key] = _round(
- millisecondsToSeconds(res[j]['avg(pageLoad)']),
- 2,
- )
- }
- } else {
- for (let j = 0; j < size; ++j) {
- const key = res[j][col]
- params[col][key] = res[j]['count()']
- }
- }
+ params[col] = res
})
await Promise.all(paramsPromises)
@@ -912,6 +972,7 @@ export class AnalyticsService {
let format
switch (timeBucket) {
+ case TimeBucketType.MINUTE:
case TimeBucketType.HOUR:
format = 'YYYY-MM-DD HH:mm:ss'
break
@@ -924,6 +985,10 @@ export class AnalyticsService {
format = 'YYYY-MM'
break
+ case TimeBucketType.YEAR:
+ format = 'YYYY'
+ break
+
default:
throw new BadRequestException(
`The provided time bucket (${timeBucket}) is incorrect`,
@@ -956,7 +1021,7 @@ export class AnalyticsService {
}
generateDateString(row: { [key: string]: number }): string {
- const { year, month, day, hour } = row
+ const { year, month, day, hour, minute } = row
let dateString = `${year}-${month < 10 ? `0${month}` : month}`
@@ -969,10 +1034,17 @@ export class AnalyticsService {
}
if (typeof hour === 'number') {
+ const strMinute =
+ typeof minute === 'number'
+ ? minute < 10
+ ? `0${minute}`
+ : minute
+ : '00'
+
if (hour < 10) {
- dateString += ` 0${hour}:00:00`
+ dateString += ` 0${hour}:${strMinute}:00`
} else {
- dateString += ` ${hour}:00:00`
+ dateString += ` ${hour}:${strMinute}:00`
}
}
@@ -1033,6 +1105,17 @@ export class AnalyticsService {
}
getGroupSubquery(timeBucket: TimeBucketType): [string, string] {
+ if (timeBucket === TimeBucketType.MINUTE) {
+ return [
+ `toYear(tz_created) as year,
+ toMonth(tz_created) as month,
+ toDayOfMonth(tz_created) as day,
+ toHour(tz_created) as hour,
+ toMinute(tz_created) as minute`,
+ 'year, month, day, hour, minute',
+ ]
+ }
+
if (timeBucket === TimeBucketType.HOUR) {
return [
`toYear(tz_created) as year,
@@ -1051,6 +1134,10 @@ export class AnalyticsService {
]
}
+ if (timeBucket === TimeBucketType.YEAR) {
+ return [`toYear(tz_created) as year`, 'year']
+ }
+
return [
`toYear(tz_created) as year,
toMonth(tz_created) as month,
@@ -1439,6 +1526,8 @@ export class AnalyticsService {
const timeBucketFunc = timeBucketConversion[timeBucket]
const [selector, groupBy] = this.getGroupSubquery(timeBucket)
+ const tzFromDate = `toTimeZone(parseDateTimeBestEffort({groupFrom:String}), '${safeTimezone}')`
+ const tzToDate = `toTimeZone(parseDateTimeBestEffort({groupTo:String}), '${safeTimezone}')`
const query = `
SELECT
@@ -1447,11 +1536,11 @@ export class AnalyticsService {
count() as count
FROM (
SELECT *,
- ${timeBucketFunc}(created) as tz_created
+ ${timeBucketFunc}(toTimeZone(created, '${safeTimezone}')) as tz_created
FROM customEV
WHERE
pid = {pid:FixedString(12)}
- AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ AND created BETWEEN ${tzFromDate} AND ${tzToDate}
${filtersQuery}
) as subquery
GROUP BY ${groupBy}, ev
diff --git a/apps/selfhosted/src/analytics/dto/get-filters.dto.ts b/apps/selfhosted/src/analytics/dto/get-filters.dto.ts
new file mode 100644
index 000000000..408bf0fc5
--- /dev/null
+++ b/apps/selfhosted/src/analytics/dto/get-filters.dto.ts
@@ -0,0 +1,20 @@
+import { ApiProperty } from '@nestjs/swagger'
+import { IsNotEmpty } from 'class-validator'
+
+export class GetFiltersDto {
+ @ApiProperty({
+ example: 'aUn1quEid-3',
+ required: true,
+ description: 'The project ID',
+ })
+ @IsNotEmpty()
+ pid: string
+
+ @ApiProperty({
+ example: 'br',
+ required: true,
+ description: 'Params type. Example: br, os, cc, etc.',
+ })
+ @IsNotEmpty()
+ type: string
+}
diff --git a/apps/selfhosted/src/analytics/dto/getData.dto.ts b/apps/selfhosted/src/analytics/dto/getData.dto.ts
index 53a39d5bc..c51c3d227 100644
--- a/apps/selfhosted/src/analytics/dto/getData.dto.ts
+++ b/apps/selfhosted/src/analytics/dto/getData.dto.ts
@@ -3,9 +3,11 @@ import { IsNotEmpty } from 'class-validator'
import { DEFAULT_TIMEZONE } from '../../user/entities/user.entity'
export enum TimeBucketType {
+ MINUTE = 'minute',
HOUR = 'hour',
DAY = 'day',
MONTH = 'month',
+ YEAR = 'year',
}
// eslint-disable-next-line @typescript-eslint/naming-convention
diff --git a/apps/selfhosted/src/auth/auth.controller.ts b/apps/selfhosted/src/auth/auth.controller.ts
index f335e8853..6d46eedbf 100644
--- a/apps/selfhosted/src/auth/auth.controller.ts
+++ b/apps/selfhosted/src/auth/auth.controller.ts
@@ -16,7 +16,7 @@ import {
import { ApiTags, ApiOperation, ApiOkResponse } from '@nestjs/swagger'
import { I18nValidationExceptionFilter, I18n, I18nContext } from 'nestjs-i18n'
-import { checkRateLimit } from '../common/utils'
+import { checkRateLimit, getIPFromHeaders } from '../common/utils'
import { generateSelfhostedUser } from '../user/entities/user.entity'
import { AuthService } from './auth.service'
import { Public, CurrentUserId, CurrentUser } from './decorators'
@@ -50,10 +50,9 @@ export class AuthController {
@Body() body: LoginRequestDto,
@I18n() i18n: I18nContext,
@Headers() headers: unknown,
- @Ip() requestIp: string,
+ @Ip() reqIP: string,
): Promise<LoginResponseDto> {
- const ip =
- headers['x-forwarded-for'] || headers['cf-connecting-ip'] || requestIp
+ const ip = getIPFromHeaders(headers) || reqIP || ''
await checkRateLimit(ip, 'login', 10, 1800)
diff --git a/apps/selfhosted/src/common/constants.ts b/apps/selfhosted/src/common/constants.ts
index 1285c4caf..f9cb8de4c 100644
--- a/apps/selfhosted/src/common/constants.ts
+++ b/apps/selfhosted/src/common/constants.ts
@@ -44,6 +44,7 @@ const clickhouse = new ClickHouse({
})
const { JWT_ACCESS_TOKEN_SECRET } = process.env
+const isProxiedByCloudflare = process.env.CLOUDFLARE_PROXY_ENABLED === 'true'
const isDevelopment = process.env.NODE_ENV === 'development'
const SELFHOSTED_EMAIL = process.env.EMAIL
@@ -80,6 +81,23 @@ const UNIQUE_SESSION_LIFE_TIME = 1800
// 35 seconds
const HEARTBEAT_SID_LIFE_TIME = 35
+const TRAFFIC_COLUMNS = [
+ 'cc',
+ 'rg',
+ 'ct',
+ 'pg',
+ 'lc',
+ 'br',
+ 'os',
+ 'dv',
+ 'ref',
+ 'so',
+ 'me',
+ 'ca',
+]
+
+const PERFORMANCE_COLUMNS = ['cc', 'rg', 'ct', 'pg', 'dv', 'br']
+
export {
clickhouse,
redis,
@@ -104,4 +122,7 @@ export {
isDevelopment,
DEFAULT_SELFHOSTED_UUID,
JWT_ACCESS_TOKEN_SECRET,
+ TRAFFIC_COLUMNS,
+ PERFORMANCE_COLUMNS,
+ isProxiedByCloudflare,
}
diff --git a/apps/selfhosted/src/common/utils.ts b/apps/selfhosted/src/common/utils.ts
index 2567d8209..33cd7ec16 100644
--- a/apps/selfhosted/src/common/utils.ts
+++ b/apps/selfhosted/src/common/utils.ts
@@ -1,4 +1,5 @@
import { NotFoundException, HttpException } from '@nestjs/common'
+import timezones from 'countries-and-timezones'
import { hash } from 'blake3'
import { v5 as uuidv5 } from 'uuid'
import * as _join from 'lodash/join'
@@ -7,6 +8,7 @@ import * as _values from 'lodash/values'
import * as _reduce from 'lodash/reduce'
import * as _keys from 'lodash/keys'
import * as _toNumber from 'lodash/toNumber'
+import * as _split from 'lodash/split'
import * as _isEmpty from 'lodash/isEmpty'
import * as _head from 'lodash/head'
import * as _round from 'lodash/round'
@@ -21,6 +23,7 @@ import {
SELFHOSTED_EMAIL,
UUIDV5_NAMESPACE,
isDevelopment,
+ isProxiedByCloudflare,
} from './constants'
import { Project } from '../project/entity/project.entity'
@@ -129,7 +132,7 @@ const deleteProjectClickhouse = async id => {
return clickhouse.query(query).toPromise()
}
-const createProjectClickhouse = async (project: Project) => {
+const createProjectClickhouse = async (project: Partial<Project>) => {
const paramsData = {
params: {
...project,
@@ -281,6 +284,71 @@ const getSelfhostedUUID = (): string => {
}
}
+interface IPGeoDetails {
+ country?: string
+ region?: string
+ city?: string
+}
+
+const getGeoDetails = (
+ ip: string,
+ tz?: string,
+ headers?: unknown,
+): IPGeoDetails => {
+ // TODO: Add support for DBIP for self-hosted
+
+ // Stage 1: Using IP address based geo lookup
+ // const data = lookup.get(ip)
+
+ // const country = data?.country?.iso_code
+ // // TODO: Add city overrides, for example, Colinton -> Edinburgh, etc.
+ // const city = data?.city?.names?.en
+ // const region = data?.subdivisions?.[0]?.names?.en
+
+ // if (country) {
+ // return {
+ // country,
+ // city,
+ // region,
+ // }
+ // }
+
+ // Stage 2: If Cloudflare is enabled, use their headers
+ if (isProxiedByCloudflare && headers?.['cf-ipcountry'] !== 'XX') {
+ return headers['cf-ipcountry']
+ }
+
+ // Stage 3: Using timezone based geo lookup as a fallback
+ const tzCountry = timezones.getCountryForTimezone(tz)?.id || null
+
+ return {
+ country: tzCountry,
+ city: null,
+ region: null,
+ }
+}
+
+const getIPFromHeaders = (headers: any) => {
+ if (isProxiedByCloudflare && headers['cf-connecting-ip']) {
+ return headers['cf-connecting-ip']
+ }
+
+ // Get IP based on the NGINX configuration
+ let ip = headers['x-real-ip']
+
+ if (ip) {
+ return ip
+ }
+
+ ip = headers['x-forwarded-for'] || null
+
+ if (!ip) {
+ return null
+ }
+
+ return _split(ip, ',')[0]
+}
+
export {
checkRateLimit,
createProjectClickhouse,
@@ -296,4 +364,6 @@ export {
updateUserClickhouse,
getUserClickhouse,
createUserClickhouse,
+ getGeoDetails,
+ getIPFromHeaders,
}
diff --git a/apps/selfhosted/src/project/project.controller.ts b/apps/selfhosted/src/project/project.controller.ts
index c27d4678a..f474f065b 100644
--- a/apps/selfhosted/src/project/project.controller.ts
+++ b/apps/selfhosted/src/project/project.controller.ts
@@ -110,7 +110,7 @@ export class ProjectController {
const project = new Project()
project.id = pid
- projectDTO.name = _trim(projectDTO.name)
+ project.name = _trim(projectDTO.name)
await createProjectClickhouse(project)
diff --git a/docker-compose.yml b/docker-compose.yml
index 80c43e980..4fae851fe 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -11,7 +11,7 @@ services:
environment:
- API_URL=http://localhost:8080/
- REACT_APP_AIAPI_URL=https://aiapi.swetrix.com/
- - REACT_APP_SELFHOSTED=true
+ - SELFHOSTED=true
# This is the analytics API
swetrix-api:
@@ -28,6 +28,7 @@ services:
- API_ORIGINS=
- PASSWORD=12345678
+ - CLOUDFLARE_PROXY_ENABLED=false
links:
- 'redis'
- 'clickhouse'