Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

initial selfhosted src

Blue Moused28ee47945a9
diff --git a/selfhosted-src/analytics/analytics.controller.ts b/selfhosted-src/analytics/analytics.controller.ts
new file mode 100644
index 000000000..a71997ed9
--- /dev/null
+++ b/selfhosted-src/analytics/analytics.controller.ts
@@ -0,0 +1,1069 @@
+import * as _isEmpty from 'lodash/isEmpty'
+import * as _isArray from 'lodash/isArray'
+import * as _toNumber from 'lodash/toNumber'
+import * as _pick from 'lodash/pick'
+import * as _map from 'lodash/map'
+import * as _uniqBy from 'lodash/uniqBy'
+import * as _round from 'lodash/round'
+import * as dayjs from 'dayjs'
+import * as utc from 'dayjs/plugin/utc'
+import * as dayjsTimezone from 'dayjs/plugin/timezone'
+import { hash } from 'blake3'
+import ct from 'countries-and-timezones'
+import {
+ Controller,
+ Body,
+ Query,
+ UseGuards,
+ Get,
+ Post,
+ Headers,
+ BadRequestException,
+ InternalServerErrorException,
+ UnprocessableEntityException,
+ Ip,
+ ForbiddenException,
+ Response,
+} from '@nestjs/common'
+import { ApiTags } from '@nestjs/swagger'
+import * as UAParser from 'ua-parser-js'
+import * as isbot from 'isbot'
+
+import { OptionalJwtAccessTokenGuard } from 'selfhosted-src/auth/guards'
+import { Auth, Public } from 'selfhosted-src/auth/decorators'
+import {
+ AnalyticsService,
+ getSessionKey,
+ getHeartbeatKey,
+ DataType,
+} from './analytics.service'
+import { TaskManagerService } from '../task-manager/task-manager.service'
+import { CurrentUserId } from '../auth/decorators/current-user-id.decorator'
+import { DEFAULT_TIMEZONE } from '../user/entities/user.entity'
+import { RolesGuard } from '../auth/guards/roles.guard'
+import { PageviewsDTO } from './dto/pageviews.dto'
+import { EventsDTO } from './dto/events.dto'
+import { AnalyticsGET_DTO } from './dto/getData.dto'
+import { GetUserFlowDTO } from './dto/getUserFlow.dto'
+import { AppLoggerService } from '../logger/logger.service'
+import {
+ REDIS_LOG_DATA_CACHE_KEY,
+ REDIS_LOG_PERF_CACHE_KEY,
+ redis,
+ REDIS_LOG_CUSTOM_CACHE_KEY,
+ HEARTBEAT_SID_LIFE_TIME,
+ REDIS_USERS_COUNT_KEY,
+ REDIS_PROJECTS_COUNT_KEY,
+ REDIS_PAGEVIEWS_COUNT_KEY,
+ REDIS_SESSION_SALT_KEY,
+ REDIS_PERFORMANCE_COUNT_KEY,
+ clickhouse,
+} from '../common/constants'
+import { BotDetection } from '../common/decorators/bot-detection.decorator'
+import { BotDetectionGuard } from '../common/guards/bot-detection.guard'
+import { GetCustomEventsDto } from './dto/get-custom-events.dto'
+import { IUserFlow } from './interfaces'
+
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+const mysql = require('mysql2')
+
+dayjs.extend(utc)
+dayjs.extend(dayjsTimezone)
+
+const getSessionKeyCustom = (
+ ip: string,
+ ua: string,
+ pid: string,
+ ev: string,
+ salt = '',
+) => `cses_${hash(`${ua}${ip}${pid}${ev}${salt}`).toString('hex')}`
+
+const analyticsDTO = (
+ sid: string,
+ pid: string,
+ pg: string,
+ prev: string,
+ dv: string,
+ br: string,
+ os: string,
+ lc: string,
+ ref: string,
+ so: string,
+ me: string,
+ ca: string,
+ cc: string,
+ sdur: number | string,
+ unique: number,
+): Array<string | number> => {
+ return [
+ sid,
+ pid,
+ pg,
+ prev,
+ dv,
+ br,
+ os,
+ lc,
+ ref,
+ so,
+ me,
+ ca,
+ cc,
+ sdur,
+ unique,
+ dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
+ ]
+}
+
+const performanceDTO = (
+ pid: string,
+ pg: string,
+ dv: string,
+ br: string,
+ cc: string,
+ dns: number,
+ tls: number,
+ conn: number,
+ response: number,
+ render: number,
+ domLoad: number,
+ pageLoad: number,
+ ttfb: number,
+): Array<string | number> => {
+ return [
+ pid,
+ pg,
+ dv,
+ br,
+ cc,
+ _round(dns),
+ _round(tls),
+ _round(conn),
+ _round(response),
+ _round(render),
+ _round(domLoad),
+ _round(pageLoad),
+ _round(ttfb),
+ dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
+ ]
+}
+
+const customLogDTO = (
+ pid: string,
+ ev: string,
+ pg: string,
+ dv: string,
+ br: string,
+ os: string,
+ lc: string,
+ ref: string,
+ so: string,
+ me: string,
+ ca: string,
+ cc: string,
+): Array<string | number> => {
+ return [
+ pid,
+ ev,
+ pg,
+ dv,
+ br,
+ os,
+ lc,
+ ref,
+ so,
+ me,
+ ca,
+ cc,
+ dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),
+ ]
+}
+
+export const getElValue = el => {
+ if (el === undefined || el === null || el === 'NULL') return 'NULL'
+ return mysql.escape(el)
+}
+
+// Performance object validator: none of the values cannot be bigger than 1000 * 60 * 5 (5 minutes) and are >= 0
+const MAX_PERFORMANCE_VALUE = 1000 * 60 * 5
+const isPerformanceValid = (perf: any) => {
+ return (
+ perf.dns <= MAX_PERFORMANCE_VALUE &&
+ perf.tls <= MAX_PERFORMANCE_VALUE &&
+ perf.conn <= MAX_PERFORMANCE_VALUE &&
+ perf.response <= MAX_PERFORMANCE_VALUE &&
+ perf.render <= MAX_PERFORMANCE_VALUE &&
+ perf.dom_load <= MAX_PERFORMANCE_VALUE &&
+ perf.page_load <= MAX_PERFORMANCE_VALUE &&
+ perf.ttfb <= MAX_PERFORMANCE_VALUE &&
+ perf.dns >= 0 &&
+ perf.tls >= 0 &&
+ perf.conn >= 0 &&
+ perf.response >= 0 &&
+ perf.render >= 0 &&
+ perf.dom_load >= 0 &&
+ perf.page_load >= 0 &&
+ perf.ttfb >= 0
+ )
+}
+
+const getPIDsArray = (pids, pid) => {
+ const pidsEmpty = _isEmpty(pids)
+ const pidEmpty = _isEmpty(pid)
+ if (pidsEmpty && pidEmpty)
+ throw new BadRequestException(
+ "An array of Project ID's (pids) or a Project ID (pid) has to be provided",
+ )
+ else if (!pidsEmpty && !pidEmpty)
+ throw new BadRequestException(
+ "Please provide either an array of Project ID's (pids) or a Project ID (pid), but not both",
+ )
+ else if (!pidEmpty) {
+ pids = JSON.stringify([pid])
+ }
+
+ try {
+ pids = JSON.parse(pids)
+ } catch (e) {
+ throw new UnprocessableEntityException(
+ "Cannot process the provided array of Project ID's",
+ )
+ }
+
+ if (!_isArray(pids)) {
+ throw new UnprocessableEntityException(
+ "An array of Project ID's has to be provided as a 'pids' param",
+ )
+ }
+
+ return pids
+}
+
+// needed for serving 1x1 px GIF
+const TRANSPARENT_GIF_BUFFER = Buffer.from(
+ 'R0lGODlhAQABAIAAAP///wAAACwAAAAAAQABAAACAkQBADs=',
+ 'base64',
+)
+
+@ApiTags('Analytics')
+@UseGuards(OptionalJwtAccessTokenGuard, RolesGuard)
+@Controller(['log', 'v1/log'])
+export class AnalyticsController {
+ constructor(
+ private readonly analyticsService: AnalyticsService,
+ private readonly logger: AppLoggerService,
+ private readonly taskManagerService: TaskManagerService,
+ ) {}
+
+ @Get()
+ @Auth([], true, true)
+ async getData(
+ @Query() data: AnalyticsGET_DTO,
+ @CurrentUserId() uid: string,
+ isCaptcha = false,
+ ): Promise<any> {
+ const {
+ pid,
+ period,
+ timeBucket,
+ from,
+ to,
+ filters,
+ timezone = DEFAULT_TIMEZONE,
+ } = data
+ this.analyticsService.validatePID(pid)
+
+ if (!_isEmpty(period)) {
+ this.analyticsService.validatePeriod(period)
+ }
+
+ this.analyticsService.validateTimebucket(timeBucket)
+ const [filtersQuery, filtersParams, parsedFilters] =
+ this.analyticsService.getFiltersQuery(
+ filters,
+ isCaptcha ? DataType.CAPTCHA : DataType.ANALYTICS,
+ )
+
+ const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
+ const { groupFrom, groupTo, groupFromUTC, groupToUTC } =
+ this.analyticsService.getGroupFromTo(
+ from,
+ to,
+ timeBucket,
+ period,
+ safeTimezone,
+ )
+ await this.analyticsService.checkProjectAccess(pid, uid)
+
+ let queryCustoms = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`
+ let subQuery = `FROM ${
+ isCaptcha ? 'captcha' : 'analytics'
+ } WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
+ let customEVFilterApplied = false
+
+ if (filtersParams?.ev && !isCaptcha) {
+ customEVFilterApplied = true
+ queryCustoms = `SELECT ev, count() FROM customEV WHERE ${
+ filtersParams.ev_exclusive ? 'NOT' : ''
+ } ev = {ev:String} AND pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`
+
+ subQuery = `FROM customEV WHERE ${
+ filtersParams.ev_exclusive ? 'NOT' : ''
+ } ev = {ev:String} AND pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
+ }
+
+ const paramsData = {
+ params: {
+ pid,
+ groupFrom: groupFromUTC,
+ groupTo: groupToUTC,
+ ...filtersParams,
+ },
+ }
+
+ let result: object | void
+
+ if (isCaptcha) {
+ result = await this.analyticsService.groupCaptchaByTimeBucket(
+ timeBucket,
+ groupFrom,
+ groupTo,
+ subQuery,
+ filtersQuery,
+ paramsData,
+ safeTimezone,
+ )
+ } else {
+ result = await this.analyticsService.groupByTimeBucket(
+ timeBucket,
+ groupFrom,
+ groupTo,
+ subQuery,
+ filtersQuery,
+ paramsData,
+ safeTimezone,
+ customEVFilterApplied,
+ parsedFilters,
+ )
+ }
+
+ if (isCaptcha) {
+ return {
+ ...result,
+ appliedFilters: parsedFilters,
+ }
+ }
+
+ const customs = await this.analyticsService.processCustomEV(
+ queryCustoms,
+ paramsData,
+ )
+
+ return {
+ ...result,
+ customs,
+ appliedFilters: parsedFilters,
+ }
+ }
+
+ @Get('chart')
+ @Auth([], true, true)
+ async getChartData(
+ @Query() data: AnalyticsGET_DTO,
+ @CurrentUserId() uid: string,
+ ): Promise<any> {
+ const {
+ pid,
+ period,
+ timeBucket,
+ from,
+ to,
+ filters,
+ timezone = DEFAULT_TIMEZONE,
+ } = data
+ this.analyticsService.validatePID(pid)
+
+ if (!_isEmpty(period)) {
+ this.analyticsService.validatePeriod(period)
+ }
+
+ this.analyticsService.validateTimebucket(timeBucket)
+ const [filtersQuery, filtersParams, parsedFilters] =
+ this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)
+
+ const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
+ const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
+ from,
+ to,
+ timeBucket,
+ period,
+ safeTimezone,
+ )
+ await this.analyticsService.checkProjectAccess(pid, uid)
+
+ let subQuery = `FROM analytics WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
+ let customEVFilterApplied = false
+
+ if (filtersParams?.ev) {
+ customEVFilterApplied = true
+ subQuery = `FROM customEV WHERE ${
+ filtersParams.ev_exclusive ? 'NOT' : ''
+ } ev = {ev:String} AND pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
+ }
+
+ const paramsData = {
+ params: {
+ pid,
+ groupFrom,
+ groupTo,
+ ...filtersParams,
+ },
+ }
+
+ const result = await this.analyticsService.groupChartByTimeBucket(
+ timeBucket,
+ groupFrom,
+ groupTo,
+ subQuery,
+ filtersQuery,
+ paramsData,
+ safeTimezone,
+ customEVFilterApplied,
+ )
+
+ return {
+ ...result,
+ appliedFilters: parsedFilters,
+ }
+ }
+
+ @Get('performance')
+ @Auth([], true, true)
+ async getPerfData(
+ @Query() data: AnalyticsGET_DTO,
+ @CurrentUserId() uid: string,
+ ): Promise<any> {
+ const {
+ pid,
+ period,
+ timeBucket,
+ from,
+ to,
+ filters,
+ timezone = DEFAULT_TIMEZONE,
+ } = data
+ this.analyticsService.validatePID(pid)
+
+ if (!_isEmpty(period)) {
+ this.analyticsService.validatePeriod(period)
+ }
+
+ this.analyticsService.validateTimebucket(timeBucket)
+ const [filtersQuery, filtersParams, parsedFilters] =
+ this.analyticsService.getFiltersQuery(filters, DataType.PERFORMANCE)
+
+ const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
+ const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
+ from,
+ to,
+ timeBucket,
+ period,
+ safeTimezone,
+ )
+ await this.analyticsService.checkProjectAccess(pid, uid)
+
+ const subQuery = `FROM performance WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`
+
+ const paramsData = {
+ params: {
+ pid,
+ groupFrom,
+ groupTo,
+ ...filtersParams,
+ },
+ }
+
+ const result = await this.analyticsService.groupPerfByTimeBucket(
+ timeBucket,
+ groupFrom,
+ groupTo,
+ subQuery,
+ filtersQuery,
+ paramsData,
+ safeTimezone,
+ )
+
+ return {
+ ...result,
+ appliedFilters: parsedFilters,
+ }
+ }
+
+ @Get('performance/chart')
+ @Auth([], true, true)
+ async getPerfChartData(
+ @Query() data: AnalyticsGET_DTO,
+ @CurrentUserId() uid: string,
+ ): Promise<any> {
+ const {
+ pid,
+ period,
+ timeBucket,
+ from,
+ to,
+ filters,
+ timezone = DEFAULT_TIMEZONE,
+ } = data
+ this.analyticsService.validatePID(pid)
+
+ if (!_isEmpty(period)) {
+ this.analyticsService.validatePeriod(period)
+ }
+
+ this.analyticsService.validateTimebucket(timeBucket)
+ const [filtersQuery, filtersParams, parsedFilters] =
+ this.analyticsService.getFiltersQuery(filters, DataType.PERFORMANCE)
+
+ const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
+ const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
+ from,
+ to,
+ timeBucket,
+ period,
+ safeTimezone,
+ )
+ await this.analyticsService.checkProjectAccess(pid, uid)
+
+ const paramsData = {
+ params: {
+ pid,
+ groupFrom,
+ groupTo,
+ ...filtersParams,
+ },
+ }
+
+ const chart = await this.analyticsService.getPerfChartData(
+ timeBucket,
+ from,
+ to,
+ filtersQuery,
+ paramsData,
+ safeTimezone,
+ )
+
+ return {
+ chart,
+ appliedFilters: parsedFilters,
+ }
+ }
+
+ @Get('captcha')
+ @Auth([], true, true)
+ async getCaptchaData(
+ @Query() data: AnalyticsGET_DTO,
+ @CurrentUserId() uid: string,
+ ): Promise<any> {
+ return this.getData(data, uid, true)
+ }
+
+ @Get('user-flow')
+ @Auth([], true, true)
+ async getUserFlow(
+ @Query() data: GetUserFlowDTO,
+ @CurrentUserId() uid: string,
+ ): Promise<IUserFlow | { appliedFilters: any[] }> {
+ const { pid, period, from, to, timezone = DEFAULT_TIMEZONE, filters } = data
+ this.analyticsService.validatePID(pid)
+
+ if (!_isEmpty(period)) {
+ this.analyticsService.validatePeriod(period)
+ }
+
+ await this.analyticsService.checkProjectAccess(pid, uid)
+
+ const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
+ const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
+ from,
+ to,
+ null,
+ period,
+ safeTimezone,
+ )
+
+ const [filtersQuery, filtersParams, parsedFilters] =
+ this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)
+
+ const params = {
+ pid,
+ groupFrom,
+ groupTo,
+ ...filtersParams,
+ }
+
+ const flow = await this.analyticsService.getUserFlow(params, filtersQuery)
+
+ return {
+ ...flow,
+ appliedFilters: parsedFilters,
+ }
+ }
+
+ @Get('birdseye')
+ @Auth([], true, true)
+ // returns overall short statistics per project
+ async getOverallStats(
+ @Query() data,
+ @CurrentUserId() uid: string,
+ ): Promise<any> {
+ const { pids, pid } = data
+ const pidsArray = getPIDsArray(pids, pid)
+
+ const validationPromises = _map(pidsArray, async currentPID => {
+ this.analyticsService.validatePID(currentPID)
+ await this.analyticsService.checkProjectAccess(currentPID, uid)
+ })
+
+ await Promise.all(validationPromises)
+
+ return this.analyticsService.getSummary(pidsArray, 'w')
+ }
+
+ @Get('captcha/birdseye')
+ @Auth([], true, true)
+ // returns overall short statistics per CAPTCHA project
+ async getCaptchaOverallStats(
+ @Query() data,
+ @CurrentUserId() uid: string,
+ ): Promise<any> {
+ const { pids, pid } = data
+ const pidsArray = getPIDsArray(pids, pid)
+
+ const validationPromises = _map(pidsArray, async currentPID => {
+ this.analyticsService.validatePID(currentPID)
+ await this.analyticsService.checkProjectAccess(currentPID, uid)
+ })
+
+ await Promise.all(validationPromises)
+
+ return this.analyticsService.getCaptchaSummary(pidsArray, 'w')
+ }
+
+ @Get('hb')
+ async getHeartBeatStats(
+ @Query() data,
+ @CurrentUserId() uid: string,
+ ): Promise<object> {
+ const { pids, pid } = data
+ const pidsArray = getPIDsArray(pids, pid)
+
+ const validationPromises = _map(pidsArray, async currentPID => {
+ this.analyticsService.validatePID(currentPID)
+ await this.analyticsService.checkProjectAccess(currentPID, uid)
+ })
+
+ await Promise.all(validationPromises)
+
+ const result = {}
+
+ const keyCountPromises = _map(pidsArray, async currentPID => {
+ result[currentPID] = await this.analyticsService.getOnlineUserCount(
+ currentPID,
+ )
+ })
+
+ await Promise.all(keyCountPromises).catch(reason => {
+ this.logger.error(`[GET /analytics/hb] ${reason}`)
+ throw new InternalServerErrorException(
+ 'An error occured while calculating heartbeat statistics',
+ )
+ })
+
+ return result
+ }
+
+ @Get('liveVisitors')
+ @Auth([], true, true)
+ async getLiveVisitors(
+ @Query() data,
+ @CurrentUserId() uid: string,
+ ): Promise<object> {
+ const { pid } = data
+
+ this.analyticsService.validatePID(pid)
+ await this.analyticsService.checkProjectAccess(pid, uid)
+
+ const keys = await redis.keys(`sd:*:${pid}`)
+
+ if (_isEmpty(keys)) {
+ return []
+ }
+
+ const sids = _map(keys, key => key.split(':')[1])
+
+ const query = `SELECT sid, dv, br, os, cc FROM analytics WHERE sid IN (${sids
+ .map(el => `'${el}'`)
+ .join(',')})`
+ const result = await clickhouse.query(query).toPromise()
+ const processed = _map(_uniqBy(result, 'sid'), el =>
+ _pick(el, ['dv', 'br', 'os', 'cc']),
+ )
+
+ return processed
+ }
+
+ // Log custom event
+ @Post('custom')
+ @UseGuards(BotDetectionGuard)
+ @BotDetection()
+ @Public()
+ async logCustom(
+ @Body() eventsDTO: EventsDTO,
+ @Headers() headers,
+ @Ip() reqIP,
+ ): Promise<any> {
+ const { 'user-agent': userAgent, origin } = headers
+
+ const ip =
+ headers['cf-connecting-ip'] || headers['x-forwarded-for'] || reqIP || ''
+
+ await this.analyticsService.validate(eventsDTO, origin, 'custom', ip)
+
+ if (eventsDTO.unique) {
+ const salt = await redis.get(REDIS_SESSION_SALT_KEY)
+ const sessionHash = getSessionKeyCustom(
+ ip,
+ userAgent,
+ eventsDTO.pid,
+ eventsDTO.ev,
+ salt,
+ )
+ const unique = await this.analyticsService.isUnique(sessionHash)
+
+ if (!unique) {
+ throw new ForbiddenException(
+ 'The unique option provided, while the custom event have already been created for this session',
+ )
+ }
+ }
+
+ const ua = UAParser(userAgent)
+ const dv = ua.device.type || 'desktop'
+ const br = ua.browser.name
+ const os = ua.os.name
+ // trying to get country from timezome, otherwise using CloudFlare's IP based country code as a fallback
+ const cc =
+ ct.getCountryForTimezone(eventsDTO.tz)?.id ||
+ (headers['cf-ipcountry'] === 'XX' ? 'NULL' : headers['cf-ipcountry'])
+
+ const dto = customLogDTO(
+ eventsDTO.pid,
+ eventsDTO.ev,
+ eventsDTO.pg,
+ dv,
+ br,
+ os,
+ eventsDTO.lc,
+ eventsDTO.ref,
+ eventsDTO.so,
+ eventsDTO.me,
+ eventsDTO.ca,
+ cc,
+ )
+
+ try {
+ const values = `(${dto.map(getElValue).join(',')})`
+ await redis.rpush(REDIS_LOG_CUSTOM_CACHE_KEY, values)
+ return
+ } catch (e) {
+ this.logger.error(e)
+ throw new InternalServerErrorException(
+ 'Error occured while saving the custom event',
+ )
+ }
+ }
+
+ @Post('hb')
+ @UseGuards(BotDetectionGuard)
+ @BotDetection()
+ @Auth([], true, true)
+ async heartbeat(
+ @Body() logDTO: PageviewsDTO,
+ @Headers() headers,
+ @Ip() reqIP,
+ ): Promise<any> {
+ const { 'user-agent': userAgent } = headers
+ const { pid } = logDTO
+ const ip =
+ headers['cf-connecting-ip'] || headers['x-forwarded-for'] || reqIP || ''
+
+ const sessionID = await this.analyticsService.getSessionHash(
+ pid,
+ userAgent,
+ ip,
+ )
+
+ await redis.set(
+ getHeartbeatKey(pid, sessionID),
+ 1,
+ 'EX',
+ HEARTBEAT_SID_LIFE_TIME,
+ )
+ await this.analyticsService.processInteractionSD(sessionID, pid)
+ }
+
+ // Log pageview event
+ @Post()
+ @UseGuards(BotDetectionGuard)
+ @BotDetection()
+ @Public()
+ async log(
+ @Body() logDTO: PageviewsDTO,
+ @Headers() headers,
+ @Ip() reqIP,
+ ): Promise<any> {
+ const { 'user-agent': userAgent, origin } = headers
+
+ const ip =
+ headers['cf-connecting-ip'] || headers['x-forwarded-for'] || reqIP || ''
+
+ await this.analyticsService.validate(logDTO, origin, 'log', ip)
+
+ const salt = await redis.get(REDIS_SESSION_SALT_KEY)
+ const sessionHash = getSessionKey(ip, userAgent, logDTO.pid, salt)
+ const unique = await this.analyticsService.isUnique(sessionHash)
+
+ await this.analyticsService.processInteractionSD(sessionHash, logDTO.pid)
+
+ if (unique && logDTO.unique) {
+ throw new ForbiddenException(
+ 'The event was not saved because it was not unique while unique only param is provided',
+ )
+ }
+
+ // trying to get country from timezome, otherwise using CloudFlare's IP based country code as a fallback
+ const cc =
+ ct.getCountryForTimezone(logDTO.tz)?.id ||
+ (headers['cf-ipcountry'] === 'XX' ? 'NULL' : headers['cf-ipcountry'])
+
+ const ua = UAParser(userAgent)
+ const dv = ua.device.type || 'desktop'
+ const br = ua.browser.name
+ const os = ua.os.name
+
+ const dto = analyticsDTO(
+ sessionHash,
+ logDTO.pid,
+ logDTO.pg,
+ logDTO.prev,
+ dv,
+ br,
+ os,
+ logDTO.lc,
+ logDTO.ref,
+ logDTO.so,
+ logDTO.me,
+ logDTO.ca,
+ cc,
+ 0,
+ Number(unique),
+ )
+
+ let perfDTO: Array<string | number> = []
+
+ if (!_isEmpty(logDTO.perf) && isPerformanceValid(logDTO.perf)) {
+ const {
+ dns,
+ tls,
+ conn,
+ response,
+ render,
+ dom_load: domLoad,
+ page_load: pageLoad,
+ ttfb,
+ } = logDTO.perf
+
+ perfDTO = performanceDTO(
+ logDTO.pid,
+ logDTO.pg,
+ dv,
+ br,
+ cc,
+ dns,
+ tls,
+ conn,
+ response,
+ render,
+ domLoad,
+ pageLoad,
+ ttfb,
+ )
+ }
+
+ const values = `(${dto.map(getElValue).join(',')})`
+ const perfValues = `(${perfDTO.map(getElValue).join(',')})`
+ try {
+ await redis.rpush(REDIS_LOG_DATA_CACHE_KEY, values)
+
+ if (!_isEmpty(perfDTO)) {
+ await redis.rpush(REDIS_LOG_PERF_CACHE_KEY, perfValues)
+ }
+
+ return
+ } catch (e) {
+ this.logger.error(e)
+ throw new InternalServerErrorException(
+ 'Error occured while saving the log data',
+ )
+ }
+ }
+
+ // Fallback for logging pageviews for users with JavaScript disabled
+ // Returns 1x1 transparent gif
+ @Get('noscript')
+ @Public()
+ async noscript(
+ @Query() data,
+ @Headers() headers,
+ @Ip() reqIP,
+ @Response() res,
+ ): Promise<any> {
+ const { 'user-agent': userAgent, origin } = headers
+ const { pid } = data
+
+ // todo: create a decorator for bot traffic detection
+ if (isbot(userAgent)) {
+ res.writeHead(200, { 'Content-Type': 'image/gif' })
+ return res.end(TRANSPARENT_GIF_BUFFER, 'binary')
+ }
+
+ const logDTO: PageviewsDTO = {
+ pid,
+ }
+
+ await this.analyticsService.validate(logDTO, origin)
+
+ const ip =
+ headers['cf-connecting-ip'] || headers['x-forwarded-for'] || reqIP || ''
+ const salt = await redis.get(REDIS_SESSION_SALT_KEY)
+ const sessionHash = getSessionKey(ip, userAgent, logDTO.pid, salt)
+ const unique = await this.analyticsService.isUnique(sessionHash)
+
+ await this.analyticsService.processInteractionSD(sessionHash, logDTO.pid)
+
+ const ua = UAParser(userAgent)
+ const dv = ua.device.type || 'desktop'
+ const br = ua.browser.name
+ const os = ua.os.name
+
+ // using CloudFlare's IP based country code
+ const cc =
+ headers['cf-ipcountry'] === 'XX' ? 'NULL' : headers['cf-ipcountry']
+ const dto = analyticsDTO(
+ sessionHash,
+ logDTO.pid,
+ 'NULL',
+ 'NULL',
+ dv,
+ br,
+ os,
+ 'NULL',
+ 'NULL',
+ 'NULL',
+ 'NULL',
+ 'NULL',
+ cc,
+ 0,
+ Number(unique),
+ )
+
+ const values = `(${dto.map(getElValue).join(',')})`
+ try {
+ await redis.rpush(REDIS_LOG_DATA_CACHE_KEY, values)
+ } catch (e) {
+ this.logger.error(e)
+ }
+
+ res.writeHead(200, { 'Content-Type': 'image/gif' })
+ return res.end(TRANSPARENT_GIF_BUFFER, 'binary')
+ }
+
+ @Get('custom-events')
+ @Auth([], true, true)
+ async getCustomEvents(
+ @Query() data: GetCustomEventsDto,
+ @CurrentUserId() uid: string,
+ ): Promise<any> {
+ const {
+ pid,
+ period,
+ timeBucket,
+ from,
+ to,
+ filters,
+ timezone = DEFAULT_TIMEZONE,
+ customEvents,
+ } = data
+ this.analyticsService.validatePID(pid)
+
+ if (!_isEmpty(period)) {
+ this.analyticsService.validatePeriod(period)
+ }
+
+ this.analyticsService.validateTimebucket(timeBucket)
+ const [filtersQuery, filtersParams, parsedFilters] =
+ this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)
+ await this.analyticsService.checkProjectAccess(pid, uid)
+
+ const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
+ const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
+ from,
+ to,
+ timeBucket,
+ period,
+ safeTimezone,
+ )
+
+ const paramsData = {
+ params: {
+ pid,
+ groupFrom,
+ groupTo,
+ ...filtersParams,
+ },
+ }
+
+ const result: any = await this.analyticsService.groupCustomEVByTimeBucket(
+ timeBucket,
+ groupFrom,
+ groupTo,
+ filtersQuery,
+ paramsData,
+ safeTimezone,
+ )
+
+ let customEventss = customEvents
+
+ if (filters) {
+ try {
+ customEventss = JSON.parse(customEvents)
+ // eslint-disable-next-line no-empty
+ } catch {}
+ }
+
+ if (customEventss.length > 0) {
+ for (const key in result.chart.events) {
+ if (!customEventss.includes(key)) {
+ delete result.chart.events[key]
+ }
+ }
+ } else {
+ result.chart = {}
+ }
+
+ return {
+ ...result,
+ appliedFilters: parsedFilters,
+ }
+ }
+}
diff --git a/selfhosted-src/analytics/analytics.module.ts b/selfhosted-src/analytics/analytics.module.ts
new file mode 100644
index 000000000..b9d46551a
--- /dev/null
+++ b/selfhosted-src/analytics/analytics.module.ts
@@ -0,0 +1,21 @@
+import { Module, forwardRef } from '@nestjs/common'
+
+import { AnalyticsService } from './analytics.service'
+import { AnalyticsController } from './analytics.controller'
+import { UserModule } from '../user/user.module'
+import { AppLoggerModule } from '../logger/logger.module'
+import { ProjectModule } from '../project/project.module'
+import { TaskManagerModule } from '../task-manager/task-manager.module'
+
+@Module({
+ imports: [
+ forwardRef(() => UserModule),
+ TaskManagerModule,
+ AppLoggerModule,
+ ProjectModule,
+ ],
+ providers: [AnalyticsService],
+ exports: [AnalyticsService],
+ controllers: [AnalyticsController],
+})
+export class AnalyticsModule {}
diff --git a/selfhosted-src/analytics/analytics.service.ts b/selfhosted-src/analytics/analytics.service.ts
new file mode 100644
index 000000000..8bb084a26
--- /dev/null
+++ b/selfhosted-src/analytics/analytics.service.ts
@@ -0,0 +1,1494 @@
+import * as _isEmpty from 'lodash/isEmpty'
+import * as _split from 'lodash/split'
+import * as _size from 'lodash/size'
+import * as _includes from 'lodash/includes'
+import * as _map from 'lodash/map'
+import * as _toUpper from 'lodash/toUpper'
+import * as _join from 'lodash/join'
+import * as _last from 'lodash/last'
+import * as _some from 'lodash/some'
+import * as _find from 'lodash/find'
+import * as _now from 'lodash/now'
+import * as _values from 'lodash/values'
+import * as _round from 'lodash/round'
+import * as _filter from 'lodash/filter'
+import * as dayjs from 'dayjs'
+import * as utc from 'dayjs/plugin/utc'
+import * as dayjsTimezone from 'dayjs/plugin/timezone'
+import * as isSameOrBefore from 'dayjs/plugin/isSameOrBefore'
+import * as ipRangeCheck from 'ip-range-check'
+import validator from 'validator'
+import { hash } from 'blake3'
+import {
+ Injectable,
+ BadRequestException,
+ InternalServerErrorException,
+ UnprocessableEntityException,
+ PreconditionFailedException,
+} from '@nestjs/common'
+
+import {
+ DEFAULT_TIMEZONE,
+} from '../user/entities/user.entity'
+import {
+ redis,
+ isValidPID,
+ UNIQUE_SESSION_LIFE_TIME,
+ clickhouse,
+ REDIS_SESSION_SALT_KEY,
+} from '../common/constants'
+import {
+ calculateRelativePercentage,
+ millisecondsToSeconds,
+} from '../common/utils'
+import { PageviewsDTO } from './dto/pageviews.dto'
+import { EventsDTO } from './dto/events.dto'
+import { ProjectService } from '../project/project.service'
+import { Project } from '../project/entity/project.entity'
+import { TimeBucketType } from './dto/getData.dto'
+import {
+ PerformanceCHResponse,
+ CustomsCHResponse,
+ CustomsCHAggregatedResponse,
+ TrafficCEFilterCHResponse,
+ TrafficCHResponse,
+ IGetGroupFromTo,
+ GetFiltersQuery,
+ IUserFlowNode,
+ IUserFlowLink,
+ IUserFlow,
+ IBuildUserFlow,
+ IExtractChartData,
+ IGenerateXAxis,
+} from './interfaces'
+
+dayjs.extend(utc)
+dayjs.extend(dayjsTimezone)
+dayjs.extend(isSameOrBefore)
+
+export const getSessionKey = (ip: string, ua: string, pid: string, salt = '') =>
+ `ses_${hash(`${ua}${ip}${pid}${salt}`).toString('hex')}`
+
+export const getHeartbeatKey = (pid: string, sessionID: string) =>
+ `hb:${pid}:${sessionID}`
+
+const getSessionDurationKey = (sessionHash: string, pid: string) =>
+ `sd:${sessionHash}:${pid}`
+
+const GMT_0_TIMEZONES = [
+ 'Atlantic/Azores',
+ 'Etc/GMT',
+ // 'Africa/Casablanca',
+]
+
+const cols = ['cc', 'pg', 'lc', 'br', 'os', 'dv', 'ref', 'so', 'me', 'ca']
+
+const captchaColumns = ['cc', 'br', 'os', 'dv']
+const perfColumns = ['cc', 'pg', 'dv', 'br']
+
+const validPeriods = [
+ 'today',
+ 'yesterday',
+ '1d',
+ '7d',
+ '4w',
+ '3M',
+ '12M',
+ '24M',
+]
+
+const validTimebuckets = [
+ TimeBucketType.HOUR,
+ TimeBucketType.DAY,
+ TimeBucketType.MONTH,
+]
+
+// mapping of allowed timebuckets per difference between days
+// (e.g. if difference is lower than (lt) (including) -> then the specified timebuckets are allowed to be applied)
+const timeBucketToDays = [
+ {
+ lt: 7,
+ tb: [TimeBucketType.HOUR, TimeBucketType.DAY, TimeBucketType.MONTH],
+ }, // 7 days
+ {
+ lt: 28,
+ tb: [TimeBucketType.DAY, TimeBucketType.MONTH],
+ }, // 4 weeks
+ { lt: 366, tb: [TimeBucketType.MONTH] }, // 12 months
+ { lt: 732, tb: [TimeBucketType.MONTH] }, // 24 months
+]
+
+// Smaller than 64 characters, must start with an English letter and contain only letters (a-z A-Z), numbers (0-9), underscores (_) and dots (.)
+// eslint-disable-next-line no-useless-escape
+const customEVvalidate = /^[a-zA-Z](?:[\w\.]){0,62}$/
+
+const timeBucketConversion = {
+ hour: 'toStartOfHour',
+ day: 'toStartOfDay',
+ month: 'toStartOfMonth',
+}
+
+const isValidTimezone = (timezone: string): boolean => {
+ if (_isEmpty(timezone)) {
+ return false
+ }
+
+ try {
+ dayjs.tz('2013-11-18 11:55', timezone)
+ return true
+ } catch {
+ return false
+ }
+}
+
+export const isValidDate = (date: string, format = 'YYYY-MM-DD'): boolean => {
+ if (_isEmpty(date)) {
+ return false
+ }
+
+ return dayjs(date, format).format(format) === date
+}
+
+const checkIfTBAllowed = (
+ timeBucket: TimeBucketType,
+ from: string,
+ to: string,
+): void => {
+ const diff = dayjs(to).diff(dayjs(from), 'days')
+
+ const tbMap = _find(timeBucketToDays, ({ lt }) => diff <= lt)
+
+ if (_isEmpty(tbMap)) {
+ throw new PreconditionFailedException(
+ "The difference between 'from' and 'to' is greater than allowed",
+ )
+ }
+
+ const { tb } = tbMap
+
+ if (!_includes(tb, timeBucket)) {
+ throw new PreconditionFailedException(
+ "The specified 'timeBucket' parameter cannot be applied to the date range",
+ )
+ }
+}
+
+const generateParamsQuery = (
+ col: string,
+ subQuery: string,
+ customEVFilterApplied: boolean,
+ isPageInclusiveFilterSet: boolean,
+ isCaptcha?: boolean,
+ isPerformance?: boolean,
+): string => {
+ if (isPerformance) {
+ return `SELECT ${col}, avg(pageLoad) ${subQuery} AND ${col} IS NOT NULL GROUP BY ${col}`
+ }
+
+ if (isCaptcha) {
+ return `SELECT ${col}, count(*) ${subQuery} AND ${col} IS NOT NULL GROUP BY ${col}`
+ }
+
+ if (customEVFilterApplied) {
+ return `SELECT ${col}, count(*) ${subQuery} AND ${col} IS NOT NULL GROUP BY ${col}`
+ }
+
+ if (col === 'pg' || isPageInclusiveFilterSet) {
+ return `SELECT ${col}, count(*) ${subQuery} AND ${col} IS NOT NULL GROUP BY ${col}`
+ }
+
+ return `SELECT ${col}, count(*) ${subQuery} AND ${col} IS NOT NULL AND unique='1' GROUP BY ${col}`
+}
+
+export enum DataType {
+ ANALYTICS = 'analytics',
+ PERFORMANCE = 'performance',
+ CAPTCHA = 'captcha',
+}
+
+@Injectable()
+export class AnalyticsService {
+ constructor(private readonly projectService: ProjectService) {}
+
+ async checkProjectAccess(pid: string, uid: string | null): Promise<void> {
+ // const project = await this.projectService.getRedisProject(pid)
+ // this.projectService.allowedToView(project, uid)
+ }
+
+ checkOrigin(project: Project, origin: string): void {
+ // For some reasons the project.origins sometimes may look like [''], let's filter it out
+ // TODO: Properly validate the origins on project update
+ const origins = _filter(project.origins, Boolean)
+
+ if (!_isEmpty(origins) && !_isEmpty(origin)) {
+ if (origin === 'null') {
+ if (!_includes(origins, 'null')) {
+ throw new BadRequestException(
+ "'null' origin is not added to your project's whitelist. To send requests from this origin either add it to your origins policy or leave it empty.",
+ )
+ }
+ } else {
+ const { hostname } = new URL(origin)
+ if (!_includes(origins, hostname)) {
+ throw new BadRequestException(
+ "This origin is prohibited by the project's origins policy",
+ )
+ }
+ }
+ }
+ }
+
+ checkIpBlacklist(project: Project, ip: string): void {
+ // For some reasons the project.ipBlacklist sometimes may look like [''], let's filter it out
+ // TODO: Properly validate the ipBlacklist on project update
+ const ipBlacklist = _filter(project.ipBlacklist, Boolean)
+
+ if (!_isEmpty(ipBlacklist) && ipRangeCheck(ip, ipBlacklist)) {
+ throw new BadRequestException(
+ 'Incoming analytics is disabled for this IP address',
+ )
+ }
+ }
+
+ validatePID(pid: string): void {
+ if (_isEmpty(pid)) {
+ throw new BadRequestException('The Project ID (pid) has to be provided')
+ }
+
+ if (!isValidPID(pid)) {
+ throw new BadRequestException(
+ 'The provided Project ID (pid) is incorrect',
+ )
+ }
+ }
+
+ async validate(
+ logDTO: PageviewsDTO | EventsDTO,
+ origin: string,
+ type: 'custom' | 'log' = 'log',
+ ip?: string,
+ ): Promise<string | null> {
+ if (_isEmpty(logDTO)) {
+ throw new BadRequestException('The request cannot be empty')
+ }
+
+ const { pid } = logDTO
+ this.validatePID(pid)
+
+ if (type === 'custom') {
+ const { ev } = <EventsDTO>logDTO
+
+ if (_isEmpty(ev)) {
+ throw new BadRequestException('Empty custom events are not allowed')
+ }
+
+ if (!customEVvalidate.test(ev)) {
+ throw new BadRequestException(
+ 'An incorrect event name (ev) is provided',
+ )
+ }
+ }
+
+ // 'tz' does not need validation as it's based on getCountryForTimezone detection
+ const { lc } = logDTO
+
+ // validate locale ('lc' param)
+ if (!_isEmpty(lc)) {
+ if (validator.isLocale(lc)) {
+ // uppercase the locale after '-' char, so for example both 'en-gb' and 'en-GB' in result will be 'en-GB'
+ const lcParted = _split(lc, '-')
+
+ if (_size(lcParted) > 1) {
+ lcParted[1] = _toUpper(lcParted[1])
+ }
+
+ logDTO.lc = _join(lcParted, '-')
+ } else {
+ logDTO.lc = 'NULL'
+ }
+ }
+
+ const project = await this.projectService.getRedisProject(pid)
+
+ this.checkIpBlacklist(project, ip)
+
+ if (!project.active) {
+ throw new BadRequestException(
+ 'Incoming analytics is disabled for this project',
+ )
+ }
+
+ this.checkOrigin(project, origin)
+
+ return null
+ }
+
+ getDataTypeColumns(dataType: DataType): string[] {
+ if (dataType === DataType.ANALYTICS) {
+ return cols
+ }
+
+ if (dataType === DataType.PERFORMANCE) {
+ return perfColumns
+ }
+
+ return captchaColumns
+ }
+
+ getGroupFromTo(
+ from: string,
+ to: string,
+ timeBucket: TimeBucketType | null,
+ period: string,
+ safeTimezone: string,
+ ): IGetGroupFromTo {
+ let groupFrom: dayjs.Dayjs
+ let groupTo: dayjs.Dayjs
+ const formatFrom = 'YYYY-MM-DD HH:mm:ss'
+ const formatTo = 'YYYY-MM-DD HH:mm:ss'
+ const djsNow = _includes(GMT_0_TIMEZONES, safeTimezone)
+ ? dayjs.utc()
+ : dayjs().tz(safeTimezone)
+
+ if (!_isEmpty(from) && !_isEmpty(to)) {
+ if (!isValidDate(from)) {
+ throw new PreconditionFailedException(
+ "The timeframe 'from' parameter is invalid",
+ )
+ }
+
+ if (!isValidDate(to)) {
+ throw new PreconditionFailedException(
+ "The timeframe 'to' parameter is invalid",
+ )
+ }
+
+ if (dayjs.utc(from).isAfter(dayjs.utc(to), 'second')) {
+ throw new PreconditionFailedException(
+ "The timeframe 'from' parameter cannot be greater than 'to'",
+ )
+ }
+
+ if (!_isEmpty(timeBucket)) {
+ checkIfTBAllowed(timeBucket, from, to)
+ }
+
+ groupFrom = dayjs.utc()
+
+ if (from === to) {
+ groupTo = dayjs.utc().add(1, 'day')
+ } else {
+ groupTo = dayjs.utc()
+ }
+ } else if (!_isEmpty(period)) {
+ if (period === 'today') {
+ groupFrom = djsNow.startOf('d')
+ groupTo = djsNow
+ } else if (period === 'yesterday') {
+ groupFrom = djsNow.subtract(1, 'day').startOf('d')
+ groupTo = djsNow.subtract(1, 'day').endOf('d')
+ } else {
+ if (period === '1d') {
+ groupFrom = djsNow.subtract(parseInt(period, 10), _last(period))
+ } else {
+ groupFrom = djsNow.subtract(parseInt(period, 10) - 1, _last(period))
+ }
+
+ groupFrom = groupFrom.startOf(timeBucket)
+ groupTo = djsNow
+ }
+
+ if (!_isEmpty(timeBucket)) {
+ checkIfTBAllowed(
+ timeBucket,
+ groupFrom.format(formatFrom),
+ groupTo.format(formatTo),
+ )
+ }
+ } else {
+ throw new BadRequestException(
+ 'The timeframe (either from/to pair or period) has to be provided',
+ )
+ }
+
+ const groupFromFormatted = groupFrom.format(formatFrom)
+ const groupToFormatted = groupTo.format(formatTo)
+
+ return {
+ // UTC time
+ groupFrom: groupFromFormatted,
+ groupTo: groupToFormatted,
+ // Timezone shifted time
+ groupFromUTC: groupFrom.utc().format(formatFrom),
+ groupToUTC: groupTo.utc().format(formatFrom),
+ }
+ }
+
+ removeCyclicDependencies(links: IUserFlowLink[]): IUserFlowLink[] {
+ const visited = new Set<string>()
+
+ return links.filter(link => {
+ const key = `${link.source}_${link.target}`
+ if (visited.has(key)) {
+ return false
+ }
+ visited.add(key)
+ return true
+ })
+ }
+
+ buildUserFlow(links: IUserFlowLink[]): IBuildUserFlow {
+ const nodes: IUserFlowNode[] = Array.from(
+ new Set(
+ links
+ .map((link: IUserFlowLink) => link.source)
+ .concat(links.map((link: IUserFlowLink) => link.target)),
+ ),
+ ).map((node: any) => ({ id: node }))
+
+ return { nodes, links }
+ }
+
+ async getUserFlow(params: unknown, filtersQuery: string): Promise<IUserFlow> {
+ const query = `
+ SELECT
+ pg AS source,
+ prev AS target,
+ count() AS value
+ FROM analytics
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ AND pg != prev
+ ${filtersQuery}
+ GROUP BY
+ pg,
+ prev
+ `
+
+ const results = <IUserFlowLink[]>(
+ await clickhouse.query(query, { params }).toPromise()
+ )
+
+ if (_isEmpty(results)) {
+ const empty = { nodes: [], links: [] }
+ return {
+ ascending: empty,
+ descending: empty,
+ }
+ }
+
+ const ascendingLinks: IUserFlowLink[] = []
+ const descendingLinks: IUserFlowLink[] = []
+
+ this.removeCyclicDependencies(results).forEach((row: any) => {
+ const link: IUserFlowLink = {
+ source: row.source,
+ target: row.target,
+ value: row.value,
+ }
+ if (link.source < link.target) {
+ ascendingLinks.push(link)
+ } else {
+ descendingLinks.push(link)
+ }
+ })
+
+ return {
+ ascending: this.buildUserFlow(ascendingLinks),
+ descending: this.buildUserFlow(descendingLinks),
+ }
+ }
+
+ async getSessionHash(
+ pid: string,
+ userAgent: string,
+ ip: string,
+ ): Promise<string> {
+ this.validatePID(pid)
+
+ const salt = await redis.get(REDIS_SESSION_SALT_KEY)
+
+ return getSessionKey(ip, userAgent, pid, salt)
+ }
+
+ async isSessionDurationOpen(sdKey: string): Promise<Array<string | boolean>> {
+ const sd = await redis.get(sdKey)
+ return [sd, Boolean(sd)]
+ }
+
+ // Processes interaction for session duration
+ async processInteractionSD(sessionHash: string, pid: string): Promise<void> {
+ const sdKey = getSessionDurationKey(sessionHash, pid)
+ const [sd, isOpened] = await this.isSessionDurationOpen(sdKey)
+ const now = _now()
+
+ // the value is START_UNIX_TIMESTAMP:LAST_INTERACTION_UNIX_TIMESTAMP
+ if (isOpened) {
+ const [start] = _split(sd, ':')
+ await redis.set(sdKey, `${start}:${now}`, 'EX', UNIQUE_SESSION_LIFE_TIME)
+ } else {
+ await redis.set(sdKey, `${now}:${now}`, 'EX', UNIQUE_SESSION_LIFE_TIME)
+ }
+ }
+
+ validatePeriod(period: string): void {
+ if (!_includes(validPeriods, period)) {
+ throw new UnprocessableEntityException('The provided period is incorrect')
+ }
+ }
+
+ // returns SQL filters query in a format like 'AND col=value AND ...'
+ getFiltersQuery(filters: string, dataType: DataType): GetFiltersQuery {
+ let parsed = []
+ let query = ''
+ let params = {}
+
+ if (_isEmpty(filters)) {
+ return [query, params, parsed]
+ }
+
+ try {
+ parsed = JSON.parse(filters)
+ } catch (e) {
+ console.error(`Cannot parse the filters array: ${filters}`)
+ return [query, params, parsed]
+ }
+
+ if (_isEmpty(parsed)) {
+ return [query, params, parsed]
+ }
+
+ const columns = this.getDataTypeColumns(dataType)
+
+ for (let i = 0; i < _size(parsed); ++i) {
+ const { column, filter, isExclusive } = parsed[i]
+
+ if (column === 'ev') {
+ params = {
+ ...params,
+ [column]: filter,
+ [`${column}_exclusive`]: isExclusive,
+ }
+
+ continue
+ }
+
+ if (!_includes(columns, column)) {
+ throw new UnprocessableEntityException(
+ `The provided filter (${column}) is not supported`,
+ )
+ }
+ // working only on 1 filter per 1 column
+ const colFilter = `cf_${column}`
+
+ params = {
+ ...params,
+ [colFilter]: filter,
+ }
+ query += ` ${
+ isExclusive ? 'AND NOT' : 'AND'
+ } ${column}={${colFilter}:String}`
+ }
+
+ return [query, params, parsed]
+ }
+
+ validateTimebucket(tb: TimeBucketType): void {
+ if (!_includes(validTimebuckets, tb)) {
+ throw new UnprocessableEntityException(
+ 'The provided timebucket is incorrect',
+ )
+ }
+ }
+
+ async isUnique(sessionHash: string) {
+ const session = await redis.get(sessionHash)
+ await redis.set(sessionHash, 1, 'EX', UNIQUE_SESSION_LIFE_TIME)
+ return !session
+ }
+
+ async getSummary(
+ pids: string[],
+ period: 'w' | 'M' = 'w',
+ amountToSubtract = 1,
+ ) {
+ return this.getSummaryStats(pids, 'analytics', period, amountToSubtract)
+ }
+
+ async getCaptchaSummary(pids: string[], period: 'w' | 'M' = 'w') {
+ return this.getSummaryStats(pids, 'captcha', period)
+ }
+
+ async getSummaryStats(
+ pids: string[],
+ tableName: 'analytics' | 'captcha',
+ period: 'w' | 'M' = 'w',
+ amountToSubtract = 1,
+ ) {
+ const result = {}
+
+ const promises = pids.map(async pid => {
+ if (!isValidPID(pid)) {
+ throw new BadRequestException(
+ `The provided Project ID (${pid}) is incorrect`,
+ )
+ }
+
+ const now = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')
+ const periodRaw = dayjs.utc().subtract(amountToSubtract, period)
+ const periodFormatted = periodRaw.format('YYYY-MM-DD HH:mm:ss')
+ const periodSubtracted = periodRaw
+ .subtract(amountToSubtract, period)
+ .format('YYYY-MM-DD HH:mm:ss')
+
+ const queryThisWeek = `SELECT ${
+ tableName === 'analytics' ? 'unique,' : ''
+ } count() FROM ${tableName} WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodFormatted:String} AND {now:String}${
+ tableName === 'analytics' ? ' GROUP BY unique' : ''
+ }`
+ const queryLastWeek = `SELECT ${
+ tableName === 'analytics' ? 'unique,' : ''
+ } count() FROM ${tableName} WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String}${
+ tableName === 'analytics' ? ' GROUP BY unique' : ''
+ }`
+
+ const paramsData = {
+ params: {
+ pid,
+ periodFormatted,
+ periodSubtracted,
+ now,
+ },
+ }
+
+ try {
+ const thisWeekResult = await clickhouse
+ .query(queryThisWeek, paramsData)
+ .toPromise()
+ const lastWeekResult = await clickhouse
+ .query(queryLastWeek, paramsData)
+ .toPromise()
+
+ if (tableName === 'analytics') {
+ const thisWeekUnique =
+ _find(thisWeekResult, ({ unique }) => unique)?.['count()'] || 0
+ const thisWeekPV =
+ (_find(thisWeekResult, ({ unique }) => !unique)?.['count()'] || 0) +
+ thisWeekUnique
+ const lastWeekUnique =
+ _find(lastWeekResult, ({ unique }) => unique)?.['count()'] || 0
+ const lastWeekPV =
+ (_find(lastWeekResult, ({ unique }) => !unique)?.['count()'] || 0) +
+ lastWeekUnique
+
+ result[pid] = {
+ thisWeek: thisWeekPV,
+ lastWeek: lastWeekPV,
+ thisWeekUnique,
+ lastWeekUnique,
+ percChange: calculateRelativePercentage(lastWeekPV, thisWeekPV),
+ percChangeUnique: calculateRelativePercentage(
+ lastWeekUnique,
+ thisWeekUnique,
+ ),
+ }
+ } else {
+ const thisWeek = thisWeekResult?.['count()'] || 0
+ const lastWeek = lastWeekResult?.['count()'] || 0
+
+ result[pid] = {
+ thisWeek,
+ lastWeek,
+ percChange: calculateRelativePercentage(lastWeek, thisWeek),
+ }
+ }
+ } catch {
+ throw new InternalServerErrorException(
+ "Can't process the provided PID. Please, try again later.",
+ )
+ }
+ })
+
+ await Promise.all(promises)
+
+ return result
+ }
+
+ async generateParams(
+ parsedFilters: Array<{ [key: string]: string }>,
+ subQuery: string,
+ customEVFilterApplied: boolean,
+ paramsData: any,
+ isCaptcha: boolean,
+ isPerformance: boolean,
+ ): Promise<any> {
+ const params = {}
+
+ // We need this to display all the pageview related data (e.g. country, browser) when user applies an inclusive filter on the Page column
+ const isPageInclusiveFilterSet =
+ isCaptcha || isPerformance
+ ? false
+ : !_isEmpty(
+ _find(
+ parsedFilters,
+ filter => filter.column === 'pg' && !filter.isExclusive,
+ ),
+ )
+
+ let columns = cols
+
+ if (isCaptcha) {
+ columns = captchaColumns
+ }
+
+ if (isPerformance) {
+ columns = perfColumns
+ }
+
+ const paramsPromises = _map(columns, async col => {
+ const query = generateParamsQuery(
+ col,
+ subQuery,
+ customEVFilterApplied,
+ isPageInclusiveFilterSet,
+ isCaptcha,
+ isPerformance,
+ )
+ const res = await clickhouse.query(query, paramsData).toPromise()
+
+ params[col] = {}
+
+ const size = _size(res)
+ for (let j = 0; j < size; ++j) {
+ const key = res[j][col]
+
+ if (isPerformance) {
+ params[col][key] = _round(
+ millisecondsToSeconds(res[j]['avg(pageLoad)']),
+ 2,
+ )
+ continue
+ }
+
+ params[col][key] = res[j]['count()']
+ }
+ })
+
+ await Promise.all(paramsPromises)
+
+ return params
+ }
+
+ async calculateAverageSessionDuration(
+ subQuery: string,
+ paramsData: any,
+ ): Promise<number> {
+ const avgSdurQuery = `SELECT avg(sdur) ${subQuery} AND sdur IS NOT NULL AND unique='1'`
+ const avgSdurObject = await clickhouse
+ .query(avgSdurQuery, paramsData)
+ .toPromise()
+
+ return _round(avgSdurObject[0]['avg(sdur)'])
+ }
+
+ generateXAxis(
+ timeBucket: TimeBucketType,
+ from: string, // timezone is already applied to the from and to parameters
+ to: string,
+ safeTimezone: string,
+ ): IGenerateXAxis {
+ const iterateTo = _includes(GMT_0_TIMEZONES, safeTimezone)
+ ? dayjs.utc(to)
+ : dayjs.tz(to, safeTimezone)
+ let djsFrom = _includes(GMT_0_TIMEZONES, safeTimezone)
+ ? dayjs.utc(from)
+ : dayjs.tz(from, safeTimezone)
+
+ let format
+
+ switch (timeBucket) {
+ case TimeBucketType.HOUR:
+ format = 'YYYY-MM-DD HH:mm:ss'
+ break
+
+ case TimeBucketType.DAY:
+ format = 'YYYY-MM-DD'
+ break
+
+ case TimeBucketType.MONTH:
+ format = 'YYYY-MM'
+ break
+
+ default:
+ throw new BadRequestException(
+ `The provided time bucket (${timeBucket}) is incorrect`,
+ )
+ }
+
+ // UTC dates
+ const x = []
+
+ // Timezone shifted dates (a hack to map Clickhouse timezone processed data to UTC dates)
+ const xShifted = []
+
+ while (djsFrom.isSameOrBefore(iterateTo, timeBucket)) {
+ x.push(djsFrom.utc().format(format))
+ xShifted.push(djsFrom.format(format))
+ djsFrom = djsFrom.add(1, timeBucket)
+ }
+
+ if (_includes(GMT_0_TIMEZONES, safeTimezone)) {
+ return {
+ x,
+ xShifted: x,
+ }
+ }
+
+ return {
+ x,
+ xShifted,
+ }
+ }
+
+ generateDateString(row: { [key: string]: number }): string {
+ const { year, month, day, hour } = row
+
+ let dateString = `${year}-${month < 10 ? `0${month}` : month}`
+
+ if (typeof day === 'number') {
+ if (day < 10) {
+ dateString += `-0${day}`
+ } else {
+ dateString += `-${day}`
+ }
+ }
+
+ if (typeof hour === 'number') {
+ if (hour < 10) {
+ dateString += ` 0${hour}:00:00`
+ } else {
+ dateString += ` ${hour}:00:00`
+ }
+ }
+
+ return dateString
+ }
+
+ extractChartData(result, x: string[]): IExtractChartData {
+ const visits = Array(x.length).fill(0)
+ const uniques = Array(x.length).fill(0)
+ const sdur = Array(x.length).fill(0)
+
+ for (let row = 0; row < _size(result); ++row) {
+ const dateString = this.generateDateString(result[row])
+ const index = x.indexOf(dateString)
+
+ if (index !== -1) {
+ visits[index] = result[row].pageviews
+ uniques[index] = result[row].uniques
+ sdur[index] = _round(result[row].sdur)
+ }
+ }
+
+ return {
+ visits,
+ uniques,
+ sdur,
+ }
+ }
+
+ extractCustomEventsChartData(queryResult, x: string[]): any {
+ const result = {}
+
+ for (let row = 0; row < _size(queryResult); ++row) {
+ const { ev = '_unknown_event' } = queryResult[row]
+
+ const dateString = this.generateDateString(queryResult[row])
+
+ const index = x.indexOf(dateString)
+
+ if (index !== -1) {
+ if (!result[ev]) {
+ result[ev] = Array(x.length).fill(0)
+ }
+
+ result[ev][index] = queryResult[row].count
+ }
+ }
+
+ return result
+ }
+
+ getSafeTimezone(timezone: string): string {
+ if (timezone === DEFAULT_TIMEZONE || !isValidTimezone(timezone)) {
+ return DEFAULT_TIMEZONE
+ }
+
+ return timezone
+ }
+
+ getGroupSubquery(timeBucket: TimeBucketType): [string, string] {
+ if (timeBucket === TimeBucketType.HOUR) {
+ return [
+ `toYear(tz_created) as year,
+ toMonth(tz_created) as month,
+ toDayOfMonth(tz_created) as day,
+ toHour(tz_created) as hour`,
+ 'year, month, day, hour',
+ ]
+ }
+
+ if (timeBucket === TimeBucketType.MONTH) {
+ return [
+ `toYear(tz_created) as year,
+ toMonth(tz_created) as month`,
+ 'year, month',
+ ]
+ }
+
+ return [
+ `toYear(tz_created) as year,
+ toMonth(tz_created) as month,
+ toDayOfMonth(tz_created) as day`,
+ 'year, month, day',
+ ]
+ }
+
+ generateAnalyticsAggregationQuery(
+ timeBucket: TimeBucketType,
+ filtersQuery: string,
+ ): string {
+ const timeBucketFunc = timeBucketConversion[timeBucket]
+ const [selector, groupBy] = this.getGroupSubquery(timeBucket)
+
+ return `
+ SELECT
+ ${selector},
+ avg(sdur) as sdur,
+ count() as pageviews,
+ sumIf(1, unique = 1) as uniques
+ FROM (
+ SELECT *,
+ ${timeBucketFunc}(created) as tz_created
+ FROM analytics
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ ${filtersQuery}
+ ) as subquery
+ GROUP BY ${groupBy}
+ ORDER BY ${groupBy}
+ `
+ }
+
+ generateCustomEventsAggregationQuery(
+ timeBucket: TimeBucketType,
+ filtersQuery: string,
+ paramsData: any,
+ ): string {
+ const timeBucketFunc = timeBucketConversion[timeBucket]
+ const [selector, groupBy] = this.getGroupSubquery(timeBucket)
+
+ return `
+ SELECT
+ ${selector},
+ count() as count
+ FROM (
+ SELECT *,
+ ${timeBucketFunc}(created) as tz_created
+ FROM customEV
+ WHERE ${paramsData.params.ev_exclusive ? 'NOT' : ''} ev = {ev:String}
+ AND pid = {pid:FixedString(12)}
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ ${filtersQuery}
+ ) as subquery
+ GROUP BY ${groupBy}
+ ORDER BY ${groupBy}
+ `
+ }
+
+ generatePerformanceAggregationQuery(
+ timeBucket: TimeBucketType,
+ filtersQuery: string,
+ ): string {
+ const timeBucketFunc = timeBucketConversion[timeBucket]
+ const [selector, groupBy] = this.getGroupSubquery(timeBucket)
+
+ return `
+ SELECT
+ ${selector},
+ avg(dns) as dns,
+ avg(tls) as tls,
+ avg(conn) as conn,
+ avg(response) as response,
+ avg(render) as render,
+ avg(domLoad) as domLoad,
+ avg(ttfb) as ttfb
+ FROM (
+ SELECT *,
+ ${timeBucketFunc}(created) as tz_created
+ FROM performance
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ ${filtersQuery}
+ ) as subquery
+ GROUP BY ${groupBy}
+ ORDER BY ${groupBy}
+ `
+ }
+
+ generateCaptchaAggregationQuery(
+ timeBucket: TimeBucketType,
+ filtersQuery: string,
+ ): string {
+ const timeBucketFunc = timeBucketConversion[timeBucket]
+ const [selector, groupBy] = this.getGroupSubquery(timeBucket)
+
+ return `
+ SELECT
+ ${selector},
+ count() as count
+ FROM (
+ SELECT *,
+ ${timeBucketFunc}(created) as tz_created
+ FROM captcha
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ ${filtersQuery}
+ ) as subquery
+ GROUP BY ${groupBy}
+ ORDER BY ${groupBy}
+ `
+ }
+
+ async groupByTimeBucket(
+ timeBucket: TimeBucketType,
+ from: string,
+ to: string,
+ subQuery: string,
+ filtersQuery: string,
+ paramsData: any,
+ safeTimezone: string,
+ customEVFilterApplied: boolean,
+ parsedFilters: Array<{ [key: string]: string }>,
+ ): Promise<object | void> {
+ let params: unknown = {}
+ let chart: unknown = {}
+ let avgSdur = 0
+
+ const promises = [
+ // Getting params
+ (async () => {
+ params = await this.groupParamsByTimeBucket(
+ subQuery,
+ paramsData,
+ customEVFilterApplied,
+ parsedFilters,
+ )
+
+ if (!_some(_values(params), val => !_isEmpty(val))) {
+ throw new BadRequestException(
+ 'The are no parameters for the specified time frames',
+ )
+ }
+ })(),
+
+ // Getting chart & average session duration data
+ (async () => {
+ const groupedData = await this.groupChartByTimeBucket(
+ timeBucket,
+ from,
+ to,
+ subQuery,
+ filtersQuery,
+ paramsData,
+ safeTimezone,
+ customEVFilterApplied,
+ )
+
+ // @ts-ignore
+ chart = groupedData.chart
+
+ // @ts-ignore
+ avgSdur = groupedData.avgSdur
+ })(),
+ ]
+
+ await Promise.all(promises)
+
+ return {
+ params,
+ chart,
+ avgSdur,
+ }
+ }
+
+ async groupParamsByTimeBucket(
+ subQuery: string,
+ paramsData: any,
+ customEVFilterApplied: boolean,
+ parsedFilters: Array<{ [key: string]: string }>,
+ ): Promise<object | void> {
+ return this.generateParams(
+ parsedFilters,
+ subQuery,
+ customEVFilterApplied,
+ paramsData,
+ false,
+ false,
+ )
+ }
+
+ async groupChartByTimeBucket(
+ timeBucket: TimeBucketType,
+ from: string,
+ to: string,
+ subQuery: string,
+ filtersQuery: string,
+ paramsData: any,
+ safeTimezone: string,
+ customEVFilterApplied: boolean,
+ ): Promise<object | void> {
+ const avgSdur = customEVFilterApplied
+ ? 0
+ : await this.calculateAverageSessionDuration(subQuery, paramsData)
+
+ const { x, xShifted } = this.generateXAxis(
+ timeBucket,
+ from,
+ to,
+ safeTimezone,
+ )
+
+ if (customEVFilterApplied) {
+ const query = this.generateCustomEventsAggregationQuery(
+ timeBucket,
+ filtersQuery,
+ paramsData,
+ )
+
+ const result = <Array<TrafficCEFilterCHResponse>>(
+ await clickhouse.query(query, paramsData).toPromise()
+ )
+
+ const uniques =
+ this.extractCustomEventsChartData(result, x)?._unknown_event || []
+
+ const sdur = Array(_size(x)).fill(0)
+
+ return Promise.resolve({
+ chart: {
+ x: xShifted,
+ visits: uniques,
+ uniques,
+ sdur,
+ },
+ avgSdur,
+ })
+ }
+ const query = this.generateAnalyticsAggregationQuery(
+ timeBucket,
+ filtersQuery,
+ )
+
+ const result = <Array<TrafficCHResponse>>(
+ await clickhouse.query(query, paramsData).toPromise()
+ )
+
+ const { visits, uniques, sdur } = this.extractChartData(result, x)
+
+ return Promise.resolve({
+ chart: {
+ x: xShifted,
+ visits,
+ uniques,
+ sdur,
+ },
+ avgSdur,
+ })
+ }
+
+ extractCaptchaChartData(result, x: string[]): any {
+ const count = Array(x.length).fill(0)
+
+ for (let row = 0; row < _size(result); ++row) {
+ const dateString = this.generateDateString(result[row])
+
+ const index = x.indexOf(dateString)
+
+ if (index !== -1) {
+ count[index] = result[row].count
+ }
+ }
+
+ return {
+ count,
+ }
+ }
+
+ async groupCaptchaByTimeBucket(
+ timeBucket: TimeBucketType,
+ from: string,
+ to: string,
+ subQuery: string,
+ filtersQuery: string,
+ paramsData: object,
+ safeTimezone: string,
+ ): Promise<object | void> {
+ let params: unknown = {}
+ let chart: unknown = {}
+
+ const promises = [
+ // Getting params
+ (async () => {
+ params = await this.generateParams(
+ null,
+ subQuery,
+ false,
+ paramsData,
+ true,
+ false,
+ )
+
+ if (!_some(_values(params), val => !_isEmpty(val))) {
+ throw new BadRequestException(
+ 'The are no parameters for the specified time frames',
+ )
+ }
+ })(),
+
+ // Getting CAPTCHA chart data
+ (async () => {
+ const { x, xShifted } = this.generateXAxis(
+ timeBucket,
+ from,
+ to,
+ safeTimezone,
+ )
+
+ const query = this.generateCaptchaAggregationQuery(
+ timeBucket,
+ filtersQuery,
+ )
+
+ const result = <Array<TrafficCEFilterCHResponse>>(
+ await clickhouse.query(query, paramsData).toPromise()
+ )
+ const { count } = this.extractCaptchaChartData(result, x)
+
+ chart = {
+ x: xShifted,
+ results: count,
+ }
+ })(),
+ ]
+
+ await Promise.all(promises)
+
+ return Promise.resolve({
+ params,
+ chart,
+ })
+ }
+
+ extractPerformanceChartData(result, x: string[]): any {
+ const dns = Array(x.length).fill(0)
+ const tls = Array(x.length).fill(0)
+ const conn = Array(x.length).fill(0)
+ const response = Array(x.length).fill(0)
+ const render = Array(x.length).fill(0)
+ const domLoad = Array(x.length).fill(0)
+ const ttfb = Array(x.length).fill(0)
+
+ for (let row = 0; row < _size(result); ++row) {
+ const dateString = this.generateDateString(result[row])
+
+ const index = x.indexOf(dateString)
+
+ if (index !== -1) {
+ dns[index] = _round(millisecondsToSeconds(result[row].dns), 2)
+ tls[index] = _round(millisecondsToSeconds(result[row].tls), 2)
+ conn[index] = _round(millisecondsToSeconds(result[row].conn), 2)
+ response[index] = _round(millisecondsToSeconds(result[row].response), 2)
+ render[index] = _round(millisecondsToSeconds(result[row].render), 2)
+ domLoad[index] = _round(millisecondsToSeconds(result[row].domLoad), 2)
+ ttfb[index] = _round(millisecondsToSeconds(result[row].ttfb), 2)
+ }
+ }
+
+ return {
+ dns,
+ tls,
+ conn,
+ response,
+ render,
+ domLoad,
+ ttfb,
+ }
+ }
+
+ async getPerfChartData(
+ timeBucket: TimeBucketType,
+ from: string,
+ to: string,
+ filtersQuery: string,
+ paramsData: object,
+ safeTimezone: string,
+ ) {
+ const { x, xShifted } = this.generateXAxis(
+ timeBucket,
+ from,
+ to,
+ safeTimezone,
+ )
+
+ const query = this.generatePerformanceAggregationQuery(
+ timeBucket,
+ filtersQuery,
+ )
+
+ const result = <Array<PerformanceCHResponse>>(
+ await clickhouse.query(query, paramsData).toPromise()
+ )
+
+ return {
+ x: xShifted,
+ ...this.extractPerformanceChartData(result, x),
+ }
+ }
+
+ async groupPerfByTimeBucket(
+ timeBucket: TimeBucketType,
+ from: string,
+ to: string,
+ subQuery: string,
+ filtersQuery: string,
+ paramsData: object,
+ safeTimezone: string,
+ ): Promise<object | void> {
+ let params: unknown = {}
+ let chart: unknown = {}
+
+ const promises = [
+ // Getting params
+ (async () => {
+ params = await this.generateParams(
+ null,
+ subQuery,
+ false,
+ paramsData,
+ false,
+ true,
+ )
+
+ if (!_some(_values(params), val => !_isEmpty(val))) {
+ throw new BadRequestException(
+ 'The are no parameters for the specified time frames',
+ )
+ }
+ })(),
+
+ // Getting chart data
+ (async () => {
+ chart = await this.getPerfChartData(
+ timeBucket,
+ from,
+ to,
+ filtersQuery,
+ paramsData,
+ safeTimezone,
+ )
+ })(),
+ ]
+
+ await Promise.all(promises)
+
+ return Promise.resolve({
+ params,
+ chart,
+ })
+ }
+
+ async processCustomEV(query: string, params: object): Promise<object> {
+ const result = {}
+
+ const rawCustoms = <Array<CustomsCHResponse>>(
+ await clickhouse.query(query, params).toPromise()
+ )
+ const size = _size(rawCustoms)
+
+ for (let i = 0; i < size; ++i) {
+ const { ev, 'count()': c } = rawCustoms[i]
+ result[ev] = c
+ }
+
+ return result
+ }
+
+ async getOnlineUserCount(pid: string): Promise<number> {
+ // @ts-ignore
+ return redis.countKeysByPattern(`hb:${pid}:*`)
+ }
+
+ async groupCustomEVByTimeBucket(
+ timeBucket: TimeBucketType,
+ from: string,
+ to: string,
+ filtersQuery: string,
+ paramsData: object,
+ safeTimezone: string,
+ ): Promise<object | void> {
+ const { x, xShifted } = this.generateXAxis(
+ timeBucket,
+ from,
+ to,
+ safeTimezone,
+ )
+
+ const timeBucketFunc = timeBucketConversion[timeBucket]
+ const [selector, groupBy] = this.getGroupSubquery(timeBucket)
+
+ const query = `
+ SELECT
+ ${selector},
+ ev,
+ count() as count
+ FROM (
+ SELECT *,
+ ${timeBucketFunc}(created) as tz_created
+ FROM customEV
+ WHERE
+ pid = {pid:FixedString(12)}
+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}
+ ${filtersQuery}
+ ) as subquery
+ GROUP BY ${groupBy}, ev
+ ORDER BY ${groupBy}
+ `
+
+ const result = <Array<CustomsCHAggregatedResponse>>(
+ await clickhouse.query(query, paramsData).toPromise()
+ )
+
+ const events = this.extractCustomEventsChartData(result, x)
+
+ return Promise.resolve({
+ chart: {
+ x: xShifted,
+ events,
+ },
+ })
+ }
+
+ async getOnlineCountByProjectId(projectId: string) {
+ // @ts-ignore
+ return redis.countKeysByPattern(`hb:${projectId}:*`)
+ }
+
+ async getStatsByProjectId(projectId: string) {
+ return this.getSummaryStats([projectId], 'analytics', 'w', 1)
+ }
+}
diff --git a/selfhosted-src/analytics/dto/events.dto.ts b/selfhosted-src/analytics/dto/events.dto.ts
new file mode 100644
index 000000000..7cec51ad4
--- /dev/null
+++ b/selfhosted-src/analytics/dto/events.dto.ts
@@ -0,0 +1,69 @@
+import { ApiProperty } from '@nestjs/swagger'
+import { IsNotEmpty } from 'class-validator'
+
+export class EventsDTO {
+ @ApiProperty({
+ example: 'aUn1quEid-3',
+ required: true,
+ description: 'The project ID',
+ })
+ @IsNotEmpty()
+ pid: string
+
+ @ApiProperty({
+ example: 'user-subscribed',
+ description: 'Event name',
+ maxLength: 64,
+ })
+ @IsNotEmpty()
+ ev: string
+
+ @ApiProperty({
+ description:
+ 'If true, only 1 event with the same ID will be saved per user session',
+ })
+ unique: object
+
+ // Tracking metrics
+ @ApiProperty({
+ example: 'Europe/Kiev',
+ description: "User's timezone",
+ })
+ tz?: string
+
+ @ApiProperty({
+ example: '/articles/my-awesome-article-1',
+ description: 'A page that user sent data from',
+ })
+ pg?: string
+
+ @ApiProperty({
+ example: 'en-GB',
+ description: "User's locale",
+ })
+ lc?: string
+
+ @ApiProperty({
+ example: 'https://example.com',
+ description: 'The referrer',
+ })
+ ref?: string
+
+ @ApiProperty({
+ example: 'duckduckgo',
+ description: 'utm_source URL parameter',
+ })
+ so?: string
+
+ @ApiProperty({
+ example: 'cpc',
+ description: 'utm_medium URL parameter',
+ })
+ me?: string
+
+ @ApiProperty({
+ example: 'spring_sale',
+ description: 'utm_campaign URL parameter',
+ })
+ ca?: string
+}
diff --git a/selfhosted-src/analytics/dto/get-custom-events.dto.ts b/selfhosted-src/analytics/dto/get-custom-events.dto.ts
new file mode 100644
index 000000000..76532db9d
--- /dev/null
+++ b/selfhosted-src/analytics/dto/get-custom-events.dto.ts
@@ -0,0 +1,14 @@
+import { PickType } from '@nestjs/swagger'
+import { AnalyticsGET_DTO } from './getData.dto'
+
+export class GetCustomEventsDto extends PickType(AnalyticsGET_DTO, [
+ 'pid',
+ 'period',
+ 'timeBucket',
+ 'from',
+ 'to',
+ 'filters',
+ 'timezone',
+] as const) {
+ customEvents: string
+}
diff --git a/selfhosted-src/analytics/dto/getData.dto.ts b/selfhosted-src/analytics/dto/getData.dto.ts
new file mode 100644
index 000000000..53a39d5bc
--- /dev/null
+++ b/selfhosted-src/analytics/dto/getData.dto.ts
@@ -0,0 +1,41 @@
+import { ApiProperty } from '@nestjs/swagger'
+import { IsNotEmpty } from 'class-validator'
+import { DEFAULT_TIMEZONE } from '../../user/entities/user.entity'
+
+export enum TimeBucketType {
+ HOUR = 'hour',
+ DAY = 'day',
+ MONTH = 'month',
+}
+
+// eslint-disable-next-line @typescript-eslint/naming-convention
+export class AnalyticsGET_DTO {
+ @ApiProperty()
+ @IsNotEmpty()
+ pid: string
+
+ @ApiProperty()
+ period: string
+
+ @ApiProperty({ enum: TimeBucketType })
+ @IsNotEmpty()
+ timeBucket: TimeBucketType
+
+ @ApiProperty()
+ from: string
+
+ @ApiProperty()
+ to: string
+
+ @ApiProperty({
+ description:
+ 'A stringified array of properties to filter [{ column, filter, isExclusive }]',
+ })
+ filters: string
+
+ @ApiProperty({
+ description: 'Timezone to display data in',
+ default: DEFAULT_TIMEZONE,
+ })
+ timezone: string
+}
diff --git a/selfhosted-src/analytics/dto/getUserFlow.dto.ts b/selfhosted-src/analytics/dto/getUserFlow.dto.ts
new file mode 100644
index 000000000..63fd88641
--- /dev/null
+++ b/selfhosted-src/analytics/dto/getUserFlow.dto.ts
@@ -0,0 +1,31 @@
+import { ApiProperty } from '@nestjs/swagger'
+import { IsNotEmpty } from 'class-validator'
+import { DEFAULT_TIMEZONE } from '../../user/entities/user.entity'
+
+// eslint-disable-next-line @typescript-eslint/naming-convention
+export class GetUserFlowDTO {
+ @ApiProperty()
+ @IsNotEmpty()
+ pid: string
+
+ @ApiProperty()
+ period: string
+
+ @ApiProperty()
+ from: string
+
+ @ApiProperty()
+ to: string
+
+ @ApiProperty({
+ description: 'Timezone to display data in',
+ default: DEFAULT_TIMEZONE,
+ })
+ timezone: string
+
+ @ApiProperty({
+ description:
+ 'A stringified array of properties to filter [{ column, filter, isExclusive }]',
+ })
+ filters: string
+}
diff --git a/selfhosted-src/analytics/dto/pageviews.dto.ts b/selfhosted-src/analytics/dto/pageviews.dto.ts
new file mode 100644
index 000000000..a8ea10615
--- /dev/null
+++ b/selfhosted-src/analytics/dto/pageviews.dto.ts
@@ -0,0 +1,91 @@
+import { ApiProperty } from '@nestjs/swagger'
+import { IsNotEmpty } from 'class-validator'
+
+export class PageviewsDTO {
+ @ApiProperty({
+ example: 'aUn1quEid-3',
+ required: true,
+ description: 'The project ID',
+ })
+ @IsNotEmpty()
+ pid: string
+
+ // Tracking metrics
+ @ApiProperty({
+ example: 'Europe/Kiev',
+ description: "User's timezone",
+ })
+ tz?: string
+
+ @ApiProperty({
+ example: '/articles/my-awesome-article-1',
+ description: 'A page that user sent data from',
+ })
+ pg?: string
+
+ @ApiProperty({
+ example: '/home',
+ description: 'Previous page user was on',
+ })
+ prev?: string
+
+ @ApiProperty({
+ example: 'en-GB',
+ description: "User's locale",
+ })
+ lc?: string
+
+ @ApiProperty({
+ example: 'https://example.com',
+ description: 'The referrer',
+ })
+ ref?: string
+
+ @ApiProperty({
+ example: 'duckduckgo',
+ description: 'utm_source URL parameter',
+ })
+ so?: string
+
+ @ApiProperty({
+ example: 'cpc',
+ description: 'utm_medium URL parameter',
+ })
+ me?: string
+
+ @ApiProperty({
+ example: 'spring_sale',
+ description: 'utm_campaign URL parameter',
+ })
+ ca?: string
+
+ @ApiProperty({
+ example: false,
+ description: 'If true, only unique events will be saved',
+ })
+ unique?: boolean
+
+ @ApiProperty({
+ example: {
+ conn: 120,
+ dns: 170,
+ dom_load: 975.5,
+ page_load: 1515,
+ render: 532.25,
+ response: 170,
+ tls: 0,
+ ttfb: 78.3,
+ },
+ description: 'Performance metrics',
+ })
+ perf?: {
+ dns: number
+ tls: number
+ conn: number
+ response: number
+ render: number
+ dom_load: number
+ page_load: number
+ ttfb: number
+ }
+}
diff --git a/selfhosted-src/analytics/interfaces/index.ts b/selfhosted-src/analytics/interfaces/index.ts
new file mode 100644
index 000000000..0844bcaf6
--- /dev/null
+++ b/selfhosted-src/analytics/interfaces/index.ts
@@ -0,0 +1,93 @@
+export interface TrafficCEFilterCHResponse {
+ year: number
+ month: number
+ day?: number
+ hour?: number
+ count: number
+}
+
+export interface TrafficCHResponse {
+ year: number
+ month: number
+ day?: number
+ hour?: number
+ sdur: number
+ uniques: number
+ pageviews: number
+}
+
+export interface PerformanceCHResponse {
+ year: number
+ month: number
+ day?: number
+ hour?: number
+ dns: number
+ tls: number
+ conn: number
+ response: number
+ render: number
+ domLoad: number
+ ttfb: number
+}
+
+export interface CustomsCHResponse {
+ ev: string
+ 'count()': number
+ index: number
+}
+
+export interface CustomsCHAggregatedResponse {
+ year: number
+ month: number
+ day?: number
+ hour?: number
+ ev: string
+ count: number
+}
+
+export interface IGetGroupFromTo {
+ groupFrom: string
+ groupTo: string
+ groupFromUTC: string
+ groupToUTC: string
+}
+
+export interface GetFiltersQuery extends Array<string | object> {
+ // SQL query
+ 0: string
+ // an object that has structure like { cf_pg: '/signup', ev_exclusive: false }
+ 1: { [key: string]: string | boolean }
+ // an array of objects like [{ "column":"pg", "filter":"/signup", "isExclusive":true }]
+ 2: Array<{ [key: string]: string }> | []
+}
+
+export interface IUserFlowNode {
+ id: string
+}
+
+export interface IUserFlowLink {
+ source: string
+ target: string
+ value: number
+}
+
+export interface IGenerateXAxis {
+ x: string[]
+ xShifted: string[]
+}
+
+export interface IBuildUserFlow {
+ nodes: IUserFlowNode[]
+ links: IUserFlowLink[]
+}
+
+export interface IUserFlow {
+ ascending: IBuildUserFlow
+ descending: IBuildUserFlow
+}
+
+export interface IExtractChartData {
+ visits: number[]
+ uniques: number[]
+ sdur: number[]
+}
diff --git a/selfhosted-src/app.module.ts b/selfhosted-src/app.module.ts
new file mode 100644
index 000000000..89832f892
--- /dev/null
+++ b/selfhosted-src/app.module.ts
@@ -0,0 +1,36 @@
+import { Module } from '@nestjs/common'
+import { ConfigModule } from '@nestjs/config'
+import { ScheduleModule } from '@nestjs/schedule'
+import { NestjsFormDataModule } from 'nestjs-form-data'
+
+import { I18nModule } from 'nestjs-i18n'
+import { UserModule } from './user/user.module'
+import { AnalyticsModule } from './analytics/analytics.module'
+import { ProjectModule } from './project/project.module'
+import { TaskManagerModule } from './task-manager/task-manager.module'
+import { PingModule } from './ping/ping.module'
+import { getI18nConfig } from './configs'
+import { AuthModule } from './auth/auth.module'
+
+const imports = [
+ ConfigModule.forRoot({
+ cache: true,
+ envFilePath: '.env',
+ expandVariables: true,
+ isGlobal: true,
+ }),
+ I18nModule.forRootAsync(getI18nConfig()),
+ ScheduleModule.forRoot(),
+ NestjsFormDataModule.config({ isGlobal: true }),
+ TaskManagerModule,
+ UserModule,
+ ProjectModule,
+ AnalyticsModule,
+ PingModule,
+ AuthModule,
+]
+
+@Module({
+ imports,
+})
+export class AppModule {}
diff --git a/selfhosted-src/auth/auth.controller.ts b/selfhosted-src/auth/auth.controller.ts
new file mode 100644
index 000000000..dc6d3d89c
--- /dev/null
+++ b/selfhosted-src/auth/auth.controller.ts
@@ -0,0 +1,154 @@
+import {
+ Controller,
+ UseFilters,
+ UseGuards,
+ UsePipes,
+ ValidationPipe,
+ Post,
+ Body,
+ Ip,
+ ConflictException,
+ HttpCode,
+ HttpStatus,
+ UnauthorizedException,
+ Headers,
+} from '@nestjs/common'
+import {
+ ApiTags,
+ ApiOperation,
+ ApiOkResponse,
+} from '@nestjs/swagger'
+import { I18nValidationExceptionFilter, I18n, I18nContext } from 'nestjs-i18n'
+
+import { checkRateLimit } from 'selfhosted-src/common/utils'
+import {
+ generateSelfhostedUser,
+} from 'selfhosted-src/user/entities/user.entity'
+import { AuthService } from './auth.service'
+import { Public, CurrentUserId, CurrentUser } from './decorators'
+import {
+ LoginResponseDto,
+ LoginRequestDto,
+} from './dtos'
+import { JwtAccessTokenGuard, JwtRefreshTokenGuard } from './guards'
+
+@ApiTags('Auth')
+@Controller({ path: 'auth', version: '1' })
+@UseFilters(new I18nValidationExceptionFilter())
+@UseGuards(JwtAccessTokenGuard)
+@UsePipes(
+ new ValidationPipe({
+ forbidNonWhitelisted: true,
+ forbidUnknownValues: true,
+ transform: true,
+ whitelist: true,
+ }),
+)
+export class AuthController {
+ constructor(
+ private readonly authService: AuthService,
+ ) {}
+
+ @ApiOperation({ summary: 'Login a user' })
+ @ApiOkResponse({
+ description: 'User logged in',
+ type: LoginResponseDto,
+ })
+ @Public()
+ @Post('login')
+ @HttpCode(HttpStatus.OK)
+ public async login(
+ @Body() body: LoginRequestDto,
+ @I18n() i18n: I18nContext,
+ @Headers() headers: unknown,
+ @Ip() requestIp: string,
+ ): Promise<LoginResponseDto> {
+ const ip =
+ headers['x-forwarded-for'] || headers['cf-connecting-ip'] || requestIp
+
+ await checkRateLimit(ip, 'login', 10, 1800)
+
+ let user = await this.authService.validateUser(body.email, body.password)
+
+ if (!user) {
+ throw new ConflictException(i18n.t('auth.invalidCredentials'))
+ }
+
+ const jwtTokens = await this.authService.generateJwtTokens(
+ user.id,
+ !user.isTwoFactorAuthenticationEnabled,
+ )
+
+ return {
+ ...jwtTokens,
+ user,
+ }
+ }
+
+ @ApiOperation({ summary: 'Refresh a token' })
+ @ApiOkResponse({
+ description: 'Token refreshed',
+ })
+ @Public()
+ @UseGuards(JwtRefreshTokenGuard)
+ @Post('refresh-token')
+ @HttpCode(200)
+ public async refreshToken(
+ @CurrentUserId() userId: string,
+ @CurrentUser('refreshToken') refreshToken: string,
+ @I18n() i18n: I18nContext,
+ ): Promise<{ accessToken: string }> {
+ const user = generateSelfhostedUser()
+
+ if (!user) {
+ throw new UnauthorizedException()
+ }
+
+ const isRefreshTokenValid = await this.authService.checkRefreshToken(
+ user.id,
+ refreshToken,
+ )
+
+ if (!isRefreshTokenValid) {
+ throw new ConflictException(i18n.t('auth.invalidRefreshToken'))
+ }
+
+ const accessToken = await this.authService.generateJwtAccessToken(
+ user.id,
+ user.isTwoFactorAuthenticationEnabled,
+ )
+
+ return { accessToken }
+ }
+
+ @ApiOperation({ summary: 'Logout' })
+ @ApiOkResponse({
+ description: 'Logged out',
+ })
+ @Public()
+ @UseGuards(JwtRefreshTokenGuard)
+ @Post('logout')
+ @HttpCode(200)
+ public async logout(
+ @CurrentUserId() userId: string,
+ @CurrentUser('refreshToken') refreshToken: string,
+ @I18n() i18n: I18nContext,
+ ): Promise<void> {
+ const user = generateSelfhostedUser()
+
+ if (!user) {
+ throw new UnauthorizedException()
+ }
+
+ const isRefreshTokenValid = await this.authService.checkRefreshToken(
+ user.id,
+ refreshToken,
+ )
+
+ if (!isRefreshTokenValid) {
+ throw new ConflictException(i18n.t('auth.invalidRefreshToken'))
+ }
+
+ await this.authService.logout(user.id, refreshToken)
+ }
+}
diff --git a/selfhosted-src/auth/auth.module.ts b/selfhosted-src/auth/auth.module.ts
new file mode 100644
index 000000000..966cffd20
--- /dev/null
+++ b/selfhosted-src/auth/auth.module.ts
@@ -0,0 +1,26 @@
+import { Module } from '@nestjs/common'
+import { JwtModule } from '@nestjs/jwt'
+import { PassportModule } from '@nestjs/passport'
+import { AuthController } from './auth.controller'
+import { AuthService } from './auth.service'
+import {
+ // ApiKeyStrategy,
+ JwtAccessTokenStrategy,
+ JwtRefreshTokenStrategy,
+} from './strategies'
+
+@Module({
+ imports: [
+ JwtModule.register({}),
+ PassportModule,
+ ],
+ controllers: [AuthController],
+ providers: [
+ AuthService,
+ JwtAccessTokenStrategy,
+ JwtRefreshTokenStrategy,
+ // ApiKeyStrategy,
+ ],
+ exports: [AuthService],
+})
+export class AuthModule {}
diff --git a/selfhosted-src/auth/auth.service.ts b/selfhosted-src/auth/auth.service.ts
new file mode 100644
index 000000000..0a986265d
--- /dev/null
+++ b/selfhosted-src/auth/auth.service.ts
@@ -0,0 +1,117 @@
+import {
+ Injectable,
+} from '@nestjs/common'
+import { ConfigService } from '@nestjs/config'
+import { JwtService } from '@nestjs/jwt'
+import { genSalt, hash } from 'bcrypt'
+import * as _isEmpty from 'lodash/isEmpty'
+
+import {
+ SelfhostedUser,
+ generateSelfhostedUser,
+} from 'selfhosted-src/user/entities/user.entity'
+import {
+ saveRefreshTokenClickhouse,
+ findRefreshTokenClickhouse,
+ deleteRefreshTokenClickhouse,
+} from 'selfhosted-src/common/utils'
+import {
+ SELFHOSTED_EMAIL,
+ SELFHOSTED_PASSWORD,
+ JWT_ACCESS_TOKEN_SECRET,
+} from 'selfhosted-src/common/constants'
+
+@Injectable()
+export class AuthService {
+ constructor(
+ private readonly configService: ConfigService,
+ private readonly jwtService: JwtService,
+ ) {}
+
+ public async hashPassword(password: string): Promise<string> {
+ const salt = await genSalt(10)
+ return hash(password, salt)
+ }
+
+ public async generateJwtAccessToken(
+ userId: string,
+ isSecondFactorAuthenticated = false,
+ ) {
+ return this.jwtService.signAsync(
+ {
+ sub: userId,
+ isSecondFactorAuthenticated,
+ },
+ {
+ algorithm: 'HS256',
+ expiresIn: 60 * 30,
+ secret: JWT_ACCESS_TOKEN_SECRET,
+ },
+ )
+ }
+
+ public async validateUser(
+ email: string,
+ password: string,
+ ): Promise<SelfhostedUser | null> {
+ if (email !== SELFHOSTED_EMAIL || password !== SELFHOSTED_PASSWORD) {
+ return null
+ }
+
+ return generateSelfhostedUser()
+ }
+
+ private async generateJwtRefreshToken(
+ userId: string,
+ isSecondFactorAuthenticated = false,
+ ) {
+ const refreshToken = await this.jwtService.signAsync(
+ {
+ sub: userId,
+ isSecondFactorAuthenticated,
+ },
+ {
+ algorithm: 'HS256',
+ expiresIn: 60 * 60 * 24 * 30,
+ secret: this.configService.get('JWT_REFRESH_TOKEN_SECRET'),
+ },
+ )
+
+ await saveRefreshTokenClickhouse(userId, refreshToken)
+
+ return refreshToken
+ }
+
+ public async checkRefreshToken(
+ userId: string,
+ refreshToken: string,
+ ): Promise<boolean> {
+ const tokens = await findRefreshTokenClickhouse(userId, refreshToken)
+ return !_isEmpty(tokens)
+ }
+
+ public async generateJwtTokens(
+ userId: string,
+ isSecondFactorAuthenticated = false,
+ ) {
+ const accessToken = await this.generateJwtAccessToken(
+ userId,
+ isSecondFactorAuthenticated,
+ )
+
+ let refreshToken = 'NOT_AVAILABLE'
+
+ if (isSecondFactorAuthenticated) {
+ refreshToken = await this.generateJwtRefreshToken(
+ userId,
+ isSecondFactorAuthenticated,
+ )
+ }
+
+ return { accessToken, refreshToken }
+ }
+
+ async logout(userId: string, refreshToken: string) {
+ await deleteRefreshTokenClickhouse(userId, refreshToken)
+ }
+}
diff --git a/selfhosted-src/auth/decorators/auth.decorator.ts b/selfhosted-src/auth/decorators/auth.decorator.ts
new file mode 100644
index 000000000..c5491076b
--- /dev/null
+++ b/selfhosted-src/auth/decorators/auth.decorator.ts
@@ -0,0 +1,24 @@
+import { applyDecorators, SetMetadata, UseGuards } from '@nestjs/common'
+import {
+ ApiKeyRateLimitGuard,
+ JwtAccessTokenGuard,
+ MultiAuthGuard,
+ RolesGuard,
+} from 'selfhosted-src/auth/guards'
+import { UserType } from 'selfhosted-src/user/entities/user.entity'
+import { ROLES_KEY } from './roles.decorator'
+
+export const IS_OPTIONAL_AUTH_KEY = 'isOptionalAuth'
+export function Auth(
+ roles: UserType[],
+ isApiKeyAuth = false,
+ isOptionalAuth = false,
+) {
+ return applyDecorators(
+ SetMetadata(ROLES_KEY, roles),
+ SetMetadata(IS_OPTIONAL_AUTH_KEY, isOptionalAuth),
+ isApiKeyAuth
+ ? UseGuards(MultiAuthGuard, ApiKeyRateLimitGuard, RolesGuard)
+ : UseGuards(JwtAccessTokenGuard, RolesGuard),
+ )
+}
diff --git a/selfhosted-src/auth/decorators/current-user-id.decorator.ts b/selfhosted-src/auth/decorators/current-user-id.decorator.ts
new file mode 100644
index 000000000..629fd4246
--- /dev/null
+++ b/selfhosted-src/auth/decorators/current-user-id.decorator.ts
@@ -0,0 +1,9 @@
+import { createParamDecorator, ExecutionContext } from '@nestjs/common'
+
+export const CurrentUserId = createParamDecorator(
+ (_: unknown, context: ExecutionContext) => {
+ const request = context.switchToHttp().getRequest()
+ const { user } = request
+ return user && (user.sub || user.id) ? user.sub || user.id : null
+ },
+)
diff --git a/selfhosted-src/auth/decorators/current-user.decorator.ts b/selfhosted-src/auth/decorators/current-user.decorator.ts
new file mode 100644
index 000000000..9c9d258d4
--- /dev/null
+++ b/selfhosted-src/auth/decorators/current-user.decorator.ts
@@ -0,0 +1,14 @@
+import { createParamDecorator, ExecutionContext } from '@nestjs/common'
+import { IJwtRefreshTokenPayload } from '../interfaces'
+
+export const CurrentUser = createParamDecorator(
+ (
+ data: keyof IJwtRefreshTokenPayload | undefined,
+ context: ExecutionContext,
+ ) => {
+ const request = context.switchToHttp().getRequest()
+ const { user } = request
+
+ return data ? user && user[data] : user
+ },
+)
diff --git a/selfhosted-src/auth/decorators/index.ts b/selfhosted-src/auth/decorators/index.ts
new file mode 100644
index 000000000..377aac6a6
--- /dev/null
+++ b/selfhosted-src/auth/decorators/index.ts
@@ -0,0 +1,6 @@
+export * from './current-user-id.decorator'
+export * from './public.decorator'
+export * from './current-user.decorator'
+export * from './two-fa-not-required.decorator'
+export * from './roles.decorator'
+export * from './auth.decorator'
diff --git a/selfhosted-src/auth/decorators/public.decorator.ts b/selfhosted-src/auth/decorators/public.decorator.ts
new file mode 100644
index 000000000..91c2399b8
--- /dev/null
+++ b/selfhosted-src/auth/decorators/public.decorator.ts
@@ -0,0 +1,4 @@
+import { SetMetadata } from '@nestjs/common'
+
+export const IS_PUBLIC_KEY = 'isPublic'
+export const Public = () => SetMetadata(IS_PUBLIC_KEY, true)
diff --git a/selfhosted-src/auth/decorators/roles.decorator.ts b/selfhosted-src/auth/decorators/roles.decorator.ts
new file mode 100644
index 000000000..fb70bbeb1
--- /dev/null
+++ b/selfhosted-src/auth/decorators/roles.decorator.ts
@@ -0,0 +1,5 @@
+import { SetMetadata } from '@nestjs/common'
+import { UserType } from 'selfhosted-src/user/entities/user.entity'
+
+export const ROLES_KEY = 'roles'
+export const Roles = (...roles: UserType[]) => SetMetadata(ROLES_KEY, roles)
diff --git a/selfhosted-src/auth/decorators/two-fa-not-required.decorator.ts b/selfhosted-src/auth/decorators/two-fa-not-required.decorator.ts
new file mode 100644
index 000000000..7f0839d32
--- /dev/null
+++ b/selfhosted-src/auth/decorators/two-fa-not-required.decorator.ts
@@ -0,0 +1,5 @@
+import { SetMetadata } from '@nestjs/common'
+
+export const IS_TWO_FA_NOT_REQUIRED_KEY = 'isTwoFaNotRequired'
+export const TwoFaNotRequired = () =>
+ SetMetadata(IS_TWO_FA_NOT_REQUIRED_KEY, true)
diff --git a/selfhosted-src/auth/dtos/change-password.dto.ts b/selfhosted-src/auth/dtos/change-password.dto.ts
new file mode 100644
index 000000000..2f8e41311
--- /dev/null
+++ b/selfhosted-src/auth/dtos/change-password.dto.ts
@@ -0,0 +1,24 @@
+import { ApiProperty } from '@nestjs/swagger'
+import { MaxLength, MinLength } from 'class-validator'
+
+export class ChangePasswordDto {
+ @ApiProperty({
+ description: 'User password',
+ example: '%d7*c4W45p',
+ maxLength: 72,
+ minLength: 8,
+ })
+ @MaxLength(50, { message: 'Max length is $constraint1 characters' })
+ @MinLength(8, { message: 'Min length is $constraint1 characters' })
+ public readonly oldPassword: string
+
+ @ApiProperty({
+ description: 'User password',
+ example: '%d7*c4W45p',
+ maxLength: 72,
+ minLength: 8,
+ })
+ @MaxLength(50, { message: 'Max length is $constraint1 characters' })
+ @MinLength(8, { message: 'Min length is $constraint1 characters' })
+ public readonly newPassword: string
+}
diff --git a/selfhosted-src/auth/dtos/confirm-change-email.dto.ts b/selfhosted-src/auth/dtos/confirm-change-email.dto.ts
new file mode 100644
index 000000000..978d62050
--- /dev/null
+++ b/selfhosted-src/auth/dtos/confirm-change-email.dto.ts
@@ -0,0 +1,11 @@
+import { ApiProperty } from '@nestjs/swagger'
+import { IsUUID } from 'class-validator'
+
+export class ConfirmChangeEmailDto {
+ @ApiProperty({
+ description: 'The token to confirm the change email',
+ example: 'a1b2c3d4-e5f6-g7h8-i9j0-k1l2m3n4o5p6',
+ })
+ @IsUUID('4', { message: 'Please enter the valid UUID.' })
+ public readonly token: string
+}
diff --git a/selfhosted-src/auth/dtos/confirm-reset-password.dto.ts b/selfhosted-src/auth/dtos/confirm-reset-password.dto.ts
new file mode 100644
index 000000000..4d367f1f6
--- /dev/null
+++ b/selfhosted-src/auth/dtos/confirm-reset-password.dto.ts
@@ -0,0 +1,11 @@
+import { ApiProperty } from '@nestjs/swagger'
+import { IsUUID } from 'class-validator'
+
+export class ConfirmResetPasswordDto {
+ @ApiProperty({
+ description: 'The token to confirm the reset password',
+ example: 'a1b2c3d4-e5f6-g7h8-i9j0-k1l2m3n4o5p6',
+ })
+ @IsUUID('4', { message: 'Please enter the valid UUID.' })
+ public readonly token: string
+}
diff --git a/selfhosted-src/auth/dtos/index.ts b/selfhosted-src/auth/dtos/index.ts
new file mode 100644
index 000000000..7afca67e8
--- /dev/null
+++ b/selfhosted-src/auth/dtos/index.ts
@@ -0,0 +1,14 @@
+export * from './register.dto'
+export * from './login.dto'
+export * from './verify-email.dto'
+export * from './request-reset-password.dto'
+export * from './confirm-reset-password.dto'
+export * from './reset-password.dto'
+export * from './change-password.dto'
+export * from './request-change-email.dto'
+export * from './confirm-change-email.dto'
+export * from './process-sso-code.dto'
+export * from './sso-generate.dto'
+export * from './sso-get-jwt-by-hash.dto'
+export * from './sso-link.dto'
+export * from './sso-unlink.dto'
diff --git a/selfhosted-src/auth/dtos/login.dto.ts b/selfhosted-src/auth/dtos/login.dto.ts
new file mode 100644
index 000000000..62ed65270
--- /dev/null
+++ b/selfhosted-src/auth/dtos/login.dto.ts
@@ -0,0 +1,23 @@
+import { ApiProperty, OmitType } from '@nestjs/swagger'
+import { RegisterRequestDto } from './register.dto'
+
+export class LoginRequestDto extends OmitType(RegisterRequestDto, [
+ 'checkIfLeaked',
+] as const) {}
+
+export class LoginResponseDto {
+ @ApiProperty({
+ description: 'Access token (JWT)',
+ })
+ public readonly accessToken: string
+
+ @ApiProperty({
+ description: 'Refresh token (JWT)',
+ })
+ public readonly refreshToken: string
+
+ @ApiProperty({
+ description: 'User entity',
+ })
+ user: object
+}
diff --git a/selfhosted-src/auth/dtos/process-sso-code.dto.ts b/selfhosted-src/auth/dtos/process-sso-code.dto.ts
new file mode 100644
index 000000000..cc05d0935
--- /dev/null
+++ b/selfhosted-src/auth/dtos/process-sso-code.dto.ts
@@ -0,0 +1,18 @@
+import { ApiProperty } from '@nestjs/swagger'
+import { IsString } from 'class-validator'
+
+export class ProcessSSOCodeDto {
+ @ApiProperty({
+ description: 'Google auth token',
+ example: '4/0AVHE...',
+ })
+ @IsString()
+ token: string
+
+ @ApiProperty({
+ description: 'SSO session identifier',
+ example: '123e4567-e89b-12d3-a456-426614174000',
+ })
+ @IsString()
+ hash: string
+}
diff --git a/selfhosted-src/auth/dtos/register.dto.ts b/selfhosted-src/auth/dtos/register.dto.ts
new file mode 100644
index 000000000..05c4d0e39
--- /dev/null
+++ b/selfhosted-src/auth/dtos/register.dto.ts
@@ -0,0 +1,54 @@
+import { ApiProperty } from '@nestjs/swagger'
+import {
+ IsEmail,
+ IsNotEmpty,
+ IsBoolean,
+ MaxLength,
+ MinLength,
+} from 'class-validator'
+
+export class RegisterRequestDto {
+ @ApiProperty({
+ description: 'User email',
+ example: '[email protected]',
+ maxLength: 254,
+ minLength: 6,
+ })
+ @IsEmail({}, { message: 'Please enter the valid email.' })
+ public readonly email: string
+
+ @ApiProperty({
+ description: 'User password',
+ example: '%d7*c4W45p',
+ maxLength: 72,
+ minLength: 8,
+ })
+ @MaxLength(50, { message: 'Max length is $constraint1 characters' })
+ @MinLength(8, { message: 'Min length is $constraint1 characters' })
+ public readonly password: string
+
+ @ApiProperty({
+ description: 'Check if password is leaked',
+ example: true,
+ })
+ @IsNotEmpty({ message: 'This field is required.' })
+ @IsBoolean({ message: 'Please enter the valid value (true or false).' })
+ public readonly checkIfLeaked: boolean
+}
+
+export class RegisterResponseDto {
+ @ApiProperty({
+ description: 'Access token (JWT)',
+ })
+ public readonly accessToken: string
+
+ @ApiProperty({
+ description: 'Refresh token (JWT)',
+ })
+ public readonly refreshToken: string
+
+ @ApiProperty({
+ description: 'User entity',
+ })
+ user: object
+}
diff --git a/selfhosted-src/auth/dtos/request-change-email.dto.ts b/selfhosted-src/auth/dtos/request-change-email.dto.ts
new file mode 100644
index 000000000..6c1bde5db
--- /dev/null
+++ b/selfhosted-src/auth/dtos/request-change-email.dto.ts
@@ -0,0 +1,23 @@
+import { ApiProperty } from '@nestjs/swagger'
+import { IsEmail, MaxLength, MinLength } from 'class-validator'
+
+export class RequestChangeEmailDto {
+ @ApiProperty({
+ description: 'User email',
+ example: '[email protected]',
+ maxLength: 254,
+ minLength: 6,
+ })
+ @IsEmail({}, { message: 'Please enter the valid email.' })
+ public readonly newEmail: string
+
+ @ApiProperty({
+ description: 'User password',
+ example: '%d7*c4W45p',
+ maxLength: 72,
+ minLength: 8,
+ })
+ @MaxLength(50, { message: 'Max length is $constraint1 characters' })
+ @MinLength(8, { message: 'Min length is $constraint1 characters' })
+ public readonly password: string
+}
diff --git a/selfhosted-src/auth/dtos/request-reset-password.dto.ts b/selfhosted-src/auth/dtos/request-reset-password.dto.ts
new file mode 100644
index 000000000..a2a09a383
--- /dev/null
+++ b/selfhosted-src/auth/dtos/request-reset-password.dto.ts
@@ -0,0 +1,13 @@
+import { ApiProperty } from '@nestjs/swagger'
+import { IsEmail } from 'class-validator'
+
+export class RequestResetPasswordDto {
+ @ApiProperty({
+ description: 'User email',
+ example: '[email protected]',
+ maxLength: 254,
+ minLength: 6,
+ })
+ @IsEmail({}, { message: 'Please enter the valid email.' })
+ public readonly email: string
+}
diff --git a/selfhosted-src/auth/dtos/reset-password.dto.ts b/selfhosted-src/auth/dtos/reset-password.dto.ts
new file mode 100644
index 000000000..14df63e68
--- /dev/null
+++ b/selfhosted-src/auth/dtos/reset-password.dto.ts
@@ -0,0 +1,14 @@
+import { ApiProperty } from '@nestjs/swagger'
+import { MaxLength, MinLength } from 'class-validator'
+
+export class ResetPasswordDto {
+ @ApiProperty({
+ description: 'User password',
+ example: '%d7*c4W45p',
+ maxLength: 72,
+ minLength: 8,
+ })
+ @MaxLength(50, { message: 'Max length is $constraint1 characters' })
+ @MinLength(8, { message: 'Min length is $constraint1 characters' })
+ public readonly newPassword: string
+}
diff --git a/selfhosted-src/auth/dtos/sso-generate.dto.ts b/selfhosted-src/auth/dtos/sso-generate.dto.ts
new file mode 100644
index 000000000..c0e43ab99
--- /dev/null
+++ b/selfhosted-src/auth/dtos/sso-generate.dto.ts
@@ -0,0 +1,16 @@
+import { ApiProperty } from '@nestjs/swagger'
+import { IsEnum } from 'class-validator'
+
+export enum SSOProviders {
+ GOOGLE = 'google',
+ GITHUB = 'github',
+}
+
+export class SSOGenerateDto {
+ @ApiProperty({
+ description: 'SSO provider name',
+ enum: SSOProviders,
+ })
+ @IsEnum(SSOProviders)
+ provider: SSOProviders
+}
diff --git a/selfhosted-src/auth/dtos/sso-get-jwt-by-hash.dto.ts b/selfhosted-src/auth/dtos/sso-get-jwt-by-hash.dto.ts
new file mode 100644
index 000000000..cc58875d2
--- /dev/null
+++ b/selfhosted-src/auth/dtos/sso-get-jwt-by-hash.dto.ts
@@ -0,0 +1,20 @@
+import { ApiProperty } from '@nestjs/swagger'
+import { IsEnum, IsString } from 'class-validator'
+
+import { SSOProviders } from './sso-generate.dto'
+
+export class SSOGetJWTByHashDto {
+ @ApiProperty({
+ description: 'SSO session identifier',
+ example: '123e4567-e89b-12d3-a456-426614174000',
+ })
+ @IsString()
+ hash: string
+
+ @ApiProperty({
+ description: 'SSO provider name',
+ enum: SSOProviders,
+ })
+ @IsEnum(SSOProviders)
+ provider: SSOProviders
+}
diff --git a/selfhosted-src/auth/dtos/sso-link.dto.ts b/selfhosted-src/auth/dtos/sso-link.dto.ts
new file mode 100644
index 000000000..5e244573e
--- /dev/null
+++ b/selfhosted-src/auth/dtos/sso-link.dto.ts
@@ -0,0 +1,20 @@
+import { ApiProperty } from '@nestjs/swagger'
+import { IsEnum, IsString } from 'class-validator'
+
+import { SSOProviders } from './sso-generate.dto'
+
+export class SSOLinkDto {
+ @ApiProperty({
+ description: 'SSO session identifier',
+ example: '123e4567-e89b-12d3-a456-426614174000',
+ })
+ @IsString()
+ hash: string
+
+ @ApiProperty({
+ description: 'SSO provider name',
+ enum: SSOProviders,
+ })
+ @IsEnum(SSOProviders)
+ provider: SSOProviders
+}
diff --git a/selfhosted-src/auth/dtos/sso-unlink.dto.ts b/selfhosted-src/auth/dtos/sso-unlink.dto.ts
new file mode 100644
index 000000000..f45697a9b
--- /dev/null
+++ b/selfhosted-src/auth/dtos/sso-unlink.dto.ts
@@ -0,0 +1,13 @@
+import { ApiProperty } from '@nestjs/swagger'
+import { IsEnum } from 'class-validator'
+
+import { SSOProviders } from './sso-generate.dto'
+
+export class SSOUnlinkDto {
+ @ApiProperty({
+ description: 'SSO provider name',
+ enum: SSOProviders,
+ })
+ @IsEnum(SSOProviders)
+ provider: SSOProviders
+}
diff --git a/selfhosted-src/auth/dtos/verify-email.dto.ts b/selfhosted-src/auth/dtos/verify-email.dto.ts
new file mode 100644
index 000000000..e03420979
--- /dev/null
+++ b/selfhosted-src/auth/dtos/verify-email.dto.ts
@@ -0,0 +1,11 @@
+import { ApiProperty } from '@nestjs/swagger'
+import { IsUUID } from 'class-validator'
+
+export class VerifyEmailDto {
+ @ApiProperty({
+ description: 'The token to verify the email',
+ example: 'a1b2c3d4-e5f6-g7h8-i9j0-k1l2m3n4o5p6',
+ })
+ @IsUUID('4', { message: 'Please enter the valid UUID.' })
+ public readonly token: string
+}
diff --git a/selfhosted-src/auth/guards/api-key-rate-limit.guard.ts b/selfhosted-src/auth/guards/api-key-rate-limit.guard.ts
new file mode 100644
index 000000000..35f6e4fbd
--- /dev/null
+++ b/selfhosted-src/auth/guards/api-key-rate-limit.guard.ts
@@ -0,0 +1,21 @@
+import { CanActivate, ExecutionContext, Injectable } from '@nestjs/common'
+import { checkRateLimitForApiKey } from 'selfhosted-src/common/utils'
+import { ACCOUNT_PLANS, PlanCode } from 'selfhosted-src/user/entities/user.entity'
+
+@Injectable()
+export class ApiKeyRateLimitGuard implements CanActivate {
+ async canActivate(context: ExecutionContext) {
+ const request = context.switchToHttp().getRequest()
+ const { user } = request
+
+ if (request.headers['x-api-key']) {
+ if (!user) return false
+
+ const reqAmount =
+ ACCOUNT_PLANS[user.planCode as PlanCode].maxApiKeyRequestsPerHour
+ return checkRateLimitForApiKey(user.apiKey, reqAmount)
+ }
+
+ return true
+ }
+}
diff --git a/selfhosted-src/auth/guards/api-key.guard.ts b/selfhosted-src/auth/guards/api-key.guard.ts
new file mode 100644
index 000000000..43e58ce2d
--- /dev/null
+++ b/selfhosted-src/auth/guards/api-key.guard.ts
@@ -0,0 +1,5 @@
+import { Injectable } from '@nestjs/common'
+import { AuthGuard } from '@nestjs/passport'
+
+@Injectable()
+export class ApiKeyGuard extends AuthGuard('api-key') {}
diff --git a/selfhosted-src/auth/guards/index.ts b/selfhosted-src/auth/guards/index.ts
new file mode 100644
index 000000000..0658afa44
--- /dev/null
+++ b/selfhosted-src/auth/guards/index.ts
@@ -0,0 +1,7 @@
+export * from './jwt-access-token.guard'
+export * from './jwt-refresh-token.guard'
+export * from './roles.guard'
+export * from './optional-jwt-access-token.guard'
+export * from './api-key.guard'
+export * from './api-key-rate-limit.guard'
+export * from './multi-auth.guard'
diff --git a/selfhosted-src/auth/guards/jwt-access-token.guard.ts b/selfhosted-src/auth/guards/jwt-access-token.guard.ts
new file mode 100644
index 000000000..acf11ae8b
--- /dev/null
+++ b/selfhosted-src/auth/guards/jwt-access-token.guard.ts
@@ -0,0 +1,22 @@
+import { Injectable, ExecutionContext } from '@nestjs/common'
+import { Reflector } from '@nestjs/core'
+import { AuthGuard } from '@nestjs/passport'
+import { IS_PUBLIC_KEY } from '../decorators'
+
+@Injectable()
+export class JwtAccessTokenGuard extends AuthGuard('jwt-access-token') {
+ constructor(private readonly reflector: Reflector) {
+ super()
+ }
+
+ canActivate(context: ExecutionContext) {
+ const isPublic = this.reflector.getAllAndOverride<boolean>(IS_PUBLIC_KEY, [
+ context.getHandler(),
+ context.getClass(),
+ ])
+
+ if (isPublic) return true
+
+ return super.canActivate(context)
+ }
+}
diff --git a/selfhosted-src/auth/guards/jwt-refresh-token.guard.ts b/selfhosted-src/auth/guards/jwt-refresh-token.guard.ts
new file mode 100644
index 000000000..7a9074a2d
--- /dev/null
+++ b/selfhosted-src/auth/guards/jwt-refresh-token.guard.ts
@@ -0,0 +1,7 @@
+import { AuthGuard } from '@nestjs/passport'
+
+export class JwtRefreshTokenGuard extends AuthGuard('jwt-refresh-token') {
+ // constructor() {
+ // super()
+ // }
+}
diff --git a/selfhosted-src/auth/guards/multi-auth.guard.ts b/selfhosted-src/auth/guards/multi-auth.guard.ts
new file mode 100644
index 000000000..96503daa0
--- /dev/null
+++ b/selfhosted-src/auth/guards/multi-auth.guard.ts
@@ -0,0 +1,28 @@
+import { ExecutionContext, Injectable } from '@nestjs/common'
+import { Reflector } from '@nestjs/core'
+import { AuthGuard } from '@nestjs/passport'
+import { IS_OPTIONAL_AUTH_KEY } from '../decorators'
+
+@Injectable()
+export class MultiAuthGuard extends AuthGuard(['jwt-access-token', 'api-key']) {
+ constructor(private readonly reflector: Reflector) {
+ super()
+ }
+
+ handleRequest(
+ err: any,
+ user: any,
+ info: any,
+ context: ExecutionContext,
+ status?: any,
+ ) {
+ const isOptionalAuth = this.reflector.getAllAndOverride<boolean>(
+ IS_OPTIONAL_AUTH_KEY,
+ [context.getHandler(), context.getClass()],
+ )
+
+ if (isOptionalAuth) return user
+
+ return super.handleRequest(err, user, info, context, status)
+ }
+}
diff --git a/selfhosted-src/auth/guards/optional-jwt-access-token.guard.ts b/selfhosted-src/auth/guards/optional-jwt-access-token.guard.ts
new file mode 100644
index 000000000..f4747504e
--- /dev/null
+++ b/selfhosted-src/auth/guards/optional-jwt-access-token.guard.ts
@@ -0,0 +1,9 @@
+import { Injectable } from '@nestjs/common'
+import { AuthGuard } from '@nestjs/passport'
+
+@Injectable()
+export class OptionalJwtAccessTokenGuard extends AuthGuard('jwt-access-token') {
+ handleRequest(err: any, user: any) {
+ return !err && user ? user : { id: null }
+ }
+}
diff --git a/selfhosted-src/auth/guards/roles.guard.ts b/selfhosted-src/auth/guards/roles.guard.ts
new file mode 100644
index 000000000..e6c608829
--- /dev/null
+++ b/selfhosted-src/auth/guards/roles.guard.ts
@@ -0,0 +1,98 @@
+import { Injectable, CanActivate, ExecutionContext } from '@nestjs/common'
+import { Reflector } from '@nestjs/core'
+import { ExtractJwt } from 'passport-jwt'
+import { verify } from 'jsonwebtoken'
+
+import { UserType, generateSelfhostedUser } from 'selfhosted-src/user/entities/user.entity'
+import { UserService } from 'selfhosted-src/user/user.service'
+import { isSelfhosted, JWT_ACCESS_TOKEN_SECRET } from 'selfhosted-src/common/constants'
+import { IS_TWO_FA_NOT_REQUIRED_KEY, ROLES_KEY } from '../decorators'
+
+@Injectable()
+export class RolesGuard implements CanActivate {
+ constructor(
+ private readonly reflector: Reflector,
+ private readonly userService: UserService,
+ ) {}
+
+ async canActivate(context: ExecutionContext) {
+ const roles = this.reflector.get<UserType[]>(
+ ROLES_KEY,
+ context.getHandler(),
+ )
+
+ const isTwoFaNotRequired = this.reflector.get<boolean>(
+ IS_TWO_FA_NOT_REQUIRED_KEY,
+ context.getHandler(),
+ )
+
+ if (!roles || roles.length === 0) return true
+
+ const request = context.switchToHttp().getRequest()
+ const userFromRequest = request.user
+
+ const user = isSelfhosted
+ ? generateSelfhostedUser()
+ : await this.userService.findUserById(userFromRequest.id)
+
+ // this is a temp measure as well due to some fucking bug related to undefined user, will revert it when I find the cause
+ let hasRole
+ try {
+ hasRole = user.roles.some(role => roles.includes(role))
+ } catch (error) {
+ console.error(`[AUTH ERROR - ROLES GUARD] ${error.message}`)
+ console.error(user, roles)
+ console.error(request?.path, request?.originalUrl, request?.method)
+ }
+
+ if (!hasRole) return false
+
+ let token = ''
+ if (request.cookies.token) {
+ token = request.cookies.token
+ } else {
+ const extract = ExtractJwt.fromAuthHeaderAsBearerToken()
+ token = extract(request)
+ }
+
+ if (isSelfhosted) {
+ try {
+ const decoded: any = verify(token, JWT_ACCESS_TOKEN_SECRET)
+
+ // If the token is not decoded, it means it's invalid
+ if (!decoded) {
+ return false
+ }
+ } catch {
+ return false
+ }
+
+ return true
+ }
+
+ if (isTwoFaNotRequired) {
+ return true
+ }
+
+ try {
+ const decoded: any = verify(token, JWT_ACCESS_TOKEN_SECRET)
+
+ // If the token is not decoded, it means it's invalid
+ if (!decoded) {
+ return false
+ }
+
+ // If the user has 2FA enabled, but the token is temporary (meant to be used for 2FA routes only) then return false
+ if (
+ user.isTwoFactorAuthenticationEnabled &&
+ !decoded.isSecondFactorAuthenticated
+ ) {
+ return false
+ }
+ } catch {
+ return false
+ }
+
+ return true
+ }
+}
diff --git a/selfhosted-src/auth/interfaces/index.ts b/selfhosted-src/auth/interfaces/index.ts
new file mode 100644
index 000000000..765786217
--- /dev/null
+++ b/selfhosted-src/auth/interfaces/index.ts
@@ -0,0 +1 @@
+export * from './jwt-payload.interface'
diff --git a/selfhosted-src/auth/interfaces/jwt-payload.interface.ts b/selfhosted-src/auth/interfaces/jwt-payload.interface.ts
new file mode 100644
index 000000000..ed0c28c74
--- /dev/null
+++ b/selfhosted-src/auth/interfaces/jwt-payload.interface.ts
@@ -0,0 +1,8 @@
+export interface IJwtPayload {
+ sub: string
+ isSecondFactorAuthenticated: boolean
+}
+
+export interface IJwtRefreshTokenPayload extends IJwtPayload {
+ refreshToken: string
+}
diff --git a/selfhosted-src/auth/strategies/api-key.strategy.ts b/selfhosted-src/auth/strategies/api-key.strategy.ts
new file mode 100644
index 000000000..e87cae9a9
--- /dev/null
+++ b/selfhosted-src/auth/strategies/api-key.strategy.ts
@@ -0,0 +1,23 @@
+import { Injectable, UnauthorizedException } from '@nestjs/common'
+import { PassportStrategy } from '@nestjs/passport'
+import { HeaderAPIKeyStrategy } from 'passport-headerapikey'
+import { AuthService } from '../auth.service'
+
+@Injectable()
+export class ApiKeyStrategy extends PassportStrategy(
+ HeaderAPIKeyStrategy,
+ 'api-key',
+) {
+ constructor(private readonly authService: AuthService) {
+ super(
+ { header: 'X-Api-Key', prefix: '' },
+ true,
+ // eslint-disable-next-line consistent-return
+ async (apiKey: string, done: any) => {
+ const user = await this.authService.validateApiKey(apiKey)
+ if (!user) return done(new UnauthorizedException(), false)
+ done(null, user)
+ },
+ )
+ }
+}
diff --git a/selfhosted-src/auth/strategies/index.ts b/selfhosted-src/auth/strategies/index.ts
new file mode 100644
index 000000000..f544f42d4
--- /dev/null
+++ b/selfhosted-src/auth/strategies/index.ts
@@ -0,0 +1,3 @@
+export * from './jwt-access-token.strategy'
+export * from './jwt-refresh-token.strategy'
+export * from './api-key.strategy'
diff --git a/selfhosted-src/auth/strategies/jwt-access-token.strategy.ts b/selfhosted-src/auth/strategies/jwt-access-token.strategy.ts
new file mode 100644
index 000000000..ede9ce01a
--- /dev/null
+++ b/selfhosted-src/auth/strategies/jwt-access-token.strategy.ts
@@ -0,0 +1,35 @@
+import { Injectable, UnauthorizedException } from '@nestjs/common'
+import { ConfigService } from '@nestjs/config'
+import { PassportStrategy } from '@nestjs/passport'
+import { Strategy, ExtractJwt } from 'passport-jwt'
+
+import {
+ SELFHOSTED_UUID,
+ JWT_ACCESS_TOKEN_SECRET,
+} from 'selfhosted-src/common/constants'
+import { generateSelfhostedUser } from 'selfhosted-src/user/entities/user.entity'
+import { IJwtPayload } from '../interfaces'
+
+@Injectable()
+export class JwtAccessTokenStrategy extends PassportStrategy(
+ Strategy,
+ 'jwt-access-token',
+) {
+ constructor(
+ public readonly configService: ConfigService,
+ ) {
+ super({
+ jwtFromRequest: ExtractJwt.fromAuthHeaderAsBearerToken(),
+ ignoreExpiration: false,
+ secretOrKey: JWT_ACCESS_TOKEN_SECRET,
+ })
+ }
+
+ public async validate(payload: IJwtPayload) {
+ if (payload.sub !== SELFHOSTED_UUID) {
+ throw new UnauthorizedException()
+ }
+
+ return generateSelfhostedUser()
+ }
+}
diff --git a/selfhosted-src/auth/strategies/jwt-refresh-token.strategy.ts b/selfhosted-src/auth/strategies/jwt-refresh-token.strategy.ts
new file mode 100644
index 000000000..ecb72b03b
--- /dev/null
+++ b/selfhosted-src/auth/strategies/jwt-refresh-token.strategy.ts
@@ -0,0 +1,28 @@
+import { ForbiddenException, Injectable } from '@nestjs/common'
+import { ConfigService } from '@nestjs/config'
+import { PassportStrategy } from '@nestjs/passport'
+import { Request } from 'express'
+import { ExtractJwt, Strategy } from 'passport-jwt'
+import { IJwtPayload } from '../interfaces'
+
+@Injectable()
+export class JwtRefreshTokenStrategy extends PassportStrategy(
+ Strategy,
+ 'jwt-refresh-token',
+) {
+ constructor(public readonly configService: ConfigService) {
+ super({
+ jwtFromRequest: ExtractJwt.fromAuthHeaderAsBearerToken(),
+ secretOrKey: configService.get('JWT_REFRESH_TOKEN_SECRET'),
+ passReqToCallback: true,
+ })
+ }
+
+ public validate(request: Request, payload: IJwtPayload) {
+ const refreshToken = request.headers.authorization.split(' ')[1]
+
+ if (!refreshToken) throw new ForbiddenException()
+
+ return { ...payload, refreshToken }
+ }
+}
diff --git a/selfhosted-src/common/constants.ts b/selfhosted-src/common/constants.ts
new file mode 100644
index 000000000..fb60c0988
--- /dev/null
+++ b/selfhosted-src/common/constants.ts
@@ -0,0 +1,155 @@
+import { ClickHouse } from 'clickhouse'
+import Redis from 'ioredis'
+import { hash } from 'blake3'
+import * as _toNumber from 'lodash/toNumber'
+
+import { getSelfhostedUUID } from './utils'
+
+// eslint-disable-next-line @typescript-eslint/no-var-requires
+require('dotenv').config()
+
+const { CLICKHOUSE_DATABASE } = process.env
+
+const redis = new Redis(
+ _toNumber(process.env.REDIS_PORT),
+ process.env.REDIS_HOST,
+ {
+ password: process.env.REDIS_PASSWORD,
+ username: process.env.REDIS_USER,
+ },
+)
+
+redis.defineCommand('countKeysByPattern', {
+ numberOfKeys: 0,
+ lua: "return #redis.call('keys', ARGV[1])",
+})
+
+const clickhouse = new ClickHouse({
+ url: process.env.CLICKHOUSE_HOST,
+ port: _toNumber(process.env.CLICKHOUSE_PORT),
+ debug: false,
+ basicAuth: {
+ username: process.env.CLICKHOUSE_USER,
+ password: process.env.CLICKHOUSE_PASSWORD,
+ },
+ isUseGzip: false,
+ format: 'json',
+ raw: false,
+ config: {
+ session_timeout: 60,
+ output_format_json_quote_64bit_integers: 0,
+ enable_http_compression: 0,
+ database: CLICKHOUSE_DATABASE,
+ log_queries: 0,
+ },
+})
+
+const { JWT_ACCESS_TOKEN_SECRET } = process.env
+const isSelfhosted = Boolean(process.env.SELFHOSTED)
+const isNewRelicEnabled = Boolean(process.env.USE_NEW_RELIC)
+const isDevelopment = process.env.NODE_ENV === 'development'
+const PRODUCTION_ORIGIN = process.env.CLIENT_URL || 'https://swetrix.com'
+
+const SELFHOSTED_EMAIL = process.env.EMAIL
+const SELFHOSTED_PASSWORD = process.env.PASSWORD
+const UUIDV5_NAMESPACE = '912c64c1-73fd-42b6-859f-785f839a9f68'
+const DEFAULT_SELFHOSTED_UUID = 'deadbeef-dead-beef-dead-beefdeadbeef'
+
+const SELFHOSTED_UUID = isSelfhosted ? getSelfhostedUUID() : ''
+
+const { TWO_FACTOR_AUTHENTICATION_APP_NAME } = process.env
+
+const ORIGINS_REGEX =
+ /^(?=.{1,255}$)[0-9A-Za-z:](?:(?:[0-9A-Za-z:]|-){0,61}[0-9A-Za-z])?(?:\.[0-9A-Za-z:](?:(?:[0-9A-Za-z]|-){0,61}[0-9A-Za-z])?)*\.?$/
+const IP_REGEX =
+ /^(([12]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])(\.|\/)){4}([1-2]?[0-9]|3[0-2])$/
+const PID_REGEX = /^(?!.*--)[a-zA-Z0-9-]{12}$/
+const isValidPID = (pid: string) => PID_REGEX.test(pid)
+
+// redis keys
+const getRedisProjectKey = (pid: string) => `pid_${pid}`
+const getRedisUserCountKey = (uid: string) => `user_c_${uid}`
+const getRedisCaptchaKey = (token: string) => `captcha_${hash(token)}`
+
+const REDIS_LOG_DATA_CACHE_KEY = 'log_cache'
+const REDIS_LOG_CAPTCHA_CACHE_KEY = 'log:captcha'
+const REDIS_LOG_PERF_CACHE_KEY = 'perf_cache'
+const REDIS_LOG_CUSTOM_CACHE_KEY = 'log_custom_cache_v2'
+const REDIS_SESSION_SALT_KEY = 'log_salt' // is updated every 24 hours
+const REDIS_USERS_COUNT_KEY = 'stats:users_count'
+const REDIS_PROJECTS_COUNT_KEY = 'stats:projects_count'
+const REDIS_PAGEVIEWS_COUNT_KEY = 'stats:pageviews'
+const REDIS_PERFORMANCE_COUNT_KEY = 'stats:performance'
+const REDIS_SSO_UUID = 'sso:uuid'
+
+// Captcha service
+const { CAPTCHA_SALT } = process.env
+const { CAPTCHA_ENCRYPTION_KEY } = process.env
+
+// 3600 sec -> 1 hour
+const redisProjectCacheTimeout = 3600
+
+// 15 minutes
+const redisProjectCountCacheTimeout = 900
+
+// 30 minues -> the amount of time analytics requests within one session are counted as non-unique
+const UNIQUE_SESSION_LIFE_TIME = 1800
+
+// 35 seconds
+const HEARTBEAT_SID_LIFE_TIME = 35
+
+// how often can user request a fresh GDPR export of their data; in days.
+const GDPR_EXPORT_TIMEFRAME = 14
+
+// send email warning when 85% of events in tier are used
+const SEND_WARNING_AT_PERC = 85
+
+const PROJECT_INVITE_EXPIRE = 48
+
+const CAPTCHA_COOKIE_KEY = 'swetrix-captcha-token'
+const CAPTCHA_TOKEN_LIFETIME = 300 // seconds (5 minutes).
+const CAPTCHA_SECRET_KEY_LENGTH = 50
+
+export {
+ clickhouse,
+ redis,
+ isValidPID,
+ getRedisProjectKey,
+ redisProjectCacheTimeout,
+ UNIQUE_SESSION_LIFE_TIME,
+ REDIS_LOG_DATA_CACHE_KEY,
+ REDIS_LOG_CAPTCHA_CACHE_KEY,
+ GDPR_EXPORT_TIMEFRAME,
+ getRedisUserCountKey,
+ redisProjectCountCacheTimeout,
+ REDIS_LOG_CUSTOM_CACHE_KEY,
+ REDIS_SESSION_SALT_KEY,
+ HEARTBEAT_SID_LIFE_TIME,
+ isSelfhosted,
+ UUIDV5_NAMESPACE,
+ SELFHOSTED_EMAIL,
+ SELFHOSTED_PASSWORD,
+ SELFHOSTED_UUID,
+ REDIS_USERS_COUNT_KEY,
+ REDIS_PROJECTS_COUNT_KEY,
+ REDIS_PAGEVIEWS_COUNT_KEY,
+ SEND_WARNING_AT_PERC,
+ PROJECT_INVITE_EXPIRE,
+ TWO_FACTOR_AUTHENTICATION_APP_NAME,
+ IP_REGEX,
+ isNewRelicEnabled,
+ ORIGINS_REGEX,
+ REDIS_LOG_PERF_CACHE_KEY,
+ REDIS_PERFORMANCE_COUNT_KEY,
+ CAPTCHA_SALT,
+ CAPTCHA_ENCRYPTION_KEY,
+ isDevelopment,
+ getRedisCaptchaKey,
+ CAPTCHA_COOKIE_KEY,
+ CAPTCHA_TOKEN_LIFETIME,
+ CAPTCHA_SECRET_KEY_LENGTH,
+ PRODUCTION_ORIGIN,
+ DEFAULT_SELFHOSTED_UUID,
+ REDIS_SSO_UUID,
+ JWT_ACCESS_TOKEN_SECRET,
+}
diff --git a/selfhosted-src/common/decorators/bot-detection.decorator.ts b/selfhosted-src/common/decorators/bot-detection.decorator.ts
new file mode 100644
index 000000000..f3be8c233
--- /dev/null
+++ b/selfhosted-src/common/decorators/bot-detection.decorator.ts
@@ -0,0 +1,4 @@
+import { SetMetadata } from '@nestjs/common'
+
+export const IS_BOT_DETECTION_ENABLED = 'isBotDetectionEnabled'
+export const BotDetection = () => SetMetadata(IS_BOT_DETECTION_ENABLED, true)
diff --git a/selfhosted-src/common/guards/bot-detection.guard.ts b/selfhosted-src/common/guards/bot-detection.guard.ts
new file mode 100644
index 000000000..90a940134
--- /dev/null
+++ b/selfhosted-src/common/guards/bot-detection.guard.ts
@@ -0,0 +1,27 @@
+import {
+ CanActivate,
+ ExecutionContext,
+ ForbiddenException,
+ Injectable,
+} from '@nestjs/common'
+import { Reflector } from '@nestjs/core'
+import * as isbot from 'isbot'
+import { IS_BOT_DETECTION_ENABLED } from '../decorators/bot-detection.decorator'
+
+@Injectable()
+export class BotDetectionGuard implements CanActivate {
+ constructor(private readonly reflector: Reflector) {}
+
+ canActivate(context: ExecutionContext): boolean {
+ this.reflector.get(IS_BOT_DETECTION_ENABLED, context.getHandler())
+
+ const { headers } = context.switchToHttp().getRequest()
+ const { 'user-agent': userAgent } = headers
+
+ if (isbot(userAgent)) {
+ throw new ForbiddenException('Bot traffic is ignored')
+ }
+
+ return true
+ }
+}
diff --git a/selfhosted-src/common/pagination/index.ts b/selfhosted-src/common/pagination/index.ts
new file mode 100644
index 000000000..f276090bf
--- /dev/null
+++ b/selfhosted-src/common/pagination/index.ts
@@ -0,0 +1,3 @@
+export * from './pagination.options.interface'
+export * from './pagination.results.interface'
+export * from './pagination'
diff --git a/selfhosted-src/common/pagination/pagination.options.interface.ts b/selfhosted-src/common/pagination/pagination.options.interface.ts
new file mode 100644
index 000000000..37378118e
--- /dev/null
+++ b/selfhosted-src/common/pagination/pagination.options.interface.ts
@@ -0,0 +1,4 @@
+export interface PaginationOptionsInterface {
+ take: number
+ skip: number
+}
diff --git a/selfhosted-src/common/pagination/pagination.results.interface.ts b/selfhosted-src/common/pagination/pagination.results.interface.ts
new file mode 100644
index 000000000..fc38ee98a
--- /dev/null
+++ b/selfhosted-src/common/pagination/pagination.results.interface.ts
@@ -0,0 +1,6 @@
+export interface PaginationResultInterface<PaginationEntity> {
+ results: PaginationEntity[]
+ total: number
+ next?: string
+ previous?: string
+}
diff --git a/selfhosted-src/common/pagination/pagination.ts b/selfhosted-src/common/pagination/pagination.ts
new file mode 100644
index 000000000..59e8287ef
--- /dev/null
+++ b/selfhosted-src/common/pagination/pagination.ts
@@ -0,0 +1,15 @@
+import { PaginationResultInterface } from './pagination.results.interface'
+
+export class Pagination<PaginationEntity> {
+ public results: PaginationEntity[]
+
+ public page_total: number
+
+ public total: number
+
+ constructor(paginationResults: PaginationResultInterface<PaginationEntity>) {
+ this.results = paginationResults.results
+ this.page_total = paginationResults.results.length
+ this.total = paginationResults.total
+ }
+}
diff --git a/selfhosted-src/common/utils.ts b/selfhosted-src/common/utils.ts
new file mode 100644
index 000000000..7ce4eda7d
--- /dev/null
+++ b/selfhosted-src/common/utils.ts
@@ -0,0 +1,240 @@
+import { NotFoundException, HttpException } from '@nestjs/common'
+import { hash } from 'blake3'
+import { v5 as uuidv5 } from 'uuid'
+import * as randomstring from 'randomstring'
+import * as _sample from 'lodash/sample'
+import * as _join from 'lodash/join'
+import * as _filter from 'lodash/filter'
+import * as _values from 'lodash/values'
+import * as _reduce from 'lodash/reduce'
+import * as _keys from 'lodash/keys'
+import * as _toNumber from 'lodash/toNumber'
+import * as _isEmpty from 'lodash/isEmpty'
+import * as _head from 'lodash/head'
+import * as _round from 'lodash/round'
+import * as dayjs from 'dayjs'
+import * as utc from 'dayjs/plugin/utc'
+import * as _map from 'lodash/map'
+
+import {
+ clickhouse,
+ redis,
+ DEFAULT_SELFHOSTED_UUID,
+ SELFHOSTED_EMAIL,
+ UUIDV5_NAMESPACE,
+ isDevelopment,
+} from './constants'
+import { Project } from '../project/entity/project.entity'
+
+dayjs.extend(utc)
+
+const RATE_LIMIT_REQUESTS_AMOUNT = 3
+const RATE_LIMIT_TIMEOUT = 86400 // 24 hours
+
+const allowedToUpdateKeys = [
+ 'name',
+ 'origins',
+ 'ipBlacklist',
+ 'active',
+ 'public',
+]
+
+const getRateLimitHash = (ipOrApiKey: string, salt = '') =>
+ `rl:${hash(`${ipOrApiKey}${salt}`).toString('hex')}`
+
+// 'action' is used as a salt to differ rate limiting routes
+const checkRateLimit = async (
+ ip: string,
+ action: string,
+ reqAmount: number = RATE_LIMIT_REQUESTS_AMOUNT,
+ reqTimeout: number = RATE_LIMIT_TIMEOUT,
+): Promise<void> => {
+ if (isDevelopment) {
+ return
+ }
+
+ const rlHash = getRateLimitHash(ip, action)
+ const rlCount: number = _toNumber(await redis.get(rlHash)) || 0
+
+ if (rlCount >= reqAmount) {
+ throw new HttpException('Too many requests, please try again later', 429)
+ }
+ await redis.set(rlHash, 1 + rlCount, 'EX', reqTimeout)
+}
+
+const RATE_LIMIT_FOR_API_KEY_TIMEOUT = 60 * 60 // 1 hour
+export const checkRateLimitForApiKey = async (
+ apiKey: string,
+ reqAmount: number,
+): Promise<boolean> => {
+ const rlHash = getRateLimitHash(apiKey)
+ const rlCount: number = _toNumber(await redis.get(rlHash)) || 0
+
+ if (rlCount >= reqAmount) {
+ throw new HttpException('Too many requests, please try again later', 429)
+ }
+ await redis.set(rlHash, 1 + rlCount, 'EX', RATE_LIMIT_FOR_API_KEY_TIMEOUT)
+ return true
+}
+
+const getProjectsClickhouse = async (id = null) => {
+ if (!id) {
+ const query = 'SELECT * FROM project;'
+ return clickhouse.query(query).toPromise()
+ }
+
+ const paramsData = {
+ params: {
+ id,
+ },
+ }
+
+ const query = `SELECT * FROM project WHERE id = {id:FixedString(12)};`
+ const project = await clickhouse.query(query, paramsData).toPromise()
+
+ if (_isEmpty(project)) {
+ throw new NotFoundException(`Project ${id} was not found in the database`)
+ }
+
+ return _head(project)
+}
+
+const updateProjectClickhouse = async (project: object) => {
+ const filtered = _reduce(
+ _filter(_keys(project), key => allowedToUpdateKeys.includes(key)),
+ (obj, key) => {
+ obj[key] = project[key]
+ return obj
+ },
+ {},
+ )
+ const columns = _keys(filtered)
+ const values = _values(filtered)
+ // @ts-ignore
+ const query = `ALTER table project UPDATE ${_join(
+ _map(columns, (col, id) => `${col}='${values[id]}'`),
+ ', ',
+ // @ts-ignore
+ )} WHERE id='${project.id}'`
+ return clickhouse.query(query).toPromise()
+}
+
+/**
+ * Checking the % change in one number relative to the other
+ * @param oldVal The initial value
+ * @param newVal The value that changed
+ * @param round Numbers after floating point
+ */
+const calculateRelativePercentage = (
+ oldVal: number,
+ newVal: number,
+ round = 2,
+) => {
+ if (oldVal === newVal) return 0
+ if (oldVal === 0) return 100
+ if (newVal === 0) return -100
+
+ if (newVal > oldVal) {
+ return _round((newVal / oldVal) * 100, round)
+ }
+
+ return _round((1 - newVal / oldVal) * -100, round)
+}
+
+const deleteProjectClickhouse = async id => {
+ const query = `ALTER table project DELETE WHERE id='${id}'`
+ return clickhouse.query(query).toPromise()
+}
+
+const createProjectClickhouse = async (project: Project) => {
+ const paramsData = {
+ params: {
+ ...project,
+ },
+ }
+ const query = `INSERT INTO project (*) VALUES ({id:FixedString(12)},{name:String},'', '',1,0,'${dayjs
+ .utc()
+ .format('YYYY-MM-DD HH:mm:ss')}')`
+ return clickhouse.query(query, paramsData).toPromise()
+}
+
+const saveRefreshTokenClickhouse = async (
+ userId: string,
+ refreshToken: string,
+) => {
+ const paramsData = {
+ params: {
+ userId,
+ refreshToken,
+ },
+ }
+ const query =
+ 'INSERT INTO refresh_token (*) VALUES ({userId:String},{refreshToken:String})'
+
+ return clickhouse.query(query, paramsData).toPromise()
+}
+
+const findRefreshTokenClickhouse = async (
+ userId: string,
+ refreshToken: string,
+) => {
+ const paramsData = {
+ params: {
+ userId,
+ refreshToken,
+ },
+ }
+ const query =
+ 'SELECT * FROM refresh_token WHERE userId = {userId:String} AND refreshToken = {refreshToken:String}'
+ return clickhouse.query(query, paramsData).toPromise()
+}
+
+const deleteRefreshTokenClickhouse = async (
+ userId: string,
+ refreshToken: string,
+) => {
+ const paramsData = {
+ params: {
+ userId,
+ refreshToken,
+ },
+ }
+ const query = `ALTER table refresh_token DELETE WHERE userId = {userId:String} AND refreshToken = {refreshToken:String}`
+ return clickhouse.query(query, paramsData).toPromise()
+}
+
+const generateRecoveryCode = () =>
+ randomstring.generate({
+ length: 30,
+ charset: 'alphabetic',
+ capitalization: 'uppercase',
+ })
+
+const millisecondsToSeconds = (milliseconds: number) => milliseconds / 1000
+
+const generateRandomString = (length: number): string =>
+ randomstring.generate(length)
+
+const getSelfhostedUUID = (): string => {
+ try {
+ return uuidv5(SELFHOSTED_EMAIL, UUIDV5_NAMESPACE)
+ } catch {
+ return DEFAULT_SELFHOSTED_UUID
+ }
+}
+
+export {
+ checkRateLimit,
+ createProjectClickhouse,
+ getProjectsClickhouse,
+ updateProjectClickhouse,
+ deleteProjectClickhouse,
+ generateRecoveryCode,
+ calculateRelativePercentage,
+ millisecondsToSeconds,
+ generateRandomString,
+ getSelfhostedUUID,
+ saveRefreshTokenClickhouse,
+ findRefreshTokenClickhouse,
+ deleteRefreshTokenClickhouse,
+}
diff --git a/selfhosted-src/configs/i18n.config.ts b/selfhosted-src/configs/i18n.config.ts
new file mode 100644
index 000000000..0288616c1
--- /dev/null
+++ b/selfhosted-src/configs/i18n.config.ts
@@ -0,0 +1,16 @@
+import { ConfigModule, ConfigService } from '@nestjs/config'
+import { AcceptLanguageResolver, I18nAsyncOptions } from 'nestjs-i18n'
+import { join } from 'path'
+
+export const getI18nConfig = (): I18nAsyncOptions => ({
+ imports: [ConfigModule],
+ inject: [ConfigService],
+ resolvers: [AcceptLanguageResolver],
+ useFactory: (configService: ConfigService) => ({
+ fallbackLanguage: 'en',
+ loaderOptions: {
+ path: join(__dirname, '..', 'i18n'),
+ watch: configService.get('NODE_ENV') === 'development',
+ },
+ }),
+})
diff --git a/selfhosted-src/configs/index.ts b/selfhosted-src/configs/index.ts
new file mode 100644
index 000000000..3113e360d
--- /dev/null
+++ b/selfhosted-src/configs/index.ts
@@ -0,0 +1 @@
+export * from './i18n.config'
diff --git a/selfhosted-src/i18n/en/auth.json b/selfhosted-src/i18n/en/auth.json
new file mode 100644
index 000000000..5f090f3b4
--- /dev/null
+++ b/selfhosted-src/i18n/en/auth.json
@@ -0,0 +1,13 @@
+{
+ "leakedPassword": "The password is in the leak database.",
+ "invalidCredentials": "Incorrect email or password.",
+ "invalidVerificationToken": "Invalid verification token.",
+ "accountNotExists": "An account with this email does not exist.",
+ "invalidResetPasswordToken": "Invalid reset password token.",
+ "invalidPassword": "Invalid password.",
+ "emailAlreadyVerified": "Email already verified.",
+ "maxEmailRequestsReached": "Maximum number of email requests reached.",
+ "emailAlreadyTaken": "Email already taken.",
+ "invalidChangeEmailToken": "Invalid change email token.",
+ "invalidRefreshToken": "Invalid refresh token."
+}
diff --git a/selfhosted-src/i18n/en/user.json b/selfhosted-src/i18n/en/user.json
new file mode 100644
index 000000000..b8acbdf29
--- /dev/null
+++ b/selfhosted-src/i18n/en/user.json
@@ -0,0 +1,3 @@
+{
+ "emailAlreadyUsed": "Email already used."
+}
diff --git a/selfhosted-src/i18n/en/validation.json b/selfhosted-src/i18n/en/validation.json
new file mode 100644
index 000000000..097d29c02
--- /dev/null
+++ b/selfhosted-src/i18n/en/validation.json
@@ -0,0 +1,7 @@
+{
+ "isEmail": "Please enter the valid email.",
+ "isPassword": "Please enter the valid password.",
+ "isNotEmpty": "This field is required.",
+ "isBoolean": "Please enter the valid value (true or false).",
+ "isUuid": "Please enter the valid UUID."
+}
diff --git a/selfhosted-src/logger/logger.module.ts b/selfhosted-src/logger/logger.module.ts
new file mode 100644
index 000000000..300c623a3
--- /dev/null
+++ b/selfhosted-src/logger/logger.module.ts
@@ -0,0 +1,8 @@
+import { Module } from '@nestjs/common'
+import { AppLoggerService } from './logger.service'
+
+@Module({
+ providers: [AppLoggerService],
+ exports: [AppLoggerService],
+})
+export class AppLoggerModule {}
diff --git a/selfhosted-src/logger/logger.service.ts b/selfhosted-src/logger/logger.service.ts
new file mode 100644
index 000000000..cd4939346
--- /dev/null
+++ b/selfhosted-src/logger/logger.service.ts
@@ -0,0 +1,11 @@
+import { Logger } from '@nestjs/common'
+
+import { isDevelopment } from '../common/constants'
+
+export class AppLoggerService extends Logger {
+ log(value, route, forceLog = false) {
+ if (isDevelopment || forceLog) {
+ Logger.log(value, route)
+ }
+ }
+}
diff --git a/selfhosted-src/main.ts b/selfhosted-src/main.ts
new file mode 100644
index 000000000..60c48ebcb
--- /dev/null
+++ b/selfhosted-src/main.ts
@@ -0,0 +1,56 @@
+import { NestFactory } from '@nestjs/core'
+import { ValidationPipe, VersioningType } from '@nestjs/common'
+import * as cookieParser from 'cookie-parser'
+import { SwaggerModule, DocumentBuilder } from '@nestjs/swagger'
+
+import { isDevelopment } from './common/constants'
+import { AppModule } from './app.module'
+
+async function bootstrap() {
+ const app = await NestFactory.create(AppModule)
+ app.use(cookieParser())
+ app.useGlobalPipes(new ValidationPipe())
+
+ app.enableVersioning({
+ type: VersioningType.URI,
+ })
+
+ if (isDevelopment) {
+ const config = new DocumentBuilder()
+ .setTitle('Swetrix API')
+ .setDescription('Swetrix Selfhosted API')
+ .setVersion(process.env.npm_package_version)
+ .build()
+ const document = SwaggerModule.createDocument(app, config)
+ SwaggerModule.setup('api', app, document)
+ }
+
+ // eslint-disable-next-line consistent-return
+ app.use(async (req, res, next) => {
+ res.header(
+ 'Cross-Origin-Embedder-Policy',
+ "require-corp; report-to='default'",
+ )
+ res.header('Cross-Origin-Opener-Policy', "same-site; report-to='default'")
+ res.header('Cross-Origin-Resource-Policy', 'same-site')
+ res.header('Permissions-Policy', 'interest-cohort=()')
+ res.header('Referrer-Policy', 'strict-origin-when-cross-origin')
+ res.header('X-Frame-Options', 'DENY')
+ res.header('X-Powered-By', 'Mountain Dew')
+ res.header('X-XSS-Protection', '1; mode=block')
+ res.header('Access-Control-Allow-Origin', process.env.API_ORIGINS || '*')
+ res.header('Access-Control-Allow-Methods', 'GET,POST,PUT,DELETE,PATCH')
+ res.header('Access-Control-Allow-Headers', '*')
+
+ if (req.method === 'OPTIONS') {
+ // TODO: INVESTIGATE
+ // eslint-disable-next-line @typescript-eslint/return-await
+ return await res.sendStatus(204)
+ }
+
+ next()
+ })
+
+ await app.listen(5005)
+}
+bootstrap()
diff --git a/selfhosted-src/ping/ping.controller.ts b/selfhosted-src/ping/ping.controller.ts
new file mode 100644
index 000000000..76ea5f035
--- /dev/null
+++ b/selfhosted-src/ping/ping.controller.ts
@@ -0,0 +1,9 @@
+import { Controller, Get } from '@nestjs/common'
+import { ApiTags } from '@nestjs/swagger'
+
+@ApiTags('Ping')
+@Controller('ping')
+export class PingController {
+ @Get('/')
+ get(): any {}
+}
diff --git a/selfhosted-src/ping/ping.module.ts b/selfhosted-src/ping/ping.module.ts
new file mode 100644
index 000000000..19a1f77e9
--- /dev/null
+++ b/selfhosted-src/ping/ping.module.ts
@@ -0,0 +1,8 @@
+import { Module } from '@nestjs/common'
+
+import { PingController } from './ping.controller'
+
+@Module({
+ controllers: [PingController],
+})
+export class PingModule {}
diff --git a/selfhosted-src/project/dto/project.dto.ts b/selfhosted-src/project/dto/project.dto.ts
new file mode 100644
index 000000000..38f36e056
--- /dev/null
+++ b/selfhosted-src/project/dto/project.dto.ts
@@ -0,0 +1,56 @@
+import { ApiProperty } from '@nestjs/swagger'
+import { IsNotEmpty, Length } from 'class-validator'
+
+export class ProjectDTO {
+ @ApiProperty({
+ example: 'Your awesome project',
+ required: true,
+ description: 'A display name for your project',
+ })
+ @IsNotEmpty()
+ @Length(1, 50)
+ name: string
+
+ @ApiProperty({
+ example: 'aUn1quEid-3',
+ required: true,
+ description: 'A unique project ID',
+ })
+ @IsNotEmpty()
+ id: string
+
+ @ApiProperty({
+ example: 'localhost:3000,example.com',
+ required: false,
+ description: 'An array allowed origins',
+ })
+ origins: string[] | null
+
+ @ApiProperty({
+ example: '::1,127.0.0.1,192.168.0.1/32',
+ required: false,
+ description: 'Array of blocked IP addresses',
+ })
+ ipBlacklist: string[] | null
+
+ @ApiProperty({
+ required: false,
+ description:
+ "The project's state. If enabled - all the incoming analytics data will be saved.",
+ })
+ active: boolean
+
+ @ApiProperty({
+ required: false,
+ description:
+ "When true, anyone on the internet (including Google) would be able to see the project's Dashboard.",
+ })
+ public: boolean
+
+ @ApiProperty({
+ required: false,
+ description:
+ 'When true, created a new Captcha Project without Analytics Projects.',
+ })
+ isCaptcha: boolean
+}
diff --git a/selfhosted-src/project/entity/index.ts b/selfhosted-src/project/entity/index.ts
new file mode 100644
index 000000000..558362f22
--- /dev/null
+++ b/selfhosted-src/project/entity/index.ts
@@ -0,0 +1 @@
+export * from './project.entity'
diff --git a/selfhosted-src/project/entity/project.entity.ts b/selfhosted-src/project/entity/project.entity.ts
new file mode 100644
index 000000000..8fb9117aa
--- /dev/null
+++ b/selfhosted-src/project/entity/project.entity.ts
@@ -0,0 +1,9 @@
+export class Project {
+ id: string
+ name: string
+ origins: string[]
+ ipBlacklist: string[]
+ active: boolean
+ public: boolean
+ created: Date
+}
diff --git a/selfhosted-src/project/project.controller.ts b/selfhosted-src/project/project.controller.ts
new file mode 100644
index 000000000..fa3f3cfc7
--- /dev/null
+++ b/selfhosted-src/project/project.controller.ts
@@ -0,0 +1,289 @@
+import {
+ Controller,
+ Body,
+ Query,
+ Param,
+ UseGuards,
+ Get,
+ Post,
+ Put,
+ Delete,
+ BadRequestException,
+ HttpCode,
+ NotFoundException,
+} from '@nestjs/common'
+import { ApiTags, ApiQuery, ApiResponse } from '@nestjs/swagger'
+import * as _isEmpty from 'lodash/isEmpty'
+import * as _map from 'lodash/map'
+import * as _trim from 'lodash/trim'
+import * as _size from 'lodash/size'
+import * as _split from 'lodash/split'
+import * as _head from 'lodash/head'
+import * as dayjs from 'dayjs'
+
+import { JwtAccessTokenGuard } from 'selfhosted-src/auth/guards'
+import { Auth } from 'selfhosted-src/auth/decorators'
+import { isValidDate } from 'selfhosted-src/analytics/analytics.service'
+import {
+ ProjectService,
+ deleteProjectRedis,
+} from './project.service'
+import { UserType } from '../user/entities/user.entity'
+import { Roles } from '../auth/decorators/roles.decorator'
+import { RolesGuard } from '../auth/guards/roles.guard'
+import { Pagination } from '../common/pagination/pagination'
+import { Project } from './entity/project.entity'
+import { CurrentUserId } from '../auth/decorators/current-user-id.decorator'
+import { ProjectDTO } from './dto/project.dto'
+import { AppLoggerService } from '../logger/logger.service'
+import {
+ isValidPID,
+ clickhouse,
+} from '../common/constants'
+import {
+ getProjectsClickhouse,
+ createProjectClickhouse,
+ updateProjectClickhouse,
+ deleteProjectClickhouse,
+} from '../common/utils'
+
+@ApiTags('Project')
+@Controller('project')
+export class ProjectController {
+ constructor(
+ private readonly projectService: ProjectService,
+ private readonly logger: AppLoggerService,
+ ) {}
+
+ @Get('/')
+ @ApiQuery({ name: 'take', required: false })
+ @ApiQuery({ name: 'skip', required: false })
+ @ApiQuery({ name: 'isCaptcha', required: false, type: Boolean })
+ @ApiQuery({ name: 'relatedonly', required: false, type: Boolean })
+ @ApiResponse({ status: 200, type: [Project] })
+ @Auth([UserType.CUSTOMER, UserType.ADMIN], true)
+ async get(
+ @CurrentUserId() userId: string,
+ @Query('take') take: number | undefined,
+ @Query('skip') skip: number | undefined,
+ ): Promise<Pagination<Project> | Project[] | object> {
+ this.logger.log({ userId, take, skip }, 'GET /project')
+
+ const results = await getProjectsClickhouse()
+ const formatted = _map(results, this.projectService.formatFromClickhouse)
+ return {
+ results: formatted,
+ page_total: _size(formatted),
+ total: _size(formatted),
+ totalMonthlyEvents: 0, // not needed as it's selfhosed
+ }
+ }
+
+ @Get('/names')
+ @ApiResponse({ status: 200, type: [Project] })
+ @Auth([UserType.CUSTOMER, UserType.ADMIN], true)
+ async getNames(@CurrentUserId() userId: string): Promise<Project[]> {
+ this.logger.log({ userId }, 'GET /project/names')
+
+ const results = await getProjectsClickhouse()
+ const formatted = _map(results, this.projectService.formatFromClickhouse)
+ return formatted
+ }
+
+ @Post('/')
+ @ApiResponse({ status: 201, type: Project })
+ @UseGuards(JwtAccessTokenGuard, RolesGuard)
+ @Roles(UserType.CUSTOMER, UserType.ADMIN)
+ async create(
+ @Body() projectDTO: ProjectDTO,
+ @CurrentUserId() userId: string,
+ ): Promise<Project> {
+ this.logger.log({ projectDTO, userId }, 'POST /project')
+
+ const projects = await getProjectsClickhouse()
+
+ this.projectService.validateProject(projectDTO)
+ this.projectService.checkIfIDUniqueClickhouse(projects, projectDTO.id)
+
+ const project = new Project()
+ Object.assign(project, projectDTO)
+ project.origins = _map(projectDTO.origins, _trim)
+ project.ipBlacklist = _map(projectDTO.ipBlacklist, _trim)
+ project.active = true
+
+ await createProjectClickhouse(project)
+
+ return project
+ }
+
+ @Delete('/reset/:id')
+ @HttpCode(204)
+ @UseGuards(JwtAccessTokenGuard, RolesGuard)
+ @Roles(UserType.CUSTOMER, UserType.ADMIN)
+ @ApiResponse({ status: 204, description: 'Empty body' })
+ async reset(
+ @Param('id') id: string,
+ @CurrentUserId() uid: string,
+ ): Promise<any> {
+ this.logger.log({ uid, id }, 'DELETE /project/reset/:id')
+ if (!isValidPID(id)) {
+ throw new BadRequestException(
+ 'The provided Project ID (pid) is incorrect',
+ )
+ }
+
+ const query1 = `ALTER table analytics DELETE WHERE pid='${id}'`
+ const query2 = `ALTER table customEV DELETE WHERE pid='${id}'`
+
+ try {
+ await clickhouse.query(query1).toPromise()
+ await clickhouse.query(query2).toPromise()
+ return 'Project resetted successfully'
+ } catch (e) {
+ this.logger.error(e)
+ return 'Error while resetting your project'
+ }
+ }
+
+ @Delete('/partially/:pid')
+ @ApiQuery({
+ name: 'from',
+ required: true,
+ description: 'Date in ISO format',
+ example: '2020-01-01T00:00:00.000Z',
+ type: 'string',
+ })
+ @ApiQuery({
+ name: 'to',
+ required: true,
+ description: 'Date in ISO format',
+ example: '2020-01-01T00:00:00.000Z',
+ type: 'string',
+ })
+ @ApiResponse({ status: 200 })
+ @UseGuards(JwtAccessTokenGuard, RolesGuard)
+ @Auth([UserType.ADMIN, UserType.CUSTOMER])
+ async deletePartially(
+ @Param('pid') pid: string,
+ @Query('from') from: string,
+ @Query('to') to: string,
+ @CurrentUserId() uid: string,
+ ): Promise<void> {
+ this.logger.log({ from, to, pid }, 'DELETE /partially/:id')
+
+ if (!isValidPID(pid)) {
+ throw new BadRequestException(
+ 'The provided Project ID (pid) is incorrect',
+ )
+ }
+
+ from = _head(_split(from, 'T'))
+ to = _head(_split(to, 'T'))
+
+ if (!isValidDate(from)) {
+ throw new BadRequestException("The provided 'from' date is incorrect")
+ }
+
+ if (!isValidDate(to)) {
+ throw new BadRequestException("The provided 'to' date is incorrect")
+ }
+
+ // Checking if project exists
+ getProjectsClickhouse(pid)
+
+ from = dayjs(from).format('YYYY-MM-DD')
+ to = dayjs(to).format('YYYY-MM-DD 23:59:59')
+
+ await this.projectService.removeDataFromClickhouse(pid, from, to)
+ }
+
+ @Delete('/:id')
+ @HttpCode(204)
+ @UseGuards(JwtAccessTokenGuard, RolesGuard)
+ @Roles(UserType.CUSTOMER, UserType.ADMIN)
+ @ApiResponse({ status: 204, description: 'Empty body' })
+ async delete(
+ @Param('id') id: string,
+ @CurrentUserId() uid: string,
+ ): Promise<any> {
+ this.logger.log({ uid, id }, 'DELETE /project/:id')
+ if (!isValidPID(id)) {
+ throw new BadRequestException(
+ 'The provided Project ID (pid) is incorrect',
+ )
+ }
+
+ await deleteProjectClickhouse(id)
+
+ const query1 = `ALTER table analytics DELETE WHERE pid='${id}'`
+ const query2 = `ALTER table customEV DELETE WHERE pid='${id}'`
+
+ try {
+ await clickhouse.query(query1).toPromise()
+ await clickhouse.query(query2).toPromise()
+ return 'Project deleted successfully'
+ } catch (e) {
+ this.logger.error(e)
+ return 'Error while deleting your project'
+ }
+ }
+
+ @Put('/:id')
+ @HttpCode(200)
+ @UseGuards(JwtAccessTokenGuard, RolesGuard)
+ @Roles(UserType.CUSTOMER, UserType.ADMIN)
+ @ApiResponse({ status: 200, type: Project })
+ async update(
+ @Param('id') id: string,
+ @Body() projectDTO: ProjectDTO,
+ @CurrentUserId() uid: string,
+ ): Promise<any> {
+ this.logger.log({ projectDTO, uid, id }, 'PUT /project/:id')
+ this.projectService.validateProject(projectDTO)
+ const project = await getProjectsClickhouse(id)
+
+ if (_isEmpty(project)) {
+ throw new NotFoundException()
+ }
+
+ project.active = projectDTO.active
+ project.origins = _map(projectDTO.origins, _trim)
+ project.ipBlacklist = _map(projectDTO.ipBlacklist, _trim)
+ project.name = projectDTO.name
+ project.public = projectDTO.public
+
+ await updateProjectClickhouse(
+ this.projectService.formatToClickhouse(project),
+ )
+
+ // await updateProjectRedis(id, project)
+ await deleteProjectRedis(id)
+
+ return project
+ }
+
+ @Get('/:id')
+ @Auth([], true, true)
+ @ApiResponse({ status: 200, type: Project })
+ async getOne(
+ @Param('id') id: string,
+ @CurrentUserId() uid: string,
+ ): Promise<Project | object> {
+ this.logger.log({ id }, 'GET /project/:id')
+ if (!isValidPID(id)) {
+ throw new BadRequestException(
+ 'The provided Project ID (pid) is incorrect',
+ )
+ }
+
+ const project = await getProjectsClickhouse(id)
+
+ if (_isEmpty(project)) {
+ throw new NotFoundException('Project was not found in the database')
+ }
+
+ this.projectService.allowedToView(project, uid)
+
+ return this.projectService.formatFromClickhouse(project)
+ }
+}
diff --git a/selfhosted-src/project/project.module.ts b/selfhosted-src/project/project.module.ts
new file mode 100644
index 000000000..9566bc47e
--- /dev/null
+++ b/selfhosted-src/project/project.module.ts
@@ -0,0 +1,15 @@
+import { Module } from '@nestjs/common'
+
+import { ProjectService } from './project.service'
+import { ProjectController } from './project.controller'
+import { AppLoggerModule } from '../logger/logger.module'
+
+@Module({
+ imports: [
+ AppLoggerModule,
+ ],
+ providers: [ProjectService],
+ exports: [ProjectService],
+ controllers: [ProjectController],
+})
+export class ProjectModule {}
diff --git a/selfhosted-src/project/project.service.ts b/selfhosted-src/project/project.service.ts
new file mode 100644
index 000000000..33c899d17
--- /dev/null
+++ b/selfhosted-src/project/project.service.ts
@@ -0,0 +1,189 @@
+import {
+ ForbiddenException,
+ Injectable,
+ BadRequestException,
+ UnprocessableEntityException,
+ InternalServerErrorException,
+ ConflictException,
+} from '@nestjs/common'
+import * as _isEmpty from 'lodash/isEmpty'
+import * as _isString from 'lodash/isString'
+import * as _size from 'lodash/size'
+import * as _join from 'lodash/join'
+import * as _find from 'lodash/find'
+import * as _map from 'lodash/map'
+import * as _pick from 'lodash/pick'
+import * as _isNull from 'lodash/isNull'
+import * as _split from 'lodash/split'
+import * as _trim from 'lodash/trim'
+import * as _findIndex from 'lodash/findIndex'
+// @ts-ignore
+import * as validateIP from 'validate-ip-node'
+
+import { Project } from './entity/project.entity'
+import { ProjectDTO } from './dto/project.dto'
+import {
+ isValidPID,
+ redis,
+ clickhouse,
+ IP_REGEX,
+ ORIGINS_REGEX,
+ getRedisProjectKey,
+ redisProjectCacheTimeout,
+} from '../common/constants'
+import { getProjectsClickhouse } from '../common/utils'
+
+export const deleteProjectRedis = async (id: string) => {
+ const key = getRedisProjectKey(id)
+
+ try {
+ await redis.del(key)
+ } catch (e) {
+ console.error(`Error deleting project ${id} from redis: ${e}`)
+ }
+}
+
+@Injectable()
+export class ProjectService {
+ constructor() {}
+
+ async getRedisProject(pid: string): Promise<Project | null> {
+ const pidKey = getRedisProjectKey(pid)
+ let project: string | Project = await redis.get(pidKey)
+
+ if (_isEmpty(project)) {
+ project = this.formatFromClickhouse(await getProjectsClickhouse(pid))
+
+ if (_isEmpty(project))
+ throw new BadRequestException(
+ 'The provided Project ID (pid) is incorrect',
+ )
+
+ await redis.set(
+ pidKey,
+ JSON.stringify(project),
+ 'EX',
+ redisProjectCacheTimeout,
+ )
+ } else {
+ try {
+ project = JSON.parse(project)
+ } catch {
+ throw new InternalServerErrorException('Error while processing project')
+ }
+ }
+
+ // @ts-ignore
+ return project
+ }
+
+ allowedToView(project: Project, uid: string | null): void {
+ if (
+ project.public ||
+ uid
+ ) {
+ return null
+ }
+
+ throw new ForbiddenException(
+ `You are not allowed to view "${project?.name}" project`,
+ )
+ }
+
+ checkIfIDUniqueClickhouse(projects: Array<object>, projectID: string): void {
+ if (_find(projects, ({ id }) => id === projectID)) {
+ throw new BadRequestException('Selected project ID is already in use')
+ }
+ }
+
+ async removeDataFromClickhouse(
+ pid: string,
+ from: string,
+ to: string,
+ ): Promise<void> {
+ const queryAnalytics =
+ 'ALTER TABLE analytics DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}'
+ const queryCustomEvents =
+ 'ALTER TABLE customEV DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}'
+ const queryPerformance =
+ 'ALTER TABLE performance DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}'
+ const params = {
+ params: {
+ pid,
+ from,
+ to,
+ },
+ }
+
+ await Promise.all([
+ clickhouse.query(queryAnalytics, params).toPromise(),
+ clickhouse.query(queryCustomEvents, params).toPromise(),
+ clickhouse.query(queryPerformance, params).toPromise(),
+ ])
+ }
+
+ formatToClickhouse(project: any): object {
+ const updProject = { ...project }
+ updProject.active = Number(updProject.active)
+ updProject.public = Number(updProject.public)
+
+ if (!_isNull(updProject.origins)) {
+ updProject.origins = _isString(updProject.origins)
+ ? updProject.origins
+ : _join(updProject.origins, ',')
+ }
+
+ if (!_isNull(updProject.ipBlacklist)) {
+ updProject.ipBlacklist = _isString(updProject.ipBlacklist)
+ ? updProject.ipBlacklist
+ : _join(updProject.ipBlacklist, ',')
+ }
+
+ return updProject
+ }
+
+ formatFromClickhouse(project: any): Project {
+ const updProject = { ...project }
+ updProject.active = Boolean(updProject.active)
+ updProject.public = Boolean(updProject.public)
+
+ updProject.origins = _isNull(updProject.origins)
+ ? []
+ : _split(updProject.origins, ',')
+
+ updProject.ipBlacklist = _isNull(updProject.ipBlacklist)
+ ? []
+ : _split(updProject.ipBlacklist, ',')
+
+ return updProject
+ }
+
+ validateProject(projectDTO: ProjectDTO) {
+ if (!isValidPID(projectDTO.id))
+ throw new UnprocessableEntityException(
+ 'The provided Project ID (pid) is incorrect',
+ )
+ if (_size(projectDTO.name) > 50)
+ throw new UnprocessableEntityException('The project name is too long')
+ if (_size(_join(projectDTO.origins, ',')) > 300)
+ throw new UnprocessableEntityException(
+ 'The list of allowed origins has to be smaller than 300 symbols',
+ )
+ if (_size(_join(projectDTO.ipBlacklist, ',')) > 300)
+ throw new UnprocessableEntityException(
+ 'The list of allowed blacklisted IP addresses must be less than 300 characters.',
+ )
+
+ _map(projectDTO.origins, host => {
+ if (!ORIGINS_REGEX.test(_trim(host))) {
+ throw new ConflictException(`Host ${host} is not correct`)
+ }
+ })
+
+ _map(projectDTO.ipBlacklist, ip => {
+ if (!validateIP(_trim(ip)) && !IP_REGEX.test(_trim(ip))) {
+ throw new ConflictException(`IP address ${ip} is not correct`)
+ }
+ })
+ }
+}
diff --git a/selfhosted-src/task-manager/task-manager.module.ts b/selfhosted-src/task-manager/task-manager.module.ts
new file mode 100644
index 000000000..8ced816d5
--- /dev/null
+++ b/selfhosted-src/task-manager/task-manager.module.ts
@@ -0,0 +1,12 @@
+import { Module } from '@nestjs/common'
+import { TaskManagerService } from './task-manager.service'
+import { AppLoggerModule } from '../logger/logger.module'
+
+@Module({
+ imports: [
+ AppLoggerModule,
+ ],
+ providers: [TaskManagerService],
+ exports: [TaskManagerService],
+})
+export class TaskManagerModule {}
diff --git a/selfhosted-src/task-manager/task-manager.service.ts b/selfhosted-src/task-manager/task-manager.service.ts
new file mode 100644
index 000000000..052a0a9c0
--- /dev/null
+++ b/selfhosted-src/task-manager/task-manager.service.ts
@@ -0,0 +1,141 @@
+import { Injectable } from '@nestjs/common'
+import { Cron, CronExpression } from '@nestjs/schedule'
+import * as bcrypt from 'bcrypt'
+import * as dayjs from 'dayjs'
+import * as utc from 'dayjs/plugin/utc'
+import * as _isEmpty from 'lodash/isEmpty'
+import * as _join from 'lodash/join'
+import * as _size from 'lodash/size'
+import * as _map from 'lodash/map'
+import * as _now from 'lodash/now'
+
+import {
+ clickhouse,
+ redis,
+ REDIS_LOG_DATA_CACHE_KEY,
+ REDIS_LOG_CUSTOM_CACHE_KEY,
+ REDIS_SESSION_SALT_KEY,
+ REDIS_LOG_PERF_CACHE_KEY,
+ REDIS_LOG_CAPTCHA_CACHE_KEY,
+} from '../common/constants'
+import { AppLoggerService } from '../logger/logger.service'
+
+dayjs.extend(utc)
+
+@Injectable()
+export class TaskManagerService {
+ constructor(
+ private readonly logger: AppLoggerService,
+ ) {}
+
+ @Cron(CronExpression.EVERY_MINUTE)
+ async saveLogData(): Promise<void> {
+ const data = await redis.lrange(REDIS_LOG_DATA_CACHE_KEY, 0, -1)
+ const customData = await redis.lrange(REDIS_LOG_CUSTOM_CACHE_KEY, 0, -1)
+ const perfData = await redis.lrange(REDIS_LOG_PERF_CACHE_KEY, 0, -1)
+ const captchaData = await redis.lrange(REDIS_LOG_CAPTCHA_CACHE_KEY, 0, -1)
+
+ if (!_isEmpty(data)) {
+ await redis.del(REDIS_LOG_DATA_CACHE_KEY)
+ const query = `INSERT INTO analytics (*) VALUES ${_join(data, ',')}`
+ try {
+ await clickhouse.query(query).toPromise()
+ } catch (e) {
+ console.error(`[CRON WORKER] Error whilst saving log data: ${e}`)
+ }
+ }
+
+ if (!_isEmpty(captchaData)) {
+ await redis.del(REDIS_LOG_CAPTCHA_CACHE_KEY)
+ const query = `INSERT INTO captcha (*) VALUES ${_join(captchaData, ',')}`
+ try {
+ await clickhouse.query(query).toPromise()
+ } catch (e) {
+ console.error(
+ `[CRON WORKER] Error whilst saving CAPTCHA log data: ${e}`,
+ )
+ }
+ }
+
+ if (!_isEmpty(customData)) {
+ await redis.del(REDIS_LOG_CUSTOM_CACHE_KEY)
+ const query = `INSERT INTO customEV (*) VALUES ${_join(customData, ',')}`
+
+ try {
+ await clickhouse.query(query).toPromise()
+ } catch (e) {
+ console.error(
+ `[CRON WORKER] Error whilst saving custom events data: ${e}`,
+ )
+ }
+ }
+
+ if (!_isEmpty(perfData)) {
+ await redis.del(REDIS_LOG_PERF_CACHE_KEY)
+
+ const query = `INSERT INTO performance (*) VALUES ${_join(perfData, ',')}`
+
+ try {
+ await clickhouse.query(query).toPromise()
+ } catch (e) {
+ console.error(
+ `[CRON WORKER] Error whilst saving performance data: ${e}`,
+ )
+ }
+ }
+ }
+
+ @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)
+ async generateSessionSalt(): Promise<void> {
+ const salt = await bcrypt.genSalt(10)
+ await redis.set(REDIS_SESSION_SALT_KEY, salt, 'EX', 87000)
+ }
+
+ @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)
+ async cleanUpSessions(): Promise<void> {
+ const delSidQuery = `ALTER TABLE analytics UPDATE sid = NULL WHERE created < '${dayjs
+ .utc()
+ .subtract(20, 'm')
+ .format('YYYY-MM-DD HH:mm:ss')}'`
+
+ await clickhouse.query(delSidQuery).toPromise()
+ }
+
+ @Cron(CronExpression.EVERY_MINUTE)
+ async processSessionDuration(): Promise<void> {
+ const keys = await redis.keys('sd:*')
+ const toSave = []
+ const now = _now()
+
+ const promises = _map(keys, async key => {
+ const [start, last] = (await redis.get(key)).split(':')
+ const duration = now - Number(last)
+
+ // storing to the DB if last interaction was more than 1 minute ago
+ if (duration > 60000) {
+ toSave.push([key, Number(last) - Number(start)])
+ }
+ })
+
+ await Promise.allSettled(promises).catch(reason => {
+ this.logger.error(
+ `[CRON WORKER](processSessionDuration) Error occured: ${reason}`,
+ )
+ })
+
+ if (_size(toSave) > 0) {
+ await redis.del(..._map(toSave, ([key]) => key))
+
+ const setSdurQuery = `ALTER TABLE analytics UPDATE sdur = sdur + CASE ${_map(
+ toSave,
+ ([key, duration]) =>
+ `WHEN sid = '${key.split(':')[1]}' THEN ${duration / 1000}`, // converting to seconds
+ ).join(' ')} END WHERE sid IN (${_map(
+ toSave,
+ ([key]) => `'${key.split(':')[1]}'`,
+ ).join(',')})`
+
+ await clickhouse.query(setSdurQuery).toPromise()
+ }
+ }
+}
diff --git a/selfhosted-src/user/entities/user.entity.ts b/selfhosted-src/user/entities/user.entity.ts
new file mode 100644
index 000000000..217cb53b2
--- /dev/null
+++ b/selfhosted-src/user/entities/user.entity.ts
@@ -0,0 +1,35 @@
+import { SELFHOSTED_EMAIL, SELFHOSTED_UUID } from '../../common/constants'
+
+export enum UserType {
+ CUSTOMER = 'customer',
+ ADMIN = 'admin',
+}
+
+export enum TimeFormat {
+ '12-hour' = '12-hour',
+ '24-hour' = '24-hour',
+}
+
+export const MAX_EMAIL_REQUESTS = 4 // 1 confirmation email on sign up + 3 additional ones
+
+export const DEFAULT_TIMEZONE = 'Etc/GMT'
+
+export const TRIAL_DURATION = 14 // days
+
+export interface SelfhostedUser {
+ id: string
+ email: string
+ isTwoFactorAuthenticationEnabled: boolean
+ planCode: 'enterprise'
+ roles: UserType[]
+ isActive: boolean
+}
+
+export const generateSelfhostedUser = (): SelfhostedUser => ({
+ id: SELFHOSTED_UUID,
+ email: SELFHOSTED_EMAIL,
+ isTwoFactorAuthenticationEnabled: false,
+ planCode: 'enterprise',
+ roles: [UserType.ADMIN, UserType.CUSTOMER],
+ isActive: true,
+})
diff --git a/selfhosted-src/user/user.controller.ts b/selfhosted-src/user/user.controller.ts
new file mode 100644
index 000000000..281fbb477
--- /dev/null
+++ b/selfhosted-src/user/user.controller.ts
@@ -0,0 +1,35 @@
+import {
+ Controller,
+ Get,
+ UseGuards,
+} from '@nestjs/common'
+import { ApiTags } from '@nestjs/swagger'
+
+import { JwtAccessTokenGuard } from 'selfhosted-src/auth/guards'
+import {
+ UserType,
+ SelfhostedUser,
+ generateSelfhostedUser,
+} from './entities/user.entity'
+import { Roles } from '../auth/decorators/roles.decorator'
+import { RolesGuard } from '../auth/guards/roles.guard'
+import { CurrentUserId } from '../auth/decorators/current-user-id.decorator'
+import { AppLoggerService } from '../logger/logger.service'
+
+@ApiTags('User')
+@Controller('user')
+@UseGuards(JwtAccessTokenGuard, RolesGuard)
+export class UserController {
+ constructor(
+ private readonly logger: AppLoggerService,
+ ) {}
+
+ @Get('/me')
+ @UseGuards(RolesGuard)
+ @Roles(UserType.CUSTOMER, UserType.ADMIN)
+ async me(@CurrentUserId() user_id: string): Promise<SelfhostedUser> {
+ this.logger.log({ user_id }, 'GET /user/me')
+
+ return generateSelfhostedUser()
+ }
+}
diff --git a/selfhosted-src/user/user.module.ts b/selfhosted-src/user/user.module.ts
new file mode 100644
index 000000000..ea523bc19
--- /dev/null
+++ b/selfhosted-src/user/user.module.ts
@@ -0,0 +1,12 @@
+import { Module } from '@nestjs/common'
+
+import { UserController } from './user.controller'
+import { UserService } from './user.service'
+
+@Module({
+ imports: [],
+ providers: [UserService],
+ exports: [UserService],
+ controllers: [UserController],
+})
+export class UserModule {}
diff --git a/selfhosted-src/user/user.service.ts b/selfhosted-src/user/user.service.ts
new file mode 100644
index 000000000..5de22cd66
--- /dev/null
+++ b/selfhosted-src/user/user.service.ts
@@ -0,0 +1,7 @@
+import { Injectable } from '@nestjs/common'
+
+@Injectable()
+export class UserService {
+ constructor() {}
+
+}