📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
initial selfhosted src
Blue Moused28ee47945a9
diff --git a/selfhosted-src/analytics/analytics.controller.ts b/selfhosted-src/analytics/analytics.controller.tsnew file mode 100644index 000000000..a71997ed9--- /dev/null+++ b/selfhosted-src/analytics/analytics.controller.ts@@ -0,0 +1,1069 @@+import * as _isEmpty from 'lodash/isEmpty'+import * as _isArray from 'lodash/isArray'+import * as _toNumber from 'lodash/toNumber'+import * as _pick from 'lodash/pick'+import * as _map from 'lodash/map'+import * as _uniqBy from 'lodash/uniqBy'+import * as _round from 'lodash/round'+import * as dayjs from 'dayjs'+import * as utc from 'dayjs/plugin/utc'+import * as dayjsTimezone from 'dayjs/plugin/timezone'+import { hash } from 'blake3'+import ct from 'countries-and-timezones'+import {+ Controller,+ Body,+ Query,+ UseGuards,+ Get,+ Post,+ Headers,+ BadRequestException,+ InternalServerErrorException,+ UnprocessableEntityException,+ Ip,+ ForbiddenException,+ Response,+} from '@nestjs/common'+import { ApiTags } from '@nestjs/swagger'+import * as UAParser from 'ua-parser-js'+import * as isbot from 'isbot'++import { OptionalJwtAccessTokenGuard } from 'selfhosted-src/auth/guards'+import { Auth, Public } from 'selfhosted-src/auth/decorators'+import {+ AnalyticsService,+ getSessionKey,+ getHeartbeatKey,+ DataType,+} from './analytics.service'+import { TaskManagerService } from '../task-manager/task-manager.service'+import { CurrentUserId } from '../auth/decorators/current-user-id.decorator'+import { DEFAULT_TIMEZONE } from '../user/entities/user.entity'+import { RolesGuard } from '../auth/guards/roles.guard'+import { PageviewsDTO } from './dto/pageviews.dto'+import { EventsDTO } from './dto/events.dto'+import { AnalyticsGET_DTO } from './dto/getData.dto'+import { GetUserFlowDTO } from './dto/getUserFlow.dto'+import { AppLoggerService } from '../logger/logger.service'+import {+ REDIS_LOG_DATA_CACHE_KEY,+ REDIS_LOG_PERF_CACHE_KEY,+ redis,+ REDIS_LOG_CUSTOM_CACHE_KEY,+ HEARTBEAT_SID_LIFE_TIME,+ REDIS_USERS_COUNT_KEY,+ REDIS_PROJECTS_COUNT_KEY,+ REDIS_PAGEVIEWS_COUNT_KEY,+ REDIS_SESSION_SALT_KEY,+ REDIS_PERFORMANCE_COUNT_KEY,+ clickhouse,+} from '../common/constants'+import { BotDetection } from '../common/decorators/bot-detection.decorator'+import { BotDetectionGuard } from '../common/guards/bot-detection.guard'+import { GetCustomEventsDto } from './dto/get-custom-events.dto'+import { IUserFlow } from './interfaces'++// eslint-disable-next-line @typescript-eslint/no-var-requires+const mysql = require('mysql2')++dayjs.extend(utc)+dayjs.extend(dayjsTimezone)++const getSessionKeyCustom = (+ ip: string,+ ua: string,+ pid: string,+ ev: string,+ salt = '',+) => `cses_${hash(`${ua}${ip}${pid}${ev}${salt}`).toString('hex')}`++const analyticsDTO = (+ sid: string,+ pid: string,+ pg: string,+ prev: string,+ dv: string,+ br: string,+ os: string,+ lc: string,+ ref: string,+ so: string,+ me: string,+ ca: string,+ cc: string,+ sdur: number | string,+ unique: number,+): Array<string | number> => {+ return [+ sid,+ pid,+ pg,+ prev,+ dv,+ br,+ os,+ lc,+ ref,+ so,+ me,+ ca,+ cc,+ sdur,+ unique,+ dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),+ ]+}++const performanceDTO = (+ pid: string,+ pg: string,+ dv: string,+ br: string,+ cc: string,+ dns: number,+ tls: number,+ conn: number,+ response: number,+ render: number,+ domLoad: number,+ pageLoad: number,+ ttfb: number,+): Array<string | number> => {+ return [+ pid,+ pg,+ dv,+ br,+ cc,+ _round(dns),+ _round(tls),+ _round(conn),+ _round(response),+ _round(render),+ _round(domLoad),+ _round(pageLoad),+ _round(ttfb),+ dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),+ ]+}++const customLogDTO = (+ pid: string,+ ev: string,+ pg: string,+ dv: string,+ br: string,+ os: string,+ lc: string,+ ref: string,+ so: string,+ me: string,+ ca: string,+ cc: string,+): Array<string | number> => {+ return [+ pid,+ ev,+ pg,+ dv,+ br,+ os,+ lc,+ ref,+ so,+ me,+ ca,+ cc,+ dayjs.utc().format('YYYY-MM-DD HH:mm:ss'),+ ]+}++export const getElValue = el => {+ if (el === undefined || el === null || el === 'NULL') return 'NULL'+ return mysql.escape(el)+}++// Performance object validator: none of the values cannot be bigger than 1000 * 60 * 5 (5 minutes) and are >= 0+const MAX_PERFORMANCE_VALUE = 1000 * 60 * 5+const isPerformanceValid = (perf: any) => {+ return (+ perf.dns <= MAX_PERFORMANCE_VALUE &&+ perf.tls <= MAX_PERFORMANCE_VALUE &&+ perf.conn <= MAX_PERFORMANCE_VALUE &&+ perf.response <= MAX_PERFORMANCE_VALUE &&+ perf.render <= MAX_PERFORMANCE_VALUE &&+ perf.dom_load <= MAX_PERFORMANCE_VALUE &&+ perf.page_load <= MAX_PERFORMANCE_VALUE &&+ perf.ttfb <= MAX_PERFORMANCE_VALUE &&+ perf.dns >= 0 &&+ perf.tls >= 0 &&+ perf.conn >= 0 &&+ perf.response >= 0 &&+ perf.render >= 0 &&+ perf.dom_load >= 0 &&+ perf.page_load >= 0 &&+ perf.ttfb >= 0+ )+}++const getPIDsArray = (pids, pid) => {+ const pidsEmpty = _isEmpty(pids)+ const pidEmpty = _isEmpty(pid)+ if (pidsEmpty && pidEmpty)+ throw new BadRequestException(+ "An array of Project ID's (pids) or a Project ID (pid) has to be provided",+ )+ else if (!pidsEmpty && !pidEmpty)+ throw new BadRequestException(+ "Please provide either an array of Project ID's (pids) or a Project ID (pid), but not both",+ )+ else if (!pidEmpty) {+ pids = JSON.stringify([pid])+ }++ try {+ pids = JSON.parse(pids)+ } catch (e) {+ throw new UnprocessableEntityException(+ "Cannot process the provided array of Project ID's",+ )+ }++ if (!_isArray(pids)) {+ throw new UnprocessableEntityException(+ "An array of Project ID's has to be provided as a 'pids' param",+ )+ }++ return pids+}++// needed for serving 1x1 px GIF+const TRANSPARENT_GIF_BUFFER = Buffer.from(+ 'R0lGODlhAQABAIAAAP///wAAACwAAAAAAQABAAACAkQBADs=',+ 'base64',+)++@ApiTags('Analytics')+@UseGuards(OptionalJwtAccessTokenGuard, RolesGuard)+@Controller(['log', 'v1/log'])+export class AnalyticsController {+ constructor(+ private readonly analyticsService: AnalyticsService,+ private readonly logger: AppLoggerService,+ private readonly taskManagerService: TaskManagerService,+ ) {}++ @Get()+ @Auth([], true, true)+ async getData(+ @Query() data: AnalyticsGET_DTO,+ @CurrentUserId() uid: string,+ isCaptcha = false,+ ): Promise<any> {+ const {+ pid,+ period,+ timeBucket,+ from,+ to,+ filters,+ timezone = DEFAULT_TIMEZONE,+ } = data+ this.analyticsService.validatePID(pid)++ if (!_isEmpty(period)) {+ this.analyticsService.validatePeriod(period)+ }++ this.analyticsService.validateTimebucket(timeBucket)+ const [filtersQuery, filtersParams, parsedFilters] =+ this.analyticsService.getFiltersQuery(+ filters,+ isCaptcha ? DataType.CAPTCHA : DataType.ANALYTICS,+ )++ const safeTimezone = this.analyticsService.getSafeTimezone(timezone)+ const { groupFrom, groupTo, groupFromUTC, groupToUTC } =+ this.analyticsService.getGroupFromTo(+ from,+ to,+ timeBucket,+ period,+ safeTimezone,+ )+ await this.analyticsService.checkProjectAccess(pid, uid)++ let queryCustoms = `SELECT ev, count() FROM customEV WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`+ let subQuery = `FROM ${+ isCaptcha ? 'captcha' : 'analytics'+ } WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`+ let customEVFilterApplied = false++ if (filtersParams?.ev && !isCaptcha) {+ customEVFilterApplied = true+ queryCustoms = `SELECT ev, count() FROM customEV WHERE ${+ filtersParams.ev_exclusive ? 'NOT' : ''+ } ev = {ev:String} AND pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String} GROUP BY ev`++ subQuery = `FROM customEV WHERE ${+ filtersParams.ev_exclusive ? 'NOT' : ''+ } ev = {ev:String} AND pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`+ }++ const paramsData = {+ params: {+ pid,+ groupFrom: groupFromUTC,+ groupTo: groupToUTC,+ ...filtersParams,+ },+ }++ let result: object | void++ if (isCaptcha) {+ result = await this.analyticsService.groupCaptchaByTimeBucket(+ timeBucket,+ groupFrom,+ groupTo,+ subQuery,+ filtersQuery,+ paramsData,+ safeTimezone,+ )+ } else {+ result = await this.analyticsService.groupByTimeBucket(+ timeBucket,+ groupFrom,+ groupTo,+ subQuery,+ filtersQuery,+ paramsData,+ safeTimezone,+ customEVFilterApplied,+ parsedFilters,+ )+ }++ if (isCaptcha) {+ return {+ ...result,+ appliedFilters: parsedFilters,+ }+ }++ const customs = await this.analyticsService.processCustomEV(+ queryCustoms,+ paramsData,+ )++ return {+ ...result,+ customs,+ appliedFilters: parsedFilters,+ }+ }++ @Get('chart')+ @Auth([], true, true)+ async getChartData(+ @Query() data: AnalyticsGET_DTO,+ @CurrentUserId() uid: string,+ ): Promise<any> {+ const {+ pid,+ period,+ timeBucket,+ from,+ to,+ filters,+ timezone = DEFAULT_TIMEZONE,+ } = data+ this.analyticsService.validatePID(pid)++ if (!_isEmpty(period)) {+ this.analyticsService.validatePeriod(period)+ }++ this.analyticsService.validateTimebucket(timeBucket)+ const [filtersQuery, filtersParams, parsedFilters] =+ this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)++ const safeTimezone = this.analyticsService.getSafeTimezone(timezone)+ const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(+ from,+ to,+ timeBucket,+ period,+ safeTimezone,+ )+ await this.analyticsService.checkProjectAccess(pid, uid)++ let subQuery = `FROM analytics WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`+ let customEVFilterApplied = false++ if (filtersParams?.ev) {+ customEVFilterApplied = true+ subQuery = `FROM customEV WHERE ${+ filtersParams.ev_exclusive ? 'NOT' : ''+ } ev = {ev:String} AND pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`+ }++ const paramsData = {+ params: {+ pid,+ groupFrom,+ groupTo,+ ...filtersParams,+ },+ }++ const result = await this.analyticsService.groupChartByTimeBucket(+ timeBucket,+ groupFrom,+ groupTo,+ subQuery,+ filtersQuery,+ paramsData,+ safeTimezone,+ customEVFilterApplied,+ )++ return {+ ...result,+ appliedFilters: parsedFilters,+ }+ }++ @Get('performance')+ @Auth([], true, true)+ async getPerfData(+ @Query() data: AnalyticsGET_DTO,+ @CurrentUserId() uid: string,+ ): Promise<any> {+ const {+ pid,+ period,+ timeBucket,+ from,+ to,+ filters,+ timezone = DEFAULT_TIMEZONE,+ } = data+ this.analyticsService.validatePID(pid)++ if (!_isEmpty(period)) {+ this.analyticsService.validatePeriod(period)+ }++ this.analyticsService.validateTimebucket(timeBucket)+ const [filtersQuery, filtersParams, parsedFilters] =+ this.analyticsService.getFiltersQuery(filters, DataType.PERFORMANCE)++ const safeTimezone = this.analyticsService.getSafeTimezone(timezone)+ const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(+ from,+ to,+ timeBucket,+ period,+ safeTimezone,+ )+ await this.analyticsService.checkProjectAccess(pid, uid)++ const subQuery = `FROM performance WHERE pid = {pid:FixedString(12)} ${filtersQuery} AND created BETWEEN {groupFrom:String} AND {groupTo:String}`++ const paramsData = {+ params: {+ pid,+ groupFrom,+ groupTo,+ ...filtersParams,+ },+ }++ const result = await this.analyticsService.groupPerfByTimeBucket(+ timeBucket,+ groupFrom,+ groupTo,+ subQuery,+ filtersQuery,+ paramsData,+ safeTimezone,+ )++ return {+ ...result,+ appliedFilters: parsedFilters,+ }+ }++ @Get('performance/chart')+ @Auth([], true, true)+ async getPerfChartData(+ @Query() data: AnalyticsGET_DTO,+ @CurrentUserId() uid: string,+ ): Promise<any> {+ const {+ pid,+ period,+ timeBucket,+ from,+ to,+ filters,+ timezone = DEFAULT_TIMEZONE,+ } = data+ this.analyticsService.validatePID(pid)++ if (!_isEmpty(period)) {+ this.analyticsService.validatePeriod(period)+ }++ this.analyticsService.validateTimebucket(timeBucket)+ const [filtersQuery, filtersParams, parsedFilters] =+ this.analyticsService.getFiltersQuery(filters, DataType.PERFORMANCE)++ const safeTimezone = this.analyticsService.getSafeTimezone(timezone)+ const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(+ from,+ to,+ timeBucket,+ period,+ safeTimezone,+ )+ await this.analyticsService.checkProjectAccess(pid, uid)++ const paramsData = {+ params: {+ pid,+ groupFrom,+ groupTo,+ ...filtersParams,+ },+ }++ const chart = await this.analyticsService.getPerfChartData(+ timeBucket,+ from,+ to,+ filtersQuery,+ paramsData,+ safeTimezone,+ )++ return {+ chart,+ appliedFilters: parsedFilters,+ }+ }++ @Get('captcha')+ @Auth([], true, true)+ async getCaptchaData(+ @Query() data: AnalyticsGET_DTO,+ @CurrentUserId() uid: string,+ ): Promise<any> {+ return this.getData(data, uid, true)+ }++ @Get('user-flow')+ @Auth([], true, true)+ async getUserFlow(+ @Query() data: GetUserFlowDTO,+ @CurrentUserId() uid: string,+ ): Promise<IUserFlow | { appliedFilters: any[] }> {+ const { pid, period, from, to, timezone = DEFAULT_TIMEZONE, filters } = data+ this.analyticsService.validatePID(pid)++ if (!_isEmpty(period)) {+ this.analyticsService.validatePeriod(period)+ }++ await this.analyticsService.checkProjectAccess(pid, uid)++ const safeTimezone = this.analyticsService.getSafeTimezone(timezone)+ const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(+ from,+ to,+ null,+ period,+ safeTimezone,+ )++ const [filtersQuery, filtersParams, parsedFilters] =+ this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)++ const params = {+ pid,+ groupFrom,+ groupTo,+ ...filtersParams,+ }++ const flow = await this.analyticsService.getUserFlow(params, filtersQuery)++ return {+ ...flow,+ appliedFilters: parsedFilters,+ }+ }++ @Get('birdseye')+ @Auth([], true, true)+ // returns overall short statistics per project+ async getOverallStats(+ @Query() data,+ @CurrentUserId() uid: string,+ ): Promise<any> {+ const { pids, pid } = data+ const pidsArray = getPIDsArray(pids, pid)++ const validationPromises = _map(pidsArray, async currentPID => {+ this.analyticsService.validatePID(currentPID)+ await this.analyticsService.checkProjectAccess(currentPID, uid)+ })++ await Promise.all(validationPromises)++ return this.analyticsService.getSummary(pidsArray, 'w')+ }++ @Get('captcha/birdseye')+ @Auth([], true, true)+ // returns overall short statistics per CAPTCHA project+ async getCaptchaOverallStats(+ @Query() data,+ @CurrentUserId() uid: string,+ ): Promise<any> {+ const { pids, pid } = data+ const pidsArray = getPIDsArray(pids, pid)++ const validationPromises = _map(pidsArray, async currentPID => {+ this.analyticsService.validatePID(currentPID)+ await this.analyticsService.checkProjectAccess(currentPID, uid)+ })++ await Promise.all(validationPromises)++ return this.analyticsService.getCaptchaSummary(pidsArray, 'w')+ }++ @Get('hb')+ async getHeartBeatStats(+ @Query() data,+ @CurrentUserId() uid: string,+ ): Promise<object> {+ const { pids, pid } = data+ const pidsArray = getPIDsArray(pids, pid)++ const validationPromises = _map(pidsArray, async currentPID => {+ this.analyticsService.validatePID(currentPID)+ await this.analyticsService.checkProjectAccess(currentPID, uid)+ })++ await Promise.all(validationPromises)++ const result = {}++ const keyCountPromises = _map(pidsArray, async currentPID => {+ result[currentPID] = await this.analyticsService.getOnlineUserCount(+ currentPID,+ )+ })++ await Promise.all(keyCountPromises).catch(reason => {+ this.logger.error(`[GET /analytics/hb] ${reason}`)+ throw new InternalServerErrorException(+ 'An error occured while calculating heartbeat statistics',+ )+ })++ return result+ }++ @Get('liveVisitors')+ @Auth([], true, true)+ async getLiveVisitors(+ @Query() data,+ @CurrentUserId() uid: string,+ ): Promise<object> {+ const { pid } = data++ this.analyticsService.validatePID(pid)+ await this.analyticsService.checkProjectAccess(pid, uid)++ const keys = await redis.keys(`sd:*:${pid}`)++ if (_isEmpty(keys)) {+ return []+ }++ const sids = _map(keys, key => key.split(':')[1])++ const query = `SELECT sid, dv, br, os, cc FROM analytics WHERE sid IN (${sids+ .map(el => `'${el}'`)+ .join(',')})`+ const result = await clickhouse.query(query).toPromise()+ const processed = _map(_uniqBy(result, 'sid'), el =>+ _pick(el, ['dv', 'br', 'os', 'cc']),+ )++ return processed+ }++ // Log custom event+ @Post('custom')+ @UseGuards(BotDetectionGuard)+ @BotDetection()+ @Public()+ async logCustom(+ @Body() eventsDTO: EventsDTO,+ @Headers() headers,+ @Ip() reqIP,+ ): Promise<any> {+ const { 'user-agent': userAgent, origin } = headers++ const ip =+ headers['cf-connecting-ip'] || headers['x-forwarded-for'] || reqIP || ''++ await this.analyticsService.validate(eventsDTO, origin, 'custom', ip)++ if (eventsDTO.unique) {+ const salt = await redis.get(REDIS_SESSION_SALT_KEY)+ const sessionHash = getSessionKeyCustom(+ ip,+ userAgent,+ eventsDTO.pid,+ eventsDTO.ev,+ salt,+ )+ const unique = await this.analyticsService.isUnique(sessionHash)++ if (!unique) {+ throw new ForbiddenException(+ 'The unique option provided, while the custom event have already been created for this session',+ )+ }+ }++ const ua = UAParser(userAgent)+ const dv = ua.device.type || 'desktop'+ const br = ua.browser.name+ const os = ua.os.name+ // trying to get country from timezome, otherwise using CloudFlare's IP based country code as a fallback+ const cc =+ ct.getCountryForTimezone(eventsDTO.tz)?.id ||+ (headers['cf-ipcountry'] === 'XX' ? 'NULL' : headers['cf-ipcountry'])++ const dto = customLogDTO(+ eventsDTO.pid,+ eventsDTO.ev,+ eventsDTO.pg,+ dv,+ br,+ os,+ eventsDTO.lc,+ eventsDTO.ref,+ eventsDTO.so,+ eventsDTO.me,+ eventsDTO.ca,+ cc,+ )++ try {+ const values = `(${dto.map(getElValue).join(',')})`+ await redis.rpush(REDIS_LOG_CUSTOM_CACHE_KEY, values)+ return+ } catch (e) {+ this.logger.error(e)+ throw new InternalServerErrorException(+ 'Error occured while saving the custom event',+ )+ }+ }++ @Post('hb')+ @UseGuards(BotDetectionGuard)+ @BotDetection()+ @Auth([], true, true)+ async heartbeat(+ @Body() logDTO: PageviewsDTO,+ @Headers() headers,+ @Ip() reqIP,+ ): Promise<any> {+ const { 'user-agent': userAgent } = headers+ const { pid } = logDTO+ const ip =+ headers['cf-connecting-ip'] || headers['x-forwarded-for'] || reqIP || ''++ const sessionID = await this.analyticsService.getSessionHash(+ pid,+ userAgent,+ ip,+ )++ await redis.set(+ getHeartbeatKey(pid, sessionID),+ 1,+ 'EX',+ HEARTBEAT_SID_LIFE_TIME,+ )+ await this.analyticsService.processInteractionSD(sessionID, pid)+ }++ // Log pageview event+ @Post()+ @UseGuards(BotDetectionGuard)+ @BotDetection()+ @Public()+ async log(+ @Body() logDTO: PageviewsDTO,+ @Headers() headers,+ @Ip() reqIP,+ ): Promise<any> {+ const { 'user-agent': userAgent, origin } = headers++ const ip =+ headers['cf-connecting-ip'] || headers['x-forwarded-for'] || reqIP || ''++ await this.analyticsService.validate(logDTO, origin, 'log', ip)++ const salt = await redis.get(REDIS_SESSION_SALT_KEY)+ const sessionHash = getSessionKey(ip, userAgent, logDTO.pid, salt)+ const unique = await this.analyticsService.isUnique(sessionHash)++ await this.analyticsService.processInteractionSD(sessionHash, logDTO.pid)++ if (unique && logDTO.unique) {+ throw new ForbiddenException(+ 'The event was not saved because it was not unique while unique only param is provided',+ )+ }++ // trying to get country from timezome, otherwise using CloudFlare's IP based country code as a fallback+ const cc =+ ct.getCountryForTimezone(logDTO.tz)?.id ||+ (headers['cf-ipcountry'] === 'XX' ? 'NULL' : headers['cf-ipcountry'])++ const ua = UAParser(userAgent)+ const dv = ua.device.type || 'desktop'+ const br = ua.browser.name+ const os = ua.os.name++ const dto = analyticsDTO(+ sessionHash,+ logDTO.pid,+ logDTO.pg,+ logDTO.prev,+ dv,+ br,+ os,+ logDTO.lc,+ logDTO.ref,+ logDTO.so,+ logDTO.me,+ logDTO.ca,+ cc,+ 0,+ Number(unique),+ )++ let perfDTO: Array<string | number> = []++ if (!_isEmpty(logDTO.perf) && isPerformanceValid(logDTO.perf)) {+ const {+ dns,+ tls,+ conn,+ response,+ render,+ dom_load: domLoad,+ page_load: pageLoad,+ ttfb,+ } = logDTO.perf++ perfDTO = performanceDTO(+ logDTO.pid,+ logDTO.pg,+ dv,+ br,+ cc,+ dns,+ tls,+ conn,+ response,+ render,+ domLoad,+ pageLoad,+ ttfb,+ )+ }++ const values = `(${dto.map(getElValue).join(',')})`+ const perfValues = `(${perfDTO.map(getElValue).join(',')})`+ try {+ await redis.rpush(REDIS_LOG_DATA_CACHE_KEY, values)++ if (!_isEmpty(perfDTO)) {+ await redis.rpush(REDIS_LOG_PERF_CACHE_KEY, perfValues)+ }++ return+ } catch (e) {+ this.logger.error(e)+ throw new InternalServerErrorException(+ 'Error occured while saving the log data',+ )+ }+ }++ // Fallback for logging pageviews for users with JavaScript disabled+ // Returns 1x1 transparent gif+ @Get('noscript')+ @Public()+ async noscript(+ @Query() data,+ @Headers() headers,+ @Ip() reqIP,+ @Response() res,+ ): Promise<any> {+ const { 'user-agent': userAgent, origin } = headers+ const { pid } = data++ // todo: create a decorator for bot traffic detection+ if (isbot(userAgent)) {+ res.writeHead(200, { 'Content-Type': 'image/gif' })+ return res.end(TRANSPARENT_GIF_BUFFER, 'binary')+ }++ const logDTO: PageviewsDTO = {+ pid,+ }++ await this.analyticsService.validate(logDTO, origin)++ const ip =+ headers['cf-connecting-ip'] || headers['x-forwarded-for'] || reqIP || ''+ const salt = await redis.get(REDIS_SESSION_SALT_KEY)+ const sessionHash = getSessionKey(ip, userAgent, logDTO.pid, salt)+ const unique = await this.analyticsService.isUnique(sessionHash)++ await this.analyticsService.processInteractionSD(sessionHash, logDTO.pid)++ const ua = UAParser(userAgent)+ const dv = ua.device.type || 'desktop'+ const br = ua.browser.name+ const os = ua.os.name++ // using CloudFlare's IP based country code+ const cc =+ headers['cf-ipcountry'] === 'XX' ? 'NULL' : headers['cf-ipcountry']+ const dto = analyticsDTO(+ sessionHash,+ logDTO.pid,+ 'NULL',+ 'NULL',+ dv,+ br,+ os,+ 'NULL',+ 'NULL',+ 'NULL',+ 'NULL',+ 'NULL',+ cc,+ 0,+ Number(unique),+ )++ const values = `(${dto.map(getElValue).join(',')})`+ try {+ await redis.rpush(REDIS_LOG_DATA_CACHE_KEY, values)+ } catch (e) {+ this.logger.error(e)+ }++ res.writeHead(200, { 'Content-Type': 'image/gif' })+ return res.end(TRANSPARENT_GIF_BUFFER, 'binary')+ }++ @Get('custom-events')+ @Auth([], true, true)+ async getCustomEvents(+ @Query() data: GetCustomEventsDto,+ @CurrentUserId() uid: string,+ ): Promise<any> {+ const {+ pid,+ period,+ timeBucket,+ from,+ to,+ filters,+ timezone = DEFAULT_TIMEZONE,+ customEvents,+ } = data+ this.analyticsService.validatePID(pid)++ if (!_isEmpty(period)) {+ this.analyticsService.validatePeriod(period)+ }++ this.analyticsService.validateTimebucket(timeBucket)+ const [filtersQuery, filtersParams, parsedFilters] =+ this.analyticsService.getFiltersQuery(filters, DataType.ANALYTICS)+ await this.analyticsService.checkProjectAccess(pid, uid)++ const safeTimezone = this.analyticsService.getSafeTimezone(timezone)+ const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(+ from,+ to,+ timeBucket,+ period,+ safeTimezone,+ )++ const paramsData = {+ params: {+ pid,+ groupFrom,+ groupTo,+ ...filtersParams,+ },+ }++ const result: any = await this.analyticsService.groupCustomEVByTimeBucket(+ timeBucket,+ groupFrom,+ groupTo,+ filtersQuery,+ paramsData,+ safeTimezone,+ )++ let customEventss = customEvents++ if (filters) {+ try {+ customEventss = JSON.parse(customEvents)+ // eslint-disable-next-line no-empty+ } catch {}+ }++ if (customEventss.length > 0) {+ for (const key in result.chart.events) {+ if (!customEventss.includes(key)) {+ delete result.chart.events[key]+ }+ }+ } else {+ result.chart = {}+ }++ return {+ ...result,+ appliedFilters: parsedFilters,+ }+ }+}diff --git a/selfhosted-src/analytics/analytics.module.ts b/selfhosted-src/analytics/analytics.module.tsnew file mode 100644index 000000000..b9d46551a--- /dev/null+++ b/selfhosted-src/analytics/analytics.module.ts@@ -0,0 +1,21 @@+import { Module, forwardRef } from '@nestjs/common'++import { AnalyticsService } from './analytics.service'+import { AnalyticsController } from './analytics.controller'+import { UserModule } from '../user/user.module'+import { AppLoggerModule } from '../logger/logger.module'+import { ProjectModule } from '../project/project.module'+import { TaskManagerModule } from '../task-manager/task-manager.module'++@Module({+ imports: [+ forwardRef(() => UserModule),+ TaskManagerModule,+ AppLoggerModule,+ ProjectModule,+ ],+ providers: [AnalyticsService],+ exports: [AnalyticsService],+ controllers: [AnalyticsController],+})+export class AnalyticsModule {}diff --git a/selfhosted-src/analytics/analytics.service.ts b/selfhosted-src/analytics/analytics.service.tsnew file mode 100644index 000000000..8bb084a26--- /dev/null+++ b/selfhosted-src/analytics/analytics.service.ts@@ -0,0 +1,1494 @@+import * as _isEmpty from 'lodash/isEmpty'+import * as _split from 'lodash/split'+import * as _size from 'lodash/size'+import * as _includes from 'lodash/includes'+import * as _map from 'lodash/map'+import * as _toUpper from 'lodash/toUpper'+import * as _join from 'lodash/join'+import * as _last from 'lodash/last'+import * as _some from 'lodash/some'+import * as _find from 'lodash/find'+import * as _now from 'lodash/now'+import * as _values from 'lodash/values'+import * as _round from 'lodash/round'+import * as _filter from 'lodash/filter'+import * as dayjs from 'dayjs'+import * as utc from 'dayjs/plugin/utc'+import * as dayjsTimezone from 'dayjs/plugin/timezone'+import * as isSameOrBefore from 'dayjs/plugin/isSameOrBefore'+import * as ipRangeCheck from 'ip-range-check'+import validator from 'validator'+import { hash } from 'blake3'+import {+ Injectable,+ BadRequestException,+ InternalServerErrorException,+ UnprocessableEntityException,+ PreconditionFailedException,+} from '@nestjs/common'++import {+ DEFAULT_TIMEZONE,+} from '../user/entities/user.entity'+import {+ redis,+ isValidPID,+ UNIQUE_SESSION_LIFE_TIME,+ clickhouse,+ REDIS_SESSION_SALT_KEY,+} from '../common/constants'+import {+ calculateRelativePercentage,+ millisecondsToSeconds,+} from '../common/utils'+import { PageviewsDTO } from './dto/pageviews.dto'+import { EventsDTO } from './dto/events.dto'+import { ProjectService } from '../project/project.service'+import { Project } from '../project/entity/project.entity'+import { TimeBucketType } from './dto/getData.dto'+import {+ PerformanceCHResponse,+ CustomsCHResponse,+ CustomsCHAggregatedResponse,+ TrafficCEFilterCHResponse,+ TrafficCHResponse,+ IGetGroupFromTo,+ GetFiltersQuery,+ IUserFlowNode,+ IUserFlowLink,+ IUserFlow,+ IBuildUserFlow,+ IExtractChartData,+ IGenerateXAxis,+} from './interfaces'++dayjs.extend(utc)+dayjs.extend(dayjsTimezone)+dayjs.extend(isSameOrBefore)++export const getSessionKey = (ip: string, ua: string, pid: string, salt = '') =>+ `ses_${hash(`${ua}${ip}${pid}${salt}`).toString('hex')}`++export const getHeartbeatKey = (pid: string, sessionID: string) =>+ `hb:${pid}:${sessionID}`++const getSessionDurationKey = (sessionHash: string, pid: string) =>+ `sd:${sessionHash}:${pid}`++const GMT_0_TIMEZONES = [+ 'Atlantic/Azores',+ 'Etc/GMT',+ // 'Africa/Casablanca',+]++const cols = ['cc', 'pg', 'lc', 'br', 'os', 'dv', 'ref', 'so', 'me', 'ca']++const captchaColumns = ['cc', 'br', 'os', 'dv']+const perfColumns = ['cc', 'pg', 'dv', 'br']++const validPeriods = [+ 'today',+ 'yesterday',+ '1d',+ '7d',+ '4w',+ '3M',+ '12M',+ '24M',+]++const validTimebuckets = [+ TimeBucketType.HOUR,+ TimeBucketType.DAY,+ TimeBucketType.MONTH,+]++// mapping of allowed timebuckets per difference between days+// (e.g. if difference is lower than (lt) (including) -> then the specified timebuckets are allowed to be applied)+const timeBucketToDays = [+ {+ lt: 7,+ tb: [TimeBucketType.HOUR, TimeBucketType.DAY, TimeBucketType.MONTH],+ }, // 7 days+ {+ lt: 28,+ tb: [TimeBucketType.DAY, TimeBucketType.MONTH],+ }, // 4 weeks+ { lt: 366, tb: [TimeBucketType.MONTH] }, // 12 months+ { lt: 732, tb: [TimeBucketType.MONTH] }, // 24 months+]++// Smaller than 64 characters, must start with an English letter and contain only letters (a-z A-Z), numbers (0-9), underscores (_) and dots (.)+// eslint-disable-next-line no-useless-escape+const customEVvalidate = /^[a-zA-Z](?:[\w\.]){0,62}$/++const timeBucketConversion = {+ hour: 'toStartOfHour',+ day: 'toStartOfDay',+ month: 'toStartOfMonth',+}++const isValidTimezone = (timezone: string): boolean => {+ if (_isEmpty(timezone)) {+ return false+ }++ try {+ dayjs.tz('2013-11-18 11:55', timezone)+ return true+ } catch {+ return false+ }+}++export const isValidDate = (date: string, format = 'YYYY-MM-DD'): boolean => {+ if (_isEmpty(date)) {+ return false+ }++ return dayjs(date, format).format(format) === date+}++const checkIfTBAllowed = (+ timeBucket: TimeBucketType,+ from: string,+ to: string,+): void => {+ const diff = dayjs(to).diff(dayjs(from), 'days')++ const tbMap = _find(timeBucketToDays, ({ lt }) => diff <= lt)++ if (_isEmpty(tbMap)) {+ throw new PreconditionFailedException(+ "The difference between 'from' and 'to' is greater than allowed",+ )+ }++ const { tb } = tbMap++ if (!_includes(tb, timeBucket)) {+ throw new PreconditionFailedException(+ "The specified 'timeBucket' parameter cannot be applied to the date range",+ )+ }+}++const generateParamsQuery = (+ col: string,+ subQuery: string,+ customEVFilterApplied: boolean,+ isPageInclusiveFilterSet: boolean,+ isCaptcha?: boolean,+ isPerformance?: boolean,+): string => {+ if (isPerformance) {+ return `SELECT ${col}, avg(pageLoad) ${subQuery} AND ${col} IS NOT NULL GROUP BY ${col}`+ }++ if (isCaptcha) {+ return `SELECT ${col}, count(*) ${subQuery} AND ${col} IS NOT NULL GROUP BY ${col}`+ }++ if (customEVFilterApplied) {+ return `SELECT ${col}, count(*) ${subQuery} AND ${col} IS NOT NULL GROUP BY ${col}`+ }++ if (col === 'pg' || isPageInclusiveFilterSet) {+ return `SELECT ${col}, count(*) ${subQuery} AND ${col} IS NOT NULL GROUP BY ${col}`+ }++ return `SELECT ${col}, count(*) ${subQuery} AND ${col} IS NOT NULL AND unique='1' GROUP BY ${col}`+}++export enum DataType {+ ANALYTICS = 'analytics',+ PERFORMANCE = 'performance',+ CAPTCHA = 'captcha',+}++@Injectable()+export class AnalyticsService {+ constructor(private readonly projectService: ProjectService) {}++ async checkProjectAccess(pid: string, uid: string | null): Promise<void> {+ // const project = await this.projectService.getRedisProject(pid)+ // this.projectService.allowedToView(project, uid)+ }++ checkOrigin(project: Project, origin: string): void {+ // For some reasons the project.origins sometimes may look like [''], let's filter it out+ // TODO: Properly validate the origins on project update+ const origins = _filter(project.origins, Boolean)++ if (!_isEmpty(origins) && !_isEmpty(origin)) {+ if (origin === 'null') {+ if (!_includes(origins, 'null')) {+ throw new BadRequestException(+ "'null' origin is not added to your project's whitelist. To send requests from this origin either add it to your origins policy or leave it empty.",+ )+ }+ } else {+ const { hostname } = new URL(origin)+ if (!_includes(origins, hostname)) {+ throw new BadRequestException(+ "This origin is prohibited by the project's origins policy",+ )+ }+ }+ }+ }++ checkIpBlacklist(project: Project, ip: string): void {+ // For some reasons the project.ipBlacklist sometimes may look like [''], let's filter it out+ // TODO: Properly validate the ipBlacklist on project update+ const ipBlacklist = _filter(project.ipBlacklist, Boolean)++ if (!_isEmpty(ipBlacklist) && ipRangeCheck(ip, ipBlacklist)) {+ throw new BadRequestException(+ 'Incoming analytics is disabled for this IP address',+ )+ }+ }++ validatePID(pid: string): void {+ if (_isEmpty(pid)) {+ throw new BadRequestException('The Project ID (pid) has to be provided')+ }++ if (!isValidPID(pid)) {+ throw new BadRequestException(+ 'The provided Project ID (pid) is incorrect',+ )+ }+ }++ async validate(+ logDTO: PageviewsDTO | EventsDTO,+ origin: string,+ type: 'custom' | 'log' = 'log',+ ip?: string,+ ): Promise<string | null> {+ if (_isEmpty(logDTO)) {+ throw new BadRequestException('The request cannot be empty')+ }++ const { pid } = logDTO+ this.validatePID(pid)++ if (type === 'custom') {+ const { ev } = <EventsDTO>logDTO++ if (_isEmpty(ev)) {+ throw new BadRequestException('Empty custom events are not allowed')+ }++ if (!customEVvalidate.test(ev)) {+ throw new BadRequestException(+ 'An incorrect event name (ev) is provided',+ )+ }+ }++ // 'tz' does not need validation as it's based on getCountryForTimezone detection+ const { lc } = logDTO++ // validate locale ('lc' param)+ if (!_isEmpty(lc)) {+ if (validator.isLocale(lc)) {+ // uppercase the locale after '-' char, so for example both 'en-gb' and 'en-GB' in result will be 'en-GB'+ const lcParted = _split(lc, '-')++ if (_size(lcParted) > 1) {+ lcParted[1] = _toUpper(lcParted[1])+ }++ logDTO.lc = _join(lcParted, '-')+ } else {+ logDTO.lc = 'NULL'+ }+ }++ const project = await this.projectService.getRedisProject(pid)++ this.checkIpBlacklist(project, ip)++ if (!project.active) {+ throw new BadRequestException(+ 'Incoming analytics is disabled for this project',+ )+ }++ this.checkOrigin(project, origin)++ return null+ }++ getDataTypeColumns(dataType: DataType): string[] {+ if (dataType === DataType.ANALYTICS) {+ return cols+ }++ if (dataType === DataType.PERFORMANCE) {+ return perfColumns+ }++ return captchaColumns+ }++ getGroupFromTo(+ from: string,+ to: string,+ timeBucket: TimeBucketType | null,+ period: string,+ safeTimezone: string,+ ): IGetGroupFromTo {+ let groupFrom: dayjs.Dayjs+ let groupTo: dayjs.Dayjs+ const formatFrom = 'YYYY-MM-DD HH:mm:ss'+ const formatTo = 'YYYY-MM-DD HH:mm:ss'+ const djsNow = _includes(GMT_0_TIMEZONES, safeTimezone)+ ? dayjs.utc()+ : dayjs().tz(safeTimezone)++ if (!_isEmpty(from) && !_isEmpty(to)) {+ if (!isValidDate(from)) {+ throw new PreconditionFailedException(+ "The timeframe 'from' parameter is invalid",+ )+ }++ if (!isValidDate(to)) {+ throw new PreconditionFailedException(+ "The timeframe 'to' parameter is invalid",+ )+ }++ if (dayjs.utc(from).isAfter(dayjs.utc(to), 'second')) {+ throw new PreconditionFailedException(+ "The timeframe 'from' parameter cannot be greater than 'to'",+ )+ }++ if (!_isEmpty(timeBucket)) {+ checkIfTBAllowed(timeBucket, from, to)+ }++ groupFrom = dayjs.utc()++ if (from === to) {+ groupTo = dayjs.utc().add(1, 'day')+ } else {+ groupTo = dayjs.utc()+ }+ } else if (!_isEmpty(period)) {+ if (period === 'today') {+ groupFrom = djsNow.startOf('d')+ groupTo = djsNow+ } else if (period === 'yesterday') {+ groupFrom = djsNow.subtract(1, 'day').startOf('d')+ groupTo = djsNow.subtract(1, 'day').endOf('d')+ } else {+ if (period === '1d') {+ groupFrom = djsNow.subtract(parseInt(period, 10), _last(period))+ } else {+ groupFrom = djsNow.subtract(parseInt(period, 10) - 1, _last(period))+ }++ groupFrom = groupFrom.startOf(timeBucket)+ groupTo = djsNow+ }++ if (!_isEmpty(timeBucket)) {+ checkIfTBAllowed(+ timeBucket,+ groupFrom.format(formatFrom),+ groupTo.format(formatTo),+ )+ }+ } else {+ throw new BadRequestException(+ 'The timeframe (either from/to pair or period) has to be provided',+ )+ }++ const groupFromFormatted = groupFrom.format(formatFrom)+ const groupToFormatted = groupTo.format(formatTo)++ return {+ // UTC time+ groupFrom: groupFromFormatted,+ groupTo: groupToFormatted,+ // Timezone shifted time+ groupFromUTC: groupFrom.utc().format(formatFrom),+ groupToUTC: groupTo.utc().format(formatFrom),+ }+ }++ removeCyclicDependencies(links: IUserFlowLink[]): IUserFlowLink[] {+ const visited = new Set<string>()++ return links.filter(link => {+ const key = `${link.source}_${link.target}`+ if (visited.has(key)) {+ return false+ }+ visited.add(key)+ return true+ })+ }++ buildUserFlow(links: IUserFlowLink[]): IBuildUserFlow {+ const nodes: IUserFlowNode[] = Array.from(+ new Set(+ links+ .map((link: IUserFlowLink) => link.source)+ .concat(links.map((link: IUserFlowLink) => link.target)),+ ),+ ).map((node: any) => ({ id: node }))++ return { nodes, links }+ }++ async getUserFlow(params: unknown, filtersQuery: string): Promise<IUserFlow> {+ const query = `+ SELECT+ pg AS source,+ prev AS target,+ count() AS value+ FROM analytics+ WHERE+ pid = {pid:FixedString(12)}+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}+ AND pg != prev+ ${filtersQuery}+ GROUP BY+ pg,+ prev+ `++ const results = <IUserFlowLink[]>(+ await clickhouse.query(query, { params }).toPromise()+ )++ if (_isEmpty(results)) {+ const empty = { nodes: [], links: [] }+ return {+ ascending: empty,+ descending: empty,+ }+ }++ const ascendingLinks: IUserFlowLink[] = []+ const descendingLinks: IUserFlowLink[] = []++ this.removeCyclicDependencies(results).forEach((row: any) => {+ const link: IUserFlowLink = {+ source: row.source,+ target: row.target,+ value: row.value,+ }+ if (link.source < link.target) {+ ascendingLinks.push(link)+ } else {+ descendingLinks.push(link)+ }+ })++ return {+ ascending: this.buildUserFlow(ascendingLinks),+ descending: this.buildUserFlow(descendingLinks),+ }+ }++ async getSessionHash(+ pid: string,+ userAgent: string,+ ip: string,+ ): Promise<string> {+ this.validatePID(pid)++ const salt = await redis.get(REDIS_SESSION_SALT_KEY)++ return getSessionKey(ip, userAgent, pid, salt)+ }++ async isSessionDurationOpen(sdKey: string): Promise<Array<string | boolean>> {+ const sd = await redis.get(sdKey)+ return [sd, Boolean(sd)]+ }++ // Processes interaction for session duration+ async processInteractionSD(sessionHash: string, pid: string): Promise<void> {+ const sdKey = getSessionDurationKey(sessionHash, pid)+ const [sd, isOpened] = await this.isSessionDurationOpen(sdKey)+ const now = _now()++ // the value is START_UNIX_TIMESTAMP:LAST_INTERACTION_UNIX_TIMESTAMP+ if (isOpened) {+ const [start] = _split(sd, ':')+ await redis.set(sdKey, `${start}:${now}`, 'EX', UNIQUE_SESSION_LIFE_TIME)+ } else {+ await redis.set(sdKey, `${now}:${now}`, 'EX', UNIQUE_SESSION_LIFE_TIME)+ }+ }++ validatePeriod(period: string): void {+ if (!_includes(validPeriods, period)) {+ throw new UnprocessableEntityException('The provided period is incorrect')+ }+ }++ // returns SQL filters query in a format like 'AND col=value AND ...'+ getFiltersQuery(filters: string, dataType: DataType): GetFiltersQuery {+ let parsed = []+ let query = ''+ let params = {}++ if (_isEmpty(filters)) {+ return [query, params, parsed]+ }++ try {+ parsed = JSON.parse(filters)+ } catch (e) {+ console.error(`Cannot parse the filters array: ${filters}`)+ return [query, params, parsed]+ }++ if (_isEmpty(parsed)) {+ return [query, params, parsed]+ }++ const columns = this.getDataTypeColumns(dataType)++ for (let i = 0; i < _size(parsed); ++i) {+ const { column, filter, isExclusive } = parsed[i]++ if (column === 'ev') {+ params = {+ ...params,+ [column]: filter,+ [`${column}_exclusive`]: isExclusive,+ }++ continue+ }++ if (!_includes(columns, column)) {+ throw new UnprocessableEntityException(+ `The provided filter (${column}) is not supported`,+ )+ }+ // working only on 1 filter per 1 column+ const colFilter = `cf_${column}`++ params = {+ ...params,+ [colFilter]: filter,+ }+ query += ` ${+ isExclusive ? 'AND NOT' : 'AND'+ } ${column}={${colFilter}:String}`+ }++ return [query, params, parsed]+ }++ validateTimebucket(tb: TimeBucketType): void {+ if (!_includes(validTimebuckets, tb)) {+ throw new UnprocessableEntityException(+ 'The provided timebucket is incorrect',+ )+ }+ }++ async isUnique(sessionHash: string) {+ const session = await redis.get(sessionHash)+ await redis.set(sessionHash, 1, 'EX', UNIQUE_SESSION_LIFE_TIME)+ return !session+ }++ async getSummary(+ pids: string[],+ period: 'w' | 'M' = 'w',+ amountToSubtract = 1,+ ) {+ return this.getSummaryStats(pids, 'analytics', period, amountToSubtract)+ }++ async getCaptchaSummary(pids: string[], period: 'w' | 'M' = 'w') {+ return this.getSummaryStats(pids, 'captcha', period)+ }++ async getSummaryStats(+ pids: string[],+ tableName: 'analytics' | 'captcha',+ period: 'w' | 'M' = 'w',+ amountToSubtract = 1,+ ) {+ const result = {}++ const promises = pids.map(async pid => {+ if (!isValidPID(pid)) {+ throw new BadRequestException(+ `The provided Project ID (${pid}) is incorrect`,+ )+ }++ const now = dayjs.utc().format('YYYY-MM-DD HH:mm:ss')+ const periodRaw = dayjs.utc().subtract(amountToSubtract, period)+ const periodFormatted = periodRaw.format('YYYY-MM-DD HH:mm:ss')+ const periodSubtracted = periodRaw+ .subtract(amountToSubtract, period)+ .format('YYYY-MM-DD HH:mm:ss')++ const queryThisWeek = `SELECT ${+ tableName === 'analytics' ? 'unique,' : ''+ } count() FROM ${tableName} WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodFormatted:String} AND {now:String}${+ tableName === 'analytics' ? ' GROUP BY unique' : ''+ }`+ const queryLastWeek = `SELECT ${+ tableName === 'analytics' ? 'unique,' : ''+ } count() FROM ${tableName} WHERE pid = {pid:FixedString(12)} AND created BETWEEN {periodSubtracted:String} AND {periodFormatted:String}${+ tableName === 'analytics' ? ' GROUP BY unique' : ''+ }`++ const paramsData = {+ params: {+ pid,+ periodFormatted,+ periodSubtracted,+ now,+ },+ }++ try {+ const thisWeekResult = await clickhouse+ .query(queryThisWeek, paramsData)+ .toPromise()+ const lastWeekResult = await clickhouse+ .query(queryLastWeek, paramsData)+ .toPromise()++ if (tableName === 'analytics') {+ const thisWeekUnique =+ _find(thisWeekResult, ({ unique }) => unique)?.['count()'] || 0+ const thisWeekPV =+ (_find(thisWeekResult, ({ unique }) => !unique)?.['count()'] || 0) ++ thisWeekUnique+ const lastWeekUnique =+ _find(lastWeekResult, ({ unique }) => unique)?.['count()'] || 0+ const lastWeekPV =+ (_find(lastWeekResult, ({ unique }) => !unique)?.['count()'] || 0) ++ lastWeekUnique++ result[pid] = {+ thisWeek: thisWeekPV,+ lastWeek: lastWeekPV,+ thisWeekUnique,+ lastWeekUnique,+ percChange: calculateRelativePercentage(lastWeekPV, thisWeekPV),+ percChangeUnique: calculateRelativePercentage(+ lastWeekUnique,+ thisWeekUnique,+ ),+ }+ } else {+ const thisWeek = thisWeekResult?.['count()'] || 0+ const lastWeek = lastWeekResult?.['count()'] || 0++ result[pid] = {+ thisWeek,+ lastWeek,+ percChange: calculateRelativePercentage(lastWeek, thisWeek),+ }+ }+ } catch {+ throw new InternalServerErrorException(+ "Can't process the provided PID. Please, try again later.",+ )+ }+ })++ await Promise.all(promises)++ return result+ }++ async generateParams(+ parsedFilters: Array<{ [key: string]: string }>,+ subQuery: string,+ customEVFilterApplied: boolean,+ paramsData: any,+ isCaptcha: boolean,+ isPerformance: boolean,+ ): Promise<any> {+ const params = {}++ // We need this to display all the pageview related data (e.g. country, browser) when user applies an inclusive filter on the Page column+ const isPageInclusiveFilterSet =+ isCaptcha || isPerformance+ ? false+ : !_isEmpty(+ _find(+ parsedFilters,+ filter => filter.column === 'pg' && !filter.isExclusive,+ ),+ )++ let columns = cols++ if (isCaptcha) {+ columns = captchaColumns+ }++ if (isPerformance) {+ columns = perfColumns+ }++ const paramsPromises = _map(columns, async col => {+ const query = generateParamsQuery(+ col,+ subQuery,+ customEVFilterApplied,+ isPageInclusiveFilterSet,+ isCaptcha,+ isPerformance,+ )+ const res = await clickhouse.query(query, paramsData).toPromise()++ params[col] = {}++ const size = _size(res)+ for (let j = 0; j < size; ++j) {+ const key = res[j][col]++ if (isPerformance) {+ params[col][key] = _round(+ millisecondsToSeconds(res[j]['avg(pageLoad)']),+ 2,+ )+ continue+ }++ params[col][key] = res[j]['count()']+ }+ })++ await Promise.all(paramsPromises)++ return params+ }++ async calculateAverageSessionDuration(+ subQuery: string,+ paramsData: any,+ ): Promise<number> {+ const avgSdurQuery = `SELECT avg(sdur) ${subQuery} AND sdur IS NOT NULL AND unique='1'`+ const avgSdurObject = await clickhouse+ .query(avgSdurQuery, paramsData)+ .toPromise()++ return _round(avgSdurObject[0]['avg(sdur)'])+ }++ generateXAxis(+ timeBucket: TimeBucketType,+ from: string, // timezone is already applied to the from and to parameters+ to: string,+ safeTimezone: string,+ ): IGenerateXAxis {+ const iterateTo = _includes(GMT_0_TIMEZONES, safeTimezone)+ ? dayjs.utc(to)+ : dayjs.tz(to, safeTimezone)+ let djsFrom = _includes(GMT_0_TIMEZONES, safeTimezone)+ ? dayjs.utc(from)+ : dayjs.tz(from, safeTimezone)++ let format++ switch (timeBucket) {+ case TimeBucketType.HOUR:+ format = 'YYYY-MM-DD HH:mm:ss'+ break++ case TimeBucketType.DAY:+ format = 'YYYY-MM-DD'+ break++ case TimeBucketType.MONTH:+ format = 'YYYY-MM'+ break++ default:+ throw new BadRequestException(+ `The provided time bucket (${timeBucket}) is incorrect`,+ )+ }++ // UTC dates+ const x = []++ // Timezone shifted dates (a hack to map Clickhouse timezone processed data to UTC dates)+ const xShifted = []++ while (djsFrom.isSameOrBefore(iterateTo, timeBucket)) {+ x.push(djsFrom.utc().format(format))+ xShifted.push(djsFrom.format(format))+ djsFrom = djsFrom.add(1, timeBucket)+ }++ if (_includes(GMT_0_TIMEZONES, safeTimezone)) {+ return {+ x,+ xShifted: x,+ }+ }++ return {+ x,+ xShifted,+ }+ }++ generateDateString(row: { [key: string]: number }): string {+ const { year, month, day, hour } = row++ let dateString = `${year}-${month < 10 ? `0${month}` : month}`++ if (typeof day === 'number') {+ if (day < 10) {+ dateString += `-0${day}`+ } else {+ dateString += `-${day}`+ }+ }++ if (typeof hour === 'number') {+ if (hour < 10) {+ dateString += ` 0${hour}:00:00`+ } else {+ dateString += ` ${hour}:00:00`+ }+ }++ return dateString+ }++ extractChartData(result, x: string[]): IExtractChartData {+ const visits = Array(x.length).fill(0)+ const uniques = Array(x.length).fill(0)+ const sdur = Array(x.length).fill(0)++ for (let row = 0; row < _size(result); ++row) {+ const dateString = this.generateDateString(result[row])+ const index = x.indexOf(dateString)++ if (index !== -1) {+ visits[index] = result[row].pageviews+ uniques[index] = result[row].uniques+ sdur[index] = _round(result[row].sdur)+ }+ }++ return {+ visits,+ uniques,+ sdur,+ }+ }++ extractCustomEventsChartData(queryResult, x: string[]): any {+ const result = {}++ for (let row = 0; row < _size(queryResult); ++row) {+ const { ev = '_unknown_event' } = queryResult[row]++ const dateString = this.generateDateString(queryResult[row])++ const index = x.indexOf(dateString)++ if (index !== -1) {+ if (!result[ev]) {+ result[ev] = Array(x.length).fill(0)+ }++ result[ev][index] = queryResult[row].count+ }+ }++ return result+ }++ getSafeTimezone(timezone: string): string {+ if (timezone === DEFAULT_TIMEZONE || !isValidTimezone(timezone)) {+ return DEFAULT_TIMEZONE+ }++ return timezone+ }++ getGroupSubquery(timeBucket: TimeBucketType): [string, string] {+ if (timeBucket === TimeBucketType.HOUR) {+ return [+ `toYear(tz_created) as year,+ toMonth(tz_created) as month,+ toDayOfMonth(tz_created) as day,+ toHour(tz_created) as hour`,+ 'year, month, day, hour',+ ]+ }++ if (timeBucket === TimeBucketType.MONTH) {+ return [+ `toYear(tz_created) as year,+ toMonth(tz_created) as month`,+ 'year, month',+ ]+ }++ return [+ `toYear(tz_created) as year,+ toMonth(tz_created) as month,+ toDayOfMonth(tz_created) as day`,+ 'year, month, day',+ ]+ }++ generateAnalyticsAggregationQuery(+ timeBucket: TimeBucketType,+ filtersQuery: string,+ ): string {+ const timeBucketFunc = timeBucketConversion[timeBucket]+ const [selector, groupBy] = this.getGroupSubquery(timeBucket)++ return `+ SELECT+ ${selector},+ avg(sdur) as sdur,+ count() as pageviews,+ sumIf(1, unique = 1) as uniques+ FROM (+ SELECT *,+ ${timeBucketFunc}(created) as tz_created+ FROM analytics+ WHERE+ pid = {pid:FixedString(12)}+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}+ ${filtersQuery}+ ) as subquery+ GROUP BY ${groupBy}+ ORDER BY ${groupBy}+ `+ }++ generateCustomEventsAggregationQuery(+ timeBucket: TimeBucketType,+ filtersQuery: string,+ paramsData: any,+ ): string {+ const timeBucketFunc = timeBucketConversion[timeBucket]+ const [selector, groupBy] = this.getGroupSubquery(timeBucket)++ return `+ SELECT+ ${selector},+ count() as count+ FROM (+ SELECT *,+ ${timeBucketFunc}(created) as tz_created+ FROM customEV+ WHERE ${paramsData.params.ev_exclusive ? 'NOT' : ''} ev = {ev:String}+ AND pid = {pid:FixedString(12)}+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}+ ${filtersQuery}+ ) as subquery+ GROUP BY ${groupBy}+ ORDER BY ${groupBy}+ `+ }++ generatePerformanceAggregationQuery(+ timeBucket: TimeBucketType,+ filtersQuery: string,+ ): string {+ const timeBucketFunc = timeBucketConversion[timeBucket]+ const [selector, groupBy] = this.getGroupSubquery(timeBucket)++ return `+ SELECT+ ${selector},+ avg(dns) as dns,+ avg(tls) as tls,+ avg(conn) as conn,+ avg(response) as response,+ avg(render) as render,+ avg(domLoad) as domLoad,+ avg(ttfb) as ttfb+ FROM (+ SELECT *,+ ${timeBucketFunc}(created) as tz_created+ FROM performance+ WHERE+ pid = {pid:FixedString(12)}+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}+ ${filtersQuery}+ ) as subquery+ GROUP BY ${groupBy}+ ORDER BY ${groupBy}+ `+ }++ generateCaptchaAggregationQuery(+ timeBucket: TimeBucketType,+ filtersQuery: string,+ ): string {+ const timeBucketFunc = timeBucketConversion[timeBucket]+ const [selector, groupBy] = this.getGroupSubquery(timeBucket)++ return `+ SELECT+ ${selector},+ count() as count+ FROM (+ SELECT *,+ ${timeBucketFunc}(created) as tz_created+ FROM captcha+ WHERE+ pid = {pid:FixedString(12)}+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}+ ${filtersQuery}+ ) as subquery+ GROUP BY ${groupBy}+ ORDER BY ${groupBy}+ `+ }++ async groupByTimeBucket(+ timeBucket: TimeBucketType,+ from: string,+ to: string,+ subQuery: string,+ filtersQuery: string,+ paramsData: any,+ safeTimezone: string,+ customEVFilterApplied: boolean,+ parsedFilters: Array<{ [key: string]: string }>,+ ): Promise<object | void> {+ let params: unknown = {}+ let chart: unknown = {}+ let avgSdur = 0++ const promises = [+ // Getting params+ (async () => {+ params = await this.groupParamsByTimeBucket(+ subQuery,+ paramsData,+ customEVFilterApplied,+ parsedFilters,+ )++ if (!_some(_values(params), val => !_isEmpty(val))) {+ throw new BadRequestException(+ 'The are no parameters for the specified time frames',+ )+ }+ })(),++ // Getting chart & average session duration data+ (async () => {+ const groupedData = await this.groupChartByTimeBucket(+ timeBucket,+ from,+ to,+ subQuery,+ filtersQuery,+ paramsData,+ safeTimezone,+ customEVFilterApplied,+ )++ // @ts-ignore+ chart = groupedData.chart++ // @ts-ignore+ avgSdur = groupedData.avgSdur+ })(),+ ]++ await Promise.all(promises)++ return {+ params,+ chart,+ avgSdur,+ }+ }++ async groupParamsByTimeBucket(+ subQuery: string,+ paramsData: any,+ customEVFilterApplied: boolean,+ parsedFilters: Array<{ [key: string]: string }>,+ ): Promise<object | void> {+ return this.generateParams(+ parsedFilters,+ subQuery,+ customEVFilterApplied,+ paramsData,+ false,+ false,+ )+ }++ async groupChartByTimeBucket(+ timeBucket: TimeBucketType,+ from: string,+ to: string,+ subQuery: string,+ filtersQuery: string,+ paramsData: any,+ safeTimezone: string,+ customEVFilterApplied: boolean,+ ): Promise<object | void> {+ const avgSdur = customEVFilterApplied+ ? 0+ : await this.calculateAverageSessionDuration(subQuery, paramsData)++ const { x, xShifted } = this.generateXAxis(+ timeBucket,+ from,+ to,+ safeTimezone,+ )++ if (customEVFilterApplied) {+ const query = this.generateCustomEventsAggregationQuery(+ timeBucket,+ filtersQuery,+ paramsData,+ )++ const result = <Array<TrafficCEFilterCHResponse>>(+ await clickhouse.query(query, paramsData).toPromise()+ )++ const uniques =+ this.extractCustomEventsChartData(result, x)?._unknown_event || []++ const sdur = Array(_size(x)).fill(0)++ return Promise.resolve({+ chart: {+ x: xShifted,+ visits: uniques,+ uniques,+ sdur,+ },+ avgSdur,+ })+ }+ const query = this.generateAnalyticsAggregationQuery(+ timeBucket,+ filtersQuery,+ )++ const result = <Array<TrafficCHResponse>>(+ await clickhouse.query(query, paramsData).toPromise()+ )++ const { visits, uniques, sdur } = this.extractChartData(result, x)++ return Promise.resolve({+ chart: {+ x: xShifted,+ visits,+ uniques,+ sdur,+ },+ avgSdur,+ })+ }++ extractCaptchaChartData(result, x: string[]): any {+ const count = Array(x.length).fill(0)++ for (let row = 0; row < _size(result); ++row) {+ const dateString = this.generateDateString(result[row])++ const index = x.indexOf(dateString)++ if (index !== -1) {+ count[index] = result[row].count+ }+ }++ return {+ count,+ }+ }++ async groupCaptchaByTimeBucket(+ timeBucket: TimeBucketType,+ from: string,+ to: string,+ subQuery: string,+ filtersQuery: string,+ paramsData: object,+ safeTimezone: string,+ ): Promise<object | void> {+ let params: unknown = {}+ let chart: unknown = {}++ const promises = [+ // Getting params+ (async () => {+ params = await this.generateParams(+ null,+ subQuery,+ false,+ paramsData,+ true,+ false,+ )++ if (!_some(_values(params), val => !_isEmpty(val))) {+ throw new BadRequestException(+ 'The are no parameters for the specified time frames',+ )+ }+ })(),++ // Getting CAPTCHA chart data+ (async () => {+ const { x, xShifted } = this.generateXAxis(+ timeBucket,+ from,+ to,+ safeTimezone,+ )++ const query = this.generateCaptchaAggregationQuery(+ timeBucket,+ filtersQuery,+ )++ const result = <Array<TrafficCEFilterCHResponse>>(+ await clickhouse.query(query, paramsData).toPromise()+ )+ const { count } = this.extractCaptchaChartData(result, x)++ chart = {+ x: xShifted,+ results: count,+ }+ })(),+ ]++ await Promise.all(promises)++ return Promise.resolve({+ params,+ chart,+ })+ }++ extractPerformanceChartData(result, x: string[]): any {+ const dns = Array(x.length).fill(0)+ const tls = Array(x.length).fill(0)+ const conn = Array(x.length).fill(0)+ const response = Array(x.length).fill(0)+ const render = Array(x.length).fill(0)+ const domLoad = Array(x.length).fill(0)+ const ttfb = Array(x.length).fill(0)++ for (let row = 0; row < _size(result); ++row) {+ const dateString = this.generateDateString(result[row])++ const index = x.indexOf(dateString)++ if (index !== -1) {+ dns[index] = _round(millisecondsToSeconds(result[row].dns), 2)+ tls[index] = _round(millisecondsToSeconds(result[row].tls), 2)+ conn[index] = _round(millisecondsToSeconds(result[row].conn), 2)+ response[index] = _round(millisecondsToSeconds(result[row].response), 2)+ render[index] = _round(millisecondsToSeconds(result[row].render), 2)+ domLoad[index] = _round(millisecondsToSeconds(result[row].domLoad), 2)+ ttfb[index] = _round(millisecondsToSeconds(result[row].ttfb), 2)+ }+ }++ return {+ dns,+ tls,+ conn,+ response,+ render,+ domLoad,+ ttfb,+ }+ }++ async getPerfChartData(+ timeBucket: TimeBucketType,+ from: string,+ to: string,+ filtersQuery: string,+ paramsData: object,+ safeTimezone: string,+ ) {+ const { x, xShifted } = this.generateXAxis(+ timeBucket,+ from,+ to,+ safeTimezone,+ )++ const query = this.generatePerformanceAggregationQuery(+ timeBucket,+ filtersQuery,+ )++ const result = <Array<PerformanceCHResponse>>(+ await clickhouse.query(query, paramsData).toPromise()+ )++ return {+ x: xShifted,+ ...this.extractPerformanceChartData(result, x),+ }+ }++ async groupPerfByTimeBucket(+ timeBucket: TimeBucketType,+ from: string,+ to: string,+ subQuery: string,+ filtersQuery: string,+ paramsData: object,+ safeTimezone: string,+ ): Promise<object | void> {+ let params: unknown = {}+ let chart: unknown = {}++ const promises = [+ // Getting params+ (async () => {+ params = await this.generateParams(+ null,+ subQuery,+ false,+ paramsData,+ false,+ true,+ )++ if (!_some(_values(params), val => !_isEmpty(val))) {+ throw new BadRequestException(+ 'The are no parameters for the specified time frames',+ )+ }+ })(),++ // Getting chart data+ (async () => {+ chart = await this.getPerfChartData(+ timeBucket,+ from,+ to,+ filtersQuery,+ paramsData,+ safeTimezone,+ )+ })(),+ ]++ await Promise.all(promises)++ return Promise.resolve({+ params,+ chart,+ })+ }++ async processCustomEV(query: string, params: object): Promise<object> {+ const result = {}++ const rawCustoms = <Array<CustomsCHResponse>>(+ await clickhouse.query(query, params).toPromise()+ )+ const size = _size(rawCustoms)++ for (let i = 0; i < size; ++i) {+ const { ev, 'count()': c } = rawCustoms[i]+ result[ev] = c+ }++ return result+ }++ async getOnlineUserCount(pid: string): Promise<number> {+ // @ts-ignore+ return redis.countKeysByPattern(`hb:${pid}:*`)+ }++ async groupCustomEVByTimeBucket(+ timeBucket: TimeBucketType,+ from: string,+ to: string,+ filtersQuery: string,+ paramsData: object,+ safeTimezone: string,+ ): Promise<object | void> {+ const { x, xShifted } = this.generateXAxis(+ timeBucket,+ from,+ to,+ safeTimezone,+ )++ const timeBucketFunc = timeBucketConversion[timeBucket]+ const [selector, groupBy] = this.getGroupSubquery(timeBucket)++ const query = `+ SELECT+ ${selector},+ ev,+ count() as count+ FROM (+ SELECT *,+ ${timeBucketFunc}(created) as tz_created+ FROM customEV+ WHERE+ pid = {pid:FixedString(12)}+ AND created BETWEEN {groupFrom:String} AND {groupTo:String}+ ${filtersQuery}+ ) as subquery+ GROUP BY ${groupBy}, ev+ ORDER BY ${groupBy}+ `++ const result = <Array<CustomsCHAggregatedResponse>>(+ await clickhouse.query(query, paramsData).toPromise()+ )++ const events = this.extractCustomEventsChartData(result, x)++ return Promise.resolve({+ chart: {+ x: xShifted,+ events,+ },+ })+ }++ async getOnlineCountByProjectId(projectId: string) {+ // @ts-ignore+ return redis.countKeysByPattern(`hb:${projectId}:*`)+ }++ async getStatsByProjectId(projectId: string) {+ return this.getSummaryStats([projectId], 'analytics', 'w', 1)+ }+}diff --git a/selfhosted-src/analytics/dto/events.dto.ts b/selfhosted-src/analytics/dto/events.dto.tsnew file mode 100644index 000000000..7cec51ad4--- /dev/null+++ b/selfhosted-src/analytics/dto/events.dto.ts@@ -0,0 +1,69 @@+import { ApiProperty } from '@nestjs/swagger'+import { IsNotEmpty } from 'class-validator'++export class EventsDTO {+ @ApiProperty({+ example: 'aUn1quEid-3',+ required: true,+ description: 'The project ID',+ })+ @IsNotEmpty()+ pid: string++ @ApiProperty({+ example: 'user-subscribed',+ description: 'Event name',+ maxLength: 64,+ })+ @IsNotEmpty()+ ev: string++ @ApiProperty({+ description:+ 'If true, only 1 event with the same ID will be saved per user session',+ })+ unique: object++ // Tracking metrics+ @ApiProperty({+ example: 'Europe/Kiev',+ description: "User's timezone",+ })+ tz?: string++ @ApiProperty({+ example: '/articles/my-awesome-article-1',+ description: 'A page that user sent data from',+ })+ pg?: string++ @ApiProperty({+ example: 'en-GB',+ description: "User's locale",+ })+ lc?: string++ @ApiProperty({+ example: 'https://example.com',+ description: 'The referrer',+ })+ ref?: string++ @ApiProperty({+ example: 'duckduckgo',+ description: 'utm_source URL parameter',+ })+ so?: string++ @ApiProperty({+ example: 'cpc',+ description: 'utm_medium URL parameter',+ })+ me?: string++ @ApiProperty({+ example: 'spring_sale',+ description: 'utm_campaign URL parameter',+ })+ ca?: string+}diff --git a/selfhosted-src/analytics/dto/get-custom-events.dto.ts b/selfhosted-src/analytics/dto/get-custom-events.dto.tsnew file mode 100644index 000000000..76532db9d--- /dev/null+++ b/selfhosted-src/analytics/dto/get-custom-events.dto.ts@@ -0,0 +1,14 @@+import { PickType } from '@nestjs/swagger'+import { AnalyticsGET_DTO } from './getData.dto'++export class GetCustomEventsDto extends PickType(AnalyticsGET_DTO, [+ 'pid',+ 'period',+ 'timeBucket',+ 'from',+ 'to',+ 'filters',+ 'timezone',+] as const) {+ customEvents: string+}diff --git a/selfhosted-src/analytics/dto/getData.dto.ts b/selfhosted-src/analytics/dto/getData.dto.tsnew file mode 100644index 000000000..53a39d5bc--- /dev/null+++ b/selfhosted-src/analytics/dto/getData.dto.ts@@ -0,0 +1,41 @@+import { ApiProperty } from '@nestjs/swagger'+import { IsNotEmpty } from 'class-validator'+import { DEFAULT_TIMEZONE } from '../../user/entities/user.entity'++export enum TimeBucketType {+ HOUR = 'hour',+ DAY = 'day',+ MONTH = 'month',+}++// eslint-disable-next-line @typescript-eslint/naming-convention+export class AnalyticsGET_DTO {+ @ApiProperty()+ @IsNotEmpty()+ pid: string++ @ApiProperty()+ period: string++ @ApiProperty({ enum: TimeBucketType })+ @IsNotEmpty()+ timeBucket: TimeBucketType++ @ApiProperty()+ from: string++ @ApiProperty()+ to: string++ @ApiProperty({+ description:+ 'A stringified array of properties to filter [{ column, filter, isExclusive }]',+ })+ filters: string++ @ApiProperty({+ description: 'Timezone to display data in',+ default: DEFAULT_TIMEZONE,+ })+ timezone: string+}diff --git a/selfhosted-src/analytics/dto/getUserFlow.dto.ts b/selfhosted-src/analytics/dto/getUserFlow.dto.tsnew file mode 100644index 000000000..63fd88641--- /dev/null+++ b/selfhosted-src/analytics/dto/getUserFlow.dto.ts@@ -0,0 +1,31 @@+import { ApiProperty } from '@nestjs/swagger'+import { IsNotEmpty } from 'class-validator'+import { DEFAULT_TIMEZONE } from '../../user/entities/user.entity'++// eslint-disable-next-line @typescript-eslint/naming-convention+export class GetUserFlowDTO {+ @ApiProperty()+ @IsNotEmpty()+ pid: string++ @ApiProperty()+ period: string++ @ApiProperty()+ from: string++ @ApiProperty()+ to: string++ @ApiProperty({+ description: 'Timezone to display data in',+ default: DEFAULT_TIMEZONE,+ })+ timezone: string++ @ApiProperty({+ description:+ 'A stringified array of properties to filter [{ column, filter, isExclusive }]',+ })+ filters: string+}diff --git a/selfhosted-src/analytics/dto/pageviews.dto.ts b/selfhosted-src/analytics/dto/pageviews.dto.tsnew file mode 100644index 000000000..a8ea10615--- /dev/null+++ b/selfhosted-src/analytics/dto/pageviews.dto.ts@@ -0,0 +1,91 @@+import { ApiProperty } from '@nestjs/swagger'+import { IsNotEmpty } from 'class-validator'++export class PageviewsDTO {+ @ApiProperty({+ example: 'aUn1quEid-3',+ required: true,+ description: 'The project ID',+ })+ @IsNotEmpty()+ pid: string++ // Tracking metrics+ @ApiProperty({+ example: 'Europe/Kiev',+ description: "User's timezone",+ })+ tz?: string++ @ApiProperty({+ example: '/articles/my-awesome-article-1',+ description: 'A page that user sent data from',+ })+ pg?: string++ @ApiProperty({+ example: '/home',+ description: 'Previous page user was on',+ })+ prev?: string++ @ApiProperty({+ example: 'en-GB',+ description: "User's locale",+ })+ lc?: string++ @ApiProperty({+ example: 'https://example.com',+ description: 'The referrer',+ })+ ref?: string++ @ApiProperty({+ example: 'duckduckgo',+ description: 'utm_source URL parameter',+ })+ so?: string++ @ApiProperty({+ example: 'cpc',+ description: 'utm_medium URL parameter',+ })+ me?: string++ @ApiProperty({+ example: 'spring_sale',+ description: 'utm_campaign URL parameter',+ })+ ca?: string++ @ApiProperty({+ example: false,+ description: 'If true, only unique events will be saved',+ })+ unique?: boolean++ @ApiProperty({+ example: {+ conn: 120,+ dns: 170,+ dom_load: 975.5,+ page_load: 1515,+ render: 532.25,+ response: 170,+ tls: 0,+ ttfb: 78.3,+ },+ description: 'Performance metrics',+ })+ perf?: {+ dns: number+ tls: number+ conn: number+ response: number+ render: number+ dom_load: number+ page_load: number+ ttfb: number+ }+}diff --git a/selfhosted-src/analytics/interfaces/index.ts b/selfhosted-src/analytics/interfaces/index.tsnew file mode 100644index 000000000..0844bcaf6--- /dev/null+++ b/selfhosted-src/analytics/interfaces/index.ts@@ -0,0 +1,93 @@+export interface TrafficCEFilterCHResponse {+ year: number+ month: number+ day?: number+ hour?: number+ count: number+}++export interface TrafficCHResponse {+ year: number+ month: number+ day?: number+ hour?: number+ sdur: number+ uniques: number+ pageviews: number+}++export interface PerformanceCHResponse {+ year: number+ month: number+ day?: number+ hour?: number+ dns: number+ tls: number+ conn: number+ response: number+ render: number+ domLoad: number+ ttfb: number+}++export interface CustomsCHResponse {+ ev: string+ 'count()': number+ index: number+}++export interface CustomsCHAggregatedResponse {+ year: number+ month: number+ day?: number+ hour?: number+ ev: string+ count: number+}++export interface IGetGroupFromTo {+ groupFrom: string+ groupTo: string+ groupFromUTC: string+ groupToUTC: string+}++export interface GetFiltersQuery extends Array<string | object> {+ // SQL query+ 0: string+ // an object that has structure like { cf_pg: '/signup', ev_exclusive: false }+ 1: { [key: string]: string | boolean }+ // an array of objects like [{ "column":"pg", "filter":"/signup", "isExclusive":true }]+ 2: Array<{ [key: string]: string }> | []+}++export interface IUserFlowNode {+ id: string+}++export interface IUserFlowLink {+ source: string+ target: string+ value: number+}++export interface IGenerateXAxis {+ x: string[]+ xShifted: string[]+}++export interface IBuildUserFlow {+ nodes: IUserFlowNode[]+ links: IUserFlowLink[]+}++export interface IUserFlow {+ ascending: IBuildUserFlow+ descending: IBuildUserFlow+}++export interface IExtractChartData {+ visits: number[]+ uniques: number[]+ sdur: number[]+}diff --git a/selfhosted-src/app.module.ts b/selfhosted-src/app.module.tsnew file mode 100644index 000000000..89832f892--- /dev/null+++ b/selfhosted-src/app.module.ts@@ -0,0 +1,36 @@+import { Module } from '@nestjs/common'+import { ConfigModule } from '@nestjs/config'+import { ScheduleModule } from '@nestjs/schedule'+import { NestjsFormDataModule } from 'nestjs-form-data'++import { I18nModule } from 'nestjs-i18n'+import { UserModule } from './user/user.module'+import { AnalyticsModule } from './analytics/analytics.module'+import { ProjectModule } from './project/project.module'+import { TaskManagerModule } from './task-manager/task-manager.module'+import { PingModule } from './ping/ping.module'+import { getI18nConfig } from './configs'+import { AuthModule } from './auth/auth.module'++const imports = [+ ConfigModule.forRoot({+ cache: true,+ envFilePath: '.env',+ expandVariables: true,+ isGlobal: true,+ }),+ I18nModule.forRootAsync(getI18nConfig()),+ ScheduleModule.forRoot(),+ NestjsFormDataModule.config({ isGlobal: true }),+ TaskManagerModule,+ UserModule,+ ProjectModule,+ AnalyticsModule,+ PingModule,+ AuthModule,+]++@Module({+ imports,+})+export class AppModule {}diff --git a/selfhosted-src/auth/auth.controller.ts b/selfhosted-src/auth/auth.controller.tsnew file mode 100644index 000000000..dc6d3d89c--- /dev/null+++ b/selfhosted-src/auth/auth.controller.ts@@ -0,0 +1,154 @@+import {+ Controller,+ UseFilters,+ UseGuards,+ UsePipes,+ ValidationPipe,+ Post,+ Body,+ Ip,+ ConflictException,+ HttpCode,+ HttpStatus,+ UnauthorizedException,+ Headers,+} from '@nestjs/common'+import {+ ApiTags,+ ApiOperation,+ ApiOkResponse,+} from '@nestjs/swagger'+import { I18nValidationExceptionFilter, I18n, I18nContext } from 'nestjs-i18n'++import { checkRateLimit } from 'selfhosted-src/common/utils'+import {+ generateSelfhostedUser,+} from 'selfhosted-src/user/entities/user.entity'+import { AuthService } from './auth.service'+import { Public, CurrentUserId, CurrentUser } from './decorators'+import {+ LoginResponseDto,+ LoginRequestDto,+} from './dtos'+import { JwtAccessTokenGuard, JwtRefreshTokenGuard } from './guards'++@ApiTags('Auth')+@Controller({ path: 'auth', version: '1' })+@UseFilters(new I18nValidationExceptionFilter())+@UseGuards(JwtAccessTokenGuard)+@UsePipes(+ new ValidationPipe({+ forbidNonWhitelisted: true,+ forbidUnknownValues: true,+ transform: true,+ whitelist: true,+ }),+)+export class AuthController {+ constructor(+ private readonly authService: AuthService,+ ) {}++ @ApiOperation({ summary: 'Login a user' })+ @ApiOkResponse({+ description: 'User logged in',+ type: LoginResponseDto,+ })+ @Public()+ @Post('login')+ @HttpCode(HttpStatus.OK)+ public async login(+ @Body() body: LoginRequestDto,+ @I18n() i18n: I18nContext,+ @Headers() headers: unknown,+ @Ip() requestIp: string,+ ): Promise<LoginResponseDto> {+ const ip =+ headers['x-forwarded-for'] || headers['cf-connecting-ip'] || requestIp++ await checkRateLimit(ip, 'login', 10, 1800)++ let user = await this.authService.validateUser(body.email, body.password)++ if (!user) {+ throw new ConflictException(i18n.t('auth.invalidCredentials'))+ }++ const jwtTokens = await this.authService.generateJwtTokens(+ user.id,+ !user.isTwoFactorAuthenticationEnabled,+ )++ return {+ ...jwtTokens,+ user,+ }+ }++ @ApiOperation({ summary: 'Refresh a token' })+ @ApiOkResponse({+ description: 'Token refreshed',+ })+ @Public()+ @UseGuards(JwtRefreshTokenGuard)+ @Post('refresh-token')+ @HttpCode(200)+ public async refreshToken(+ @CurrentUserId() userId: string,+ @CurrentUser('refreshToken') refreshToken: string,+ @I18n() i18n: I18nContext,+ ): Promise<{ accessToken: string }> {+ const user = generateSelfhostedUser()++ if (!user) {+ throw new UnauthorizedException()+ }++ const isRefreshTokenValid = await this.authService.checkRefreshToken(+ user.id,+ refreshToken,+ )++ if (!isRefreshTokenValid) {+ throw new ConflictException(i18n.t('auth.invalidRefreshToken'))+ }++ const accessToken = await this.authService.generateJwtAccessToken(+ user.id,+ user.isTwoFactorAuthenticationEnabled,+ )++ return { accessToken }+ }++ @ApiOperation({ summary: 'Logout' })+ @ApiOkResponse({+ description: 'Logged out',+ })+ @Public()+ @UseGuards(JwtRefreshTokenGuard)+ @Post('logout')+ @HttpCode(200)+ public async logout(+ @CurrentUserId() userId: string,+ @CurrentUser('refreshToken') refreshToken: string,+ @I18n() i18n: I18nContext,+ ): Promise<void> {+ const user = generateSelfhostedUser()++ if (!user) {+ throw new UnauthorizedException()+ }++ const isRefreshTokenValid = await this.authService.checkRefreshToken(+ user.id,+ refreshToken,+ )++ if (!isRefreshTokenValid) {+ throw new ConflictException(i18n.t('auth.invalidRefreshToken'))+ }++ await this.authService.logout(user.id, refreshToken)+ }+}diff --git a/selfhosted-src/auth/auth.module.ts b/selfhosted-src/auth/auth.module.tsnew file mode 100644index 000000000..966cffd20--- /dev/null+++ b/selfhosted-src/auth/auth.module.ts@@ -0,0 +1,26 @@+import { Module } from '@nestjs/common'+import { JwtModule } from '@nestjs/jwt'+import { PassportModule } from '@nestjs/passport'+import { AuthController } from './auth.controller'+import { AuthService } from './auth.service'+import {+ // ApiKeyStrategy,+ JwtAccessTokenStrategy,+ JwtRefreshTokenStrategy,+} from './strategies'++@Module({+ imports: [+ JwtModule.register({}),+ PassportModule,+ ],+ controllers: [AuthController],+ providers: [+ AuthService,+ JwtAccessTokenStrategy,+ JwtRefreshTokenStrategy,+ // ApiKeyStrategy,+ ],+ exports: [AuthService],+})+export class AuthModule {}diff --git a/selfhosted-src/auth/auth.service.ts b/selfhosted-src/auth/auth.service.tsnew file mode 100644index 000000000..0a986265d--- /dev/null+++ b/selfhosted-src/auth/auth.service.ts@@ -0,0 +1,117 @@+import {+ Injectable,+} from '@nestjs/common'+import { ConfigService } from '@nestjs/config'+import { JwtService } from '@nestjs/jwt'+import { genSalt, hash } from 'bcrypt'+import * as _isEmpty from 'lodash/isEmpty'++import {+ SelfhostedUser,+ generateSelfhostedUser,+} from 'selfhosted-src/user/entities/user.entity'+import {+ saveRefreshTokenClickhouse,+ findRefreshTokenClickhouse,+ deleteRefreshTokenClickhouse,+} from 'selfhosted-src/common/utils'+import {+ SELFHOSTED_EMAIL,+ SELFHOSTED_PASSWORD,+ JWT_ACCESS_TOKEN_SECRET,+} from 'selfhosted-src/common/constants'++@Injectable()+export class AuthService {+ constructor(+ private readonly configService: ConfigService,+ private readonly jwtService: JwtService,+ ) {}++ public async hashPassword(password: string): Promise<string> {+ const salt = await genSalt(10)+ return hash(password, salt)+ }++ public async generateJwtAccessToken(+ userId: string,+ isSecondFactorAuthenticated = false,+ ) {+ return this.jwtService.signAsync(+ {+ sub: userId,+ isSecondFactorAuthenticated,+ },+ {+ algorithm: 'HS256',+ expiresIn: 60 * 30,+ secret: JWT_ACCESS_TOKEN_SECRET,+ },+ )+ }++ public async validateUser(+ email: string,+ password: string,+ ): Promise<SelfhostedUser | null> {+ if (email !== SELFHOSTED_EMAIL || password !== SELFHOSTED_PASSWORD) {+ return null+ }++ return generateSelfhostedUser()+ }++ private async generateJwtRefreshToken(+ userId: string,+ isSecondFactorAuthenticated = false,+ ) {+ const refreshToken = await this.jwtService.signAsync(+ {+ sub: userId,+ isSecondFactorAuthenticated,+ },+ {+ algorithm: 'HS256',+ expiresIn: 60 * 60 * 24 * 30,+ secret: this.configService.get('JWT_REFRESH_TOKEN_SECRET'),+ },+ )++ await saveRefreshTokenClickhouse(userId, refreshToken)++ return refreshToken+ }++ public async checkRefreshToken(+ userId: string,+ refreshToken: string,+ ): Promise<boolean> {+ const tokens = await findRefreshTokenClickhouse(userId, refreshToken)+ return !_isEmpty(tokens)+ }++ public async generateJwtTokens(+ userId: string,+ isSecondFactorAuthenticated = false,+ ) {+ const accessToken = await this.generateJwtAccessToken(+ userId,+ isSecondFactorAuthenticated,+ )++ let refreshToken = 'NOT_AVAILABLE'++ if (isSecondFactorAuthenticated) {+ refreshToken = await this.generateJwtRefreshToken(+ userId,+ isSecondFactorAuthenticated,+ )+ }++ return { accessToken, refreshToken }+ }++ async logout(userId: string, refreshToken: string) {+ await deleteRefreshTokenClickhouse(userId, refreshToken)+ }+}diff --git a/selfhosted-src/auth/decorators/auth.decorator.ts b/selfhosted-src/auth/decorators/auth.decorator.tsnew file mode 100644index 000000000..c5491076b--- /dev/null+++ b/selfhosted-src/auth/decorators/auth.decorator.ts@@ -0,0 +1,24 @@+import { applyDecorators, SetMetadata, UseGuards } from '@nestjs/common'+import {+ ApiKeyRateLimitGuard,+ JwtAccessTokenGuard,+ MultiAuthGuard,+ RolesGuard,+} from 'selfhosted-src/auth/guards'+import { UserType } from 'selfhosted-src/user/entities/user.entity'+import { ROLES_KEY } from './roles.decorator'++export const IS_OPTIONAL_AUTH_KEY = 'isOptionalAuth'+export function Auth(+ roles: UserType[],+ isApiKeyAuth = false,+ isOptionalAuth = false,+) {+ return applyDecorators(+ SetMetadata(ROLES_KEY, roles),+ SetMetadata(IS_OPTIONAL_AUTH_KEY, isOptionalAuth),+ isApiKeyAuth+ ? UseGuards(MultiAuthGuard, ApiKeyRateLimitGuard, RolesGuard)+ : UseGuards(JwtAccessTokenGuard, RolesGuard),+ )+}diff --git a/selfhosted-src/auth/decorators/current-user-id.decorator.ts b/selfhosted-src/auth/decorators/current-user-id.decorator.tsnew file mode 100644index 000000000..629fd4246--- /dev/null+++ b/selfhosted-src/auth/decorators/current-user-id.decorator.ts@@ -0,0 +1,9 @@+import { createParamDecorator, ExecutionContext } from '@nestjs/common'++export const CurrentUserId = createParamDecorator(+ (_: unknown, context: ExecutionContext) => {+ const request = context.switchToHttp().getRequest()+ const { user } = request+ return user && (user.sub || user.id) ? user.sub || user.id : null+ },+)diff --git a/selfhosted-src/auth/decorators/current-user.decorator.ts b/selfhosted-src/auth/decorators/current-user.decorator.tsnew file mode 100644index 000000000..9c9d258d4--- /dev/null+++ b/selfhosted-src/auth/decorators/current-user.decorator.ts@@ -0,0 +1,14 @@+import { createParamDecorator, ExecutionContext } from '@nestjs/common'+import { IJwtRefreshTokenPayload } from '../interfaces'++export const CurrentUser = createParamDecorator(+ (+ data: keyof IJwtRefreshTokenPayload | undefined,+ context: ExecutionContext,+ ) => {+ const request = context.switchToHttp().getRequest()+ const { user } = request++ return data ? user && user[data] : user+ },+)diff --git a/selfhosted-src/auth/decorators/index.ts b/selfhosted-src/auth/decorators/index.tsnew file mode 100644index 000000000..377aac6a6--- /dev/null+++ b/selfhosted-src/auth/decorators/index.ts@@ -0,0 +1,6 @@+export * from './current-user-id.decorator'+export * from './public.decorator'+export * from './current-user.decorator'+export * from './two-fa-not-required.decorator'+export * from './roles.decorator'+export * from './auth.decorator'diff --git a/selfhosted-src/auth/decorators/public.decorator.ts b/selfhosted-src/auth/decorators/public.decorator.tsnew file mode 100644index 000000000..91c2399b8--- /dev/null+++ b/selfhosted-src/auth/decorators/public.decorator.ts@@ -0,0 +1,4 @@+import { SetMetadata } from '@nestjs/common'++export const IS_PUBLIC_KEY = 'isPublic'+export const Public = () => SetMetadata(IS_PUBLIC_KEY, true)diff --git a/selfhosted-src/auth/decorators/roles.decorator.ts b/selfhosted-src/auth/decorators/roles.decorator.tsnew file mode 100644index 000000000..fb70bbeb1--- /dev/null+++ b/selfhosted-src/auth/decorators/roles.decorator.ts@@ -0,0 +1,5 @@+import { SetMetadata } from '@nestjs/common'+import { UserType } from 'selfhosted-src/user/entities/user.entity'++export const ROLES_KEY = 'roles'+export const Roles = (...roles: UserType[]) => SetMetadata(ROLES_KEY, roles)diff --git a/selfhosted-src/auth/decorators/two-fa-not-required.decorator.ts b/selfhosted-src/auth/decorators/two-fa-not-required.decorator.tsnew file mode 100644index 000000000..7f0839d32--- /dev/null+++ b/selfhosted-src/auth/decorators/two-fa-not-required.decorator.ts@@ -0,0 +1,5 @@+import { SetMetadata } from '@nestjs/common'++export const IS_TWO_FA_NOT_REQUIRED_KEY = 'isTwoFaNotRequired'+export const TwoFaNotRequired = () =>+ SetMetadata(IS_TWO_FA_NOT_REQUIRED_KEY, true)diff --git a/selfhosted-src/auth/dtos/change-password.dto.ts b/selfhosted-src/auth/dtos/change-password.dto.tsnew file mode 100644index 000000000..2f8e41311--- /dev/null+++ b/selfhosted-src/auth/dtos/change-password.dto.ts@@ -0,0 +1,24 @@+import { ApiProperty } from '@nestjs/swagger'+import { MaxLength, MinLength } from 'class-validator'++export class ChangePasswordDto {+ @ApiProperty({+ description: 'User password',+ example: '%d7*c4W45p',+ maxLength: 72,+ minLength: 8,+ })+ @MaxLength(50, { message: 'Max length is $constraint1 characters' })+ @MinLength(8, { message: 'Min length is $constraint1 characters' })+ public readonly oldPassword: string++ @ApiProperty({+ description: 'User password',+ example: '%d7*c4W45p',+ maxLength: 72,+ minLength: 8,+ })+ @MaxLength(50, { message: 'Max length is $constraint1 characters' })+ @MinLength(8, { message: 'Min length is $constraint1 characters' })+ public readonly newPassword: string+}diff --git a/selfhosted-src/auth/dtos/confirm-change-email.dto.ts b/selfhosted-src/auth/dtos/confirm-change-email.dto.tsnew file mode 100644index 000000000..978d62050--- /dev/null+++ b/selfhosted-src/auth/dtos/confirm-change-email.dto.ts@@ -0,0 +1,11 @@+import { ApiProperty } from '@nestjs/swagger'+import { IsUUID } from 'class-validator'++export class ConfirmChangeEmailDto {+ @ApiProperty({+ description: 'The token to confirm the change email',+ example: 'a1b2c3d4-e5f6-g7h8-i9j0-k1l2m3n4o5p6',+ })+ @IsUUID('4', { message: 'Please enter the valid UUID.' })+ public readonly token: string+}diff --git a/selfhosted-src/auth/dtos/confirm-reset-password.dto.ts b/selfhosted-src/auth/dtos/confirm-reset-password.dto.tsnew file mode 100644index 000000000..4d367f1f6--- /dev/null+++ b/selfhosted-src/auth/dtos/confirm-reset-password.dto.ts@@ -0,0 +1,11 @@+import { ApiProperty } from '@nestjs/swagger'+import { IsUUID } from 'class-validator'++export class ConfirmResetPasswordDto {+ @ApiProperty({+ description: 'The token to confirm the reset password',+ example: 'a1b2c3d4-e5f6-g7h8-i9j0-k1l2m3n4o5p6',+ })+ @IsUUID('4', { message: 'Please enter the valid UUID.' })+ public readonly token: string+}diff --git a/selfhosted-src/auth/dtos/index.ts b/selfhosted-src/auth/dtos/index.tsnew file mode 100644index 000000000..7afca67e8--- /dev/null+++ b/selfhosted-src/auth/dtos/index.ts@@ -0,0 +1,14 @@+export * from './register.dto'+export * from './login.dto'+export * from './verify-email.dto'+export * from './request-reset-password.dto'+export * from './confirm-reset-password.dto'+export * from './reset-password.dto'+export * from './change-password.dto'+export * from './request-change-email.dto'+export * from './confirm-change-email.dto'+export * from './process-sso-code.dto'+export * from './sso-generate.dto'+export * from './sso-get-jwt-by-hash.dto'+export * from './sso-link.dto'+export * from './sso-unlink.dto'diff --git a/selfhosted-src/auth/dtos/login.dto.ts b/selfhosted-src/auth/dtos/login.dto.tsnew file mode 100644index 000000000..62ed65270--- /dev/null+++ b/selfhosted-src/auth/dtos/login.dto.ts@@ -0,0 +1,23 @@+import { ApiProperty, OmitType } from '@nestjs/swagger'+import { RegisterRequestDto } from './register.dto'++export class LoginRequestDto extends OmitType(RegisterRequestDto, [+ 'checkIfLeaked',+] as const) {}++export class LoginResponseDto {+ @ApiProperty({+ description: 'Access token (JWT)',+ })+ public readonly accessToken: string++ @ApiProperty({+ description: 'Refresh token (JWT)',+ })+ public readonly refreshToken: string++ @ApiProperty({+ description: 'User entity',+ })+ user: object+}diff --git a/selfhosted-src/auth/dtos/process-sso-code.dto.ts b/selfhosted-src/auth/dtos/process-sso-code.dto.tsnew file mode 100644index 000000000..cc05d0935--- /dev/null+++ b/selfhosted-src/auth/dtos/process-sso-code.dto.ts@@ -0,0 +1,18 @@+import { ApiProperty } from '@nestjs/swagger'+import { IsString } from 'class-validator'++export class ProcessSSOCodeDto {+ @ApiProperty({+ description: 'Google auth token',+ example: '4/0AVHE...',+ })+ @IsString()+ token: string++ @ApiProperty({+ description: 'SSO session identifier',+ example: '123e4567-e89b-12d3-a456-426614174000',+ })+ @IsString()+ hash: string+}diff --git a/selfhosted-src/auth/dtos/register.dto.ts b/selfhosted-src/auth/dtos/register.dto.tsnew file mode 100644index 000000000..05c4d0e39--- /dev/null+++ b/selfhosted-src/auth/dtos/register.dto.ts@@ -0,0 +1,54 @@+import { ApiProperty } from '@nestjs/swagger'+import {+ IsEmail,+ IsNotEmpty,+ IsBoolean,+ MaxLength,+ MinLength,+} from 'class-validator'++export class RegisterRequestDto {+ @ApiProperty({+ description: 'User email',+ example: '[email protected]',+ maxLength: 254,+ minLength: 6,+ })+ @IsEmail({}, { message: 'Please enter the valid email.' })+ public readonly email: string++ @ApiProperty({+ description: 'User password',+ example: '%d7*c4W45p',+ maxLength: 72,+ minLength: 8,+ })+ @MaxLength(50, { message: 'Max length is $constraint1 characters' })+ @MinLength(8, { message: 'Min length is $constraint1 characters' })+ public readonly password: string++ @ApiProperty({+ description: 'Check if password is leaked',+ example: true,+ })+ @IsNotEmpty({ message: 'This field is required.' })+ @IsBoolean({ message: 'Please enter the valid value (true or false).' })+ public readonly checkIfLeaked: boolean+}++export class RegisterResponseDto {+ @ApiProperty({+ description: 'Access token (JWT)',+ })+ public readonly accessToken: string++ @ApiProperty({+ description: 'Refresh token (JWT)',+ })+ public readonly refreshToken: string++ @ApiProperty({+ description: 'User entity',+ })+ user: object+}diff --git a/selfhosted-src/auth/dtos/request-change-email.dto.ts b/selfhosted-src/auth/dtos/request-change-email.dto.tsnew file mode 100644index 000000000..6c1bde5db--- /dev/null+++ b/selfhosted-src/auth/dtos/request-change-email.dto.ts@@ -0,0 +1,23 @@+import { ApiProperty } from '@nestjs/swagger'+import { IsEmail, MaxLength, MinLength } from 'class-validator'++export class RequestChangeEmailDto {+ @ApiProperty({+ description: 'User email',+ example: '[email protected]',+ maxLength: 254,+ minLength: 6,+ })+ @IsEmail({}, { message: 'Please enter the valid email.' })+ public readonly newEmail: string++ @ApiProperty({+ description: 'User password',+ example: '%d7*c4W45p',+ maxLength: 72,+ minLength: 8,+ })+ @MaxLength(50, { message: 'Max length is $constraint1 characters' })+ @MinLength(8, { message: 'Min length is $constraint1 characters' })+ public readonly password: string+}diff --git a/selfhosted-src/auth/dtos/request-reset-password.dto.ts b/selfhosted-src/auth/dtos/request-reset-password.dto.tsnew file mode 100644index 000000000..a2a09a383--- /dev/null+++ b/selfhosted-src/auth/dtos/request-reset-password.dto.ts@@ -0,0 +1,13 @@+import { ApiProperty } from '@nestjs/swagger'+import { IsEmail } from 'class-validator'++export class RequestResetPasswordDto {+ @ApiProperty({+ description: 'User email',+ example: '[email protected]',+ maxLength: 254,+ minLength: 6,+ })+ @IsEmail({}, { message: 'Please enter the valid email.' })+ public readonly email: string+}diff --git a/selfhosted-src/auth/dtos/reset-password.dto.ts b/selfhosted-src/auth/dtos/reset-password.dto.tsnew file mode 100644index 000000000..14df63e68--- /dev/null+++ b/selfhosted-src/auth/dtos/reset-password.dto.ts@@ -0,0 +1,14 @@+import { ApiProperty } from '@nestjs/swagger'+import { MaxLength, MinLength } from 'class-validator'++export class ResetPasswordDto {+ @ApiProperty({+ description: 'User password',+ example: '%d7*c4W45p',+ maxLength: 72,+ minLength: 8,+ })+ @MaxLength(50, { message: 'Max length is $constraint1 characters' })+ @MinLength(8, { message: 'Min length is $constraint1 characters' })+ public readonly newPassword: string+}diff --git a/selfhosted-src/auth/dtos/sso-generate.dto.ts b/selfhosted-src/auth/dtos/sso-generate.dto.tsnew file mode 100644index 000000000..c0e43ab99--- /dev/null+++ b/selfhosted-src/auth/dtos/sso-generate.dto.ts@@ -0,0 +1,16 @@+import { ApiProperty } from '@nestjs/swagger'+import { IsEnum } from 'class-validator'++export enum SSOProviders {+ GOOGLE = 'google',+ GITHUB = 'github',+}++export class SSOGenerateDto {+ @ApiProperty({+ description: 'SSO provider name',+ enum: SSOProviders,+ })+ @IsEnum(SSOProviders)+ provider: SSOProviders+}diff --git a/selfhosted-src/auth/dtos/sso-get-jwt-by-hash.dto.ts b/selfhosted-src/auth/dtos/sso-get-jwt-by-hash.dto.tsnew file mode 100644index 000000000..cc58875d2--- /dev/null+++ b/selfhosted-src/auth/dtos/sso-get-jwt-by-hash.dto.ts@@ -0,0 +1,20 @@+import { ApiProperty } from '@nestjs/swagger'+import { IsEnum, IsString } from 'class-validator'++import { SSOProviders } from './sso-generate.dto'++export class SSOGetJWTByHashDto {+ @ApiProperty({+ description: 'SSO session identifier',+ example: '123e4567-e89b-12d3-a456-426614174000',+ })+ @IsString()+ hash: string++ @ApiProperty({+ description: 'SSO provider name',+ enum: SSOProviders,+ })+ @IsEnum(SSOProviders)+ provider: SSOProviders+}diff --git a/selfhosted-src/auth/dtos/sso-link.dto.ts b/selfhosted-src/auth/dtos/sso-link.dto.tsnew file mode 100644index 000000000..5e244573e--- /dev/null+++ b/selfhosted-src/auth/dtos/sso-link.dto.ts@@ -0,0 +1,20 @@+import { ApiProperty } from '@nestjs/swagger'+import { IsEnum, IsString } from 'class-validator'++import { SSOProviders } from './sso-generate.dto'++export class SSOLinkDto {+ @ApiProperty({+ description: 'SSO session identifier',+ example: '123e4567-e89b-12d3-a456-426614174000',+ })+ @IsString()+ hash: string++ @ApiProperty({+ description: 'SSO provider name',+ enum: SSOProviders,+ })+ @IsEnum(SSOProviders)+ provider: SSOProviders+}diff --git a/selfhosted-src/auth/dtos/sso-unlink.dto.ts b/selfhosted-src/auth/dtos/sso-unlink.dto.tsnew file mode 100644index 000000000..f45697a9b--- /dev/null+++ b/selfhosted-src/auth/dtos/sso-unlink.dto.ts@@ -0,0 +1,13 @@+import { ApiProperty } from '@nestjs/swagger'+import { IsEnum } from 'class-validator'++import { SSOProviders } from './sso-generate.dto'++export class SSOUnlinkDto {+ @ApiProperty({+ description: 'SSO provider name',+ enum: SSOProviders,+ })+ @IsEnum(SSOProviders)+ provider: SSOProviders+}diff --git a/selfhosted-src/auth/dtos/verify-email.dto.ts b/selfhosted-src/auth/dtos/verify-email.dto.tsnew file mode 100644index 000000000..e03420979--- /dev/null+++ b/selfhosted-src/auth/dtos/verify-email.dto.ts@@ -0,0 +1,11 @@+import { ApiProperty } from '@nestjs/swagger'+import { IsUUID } from 'class-validator'++export class VerifyEmailDto {+ @ApiProperty({+ description: 'The token to verify the email',+ example: 'a1b2c3d4-e5f6-g7h8-i9j0-k1l2m3n4o5p6',+ })+ @IsUUID('4', { message: 'Please enter the valid UUID.' })+ public readonly token: string+}diff --git a/selfhosted-src/auth/guards/api-key-rate-limit.guard.ts b/selfhosted-src/auth/guards/api-key-rate-limit.guard.tsnew file mode 100644index 000000000..35f6e4fbd--- /dev/null+++ b/selfhosted-src/auth/guards/api-key-rate-limit.guard.ts@@ -0,0 +1,21 @@+import { CanActivate, ExecutionContext, Injectable } from '@nestjs/common'+import { checkRateLimitForApiKey } from 'selfhosted-src/common/utils'+import { ACCOUNT_PLANS, PlanCode } from 'selfhosted-src/user/entities/user.entity'++@Injectable()+export class ApiKeyRateLimitGuard implements CanActivate {+ async canActivate(context: ExecutionContext) {+ const request = context.switchToHttp().getRequest()+ const { user } = request++ if (request.headers['x-api-key']) {+ if (!user) return false++ const reqAmount =+ ACCOUNT_PLANS[user.planCode as PlanCode].maxApiKeyRequestsPerHour+ return checkRateLimitForApiKey(user.apiKey, reqAmount)+ }++ return true+ }+}diff --git a/selfhosted-src/auth/guards/api-key.guard.ts b/selfhosted-src/auth/guards/api-key.guard.tsnew file mode 100644index 000000000..43e58ce2d--- /dev/null+++ b/selfhosted-src/auth/guards/api-key.guard.ts@@ -0,0 +1,5 @@+import { Injectable } from '@nestjs/common'+import { AuthGuard } from '@nestjs/passport'++@Injectable()+export class ApiKeyGuard extends AuthGuard('api-key') {}diff --git a/selfhosted-src/auth/guards/index.ts b/selfhosted-src/auth/guards/index.tsnew file mode 100644index 000000000..0658afa44--- /dev/null+++ b/selfhosted-src/auth/guards/index.ts@@ -0,0 +1,7 @@+export * from './jwt-access-token.guard'+export * from './jwt-refresh-token.guard'+export * from './roles.guard'+export * from './optional-jwt-access-token.guard'+export * from './api-key.guard'+export * from './api-key-rate-limit.guard'+export * from './multi-auth.guard'diff --git a/selfhosted-src/auth/guards/jwt-access-token.guard.ts b/selfhosted-src/auth/guards/jwt-access-token.guard.tsnew file mode 100644index 000000000..acf11ae8b--- /dev/null+++ b/selfhosted-src/auth/guards/jwt-access-token.guard.ts@@ -0,0 +1,22 @@+import { Injectable, ExecutionContext } from '@nestjs/common'+import { Reflector } from '@nestjs/core'+import { AuthGuard } from '@nestjs/passport'+import { IS_PUBLIC_KEY } from '../decorators'++@Injectable()+export class JwtAccessTokenGuard extends AuthGuard('jwt-access-token') {+ constructor(private readonly reflector: Reflector) {+ super()+ }++ canActivate(context: ExecutionContext) {+ const isPublic = this.reflector.getAllAndOverride<boolean>(IS_PUBLIC_KEY, [+ context.getHandler(),+ context.getClass(),+ ])++ if (isPublic) return true++ return super.canActivate(context)+ }+}diff --git a/selfhosted-src/auth/guards/jwt-refresh-token.guard.ts b/selfhosted-src/auth/guards/jwt-refresh-token.guard.tsnew file mode 100644index 000000000..7a9074a2d--- /dev/null+++ b/selfhosted-src/auth/guards/jwt-refresh-token.guard.ts@@ -0,0 +1,7 @@+import { AuthGuard } from '@nestjs/passport'++export class JwtRefreshTokenGuard extends AuthGuard('jwt-refresh-token') {+ // constructor() {+ // super()+ // }+}diff --git a/selfhosted-src/auth/guards/multi-auth.guard.ts b/selfhosted-src/auth/guards/multi-auth.guard.tsnew file mode 100644index 000000000..96503daa0--- /dev/null+++ b/selfhosted-src/auth/guards/multi-auth.guard.ts@@ -0,0 +1,28 @@+import { ExecutionContext, Injectable } from '@nestjs/common'+import { Reflector } from '@nestjs/core'+import { AuthGuard } from '@nestjs/passport'+import { IS_OPTIONAL_AUTH_KEY } from '../decorators'++@Injectable()+export class MultiAuthGuard extends AuthGuard(['jwt-access-token', 'api-key']) {+ constructor(private readonly reflector: Reflector) {+ super()+ }++ handleRequest(+ err: any,+ user: any,+ info: any,+ context: ExecutionContext,+ status?: any,+ ) {+ const isOptionalAuth = this.reflector.getAllAndOverride<boolean>(+ IS_OPTIONAL_AUTH_KEY,+ [context.getHandler(), context.getClass()],+ )++ if (isOptionalAuth) return user++ return super.handleRequest(err, user, info, context, status)+ }+}diff --git a/selfhosted-src/auth/guards/optional-jwt-access-token.guard.ts b/selfhosted-src/auth/guards/optional-jwt-access-token.guard.tsnew file mode 100644index 000000000..f4747504e--- /dev/null+++ b/selfhosted-src/auth/guards/optional-jwt-access-token.guard.ts@@ -0,0 +1,9 @@+import { Injectable } from '@nestjs/common'+import { AuthGuard } from '@nestjs/passport'++@Injectable()+export class OptionalJwtAccessTokenGuard extends AuthGuard('jwt-access-token') {+ handleRequest(err: any, user: any) {+ return !err && user ? user : { id: null }+ }+}diff --git a/selfhosted-src/auth/guards/roles.guard.ts b/selfhosted-src/auth/guards/roles.guard.tsnew file mode 100644index 000000000..e6c608829--- /dev/null+++ b/selfhosted-src/auth/guards/roles.guard.ts@@ -0,0 +1,98 @@+import { Injectable, CanActivate, ExecutionContext } from '@nestjs/common'+import { Reflector } from '@nestjs/core'+import { ExtractJwt } from 'passport-jwt'+import { verify } from 'jsonwebtoken'++import { UserType, generateSelfhostedUser } from 'selfhosted-src/user/entities/user.entity'+import { UserService } from 'selfhosted-src/user/user.service'+import { isSelfhosted, JWT_ACCESS_TOKEN_SECRET } from 'selfhosted-src/common/constants'+import { IS_TWO_FA_NOT_REQUIRED_KEY, ROLES_KEY } from '../decorators'++@Injectable()+export class RolesGuard implements CanActivate {+ constructor(+ private readonly reflector: Reflector,+ private readonly userService: UserService,+ ) {}++ async canActivate(context: ExecutionContext) {+ const roles = this.reflector.get<UserType[]>(+ ROLES_KEY,+ context.getHandler(),+ )++ const isTwoFaNotRequired = this.reflector.get<boolean>(+ IS_TWO_FA_NOT_REQUIRED_KEY,+ context.getHandler(),+ )++ if (!roles || roles.length === 0) return true++ const request = context.switchToHttp().getRequest()+ const userFromRequest = request.user++ const user = isSelfhosted+ ? generateSelfhostedUser()+ : await this.userService.findUserById(userFromRequest.id)++ // this is a temp measure as well due to some fucking bug related to undefined user, will revert it when I find the cause+ let hasRole+ try {+ hasRole = user.roles.some(role => roles.includes(role))+ } catch (error) {+ console.error(`[AUTH ERROR - ROLES GUARD] ${error.message}`)+ console.error(user, roles)+ console.error(request?.path, request?.originalUrl, request?.method)+ }++ if (!hasRole) return false++ let token = ''+ if (request.cookies.token) {+ token = request.cookies.token+ } else {+ const extract = ExtractJwt.fromAuthHeaderAsBearerToken()+ token = extract(request)+ }++ if (isSelfhosted) {+ try {+ const decoded: any = verify(token, JWT_ACCESS_TOKEN_SECRET)++ // If the token is not decoded, it means it's invalid+ if (!decoded) {+ return false+ }+ } catch {+ return false+ }++ return true+ }++ if (isTwoFaNotRequired) {+ return true+ }++ try {+ const decoded: any = verify(token, JWT_ACCESS_TOKEN_SECRET)++ // If the token is not decoded, it means it's invalid+ if (!decoded) {+ return false+ }++ // If the user has 2FA enabled, but the token is temporary (meant to be used for 2FA routes only) then return false+ if (+ user.isTwoFactorAuthenticationEnabled &&+ !decoded.isSecondFactorAuthenticated+ ) {+ return false+ }+ } catch {+ return false+ }++ return true+ }+}diff --git a/selfhosted-src/auth/interfaces/index.ts b/selfhosted-src/auth/interfaces/index.tsnew file mode 100644index 000000000..765786217--- /dev/null+++ b/selfhosted-src/auth/interfaces/index.ts@@ -0,0 +1 @@+export * from './jwt-payload.interface'diff --git a/selfhosted-src/auth/interfaces/jwt-payload.interface.ts b/selfhosted-src/auth/interfaces/jwt-payload.interface.tsnew file mode 100644index 000000000..ed0c28c74--- /dev/null+++ b/selfhosted-src/auth/interfaces/jwt-payload.interface.ts@@ -0,0 +1,8 @@+export interface IJwtPayload {+ sub: string+ isSecondFactorAuthenticated: boolean+}++export interface IJwtRefreshTokenPayload extends IJwtPayload {+ refreshToken: string+}diff --git a/selfhosted-src/auth/strategies/api-key.strategy.ts b/selfhosted-src/auth/strategies/api-key.strategy.tsnew file mode 100644index 000000000..e87cae9a9--- /dev/null+++ b/selfhosted-src/auth/strategies/api-key.strategy.ts@@ -0,0 +1,23 @@+import { Injectable, UnauthorizedException } from '@nestjs/common'+import { PassportStrategy } from '@nestjs/passport'+import { HeaderAPIKeyStrategy } from 'passport-headerapikey'+import { AuthService } from '../auth.service'++@Injectable()+export class ApiKeyStrategy extends PassportStrategy(+ HeaderAPIKeyStrategy,+ 'api-key',+) {+ constructor(private readonly authService: AuthService) {+ super(+ { header: 'X-Api-Key', prefix: '' },+ true,+ // eslint-disable-next-line consistent-return+ async (apiKey: string, done: any) => {+ const user = await this.authService.validateApiKey(apiKey)+ if (!user) return done(new UnauthorizedException(), false)+ done(null, user)+ },+ )+ }+}diff --git a/selfhosted-src/auth/strategies/index.ts b/selfhosted-src/auth/strategies/index.tsnew file mode 100644index 000000000..f544f42d4--- /dev/null+++ b/selfhosted-src/auth/strategies/index.ts@@ -0,0 +1,3 @@+export * from './jwt-access-token.strategy'+export * from './jwt-refresh-token.strategy'+export * from './api-key.strategy'diff --git a/selfhosted-src/auth/strategies/jwt-access-token.strategy.ts b/selfhosted-src/auth/strategies/jwt-access-token.strategy.tsnew file mode 100644index 000000000..ede9ce01a--- /dev/null+++ b/selfhosted-src/auth/strategies/jwt-access-token.strategy.ts@@ -0,0 +1,35 @@+import { Injectable, UnauthorizedException } from '@nestjs/common'+import { ConfigService } from '@nestjs/config'+import { PassportStrategy } from '@nestjs/passport'+import { Strategy, ExtractJwt } from 'passport-jwt'++import {+ SELFHOSTED_UUID,+ JWT_ACCESS_TOKEN_SECRET,+} from 'selfhosted-src/common/constants'+import { generateSelfhostedUser } from 'selfhosted-src/user/entities/user.entity'+import { IJwtPayload } from '../interfaces'++@Injectable()+export class JwtAccessTokenStrategy extends PassportStrategy(+ Strategy,+ 'jwt-access-token',+) {+ constructor(+ public readonly configService: ConfigService,+ ) {+ super({+ jwtFromRequest: ExtractJwt.fromAuthHeaderAsBearerToken(),+ ignoreExpiration: false,+ secretOrKey: JWT_ACCESS_TOKEN_SECRET,+ })+ }++ public async validate(payload: IJwtPayload) {+ if (payload.sub !== SELFHOSTED_UUID) {+ throw new UnauthorizedException()+ }++ return generateSelfhostedUser()+ }+}diff --git a/selfhosted-src/auth/strategies/jwt-refresh-token.strategy.ts b/selfhosted-src/auth/strategies/jwt-refresh-token.strategy.tsnew file mode 100644index 000000000..ecb72b03b--- /dev/null+++ b/selfhosted-src/auth/strategies/jwt-refresh-token.strategy.ts@@ -0,0 +1,28 @@+import { ForbiddenException, Injectable } from '@nestjs/common'+import { ConfigService } from '@nestjs/config'+import { PassportStrategy } from '@nestjs/passport'+import { Request } from 'express'+import { ExtractJwt, Strategy } from 'passport-jwt'+import { IJwtPayload } from '../interfaces'++@Injectable()+export class JwtRefreshTokenStrategy extends PassportStrategy(+ Strategy,+ 'jwt-refresh-token',+) {+ constructor(public readonly configService: ConfigService) {+ super({+ jwtFromRequest: ExtractJwt.fromAuthHeaderAsBearerToken(),+ secretOrKey: configService.get('JWT_REFRESH_TOKEN_SECRET'),+ passReqToCallback: true,+ })+ }++ public validate(request: Request, payload: IJwtPayload) {+ const refreshToken = request.headers.authorization.split(' ')[1]++ if (!refreshToken) throw new ForbiddenException()++ return { ...payload, refreshToken }+ }+}diff --git a/selfhosted-src/common/constants.ts b/selfhosted-src/common/constants.tsnew file mode 100644index 000000000..fb60c0988--- /dev/null+++ b/selfhosted-src/common/constants.ts@@ -0,0 +1,155 @@+import { ClickHouse } from 'clickhouse'+import Redis from 'ioredis'+import { hash } from 'blake3'+import * as _toNumber from 'lodash/toNumber'++import { getSelfhostedUUID } from './utils'++// eslint-disable-next-line @typescript-eslint/no-var-requires+require('dotenv').config()++const { CLICKHOUSE_DATABASE } = process.env++const redis = new Redis(+ _toNumber(process.env.REDIS_PORT),+ process.env.REDIS_HOST,+ {+ password: process.env.REDIS_PASSWORD,+ username: process.env.REDIS_USER,+ },+)++redis.defineCommand('countKeysByPattern', {+ numberOfKeys: 0,+ lua: "return #redis.call('keys', ARGV[1])",+})++const clickhouse = new ClickHouse({+ url: process.env.CLICKHOUSE_HOST,+ port: _toNumber(process.env.CLICKHOUSE_PORT),+ debug: false,+ basicAuth: {+ username: process.env.CLICKHOUSE_USER,+ password: process.env.CLICKHOUSE_PASSWORD,+ },+ isUseGzip: false,+ format: 'json',+ raw: false,+ config: {+ session_timeout: 60,+ output_format_json_quote_64bit_integers: 0,+ enable_http_compression: 0,+ database: CLICKHOUSE_DATABASE,+ log_queries: 0,+ },+})++const { JWT_ACCESS_TOKEN_SECRET } = process.env+const isSelfhosted = Boolean(process.env.SELFHOSTED)+const isNewRelicEnabled = Boolean(process.env.USE_NEW_RELIC)+const isDevelopment = process.env.NODE_ENV === 'development'+const PRODUCTION_ORIGIN = process.env.CLIENT_URL || 'https://swetrix.com'++const SELFHOSTED_EMAIL = process.env.EMAIL+const SELFHOSTED_PASSWORD = process.env.PASSWORD+const UUIDV5_NAMESPACE = '912c64c1-73fd-42b6-859f-785f839a9f68'+const DEFAULT_SELFHOSTED_UUID = 'deadbeef-dead-beef-dead-beefdeadbeef'++const SELFHOSTED_UUID = isSelfhosted ? getSelfhostedUUID() : ''++const { TWO_FACTOR_AUTHENTICATION_APP_NAME } = process.env++const ORIGINS_REGEX =+ /^(?=.{1,255}$)[0-9A-Za-z:](?:(?:[0-9A-Za-z:]|-){0,61}[0-9A-Za-z])?(?:\.[0-9A-Za-z:](?:(?:[0-9A-Za-z]|-){0,61}[0-9A-Za-z])?)*\.?$/+const IP_REGEX =+ /^(([12]?[0-9]{1,2}|2[0-4][0-9]|25[0-5])(\.|\/)){4}([1-2]?[0-9]|3[0-2])$/+const PID_REGEX = /^(?!.*--)[a-zA-Z0-9-]{12}$/+const isValidPID = (pid: string) => PID_REGEX.test(pid)++// redis keys+const getRedisProjectKey = (pid: string) => `pid_${pid}`+const getRedisUserCountKey = (uid: string) => `user_c_${uid}`+const getRedisCaptchaKey = (token: string) => `captcha_${hash(token)}`++const REDIS_LOG_DATA_CACHE_KEY = 'log_cache'+const REDIS_LOG_CAPTCHA_CACHE_KEY = 'log:captcha'+const REDIS_LOG_PERF_CACHE_KEY = 'perf_cache'+const REDIS_LOG_CUSTOM_CACHE_KEY = 'log_custom_cache_v2'+const REDIS_SESSION_SALT_KEY = 'log_salt' // is updated every 24 hours+const REDIS_USERS_COUNT_KEY = 'stats:users_count'+const REDIS_PROJECTS_COUNT_KEY = 'stats:projects_count'+const REDIS_PAGEVIEWS_COUNT_KEY = 'stats:pageviews'+const REDIS_PERFORMANCE_COUNT_KEY = 'stats:performance'+const REDIS_SSO_UUID = 'sso:uuid'++// Captcha service+const { CAPTCHA_SALT } = process.env+const { CAPTCHA_ENCRYPTION_KEY } = process.env++// 3600 sec -> 1 hour+const redisProjectCacheTimeout = 3600++// 15 minutes+const redisProjectCountCacheTimeout = 900++// 30 minues -> the amount of time analytics requests within one session are counted as non-unique+const UNIQUE_SESSION_LIFE_TIME = 1800++// 35 seconds+const HEARTBEAT_SID_LIFE_TIME = 35++// how often can user request a fresh GDPR export of their data; in days.+const GDPR_EXPORT_TIMEFRAME = 14++// send email warning when 85% of events in tier are used+const SEND_WARNING_AT_PERC = 85++const PROJECT_INVITE_EXPIRE = 48++const CAPTCHA_COOKIE_KEY = 'swetrix-captcha-token'+const CAPTCHA_TOKEN_LIFETIME = 300 // seconds (5 minutes).+const CAPTCHA_SECRET_KEY_LENGTH = 50++export {+ clickhouse,+ redis,+ isValidPID,+ getRedisProjectKey,+ redisProjectCacheTimeout,+ UNIQUE_SESSION_LIFE_TIME,+ REDIS_LOG_DATA_CACHE_KEY,+ REDIS_LOG_CAPTCHA_CACHE_KEY,+ GDPR_EXPORT_TIMEFRAME,+ getRedisUserCountKey,+ redisProjectCountCacheTimeout,+ REDIS_LOG_CUSTOM_CACHE_KEY,+ REDIS_SESSION_SALT_KEY,+ HEARTBEAT_SID_LIFE_TIME,+ isSelfhosted,+ UUIDV5_NAMESPACE,+ SELFHOSTED_EMAIL,+ SELFHOSTED_PASSWORD,+ SELFHOSTED_UUID,+ REDIS_USERS_COUNT_KEY,+ REDIS_PROJECTS_COUNT_KEY,+ REDIS_PAGEVIEWS_COUNT_KEY,+ SEND_WARNING_AT_PERC,+ PROJECT_INVITE_EXPIRE,+ TWO_FACTOR_AUTHENTICATION_APP_NAME,+ IP_REGEX,+ isNewRelicEnabled,+ ORIGINS_REGEX,+ REDIS_LOG_PERF_CACHE_KEY,+ REDIS_PERFORMANCE_COUNT_KEY,+ CAPTCHA_SALT,+ CAPTCHA_ENCRYPTION_KEY,+ isDevelopment,+ getRedisCaptchaKey,+ CAPTCHA_COOKIE_KEY,+ CAPTCHA_TOKEN_LIFETIME,+ CAPTCHA_SECRET_KEY_LENGTH,+ PRODUCTION_ORIGIN,+ DEFAULT_SELFHOSTED_UUID,+ REDIS_SSO_UUID,+ JWT_ACCESS_TOKEN_SECRET,+}diff --git a/selfhosted-src/common/decorators/bot-detection.decorator.ts b/selfhosted-src/common/decorators/bot-detection.decorator.tsnew file mode 100644index 000000000..f3be8c233--- /dev/null+++ b/selfhosted-src/common/decorators/bot-detection.decorator.ts@@ -0,0 +1,4 @@+import { SetMetadata } from '@nestjs/common'++export const IS_BOT_DETECTION_ENABLED = 'isBotDetectionEnabled'+export const BotDetection = () => SetMetadata(IS_BOT_DETECTION_ENABLED, true)diff --git a/selfhosted-src/common/guards/bot-detection.guard.ts b/selfhosted-src/common/guards/bot-detection.guard.tsnew file mode 100644index 000000000..90a940134--- /dev/null+++ b/selfhosted-src/common/guards/bot-detection.guard.ts@@ -0,0 +1,27 @@+import {+ CanActivate,+ ExecutionContext,+ ForbiddenException,+ Injectable,+} from '@nestjs/common'+import { Reflector } from '@nestjs/core'+import * as isbot from 'isbot'+import { IS_BOT_DETECTION_ENABLED } from '../decorators/bot-detection.decorator'++@Injectable()+export class BotDetectionGuard implements CanActivate {+ constructor(private readonly reflector: Reflector) {}++ canActivate(context: ExecutionContext): boolean {+ this.reflector.get(IS_BOT_DETECTION_ENABLED, context.getHandler())++ const { headers } = context.switchToHttp().getRequest()+ const { 'user-agent': userAgent } = headers++ if (isbot(userAgent)) {+ throw new ForbiddenException('Bot traffic is ignored')+ }++ return true+ }+}diff --git a/selfhosted-src/common/pagination/index.ts b/selfhosted-src/common/pagination/index.tsnew file mode 100644index 000000000..f276090bf--- /dev/null+++ b/selfhosted-src/common/pagination/index.ts@@ -0,0 +1,3 @@+export * from './pagination.options.interface'+export * from './pagination.results.interface'+export * from './pagination'diff --git a/selfhosted-src/common/pagination/pagination.options.interface.ts b/selfhosted-src/common/pagination/pagination.options.interface.tsnew file mode 100644index 000000000..37378118e--- /dev/null+++ b/selfhosted-src/common/pagination/pagination.options.interface.ts@@ -0,0 +1,4 @@+export interface PaginationOptionsInterface {+ take: number+ skip: number+}diff --git a/selfhosted-src/common/pagination/pagination.results.interface.ts b/selfhosted-src/common/pagination/pagination.results.interface.tsnew file mode 100644index 000000000..fc38ee98a--- /dev/null+++ b/selfhosted-src/common/pagination/pagination.results.interface.ts@@ -0,0 +1,6 @@+export interface PaginationResultInterface<PaginationEntity> {+ results: PaginationEntity[]+ total: number+ next?: string+ previous?: string+}diff --git a/selfhosted-src/common/pagination/pagination.ts b/selfhosted-src/common/pagination/pagination.tsnew file mode 100644index 000000000..59e8287ef--- /dev/null+++ b/selfhosted-src/common/pagination/pagination.ts@@ -0,0 +1,15 @@+import { PaginationResultInterface } from './pagination.results.interface'++export class Pagination<PaginationEntity> {+ public results: PaginationEntity[]++ public page_total: number++ public total: number++ constructor(paginationResults: PaginationResultInterface<PaginationEntity>) {+ this.results = paginationResults.results+ this.page_total = paginationResults.results.length+ this.total = paginationResults.total+ }+}diff --git a/selfhosted-src/common/utils.ts b/selfhosted-src/common/utils.tsnew file mode 100644index 000000000..7ce4eda7d--- /dev/null+++ b/selfhosted-src/common/utils.ts@@ -0,0 +1,240 @@+import { NotFoundException, HttpException } from '@nestjs/common'+import { hash } from 'blake3'+import { v5 as uuidv5 } from 'uuid'+import * as randomstring from 'randomstring'+import * as _sample from 'lodash/sample'+import * as _join from 'lodash/join'+import * as _filter from 'lodash/filter'+import * as _values from 'lodash/values'+import * as _reduce from 'lodash/reduce'+import * as _keys from 'lodash/keys'+import * as _toNumber from 'lodash/toNumber'+import * as _isEmpty from 'lodash/isEmpty'+import * as _head from 'lodash/head'+import * as _round from 'lodash/round'+import * as dayjs from 'dayjs'+import * as utc from 'dayjs/plugin/utc'+import * as _map from 'lodash/map'++import {+ clickhouse,+ redis,+ DEFAULT_SELFHOSTED_UUID,+ SELFHOSTED_EMAIL,+ UUIDV5_NAMESPACE,+ isDevelopment,+} from './constants'+import { Project } from '../project/entity/project.entity'++dayjs.extend(utc)++const RATE_LIMIT_REQUESTS_AMOUNT = 3+const RATE_LIMIT_TIMEOUT = 86400 // 24 hours++const allowedToUpdateKeys = [+ 'name',+ 'origins',+ 'ipBlacklist',+ 'active',+ 'public',+]++const getRateLimitHash = (ipOrApiKey: string, salt = '') =>+ `rl:${hash(`${ipOrApiKey}${salt}`).toString('hex')}`++// 'action' is used as a salt to differ rate limiting routes+const checkRateLimit = async (+ ip: string,+ action: string,+ reqAmount: number = RATE_LIMIT_REQUESTS_AMOUNT,+ reqTimeout: number = RATE_LIMIT_TIMEOUT,+): Promise<void> => {+ if (isDevelopment) {+ return+ }++ const rlHash = getRateLimitHash(ip, action)+ const rlCount: number = _toNumber(await redis.get(rlHash)) || 0++ if (rlCount >= reqAmount) {+ throw new HttpException('Too many requests, please try again later', 429)+ }+ await redis.set(rlHash, 1 + rlCount, 'EX', reqTimeout)+}++const RATE_LIMIT_FOR_API_KEY_TIMEOUT = 60 * 60 // 1 hour+export const checkRateLimitForApiKey = async (+ apiKey: string,+ reqAmount: number,+): Promise<boolean> => {+ const rlHash = getRateLimitHash(apiKey)+ const rlCount: number = _toNumber(await redis.get(rlHash)) || 0++ if (rlCount >= reqAmount) {+ throw new HttpException('Too many requests, please try again later', 429)+ }+ await redis.set(rlHash, 1 + rlCount, 'EX', RATE_LIMIT_FOR_API_KEY_TIMEOUT)+ return true+}++const getProjectsClickhouse = async (id = null) => {+ if (!id) {+ const query = 'SELECT * FROM project;'+ return clickhouse.query(query).toPromise()+ }++ const paramsData = {+ params: {+ id,+ },+ }++ const query = `SELECT * FROM project WHERE id = {id:FixedString(12)};`+ const project = await clickhouse.query(query, paramsData).toPromise()++ if (_isEmpty(project)) {+ throw new NotFoundException(`Project ${id} was not found in the database`)+ }++ return _head(project)+}++const updateProjectClickhouse = async (project: object) => {+ const filtered = _reduce(+ _filter(_keys(project), key => allowedToUpdateKeys.includes(key)),+ (obj, key) => {+ obj[key] = project[key]+ return obj+ },+ {},+ )+ const columns = _keys(filtered)+ const values = _values(filtered)+ // @ts-ignore+ const query = `ALTER table project UPDATE ${_join(+ _map(columns, (col, id) => `${col}='${values[id]}'`),+ ', ',+ // @ts-ignore+ )} WHERE id='${project.id}'`+ return clickhouse.query(query).toPromise()+}++/**+ * Checking the % change in one number relative to the other+ * @param oldVal The initial value+ * @param newVal The value that changed+ * @param round Numbers after floating point+ */+const calculateRelativePercentage = (+ oldVal: number,+ newVal: number,+ round = 2,+) => {+ if (oldVal === newVal) return 0+ if (oldVal === 0) return 100+ if (newVal === 0) return -100++ if (newVal > oldVal) {+ return _round((newVal / oldVal) * 100, round)+ }++ return _round((1 - newVal / oldVal) * -100, round)+}++const deleteProjectClickhouse = async id => {+ const query = `ALTER table project DELETE WHERE id='${id}'`+ return clickhouse.query(query).toPromise()+}++const createProjectClickhouse = async (project: Project) => {+ const paramsData = {+ params: {+ ...project,+ },+ }+ const query = `INSERT INTO project (*) VALUES ({id:FixedString(12)},{name:String},'', '',1,0,'${dayjs+ .utc()+ .format('YYYY-MM-DD HH:mm:ss')}')`+ return clickhouse.query(query, paramsData).toPromise()+}++const saveRefreshTokenClickhouse = async (+ userId: string,+ refreshToken: string,+) => {+ const paramsData = {+ params: {+ userId,+ refreshToken,+ },+ }+ const query =+ 'INSERT INTO refresh_token (*) VALUES ({userId:String},{refreshToken:String})'++ return clickhouse.query(query, paramsData).toPromise()+}++const findRefreshTokenClickhouse = async (+ userId: string,+ refreshToken: string,+) => {+ const paramsData = {+ params: {+ userId,+ refreshToken,+ },+ }+ const query =+ 'SELECT * FROM refresh_token WHERE userId = {userId:String} AND refreshToken = {refreshToken:String}'+ return clickhouse.query(query, paramsData).toPromise()+}++const deleteRefreshTokenClickhouse = async (+ userId: string,+ refreshToken: string,+) => {+ const paramsData = {+ params: {+ userId,+ refreshToken,+ },+ }+ const query = `ALTER table refresh_token DELETE WHERE userId = {userId:String} AND refreshToken = {refreshToken:String}`+ return clickhouse.query(query, paramsData).toPromise()+}++const generateRecoveryCode = () =>+ randomstring.generate({+ length: 30,+ charset: 'alphabetic',+ capitalization: 'uppercase',+ })++const millisecondsToSeconds = (milliseconds: number) => milliseconds / 1000++const generateRandomString = (length: number): string =>+ randomstring.generate(length)++const getSelfhostedUUID = (): string => {+ try {+ return uuidv5(SELFHOSTED_EMAIL, UUIDV5_NAMESPACE)+ } catch {+ return DEFAULT_SELFHOSTED_UUID+ }+}++export {+ checkRateLimit,+ createProjectClickhouse,+ getProjectsClickhouse,+ updateProjectClickhouse,+ deleteProjectClickhouse,+ generateRecoveryCode,+ calculateRelativePercentage,+ millisecondsToSeconds,+ generateRandomString,+ getSelfhostedUUID,+ saveRefreshTokenClickhouse,+ findRefreshTokenClickhouse,+ deleteRefreshTokenClickhouse,+}diff --git a/selfhosted-src/configs/i18n.config.ts b/selfhosted-src/configs/i18n.config.tsnew file mode 100644index 000000000..0288616c1--- /dev/null+++ b/selfhosted-src/configs/i18n.config.ts@@ -0,0 +1,16 @@+import { ConfigModule, ConfigService } from '@nestjs/config'+import { AcceptLanguageResolver, I18nAsyncOptions } from 'nestjs-i18n'+import { join } from 'path'++export const getI18nConfig = (): I18nAsyncOptions => ({+ imports: [ConfigModule],+ inject: [ConfigService],+ resolvers: [AcceptLanguageResolver],+ useFactory: (configService: ConfigService) => ({+ fallbackLanguage: 'en',+ loaderOptions: {+ path: join(__dirname, '..', 'i18n'),+ watch: configService.get('NODE_ENV') === 'development',+ },+ }),+})diff --git a/selfhosted-src/configs/index.ts b/selfhosted-src/configs/index.tsnew file mode 100644index 000000000..3113e360d--- /dev/null+++ b/selfhosted-src/configs/index.ts@@ -0,0 +1 @@+export * from './i18n.config'diff --git a/selfhosted-src/i18n/en/auth.json b/selfhosted-src/i18n/en/auth.jsonnew file mode 100644index 000000000..5f090f3b4--- /dev/null+++ b/selfhosted-src/i18n/en/auth.json@@ -0,0 +1,13 @@+{+ "leakedPassword": "The password is in the leak database.",+ "invalidCredentials": "Incorrect email or password.",+ "invalidVerificationToken": "Invalid verification token.",+ "accountNotExists": "An account with this email does not exist.",+ "invalidResetPasswordToken": "Invalid reset password token.",+ "invalidPassword": "Invalid password.",+ "emailAlreadyVerified": "Email already verified.",+ "maxEmailRequestsReached": "Maximum number of email requests reached.",+ "emailAlreadyTaken": "Email already taken.",+ "invalidChangeEmailToken": "Invalid change email token.",+ "invalidRefreshToken": "Invalid refresh token."+}diff --git a/selfhosted-src/i18n/en/user.json b/selfhosted-src/i18n/en/user.jsonnew file mode 100644index 000000000..b8acbdf29--- /dev/null+++ b/selfhosted-src/i18n/en/user.json@@ -0,0 +1,3 @@+{+ "emailAlreadyUsed": "Email already used."+}diff --git a/selfhosted-src/i18n/en/validation.json b/selfhosted-src/i18n/en/validation.jsonnew file mode 100644index 000000000..097d29c02--- /dev/null+++ b/selfhosted-src/i18n/en/validation.json@@ -0,0 +1,7 @@+{+ "isEmail": "Please enter the valid email.",+ "isPassword": "Please enter the valid password.",+ "isNotEmpty": "This field is required.",+ "isBoolean": "Please enter the valid value (true or false).",+ "isUuid": "Please enter the valid UUID."+}diff --git a/selfhosted-src/logger/logger.module.ts b/selfhosted-src/logger/logger.module.tsnew file mode 100644index 000000000..300c623a3--- /dev/null+++ b/selfhosted-src/logger/logger.module.ts@@ -0,0 +1,8 @@+import { Module } from '@nestjs/common'+import { AppLoggerService } from './logger.service'++@Module({+ providers: [AppLoggerService],+ exports: [AppLoggerService],+})+export class AppLoggerModule {}diff --git a/selfhosted-src/logger/logger.service.ts b/selfhosted-src/logger/logger.service.tsnew file mode 100644index 000000000..cd4939346--- /dev/null+++ b/selfhosted-src/logger/logger.service.ts@@ -0,0 +1,11 @@+import { Logger } from '@nestjs/common'++import { isDevelopment } from '../common/constants'++export class AppLoggerService extends Logger {+ log(value, route, forceLog = false) {+ if (isDevelopment || forceLog) {+ Logger.log(value, route)+ }+ }+}diff --git a/selfhosted-src/main.ts b/selfhosted-src/main.tsnew file mode 100644index 000000000..60c48ebcb--- /dev/null+++ b/selfhosted-src/main.ts@@ -0,0 +1,56 @@+import { NestFactory } from '@nestjs/core'+import { ValidationPipe, VersioningType } from '@nestjs/common'+import * as cookieParser from 'cookie-parser'+import { SwaggerModule, DocumentBuilder } from '@nestjs/swagger'++import { isDevelopment } from './common/constants'+import { AppModule } from './app.module'++async function bootstrap() {+ const app = await NestFactory.create(AppModule)+ app.use(cookieParser())+ app.useGlobalPipes(new ValidationPipe())++ app.enableVersioning({+ type: VersioningType.URI,+ })++ if (isDevelopment) {+ const config = new DocumentBuilder()+ .setTitle('Swetrix API')+ .setDescription('Swetrix Selfhosted API')+ .setVersion(process.env.npm_package_version)+ .build()+ const document = SwaggerModule.createDocument(app, config)+ SwaggerModule.setup('api', app, document)+ }++ // eslint-disable-next-line consistent-return+ app.use(async (req, res, next) => {+ res.header(+ 'Cross-Origin-Embedder-Policy',+ "require-corp; report-to='default'",+ )+ res.header('Cross-Origin-Opener-Policy', "same-site; report-to='default'")+ res.header('Cross-Origin-Resource-Policy', 'same-site')+ res.header('Permissions-Policy', 'interest-cohort=()')+ res.header('Referrer-Policy', 'strict-origin-when-cross-origin')+ res.header('X-Frame-Options', 'DENY')+ res.header('X-Powered-By', 'Mountain Dew')+ res.header('X-XSS-Protection', '1; mode=block')+ res.header('Access-Control-Allow-Origin', process.env.API_ORIGINS || '*')+ res.header('Access-Control-Allow-Methods', 'GET,POST,PUT,DELETE,PATCH')+ res.header('Access-Control-Allow-Headers', '*')++ if (req.method === 'OPTIONS') {+ // TODO: INVESTIGATE+ // eslint-disable-next-line @typescript-eslint/return-await+ return await res.sendStatus(204)+ }++ next()+ })++ await app.listen(5005)+}+bootstrap()diff --git a/selfhosted-src/ping/ping.controller.ts b/selfhosted-src/ping/ping.controller.tsnew file mode 100644index 000000000..76ea5f035--- /dev/null+++ b/selfhosted-src/ping/ping.controller.ts@@ -0,0 +1,9 @@+import { Controller, Get } from '@nestjs/common'+import { ApiTags } from '@nestjs/swagger'++@ApiTags('Ping')+@Controller('ping')+export class PingController {+ @Get('/')+ get(): any {}+}diff --git a/selfhosted-src/ping/ping.module.ts b/selfhosted-src/ping/ping.module.tsnew file mode 100644index 000000000..19a1f77e9--- /dev/null+++ b/selfhosted-src/ping/ping.module.ts@@ -0,0 +1,8 @@+import { Module } from '@nestjs/common'++import { PingController } from './ping.controller'++@Module({+ controllers: [PingController],+})+export class PingModule {}diff --git a/selfhosted-src/project/dto/project.dto.ts b/selfhosted-src/project/dto/project.dto.tsnew file mode 100644index 000000000..38f36e056--- /dev/null+++ b/selfhosted-src/project/dto/project.dto.ts@@ -0,0 +1,56 @@+import { ApiProperty } from '@nestjs/swagger'+import { IsNotEmpty, Length } from 'class-validator'++export class ProjectDTO {+ @ApiProperty({+ example: 'Your awesome project',+ required: true,+ description: 'A display name for your project',+ })+ @IsNotEmpty()+ @Length(1, 50)+ name: string++ @ApiProperty({+ example: 'aUn1quEid-3',+ required: true,+ description: 'A unique project ID',+ })+ @IsNotEmpty()+ id: string++ @ApiProperty({+ example: 'localhost:3000,example.com',+ required: false,+ description: 'An array allowed origins',+ })+ origins: string[] | null++ @ApiProperty({+ example: '::1,127.0.0.1,192.168.0.1/32',+ required: false,+ description: 'Array of blocked IP addresses',+ })+ ipBlacklist: string[] | null++ @ApiProperty({+ required: false,+ description:+ "The project's state. If enabled - all the incoming analytics data will be saved.",+ })+ active: boolean++ @ApiProperty({+ required: false,+ description:+ "When true, anyone on the internet (including Google) would be able to see the project's Dashboard.",+ })+ public: boolean++ @ApiProperty({+ required: false,+ description:+ 'When true, created a new Captcha Project without Analytics Projects.',+ })+ isCaptcha: boolean+}diff --git a/selfhosted-src/project/entity/index.ts b/selfhosted-src/project/entity/index.tsnew file mode 100644index 000000000..558362f22--- /dev/null+++ b/selfhosted-src/project/entity/index.ts@@ -0,0 +1 @@+export * from './project.entity'diff --git a/selfhosted-src/project/entity/project.entity.ts b/selfhosted-src/project/entity/project.entity.tsnew file mode 100644index 000000000..8fb9117aa--- /dev/null+++ b/selfhosted-src/project/entity/project.entity.ts@@ -0,0 +1,9 @@+export class Project {+ id: string+ name: string+ origins: string[]+ ipBlacklist: string[]+ active: boolean+ public: boolean+ created: Date+}diff --git a/selfhosted-src/project/project.controller.ts b/selfhosted-src/project/project.controller.tsnew file mode 100644index 000000000..fa3f3cfc7--- /dev/null+++ b/selfhosted-src/project/project.controller.ts@@ -0,0 +1,289 @@+import {+ Controller,+ Body,+ Query,+ Param,+ UseGuards,+ Get,+ Post,+ Put,+ Delete,+ BadRequestException,+ HttpCode,+ NotFoundException,+} from '@nestjs/common'+import { ApiTags, ApiQuery, ApiResponse } from '@nestjs/swagger'+import * as _isEmpty from 'lodash/isEmpty'+import * as _map from 'lodash/map'+import * as _trim from 'lodash/trim'+import * as _size from 'lodash/size'+import * as _split from 'lodash/split'+import * as _head from 'lodash/head'+import * as dayjs from 'dayjs'++import { JwtAccessTokenGuard } from 'selfhosted-src/auth/guards'+import { Auth } from 'selfhosted-src/auth/decorators'+import { isValidDate } from 'selfhosted-src/analytics/analytics.service'+import {+ ProjectService,+ deleteProjectRedis,+} from './project.service'+import { UserType } from '../user/entities/user.entity'+import { Roles } from '../auth/decorators/roles.decorator'+import { RolesGuard } from '../auth/guards/roles.guard'+import { Pagination } from '../common/pagination/pagination'+import { Project } from './entity/project.entity'+import { CurrentUserId } from '../auth/decorators/current-user-id.decorator'+import { ProjectDTO } from './dto/project.dto'+import { AppLoggerService } from '../logger/logger.service'+import {+ isValidPID,+ clickhouse,+} from '../common/constants'+import {+ getProjectsClickhouse,+ createProjectClickhouse,+ updateProjectClickhouse,+ deleteProjectClickhouse,+} from '../common/utils'++@ApiTags('Project')+@Controller('project')+export class ProjectController {+ constructor(+ private readonly projectService: ProjectService,+ private readonly logger: AppLoggerService,+ ) {}++ @Get('/')+ @ApiQuery({ name: 'take', required: false })+ @ApiQuery({ name: 'skip', required: false })+ @ApiQuery({ name: 'isCaptcha', required: false, type: Boolean })+ @ApiQuery({ name: 'relatedonly', required: false, type: Boolean })+ @ApiResponse({ status: 200, type: [Project] })+ @Auth([UserType.CUSTOMER, UserType.ADMIN], true)+ async get(+ @CurrentUserId() userId: string,+ @Query('take') take: number | undefined,+ @Query('skip') skip: number | undefined,+ ): Promise<Pagination<Project> | Project[] | object> {+ this.logger.log({ userId, take, skip }, 'GET /project')++ const results = await getProjectsClickhouse()+ const formatted = _map(results, this.projectService.formatFromClickhouse)+ return {+ results: formatted,+ page_total: _size(formatted),+ total: _size(formatted),+ totalMonthlyEvents: 0, // not needed as it's selfhosed+ }+ }++ @Get('/names')+ @ApiResponse({ status: 200, type: [Project] })+ @Auth([UserType.CUSTOMER, UserType.ADMIN], true)+ async getNames(@CurrentUserId() userId: string): Promise<Project[]> {+ this.logger.log({ userId }, 'GET /project/names')++ const results = await getProjectsClickhouse()+ const formatted = _map(results, this.projectService.formatFromClickhouse)+ return formatted+ }++ @Post('/')+ @ApiResponse({ status: 201, type: Project })+ @UseGuards(JwtAccessTokenGuard, RolesGuard)+ @Roles(UserType.CUSTOMER, UserType.ADMIN)+ async create(+ @Body() projectDTO: ProjectDTO,+ @CurrentUserId() userId: string,+ ): Promise<Project> {+ this.logger.log({ projectDTO, userId }, 'POST /project')++ const projects = await getProjectsClickhouse()++ this.projectService.validateProject(projectDTO)+ this.projectService.checkIfIDUniqueClickhouse(projects, projectDTO.id)++ const project = new Project()+ Object.assign(project, projectDTO)+ project.origins = _map(projectDTO.origins, _trim)+ project.ipBlacklist = _map(projectDTO.ipBlacklist, _trim)+ project.active = true++ await createProjectClickhouse(project)++ return project+ }++ @Delete('/reset/:id')+ @HttpCode(204)+ @UseGuards(JwtAccessTokenGuard, RolesGuard)+ @Roles(UserType.CUSTOMER, UserType.ADMIN)+ @ApiResponse({ status: 204, description: 'Empty body' })+ async reset(+ @Param('id') id: string,+ @CurrentUserId() uid: string,+ ): Promise<any> {+ this.logger.log({ uid, id }, 'DELETE /project/reset/:id')+ if (!isValidPID(id)) {+ throw new BadRequestException(+ 'The provided Project ID (pid) is incorrect',+ )+ }++ const query1 = `ALTER table analytics DELETE WHERE pid='${id}'`+ const query2 = `ALTER table customEV DELETE WHERE pid='${id}'`++ try {+ await clickhouse.query(query1).toPromise()+ await clickhouse.query(query2).toPromise()+ return 'Project resetted successfully'+ } catch (e) {+ this.logger.error(e)+ return 'Error while resetting your project'+ }+ }++ @Delete('/partially/:pid')+ @ApiQuery({+ name: 'from',+ required: true,+ description: 'Date in ISO format',+ example: '2020-01-01T00:00:00.000Z',+ type: 'string',+ })+ @ApiQuery({+ name: 'to',+ required: true,+ description: 'Date in ISO format',+ example: '2020-01-01T00:00:00.000Z',+ type: 'string',+ })+ @ApiResponse({ status: 200 })+ @UseGuards(JwtAccessTokenGuard, RolesGuard)+ @Auth([UserType.ADMIN, UserType.CUSTOMER])+ async deletePartially(+ @Param('pid') pid: string,+ @Query('from') from: string,+ @Query('to') to: string,+ @CurrentUserId() uid: string,+ ): Promise<void> {+ this.logger.log({ from, to, pid }, 'DELETE /partially/:id')++ if (!isValidPID(pid)) {+ throw new BadRequestException(+ 'The provided Project ID (pid) is incorrect',+ )+ }++ from = _head(_split(from, 'T'))+ to = _head(_split(to, 'T'))++ if (!isValidDate(from)) {+ throw new BadRequestException("The provided 'from' date is incorrect")+ }++ if (!isValidDate(to)) {+ throw new BadRequestException("The provided 'to' date is incorrect")+ }++ // Checking if project exists+ getProjectsClickhouse(pid)++ from = dayjs(from).format('YYYY-MM-DD')+ to = dayjs(to).format('YYYY-MM-DD 23:59:59')++ await this.projectService.removeDataFromClickhouse(pid, from, to)+ }++ @Delete('/:id')+ @HttpCode(204)+ @UseGuards(JwtAccessTokenGuard, RolesGuard)+ @Roles(UserType.CUSTOMER, UserType.ADMIN)+ @ApiResponse({ status: 204, description: 'Empty body' })+ async delete(+ @Param('id') id: string,+ @CurrentUserId() uid: string,+ ): Promise<any> {+ this.logger.log({ uid, id }, 'DELETE /project/:id')+ if (!isValidPID(id)) {+ throw new BadRequestException(+ 'The provided Project ID (pid) is incorrect',+ )+ }++ await deleteProjectClickhouse(id)++ const query1 = `ALTER table analytics DELETE WHERE pid='${id}'`+ const query2 = `ALTER table customEV DELETE WHERE pid='${id}'`++ try {+ await clickhouse.query(query1).toPromise()+ await clickhouse.query(query2).toPromise()+ return 'Project deleted successfully'+ } catch (e) {+ this.logger.error(e)+ return 'Error while deleting your project'+ }+ }++ @Put('/:id')+ @HttpCode(200)+ @UseGuards(JwtAccessTokenGuard, RolesGuard)+ @Roles(UserType.CUSTOMER, UserType.ADMIN)+ @ApiResponse({ status: 200, type: Project })+ async update(+ @Param('id') id: string,+ @Body() projectDTO: ProjectDTO,+ @CurrentUserId() uid: string,+ ): Promise<any> {+ this.logger.log({ projectDTO, uid, id }, 'PUT /project/:id')+ this.projectService.validateProject(projectDTO)+ const project = await getProjectsClickhouse(id)++ if (_isEmpty(project)) {+ throw new NotFoundException()+ }++ project.active = projectDTO.active+ project.origins = _map(projectDTO.origins, _trim)+ project.ipBlacklist = _map(projectDTO.ipBlacklist, _trim)+ project.name = projectDTO.name+ project.public = projectDTO.public++ await updateProjectClickhouse(+ this.projectService.formatToClickhouse(project),+ )++ // await updateProjectRedis(id, project)+ await deleteProjectRedis(id)++ return project+ }++ @Get('/:id')+ @Auth([], true, true)+ @ApiResponse({ status: 200, type: Project })+ async getOne(+ @Param('id') id: string,+ @CurrentUserId() uid: string,+ ): Promise<Project | object> {+ this.logger.log({ id }, 'GET /project/:id')+ if (!isValidPID(id)) {+ throw new BadRequestException(+ 'The provided Project ID (pid) is incorrect',+ )+ }++ const project = await getProjectsClickhouse(id)++ if (_isEmpty(project)) {+ throw new NotFoundException('Project was not found in the database')+ }++ this.projectService.allowedToView(project, uid)++ return this.projectService.formatFromClickhouse(project)+ }+}diff --git a/selfhosted-src/project/project.module.ts b/selfhosted-src/project/project.module.tsnew file mode 100644index 000000000..9566bc47e--- /dev/null+++ b/selfhosted-src/project/project.module.ts@@ -0,0 +1,15 @@+import { Module } from '@nestjs/common'++import { ProjectService } from './project.service'+import { ProjectController } from './project.controller'+import { AppLoggerModule } from '../logger/logger.module'++@Module({+ imports: [+ AppLoggerModule,+ ],+ providers: [ProjectService],+ exports: [ProjectService],+ controllers: [ProjectController],+})+export class ProjectModule {}diff --git a/selfhosted-src/project/project.service.ts b/selfhosted-src/project/project.service.tsnew file mode 100644index 000000000..33c899d17--- /dev/null+++ b/selfhosted-src/project/project.service.ts@@ -0,0 +1,189 @@+import {+ ForbiddenException,+ Injectable,+ BadRequestException,+ UnprocessableEntityException,+ InternalServerErrorException,+ ConflictException,+} from '@nestjs/common'+import * as _isEmpty from 'lodash/isEmpty'+import * as _isString from 'lodash/isString'+import * as _size from 'lodash/size'+import * as _join from 'lodash/join'+import * as _find from 'lodash/find'+import * as _map from 'lodash/map'+import * as _pick from 'lodash/pick'+import * as _isNull from 'lodash/isNull'+import * as _split from 'lodash/split'+import * as _trim from 'lodash/trim'+import * as _findIndex from 'lodash/findIndex'+// @ts-ignore+import * as validateIP from 'validate-ip-node'++import { Project } from './entity/project.entity'+import { ProjectDTO } from './dto/project.dto'+import {+ isValidPID,+ redis,+ clickhouse,+ IP_REGEX,+ ORIGINS_REGEX,+ getRedisProjectKey,+ redisProjectCacheTimeout,+} from '../common/constants'+import { getProjectsClickhouse } from '../common/utils'++export const deleteProjectRedis = async (id: string) => {+ const key = getRedisProjectKey(id)++ try {+ await redis.del(key)+ } catch (e) {+ console.error(`Error deleting project ${id} from redis: ${e}`)+ }+}++@Injectable()+export class ProjectService {+ constructor() {}++ async getRedisProject(pid: string): Promise<Project | null> {+ const pidKey = getRedisProjectKey(pid)+ let project: string | Project = await redis.get(pidKey)++ if (_isEmpty(project)) {+ project = this.formatFromClickhouse(await getProjectsClickhouse(pid))++ if (_isEmpty(project))+ throw new BadRequestException(+ 'The provided Project ID (pid) is incorrect',+ )++ await redis.set(+ pidKey,+ JSON.stringify(project),+ 'EX',+ redisProjectCacheTimeout,+ )+ } else {+ try {+ project = JSON.parse(project)+ } catch {+ throw new InternalServerErrorException('Error while processing project')+ }+ }++ // @ts-ignore+ return project+ }++ allowedToView(project: Project, uid: string | null): void {+ if (+ project.public ||+ uid+ ) {+ return null+ }++ throw new ForbiddenException(+ `You are not allowed to view "${project?.name}" project`,+ )+ }++ checkIfIDUniqueClickhouse(projects: Array<object>, projectID: string): void {+ if (_find(projects, ({ id }) => id === projectID)) {+ throw new BadRequestException('Selected project ID is already in use')+ }+ }++ async removeDataFromClickhouse(+ pid: string,+ from: string,+ to: string,+ ): Promise<void> {+ const queryAnalytics =+ 'ALTER TABLE analytics DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}'+ const queryCustomEvents =+ 'ALTER TABLE customEV DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}'+ const queryPerformance =+ 'ALTER TABLE performance DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}'+ const params = {+ params: {+ pid,+ from,+ to,+ },+ }++ await Promise.all([+ clickhouse.query(queryAnalytics, params).toPromise(),+ clickhouse.query(queryCustomEvents, params).toPromise(),+ clickhouse.query(queryPerformance, params).toPromise(),+ ])+ }++ formatToClickhouse(project: any): object {+ const updProject = { ...project }+ updProject.active = Number(updProject.active)+ updProject.public = Number(updProject.public)++ if (!_isNull(updProject.origins)) {+ updProject.origins = _isString(updProject.origins)+ ? updProject.origins+ : _join(updProject.origins, ',')+ }++ if (!_isNull(updProject.ipBlacklist)) {+ updProject.ipBlacklist = _isString(updProject.ipBlacklist)+ ? updProject.ipBlacklist+ : _join(updProject.ipBlacklist, ',')+ }++ return updProject+ }++ formatFromClickhouse(project: any): Project {+ const updProject = { ...project }+ updProject.active = Boolean(updProject.active)+ updProject.public = Boolean(updProject.public)++ updProject.origins = _isNull(updProject.origins)+ ? []+ : _split(updProject.origins, ',')++ updProject.ipBlacklist = _isNull(updProject.ipBlacklist)+ ? []+ : _split(updProject.ipBlacklist, ',')++ return updProject+ }++ validateProject(projectDTO: ProjectDTO) {+ if (!isValidPID(projectDTO.id))+ throw new UnprocessableEntityException(+ 'The provided Project ID (pid) is incorrect',+ )+ if (_size(projectDTO.name) > 50)+ throw new UnprocessableEntityException('The project name is too long')+ if (_size(_join(projectDTO.origins, ',')) > 300)+ throw new UnprocessableEntityException(+ 'The list of allowed origins has to be smaller than 300 symbols',+ )+ if (_size(_join(projectDTO.ipBlacklist, ',')) > 300)+ throw new UnprocessableEntityException(+ 'The list of allowed blacklisted IP addresses must be less than 300 characters.',+ )++ _map(projectDTO.origins, host => {+ if (!ORIGINS_REGEX.test(_trim(host))) {+ throw new ConflictException(`Host ${host} is not correct`)+ }+ })++ _map(projectDTO.ipBlacklist, ip => {+ if (!validateIP(_trim(ip)) && !IP_REGEX.test(_trim(ip))) {+ throw new ConflictException(`IP address ${ip} is not correct`)+ }+ })+ }+}diff --git a/selfhosted-src/task-manager/task-manager.module.ts b/selfhosted-src/task-manager/task-manager.module.tsnew file mode 100644index 000000000..8ced816d5--- /dev/null+++ b/selfhosted-src/task-manager/task-manager.module.ts@@ -0,0 +1,12 @@+import { Module } from '@nestjs/common'+import { TaskManagerService } from './task-manager.service'+import { AppLoggerModule } from '../logger/logger.module'++@Module({+ imports: [+ AppLoggerModule,+ ],+ providers: [TaskManagerService],+ exports: [TaskManagerService],+})+export class TaskManagerModule {}diff --git a/selfhosted-src/task-manager/task-manager.service.ts b/selfhosted-src/task-manager/task-manager.service.tsnew file mode 100644index 000000000..052a0a9c0--- /dev/null+++ b/selfhosted-src/task-manager/task-manager.service.ts@@ -0,0 +1,141 @@+import { Injectable } from '@nestjs/common'+import { Cron, CronExpression } from '@nestjs/schedule'+import * as bcrypt from 'bcrypt'+import * as dayjs from 'dayjs'+import * as utc from 'dayjs/plugin/utc'+import * as _isEmpty from 'lodash/isEmpty'+import * as _join from 'lodash/join'+import * as _size from 'lodash/size'+import * as _map from 'lodash/map'+import * as _now from 'lodash/now'++import {+ clickhouse,+ redis,+ REDIS_LOG_DATA_CACHE_KEY,+ REDIS_LOG_CUSTOM_CACHE_KEY,+ REDIS_SESSION_SALT_KEY,+ REDIS_LOG_PERF_CACHE_KEY,+ REDIS_LOG_CAPTCHA_CACHE_KEY,+} from '../common/constants'+import { AppLoggerService } from '../logger/logger.service'++dayjs.extend(utc)++@Injectable()+export class TaskManagerService {+ constructor(+ private readonly logger: AppLoggerService,+ ) {}++ @Cron(CronExpression.EVERY_MINUTE)+ async saveLogData(): Promise<void> {+ const data = await redis.lrange(REDIS_LOG_DATA_CACHE_KEY, 0, -1)+ const customData = await redis.lrange(REDIS_LOG_CUSTOM_CACHE_KEY, 0, -1)+ const perfData = await redis.lrange(REDIS_LOG_PERF_CACHE_KEY, 0, -1)+ const captchaData = await redis.lrange(REDIS_LOG_CAPTCHA_CACHE_KEY, 0, -1)++ if (!_isEmpty(data)) {+ await redis.del(REDIS_LOG_DATA_CACHE_KEY)+ const query = `INSERT INTO analytics (*) VALUES ${_join(data, ',')}`+ try {+ await clickhouse.query(query).toPromise()+ } catch (e) {+ console.error(`[CRON WORKER] Error whilst saving log data: ${e}`)+ }+ }++ if (!_isEmpty(captchaData)) {+ await redis.del(REDIS_LOG_CAPTCHA_CACHE_KEY)+ const query = `INSERT INTO captcha (*) VALUES ${_join(captchaData, ',')}`+ try {+ await clickhouse.query(query).toPromise()+ } catch (e) {+ console.error(+ `[CRON WORKER] Error whilst saving CAPTCHA log data: ${e}`,+ )+ }+ }++ if (!_isEmpty(customData)) {+ await redis.del(REDIS_LOG_CUSTOM_CACHE_KEY)+ const query = `INSERT INTO customEV (*) VALUES ${_join(customData, ',')}`++ try {+ await clickhouse.query(query).toPromise()+ } catch (e) {+ console.error(+ `[CRON WORKER] Error whilst saving custom events data: ${e}`,+ )+ }+ }++ if (!_isEmpty(perfData)) {+ await redis.del(REDIS_LOG_PERF_CACHE_KEY)++ const query = `INSERT INTO performance (*) VALUES ${_join(perfData, ',')}`++ try {+ await clickhouse.query(query).toPromise()+ } catch (e) {+ console.error(+ `[CRON WORKER] Error whilst saving performance data: ${e}`,+ )+ }+ }+ }++ @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)+ async generateSessionSalt(): Promise<void> {+ const salt = await bcrypt.genSalt(10)+ await redis.set(REDIS_SESSION_SALT_KEY, salt, 'EX', 87000)+ }++ @Cron(CronExpression.EVERY_DAY_AT_MIDNIGHT)+ async cleanUpSessions(): Promise<void> {+ const delSidQuery = `ALTER TABLE analytics UPDATE sid = NULL WHERE created < '${dayjs+ .utc()+ .subtract(20, 'm')+ .format('YYYY-MM-DD HH:mm:ss')}'`++ await clickhouse.query(delSidQuery).toPromise()+ }++ @Cron(CronExpression.EVERY_MINUTE)+ async processSessionDuration(): Promise<void> {+ const keys = await redis.keys('sd:*')+ const toSave = []+ const now = _now()++ const promises = _map(keys, async key => {+ const [start, last] = (await redis.get(key)).split(':')+ const duration = now - Number(last)++ // storing to the DB if last interaction was more than 1 minute ago+ if (duration > 60000) {+ toSave.push([key, Number(last) - Number(start)])+ }+ })++ await Promise.allSettled(promises).catch(reason => {+ this.logger.error(+ `[CRON WORKER](processSessionDuration) Error occured: ${reason}`,+ )+ })++ if (_size(toSave) > 0) {+ await redis.del(..._map(toSave, ([key]) => key))++ const setSdurQuery = `ALTER TABLE analytics UPDATE sdur = sdur + CASE ${_map(+ toSave,+ ([key, duration]) =>+ `WHEN sid = '${key.split(':')[1]}' THEN ${duration / 1000}`, // converting to seconds+ ).join(' ')} END WHERE sid IN (${_map(+ toSave,+ ([key]) => `'${key.split(':')[1]}'`,+ ).join(',')})`++ await clickhouse.query(setSdurQuery).toPromise()+ }+ }+}diff --git a/selfhosted-src/user/entities/user.entity.ts b/selfhosted-src/user/entities/user.entity.tsnew file mode 100644index 000000000..217cb53b2--- /dev/null+++ b/selfhosted-src/user/entities/user.entity.ts@@ -0,0 +1,35 @@+import { SELFHOSTED_EMAIL, SELFHOSTED_UUID } from '../../common/constants'++export enum UserType {+ CUSTOMER = 'customer',+ ADMIN = 'admin',+}++export enum TimeFormat {+ '12-hour' = '12-hour',+ '24-hour' = '24-hour',+}++export const MAX_EMAIL_REQUESTS = 4 // 1 confirmation email on sign up + 3 additional ones++export const DEFAULT_TIMEZONE = 'Etc/GMT'++export const TRIAL_DURATION = 14 // days++export interface SelfhostedUser {+ id: string+ email: string+ isTwoFactorAuthenticationEnabled: boolean+ planCode: 'enterprise'+ roles: UserType[]+ isActive: boolean+}++export const generateSelfhostedUser = (): SelfhostedUser => ({+ id: SELFHOSTED_UUID,+ email: SELFHOSTED_EMAIL,+ isTwoFactorAuthenticationEnabled: false,+ planCode: 'enterprise',+ roles: [UserType.ADMIN, UserType.CUSTOMER],+ isActive: true,+})diff --git a/selfhosted-src/user/user.controller.ts b/selfhosted-src/user/user.controller.tsnew file mode 100644index 000000000..281fbb477--- /dev/null+++ b/selfhosted-src/user/user.controller.ts@@ -0,0 +1,35 @@+import {+ Controller,+ Get,+ UseGuards,+} from '@nestjs/common'+import { ApiTags } from '@nestjs/swagger'++import { JwtAccessTokenGuard } from 'selfhosted-src/auth/guards'+import {+ UserType,+ SelfhostedUser,+ generateSelfhostedUser,+} from './entities/user.entity'+import { Roles } from '../auth/decorators/roles.decorator'+import { RolesGuard } from '../auth/guards/roles.guard'+import { CurrentUserId } from '../auth/decorators/current-user-id.decorator'+import { AppLoggerService } from '../logger/logger.service'++@ApiTags('User')+@Controller('user')+@UseGuards(JwtAccessTokenGuard, RolesGuard)+export class UserController {+ constructor(+ private readonly logger: AppLoggerService,+ ) {}++ @Get('/me')+ @UseGuards(RolesGuard)+ @Roles(UserType.CUSTOMER, UserType.ADMIN)+ async me(@CurrentUserId() user_id: string): Promise<SelfhostedUser> {+ this.logger.log({ user_id }, 'GET /user/me')++ return generateSelfhostedUser()+ }+}diff --git a/selfhosted-src/user/user.module.ts b/selfhosted-src/user/user.module.tsnew file mode 100644index 000000000..ea523bc19--- /dev/null+++ b/selfhosted-src/user/user.module.ts@@ -0,0 +1,12 @@+import { Module } from '@nestjs/common'++import { UserController } from './user.controller'+import { UserService } from './user.service'++@Module({+ imports: [],+ providers: [UserService],+ exports: [UserService],+ controllers: [UserController],+})+export class UserModule {}diff --git a/selfhosted-src/user/user.service.ts b/selfhosted-src/user/user.service.tsnew file mode 100644index 000000000..5de22cd66--- /dev/null+++ b/selfhosted-src/user/user.service.ts@@ -0,0 +1,7 @@+import { Injectable } from '@nestjs/common'++@Injectable()+export class UserService {+ constructor() {}++}