📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
(improvement) Preparing for a new selfhosted release (#235)
Andrii Romasiun06587de65afd
diff --git a/.env.example b/.env.exampleindex 1e02c3cee..6c761faa6 100644--- a/.env.example+++ b/.env.example@@ -44,6 +44,7 @@ SMTP_MOCK=true# SelfhostedPASSWORD=12345678+API_KEY=# Swetrix CDN for storing filesCDN_URL=http://localhost:5006diff --git a/Dockerfile b/Dockerfileindex 427bc4584..dbaa9d5b6 100644--- a/Dockerfile+++ b/Dockerfile@@ -20,7 +20,8 @@ ENV TZ=UTC \CLICKHOUSE_DATABASE=analytics \API_ORIGINS=\- PASSWORD=12345678+ PASSWORD=12345678 \+ API_KEY=RUN apk add --no-cache tzdata && cp /usr/share/zoneinfo/$TZ /etc/localtimeWORKDIR /appCOPY --from=build /build/package*.json ./diff --git a/apps/production/src/analytics/analytics.service.ts b/apps/production/src/analytics/analytics.service.tsindex 66b79a16e..6671c37cf 100644--- a/apps/production/src/analytics/analytics.service.ts+++ b/apps/production/src/analytics/analytics.service.ts@@ -824,7 +824,7 @@ export class AnalyticsService {let diff = nullif (from && to) {- diff = dayjs(to).diff(dayjs(from?.[0].created || to), 'days')+ diff = dayjs(to).diff(dayjs(from?.[0]?.created || to), 'days')const tbMap = _find(timeBucketToDays, ({ lt }) => diff <= lt)diff --git a/apps/production/src/project/project.service.ts b/apps/production/src/project/project.service.tsindex 0f22654c0..cab9588ef 100644--- a/apps/production/src/project/project.service.ts+++ b/apps/production/src/project/project.service.ts@@ -838,9 +838,11 @@ export class ProjectService {})}- async getPIDsWhereAnalyticsDataExists(projectIds: string[]) {+ async getPIDsWhereAnalyticsDataExists(+ projectIds: string[],+ ): Promise<string[]> {if (_isEmpty(projectIds)) {- return {}+ return []}const params = _reduce(@@ -895,9 +897,9 @@ export class ProjectService {return _map(result, ({ pid }) => pid)}- async getPIDsWhereErrorsDataExists(projectIds: string[]) {+ async getPIDsWhereErrorsDataExists(projectIds: string[]): Promise<string[]> {if (_isEmpty(projectIds)) {- return {}+ return []}const params = _reduce(diff --git a/apps/selfhosted/src/analytics/analytics.controller.ts b/apps/selfhosted/src/analytics/analytics.controller.tsindex 1b1725c87..3af2f00a4 100644--- a/apps/selfhosted/src/analytics/analytics.controller.ts+++ b/apps/selfhosted/src/analytics/analytics.controller.ts@@ -6,9 +6,6 @@ import * as _uniqBy from 'lodash/uniqBy'import * as _round from 'lodash/round'import * as _includes from 'lodash/includes'import * as _keys from 'lodash/keys'-import * as _size from 'lodash/size'-import * as _some from 'lodash/some'-import * as _isString from 'lodash/isString'import * as _values from 'lodash/values'import * as dayjs from 'dayjs'import * as utc from 'dayjs/plugin/utc'@@ -62,8 +59,6 @@ import {REDIS_LOG_CUSTOM_CACHE_KEY,HEARTBEAT_SID_LIFE_TIME,REDIS_SESSION_SALT_KEY,- MIN_PAGES_IN_FUNNEL,- MAX_PAGES_IN_FUNNEL,clickhouse,REDIS_LOG_ERROR_CACHE_KEY,} from '../common/constants'@@ -348,44 +343,6 @@ const getEIDsArray = (eids, eid) => {return [eid]}-const getPagesArray = (rawPages: string): string[] => {- try {- const pages = JSON.parse(rawPages)-- if (!_isArray(pages)) {- throw new UnprocessableEntityException(- 'An array of pages has to be provided as a pages param',- )- }-- const size = _size(pages)-- if (size < MIN_PAGES_IN_FUNNEL) {- throw new UnprocessableEntityException(- `A minimum of ${MIN_PAGES_IN_FUNNEL} pages or events has to be provided`,- )- }-- if (size > MAX_PAGES_IN_FUNNEL) {- throw new UnprocessableEntityException(- `A maximum of ${MAX_PAGES_IN_FUNNEL} pages or events can be provided`,- )- }-- if (_some(pages, page => !_isString(page))) {- throw new UnprocessableEntityException(- 'Pages array must contain string values only',- )- }-- return pages- } catch (e) {- throw new UnprocessableEntityException(- 'Cannot process the provided array of pages',- )- }-}-// needed for serving 1x1 px GIFconst TRANSPARENT_GIF_BUFFER = Buffer.from('R0lGODlhAQABAIAAAP///wAAACwAAAAAAQABAAACAkQBADs=',@@ -406,6 +363,7 @@ export class AnalyticsController {async getData(@Query() data: AnalyticsGET_DTO,@CurrentUserId() uid: string,+ @Headers() headers: { 'x-password'?: string },): Promise<any> {const {pid,@@ -423,7 +381,11 @@ export class AnalyticsController {this.analyticsService.validatePeriod(period)}- await this.analyticsService.checkProjectAccess(pid, uid)+ await this.analyticsService.checkProjectAccess(+ pid,+ uid,+ headers['x-password'],+ )let newTimebucket = timeBucketlet allowedTumebucketForPeriodAll@@ -509,17 +471,34 @@ export class AnalyticsController {async getFunnel(@Query() data: GetFunnelsDTO,@CurrentUserId() uid: string,+ @Headers() headers: { 'x-password'?: string },): Promise<IGetFunnel> {- const { pid, period, from, to, timezone = DEFAULT_TIMEZONE, pages } = data+ const {+ pid,+ period,+ from,+ to,+ timezone = DEFAULT_TIMEZONE,+ pages,+ funnelId,+ } = datathis.analyticsService.validatePID(pid)if (!_isEmpty(period)) {this.analyticsService.validatePeriod(period)}- const pagesArr = getPagesArray(pages)+ const pagesArr = await this.analyticsService.getPagesArray(+ pages,+ funnelId,+ pid,+ )- await this.analyticsService.checkProjectAccess(pid, uid)+ await this.analyticsService.checkProjectAccess(+ pid,+ uid,+ headers['x-password'],+ )const safeTimezone = this.analyticsService.getSafeTimezone(timezone)const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(@@ -565,6 +544,7 @@ export class AnalyticsController {async getCustomEventMetadata(@Query() data: GetCustomEventMetadata,@CurrentUserId() uid: string,+ @Headers() headers: { 'x-password'?: string },): Promise<IAggregatedMetadata[]> {const { pid, period } = datathis.analyticsService.validatePID(pid)@@ -573,7 +553,11 @@ export class AnalyticsController {this.analyticsService.validatePeriod(period)}- await this.analyticsService.checkProjectAccess(pid, uid)+ await this.analyticsService.checkProjectAccess(+ pid,+ uid,+ headers['x-password'],+ )return this.analyticsService.getCustomEventMetadata(data)}@@ -583,11 +567,16 @@ export class AnalyticsController {async getFilters(@Query() data: GetFiltersDto,@CurrentUserId() uid: string,+ @Headers() headers: { 'x-password'?: string },): Promise<any> {const { pid, type } = datathis.analyticsService.validatePID(pid)- await this.analyticsService.checkProjectAccess(pid, uid)+ await this.analyticsService.checkProjectAccess(+ pid,+ uid,+ headers['x-password'],+ )return this.analyticsService.getFilters(pid, type)}@@ -597,6 +586,7 @@ export class AnalyticsController {async getChartData(@Query() data: AnalyticsGET_DTO,@CurrentUserId() uid: string,+ @Headers() headers: { 'x-password'?: string },): Promise<any> {const {pid,@@ -626,7 +616,11 @@ export class AnalyticsController {period,safeTimezone,)- await this.analyticsService.checkProjectAccess(pid, uid)+ await this.analyticsService.checkProjectAccess(+ pid,+ uid,+ headers['x-password'],+ )const paramsData = {params: {@@ -659,6 +653,7 @@ export class AnalyticsController {async getPerfData(@Query() data: AnalyticsGET_DTO & { measure: PerfMeasure },@CurrentUserId() uid: string,+ @Headers() headers: { 'x-password'?: string },): Promise<any> {const {pid,@@ -678,7 +673,11 @@ export class AnalyticsController {this.analyticsService.validatePeriod(period)}- await this.analyticsService.checkProjectAccess(pid, uid)+ await this.analyticsService.checkProjectAccess(+ pid,+ uid,+ headers['x-password'],+ )let newTimeBucket = timeBucketlet allowedTumebucketForPeriodAll@@ -747,6 +746,7 @@ export class AnalyticsController {async getPerfChartData(@Query() data: AnalyticsGET_DTO & { measure: PerfMeasure },@CurrentUserId() uid: string,+ @Headers() headers: { 'x-password'?: string },): Promise<any> {const {pid,@@ -778,7 +778,11 @@ export class AnalyticsController {period,safeTimezone,)- await this.analyticsService.checkProjectAccess(pid, uid)+ await this.analyticsService.checkProjectAccess(+ pid,+ uid,+ headers['x-password'],+ )const paramsData = {params: {@@ -810,6 +814,7 @@ export class AnalyticsController {async getUserFlow(@Query() data: GetUserFlowDTO,@CurrentUserId() uid: string,+ @Headers() headers: { 'x-password'?: string },): Promise<IUserFlow | { appliedFilters: any[] }> {const { pid, period, from, to, timezone = DEFAULT_TIMEZONE, filters } = datathis.analyticsService.validatePID(pid)@@ -818,7 +823,11 @@ export class AnalyticsController {this.analyticsService.validatePeriod(period)}- await this.analyticsService.checkProjectAccess(pid, uid)+ await this.analyticsService.checkProjectAccess(+ pid,+ uid,+ headers['x-password'],+ )let diff@@ -866,6 +875,7 @@ export class AnalyticsController {async getOverallStats(@Query() data,@CurrentUserId() uid: string,+ @Headers() headers: { 'x-password'?: string },): Promise<any> {const {pids,@@ -880,7 +890,11 @@ export class AnalyticsController {const validationPromises = _map(pidsArray, async currentPID => {this.analyticsService.validatePID(currentPID)- await this.analyticsService.checkProjectAccess(currentPID, uid)+ await this.analyticsService.checkProjectAccess(+ currentPID,+ uid,+ headers['x-password'],+ )})await Promise.all(validationPromises)@@ -900,6 +914,7 @@ export class AnalyticsController {async getPerformanceOverallStats(@Query() data,@CurrentUserId() uid: string,+ @Headers() headers: { 'x-password'?: string },): Promise<any> {const {pids,@@ -921,7 +936,11 @@ export class AnalyticsController {const validationPromises = _map(pidsArray, async currentPID => {this.analyticsService.validatePID(currentPID)- await this.analyticsService.checkProjectAccess(currentPID, uid)+ await this.analyticsService.checkProjectAccess(+ currentPID,+ uid,+ headers['x-password'],+ )})await Promise.all(validationPromises)@@ -941,13 +960,18 @@ export class AnalyticsController {async getHeartBeatStats(@Query() data,@CurrentUserId() uid: string,+ @Headers() headers: { 'x-password'?: string },): Promise<object> {const { pids, pid } = dataconst pidsArray = getPIDsArray(pids, pid)const validationPromises = _map(pidsArray, async currentPID => {this.analyticsService.validatePID(currentPID)- await this.analyticsService.checkProjectAccess(currentPID, uid)+ await this.analyticsService.checkProjectAccess(+ currentPID,+ uid,+ headers['x-password'],+ )})await Promise.all(validationPromises)@@ -974,11 +998,16 @@ export class AnalyticsController {async getLiveVisitors(@Query() data,@CurrentUserId() uid: string,+ @Headers() headers: { 'x-password'?: string },): Promise<object> {const { pid } = datathis.analyticsService.validatePID(pid)- await this.analyticsService.checkProjectAccess(pid, uid)+ await this.analyticsService.checkProjectAccess(+ pid,+ uid,+ headers['x-password'],+ )const keys = await redis.keys(`sd:*:${pid}`)@@ -1297,6 +1326,7 @@ export class AnalyticsController {async getSessions(@Query() data: GetSessionsDto,@CurrentUserId() uid: string,+ @Headers() headers: { 'x-password'?: string },): Promise<any> {const { pid, period, from, to, filters, timezone = DEFAULT_TIMEZONE } = datathis.analyticsService.validatePID(pid)@@ -1305,7 +1335,11 @@ export class AnalyticsController {this.analyticsService.validatePeriod(period)}- await this.analyticsService.checkProjectAccess(pid, uid)+ await this.analyticsService.checkProjectAccess(+ pid,+ uid,+ headers['x-password'],+ )const take = this.analyticsService.getSafeNumber(data.take, 30)const skip = this.analyticsService.getSafeNumber(data.skip, 0)@@ -1378,6 +1412,7 @@ export class AnalyticsController {async getSession(@Query() data: GetSessionDto,@CurrentUserId() uid: string,+ @Headers() headers: { 'x-password'?: string },): Promise<any> {const {pid,@@ -1387,7 +1422,11 @@ export class AnalyticsController {} = datathis.analyticsService.validatePID(pid)- await this.analyticsService.checkProjectAccess(pid, uid)+ await this.analyticsService.checkProjectAccess(+ pid,+ uid,+ headers['x-password'],+ )const safeTimezone = this.analyticsService.getSafeTimezone(timezone)const result = await this.analyticsService.getSessionDetails(@@ -1404,6 +1443,7 @@ export class AnalyticsController {async getCustomEvents(@Query() data: GetCustomEventsDto,@CurrentUserId() uid: string,+ @Headers() headers: { 'x-password'?: string },): Promise<any> {const {pid,@@ -1421,7 +1461,11 @@ export class AnalyticsController {this.analyticsService.validatePeriod(period)}- await this.analyticsService.checkProjectAccess(pid, uid)+ await this.analyticsService.checkProjectAccess(+ pid,+ uid,+ headers['x-password'],+ )let newTimeBucket = timeBucketlet diff@@ -1505,11 +1549,16 @@ export class AnalyticsController {async getErrorsFilters(@Query() data: GetFiltersDto,@CurrentUserId() uid: string,+ @Headers() headers: { 'x-password'?: string },): Promise<any> {const { pid, type } = datathis.analyticsService.validatePID(pid)- await this.analyticsService.checkProjectAccess(pid, uid)+ await this.analyticsService.checkProjectAccess(+ pid,+ uid,+ headers['x-password'],+ )return this.analyticsService.getErrorsFilters(pid, type)}@@ -1587,7 +1636,11 @@ export class AnalyticsController {await checkRateLimit(ip, 'error-status', 100, 1800)this.analyticsService.validatePID(pid)- await this.analyticsService.checkProjectAccess(pid, uid)+ await this.analyticsService.checkProjectAccess(+ pid,+ uid,+ headers['x-password'],+ )const eids = getEIDsArray(unprocessedEids, eid)@@ -1601,6 +1654,7 @@ export class AnalyticsController {async getErrors(@Query() data: GetErrorsDto,@CurrentUserId() uid: string,+ @Headers() headers: { 'x-password'?: string },): Promise<any> {const {pid,@@ -1617,7 +1671,11 @@ export class AnalyticsController {this.analyticsService.validatePeriod(period)}- await this.analyticsService.checkProjectAccess(pid, uid)+ await this.analyticsService.checkProjectAccess(+ pid,+ uid,+ headers['x-password'],+ )const take = this.analyticsService.getSafeNumber(data.take, 30)const skip = this.analyticsService.getSafeNumber(data.skip, 0)@@ -1690,6 +1748,7 @@ export class AnalyticsController {async getError(@Query() data: GetErrorDTO,@CurrentUserId() uid: string,+ @Headers() headers: { 'x-password'?: string },): Promise<any> {const {pid,@@ -1706,7 +1765,11 @@ export class AnalyticsController {this.analyticsService.validatePeriod(period)}- await this.analyticsService.checkProjectAccess(pid, uid)+ await this.analyticsService.checkProjectAccess(+ pid,+ uid,+ headers['x-password'],+ )let timeBucketlet diffdiff --git a/apps/selfhosted/src/analytics/analytics.service.ts b/apps/selfhosted/src/analytics/analytics.service.tsindex 4d986e33a..2eb1cff05 100644--- a/apps/selfhosted/src/analytics/analytics.service.ts+++ b/apps/selfhosted/src/analytics/analytics.service.ts@@ -46,8 +46,14 @@ import {TRAFFIC_COLUMNS,PERFORMANCE_COLUMNS,ERROR_COLUMNS,+ MIN_PAGES_IN_FUNNEL,+ MAX_PAGES_IN_FUNNEL,} from '../common/constants'-import { millisecondsToSeconds, sumArrays } from '../common/utils'+import {+ getFunnelsClickhouse,+ millisecondsToSeconds,+ sumArrays,+} from '../common/utils'import { PageviewsDTO } from './dto/pageviews.dto'import { EventsDTO } from './dto/events.dto'import { ProjectService } from '../project/project.service'@@ -332,9 +338,13 @@ const isValidOrigin = (origins: string[], origin: string) => {export class AnalyticsService {constructor(private readonly projectService: ProjectService) {}- async checkProjectAccess(pid: string, uid: string | null): Promise<void> {+ async checkProjectAccess(+ pid: string,+ uid: string | null,+ password: string | null,+ ): Promise<void> {const project = await this.projectService.getRedisProject(pid)- this.projectService.allowedToView(project, uid)+ this.projectService.allowedToView(project, uid, password)}checkOrigin(project: Project, origin: string): void {@@ -705,6 +715,62 @@ export class AnalyticsService {}}+ async getPagesArray(+ rawPages?: string,+ funnelId?: string,+ projectId?: string,+ ): Promise<string[]> {+ if (funnelId && projectId) {+ const funnel = this.projectService.formatFunnelFromClickhouse(+ await getFunnelsClickhouse(projectId, funnelId),+ )++ if (!funnel || _isEmpty(funnel)) {+ throw new UnprocessableEntityException(+ 'The provided funnelId is incorrect',+ )+ }++ return funnel.steps+ }++ try {+ const pages = JSON.parse(rawPages)++ if (!_isArray(pages)) {+ throw new UnprocessableEntityException(+ 'An array of pages has to be provided as a pages param',+ )+ }++ const size = _size(pages)++ if (size < MIN_PAGES_IN_FUNNEL) {+ throw new UnprocessableEntityException(+ `A minimum of ${MIN_PAGES_IN_FUNNEL} pages or events has to be provided`,+ )+ }++ if (size > MAX_PAGES_IN_FUNNEL) {+ throw new UnprocessableEntityException(+ `A maximum of ${MAX_PAGES_IN_FUNNEL} pages or events can be provided`,+ )+ }++ if (_some(pages, page => !_isString(page))) {+ throw new UnprocessableEntityException(+ 'Pages array must contain string values only',+ )+ }++ return pages+ } catch (e) {+ throw new UnprocessableEntityException(+ 'Cannot process the provided array of pages',+ )+ }+ }+async getTimeBucketForAllTime(pid: string,period: string,@@ -731,7 +797,7 @@ export class AnalyticsService {let diff = nullif (from && to) {- diff = dayjs(to).diff(dayjs(from?.[0].created || to), 'days')+ diff = dayjs(to).diff(dayjs(from?.[0]?.created || to), 'days')const tbMap = _find(timeBucketToDays, ({ lt }) => diff <= lt)@@ -932,7 +998,7 @@ export class AnalyticsService {let dropoff = 0let eventsPerc = 100let eventsPercStep = 100- let dropoffPerc = 0+ let dropoffPercStep = 0if (index > 0) {const prev = data[index - 1]@@ -940,7 +1006,7 @@ export class AnalyticsService {dropoff = prev.c - row.ceventsPerc = _round((row.c / data[0].c) * 100, 2)eventsPercStep = _round((row.c / prev.c) * 100, 2)- dropoffPerc = _round((dropoff / prev.c) * 100, 2)+ dropoffPercStep = _round((dropoff / prev.c) * 100, 2)}return {@@ -949,7 +1015,7 @@ export class AnalyticsService {eventsPerc,eventsPercStep,dropoff,- dropoffPerc,+ dropoffPercStep,}})diff --git a/apps/selfhosted/src/analytics/dto/getFunnels.dto.ts b/apps/selfhosted/src/analytics/dto/getFunnels.dto.tsindex afde79179..197895faa 100644--- a/apps/selfhosted/src/analytics/dto/getFunnels.dto.ts+++ b/apps/selfhosted/src/analytics/dto/getFunnels.dto.ts@@ -27,4 +27,9 @@ export class GetFunnelsDTO {description: 'A stringified array of pages to generate funnel for',})pages: string++ @ApiProperty({+ description: 'Funnel ID',+ })+ funnelId?: string}diff --git a/apps/selfhosted/src/analytics/interfaces/index.ts b/apps/selfhosted/src/analytics/interfaces/index.tsindex 88de841f8..770ada1ad 100644--- a/apps/selfhosted/src/analytics/interfaces/index.ts+++ b/apps/selfhosted/src/analytics/interfaces/index.ts@@ -111,7 +111,7 @@ export interface IFunnel {eventsPerc: numbereventsPercStep: numberdropoff: number- dropoffPerc: number+ dropoffPercStep: number}export interface IGetFunnel {diff --git a/apps/selfhosted/src/auth/auth.module.ts b/apps/selfhosted/src/auth/auth.module.tsindex 05e51b708..fb56a08e3 100644--- a/apps/selfhosted/src/auth/auth.module.ts+++ b/apps/selfhosted/src/auth/auth.module.ts@@ -4,7 +4,7 @@ import { PassportModule } from '@nestjs/passport'import { AuthController } from './auth.controller'import { AuthService } from './auth.service'import {- // ApiKeyStrategy,+ ApiKeyStrategy,JwtAccessTokenStrategy,JwtRefreshTokenStrategy,} from './strategies'@@ -16,7 +16,7 @@ import {AuthService,JwtAccessTokenStrategy,JwtRefreshTokenStrategy,- // ApiKeyStrategy,+ ApiKeyStrategy,],exports: [AuthService],})diff --git a/apps/selfhosted/src/auth/auth.service.ts b/apps/selfhosted/src/auth/auth.service.tsindex 5284b63f9..ebf5e8f95 100644--- a/apps/selfhosted/src/auth/auth.service.ts+++ b/apps/selfhosted/src/auth/auth.service.ts@@ -15,6 +15,7 @@ import {JWT_ACCESS_TOKEN_SECRET,JWT_ACCESS_TOKEN_LIFETIME,JWT_REFRESH_TOKEN_LIFETIME,+ SELFHOSTED_API_KEY,} from '../common/constants'import { getSelfhostedUser, SelfhostedUser } from '../user/entities/user.entity'@@ -111,4 +112,8 @@ export class AuthService {async logout(userId: string, refreshToken: string) {await deleteRefreshTokenClickhouse(userId, refreshToken)}++ isApiKeyValid(apiKey: string): boolean {+ return apiKey === SELFHOSTED_API_KEY+ }}diff --git a/apps/selfhosted/src/auth/decorators/auth.decorator.ts b/apps/selfhosted/src/auth/decorators/auth.decorator.tsindex 25a30e4b9..b08e90931 100644--- a/apps/selfhosted/src/auth/decorators/auth.decorator.ts+++ b/apps/selfhosted/src/auth/decorators/auth.decorator.ts@@ -1,10 +1,6 @@import { applyDecorators, SetMetadata, UseGuards } from '@nestjs/common'-import {- JwtAccessTokenGuard,- MultiAuthGuard,- RolesGuard,-} from 'src/auth/guards'-import { UserType } from 'src/user/entities/user.entity'+import { JwtAccessTokenGuard, MultiAuthGuard, RolesGuard } from '../guards'+import { UserType } from '../../user/entities/user.entity'import { ROLES_KEY } from './roles.decorator'export const IS_OPTIONAL_AUTH_KEY = 'isOptionalAuth'diff --git a/apps/selfhosted/src/auth/guards/api-key.guard.ts b/apps/selfhosted/src/auth/guards/api-key.guard.tsnew file mode 100644index 000000000..43e58ce2d--- /dev/null+++ b/apps/selfhosted/src/auth/guards/api-key.guard.ts@@ -0,0 +1,5 @@+import { Injectable } from '@nestjs/common'+import { AuthGuard } from '@nestjs/passport'++@Injectable()+export class ApiKeyGuard extends AuthGuard('api-key') {}diff --git a/apps/selfhosted/src/auth/guards/multi-auth.guard.ts b/apps/selfhosted/src/auth/guards/multi-auth.guard.tsindex 3426269d9..96503daa0 100644--- a/apps/selfhosted/src/auth/guards/multi-auth.guard.ts+++ b/apps/selfhosted/src/auth/guards/multi-auth.guard.ts@@ -4,7 +4,7 @@ import { AuthGuard } from '@nestjs/passport'import { IS_OPTIONAL_AUTH_KEY } from '../decorators'@Injectable()-export class MultiAuthGuard extends AuthGuard(['jwt-access-token']) {+export class MultiAuthGuard extends AuthGuard(['jwt-access-token', 'api-key']) {constructor(private readonly reflector: Reflector) {super()}diff --git a/apps/selfhosted/src/auth/guards/roles.guard.ts b/apps/selfhosted/src/auth/guards/roles.guard.tsindex 210476d16..1a34aab68 100644--- a/apps/selfhosted/src/auth/guards/roles.guard.ts+++ b/apps/selfhosted/src/auth/guards/roles.guard.ts@@ -3,8 +3,8 @@ import { Reflector } from '@nestjs/core'import { ExtractJwt } from 'passport-jwt'import { verify } from 'jsonwebtoken'-import { UserType } from 'src/user/entities/user.entity'-import { JWT_ACCESS_TOKEN_SECRET } from 'src/common/constants'+import { UserType } from '../../user/entities/user.entity'+import { JWT_ACCESS_TOKEN_SECRET } from '../../common/constants'import { ROLES_KEY } from '../decorators'@Injectable()diff --git a/apps/selfhosted/src/auth/strategies/api-key.strategy.ts b/apps/selfhosted/src/auth/strategies/api-key.strategy.tsnew file mode 100644index 000000000..9e2a4df31--- /dev/null+++ b/apps/selfhosted/src/auth/strategies/api-key.strategy.ts@@ -0,0 +1,26 @@+import { Injectable, UnauthorizedException } from '@nestjs/common'+import { PassportStrategy } from '@nestjs/passport'+import { HeaderAPIKeyStrategy } from 'passport-headerapikey'+import { AuthService } from '../auth.service'+import { getSelfhostedUser } from '../../user/entities/user.entity'++@Injectable()+export class ApiKeyStrategy extends PassportStrategy(+ HeaderAPIKeyStrategy,+ 'api-key',+) {+ constructor(private readonly authService: AuthService) {+ super(+ { header: 'X-Api-Key', prefix: '' },+ true,+ // eslint-disable-next-line consistent-return+ async (apiKey: string, done: any) => {+ const isValid = this.authService.isApiKeyValid(apiKey)+ if (!isValid) return done(new UnauthorizedException(), false)++ const user = await getSelfhostedUser()+ done(null, user)+ },+ )+ }+}diff --git a/apps/selfhosted/src/auth/strategies/index.ts b/apps/selfhosted/src/auth/strategies/index.tsindex 20e4bbdce..f544f42d4 100644--- a/apps/selfhosted/src/auth/strategies/index.ts+++ b/apps/selfhosted/src/auth/strategies/index.ts@@ -1,2 +1,3 @@export * from './jwt-access-token.strategy'export * from './jwt-refresh-token.strategy'+export * from './api-key.strategy'diff --git a/apps/selfhosted/src/common/constants.ts b/apps/selfhosted/src/common/constants.tsindex 89220bfc3..226134eca 100644--- a/apps/selfhosted/src/common/constants.ts+++ b/apps/selfhosted/src/common/constants.ts@@ -56,6 +56,8 @@ const isDevelopment = process.env.NODE_ENV === 'development'const SELFHOSTED_EMAIL = process.env.EMAILconst SELFHOSTED_PASSWORD = process.env.PASSWORD+const SELFHOSTED_API_KEY = process.env.API_KEY+const SELFHOSTED_API_AUTH_ENABLED = !!SELFHOSTED_API_KEYconst UUIDV5_NAMESPACE = '912c64c1-73fd-42b6-859f-785f839a9f68'const DEFAULT_SELFHOSTED_UUID = 'deadbeef-dead-beef-dead-beefdeadbeef'@@ -129,6 +131,8 @@ export {UUIDV5_NAMESPACE,SELFHOSTED_EMAIL,SELFHOSTED_PASSWORD,+ SELFHOSTED_API_KEY,+ SELFHOSTED_API_AUTH_ENABLED,SELFHOSTED_UUID,REDIS_USERS_COUNT_KEY,REDIS_PROJECTS_COUNT_KEY,diff --git a/apps/selfhosted/src/common/utils.ts b/apps/selfhosted/src/common/utils.tsindex 20cf585de..510346f4a 100644--- a/apps/selfhosted/src/common/utils.ts+++ b/apps/selfhosted/src/common/utils.ts@@ -10,6 +10,7 @@ import * as _keys from 'lodash/keys'import * as _toNumber from 'lodash/toNumber'import * as _split from 'lodash/split'import * as _isEmpty from 'lodash/isEmpty'+import * as _isNil from 'lodash/isNil'import * as _head from 'lodash/head'import * as _round from 'lodash/round'import * as _size from 'lodash/size'@@ -28,20 +29,25 @@ import {} from './constants'import { DEFAULT_TIMEZONE, TimeFormat } from '../user/entities/user.entity'import { Project } from '../project/entity/project.entity'+import { Funnel } from '../project/entity/funnel.entity'dayjs.extend(utc)const RATE_LIMIT_REQUESTS_AMOUNT = 3const RATE_LIMIT_TIMEOUT = 86400 // 24 hours-const allowedToUpdateKeys = [+const ALLOWED_KEYS = ['name','origins','ipBlacklist','active','public',+ 'isPasswordProtected',+ 'passwordHash',]+const ALLOWED_FUNNEL_KEYS = ['name', 'steps']+const getRateLimitHash = (ipOrApiKey: string, salt = '') =>`rl:${hash(`${ipOrApiKey}${salt}`).toString('hex')}`@@ -65,9 +71,86 @@ const checkRateLimit = async (await redis.set(rlHash, 1 + rlCount, 'EX', reqTimeout)}+const getFunnelsClickhouse = async (projectId: string, funnelId = null) => {+ const paramsData = {+ params: {+ projectId,+ funnelId,+ },+ }++ if (!funnelId) {+ const query =+ 'SELECT * FROM funnel WHERE projectId = {projectId:FixedString(12)}'+ return clickhouse.query(query, paramsData).toPromise()+ }++ const query = `+ SELECT+ *+ FROM funnel+ WHERE+ projectId = {projectId:FixedString(12)}+ AND id = {funnelId:String}`++ const funnel = await clickhouse.query(query, paramsData).toPromise()++ if (_isEmpty(funnel)) {+ throw new NotFoundException(+ `Funnel ${funnelId} was not found in the database`,+ )+ }++ return _head(funnel)+}++const updateFunnelClickhouse = async (funnel: any) => {+ const filtered = _reduce(+ _filter(_keys(funnel), key => ALLOWED_FUNNEL_KEYS.includes(key)),+ (obj, key) => {+ obj[key] = funnel[key]+ return obj+ },+ {},+ )+ const columns = _keys(filtered)+ const values = _values(filtered)+ const query = `ALTER table funnel UPDATE ${_join(+ _map(columns, (col, id) => `${col}='${values[id]}'`),+ ', ',+ )} WHERE id='${funnel.id}'`++ return clickhouse.query(query).toPromise()+}++const deleteFunnelClickhouse = async (id: string) => {+ const paramsData = {+ params: {+ id,+ },+ }++ const query = `ALTER table funnel DELETE WHERE id = {id:String}`+ return clickhouse.query(query, paramsData).toPromise()+}++const createFunnelClickhouse = async (funnel: Partial<Funnel>) => {+ const paramsData = {+ params: {+ ...funnel,+ },+ }++ const query = `INSERT INTO funnel (*) VALUES ({id:String},{name:String},{steps:String},{projectId:FixedString(12)},'${dayjs+ .utc()+ .format('YYYY-MM-DD HH:mm:ss')}')`++ return clickhouse.query(query, paramsData).toPromise()+}+const getProjectsClickhouse = async (id = null) => {if (!id) {- const query = 'SELECT * FROM project;'+ const query = 'SELECT * FROM project ORDER BY created ASC;'return clickhouse.query(query).toPromise()}@@ -87,9 +170,9 @@ const getProjectsClickhouse = async (id = null) => {return _head(project)}-const updateProjectClickhouse = async (project: object) => {+const updateProjectClickhouse = async (project: any) => {const filtered = _reduce(- _filter(_keys(project), key => allowedToUpdateKeys.includes(key)),+ _filter(_keys(project), key => ALLOWED_KEYS.includes(key)),(obj, key) => {obj[key] = project[key]return obj@@ -98,11 +181,9 @@ const updateProjectClickhouse = async (project: object) => {)const columns = _keys(filtered)const values = _values(filtered)- // @ts-ignoreconst query = `ALTER table project UPDATE ${_join(_map(columns, (col, id) => `${col}='${values[id]}'`),', ',- // @ts-ignore)} WHERE id='${project.id}'`return clickhouse.query(query).toPromise()}@@ -129,9 +210,15 @@ const calculateRelativePercentage = (return _round((1 - newVal / oldVal) * -100, round)}-const deleteProjectClickhouse = async id => {- const query = `ALTER table project DELETE WHERE id='${id}'`- return clickhouse.query(query).toPromise()+const deleteProjectClickhouse = async (id: string) => {+ const paramsData = {+ params: {+ id,+ },+ }++ const query = `ALTER table project DELETE WHERE id = {id:FixedString(12)}`+ return clickhouse.query(query, paramsData).toPromise()}const createProjectClickhouse = async (project: Partial<Project>) => {@@ -261,17 +348,17 @@ const updateUserClickhouse = async (user: IClickhouseUser) => {let query = 'ALTER table sfuser UPDATE 'let separator = ''- if (user.timezone) {+ if (!_isNil(user.timezone)) {query += `${separator}timezone={timezone:String}`separator = ', '}- if (user.timeFormat) {+ if (!_isNil(user.timeFormat)) {query += `${separator}timeFormat={timeFormat:String}`separator = ', '}- if (user.showLiveVisitorsInTitle) {+ if (!_isNil(user.showLiveVisitorsInTitle)) {query += `${separator}showLiveVisitorsInTitle={showLiveVisitorsInTitle:Int8}`separator = ', '}@@ -385,4 +472,8 @@ export {getGeoDetails,getIPFromHeaders,sumArrays,+ getFunnelsClickhouse,+ updateFunnelClickhouse,+ deleteFunnelClickhouse,+ createFunnelClickhouse,}diff --git a/apps/selfhosted/src/project/dto/funnel-create.dto.ts b/apps/selfhosted/src/project/dto/funnel-create.dto.tsnew file mode 100644index 000000000..6d2ab4440--- /dev/null+++ b/apps/selfhosted/src/project/dto/funnel-create.dto.ts@@ -0,0 +1,38 @@+import { ApiProperty } from '@nestjs/swagger'+import { IsNotEmpty, Length, ArrayMinSize, ArrayMaxSize } from 'class-validator'+import {+ MAX_PAGES_IN_FUNNEL,+ MIN_PAGES_IN_FUNNEL,+} from '../../common/constants'++export class FunnelCreateDTO {+ @ApiProperty({+ example: 'User sign up funnel',+ required: true,+ description: 'A display name for your funnel',+ })+ @IsNotEmpty()+ @Length(1, 50)+ name: string++ @ApiProperty({+ example: 'aUn1quEid-3',+ required: true,+ description: 'Funnel project ID',+ })+ @IsNotEmpty()+ pid: string++ @ApiProperty({+ example: ['/', '/signup', '/dashboard'],+ required: true,+ description: 'Steps of the funnel',+ })+ @ArrayMinSize(MIN_PAGES_IN_FUNNEL, {+ message: `A funnel must have at least ${MIN_PAGES_IN_FUNNEL} steps`,+ })+ @ArrayMaxSize(MAX_PAGES_IN_FUNNEL, {+ message: `A funnel must have no more than ${MAX_PAGES_IN_FUNNEL} steps`,+ })+ steps: string[] | null+}diff --git a/apps/selfhosted/src/project/dto/funnel-update.dto.ts b/apps/selfhosted/src/project/dto/funnel-update.dto.tsnew file mode 100644index 000000000..31a7a3c3c--- /dev/null+++ b/apps/selfhosted/src/project/dto/funnel-update.dto.ts@@ -0,0 +1,47 @@+import { ApiProperty } from '@nestjs/swagger'+import {+ IsNotEmpty,+ Length,+ ArrayMinSize,+ ArrayMaxSize,+ IsUUID,+} from 'class-validator'+import {+ MAX_PAGES_IN_FUNNEL,+ MIN_PAGES_IN_FUNNEL,+} from '../../common/constants'++export class FunnelUpdateDTO {+ @IsUUID('4')+ id: string++ @ApiProperty({+ example: 'aUn1quEid-3',+ required: true,+ description: 'Funnel project ID',+ })+ @IsNotEmpty()+ pid: string++ @ApiProperty({+ example: 'User sign up funnel',+ required: true,+ description: 'A display name for your funnel',+ })+ @IsNotEmpty()+ @Length(1, 50)+ name: string++ @ApiProperty({+ example: ['/', '/signup', '/dashboard'],+ required: true,+ description: 'Steps of the funnel',+ })+ @ArrayMinSize(MIN_PAGES_IN_FUNNEL, {+ message: `A funnel must have at least ${MIN_PAGES_IN_FUNNEL} steps`,+ })+ @ArrayMaxSize(MAX_PAGES_IN_FUNNEL, {+ message: `A funnel must have no more than ${MAX_PAGES_IN_FUNNEL} steps`,+ })+ steps: string[] | null+}diff --git a/apps/selfhosted/src/project/dto/index.ts b/apps/selfhosted/src/project/dto/index.tsindex 2c80f6996..1308d4594 100644--- a/apps/selfhosted/src/project/dto/index.ts+++ b/apps/selfhosted/src/project/dto/index.ts@@ -1,2 +1,6 @@export * from './create-project.dto'export * from './project.dto'+export * from './update-project.dto'+export * from './project-password.dto'+export * from './funnel-create.dto'+export * from './funnel-update.dto'diff --git a/apps/selfhosted/src/project/dto/project-password.dto.ts b/apps/selfhosted/src/project/dto/project-password.dto.tsnew file mode 100644index 000000000..9c033b58c--- /dev/null+++ b/apps/selfhosted/src/project/dto/project-password.dto.ts@@ -0,0 +1,29 @@+import {+ IsOptional,+ IsString,+ MaxLength,+ MinLength,+ ValidateIf,+} from 'class-validator'++export const MAX_PROJECT_PASSWORD_LENGTH = 80++export class ProjectPasswordDto {+ @IsString()+ @MaxLength(MAX_PROJECT_PASSWORD_LENGTH, {+ message: 'Max length is $constraint1 characters',+ })+ @MinLength(1, { message: 'Min length is $constraint1 characters' })+ @IsOptional()+ password?: string | null++ @IsOptional()+ @ValidateIf(o => o.isPasswordProtected !== null)+ isPasswordProtected?: boolean | null+}++export class GetProtectedProjectDto {+ @IsOptional()+ @ValidateIf(o => o.password !== null)+ password?: string | null+}diff --git a/apps/selfhosted/src/project/dto/update-project.dto.ts b/apps/selfhosted/src/project/dto/update-project.dto.tsnew file mode 100644index 000000000..a6dbcde0c--- /dev/null+++ b/apps/selfhosted/src/project/dto/update-project.dto.ts@@ -0,0 +1,23 @@+import { IntersectionType, PartialType } from '@nestjs/mapped-types'+import { ApiProperty } from '@nestjs/swagger'+import { ProjectDTO } from './project.dto'+import { ProjectPasswordDto } from './project-password.dto'++export class UpdateProjectDto extends IntersectionType(+ PartialType(ProjectDTO),+ ProjectPasswordDto,+) {+ @ApiProperty({+ required: false,+ description:+ "The project's state. If enabled - all the incoming analytics data will be saved.",+ })+ active?: boolean++ @ApiProperty({+ required: false,+ description:+ "When true, anyone on the internet (including Google) would be able to see the project's Dashboard.",+ })+ public?: boolean+}diff --git a/apps/selfhosted/src/project/entity/funnel.entity.ts b/apps/selfhosted/src/project/entity/funnel.entity.tsnew file mode 100644index 000000000..eb3614c2f--- /dev/null+++ b/apps/selfhosted/src/project/entity/funnel.entity.ts@@ -0,0 +1,11 @@+export class Funnel {+ id: string++ name: string++ steps: string[]++ created: Date++ projectId: string+}diff --git a/apps/selfhosted/src/project/entity/project.entity.ts b/apps/selfhosted/src/project/entity/project.entity.tsindex dbe0df792..420ee292c 100644--- a/apps/selfhosted/src/project/entity/project.entity.ts+++ b/apps/selfhosted/src/project/entity/project.entity.ts@@ -11,5 +11,9 @@ export class Project {public: boolean+ isPasswordProtected: boolean++ passwordHash?: string+created: Date}diff --git a/apps/selfhosted/src/project/project.controller.ts b/apps/selfhosted/src/project/project.controller.tsindex 75f86f2ff..7c7088f63 100644--- a/apps/selfhosted/src/project/project.controller.ts+++ b/apps/selfhosted/src/project/project.controller.ts@@ -11,16 +11,24 @@ import {BadRequestException,HttpCode,NotFoundException,+ Headers,+ UnauthorizedException,+ Patch,} from '@nestjs/common'+import { v4 as uuidv4 } from 'uuid'import { ApiTags, ApiQuery, ApiResponse, ApiBearerAuth } from '@nestjs/swagger'import * as _isEmpty from 'lodash/isEmpty'import * as _map from 'lodash/map'import * as _trim from 'lodash/trim'import * as _size from 'lodash/size'import * as _split from 'lodash/split'+import * as _isBoolean from 'lodash/isBoolean'+import * as _omit from 'lodash/omit'+import * as _reduce from 'lodash/reduce'import * as _head from 'lodash/head'import * as _includes from 'lodash/includes'import * as dayjs from 'dayjs'+import { hash } from 'bcrypt'import { JwtAccessTokenGuard } from '../auth/guards'import { Auth } from '../auth/decorators'@@ -36,7 +44,13 @@ import { RolesGuard } from '../auth/guards/roles.guard'import { Pagination } from '../common/pagination/pagination'import { Project } from './entity/project.entity'import { CurrentUserId } from '../auth/decorators/current-user-id.decorator'-import { ProjectDTO, CreateProjectDTO } from './dto'+import {+ ProjectDTO,+ CreateProjectDTO,+ UpdateProjectDto,+ FunnelCreateDTO,+ FunnelUpdateDTO,+} from './dto'import { AppLoggerService } from '../logger/logger.service'import { isValidPID, clickhouse } from '../common/constants'import {@@ -44,7 +58,12 @@ import {createProjectClickhouse,updateProjectClickhouse,deleteProjectClickhouse,+ getFunnelsClickhouse,+ deleteFunnelClickhouse,+ createFunnelClickhouse,+ updateFunnelClickhouse,} from '../common/utils'+import { Funnel } from './entity/funnel.entity'@ApiTags('Project')@Controller('project')@@ -81,8 +100,35 @@ export class ProjectController {_map(formatted, ({ id }) => id),)+ const funnelsData = await Promise.allSettled(+ pidsWithData.map(async pid => {+ return {+ pid,+ data: await getFunnelsClickhouse(pid),+ }+ }),+ )++ const funnelsMap = _reduce(+ funnelsData,+ (acc, { status, value }) => {+ if (status !== 'fulfilled') {+ return acc+ }++ return {+ ...acc,+ [value.pid]: this.projectService.formatFunnelsFromClickhouse(+ value.data,+ ),+ }+ },+ {},+ )+const results = _map(formatted, p => ({- ...p,+ ..._omit(p, ['passwordHash']),+ funnels: funnelsMap[p.id],isOwner: true,isLocked: false,isDataExists: _includes(pidsWithData, p?.id),@@ -168,6 +214,153 @@ export class ProjectController {}}+ @Post('/funnel')+ @ApiResponse({ status: 201 })+ @Auth([], true)+ async createFunnel(+ @Body() funnelDTO: FunnelCreateDTO,+ @CurrentUserId() userId: string,+ ): Promise<any> {+ this.logger.log({ funnelDTO, userId }, 'POST /project/funnel')++ if (!userId) {+ throw new UnauthorizedException('Please auth first')+ }++ const project = getProjectsClickhouse(funnelDTO.pid)++ if (!project) {+ throw new NotFoundException('Project not found.')+ }++ const funnel = new Funnel()+ funnel.id = uuidv4()+ funnel.name = funnelDTO.name+ funnel.steps = _map(funnelDTO.steps, _trim)+ funnel.projectId = funnelDTO.pid++ const formatted = this.projectService.formatFunnelToClickhouse(funnel)++ await createFunnelClickhouse(formatted)++ return {+ ...funnel,+ pid: funnelDTO.pid,+ project: _omit(this.projectService.formatFromClickhouse(project), [+ 'passwordHash',+ ]),+ }+ }++ @Patch('/funnel')+ @ApiResponse({ status: 200 })+ @Auth([], true)+ async updateFunnel(+ @Body() funnelDTO: FunnelUpdateDTO,+ @CurrentUserId() userId: string,+ ): Promise<void> {+ this.logger.log({ funnelDTO, userId }, 'PATCH /project/funnel')++ if (!userId) {+ throw new UnauthorizedException('Please auth first')+ }++ const project = getProjectsClickhouse(funnelDTO.pid)++ if (!project) {+ throw new NotFoundException('Project not found.')+ }++ const oldFunnel = this.projectService.formatFunnelFromClickhouse(+ await getFunnelsClickhouse(funnelDTO.pid, funnelDTO.id),+ )++ if (!oldFunnel) {+ throw new NotFoundException('Funnel not found.')+ }++ await updateFunnelClickhouse(+ this.projectService.formatFunnelToClickhouse({+ id: funnelDTO.id,+ name: funnelDTO.name,+ steps: funnelDTO.steps,+ } as Funnel),+ )+ }++ @Delete('/funnel/:id/:pid')+ @ApiResponse({ status: 200 })+ @Auth([], true)+ async deleteFunnel(+ @Param('id') id: string,+ @Param('pid') pid: string,+ @CurrentUserId() userId: string,+ ): Promise<void> {+ this.logger.log({ id, userId }, 'PATCH /project/funnel')++ if (!userId) {+ throw new UnauthorizedException('Please auth first')+ }++ const oldFunnel = getFunnelsClickhouse(pid, id)++ if (!oldFunnel) {+ throw new NotFoundException('Funnel not found.')+ }++ await deleteFunnelClickhouse(id)+ }++ @Get('/funnels/:pid')+ @ApiResponse({ status: 200 })+ @Auth([], true)+ async getFunnels(+ @Param('pid') pid: string,+ @CurrentUserId() userId: string,+ @Headers() headers: { 'x-password'?: string },+ ): Promise<any> {+ this.logger.log({ pid, userId }, 'PATCH /project/funnel')++ if (!userId) {+ throw new UnauthorizedException('Please auth first')+ }++ const project = await getProjectsClickhouse(pid)++ if (!project) {+ throw new NotFoundException('Project not found.')+ }++ this.projectService.allowedToView(project, userId, headers['x-password'])++ return getFunnelsClickhouse(pid)+ }++ @Get('password/:projectId')+ @Auth([], true, true)+ @ApiResponse({ status: 200, type: Project })+ async checkPassword(+ @Param('projectId') projectId: string,+ @CurrentUserId() userId: string,+ @Headers() headers: { 'x-password'?: string },+ ): Promise<boolean> {+ this.logger.log({ projectId }, 'GET /project/password/:projectId')++ const project = await getProjectsClickhouse(projectId)++ if (!project) {+ throw new NotFoundException('Project not found.')+ }++ try {+ this.projectService.allowedToView(project, userId, headers['x-password'])+ } catch {+ return false+ }++ return true+ }+@Delete('/partially/:pid')@ApiQuery({name: 'from',@@ -259,7 +452,7 @@ export class ProjectController {@ApiResponse({ status: 200, type: Project })async update(@Param('id') id: string,- @Body() projectDTO: ProjectDTO,+ @Body() projectDTO: UpdateProjectDto,@CurrentUserId() uid: string,): Promise<any> {this.logger.log({ projectDTO, uid, id }, 'PUT /project/:id')@@ -270,11 +463,37 @@ export class ProjectController {throw new NotFoundException()}- project.active = projectDTO.active- project.origins = _map(projectDTO.origins, _trim)- project.ipBlacklist = _map(projectDTO.ipBlacklist, _trim)- project.name = projectDTO.name- project.public = projectDTO.public+ if (_isBoolean(projectDTO.public)) {+ project.public = projectDTO.public+ }++ if (_isBoolean(projectDTO.active)) {+ project.active = projectDTO.active+ }++ if (projectDTO.origins) {+ project.origins = _map(projectDTO.origins, _trim)+ }++ if (projectDTO.ipBlacklist) {+ project.ipBlacklist = _map(projectDTO.ipBlacklist, _trim)+ }++ if (projectDTO.name) {+ project.name = _trim(projectDTO.name)+ }++ if (_isBoolean(projectDTO.isPasswordProtected)) {+ if (projectDTO.isPasswordProtected) {+ if (projectDTO.password) {+ project.isPasswordProtected = true+ project.passwordHash = await hash(projectDTO.password, 10)+ }+ } else {+ project.isPasswordProtected = false+ project.passwordHash = null+ }+ }await updateProjectClickhouse(this.projectService.formatToClickhouse(project),@@ -283,7 +502,7 @@ export class ProjectController {// await updateProjectRedis(id, project)await deleteProjectRedis(id)- return project+ return _omit(project, ['passwordHash'])}@Get('/:id')@@ -292,6 +511,7 @@ export class ProjectController {async getOne(@Param('id') id: string,@CurrentUserId() uid: string,+ @Headers() headers: { 'x-password'?: string },): Promise<Project | object> {this.logger.log({ id }, 'GET /project/:id')if (!isValidPID(id)) {@@ -306,7 +526,14 @@ export class ProjectController {throw new NotFoundException('Project was not found in the database')}- this.projectService.allowedToView(project, uid)+ if (project.isPasswordProtected && _isEmpty(headers['x-password'])) {+ return {+ isPasswordProtected: true,+ id: project.id,+ }+ }++ this.projectService.allowedToView(project, uid, headers['x-password'])const isDataExists = !_isEmpty(await this.projectService.getPIDsWhereAnalyticsDataExists([id]),@@ -316,8 +543,11 @@ export class ProjectController {await this.projectService.getPIDsWhereErrorsDataExists([id]),)+ const funnels = await getFunnelsClickhouse(id)+return this.projectService.formatFromClickhouse({- ...project,+ ..._omit(project, ['passwordHash']),+ funnels: this.projectService.formatFunnelsFromClickhouse(funnels),isDataExists,isErrorDataExists,})diff --git a/apps/selfhosted/src/project/project.service.ts b/apps/selfhosted/src/project/project.service.tsindex e845cffbe..ed8c7504c 100644--- a/apps/selfhosted/src/project/project.service.ts+++ b/apps/selfhosted/src/project/project.service.ts@@ -18,6 +18,7 @@ import * as _isNull from 'lodash/isNull'import * as _split from 'lodash/split'import * as _trim from 'lodash/trim'import * as _reduce from 'lodash/reduce'+import { compareSync } from 'bcrypt'import { Project } from './entity/project.entity'import { ProjectDTO } from './dto/project.dto'@@ -31,6 +32,8 @@ import {redisProjectCacheTimeout,} from '../common/constants'import { getProjectsClickhouse } from '../common/utils'+import { MAX_PROJECT_PASSWORD_LENGTH, UpdateProjectDto } from './dto'+import { Funnel } from './entity/funnel.entity'// A list of characters that can be used in a Project IDconst LEGAL_PID_CHARACTERS =@@ -79,7 +82,22 @@ export class ProjectService {return project}- allowedToView(project: Project, uid: string | null): void {+ allowedToView(+ project: Project,+ uid: string | null,+ password?: string | null,+ ): void {+ if (project.isPasswordProtected && password) {+ if (+ _size(password) <= MAX_PROJECT_PASSWORD_LENGTH &&+ compareSync(password, project.passwordHash)+ ) {+ return null+ }++ throw new ConflictException('Incorrect password')+ }+if (project.public || uid) {return null}@@ -99,9 +117,11 @@ export class ProjectService {}}- async getPIDsWhereAnalyticsDataExists(projectIds: string[]) {+ async getPIDsWhereAnalyticsDataExists(+ projectIds: string[],+ ): Promise<string[]> {if (_isEmpty(projectIds)) {- return {}+ return []}const params = _reduce(@@ -156,9 +176,9 @@ export class ProjectService {return _map(result, ({ pid }) => pid)}- async getPIDsWhereErrorsDataExists(projectIds: string[]) {+ async getPIDsWhereErrorsDataExists(projectIds: string[]): Promise<string[]> {if (_isEmpty(projectIds)) {- return {}+ return []}const params = _reduce(@@ -230,6 +250,7 @@ export class ProjectService {const updProject = { ...project }updProject.active = Number(updProject.active)updProject.public = Number(updProject.public)+ updProject.isPasswordProtected = Number(updProject.isPasswordProtected)if (!_isNull(updProject.origins)) {updProject.origins = _isString(updProject.origins)@@ -250,6 +271,7 @@ export class ProjectService {const updProject = { ...project }updProject.active = Boolean(updProject.active)updProject.public = Boolean(updProject.public)+ updProject.isPasswordProtected = Boolean(updProject.isPasswordProtected)updProject.origins = _isNull(updProject.origins)? []@@ -262,7 +284,38 @@ export class ProjectService {return updProject}- validateProject(projectDTO: ProjectDTO, creatingProject = false) {+ formatFunnelToClickhouse(funnel: Funnel): any {+ const updFunnel = { ...funnel }+ updFunnel.name = _trim(funnel.name)+ updFunnel.steps = _isString(updFunnel.steps)+ ? updFunnel.steps+ : _join(updFunnel.steps, ',')++ return updFunnel+ }++ formatFunnelsFromClickhouse(funnels: any[]): Funnel[] {+ if (_isEmpty(funnels)) {+ return []+ }++ return _map(funnels, this.formatFunnelFromClickhouse)+ }++ formatFunnelFromClickhouse(funnel: any): Funnel {+ const updFunnel = { ...funnel }++ updFunnel.steps = _isNull(updFunnel.steps)+ ? []+ : _split(updFunnel.steps, ',')++ return updFunnel+ }++ validateProject(+ projectDTO: ProjectDTO | UpdateProjectDto,+ creatingProject = false,+ ) {if (_size(projectDTO.name) > 50)throw new UnprocessableEntityException('The project name is too long')diff --git a/docker-compose-dev.yml b/docker-compose-dev.ymlindex 02ae8f9ec..beca87ac0 100644--- a/docker-compose-dev.yml+++ b/docker-compose-dev.yml@@ -1,11 +1,11 @@version: '3.8'services:redis:- image: redis:7.0.11-alpine+ image: redis:7.2-alpineports:- 6379:6379clickhouse:- image: clickhouse/clickhouse-server:23.4.2.11-alpine+ image: clickhouse/clickhouse-server:23.8-alpineenvironment:CLICKHOUSE_DB: 'analytics'ports:diff --git a/docker-compose-prod.yml b/docker-compose-prod.ymlindex 438a2dafc..3af77d322 100644--- a/docker-compose-prod.yml+++ b/docker-compose-prod.yml@@ -58,7 +58,7 @@ services:- 'mariadb'- 'clickhouse'redis:- image: redis:7.0.11-alpine+ image: redis:7.2-alpinerestart: alwaysenvironment:- REDIS_PORT=6379@@ -77,7 +77,7 @@ services:volumes:- swetrix-users-data:/var/lib/mysqlclickhouse:- image: clickhouse/clickhouse-server:23.4.2.11-alpine+ image: clickhouse/clickhouse-server:23.8-alpinecontainer_name: clickhouseenvironment:- CLICKHOUSE_DATABASE=analyticsdiff --git a/docker-compose.yml b/docker-compose.ymlindex 4fae851fe..adbcaffba 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -36,7 +36,7 @@ services:- 'redis'- 'clickhouse'redis:- image: redis:7.0.11-alpine+ image: redis:7.2-alpinerestart: alwaysenvironment:- REDIS_PORT=6379@@ -47,7 +47,7 @@ services:# volumes:# - '/opt/redis-volume:/data'clickhouse:- image: clickhouse/clickhouse-server:23.4.2.11-alpine+ image: clickhouse/clickhouse-server:23.8-alpinecontainer_name: clickhouseenvironment:- CLICKHOUSE_DATABASE=analyticsdiff --git a/migrations/clickhouse/2023_05_19.js b/migrations/clickhouse/2023_05_19.jsindex ca8201f23..1887deed3 100644--- a/migrations/clickhouse/2023_05_19.js+++ b/migrations/clickhouse/2023_05_19.js@@ -100,7 +100,7 @@ const queries = [)ENGINE = MergeTree()PARTITION BY toYYYYMM(created)- ORDER BY (pid, created);`+ ORDER BY (pid, created);`,`INSERT INTO ${dbName}.captcha_temp (*) SELECT * FROM ${dbName}.captcha`,diff --git a/migrations/clickhouse/initialise_selfhosted.js b/migrations/clickhouse/initialise_selfhosted.jsindex 17238a469..6517518dc 100644--- a/migrations/clickhouse/initialise_selfhosted.js+++ b/migrations/clickhouse/initialise_selfhosted.js@@ -11,6 +11,8 @@ const CLICKHOUSE_INIT_QUERIES = [ipBlacklist Nullable(String),active Int8,public Int8,+ isPasswordProtected Int8,+ passwordHash Nullable(String),created DateTime)ENGINE = MergeTree()@@ -34,6 +36,17 @@ const CLICKHOUSE_INIT_QUERIES = [)ENGINE = MergeTree()PRIMARY KEY id;`,++ `CREATE TABLE IF NOT EXISTS ${dbName}.funnel+ (+ id String,+ name String,+ steps String,+ projectId FixedString(12),+ created DateTime+ )+ ENGINE = MergeTree()+ PRIMARY KEY id;`,]const initialiseSelfhosted = async () => {@@ -41,7 +54,9 @@ const initialiseSelfhosted = async () => {await initialiseDatabase()await queriesRunner(CLICKHOUSE_INIT_QUERIES)} catch (reason) {- console.error(`[ERROR] Error occured whilst initialising the database: ${reason}`)+ console.error(+ `[ERROR] Error occured whilst initialising the database: ${reason}`,+ )}}diff --git a/migrations/clickhouse/selfhosted_2023_05_28.js b/migrations/clickhouse/selfhosted_2023_05_28.jsindex 6c034d0f3..453632c8b 100644--- a/migrations/clickhouse/selfhosted_2023_05_28.js+++ b/migrations/clickhouse/selfhosted_2023_05_28.js@@ -7,7 +7,7 @@ const queries = [(id String,timezone Nullable(String),- timeFormat Nullable(String)+ timeFormat Nullable(String),showLiveVisitorsInTitle Int8)ENGINE = MergeTree()diff --git a/migrations/clickhouse/selfhosted_2023_10_11.js b/migrations/clickhouse/selfhosted_2023_10_11.jsindex 1275bd507..dbf1821d1 100644--- a/migrations/clickhouse/selfhosted_2023_10_11.js+++ b/migrations/clickhouse/selfhosted_2023_10_11.js@@ -1,7 +1,7 @@const { queriesRunner, dbName } = require('./setup')const queries = [- // Added 'showLiveVisitorsInTitle' column+ // Made 'showLiveVisitorsInTitle' column nullable`DROP TABLE IF EXISTS ${dbName}.sfuser_temp`,`CREATE TABLE IF NOT EXISTS ${dbName}.sfuser_temp(diff --git a/migrations/clickhouse/selfhosted_2024_05_13.js b/migrations/clickhouse/selfhosted_2024_05_13.jsnew file mode 100644index 000000000..d07192b38--- /dev/null+++ b/migrations/clickhouse/selfhosted_2024_05_13.js@@ -0,0 +1,41 @@+const { queriesRunner, dbName } = require('./setup')++const queries = [+ // Added password-related columns+ `DROP TABLE IF EXISTS ${dbName}.project_temp`,+ `CREATE TABLE IF NOT EXISTS ${dbName}.project_temp+ (+ id FixedString(12),+ name String,+ origins Nullable(String),+ ipBlacklist Nullable(String),+ active Int8,+ public Int8,+ isPasswordProtected Int8,+ passwordHash Nullable(String),+ created DateTime+ )+ ENGINE = MergeTree()+ PARTITION BY toYYYYMM(created)+ ORDER BY (created);`,++ `INSERT INTO ${dbName}.project_temp (*)+ SELECT id, name, origins, ipBlacklist, active, public, 0, NULL, created FROM ${dbName}.project`,++ `DROP TABLE ${dbName}.project`,+ `RENAME TABLE ${dbName}.project_temp TO ${dbName}.project`,++ // Added funnel table+ `CREATE TABLE IF NOT EXISTS ${dbName}.funnel+ (+ id String,+ name String,+ steps String,+ projectId FixedString(12),+ created DateTime+ )+ ENGINE = MergeTree()+ PRIMARY KEY id;`,+]++queriesRunner(queries)diff --git a/migrations/clickhouse/selfhosted_v2_to_v3.js b/migrations/clickhouse/selfhosted_v2_to_v3.jsnew file mode 100644index 000000000..595fb947c--- /dev/null+++ b/migrations/clickhouse/selfhosted_v2_to_v3.js@@ -0,0 +1,48 @@+const childProcess = require('child_process')+const path = require('path')++function runScript(scriptPath) {+ return new Promise((resolve, reject) => {+ const fullPath = path.join(__dirname, scriptPath)+ console.log('[INFO] Running script: ' + fullPath)++ const process = childProcess.fork(fullPath)+ let invoked = false++ // listen for errors as they may prevent the exit event from firing+ process.on('error', function (err) {+ if (invoked) return+ invoked = true+ reject(err)+ })++ // execute the callback once the process has finished running+ process.on('exit', function (code) {+ if (invoked) return+ invoked = true+ const err = code === 0 ? null : new Error('exit code ' + code)+ err ? reject(err) : resolve()+ })+ })+}++async function runMigrations() {+ try {+ await runScript('./initialise_database.js')+ await runScript('./initialise_selfhosted.js')+ await runScript('./2023_05_19.js')+ await runScript('./2023_06_26.js')+ await runScript('./2023_10_13.js')+ await runScript('./2023_10_18.js')+ await runScript('./2024_03_30.js')+ await runScript('./selfhosted_2023_05_28.js')+ await runScript('./selfhosted_2023_10_11.js')+ await runScript('./selfhosted_2024_05_13.js')++ console.log('[INFO] SELFHOSTED MIGRATIONS FINISHED')+ } catch (err) {+ console.error('Error running migration:', err)+ }+}++runMigrations()