Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

(improvement) Preparing for a new selfhosted release (#235)

Andrii Romasiun06587de65afd
diff --git a/.env.example b/.env.example
index 1e02c3cee..6c761faa6 100644
--- a/.env.example
+++ b/.env.example
@@ -44,6 +44,7 @@ SMTP_MOCK=true
# Selfhosted
PASSWORD=12345678
+API_KEY=
# Swetrix CDN for storing files
CDN_URL=http://localhost:5006
diff --git a/Dockerfile b/Dockerfile
index 427bc4584..dbaa9d5b6 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -20,7 +20,8 @@ ENV TZ=UTC \
CLICKHOUSE_DATABASE=analytics \
API_ORIGINS=\
- PASSWORD=12345678
+ PASSWORD=12345678 \
+ API_KEY=
RUN apk add --no-cache tzdata && cp /usr/share/zoneinfo/$TZ /etc/localtime
WORKDIR /app
COPY --from=build /build/package*.json ./
diff --git a/apps/production/src/analytics/analytics.service.ts b/apps/production/src/analytics/analytics.service.ts
index 66b79a16e..6671c37cf 100644
--- a/apps/production/src/analytics/analytics.service.ts
+++ b/apps/production/src/analytics/analytics.service.ts
@@ -824,7 +824,7 @@ export class AnalyticsService {
let diff = null
if (from && to) {
- diff = dayjs(to).diff(dayjs(from?.[0].created || to), 'days')
+ diff = dayjs(to).diff(dayjs(from?.[0]?.created || to), 'days')
const tbMap = _find(timeBucketToDays, ({ lt }) => diff <= lt)
diff --git a/apps/production/src/project/project.service.ts b/apps/production/src/project/project.service.ts
index 0f22654c0..cab9588ef 100644
--- a/apps/production/src/project/project.service.ts
+++ b/apps/production/src/project/project.service.ts
@@ -838,9 +838,11 @@ export class ProjectService {
})
}
- async getPIDsWhereAnalyticsDataExists(projectIds: string[]) {
+ async getPIDsWhereAnalyticsDataExists(
+ projectIds: string[],
+ ): Promise<string[]> {
if (_isEmpty(projectIds)) {
- return {}
+ return []
}
const params = _reduce(
@@ -895,9 +897,9 @@ export class ProjectService {
return _map(result, ({ pid }) => pid)
}
- async getPIDsWhereErrorsDataExists(projectIds: string[]) {
+ async getPIDsWhereErrorsDataExists(projectIds: string[]): Promise<string[]> {
if (_isEmpty(projectIds)) {
- return {}
+ return []
}
const params = _reduce(
diff --git a/apps/selfhosted/src/analytics/analytics.controller.ts b/apps/selfhosted/src/analytics/analytics.controller.ts
index 1b1725c87..3af2f00a4 100644
--- a/apps/selfhosted/src/analytics/analytics.controller.ts
+++ b/apps/selfhosted/src/analytics/analytics.controller.ts
@@ -6,9 +6,6 @@ import * as _uniqBy from 'lodash/uniqBy'
import * as _round from 'lodash/round'
import * as _includes from 'lodash/includes'
import * as _keys from 'lodash/keys'
-import * as _size from 'lodash/size'
-import * as _some from 'lodash/some'
-import * as _isString from 'lodash/isString'
import * as _values from 'lodash/values'
import * as dayjs from 'dayjs'
import * as utc from 'dayjs/plugin/utc'
@@ -62,8 +59,6 @@ import {
REDIS_LOG_CUSTOM_CACHE_KEY,
HEARTBEAT_SID_LIFE_TIME,
REDIS_SESSION_SALT_KEY,
- MIN_PAGES_IN_FUNNEL,
- MAX_PAGES_IN_FUNNEL,
clickhouse,
REDIS_LOG_ERROR_CACHE_KEY,
} from '../common/constants'
@@ -348,44 +343,6 @@ const getEIDsArray = (eids, eid) => {
return [eid]
}
-const getPagesArray = (rawPages: string): string[] => {
- try {
- const pages = JSON.parse(rawPages)
-
- if (!_isArray(pages)) {
- throw new UnprocessableEntityException(
- 'An array of pages has to be provided as a pages param',
- )
- }
-
- const size = _size(pages)
-
- if (size < MIN_PAGES_IN_FUNNEL) {
- throw new UnprocessableEntityException(
- `A minimum of ${MIN_PAGES_IN_FUNNEL} pages or events has to be provided`,
- )
- }
-
- if (size > MAX_PAGES_IN_FUNNEL) {
- throw new UnprocessableEntityException(
- `A maximum of ${MAX_PAGES_IN_FUNNEL} pages or events can be provided`,
- )
- }
-
- if (_some(pages, page => !_isString(page))) {
- throw new UnprocessableEntityException(
- 'Pages array must contain string values only',
- )
- }
-
- return pages
- } catch (e) {
- throw new UnprocessableEntityException(
- 'Cannot process the provided array of pages',
- )
- }
-}
-
// needed for serving 1x1 px GIF
const TRANSPARENT_GIF_BUFFER = Buffer.from(
'R0lGODlhAQABAIAAAP///wAAACwAAAAAAQABAAACAkQBADs=',
@@ -406,6 +363,7 @@ export class AnalyticsController {
async getData(
@Query() data: AnalyticsGET_DTO,
@CurrentUserId() uid: string,
+ @Headers() headers: { 'x-password'?: string },
): Promise<any> {
const {
pid,
@@ -423,7 +381,11 @@ export class AnalyticsController {
this.analyticsService.validatePeriod(period)
}
- await this.analyticsService.checkProjectAccess(pid, uid)
+ await this.analyticsService.checkProjectAccess(
+ pid,
+ uid,
+ headers['x-password'],
+ )
let newTimebucket = timeBucket
let allowedTumebucketForPeriodAll
@@ -509,17 +471,34 @@ export class AnalyticsController {
async getFunnel(
@Query() data: GetFunnelsDTO,
@CurrentUserId() uid: string,
+ @Headers() headers: { 'x-password'?: string },
): Promise<IGetFunnel> {
- const { pid, period, from, to, timezone = DEFAULT_TIMEZONE, pages } = data
+ const {
+ pid,
+ period,
+ from,
+ to,
+ timezone = DEFAULT_TIMEZONE,
+ pages,
+ funnelId,
+ } = data
this.analyticsService.validatePID(pid)
if (!_isEmpty(period)) {
this.analyticsService.validatePeriod(period)
}
- const pagesArr = getPagesArray(pages)
+ const pagesArr = await this.analyticsService.getPagesArray(
+ pages,
+ funnelId,
+ pid,
+ )
- await this.analyticsService.checkProjectAccess(pid, uid)
+ await this.analyticsService.checkProjectAccess(
+ pid,
+ uid,
+ headers['x-password'],
+ )
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const { groupFrom, groupTo } = this.analyticsService.getGroupFromTo(
@@ -565,6 +544,7 @@ export class AnalyticsController {
async getCustomEventMetadata(
@Query() data: GetCustomEventMetadata,
@CurrentUserId() uid: string,
+ @Headers() headers: { 'x-password'?: string },
): Promise<IAggregatedMetadata[]> {
const { pid, period } = data
this.analyticsService.validatePID(pid)
@@ -573,7 +553,11 @@ export class AnalyticsController {
this.analyticsService.validatePeriod(period)
}
- await this.analyticsService.checkProjectAccess(pid, uid)
+ await this.analyticsService.checkProjectAccess(
+ pid,
+ uid,
+ headers['x-password'],
+ )
return this.analyticsService.getCustomEventMetadata(data)
}
@@ -583,11 +567,16 @@ export class AnalyticsController {
async getFilters(
@Query() data: GetFiltersDto,
@CurrentUserId() uid: string,
+ @Headers() headers: { 'x-password'?: string },
): Promise<any> {
const { pid, type } = data
this.analyticsService.validatePID(pid)
- await this.analyticsService.checkProjectAccess(pid, uid)
+ await this.analyticsService.checkProjectAccess(
+ pid,
+ uid,
+ headers['x-password'],
+ )
return this.analyticsService.getFilters(pid, type)
}
@@ -597,6 +586,7 @@ export class AnalyticsController {
async getChartData(
@Query() data: AnalyticsGET_DTO,
@CurrentUserId() uid: string,
+ @Headers() headers: { 'x-password'?: string },
): Promise<any> {
const {
pid,
@@ -626,7 +616,11 @@ export class AnalyticsController {
period,
safeTimezone,
)
- await this.analyticsService.checkProjectAccess(pid, uid)
+ await this.analyticsService.checkProjectAccess(
+ pid,
+ uid,
+ headers['x-password'],
+ )
const paramsData = {
params: {
@@ -659,6 +653,7 @@ export class AnalyticsController {
async getPerfData(
@Query() data: AnalyticsGET_DTO & { measure: PerfMeasure },
@CurrentUserId() uid: string,
+ @Headers() headers: { 'x-password'?: string },
): Promise<any> {
const {
pid,
@@ -678,7 +673,11 @@ export class AnalyticsController {
this.analyticsService.validatePeriod(period)
}
- await this.analyticsService.checkProjectAccess(pid, uid)
+ await this.analyticsService.checkProjectAccess(
+ pid,
+ uid,
+ headers['x-password'],
+ )
let newTimeBucket = timeBucket
let allowedTumebucketForPeriodAll
@@ -747,6 +746,7 @@ export class AnalyticsController {
async getPerfChartData(
@Query() data: AnalyticsGET_DTO & { measure: PerfMeasure },
@CurrentUserId() uid: string,
+ @Headers() headers: { 'x-password'?: string },
): Promise<any> {
const {
pid,
@@ -778,7 +778,11 @@ export class AnalyticsController {
period,
safeTimezone,
)
- await this.analyticsService.checkProjectAccess(pid, uid)
+ await this.analyticsService.checkProjectAccess(
+ pid,
+ uid,
+ headers['x-password'],
+ )
const paramsData = {
params: {
@@ -810,6 +814,7 @@ export class AnalyticsController {
async getUserFlow(
@Query() data: GetUserFlowDTO,
@CurrentUserId() uid: string,
+ @Headers() headers: { 'x-password'?: string },
): Promise<IUserFlow | { appliedFilters: any[] }> {
const { pid, period, from, to, timezone = DEFAULT_TIMEZONE, filters } = data
this.analyticsService.validatePID(pid)
@@ -818,7 +823,11 @@ export class AnalyticsController {
this.analyticsService.validatePeriod(period)
}
- await this.analyticsService.checkProjectAccess(pid, uid)
+ await this.analyticsService.checkProjectAccess(
+ pid,
+ uid,
+ headers['x-password'],
+ )
let diff
@@ -866,6 +875,7 @@ export class AnalyticsController {
async getOverallStats(
@Query() data,
@CurrentUserId() uid: string,
+ @Headers() headers: { 'x-password'?: string },
): Promise<any> {
const {
pids,
@@ -880,7 +890,11 @@ export class AnalyticsController {
const validationPromises = _map(pidsArray, async currentPID => {
this.analyticsService.validatePID(currentPID)
- await this.analyticsService.checkProjectAccess(currentPID, uid)
+ await this.analyticsService.checkProjectAccess(
+ currentPID,
+ uid,
+ headers['x-password'],
+ )
})
await Promise.all(validationPromises)
@@ -900,6 +914,7 @@ export class AnalyticsController {
async getPerformanceOverallStats(
@Query() data,
@CurrentUserId() uid: string,
+ @Headers() headers: { 'x-password'?: string },
): Promise<any> {
const {
pids,
@@ -921,7 +936,11 @@ export class AnalyticsController {
const validationPromises = _map(pidsArray, async currentPID => {
this.analyticsService.validatePID(currentPID)
- await this.analyticsService.checkProjectAccess(currentPID, uid)
+ await this.analyticsService.checkProjectAccess(
+ currentPID,
+ uid,
+ headers['x-password'],
+ )
})
await Promise.all(validationPromises)
@@ -941,13 +960,18 @@ export class AnalyticsController {
async getHeartBeatStats(
@Query() data,
@CurrentUserId() uid: string,
+ @Headers() headers: { 'x-password'?: string },
): Promise<object> {
const { pids, pid } = data
const pidsArray = getPIDsArray(pids, pid)
const validationPromises = _map(pidsArray, async currentPID => {
this.analyticsService.validatePID(currentPID)
- await this.analyticsService.checkProjectAccess(currentPID, uid)
+ await this.analyticsService.checkProjectAccess(
+ currentPID,
+ uid,
+ headers['x-password'],
+ )
})
await Promise.all(validationPromises)
@@ -974,11 +998,16 @@ export class AnalyticsController {
async getLiveVisitors(
@Query() data,
@CurrentUserId() uid: string,
+ @Headers() headers: { 'x-password'?: string },
): Promise<object> {
const { pid } = data
this.analyticsService.validatePID(pid)
- await this.analyticsService.checkProjectAccess(pid, uid)
+ await this.analyticsService.checkProjectAccess(
+ pid,
+ uid,
+ headers['x-password'],
+ )
const keys = await redis.keys(`sd:*:${pid}`)
@@ -1297,6 +1326,7 @@ export class AnalyticsController {
async getSessions(
@Query() data: GetSessionsDto,
@CurrentUserId() uid: string,
+ @Headers() headers: { 'x-password'?: string },
): Promise<any> {
const { pid, period, from, to, filters, timezone = DEFAULT_TIMEZONE } = data
this.analyticsService.validatePID(pid)
@@ -1305,7 +1335,11 @@ export class AnalyticsController {
this.analyticsService.validatePeriod(period)
}
- await this.analyticsService.checkProjectAccess(pid, uid)
+ await this.analyticsService.checkProjectAccess(
+ pid,
+ uid,
+ headers['x-password'],
+ )
const take = this.analyticsService.getSafeNumber(data.take, 30)
const skip = this.analyticsService.getSafeNumber(data.skip, 0)
@@ -1378,6 +1412,7 @@ export class AnalyticsController {
async getSession(
@Query() data: GetSessionDto,
@CurrentUserId() uid: string,
+ @Headers() headers: { 'x-password'?: string },
): Promise<any> {
const {
pid,
@@ -1387,7 +1422,11 @@ export class AnalyticsController {
} = data
this.analyticsService.validatePID(pid)
- await this.analyticsService.checkProjectAccess(pid, uid)
+ await this.analyticsService.checkProjectAccess(
+ pid,
+ uid,
+ headers['x-password'],
+ )
const safeTimezone = this.analyticsService.getSafeTimezone(timezone)
const result = await this.analyticsService.getSessionDetails(
@@ -1404,6 +1443,7 @@ export class AnalyticsController {
async getCustomEvents(
@Query() data: GetCustomEventsDto,
@CurrentUserId() uid: string,
+ @Headers() headers: { 'x-password'?: string },
): Promise<any> {
const {
pid,
@@ -1421,7 +1461,11 @@ export class AnalyticsController {
this.analyticsService.validatePeriod(period)
}
- await this.analyticsService.checkProjectAccess(pid, uid)
+ await this.analyticsService.checkProjectAccess(
+ pid,
+ uid,
+ headers['x-password'],
+ )
let newTimeBucket = timeBucket
let diff
@@ -1505,11 +1549,16 @@ export class AnalyticsController {
async getErrorsFilters(
@Query() data: GetFiltersDto,
@CurrentUserId() uid: string,
+ @Headers() headers: { 'x-password'?: string },
): Promise<any> {
const { pid, type } = data
this.analyticsService.validatePID(pid)
- await this.analyticsService.checkProjectAccess(pid, uid)
+ await this.analyticsService.checkProjectAccess(
+ pid,
+ uid,
+ headers['x-password'],
+ )
return this.analyticsService.getErrorsFilters(pid, type)
}
@@ -1587,7 +1636,11 @@ export class AnalyticsController {
await checkRateLimit(ip, 'error-status', 100, 1800)
this.analyticsService.validatePID(pid)
- await this.analyticsService.checkProjectAccess(pid, uid)
+ await this.analyticsService.checkProjectAccess(
+ pid,
+ uid,
+ headers['x-password'],
+ )
const eids = getEIDsArray(unprocessedEids, eid)
@@ -1601,6 +1654,7 @@ export class AnalyticsController {
async getErrors(
@Query() data: GetErrorsDto,
@CurrentUserId() uid: string,
+ @Headers() headers: { 'x-password'?: string },
): Promise<any> {
const {
pid,
@@ -1617,7 +1671,11 @@ export class AnalyticsController {
this.analyticsService.validatePeriod(period)
}
- await this.analyticsService.checkProjectAccess(pid, uid)
+ await this.analyticsService.checkProjectAccess(
+ pid,
+ uid,
+ headers['x-password'],
+ )
const take = this.analyticsService.getSafeNumber(data.take, 30)
const skip = this.analyticsService.getSafeNumber(data.skip, 0)
@@ -1690,6 +1748,7 @@ export class AnalyticsController {
async getError(
@Query() data: GetErrorDTO,
@CurrentUserId() uid: string,
+ @Headers() headers: { 'x-password'?: string },
): Promise<any> {
const {
pid,
@@ -1706,7 +1765,11 @@ export class AnalyticsController {
this.analyticsService.validatePeriod(period)
}
- await this.analyticsService.checkProjectAccess(pid, uid)
+ await this.analyticsService.checkProjectAccess(
+ pid,
+ uid,
+ headers['x-password'],
+ )
let timeBucket
let diff
diff --git a/apps/selfhosted/src/analytics/analytics.service.ts b/apps/selfhosted/src/analytics/analytics.service.ts
index 4d986e33a..2eb1cff05 100644
--- a/apps/selfhosted/src/analytics/analytics.service.ts
+++ b/apps/selfhosted/src/analytics/analytics.service.ts
@@ -46,8 +46,14 @@ import {
TRAFFIC_COLUMNS,
PERFORMANCE_COLUMNS,
ERROR_COLUMNS,
+ MIN_PAGES_IN_FUNNEL,
+ MAX_PAGES_IN_FUNNEL,
} from '../common/constants'
-import { millisecondsToSeconds, sumArrays } from '../common/utils'
+import {
+ getFunnelsClickhouse,
+ millisecondsToSeconds,
+ sumArrays,
+} from '../common/utils'
import { PageviewsDTO } from './dto/pageviews.dto'
import { EventsDTO } from './dto/events.dto'
import { ProjectService } from '../project/project.service'
@@ -332,9 +338,13 @@ const isValidOrigin = (origins: string[], origin: string) => {
export class AnalyticsService {
constructor(private readonly projectService: ProjectService) {}
- async checkProjectAccess(pid: string, uid: string | null): Promise<void> {
+ async checkProjectAccess(
+ pid: string,
+ uid: string | null,
+ password: string | null,
+ ): Promise<void> {
const project = await this.projectService.getRedisProject(pid)
- this.projectService.allowedToView(project, uid)
+ this.projectService.allowedToView(project, uid, password)
}
checkOrigin(project: Project, origin: string): void {
@@ -705,6 +715,62 @@ export class AnalyticsService {
}
}
+ async getPagesArray(
+ rawPages?: string,
+ funnelId?: string,
+ projectId?: string,
+ ): Promise<string[]> {
+ if (funnelId && projectId) {
+ const funnel = this.projectService.formatFunnelFromClickhouse(
+ await getFunnelsClickhouse(projectId, funnelId),
+ )
+
+ if (!funnel || _isEmpty(funnel)) {
+ throw new UnprocessableEntityException(
+ 'The provided funnelId is incorrect',
+ )
+ }
+
+ return funnel.steps
+ }
+
+ try {
+ const pages = JSON.parse(rawPages)
+
+ if (!_isArray(pages)) {
+ throw new UnprocessableEntityException(
+ 'An array of pages has to be provided as a pages param',
+ )
+ }
+
+ const size = _size(pages)
+
+ if (size < MIN_PAGES_IN_FUNNEL) {
+ throw new UnprocessableEntityException(
+ `A minimum of ${MIN_PAGES_IN_FUNNEL} pages or events has to be provided`,
+ )
+ }
+
+ if (size > MAX_PAGES_IN_FUNNEL) {
+ throw new UnprocessableEntityException(
+ `A maximum of ${MAX_PAGES_IN_FUNNEL} pages or events can be provided`,
+ )
+ }
+
+ if (_some(pages, page => !_isString(page))) {
+ throw new UnprocessableEntityException(
+ 'Pages array must contain string values only',
+ )
+ }
+
+ return pages
+ } catch (e) {
+ throw new UnprocessableEntityException(
+ 'Cannot process the provided array of pages',
+ )
+ }
+ }
+
async getTimeBucketForAllTime(
pid: string,
period: string,
@@ -731,7 +797,7 @@ export class AnalyticsService {
let diff = null
if (from && to) {
- diff = dayjs(to).diff(dayjs(from?.[0].created || to), 'days')
+ diff = dayjs(to).diff(dayjs(from?.[0]?.created || to), 'days')
const tbMap = _find(timeBucketToDays, ({ lt }) => diff <= lt)
@@ -932,7 +998,7 @@ export class AnalyticsService {
let dropoff = 0
let eventsPerc = 100
let eventsPercStep = 100
- let dropoffPerc = 0
+ let dropoffPercStep = 0
if (index > 0) {
const prev = data[index - 1]
@@ -940,7 +1006,7 @@ export class AnalyticsService {
dropoff = prev.c - row.c
eventsPerc = _round((row.c / data[0].c) * 100, 2)
eventsPercStep = _round((row.c / prev.c) * 100, 2)
- dropoffPerc = _round((dropoff / prev.c) * 100, 2)
+ dropoffPercStep = _round((dropoff / prev.c) * 100, 2)
}
return {
@@ -949,7 +1015,7 @@ export class AnalyticsService {
eventsPerc,
eventsPercStep,
dropoff,
- dropoffPerc,
+ dropoffPercStep,
}
})
diff --git a/apps/selfhosted/src/analytics/dto/getFunnels.dto.ts b/apps/selfhosted/src/analytics/dto/getFunnels.dto.ts
index afde79179..197895faa 100644
--- a/apps/selfhosted/src/analytics/dto/getFunnels.dto.ts
+++ b/apps/selfhosted/src/analytics/dto/getFunnels.dto.ts
@@ -27,4 +27,9 @@ export class GetFunnelsDTO {
description: 'A stringified array of pages to generate funnel for',
})
pages: string
+
+ @ApiProperty({
+ description: 'Funnel ID',
+ })
+ funnelId?: string
}
diff --git a/apps/selfhosted/src/analytics/interfaces/index.ts b/apps/selfhosted/src/analytics/interfaces/index.ts
index 88de841f8..770ada1ad 100644
--- a/apps/selfhosted/src/analytics/interfaces/index.ts
+++ b/apps/selfhosted/src/analytics/interfaces/index.ts
@@ -111,7 +111,7 @@ export interface IFunnel {
eventsPerc: number
eventsPercStep: number
dropoff: number
- dropoffPerc: number
+ dropoffPercStep: number
}
export interface IGetFunnel {
diff --git a/apps/selfhosted/src/auth/auth.module.ts b/apps/selfhosted/src/auth/auth.module.ts
index 05e51b708..fb56a08e3 100644
--- a/apps/selfhosted/src/auth/auth.module.ts
+++ b/apps/selfhosted/src/auth/auth.module.ts
@@ -4,7 +4,7 @@ import { PassportModule } from '@nestjs/passport'
import { AuthController } from './auth.controller'
import { AuthService } from './auth.service'
import {
- // ApiKeyStrategy,
+ ApiKeyStrategy,
JwtAccessTokenStrategy,
JwtRefreshTokenStrategy,
} from './strategies'
@@ -16,7 +16,7 @@ import {
AuthService,
JwtAccessTokenStrategy,
JwtRefreshTokenStrategy,
- // ApiKeyStrategy,
+ ApiKeyStrategy,
],
exports: [AuthService],
})
diff --git a/apps/selfhosted/src/auth/auth.service.ts b/apps/selfhosted/src/auth/auth.service.ts
index 5284b63f9..ebf5e8f95 100644
--- a/apps/selfhosted/src/auth/auth.service.ts
+++ b/apps/selfhosted/src/auth/auth.service.ts
@@ -15,6 +15,7 @@ import {
JWT_ACCESS_TOKEN_SECRET,
JWT_ACCESS_TOKEN_LIFETIME,
JWT_REFRESH_TOKEN_LIFETIME,
+ SELFHOSTED_API_KEY,
} from '../common/constants'
import { getSelfhostedUser, SelfhostedUser } from '../user/entities/user.entity'
@@ -111,4 +112,8 @@ export class AuthService {
async logout(userId: string, refreshToken: string) {
await deleteRefreshTokenClickhouse(userId, refreshToken)
}
+
+ isApiKeyValid(apiKey: string): boolean {
+ return apiKey === SELFHOSTED_API_KEY
+ }
}
diff --git a/apps/selfhosted/src/auth/decorators/auth.decorator.ts b/apps/selfhosted/src/auth/decorators/auth.decorator.ts
index 25a30e4b9..b08e90931 100644
--- a/apps/selfhosted/src/auth/decorators/auth.decorator.ts
+++ b/apps/selfhosted/src/auth/decorators/auth.decorator.ts
@@ -1,10 +1,6 @@
import { applyDecorators, SetMetadata, UseGuards } from '@nestjs/common'
-import {
- JwtAccessTokenGuard,
- MultiAuthGuard,
- RolesGuard,
-} from 'src/auth/guards'
-import { UserType } from 'src/user/entities/user.entity'
+import { JwtAccessTokenGuard, MultiAuthGuard, RolesGuard } from '../guards'
+import { UserType } from '../../user/entities/user.entity'
import { ROLES_KEY } from './roles.decorator'
export const IS_OPTIONAL_AUTH_KEY = 'isOptionalAuth'
diff --git a/apps/selfhosted/src/auth/guards/api-key.guard.ts b/apps/selfhosted/src/auth/guards/api-key.guard.ts
new file mode 100644
index 000000000..43e58ce2d
--- /dev/null
+++ b/apps/selfhosted/src/auth/guards/api-key.guard.ts
@@ -0,0 +1,5 @@
+import { Injectable } from '@nestjs/common'
+import { AuthGuard } from '@nestjs/passport'
+
+@Injectable()
+export class ApiKeyGuard extends AuthGuard('api-key') {}
diff --git a/apps/selfhosted/src/auth/guards/multi-auth.guard.ts b/apps/selfhosted/src/auth/guards/multi-auth.guard.ts
index 3426269d9..96503daa0 100644
--- a/apps/selfhosted/src/auth/guards/multi-auth.guard.ts
+++ b/apps/selfhosted/src/auth/guards/multi-auth.guard.ts
@@ -4,7 +4,7 @@ import { AuthGuard } from '@nestjs/passport'
import { IS_OPTIONAL_AUTH_KEY } from '../decorators'
@Injectable()
-export class MultiAuthGuard extends AuthGuard(['jwt-access-token']) {
+export class MultiAuthGuard extends AuthGuard(['jwt-access-token', 'api-key']) {
constructor(private readonly reflector: Reflector) {
super()
}
diff --git a/apps/selfhosted/src/auth/guards/roles.guard.ts b/apps/selfhosted/src/auth/guards/roles.guard.ts
index 210476d16..1a34aab68 100644
--- a/apps/selfhosted/src/auth/guards/roles.guard.ts
+++ b/apps/selfhosted/src/auth/guards/roles.guard.ts
@@ -3,8 +3,8 @@ import { Reflector } from '@nestjs/core'
import { ExtractJwt } from 'passport-jwt'
import { verify } from 'jsonwebtoken'
-import { UserType } from 'src/user/entities/user.entity'
-import { JWT_ACCESS_TOKEN_SECRET } from 'src/common/constants'
+import { UserType } from '../../user/entities/user.entity'
+import { JWT_ACCESS_TOKEN_SECRET } from '../../common/constants'
import { ROLES_KEY } from '../decorators'
@Injectable()
diff --git a/apps/selfhosted/src/auth/strategies/api-key.strategy.ts b/apps/selfhosted/src/auth/strategies/api-key.strategy.ts
new file mode 100644
index 000000000..9e2a4df31
--- /dev/null
+++ b/apps/selfhosted/src/auth/strategies/api-key.strategy.ts
@@ -0,0 +1,26 @@
+import { Injectable, UnauthorizedException } from '@nestjs/common'
+import { PassportStrategy } from '@nestjs/passport'
+import { HeaderAPIKeyStrategy } from 'passport-headerapikey'
+import { AuthService } from '../auth.service'
+import { getSelfhostedUser } from '../../user/entities/user.entity'
+
+@Injectable()
+export class ApiKeyStrategy extends PassportStrategy(
+ HeaderAPIKeyStrategy,
+ 'api-key',
+) {
+ constructor(private readonly authService: AuthService) {
+ super(
+ { header: 'X-Api-Key', prefix: '' },
+ true,
+ // eslint-disable-next-line consistent-return
+ async (apiKey: string, done: any) => {
+ const isValid = this.authService.isApiKeyValid(apiKey)
+ if (!isValid) return done(new UnauthorizedException(), false)
+
+ const user = await getSelfhostedUser()
+ done(null, user)
+ },
+ )
+ }
+}
diff --git a/apps/selfhosted/src/auth/strategies/index.ts b/apps/selfhosted/src/auth/strategies/index.ts
index 20e4bbdce..f544f42d4 100644
--- a/apps/selfhosted/src/auth/strategies/index.ts
+++ b/apps/selfhosted/src/auth/strategies/index.ts
@@ -1,2 +1,3 @@
export * from './jwt-access-token.strategy'
export * from './jwt-refresh-token.strategy'
+export * from './api-key.strategy'
diff --git a/apps/selfhosted/src/common/constants.ts b/apps/selfhosted/src/common/constants.ts
index 89220bfc3..226134eca 100644
--- a/apps/selfhosted/src/common/constants.ts
+++ b/apps/selfhosted/src/common/constants.ts
@@ -56,6 +56,8 @@ const isDevelopment = process.env.NODE_ENV === 'development'
const SELFHOSTED_EMAIL = process.env.EMAIL
const SELFHOSTED_PASSWORD = process.env.PASSWORD
+const SELFHOSTED_API_KEY = process.env.API_KEY
+const SELFHOSTED_API_AUTH_ENABLED = !!SELFHOSTED_API_KEY
const UUIDV5_NAMESPACE = '912c64c1-73fd-42b6-859f-785f839a9f68'
const DEFAULT_SELFHOSTED_UUID = 'deadbeef-dead-beef-dead-beefdeadbeef'
@@ -129,6 +131,8 @@ export {
UUIDV5_NAMESPACE,
SELFHOSTED_EMAIL,
SELFHOSTED_PASSWORD,
+ SELFHOSTED_API_KEY,
+ SELFHOSTED_API_AUTH_ENABLED,
SELFHOSTED_UUID,
REDIS_USERS_COUNT_KEY,
REDIS_PROJECTS_COUNT_KEY,
diff --git a/apps/selfhosted/src/common/utils.ts b/apps/selfhosted/src/common/utils.ts
index 20cf585de..510346f4a 100644
--- a/apps/selfhosted/src/common/utils.ts
+++ b/apps/selfhosted/src/common/utils.ts
@@ -10,6 +10,7 @@ import * as _keys from 'lodash/keys'
import * as _toNumber from 'lodash/toNumber'
import * as _split from 'lodash/split'
import * as _isEmpty from 'lodash/isEmpty'
+import * as _isNil from 'lodash/isNil'
import * as _head from 'lodash/head'
import * as _round from 'lodash/round'
import * as _size from 'lodash/size'
@@ -28,20 +29,25 @@ import {
} from './constants'
import { DEFAULT_TIMEZONE, TimeFormat } from '../user/entities/user.entity'
import { Project } from '../project/entity/project.entity'
+import { Funnel } from '../project/entity/funnel.entity'
dayjs.extend(utc)
const RATE_LIMIT_REQUESTS_AMOUNT = 3
const RATE_LIMIT_TIMEOUT = 86400 // 24 hours
-const allowedToUpdateKeys = [
+const ALLOWED_KEYS = [
'name',
'origins',
'ipBlacklist',
'active',
'public',
+ 'isPasswordProtected',
+ 'passwordHash',
]
+const ALLOWED_FUNNEL_KEYS = ['name', 'steps']
+
const getRateLimitHash = (ipOrApiKey: string, salt = '') =>
`rl:${hash(`${ipOrApiKey}${salt}`).toString('hex')}`
@@ -65,9 +71,86 @@ const checkRateLimit = async (
await redis.set(rlHash, 1 + rlCount, 'EX', reqTimeout)
}
+const getFunnelsClickhouse = async (projectId: string, funnelId = null) => {
+ const paramsData = {
+ params: {
+ projectId,
+ funnelId,
+ },
+ }
+
+ if (!funnelId) {
+ const query =
+ 'SELECT * FROM funnel WHERE projectId = {projectId:FixedString(12)}'
+ return clickhouse.query(query, paramsData).toPromise()
+ }
+
+ const query = `
+ SELECT
+ *
+ FROM funnel
+ WHERE
+ projectId = {projectId:FixedString(12)}
+ AND id = {funnelId:String}`
+
+ const funnel = await clickhouse.query(query, paramsData).toPromise()
+
+ if (_isEmpty(funnel)) {
+ throw new NotFoundException(
+ `Funnel ${funnelId} was not found in the database`,
+ )
+ }
+
+ return _head(funnel)
+}
+
+const updateFunnelClickhouse = async (funnel: any) => {
+ const filtered = _reduce(
+ _filter(_keys(funnel), key => ALLOWED_FUNNEL_KEYS.includes(key)),
+ (obj, key) => {
+ obj[key] = funnel[key]
+ return obj
+ },
+ {},
+ )
+ const columns = _keys(filtered)
+ const values = _values(filtered)
+ const query = `ALTER table funnel UPDATE ${_join(
+ _map(columns, (col, id) => `${col}='${values[id]}'`),
+ ', ',
+ )} WHERE id='${funnel.id}'`
+
+ return clickhouse.query(query).toPromise()
+}
+
+const deleteFunnelClickhouse = async (id: string) => {
+ const paramsData = {
+ params: {
+ id,
+ },
+ }
+
+ const query = `ALTER table funnel DELETE WHERE id = {id:String}`
+ return clickhouse.query(query, paramsData).toPromise()
+}
+
+const createFunnelClickhouse = async (funnel: Partial<Funnel>) => {
+ const paramsData = {
+ params: {
+ ...funnel,
+ },
+ }
+
+ const query = `INSERT INTO funnel (*) VALUES ({id:String},{name:String},{steps:String},{projectId:FixedString(12)},'${dayjs
+ .utc()
+ .format('YYYY-MM-DD HH:mm:ss')}')`
+
+ return clickhouse.query(query, paramsData).toPromise()
+}
+
const getProjectsClickhouse = async (id = null) => {
if (!id) {
- const query = 'SELECT * FROM project;'
+ const query = 'SELECT * FROM project ORDER BY created ASC;'
return clickhouse.query(query).toPromise()
}
@@ -87,9 +170,9 @@ const getProjectsClickhouse = async (id = null) => {
return _head(project)
}
-const updateProjectClickhouse = async (project: object) => {
+const updateProjectClickhouse = async (project: any) => {
const filtered = _reduce(
- _filter(_keys(project), key => allowedToUpdateKeys.includes(key)),
+ _filter(_keys(project), key => ALLOWED_KEYS.includes(key)),
(obj, key) => {
obj[key] = project[key]
return obj
@@ -98,11 +181,9 @@ const updateProjectClickhouse = async (project: object) => {
)
const columns = _keys(filtered)
const values = _values(filtered)
- // @ts-ignore
const query = `ALTER table project UPDATE ${_join(
_map(columns, (col, id) => `${col}='${values[id]}'`),
', ',
- // @ts-ignore
)} WHERE id='${project.id}'`
return clickhouse.query(query).toPromise()
}
@@ -129,9 +210,15 @@ const calculateRelativePercentage = (
return _round((1 - newVal / oldVal) * -100, round)
}
-const deleteProjectClickhouse = async id => {
- const query = `ALTER table project DELETE WHERE id='${id}'`
- return clickhouse.query(query).toPromise()
+const deleteProjectClickhouse = async (id: string) => {
+ const paramsData = {
+ params: {
+ id,
+ },
+ }
+
+ const query = `ALTER table project DELETE WHERE id = {id:FixedString(12)}`
+ return clickhouse.query(query, paramsData).toPromise()
}
const createProjectClickhouse = async (project: Partial<Project>) => {
@@ -261,17 +348,17 @@ const updateUserClickhouse = async (user: IClickhouseUser) => {
let query = 'ALTER table sfuser UPDATE '
let separator = ''
- if (user.timezone) {
+ if (!_isNil(user.timezone)) {
query += `${separator}timezone={timezone:String}`
separator = ', '
}
- if (user.timeFormat) {
+ if (!_isNil(user.timeFormat)) {
query += `${separator}timeFormat={timeFormat:String}`
separator = ', '
}
- if (user.showLiveVisitorsInTitle) {
+ if (!_isNil(user.showLiveVisitorsInTitle)) {
query += `${separator}showLiveVisitorsInTitle={showLiveVisitorsInTitle:Int8}`
separator = ', '
}
@@ -385,4 +472,8 @@ export {
getGeoDetails,
getIPFromHeaders,
sumArrays,
+ getFunnelsClickhouse,
+ updateFunnelClickhouse,
+ deleteFunnelClickhouse,
+ createFunnelClickhouse,
}
diff --git a/apps/selfhosted/src/project/dto/funnel-create.dto.ts b/apps/selfhosted/src/project/dto/funnel-create.dto.ts
new file mode 100644
index 000000000..6d2ab4440
--- /dev/null
+++ b/apps/selfhosted/src/project/dto/funnel-create.dto.ts
@@ -0,0 +1,38 @@
+import { ApiProperty } from '@nestjs/swagger'
+import { IsNotEmpty, Length, ArrayMinSize, ArrayMaxSize } from 'class-validator'
+import {
+ MAX_PAGES_IN_FUNNEL,
+ MIN_PAGES_IN_FUNNEL,
+} from '../../common/constants'
+
+export class FunnelCreateDTO {
+ @ApiProperty({
+ example: 'User sign up funnel',
+ required: true,
+ description: 'A display name for your funnel',
+ })
+ @IsNotEmpty()
+ @Length(1, 50)
+ name: string
+
+ @ApiProperty({
+ example: 'aUn1quEid-3',
+ required: true,
+ description: 'Funnel project ID',
+ })
+ @IsNotEmpty()
+ pid: string
+
+ @ApiProperty({
+ example: ['/', '/signup', '/dashboard'],
+ required: true,
+ description: 'Steps of the funnel',
+ })
+ @ArrayMinSize(MIN_PAGES_IN_FUNNEL, {
+ message: `A funnel must have at least ${MIN_PAGES_IN_FUNNEL} steps`,
+ })
+ @ArrayMaxSize(MAX_PAGES_IN_FUNNEL, {
+ message: `A funnel must have no more than ${MAX_PAGES_IN_FUNNEL} steps`,
+ })
+ steps: string[] | null
+}
diff --git a/apps/selfhosted/src/project/dto/funnel-update.dto.ts b/apps/selfhosted/src/project/dto/funnel-update.dto.ts
new file mode 100644
index 000000000..31a7a3c3c
--- /dev/null
+++ b/apps/selfhosted/src/project/dto/funnel-update.dto.ts
@@ -0,0 +1,47 @@
+import { ApiProperty } from '@nestjs/swagger'
+import {
+ IsNotEmpty,
+ Length,
+ ArrayMinSize,
+ ArrayMaxSize,
+ IsUUID,
+} from 'class-validator'
+import {
+ MAX_PAGES_IN_FUNNEL,
+ MIN_PAGES_IN_FUNNEL,
+} from '../../common/constants'
+
+export class FunnelUpdateDTO {
+ @IsUUID('4')
+ id: string
+
+ @ApiProperty({
+ example: 'aUn1quEid-3',
+ required: true,
+ description: 'Funnel project ID',
+ })
+ @IsNotEmpty()
+ pid: string
+
+ @ApiProperty({
+ example: 'User sign up funnel',
+ required: true,
+ description: 'A display name for your funnel',
+ })
+ @IsNotEmpty()
+ @Length(1, 50)
+ name: string
+
+ @ApiProperty({
+ example: ['/', '/signup', '/dashboard'],
+ required: true,
+ description: 'Steps of the funnel',
+ })
+ @ArrayMinSize(MIN_PAGES_IN_FUNNEL, {
+ message: `A funnel must have at least ${MIN_PAGES_IN_FUNNEL} steps`,
+ })
+ @ArrayMaxSize(MAX_PAGES_IN_FUNNEL, {
+ message: `A funnel must have no more than ${MAX_PAGES_IN_FUNNEL} steps`,
+ })
+ steps: string[] | null
+}
diff --git a/apps/selfhosted/src/project/dto/index.ts b/apps/selfhosted/src/project/dto/index.ts
index 2c80f6996..1308d4594 100644
--- a/apps/selfhosted/src/project/dto/index.ts
+++ b/apps/selfhosted/src/project/dto/index.ts
@@ -1,2 +1,6 @@
export * from './create-project.dto'
export * from './project.dto'
+export * from './update-project.dto'
+export * from './project-password.dto'
+export * from './funnel-create.dto'
+export * from './funnel-update.dto'
diff --git a/apps/selfhosted/src/project/dto/project-password.dto.ts b/apps/selfhosted/src/project/dto/project-password.dto.ts
new file mode 100644
index 000000000..9c033b58c
--- /dev/null
+++ b/apps/selfhosted/src/project/dto/project-password.dto.ts
@@ -0,0 +1,29 @@
+import {
+ IsOptional,
+ IsString,
+ MaxLength,
+ MinLength,
+ ValidateIf,
+} from 'class-validator'
+
+export const MAX_PROJECT_PASSWORD_LENGTH = 80
+
+export class ProjectPasswordDto {
+ @IsString()
+ @MaxLength(MAX_PROJECT_PASSWORD_LENGTH, {
+ message: 'Max length is $constraint1 characters',
+ })
+ @MinLength(1, { message: 'Min length is $constraint1 characters' })
+ @IsOptional()
+ password?: string | null
+
+ @IsOptional()
+ @ValidateIf(o => o.isPasswordProtected !== null)
+ isPasswordProtected?: boolean | null
+}
+
+export class GetProtectedProjectDto {
+ @IsOptional()
+ @ValidateIf(o => o.password !== null)
+ password?: string | null
+}
diff --git a/apps/selfhosted/src/project/dto/update-project.dto.ts b/apps/selfhosted/src/project/dto/update-project.dto.ts
new file mode 100644
index 000000000..a6dbcde0c
--- /dev/null
+++ b/apps/selfhosted/src/project/dto/update-project.dto.ts
@@ -0,0 +1,23 @@
+import { IntersectionType, PartialType } from '@nestjs/mapped-types'
+import { ApiProperty } from '@nestjs/swagger'
+import { ProjectDTO } from './project.dto'
+import { ProjectPasswordDto } from './project-password.dto'
+
+export class UpdateProjectDto extends IntersectionType(
+ PartialType(ProjectDTO),
+ ProjectPasswordDto,
+) {
+ @ApiProperty({
+ required: false,
+ description:
+ "The project's state. If enabled - all the incoming analytics data will be saved.",
+ })
+ active?: boolean
+
+ @ApiProperty({
+ required: false,
+ description:
+ "When true, anyone on the internet (including Google) would be able to see the project's Dashboard.",
+ })
+ public?: boolean
+}
diff --git a/apps/selfhosted/src/project/entity/funnel.entity.ts b/apps/selfhosted/src/project/entity/funnel.entity.ts
new file mode 100644
index 000000000..eb3614c2f
--- /dev/null
+++ b/apps/selfhosted/src/project/entity/funnel.entity.ts
@@ -0,0 +1,11 @@
+export class Funnel {
+ id: string
+
+ name: string
+
+ steps: string[]
+
+ created: Date
+
+ projectId: string
+}
diff --git a/apps/selfhosted/src/project/entity/project.entity.ts b/apps/selfhosted/src/project/entity/project.entity.ts
index dbe0df792..420ee292c 100644
--- a/apps/selfhosted/src/project/entity/project.entity.ts
+++ b/apps/selfhosted/src/project/entity/project.entity.ts
@@ -11,5 +11,9 @@ export class Project {
public: boolean
+ isPasswordProtected: boolean
+
+ passwordHash?: string
+
created: Date
}
diff --git a/apps/selfhosted/src/project/project.controller.ts b/apps/selfhosted/src/project/project.controller.ts
index 75f86f2ff..7c7088f63 100644
--- a/apps/selfhosted/src/project/project.controller.ts
+++ b/apps/selfhosted/src/project/project.controller.ts
@@ -11,16 +11,24 @@ import {
BadRequestException,
HttpCode,
NotFoundException,
+ Headers,
+ UnauthorizedException,
+ Patch,
} from '@nestjs/common'
+import { v4 as uuidv4 } from 'uuid'
import { ApiTags, ApiQuery, ApiResponse, ApiBearerAuth } from '@nestjs/swagger'
import * as _isEmpty from 'lodash/isEmpty'
import * as _map from 'lodash/map'
import * as _trim from 'lodash/trim'
import * as _size from 'lodash/size'
import * as _split from 'lodash/split'
+import * as _isBoolean from 'lodash/isBoolean'
+import * as _omit from 'lodash/omit'
+import * as _reduce from 'lodash/reduce'
import * as _head from 'lodash/head'
import * as _includes from 'lodash/includes'
import * as dayjs from 'dayjs'
+import { hash } from 'bcrypt'
import { JwtAccessTokenGuard } from '../auth/guards'
import { Auth } from '../auth/decorators'
@@ -36,7 +44,13 @@ import { RolesGuard } from '../auth/guards/roles.guard'
import { Pagination } from '../common/pagination/pagination'
import { Project } from './entity/project.entity'
import { CurrentUserId } from '../auth/decorators/current-user-id.decorator'
-import { ProjectDTO, CreateProjectDTO } from './dto'
+import {
+ ProjectDTO,
+ CreateProjectDTO,
+ UpdateProjectDto,
+ FunnelCreateDTO,
+ FunnelUpdateDTO,
+} from './dto'
import { AppLoggerService } from '../logger/logger.service'
import { isValidPID, clickhouse } from '../common/constants'
import {
@@ -44,7 +58,12 @@ import {
createProjectClickhouse,
updateProjectClickhouse,
deleteProjectClickhouse,
+ getFunnelsClickhouse,
+ deleteFunnelClickhouse,
+ createFunnelClickhouse,
+ updateFunnelClickhouse,
} from '../common/utils'
+import { Funnel } from './entity/funnel.entity'
@ApiTags('Project')
@Controller('project')
@@ -81,8 +100,35 @@ export class ProjectController {
_map(formatted, ({ id }) => id),
)
+ const funnelsData = await Promise.allSettled(
+ pidsWithData.map(async pid => {
+ return {
+ pid,
+ data: await getFunnelsClickhouse(pid),
+ }
+ }),
+ )
+
+ const funnelsMap = _reduce(
+ funnelsData,
+ (acc, { status, value }) => {
+ if (status !== 'fulfilled') {
+ return acc
+ }
+
+ return {
+ ...acc,
+ [value.pid]: this.projectService.formatFunnelsFromClickhouse(
+ value.data,
+ ),
+ }
+ },
+ {},
+ )
+
const results = _map(formatted, p => ({
- ...p,
+ ..._omit(p, ['passwordHash']),
+ funnels: funnelsMap[p.id],
isOwner: true,
isLocked: false,
isDataExists: _includes(pidsWithData, p?.id),
@@ -168,6 +214,153 @@ export class ProjectController {
}
}
+ @Post('/funnel')
+ @ApiResponse({ status: 201 })
+ @Auth([], true)
+ async createFunnel(
+ @Body() funnelDTO: FunnelCreateDTO,
+ @CurrentUserId() userId: string,
+ ): Promise<any> {
+ this.logger.log({ funnelDTO, userId }, 'POST /project/funnel')
+
+ if (!userId) {
+ throw new UnauthorizedException('Please auth first')
+ }
+
+ const project = getProjectsClickhouse(funnelDTO.pid)
+
+ if (!project) {
+ throw new NotFoundException('Project not found.')
+ }
+
+ const funnel = new Funnel()
+ funnel.id = uuidv4()
+ funnel.name = funnelDTO.name
+ funnel.steps = _map(funnelDTO.steps, _trim)
+ funnel.projectId = funnelDTO.pid
+
+ const formatted = this.projectService.formatFunnelToClickhouse(funnel)
+
+ await createFunnelClickhouse(formatted)
+
+ return {
+ ...funnel,
+ pid: funnelDTO.pid,
+ project: _omit(this.projectService.formatFromClickhouse(project), [
+ 'passwordHash',
+ ]),
+ }
+ }
+
+ @Patch('/funnel')
+ @ApiResponse({ status: 200 })
+ @Auth([], true)
+ async updateFunnel(
+ @Body() funnelDTO: FunnelUpdateDTO,
+ @CurrentUserId() userId: string,
+ ): Promise<void> {
+ this.logger.log({ funnelDTO, userId }, 'PATCH /project/funnel')
+
+ if (!userId) {
+ throw new UnauthorizedException('Please auth first')
+ }
+
+ const project = getProjectsClickhouse(funnelDTO.pid)
+
+ if (!project) {
+ throw new NotFoundException('Project not found.')
+ }
+
+ const oldFunnel = this.projectService.formatFunnelFromClickhouse(
+ await getFunnelsClickhouse(funnelDTO.pid, funnelDTO.id),
+ )
+
+ if (!oldFunnel) {
+ throw new NotFoundException('Funnel not found.')
+ }
+
+ await updateFunnelClickhouse(
+ this.projectService.formatFunnelToClickhouse({
+ id: funnelDTO.id,
+ name: funnelDTO.name,
+ steps: funnelDTO.steps,
+ } as Funnel),
+ )
+ }
+
+ @Delete('/funnel/:id/:pid')
+ @ApiResponse({ status: 200 })
+ @Auth([], true)
+ async deleteFunnel(
+ @Param('id') id: string,
+ @Param('pid') pid: string,
+ @CurrentUserId() userId: string,
+ ): Promise<void> {
+ this.logger.log({ id, userId }, 'PATCH /project/funnel')
+
+ if (!userId) {
+ throw new UnauthorizedException('Please auth first')
+ }
+
+ const oldFunnel = getFunnelsClickhouse(pid, id)
+
+ if (!oldFunnel) {
+ throw new NotFoundException('Funnel not found.')
+ }
+
+ await deleteFunnelClickhouse(id)
+ }
+
+ @Get('/funnels/:pid')
+ @ApiResponse({ status: 200 })
+ @Auth([], true)
+ async getFunnels(
+ @Param('pid') pid: string,
+ @CurrentUserId() userId: string,
+ @Headers() headers: { 'x-password'?: string },
+ ): Promise<any> {
+ this.logger.log({ pid, userId }, 'PATCH /project/funnel')
+
+ if (!userId) {
+ throw new UnauthorizedException('Please auth first')
+ }
+
+ const project = await getProjectsClickhouse(pid)
+
+ if (!project) {
+ throw new NotFoundException('Project not found.')
+ }
+
+ this.projectService.allowedToView(project, userId, headers['x-password'])
+
+ return getFunnelsClickhouse(pid)
+ }
+
+ @Get('password/:projectId')
+ @Auth([], true, true)
+ @ApiResponse({ status: 200, type: Project })
+ async checkPassword(
+ @Param('projectId') projectId: string,
+ @CurrentUserId() userId: string,
+ @Headers() headers: { 'x-password'?: string },
+ ): Promise<boolean> {
+ this.logger.log({ projectId }, 'GET /project/password/:projectId')
+
+ const project = await getProjectsClickhouse(projectId)
+
+ if (!project) {
+ throw new NotFoundException('Project not found.')
+ }
+
+ try {
+ this.projectService.allowedToView(project, userId, headers['x-password'])
+ } catch {
+ return false
+ }
+
+ return true
+ }
+
@Delete('/partially/:pid')
@ApiQuery({
name: 'from',
@@ -259,7 +452,7 @@ export class ProjectController {
@ApiResponse({ status: 200, type: Project })
async update(
@Param('id') id: string,
- @Body() projectDTO: ProjectDTO,
+ @Body() projectDTO: UpdateProjectDto,
@CurrentUserId() uid: string,
): Promise<any> {
this.logger.log({ projectDTO, uid, id }, 'PUT /project/:id')
@@ -270,11 +463,37 @@ export class ProjectController {
throw new NotFoundException()
}
- project.active = projectDTO.active
- project.origins = _map(projectDTO.origins, _trim)
- project.ipBlacklist = _map(projectDTO.ipBlacklist, _trim)
- project.name = projectDTO.name
- project.public = projectDTO.public
+ if (_isBoolean(projectDTO.public)) {
+ project.public = projectDTO.public
+ }
+
+ if (_isBoolean(projectDTO.active)) {
+ project.active = projectDTO.active
+ }
+
+ if (projectDTO.origins) {
+ project.origins = _map(projectDTO.origins, _trim)
+ }
+
+ if (projectDTO.ipBlacklist) {
+ project.ipBlacklist = _map(projectDTO.ipBlacklist, _trim)
+ }
+
+ if (projectDTO.name) {
+ project.name = _trim(projectDTO.name)
+ }
+
+ if (_isBoolean(projectDTO.isPasswordProtected)) {
+ if (projectDTO.isPasswordProtected) {
+ if (projectDTO.password) {
+ project.isPasswordProtected = true
+ project.passwordHash = await hash(projectDTO.password, 10)
+ }
+ } else {
+ project.isPasswordProtected = false
+ project.passwordHash = null
+ }
+ }
await updateProjectClickhouse(
this.projectService.formatToClickhouse(project),
@@ -283,7 +502,7 @@ export class ProjectController {
// await updateProjectRedis(id, project)
await deleteProjectRedis(id)
- return project
+ return _omit(project, ['passwordHash'])
}
@Get('/:id')
@@ -292,6 +511,7 @@ export class ProjectController {
async getOne(
@Param('id') id: string,
@CurrentUserId() uid: string,
+ @Headers() headers: { 'x-password'?: string },
): Promise<Project | object> {
this.logger.log({ id }, 'GET /project/:id')
if (!isValidPID(id)) {
@@ -306,7 +526,14 @@ export class ProjectController {
throw new NotFoundException('Project was not found in the database')
}
- this.projectService.allowedToView(project, uid)
+ if (project.isPasswordProtected && _isEmpty(headers['x-password'])) {
+ return {
+ isPasswordProtected: true,
+ id: project.id,
+ }
+ }
+
+ this.projectService.allowedToView(project, uid, headers['x-password'])
const isDataExists = !_isEmpty(
await this.projectService.getPIDsWhereAnalyticsDataExists([id]),
@@ -316,8 +543,11 @@ export class ProjectController {
await this.projectService.getPIDsWhereErrorsDataExists([id]),
)
+ const funnels = await getFunnelsClickhouse(id)
+
return this.projectService.formatFromClickhouse({
- ...project,
+ ..._omit(project, ['passwordHash']),
+ funnels: this.projectService.formatFunnelsFromClickhouse(funnels),
isDataExists,
isErrorDataExists,
})
diff --git a/apps/selfhosted/src/project/project.service.ts b/apps/selfhosted/src/project/project.service.ts
index e845cffbe..ed8c7504c 100644
--- a/apps/selfhosted/src/project/project.service.ts
+++ b/apps/selfhosted/src/project/project.service.ts
@@ -18,6 +18,7 @@ import * as _isNull from 'lodash/isNull'
import * as _split from 'lodash/split'
import * as _trim from 'lodash/trim'
import * as _reduce from 'lodash/reduce'
+import { compareSync } from 'bcrypt'
import { Project } from './entity/project.entity'
import { ProjectDTO } from './dto/project.dto'
@@ -31,6 +32,8 @@ import {
redisProjectCacheTimeout,
} from '../common/constants'
import { getProjectsClickhouse } from '../common/utils'
+import { MAX_PROJECT_PASSWORD_LENGTH, UpdateProjectDto } from './dto'
+import { Funnel } from './entity/funnel.entity'
// A list of characters that can be used in a Project ID
const LEGAL_PID_CHARACTERS =
@@ -79,7 +82,22 @@ export class ProjectService {
return project
}
- allowedToView(project: Project, uid: string | null): void {
+ allowedToView(
+ project: Project,
+ uid: string | null,
+ password?: string | null,
+ ): void {
+ if (project.isPasswordProtected && password) {
+ if (
+ _size(password) <= MAX_PROJECT_PASSWORD_LENGTH &&
+ compareSync(password, project.passwordHash)
+ ) {
+ return null
+ }
+
+ throw new ConflictException('Incorrect password')
+ }
+
if (project.public || uid) {
return null
}
@@ -99,9 +117,11 @@ export class ProjectService {
}
}
- async getPIDsWhereAnalyticsDataExists(projectIds: string[]) {
+ async getPIDsWhereAnalyticsDataExists(
+ projectIds: string[],
+ ): Promise<string[]> {
if (_isEmpty(projectIds)) {
- return {}
+ return []
}
const params = _reduce(
@@ -156,9 +176,9 @@ export class ProjectService {
return _map(result, ({ pid }) => pid)
}
- async getPIDsWhereErrorsDataExists(projectIds: string[]) {
+ async getPIDsWhereErrorsDataExists(projectIds: string[]): Promise<string[]> {
if (_isEmpty(projectIds)) {
- return {}
+ return []
}
const params = _reduce(
@@ -230,6 +250,7 @@ export class ProjectService {
const updProject = { ...project }
updProject.active = Number(updProject.active)
updProject.public = Number(updProject.public)
+ updProject.isPasswordProtected = Number(updProject.isPasswordProtected)
if (!_isNull(updProject.origins)) {
updProject.origins = _isString(updProject.origins)
@@ -250,6 +271,7 @@ export class ProjectService {
const updProject = { ...project }
updProject.active = Boolean(updProject.active)
updProject.public = Boolean(updProject.public)
+ updProject.isPasswordProtected = Boolean(updProject.isPasswordProtected)
updProject.origins = _isNull(updProject.origins)
? []
@@ -262,7 +284,38 @@ export class ProjectService {
return updProject
}
- validateProject(projectDTO: ProjectDTO, creatingProject = false) {
+ formatFunnelToClickhouse(funnel: Funnel): any {
+ const updFunnel = { ...funnel }
+ updFunnel.name = _trim(funnel.name)
+ updFunnel.steps = _isString(updFunnel.steps)
+ ? updFunnel.steps
+ : _join(updFunnel.steps, ',')
+
+ return updFunnel
+ }
+
+ formatFunnelsFromClickhouse(funnels: any[]): Funnel[] {
+ if (_isEmpty(funnels)) {
+ return []
+ }
+
+ return _map(funnels, this.formatFunnelFromClickhouse)
+ }
+
+ formatFunnelFromClickhouse(funnel: any): Funnel {
+ const updFunnel = { ...funnel }
+
+ updFunnel.steps = _isNull(updFunnel.steps)
+ ? []
+ : _split(updFunnel.steps, ',')
+
+ return updFunnel
+ }
+
+ validateProject(
+ projectDTO: ProjectDTO | UpdateProjectDto,
+ creatingProject = false,
+ ) {
if (_size(projectDTO.name) > 50)
throw new UnprocessableEntityException('The project name is too long')
diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
index 02ae8f9ec..beca87ac0 100644
--- a/docker-compose-dev.yml
+++ b/docker-compose-dev.yml
@@ -1,11 +1,11 @@
version: '3.8'
services:
redis:
- image: redis:7.0.11-alpine
+ image: redis:7.2-alpine
ports:
- 6379:6379
clickhouse:
- image: clickhouse/clickhouse-server:23.4.2.11-alpine
+ image: clickhouse/clickhouse-server:23.8-alpine
environment:
CLICKHOUSE_DB: 'analytics'
ports:
diff --git a/docker-compose-prod.yml b/docker-compose-prod.yml
index 438a2dafc..3af77d322 100644
--- a/docker-compose-prod.yml
+++ b/docker-compose-prod.yml
@@ -58,7 +58,7 @@ services:
- 'mariadb'
- 'clickhouse'
redis:
- image: redis:7.0.11-alpine
+ image: redis:7.2-alpine
restart: always
environment:
- REDIS_PORT=6379
@@ -77,7 +77,7 @@ services:
volumes:
- swetrix-users-data:/var/lib/mysql
clickhouse:
- image: clickhouse/clickhouse-server:23.4.2.11-alpine
+ image: clickhouse/clickhouse-server:23.8-alpine
container_name: clickhouse
environment:
- CLICKHOUSE_DATABASE=analytics
diff --git a/docker-compose.yml b/docker-compose.yml
index 4fae851fe..adbcaffba 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -36,7 +36,7 @@ services:
- 'redis'
- 'clickhouse'
redis:
- image: redis:7.0.11-alpine
+ image: redis:7.2-alpine
restart: always
environment:
- REDIS_PORT=6379
@@ -47,7 +47,7 @@ services:
# volumes:
# - '/opt/redis-volume:/data'
clickhouse:
- image: clickhouse/clickhouse-server:23.4.2.11-alpine
+ image: clickhouse/clickhouse-server:23.8-alpine
container_name: clickhouse
environment:
- CLICKHOUSE_DATABASE=analytics
diff --git a/migrations/clickhouse/2023_05_19.js b/migrations/clickhouse/2023_05_19.js
index ca8201f23..1887deed3 100644
--- a/migrations/clickhouse/2023_05_19.js
+++ b/migrations/clickhouse/2023_05_19.js
@@ -100,7 +100,7 @@ const queries = [
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(created)
- ORDER BY (pid, created);`
+ ORDER BY (pid, created);`,
`INSERT INTO ${dbName}.captcha_temp (*) SELECT * FROM ${dbName}.captcha`,
diff --git a/migrations/clickhouse/initialise_selfhosted.js b/migrations/clickhouse/initialise_selfhosted.js
index 17238a469..6517518dc 100644
--- a/migrations/clickhouse/initialise_selfhosted.js
+++ b/migrations/clickhouse/initialise_selfhosted.js
@@ -11,6 +11,8 @@ const CLICKHOUSE_INIT_QUERIES = [
ipBlacklist Nullable(String),
active Int8,
public Int8,
+ isPasswordProtected Int8,
+ passwordHash Nullable(String),
created DateTime
)
ENGINE = MergeTree()
@@ -34,6 +36,17 @@ const CLICKHOUSE_INIT_QUERIES = [
)
ENGINE = MergeTree()
PRIMARY KEY id;`,
+
+ `CREATE TABLE IF NOT EXISTS ${dbName}.funnel
+ (
+ id String,
+ name String,
+ steps String,
+ projectId FixedString(12),
+ created DateTime
+ )
+ ENGINE = MergeTree()
+ PRIMARY KEY id;`,
]
const initialiseSelfhosted = async () => {
@@ -41,7 +54,9 @@ const initialiseSelfhosted = async () => {
await initialiseDatabase()
await queriesRunner(CLICKHOUSE_INIT_QUERIES)
} catch (reason) {
- console.error(`[ERROR] Error occured whilst initialising the database: ${reason}`)
+ console.error(
+ `[ERROR] Error occured whilst initialising the database: ${reason}`,
+ )
}
}
diff --git a/migrations/clickhouse/selfhosted_2023_05_28.js b/migrations/clickhouse/selfhosted_2023_05_28.js
index 6c034d0f3..453632c8b 100644
--- a/migrations/clickhouse/selfhosted_2023_05_28.js
+++ b/migrations/clickhouse/selfhosted_2023_05_28.js
@@ -7,7 +7,7 @@ const queries = [
(
id String,
timezone Nullable(String),
- timeFormat Nullable(String)
+ timeFormat Nullable(String),
showLiveVisitorsInTitle Int8
)
ENGINE = MergeTree()
diff --git a/migrations/clickhouse/selfhosted_2023_10_11.js b/migrations/clickhouse/selfhosted_2023_10_11.js
index 1275bd507..dbf1821d1 100644
--- a/migrations/clickhouse/selfhosted_2023_10_11.js
+++ b/migrations/clickhouse/selfhosted_2023_10_11.js
@@ -1,7 +1,7 @@
const { queriesRunner, dbName } = require('./setup')
const queries = [
- // Added 'showLiveVisitorsInTitle' column
+ // Made 'showLiveVisitorsInTitle' column nullable
`DROP TABLE IF EXISTS ${dbName}.sfuser_temp`,
`CREATE TABLE IF NOT EXISTS ${dbName}.sfuser_temp
(
diff --git a/migrations/clickhouse/selfhosted_2024_05_13.js b/migrations/clickhouse/selfhosted_2024_05_13.js
new file mode 100644
index 000000000..d07192b38
--- /dev/null
+++ b/migrations/clickhouse/selfhosted_2024_05_13.js
@@ -0,0 +1,41 @@
+const { queriesRunner, dbName } = require('./setup')
+
+const queries = [
+ // Added password-related columns
+ `DROP TABLE IF EXISTS ${dbName}.project_temp`,
+ `CREATE TABLE IF NOT EXISTS ${dbName}.project_temp
+ (
+ id FixedString(12),
+ name String,
+ origins Nullable(String),
+ ipBlacklist Nullable(String),
+ active Int8,
+ public Int8,
+ isPasswordProtected Int8,
+ passwordHash Nullable(String),
+ created DateTime
+ )
+ ENGINE = MergeTree()
+ PARTITION BY toYYYYMM(created)
+ ORDER BY (created);`,
+
+ `INSERT INTO ${dbName}.project_temp (*)
+ SELECT id, name, origins, ipBlacklist, active, public, 0, NULL, created FROM ${dbName}.project`,
+
+ `DROP TABLE ${dbName}.project`,
+ `RENAME TABLE ${dbName}.project_temp TO ${dbName}.project`,
+
+ // Added funnel table
+ `CREATE TABLE IF NOT EXISTS ${dbName}.funnel
+ (
+ id String,
+ name String,
+ steps String,
+ projectId FixedString(12),
+ created DateTime
+ )
+ ENGINE = MergeTree()
+ PRIMARY KEY id;`,
+]
+
+queriesRunner(queries)
diff --git a/migrations/clickhouse/selfhosted_v2_to_v3.js b/migrations/clickhouse/selfhosted_v2_to_v3.js
new file mode 100644
index 000000000..595fb947c
--- /dev/null
+++ b/migrations/clickhouse/selfhosted_v2_to_v3.js
@@ -0,0 +1,48 @@
+const childProcess = require('child_process')
+const path = require('path')
+
+function runScript(scriptPath) {
+ return new Promise((resolve, reject) => {
+ const fullPath = path.join(__dirname, scriptPath)
+ console.log('[INFO] Running script: ' + fullPath)
+
+ const process = childProcess.fork(fullPath)
+ let invoked = false
+
+ // listen for errors as they may prevent the exit event from firing
+ process.on('error', function (err) {
+ if (invoked) return
+ invoked = true
+ reject(err)
+ })
+
+ // execute the callback once the process has finished running
+ process.on('exit', function (code) {
+ if (invoked) return
+ invoked = true
+ const err = code === 0 ? null : new Error('exit code ' + code)
+ err ? reject(err) : resolve()
+ })
+ })
+}
+
+async function runMigrations() {
+ try {
+ await runScript('./initialise_database.js')
+ await runScript('./initialise_selfhosted.js')
+ await runScript('./2023_05_19.js')
+ await runScript('./2023_06_26.js')
+ await runScript('./2023_10_13.js')
+ await runScript('./2023_10_18.js')
+ await runScript('./2024_03_30.js')
+ await runScript('./selfhosted_2023_05_28.js')
+ await runScript('./selfhosted_2023_10_11.js')
+ await runScript('./selfhosted_2024_05_13.js')
+
+ console.log('[INFO] SELFHOSTED MIGRATIONS FINISHED')
+ } catch (err) {
+ console.error('Error running migration:', err)
+ }
+}
+
+runMigrations()