📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
import {
ForbiddenException,
Injectable,
BadRequestException,
UnprocessableEntityException,
InternalServerErrorException,
ConflictException,
} from '@nestjs/common'
import { randomUUID } from 'crypto'
import _isEmpty from 'lodash/isEmpty'
import _isString from 'lodash/isString'
import _size from 'lodash/size'
import _join from 'lodash/join'
import _filter from 'lodash/filter'
import _includes from 'lodash/includes'
import _find from 'lodash/find'
import _map from 'lodash/map'
import _isNull from 'lodash/isNull'
import _split from 'lodash/split'
import _trim from 'lodash/trim'
import _reduce from 'lodash/reduce'
import _findIndex from 'lodash/findIndex'
import { compareSync } from 'bcrypt'
import { Project, ProjectWithShare, Role } from './entity/project.entity'
import { ProjectDTO } from './dto/project.dto'
import {
isValidPID,
redis,
ORIGINS_REGEX,
getRedisProjectKey,
redisProjectCacheTimeout,
V2_VIEW_FILTER_DIMENSIONS,
} from '../common/constants'
import { clickhouse } from '../common/integrations/clickhouse'
import { isValidIpRange } from '../common/ip-range'
import {
createProjectShareClickhouse,
findProjectSharesByProjectClickhouse,
getProjectClickhouse,
updateProjectClickhouse,
} from '../common/utils'
import { MAX_PROJECT_PASSWORD_LENGTH, UpdateProjectDto } from './dto'
import { Funnel } from './entity/funnel.entity'
import { ProjectViewEntity } from './entity/project-view.entity'
import {
CreateProjectViewDto,
Filter,
ProjectViewCustomEventDto,
} from './dto/create-project-view.dto'
import { ProjectViewCustomEventEntity } from './entity/project-view-custom-event.entity'
import { ClickhouseFunnel } from '../common/types'
// A list of characters that can be used in a Project ID
export const LEGAL_PID_CHARACTERS =
'1234567890QWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm'
export const PID_LENGTH = 12
export const deleteProjectRedis = async (id: string) => {
const key = getRedisProjectKey(id)
try {
await redis.del(key)
} catch (reason) {
console.error(`Error deleting project ${id} from redis: ${reason}`)
}
}
@Injectable()
export class ProjectService {
async getRedisProject(pid: string) {
const pidKey = getRedisProjectKey(pid)
let project: string | Project = await redis.get(pidKey)
if (_isEmpty(project)) {
project = this.formatFromClickhouse(await this.getFullProject(pid))
if (_isEmpty(project))
throw new BadRequestException(
'The provided Project ID (pid) is incorrect',
)
await redis.set(
pidKey,
JSON.stringify(project),
'EX',
redisProjectCacheTimeout,
)
} else {
try {
project = JSON.parse(project)
} catch {
throw new InternalServerErrorException('Error while processing project')
}
}
return project as ProjectWithShare | null
}
allowedToView(
project: ProjectWithShare,
uid: string | null,
password?: string | null,
): void {
if (
project.public ||
uid === project.adminId ||
_findIndex(
project.share,
({ user, confirmed }) => user?.id === uid && confirmed === true,
) !== -1
) {
return null
}
if (project.isPasswordProtected && password) {
if (
_size(password) <= MAX_PROJECT_PASSWORD_LENGTH &&
compareSync(password, project.passwordHash)
) {
return null
}
throw new ConflictException('Incorrect password')
}
if (project.isPasswordProtected) {
throw new ForbiddenException('This project is password protected')
}
throw new ForbiddenException('You are not allowed to view this project')
}
// Instead of passing relations to the findOne method every time, this method returns a project
// with necessary relations needed for the allowedToView and allowedToManage methods
async getFullProject(pid: string): Promise<ProjectWithShare | null> {
const project = await getProjectClickhouse(pid)
if (_isEmpty(project)) {
return null
}
const shares = await findProjectSharesByProjectClickhouse(pid)
const share = (shares || []).map((row: any) => ({
id: row.id,
role: row.role,
confirmed: Boolean(row.confirmed),
user: {
id: row.userId,
email: row.email,
},
}))
return {
...project,
share,
}
}
async getOwnProject(projectId: string, userId: string) {
const project = await this.getFullProject(projectId)
if (!project) {
return project
}
this.allowedToManage(project, userId)
return project
}
async confirmTransferProject(
projectId: string,
userId: string,
oldAdminId: string,
) {
await updateProjectClickhouse(
{
id: projectId,
adminId: userId,
},
{ ignoreAllowedKeys: true },
)
await createProjectShareClickhouse({
id: randomUUID(),
userId: oldAdminId,
projectId: projectId,
confirmed: 1,
role: Role.admin,
})
await deleteProjectRedis(projectId)
}
allowedToManage(
project: ProjectWithShare,
uid: string,
message = 'You are not allowed to manage this project',
): void {
if (
uid === project.adminId ||
_findIndex(
project.share,
(share) =>
share.user?.id === uid &&
share.role === Role.admin &&
share.confirmed === true,
) !== -1
) {
return null
}
throw new ForbiddenException(message)
}
isPIDUnique(projects: Project[], pid: string): boolean {
return !_find(projects, ({ id }) => id === pid)
}
checkIfIDUnique(projects: Project[], pid: string): void {
const isUnique = this.isPIDUnique(projects, pid)
if (!isUnique) {
throw new BadRequestException('Selected project ID is already in use')
}
}
async getPIDsWhereAnalyticsDataExists(
projectIds: string[],
): Promise<string[]> {
if (_isEmpty(projectIds)) {
return []
}
const params = _reduce(
projectIds,
(acc, curr, index) => ({
...acc,
[`pid_${index}`]: curr,
}),
{},
)
const pids = _join(
_map(params, (val, key) => `{${key}:FixedString(12)}`),
',',
)
const query = `
SELECT DISTINCT pid
FROM events
WHERE pid IN (${pids})
AND type IN ('pageview', 'custom_event', 'outbound')
`
const { data } = await clickhouse
.query({
query,
query_params: params,
})
.then((resultSet) => resultSet.json<{ pid: string }>())
return _map(data, ({ pid }) => pid)
}
async getPIDsWhereErrorsDataExists(projectIds: string[]): Promise<string[]> {
if (_isEmpty(projectIds)) {
return []
}
const params = _reduce(
projectIds,
(acc, curr, index) => ({
...acc,
[`pid_${index}`]: curr,
}),
{},
)
const pids = _join(
_map(params, (val, key) => `{${key}:FixedString(12)}`),
',',
)
const query = `
SELECT DISTINCT pid
FROM events
WHERE pid IN (${pids})
AND type = 'error'
`
const { data } = await clickhouse
.query({
query,
query_params: params,
})
.then((resultSet) => resultSet.json<{ pid: string }>())
return _map(data, ({ pid }) => pid)
}
async getPIDsWhereCaptchaDataExists(projectIds: string[]): Promise<string[]> {
if (_isEmpty(projectIds)) {
return []
}
const params = _reduce(
projectIds,
(acc, curr, index) => ({
...acc,
[`pid_${index}`]: curr,
}),
{},
)
const pids = _join(
_map(params, (val, key) => `{${key}:FixedString(12)}`),
',',
)
const query = `
SELECT DISTINCT pid
FROM events
WHERE pid IN (${pids})
AND type = 'captcha'
`
const { data } = await clickhouse
.query({
query,
query_params: params,
})
.then((resultSet) => resultSet.json<{ pid: string }>())
return _map(data, ({ pid }) => pid)
}
async removeDataFromClickhouse(
pid: string,
from: string,
to: string,
): Promise<void> {
const queries = [
"ALTER TABLE events DELETE WHERE pid = {pid:FixedString(12)} AND type IN ('pageview', 'custom_event', 'performance', 'error', 'captcha', 'outbound') AND created BETWEEN {from:String} AND {to:String}",
'ALTER TABLE error_statuses DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',
]
const params = {
pid,
from,
to,
}
await Promise.all(
_map(queries, (query) =>
clickhouse.command({
query,
query_params: params,
}),
),
)
}
formatToClickhouse(project: any) {
const updProject = { ...project }
updProject.active = Number(updProject.active)
updProject.public = Number(updProject.public)
updProject.isPasswordProtected = Number(updProject.isPasswordProtected)
if (updProject.isCaptchaProject !== undefined) {
updProject.isCaptchaProject = Number(updProject.isCaptchaProject)
}
if (!_isNull(updProject.origins)) {
updProject.origins = _isString(updProject.origins)
? updProject.origins
: _join(updProject.origins, ',')
}
if (!_isNull(updProject.ipBlacklist)) {
updProject.ipBlacklist = _isString(updProject.ipBlacklist)
? updProject.ipBlacklist
: _join(updProject.ipBlacklist, ',')
}
if (!_isNull(updProject.ipWhitelist)) {
updProject.ipWhitelist = _isString(updProject.ipWhitelist)
? updProject.ipWhitelist
: _join(updProject.ipWhitelist, ',')
}
if (!_isNull(updProject.countryBlacklist)) {
updProject.countryBlacklist = _isString(updProject.countryBlacklist)
? updProject.countryBlacklist
: _join(updProject.countryBlacklist, ',')
}
if (updProject.brandKeywords !== undefined) {
if (
updProject.brandKeywords === null ||
(Array.isArray(updProject.brandKeywords) &&
updProject.brandKeywords.length === 0)
) {
updProject.brandKeywords = null
} else if (Array.isArray(updProject.brandKeywords)) {
updProject.brandKeywords = JSON.stringify(updProject.brandKeywords)
}
}
return updProject
}
formatFromClickhouse(project: any): Project {
const updProject = { ...project }
updProject.active = Boolean(updProject.active)
updProject.public = Boolean(updProject.public)
updProject.isPasswordProtected = Boolean(updProject.isPasswordProtected)
updProject.isCaptchaProject = Boolean(updProject.isCaptchaProject)
updProject.origins = _isEmpty(updProject.origins)
? []
: _split(updProject.origins, ',')
updProject.ipBlacklist = _isEmpty(updProject.ipBlacklist)
? []
: _split(updProject.ipBlacklist, ',')
updProject.ipWhitelist = _isEmpty(updProject.ipWhitelist)
? []
: _split(updProject.ipWhitelist, ',')
updProject.countryBlacklist = _isEmpty(updProject.countryBlacklist)
? []
: _split(updProject.countryBlacklist, ',')
if (_isString(updProject.brandKeywords)) {
const trimmed = updProject.brandKeywords.trim()
if (trimmed.length === 0) {
updProject.brandKeywords = null
} else {
try {
const parsed = JSON.parse(trimmed)
updProject.brandKeywords =
Array.isArray(parsed) && parsed.length > 0 ? parsed : null
} catch {
updProject.brandKeywords = null
}
}
} else if (!Array.isArray(updProject.brandKeywords)) {
updProject.brandKeywords = null
}
return updProject
}
filterUnsupportedColumns(
filters: CreateProjectViewDto['filters'],
): CreateProjectViewDto['filters'] {
if (!filters) {
return []
}
return _filter(filters, ({ dimension }) =>
_includes(V2_VIEW_FILTER_DIMENSIONS, dimension),
)
}
formatCustomEventToClickhose(
viewId: string,
customEvent: ProjectViewCustomEventEntity | ProjectViewCustomEventDto,
) {
if (!customEvent) {
return null
}
return {
viewId,
// @ts-expect-error
id: customEvent.id,
customEventName: customEvent.customEventName,
metaKey: customEvent.metaKey || null,
metaValue: customEvent.metaValue || null,
metaValueType: customEvent.metaValueType,
metricKey: customEvent.metricKey,
}
}
formatCustomEventsToClickhouse(
viewId: string,
customEvents: (ProjectViewCustomEventEntity | ProjectViewCustomEventDto)[],
) {
if (_isEmpty(customEvents)) {
return []
}
return _map(customEvents, (customEvent) =>
this.formatCustomEventToClickhose(viewId, customEvent),
)
}
formatViewToClickhouse(
view: (ProjectViewEntity | CreateProjectViewDto) & {
id: string
projectId: string
},
) {
const { id, projectId, type, customEvents, name, filters } = view
return {
id,
projectId,
type,
customEvents: this.formatCustomEventsToClickhouse(id, customEvents),
name,
filters: JSON.stringify(
this.filterUnsupportedColumns(filters as Filter[]),
null,
2,
),
}
}
formatViewsFromClickhouse(views: any[]): ProjectViewEntity[] {
if (_isEmpty(views)) {
return []
}
return _map(views, this.formatViewFromClickhouse)
}
formatViewFromClickhouse(view: any): ProjectViewEntity {
const updView = { ...view }
updView.name = _trim(updView.name)
try {
updView.filters = JSON.parse(view.filters)
} catch (reason) {
console.error(
'[ERROR] formatViewFromClickhouse, failed to parse filters:',
reason,
)
console.error('VIEW:', view)
updView.filters = []
}
updView.customEvents = _map(view.customEvents, (customEvent) => ({
...customEvent,
createdAt: '1970-01-01T00:00:00.000Z',
updatedAt: '1970-01-01T00:00:00.000Z',
}))
return updView
}
formatFunnelToClickhouse(funnel: Funnel): ClickhouseFunnel {
const updFunnel = { ...funnel } as unknown as ClickhouseFunnel
updFunnel.name = _trim(funnel.name)
updFunnel.steps = _isString(updFunnel.steps)
? updFunnel.steps
: _join(updFunnel.steps, ',')
return updFunnel
}
formatFunnelsFromClickhouse(funnels: ClickhouseFunnel[]): Funnel[] {
if (_isEmpty(funnels)) {
return []
}
return _map(funnels, this.formatFunnelFromClickhouse)
}
formatFunnelFromClickhouse(funnel: any): Funnel {
const updFunnel = { ...funnel }
updFunnel.steps = _isNull(updFunnel.steps)
? []
: _split(updFunnel.steps, ',')
return updFunnel
}
validateOrigins(projectDTO: ProjectDTO | UpdateProjectDto) {
if (_size(_join(projectDTO.origins, ',')) > 300)
throw new UnprocessableEntityException(
'The list of allowed origins has to be smaller than 300 symbols',
)
_map(projectDTO.origins, (host) => {
if (!ORIGINS_REGEX.test(_trim(host))) {
throw new ConflictException(`Host ${host} is not correct`)
}
})
}
validateIPBlacklist(projectDTO: ProjectDTO | UpdateProjectDto) {
if (_size(_join(projectDTO.ipBlacklist, ',')) > 300)
throw new UnprocessableEntityException(
'The list of allowed blacklisted IP addresses must be less than 300 characters.',
)
_map(projectDTO.ipBlacklist, (ip) => {
if (!isValidIpRange(_trim(ip))) {
throw new ConflictException(`IP address ${ip} is not correct`)
}
})
}
validateIPWhitelist(projectDTO: ProjectDTO | UpdateProjectDto) {
if (!projectDTO.ipWhitelist) {
return
}
if (!Array.isArray(projectDTO.ipWhitelist))
throw new UnprocessableEntityException(
'The list of whitelisted IP addresses must be an array.',
)
if (_size(_join(projectDTO.ipWhitelist, ',')) > 300)
throw new UnprocessableEntityException(
'The list of whitelisted IP addresses must be less than 300 characters.',
)
_map(projectDTO.ipWhitelist, (ip) => {
if (!isValidIpRange(_trim(ip))) {
throw new ConflictException(`IP address ${ip} is not correct`)
}
})
}
validateCountryBlacklist(projectDTO: ProjectDTO | UpdateProjectDto) {
if (!projectDTO.countryBlacklist) {
return
}
if (_size(projectDTO.countryBlacklist) > 250)
throw new UnprocessableEntityException(
'The list of blocked countries cannot exceed 250 entries.',
)
const countryCodeRegex = /^[A-Z]{2}$/
_map(projectDTO.countryBlacklist, (code) => {
const trimmedCode = _trim(code).toUpperCase()
if (!countryCodeRegex.test(trimmedCode)) {
throw new ConflictException(
`Country code "${code}" is not a valid ISO 3166-1 alpha-2 code`,
)
}
})
}
validateProject(
projectDTO: ProjectDTO | UpdateProjectDto,
creatingProject = false,
) {
if (_size(projectDTO.name) > 50)
throw new UnprocessableEntityException('The project name is too long')
if (creatingProject) {
return
}
if (projectDTO?.id && !isValidPID(projectDTO.id))
throw new UnprocessableEntityException(
'The provided Project ID (pid) is incorrect',
)
this.validateOrigins(projectDTO)
this.validateIPBlacklist(projectDTO)
this.validateIPWhitelist(projectDTO)
this.validateCountryBlacklist(projectDTO)
}
async getPIDsWhereSessionReplayDataExists(
projectIds: string[],
): Promise<string[]> {
if (_isEmpty(projectIds)) {
return []
}
const params = _reduce(
projectIds,
(acc, curr, index) => ({
...acc,
[`pid_${index}`]: curr,
}),
{},
)
const pids = _join(
_map(params, (val, key) => `{${key}:FixedString(12)}`),
',',
)
const query = `
SELECT DISTINCT pid
FROM session_replay_chunks
WHERE pid IN (${pids})
`
const { data } = await clickhouse
.query({
query,
query_params: params,
})
.then((resultSet) => resultSet.json<{ pid: string }>())
return _map(data, ({ pid }) => pid)
}
}