Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

17.9 KB
Raw Blame
import {
  ForbiddenException,
  Injectable,
  BadRequestException,
  UnprocessableEntityException,
  InternalServerErrorException,
  ConflictException,
} from '@nestjs/common'
import { randomUUID } from 'crypto'
import _isEmpty from 'lodash/isEmpty'
import _isString from 'lodash/isString'
import _size from 'lodash/size'
import _join from 'lodash/join'
import _filter from 'lodash/filter'
import _includes from 'lodash/includes'
import _find from 'lodash/find'
import _map from 'lodash/map'
import _isNull from 'lodash/isNull'
import _split from 'lodash/split'
import _trim from 'lodash/trim'
import _reduce from 'lodash/reduce'
import _findIndex from 'lodash/findIndex'
import { compareSync } from 'bcrypt'

import { Project, ProjectWithShare, Role } from './entity/project.entity'
import { ProjectDTO } from './dto/project.dto'
import {
  isValidPID,
  redis,
  ORIGINS_REGEX,
  getRedisProjectKey,
  redisProjectCacheTimeout,
  V2_VIEW_FILTER_DIMENSIONS,
} from '../common/constants'
import { clickhouse } from '../common/integrations/clickhouse'
import { isValidIpRange } from '../common/ip-range'
import {
  createProjectShareClickhouse,
  findProjectSharesByProjectClickhouse,
  getProjectClickhouse,
  updateProjectClickhouse,
} from '../common/utils'
import { MAX_PROJECT_PASSWORD_LENGTH, UpdateProjectDto } from './dto'
import { Funnel } from './entity/funnel.entity'
import { ProjectViewEntity } from './entity/project-view.entity'
import {
  CreateProjectViewDto,
  Filter,
  ProjectViewCustomEventDto,
} from './dto/create-project-view.dto'
import { ProjectViewCustomEventEntity } from './entity/project-view-custom-event.entity'
import { ClickhouseFunnel } from '../common/types'

// A list of characters that can be used in a Project ID
export const LEGAL_PID_CHARACTERS =
  '1234567890QWERTYUIOPASDFGHJKLZXCVBNMqwertyuiopasdfghjklzxcvbnm'
export const PID_LENGTH = 12

export const deleteProjectRedis = async (id: string) => {
  const key = getRedisProjectKey(id)

  try {
    await redis.del(key)
  } catch (reason) {
    console.error(`Error deleting project ${id} from redis: ${reason}`)
  }
}

@Injectable()
export class ProjectService {
  async getRedisProject(pid: string) {
    const pidKey = getRedisProjectKey(pid)
    let project: string | Project = await redis.get(pidKey)

    if (_isEmpty(project)) {
      project = this.formatFromClickhouse(await this.getFullProject(pid))

      if (_isEmpty(project))
        throw new BadRequestException(
          'The provided Project ID (pid) is incorrect',
        )

      await redis.set(
        pidKey,
        JSON.stringify(project),
        'EX',
        redisProjectCacheTimeout,
      )
    } else {
      try {
        project = JSON.parse(project)
      } catch {
        throw new InternalServerErrorException('Error while processing project')
      }
    }

    return project as ProjectWithShare | null
  }

  allowedToView(
    project: ProjectWithShare,
    uid: string | null,
    password?: string | null,
  ): void {
    if (
      project.public ||
      uid === project.adminId ||
      _findIndex(
        project.share,
        ({ user, confirmed }) => user?.id === uid && confirmed === true,
      ) !== -1
    ) {
      return null
    }

    if (project.isPasswordProtected && password) {
      if (
        _size(password) <= MAX_PROJECT_PASSWORD_LENGTH &&
        compareSync(password, project.passwordHash)
      ) {
        return null
      }

      throw new ConflictException('Incorrect password')
    }

    if (project.isPasswordProtected) {
      throw new ForbiddenException('This project is password protected')
    }

    throw new ForbiddenException('You are not allowed to view this project')
  }

  // Instead of passing relations to the findOne method every time, this method returns a project
  // with necessary relations needed for the allowedToView and allowedToManage methods
  async getFullProject(pid: string): Promise<ProjectWithShare | null> {
    const project = await getProjectClickhouse(pid)

    if (_isEmpty(project)) {
      return null
    }

    const shares = await findProjectSharesByProjectClickhouse(pid)
    const share = (shares || []).map((row: any) => ({
      id: row.id,
      role: row.role,
      confirmed: Boolean(row.confirmed),
      user: {
        id: row.userId,
        email: row.email,
      },
    }))

    return {
      ...project,
      share,
    }
  }

  async getOwnProject(projectId: string, userId: string) {
    const project = await this.getFullProject(projectId)

    if (!project) {
      return project
    }

    this.allowedToManage(project, userId)

    return project
  }

  async confirmTransferProject(
    projectId: string,
    userId: string,
    oldAdminId: string,
  ) {
    await updateProjectClickhouse(
      {
        id: projectId,
        adminId: userId,
      },
      { ignoreAllowedKeys: true },
    )

    await createProjectShareClickhouse({
      id: randomUUID(),
      userId: oldAdminId,
      projectId: projectId,
      confirmed: 1,
      role: Role.admin,
    })

    await deleteProjectRedis(projectId)
  }

  allowedToManage(
    project: ProjectWithShare,
    uid: string,
    message = 'You are not allowed to manage this project',
  ): void {
    if (
      uid === project.adminId ||
      _findIndex(
        project.share,
        (share) =>
          share.user?.id === uid &&
          share.role === Role.admin &&
          share.confirmed === true,
      ) !== -1
    ) {
      return null
    }

    throw new ForbiddenException(message)
  }

  isPIDUnique(projects: Project[], pid: string): boolean {
    return !_find(projects, ({ id }) => id === pid)
  }

  checkIfIDUnique(projects: Project[], pid: string): void {
    const isUnique = this.isPIDUnique(projects, pid)

    if (!isUnique) {
      throw new BadRequestException('Selected project ID is already in use')
    }
  }

  async getPIDsWhereAnalyticsDataExists(
    projectIds: string[],
  ): Promise<string[]> {
    if (_isEmpty(projectIds)) {
      return []
    }

    const params = _reduce(
      projectIds,
      (acc, curr, index) => ({
        ...acc,
        [`pid_${index}`]: curr,
      }),
      {},
    )

    const pids = _join(
      _map(params, (val, key) => `{${key}:FixedString(12)}`),
      ',',
    )

    const query = `
      SELECT DISTINCT pid
      FROM events
      WHERE pid IN (${pids})
        AND type IN ('pageview', 'custom_event', 'outbound')
    `

    const { data } = await clickhouse
      .query({
        query,
        query_params: params,
      })
      .then((resultSet) => resultSet.json<{ pid: string }>())

    return _map(data, ({ pid }) => pid)
  }

  async getPIDsWhereErrorsDataExists(projectIds: string[]): Promise<string[]> {
    if (_isEmpty(projectIds)) {
      return []
    }

    const params = _reduce(
      projectIds,
      (acc, curr, index) => ({
        ...acc,
        [`pid_${index}`]: curr,
      }),
      {},
    )

    const pids = _join(
      _map(params, (val, key) => `{${key}:FixedString(12)}`),
      ',',
    )

    const query = `
      SELECT DISTINCT pid
      FROM events
      WHERE pid IN (${pids})
        AND type = 'error'
    `

    const { data } = await clickhouse
      .query({
        query,
        query_params: params,
      })
      .then((resultSet) => resultSet.json<{ pid: string }>())

    return _map(data, ({ pid }) => pid)
  }

  async getPIDsWhereCaptchaDataExists(projectIds: string[]): Promise<string[]> {
    if (_isEmpty(projectIds)) {
      return []
    }

    const params = _reduce(
      projectIds,
      (acc, curr, index) => ({
        ...acc,
        [`pid_${index}`]: curr,
      }),
      {},
    )

    const pids = _join(
      _map(params, (val, key) => `{${key}:FixedString(12)}`),
      ',',
    )

    const query = `
      SELECT DISTINCT pid
      FROM events
      WHERE pid IN (${pids})
        AND type = 'captcha'
    `

    const { data } = await clickhouse
      .query({
        query,
        query_params: params,
      })
      .then((resultSet) => resultSet.json<{ pid: string }>())

    return _map(data, ({ pid }) => pid)
  }

  async removeDataFromClickhouse(
    pid: string,
    from: string,
    to: string,
  ): Promise<void> {
    const queries = [
      "ALTER TABLE events DELETE WHERE pid = {pid:FixedString(12)} AND type IN ('pageview', 'custom_event', 'performance', 'error', 'captcha', 'outbound') AND created BETWEEN {from:String} AND {to:String}",
      'ALTER TABLE error_statuses DELETE WHERE pid = {pid:FixedString(12)} AND created BETWEEN {from:String} AND {to:String}',
    ]

    const params = {
      pid,
      from,
      to,
    }

    await Promise.all(
      _map(queries, (query) =>
        clickhouse.command({
          query,
          query_params: params,
        }),
      ),
    )
  }

  formatToClickhouse(project: any) {
    const updProject = { ...project }
    updProject.active = Number(updProject.active)
    updProject.public = Number(updProject.public)
    updProject.isPasswordProtected = Number(updProject.isPasswordProtected)

    if (updProject.isCaptchaProject !== undefined) {
      updProject.isCaptchaProject = Number(updProject.isCaptchaProject)
    }

    if (!_isNull(updProject.origins)) {
      updProject.origins = _isString(updProject.origins)
        ? updProject.origins
        : _join(updProject.origins, ',')
    }

    if (!_isNull(updProject.ipBlacklist)) {
      updProject.ipBlacklist = _isString(updProject.ipBlacklist)
        ? updProject.ipBlacklist
        : _join(updProject.ipBlacklist, ',')
    }

    if (!_isNull(updProject.ipWhitelist)) {
      updProject.ipWhitelist = _isString(updProject.ipWhitelist)
        ? updProject.ipWhitelist
        : _join(updProject.ipWhitelist, ',')
    }

    if (!_isNull(updProject.countryBlacklist)) {
      updProject.countryBlacklist = _isString(updProject.countryBlacklist)
        ? updProject.countryBlacklist
        : _join(updProject.countryBlacklist, ',')
    }

    if (updProject.brandKeywords !== undefined) {
      if (
        updProject.brandKeywords === null ||
        (Array.isArray(updProject.brandKeywords) &&
          updProject.brandKeywords.length === 0)
      ) {
        updProject.brandKeywords = null
      } else if (Array.isArray(updProject.brandKeywords)) {
        updProject.brandKeywords = JSON.stringify(updProject.brandKeywords)
      }
    }

    return updProject
  }

  formatFromClickhouse(project: any): Project {
    const updProject = { ...project }
    updProject.active = Boolean(updProject.active)
    updProject.public = Boolean(updProject.public)
    updProject.isPasswordProtected = Boolean(updProject.isPasswordProtected)
    updProject.isCaptchaProject = Boolean(updProject.isCaptchaProject)

    updProject.origins = _isEmpty(updProject.origins)
      ? []
      : _split(updProject.origins, ',')

    updProject.ipBlacklist = _isEmpty(updProject.ipBlacklist)
      ? []
      : _split(updProject.ipBlacklist, ',')

    updProject.ipWhitelist = _isEmpty(updProject.ipWhitelist)
      ? []
      : _split(updProject.ipWhitelist, ',')

    updProject.countryBlacklist = _isEmpty(updProject.countryBlacklist)
      ? []
      : _split(updProject.countryBlacklist, ',')

    if (_isString(updProject.brandKeywords)) {
      const trimmed = updProject.brandKeywords.trim()

      if (trimmed.length === 0) {
        updProject.brandKeywords = null
      } else {
        try {
          const parsed = JSON.parse(trimmed)
          updProject.brandKeywords =
            Array.isArray(parsed) && parsed.length > 0 ? parsed : null
        } catch {
          updProject.brandKeywords = null
        }
      }
    } else if (!Array.isArray(updProject.brandKeywords)) {
      updProject.brandKeywords = null
    }

    return updProject
  }

  filterUnsupportedColumns(
    filters: CreateProjectViewDto['filters'],
  ): CreateProjectViewDto['filters'] {
    if (!filters) {
      return []
    }

    return _filter(filters, ({ dimension }) =>
      _includes(V2_VIEW_FILTER_DIMENSIONS, dimension),
    )
  }

  formatCustomEventToClickhose(
    viewId: string,
    customEvent: ProjectViewCustomEventEntity | ProjectViewCustomEventDto,
  ) {
    if (!customEvent) {
      return null
    }

    return {
      viewId,
      // @ts-expect-error
      id: customEvent.id,
      customEventName: customEvent.customEventName,
      metaKey: customEvent.metaKey || null,
      metaValue: customEvent.metaValue || null,
      metaValueType: customEvent.metaValueType,
      metricKey: customEvent.metricKey,
    }
  }

  formatCustomEventsToClickhouse(
    viewId: string,
    customEvents: (ProjectViewCustomEventEntity | ProjectViewCustomEventDto)[],
  ) {
    if (_isEmpty(customEvents)) {
      return []
    }

    return _map(customEvents, (customEvent) =>
      this.formatCustomEventToClickhose(viewId, customEvent),
    )
  }

  formatViewToClickhouse(
    view: (ProjectViewEntity | CreateProjectViewDto) & {
      id: string
      projectId: string
    },
  ) {
    const { id, projectId, type, customEvents, name, filters } = view

    return {
      id,
      projectId,
      type,
      customEvents: this.formatCustomEventsToClickhouse(id, customEvents),
      name,
      filters: JSON.stringify(
        this.filterUnsupportedColumns(filters as Filter[]),
        null,
        2,
      ),
    }
  }

  formatViewsFromClickhouse(views: any[]): ProjectViewEntity[] {
    if (_isEmpty(views)) {
      return []
    }

    return _map(views, this.formatViewFromClickhouse)
  }

  formatViewFromClickhouse(view: any): ProjectViewEntity {
    const updView = { ...view }
    updView.name = _trim(updView.name)

    try {
      updView.filters = JSON.parse(view.filters)
    } catch (reason) {
      console.error(
        '[ERROR] formatViewFromClickhouse, failed to parse filters:',
        reason,
      )
      console.error('VIEW:', view)
      updView.filters = []
    }

    updView.customEvents = _map(view.customEvents, (customEvent) => ({
      ...customEvent,
      createdAt: '1970-01-01T00:00:00.000Z',
      updatedAt: '1970-01-01T00:00:00.000Z',
    }))

    return updView
  }

  formatFunnelToClickhouse(funnel: Funnel): ClickhouseFunnel {
    const updFunnel = { ...funnel } as unknown as ClickhouseFunnel
    updFunnel.name = _trim(funnel.name)
    updFunnel.steps = _isString(updFunnel.steps)
      ? updFunnel.steps
      : _join(updFunnel.steps, ',')

    return updFunnel
  }

  formatFunnelsFromClickhouse(funnels: ClickhouseFunnel[]): Funnel[] {
    if (_isEmpty(funnels)) {
      return []
    }

    return _map(funnels, this.formatFunnelFromClickhouse)
  }

  formatFunnelFromClickhouse(funnel: any): Funnel {
    const updFunnel = { ...funnel }

    updFunnel.steps = _isNull(updFunnel.steps)
      ? []
      : _split(updFunnel.steps, ',')

    return updFunnel
  }

  validateOrigins(projectDTO: ProjectDTO | UpdateProjectDto) {
    if (_size(_join(projectDTO.origins, ',')) > 300)
      throw new UnprocessableEntityException(
        'The list of allowed origins has to be smaller than 300 symbols',
      )

    _map(projectDTO.origins, (host) => {
      if (!ORIGINS_REGEX.test(_trim(host))) {
        throw new ConflictException(`Host ${host} is not correct`)
      }
    })
  }

  validateIPBlacklist(projectDTO: ProjectDTO | UpdateProjectDto) {
    if (_size(_join(projectDTO.ipBlacklist, ',')) > 300)
      throw new UnprocessableEntityException(
        'The list of allowed blacklisted IP addresses must be less than 300 characters.',
      )
    _map(projectDTO.ipBlacklist, (ip) => {
      if (!isValidIpRange(_trim(ip))) {
        throw new ConflictException(`IP address ${ip} is not correct`)
      }
    })
  }

  validateIPWhitelist(projectDTO: ProjectDTO | UpdateProjectDto) {
    if (!projectDTO.ipWhitelist) {
      return
    }

    if (!Array.isArray(projectDTO.ipWhitelist))
      throw new UnprocessableEntityException(
        'The list of whitelisted IP addresses must be an array.',
      )

    if (_size(_join(projectDTO.ipWhitelist, ',')) > 300)
      throw new UnprocessableEntityException(
        'The list of whitelisted IP addresses must be less than 300 characters.',
      )
    _map(projectDTO.ipWhitelist, (ip) => {
      if (!isValidIpRange(_trim(ip))) {
        throw new ConflictException(`IP address ${ip} is not correct`)
      }
    })
  }

  validateCountryBlacklist(projectDTO: ProjectDTO | UpdateProjectDto) {
    if (!projectDTO.countryBlacklist) {
      return
    }

    if (_size(projectDTO.countryBlacklist) > 250)
      throw new UnprocessableEntityException(
        'The list of blocked countries cannot exceed 250 entries.',
      )

    const countryCodeRegex = /^[A-Z]{2}$/
    _map(projectDTO.countryBlacklist, (code) => {
      const trimmedCode = _trim(code).toUpperCase()
      if (!countryCodeRegex.test(trimmedCode)) {
        throw new ConflictException(
          `Country code "${code}" is not a valid ISO 3166-1 alpha-2 code`,
        )
      }
    })
  }

  validateProject(
    projectDTO: ProjectDTO | UpdateProjectDto,
    creatingProject = false,
  ) {
    if (_size(projectDTO.name) > 50)
      throw new UnprocessableEntityException('The project name is too long')

    if (creatingProject) {
      return
    }

    if (projectDTO?.id && !isValidPID(projectDTO.id))
      throw new UnprocessableEntityException(
        'The provided Project ID (pid) is incorrect',
      )

    this.validateOrigins(projectDTO)
    this.validateIPBlacklist(projectDTO)
    this.validateIPWhitelist(projectDTO)
    this.validateCountryBlacklist(projectDTO)
  }

  async getPIDsWhereSessionReplayDataExists(
    projectIds: string[],
  ): Promise<string[]> {
    if (_isEmpty(projectIds)) {
      return []
    }

    const params = _reduce(
      projectIds,
      (acc, curr, index) => ({
        ...acc,
        [`pid_${index}`]: curr,
      }),
      {},
    )

    const pids = _join(
      _map(params, (val, key) => `{${key}:FixedString(12)}`),
      ',',
    )

    const query = `
      SELECT DISTINCT pid
      FROM session_replay_chunks
      WHERE pid IN (${pids})
    `

    const { data } = await clickhouse
      .query({
        query,
        query_params: params,
      })
      .then((resultSet) => resultSet.json<{ pid: string }>())

    return _map(data, ({ pid }) => pid)
  }
}