📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
Merge branch 'main' into feature/captcha
Andrii R1debb522b636
diff --cc package.jsonindex 41d5d2236,c6f207765..f0ff7f2a1--- a/package.json+++ b/package.json@@@ -44,9 -44,8 +44,9 @@@"class-validator": "^0.14.0","clickhouse": "^2.6.0","cookie-parser": "^1.4.6",- "countries-and-timezones": "^3.4.0",+ "countries-and-timezones": "^3.4.1","cross-env": "^7.0.3",+ "crypto-js": "^4.1.1","dayjs": "^1.11.7","dotenv": "^16.0.3","express": "^4.18.2",@@@ -72,24 -71,23 +72,24 @@@"reflect-metadata": "^0.1.13","rimraf": "^3.0.2","rxjs": "^7.8.0",- "telegraf": "^4.12.2",+ "svg-captcha": "^1.4.0",+ "telegraf": "^4.11.2","typeorm": "^0.2.45",- "ua-parser-js": "^1.0.32",+ "ua-parser-js": "^1.0.34","uuid": "^9.0.0","validate-ip-node": "^1.0.8",- "validator": "^13.7.0"+ "validator": "^13.9.0"},"devDependencies": {- "@nestjs/cli": "^9.1.5",- "@nestjs/testing": "^9.2.1",+ "@nestjs/cli": "^9.3.0",+ "@nestjs/testing": "^9.3.10","@types/bcrypt": "^5.0.0",- "@types/express": "^4.17.15",+ "@types/express": "^4.17.17","@types/ioredis": "^4.28.10",- "@types/jest": "^29.2.4",+ "@types/jest": "^29.5.0","@types/multer": "^1.4.7","@types/newrelic": "^9.4.0",- "@types/node": "^18.11.18",+ "@types/node": "^18.15.3","@types/passport-jwt": "^3.0.8","@types/supertest": "^2.0.12","@types/ua-parser-js": "^0.7.36",diff --cc src/app.module.tsindex ea7123bcb,26e0c1d25..53738e1d6--- a/src/app.module.ts+++ b/src/app.module.ts@@@ -20,7 -20,7 +20,7 @@@ import { AlertModule } from './alert/alimport { I18nModule } from 'nestjs-i18n'import { getI18nConfig } from './configs'import { AuthModule } from './auth/auth.module'- import { CaptchaModule } from './captcha/captcha.module'-import { isDevelopment, isTgTokenPresent } from './common/constants'++import { CaptchaModule, isDevelopment, isTgTokenPresent } from './captcha/captcha.module'const modules = [ConfigModule.forRoot({ envFilePath: '.env', isGlobal: true }),@@@ -50,10 -50,9 +50,10 @@@MarketplaceModule,AlertModule,AuthModule,+ CaptchaModule,]- if (process.env.TG_BOT_TOKEN) {+ if (isTgTokenPresent) {modules.push(TelegrafModule.forRoot({token: process.env.TG_BOT_TOKEN,diff --cc src/common/constants.tsindex 98ace0008,190f73ee2..c3bac4656--- a/src/common/constants.ts+++ b/src/common/constants.ts@@@ -40,8 -39,10 +40,11 @@@ const clickhouse = new ClickHouse(})const isSelfhosted = Boolean(process.env.SELFHOSTED)+ const isTgTokenPresent = Boolean(process.env.TG_BOT_TOKEN)+ const isDevelopment = process.env.NODE_ENV === 'development'const isNewRelicEnabled = Boolean(process.env.USE_NEW_RELIC)+const isDevelopment = process.env.NODE_ENV === 'development'+ const PRODUCTION_ORIGIN = process.env.CLIENT_URL || 'https://swetrix.com'const CLICKHOUSE_INIT_QUERIES = ['CREATE DATABASE IF NOT EXISTS analytics',@@@ -239,12 -229,7 +242,14 @@@ exportORIGINS_REGEX,REDIS_LOG_PERF_CACHE_KEY,REDIS_PERFORMANCE_COUNT_KEY,+ CAPTCHA_SALT,+ CAPTCHA_ENCRYPTION_KEY,isDevelopment,+ getRedisCaptchaKey,+ CAPTCHA_COOKIE_KEY,+ CAPTCHA_TOKEN_LIFETIME,+ PID_REGEX,+ CAPTCHA_SECRET_KEY_LENGTH,+ PRODUCTION_ORIGIN,+ isTgTokenPresent,}diff --cc src/project/entity/project.entity.tsindex 5dc710173,d234c1ab3..ca867cb39--- a/src/project/entity/project.entity.ts+++ b/src/project/entity/project.entity.ts@@@ -1,91 -1,78 +1,96 @@@-import {- Entity,- Column,- PrimaryColumn,- ManyToOne,- OneToMany,- JoinTable,-} from 'typeorm'-import { ApiProperty } from '@nestjs/swagger'--import { User } from '../../user/entities/user.entity'-import { ProjectShare } from './project-share.entity'-import { Alert } from 'src/alert/entity/alert.entity'-import { ExtensionToProject } from '../../marketplace/extensions/entities/extension-to-project.entity'-import { ProjectSubscriber } from './project-subscriber.entity'--// In case of modifying some properties here, make sure to also edit them in common/constants.ts -> selfhosted -> clickhouse-// and to add them to the GDPR data export email template-@Entity()-export class Project {- @ApiProperty()- @PrimaryColumn('varchar', {- unique: true,- length: 12,- })- id: string-- @ApiProperty()- @Column('varchar', { length: 50 })- name: string-- @ApiProperty()- @Column('simple-array')- origins: string[]-- @ApiProperty()- @Column('simple-array', { nullable: true, default: null })- ipBlacklist: string[]-- @ApiProperty()- @Column({- default: true,- })- active: boolean-- @ApiProperty()- @Column({- default: false,- })- public: boolean-- @ApiProperty({ type: () => User })- @ManyToOne(() => User, user => user.projects)- admin: User-- @ApiProperty({ type: () => ProjectShare })- @OneToMany(() => ProjectShare, share => share.project)- share: ProjectShare[]-- @ApiProperty({ type: () => Alert })- @OneToMany(() => Alert, alert => alert.project)- alerts: Alert[]-- @Column({ type: 'timestamp', default: () => 'CURRENT_TIMESTAMP' })- created: Date-- @OneToMany(- () => ExtensionToProject,- extensionToProject => extensionToProject.project,- )- @JoinTable()- extensions: ExtensionToProject[]-- @OneToMany(() => ProjectSubscriber, subscriber => subscriber.project, {- cascade: true,- })- subscribers: ProjectSubscriber[]-}+import {+ Entity,+ Column,+ PrimaryColumn,+ ManyToOne,+ OneToMany,+ JoinTable,+} from 'typeorm'+import { ApiProperty } from '@nestjs/swagger'++import { User } from '../../user/entities/user.entity'+import { ProjectShare } from './project-share.entity'+import { Alert } from 'src/alert/entity/alert.entity'+import { ExtensionToProject } from '../../marketplace/extensions/entities/extension-to-project.entity'++import { ProjectSubscriber } from './project-subscriber.entity'+import { CAPTCHA_SECRET_KEY_LENGTH } from '../../common/constants'++// In case of modifying some properties here, make sure to also edit them in common/constants.ts -> selfhosted -> clickhouse+// and to add them to the GDPR data export email template+@Entity()+export class Project {+ @ApiProperty()+ @PrimaryColumn('varchar', {+ unique: true,+ length: 12,+ })+ id: string++ @ApiProperty()+ @Column('varchar', { length: 50 })+ name: string++ @ApiProperty()+ @Column('simple-array')+ origins: string[]++ @ApiProperty()+ @Column('simple-array', { nullable: true, default: null })+ ipBlacklist: string[]++ @ApiProperty()+ @Column({+ default: true,+ })+ active: boolean++ @ApiProperty()+ @Column({+ default: false,+ })+ public: boolean-+++ // Swetrix CAPTCHA related stuff-+ @ApiProperty()+ @Column('boolean', { default: true })+ isAnalyticsProject: boolean++ @ApiProperty()+ @Column('boolean', { default: false })+ isCaptchaProject: boolean++ @ApiProperty()+ @Column('boolean', { default: false })+ isCaptchaEnabled: boolean++ @ApiProperty()+ @Column('varchar', { default: null, length: CAPTCHA_SECRET_KEY_LENGTH })+ captchaSecretKey: string++ @ApiProperty({ type: () => User })+ @ManyToOne(() => User, user => user.projects)+ admin: User++ @ApiProperty({ type: () => ProjectShare })+ @OneToMany(() => ProjectShare, share => share.project)+ share: ProjectShare[]++ @ApiProperty({ type: () => Alert })+ @OneToMany(() => Alert, alert => alert.project)+ alerts: Alert[]++ @Column({ type: 'timestamp', default: () => 'CURRENT_TIMESTAMP' })+ created: Date++ @OneToMany(+ () => ExtensionToProject,+ extensionToProject => extensionToProject.project,+ )+ @JoinTable()+ extensions: ExtensionToProject[]++++ @OneToMany(() => ProjectSubscriber, subscriber => subscriber.project, {++ cascade: true,++ })++ subscribers: ProjectSubscriber[]+}diff --cc src/project/project.controller.tsindex 914e8c7c7,3da2a5789..801fc756f--- a/src/project/project.controller.ts+++ b/src/project/project.controller.ts@@@ -47,7 -48,8 +49,9 @@@ importclickhouse,isSelfhosted,PROJECT_INVITE_EXPIRE,+ CAPTCHA_SECRET_KEY_LENGTH,+ isDevelopment,+ PRODUCTION_ORIGIN,} from '../common/constants'import {getProjectsClickhouse,diff --cc src/project/project.service.tsindex 3f5628c7b,96fa6c091..7ea121cd6--- a/src/project/project.service.ts+++ b/src/project/project.service.ts@@@ -1,481 -1,560 +1,596 @@@-import {- ForbiddenException,- Injectable,- BadRequestException,- UnprocessableEntityException,- InternalServerErrorException,- ConflictException,-} from '@nestjs/common'-import { InjectRepository } from '@nestjs/typeorm'-import { Repository } from 'typeorm'-import * as _isEmpty from 'lodash/isEmpty'-import * as _isString from 'lodash/isString'-import * as _size from 'lodash/size'-import * as _join from 'lodash/join'-import * as _find from 'lodash/find'-import * as _map from 'lodash/map'-import * as _pick from 'lodash/pick'-import * as _trim from 'lodash/trim'-import * as _findIndex from 'lodash/findIndex'-import * as _includes from 'lodash/includes'-import * as dayjs from 'dayjs'-import * as utc from 'dayjs/plugin/utc'-// @ts-ignore-import * as validateIP from 'validate-ip-node'--import { Pagination, PaginationOptionsInterface } from '../common/pagination'-import { Project } from './entity/project.entity'-import { ProjectShare } from './entity/project-share.entity'-import { ProjectDTO } from './dto/project.dto'-import { UserType } from '../user/entities/user.entity'-import { UserService } from 'src/user/user.service'-import { Role } from '../project/entity/project-share.entity'-import {- isValidPID,- redisProjectCountCacheTimeout,- getRedisUserCountKey,- redis,- clickhouse,- isSelfhosted,- IP_REGEX,- ORIGINS_REGEX,- getRedisProjectKey,-} from '../common/constants'-import { ProjectSubscriber } from './entity'-import { ActionTokensService } from 'src/action-tokens/action-tokens.service'-import { ActionTokenType } from 'src/action-tokens/action-token.entity'-import { MailerService } from 'src/mailer/mailer.service'-import { LetterTemplate } from 'src/mailer/letter'-import { AddSubscriberType } from './types'-import { GetSubscribersQueriesDto, UpdateSubscriberBodyDto } from './dto'-import { ReportFrequency } from './enums'--dayjs.extend(utc)--// const updateProjectRedis = async (id: string, project: Project) => {-// const key = getRedisProjectKey(id)--// try {-// await redis.set(-// key,-// JSON.stringify(project),-// 'EX',-// redisProjectCacheTimeout,-// )-// } catch {-// await redis.del(key)-// }-// }--export const deleteProjectRedis = async (id: string) => {- const key = getRedisProjectKey(id)-- try {- await redis.del(key)- } catch (e) {- console.error(`Error deleting project ${id} from redis: ${e}`)- }-}--export const deleteProjectsRedis = async (ids: string[]) => {- await Promise.all(_map(ids, deleteProjectRedis))-}--export const processProjectUser = (project: Project): Project => {- const { share } = project-- for (let j = 0; j < _size(share); ++j) {- const { user } = share[j]-- if (user) {- share[j].user = _pick(user, ['email'])- }- }-- return project-}--export const processProjectsUser = (projects: Project[]): Project[] => {- for (let i = 0; i < _size(projects); ++i) {- projects[i] = processProjectUser(projects[i])- }-- return projects-}--@Injectable()-export class ProjectService {- constructor(- @InjectRepository(Project)- private projectsRepository: Repository<Project>,- @InjectRepository(ProjectShare)- private projectShareRepository: Repository<ProjectShare>,- private userService: UserService,- @InjectRepository(ProjectSubscriber)- private readonly projectSubscriberRepository: Repository<ProjectSubscriber>,- private readonly actionTokens: ActionTokensService,- private readonly mailerService: MailerService,- ) {}-- async paginate(- options: PaginationOptionsInterface,- where: Record<string, unknown> | undefined,- ): Promise<Pagination<Project>> {- const [results, total] = await this.projectsRepository.findAndCount({- take: options.take || 100,- skip: options.skip || 0,- where,- order: {- name: 'ASC',- },- relations: ['share', 'share.user'],- })-- return new Pagination<Project>({- results: processProjectsUser(results),- total,- })- }-- async paginateShared(- options: PaginationOptionsInterface,- where: Record<string, unknown> | undefined,- ): Promise<Pagination<ProjectShare>> {- const [results, total] = await this.projectShareRepository.findAndCount({- take: options.take || 100,- skip: options.skip || 0,- where,- order: {- project: 'ASC',- },- relations: ['project'],- })-- return new Pagination<ProjectShare>({- // results: processProjectsUser(results),- results,- total,- })- }-- async count(): Promise<number> {- return await this.projectsRepository.count()- }-- async create(project: ProjectDTO | Project): Promise<Project> {- return this.projectsRepository.save(project)- }-- async update(- id: string,- projectDTO: ProjectDTO,- ): Promise<any> {- return this.projectsRepository.update(id, projectDTO)- }-- async delete(id: string): Promise<any> {- return this.projectsRepository.delete(id)- }-- async deleteMultiple(pids: string[]): Promise<any> {- return (- this.projectsRepository- .createQueryBuilder()- .delete()- // TODO: !!! Enforce Prepared Statements and Parameterization- .where(`id IN (${pids})`)- .execute()- )- }-- async deleteMultipleShare(where: string): Promise<any> {- return this.projectShareRepository- .createQueryBuilder()- .delete()- .where(where)- .execute()- }-- async createShare(share: ProjectShare): Promise<ProjectShare> {- return this.projectShareRepository.save(share)- }-- async deleteShare(id: string): Promise<any> {- return this.projectShareRepository.delete(id)- }-- async updateShare(id: string, share: ProjectShare | Object): Promise<any> {- return this.projectShareRepository.update(id, share)- }-- async findShare(params: object): Promise<ProjectShare[]> {- return this.projectShareRepository.find(params)- }-- async findOneShare(- id: string,- params: Object = {},- ): Promise<ProjectShare | null> {- return this.projectShareRepository.findOne(id, params)- }-- findOneWithRelations(id: string): Promise<Project | null> {- return this.projectsRepository.findOne(id, { relations: ['admin'] })- }-- findOne(id: string, params: Object = {}): Promise<Project | null> {- return this.projectsRepository.findOne(id, params)- }-- findWhere(- where: Record<string, unknown>,- relations?: string[],- ): Promise<Project[]> {- return this.projectsRepository.find({ where, relations })- }-- find(params: object): Promise<Project[]> {- return this.projectsRepository.find(params)- }-- findOneWhere(- where: Record<string, unknown>,- params: object = {},- ): Promise<Project> {- return this.projectsRepository.findOne({ where, ...params })- }-- allowedToView(project: Project, uid: string | null): void {- if (- project.public ||- uid === project.admin?.id ||- _findIndex(project.share, ({ user }) => user?.id === uid) !== -1- ) {- return- } else {- throw new ForbiddenException('You are not allowed to view this project')- }- }-- allowedToManage(- project: Project,- uid: string,- roles: Array<UserType> = [],- message: string = 'You are not allowed to manage this project'- ): void {- if (- uid === project.admin?.id ||- _includes(roles, UserType.ADMIN) ||- _findIndex(- project.share,- share => share.user?.id === uid && share.role === Role.admin,- ) !== -1- ) {- return- } else {- throw new ForbiddenException(message)- }- }-- async checkIfIDUnique(projectID: string): Promise<void> {- const project = await this.findOne(projectID)-- if (project) {- throw new BadRequestException('Selected project ID is already in use')- }- }-- checkIfIDUniqueClickhouse(projects: Array<object>, projectID: string): void {- if (_find(projects, ({ id }) => id === projectID)) {- throw new BadRequestException('Selected project ID is already in use')- }- }-- formatToClickhouse(project: Project): object {- const updProject = { ...project }- // @ts-ignore- updProject.active = Number(updProject.active)- // @ts-ignore- updProject.public = Number(updProject.public)- // @ts-ignore- updProject.origins = _isString(updProject.origins)- ? updProject.origins- : _join(updProject.origins, ',')-- return updProject- }-- formatFromClickhouse(project: object): object {- const updProject = { ...project }- // @ts-ignore- updProject.active = Boolean(updProject.active)- // @ts-ignore- updProject.public = Boolean(updProject.public)-- return updProject- }-- validateProject(projectDTO: ProjectDTO) {- if (!isValidPID(projectDTO.id))- throw new UnprocessableEntityException(- 'The provided Project ID (pid) is incorrect',- )- if (_size(projectDTO.name) > 50)- throw new UnprocessableEntityException('The project name is too long')- if (_size(_join(projectDTO.origins, ',')) > 300)- throw new UnprocessableEntityException(- 'The list of allowed origins has to be smaller than 300 symbols',- )- if (_size(_join(projectDTO.ipBlacklist, ',')) > 300)- throw new UnprocessableEntityException(- 'The list of allowed blacklisted IP addresses must be less than 300 characters.',- )-- _map(projectDTO.origins, host => {- if (!ORIGINS_REGEX.test(_trim(host))) {- throw new ConflictException(`Host ${host} is not correct`)- }- })-- _map(projectDTO.ipBlacklist, ip => {- if (!validateIP(_trim(ip)) && !IP_REGEX.test(_trim(ip))) {- throw new ConflictException(`IP address ${ip} is not correct`)- }- })- }-- // Returns amount of exirsting events starting from month- async getRedisCount(uid: string): Promise<number | null> {- const countKey = getRedisUserCountKey(uid)- let count: string | number = await redis.get(countKey)-- if (_isEmpty(count)) {- const monthStart = dayjs- .utc()- .startOf('month')- .format('YYYY-MM-DD HH:mm:ss')- const monthEnd = dayjs.utc().endOf('month').format('YYYY-MM-DD HH:mm:ss')-- let pids-- if (isSelfhosted) {- // selfhosted has no limits- return 0- } else {- pids = await this.find({- where: {- admin: uid,- },- select: ['id'],- })- }-- if (_isEmpty(pids)) {- return 0- }-- const count_ev_query = `SELECT COUNT() FROM analytics WHERE pid IN (${_join(- _map(pids, el => `'${el.id}'`),- ',',- )}) AND created BETWEEN '${monthStart}' AND '${monthEnd}'`- const count_custom_ev_query = `SELECT COUNT() FROM customEV WHERE pid IN (${_join(- _map(pids, el => `'${el.id}'`),- ',',- )}) AND created BETWEEN '${monthStart}' AND '${monthEnd}'`-- const pageviews = (await clickhouse.query(count_ev_query).toPromise())[0][- 'count()'- ]- const customEvents = (- await clickhouse.query(count_custom_ev_query).toPromise()- )[0]['count()']-- count = pageviews + customEvents-- await redis.set(- countKey,- `${pageviews}`,- 'EX',- redisProjectCountCacheTimeout,- )- } else {- try {- // @ts-ignore- count = Number(count)- } catch (e) {- count = 0- console.error(e)- throw new InternalServerErrorException('Error while processing project')- }- }-- // @ts-ignore- return count- }-- async clearProjectsRedisCache(uid: string): Promise<void> {- const projects = await this.findWhere({- admin: uid,- })-- if (_isEmpty(projects)) {- return- }-- const pids = _map(projects, 'id')-- await deleteProjectsRedis(pids)- }-- async clearProjectsRedisCacheByEmail(email: string): Promise<void> {- const user = await this.userService.findOneWhere({ email })-- if (!user) {- return- }-- await this.clearProjectsRedisCache(user.id)- }-- async getProject(projectId: string, userId: string) {- return await this.projectsRepository.findOne({- where: {- id: projectId,- admin: { id: userId },- },- })- }-- async getSubscriberByEmail(projectId: string, email: string) {- return await this.projectSubscriberRepository.findOne({- where: { projectId, email },- })- }-- async addSubscriber(data: AddSubscriberType) {- const subscriber = await this.projectSubscriberRepository.save({ ...data })-- await this.sendSubscriberInvite(- data, subscriber.id,- )-- return subscriber- }-- async sendSubscriberInvite(- data: AddSubscriberType,- subscriberId: string,- ) {- const {- userId, projectId, projectName, email, origin- } = data-- const actionToken = await this.actionTokens.createActionToken(- userId,- ActionTokenType.ADDING_PROJECT_SUBSCRIBER,- `${projectId}:${subscriberId}`,- )-- const url = `${origin}/projects/${projectId}/subscribers/invite?token=${actionToken.id}`-- await this.mailerService.sendEmail(- email,- LetterTemplate.ProjectSubscriberInvitation,- {- url, projectName,- },- )- }-- async getProjectById(projectId: string) {- return await this.projectsRepository.findOne({- where: { id: projectId },- })- }-- async getSubscriber(projectId: string, subscriberId: string) {- return await this.projectSubscriberRepository.findOne({- where: { id: subscriberId, projectId },- })- }-- async confirmSubscriber(- projectId: string,- subscriberId: string,- token: string,- ) {- await this.projectSubscriberRepository.update(- { id: subscriberId, projectId },- { isConfirmed: true },- )- await this.actionTokens.deleteActionToken(token)- }-- async getSubscribers(projectId: string, queries: GetSubscribersQueriesDto) {- const [subscribers, count] =- await this.projectSubscriberRepository.findAndCount({- skip: Number(queries.offset) || 0,- take: Number(queries.limit) > 100 ? 100 : Number(queries.limit) || 100,- where: { projectId },- })-- return { subscribers, count }- }-- async updateSubscriber(- projectId: string,- subscriberId: string,- data: UpdateSubscriberBodyDto,- ) {- await this.projectSubscriberRepository.update(- { id: subscriberId, projectId },- data,- )-- return await this.getSubscriber(projectId, subscriberId)- }-- async removeSubscriber(projectId: string, subscriberId: string) {- await this.projectSubscriberRepository.delete({- id: subscriberId,- projectId,- })- }-- async getSubscribersForReports(reportFrequency: ReportFrequency) {- return await this.projectSubscriberRepository.find({- relations: ['project'],- where: { reportFrequency, isConfirmed: true },- })- }-- async getSubscriberProjects(subscriberId: string) {- const projects = await this.projectSubscriberRepository.find({- relations: ['project'],- where: { id: subscriberId },- })-- return projects.map(project => project.project)- }-}+import {+ ForbiddenException,+ Injectable,+ BadRequestException,+ UnprocessableEntityException,+ InternalServerErrorException,+ ConflictException,+} from '@nestjs/common'+import { InjectRepository } from '@nestjs/typeorm'+import { Repository } from 'typeorm'+import * as _isEmpty from 'lodash/isEmpty'+import * as _isString from 'lodash/isString'+import * as _size from 'lodash/size'+import * as _join from 'lodash/join'+import * as _find from 'lodash/find'+import * as _map from 'lodash/map'+import * as _pick from 'lodash/pick'+import * as _trim from 'lodash/trim'+import * as _findIndex from 'lodash/findIndex'+import * as _includes from 'lodash/includes'+import * as dayjs from 'dayjs'+import * as utc from 'dayjs/plugin/utc'+// @ts-ignore+import * as validateIP from 'validate-ip-node'++import { Pagination, PaginationOptionsInterface } from '../common/pagination'+import { Project } from './entity/project.entity'+import { ProjectShare } from './entity/project-share.entity'+import { ProjectDTO } from './dto/project.dto'+import { UserType } from '../user/entities/user.entity'+import { UserService } from 'src/user/user.service'+import { Role } from '../project/entity/project-share.entity'+import {+ isValidPID,+ redisProjectCountCacheTimeout,+ getRedisUserCountKey,+ redis,+ clickhouse,+ isSelfhosted,+ IP_REGEX,+ ORIGINS_REGEX,+ getRedisProjectKey,+ redisProjectCacheTimeout,+} from '../common/constants'+import {+ getProjectsClickhouse,+} from '../common/utils'++import { ProjectSubscriber } from './entity'++import { ActionTokensService } from 'src/action-tokens/action-tokens.service'++import { ActionTokenType } from 'src/action-tokens/action-token.entity'++import { MailerService } from 'src/mailer/mailer.service'++import { LetterTemplate } from 'src/mailer/letter'++import { AddSubscriberType } from './types'++import { GetSubscribersQueriesDto, UpdateSubscriberBodyDto } from './dto'++import { ReportFrequency } from './enums'++dayjs.extend(utc)++// const updateProjectRedis = async (id: string, project: Project) => {+// const key = getRedisProjectKey(id)++// try {+// await redis.set(+// key,+// JSON.stringify(project),+// 'EX',+// redisProjectCacheTimeout,+// )+// } catch {+// await redis.del(key)+// }+// }++export const deleteProjectRedis = async (id: string) => {+ const key = getRedisProjectKey(id)++ try {+ await redis.del(key)+ } catch (e) {+ console.error(`Error deleting project ${id} from redis: ${e}`)+ }+}++export const deleteProjectsRedis = async (ids: string[]) => {+ await Promise.all(_map(ids, deleteProjectRedis))+}++export const processProjectUser = (project: Project): Project => {+ const { share } = project++ for (let j = 0; j < _size(share); ++j) {+ const { user } = share[j]++ if (user) {+ share[j].user = _pick(user, ['email'])+ }+ }++ return project+}++export const processProjectsUser = (projects: Project[]): Project[] => {+ for (let i = 0; i < _size(projects); ++i) {+ projects[i] = processProjectUser(projects[i])+ }++ return projects+}++@Injectable()+export class ProjectService {+ constructor(+ @InjectRepository(Project)+ private projectsRepository: Repository<Project>,+ @InjectRepository(ProjectShare)+ private projectShareRepository: Repository<ProjectShare>,+ private userService: UserService,++ @InjectRepository(ProjectSubscriber)++ private readonly projectSubscriberRepository: Repository<ProjectSubscriber>,++ private readonly actionTokens: ActionTokensService,++ private readonly mailerService: MailerService,++ ) {}+ ) { }++ async getRedisProject(pid: string): Promise<Project | null> {+ const pidKey = getRedisProjectKey(pid)+ let project: string | Project = await redis.get(pidKey)++ if (_isEmpty(project)) {+ if (isSelfhosted) {+ project = await getProjectsClickhouse(pid)+ } else {+ // todo: optimise the relations - select+ // select only required columns+ // https://stackoverflow.com/questions/59645009/how-to-return-only-some-columns-of-a-relations-with-typeorm+ project = await this.findOne(pid, {+ relations: ['admin'],+ select: ['origins', 'active', 'admin', 'public', 'ipBlacklist', 'captchaSecretKey', 'isCaptchaEnabled'],+ })+ }+ if (_isEmpty(project))+ throw new BadRequestException(+ 'The provided Project ID (pid) is incorrect',+ )++ if (!isSelfhosted) {+ const share = await this.findShare({+ where: {+ project: pid,+ },+ relations: ['user'],+ })+ // @ts-ignore+ project = { ...project, share }+ }++ await redis.set(+ pidKey,+ JSON.stringify(project),+ 'EX',+ redisProjectCacheTimeout,+ )+ } else {+ try {+ project = JSON.parse(project)+ } catch {+ throw new InternalServerErrorException('Error while processing project')+ }+ }++ // @ts-ignore+ return project+ }++ async paginate(+ options: PaginationOptionsInterface,+ where: Record<string, unknown> | undefined,+ ): Promise<Pagination<Project>> {+ const [results, total] = await this.projectsRepository.findAndCount({+ take: options.take || 100,+ skip: options.skip || 0,+ where,+ order: {+ name: 'ASC',+ },+ relations: ['share', 'share.user'],+ })++ return new Pagination<Project>({+ results: processProjectsUser(results),+ total,+ })+ }++ async paginateShared(+ options: PaginationOptionsInterface,+ where: Record<string, unknown> | undefined,+ ): Promise<Pagination<ProjectShare>> {+ const [results, total] = await this.projectShareRepository.findAndCount({+ take: options.take || 100,+ skip: options.skip || 0,+ where,+ order: {+ project: 'ASC',+ },+ relations: ['project'],+ })++ return new Pagination<ProjectShare>({+ // results: processProjectsUser(results),+ results,+ total,+ })+ }++ async count(): Promise<number> {+ return await this.projectsRepository.count()+ }++ async create(project: ProjectDTO | Project): Promise<Project> {+ return this.projectsRepository.save(project)+ }++ async update(+ id: string,+ projectDTO: ProjectDTO | Project,+ ): Promise<any> {+ return this.projectsRepository.update(id, projectDTO)+ }++ async delete(id: string): Promise<any> {+ return this.projectsRepository.delete(id)+ }++ async deleteMultiple(pids: string[]): Promise<any> {+ return (+ this.projectsRepository+ .createQueryBuilder()+ .delete()+ // TODO: !!! Enforce Prepared Statements and Parameterization+ .where(`id IN (${pids})`)+ .execute()+ )+ }++ async deleteMultipleShare(where: string): Promise<any> {+ return this.projectShareRepository+ .createQueryBuilder()+ .delete()+ .where(where)+ .execute()+ }++ async createShare(share: ProjectShare): Promise<ProjectShare> {+ return this.projectShareRepository.save(share)+ }++ async deleteShare(id: string): Promise<any> {+ return this.projectShareRepository.delete(id)+ }++ async updateShare(id: string, share: ProjectShare | Object): Promise<any> {+ return this.projectShareRepository.update(id, share)+ }++ async findShare(params: object): Promise<ProjectShare[]> {+ return this.projectShareRepository.find(params)+ }++ async findOneShare(+ id: string,+ params: Object = {},+ ): Promise<ProjectShare | null> {+ return this.projectShareRepository.findOne(id, params)+ }++ findOneWithRelations(id: string): Promise<Project | null> {+ return this.projectsRepository.findOne(id, { relations: ['admin'] })+ }++ findOne(id: string, params: Object = {}): Promise<Project | null> {+ return this.projectsRepository.findOne(id, params)+ }++ findWhere(+ where: Record<string, unknown>,+ relations?: string[],+ ): Promise<Project[]> {+ return this.projectsRepository.find({ where, relations })+ }++ find(params: object): Promise<Project[]> {+ return this.projectsRepository.find(params)+ }++ findOneWhere(+ where: Record<string, unknown>,+ params: object = {},+ ): Promise<Project> {+ return this.projectsRepository.findOne({ where, ...params })+ }++ allowedToView(project: Project, uid: string | null): void {+ if (+ project.public ||+ uid === project.admin?.id ||+ _findIndex(project.share, ({ user }) => user?.id === uid) !== -1+ ) {+ return+ } else {+ throw new ForbiddenException('You are not allowed to view this project')+ }+ }++ allowedToManage(+ project: Project,+ uid: string,+ roles: Array<UserType> = [],+ message: string = 'You are not allowed to manage this project'+ ): void {+ if (+ uid === project.admin?.id ||+ _includes(roles, UserType.ADMIN) ||+ _findIndex(+ project.share,+ share => share.user?.id === uid && share.role === Role.admin,+ ) !== -1+ ) {+ return+ } else {+ throw new ForbiddenException(message)+ }+ }++ async checkIfIDUnique(projectID: string): Promise<void> {+ const project = await this.findOne(projectID)++ if (project) {+ throw new BadRequestException('Selected project ID is already in use')+ }+ }++ checkIfIDUniqueClickhouse(projects: Array<object>, projectID: string): void {+ if (_find(projects, ({ id }) => id === projectID)) {+ throw new BadRequestException('Selected project ID is already in use')+ }+ }++ formatToClickhouse(project: Project): object {+ const updProject = { ...project }+ // @ts-ignore+ updProject.active = Number(updProject.active)+ // @ts-ignore+ updProject.public = Number(updProject.public)+ // @ts-ignore+ updProject.origins = _isString(updProject.origins)+ ? updProject.origins+ : _join(updProject.origins, ',')++ return updProject+ }++ formatFromClickhouse(project: object): object {+ const updProject = { ...project }+ // @ts-ignore+ updProject.active = Boolean(updProject.active)+ // @ts-ignore+ updProject.public = Boolean(updProject.public)++ return updProject+ }++ validateProject(projectDTO: ProjectDTO) {+ if (!isValidPID(projectDTO.id))+ throw new UnprocessableEntityException(+ 'The provided Project ID (pid) is incorrect',+ )+ if (_size(projectDTO.name) > 50)+ throw new UnprocessableEntityException('The project name is too long')+ if (_size(_join(projectDTO.origins, ',')) > 300)+ throw new UnprocessableEntityException(+ 'The list of allowed origins has to be smaller than 300 symbols',+ )+ if (_size(_join(projectDTO.ipBlacklist, ',')) > 300)+ throw new UnprocessableEntityException(+ 'The list of allowed blacklisted IP addresses must be less than 300 characters.',+ )++ _map(projectDTO.origins, host => {+ if (!ORIGINS_REGEX.test(_trim(host))) {+ throw new ConflictException(`Host ${host} is not correct`)+ }+ })++ _map(projectDTO.ipBlacklist, ip => {+ if (!validateIP(_trim(ip)) && !IP_REGEX.test(_trim(ip))) {+ throw new ConflictException(`IP address ${ip} is not correct`)+ }+ })+ }++ // Returns amount of exirsting events starting from month+ async getRedisCount(uid: string): Promise<number | null> {+ const countKey = getRedisUserCountKey(uid)+ let count: string | number = await redis.get(countKey)++ if (_isEmpty(count)) {+ const monthStart = dayjs+ .utc()+ .startOf('month')+ .format('YYYY-MM-DD HH:mm:ss')+ const monthEnd = dayjs.utc().endOf('month').format('YYYY-MM-DD HH:mm:ss')++ let pids++ if (isSelfhosted) {+ // selfhosted has no limits+ return 0+ } else {+ pids = await this.find({+ where: {+ admin: uid,+ },+ select: ['id'],+ })+ }++ if (_isEmpty(pids)) {+ return 0+ }++ const count_ev_query = `SELECT COUNT() FROM analytics WHERE pid IN (${_join(+ _map(pids, el => `'${el.id}'`),+ ',',+ )}) AND created BETWEEN '${monthStart}' AND '${monthEnd}'`+ const count_custom_ev_query = `SELECT COUNT() FROM customEV WHERE pid IN (${_join(+ _map(pids, el => `'${el.id}'`),+ ',',+ )}) AND created BETWEEN '${monthStart}' AND '${monthEnd}'`++ const pageviews = (await clickhouse.query(count_ev_query).toPromise())[0][+ 'count()'+ ]+ const customEvents = (+ await clickhouse.query(count_custom_ev_query).toPromise()+ )[0]['count()']++ count = pageviews + customEvents++ await redis.set(+ countKey,+ `${pageviews}`,+ 'EX',+ redisProjectCountCacheTimeout,+ )+ } else {+ try {+ // @ts-ignore+ count = Number(count)+ } catch (e) {+ count = 0+ console.error(e)+ throw new InternalServerErrorException('Error while processing project')+ }+ }++ // @ts-ignore+ return count+ }++ async clearProjectsRedisCache(uid: string): Promise<void> {+ const projects = await this.findWhere({+ admin: uid,+ })++ if (_isEmpty(projects)) {+ return+ }++ const pids = _map(projects, 'id')++ await deleteProjectsRedis(pids)+ }++ async clearProjectsRedisCacheByEmail(email: string): Promise<void> {+ const user = await this.userService.findOneWhere({ email })++ if (!user) {+ return+ }++ await this.clearProjectsRedisCache(user.id)+ }++++ async getProject(projectId: string, userId: string) {++ return await this.projectsRepository.findOne({++ where: {++ id: projectId,++ admin: { id: userId },++ },++ })++ }++ async getSubscriberByEmail(projectId: string, email: string) {++ return await this.projectSubscriberRepository.findOne({++ where: { projectId, email },++ })++ }++ async addSubscriber(data: AddSubscriberType) {++ const subscriber = await this.projectSubscriberRepository.save({ ...data })++ await this.sendSubscriberInvite(++ data, subscriber.id,++ )++ return subscriber++ }++ async sendSubscriberInvite(++ data: AddSubscriberType,++ subscriberId: string,++ ) {++ const {++ userId, projectId, projectName, email, origin++ } = data++ const actionToken = await this.actionTokens.createActionToken(++ userId,++ ActionTokenType.ADDING_PROJECT_SUBSCRIBER,++ `${projectId}:${subscriberId}`,++ )++ const url = `${origin}/projects/${projectId}/subscribers/invite?token=${actionToken.id}`++ await this.mailerService.sendEmail(++ email,++ LetterTemplate.ProjectSubscriberInvitation,++ {++ url, projectName,++ },++ )++ }++ async getProjectById(projectId: string) {++ return await this.projectsRepository.findOne({++ where: { id: projectId },++ })++ }++ async getSubscriber(projectId: string, subscriberId: string) {++ return await this.projectSubscriberRepository.findOne({++ where: { id: subscriberId, projectId },++ })++ }++ async confirmSubscriber(++ projectId: string,++ subscriberId: string,++ token: string,++ ) {++ await this.projectSubscriberRepository.update(++ { id: subscriberId, projectId },++ { isConfirmed: true },++ )++ await this.actionTokens.deleteActionToken(token)++ }++ async getSubscribers(projectId: string, queries: GetSubscribersQueriesDto) {++ const [subscribers, count] =++ await this.projectSubscriberRepository.findAndCount({++ skip: Number(queries.offset) || 0,++ take: Number(queries.limit) > 100 ? 100 : Number(queries.limit) || 100,++ where: { projectId },++ })++ return { subscribers, count }++ }++ async updateSubscriber(++ projectId: string,++ subscriberId: string,++ data: UpdateSubscriberBodyDto,++ ) {++ await this.projectSubscriberRepository.update(++ { id: subscriberId, projectId },++ data,++ )++ return await this.getSubscriber(projectId, subscriberId)++ }++ async removeSubscriber(projectId: string, subscriberId: string) {++ await this.projectSubscriberRepository.delete({++ id: subscriberId,++ projectId,++ })++ }++ async getSubscribersForReports(reportFrequency: ReportFrequency) {++ return await this.projectSubscriberRepository.find({++ relations: ['project'],++ where: { reportFrequency, isConfirmed: true },++ })++ }++ async getSubscriberProjects(subscriberId: string) {++ const projects = await this.projectSubscriberRepository.find({++ relations: ['project'],++ where: { id: subscriberId },++ })++ return projects.map(project => project.project)++ }+}