Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

Merge branch 'main' into feature/captcha

Andrii R1debb522b636
diff --cc package.json
index 41d5d2236,c6f207765..f0ff7f2a1
--- a/package.json
+++ b/package.json
@@@ -44,9 -44,8 +44,9 @@@
"class-validator": "^0.14.0",
"clickhouse": "^2.6.0",
"cookie-parser": "^1.4.6",
- "countries-and-timezones": "^3.4.0",
+ "countries-and-timezones": "^3.4.1",
"cross-env": "^7.0.3",
+ "crypto-js": "^4.1.1",
"dayjs": "^1.11.7",
"dotenv": "^16.0.3",
"express": "^4.18.2",
@@@ -72,24 -71,23 +72,24 @@@
"reflect-metadata": "^0.1.13",
"rimraf": "^3.0.2",
"rxjs": "^7.8.0",
- "telegraf": "^4.12.2",
+ "svg-captcha": "^1.4.0",
+ "telegraf": "^4.11.2",
"typeorm": "^0.2.45",
- "ua-parser-js": "^1.0.32",
+ "ua-parser-js": "^1.0.34",
"uuid": "^9.0.0",
"validate-ip-node": "^1.0.8",
- "validator": "^13.7.0"
+ "validator": "^13.9.0"
},
"devDependencies": {
- "@nestjs/cli": "^9.1.5",
- "@nestjs/testing": "^9.2.1",
+ "@nestjs/cli": "^9.3.0",
+ "@nestjs/testing": "^9.3.10",
"@types/bcrypt": "^5.0.0",
- "@types/express": "^4.17.15",
+ "@types/express": "^4.17.17",
"@types/ioredis": "^4.28.10",
- "@types/jest": "^29.2.4",
+ "@types/jest": "^29.5.0",
"@types/multer": "^1.4.7",
"@types/newrelic": "^9.4.0",
- "@types/node": "^18.11.18",
+ "@types/node": "^18.15.3",
"@types/passport-jwt": "^3.0.8",
"@types/supertest": "^2.0.12",
"@types/ua-parser-js": "^0.7.36",
diff --cc src/app.module.ts
index ea7123bcb,26e0c1d25..53738e1d6
--- a/src/app.module.ts
+++ b/src/app.module.ts
@@@ -20,7 -20,7 +20,7 @@@ import { AlertModule } from './alert/al
import { I18nModule } from 'nestjs-i18n'
import { getI18nConfig } from './configs'
import { AuthModule } from './auth/auth.module'
- import { CaptchaModule } from './captcha/captcha.module'
-import { isDevelopment, isTgTokenPresent } from './common/constants'
++import { CaptchaModule, isDevelopment, isTgTokenPresent } from './captcha/captcha.module'
const modules = [
ConfigModule.forRoot({ envFilePath: '.env', isGlobal: true }),
@@@ -50,10 -50,9 +50,10 @@@
MarketplaceModule,
AlertModule,
AuthModule,
+ CaptchaModule,
]
- if (process.env.TG_BOT_TOKEN) {
+ if (isTgTokenPresent) {
modules.push(
TelegrafModule.forRoot({
token: process.env.TG_BOT_TOKEN,
diff --cc src/common/constants.ts
index 98ace0008,190f73ee2..c3bac4656
--- a/src/common/constants.ts
+++ b/src/common/constants.ts
@@@ -40,8 -39,10 +40,11 @@@ const clickhouse = new ClickHouse(
})
const isSelfhosted = Boolean(process.env.SELFHOSTED)
+ const isTgTokenPresent = Boolean(process.env.TG_BOT_TOKEN)
+ const isDevelopment = process.env.NODE_ENV === 'development'
const isNewRelicEnabled = Boolean(process.env.USE_NEW_RELIC)
+const isDevelopment = process.env.NODE_ENV === 'development'
+ const PRODUCTION_ORIGIN = process.env.CLIENT_URL || 'https://swetrix.com'
const CLICKHOUSE_INIT_QUERIES = [
'CREATE DATABASE IF NOT EXISTS analytics',
@@@ -239,12 -229,7 +242,14 @@@ export
ORIGINS_REGEX,
REDIS_LOG_PERF_CACHE_KEY,
REDIS_PERFORMANCE_COUNT_KEY,
+ CAPTCHA_SALT,
+ CAPTCHA_ENCRYPTION_KEY,
isDevelopment,
+ getRedisCaptchaKey,
+ CAPTCHA_COOKIE_KEY,
+ CAPTCHA_TOKEN_LIFETIME,
+ PID_REGEX,
+ CAPTCHA_SECRET_KEY_LENGTH,
+ PRODUCTION_ORIGIN,
+ isTgTokenPresent,
}
diff --cc src/project/entity/project.entity.ts
index 5dc710173,d234c1ab3..ca867cb39
--- a/src/project/entity/project.entity.ts
+++ b/src/project/entity/project.entity.ts
@@@ -1,91 -1,78 +1,96 @@@
-import {
- Entity,
- Column,
- PrimaryColumn,
- ManyToOne,
- OneToMany,
- JoinTable,
-} from 'typeorm'
-import { ApiProperty } from '@nestjs/swagger'
-
-import { User } from '../../user/entities/user.entity'
-import { ProjectShare } from './project-share.entity'
-import { Alert } from 'src/alert/entity/alert.entity'
-import { ExtensionToProject } from '../../marketplace/extensions/entities/extension-to-project.entity'
-import { ProjectSubscriber } from './project-subscriber.entity'
-
-// In case of modifying some properties here, make sure to also edit them in common/constants.ts -> selfhosted -> clickhouse
-// and to add them to the GDPR data export email template
-@Entity()
-export class Project {
- @ApiProperty()
- @PrimaryColumn('varchar', {
- unique: true,
- length: 12,
- })
- id: string
-
- @ApiProperty()
- @Column('varchar', { length: 50 })
- name: string
-
- @ApiProperty()
- @Column('simple-array')
- origins: string[]
-
- @ApiProperty()
- @Column('simple-array', { nullable: true, default: null })
- ipBlacklist: string[]
-
- @ApiProperty()
- @Column({
- default: true,
- })
- active: boolean
-
- @ApiProperty()
- @Column({
- default: false,
- })
- public: boolean
-
- @ApiProperty({ type: () => User })
- @ManyToOne(() => User, user => user.projects)
- admin: User
-
- @ApiProperty({ type: () => ProjectShare })
- @OneToMany(() => ProjectShare, share => share.project)
- share: ProjectShare[]
-
- @ApiProperty({ type: () => Alert })
- @OneToMany(() => Alert, alert => alert.project)
- alerts: Alert[]
-
- @Column({ type: 'timestamp', default: () => 'CURRENT_TIMESTAMP' })
- created: Date
-
- @OneToMany(
- () => ExtensionToProject,
- extensionToProject => extensionToProject.project,
- )
- @JoinTable()
- extensions: ExtensionToProject[]
-
- @OneToMany(() => ProjectSubscriber, subscriber => subscriber.project, {
- cascade: true,
- })
- subscribers: ProjectSubscriber[]
-}
+import {
+ Entity,
+ Column,
+ PrimaryColumn,
+ ManyToOne,
+ OneToMany,
+ JoinTable,
+} from 'typeorm'
+import { ApiProperty } from '@nestjs/swagger'
+
+import { User } from '../../user/entities/user.entity'
+import { ProjectShare } from './project-share.entity'
+import { Alert } from 'src/alert/entity/alert.entity'
+import { ExtensionToProject } from '../../marketplace/extensions/entities/extension-to-project.entity'
++import { ProjectSubscriber } from './project-subscriber.entity'
+import { CAPTCHA_SECRET_KEY_LENGTH } from '../../common/constants'
+
+// In case of modifying some properties here, make sure to also edit them in common/constants.ts -> selfhosted -> clickhouse
+// and to add them to the GDPR data export email template
+@Entity()
+export class Project {
+ @ApiProperty()
+ @PrimaryColumn('varchar', {
+ unique: true,
+ length: 12,
+ })
+ id: string
+
+ @ApiProperty()
+ @Column('varchar', { length: 50 })
+ name: string
+
+ @ApiProperty()
+ @Column('simple-array')
+ origins: string[]
+
+ @ApiProperty()
+ @Column('simple-array', { nullable: true, default: null })
+ ipBlacklist: string[]
+
+ @ApiProperty()
+ @Column({
+ default: true,
+ })
+ active: boolean
+
+ @ApiProperty()
+ @Column({
+ default: false,
+ })
+ public: boolean
-
++
+ // Swetrix CAPTCHA related stuff
-
+ @ApiProperty()
+ @Column('boolean', { default: true })
+ isAnalyticsProject: boolean
+
+ @ApiProperty()
+ @Column('boolean', { default: false })
+ isCaptchaProject: boolean
+
+ @ApiProperty()
+ @Column('boolean', { default: false })
+ isCaptchaEnabled: boolean
+
+ @ApiProperty()
+ @Column('varchar', { default: null, length: CAPTCHA_SECRET_KEY_LENGTH })
+ captchaSecretKey: string
+
+ @ApiProperty({ type: () => User })
+ @ManyToOne(() => User, user => user.projects)
+ admin: User
+
+ @ApiProperty({ type: () => ProjectShare })
+ @OneToMany(() => ProjectShare, share => share.project)
+ share: ProjectShare[]
+
+ @ApiProperty({ type: () => Alert })
+ @OneToMany(() => Alert, alert => alert.project)
+ alerts: Alert[]
+
+ @Column({ type: 'timestamp', default: () => 'CURRENT_TIMESTAMP' })
+ created: Date
+
+ @OneToMany(
+ () => ExtensionToProject,
+ extensionToProject => extensionToProject.project,
+ )
+ @JoinTable()
+ extensions: ExtensionToProject[]
++
++ @OneToMany(() => ProjectSubscriber, subscriber => subscriber.project, {
++ cascade: true,
++ })
++ subscribers: ProjectSubscriber[]
+}
diff --cc src/project/project.controller.ts
index 914e8c7c7,3da2a5789..801fc756f
--- a/src/project/project.controller.ts
+++ b/src/project/project.controller.ts
@@@ -47,7 -48,8 +49,9 @@@ import
clickhouse,
isSelfhosted,
PROJECT_INVITE_EXPIRE,
+ CAPTCHA_SECRET_KEY_LENGTH,
+ isDevelopment,
+ PRODUCTION_ORIGIN,
} from '../common/constants'
import {
getProjectsClickhouse,
diff --cc src/project/project.service.ts
index 3f5628c7b,96fa6c091..7ea121cd6
--- a/src/project/project.service.ts
+++ b/src/project/project.service.ts
@@@ -1,481 -1,560 +1,596 @@@
-import {
- ForbiddenException,
- Injectable,
- BadRequestException,
- UnprocessableEntityException,
- InternalServerErrorException,
- ConflictException,
-} from '@nestjs/common'
-import { InjectRepository } from '@nestjs/typeorm'
-import { Repository } from 'typeorm'
-import * as _isEmpty from 'lodash/isEmpty'
-import * as _isString from 'lodash/isString'
-import * as _size from 'lodash/size'
-import * as _join from 'lodash/join'
-import * as _find from 'lodash/find'
-import * as _map from 'lodash/map'
-import * as _pick from 'lodash/pick'
-import * as _trim from 'lodash/trim'
-import * as _findIndex from 'lodash/findIndex'
-import * as _includes from 'lodash/includes'
-import * as dayjs from 'dayjs'
-import * as utc from 'dayjs/plugin/utc'
-// @ts-ignore
-import * as validateIP from 'validate-ip-node'
-
-import { Pagination, PaginationOptionsInterface } from '../common/pagination'
-import { Project } from './entity/project.entity'
-import { ProjectShare } from './entity/project-share.entity'
-import { ProjectDTO } from './dto/project.dto'
-import { UserType } from '../user/entities/user.entity'
-import { UserService } from 'src/user/user.service'
-import { Role } from '../project/entity/project-share.entity'
-import {
- isValidPID,
- redisProjectCountCacheTimeout,
- getRedisUserCountKey,
- redis,
- clickhouse,
- isSelfhosted,
- IP_REGEX,
- ORIGINS_REGEX,
- getRedisProjectKey,
-} from '../common/constants'
-import { ProjectSubscriber } from './entity'
-import { ActionTokensService } from 'src/action-tokens/action-tokens.service'
-import { ActionTokenType } from 'src/action-tokens/action-token.entity'
-import { MailerService } from 'src/mailer/mailer.service'
-import { LetterTemplate } from 'src/mailer/letter'
-import { AddSubscriberType } from './types'
-import { GetSubscribersQueriesDto, UpdateSubscriberBodyDto } from './dto'
-import { ReportFrequency } from './enums'
-
-dayjs.extend(utc)
-
-// const updateProjectRedis = async (id: string, project: Project) => {
-// const key = getRedisProjectKey(id)
-
-// try {
-// await redis.set(
-// key,
-// JSON.stringify(project),
-// 'EX',
-// redisProjectCacheTimeout,
-// )
-// } catch {
-// await redis.del(key)
-// }
-// }
-
-export const deleteProjectRedis = async (id: string) => {
- const key = getRedisProjectKey(id)
-
- try {
- await redis.del(key)
- } catch (e) {
- console.error(`Error deleting project ${id} from redis: ${e}`)
- }
-}
-
-export const deleteProjectsRedis = async (ids: string[]) => {
- await Promise.all(_map(ids, deleteProjectRedis))
-}
-
-export const processProjectUser = (project: Project): Project => {
- const { share } = project
-
- for (let j = 0; j < _size(share); ++j) {
- const { user } = share[j]
-
- if (user) {
- share[j].user = _pick(user, ['email'])
- }
- }
-
- return project
-}
-
-export const processProjectsUser = (projects: Project[]): Project[] => {
- for (let i = 0; i < _size(projects); ++i) {
- projects[i] = processProjectUser(projects[i])
- }
-
- return projects
-}
-
-@Injectable()
-export class ProjectService {
- constructor(
- @InjectRepository(Project)
- private projectsRepository: Repository<Project>,
- @InjectRepository(ProjectShare)
- private projectShareRepository: Repository<ProjectShare>,
- private userService: UserService,
- @InjectRepository(ProjectSubscriber)
- private readonly projectSubscriberRepository: Repository<ProjectSubscriber>,
- private readonly actionTokens: ActionTokensService,
- private readonly mailerService: MailerService,
- ) {}
-
- async paginate(
- options: PaginationOptionsInterface,
- where: Record<string, unknown> | undefined,
- ): Promise<Pagination<Project>> {
- const [results, total] = await this.projectsRepository.findAndCount({
- take: options.take || 100,
- skip: options.skip || 0,
- where,
- order: {
- name: 'ASC',
- },
- relations: ['share', 'share.user'],
- })
-
- return new Pagination<Project>({
- results: processProjectsUser(results),
- total,
- })
- }
-
- async paginateShared(
- options: PaginationOptionsInterface,
- where: Record<string, unknown> | undefined,
- ): Promise<Pagination<ProjectShare>> {
- const [results, total] = await this.projectShareRepository.findAndCount({
- take: options.take || 100,
- skip: options.skip || 0,
- where,
- order: {
- project: 'ASC',
- },
- relations: ['project'],
- })
-
- return new Pagination<ProjectShare>({
- // results: processProjectsUser(results),
- results,
- total,
- })
- }
-
- async count(): Promise<number> {
- return await this.projectsRepository.count()
- }
-
- async create(project: ProjectDTO | Project): Promise<Project> {
- return this.projectsRepository.save(project)
- }
-
- async update(
- id: string,
- projectDTO: ProjectDTO,
- ): Promise<any> {
- return this.projectsRepository.update(id, projectDTO)
- }
-
- async delete(id: string): Promise<any> {
- return this.projectsRepository.delete(id)
- }
-
- async deleteMultiple(pids: string[]): Promise<any> {
- return (
- this.projectsRepository
- .createQueryBuilder()
- .delete()
- // TODO: !!! Enforce Prepared Statements and Parameterization
- .where(`id IN (${pids})`)
- .execute()
- )
- }
-
- async deleteMultipleShare(where: string): Promise<any> {
- return this.projectShareRepository
- .createQueryBuilder()
- .delete()
- .where(where)
- .execute()
- }
-
- async createShare(share: ProjectShare): Promise<ProjectShare> {
- return this.projectShareRepository.save(share)
- }
-
- async deleteShare(id: string): Promise<any> {
- return this.projectShareRepository.delete(id)
- }
-
- async updateShare(id: string, share: ProjectShare | Object): Promise<any> {
- return this.projectShareRepository.update(id, share)
- }
-
- async findShare(params: object): Promise<ProjectShare[]> {
- return this.projectShareRepository.find(params)
- }
-
- async findOneShare(
- id: string,
- params: Object = {},
- ): Promise<ProjectShare | null> {
- return this.projectShareRepository.findOne(id, params)
- }
-
- findOneWithRelations(id: string): Promise<Project | null> {
- return this.projectsRepository.findOne(id, { relations: ['admin'] })
- }
-
- findOne(id: string, params: Object = {}): Promise<Project | null> {
- return this.projectsRepository.findOne(id, params)
- }
-
- findWhere(
- where: Record<string, unknown>,
- relations?: string[],
- ): Promise<Project[]> {
- return this.projectsRepository.find({ where, relations })
- }
-
- find(params: object): Promise<Project[]> {
- return this.projectsRepository.find(params)
- }
-
- findOneWhere(
- where: Record<string, unknown>,
- params: object = {},
- ): Promise<Project> {
- return this.projectsRepository.findOne({ where, ...params })
- }
-
- allowedToView(project: Project, uid: string | null): void {
- if (
- project.public ||
- uid === project.admin?.id ||
- _findIndex(project.share, ({ user }) => user?.id === uid) !== -1
- ) {
- return
- } else {
- throw new ForbiddenException('You are not allowed to view this project')
- }
- }
-
- allowedToManage(
- project: Project,
- uid: string,
- roles: Array<UserType> = [],
- message: string = 'You are not allowed to manage this project'
- ): void {
- if (
- uid === project.admin?.id ||
- _includes(roles, UserType.ADMIN) ||
- _findIndex(
- project.share,
- share => share.user?.id === uid && share.role === Role.admin,
- ) !== -1
- ) {
- return
- } else {
- throw new ForbiddenException(message)
- }
- }
-
- async checkIfIDUnique(projectID: string): Promise<void> {
- const project = await this.findOne(projectID)
-
- if (project) {
- throw new BadRequestException('Selected project ID is already in use')
- }
- }
-
- checkIfIDUniqueClickhouse(projects: Array<object>, projectID: string): void {
- if (_find(projects, ({ id }) => id === projectID)) {
- throw new BadRequestException('Selected project ID is already in use')
- }
- }
-
- formatToClickhouse(project: Project): object {
- const updProject = { ...project }
- // @ts-ignore
- updProject.active = Number(updProject.active)
- // @ts-ignore
- updProject.public = Number(updProject.public)
- // @ts-ignore
- updProject.origins = _isString(updProject.origins)
- ? updProject.origins
- : _join(updProject.origins, ',')
-
- return updProject
- }
-
- formatFromClickhouse(project: object): object {
- const updProject = { ...project }
- // @ts-ignore
- updProject.active = Boolean(updProject.active)
- // @ts-ignore
- updProject.public = Boolean(updProject.public)
-
- return updProject
- }
-
- validateProject(projectDTO: ProjectDTO) {
- if (!isValidPID(projectDTO.id))
- throw new UnprocessableEntityException(
- 'The provided Project ID (pid) is incorrect',
- )
- if (_size(projectDTO.name) > 50)
- throw new UnprocessableEntityException('The project name is too long')
- if (_size(_join(projectDTO.origins, ',')) > 300)
- throw new UnprocessableEntityException(
- 'The list of allowed origins has to be smaller than 300 symbols',
- )
- if (_size(_join(projectDTO.ipBlacklist, ',')) > 300)
- throw new UnprocessableEntityException(
- 'The list of allowed blacklisted IP addresses must be less than 300 characters.',
- )
-
- _map(projectDTO.origins, host => {
- if (!ORIGINS_REGEX.test(_trim(host))) {
- throw new ConflictException(`Host ${host} is not correct`)
- }
- })
-
- _map(projectDTO.ipBlacklist, ip => {
- if (!validateIP(_trim(ip)) && !IP_REGEX.test(_trim(ip))) {
- throw new ConflictException(`IP address ${ip} is not correct`)
- }
- })
- }
-
- // Returns amount of exirsting events starting from month
- async getRedisCount(uid: string): Promise<number | null> {
- const countKey = getRedisUserCountKey(uid)
- let count: string | number = await redis.get(countKey)
-
- if (_isEmpty(count)) {
- const monthStart = dayjs
- .utc()
- .startOf('month')
- .format('YYYY-MM-DD HH:mm:ss')
- const monthEnd = dayjs.utc().endOf('month').format('YYYY-MM-DD HH:mm:ss')
-
- let pids
-
- if (isSelfhosted) {
- // selfhosted has no limits
- return 0
- } else {
- pids = await this.find({
- where: {
- admin: uid,
- },
- select: ['id'],
- })
- }
-
- if (_isEmpty(pids)) {
- return 0
- }
-
- const count_ev_query = `SELECT COUNT() FROM analytics WHERE pid IN (${_join(
- _map(pids, el => `'${el.id}'`),
- ',',
- )}) AND created BETWEEN '${monthStart}' AND '${monthEnd}'`
- const count_custom_ev_query = `SELECT COUNT() FROM customEV WHERE pid IN (${_join(
- _map(pids, el => `'${el.id}'`),
- ',',
- )}) AND created BETWEEN '${monthStart}' AND '${monthEnd}'`
-
- const pageviews = (await clickhouse.query(count_ev_query).toPromise())[0][
- 'count()'
- ]
- const customEvents = (
- await clickhouse.query(count_custom_ev_query).toPromise()
- )[0]['count()']
-
- count = pageviews + customEvents
-
- await redis.set(
- countKey,
- `${pageviews}`,
- 'EX',
- redisProjectCountCacheTimeout,
- )
- } else {
- try {
- // @ts-ignore
- count = Number(count)
- } catch (e) {
- count = 0
- console.error(e)
- throw new InternalServerErrorException('Error while processing project')
- }
- }
-
- // @ts-ignore
- return count
- }
-
- async clearProjectsRedisCache(uid: string): Promise<void> {
- const projects = await this.findWhere({
- admin: uid,
- })
-
- if (_isEmpty(projects)) {
- return
- }
-
- const pids = _map(projects, 'id')
-
- await deleteProjectsRedis(pids)
- }
-
- async clearProjectsRedisCacheByEmail(email: string): Promise<void> {
- const user = await this.userService.findOneWhere({ email })
-
- if (!user) {
- return
- }
-
- await this.clearProjectsRedisCache(user.id)
- }
-
- async getProject(projectId: string, userId: string) {
- return await this.projectsRepository.findOne({
- where: {
- id: projectId,
- admin: { id: userId },
- },
- })
- }
-
- async getSubscriberByEmail(projectId: string, email: string) {
- return await this.projectSubscriberRepository.findOne({
- where: { projectId, email },
- })
- }
-
- async addSubscriber(data: AddSubscriberType) {
- const subscriber = await this.projectSubscriberRepository.save({ ...data })
-
- await this.sendSubscriberInvite(
- data, subscriber.id,
- )
-
- return subscriber
- }
-
- async sendSubscriberInvite(
- data: AddSubscriberType,
- subscriberId: string,
- ) {
- const {
- userId, projectId, projectName, email, origin
- } = data
-
- const actionToken = await this.actionTokens.createActionToken(
- userId,
- ActionTokenType.ADDING_PROJECT_SUBSCRIBER,
- `${projectId}:${subscriberId}`,
- )
-
- const url = `${origin}/projects/${projectId}/subscribers/invite?token=${actionToken.id}`
-
- await this.mailerService.sendEmail(
- email,
- LetterTemplate.ProjectSubscriberInvitation,
- {
- url, projectName,
- },
- )
- }
-
- async getProjectById(projectId: string) {
- return await this.projectsRepository.findOne({
- where: { id: projectId },
- })
- }
-
- async getSubscriber(projectId: string, subscriberId: string) {
- return await this.projectSubscriberRepository.findOne({
- where: { id: subscriberId, projectId },
- })
- }
-
- async confirmSubscriber(
- projectId: string,
- subscriberId: string,
- token: string,
- ) {
- await this.projectSubscriberRepository.update(
- { id: subscriberId, projectId },
- { isConfirmed: true },
- )
- await this.actionTokens.deleteActionToken(token)
- }
-
- async getSubscribers(projectId: string, queries: GetSubscribersQueriesDto) {
- const [subscribers, count] =
- await this.projectSubscriberRepository.findAndCount({
- skip: Number(queries.offset) || 0,
- take: Number(queries.limit) > 100 ? 100 : Number(queries.limit) || 100,
- where: { projectId },
- })
-
- return { subscribers, count }
- }
-
- async updateSubscriber(
- projectId: string,
- subscriberId: string,
- data: UpdateSubscriberBodyDto,
- ) {
- await this.projectSubscriberRepository.update(
- { id: subscriberId, projectId },
- data,
- )
-
- return await this.getSubscriber(projectId, subscriberId)
- }
-
- async removeSubscriber(projectId: string, subscriberId: string) {
- await this.projectSubscriberRepository.delete({
- id: subscriberId,
- projectId,
- })
- }
-
- async getSubscribersForReports(reportFrequency: ReportFrequency) {
- return await this.projectSubscriberRepository.find({
- relations: ['project'],
- where: { reportFrequency, isConfirmed: true },
- })
- }
-
- async getSubscriberProjects(subscriberId: string) {
- const projects = await this.projectSubscriberRepository.find({
- relations: ['project'],
- where: { id: subscriberId },
- })
-
- return projects.map(project => project.project)
- }
-}
+import {
+ ForbiddenException,
+ Injectable,
+ BadRequestException,
+ UnprocessableEntityException,
+ InternalServerErrorException,
+ ConflictException,
+} from '@nestjs/common'
+import { InjectRepository } from '@nestjs/typeorm'
+import { Repository } from 'typeorm'
+import * as _isEmpty from 'lodash/isEmpty'
+import * as _isString from 'lodash/isString'
+import * as _size from 'lodash/size'
+import * as _join from 'lodash/join'
+import * as _find from 'lodash/find'
+import * as _map from 'lodash/map'
+import * as _pick from 'lodash/pick'
+import * as _trim from 'lodash/trim'
+import * as _findIndex from 'lodash/findIndex'
+import * as _includes from 'lodash/includes'
+import * as dayjs from 'dayjs'
+import * as utc from 'dayjs/plugin/utc'
+// @ts-ignore
+import * as validateIP from 'validate-ip-node'
+
+import { Pagination, PaginationOptionsInterface } from '../common/pagination'
+import { Project } from './entity/project.entity'
+import { ProjectShare } from './entity/project-share.entity'
+import { ProjectDTO } from './dto/project.dto'
+import { UserType } from '../user/entities/user.entity'
+import { UserService } from 'src/user/user.service'
+import { Role } from '../project/entity/project-share.entity'
+import {
+ isValidPID,
+ redisProjectCountCacheTimeout,
+ getRedisUserCountKey,
+ redis,
+ clickhouse,
+ isSelfhosted,
+ IP_REGEX,
+ ORIGINS_REGEX,
+ getRedisProjectKey,
+ redisProjectCacheTimeout,
+} from '../common/constants'
+import {
+ getProjectsClickhouse,
+} from '../common/utils'
++import { ProjectSubscriber } from './entity'
++import { ActionTokensService } from 'src/action-tokens/action-tokens.service'
++import { ActionTokenType } from 'src/action-tokens/action-token.entity'
++import { MailerService } from 'src/mailer/mailer.service'
++import { LetterTemplate } from 'src/mailer/letter'
++import { AddSubscriberType } from './types'
++import { GetSubscribersQueriesDto, UpdateSubscriberBodyDto } from './dto'
++import { ReportFrequency } from './enums'
+
+dayjs.extend(utc)
+
+// const updateProjectRedis = async (id: string, project: Project) => {
+// const key = getRedisProjectKey(id)
+
+// try {
+// await redis.set(
+// key,
+// JSON.stringify(project),
+// 'EX',
+// redisProjectCacheTimeout,
+// )
+// } catch {
+// await redis.del(key)
+// }
+// }
+
+export const deleteProjectRedis = async (id: string) => {
+ const key = getRedisProjectKey(id)
+
+ try {
+ await redis.del(key)
+ } catch (e) {
+ console.error(`Error deleting project ${id} from redis: ${e}`)
+ }
+}
+
+export const deleteProjectsRedis = async (ids: string[]) => {
+ await Promise.all(_map(ids, deleteProjectRedis))
+}
+
+export const processProjectUser = (project: Project): Project => {
+ const { share } = project
+
+ for (let j = 0; j < _size(share); ++j) {
+ const { user } = share[j]
+
+ if (user) {
+ share[j].user = _pick(user, ['email'])
+ }
+ }
+
+ return project
+}
+
+export const processProjectsUser = (projects: Project[]): Project[] => {
+ for (let i = 0; i < _size(projects); ++i) {
+ projects[i] = processProjectUser(projects[i])
+ }
+
+ return projects
+}
+
+@Injectable()
+export class ProjectService {
+ constructor(
+ @InjectRepository(Project)
+ private projectsRepository: Repository<Project>,
+ @InjectRepository(ProjectShare)
+ private projectShareRepository: Repository<ProjectShare>,
+ private userService: UserService,
++ @InjectRepository(ProjectSubscriber)
++ private readonly projectSubscriberRepository: Repository<ProjectSubscriber>,
++ private readonly actionTokens: ActionTokensService,
++ private readonly mailerService: MailerService,
++ ) {}
+ ) { }
+
+ async getRedisProject(pid: string): Promise<Project | null> {
+ const pidKey = getRedisProjectKey(pid)
+ let project: string | Project = await redis.get(pidKey)
+
+ if (_isEmpty(project)) {
+ if (isSelfhosted) {
+ project = await getProjectsClickhouse(pid)
+ } else {
+ // todo: optimise the relations - select
+ // select only required columns
+ // https://stackoverflow.com/questions/59645009/how-to-return-only-some-columns-of-a-relations-with-typeorm
+ project = await this.findOne(pid, {
+ relations: ['admin'],
+ select: ['origins', 'active', 'admin', 'public', 'ipBlacklist', 'captchaSecretKey', 'isCaptchaEnabled'],
+ })
+ }
+ if (_isEmpty(project))
+ throw new BadRequestException(
+ 'The provided Project ID (pid) is incorrect',
+ )
+
+ if (!isSelfhosted) {
+ const share = await this.findShare({
+ where: {
+ project: pid,
+ },
+ relations: ['user'],
+ })
+ // @ts-ignore
+ project = { ...project, share }
+ }
+
+ await redis.set(
+ pidKey,
+ JSON.stringify(project),
+ 'EX',
+ redisProjectCacheTimeout,
+ )
+ } else {
+ try {
+ project = JSON.parse(project)
+ } catch {
+ throw new InternalServerErrorException('Error while processing project')
+ }
+ }
+
+ // @ts-ignore
+ return project
+ }
+
+ async paginate(
+ options: PaginationOptionsInterface,
+ where: Record<string, unknown> | undefined,
+ ): Promise<Pagination<Project>> {
+ const [results, total] = await this.projectsRepository.findAndCount({
+ take: options.take || 100,
+ skip: options.skip || 0,
+ where,
+ order: {
+ name: 'ASC',
+ },
+ relations: ['share', 'share.user'],
+ })
+
+ return new Pagination<Project>({
+ results: processProjectsUser(results),
+ total,
+ })
+ }
+
+ async paginateShared(
+ options: PaginationOptionsInterface,
+ where: Record<string, unknown> | undefined,
+ ): Promise<Pagination<ProjectShare>> {
+ const [results, total] = await this.projectShareRepository.findAndCount({
+ take: options.take || 100,
+ skip: options.skip || 0,
+ where,
+ order: {
+ project: 'ASC',
+ },
+ relations: ['project'],
+ })
+
+ return new Pagination<ProjectShare>({
+ // results: processProjectsUser(results),
+ results,
+ total,
+ })
+ }
+
+ async count(): Promise<number> {
+ return await this.projectsRepository.count()
+ }
+
+ async create(project: ProjectDTO | Project): Promise<Project> {
+ return this.projectsRepository.save(project)
+ }
+
+ async update(
+ id: string,
+ projectDTO: ProjectDTO | Project,
+ ): Promise<any> {
+ return this.projectsRepository.update(id, projectDTO)
+ }
+
+ async delete(id: string): Promise<any> {
+ return this.projectsRepository.delete(id)
+ }
+
+ async deleteMultiple(pids: string[]): Promise<any> {
+ return (
+ this.projectsRepository
+ .createQueryBuilder()
+ .delete()
+ // TODO: !!! Enforce Prepared Statements and Parameterization
+ .where(`id IN (${pids})`)
+ .execute()
+ )
+ }
+
+ async deleteMultipleShare(where: string): Promise<any> {
+ return this.projectShareRepository
+ .createQueryBuilder()
+ .delete()
+ .where(where)
+ .execute()
+ }
+
+ async createShare(share: ProjectShare): Promise<ProjectShare> {
+ return this.projectShareRepository.save(share)
+ }
+
+ async deleteShare(id: string): Promise<any> {
+ return this.projectShareRepository.delete(id)
+ }
+
+ async updateShare(id: string, share: ProjectShare | Object): Promise<any> {
+ return this.projectShareRepository.update(id, share)
+ }
+
+ async findShare(params: object): Promise<ProjectShare[]> {
+ return this.projectShareRepository.find(params)
+ }
+
+ async findOneShare(
+ id: string,
+ params: Object = {},
+ ): Promise<ProjectShare | null> {
+ return this.projectShareRepository.findOne(id, params)
+ }
+
+ findOneWithRelations(id: string): Promise<Project | null> {
+ return this.projectsRepository.findOne(id, { relations: ['admin'] })
+ }
+
+ findOne(id: string, params: Object = {}): Promise<Project | null> {
+ return this.projectsRepository.findOne(id, params)
+ }
+
+ findWhere(
+ where: Record<string, unknown>,
+ relations?: string[],
+ ): Promise<Project[]> {
+ return this.projectsRepository.find({ where, relations })
+ }
+
+ find(params: object): Promise<Project[]> {
+ return this.projectsRepository.find(params)
+ }
+
+ findOneWhere(
+ where: Record<string, unknown>,
+ params: object = {},
+ ): Promise<Project> {
+ return this.projectsRepository.findOne({ where, ...params })
+ }
+
+ allowedToView(project: Project, uid: string | null): void {
+ if (
+ project.public ||
+ uid === project.admin?.id ||
+ _findIndex(project.share, ({ user }) => user?.id === uid) !== -1
+ ) {
+ return
+ } else {
+ throw new ForbiddenException('You are not allowed to view this project')
+ }
+ }
+
+ allowedToManage(
+ project: Project,
+ uid: string,
+ roles: Array<UserType> = [],
+ message: string = 'You are not allowed to manage this project'
+ ): void {
+ if (
+ uid === project.admin?.id ||
+ _includes(roles, UserType.ADMIN) ||
+ _findIndex(
+ project.share,
+ share => share.user?.id === uid && share.role === Role.admin,
+ ) !== -1
+ ) {
+ return
+ } else {
+ throw new ForbiddenException(message)
+ }
+ }
+
+ async checkIfIDUnique(projectID: string): Promise<void> {
+ const project = await this.findOne(projectID)
+
+ if (project) {
+ throw new BadRequestException('Selected project ID is already in use')
+ }
+ }
+
+ checkIfIDUniqueClickhouse(projects: Array<object>, projectID: string): void {
+ if (_find(projects, ({ id }) => id === projectID)) {
+ throw new BadRequestException('Selected project ID is already in use')
+ }
+ }
+
+ formatToClickhouse(project: Project): object {
+ const updProject = { ...project }
+ // @ts-ignore
+ updProject.active = Number(updProject.active)
+ // @ts-ignore
+ updProject.public = Number(updProject.public)
+ // @ts-ignore
+ updProject.origins = _isString(updProject.origins)
+ ? updProject.origins
+ : _join(updProject.origins, ',')
+
+ return updProject
+ }
+
+ formatFromClickhouse(project: object): object {
+ const updProject = { ...project }
+ // @ts-ignore
+ updProject.active = Boolean(updProject.active)
+ // @ts-ignore
+ updProject.public = Boolean(updProject.public)
+
+ return updProject
+ }
+
+ validateProject(projectDTO: ProjectDTO) {
+ if (!isValidPID(projectDTO.id))
+ throw new UnprocessableEntityException(
+ 'The provided Project ID (pid) is incorrect',
+ )
+ if (_size(projectDTO.name) > 50)
+ throw new UnprocessableEntityException('The project name is too long')
+ if (_size(_join(projectDTO.origins, ',')) > 300)
+ throw new UnprocessableEntityException(
+ 'The list of allowed origins has to be smaller than 300 symbols',
+ )
+ if (_size(_join(projectDTO.ipBlacklist, ',')) > 300)
+ throw new UnprocessableEntityException(
+ 'The list of allowed blacklisted IP addresses must be less than 300 characters.',
+ )
+
+ _map(projectDTO.origins, host => {
+ if (!ORIGINS_REGEX.test(_trim(host))) {
+ throw new ConflictException(`Host ${host} is not correct`)
+ }
+ })
+
+ _map(projectDTO.ipBlacklist, ip => {
+ if (!validateIP(_trim(ip)) && !IP_REGEX.test(_trim(ip))) {
+ throw new ConflictException(`IP address ${ip} is not correct`)
+ }
+ })
+ }
+
+ // Returns amount of exirsting events starting from month
+ async getRedisCount(uid: string): Promise<number | null> {
+ const countKey = getRedisUserCountKey(uid)
+ let count: string | number = await redis.get(countKey)
+
+ if (_isEmpty(count)) {
+ const monthStart = dayjs
+ .utc()
+ .startOf('month')
+ .format('YYYY-MM-DD HH:mm:ss')
+ const monthEnd = dayjs.utc().endOf('month').format('YYYY-MM-DD HH:mm:ss')
+
+ let pids
+
+ if (isSelfhosted) {
+ // selfhosted has no limits
+ return 0
+ } else {
+ pids = await this.find({
+ where: {
+ admin: uid,
+ },
+ select: ['id'],
+ })
+ }
+
+ if (_isEmpty(pids)) {
+ return 0
+ }
+
+ const count_ev_query = `SELECT COUNT() FROM analytics WHERE pid IN (${_join(
+ _map(pids, el => `'${el.id}'`),
+ ',',
+ )}) AND created BETWEEN '${monthStart}' AND '${monthEnd}'`
+ const count_custom_ev_query = `SELECT COUNT() FROM customEV WHERE pid IN (${_join(
+ _map(pids, el => `'${el.id}'`),
+ ',',
+ )}) AND created BETWEEN '${monthStart}' AND '${monthEnd}'`
+
+ const pageviews = (await clickhouse.query(count_ev_query).toPromise())[0][
+ 'count()'
+ ]
+ const customEvents = (
+ await clickhouse.query(count_custom_ev_query).toPromise()
+ )[0]['count()']
+
+ count = pageviews + customEvents
+
+ await redis.set(
+ countKey,
+ `${pageviews}`,
+ 'EX',
+ redisProjectCountCacheTimeout,
+ )
+ } else {
+ try {
+ // @ts-ignore
+ count = Number(count)
+ } catch (e) {
+ count = 0
+ console.error(e)
+ throw new InternalServerErrorException('Error while processing project')
+ }
+ }
+
+ // @ts-ignore
+ return count
+ }
+
+ async clearProjectsRedisCache(uid: string): Promise<void> {
+ const projects = await this.findWhere({
+ admin: uid,
+ })
+
+ if (_isEmpty(projects)) {
+ return
+ }
+
+ const pids = _map(projects, 'id')
+
+ await deleteProjectsRedis(pids)
+ }
+
+ async clearProjectsRedisCacheByEmail(email: string): Promise<void> {
+ const user = await this.userService.findOneWhere({ email })
+
+ if (!user) {
+ return
+ }
+
+ await this.clearProjectsRedisCache(user.id)
+ }
++
++ async getProject(projectId: string, userId: string) {
++ return await this.projectsRepository.findOne({
++ where: {
++ id: projectId,
++ admin: { id: userId },
++ },
++ })
++ }
++ async getSubscriberByEmail(projectId: string, email: string) {
++ return await this.projectSubscriberRepository.findOne({
++ where: { projectId, email },
++ })
++ }
++ async addSubscriber(data: AddSubscriberType) {
++ const subscriber = await this.projectSubscriberRepository.save({ ...data })
++ await this.sendSubscriberInvite(
++ data, subscriber.id,
++ )
++ return subscriber
++ }
++ async sendSubscriberInvite(
++ data: AddSubscriberType,
++ subscriberId: string,
++ ) {
++ const {
++ userId, projectId, projectName, email, origin
++ } = data
++ const actionToken = await this.actionTokens.createActionToken(
++ userId,
++ ActionTokenType.ADDING_PROJECT_SUBSCRIBER,
++ `${projectId}:${subscriberId}`,
++ )
++ const url = `${origin}/projects/${projectId}/subscribers/invite?token=${actionToken.id}`
++ await this.mailerService.sendEmail(
++ email,
++ LetterTemplate.ProjectSubscriberInvitation,
++ {
++ url, projectName,
++ },
++ )
++ }
++ async getProjectById(projectId: string) {
++ return await this.projectsRepository.findOne({
++ where: { id: projectId },
++ })
++ }
++ async getSubscriber(projectId: string, subscriberId: string) {
++ return await this.projectSubscriberRepository.findOne({
++ where: { id: subscriberId, projectId },
++ })
++ }
++ async confirmSubscriber(
++ projectId: string,
++ subscriberId: string,
++ token: string,
++ ) {
++ await this.projectSubscriberRepository.update(
++ { id: subscriberId, projectId },
++ { isConfirmed: true },
++ )
++ await this.actionTokens.deleteActionToken(token)
++ }
++ async getSubscribers(projectId: string, queries: GetSubscribersQueriesDto) {
++ const [subscribers, count] =
++ await this.projectSubscriberRepository.findAndCount({
++ skip: Number(queries.offset) || 0,
++ take: Number(queries.limit) > 100 ? 100 : Number(queries.limit) || 100,
++ where: { projectId },
++ })
++ return { subscribers, count }
++ }
++ async updateSubscriber(
++ projectId: string,
++ subscriberId: string,
++ data: UpdateSubscriberBodyDto,
++ ) {
++ await this.projectSubscriberRepository.update(
++ { id: subscriberId, projectId },
++ data,
++ )
++ return await this.getSubscriber(projectId, subscriberId)
++ }
++ async removeSubscriber(projectId: string, subscriberId: string) {
++ await this.projectSubscriberRepository.delete({
++ id: subscriberId,
++ projectId,
++ })
++ }
++ async getSubscribersForReports(reportFrequency: ReportFrequency) {
++ return await this.projectSubscriberRepository.find({
++ relations: ['project'],
++ where: { reportFrequency, isConfirmed: true },
++ })
++ }
++ async getSubscriberProjects(subscriberId: string) {
++ const projects = await this.projectSubscriberRepository.find({
++ relations: ['project'],
++ where: { id: subscriberId },
++ })
++ return projects.map(project => project.project)
++ }
+}