π Open source, privacy-first web analytics β a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
[untested] moved alerts into a separate table
Blue Mouseb0f43943a0e7
diff --git a/src/alert/alert.controller.ts b/src/alert/alert.controller.tsnew file mode 100644index 000000000..b1195c9a7--- /dev/null+++ b/src/alert/alert.controller.ts@@ -0,0 +1,16 @@+import {+ Controller,+} from '@nestjs/common'+import { ApiTags } from '@nestjs/swagger'+import { AlertService } from './alert.service'+++@ApiTags('Alert')+@Controller('alert')+export class AlertController {+ constructor(+ private readonly alertService: AlertService,+ ) {}++ // TODO: ADD ENDPOINTS TO CREATE / UPDATE / DELETE ALERTS+}diff --git a/src/alert/alert.module.ts b/src/alert/alert.module.tsnew file mode 100644index 000000000..acc21cf0e--- /dev/null+++ b/src/alert/alert.module.ts@@ -0,0 +1,16 @@+import { Module } from '@nestjs/common'+import { TypeOrmModule } from '@nestjs/typeorm'++import { Alert } from './entity/alert.entity'+import { AlertService } from './alert.service'+import { AlertController } from './alert.controller'++@Module({+ imports: [+ TypeOrmModule.forFeature([Alert]),+ ],+ providers: [AlertService],+ exports: [AlertService],+ controllers: [AlertController],+})+export class AlertModule {}diff --git a/src/alert/alert.service.spec.ts b/src/alert/alert.service.spec.tsnew file mode 100644index 000000000..8d69f795e--- /dev/null+++ b/src/alert/alert.service.spec.ts@@ -0,0 +1,11 @@+import { AlertService } from './alert.service'++describe('AlertService', () => {+ let service: AlertService++ describe('root', () => {+ it('should be defined', () => {+ expect(service).toBeDefined()+ })+ })+})diff --git a/src/alert/alert.service.ts b/src/alert/alert.service.tsnew file mode 100644index 000000000..3b1248d09--- /dev/null+++ b/src/alert/alert.service.ts@@ -0,0 +1,67 @@+import {+ Injectable,+} from '@nestjs/common'+import { InjectRepository } from '@nestjs/typeorm'+import { Repository } from 'typeorm'+import { Alert } from './entity/alert.entity'+import { AlertDTO } from './dto/alert.dto'++@Injectable()+export class AlertService {+ constructor(+ @InjectRepository(Alert)+ private alertsReporsitory: Repository<Alert>,+ ) {}++ findOneWithRelations(id: string): Promise<Alert | null> {+ return this.alertsReporsitory.findOne(id, { relations: ['project'] })+ }++ findOne(id: string, params: Object = {}): Promise<Alert | null> {+ return this.alertsReporsitory.findOne(id, params)+ }++ findWhere(+ where: Record<string, unknown>,+ relations?: string[],+ ): Promise<Alert[]> {+ return this.alertsReporsitory.find({ where, relations })+ }++ find(params: object): Promise<Alert[]> {+ return this.alertsReporsitory.find(params)+ }++ findOneWhere(+ where: Record<string, unknown>,+ params: object = {},+ ): Promise<Alert> {+ return this.alertsReporsitory.findOne({ where, ...params })+ }++ async create(project: AlertDTO | Alert): Promise<Alert> {+ return this.alertsReporsitory.save(project)+ }++ async update(+ id: string,+ alertDTO: AlertDTO | Alert,+ ): Promise<any> {+ return this.alertsReporsitory.update(id, alertDTO)+ }++ async delete(id: string): Promise<any> {+ return this.alertsReporsitory.delete(id)+ }++ async deleteMultiple(ids: string[]): Promise<any> {+ return (+ this.alertsReporsitory+ .createQueryBuilder()+ .delete()+ // TODO: !!! Enforce Prepared Statements and Parameterization+ .where(`id IN (${ids})`)+ .execute()+ )+ }+}diff --git a/src/alert/dto/alert.dto.ts b/src/alert/dto/alert.dto.tsnew file mode 100644index 000000000..cceb6d820--- /dev/null+++ b/src/alert/dto/alert.dto.ts@@ -0,0 +1,40 @@+import { ApiProperty } from '@nestjs/swagger'++export enum QueryMetric {+ PAGE_VIEWS = 'page_views',+ UNIQUE_PAGE_VIEWS = 'unique_page_views',+ ONLINE_USERS = 'online_users',+}++export enum QueryCondition {+ GREATER_THAN = 'greater_than',+ GREATER_EQUAL_THAN = 'greater_equal_than',+ LESS_THAN = 'less_than',+ LESS_EQUAL_THAN = 'less_equal_than',+}++export enum QueryTime {+ LAST_15_MINUTES = 'last_15_minutes',+ LAST_30_MINUTES = 'last_30_minutes',+ LAST_1_HOUR = 'last_1_hour',+ LAST_4_HOURS = 'last_4_hours',+ LAST_24_HOURS = 'last_24_hours',+ LAST_48_HOURS = 'last_48_hours',+}++export class AlertDTO {+ @ApiProperty()+ queryMetric?: QueryMetric | null++ @ApiProperty()+ queryCondition?: QueryCondition | null++ @ApiProperty()+ queryValue?: number | null++ @ApiProperty()+ lastTriggered?: Date | null++ @ApiProperty()+ queryTime?: QueryTime | null+}\ No newline at end of filediff --git a/src/alert/entity/alert.entity.ts b/src/alert/entity/alert.entity.tsnew file mode 100644index 000000000..f690f9266--- /dev/null+++ b/src/alert/entity/alert.entity.ts@@ -0,0 +1,79 @@+import {+ Entity,+ Column,+ ManyToOne,+ PrimaryGeneratedColumn,+} from 'typeorm'+import { ApiProperty } from '@nestjs/swagger'++import { Project } from 'src/project/entity/project.entity'+import {+ QueryCondition,+ QueryMetric,+ QueryTime,+} from 'src/alert/dto/alert.dto'++@Entity()+export class Alert {+ @PrimaryGeneratedColumn('uuid')+ id: string++ @ManyToOne(() => Project, project => project.alerts)+ project: Project++ @ApiProperty()+ @Column('varchar', { length: 50 })+ name: string++ @ApiProperty()+ @Column({+ default: true,+ })+ active: boolean++ @Column({ type: 'timestamp', default: () => 'CURRENT_TIMESTAMP' })+ created: Date++ @ApiProperty()+ @Column({+ type: 'date',+ nullable: true,+ default: null,+ })+ lastTriggered: Date | null++ @ApiProperty()+ @Column({+ type: 'enum',+ enum: QueryMetric,+ nullable: true,+ default: null,+ })+ queryMetric: QueryMetric | null++ @ApiProperty()+ @Column({+ type: 'enum',+ enum: QueryCondition,+ nullable: true,+ default: null,+ })+ queryCondition: QueryCondition | null++ @ApiProperty()+ @Column({+ type: 'int',+ nullable: true,+ default: null,+ })+ queryValue: number | null++ @ApiProperty()+ @Column({+ type: 'enum',+ enum: QueryTime,+ nullable: true,+ default: null,+ })+ queryTime: QueryTime | null+}diff --git a/src/app.module.ts b/src/app.module.tsindex 5568d23aa..a4d4cfba7 100644--- a/src/app.module.ts+++ b/src/app.module.ts@@ -16,6 +16,7 @@ import { WebhookModule } from './webhook/webhook.module'import { PingModule } from './ping/ping.module'import { TGModule } from './tg-integration/tg.module'import { MarketplaceModule } from './marketplace/marketplace.module'+import { AlertModule } from './alert/alert.module'const modules = [ConfigModule.forRoot({ envFilePath: '.env', isGlobal: true }),@@ -42,6 +43,7 @@ const modules = [PingModule,TGModule,MarketplaceModule,+ AlertModule,]if (process.env.TG_BOT_TOKEN) {diff --git a/src/project/dto/project.dto.ts b/src/project/dto/project.dto.tsindex 6f884b76f..157395e3c 100644--- a/src/project/dto/project.dto.ts+++ b/src/project/dto/project.dto.ts@@ -1,27 +1,6 @@import { ApiProperty } from '@nestjs/swagger'import { IsNotEmpty } from 'class-validator'-export enum QueryMetric {- PAGE_VIEWS = 'page_views',- UNIQUE_PAGE_VIEWS = 'unique_page_views',-}--export enum QueryCondition {- GREATER_THAN = 'greater_than',- GREATER_EQUAL_THAN = 'greater_equal_than',- LESS_THAN = 'less_than',- LESS_EQUAL_THAN = 'less_equal_than',-}--export enum QueryTime {- LAST_15_MINUTES = 'last_15_minutes',- LAST_30_MINUTES = 'last_30_minutes',- LAST_1_HOUR = 'last_1_hour',- LAST_4_HOURS = 'last_4_hours',- LAST_24_HOURS = 'last_24_hours',- LAST_48_HOURS = 'last_48_hours',-}-export class ProjectDTO {@ApiProperty({example: 'Your awesome project',@@ -66,23 +45,4 @@ export class ProjectDTO {"When true, anyone on the internet (including Google) would be able to see the project's Dashboard.",})public: boolean-- @ApiProperty({- required: false,- description:- 'Alert if the number of online users exceeds the specified number.',- })- alertIfOnlineUsersExceeds: number | null-- @ApiProperty()- additionalAlertQueryMetric: QueryMetric | null-- @ApiProperty()- additionalAlertQueryCondition: QueryCondition | null-- @ApiProperty()- additionalAlertQueryValue: number | null-- @ApiProperty()- additionalAlertQueryTime: QueryTime | null}diff --git a/src/project/entity/project.entity.ts b/src/project/entity/project.entity.tsindex dffba0963..8c6081b63 100644--- a/src/project/entity/project.entity.ts+++ b/src/project/entity/project.entity.ts@@ -10,12 +10,8 @@ import { ApiProperty } from '@nestjs/swagger'import { User } from '../../user/entities/user.entity'import { ProjectShare } from './project-share.entity'+import { Alert } from 'src/alert/entity/alert.entity'import { ExtensionToProject } from '../../marketplace/extensions/entities/extension-to-project.entity'-import {- QueryCondition,- QueryMetric,- QueryTime,-} from 'src/project/dto/project.dto'// In case of modifying some properties here, make sure to also edit them in common/constants.ts -> selfhosted -> clickhouse// and to add them to the GDPR data export email template@@ -60,6 +56,10 @@ export class Project {@OneToMany(() => ProjectShare, share => share.project)share: ProjectShare[]+ @ApiProperty({ type: () => Alert })+ @OneToMany(() => Alert, alert => alert.project)+ alerts: Alert[]+@Column({ type: 'timestamp', default: () => 'CURRENT_TIMESTAMP' })created: Date@@ -69,62 +69,4 @@ export class Project {)@JoinTable()extensions: ExtensionToProject[]-- @ApiProperty()- @Column({- type: 'int',- nullable: true,- default: null,- })- alertIfOnlineUsersExceeds: number | null-- @ApiProperty()- @Column({- type: 'date',- nullable: true,- default: null,- })- lastSendedAlert: Date | null-- @ApiProperty()- @Column({- type: 'boolean',- default: false,- })- isAdditionalAlertEnabled: boolean-- @ApiProperty()- @Column({- type: 'enum',- enum: QueryMetric,- nullable: true,- default: null,- })- additionalAlertQueryMetric: QueryMetric | null-- @ApiProperty()- @Column({- type: 'enum',- enum: QueryCondition,- nullable: true,- default: null,- })- additionalAlertQueryCondition: QueryCondition | null-- @ApiProperty()- @Column({- type: 'int',- nullable: true,- default: null,- })- additionalAlertQueryValue: number | null-- @ApiProperty()- @Column({- type: 'enum',- enum: QueryTime,- nullable: true,- default: null,- })- additionalAlertQueryTime: QueryTime | null}diff --git a/src/project/project.controller.ts b/src/project/project.controller.tsindex 12be65d6c..9d9b2f521 100644--- a/src/project/project.controller.ts+++ b/src/project/project.controller.ts@@ -460,21 +460,6 @@ export class ProjectController {project.ipBlacklist = _map(projectDTO.ipBlacklist, _trim)project.name = projectDTO.nameproject.public = projectDTO.public- project.alertIfOnlineUsersExceeds = projectDTO.alertIfOnlineUsersExceeds- project.additionalAlertQueryMetric = projectDTO.additionalAlertQueryMetric- project.additionalAlertQueryCondition =- projectDTO.additionalAlertQueryCondition- project.additionalAlertQueryValue = projectDTO.additionalAlertQueryValue- project.additionalAlertQueryTime = projectDTO.additionalAlertQueryTime-- if (- projectDTO.additionalAlertQueryMetric &&- projectDTO.additionalAlertQueryCondition &&- projectDTO.additionalAlertQueryValue &&- projectDTO.additionalAlertQueryTime- ) {- project.isAdditionalAlertEnabled = true- }await updateProjectClickhouse(this.projectService.formatToClickhouse(project),@@ -496,21 +481,6 @@ export class ProjectController {project.ipBlacklist = _map(projectDTO.ipBlacklist, _trim)project.name = projectDTO.nameproject.public = projectDTO.public- project.alertIfOnlineUsersExceeds = projectDTO.alertIfOnlineUsersExceeds- project.additionalAlertQueryMetric = projectDTO.additionalAlertQueryMetric- project.additionalAlertQueryCondition =- projectDTO.additionalAlertQueryCondition- project.additionalAlertQueryValue = projectDTO.additionalAlertQueryValue- project.additionalAlertQueryTime = projectDTO.additionalAlertQueryTime-- if (- projectDTO.additionalAlertQueryMetric &&- projectDTO.additionalAlertQueryCondition &&- projectDTO.additionalAlertQueryValue &&- projectDTO.additionalAlertQueryTime- ) {- project.isAdditionalAlertEnabled = true- }await this.projectService.update(id, _omit(project, ['share', 'admin']))}diff --git a/src/project/project.service.ts b/src/project/project.service.tsindex 14fe0c226..54a10acf2 100644--- a/src/project/project.service.ts+++ b/src/project/project.service.ts@@ -124,7 +124,7 @@ export class ProjectService {async update(id: string,- projectDTO: { ProjectDTO?: ProjectDTO; lastSendedAlert?: Date | null },+ projectDTO: ProjectDTO,): Promise<any> {return this.projectsRepository.update(id, projectDTO)}@@ -193,6 +193,7 @@ export class ProjectService {find(params: object): Promise<Project[]> {return this.projectsRepository.find(params)}+findOneWhere(where: Record<string, unknown>,params: object = {},diff --git a/src/task-manager/task-manager.module.ts b/src/task-manager/task-manager.module.tsindex ace411d4e..da99b9db5 100644--- a/src/task-manager/task-manager.module.ts+++ b/src/task-manager/task-manager.module.ts@@ -5,6 +5,7 @@ import { UserModule } from '../user/user.module'import { ProjectModule } from '../project/project.module'import { AnalyticsModule } from '../analytics/analytics.module'import { ActionTokensModule } from '../action-tokens/action-tokens.module'+import { AlertModule } from 'src/alert/alert.module'@Module({imports: [@@ -12,6 +13,7 @@ import { ActionTokensModule } from '../action-tokens/action-tokens.module'MailerModule,UserModule,ActionTokensModule,+ AlertModule,forwardRef(() => AnalyticsModule),],providers: [TaskManagerService],diff --git a/src/task-manager/task-manager.service.ts b/src/task-manager/task-manager.service.tsindex 088c43997..af472feec 100644--- a/src/task-manager/task-manager.service.ts+++ b/src/task-manager/task-manager.service.ts@@ -1,6 +1,6 @@import { Injectable } from '@nestjs/common'import { Cron, CronExpression } from '@nestjs/schedule'-import { IsNull, LessThan, Not } from 'typeorm'+import { IsNull, LessThan, Not, In } from 'typeorm'import * as bcrypt from 'bcrypt'import * as dayjs from 'dayjs'import * as utc from 'dayjs/plugin/utc'@@ -10,11 +10,13 @@ import * as _join from 'lodash/join'import * as _size from 'lodash/size'import * as _map from 'lodash/map'import * as _now from 'lodash/now'+import * as _find from 'lodash/find'import { MailerService } from '../mailer/mailer.service'import { UserService } from '../user/user.service'import { ProjectService } from '../project/project.service'import { ActionTokensService } from '../action-tokens/action-tokens.service'+import { AlertService } from 'src/alert/alert.service'import { ActionTokenType } from '../action-tokens/action-token.entity'import { LetterTemplate } from '../mailer/letter'import { AnalyticsService } from '../analytics/analytics.service'@@ -41,7 +43,7 @@ import {QueryCondition,QueryMetric,QueryTime,-} from 'src/project/dto/project.dto'+} from 'src/alert/dto/alert.dto'dayjs.extend(utc)@@ -53,6 +55,7 @@ export class TaskManagerService {private readonly analyticsService: AnalyticsService,private readonly projectService: ProjectService,private readonly actionTokensService: ActionTokensService,+ private readonly alertService: AlertService,@InjectBot() private bot: Telegraf<TelegrafContext>,) {}@@ -343,11 +346,9 @@ export class TaskManagerService {})}}-- // every 5 minutes- @Cron('*/5 * * * *')- // // every 5 seconds- // @Cron('*/5 * * * * *')++ @Cron(CronExpression.EVERY_5_MINUTES)+ // @Cron(CronExpression.EVERY_5_SECONDS)async checkUserOnline(): Promise<void> {const projects = await this.projectService.findWhere({@@ -359,35 +360,33 @@ export class TaskManagerService {['admin'],)+ const alerts = await this.alertService.findWhere({+ project: In(_map(projects, 'id')),+ active: true,+ })+for (let i = 0; i < _size(projects); ++i) {- const project = await this.projectService.findOneWithRelations(- projects[i].id,- )+ const alert = alerts[i]+ const project = _find(projects, { id: alert.project.id })- if (project.lastSendedAlert !== null) {- const lastSendedAlert = new Date(project.lastSendedAlert)+ if (alert.lastTriggered !== null) {+ const lastTriggered = new Date(alert.lastTriggered)const now = new Date()- if (now.getTime() - lastSendedAlert.getTime() < 24 * 60 * 60 * 1000) {+ if (now.getTime() - lastTriggered.getTime() < 24 * 60 * 60 * 1000) {return}}const online = await this.analyticsService.getOnlineUserCount(- projects[i].id,+ alert.project.id,)const onlineCount = online.length- // console.log(onlineCount, project.admin.telegramChatId)-- if (onlineCount >= projects[i].alertIfOnlineUsersExceeds) {- if (!project.admin.isTelegramChatIdConfirmed) {- return- }-- await this.projectService.update(projects[i].id, {- lastSendedAlert: new Date(),+ if (onlineCount >= alert.queryValue) {+ await this.alertService.update(alert.id, {+ lastTriggered: new Date(),})this.bot.telegram.sendMessage(@@ -401,14 +400,11 @@ export class TaskManagerService {}}- // every 5 minutes- // @Cron('*/5 * * * *')- //every 5 seconds- @Cron('*/5 * * * * *')+ @Cron(CronExpression.EVERY_5_MINUTES)+ // @Cron(CronExpression.EVERY_5_SECONDS)async checkAdditionalAlerts(): Promise<void> {const projects = await this.projectService.findWhere({- isAdditionalAlertEnabled: true,admin: {isTelegramChatIdConfirmed: true,},@@ -416,90 +412,88 @@ export class TaskManagerService {['admin'],)- for (let i = 0; i < _size(projects); ++i) {- const project = await this.projectService.findOneWithRelations(- projects[i].id,- )+ const alerts = await this.alertService.findWhere({+ project: In(_map(projects, 'id')),+ active: true,+ })++ console.log(alerts)++ for (let i = 0; i < _size(alerts); ++i) {+ const alert = alerts[i]+ const project = _find(projects, { id: alert.project.id })- if (project.lastSendedAlert !== null) {- const lastSendedAlert = new Date(project.lastSendedAlert)+ if (alert.lastTriggered !== null) {+ const lastTriggered = new Date(alert.lastTriggered)const now = new Date()- if (now.getTime() - lastSendedAlert.getTime() < 24 * 60 * 60 * 1000) {+ if (now.getTime() - lastTriggered.getTime() < 24 * 60 * 60 * 1000) {return}}- const isUnique =- projects[i].additionalAlertQueryMetric === QueryMetric.UNIQUE_PAGE_VIEWS- ? 1- : 0+ const isUnique = Number(alert.queryMetric === QueryMetric.UNIQUE_PAGE_VIEWS)const time =- projects[i].additionalAlertQueryTime === QueryTime.LAST_15_MINUTES+ alert.queryTime === QueryTime.LAST_15_MINUTES? 15 * 60- : projects[i].additionalAlertQueryTime === QueryTime.LAST_30_MINUTES+ : alert.queryTime === QueryTime.LAST_30_MINUTES? 30 * 60- : projects[i].additionalAlertQueryTime === QueryTime.LAST_1_HOUR+ : alert.queryTime === QueryTime.LAST_1_HOUR? 60 * 60- : projects[i].additionalAlertQueryTime === QueryTime.LAST_4_HOURS+ : alert.queryTime === QueryTime.LAST_4_HOURS? 4 * 60 * 60- : projects[i].additionalAlertQueryTime === QueryTime.LAST_24_HOURS+ : alert.queryTime === QueryTime.LAST_24_HOURS? 24 * 60 * 60- : projects[i].additionalAlertQueryTime === QueryTime.LAST_48_HOURS+ : alert.queryTime === QueryTime.LAST_48_HOURS? 48 * 60 * 60: 0const isLess =- projects[i].additionalAlertQueryCondition === QueryCondition.LESS_THAN+ alert.queryCondition === QueryCondition.LESS_THAN? '<'- : projects[i].additionalAlertQueryCondition ===+ : alert.queryCondition ===QueryCondition.LESS_EQUAL_THAN? '<='- : projects[i].additionalAlertQueryCondition ===+ : alert.queryCondition ===QueryCondition.GREATER_THAN? '>'- : projects[i].additionalAlertQueryCondition ===+ : alert.queryCondition ===QueryCondition.GREATER_EQUAL_THAN? '>=': ''- const query = `SELECT count() FROM analytics WHERE pid = '${projects[i].id}' AND unique = '${isUnique}' AND created ${isLess} now() - ${time}`+ const query = `SELECT count() FROM analytics WHERE pid = '${alert.id}' AND unique = '${isUnique}' AND created ${isLess} now() - ${time}`const queryResult = await clickhouse.query(query).toPromise()const count = Number(queryResult[0]['count()'])- console.log(count)-- if (count >= projects[i].additionalAlertQueryValue) {- if (!project.admin.isTelegramChatIdConfirmed) {- return- }-- await this.projectService.update(projects[i].id, {- lastSendedAlert: new Date(),+ if (count >= alert.queryValue) {+ await this.alertService.update(alert.id, {+ lastTriggered: new Date(),})- const text2 = `π Your project *${projects[i].name}* has *${count}* ${- projects[i].additionalAlertQueryMetric ===+ const text2 = `π Your project *${alert.name}* has *${count}* ${+ alert.queryMetric ===QueryMetric.UNIQUE_PAGE_VIEWS? 'unique page views': 'page views'} in the last ${- projects[i].additionalAlertQueryTime === QueryTime.LAST_15_MINUTES+ alert.queryTime === QueryTime.LAST_15_MINUTES? '15 minutes'- : projects[i].additionalAlertQueryTime === QueryTime.LAST_30_MINUTES+ : alert.queryTime === QueryTime.LAST_30_MINUTES? '30 minutes'- : projects[i].additionalAlertQueryTime === QueryTime.LAST_1_HOUR+ : alert.queryTime === QueryTime.LAST_1_HOUR? '1 hour'- : projects[i].additionalAlertQueryTime === QueryTime.LAST_4_HOURS+ : alert.queryTime === QueryTime.LAST_4_HOURS? '4 hours'- : projects[i].additionalAlertQueryTime === QueryTime.LAST_24_HOURS+ : alert.queryTime === QueryTime.LAST_24_HOURS? '24 hours'- : projects[i].additionalAlertQueryTime === QueryTime.LAST_48_HOURS+ : alert.queryTime === QueryTime.LAST_48_HOURS? '48 hours': '0'}!`+this.bot.telegram.sendMessage(project.admin.telegramChatId, text2, {parse_mode: 'Markdown',})