📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!
TypeScript chartsclickhousedockergdprgoogle-analyticsmetricsmonitoringmysqlnestjsprivacyredisself-hostedselfhostedstatisticstailwindcssweb-analyticswebanalyticswebsite
Complete raw uptime functionality
Yehor Dremliuha960ee8e1e2f3
diff --git a/apps/production/src/app.module.ts b/apps/production/src/app.module.tsindex 7453b52fd..fb940a65a 100644--- a/apps/production/src/app.module.ts+++ b/apps/production/src/app.module.ts@@ -57,7 +57,7 @@ const modules = [username: process.env.MYSQL_USER,password: process.env.MYSQL_ROOT_PASSWORD,database: process.env.MYSQL_DATABASE,- synchronize: false, // process.env.NODE_ENV !== 'production',+ synchronize: true, // process.env.NODE_ENV !== 'production',entities: [`${__dirname}/**/*.entity{.ts,.js}`],}),BullModule.forRoot({diff --git a/apps/production/src/project/consumers/monitor.consumer.ts b/apps/production/src/project/consumers/monitor.consumer.tsindex 6a7457450..7607bc532 100644--- a/apps/production/src/project/consumers/monitor.consumer.ts+++ b/apps/production/src/project/consumers/monitor.consumer.ts@@ -1,9 +1,10 @@import { Process, Processor } from '@nestjs/bull'-import { Inject, Logger } from '@nestjs/common'+import { Inject, InternalServerErrorException, Logger } from '@nestjs/common'import { ClientProxy } from '@nestjs/microservices'-import { Job } from 'bullmq'+import { Job } from 'bull'import { firstValueFrom } from 'rxjs'import { CreateMonitorHttpRequestDTO } from '../dto/create-monitor.dto'+import { clickhouse } from '../../common/integrations/clickhouse'@Processor('monitor')export class MonitorConsumer {@@ -13,13 +14,36 @@ export class MonitorConsumer {@Process('http-request')async httpRequestProcess(- job: Job<CreateMonitorHttpRequestDTO>,+ job: Job<CreateMonitorHttpRequestDTO & { monitorID: string }>,): Promise<any> {this.logger.debug(`Consumer works ${JSON.stringify(job.data)}`)- const res = await firstValueFrom(- this.monitorService.send('http-request', job.data),- )+ const res: {+ responseTime: number+ statusCode: number+ timestamp: number+ region: string+ } = await firstValueFrom(this.monitorService.send('http-request', job.data))this.logger.debug(`Consumer works ${JSON.stringify(res)}`)- return res++ try {+ await clickhouse.insert({+ table: 'monitor_responses',+ format: 'JSONEachRow',+ values: [+ {+ monitorID: job.data.monitorID,+ region: res.region,+ responseTime: res.responseTime,+ timestamp: res.timestamp,+ statusCode: res.statusCode,+ },+ ],+ })+ } catch (e) {+ this.logger.error(e)+ throw new InternalServerErrorException(+ 'Error occurred while saving the monitor data',+ )+ }}}diff --git a/apps/production/src/project/project.controller.ts b/apps/production/src/project/project.controller.tsindex ebc53311a..bc27a0162 100644--- a/apps/production/src/project/project.controller.ts+++ b/apps/production/src/project/project.controller.ts@@ -2382,7 +2382,10 @@ export class ProjectController {)// Create a job in service for sending httpRequest- await this.projectService.sendHttpRequest(monitor.id, monitor)+ await this.projectService.sendHttpRequest(monitor.id, {+ ...monitor,+ timeout: monitor.timeout * 1000,+ })}@ApiOperation({ summary: 'Get monitor for the project' })@@ -2422,11 +2425,7 @@ export class ProjectController {throw new NotFoundException('Monitor group not found.')}- const monitor = await this.projectService.getMonitor(- monitorId,- monitorGroup.id,- project.id,- )+ const monitor = await this.projectService.getMonitor(monitorId)if (!monitor) {throw new NotFoundException('Monitor not found.')@@ -2473,11 +2472,7 @@ export class ProjectController {throw new NotFoundException('Monitor group not found.')}- const monitor = await this.projectService.getMonitor(- monitorId,- monitorGroup.id,- project.id,- )+ const monitor = await this.projectService.getMonitor(monitorId)if (!monitor) {throw new NotFoundException('Monitor not found.')@@ -2528,11 +2523,7 @@ export class ProjectController {throw new NotFoundException('Monitor group not found.')}- const monitor = await this.projectService.getMonitor(- monitorId,- monitorGroup.id,- project.id,- )+ const monitor = await this.projectService.getMonitor(monitorId)if (!monitor) {throw new NotFoundException('Monitor not found.')diff --git a/apps/production/src/project/project.service.ts b/apps/production/src/project/project.service.tsindex bcba0cf74..a2247e408 100644--- a/apps/production/src/project/project.service.ts+++ b/apps/production/src/project/project.service.ts@@ -1508,10 +1508,14 @@ export class ProjectService {monitorHttpRequestDto: HttpRequestOptions,) {// TODO crete interface for that stuff- await this.monitorQueue.add('http-request', monitorHttpRequestDto, {- repeat: { every: monitorHttpRequestDto.interval },- jobId: monitorID,- })+ await this.monitorQueue.add(+ 'http-request',+ { ...monitorHttpRequestDto, monitorID },+ {+ repeat: { every: monitorHttpRequestDto.interval * 1000 },+ jobId: monitorID,+ },+ )}async updateHttpRequest(@@ -1520,7 +1524,10 @@ export class ProjectService {): Promise<void> {await this.deleteHttpRequest(monitorID)- await this.sendHttpRequest(monitorID, updatedHttpRequestDto)+ await this.sendHttpRequest(monitorID, {+ ...updatedHttpRequestDto,+ timeout: updatedHttpRequestDto.timeout * 1000,+ })}async deleteHttpRequest(monitorID: string) {diff --git a/docker-compose-dev.yml b/docker-compose-dev.ymlindex f463af4b0..8e433eebc 100644--- a/docker-compose-dev.yml+++ b/docker-compose-dev.yml@@ -3,7 +3,7 @@ services:redis:image: redis:7.2-alpineports:- - 6378:6379+ - 6379:6379clickhouse:image: clickhouse/clickhouse-server:23.8-alpineenvironment:@@ -16,14 +16,4 @@ services:MYSQL_DATABASE: 'analytics'MYSQL_ALLOW_EMPTY_PASSWORD: 'yes'ports:- - 3306:3306-- redis-commander:- image: rediscommander/redis-commander:latest- container_name: redis-commander- environment:- - REDIS_HOSTS=local:redis:6378- ports:- - "8081:8081"- depends_on:- - redis+ - 3306:3306\ No newline at end of filediff --git a/kill_all_jobs_rmq.ts b/kill_all_jobs_rmq.tsnew file mode 100644index 000000000..c633efc53--- /dev/null+++ b/kill_all_jobs_rmq.ts@@ -0,0 +1,48 @@+import { Queue } from 'bullmq';++async function cleanAllJobs(queueName: string) {+ const queue = new Queue(queueName, {+ connection: {+ host: 'localhost',+ port: 6379,+ // Add other Redis connection options if needed+ },+ });++ try {+ // Clean all completed jobs+ await queue.clean(0, 1000, 'completed');+ console.log('Completed jobs cleaned');++ // Clean all waiting jobs+ await queue.clean(0, 1000, 'wait');+ console.log('Waiting jobs cleaned');++ // Clean all active jobs+ await queue.clean(0, 1000, 'active');+ console.log('Active jobs cleaned');++ // Clean all delayed jobs+ await queue.clean(0, 1000, 'delayed');+ console.log('Delayed jobs cleaned');++ // Clean all failed jobs+ await queue.clean(0, 1000, 'failed');+ console.log('Failed jobs cleaned');++ // Drain the queue (remove all jobs)+ await queue.drain();+ console.log('Queue drained');++ // Obliterate the queue (remove all data and jobs associated with the queue)+ await queue.obliterate({ force: true });+ console.log('Queue obliterated');++ } catch (error) {+ console.error('Error cleaning jobs:', error);+ } finally {+ await queue.close();+ }+}++cleanAllJobs('monitor');diff --git a/migrations/clickhouse/2024_09_13.js b/migrations/clickhouse/2024_09_13.jsnew file mode 100644index 000000000..1e8112852--- /dev/null+++ b/migrations/clickhouse/2024_09_13.js@@ -0,0 +1,19 @@+const { queriesRunner, dbName } = require('./setup');++const queries = [+ // Monitor responses table+ `CREATE TABLE IF NOT EXISTS ${dbName}.monitor_responses+ (+ monitorID FixedString(36),+ region LowCardinality(Nullable(String)),+ responseTime UInt32,+ timestamp UInt32,+ statusCode UInt32,+ created DateTime('UTC') DEFAULT now()+ )+ ENGINE = MergeTree()+ PARTITION BY toYYYYMM(created)+ ORDER BY (monitorID, created);`,+];++queriesRunner(queries);diff --git a/migrations/clickhouse/initialise_database.js b/migrations/clickhouse/initialise_database.jsindex a48c80f22..66496e3a8 100644--- a/migrations/clickhouse/initialise_database.js+++ b/migrations/clickhouse/initialise_database.js@@ -137,6 +137,19 @@ const CLICKHOUSE_INIT_QUERIES = [ENGINE = MergeTree()PARTITION BY toYYYYMM(created)ORDER BY (pid, created);`,++ `CREATE TABLE IF NOT EXISTS ${dbName}.monitor_responses+ (+ monitorID FixedString(36),+ region LowCardinality(Nullable(String)),+ responseTime UInt32,+ timestamp UInt32,+ statusCode UInt32,+ created DateTime('UTC') DEFAULT now()+ )+ ENGINE = MergeTree()+ PARTITION BY toYYYYMM(created)+ ORDER BY (monitorID, created);`,]const initialiseDatabase = async () => {