Skip to content

📈 Open source, privacy-first web analytics — a cookie-less Google Analytics alternative with built-in error tracking and performance monitoring. Self-host it or use our cloud. Get started for free!

Complete raw uptime functionality

Yehor Dremliuha960ee8e1e2f3
diff --git a/apps/production/src/app.module.ts b/apps/production/src/app.module.ts
index 7453b52fd..fb940a65a 100644
--- a/apps/production/src/app.module.ts
+++ b/apps/production/src/app.module.ts
@@ -57,7 +57,7 @@ const modules = [
username: process.env.MYSQL_USER,
password: process.env.MYSQL_ROOT_PASSWORD,
database: process.env.MYSQL_DATABASE,
- synchronize: false, // process.env.NODE_ENV !== 'production',
+ synchronize: true, // process.env.NODE_ENV !== 'production',
entities: [`${__dirname}/**/*.entity{.ts,.js}`],
}),
BullModule.forRoot({
diff --git a/apps/production/src/project/consumers/monitor.consumer.ts b/apps/production/src/project/consumers/monitor.consumer.ts
index 6a7457450..7607bc532 100644
--- a/apps/production/src/project/consumers/monitor.consumer.ts
+++ b/apps/production/src/project/consumers/monitor.consumer.ts
@@ -1,9 +1,10 @@
import { Process, Processor } from '@nestjs/bull'
-import { Inject, Logger } from '@nestjs/common'
+import { Inject, InternalServerErrorException, Logger } from '@nestjs/common'
import { ClientProxy } from '@nestjs/microservices'
-import { Job } from 'bullmq'
+import { Job } from 'bull'
import { firstValueFrom } from 'rxjs'
import { CreateMonitorHttpRequestDTO } from '../dto/create-monitor.dto'
+import { clickhouse } from '../../common/integrations/clickhouse'
@Processor('monitor')
export class MonitorConsumer {
@@ -13,13 +14,36 @@ export class MonitorConsumer {
@Process('http-request')
async httpRequestProcess(
- job: Job<CreateMonitorHttpRequestDTO>,
+ job: Job<CreateMonitorHttpRequestDTO & { monitorID: string }>,
): Promise<any> {
this.logger.debug(`Consumer works ${JSON.stringify(job.data)}`)
- const res = await firstValueFrom(
- this.monitorService.send('http-request', job.data),
- )
+ const res: {
+ responseTime: number
+ statusCode: number
+ timestamp: number
+ region: string
+ } = await firstValueFrom(this.monitorService.send('http-request', job.data))
this.logger.debug(`Consumer works ${JSON.stringify(res)}`)
- return res
+
+ try {
+ await clickhouse.insert({
+ table: 'monitor_responses',
+ format: 'JSONEachRow',
+ values: [
+ {
+ monitorID: job.data.monitorID,
+ region: res.region,
+ responseTime: res.responseTime,
+ timestamp: res.timestamp,
+ statusCode: res.statusCode,
+ },
+ ],
+ })
+ } catch (e) {
+ this.logger.error(e)
+ throw new InternalServerErrorException(
+ 'Error occurred while saving the monitor data',
+ )
+ }
}
}
diff --git a/apps/production/src/project/project.controller.ts b/apps/production/src/project/project.controller.ts
index ebc53311a..bc27a0162 100644
--- a/apps/production/src/project/project.controller.ts
+++ b/apps/production/src/project/project.controller.ts
@@ -2382,7 +2382,10 @@ export class ProjectController {
)
// Create a job in service for sending httpRequest
- await this.projectService.sendHttpRequest(monitor.id, monitor)
+ await this.projectService.sendHttpRequest(monitor.id, {
+ ...monitor,
+ timeout: monitor.timeout * 1000,
+ })
}
@ApiOperation({ summary: 'Get monitor for the project' })
@@ -2422,11 +2425,7 @@ export class ProjectController {
throw new NotFoundException('Monitor group not found.')
}
- const monitor = await this.projectService.getMonitor(
- monitorId,
- monitorGroup.id,
- project.id,
- )
+ const monitor = await this.projectService.getMonitor(monitorId)
if (!monitor) {
throw new NotFoundException('Monitor not found.')
@@ -2473,11 +2472,7 @@ export class ProjectController {
throw new NotFoundException('Monitor group not found.')
}
- const monitor = await this.projectService.getMonitor(
- monitorId,
- monitorGroup.id,
- project.id,
- )
+ const monitor = await this.projectService.getMonitor(monitorId)
if (!monitor) {
throw new NotFoundException('Monitor not found.')
@@ -2528,11 +2523,7 @@ export class ProjectController {
throw new NotFoundException('Monitor group not found.')
}
- const monitor = await this.projectService.getMonitor(
- monitorId,
- monitorGroup.id,
- project.id,
- )
+ const monitor = await this.projectService.getMonitor(monitorId)
if (!monitor) {
throw new NotFoundException('Monitor not found.')
diff --git a/apps/production/src/project/project.service.ts b/apps/production/src/project/project.service.ts
index bcba0cf74..a2247e408 100644
--- a/apps/production/src/project/project.service.ts
+++ b/apps/production/src/project/project.service.ts
@@ -1508,10 +1508,14 @@ export class ProjectService {
monitorHttpRequestDto: HttpRequestOptions,
) {
// TODO crete interface for that stuff
- await this.monitorQueue.add('http-request', monitorHttpRequestDto, {
- repeat: { every: monitorHttpRequestDto.interval },
- jobId: monitorID,
- })
+ await this.monitorQueue.add(
+ 'http-request',
+ { ...monitorHttpRequestDto, monitorID },
+ {
+ repeat: { every: monitorHttpRequestDto.interval * 1000 },
+ jobId: monitorID,
+ },
+ )
}
async updateHttpRequest(
@@ -1520,7 +1524,10 @@ export class ProjectService {
): Promise<void> {
await this.deleteHttpRequest(monitorID)
- await this.sendHttpRequest(monitorID, updatedHttpRequestDto)
+ await this.sendHttpRequest(monitorID, {
+ ...updatedHttpRequestDto,
+ timeout: updatedHttpRequestDto.timeout * 1000,
+ })
}
async deleteHttpRequest(monitorID: string) {
diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml
index f463af4b0..8e433eebc 100644
--- a/docker-compose-dev.yml
+++ b/docker-compose-dev.yml
@@ -3,7 +3,7 @@ services:
redis:
image: redis:7.2-alpine
ports:
- - 6378:6379
+ - 6379:6379
clickhouse:
image: clickhouse/clickhouse-server:23.8-alpine
environment:
@@ -16,14 +16,4 @@ services:
MYSQL_DATABASE: 'analytics'
MYSQL_ALLOW_EMPTY_PASSWORD: 'yes'
ports:
- - 3306:3306
-
- redis-commander:
- image: rediscommander/redis-commander:latest
- container_name: redis-commander
- environment:
- - REDIS_HOSTS=local:redis:6378
- ports:
- - "8081:8081"
- depends_on:
- - redis
+ - 3306:3306
\ No newline at end of file
diff --git a/kill_all_jobs_rmq.ts b/kill_all_jobs_rmq.ts
new file mode 100644
index 000000000..c633efc53
--- /dev/null
+++ b/kill_all_jobs_rmq.ts
@@ -0,0 +1,48 @@
+import { Queue } from 'bullmq';
+
+async function cleanAllJobs(queueName: string) {
+ const queue = new Queue(queueName, {
+ connection: {
+ host: 'localhost',
+ port: 6379,
+ // Add other Redis connection options if needed
+ },
+ });
+
+ try {
+ // Clean all completed jobs
+ await queue.clean(0, 1000, 'completed');
+ console.log('Completed jobs cleaned');
+
+ // Clean all waiting jobs
+ await queue.clean(0, 1000, 'wait');
+ console.log('Waiting jobs cleaned');
+
+ // Clean all active jobs
+ await queue.clean(0, 1000, 'active');
+ console.log('Active jobs cleaned');
+
+ // Clean all delayed jobs
+ await queue.clean(0, 1000, 'delayed');
+ console.log('Delayed jobs cleaned');
+
+ // Clean all failed jobs
+ await queue.clean(0, 1000, 'failed');
+ console.log('Failed jobs cleaned');
+
+ // Drain the queue (remove all jobs)
+ await queue.drain();
+ console.log('Queue drained');
+
+ // Obliterate the queue (remove all data and jobs associated with the queue)
+ await queue.obliterate({ force: true });
+ console.log('Queue obliterated');
+
+ } catch (error) {
+ console.error('Error cleaning jobs:', error);
+ } finally {
+ await queue.close();
+ }
+}
+
+cleanAllJobs('monitor');
diff --git a/migrations/clickhouse/2024_09_13.js b/migrations/clickhouse/2024_09_13.js
new file mode 100644
index 000000000..1e8112852
--- /dev/null
+++ b/migrations/clickhouse/2024_09_13.js
@@ -0,0 +1,19 @@
+const { queriesRunner, dbName } = require('./setup');
+
+const queries = [
+ // Monitor responses table
+ `CREATE TABLE IF NOT EXISTS ${dbName}.monitor_responses
+ (
+ monitorID FixedString(36),
+ region LowCardinality(Nullable(String)),
+ responseTime UInt32,
+ timestamp UInt32,
+ statusCode UInt32,
+ created DateTime('UTC') DEFAULT now()
+ )
+ ENGINE = MergeTree()
+ PARTITION BY toYYYYMM(created)
+ ORDER BY (monitorID, created);`,
+];
+
+queriesRunner(queries);
diff --git a/migrations/clickhouse/initialise_database.js b/migrations/clickhouse/initialise_database.js
index a48c80f22..66496e3a8 100644
--- a/migrations/clickhouse/initialise_database.js
+++ b/migrations/clickhouse/initialise_database.js
@@ -137,6 +137,19 @@ const CLICKHOUSE_INIT_QUERIES = [
ENGINE = MergeTree()
PARTITION BY toYYYYMM(created)
ORDER BY (pid, created);`,
+
+ `CREATE TABLE IF NOT EXISTS ${dbName}.monitor_responses
+ (
+ monitorID FixedString(36),
+ region LowCardinality(Nullable(String)),
+ responseTime UInt32,
+ timestamp UInt32,
+ statusCode UInt32,
+ created DateTime('UTC') DEFAULT now()
+ )
+ ENGINE = MergeTree()
+ PARTITION BY toYYYYMM(created)
+ ORDER BY (monitorID, created);`,
]
const initialiseDatabase = async () => {