Skip to content

Open-source status page, incident response and uptime monitoring

πŸ› Bug notifications (#983)

Thibault Le Ouay6e01156ab174
diff --git a/apps/server/src/checker/alerting.ts b/apps/server/src/checker/alerting.ts
index e7407ccab..d2c937fb1 100644
--- a/apps/server/src/checker/alerting.ts
+++ b/apps/server/src/checker/alerting.ts
@@ -6,9 +6,12 @@ import {
} from "@openstatus/db/src/schema";
import type { MonitorFlyRegion } from "@openstatus/db/src/schema/constants";
+import { Redis } from "@openstatus/upstash";
import { checkerAudit } from "../utils/audit-log";
import { providerToFunction } from "./utils";
+const redis = Redis.fromEnv();
+
export const triggerNotifications = async ({
monitorId,
statusCode,
@@ -37,6 +40,15 @@ export const triggerNotifications = async ({
.where(eq(schema.monitor.id, Number(monitorId)))
.all();
for (const notif of notifications) {
+ const key = `${monitorId}:${incidentId}:${notif.notification.provider}:${notifType}`;
+ const r = await redis.setnx(key, "1");
+ if (r === 0) {
+ console.log(`πŸ€” notification already sent for ${key}`);
+ continue;
+ }
+
+ await redis.expire(key, 60 * 60);
+
console.log(
`πŸ’Œ sending notification for ${monitorId} and chanel ${notif.notification.provider} for ${notifType}`,
);
diff --git a/apps/server/src/checker/index.ts b/apps/server/src/checker/index.ts
index 9efce61c7..7788c3e65 100644
--- a/apps/server/src/checker/index.ts
+++ b/apps/server/src/checker/index.ts
@@ -87,15 +87,15 @@ checkerRoute.post("/updateStatus", async (c) => {
// We add the new region to the set
await redis.sadd(redisKey, region);
// let's add an expire to the set
- await redis.expire(redisKey, 60 * 60 * 24);
// We get the number of regions affected
const nbAffectedRegion = await redis.scard(redisKey);
+ await redis.expire(redisKey, 60 * 60 * 24);
const monitor = selectMonitorSchema.parse(currentMonitor);
const numberOfRegions = monitor.regions.length;
- if (nbAffectedRegion > numberOfRegions / 2) {
+ if (nbAffectedRegion >= numberOfRegions / 2) {
await triggerNotifications({
monitorId,
statusCode,
@@ -126,9 +126,9 @@ checkerRoute.post("/updateStatus", async (c) => {
// We add the new region to the set
await redis.sadd(redisKey, region);
// let's add an expire to the set
- await redis.expire(redisKey, 60 * 60 * 24);
// We get the number of regions affected
const nbAffectedRegion = await redis.scard(redisKey);
+ await redis.expire(redisKey, 60 * 60 * 24);
const currentMonitor = await db
.select()
@@ -145,7 +145,7 @@ checkerRoute.post("/updateStatus", async (c) => {
);
// If the number of affected regions is greater than half of the total region, we trigger the alerting
// 4 of 6 monitor need to fail to trigger an alerting
- if (nbAffectedRegion > numberOfRegions / 2) {
+ if (nbAffectedRegion >= numberOfRegions / 2) {
// let's refetch the incident to avoid race condition
const incident = await db
.select()
@@ -218,9 +218,9 @@ checkerRoute.post("/updateStatus", async (c) => {
// // We add the new region to the set
await redis.sadd(redisKey, region);
// // let's add an expire to the set
- await redis.expire(redisKey, 60 * 60 * 24);
// // We get the number of regions affected
const nbAffectedRegion = await redis.scard(redisKey);
+ await redis.expire(redisKey, 60 * 60 * 24);
const currentMonitor = await db
.select()
@@ -237,7 +237,7 @@ checkerRoute.post("/updateStatus", async (c) => {
);
// // If the number of affected regions is greater than half of the total region, we trigger the alerting
// // 4 of 6 monitor need to fail to trigger an alerting
- if (nbAffectedRegion > numberOfRegions / 2) {
+ if (nbAffectedRegion >= numberOfRegions / 2) {
const incident = await db
.select()
.from(incidentTable)
@@ -259,6 +259,7 @@ checkerRoute.post("/updateStatus", async (c) => {
})
.where(eq(incidentTable.id, incident.id))
.run();
+
await triggerNotifications({
monitorId,
statusCode,