Open-source status page, incident response and uptime monitoring
TypeScript alertingincident-managementmcpmcp-servermonitoringmonitoring-as-codeobservabilityon-callopen-sourcesrestatus-pagestatuspagesynthetic-monitoringterraformtinybirdtursouptimeuptime-checkeruptime-monitor
feat: return triggered notifications from alerting and enrich status_update event (#1846)
Thibault Le Ouayf6580b37c0e5
diff --git a/apps/workflows/src/checker/alerting.test.ts b/apps/workflows/src/checker/alerting.test.tsindex 79bc410e4..884e6a145 100644--- a/apps/workflows/src/checker/alerting.test.ts+++ b/apps/workflows/src/checker/alerting.test.ts@@ -1,22 +1,312 @@-import { expect, mock, test } from "bun:test";--import { triggerNotifications } from "./alerting";--test.todo("should send email notification", async () => {- const fn = mock(() => {});- mock.module("./utils.ts", () => {- return {- providerToFunction: {- email: fn,- },- };- });- await triggerNotifications({- monitorId: "1",- statusCode: 400,- notifType: "alert",- cronTimestamp: 123456,- incidentId: 1,- });- expect(fn).toHaveBeenCalled();+import { afterAll, afterEach, describe, expect, mock, test } from "bun:test";+import { db, eq, inArray } from "@openstatus/db";+import {+ notification,+ notificationTrigger,+ notificationsToMonitors,+} from "@openstatus/db/src/schema";++// Mock audit log to avoid calling Tinybird+mock.module("../utils/audit-log", () => ({+ checkerAudit: {+ publishAuditLog: mock(() => Promise.resolve()),+ },+}));++// Mock notification providers to avoid real HTTP calls+const mockEmailSendAlert = mock(() => Promise.resolve());+const mockEmailSendRecovery = mock(() => Promise.resolve());+const mockEmailSendDegraded = mock(() => Promise.resolve());+const mockSlackSendAlert = mock(() => Promise.resolve());+const mockSlackSendRecovery = mock(() => Promise.resolve());+const mockSlackSendDegraded = mock(() => Promise.resolve());+const mockDiscordSendAlert = mock(() => Promise.resolve());+const mockDiscordSendRecovery = mock(() => Promise.resolve());+const mockDiscordSendDegraded = mock(() => Promise.resolve());++mock.module("./utils", () => ({+ providerToFunction: {+ email: {+ sendAlert: mockEmailSendAlert,+ sendRecovery: mockEmailSendRecovery,+ sendDegraded: mockEmailSendDegraded,+ },+ discord: {+ sendAlert: mockDiscordSendAlert,+ sendRecovery: mockDiscordSendRecovery,+ sendDegraded: mockDiscordSendDegraded,+ },+ slack: {+ sendAlert: mockSlackSendAlert,+ sendRecovery: mockSlackSendRecovery,+ sendDegraded: mockSlackSendDegraded,+ },+ sms: { sendAlert: mock(), sendRecovery: mock(), sendDegraded: mock() },+ webhook: { sendAlert: mock(), sendRecovery: mock(), sendDegraded: mock() },+ telegram: { sendAlert: mock(), sendRecovery: mock(), sendDegraded: mock() },+ pagerduty: {+ sendAlert: mock(),+ sendRecovery: mock(),+ sendDegraded: mock(),+ },+ opsgenie: { sendAlert: mock(), sendRecovery: mock(), sendDegraded: mock() },+ ntfy: { sendAlert: mock(), sendRecovery: mock(), sendDegraded: mock() },+ "google-chat": {+ sendAlert: mock(),+ sendRecovery: mock(),+ sendDegraded: mock(),+ },+ "grafana-oncall": {+ sendAlert: mock(),+ sendRecovery: mock(),+ sendDegraded: mock(),+ },+ whatsapp: { sendAlert: mock(), sendRecovery: mock(), sendDegraded: mock() },+ },+}));++// Import after mocks are set up+const { triggerNotifications } = await import("./alerting");++// Seed data has: monitor 1 (workspace 1) linked to notification 1 (email provider)+// We use unique cronTimestamp per test to avoid unique constraint conflicts++describe("triggerNotifications", () => {+ afterEach(() => {+ mockEmailSendAlert.mockClear();+ mockEmailSendRecovery.mockClear();+ mockEmailSendDegraded.mockClear();+ mockSlackSendAlert.mockClear();+ mockSlackSendRecovery.mockClear();+ mockSlackSendDegraded.mockClear();+ mockDiscordSendAlert.mockClear();+ mockDiscordSendRecovery.mockClear();+ mockDiscordSendDegraded.mockClear();+ });++ afterAll(async () => {+ // Clean up notification triggers created during tests+ await db+ .delete(notificationTrigger)+ .where(eq(notificationTrigger.monitorId, 1))+ .run();+ });++ test("should send alert notification and return triggered list", async () => {+ const cronTimestamp = 9000001;++ const result = await triggerNotifications({+ monitorId: "1",+ statusCode: 500,+ message: "Internal Server Error",+ notifType: "alert",+ cronTimestamp,+ incidentId: undefined,+ regions: ["ams"],+ latency: 1500,+ });++ expect(mockEmailSendAlert).toHaveBeenCalledTimes(1);+ expect(result).toHaveLength(1);+ expect(result[0]).toEqual({+ notificationId: 1,+ provider: "email",+ });+ });++ test("should send recovery notification and return triggered list", async () => {+ const cronTimestamp = 9000002;++ const result = await triggerNotifications({+ monitorId: "1",+ statusCode: 200,+ notifType: "recovery",+ cronTimestamp,+ regions: ["ams"],+ });++ expect(mockEmailSendRecovery).toHaveBeenCalledTimes(1);+ expect(result).toHaveLength(1);+ expect(result[0]).toEqual({+ notificationId: 1,+ provider: "email",+ });+ });++ test("should send degraded notification and return triggered list", async () => {+ const cronTimestamp = 9000003;++ const result = await triggerNotifications({+ monitorId: "1",+ statusCode: 200,+ notifType: "degraded",+ cronTimestamp,+ latency: 5000,+ regions: ["ams"],+ });++ expect(mockEmailSendDegraded).toHaveBeenCalledTimes(1);+ expect(result).toHaveLength(1);+ expect(result[0]).toEqual({+ notificationId: 1,+ provider: "email",+ });+ });++ test("should return empty list when monitor has no notifications", async () => {+ const cronTimestamp = 9000004;++ // Monitor 2 has no notifications linked in seed data+ const result = await triggerNotifications({+ monitorId: "2",+ statusCode: 500,+ notifType: "alert",+ cronTimestamp,+ });++ expect(mockEmailSendAlert).not.toHaveBeenCalled();+ expect(result).toHaveLength(0);+ });++ test("should skip duplicate notification trigger for same cronTimestamp", async () => {+ const cronTimestamp = 9000005;++ const first = await triggerNotifications({+ monitorId: "1",+ statusCode: 500,+ notifType: "alert",+ cronTimestamp,+ });++ expect(first).toHaveLength(1);+ mockEmailSendAlert.mockClear();++ // Same cronTimestamp should be skipped due to unique constraint+ const second = await triggerNotifications({+ monitorId: "1",+ statusCode: 500,+ notifType: "alert",+ cronTimestamp,+ });++ expect(mockEmailSendAlert).not.toHaveBeenCalled();+ expect(second).toHaveLength(0);+ });+});++describe("triggerNotifications with multiple providers", () => {+ const testNotificationIds: number[] = [];++ // Monitor 3 (workspace 1) exists in seed but has no notifications.+ // We link slack and discord notifications to it for this test suite.+ const testMonitorId = 3;++ afterEach(() => {+ mockEmailSendAlert.mockClear();+ mockSlackSendAlert.mockClear();+ mockDiscordSendAlert.mockClear();+ mockSlackSendRecovery.mockClear();+ mockDiscordSendRecovery.mockClear();+ });++ afterAll(async () => {+ // Clean up notification triggers+ await db+ .delete(notificationTrigger)+ .where(eq(notificationTrigger.monitorId, testMonitorId))+ .run();++ // Clean up notification-to-monitor links+ await db+ .delete(notificationsToMonitors)+ .where(eq(notificationsToMonitors.monitorId, testMonitorId))+ .run();++ // Clean up test notifications+ if (testNotificationIds.length > 0) {+ await db+ .delete(notification)+ .where(inArray(notification.id, testNotificationIds))+ .run();+ }+ });++ test("should trigger all linked providers and return each in the result", async () => {+ // Create slack notification+ const [slackNotif] = await db+ .insert(notification)+ .values({+ name: "test-slack",+ provider: "slack",+ data: '{"slack":"https://hooks.slack.com/test"}',+ workspaceId: 1,+ })+ .returning();++ // Create discord notification+ const [discordNotif] = await db+ .insert(notification)+ .values({+ name: "test-discord",+ provider: "discord",+ data: '{"discord":"https://discord.com/api/webhooks/test"}',+ workspaceId: 1,+ })+ .returning();++ testNotificationIds.push(slackNotif.id, discordNotif.id);++ // Link both to monitor 3+ await db+ .insert(notificationsToMonitors)+ .values([+ { monitorId: testMonitorId, notificationId: slackNotif.id },+ { monitorId: testMonitorId, notificationId: discordNotif.id },+ ])+ .run();++ const cronTimestamp = 9100001;++ const result = await triggerNotifications({+ monitorId: String(testMonitorId),+ statusCode: 500,+ message: "Server Error",+ notifType: "alert",+ cronTimestamp,+ regions: ["ams"],+ });++ expect(mockSlackSendAlert).toHaveBeenCalledTimes(1);+ expect(mockDiscordSendAlert).toHaveBeenCalledTimes(1);+ expect(mockEmailSendAlert).not.toHaveBeenCalled();++ expect(result).toHaveLength(2);+ expect(result).toContainEqual({+ notificationId: slackNotif.id,+ provider: "slack",+ });+ expect(result).toContainEqual({+ notificationId: discordNotif.id,+ provider: "discord",+ });+ });++ test("should trigger recovery on all linked providers", async () => {+ const cronTimestamp = 9100002;++ const result = await triggerNotifications({+ monitorId: String(testMonitorId),+ statusCode: 200,+ notifType: "recovery",+ cronTimestamp,+ regions: ["ams"],+ });++ expect(mockSlackSendRecovery).toHaveBeenCalledTimes(1);+ expect(mockDiscordSendRecovery).toHaveBeenCalledTimes(1);++ expect(result).toHaveLength(2);+ const providers = result.map((r) => r.provider).sort();+ expect(providers).toEqual(["discord", "slack"]);+ });});diff --git a/apps/workflows/src/checker/alerting.ts b/apps/workflows/src/checker/alerting.tsindex d36869868..22bc39579 100644--- a/apps/workflows/src/checker/alerting.ts+++ b/apps/workflows/src/checker/alerting.ts@@ -32,12 +32,14 @@ export const triggerNotifications = async ({incidentId?: number;regions?: string[];latency?: number;-}) => {+}): Promise<{ notificationId: number; provider: string }[]> => {logger.info("Triggering alerting", {monitor_id: monitorId,notification_type: notifType,});+ const triggered: { notificationId: number; provider: string }[] = [];+let incident: Incident | undefined;if (incidentId) {try {@@ -135,6 +137,10 @@ export const triggerNotifications = async ({logger.error("notification trigger already exists dont send again");continue;}+ triggered.push({+ notificationId: notif.notification.id,+ provider: notif.notification.provider,+ });switch (notifType) {case "alert":const alertResult = Effect.tryPromise({@@ -244,6 +250,8 @@ export const triggerNotifications = async ({},});}++ return triggered;};const insertNotificationTrigger = async ({diff --git a/apps/workflows/src/checker/index.ts b/apps/workflows/src/checker/index.tsindex d7ad49497..d87f640cd 100644--- a/apps/workflows/src/checker/index.ts+++ b/apps/workflows/src/checker/index.ts@@ -98,14 +98,6 @@ checkerRoute.post("/updateStatus", async (c) => {if (!result.success) {return c.text("Unprocessable Entity", 422);}- event.status_update = {- status: result.data.status,- message: result.data.message,- region: result.data.region,- status_code: result.data.statusCode,- cron_timestamp: result.data.cronTimestamp,- latency_ms: result.data.latency,- };const {monitorId,@@ -158,6 +150,17 @@ checkerRoute.post("/updateStatus", async (c) => {const affectedRegionsList = affectedRegions.map((r) => r.region);const affectedRegionCount = affectedRegionsList.length;+ event.status_update = {+ status: result.data.status,+ message: result.data.message,+ region: result.data.region,+ status_code: result.data.statusCode,+ cron_timestamp: result.data.cronTimestamp,+ latency_ms: result.data.latency,+ affectedRegionsCount: affectedRegionCount,+ monitorId: monitor.id,+ };+if (affectedRegionCount === 0) {return c.json({ success: true }, 200);}@@ -207,6 +210,9 @@ checkerRoute.post("/updateStatus", async (c) => {break;}+ let triggeredNotifications: { notificationId: number; provider: string }[] =+ [];+if (affectedRegionCount >= numberOfRegions / 2 || numberOfRegions === 1) {switch (status) {case "active": {@@ -228,7 +234,7 @@ checkerRoute.post("/updateStatus", async (c) => {incident = await resolveIncident({ monitorId, cronTimestamp });}- await triggerNotifications({+ triggeredNotifications = await triggerNotifications({monitorId,statusCode,message,@@ -264,7 +270,7 @@ checkerRoute.post("/updateStatus", async (c) => {});}- await triggerNotifications({+ triggeredNotifications = await triggerNotifications({monitorId,statusCode,message,@@ -320,7 +326,7 @@ checkerRoute.post("/updateStatus", async (c) => {metadata: { cronTimestamp, incidentId: newIncident.id },});- await triggerNotifications({+ triggeredNotifications = await triggerNotifications({monitorId,statusCode,message,@@ -341,5 +347,10 @@ checkerRoute.post("/updateStatus", async (c) => {}}+ (event.status_update as Record<string, unknown>).notificationTriggered =+ triggeredNotifications.length > 0;+ (event.status_update as Record<string, unknown>).notifications =+ triggeredNotifications;+return c.text("Ok", 200);});