Open-source status page, incident response and uptime monitoring
TypeScript alertingincident-managementmcpmcp-servermonitoringmonitoring-as-codeobservabilityon-callopen-sourcesrestatus-pagestatuspagesynthetic-monitoringterraformtinybirdtursouptimeuptime-checkeruptime-monitor
feat: incident audit logs (#1255)
Maximilian Kaskebe3096a2df3f
diff --git a/apps/workflows/src/checker/alerting.ts b/apps/workflows/src/checker/alerting.tsindex e7eb4e708..75486a7b6 100644--- a/apps/workflows/src/checker/alerting.ts+++ b/apps/workflows/src/checker/alerting.ts@@ -102,7 +102,12 @@ export const triggerNotifications = async ({id: `monitor:${monitorId}`,action: "notification.sent",targets: [{ id: monitorId, type: "monitor" }],- metadata: { provider: notif.notification.provider },+ metadata: {+ provider: notif.notification.provider,+ cronTimestamp,+ type: notifType,+ notificationId: notif.notification.id,+ },});//}@@ -112,7 +117,11 @@ const insertNotificationTrigger = async ({monitorId,notificationId,cronTimestamp,-}: { monitorId: number; notificationId: number; cronTimestamp: number }) => {+}: {+ monitorId: number;+ notificationId: number;+ cronTimestamp: number;+}) => {await db.insert(schema.notificationTrigger).values({diff --git a/apps/workflows/src/checker/index.ts b/apps/workflows/src/checker/index.tsindex f958daab3..6a12ef94a 100644--- a/apps/workflows/src/checker/index.ts+++ b/apps/workflows/src/checker/index.ts@@ -4,7 +4,7 @@ import { Hono } from "hono";import { z } from "zod";import { and, count, db, eq, inArray, isNull, schema } from "@openstatus/db";-import { incidentTable, workspace } from "@openstatus/db/src/schema";+import { incidentTable } from "@openstatus/db/src/schema";import {monitorStatusSchema,selectMonitorSchema,@@ -94,6 +94,51 @@ checkerRoute.post("/updateStatus", async (c) => {return c.json({ success: true }, 200);}+ // audit log the current state of the ping++ switch (status) {+ case "active":+ await checkerAudit.publishAuditLog({+ id: `monitor:${monitorId}`,+ action: "monitor.recovered",+ targets: [{ id: monitorId, type: "monitor" }],+ metadata: {+ region,+ statusCode: statusCode ?? -1,+ cronTimestamp,+ latency,+ },+ });+ break;+ case "degraded":+ await checkerAudit.publishAuditLog({+ id: `monitor:${monitorId}`,+ action: "monitor.degraded",+ targets: [{ id: monitorId, type: "monitor" }],+ metadata: {+ region,+ statusCode: statusCode ?? -1,+ cronTimestamp,+ latency,+ },+ });+ break;+ case "error":+ await checkerAudit.publishAuditLog({+ id: `monitor:${monitorId}`,+ action: "monitor.failed",+ targets: [{ id: monitorId, type: "monitor" }],+ metadata: {+ region,+ statusCode: statusCode ?? -1,+ message,+ cronTimestamp,+ latency,+ },+ });+ break;+ }+if (affectedRegion.count >= numberOfRegions / 2 || numberOfRegions === 1) {switch (status) {case "active": {@@ -130,8 +175,8 @@ checkerRoute.post("/updateStatus", async (c) => {// incident is already resolvedbreak;}-console.log(`π€ recovering incident ${incident.id}`);+await db.update(incidentTable).set({@@ -140,6 +185,13 @@ checkerRoute.post("/updateStatus", async (c) => {}).where(eq(incidentTable.id, incident.id)).run();++ await checkerAudit.publishAuditLog({+ id: `monitor:${monitorId}`,+ action: "incident.resolved",+ targets: [{ id: monitorId, type: "monitor" }],+ metadata: { cronTimestamp, incidentId: incident.id },+ });}await triggerNotifications({@@ -153,13 +205,6 @@ checkerRoute.post("/updateStatus", async (c) => {incidentId: `${cronTimestamp}`,});- await checkerAudit.publishAuditLog({- id: `monitor:${monitorId}`,- action: "monitor.recovered",- targets: [{ id: monitorId, type: "monitor" }],- metadata: { region: region, statusCode: statusCode ?? -1 },- });-break;}case "degraded":@@ -168,6 +213,7 @@ checkerRoute.post("/updateStatus", async (c) => {break;}console.log(`π update monitorStatus ${monitor.id} status: DEGRADED`);+await db.update(schema.monitor).set({ status: "degraded" })@@ -184,12 +230,6 @@ checkerRoute.post("/updateStatus", async (c) => {incidentId: `${cronTimestamp}`,});- await checkerAudit.publishAuditLog({- id: `monitor:${monitorId}`,- action: "monitor.degraded",- targets: [{ id: monitorId, type: "monitor" }],- metadata: { region, statusCode: statusCode ?? -1 },- });break;case "error":if (monitor.status === "error") {@@ -198,6 +238,7 @@ checkerRoute.post("/updateStatus", async (c) => {}console.log(`π update monitorStatus ${monitor.id} status: ERROR`);+await db.update(schema.monitor).set({ status: "error" })@@ -219,7 +260,7 @@ checkerRoute.post("/updateStatus", async (c) => {console.log("we are already in incident");break;}- const newIncident = await db+ const [newIncident] = await db.insert(incidentTable).values({monitorId: Number(monitorId),@@ -228,9 +269,17 @@ checkerRoute.post("/updateStatus", async (c) => {}).returning();- if (!newIncident[0].id) {+ if (!newIncident.id) {return;}++ await checkerAudit.publishAuditLog({+ id: `monitor:${monitorId}`,+ action: "incident.created",+ targets: [{ id: monitorId, type: "monitor" }],+ metadata: { cronTimestamp, incidentId: newIncident.id },+ });+await triggerNotifications({monitorId,statusCode,@@ -239,19 +288,13 @@ checkerRoute.post("/updateStatus", async (c) => {cronTimestamp,latency,region,- incidentId: String(newIncident[0].id),+ incidentId: String(newIncident.id),});await db.update(schema.monitor).set({ status: "error" }).where(eq(schema.monitor.id, monitor.id));- await checkerAudit.publishAuditLog({- id: `monitor:${monitorId}`,- action: "monitor.failed",- targets: [{ id: monitorId, type: "monitor" }],- metadata: { region, statusCode, message },- });} catch {console.log("incident was already created");}diff --git a/packages/tinybird/src/audit-log/action-schema.ts b/packages/tinybird/src/audit-log/action-schema.tsindex 510e80912..f69d94f0b 100644--- a/packages/tinybird/src/audit-log/action-schema.ts+++ b/packages/tinybird/src/audit-log/action-schema.ts@@ -9,6 +9,7 @@ export const monitorRecoveredSchema = z.object({metadata: z.object({region: z.string(),statusCode: z.number(),+ latency: z.number().optional(),cronTimestamp: z.number().optional(),}),});@@ -23,6 +24,7 @@ export const monitorDegradedSchema = z.object({region: z.string(),statusCode: z.number(),cronTimestamp: z.number().optional(),+ latency: z.number().optional(),}),});@@ -36,6 +38,7 @@ export const monitorFailedSchema = z.object({region: z.string(),statusCode: z.number().optional(),message: z.string().optional(),+ latency: z.number().optional(),cronTimestamp: z.number().optional(),}),});@@ -46,20 +49,27 @@ export const monitorFailedSchema = z.object({*/export const notificationSentSchema = z.object({action: z.literal("notification.sent"),- // we could use the notificationProviderSchema for more type safety- metadata: z.object({ provider: z.string() }),+ metadata: z.object({+ // we could use the notificationProviderSchema for more type safety+ provider: z.string(),+ cronTimestamp: z.number().optional(),+ type: z.enum(["alert", "recovery", "degraded"]).optional(),+ notificationId: z.number().optional(),+ }),});-// TODO: update schemas with correct metadata and description-export const incidentCreatedSchema = z.object({action: z.literal("incident.created"),- metadata: z.object({}), // tbd+ metadata: z.object({+ cronTimestamp: z.number().optional(),+ incidentId: z.number().optional(),+ }),});export const incidentResolvedSchema = z.object({action: z.literal("incident.resolved"),- metadata: z.object({}), // tbd+ metadata: z.object({+ cronTimestamp: z.number().optional(),+ incidentId: z.number().optional(),+ }),});--// ...diff --git a/packages/tinybird/src/audit-log/action-validation.ts b/packages/tinybird/src/audit-log/action-validation.tsindex 03125aefe..4e0e85f9a 100644--- a/packages/tinybird/src/audit-log/action-validation.ts+++ b/packages/tinybird/src/audit-log/action-validation.ts@@ -1,6 +1,8 @@import { z } from "zod";import {+ incidentCreatedSchema,+ incidentResolvedSchema,monitorDegradedSchema,monitorFailedSchema,monitorRecoveredSchema,@@ -25,6 +27,8 @@ export const ingestActionEventSchema = zmonitorDegradedSchema,monitorFailedSchema,notificationSentSchema,+ incidentCreatedSchema,+ incidentResolvedSchema,]),ingestBaseEventSchema,)@@ -49,6 +53,12 @@ export const pipeActionResponseData = z.intersection(monitorRecoveredSchema.shape.metadata,),}),+ monitorDegradedSchema.extend({+ metadata: z.preprocess(+ (val) => JSON.parse(String(val)),+ monitorDegradedSchema.shape.metadata,+ ),+ }),monitorFailedSchema.extend({metadata: z.preprocess((val) => JSON.parse(String(val)),@@ -61,6 +71,18 @@ export const pipeActionResponseData = z.intersection(notificationSentSchema.shape.metadata,),}),+ incidentCreatedSchema.extend({+ metadata: z.preprocess(+ (val) => JSON.parse(String(val)),+ incidentCreatedSchema.shape.metadata,+ ),+ }),+ incidentResolvedSchema.extend({+ metadata: z.preprocess(+ (val) => JSON.parse(String(val)),+ incidentResolvedSchema.shape.metadata,+ ),+ }),]),pipeBaseResponseData,);