Skip to content

Open-source status page, incident response and uptime monitoring

feat: incident audit logs (#1255)

Maximilian Kaskebe3096a2df3f
diff --git a/apps/workflows/src/checker/alerting.ts b/apps/workflows/src/checker/alerting.ts
index e7eb4e708..75486a7b6 100644
--- a/apps/workflows/src/checker/alerting.ts
+++ b/apps/workflows/src/checker/alerting.ts
@@ -102,7 +102,12 @@ export const triggerNotifications = async ({
id: `monitor:${monitorId}`,
action: "notification.sent",
targets: [{ id: monitorId, type: "monitor" }],
- metadata: { provider: notif.notification.provider },
+ metadata: {
+ provider: notif.notification.provider,
+ cronTimestamp,
+ type: notifType,
+ notificationId: notif.notification.id,
+ },
});
//
}
@@ -112,7 +117,11 @@ const insertNotificationTrigger = async ({
monitorId,
notificationId,
cronTimestamp,
-}: { monitorId: number; notificationId: number; cronTimestamp: number }) => {
+}: {
+ monitorId: number;
+ notificationId: number;
+ cronTimestamp: number;
+}) => {
await db
.insert(schema.notificationTrigger)
.values({
diff --git a/apps/workflows/src/checker/index.ts b/apps/workflows/src/checker/index.ts
index f958daab3..6a12ef94a 100644
--- a/apps/workflows/src/checker/index.ts
+++ b/apps/workflows/src/checker/index.ts
@@ -4,7 +4,7 @@ import { Hono } from "hono";
import { z } from "zod";
import { and, count, db, eq, inArray, isNull, schema } from "@openstatus/db";
-import { incidentTable, workspace } from "@openstatus/db/src/schema";
+import { incidentTable } from "@openstatus/db/src/schema";
import {
monitorStatusSchema,
selectMonitorSchema,
@@ -94,6 +94,51 @@ checkerRoute.post("/updateStatus", async (c) => {
return c.json({ success: true }, 200);
}
+ // audit log the current state of the ping
+
+ switch (status) {
+ case "active":
+ await checkerAudit.publishAuditLog({
+ id: `monitor:${monitorId}`,
+ action: "monitor.recovered",
+ targets: [{ id: monitorId, type: "monitor" }],
+ metadata: {
+ region,
+ statusCode: statusCode ?? -1,
+ cronTimestamp,
+ latency,
+ },
+ });
+ break;
+ case "degraded":
+ await checkerAudit.publishAuditLog({
+ id: `monitor:${monitorId}`,
+ action: "monitor.degraded",
+ targets: [{ id: monitorId, type: "monitor" }],
+ metadata: {
+ region,
+ statusCode: statusCode ?? -1,
+ cronTimestamp,
+ latency,
+ },
+ });
+ break;
+ case "error":
+ await checkerAudit.publishAuditLog({
+ id: `monitor:${monitorId}`,
+ action: "monitor.failed",
+ targets: [{ id: monitorId, type: "monitor" }],
+ metadata: {
+ region,
+ statusCode: statusCode ?? -1,
+ message,
+ cronTimestamp,
+ latency,
+ },
+ });
+ break;
+ }
+
if (affectedRegion.count >= numberOfRegions / 2 || numberOfRegions === 1) {
switch (status) {
case "active": {
@@ -130,8 +175,8 @@ checkerRoute.post("/updateStatus", async (c) => {
// incident is already resolved
break;
}
-
console.log(`πŸ€“ recovering incident ${incident.id}`);
+
await db
.update(incidentTable)
.set({
@@ -140,6 +185,13 @@ checkerRoute.post("/updateStatus", async (c) => {
})
.where(eq(incidentTable.id, incident.id))
.run();
+
+ await checkerAudit.publishAuditLog({
+ id: `monitor:${monitorId}`,
+ action: "incident.resolved",
+ targets: [{ id: monitorId, type: "monitor" }],
+ metadata: { cronTimestamp, incidentId: incident.id },
+ });
}
await triggerNotifications({
@@ -153,13 +205,6 @@ checkerRoute.post("/updateStatus", async (c) => {
incidentId: `${cronTimestamp}`,
});
- await checkerAudit.publishAuditLog({
- id: `monitor:${monitorId}`,
- action: "monitor.recovered",
- targets: [{ id: monitorId, type: "monitor" }],
- metadata: { region: region, statusCode: statusCode ?? -1 },
- });
-
break;
}
case "degraded":
@@ -168,6 +213,7 @@ checkerRoute.post("/updateStatus", async (c) => {
break;
}
console.log(`πŸ”„ update monitorStatus ${monitor.id} status: DEGRADED`);
+
await db
.update(schema.monitor)
.set({ status: "degraded" })
@@ -184,12 +230,6 @@ checkerRoute.post("/updateStatus", async (c) => {
incidentId: `${cronTimestamp}`,
});
- await checkerAudit.publishAuditLog({
- id: `monitor:${monitorId}`,
- action: "monitor.degraded",
- targets: [{ id: monitorId, type: "monitor" }],
- metadata: { region, statusCode: statusCode ?? -1 },
- });
break;
case "error":
if (monitor.status === "error") {
@@ -198,6 +238,7 @@ checkerRoute.post("/updateStatus", async (c) => {
}
console.log(`πŸ”„ update monitorStatus ${monitor.id} status: ERROR`);
+
await db
.update(schema.monitor)
.set({ status: "error" })
@@ -219,7 +260,7 @@ checkerRoute.post("/updateStatus", async (c) => {
console.log("we are already in incident");
break;
}
- const newIncident = await db
+ const [newIncident] = await db
.insert(incidentTable)
.values({
monitorId: Number(monitorId),
@@ -228,9 +269,17 @@ checkerRoute.post("/updateStatus", async (c) => {
})
.returning();
- if (!newIncident[0].id) {
+ if (!newIncident.id) {
return;
}
+
+ await checkerAudit.publishAuditLog({
+ id: `monitor:${monitorId}`,
+ action: "incident.created",
+ targets: [{ id: monitorId, type: "monitor" }],
+ metadata: { cronTimestamp, incidentId: newIncident.id },
+ });
+
await triggerNotifications({
monitorId,
statusCode,
@@ -239,19 +288,13 @@ checkerRoute.post("/updateStatus", async (c) => {
cronTimestamp,
latency,
region,
- incidentId: String(newIncident[0].id),
+ incidentId: String(newIncident.id),
});
await db
.update(schema.monitor)
.set({ status: "error" })
.where(eq(schema.monitor.id, monitor.id));
- await checkerAudit.publishAuditLog({
- id: `monitor:${monitorId}`,
- action: "monitor.failed",
- targets: [{ id: monitorId, type: "monitor" }],
- metadata: { region, statusCode, message },
- });
} catch {
console.log("incident was already created");
}
diff --git a/packages/tinybird/src/audit-log/action-schema.ts b/packages/tinybird/src/audit-log/action-schema.ts
index 510e80912..f69d94f0b 100644
--- a/packages/tinybird/src/audit-log/action-schema.ts
+++ b/packages/tinybird/src/audit-log/action-schema.ts
@@ -9,6 +9,7 @@ export const monitorRecoveredSchema = z.object({
metadata: z.object({
region: z.string(),
statusCode: z.number(),
+ latency: z.number().optional(),
cronTimestamp: z.number().optional(),
}),
});
@@ -23,6 +24,7 @@ export const monitorDegradedSchema = z.object({
region: z.string(),
statusCode: z.number(),
cronTimestamp: z.number().optional(),
+ latency: z.number().optional(),
}),
});
@@ -36,6 +38,7 @@ export const monitorFailedSchema = z.object({
region: z.string(),
statusCode: z.number().optional(),
message: z.string().optional(),
+ latency: z.number().optional(),
cronTimestamp: z.number().optional(),
}),
});
@@ -46,20 +49,27 @@ export const monitorFailedSchema = z.object({
*/
export const notificationSentSchema = z.object({
action: z.literal("notification.sent"),
- // we could use the notificationProviderSchema for more type safety
- metadata: z.object({ provider: z.string() }),
+ metadata: z.object({
+ // we could use the notificationProviderSchema for more type safety
+ provider: z.string(),
+ cronTimestamp: z.number().optional(),
+ type: z.enum(["alert", "recovery", "degraded"]).optional(),
+ notificationId: z.number().optional(),
+ }),
});
-// TODO: update schemas with correct metadata and description
-
export const incidentCreatedSchema = z.object({
action: z.literal("incident.created"),
- metadata: z.object({}), // tbd
+ metadata: z.object({
+ cronTimestamp: z.number().optional(),
+ incidentId: z.number().optional(),
+ }),
});
export const incidentResolvedSchema = z.object({
action: z.literal("incident.resolved"),
- metadata: z.object({}), // tbd
+ metadata: z.object({
+ cronTimestamp: z.number().optional(),
+ incidentId: z.number().optional(),
+ }),
});
-
-// ...
diff --git a/packages/tinybird/src/audit-log/action-validation.ts b/packages/tinybird/src/audit-log/action-validation.ts
index 03125aefe..4e0e85f9a 100644
--- a/packages/tinybird/src/audit-log/action-validation.ts
+++ b/packages/tinybird/src/audit-log/action-validation.ts
@@ -1,6 +1,8 @@
import { z } from "zod";
import {
+ incidentCreatedSchema,
+ incidentResolvedSchema,
monitorDegradedSchema,
monitorFailedSchema,
monitorRecoveredSchema,
@@ -25,6 +27,8 @@ export const ingestActionEventSchema = z
monitorDegradedSchema,
monitorFailedSchema,
notificationSentSchema,
+ incidentCreatedSchema,
+ incidentResolvedSchema,
]),
ingestBaseEventSchema,
)
@@ -49,6 +53,12 @@ export const pipeActionResponseData = z.intersection(
monitorRecoveredSchema.shape.metadata,
),
}),
+ monitorDegradedSchema.extend({
+ metadata: z.preprocess(
+ (val) => JSON.parse(String(val)),
+ monitorDegradedSchema.shape.metadata,
+ ),
+ }),
monitorFailedSchema.extend({
metadata: z.preprocess(
(val) => JSON.parse(String(val)),
@@ -61,6 +71,18 @@ export const pipeActionResponseData = z.intersection(
notificationSentSchema.shape.metadata,
),
}),
+ incidentCreatedSchema.extend({
+ metadata: z.preprocess(
+ (val) => JSON.parse(String(val)),
+ incidentCreatedSchema.shape.metadata,
+ ),
+ }),
+ incidentResolvedSchema.extend({
+ metadata: z.preprocess(
+ (val) => JSON.parse(String(val)),
+ incidentResolvedSchema.shape.metadata,
+ ),
+ }),
]),
pipeBaseResponseData,
);