Skip to content

Open-source status page, incident response and uptime monitoring

fix: bug incident (#1772)

Thibault Le Ouay6d27a3d3f1ce
diff --git a/apps/workflows/src/checker/index.ts b/apps/workflows/src/checker/index.ts
index 681f29ec3..675f883ed 100644
--- a/apps/workflows/src/checker/index.ts
+++ b/apps/workflows/src/checker/index.ts
@@ -29,6 +29,60 @@ const payloadSchema = z.object({
const logger = getLogger(["workflow"]);
+/**
+ * Finds an open incident (not resolved and not acknowledged) for the given monitor.
+ */
+async function findOpenIncident(monitorId: number) {
+ return db
+ .select()
+ .from(incidentTable)
+ .where(
+ and(
+ eq(incidentTable.monitorId, monitorId),
+ isNull(incidentTable.resolvedAt),
+ ),
+ )
+ .get();
+}
+
+/**
+ * Resolves an open incident by setting resolvedAt and autoResolved flag.
+ */
+async function resolveIncident(params: {
+ monitorId: string;
+ cronTimestamp: number;
+}) {
+ const { monitorId, cronTimestamp } = params;
+ const incident = await findOpenIncident(Number(monitorId));
+
+ if (!incident || incident.resolvedAt) {
+ return null;
+ }
+
+ logger.info("Recovering incident", {
+ incident_id: incident.id,
+ monitor_id: monitorId,
+ });
+
+ await db
+ .update(incidentTable)
+ .set({
+ resolvedAt: new Date(cronTimestamp),
+ autoResolved: true,
+ })
+ .where(eq(incidentTable.id, incident.id))
+ .run();
+
+ await checkerAudit.publishAuditLog({
+ id: `monitor:${monitorId}`,
+ action: "incident.resolved",
+ targets: [{ id: monitorId, type: "monitor" }],
+ metadata: { cronTimestamp, incidentId: incident.id },
+ });
+
+ return incident;
+}
+
checkerRoute.post("/updateStatus", async (c) => {
const auth = c.req.header("Authorization");
if (auth !== `Basic ${env().CRON_SECRET}`) {
@@ -152,7 +206,6 @@ checkerRoute.post("/updateStatus", async (c) => {
if (affectedRegion.count >= numberOfRegions / 2 || numberOfRegions === 1) {
switch (status) {
case "active": {
- // it's been resolved
if (monitor.status === "active") {
break;
}
@@ -166,48 +219,8 @@ checkerRoute.post("/updateStatus", async (c) => {
.set({ status: "active" })
.where(eq(schema.monitor.id, monitor.id));
- // we can't have a monitor in error without an incident
if (monitor.status === "error") {
- const incident = await db
- .select()
- .from(incidentTable)
- .where(
- and(
- eq(incidentTable.monitorId, Number(monitorId)),
- isNull(incidentTable.resolvedAt),
- isNull(incidentTable.acknowledgedAt),
- ),
- )
- .get();
-
- if (!incident) {
- // it was just a single failure not a proper incident
- break;
- }
- if (incident?.resolvedAt) {
- // incident is already resolved
- break;
- }
- logger.info("Recovering incident", {
- incident_id: incident.id,
- monitor_id: monitorId,
- });
-
- await db
- .update(incidentTable)
- .set({
- resolvedAt: new Date(cronTimestamp),
- autoResolved: true,
- })
- .where(eq(incidentTable.id, incident.id))
- .run();
-
- await checkerAudit.publishAuditLog({
- id: `monitor:${monitorId}`,
- action: "incident.resolved",
- targets: [{ id: monitorId, type: "monitor" }],
- metadata: { cronTimestamp, incidentId: incident.id },
- });
+ await resolveIncident({ monitorId, cronTimestamp });
}
await triggerNotifications({
@@ -225,9 +238,9 @@ checkerRoute.post("/updateStatus", async (c) => {
}
case "degraded":
if (monitor.status === "degraded") {
- // already degraded let's return early
break;
}
+
logger.info("Monitor status changed to degraded", {
monitor_id: monitor.id,
workspace_id: monitor.workspaceId,
@@ -237,7 +250,7 @@ checkerRoute.post("/updateStatus", async (c) => {
.update(schema.monitor)
.set({ status: "degraded" })
.where(eq(schema.monitor.id, monitor.id));
- // figure how to send the notification once
+
await triggerNotifications({
monitorId,
statusCode,
@@ -249,10 +262,12 @@ checkerRoute.post("/updateStatus", async (c) => {
incidentId: `${cronTimestamp}`,
});
+ if (monitor.status === "error") {
+ await resolveIncident({ monitorId, cronTimestamp });
+ }
break;
case "error":
if (monitor.status === "error") {
- // already in error let's return early
break;
}
@@ -267,21 +282,14 @@ checkerRoute.post("/updateStatus", async (c) => {
.where(eq(schema.monitor.id, monitor.id));
try {
- const incident = await db
- .select()
- .from(incidentTable)
- .where(
- and(
- eq(incidentTable.monitorId, Number(monitorId)),
- isNull(incidentTable.resolvedAt),
- isNull(incidentTable.acknowledgedAt),
- ),
- )
- .get();
- if (incident) {
- logger.info("we are already in incident");
+ const existingIncident = await findOpenIncident(Number(monitorId));
+ if (existingIncident) {
+ logger.info("Already in incident", {
+ incident_id: existingIncident.id,
+ });
break;
}
+
const [newIncident] = await db
.insert(incidentTable)
.values({
@@ -291,8 +299,8 @@ checkerRoute.post("/updateStatus", async (c) => {
})
.returning();
- if (!newIncident.id) {
- return;
+ if (!newIncident?.id) {
+ break;
}
await checkerAudit.publishAuditLog({
@@ -312,13 +320,8 @@ checkerRoute.post("/updateStatus", async (c) => {
region,
incidentId: String(newIncident.id),
});
-
- await db
- .update(schema.monitor)
- .set({ status: "error" })
- .where(eq(schema.monitor.id, monitor.id));
- } catch {
- logger.warning("incident was already created");
+ } catch (error) {
+ logger.warning("Failed to create incident", { error });
}
break;
@@ -328,6 +331,5 @@ checkerRoute.post("/updateStatus", async (c) => {
}
}
- // if we are in error
return c.text("Ok", 200);
});
diff --git a/apps/workflows/src/incident/index.ts b/apps/workflows/src/incident/index.ts
new file mode 100644
index 000000000..9a340a4d1
--- /dev/null
+++ b/apps/workflows/src/incident/index.ts
@@ -0,0 +1,63 @@
+import { schema } from "@openstatus/db";
+import { and, eq, inArray, isNotNull, isNull, ne } from "drizzle-orm";
+import { Hono } from "hono";
+import { env } from "../env";
+import { db } from "../lib/db";
+
+export const incidentRoute = new Hono({ strict: false });
+
+incidentRoute.use("*", async (c, next) => {
+ if (c.req.header("authorization") !== env().CRON_SECRET) {
+ return c.text("Unauthorized", 401);
+ }
+
+ return next();
+});
+
+incidentRoute.get("/cleanup", async (c) => {
+ // Find monitors that have unresolved incidents but are active
+ const unresolvedIncidentMonitorIds = db
+ .select({ monitorId: schema.incidentTable.monitorId })
+ .from(schema.incidentTable)
+ .where(
+ and(
+ isNull(schema.incidentTable.resolvedAt),
+ isNotNull(schema.incidentTable.acknowledgedAt),
+ ),
+ );
+
+ const activeMonitorsWithUnresolvedIncidents = await db
+ .select({ id: schema.monitor.id })
+ .from(schema.monitor)
+ .where(
+ and(
+ inArray(schema.monitor.id, unresolvedIncidentMonitorIds),
+ eq(schema.monitor.active, true),
+ ne(schema.monitor.status, "error"),
+ ),
+ )
+ .all();
+
+ const monitorIds = activeMonitorsWithUnresolvedIncidents.map((m) => m.id);
+
+ if (monitorIds.length === 0) {
+ return c.json({ status: "ok", updated: 0 });
+ }
+
+ // Update incidents for these monitors: set resolvedAt to now and autoResolved to true
+ const result = await db
+ .update(schema.incidentTable)
+ .set({
+ resolvedAt: new Date(),
+ autoResolved: true,
+ })
+ .where(
+ and(
+ inArray(schema.incidentTable.monitorId, monitorIds),
+ isNull(schema.incidentTable.resolvedAt),
+ ),
+ )
+ .returning({ id: schema.incidentTable.id });
+
+ return c.json({ status: "ok", updated: result.length });
+});
diff --git a/apps/workflows/src/index.ts b/apps/workflows/src/index.ts
index 5954b017d..a58d242ff 100644
--- a/apps/workflows/src/index.ts
+++ b/apps/workflows/src/index.ts
@@ -21,6 +21,7 @@ import { env } from "./env";
import { resourceFromAttributes } from "@opentelemetry/resources";
import { ATTR_DEPLOYMENT_ENVIRONMENT_NAME } from "@opentelemetry/semantic-conventions/incubating";
+import { incidentRoute } from "./incident";
const { NODE_ENV, PORT } = env();
@@ -190,6 +191,7 @@ app.route("/cron", cronRouter);
app.route("/", checkerRoute);
+app.route("/incident", incidentRoute);
if (NODE_ENV === "development") {
showRoutes(app, { verbose: true, colorize: true });
}