Open-source status page, incident response and uptime monitoring
TypeScript alertingincident-managementmcpmcp-servermonitoringmonitoring-as-codeobservabilityon-callopen-sourcesrestatus-pagestatuspagesynthetic-monitoringterraformtinybirdtursouptimeuptime-checkeruptime-monitor
π improve notification (#1216)
Thibault Le Ouay89f744f568e0
diff --git a/apps/workflows/src/checker/index.ts b/apps/workflows/src/checker/index.tsindex c1f513d9a..4f81eb8b7 100644--- a/apps/workflows/src/checker/index.ts+++ b/apps/workflows/src/checker/index.ts@@ -96,43 +96,50 @@ checkerRoute.post("/updateStatus", async (c) => {if (affectedRegion.count >= numberOfRegions / 2 || numberOfRegions === 1) {switch (status) {case "active": {- if (monitor.status !== "active") {- await db- .update(schema.monitor)- .set({ status: "active" })- .where(eq(schema.monitor.id, monitor.id));- }-- const incident = await db- .select()- .from(incidentTable)- .where(- and(- eq(incidentTable.monitorId, Number(monitorId)),- isNull(incidentTable.resolvedAt),- isNull(incidentTable.acknowledgedAt),- ),- )- .get();-- if (!incident) {- // it was just a single failure not a proper incident- break;- }- if (incident?.resolvedAt) {- // incident is already resolved+ // it's been resolved+ if (monitor.status === "active") {break;}- console.log(`π€ recovering incident ${incident.id}`);+ console.log(`π update monitorStatus ${monitor.id} status: ACTIVE`);await db- .update(incidentTable)- .set({- resolvedAt: new Date(cronTimestamp),- autoResolved: true,- })- .where(eq(incidentTable.id, incident.id))- .run();+ .update(schema.monitor)+ .set({ status: "active" })+ .where(eq(schema.monitor.id, monitor.id));++ // we can't have a monitor in error without an incident+ if (monitor.status === "error") {+ const incident = await db+ .select()+ .from(incidentTable)+ .where(+ and(+ eq(incidentTable.monitorId, Number(monitorId)),+ isNull(incidentTable.resolvedAt),+ isNull(incidentTable.acknowledgedAt),+ ),+ )+ .get();++ if (!incident) {+ // it was just a single failure not a proper incident+ break;+ }+ if (incident?.resolvedAt) {+ // incident is already resolved+ break;+ }++ console.log(`π€ recovering incident ${incident.id}`);+ await db+ .update(incidentTable)+ .set({+ resolvedAt: new Date(cronTimestamp),+ autoResolved: true,+ })+ .where(eq(incidentTable.id, incident.id))+ .run();+ }await triggerNotifications({monitorId,@@ -155,26 +162,27 @@ checkerRoute.post("/updateStatus", async (c) => {break;}case "degraded":- if (monitor.status !== "degraded") {- console.log(- `π update monitorStatus ${monitor.id} status: DEGRADED}`,- );- await db- .update(schema.monitor)- .set({ status: "degraded" })- .where(eq(schema.monitor.id, monitor.id));- // figure how to send the notification once- await triggerNotifications({- monitorId,- statusCode,- message,- notifType: "degraded",- cronTimestamp,- latency,- region,- incidentId: `${cronTimestamp}`,- });+ if (monitor.status === "degraded") {+ // already degraded let's return early+ break;}+ console.log(`π update monitorStatus ${monitor.id} status: DEGRADED`);+ await db+ .update(schema.monitor)+ .set({ status: "degraded" })+ .where(eq(schema.monitor.id, monitor.id));+ // figure how to send the notification once+ await triggerNotifications({+ monitorId,+ statusCode,+ message,+ notifType: "degraded",+ cronTimestamp,+ latency,+ region,+ incidentId: `${cronTimestamp}`,+ });+await checkerAudit.publishAuditLog({id: `monitor:${monitorId}`,action: "monitor.degraded",@@ -183,12 +191,17 @@ checkerRoute.post("/updateStatus", async (c) => {});break;case "error":- if (monitor.status !== "error") {- await db- .update(schema.monitor)- .set({ status: "error" })- .where(eq(schema.monitor.id, monitor.id));+ if (monitor.status === "error") {+ // already in error let's return early+ break;}++ console.log(`π update monitorStatus ${monitor.id} status: ERROR`);+ await db+ .update(schema.monitor)+ .set({ status: "error" })+ .where(eq(schema.monitor.id, monitor.id));+try {const incident = await db.select()@@ -258,6 +271,7 @@ checkerRoute.post("/updateStatus", async (c) => {} catch {console.log("incident was already created");}+break;default:console.log("should not happen");