Skip to content

Open-source status page, incident response and uptime monitoring

15.2 KB
Raw Blame
import { getLogger } from "@logtape/logtape";
import {
  and,
  db,
  eq,
  gte,
  inArray,
  isNull,
  lte,
  schema,
  sql,
} from "@openstatus/db";
import { monitorStatusSchema } from "@openstatus/db/src/schema/monitors/validation";
import type { Context } from "hono";
import { z } from "zod";

import { env } from "../env";
import type { Env } from "../index";
import { checkerAudit } from "../utils/audit-log";
import { triggerNotifications } from "./alerting";
import { findOpenIncident, resolveIncident } from "./incident-utils";

const logger = getLogger(["workflow"]);

const payloadSchema = z.object({
  monitorId: z.string(),
  privateLocationId: z.string(),
  status: monitorStatusSchema,
  cronTimestamp: z.number(),
  message: z.string().optional(),
  statusCode: z.number().optional(),
  latency: z.number().optional(),
});

export async function updateStatusPrivate(c: Context<Env>) {
  const auth = c.req.header("Authorization");
  if (auth !== `Basic ${env().CRON_SECRET}`) {
    logger.error("Unauthorized");
    return c.text("Unauthorized", 401);
  }

  const result = payloadSchema.safeParse(await c.req.json());
  if (!result.success) {
    return c.text("Unprocessable Entity", 422);
  }

  const {
    monitorId,
    privateLocationId,
    status,
    cronTimestamp,
    message,
    statusCode,
    latency,
  } = result.data;

  const event = c.get("event");
  const monitorIdNumber = Number(monitorId);
  const privateLocationIdNumber = Number(privateLocationId);

  // Set event data for OTel/Axiom structured logging (matches cloud checker pattern)
  // Note: Private location pings are ingested to Tinybird separately via Go code in
  // apps/private-location/internal/tinybird/client.go, not through this event object
  if (event) {
    event.status_update = {
      status: status,
      message: message,
      region: privateLocationId, // Private location ID as region string
      status_code: statusCode,
      cron_timestamp: cronTimestamp,
      latency_ms: latency,
      monitorId: monitorIdNumber,
    };
  }

  try {
    const monitor = await db
      .select()
      .from(schema.monitor)
      .where(eq(schema.monitor.id, monitorIdNumber))
      .get();

    if (!monitor || monitor.deletedAt || !monitor.active) {
      return c.json({ success: true }, 200);
    }

    const now = new Date();
    const activeMaintenance = await db
      .select({ id: schema.maintenance.id })
      .from(schema.maintenance)
      .innerJoin(
        schema.maintenancesToPageComponents,
        eq(
          schema.maintenancesToPageComponents.maintenanceId,
          schema.maintenance.id,
        ),
      )
      .innerJoin(
        schema.pageComponent,
        eq(
          schema.pageComponent.id,
          schema.maintenancesToPageComponents.pageComponentId,
        ),
      )
      .where(
        and(
          lte(schema.maintenance.from, now),
          gte(schema.maintenance.to, now),
          eq(schema.pageComponent.monitorId, monitorIdNumber),
        ),
      )
      .get();

    if (activeMaintenance) {
      return c.json({ success: true }, 200);
    }

    const attachment = await db
      .select({ name: schema.privateLocation.name })
      .from(schema.privateLocationToMonitors)
      .innerJoin(
        schema.privateLocation,
        eq(
          schema.privateLocation.id,
          schema.privateLocationToMonitors.privateLocationId,
        ),
      )
      .where(
        and(
          eq(schema.privateLocationToMonitors.monitorId, monitorIdNumber),
          eq(
            schema.privateLocationToMonitors.privateLocationId,
            privateLocationIdNumber,
          ),
          isNull(schema.privateLocationToMonitors.deletedAt),
        ),
      )
      .get();

    if (!attachment) {
      return c.json({ success: true }, 200);
    }

    const priorRow = await db
      .select({ status: schema.privateLocationMonitorStatus.status })
      .from(schema.privateLocationMonitorStatus)
      .where(
        and(
          eq(schema.privateLocationMonitorStatus.monitorId, monitorIdNumber),
          eq(
            schema.privateLocationMonitorStatus.privateLocationId,
            privateLocationIdNumber,
          ),
        ),
      )
      .get();

    const priorStatus = priorRow?.status ?? "active";

    const upserted = await db
      .insert(schema.privateLocationMonitorStatus)
      .values({
        monitorId: monitorIdNumber,
        privateLocationId: privateLocationIdNumber,
        status,
        cronTimestamp,
      })
      .onConflictDoUpdate({
        target: [
          schema.privateLocationMonitorStatus.monitorId,
          schema.privateLocationMonitorStatus.privateLocationId,
        ],
        set: { status, cronTimestamp, updatedAt: new Date() },
        setWhere: sql`excluded.cron_timestamp > ${schema.privateLocationMonitorStatus.cronTimestamp}`,
      })
      .returning();

    if (upserted.length === 0 || status === priorStatus) {
      return c.json({ success: true }, 200);
    }

    const regions = [attachment.name];

    // Check if monitor has cloud regions
    const hasCloudRegions =
      monitor.regions && monitor.regions.trim().length > 0;

    // Query all private locations for threshold check
    const allLocations = await db
      .select({ id: schema.privateLocationToMonitors.privateLocationId })
      .from(schema.privateLocationToMonitors)
      .where(
        and(
          eq(schema.privateLocationToMonitors.monitorId, monitorIdNumber),
          isNull(schema.privateLocationToMonitors.deletedAt),
        ),
      )
      .all();

    const numberOfLocations = allLocations.length;
    const locationIds = allLocations
      .map((loc) => loc.id)
      .filter((id): id is number => id !== null);

    // Count how many locations report this status
    const locationsWithStatus = await db
      .select({
        privateLocationId:
          schema.privateLocationMonitorStatus.privateLocationId,
      })
      .from(schema.privateLocationMonitorStatus)
      .where(
        and(
          eq(schema.privateLocationMonitorStatus.monitorId, monitorIdNumber),
          eq(schema.privateLocationMonitorStatus.status, status),
          inArray(
            schema.privateLocationMonitorStatus.privateLocationId,
            locationIds,
          ),
        ),
      )
      .all();

    const affectedLocationCount = locationsWithStatus.length;

    // Apply ≥50% threshold (matching cloud checker logic)
    const shouldTriggerIncident =
      !hasCloudRegions &&
      (affectedLocationCount >= numberOfLocations / 2 ||
        numberOfLocations === 1);

    let incident = null;
    let triggeredNotifications: { notificationId: number; provider: string }[] =
      [];

    switch (status) {
      case "error":
        // Update monitor status for private-only monitors
        if (
          !hasCloudRegions &&
          shouldTriggerIncident &&
          monitor.status !== "error"
        ) {
          logger.info("Monitor status changed to error", {
            monitor_id: monitor.id,
            workspace_id: monitor.workspaceId,
          });
          await db
            .update(schema.monitor)
            .set({ status: "error" })
            .where(eq(schema.monitor.id, monitorIdNumber));
        }

        // Create incident only if private-only monitor AND threshold met
        if (shouldTriggerIncident) {
          try {
            const existingIncident = await findOpenIncident(monitorIdNumber);
            if (!existingIncident) {
              const [newIncident] = await db
                .insert(schema.monitorIncidentTable)
                .values({
                  monitorId: monitorIdNumber,
                  workspaceId: monitor.workspaceId,
                  startedAt: new Date(cronTimestamp),
                })
                .returning();

              if (newIncident?.id) {
                incident = newIncident;
                await checkerAudit.publishAuditLog({
                  id: `monitor:${monitorId}`,
                  action: "incident.created",
                  targets: [{ id: monitorId, type: "monitor" }],
                  metadata: { cronTimestamp, incidentId: newIncident.id },
                });
                logger.info("Created incident", {
                  incident_id: newIncident.id,
                  monitor_id: monitorId,
                  affected_location_count: affectedLocationCount,
                  total_locations: numberOfLocations,
                });
              }
            } else {
              incident = existingIncident;
              logger.info("Already in incident", {
                incident_id: existingIncident.id,
              });
            }
          } catch (error) {
            // Check if this is a constraint violation (race condition)
            const errorMessage =
              error instanceof Error ? error.message : String(error);
            if (
              errorMessage.includes("UNIQUE constraint") ||
              errorMessage.includes("unique")
            ) {
              // Another request created the incident concurrently, fetch it
              logger.info(
                "Concurrent incident creation detected, fetching existing",
                {
                  monitor_id: monitorId,
                },
              );
              const existingIncident = await findOpenIncident(monitorIdNumber);
              if (existingIncident) {
                incident = existingIncident;
              }
            } else {
              logger.error("Failed to create incident", {
                monitor_id: monitorId,
                error_message: errorMessage,
              });
            }
          }
        }

        await checkerAudit.publishAuditLog({
          id: `monitor:${monitorId}`,
          action: "monitor.failed",
          targets: [{ id: monitorId, type: "monitor" }],
          metadata: {
            region: privateLocationId,
            statusCode: statusCode ?? -1,
            message,
            cronTimestamp,
            latency,
          },
        });
        // Trigger notifications for cloud monitors always, private-only when threshold met AND status changed
        if (
          hasCloudRegions ||
          (shouldTriggerIncident && monitor.status !== "error")
        ) {
          triggeredNotifications = await triggerNotifications({
            monitorId,
            statusCode,
            message,
            notifType: "alert",
            cronTimestamp,
            regions,
            latency,
            incidentId: incident?.id,
          });
        }
        break;
      case "degraded":
        // Update monitor status for private-only monitors
        if (
          !hasCloudRegions &&
          shouldTriggerIncident &&
          monitor.status !== "degraded"
        ) {
          logger.info("Monitor status changed to degraded", {
            monitor_id: monitor.id,
            workspace_id: monitor.workspaceId,
          });
          await db
            .update(schema.monitor)
            .set({ status: "degraded" })
            .where(eq(schema.monitor.id, monitorIdNumber));
        }

        // Resolve incident if private-only monitor AND threshold met
        if (shouldTriggerIncident) {
          try {
            const incidents = await resolveIncident({
              monitorId,
              cronTimestamp,
            });
            incident = incidents[0] ?? null;
          } catch (error) {
            logger.warning(
              "Failed to resolve incident on degraded transition",
              {
                monitor_id: monitorId,
                error_message:
                  error instanceof Error ? error.message : String(error),
              },
            );
            // Continue with notifications even if resolution fails
          }
        }

        await checkerAudit.publishAuditLog({
          id: `monitor:${monitorId}`,
          action: "monitor.degraded",
          targets: [{ id: monitorId, type: "monitor" }],
          metadata: {
            region: privateLocationId,
            statusCode: statusCode ?? -1,
            cronTimestamp,
            latency,
          },
        });
        // Trigger notifications for cloud monitors always, private-only when threshold met AND status changed
        if (
          hasCloudRegions ||
          (shouldTriggerIncident && monitor.status !== "degraded")
        ) {
          triggeredNotifications = await triggerNotifications({
            monitorId,
            statusCode,
            message,
            notifType: "degraded",
            cronTimestamp,
            regions,
            latency,
            incidentId: incident?.id,
          });
        }
        break;
      case "active":
        // Update monitor status for private-only monitors
        if (
          !hasCloudRegions &&
          shouldTriggerIncident &&
          monitor.status !== "active"
        ) {
          logger.info("Monitor status changed to active", {
            monitor_id: monitor.id,
            workspace_id: monitor.workspaceId,
          });
          await db
            .update(schema.monitor)
            .set({ status: "active" })
            .where(eq(schema.monitor.id, monitorIdNumber));
        }

        // Resolve incident if private-only monitor AND threshold met
        if (shouldTriggerIncident) {
          try {
            const incidents = await resolveIncident({
              monitorId,
              cronTimestamp,
            });
            incident = incidents[0] ?? null;
          } catch (error) {
            logger.warning("Failed to resolve incident on active transition", {
              monitor_id: monitorId,
              error_message:
                error instanceof Error ? error.message : String(error),
            });
            // Continue with notifications even if resolution fails
          }
        }

        await checkerAudit.publishAuditLog({
          id: `monitor:${monitorId}`,
          action: "monitor.recovered",
          targets: [{ id: monitorId, type: "monitor" }],
          metadata: {
            region: privateLocationId,
            statusCode: statusCode ?? -1,
            cronTimestamp,
            latency,
          },
        });
        // Trigger notifications for cloud monitors always, private-only when threshold met AND status changed
        if (
          hasCloudRegions ||
          (shouldTriggerIncident && monitor.status !== "active")
        ) {
          triggeredNotifications = await triggerNotifications({
            monitorId,
            statusCode,
            message,
            notifType: "recovery",
            cronTimestamp,
            regions,
            latency,
            incidentId: incident?.id,
          });
        }
        break;
    }

    // Add notification outcome to event for OTel logging
    if (event?.status_update) {
      (event.status_update as Record<string, unknown>).notificationTriggered =
        triggeredNotifications.length > 0;
      (event.status_update as Record<string, unknown>).notifications =
        triggeredNotifications;
    }

    return c.json({ success: true }, 200);
  } catch (error) {
    logger.error("Failed to update private location status", {
      monitor_id: monitorId,
      private_location_id: privateLocationId,
      error_message: error instanceof Error ? error.message : String(error),
    });
    return c.text("Internal Server Error", 500);
  }
}