Skip to content

Open-source status page, incident response and uptime monitoring

2.9 KB
Raw Blame
import { getLogger } from "@logtape/logtape";
import { and, db, eq, inArray, isNull, schema } from "@openstatus/db";

import { checkerAudit } from "../utils/audit-log";

const logger = getLogger(["workflow"]);

/**
 * Finds an open incident (not resolved) for the given monitor.
 */
export async function findOpenIncident(monitorId: number) {
  return db
    .select()
    .from(schema.monitorIncidentTable)
    .where(
      and(
        eq(schema.monitorIncidentTable.monitorId, monitorId),
        isNull(schema.monitorIncidentTable.resolvedAt),
      ),
    )
    .get();
}

/**
 * Finds all open incidents (not resolved) for the given monitor.
 */
export async function findAllOpenIncidents(monitorId: number) {
  return db
    .select()
    .from(schema.monitorIncidentTable)
    .where(
      and(
        eq(schema.monitorIncidentTable.monitorId, monitorId),
        isNull(schema.monitorIncidentTable.resolvedAt),
      ),
    )
    .all();
}

/**
 * Resolves all open incidents by setting resolvedAt and autoResolved flag.
 * Uses a single atomic bulk update to prevent partial state on failures.
 * Returns array of successfully resolved incidents.
 */
export async function resolveIncident(params: {
  monitorId: string;
  cronTimestamp: number;
}): Promise<(typeof schema.monitorIncidentTable.$inferSelect)[]> {
  const { monitorId, cronTimestamp } = params;

  // Find ALL open incidents for this monitor
  const incidents = await findAllOpenIncidents(Number(monitorId));

  if (incidents.length === 0) {
    return []; // No open incidents
  }

  // Extract all incident IDs for bulk update
  const incidentIds = incidents.map((i) => i.id);

  // ATOMIC BULK UPDATE: Resolve all incidents in a single query
  // This prevents partial state if operation fails midway
  const resolvedIncidents = await db
    .update(schema.monitorIncidentTable)
    .set({
      resolvedAt: new Date(cronTimestamp),
      autoResolved: true,
    })
    .where(
      and(
        inArray(schema.monitorIncidentTable.id, incidentIds),
        isNull(schema.monitorIncidentTable.resolvedAt), // Still prevents race conditions
      ),
    )
    .returning();

  // Emit audit logs for each resolved incident
  // These are best-effort; failure here doesn't affect data integrity
  for (const incident of resolvedIncidents) {
    logger.info("Recovered incident", {
      incident_id: incident.id,
      monitor_id: monitorId,
    });

    try {
      await checkerAudit.publishAuditLog({
        id: `monitor:${monitorId}`,
        action: "incident.resolved",
        targets: [{ id: monitorId, type: "monitor" }],
        metadata: { cronTimestamp, incidentId: incident.id },
      });
    } catch (error) {
      logger.error("Failed to publish audit log for incident resolution", {
        incident_id: incident.id,
        monitor_id: monitorId,
        error,
      });
      // Don't throw - incident is already resolved
    }
  }

  return resolvedIncidents;
}