Skip to content

Open-source status page, incident response and uptime monitoring

21.7 KB
Raw Blame
import { getLogger } from "@logtape/logtape";
import { db } from "@openstatus/db";
import { listExternalServices } from "@openstatus/services/external-service";
import type { ExternalServiceRow } from "@openstatus/services/external-service";
import { applyDetectedProvider } from "@openstatus/services/external-service";
import {
  type UpsertExternalComponentInput,
  upsertExternalComponentsForService,
} from "@openstatus/services/external-service-component";
import {
  type UpsertExternalIncidentInput,
  upsertExternalIncidentsForService,
} from "@openstatus/services/external-service-incident";
import {
  FetchError,
  detectProvider,
  fetchers,
} from "@openstatus/status-fetcher";
import type {
  NormalizedComponent,
  NormalizedIncident,
  StatusFetcher,
  StatusPageEntry,
  StatusResult,
} from "@openstatus/status-fetcher";
import { OSTinybird } from "@openstatus/tinybird";
import { Effect } from "effect";
import type { Context } from "hono";

import { env } from "../env";
import {
  reportBackgroundError,
  reportDetectionStory,
  reportDetectionWriteFailure,
  reportFetchFailure,
  runSentryCron,
} from "../lib/sentry";
import {
  clearProbeStamp,
  decideDetectionAction,
  isBlocked,
  isSuspicious,
  shouldProbe,
} from "./external-status-detect";

const logger = getLogger(["workflow", "external-status"]);

const tb = new OSTinybird({
  token: env().TINY_BIRD_API_KEY,
  baseUrl: env().TINYBIRD_URL,
  noop: env().TINYBIRD_NOOP,
});

// 10 per phase × 3 phases = peak 30 concurrent HTTP requests upstream; keeps
// Atlassian/Incident.io CDNs comfortable while still parallelising heavily.
const PHASE_CONCURRENCY = 10;

function toStatusPageEntry(row: ExternalServiceRow): StatusPageEntry {
  return {
    id: row.slug,
    name: row.name,
    url: row.url,
    status_page_url: row.statusPageUrl,
    provider: row.provider,
    industry: row.industry,
    description: row.description ?? undefined,
    api_config: row.apiConfig ?? undefined,
  };
}

type Snapshot = {
  id: string;
  indicator: string;
  status: string;
  status_message: string;
  fetched_at: number;
  updated_at: number;
  time_zone: string;
};

function buildSnapshot(args: {
  entry: StatusPageEntry;
  result: StatusResult;
  fetchedAt: number;
}): Snapshot {
  const { entry, result, fetchedAt } = args;
  return {
    id: entry.id,
    indicator: result.severity,
    status: result.status,
    status_message: result.description,
    fetched_at: fetchedAt,
    updated_at: result.updated_at,
    time_zone: result.timezone ?? "",
  };
}

function toUpsertInput(
  incident: NormalizedIncident,
): UpsertExternalIncidentInput {
  return {
    providerIncidentId: incident.providerIncidentId,
    name: incident.name,
    status: incident.status,
    impact: incident.impact,
    shortlink: incident.shortlink,
    startedAt: incident.startedAt,
    createdAt: incident.createdAt,
    resolvedAt: incident.resolvedAt,
    affectedComponentIds: incident.affectedComponentIds,
    raw: incident.raw,
  };
}

type ComponentSnapshot = {
  component_id: string;
  external_service_id: number;
  indicator: string;
  status: string;
  fetched_at: number;
};

function toComponentUpsertInput(
  component: NormalizedComponent,
): UpsertExternalComponentInput {
  return {
    upstreamComponentId: component.upstreamComponentId,
    name: component.name,
    description: component.description,
    groupName: component.groupName,
    position: component.position,
    indicator: component.severity,
    status: component.status,
  };
}

type PhaseCounts = {
  successCount: number;
  failureCount: number;
  skippedCount: number;
  total: number;
};

type StatusPhaseOutcome =
  | { kind: "ok"; snapshot: Snapshot }
  | { kind: "no-fetcher"; slug: string }
  | { kind: "fail"; slug: string; reason: string; error: FetchError };

type IncidentPhaseOutcome =
  | { kind: "ok"; slug: string; count: number }
  | { kind: "skip"; slug: string }
  | { kind: "fail"; slug: string; reason: string };

type ComponentPhaseOutcome =
  | { kind: "ok"; slug: string; snapshots: ComponentSnapshot[] }
  | { kind: "skip"; slug: string }
  | { kind: "fail"; slug: string; reason: string };

type Triplet = {
  row: ExternalServiceRow;
  entry: StatusPageEntry;
  fetcher: StatusFetcher | null;
};

function runStatusPhase(
  triplets: Triplet[],
  fetchedAt: number,
): Effect.Effect<StatusPhaseOutcome[]> {
  return Effect.forEach(
    triplets,
    ({ entry, fetcher }) => {
      if (!fetcher) {
        return Effect.succeed<StatusPhaseOutcome>({
          kind: "no-fetcher",
          slug: entry.id,
        });
      }
      return fetcher.fetch(entry).pipe(
        Effect.map((result): StatusPhaseOutcome => ({
          kind: "ok",
          snapshot: buildSnapshot({ entry, result, fetchedAt }),
        })),
        // Failure reporting is deferred: the detect step after this phase
        // either merges it into a detection story or reports it plain.
        Effect.catch((err: FetchError) =>
          Effect.succeed<StatusPhaseOutcome>({
            kind: "fail",
            slug: entry.id,
            reason: err.message,
            error: err,
          }),
        ),
      );
    },
    { concurrency: PHASE_CONCURRENCY },
  );
}

function runIncidentPhase(
  triplets: Triplet[],
  tickStartedAt: Date,
): Effect.Effect<IncidentPhaseOutcome[]> {
  return Effect.forEach(
    triplets,
    ({ row, entry, fetcher }) => {
      if (!fetcher || !fetcher.fetchIncidents) {
        return Effect.succeed<IncidentPhaseOutcome>({
          kind: "skip",
          slug: entry.id,
        });
      }
      return fetcher.fetchIncidents(entry).pipe(
        Effect.flatMap((incidents) =>
          Effect.tryPromise({
            try: () =>
              upsertExternalIncidentsForService({
                ctx: { db },
                externalServiceId: row.id,
                incidents: incidents.map(toUpsertInput),
                now: tickStartedAt,
              }),
            catch: (e) =>
              new FetchError({
                url: entry.status_page_url,
                fetcherName: fetcher.name,
                entryId: entry.id,
                cause: e instanceof Error ? e : new Error(String(e)),
              }),
          }).pipe(
            Effect.map((result): IncidentPhaseOutcome => ({
              kind: "ok",
              slug: entry.id,
              count: result.upserted,
            })),
          ),
        ),
        Effect.catch((err: FetchError) =>
          Effect.sync<IncidentPhaseOutcome>(() => {
            reportFetchFailure({
              phase: "incidents",
              slug: entry.id,
              error: err,
              level: isBlocked(err) ? "warning" : undefined,
            });
            return { kind: "fail", slug: entry.id, reason: err.message };
          }),
        ),
      );
    },
    { concurrency: PHASE_CONCURRENCY },
  );
}

function runComponentPhase(
  triplets: Triplet[],
  tickStartedAt: Date,
  fetchedAt: number,
): Effect.Effect<ComponentPhaseOutcome[]> {
  return Effect.forEach(
    triplets,
    ({ row, entry, fetcher }) => {
      if (!fetcher || !fetcher.fetchComponents) {
        return Effect.succeed<ComponentPhaseOutcome>({
          kind: "skip",
          slug: entry.id,
        });
      }
      return fetcher.fetchComponents(entry).pipe(
        Effect.flatMap((components) =>
          Effect.tryPromise({
            try: () =>
              upsertExternalComponentsForService({
                ctx: { db },
                externalServiceId: row.id,
                components: components.map(toComponentUpsertInput),
                now: tickStartedAt,
              }),
            catch: (e) =>
              new FetchError({
                url: entry.status_page_url,
                fetcherName: fetcher.name,
                entryId: entry.id,
                cause: e instanceof Error ? e : new Error(String(e)),
              }),
          }).pipe(
            Effect.map((result): ComponentPhaseOutcome => {
              // History rows key on our PK, so the upstream→PK map from the
              // upsert turns each normalized component into a snapshot.
              const byUpstream = new Map(
                components.map((c) => [c.upstreamComponentId, c]),
              );
              const snapshots: ComponentSnapshot[] = [];
              for (const upserted of result.upserted) {
                const c = byUpstream.get(upserted.upstreamComponentId);
                if (!c) continue;
                snapshots.push({
                  component_id: String(upserted.id),
                  external_service_id: row.id,
                  indicator: c.severity,
                  status: c.status,
                  fetched_at: fetchedAt,
                });
              }
              return { kind: "ok", slug: entry.id, snapshots };
            }),
          ),
        ),
        Effect.catch((err: FetchError) =>
          Effect.sync<ComponentPhaseOutcome>(() => {
            reportFetchFailure({
              phase: "components",
              slug: entry.id,
              error: err,
              level: isBlocked(err) ? "warning" : undefined,
            });
            return { kind: "fail", slug: entry.id, reason: err.message };
          }),
        ),
      );
    },
    { concurrency: PHASE_CONCURRENCY },
  );
}

function summarizeStatus(outcomes: StatusPhaseOutcome[]): {
  counts: PhaseCounts;
  snapshots: Snapshot[];
} {
  const snapshots: Snapshot[] = [];
  let successCount = 0;
  let failureCount = 0;
  let skippedCount = 0;
  for (const o of outcomes) {
    if (o.kind === "ok") {
      snapshots.push(o.snapshot);
      successCount++;
    } else if (o.kind === "no-fetcher") {
      skippedCount++;
      logger.warn("external-status status: no fetcher matches slug={slug}", {
        slug: o.slug,
      });
    } else {
      failureCount++;
      logger.warn(
        "external-status status: fetch failed for slug={slug}: {reason}",
        {
          slug: o.slug,
          reason: o.reason,
        },
      );
    }
  }
  return {
    counts: {
      successCount,
      failureCount,
      skippedCount,
      total: outcomes.length,
    },
    snapshots,
  };
}

function summarizeIncidents(outcomes: IncidentPhaseOutcome[]): PhaseCounts {
  let successCount = 0;
  let failureCount = 0;
  let skippedCount = 0;
  for (const o of outcomes) {
    if (o.kind === "ok") {
      successCount++;
    } else if (o.kind === "skip") {
      skippedCount++;
    } else {
      failureCount++;
      logger.warn(
        "external-status incidents: failed for slug={slug}: {reason}",
        {
          slug: o.slug,
          reason: o.reason,
        },
      );
    }
  }
  return {
    successCount,
    failureCount,
    skippedCount,
    total: outcomes.length,
  };
}

function summarizeComponents(outcomes: ComponentPhaseOutcome[]): {
  counts: PhaseCounts;
  snapshots: ComponentSnapshot[];
} {
  const snapshots: ComponentSnapshot[] = [];
  let successCount = 0;
  let failureCount = 0;
  let skippedCount = 0;
  for (const o of outcomes) {
    if (o.kind === "ok") {
      successCount++;
      snapshots.push(...o.snapshots);
    } else if (o.kind === "skip") {
      skippedCount++;
    } else {
      failureCount++;
      logger.warn(
        "external-status components: failed for slug={slug}: {reason}",
        {
          slug: o.slug,
          reason: o.reason,
        },
      );
    }
  }
  return {
    counts: {
      successCount,
      failureCount,
      skippedCount,
      total: outcomes.length,
    },
    snapshots,
  };
}

function buildTriplets(services: ExternalServiceRow[]): Triplet[] {
  return services.map((row) => {
    const entry = toStatusPageEntry(row);
    const fetcher = fetchers.find((f) => f.canHandle(entry)) ?? null;
    return { row, entry, fetcher };
  });
}

const DETECT_CONCURRENCY = 3;

type DetectItem = { triplet: Triplet; error?: FetchError };

type DetectOutcome = {
  kind:
    | "applied"
    | "config-fixed"
    | "suggested"
    | "transient"
    | "none"
    | "skipped"
    | "error";
  slug: string;
};

type DetectCounts = {
  probed: number;
  applied: number;
  configFixed: number;
  suggested: number;
  failed: number;
};

function collectDetectItems(
  triplets: Triplet[],
  statusOutcomes: StatusPhaseOutcome[],
  now: number,
): DetectItem[] {
  const items: DetectItem[] = [];
  statusOutcomes.forEach((outcome, i) => {
    const triplet = triplets[i];
    if (!triplet) return;
    if (outcome.kind === "no-fetcher") {
      if (shouldProbe(triplet.entry.id, now)) items.push({ triplet });
      return;
    }
    if (outcome.kind !== "fail") return;
    if (isSuspicious(outcome.error) && shouldProbe(triplet.entry.id, now)) {
      items.push({ triplet, error: outcome.error });
    } else {
      reportFetchFailure({
        phase: "status",
        slug: outcome.slug,
        error: outcome.error,
        level: isBlocked(outcome.error) ? "warning" : undefined,
      });
    }
  });
  return items;
}

function applyDetection(args: {
  triplet: Triplet;
  error?: FetchError;
  provider: ExternalServiceRow["provider"];
  outcome: "applied" | "config-fixed";
  evidence: string[];
  tickStartedAt: Date;
}): Effect.Effect<DetectOutcome> {
  const { triplet, error, provider, outcome, evidence, tickStartedAt } = args;
  const { row, entry } = triplet;
  return Effect.tryPromise({
    try: () =>
      applyDetectedProvider({
        ctx: { db },
        input: {
          serviceId: row.id,
          expected: {
            provider: row.provider,
            apiConfig: row.apiConfig ?? null,
          },
          set: { provider, apiConfig: null },
        },
        now: tickStartedAt,
      }),
    catch: (e) => (e instanceof Error ? e : new Error(String(e))),
  }).pipe(
    Effect.map(({ updated }): DetectOutcome => {
      if (!updated) {
        logger.warn(
          "external-status detect: concurrent edit, write skipped for slug={slug}",
          { slug: entry.id },
        );
        return { kind: "skipped", slug: entry.id };
      }
      reportDetectionStory({
        slug: entry.id,
        currentProvider: row.provider,
        fetchError: error,
        outcome:
          outcome === "applied"
            ? { kind: "applied", provider }
            : { kind: "config-cleared" },
        evidence,
      });
      return { kind: outcome, slug: entry.id };
    }),
    Effect.catch((e) =>
      Effect.sync((): DetectOutcome => {
        logger.warn(
          "external-status detect: write failed for slug={slug}: {message}",
          { slug: entry.id, message: e.message },
        );
        reportDetectionWriteFailure({ slug: entry.id, error: e });
        clearProbeStamp(entry.id);
        // The merged story never fired; keep the triggering failure visible.
        if (error) {
          reportFetchFailure({ phase: "status", slug: entry.id, error });
        }
        return { kind: "error", slug: entry.id };
      }),
    ),
  );
}

function detectAndAct(
  item: DetectItem,
  tickStartedAt: Date,
): Effect.Effect<DetectOutcome> {
  const { triplet, error } = item;
  const { row, entry } = triplet;
  return detectProvider({
    statusPageUrl: entry.status_page_url,
    currentProvider: row.provider,
    entryId: entry.id,
  }).pipe(
    Effect.flatMap((result) => {
      const action = decideDetectionAction(result, row);
      switch (action.kind) {
        case "apply":
          return applyDetection({
            triplet,
            error,
            provider: action.provider,
            outcome: "applied",
            evidence: action.evidence,
            tickStartedAt,
          });
        case "clear-config":
          return applyDetection({
            triplet,
            error,
            provider: row.provider,
            outcome: "config-fixed",
            evidence: action.evidence,
            tickStartedAt,
          });
        case "suggest":
          return Effect.sync((): DetectOutcome => {
            reportDetectionStory({
              slug: entry.id,
              currentProvider: row.provider,
              fetchError: error,
              outcome: { kind: "suggest", suggestion: action.suggestion },
              evidence: action.evidence,
            });
            return { kind: "suggested", slug: entry.id };
          });
        case "noop":
          return Effect.sync((): DetectOutcome => {
            if (action.reason === "no-evidence") {
              // Valid JSON our schema rejects means our schema is stale, not
              // that the page moved.
              reportDetectionStory({
                slug: entry.id,
                currentProvider: row.provider,
                fetchError: error,
                outcome:
                  error?.kind === "schema"
                    ? { kind: "schema-mismatch" }
                    : { kind: "none" },
                evidence: action.evidence,
              });
              return { kind: "none", slug: entry.id };
            }
            if (error) {
              reportFetchFailure({ phase: "status", slug: entry.id, error });
            }
            return { kind: "transient", slug: entry.id };
          });
      }
    }),
  );
}

function runDetectPhase(
  items: DetectItem[],
  tickStartedAt: Date,
): Effect.Effect<DetectOutcome[]> {
  return Effect.forEach(items, (item) => detectAndAct(item, tickStartedAt), {
    concurrency: DETECT_CONCURRENCY,
  });
}

function summarizeDetect(outcomes: DetectOutcome[]): DetectCounts {
  let applied = 0;
  let configFixed = 0;
  let suggested = 0;
  let failed = 0;
  for (const o of outcomes) {
    if (o.kind === "applied") applied++;
    else if (o.kind === "config-fixed") configFixed++;
    else if (o.kind === "suggested") suggested++;
    else if (o.kind === "error") failed++;
  }
  return { probed: outcomes.length, applied, configFixed, suggested, failed };
}

export async function runExternalStatusTick(): Promise<{
  status: PhaseCounts;
  incidents: PhaseCounts;
  components: PhaseCounts;
  detect: DetectCounts;
}> {
  const services = await listExternalServices({ ctx: { db } });

  const triplets = buildTriplets(services);
  const tickStartedAt = new Date();

  // The three phases are intentionally independent: each hits a different
  // upstream endpoint and store, so a failed status fetch must not suppress a
  // service's components (or vice versa). A tick can therefore persist
  // component history for a service whose status snapshot failed that tick.
  const [statusOutcomes, incidentOutcomes, componentOutcomes] =
    await Effect.runPromise(
      Effect.all(
        [
          runStatusPhase(triplets, tickStartedAt.getTime()),
          runIncidentPhase(triplets, tickStartedAt),
          runComponentPhase(triplets, tickStartedAt, tickStartedAt.getTime()),
        ],
        { concurrency: 50 },
      ),
    );

  const status = summarizeStatus(statusOutcomes);
  const incidents = summarizeIncidents(incidentOutcomes);
  const components = summarizeComponents(componentOutcomes);

  if (status.snapshots.length > 0) {
    await tb.publishExternalStatus(status.snapshots);
  }
  if (components.snapshots.length > 0) {
    await tb.publishExternalStatusComponent(components.snapshots);
  }

  // After the publishes: a slow probe fleet must not delay or drop the
  // snapshots the tick already fetched.
  const detectItems = collectDetectItems(triplets, statusOutcomes, Date.now());
  const detectOutcomes =
    detectItems.length > 0
      ? await Effect.runPromise(runDetectPhase(detectItems, tickStartedAt))
      : [];
  const detect = summarizeDetect(detectOutcomes);

  return {
    status: status.counts,
    incidents,
    components: components.counts,
    detect,
  };
}

export async function handleExternalStatusCron(c: Context) {
  const { cronCompleted, cronFailed } = runSentryCron("external-status");

  // Background chain: must not capture `c` or anything derived from it
  // (e.g. via getSentry(c)). The handler returns 200 before this resolves, and
  // a captured per-request Sentry hub stays pinned across retries — see
  // apps/workflows/plan.md.
  void Effect.runPromise(
    Effect.tryPromise({
      try: () => runExternalStatusTick(),
      catch: (e) =>
        new Error(
          `external-status tick failed: ${e instanceof Error ? e.message : String(e)}`,
        ),
    }).pipe(
      Effect.tap((res) =>
        Effect.sync(() => {
          logger.info(
            "external-status tick complete: status={statusOk}/{statusTotal} ({statusFail} failures, {statusSkip} skipped), incidents={incOk}/{incTotal} ({incFail} failures, {incSkip} skipped), components={compOk}/{compTotal} ({compFail} failures, {compSkip} skipped), detect={detProbed} probed ({detApplied} applied, {detCfg} config-fixed, {detSuggested} suggested, {detFailed} failed)",
            {
              detProbed: res.detect.probed,
              detApplied: res.detect.applied,
              detCfg: res.detect.configFixed,
              detSuggested: res.detect.suggested,
              detFailed: res.detect.failed,
              statusOk: res.status.successCount,
              statusTotal: res.status.total,
              statusFail: res.status.failureCount,
              statusSkip: res.status.skippedCount,
              incOk: res.incidents.successCount,
              incTotal: res.incidents.total,
              incFail: res.incidents.failureCount,
              incSkip: res.incidents.skippedCount,
              compOk: res.components.successCount,
              compTotal: res.components.total,
              compFail: res.components.failureCount,
              compSkip: res.components.skippedCount,
            },
          );
          void cronCompleted();
        }),
      ),
      Effect.catch((e) =>
        Effect.sync(() => {
          logger.error("external-status tick errored: {message}", {
            message: e.message,
          });
          void reportBackgroundError(e.message);
          void cronFailed();
        }),
      ),
    ),
  );

  return c.json({ success: true }, 200);
}