Skip to content

Open-source status page, incident response and uptime monitoring

3.9 KB
Raw Blame
import { getLogger } from "@logtape/logtape";
import * as Sentry from "@sentry/node";
import { Effect, Fiber, Schedule } from "effect";

import { runIncidentRemindersTick } from "./incident-reminders";
import { runOAuthPruneTick } from "./oauth-prune";
import { handleOutboxDrainCron, handleOutboxRetentionCron } from "./outbox";
import { handleStatusDriftCron } from "./status-drift";

const logger = getLogger(["workflow"]);

type ScheduledTask = {
  name: string;
  expression: string;
  run: () => Promise<unknown>;
};

/**
 * Internal maintenance only, so it runs in-process rather than needing a
 * schedule added outside this repo. Every task is safe to run on both machines
 * at once: the outbox claim is atomic, drift repair is guarded by the same
 * compare-and-swap as a live check, retention deletes are idempotent, and each
 * incident reminder window is claimed in Redis before it is sent.
 */
export const SCHEDULED_TASKS: ScheduledTask[] = [
  {
    name: "outbox-drain",
    expression: "* * * * *",
    run: handleOutboxDrainCron,
  },
  {
    name: "status-drift",
    expression: "*/5 * * * *",
    run: handleStatusDriftCron,
  },
  {
    name: "outbox-retention",
    expression: "17 3 * * *",
    run: handleOutboxRetentionCron,
  },
  {
    name: "oauth-prune",
    expression: "41 3 * * *",
    run: runOAuthPruneTick,
  },
  {
    name: "incident-reminders",
    expression: "*/10 * * * *",
    run: runIncidentRemindersTick,
  },
];

type RunningTask = Fiber.Fiber<unknown, unknown>;

let running: RunningTask[] = [];

// Interrupting a fiber does not cancel the promise `Effect.tryPromise` is
// waiting on, so the task keeps running detached. Track it so shutdown can wait
// for, e.g., a claimed incident reminder window to actually be sent.
const inFlight = new Set<Promise<unknown>>();

function track(task: ScheduledTask): Promise<unknown> {
  const promise = task.run().finally(() => {
    inFlight.delete(promise);
  });
  inFlight.add(promise);
  return promise;
}

function scheduleTask(task: ScheduledTask): RunningTask {
  const body = Effect.tryPromise({
    try: () => track(task),
    catch: (error) =>
      error instanceof Error ? error : new Error(String(error)),
  }).pipe(
    Effect.catch((error) =>
      Effect.sync(() => {
        logger.error("Scheduled task failed", {
          task: task.name,
          error_message: error.message,
        });
        Sentry.captureException(error);
      }),
    ),
  );

  return Effect.runFork(
    Effect.repeat(body, Schedule.cron(task.expression)).pipe(
      Effect.catch((error) =>
        Effect.sync(() => {
          logger.error("Scheduled task stopped", {
            task: task.name,
            error_message: String(error),
          });
          Sentry.captureException(
            new Error(`Scheduled task ${task.name} stopped: ${String(error)}`),
          );
        }),
      ),
    ),
  );
}

export function startScheduler(): void {
  if (running.length > 0) return;
  running = SCHEDULED_TASKS.map(scheduleTask);
  logger.info("Started in-process scheduler", {
    tasks: SCHEDULED_TASKS.map((task) => `${task.name}@${task.expression}`),
  });
}

export async function stopScheduler(): Promise<void> {
  const fibers = running;
  running = [];
  await Effect.runPromise(
    Effect.forEach(fibers, (fiber) => Fiber.interrupt(fiber), {
      concurrency: "unbounded",
      discard: true,
    }),
  );
}

/**
 * Resolves once every scheduled task already started has settled, or after
 * `graceMs`. Call after `stopScheduler` so no new run can start meanwhile.
 */
export async function drainScheduledTasks(graceMs: number): Promise<void> {
  if (inFlight.size === 0) return;
  let timer: ReturnType<typeof setTimeout> | undefined;
  const grace = new Promise<void>((resolve) => {
    timer = setTimeout(resolve, graceMs);
  });
  await Promise.race([
    Promise.allSettled(inFlight).then(() => undefined),
    grace,
  ]);
  if (timer !== undefined) clearTimeout(timer);
}