Open-source status page, incident response and uptime monitoring
TypeScript alertingincident-managementmcpmcp-servermonitoringmonitoring-as-codeobservabilityon-callopen-sourcesrestatus-pagestatuspagesynthetic-monitoringterraformtinybirdtursouptimeuptime-checkeruptime-monitor
import { getLogger } from "@logtape/logtape";
import * as Sentry from "@sentry/node";
import { Effect, Fiber, Schedule } from "effect";
import { runIncidentRemindersTick } from "./incident-reminders";
import { runOAuthPruneTick } from "./oauth-prune";
import { handleOutboxDrainCron, handleOutboxRetentionCron } from "./outbox";
import { handleStatusDriftCron } from "./status-drift";
const logger = getLogger(["workflow"]);
type ScheduledTask = {
name: string;
expression: string;
run: () => Promise<unknown>;
};
/**
* Internal maintenance only, so it runs in-process rather than needing a
* schedule added outside this repo. Every task is safe to run on both machines
* at once: the outbox claim is atomic, drift repair is guarded by the same
* compare-and-swap as a live check, retention deletes are idempotent, and each
* incident reminder window is claimed in Redis before it is sent.
*/
export const SCHEDULED_TASKS: ScheduledTask[] = [
{
name: "outbox-drain",
expression: "* * * * *",
run: handleOutboxDrainCron,
},
{
name: "status-drift",
expression: "*/5 * * * *",
run: handleStatusDriftCron,
},
{
name: "outbox-retention",
expression: "17 3 * * *",
run: handleOutboxRetentionCron,
},
{
name: "oauth-prune",
expression: "41 3 * * *",
run: runOAuthPruneTick,
},
{
name: "incident-reminders",
expression: "*/10 * * * *",
run: runIncidentRemindersTick,
},
];
type RunningTask = Fiber.Fiber<unknown, unknown>;
let running: RunningTask[] = [];
// Interrupting a fiber does not cancel the promise `Effect.tryPromise` is
// waiting on, so the task keeps running detached. Track it so shutdown can wait
// for, e.g., a claimed incident reminder window to actually be sent.
const inFlight = new Set<Promise<unknown>>();
function track(task: ScheduledTask): Promise<unknown> {
const promise = task.run().finally(() => {
inFlight.delete(promise);
});
inFlight.add(promise);
return promise;
}
function scheduleTask(task: ScheduledTask): RunningTask {
const body = Effect.tryPromise({
try: () => track(task),
catch: (error) =>
error instanceof Error ? error : new Error(String(error)),
}).pipe(
Effect.catch((error) =>
Effect.sync(() => {
logger.error("Scheduled task failed", {
task: task.name,
error_message: error.message,
});
Sentry.captureException(error);
}),
),
);
return Effect.runFork(
Effect.repeat(body, Schedule.cron(task.expression)).pipe(
Effect.catch((error) =>
Effect.sync(() => {
logger.error("Scheduled task stopped", {
task: task.name,
error_message: String(error),
});
Sentry.captureException(
new Error(`Scheduled task ${task.name} stopped: ${String(error)}`),
);
}),
),
),
);
}
export function startScheduler(): void {
if (running.length > 0) return;
running = SCHEDULED_TASKS.map(scheduleTask);
logger.info("Started in-process scheduler", {
tasks: SCHEDULED_TASKS.map((task) => `${task.name}@${task.expression}`),
});
}
export async function stopScheduler(): Promise<void> {
const fibers = running;
running = [];
await Effect.runPromise(
Effect.forEach(fibers, (fiber) => Fiber.interrupt(fiber), {
concurrency: "unbounded",
discard: true,
}),
);
}
/**
* Resolves once every scheduled task already started has settled, or after
* `graceMs`. Call after `stopScheduler` so no new run can start meanwhile.
*/
export async function drainScheduledTasks(graceMs: number): Promise<void> {
if (inFlight.size === 0) return;
let timer: ReturnType<typeof setTimeout> | undefined;
const grace = new Promise<void>((resolve) => {
timer = setTimeout(resolve, graceMs);
});
await Promise.race([
Promise.allSettled(inFlight).then(() => undefined),
grace,
]);
if (timer !== undefined) clearTimeout(timer);
}