Open-source status page, incident response and uptime monitoring
TypeScript alertingincident-managementmcpmcp-servermonitoringmonitoring-as-codeobservabilityon-callopen-sourcesrestatus-pagestatuspagesynthetic-monitoringterraformtinybirdtursouptimeuptime-checkeruptime-monitor
import { and, count, db, eq, isNull } from "@openstatus/db";
import {
notificationOutbox,
monitorIncidentTable,
monitor,
monitorStatusTable,
} from "@openstatus/db/src/schema";
import {
createMonitor,
createNotification,
createTestWorkspace,
linkNotificationToMonitor,
} from "@openstatus/db/src/test/factories";
import {
afterAll,
afterEach,
beforeAll,
beforeEach,
describe,
expect,
test,
vi,
type MockInstance,
} from "vitest";
import { applyStatusTransition } from "../checker/transition";
import { providerToFunction } from "../checker/utils";
import { stubPublishAuditLog } from "../test/stub-audit-log";
import { handleStatusDriftCron } from "./status-drift";
// biome-ignore lint/suspicious/noExplicitAny: heterogeneous provider stubs
type AnyStub = MockInstance;
let stubs: AnyStub[] = [];
let workspaceId: number;
beforeAll(async () => {
const { workspace } = await createTestWorkspace();
workspaceId = workspace.id;
});
beforeEach(() => {
stubs = [];
stubs.push(stubPublishAuditLog());
stubs.push(
vi
.spyOn(providerToFunction.email, "sendAlert")
.mockImplementation(() => Promise.resolve()),
);
});
afterEach(() => {
for (const s of stubs) s.mockRestore();
stubs = [];
});
afterAll(async () => {
await db.delete(monitor).where(eq(monitor.workspaceId, workspaceId)).run();
});
describe("handleStatusDriftCron", () => {
test("re-evaluates a monitor whose region write landed without its transition", async () => {
const monitorRow = await createMonitor(workspaceId, {
regions: "ams",
active: true,
});
const notif = await createNotification(workspaceId);
await linkNotificationToMonitor(notif.id, monitorRow.id);
// Exactly the state a crash between the region write and the batch leaves:
// the region says error, the monitor still says active.
await db
.insert(monitorStatusTable)
.values({
monitorId: monitorRow.id,
region: "ams",
status: "error",
cronTimestamp: Date.now(),
})
.run();
// A replay of the same check cannot recover it: the region status is
// unchanged, so the fast path short-circuits.
const replay = await applyStatusTransition({
monitorId: monitorRow.id,
region: "ams",
status: "error",
cronTimestamp: Date.now() + 1000,
deadlineSeconds: 300,
rolloutPct: 100,
});
expect(replay.kind).toBe("unchanged");
const beforeRepair = await db
.select({ status: monitor.status })
.from(monitor)
.where(eq(monitor.id, monitorRow.id))
.all();
expect(beforeRepair[0]?.status).toBe("active");
const result = await handleStatusDriftCron();
expect(result.repaired).toBeGreaterThanOrEqual(1);
const afterRepair = await db
.select({ status: monitor.status })
.from(monitor)
.where(eq(monitor.id, monitorRow.id))
.all();
expect(afterRepair[0]?.status).toBe("error");
const incidents = await db
.select({ total: count() })
.from(monitorIncidentTable)
.where(
and(
eq(monitorIncidentTable.monitorId, monitorRow.id),
isNull(monitorIncidentTable.resolvedAt),
),
)
.all();
expect(incidents[0]?.total).toBe(1);
const outbox = await db
.select({ total: count() })
.from(notificationOutbox)
.where(eq(notificationOutbox.monitorId, monitorRow.id))
.all();
expect(outbox[0]?.total).toBe(1);
});
test("a monitor below quorum is not a drift candidate", async () => {
const monitorRow = await createMonitor(workspaceId, {
regions: "ams,arn,atl,bog",
active: true,
});
await db
.insert(monitorStatusTable)
.values({
monitorId: monitorRow.id,
region: "ams",
status: "error",
cronTimestamp: Date.now(),
})
.run();
await handleStatusDriftCron();
const after = await db
.select({ status: monitor.status })
.from(monitor)
.where(eq(monitor.id, monitorRow.id))
.all();
expect(after[0]?.status).toBe("active");
});
});
describe("drift repair delivery", () => {
test("sends the notification the repair is recovering", async () => {
const sent: string[] = [];
for (const s of stubs) s.mockRestore();
stubs = [
stubPublishAuditLog(),
vi.spyOn(providerToFunction.email, "sendAlert").mockImplementation(() => {
sent.push("alert");
return Promise.resolve();
}),
];
const monitorRow = await createMonitor(workspaceId, {
regions: "ams",
active: true,
});
const notif = await createNotification(workspaceId);
await linkNotificationToMonitor(notif.id, monitorRow.id);
await db
.insert(monitorStatusTable)
.values({
monitorId: monitorRow.id,
region: "ams",
status: "error",
cronTimestamp: Date.now(),
})
.run();
// OUTBOX_ROLLOUT_PCT defaults to 0, so the repair must fall back to the
// inline sender rather than writing a row nobody delivers.
const result = await handleStatusDriftCron();
expect(result.repaired).toBeGreaterThanOrEqual(1);
expect(sent.length).toBeGreaterThanOrEqual(1);
});
});