Open-source status page, incident response and uptime monitoring
TypeScript alertingincident-managementmcpmcp-servermonitoringmonitoring-as-codeobservabilityon-callopen-sourcesrestatus-pagestatuspagesynthetic-monitoringterraformtinybirdtursouptimeuptime-checkeruptime-monitor
workflows: stale-incident reminders (#2805)
Thibault Le Ouay4f3d9049c8d9
diff --git a/apps/workflows/package.json b/apps/workflows/package.jsonindex dd8f9f09b..46624fca6 100644--- a/apps/workflows/package.json+++ b/apps/workflows/package.json@@ -44,6 +44,7 @@"@opentelemetry/sdk-logs": "catalog:","@opentelemetry/semantic-conventions": "catalog:","@sentry/deno": "catalog:",+ "@slack/web-api": "catalog:","@upstash/qstash": "catalog:","drizzle-orm": "catalog:","effect": "catalog:",diff --git a/apps/workflows/src/cron/incident-reminders.ts b/apps/workflows/src/cron/incident-reminders.tsnew file mode 100644index 000000000..31ffbe086--- /dev/null+++ b/apps/workflows/src/cron/incident-reminders.ts@@ -0,0 +1,41 @@+import { getLogger } from "@logtape/logtape";+import {+ type ReminderResult,+ remindStaleIncidents,+} from "@openstatus/services/incident";+import { Redis } from "@openstatus/upstash";+import { WebClient } from "@slack/web-api";++import { env } from "../env";++const logger = getLogger(["workflow", "incident-reminders"]);++let redis: ReturnType<typeof Redis.fromEnv> | undefined;++export async function runIncidentRemindersTick(): Promise<ReminderResult[]> {+ redis ??= Redis.fromEnv();+ const client = redis;+ const results = await remindStaleIncidents({+ // Default retries can hold the tick for ~30 min; a released claim retries+ // on the next tick instead.+ clientFor: (token) => new WebClient(token, { retryConfig: { retries: 2 } }),+ claim: async (key, ttlSeconds) =>+ (await client.set(key, "1", { nx: true, ex: ttlSeconds })) !== null,+ release: async (key) => {+ await client.del(key);+ },+ dashboardUrl:+ env().NODE_ENV === "production"+ ? "https://app.openstatus.dev"+ : "http://localhost:3001",+ });+ logger.info(+ "incident reminders sent: {sent}, failed: {failed}, skipped: {skipped}",+ {+ sent: results.filter((r) => r.sent).length,+ failed: results.filter((r) => r.target !== null && !r.sent).length,+ skipped: results.filter((r) => r.target === null).length,+ },+ );+ return results;+}diff --git a/apps/workflows/src/cron/scheduler.ts b/apps/workflows/src/cron/scheduler.tsindex 41141ee46..ccda55f92 100644--- a/apps/workflows/src/cron/scheduler.ts+++ b/apps/workflows/src/cron/scheduler.ts@@ -2,6 +2,7 @@ import { getLogger } from "@logtape/logtape";import * as Sentry from "@sentry/deno";import { Effect, Fiber, Schedule } from "effect";+import { runIncidentRemindersTick } from "./incident-reminders";import { runOAuthPruneTick } from "./oauth-prune";import { handleOutboxDrainCron, handleOutboxRetentionCron } from "./outbox";import { handleStatusDriftCron } from "./status-drift";@@ -18,7 +19,8 @@ type ScheduledTask = {* Internal maintenance only, so it runs in-process rather than needing a* schedule added outside this repo. Every task is safe to run on both machines* at once: the outbox claim is atomic, drift repair is guarded by the same- * compare-and-swap as a live check, and retention deletes are idempotent.+ * compare-and-swap as a live check, retention deletes are idempotent, and each+ * incident reminder window is claimed in Redis before it is sent.*/export const SCHEDULED_TASKS: ScheduledTask[] = [{@@ -41,6 +43,11 @@ export const SCHEDULED_TASKS: ScheduledTask[] = [expression: "41 3 * * *",run: runOAuthPruneTick,},+ {+ name: "incident-reminders",+ expression: "*/10 * * * *",+ run: runIncidentRemindersTick,+ },];type RunningTask = Fiber.Fiber<unknown, unknown>;diff --git a/packages/services/src/incident/__tests__/reminders.test.ts b/packages/services/src/incident/__tests__/reminders.test.tsnew file mode 100644index 000000000..cf89978b5--- /dev/null+++ b/packages/services/src/incident/__tests__/reminders.test.ts@@ -0,0 +1,298 @@+import { integration } from "@openstatus/db/src/schema";+import {+ createIncident,+ createIncidentEvent,+ createSlackUser,+} from "@openstatus/db/src/test/factories";+import { expect } from "@std/expect";+import { beforeAll, describe, test } from "@std/testing/bdd";++import {+ createWorkspaceFixture,+ withTestTransaction,+} from "../../../test/helpers";+import type { DB } from "../../context";+import { SLACK_BOT_SCOPES } from "../../integration/slack-scopes";+import type { Workspace } from "../../types";+import {+ reminderWindow,+ remindStaleIncidents,+ type SlackIncidentClient,+} from "../index";++const HOUR = 60 * 60 * 1000;++let workspace: Workspace;+let userId: number;++beforeAll(async () => {+ const fixture = await createWorkspaceFixture("team");+ workspace = fixture.workspace;+ userId = fixture.userId;+});++describe("reminderWindow", () => {+ const now = new Date("2026-09-28T12:00:00Z");+ const ago = (h: number) => new Date(now.getTime() - h * HOUR);++ test("fresh, then windows that double", () => {+ expect(reminderWindow("major", ago(3), now)).toBeNull();+ expect(reminderWindow("major", ago(4), now)).toBe(0);+ expect(reminderWindow("major", ago(7.9), now)).toBe(0);+ expect(reminderWindow("major", ago(8), now)).toBe(1);+ expect(reminderWindow("major", ago(16), now)).toBe(2);+ expect(reminderWindow("critical", ago(1), now)).toBe(0);+ expect(reminderWindow("minor", ago(23), now)).toBeNull();+ });+});++function fakeClient() {+ const sent: { channel: string; text: string }[] = [];+ const client = {+ chat: {+ postMessage: async (args: { channel: string; text: string }) => {+ sent.push(args);+ return { ok: true, ts: "1.1" };+ },+ },+ } as SlackIncidentClient;+ return { client, sent };+}++function memoryClaim() {+ const keys = new Set<string>();+ return {+ claim: async (key: string) => {+ if (keys.has(key)) return false;+ keys.add(key);+ return true;+ },+ release: async (key: string) => {+ keys.delete(key);+ },+ };+}++async function connect(+ tx: DB,+ teamId: string,+ scopes: readonly string[] = SLACK_BOT_SCOPES,+) {+ await tx.insert(integration).values({+ name: "slack-agent",+ workspaceId: workspace.id,+ externalId: teamId,+ credential: { botToken: "xoxb-test", botUserId: "UBOT" },+ data: { teamId, scopes: scopes.join(",") },+ });+}++describe("remindStaleIncidents", () => {+ test("bound channel first, then the commander's DM, once per window", async () => {+ await withTestTransaction(async (tx) => {+ const teamId = `T_REM_${crypto.randomUUID()}`;+ await connect(tx, teamId);+ const stale = new Date(Date.now() - 5 * HOUR);+ const inChannel = await createIncident(+ workspace.id,+ {+ declaredAt: stale,+ commanderId: userId,+ slackTeamId: teamId,+ slackChannelId: "C_REM",+ },+ tx,+ );+ const byDm = await createIncident(+ workspace.id,+ { declaredAt: stale, commanderId: userId },+ tx,+ );+ await createIncidentEvent(byDm.id, { createdAt: stale }, tx);+ await createSlackUser(+ workspace.id,+ userId,+ { slackTeamId: teamId, slackUserId: "U_COMMANDER" },+ tx,+ );+ const fresh = await createIncident(workspace.id, {}, tx);+ await createIncidentEvent(fresh.id, { createdAt: new Date() }, tx);++ const { client, sent } = fakeClient();+ const claims = memoryClaim();+ const run = () =>+ remindStaleIncidents({+ db: tx,+ clientFor: () => client,+ ...claims,+ dashboardUrl: "https://app.test",+ });++ const results = await run();+ const mine = (id: number) => results.find((r) => r.incidentId === id);+ expect(mine(inChannel.id)?.target).toEqual({+ kind: "channel",+ channel: "C_REM",+ });+ expect(mine(byDm.id)?.target).toEqual({+ kind: "dm",+ slackUserId: "U_COMMANDER",+ role: "commander",+ });+ expect(mine(fresh.id)).toBeUndefined();+ expect(sent.map((m) => m.channel).sort()).toEqual(+ ["C_REM", "U_COMMANDER"].sort(),+ );++ await run();+ expect(sent).toHaveLength(2);+ });+ });++ test("nobody linked: reported once per window with no target", async () => {+ await withTestTransaction(async (tx) => {+ const teamId = `T_REM_${crypto.randomUUID()}`;+ await connect(tx, teamId);+ const row = await createIncident(+ workspace.id,+ { declaredAt: new Date(Date.now() - 30 * HOUR), severity: "minor" },+ tx,+ );+ const { client, sent } = fakeClient();+ const claims = memoryClaim();+ const run = () =>+ remindStaleIncidents({+ db: tx,+ clientFor: () => client,+ ...claims,+ dashboardUrl: "https://app.test",+ });+ expect((await run()).find((r) => r.incidentId === row.id)).toEqual({+ incidentId: row.id,+ target: null,+ sent: false,+ });+ expect(+ (await run()).find((r) => r.incidentId === row.id),+ ).toBeUndefined();+ expect(sent).toHaveLength(0);+ });+ });++ function failingClient(error: unknown) {+ let calls = 0;+ const client = {+ chat: {+ postMessage: async () => {+ calls++;+ if (calls === 1) throw error;+ return { ok: true, ts: "1.1" };+ },+ },+ } as unknown as SlackIncidentClient;+ return { client, calls: () => calls };+ }++ async function staleChannelIncident(tx: DB) {+ const teamId = `T_REM_${crypto.randomUUID()}`;+ await connect(tx, teamId);+ return createIncident(+ workspace.id,+ {+ declaredAt: new Date(Date.now() - 5 * HOUR),+ slackTeamId: teamId,+ slackChannelId: "C_FAIL",+ },+ tx,+ );+ }++ test("transient slack failure: not sent, retried on the next tick", async () => {+ await withTestTransaction(async (tx) => {+ const row = await staleChannelIncident(tx);+ const { client, calls } = failingClient(+ Object.assign(new Error("socket hang up"), {+ code: "slack_webapi_request_error",+ }),+ );+ const claims = memoryClaim();+ const run = () =>+ remindStaleIncidents({+ db: tx,+ clientFor: () => client,+ ...claims,+ dashboardUrl: "https://app.test",+ });+ expect((await run()).find((r) => r.incidentId === row.id)?.sent).toBe(+ false,+ );+ expect((await run()).find((r) => r.incidentId === row.id)?.sent).toBe(+ true,+ );+ expect(calls()).toBe(2);+ });+ });++ test("slack platform error: not sent, window stays claimed", async () => {+ await withTestTransaction(async (tx) => {+ const row = await staleChannelIncident(tx);+ const { client, calls } = failingClient(+ Object.assign(new Error("channel_not_found"), {+ code: "slack_webapi_platform_error",+ }),+ );+ const claims = memoryClaim();+ const run = () =>+ remindStaleIncidents({+ db: tx,+ clientFor: () => client,+ ...claims,+ dashboardUrl: "https://app.test",+ });+ expect((await run()).find((r) => r.incidentId === row.id)?.sent).toBe(+ false,+ );+ expect(+ (await run()).find((r) => r.incidentId === row.id),+ ).toBeUndefined();+ expect(calls()).toBe(1);+ });+ });++ test("install without chat:write: nothing claimed or sent", async () => {+ await withTestTransaction(async (tx) => {+ const teamId = `T_REM_${crypto.randomUUID()}`;+ await connect(+ tx,+ teamId,+ SLACK_BOT_SCOPES.filter((s) => s !== "chat:write"),+ );+ const row = await createIncident(+ workspace.id,+ {+ declaredAt: new Date(Date.now() - 5 * HOUR),+ slackTeamId: teamId,+ slackChannelId: "C_NOSCOPE",+ },+ tx,+ );+ const claimed: string[] = [];+ const { client, sent } = fakeClient();+ const results = await remindStaleIncidents({+ db: tx,+ clientFor: () => client,+ claim: async (key) => {+ claimed.push(key);+ return true;+ },+ release: async () => {},+ dashboardUrl: "https://app.test",+ });+ expect(results.find((r) => r.incidentId === row.id)).toBeUndefined();+ expect(+ claimed.some((k) => k.startsWith(`incident:reminder:${row.id}:`)),+ ).toBe(false);+ expect(sent).toHaveLength(0);+ });+ });+});diff --git a/packages/services/src/incident/index.ts b/packages/services/src/incident/index.tsindex c307352e7..b5ccf26a4 100644--- a/packages/services/src/incident/index.ts+++ b/packages/services/src/incident/index.ts@@ -10,6 +10,13 @@ export {} from "./link-status-report";export { listIncidentEvents } from "./list-events";export { clearIncidentCommander } from "./members";+export {+ type ClaimOnce,+ reminderWindow,+ remindStaleIncidents,+ type ReminderResult,+ STALE_AFTER,+} from "./reminders";export {getIncident,getIncidentBySlackChannel,diff --git a/packages/services/src/incident/reminders.ts b/packages/services/src/incident/reminders.tsnew file mode 100644index 000000000..f97fc50bb--- /dev/null+++ b/packages/services/src/incident/reminders.ts@@ -0,0 +1,210 @@+import { and, db as defaultDb, desc, eq, inArray } from "@openstatus/db";+import {+ type Incident,+ type IncidentSeverity,+ incident,+ incidentEvent,+ slackUser,+ selectWorkspaceSchema,+ workspace,+} from "@openstatus/db/src/schema";++import type { DB, ServiceContext } from "../context";+import { isFeatureEnabled } from "../features";+import {+ type SlackConnection,+ getSlackConnection,+} from "../integration/slack-connection";+import { INCIDENT_FEATURE } from "./internal";+import type { SlackClientFactory } from "./slack-flow";++const HOUR = 60 * 60 * 1000;++export const STALE_AFTER: Record<IncidentSeverity, number> = {+ critical: HOUR,+ major: 4 * HOUR,+ minor: 24 * HOUR,+};++/**+ * Which reminder window an incident is in: 0 once it's stale, then 1, 2, …+ * each twice as long as the last. `null` while it's still fresh.+ */+export function reminderWindow(+ severity: IncidentSeverity,+ lastActivity: Date,+ now: Date,+): number | null {+ const threshold = STALE_AFTER[severity];+ const elapsed = now.getTime() - lastActivity.getTime();+ if (elapsed < threshold) return null;+ return Math.floor(Math.log2(elapsed / threshold));+}++export type ReminderTarget =+ | { kind: "channel"; channel: string }+ | { kind: "dm"; slackUserId: string; role: "commander" | "declarer" };++export type ReminderResult = {+ incidentId: number;+ target: ReminderTarget | null;+ /** `false` when there was nobody to remind or Slack rejected the post. */+ sent: boolean;+};++/** `true` when this caller owns the key; used so each window fires once. */+export type ClaimOnce = (key: string, ttlSeconds: number) => Promise<boolean>;+export type ReleaseClaim = (key: string) => Promise<void>;++// The Slack client already retries network, 5xx and 429 errors; anything else+// (channel_not_found, not_in_channel, …) would fail again on the next tick.+function isPermanentSlackError(error: unknown): boolean {+ return (+ typeof error === "object" &&+ error !== null &&+ "code" in error &&+ error.code === "slack_webapi_platform_error"+ );+}++async function reminderConnection(+ db: DB,+ workspaceId: number,+): Promise<SlackConnection | null> {+ const parsed = selectWorkspaceSchema.safeParse(+ await db+ .select()+ .from(workspace)+ .where(eq(workspace.id, workspaceId))+ .get(),+ );+ if (!parsed.success) return null;+ const ws = parsed.data;+ if (!isFeatureEnabled(ws, INCIDENT_FEATURE)) return null;+ if (!ws.limits["slack-agent"]) return null;+ const ctx: ServiceContext = {+ workspace: ws,+ actor: { type: "system", job: "incident-reminders" },+ db,+ };+ const connection = await getSlackConnection({ ctx });+ if (!connection || connection.missingScopes.includes("chat:write")) {+ return null;+ }+ return connection;+}++async function dmTarget(+ db: DB,+ row: Incident,+ teamId: string,+): Promise<ReminderTarget | null> {+ for (const [role, userId] of [+ ["commander", row.commanderId],+ ["declarer", row.declaredBy],+ ] as const) {+ if (userId === null) continue;+ const mapping = await db+ .select({ slackUserId: slackUser.slackUserId })+ .from(slackUser)+ .where(+ and(+ eq(slackUser.workspaceId, row.workspaceId),+ eq(slackUser.slackTeamId, teamId),+ eq(slackUser.userId, userId),+ ),+ )+ .get();+ if (mapping) return { kind: "dm", slackUserId: mapping.slackUserId, role };+ }+ return null;+}++/**+ * Nudges open and mitigated incidents that went quiet: the bound channel,+ * else a DM to the commander, else to the declarer. Safe to run on every+ * machine at once: each window is claimed before anything is sent.+ */+export async function remindStaleIncidents(args: {+ now?: Date;+ clientFor: SlackClientFactory;+ claim: ClaimOnce;+ release: ReleaseClaim;+ dashboardUrl: string;+ db?: DB;+}): Promise<ReminderResult[]> {+ const db = args.db ?? defaultDb;+ const now = args.now ?? new Date();+ const candidates = await db+ .select()+ .from(incident)+ .where(inArray(incident.status, ["open", "mitigated"]))+ .all();++ const results: ReminderResult[] = [];+ const connections = new Map<number, SlackConnection | null>();+ for (const row of candidates) {+ if (row.closedAt) continue;+ if (!connections.has(row.workspaceId)) {+ connections.set(+ row.workspaceId,+ await reminderConnection(db, row.workspaceId),+ );+ }+ const connection = connections.get(row.workspaceId);+ if (!connection) continue;++ const last = await db+ .select({ id: incidentEvent.id, createdAt: incidentEvent.createdAt })+ .from(incidentEvent)+ .where(eq(incidentEvent.incidentId, row.id))+ .orderBy(desc(incidentEvent.createdAt), desc(incidentEvent.id))+ .get();+ const lastActivity = last?.createdAt ?? row.declaredAt;+ const window = reminderWindow(row.severity, lastActivity, now);+ if (window === null) continue;++ // Claimed before resolving the target so a skip is reported once per window.+ const key = `incident:reminder:${row.id}:${last?.id ?? 0}:${window}`;+ if (!(await args.claim(key, 14 * 24 * 60 * 60))) continue;++ const target: ReminderTarget | null =+ row.slackChannelId && row.slackTeamId === connection.teamId+ ? { kind: "channel", channel: row.slackChannelId }+ : await dmTarget(db, row, connection.teamId);+ if (!target) {+ console.warn("incident reminder skipped: nobody to remind", {+ incidentId: row.id,+ });+ results.push({ incidentId: row.id, target: null, sent: false });+ continue;+ }++ const quiet = Math.round((now.getTime() - lastActivity.getTime()) / HOUR);+ const url = `${args.dashboardUrl}/incidents/${row.id}`;+ const text =+ target.kind === "channel"+ ? `:hourglass: No update on *${row.title}* for ${quiet}h. Post a note, change its status, or resolve it. <${url}|Open in openstatus>`+ : `:hourglass: You are the ${target.role} of *${row.title}* (${row.severity}), which has had no update for ${quiet}h. <${url}|Open in openstatus>`;+ const sent = await args+ .clientFor(connection.botToken)+ .chat.postMessage({+ channel:+ target.kind === "channel" ? target.channel : target.slackUserId,+ text,+ })+ .then(+ () => true,+ async (error) => {+ console.warn("incident reminder failed", {+ incidentId: row.id,+ error,+ });+ if (!isPermanentSlackError(error)) await args.release(key);+ return false;+ },+ );+ results.push({ incidentId: row.id, target, sent });+ }+ return results;+}diff --git a/pnpm-lock.yaml b/pnpm-lock.yamlindex 305644272..c15e2595d 100644--- a/pnpm-lock.yaml+++ b/pnpm-lock.yaml@@ -1890,6 +1890,9 @@ importers:'@sentry/deno':specifier: 'catalog:'version: 10.75.1+ '@slack/web-api':+ specifier: 'catalog:'+ version: 8.1.1'@upstash/qstash':specifier: 'catalog:'version: 2.11.3