Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(goalert-bot): post GoAlert alerts to Macro Alerts with threaded updates (#9216)

Wolf Mermelsteina460a8bdff31
diff --git a/bun.lock b/bun.lock
index 289490844c..b95e612e0f 100644
--- a/bun.lock
+++ b/bun.lock
@@ -496,6 +496,20 @@
"wrangler": "^4.110.0",
},
},
+ "services/bots/goalert-bot": {
+ "name": "goalert-bot",
+ "version": "0.0.1",
+ "dependencies": {
+ "@macro-inc/observability": "workspace:*",
+ "hono": "^4.6.20",
+ "zod": "^4.4.3",
+ },
+ "devDependencies": {
+ "@types/node": "^25.9.3",
+ "bun-types": "^1.3.14",
+ "wrangler": "^4.110.0",
+ },
+ },
"services/bots/stripe-payment-bot": {
"name": "stripe-payment-bot",
"version": "0.0.1",
@@ -2631,6 +2645,8 @@
"globrex": ["[email protected]", "", {}, "sha512-uHJgbwAMwNFf5mLst7IWLNg14x1CkeqglJb/K3doi4dw6q2IvAAmM/Y81kevy83wP+Sst+nutFTYOGg3d1lsxg=="],
+ "goalert-bot": ["goalert-bot@workspace:services/bots/goalert-bot"],
+
"goober": ["[email protected]", "", { "peerDependencies": { "csstype": "^3.0.10" } }, "sha512-U7veizMqxyKlM58+Z5j2ngJBH/r9siDmxpvNxSw0PylF6WQvrASJEZrxh1hidRBJc2jqoBVSyOban5u8m+6Rxg=="],
"google-auth-library": ["[email protected]", "", { "dependencies": { "base64-js": "^1.3.0", "ecdsa-sig-formatter": "^1.0.11", "gaxios": "^7.1.4", "gcp-metadata": "8.1.2", "google-logging-utils": "1.1.3", "jws": "^4.0.0" } }, "sha512-xtvUqvINPhTaBm7nXqlYPcrMHJPm1lCNdSovxnKKhTm+4JsvQ+KGVYJViLoH9Yxu8w+T0Qv5HubzYT9BLrppJg=="],
@@ -4199,6 +4215,8 @@
"get-uri/data-uri-to-buffer": ["[email protected]", "", {}, "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw=="],
+ "goalert-bot/@types/node": ["@types/[email protected]", "", { "dependencies": { "undici-types": ">=7.24.0 <7.24.7" } }, "sha512-OScDchr2fwuUmWdf4kZ9h7PcJiYDVInhJizG/biAq3cAvqwYktuy/TYGGdZNMtNTFUP7rnb0NU4TUdm82kt4Rg=="],
+
"google-auth-library/gcp-metadata": ["[email protected]", "", { "dependencies": { "gaxios": "^7.0.0", "google-logging-utils": "^1.0.0", "json-bigint": "^1.0.0" } }, "sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg=="],
"google-gax/@grpc/proto-loader": ["@grpc/[email protected]", "", { "dependencies": { "lodash.camelcase": "^4.3.0", "long": "^5.0.0", "protobufjs": "^7.2.5", "yargs": "^17.7.2" }, "bin": { "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" } }, "sha512-tMXdRCfYVixjuFK+Hk0Q1s38gV9zDiDJfWL3h1rv4Qc39oILCu1TRTDt7+fGUI8K4G1Fj125Hx/ru3azECWTyQ=="],
@@ -4629,6 +4647,8 @@
"gcp-metadata/gaxios/uuid": ["[email protected]", "", { "bin": { "uuid": "dist/bin/uuid" } }, "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA=="],
+ "goalert-bot/@types/node/undici-types": ["[email protected]", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="],
+
"google-gax/@grpc/proto-loader/yargs": ["[email protected]", "", { "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "require-directory": "^2.1.1", "string-width": "^4.2.3", "y18n": "^5.0.5", "yargs-parser": "^21.1.1" } }, "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w=="],
"google-gax/google-auth-library/gaxios": ["[email protected]", "", { "dependencies": { "extend": "^3.0.2", "https-proxy-agent": "^7.0.1", "is-stream": "^2.0.0", "node-fetch": "^2.6.9", "uuid": "^9.0.1" } }, "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ=="],
diff --git a/nix-support/node_modules-hashes.json b/nix-support/node_modules-hashes.json
index aa114f317a..51a11ffdb8 100644
--- a/nix-support/node_modules-hashes.json
+++ b/nix-support/node_modules-hashes.json
@@ -1,6 +1,6 @@
{
"nodeModules": {
- "aarch64-darwin": "sha256-KwFdy4O78IkrXvPKd0hjY8NLgVbxJPUObyd+Ny50ieo=",
- "x86_64-linux": "sha256-HV7k/t1RjAeQ/q7GhHCZnU9FlqR1mfue7y07ODHBpFk="
+ "aarch64-darwin": "sha256-DYQqGiu4+hooVYvTs+ov0HPeVV6GyFnVwjndsPhUmQY=",
+ "x86_64-linux": "sha256-7P+xA87nIIy9+e2/UuDB0oOBcpBEE38nsrKMWJ9uC4w="
}
}
diff --git a/package.json b/package.json
index b290d7b049..9d2486fde1 100644
--- a/package.json
+++ b/package.json
@@ -78,6 +78,7 @@
"packages/lexical-core",
"services/ai-editing-worker",
"services/bots/anthropic-status-bot",
+ "services/bots/goalert-bot",
"services/bots/stripe-payment-bot",
"services/lexical-service",
"packages/spreadsheet",
diff --git a/services/bots/goalert-bot/AGENTS.md b/services/bots/goalert-bot/AGENTS.md
new file mode 100644
index 0000000000..c2f5f44292
--- /dev/null
+++ b/services/bots/goalert-bot/AGENTS.md
@@ -0,0 +1,44 @@
+# goalert-bot
+
+## What it is
+
+Observability: a Cloudflare Worker that turns GoAlert outgoing webhooks into
+Macro Alerts channel posts, one thread per alert, through the Macro SDK.
+
+## Start here
+
+- `src/index.ts`: the webhook route and what each payload type does.
+- `src/format.ts`: the compact post format, parsed from Datadog and Grafana details.
+- `README.md`: payload → channel effect table, auth, setup.
+
+## Scope
+
+- Owns: formatting alerts, threading status changes, the KV map
+ `alert:<AlertID>` → posted message.
+- Not here: which alerts page → `infra/stacks/monitoring` (Datadog) and the
+ observability stack's Grafana and GoAlert config.
+- Called by: GoAlert webhook destinations only.
+
+## Checks
+
+- `bun test` (from this directory)
+- `bunx tsc --noEmit --project tsconfig.json`
+
+## Gotchas
+
+- GoAlert abandons a webhook after ~3s and never retries: keep a request to one
+ KV read, parallel channel calls, and one KV write. Don't add fetches.
+- Import `@macro-inc/observability/worker` only from `src/worker.ts`: it loads
+ `cloudflare:workers`, which Bun tests cannot resolve. Spans elsewhere use
+ `Telemetry` from `@macro-inc/observability` (`src/traced.ts`).
+- The bot is user-owned and acts as its owner (`ACTING_USER_ID`, a secret, never
+ committed). Without `requestedAs` the SDK falls back to the channel webhook,
+ which cannot thread, edit or react.
+- The resolved edit drops the `here` mention; a bot edit carrying mentions can
+ notify again.
+- The SDK is imported by relative path from `packages/sdk/src`, like
+ `anthropic-status-bot`; SDK changes reach this worker on the next deploy.
+
+## Related
+
+- Rules: `.agents/rules/observability.md` · Docs: `docs/guides/new-service.md`
diff --git a/services/bots/goalert-bot/CLAUDE.md b/services/bots/goalert-bot/CLAUDE.md
new file mode 120000
index 0000000000..47dc3e3d86
--- /dev/null
+++ b/services/bots/goalert-bot/CLAUDE.md
@@ -0,0 +1 @@
+AGENTS.md
\ No newline at end of file
diff --git a/services/bots/goalert-bot/README.md b/services/bots/goalert-bot/README.md
new file mode 100644
index 0000000000..c28664a31f
--- /dev/null
+++ b/services/bots/goalert-bot/README.md
@@ -0,0 +1,118 @@
+# goalert-bot
+
+Posts GoAlert alerts to the Macro Alerts channel, one thread per alert.
+
+GoAlert is the paging hub: Grafana and Datadog alert GoAlert, whose escalation
+policies send outgoing webhooks for each alert and status change. This worker
+receives them and posts through the [Macro SDK](../../../packages/sdk), so it
+can thread, mention, edit and react, which the channel bot webhook cannot.
+
+## How it works
+
+```
+GoAlert webhook ──POST──> /goalert/{WEBHOOK_SECRET}
+ │ parse (Alert | AlertStatus | bundles | …)
+ │ KV ALERT_MESSAGES: alert:<AlertID> → message
+ ▼
+ @macro-inc/sdk → channel.send / message.reply / react / edit
+ headers: x-macro-bot-token: BOT_TOKEN, x-macro-bot-scope: user,
+ x-macro-bot-for-macro-user-id: ACTING_USER_ID
+```
+
+| GoAlert payload | Channel effect |
+| --- | --- |
+| `Alert` (new) | Compact top-level post; pages end the headline with a `here` mention. Stored in KV for 30 days. |
+| `Alert` (already posted) | GoAlert re-escalated an open page: thread reply `⏰ Still open` (with `here` for pages). |
+| `AlertStatus` acknowledged | Thread reply `👀 Acknowledged by …`, 👀 reaction on the post. |
+| `AlertStatus` closed | Thread reply `✅ Closed by …`, ✅ reaction, and the post's headline is prefixed `✅ Resolved ·` (the `here` mention is dropped so the edit notifies nobody). |
+| `AlertStatus` other (escalated, …) | Thread reply with the log entry. |
+| `AlertStatus` for an alert never posted | One-line top-level post. |
+| `AlertBundle` / `AlertStatusBundle` | One top-level line with the count and a link. |
+| `ScheduleOnCallUsers`, `Test`, `Verification` | One top-level notice. |
+
+A new alert post:
+
+```
+🚨 **PAGE** · **<title> — <scope>** @here
+🔴 critical: 2631 (threshold 1800)
+<one-sentence description, AWS hostnames shortened>
+[Acknowledge](…) · [Graph](…) · [Runbook](…)
+```
+
+The `Notify` GoAlert service posts `🔔` instead of `🚨 **PAGE**` and mentions
+nobody. State, description and links are parsed from Datadog monitor messages
+(`alertMessage()` in `infra/stacks/monitoring/notify.ts`) and Grafana's details
+table.
+
+Notes on the GoAlert side:
+
+- GoAlert **cannot set headers**, so the URL carries the shared secret,
+ compared in constant time.
+- GoAlert **abandons a webhook after about three seconds** and does not retry.
+ A request does one KV read, its channel calls in parallel, and at most one
+ KV write. A failed channel call returns `502` so it shows in GoAlert's logs.
+- Unrecognized payloads are acknowledged (`200 {ok, ignored: true}`).
+
+## Auth
+
+The bot is **user-owned and acts as its owner** (`requestedAs(ACTING_USER_ID)`,
+user scope). The server lets a user-owned bot act only as its owner. Messages
+show the bot as sender, triggered by the owner, and the bot's channel access
+follows the owner's: the owner must be able to post in `CHANNEL_ID`. Moving to a
+team-owned bot means dropping `ACTING_USER_ID` and `requestedAs` in
+`src/channel.ts` and using the default team scope.
+
+## Tracing
+
+`src/worker.ts` wraps the worker in otel-cf-workers' `instrument()` through
+`@macro-inc/observability/worker`, configured like `services/ai-editing-worker`:
+spans go to `OTEL_EXPORTER_OTLP_ENDPOINT` (the prod OTLP proxy), with
+`DD_API_KEY` as `dd-api-key` when set. Each webhook is one
+`goalert_bot.webhook` span with `goalert.payload_type`, `goalert.alert_id`,
+`goalert.service_name` and `goalert.outcome`; children
+`goalert_bot.kv.get`/`kv.put` (`goalert.found` on reads) and
+`goalert_bot.channel.post`/`reply`/`react`/`edit` (`goalert.message_id`).
+Message bodies are never recorded. Search by `@goalert.alert_id`.
+
+## Setup
+
+Deployed as a single prod Cloudflare Worker (`goalert-bot`, no dev
+environment):
+
+```bash
+wrangler kv namespace create ALERT_MESSAGES # put the id in wrangler.jsonc
+printf '%s' '<token>' | wrangler secret put BOT_TOKEN
+printf '%s' '<owner user id>' | wrangler secret put ACTING_USER_ID
+printf '%s' '<secret>' | wrangler secret put WEBHOOK_SECRET
+printf '%s' '<key>' | wrangler secret put DD_API_KEY # optional, see Tracing
+bun run deploy # wrangler deploy
+```
+
+Then point a GoAlert webhook destination (escalation policy step or service
+notification) at
+`https://goalert-bot.macroverse.workers.dev/goalert/<WEBHOOK_SECRET>`.
+
+## Environment variables
+
+| Variable | Required | Default | Description |
+| ---------------- | -------- | ------- | --------------------------------------------------------- |
+| `BOT_TOKEN` | yes | | Bot API key (`mbot_...`, sent as `x-macro-bot-token`) |
+| `CHANNEL_ID` | yes | | Channel UUID to post to (Macro Alerts) |
+| `ACTING_USER_ID` | yes | | Secret: the bot owner's user id (`macro\|<email>`) |
+| `WEBHOOK_SECRET` | yes | | Secret path segment for the webhook URL |
+| `ALERT_MESSAGES` | worker | | KV namespace binding: `alert:<AlertID>` → posted message |
+| `ENVIRONMENT` | no | `local` | `deployment.environment` on exported spans |
+| `OTEL_EXPORTER_OTLP_ENDPOINT` | no | | OTLP/HTTP traces endpoint; unset disables export |
+| `DD_API_KEY` | no | | Secret: Datadog key sent with trace exports |
+| `MACRO_ENV` | no | `prod` | SDK environment (`prod`/`dev`/`local`), local server only |
+| `PORT` | no | `8089` | Listen port (local Bun server only) |
+
+## Development
+
+```bash
+just dev # bun run --watch src/server.ts (in-memory alert store)
+just test # bun test
+just check # tsc --noEmit
+```
+
+Formatter tests use real Datadog and Grafana alert details.
diff --git a/services/bots/goalert-bot/biome.jsonc b/services/bots/goalert-bot/biome.jsonc
new file mode 100644
index 0000000000..82a0f443c4
--- /dev/null
+++ b/services/bots/goalert-bot/biome.jsonc
@@ -0,0 +1,11 @@
+{
+ "$schema": "https://biomejs.dev/schemas/2.4.12/schema.json",
+ "root": true,
+ "extends": ["../../../biome.base.jsonc"],
+ "vcs": {
+ "root": "../../.."
+ },
+ "files": {
+ "includes": ["**/*.{ts,tsx,js,jsx}", "!**/*.d.ts", "!**/node_modules"]
+ }
+}
diff --git a/services/bots/goalert-bot/justfile b/services/bots/goalert-bot/justfile
new file mode 100644
index 0000000000..44103f8db1
--- /dev/null
+++ b/services/bots/goalert-bot/justfile
@@ -0,0 +1,8 @@
+dev:
+ bun run --watch src/server.ts
+
+test:
+ bun test
+
+check:
+ bunx tsc --noEmit --project tsconfig.json
diff --git a/services/bots/goalert-bot/package.json b/services/bots/goalert-bot/package.json
new file mode 100644
index 0000000000..06b03a89b3
--- /dev/null
+++ b/services/bots/goalert-bot/package.json
@@ -0,0 +1,23 @@
+{
+ "name": "goalert-bot",
+ "version": "0.0.1",
+ "private": true,
+ "type": "module",
+ "scripts": {
+ "dev": "bun run --watch src/server.ts",
+ "start": "bun run src/server.ts",
+ "test": "bun test",
+ "check": "tsc --noEmit --project tsconfig.json",
+ "deploy": "wrangler deploy"
+ },
+ "dependencies": {
+ "@macro-inc/observability": "workspace:*",
+ "hono": "^4.6.20",
+ "zod": "^4.4.3"
+ },
+ "devDependencies": {
+ "@types/node": "^25.9.3",
+ "bun-types": "^1.3.14",
+ "wrangler": "^4.110.0"
+ }
+}
diff --git a/services/bots/goalert-bot/src/channel.ts b/services/bots/goalert-bot/src/channel.ts
new file mode 100644
index 0000000000..f5de17453a
--- /dev/null
+++ b/services/bots/goalert-bot/src/channel.ts
@@ -0,0 +1,54 @@
+import type { Env } from '../../../../packages/sdk/src/config';
+import { Macro } from '../../../../packages/sdk/src/macro';
+import type { RichMessage } from '../../../../packages/sdk/src/mentions';
+
+export type ChannelConfig = {
+ /** UUID of the channel the bot posts to. */
+ channelId: string;
+ /** Bot API key (mbot_...), sent as x-macro-bot-token. */
+ botToken: string;
+ /** The bot owner's user id (`macro|<email>`); the bot acts as this user. */
+ actingUserId: string;
+ /** Macro environment. Defaults to prod. */
+ env?: Env;
+};
+
+/** The channel operations the bot needs. Every method throws on failure. */
+export type AlertChannel = {
+ /** Post a top-level message and return its id. */
+ post(body: RichMessage): Promise<string>;
+ /** Reply in the thread rooted at `messageId`. */
+ reply(messageId: string, body: RichMessage): Promise<void>;
+ react(messageId: string, emoji: string): Promise<void>;
+ edit(messageId: string, body: RichMessage): Promise<void>;
+};
+
+/**
+ * The alerts channel through the Macro SDK.
+ *
+ * The bot is user-owned and acts as its owner (`x-macro-bot-for-macro-user-id`,
+ * user scope), which routes posts through the entity message API: threads,
+ * mentions, edits and reactions. Messages show the bot as sender. A team-owned
+ * bot would drop `requestedAs` and use the default team scope instead.
+ */
+export function createAlertChannel(config: ChannelConfig): AlertChannel {
+ const bot = new Macro({
+ env: config.env ?? 'prod',
+ auth: { type: 'bot', token: config.botToken, scope: 'user' },
+ });
+ const macro = bot.requestedAs(bot.users.byId(config.actingUserId));
+ const channel = macro.channels.byId(config.channelId);
+
+ return {
+ post: async (body) => (await channel.send(body)).id,
+ reply: async (messageId, body) => {
+ await channel.message(messageId).reply(body);
+ },
+ react: async (messageId, emoji) => {
+ await channel.message(messageId).react(emoji);
+ },
+ edit: async (messageId, body) => {
+ await channel.message(messageId).edit(body);
+ },
+ };
+}
diff --git a/services/bots/goalert-bot/src/format.ts b/services/bots/goalert-bot/src/format.ts
new file mode 100644
index 0000000000..3667b4bf3f
--- /dev/null
+++ b/services/bots/goalert-bot/src/format.ts
@@ -0,0 +1,261 @@
+import {
+ here,
+ msg,
+ type RichMessage,
+ toBody,
+} from '../../../../packages/sdk/src/mentions';
+import type { AlertPayload, AlertStatusPayload } from './goalert';
+
+const MAX_DESCRIPTION_CHARS = 300;
+
+// Datadog titles start with bracketed tags: `[P3] [Re-Triggered on {host:x}] [PROD] Title`.
+const DATADOG_TAG = /^\s*\[([^\]]*)\]\s*/;
+const SCOPE = /on \{(.+)\}/;
+// Datadog's state line, from `alertMessage()` in infra/stacks/monitoring/notify.ts.
+const DATADOG_STATE = /^(🔴|🟠|✅)\s*(CRITICAL|WARNING|RECOVERED):\s*(.*)$/u;
+const GRAFANA_ROW = /^\|\s*([^|]+?)\s*\|\s*(.*?)\s*\|$/;
+// `macro-db-prod.ctkwyjgndnfr.us-east-1.rds.amazonaws.com` -> `macro-db-prod`.
+const AWS_HOSTNAME = /\b([a-z0-9-]+)\.[a-z0-9.-]+\.amazonaws\.com\b/g;
+const LINK_LINE =
+ /^(Runbook|Dashboard|Metric Graph|Source):\s*(https?:\/\/\S+)/;
+const NOT_A_DESCRIPTION = ['@', '`', 'http', 'Metric value', 'Paging on-call'];
+const GRAFANA_SEVERITY: Record<string, string> = {
+ critical: '🔴 critical',
+ warning: '🟠 warning',
+};
+
+/** A new-alert post, kept in parts so the headline can be rewritten on close. */
+export type CompactPost = {
+ /** Pages mention `@here`; the `Notify` service does not. */
+ page: boolean;
+ /** `🚨 **PAGE** · **<title> — <scope>**`, without the mention. */
+ headline: string;
+ /** State, description and links, one per line. */
+ lines: string[];
+};
+
+type ParsedDetails = {
+ state?: string;
+ description?: string;
+ scope?: string;
+ links: Record<string, string>;
+};
+
+/** Whether alerts on this GoAlert service page on-call. */
+export function isPage(serviceName: string): boolean {
+ return !serviceName.includes('Notify');
+}
+
+/** `host:macro-db-prod.abc.us-east-1.rds.amazonaws.com` -> `macro-db-prod`. */
+function shortScope(scope: string): string {
+ return scope
+ .split(',')
+ .map((part) => {
+ const value = part.slice(part.indexOf(':') + 1);
+ return value.includes('.amazonaws.com') ? value.split('.')[0] : value;
+ })
+ .join(', ');
+}
+
+/** The title without Datadog's bracketed tags, and the group it fired on. */
+function datadogTitle(summary: string): { title: string; scope: string } {
+ let scope = '';
+ let rest = summary;
+ for (
+ let match = DATADOG_TAG.exec(rest);
+ match;
+ match = DATADOG_TAG.exec(rest)
+ ) {
+ const on = SCOPE.exec(match[1]);
+ if (on) scope = shortScope(on[1]);
+ rest = rest.slice(match[0].length);
+ }
+ return { title: rest.trim(), scope };
+}
+
+/** The alert's cleaned-up title with the group it fired on. */
+export function alertTitle(summary: string): string {
+ const { title, scope } = datadogTitle(summary);
+ return scope ? `${title} — ${scope}` : title;
+}
+
+/** Collapse whitespace, shorten AWS hostnames, and cut at a sentence end. */
+export function firstSentences(text: string): string {
+ const flat = text.split(/\s+/).filter(Boolean).join(' ');
+ const short = flat.replace(AWS_HOSTNAME, '$1');
+ if (short.length <= MAX_DESCRIPTION_CHARS) return short;
+ const cut = short.slice(0, MAX_DESCRIPTION_CHARS);
+ const sentenceEnd = cut.lastIndexOf('. ');
+ const kept = sentenceEnd >= 0 ? cut.slice(0, sentenceEnd + 1) : cut.trimEnd();
+ return kept || cut;
+}
+
+/** State, description and links from a Datadog message or Grafana's details table. */
+function parseDetails(details: string): ParsedDetails {
+ const parsed: ParsedDetails = { links: {} };
+ const rows: Record<string, string> = {};
+ for (const raw of details.split('\n')) {
+ const line = raw.trim();
+ const row = GRAFANA_ROW.exec(line);
+ const state = DATADOG_STATE.exec(line);
+ const link = LINK_LINE.exec(line);
+ if (row) {
+ rows[row[1]] = row[2];
+ } else if (state) {
+ parsed.state = `${state[1]} ${state[2].toLowerCase()}: ${state[3]}`;
+ } else if (link) {
+ parsed.links[link[1]] ??= link[2];
+ } else if (
+ line &&
+ parsed.description === undefined &&
+ parsed.state !== undefined &&
+ !NOT_A_DESCRIPTION.some((prefix) => line.startsWith(prefix))
+ ) {
+ parsed.description = line;
+ }
+ }
+ if (Object.keys(rows).length > 0) {
+ const severity = rows.severity ?? '';
+ parsed.state = GRAFANA_SEVERITY[severity] ?? severity;
+ parsed.description = rows.description ?? '';
+ parsed.scope =
+ rows.DBInstanceIdentifier || rows.CacheClusterId || rows.QueueName || '';
+ if (rows.panel_url) parsed.links.Graph = rows.panel_url;
+ if (rows.runbook_url) parsed.links.Runbook = rows.runbook_url;
+ }
+ return parsed;
+}
+
+function markdownLinks(links: [string, string | undefined][]): string {
+ return links
+ .filter((link): link is [string, string] => Boolean(link[1]))
+ .map(([name, url]) => `[${name}](${url})`)
+ .join(' · ');
+}
+
+/**
+ * ```
+ * 🚨 **PAGE** · **macro-db-prod CPU high — macro-db-prod**
+ * 🔴 critical: 92 (threshold 85)
+ * macro-db-prod CPU averaged 92% over 15 minutes.
+ * [Acknowledge](…) · [Graph](…) · [Runbook](…)
+ * ```
+ */
+export function compactAlert(
+ payload: Pick<
+ AlertPayload,
+ 'Summary' | 'Details' | 'ServiceName' | 'GoAlertURL'
+ >
+): CompactPost {
+ const page = isPage(payload.ServiceName);
+ const parsed = parseDetails(payload.Details);
+ let title = alertTitle(payload.Summary);
+ if (parsed.scope && !title.includes(parsed.scope)) {
+ title = `${title} — ${parsed.scope}`;
+ }
+ const lines: string[] = [];
+ if (parsed.state) lines.push(parsed.state);
+ if (parsed.description) lines.push(firstSentences(parsed.description));
+ lines.push(
+ markdownLinks([
+ ['Acknowledge', payload.GoAlertURL],
+ [
+ 'Graph',
+ parsed.links.Graph ||
+ parsed.links.Dashboard ||
+ parsed.links['Metric Graph'],
+ ],
+ ['Runbook', parsed.links.Runbook],
+ ])
+ );
+ return {
+ page,
+ headline: `${page ? '🚨 **PAGE**' : '🔔'} · **${title}**`,
+ lines,
+ };
+}
+
+/** The post body: a separator, the headline (`@here` on open pages), and the rest. */
+export function renderPost(post: CompactPost, resolved = false): RichMessage {
+ const rest = post.lines.join('\n');
+ if (resolved) {
+ // The mention is dropped so the edit does not notify everyone again.
+ return toBody(`✅ Resolved · ${post.headline}\n${rest}`);
+ }
+ if (post.page) {
+ return msg`${post.headline} ${here}\n${rest}`;
+ }
+ return toBody(`${post.headline}\n${rest}`);
+}
+
+/** What an `AlertStatus` log entry did to the alert. */
+export type StatusChange = 'acknowledged' | 'closed' | 'escalated' | 'other';
+
+export function statusChange(logEntry: string): StatusChange {
+ const entry = logEntry.toLowerCase();
+ if (entry.startsWith('acknowledged')) return 'acknowledged';
+ if (entry.startsWith('closed')) return 'closed';
+ if (entry.startsWith('escalated')) return 'escalated';
+ return 'other';
+}
+
+const STATUS_EMOJI: Record<StatusChange, string> = {
+ acknowledged: '👀',
+ closed: '✅',
+ escalated: '⏫',
+ other: 'ℹ️',
+};
+
+/** One thread line for a status change: `👀 Acknowledged by Sam`. */
+export function statusReply(logEntry: string): RichMessage {
+ return toBody(`${STATUS_EMOJI[statusChange(logEntry)]} ${logEntry}`);
+}
+
+/** GoAlert re-sent an open alert (hourly re-escalation of an unacknowledged page). */
+export function stillOpenReply(
+ payload: Pick<AlertPayload, 'ServiceName' | 'GoAlertURL'>
+): RichMessage {
+ const line = `⏰ Still open · ${markdownLinks([['Acknowledge', payload.GoAlertURL]])}`;
+ return isPage(payload.ServiceName) ? msg`${line} ${here}` : toBody(line);
+}
+
+/** A status change for an alert this bot never posted, as its own one-line post. */
+export function statusPost(payload: AlertStatusPayload): RichMessage {
+ const parts = [
+ payload.LogEntry,
+ alertTitle(payload.Summary),
+ markdownLinks([['Open', payload.GoAlertURL]]),
+ ].filter(Boolean);
+ return toBody(`${parts.join(' · ')}`);
+}
+
+export function alertBundlePost(payload: {
+ ServiceName: string;
+ Count: number;
+ GoAlertURL: string;
+}): RichMessage {
+ const page = isPage(payload.ServiceName);
+ const line = `${page ? '🚨 **PAGE**' : '🔔'} · ${payload.Count} alerts on ${payload.ServiceName} · ${markdownLinks([['Acknowledge', payload.GoAlertURL]])}`;
+ return page ? msg`${line} ${here}` : toBody(`${line}`);
+}
+
+export function alertStatusBundlePost(payload: {
+ LogEntry: string;
+ Count: number;
+ GoAlertURL: string;
+}): RichMessage {
+ return toBody(
+ `${payload.LogEntry} (${payload.Count} alerts) · ${markdownLinks([['Open', payload.GoAlertURL]])}`
+ );
+}
+
+export function onCallPost(payload: {
+ ScheduleName: string;
+ ScheduleURL: string;
+ Users: { Name: string }[];
+}): RichMessage {
+ const names = payload.Users.map((user) => user.Name).join(', ');
+ return toBody(
+ `On call for ${payload.ScheduleName}: ${names || 'nobody'}\n${payload.ScheduleURL}`
+ );
+}
diff --git a/services/bots/goalert-bot/src/goalert.ts b/services/bots/goalert-bot/src/goalert.ts
new file mode 100644
index 0000000000..f002ad875d
--- /dev/null
+++ b/services/bots/goalert-bot/src/goalert.ts
@@ -0,0 +1,83 @@
+import { z } from 'zod';
+
+/**
+ * GoAlert outgoing webhook payloads (`notification/webhook` in GoAlert).
+ *
+ * Every payload carries `AppName` and a `Type` naming its kind. Schemas are
+ * loose and text fields default to empty, because GoAlert omits fields that do
+ * not apply and may add new ones.
+ */
+
+const text = z.string().default('');
+
+const alertSchema = z.looseObject({
+ Type: z.literal('Alert'),
+ AlertID: z.number(),
+ Summary: text,
+ Details: text,
+ ServiceID: text,
+ ServiceName: text,
+ Meta: z.record(z.string(), z.string()).nullish(),
+ GoAlertURL: text,
+});
+
+const alertStatusSchema = z.looseObject({
+ Type: z.literal('AlertStatus'),
+ AlertID: z.number(),
+ Summary: text,
+ ServiceID: text,
+ ServiceName: text,
+ LogEntry: text,
+ GoAlertURL: text,
+});
+
+const alertBundleSchema = z.looseObject({
+ Type: z.literal('AlertBundle'),
+ ServiceID: text,
+ ServiceName: text,
+ Count: z.number(),
+ GoAlertURL: text,
+});
+
+const alertStatusBundleSchema = z.looseObject({
+ Type: z.literal('AlertStatusBundle'),
+ ServiceID: text,
+ ServiceName: text,
+ LogEntry: text,
+ Count: z.number(),
+ GoAlertURL: text,
+});
+
+const scheduleOnCallUsersSchema = z.looseObject({
+ Type: z.literal('ScheduleOnCallUsers'),
+ ScheduleName: text,
+ ScheduleURL: text,
+ Users: z.array(z.looseObject({ Name: text })).default([]),
+});
+
+const testSchema = z.looseObject({ Type: z.literal('Test') });
+
+const verificationSchema = z.looseObject({
+ Type: z.literal('Verification'),
+ Code: z.union([z.string(), z.number()]).optional(),
+});
+
+const goAlertPayloadSchema = z.discriminatedUnion('Type', [
+ alertSchema,
+ alertStatusSchema,
+ alertBundleSchema,
+ alertStatusBundleSchema,
+ scheduleOnCallUsersSchema,
+ testSchema,
+ verificationSchema,
+]);
+
+export type AlertPayload = z.infer<typeof alertSchema>;
+export type AlertStatusPayload = z.infer<typeof alertStatusSchema>;
+export type GoAlertPayload = z.infer<typeof goAlertPayloadSchema>;
+
+/** Parse a webhook body, or `undefined` for anything this bot does not know. */
+export function parseGoAlertPayload(body: unknown): GoAlertPayload | undefined {
+ const result = goAlertPayloadSchema.safeParse(body);
+ return result.success ? result.data : undefined;
+}
diff --git a/services/bots/goalert-bot/src/index.ts b/services/bots/goalert-bot/src/index.ts
new file mode 100644
index 0000000000..402712c24a
--- /dev/null
+++ b/services/bots/goalert-bot/src/index.ts
@@ -0,0 +1,192 @@
+import { createHash, timingSafeEqual } from 'node:crypto';
+import { Telemetry } from '@macro-inc/observability';
+import { Hono } from 'hono';
+import { toBody } from '../../../../packages/sdk/src/mentions';
+import type { AlertChannel } from './channel';
+import {
+ alertBundlePost,
+ alertStatusBundlePost,
+ compactAlert,
+ onCallPost,
+ renderPost,
+ statusChange,
+ statusPost,
+ statusReply,
+ stillOpenReply,
+} from './format';
+import {
+ type AlertPayload,
+ type AlertStatusPayload,
+ type GoAlertPayload,
+ parseGoAlertPayload,
+} from './goalert';
+import type { AlertStore } from './store';
+import { tracedChannel, tracedStore } from './traced';
+
+const MAX_BODY_BYTES = 256 * 1024;
+
+export type AppConfig = {
+ /** Shared secret embedded in the webhook URL path. */
+ webhookSecret: string;
+ channel: AlertChannel;
+ store: AlertStore;
+};
+
+/** What the bot did with a webhook, for logs and the response body. */
+type Outcome =
+ | 'posted'
+ | 'still_open_replied'
+ | 'status_replied'
+ | 'status_posted'
+ | 'bundle_posted'
+ | 'notice_posted';
+
+function sameSecret(given: string, expected: string): boolean {
+ // Hashing first gives equal lengths, which timingSafeEqual requires.
+ const digest = (value: string) => createHash('sha256').update(value).digest();
+ return timingSafeEqual(digest(given), digest(expected));
+}
+
+async function handleAlert(
+ config: AppConfig,
+ payload: AlertPayload
+): Promise<Outcome> {
+ const posted = await config.store.get(payload.AlertID);
+ if (posted) {
+ await config.channel.reply(posted.messageId, stillOpenReply(payload));
+ return 'still_open_replied';
+ }
+ const post = compactAlert(payload);
+ const messageId = await config.channel.post(renderPost(post));
+ try {
+ await config.store.put(payload.AlertID, { messageId, post });
+ } catch {
+ // The post is up and the kv.put span records the failure; later status
+ // changes fall back to top-level posts.
+ }
+ return 'posted';
+}
+
+async function handleAlertStatus(
+ config: AppConfig,
+ payload: AlertStatusPayload
+): Promise<Outcome> {
+ const posted = await config.store.get(payload.AlertID);
+ if (!posted) {
+ await config.channel.post(statusPost(payload));
+ return 'status_posted';
+ }
+ const { channel } = config;
+ const change = statusChange(payload.LogEntry);
+ const updates = [
+ channel.reply(posted.messageId, statusReply(payload.LogEntry)),
+ ];
+ if (change === 'acknowledged') {
+ updates.push(channel.react(posted.messageId, '👀'));
+ } else if (change === 'closed') {
+ updates.push(
+ channel.react(posted.messageId, '✅'),
+ channel.edit(posted.messageId, renderPost(posted.post, true))
+ );
+ }
+ await Promise.all(updates);
+ return 'status_replied';
+}
+
+async function handle(
+ config: AppConfig,
+ payload: GoAlertPayload
+): Promise<Outcome> {
+ const { channel } = config;
+ if (payload.Type === 'Alert') return handleAlert(config, payload);
+ if (payload.Type === 'AlertStatus') return handleAlertStatus(config, payload);
+ if (payload.Type === 'AlertBundle') {
+ await channel.post(alertBundlePost(payload));
+ return 'bundle_posted';
+ }
+ if (payload.Type === 'AlertStatusBundle') {
+ await channel.post(alertStatusBundlePost(payload));
+ return 'bundle_posted';
+ }
+ if (payload.Type === 'ScheduleOnCallUsers') {
+ await channel.post(onCallPost(payload));
+ return 'notice_posted';
+ }
+ if (payload.Type === 'Verification') {
+ await channel.post(toBody(`GoAlert verification code: ${payload.Code}`));
+ return 'notice_posted';
+ }
+ await channel.post(toBody('GoAlert test message for the Macro Alerts bot.'));
+ return 'notice_posted';
+}
+
+/**
+ * GoAlert → Macro Alerts channel bot.
+ *
+ * GoAlert's outgoing webhooks cannot set headers, so the URL carries a shared
+ * secret: POST /goalert/{WEBHOOK_SECRET}. GoAlert abandons a webhook after
+ * about three seconds, so each request does at most a store read, the channel
+ * calls (in parallel), and a store write.
+ *
+ * Unrecognized payloads are acknowledged and ignored; a failed channel call
+ * returns 502 so it shows in GoAlert's logs.
+ */
+export function createApp(config: AppConfig) {
+ const app = new Hono();
+ const tracedConfig: AppConfig = {
+ ...config,
+ channel: tracedChannel(config.channel),
+ store: tracedStore(config.store),
+ };
+
+ app.get('/health', (c) => c.json({ ok: true }));
+
+ app.post('/goalert/:secret', async (c) => {
+ if (!sameSecret(c.req.param('secret'), config.webhookSecret)) {
+ return c.json({ error: 'unauthorized' }, 401);
+ }
+
+ const raw = await c.req.text();
+ if (raw.length > MAX_BODY_BYTES) {
+ return c.json({ error: 'body too large' }, 413);
+ }
+ let body: unknown;
+ try {
+ body = JSON.parse(raw);
+ } catch {
+ return c.json({ error: 'invalid JSON body' }, 400);
+ }
+
+ const payload = parseGoAlertPayload(body);
+ if (!payload) {
+ console.warn('Ignoring unrecognized GoAlert webhook payload');
+ return c.json({ ok: true, ignored: true });
+ }
+
+ return Telemetry.span('goalert_bot.webhook', async (span) => {
+ span.setAttr('goalert.payload_type', payload.Type);
+ if (payload.Type === 'Alert' || payload.Type === 'AlertStatus') {
+ span.setAttr('goalert.alert_id', payload.AlertID);
+ }
+ if (
+ payload.Type === 'Alert' ||
+ payload.Type === 'AlertStatus' ||
+ payload.Type === 'AlertBundle' ||
+ payload.Type === 'AlertStatusBundle'
+ ) {
+ span.setAttr('goalert.service_name', payload.ServiceName);
+ }
+ try {
+ const outcome = await handle(tracedConfig, payload);
+ span.setAttr('goalert.outcome', outcome);
+ return c.json({ ok: true, outcome });
+ } catch (error) {
+ span.setAttr('goalert.outcome', 'failed');
+ span.error(error);
+ return c.json({ error: 'failed to deliver message' }, 502);
+ }
+ });
+ });
+
+ return app;
+}
diff --git a/services/bots/goalert-bot/src/server.ts b/services/bots/goalert-bot/src/server.ts
new file mode 100644
index 0000000000..fa770def68
--- /dev/null
+++ b/services/bots/goalert-bot/src/server.ts
@@ -0,0 +1,36 @@
+import { createAlertChannel } from './channel';
+import { createApp } from './index';
+import { createMemoryAlertStore } from './store';
+
+const PORT = parseInt(process.env.PORT || '8089', 10);
+
+const requiredVars = [
+ 'BOT_TOKEN',
+ 'CHANNEL_ID',
+ 'ACTING_USER_ID',
+ 'WEBHOOK_SECRET',
+] as const;
+for (const varName of requiredVars) {
+ if (!process.env[varName]) {
+ console.error(`Missing required environment variable: ${varName}`);
+ process.exit(1);
+ }
+}
+
+const app = createApp({
+ webhookSecret: process.env.WEBHOOK_SECRET!,
+ channel: createAlertChannel({
+ channelId: process.env.CHANNEL_ID!,
+ botToken: process.env.BOT_TOKEN!,
+ actingUserId: process.env.ACTING_USER_ID!,
+ env: (process.env.MACRO_ENV as 'prod' | 'dev' | 'local') || 'prod',
+ }),
+ store: createMemoryAlertStore(),
+});
+
+const server = Bun.serve({
+ port: PORT,
+ fetch: app.fetch,
+});
+
+console.log(`goalert-bot listening on http://localhost:${server.port}`);
diff --git a/services/bots/goalert-bot/src/store.ts b/services/bots/goalert-bot/src/store.ts
new file mode 100644
index 0000000000..d0d836638b
--- /dev/null
+++ b/services/bots/goalert-bot/src/store.ts
@@ -0,0 +1,65 @@
+import { z } from 'zod';
+
+/** How long a posted alert's thread stays reachable by later status changes. */
+const ALERT_TTL_SECONDS = 30 * 24 * 60 * 60;
+
+const postedAlertSchema = z.object({
+ messageId: z.string(),
+ // Kept so a close can rewrite the headline without fetching the message.
+ post: z.object({
+ page: z.boolean(),
+ headline: z.string(),
+ lines: z.array(z.string()),
+ }),
+});
+
+/** The channel message an alert was posted as. */
+export type PostedAlert = z.infer<typeof postedAlertSchema>;
+
+/** GoAlert alert id → the message it was posted as. */
+export type AlertStore = {
+ get(alertId: number): Promise<PostedAlert | undefined>;
+ put(alertId: number, posted: PostedAlert): Promise<void>;
+};
+
+/** The subset of a Workers KV namespace binding this store uses. */
+export type KeyValueNamespace = {
+ get(key: string, type: 'json'): Promise<unknown>;
+ put(
+ key: string,
+ value: string,
+ options: { expirationTtl: number }
+ ): Promise<void>;
+};
+
+function key(alertId: number): string {
+ return `alert:${alertId}`;
+}
+
+/** Store backed by a Workers KV namespace; entries expire after 30 days. */
+export function createKvAlertStore(kv: KeyValueNamespace): AlertStore {
+ return {
+ async get(alertId) {
+ const value = await kv.get(key(alertId), 'json');
+ if (value === null) return undefined;
+ const parsed = postedAlertSchema.safeParse(value);
+ return parsed.success ? parsed.data : undefined;
+ },
+ async put(alertId, posted) {
+ await kv.put(key(alertId), JSON.stringify(posted), {
+ expirationTtl: ALERT_TTL_SECONDS,
+ });
+ },
+ };
+}
+
+/** In-process store for the local Bun server and tests. */
+export function createMemoryAlertStore(): AlertStore {
+ const entries = new Map<number, PostedAlert>();
+ return {
+ get: async (alertId) => entries.get(alertId),
+ put: async (alertId, posted) => {
+ entries.set(alertId, posted);
+ },
+ };
+}
diff --git a/services/bots/goalert-bot/src/traced.ts b/services/bots/goalert-bot/src/traced.ts
new file mode 100644
index 0000000000..db65e12f80
--- /dev/null
+++ b/services/bots/goalert-bot/src/traced.ts
@@ -0,0 +1,72 @@
+import { type Span, Telemetry } from '@macro-inc/observability';
+import type { AlertChannel } from './channel';
+import type { AlertStore } from './store';
+
+/** Run `operation` in a child span named `name`, marking it failed on error. */
+async function traced<T>(
+ name: string,
+ attributes: Record<string, string | number>,
+ operation: (span: Span) => Promise<T>
+): Promise<T> {
+ return Telemetry.span(name, async (span) => {
+ for (const [key, value] of Object.entries(attributes)) {
+ span.setAttr(key, value);
+ }
+ try {
+ return await operation(span);
+ } catch (error) {
+ span.error(error);
+ throw error;
+ }
+ });
+}
+
+/** The channel with one span per call. Bodies are not recorded. */
+export function tracedChannel(channel: AlertChannel): AlertChannel {
+ return {
+ post: (body) =>
+ traced('goalert_bot.channel.post', {}, async (span) => {
+ const messageId = await channel.post(body);
+ span.setAttr('goalert.message_id', messageId);
+ return messageId;
+ }),
+ reply: (messageId, body) =>
+ traced(
+ 'goalert_bot.channel.reply',
+ { 'goalert.message_id': messageId },
+ () => channel.reply(messageId, body)
+ ),
+ react: (messageId, emoji) =>
+ traced(
+ 'goalert_bot.channel.react',
+ { 'goalert.message_id': messageId },
+ () => channel.react(messageId, emoji)
+ ),
+ edit: (messageId, body) =>
+ traced(
+ 'goalert_bot.channel.edit',
+ { 'goalert.message_id': messageId },
+ () => channel.edit(messageId, body)
+ ),
+ };
+}
+
+/** The alert store with one span per KV call, recording hit or miss. */
+export function tracedStore(store: AlertStore): AlertStore {
+ return {
+ get: (alertId) =>
+ traced(
+ 'goalert_bot.kv.get',
+ { 'goalert.alert_id': alertId },
+ async (span) => {
+ const posted = await store.get(alertId);
+ span.setAttr('goalert.found', posted !== undefined);
+ return posted;
+ }
+ ),
+ put: (alertId, posted) =>
+ traced('goalert_bot.kv.put', { 'goalert.alert_id': alertId }, () =>
+ store.put(alertId, posted)
+ ),
+ };
+}
diff --git a/services/bots/goalert-bot/src/worker.ts b/services/bots/goalert-bot/src/worker.ts
new file mode 100644
index 0000000000..2626934260
--- /dev/null
+++ b/services/bots/goalert-bot/src/worker.ts
@@ -0,0 +1,75 @@
+import {
+ createWorkerTraceConfig,
+ instrument,
+} from '@macro-inc/observability/worker';
+import { createAlertChannel } from './channel';
+import { createApp } from './index';
+import { createKvAlertStore, type KeyValueNamespace } from './store';
+
+/**
+ * Cloudflare Worker entrypoint. Config comes from worker bindings
+ * (vars + secrets + the KV namespace); see wrangler.jsonc. For local
+ * development outside wrangler, src/server.ts provides the same app under Bun.
+ */
+type Env = {
+ BOT_TOKEN: string;
+ CHANNEL_ID: string;
+ ACTING_USER_ID: string;
+ WEBHOOK_SECRET: string;
+ ALERT_MESSAGES: KeyValueNamespace;
+ /** Tags exported spans with deployment.environment (matches DD_ENV). */
+ ENVIRONMENT: string | undefined;
+ /** OTLP/HTTP traces endpoint; unset disables trace export. */
+ OTEL_EXPORTER_OTLP_ENDPOINT: string | undefined;
+ /** Datadog API key attached to trace exports when the endpoint is a
+ * Datadog intake; secret, set via `wrangler secret put DD_API_KEY`. */
+ DD_API_KEY: string | undefined;
+};
+
+const REQUIRED = [
+ 'BOT_TOKEN',
+ 'CHANNEL_ID',
+ 'ACTING_USER_ID',
+ 'WEBHOOK_SECRET',
+ 'ALERT_MESSAGES',
+] as const satisfies readonly (keyof Env)[];
+
+// One app per isolate, so the SDK's cached bot lookup survives across requests.
+let app: ReturnType<typeof createApp> | undefined;
+
+function fetch(request: Request, env: Env): Response | Promise<Response> {
+ const missing = REQUIRED.filter((key) => !env[key]);
+ if (missing.length > 0) {
+ return Response.json(
+ { error: `missing worker bindings: ${missing.join(', ')}` },
+ { status: 500 }
+ );
+ }
+
+ app ??= createApp({
+ webhookSecret: env.WEBHOOK_SECRET,
+ channel: createAlertChannel({
+ channelId: env.CHANNEL_ID,
+ botToken: env.BOT_TOKEN,
+ actingUserId: env.ACTING_USER_ID,
+ env: 'prod',
+ }),
+ store: createKvAlertStore(env.ALERT_MESSAGES),
+ });
+
+ return app.fetch(request, env);
+}
+
+// instrument() wraps fetch with a request root span, registers the global
+// tracer provider/context manager (which Telemetry.span resolves through),
+// propagates traceparent, and flushes spans via waitUntil after the response.
+export default instrument({ fetch }, (env: Env) =>
+ createWorkerTraceConfig({
+ serviceName: 'goalert-bot',
+ environment: env.ENVIRONMENT ?? 'local',
+ tracesUrl: env.OTEL_EXPORTER_OTLP_ENDPOINT,
+ tracesHeaders: env.DD_API_KEY
+ ? { 'dd-api-key': env.DD_API_KEY }
+ : undefined,
+ })
+);
diff --git a/services/bots/goalert-bot/test/format.test.ts b/services/bots/goalert-bot/test/format.test.ts
new file mode 100644
index 0000000000..df15881c6d
--- /dev/null
+++ b/services/bots/goalert-bot/test/format.test.ts
@@ -0,0 +1,120 @@
+import { describe, expect, test } from 'bun:test';
+import {
+ compactAlert,
+ firstSentences,
+ renderPost,
+ statusPost,
+} from '../src/format';
+
+// A real Datadog monitor message and Grafana alert details, as GoAlert
+// forwards them in `Details`.
+const DATADOG_DETAILS = `https://us5.datadoghq.com/event/event?id=8876996053944573810
+
+🔴 CRITICAL: 2631.578 (threshold 1800.0)
+
+The oldest open transaction on macro-db-prod.ctkwyjgndnfr.us-east-1.rds.amazonaws.com is 2631.578 seconds old. Long transactions hold locks and stop vacuum.
+
+Runbook: https://github.com/macro-inc/macro/blob/main/docs/runbooks/macrodb.md#locks
+Dashboard: https://us5.datadoghq.com/dashboard/42x-f55-h6z/macrodb-alerts
+
+@webhook-goalert-notify
+
+\`avg(last_10m):max:postgresql.activity.xact_start_age{host:macro-db-prod*} by {host} > 1800\`
+
+Metric value: 2631.578
+
+Metric Graph: https://us5.datadoghq.com/monitors/23404621?group=host · Monitor Status: https://us5.datadoghq.com/monitors/23404621`;
+
+const GRAFANA_DETAILS = `| Label | Value |
+| ----- | ----- |
+| DBInstanceIdentifier | macro-db-prod-read-replica |
+| alertname | [PROD] macro-db-prod database load above vCPU count (warning) |
+| pages | false |
+| severity | warning |
+| Annotation | Value |
+| ---------- | ----- |
+| description | macro-db-prod-read-replica has 3.98x as many active sessions as vCPUs. Normal is under 1.3. |
+| panel_url | https://grafana.macro-internal.com/d/prod-alerts?viewPanel=4 |
+| runbook_url | https://github.com/macro-inc/macro/blob/main/docs/runbooks/macrodb.md#load |
+| summary | macro-db-prod database load above vCPU count |
+
+Source: https://grafana.macro-internal.com/alerting/grafana/macrodb-load-warn/view?orgId=1
+
+\`\`\`
+[ var='B' labels={} type='reduce' value=3.98 ]
+\`\`\``;
+
+const HERE_TAG = '<m-group-mention>{"groupAlias":"here"}</m-group-mention>';
+
+describe('compact alert posts', () => {
+ test('Datadog notify alert', () => {
+ const post = compactAlert({
+ ServiceName: 'Notify',
+ Summary:
+ '[P3] [Triggered on {host:macro-db-prod.ctkwyjgndnfr.us-east-1.rds.amazonaws.com}] [PROD] macro-db-prod long-running transaction',
+ Details: DATADOG_DETAILS,
+ GoAlertURL: 'https://oncall.macro-internal.com/alerts/130',
+ });
+ expect(renderPost(post)).toEqual({
+ content: `🔔 · **macro-db-prod long-running transaction — macro-db-prod**
+🔴 critical: 2631.578 (threshold 1800.0)
+The oldest open transaction on macro-db-prod is 2631.578 seconds old. Long transactions hold locks and stop vacuum.
+[Acknowledge](https://oncall.macro-internal.com/alerts/130) · [Graph](https://us5.datadoghq.com/dashboard/42x-f55-h6z/macrodb-alerts) · [Runbook](https://github.com/macro-inc/macro/blob/main/docs/runbooks/macrodb.md#locks)`,
+ mentions: [],
+ });
+ });
+
+ test('Grafana page mentions here', () => {
+ const post = compactAlert({
+ ServiceName: 'Grafana critical',
+ Summary: 'macro-db-prod database load above vCPU count',
+ Details: GRAFANA_DETAILS,
+ GoAlertURL: 'https://oncall.macro-internal.com/alerts/127',
+ });
+ expect(renderPost(post)).toEqual({
+ content: `🚨 **PAGE** · **macro-db-prod database load above vCPU count — macro-db-prod-read-replica** ${HERE_TAG}
+🟠 warning
+macro-db-prod-read-replica has 3.98x as many active sessions as vCPUs. Normal is under 1.3.
+[Acknowledge](https://oncall.macro-internal.com/alerts/127) · [Graph](https://grafana.macro-internal.com/d/prod-alerts?viewPanel=4) · [Runbook](https://github.com/macro-inc/macro/blob/main/docs/runbooks/macrodb.md#load)`,
+ mentions: [{ entity_type: 'group', entity_id: 'here' }],
+ });
+ });
+
+ test('resolved page keeps the post and drops the mention', () => {
+ const post = compactAlert({
+ ServiceName: 'Grafana critical',
+ Summary: 'macro-db-prod database load above vCPU count',
+ Details: GRAFANA_DETAILS,
+ GoAlertURL: 'https://oncall.macro-internal.com/alerts/127',
+ });
+ expect(renderPost(post, true)).toEqual({
+ content: `✅ Resolved · 🚨 **PAGE** · **macro-db-prod database load above vCPU count — macro-db-prod-read-replica**
+🟠 warning
+macro-db-prod-read-replica has 3.98x as many active sessions as vCPUs. Normal is under 1.3.
+[Acknowledge](https://oncall.macro-internal.com/alerts/127) · [Graph](https://grafana.macro-internal.com/d/prod-alerts?viewPanel=4) · [Runbook](https://github.com/macro-inc/macro/blob/main/docs/runbooks/macrodb.md#load)`,
+ mentions: [],
+ });
+ });
+
+ test('status change without a posted alert is one line', () => {
+ expect(
+ statusPost({
+ Type: 'AlertStatus',
+ AlertID: 48,
+ ServiceID: '',
+ ServiceName: 'Notify',
+ Summary: '[P3] [Recovered on {tablename:events}] DynamoDB reads high',
+ LogEntry: 'Closed by Datadog',
+ GoAlertURL: 'https://oncall.macro-internal.com/alerts/48',
+ })
+ ).toEqual({
+ content: `Closed by Datadog · DynamoDB reads high — events · [Open](https://oncall.macro-internal.com/alerts/48)`,
+ mentions: [],
+ });
+ });
+
+ test('long description is cut at a sentence', () => {
+ const sentence = `${'A'.repeat(200)}. `;
+ expect(firstSentences(sentence.repeat(3))).toBe(`${'A'.repeat(200)}.`);
+ });
+});
diff --git a/services/bots/goalert-bot/test/webhook.test.ts b/services/bots/goalert-bot/test/webhook.test.ts
new file mode 100644
index 0000000000..0a43e279b1
--- /dev/null
+++ b/services/bots/goalert-bot/test/webhook.test.ts
@@ -0,0 +1,261 @@
+import { describe, expect, test } from 'bun:test';
+import type { RichMessage } from '../../../../packages/sdk/src/mentions';
+import type { AlertChannel } from '../src/channel';
+import { createApp } from '../src/index';
+import { type AlertStore, createMemoryAlertStore } from '../src/store';
+
+type ChannelCall =
+ | { kind: 'post'; body: RichMessage }
+ | { kind: 'reply'; messageId: string; body: RichMessage }
+ | { kind: 'react'; messageId: string; emoji: string }
+ | { kind: 'edit'; messageId: string; body: RichMessage };
+
+/** Records every channel call; posts get ids `message-1`, `message-2`, … */
+function fakeChannel(): AlertChannel & { calls: ChannelCall[] } {
+ const calls: ChannelCall[] = [];
+ return {
+ calls,
+ post: async (body) => {
+ calls.push({ kind: 'post', body });
+ return `message-${calls.filter((call) => call.kind === 'post').length}`;
+ },
+ reply: async (messageId, body) => {
+ calls.push({ kind: 'reply', messageId, body });
+ },
+ react: async (messageId, emoji) => {
+ calls.push({ kind: 'react', messageId, emoji });
+ },
+ edit: async (messageId, body) => {
+ calls.push({ kind: 'edit', messageId, body });
+ },
+ };
+}
+
+function makeApp(channel: AlertChannel, store: AlertStore) {
+ return createApp({ webhookSecret: 'test-secret', channel, store });
+}
+
+function postWebhook(
+ app: ReturnType<typeof makeApp>,
+ body: unknown,
+ secret = 'test-secret'
+) {
+ return app.request(`/goalert/${secret}`, {
+ method: 'POST',
+ headers: { 'content-type': 'application/json' },
+ body: typeof body === 'string' ? body : JSON.stringify(body),
+ });
+}
+
+const HERE_TAG = '<m-group-mention>{"groupAlias":"here"}</m-group-mention>';
+
+const PAGE_ALERT = {
+ AppName: 'GoAlert',
+ Type: 'Alert',
+ AlertID: 127,
+ Summary: 'macro-db-prod CPU high',
+ Details: '| severity | critical |\n| description | CPU at 92%. |',
+ ServiceID: 'b4b2c1a6-0000-4000-8000-000000000001',
+ ServiceName: 'Grafana critical',
+ Meta: {},
+ GoAlertURL: 'https://oncall.macro-internal.com/alerts/127',
+};
+
+const PAGE_POST = {
+ page: true,
+ headline: '🚨 **PAGE** · **macro-db-prod CPU high**',
+ lines: [
+ '🔴 critical',
+ 'CPU at 92%.',
+ '[Acknowledge](https://oncall.macro-internal.com/alerts/127)',
+ ],
+};
+
+function statusFor(alertId: number, logEntry: string) {
+ return {
+ AppName: 'GoAlert',
+ Type: 'AlertStatus',
+ AlertID: alertId,
+ Summary: 'macro-db-prod CPU high',
+ ServiceID: 'b4b2c1a6-0000-4000-8000-000000000001',
+ ServiceName: 'Grafana critical',
+ LogEntry: logEntry,
+ GoAlertURL: `https://oncall.macro-internal.com/alerts/${alertId}`,
+ };
+}
+
+describe('goalert-bot webhook', () => {
+ test('health check', async () => {
+ const app = makeApp(fakeChannel(), createMemoryAlertStore());
+ const res = await app.request('/health');
+ expect(res.status).toBe(200);
+ expect(await res.json()).toEqual({ ok: true });
+ });
+
+ test('rejects a wrong secret without posting', async () => {
+ const channel = fakeChannel();
+ const app = makeApp(channel, createMemoryAlertStore());
+ const res = await postWebhook(app, PAGE_ALERT, 'wrong-secret');
+ expect(res.status).toBe(401);
+ expect(channel.calls).toEqual([]);
+ });
+
+ test('rejects invalid JSON', async () => {
+ const channel = fakeChannel();
+ const app = makeApp(channel, createMemoryAlertStore());
+ const res = await postWebhook(app, 'not json{');
+ expect(res.status).toBe(400);
+ expect(channel.calls).toEqual([]);
+ });
+
+ test('acknowledges and ignores unrecognized payloads', async () => {
+ const channel = fakeChannel();
+ const app = makeApp(channel, createMemoryAlertStore());
+ const res = await postWebhook(app, { Type: 'SomethingNew' });
+ expect(res.status).toBe(200);
+ expect(await res.json()).toEqual({ ok: true, ignored: true });
+ expect(channel.calls).toEqual([]);
+ });
+
+ test('new alert is posted and stored', async () => {
+ const channel = fakeChannel();
+ const store = createMemoryAlertStore();
+ const res = await postWebhook(makeApp(channel, store), PAGE_ALERT);
+ expect(res.status).toBe(200);
+ expect(await res.json()).toEqual({ ok: true, outcome: 'posted' });
+ expect(channel.calls).toEqual([
+ {
+ kind: 'post',
+ body: {
+ content: `🚨 **PAGE** · **macro-db-prod CPU high** ${HERE_TAG}\n🔴 critical\nCPU at 92%.\n[Acknowledge](https://oncall.macro-internal.com/alerts/127)`,
+ mentions: [{ entity_type: 'group', entity_id: 'here' }],
+ },
+ },
+ ]);
+ expect(await store.get(127)).toEqual({
+ messageId: 'message-1',
+ post: PAGE_POST,
+ });
+ });
+
+ test('repeated alert replies in its thread', async () => {
+ const channel = fakeChannel();
+ const store = createMemoryAlertStore();
+ await store.put(127, { messageId: 'message-9', post: PAGE_POST });
+ const res = await postWebhook(makeApp(channel, store), PAGE_ALERT);
+ expect(await res.json()).toEqual({
+ ok: true,
+ outcome: 'still_open_replied',
+ });
+ expect(channel.calls).toEqual([
+ {
+ kind: 'reply',
+ messageId: 'message-9',
+ body: {
+ content: `⏰ Still open · [Acknowledge](https://oncall.macro-internal.com/alerts/127) ${HERE_TAG}`,
+ mentions: [{ entity_type: 'group', entity_id: 'here' }],
+ },
+ },
+ ]);
+ });
+
+ test('acknowledgement replies and reacts', async () => {
+ const channel = fakeChannel();
+ const store = createMemoryAlertStore();
+ await store.put(127, { messageId: 'message-9', post: PAGE_POST });
+ const res = await postWebhook(
+ makeApp(channel, store),
+ statusFor(127, 'Acknowledged by Sam')
+ );
+ expect(await res.json()).toEqual({ ok: true, outcome: 'status_replied' });
+ expect(channel.calls).toEqual([
+ {
+ kind: 'reply',
+ messageId: 'message-9',
+ body: { content: '👀 Acknowledged by Sam', mentions: [] },
+ },
+ { kind: 'react', messageId: 'message-9', emoji: '👀' },
+ ]);
+ });
+
+ test('close replies, reacts, and marks the post resolved', async () => {
+ const channel = fakeChannel();
+ const store = createMemoryAlertStore();
+ await store.put(127, { messageId: 'message-9', post: PAGE_POST });
+ const res = await postWebhook(
+ makeApp(channel, store),
+ statusFor(127, 'Closed by Datadog')
+ );
+ expect(await res.json()).toEqual({ ok: true, outcome: 'status_replied' });
+ expect(channel.calls).toEqual([
+ {
+ kind: 'reply',
+ messageId: 'message-9',
+ body: { content: '✅ Closed by Datadog', mentions: [] },
+ },
+ { kind: 'react', messageId: 'message-9', emoji: '✅' },
+ {
+ kind: 'edit',
+ messageId: 'message-9',
+ body: {
+ content:
+ '✅ Resolved · 🚨 **PAGE** · **macro-db-prod CPU high**\n🔴 critical\nCPU at 92%.\n[Acknowledge](https://oncall.macro-internal.com/alerts/127)',
+ mentions: [],
+ },
+ },
+ ]);
+ });
+
+ test('status change for an unknown alert is posted top-level', async () => {
+ const channel = fakeChannel();
+ const res = await postWebhook(
+ makeApp(channel, createMemoryAlertStore()),
+ statusFor(48, 'Closed by Datadog')
+ );
+ expect(await res.json()).toEqual({ ok: true, outcome: 'status_posted' });
+ expect(channel.calls).toEqual([
+ {
+ kind: 'post',
+ body: {
+ content:
+ 'Closed by Datadog · macro-db-prod CPU high · [Open](https://oncall.macro-internal.com/alerts/48)',
+ mentions: [],
+ },
+ },
+ ]);
+ });
+
+ test('bundle is one top-level line', async () => {
+ const channel = fakeChannel();
+ const res = await postWebhook(makeApp(channel, createMemoryAlertStore()), {
+ AppName: 'GoAlert',
+ Type: 'AlertBundle',
+ ServiceID: 'b4b2c1a6-0000-4000-8000-000000000002',
+ ServiceName: 'Notify',
+ Count: 5,
+ GoAlertURL: 'https://oncall.macro-internal.com/services/notify/alerts',
+ });
+ expect(await res.json()).toEqual({ ok: true, outcome: 'bundle_posted' });
+ expect(channel.calls).toEqual([
+ {
+ kind: 'post',
+ body: {
+ content:
+ '🔔 · 5 alerts on Notify · [Acknowledge](https://oncall.macro-internal.com/services/notify/alerts)',
+ mentions: [],
+ },
+ },
+ ]);
+ });
+
+ test('a failed channel call returns 502', async () => {
+ const channel = fakeChannel();
+ channel.post = async () => {
+ throw new Error('channel unavailable');
+ };
+ const store = createMemoryAlertStore();
+ const res = await postWebhook(makeApp(channel, store), PAGE_ALERT);
+ expect(res.status).toBe(502);
+ expect(await store.get(127)).toBeUndefined();
+ });
+});
diff --git a/services/bots/goalert-bot/tsconfig.json b/services/bots/goalert-bot/tsconfig.json
new file mode 100644
index 0000000000..d73c3dfc9a
--- /dev/null
+++ b/services/bots/goalert-bot/tsconfig.json
@@ -0,0 +1,18 @@
+{
+ "compilerOptions": {
+ "esModuleInterop": true,
+ "skipLibCheck": true,
+ "forceConsistentCasingInFileNames": true,
+ "target": "es2022",
+ "module": "es2022",
+ "moduleResolution": "Bundler",
+ "resolveJsonModule": true,
+ "moduleDetection": "force",
+ "strict": true,
+ "noEmit": true,
+ "lib": ["es2022", "dom"],
+ "types": ["@types/node", "bun-types"]
+ },
+ "exclude": ["node_modules"],
+ "include": ["src", "test"]
+}
diff --git a/services/bots/goalert-bot/wrangler.jsonc b/services/bots/goalert-bot/wrangler.jsonc
new file mode 100644
index 0000000000..de191b943b
--- /dev/null
+++ b/services/bots/goalert-bot/wrangler.jsonc
@@ -0,0 +1,39 @@
+/**
+ * For more details on how to configure Wrangler, refer to:
+ * https://developers.cloudflare.com/workers/wrangler/configuration/
+ *
+ * Single prod deployment only — there is no dev environment.
+ *
+ * Secrets (set with `wrangler secret put <NAME>`):
+ * BOT_TOKEN - user-owned bot token (x-macro-bot-token)
+ * ACTING_USER_ID - the bot owner's user id; the bot acts as this user
+ * WEBHOOK_SECRET - shared secret embedded in the webhook URL path
+ * DD_API_KEY - optional; sent with trace exports to a Datadog intake
+ */
+{
+ "$schema": "node_modules/wrangler/config-schema.json",
+ "name": "goalert-bot",
+ "main": "src/worker.ts",
+ "compatibility_date": "2026-01-01",
+ // node:crypto for the constant-time webhook secret check, and
+ // AsyncLocalStorage for otel-cf-workers' span context.
+ "compatibility_flags": ["nodejs_compat"],
+ "observability": {
+ "enabled": true
+ },
+ "vars": {
+ // Macro Alerts.
+ "CHANNEL_ID": "019fb34d-fcfd-7548-9a0f-7084ea16416c",
+ // Tags exported spans with deployment.environment (matches DD_ENV).
+ "ENVIRONMENT": "production",
+ "OTEL_EXPORTER_OTLP_ENDPOINT": "https://macro-prox-prod.macroverse.workers.dev/i/otlp/v1/traces"
+ },
+ "kv_namespaces": [
+ {
+ // GoAlert alert id → posted message id (`alert:<AlertID>`).
+ "binding": "ALERT_MESSAGES",
+ // From `wrangler kv namespace create ALERT_MESSAGES` — see README.
+ "id": "3cfc0519b2254d6b9e64434658156d52"
+ }
+ ]
+}