Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(goalert-bot): post GoAlert alerts to Macro Alerts with threaded updates (#9216)
Wolf Mermelsteina460a8bdff31
diff --git a/bun.lock b/bun.lockindex 289490844c..b95e612e0f 100644--- a/bun.lock+++ b/bun.lock@@ -496,6 +496,20 @@"wrangler": "^4.110.0",},},+ "services/bots/goalert-bot": {+ "name": "goalert-bot",+ "version": "0.0.1",+ "dependencies": {+ "@macro-inc/observability": "workspace:*",+ "hono": "^4.6.20",+ "zod": "^4.4.3",+ },+ "devDependencies": {+ "@types/node": "^25.9.3",+ "bun-types": "^1.3.14",+ "wrangler": "^4.110.0",+ },+ },"services/bots/stripe-payment-bot": {"name": "stripe-payment-bot","version": "0.0.1",@@ -2631,6 +2645,8 @@"globrex": ["[email protected]", "", {}, "sha512-uHJgbwAMwNFf5mLst7IWLNg14x1CkeqglJb/K3doi4dw6q2IvAAmM/Y81kevy83wP+Sst+nutFTYOGg3d1lsxg=="],+ "goalert-bot": ["goalert-bot@workspace:services/bots/goalert-bot"],+"goober": ["[email protected]", "", { "peerDependencies": { "csstype": "^3.0.10" } }, "sha512-U7veizMqxyKlM58+Z5j2ngJBH/r9siDmxpvNxSw0PylF6WQvrASJEZrxh1hidRBJc2jqoBVSyOban5u8m+6Rxg=="],"google-auth-library": ["[email protected]", "", { "dependencies": { "base64-js": "^1.3.0", "ecdsa-sig-formatter": "^1.0.11", "gaxios": "^7.1.4", "gcp-metadata": "8.1.2", "google-logging-utils": "1.1.3", "jws": "^4.0.0" } }, "sha512-xtvUqvINPhTaBm7nXqlYPcrMHJPm1lCNdSovxnKKhTm+4JsvQ+KGVYJViLoH9Yxu8w+T0Qv5HubzYT9BLrppJg=="],@@ -4199,6 +4215,8 @@"get-uri/data-uri-to-buffer": ["[email protected]", "", {}, "sha512-7hvf7/GW8e86rW0ptuwS3OcBGDjIi6SZva7hCyWC0yYry2cOPmLIjXAUHI6DK2HsnwJd9ifmt57i8eV2n4YNpw=="],+ "goalert-bot/@types/node": ["@types/[email protected]", "", { "dependencies": { "undici-types": ">=7.24.0 <7.24.7" } }, "sha512-OScDchr2fwuUmWdf4kZ9h7PcJiYDVInhJizG/biAq3cAvqwYktuy/TYGGdZNMtNTFUP7rnb0NU4TUdm82kt4Rg=="],+"google-auth-library/gcp-metadata": ["[email protected]", "", { "dependencies": { "gaxios": "^7.0.0", "google-logging-utils": "^1.0.0", "json-bigint": "^1.0.0" } }, "sha512-zV/5HKTfCeKWnxG0Dmrw51hEWFGfcF2xiXqcA3+J90WDuP0SvoiSO5ORvcBsifmx/FoIjgQN3oNOGaQ5PhLFkg=="],"google-gax/@grpc/proto-loader": ["@grpc/[email protected]", "", { "dependencies": { "lodash.camelcase": "^4.3.0", "long": "^5.0.0", "protobufjs": "^7.2.5", "yargs": "^17.7.2" }, "bin": { "proto-loader-gen-types": "build/bin/proto-loader-gen-types.js" } }, "sha512-tMXdRCfYVixjuFK+Hk0Q1s38gV9zDiDJfWL3h1rv4Qc39oILCu1TRTDt7+fGUI8K4G1Fj125Hx/ru3azECWTyQ=="],@@ -4629,6 +4647,8 @@"gcp-metadata/gaxios/uuid": ["[email protected]", "", { "bin": { "uuid": "dist/bin/uuid" } }, "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA=="],+ "goalert-bot/@types/node/undici-types": ["[email protected]", "", {}, "sha512-WRNW+sJgj5OBN4/0JpHFqtqzhpbnV0GuB+OozA9gCL7a993SmU+1JBZCzLNxYsbMfIeDL+lTsphD5jN5N+n0zg=="],+"google-gax/@grpc/proto-loader/yargs": ["[email protected]", "", { "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "require-directory": "^2.1.1", "string-width": "^4.2.3", "y18n": "^5.0.5", "yargs-parser": "^21.1.1" } }, "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w=="],"google-gax/google-auth-library/gaxios": ["[email protected]", "", { "dependencies": { "extend": "^3.0.2", "https-proxy-agent": "^7.0.1", "is-stream": "^2.0.0", "node-fetch": "^2.6.9", "uuid": "^9.0.1" } }, "sha512-LDODD4TMYx7XXdpwxAVRAIAuB0bzv0s+ywFonY46k126qzQHT9ygyoa9tncmOiQmmDrik65UYsEkv3lbfqQ3yQ=="],diff --git a/nix-support/node_modules-hashes.json b/nix-support/node_modules-hashes.jsonindex aa114f317a..51a11ffdb8 100644--- a/nix-support/node_modules-hashes.json+++ b/nix-support/node_modules-hashes.json@@ -1,6 +1,6 @@{"nodeModules": {- "aarch64-darwin": "sha256-KwFdy4O78IkrXvPKd0hjY8NLgVbxJPUObyd+Ny50ieo=",- "x86_64-linux": "sha256-HV7k/t1RjAeQ/q7GhHCZnU9FlqR1mfue7y07ODHBpFk="+ "aarch64-darwin": "sha256-DYQqGiu4+hooVYvTs+ov0HPeVV6GyFnVwjndsPhUmQY=",+ "x86_64-linux": "sha256-7P+xA87nIIy9+e2/UuDB0oOBcpBEE38nsrKMWJ9uC4w="}}diff --git a/package.json b/package.jsonindex b290d7b049..9d2486fde1 100644--- a/package.json+++ b/package.json@@ -78,6 +78,7 @@"packages/lexical-core","services/ai-editing-worker","services/bots/anthropic-status-bot",+ "services/bots/goalert-bot","services/bots/stripe-payment-bot","services/lexical-service","packages/spreadsheet",diff --git a/services/bots/goalert-bot/AGENTS.md b/services/bots/goalert-bot/AGENTS.mdnew file mode 100644index 0000000000..c2f5f44292--- /dev/null+++ b/services/bots/goalert-bot/AGENTS.md@@ -0,0 +1,44 @@+# goalert-bot++## What it is++Observability: a Cloudflare Worker that turns GoAlert outgoing webhooks into+Macro Alerts channel posts, one thread per alert, through the Macro SDK.++## Start here++- `src/index.ts`: the webhook route and what each payload type does.+- `src/format.ts`: the compact post format, parsed from Datadog and Grafana details.+- `README.md`: payload → channel effect table, auth, setup.++## Scope++- Owns: formatting alerts, threading status changes, the KV map+ `alert:<AlertID>` → posted message.+- Not here: which alerts page → `infra/stacks/monitoring` (Datadog) and the+ observability stack's Grafana and GoAlert config.+- Called by: GoAlert webhook destinations only.++## Checks++- `bun test` (from this directory)+- `bunx tsc --noEmit --project tsconfig.json`++## Gotchas++- GoAlert abandons a webhook after ~3s and never retries: keep a request to one+ KV read, parallel channel calls, and one KV write. Don't add fetches.+- Import `@macro-inc/observability/worker` only from `src/worker.ts`: it loads+ `cloudflare:workers`, which Bun tests cannot resolve. Spans elsewhere use+ `Telemetry` from `@macro-inc/observability` (`src/traced.ts`).+- The bot is user-owned and acts as its owner (`ACTING_USER_ID`, a secret, never+ committed). Without `requestedAs` the SDK falls back to the channel webhook,+ which cannot thread, edit or react.+- The resolved edit drops the `here` mention; a bot edit carrying mentions can+ notify again.+- The SDK is imported by relative path from `packages/sdk/src`, like+ `anthropic-status-bot`; SDK changes reach this worker on the next deploy.++## Related++- Rules: `.agents/rules/observability.md` · Docs: `docs/guides/new-service.md`diff --git a/services/bots/goalert-bot/CLAUDE.md b/services/bots/goalert-bot/CLAUDE.mdnew file mode 120000index 0000000000..47dc3e3d86--- /dev/null+++ b/services/bots/goalert-bot/CLAUDE.md@@ -0,0 +1 @@+AGENTS.md\ No newline at end of filediff --git a/services/bots/goalert-bot/README.md b/services/bots/goalert-bot/README.mdnew file mode 100644index 0000000000..c28664a31f--- /dev/null+++ b/services/bots/goalert-bot/README.md@@ -0,0 +1,118 @@+# goalert-bot++Posts GoAlert alerts to the Macro Alerts channel, one thread per alert.++GoAlert is the paging hub: Grafana and Datadog alert GoAlert, whose escalation+policies send outgoing webhooks for each alert and status change. This worker+receives them and posts through the [Macro SDK](../../../packages/sdk), so it+can thread, mention, edit and react, which the channel bot webhook cannot.++## How it works++```+GoAlert webhook ──POST──> /goalert/{WEBHOOK_SECRET}+ │ parse (Alert | AlertStatus | bundles | …)+ │ KV ALERT_MESSAGES: alert:<AlertID> → message+ ▼+ @macro-inc/sdk → channel.send / message.reply / react / edit+ headers: x-macro-bot-token: BOT_TOKEN, x-macro-bot-scope: user,+ x-macro-bot-for-macro-user-id: ACTING_USER_ID+```++| GoAlert payload | Channel effect |+| --- | --- |+| `Alert` (new) | Compact top-level post; pages end the headline with a `here` mention. Stored in KV for 30 days. |+| `Alert` (already posted) | GoAlert re-escalated an open page: thread reply `⏰ Still open` (with `here` for pages). |+| `AlertStatus` acknowledged | Thread reply `👀 Acknowledged by …`, 👀 reaction on the post. |+| `AlertStatus` closed | Thread reply `✅ Closed by …`, ✅ reaction, and the post's headline is prefixed `✅ Resolved ·` (the `here` mention is dropped so the edit notifies nobody). |+| `AlertStatus` other (escalated, …) | Thread reply with the log entry. |+| `AlertStatus` for an alert never posted | One-line top-level post. |+| `AlertBundle` / `AlertStatusBundle` | One top-level line with the count and a link. |+| `ScheduleOnCallUsers`, `Test`, `Verification` | One top-level notice. |++A new alert post:++```+🚨 **PAGE** · **<title> — <scope>** @here+🔴 critical: 2631 (threshold 1800)+<one-sentence description, AWS hostnames shortened>+[Acknowledge](…) · [Graph](…) · [Runbook](…)+```++The `Notify` GoAlert service posts `🔔` instead of `🚨 **PAGE**` and mentions+nobody. State, description and links are parsed from Datadog monitor messages+(`alertMessage()` in `infra/stacks/monitoring/notify.ts`) and Grafana's details+table.++Notes on the GoAlert side:++- GoAlert **cannot set headers**, so the URL carries the shared secret,+ compared in constant time.+- GoAlert **abandons a webhook after about three seconds** and does not retry.+ A request does one KV read, its channel calls in parallel, and at most one+ KV write. A failed channel call returns `502` so it shows in GoAlert's logs.+- Unrecognized payloads are acknowledged (`200 {ok, ignored: true}`).++## Auth++The bot is **user-owned and acts as its owner** (`requestedAs(ACTING_USER_ID)`,+user scope). The server lets a user-owned bot act only as its owner. Messages+show the bot as sender, triggered by the owner, and the bot's channel access+follows the owner's: the owner must be able to post in `CHANNEL_ID`. Moving to a+team-owned bot means dropping `ACTING_USER_ID` and `requestedAs` in+`src/channel.ts` and using the default team scope.++## Tracing++`src/worker.ts` wraps the worker in otel-cf-workers' `instrument()` through+`@macro-inc/observability/worker`, configured like `services/ai-editing-worker`:+spans go to `OTEL_EXPORTER_OTLP_ENDPOINT` (the prod OTLP proxy), with+`DD_API_KEY` as `dd-api-key` when set. Each webhook is one+`goalert_bot.webhook` span with `goalert.payload_type`, `goalert.alert_id`,+`goalert.service_name` and `goalert.outcome`; children+`goalert_bot.kv.get`/`kv.put` (`goalert.found` on reads) and+`goalert_bot.channel.post`/`reply`/`react`/`edit` (`goalert.message_id`).+Message bodies are never recorded. Search by `@goalert.alert_id`.++## Setup++Deployed as a single prod Cloudflare Worker (`goalert-bot`, no dev+environment):++```bash+wrangler kv namespace create ALERT_MESSAGES # put the id in wrangler.jsonc+printf '%s' '<token>' | wrangler secret put BOT_TOKEN+printf '%s' '<owner user id>' | wrangler secret put ACTING_USER_ID+printf '%s' '<secret>' | wrangler secret put WEBHOOK_SECRET+printf '%s' '<key>' | wrangler secret put DD_API_KEY # optional, see Tracing+bun run deploy # wrangler deploy+```++Then point a GoAlert webhook destination (escalation policy step or service+notification) at+`https://goalert-bot.macroverse.workers.dev/goalert/<WEBHOOK_SECRET>`.++## Environment variables++| Variable | Required | Default | Description |+| ---------------- | -------- | ------- | --------------------------------------------------------- |+| `BOT_TOKEN` | yes | | Bot API key (`mbot_...`, sent as `x-macro-bot-token`) |+| `CHANNEL_ID` | yes | | Channel UUID to post to (Macro Alerts) |+| `ACTING_USER_ID` | yes | | Secret: the bot owner's user id (`macro\|<email>`) |+| `WEBHOOK_SECRET` | yes | | Secret path segment for the webhook URL |+| `ALERT_MESSAGES` | worker | | KV namespace binding: `alert:<AlertID>` → posted message |+| `ENVIRONMENT` | no | `local` | `deployment.environment` on exported spans |+| `OTEL_EXPORTER_OTLP_ENDPOINT` | no | | OTLP/HTTP traces endpoint; unset disables export |+| `DD_API_KEY` | no | | Secret: Datadog key sent with trace exports |+| `MACRO_ENV` | no | `prod` | SDK environment (`prod`/`dev`/`local`), local server only |+| `PORT` | no | `8089` | Listen port (local Bun server only) |++## Development++```bash+just dev # bun run --watch src/server.ts (in-memory alert store)+just test # bun test+just check # tsc --noEmit+```++Formatter tests use real Datadog and Grafana alert details.diff --git a/services/bots/goalert-bot/biome.jsonc b/services/bots/goalert-bot/biome.jsoncnew file mode 100644index 0000000000..82a0f443c4--- /dev/null+++ b/services/bots/goalert-bot/biome.jsonc@@ -0,0 +1,11 @@+{+ "$schema": "https://biomejs.dev/schemas/2.4.12/schema.json",+ "root": true,+ "extends": ["../../../biome.base.jsonc"],+ "vcs": {+ "root": "../../.."+ },+ "files": {+ "includes": ["**/*.{ts,tsx,js,jsx}", "!**/*.d.ts", "!**/node_modules"]+ }+}diff --git a/services/bots/goalert-bot/justfile b/services/bots/goalert-bot/justfilenew file mode 100644index 0000000000..44103f8db1--- /dev/null+++ b/services/bots/goalert-bot/justfile@@ -0,0 +1,8 @@+dev:+ bun run --watch src/server.ts++test:+ bun test++check:+ bunx tsc --noEmit --project tsconfig.jsondiff --git a/services/bots/goalert-bot/package.json b/services/bots/goalert-bot/package.jsonnew file mode 100644index 0000000000..06b03a89b3--- /dev/null+++ b/services/bots/goalert-bot/package.json@@ -0,0 +1,23 @@+{+ "name": "goalert-bot",+ "version": "0.0.1",+ "private": true,+ "type": "module",+ "scripts": {+ "dev": "bun run --watch src/server.ts",+ "start": "bun run src/server.ts",+ "test": "bun test",+ "check": "tsc --noEmit --project tsconfig.json",+ "deploy": "wrangler deploy"+ },+ "dependencies": {+ "@macro-inc/observability": "workspace:*",+ "hono": "^4.6.20",+ "zod": "^4.4.3"+ },+ "devDependencies": {+ "@types/node": "^25.9.3",+ "bun-types": "^1.3.14",+ "wrangler": "^4.110.0"+ }+}diff --git a/services/bots/goalert-bot/src/channel.ts b/services/bots/goalert-bot/src/channel.tsnew file mode 100644index 0000000000..f5de17453a--- /dev/null+++ b/services/bots/goalert-bot/src/channel.ts@@ -0,0 +1,54 @@+import type { Env } from '../../../../packages/sdk/src/config';+import { Macro } from '../../../../packages/sdk/src/macro';+import type { RichMessage } from '../../../../packages/sdk/src/mentions';++export type ChannelConfig = {+ /** UUID of the channel the bot posts to. */+ channelId: string;+ /** Bot API key (mbot_...), sent as x-macro-bot-token. */+ botToken: string;+ /** The bot owner's user id (`macro|<email>`); the bot acts as this user. */+ actingUserId: string;+ /** Macro environment. Defaults to prod. */+ env?: Env;+};++/** The channel operations the bot needs. Every method throws on failure. */+export type AlertChannel = {+ /** Post a top-level message and return its id. */+ post(body: RichMessage): Promise<string>;+ /** Reply in the thread rooted at `messageId`. */+ reply(messageId: string, body: RichMessage): Promise<void>;+ react(messageId: string, emoji: string): Promise<void>;+ edit(messageId: string, body: RichMessage): Promise<void>;+};++/**+ * The alerts channel through the Macro SDK.+ *+ * The bot is user-owned and acts as its owner (`x-macro-bot-for-macro-user-id`,+ * user scope), which routes posts through the entity message API: threads,+ * mentions, edits and reactions. Messages show the bot as sender. A team-owned+ * bot would drop `requestedAs` and use the default team scope instead.+ */+export function createAlertChannel(config: ChannelConfig): AlertChannel {+ const bot = new Macro({+ env: config.env ?? 'prod',+ auth: { type: 'bot', token: config.botToken, scope: 'user' },+ });+ const macro = bot.requestedAs(bot.users.byId(config.actingUserId));+ const channel = macro.channels.byId(config.channelId);++ return {+ post: async (body) => (await channel.send(body)).id,+ reply: async (messageId, body) => {+ await channel.message(messageId).reply(body);+ },+ react: async (messageId, emoji) => {+ await channel.message(messageId).react(emoji);+ },+ edit: async (messageId, body) => {+ await channel.message(messageId).edit(body);+ },+ };+}diff --git a/services/bots/goalert-bot/src/format.ts b/services/bots/goalert-bot/src/format.tsnew file mode 100644index 0000000000..3667b4bf3f--- /dev/null+++ b/services/bots/goalert-bot/src/format.ts@@ -0,0 +1,261 @@+import {+ here,+ msg,+ type RichMessage,+ toBody,+} from '../../../../packages/sdk/src/mentions';+import type { AlertPayload, AlertStatusPayload } from './goalert';++const MAX_DESCRIPTION_CHARS = 300;++// Datadog titles start with bracketed tags: `[P3] [Re-Triggered on {host:x}] [PROD] Title`.+const DATADOG_TAG = /^\s*\[([^\]]*)\]\s*/;+const SCOPE = /on \{(.+)\}/;+// Datadog's state line, from `alertMessage()` in infra/stacks/monitoring/notify.ts.+const DATADOG_STATE = /^(🔴|🟠|✅)\s*(CRITICAL|WARNING|RECOVERED):\s*(.*)$/u;+const GRAFANA_ROW = /^\|\s*([^|]+?)\s*\|\s*(.*?)\s*\|$/;+// `macro-db-prod.ctkwyjgndnfr.us-east-1.rds.amazonaws.com` -> `macro-db-prod`.+const AWS_HOSTNAME = /\b([a-z0-9-]+)\.[a-z0-9.-]+\.amazonaws\.com\b/g;+const LINK_LINE =+ /^(Runbook|Dashboard|Metric Graph|Source):\s*(https?:\/\/\S+)/;+const NOT_A_DESCRIPTION = ['@', '`', 'http', 'Metric value', 'Paging on-call'];+const GRAFANA_SEVERITY: Record<string, string> = {+ critical: '🔴 critical',+ warning: '🟠 warning',+};++/** A new-alert post, kept in parts so the headline can be rewritten on close. */+export type CompactPost = {+ /** Pages mention `@here`; the `Notify` service does not. */+ page: boolean;+ /** `🚨 **PAGE** · **<title> — <scope>**`, without the mention. */+ headline: string;+ /** State, description and links, one per line. */+ lines: string[];+};++type ParsedDetails = {+ state?: string;+ description?: string;+ scope?: string;+ links: Record<string, string>;+};++/** Whether alerts on this GoAlert service page on-call. */+export function isPage(serviceName: string): boolean {+ return !serviceName.includes('Notify');+}++/** `host:macro-db-prod.abc.us-east-1.rds.amazonaws.com` -> `macro-db-prod`. */+function shortScope(scope: string): string {+ return scope+ .split(',')+ .map((part) => {+ const value = part.slice(part.indexOf(':') + 1);+ return value.includes('.amazonaws.com') ? value.split('.')[0] : value;+ })+ .join(', ');+}++/** The title without Datadog's bracketed tags, and the group it fired on. */+function datadogTitle(summary: string): { title: string; scope: string } {+ let scope = '';+ let rest = summary;+ for (+ let match = DATADOG_TAG.exec(rest);+ match;+ match = DATADOG_TAG.exec(rest)+ ) {+ const on = SCOPE.exec(match[1]);+ if (on) scope = shortScope(on[1]);+ rest = rest.slice(match[0].length);+ }+ return { title: rest.trim(), scope };+}++/** The alert's cleaned-up title with the group it fired on. */+export function alertTitle(summary: string): string {+ const { title, scope } = datadogTitle(summary);+ return scope ? `${title} — ${scope}` : title;+}++/** Collapse whitespace, shorten AWS hostnames, and cut at a sentence end. */+export function firstSentences(text: string): string {+ const flat = text.split(/\s+/).filter(Boolean).join(' ');+ const short = flat.replace(AWS_HOSTNAME, '$1');+ if (short.length <= MAX_DESCRIPTION_CHARS) return short;+ const cut = short.slice(0, MAX_DESCRIPTION_CHARS);+ const sentenceEnd = cut.lastIndexOf('. ');+ const kept = sentenceEnd >= 0 ? cut.slice(0, sentenceEnd + 1) : cut.trimEnd();+ return kept || cut;+}++/** State, description and links from a Datadog message or Grafana's details table. */+function parseDetails(details: string): ParsedDetails {+ const parsed: ParsedDetails = { links: {} };+ const rows: Record<string, string> = {};+ for (const raw of details.split('\n')) {+ const line = raw.trim();+ const row = GRAFANA_ROW.exec(line);+ const state = DATADOG_STATE.exec(line);+ const link = LINK_LINE.exec(line);+ if (row) {+ rows[row[1]] = row[2];+ } else if (state) {+ parsed.state = `${state[1]} ${state[2].toLowerCase()}: ${state[3]}`;+ } else if (link) {+ parsed.links[link[1]] ??= link[2];+ } else if (+ line &&+ parsed.description === undefined &&+ parsed.state !== undefined &&+ !NOT_A_DESCRIPTION.some((prefix) => line.startsWith(prefix))+ ) {+ parsed.description = line;+ }+ }+ if (Object.keys(rows).length > 0) {+ const severity = rows.severity ?? '';+ parsed.state = GRAFANA_SEVERITY[severity] ?? severity;+ parsed.description = rows.description ?? '';+ parsed.scope =+ rows.DBInstanceIdentifier || rows.CacheClusterId || rows.QueueName || '';+ if (rows.panel_url) parsed.links.Graph = rows.panel_url;+ if (rows.runbook_url) parsed.links.Runbook = rows.runbook_url;+ }+ return parsed;+}++function markdownLinks(links: [string, string | undefined][]): string {+ return links+ .filter((link): link is [string, string] => Boolean(link[1]))+ .map(([name, url]) => `[${name}](${url})`)+ .join(' · ');+}++/**+ * ```+ * 🚨 **PAGE** · **macro-db-prod CPU high — macro-db-prod**+ * 🔴 critical: 92 (threshold 85)+ * macro-db-prod CPU averaged 92% over 15 minutes.+ * [Acknowledge](…) · [Graph](…) · [Runbook](…)+ * ```+ */+export function compactAlert(+ payload: Pick<+ AlertPayload,+ 'Summary' | 'Details' | 'ServiceName' | 'GoAlertURL'+ >+): CompactPost {+ const page = isPage(payload.ServiceName);+ const parsed = parseDetails(payload.Details);+ let title = alertTitle(payload.Summary);+ if (parsed.scope && !title.includes(parsed.scope)) {+ title = `${title} — ${parsed.scope}`;+ }+ const lines: string[] = [];+ if (parsed.state) lines.push(parsed.state);+ if (parsed.description) lines.push(firstSentences(parsed.description));+ lines.push(+ markdownLinks([+ ['Acknowledge', payload.GoAlertURL],+ [+ 'Graph',+ parsed.links.Graph ||+ parsed.links.Dashboard ||+ parsed.links['Metric Graph'],+ ],+ ['Runbook', parsed.links.Runbook],+ ])+ );+ return {+ page,+ headline: `${page ? '🚨 **PAGE**' : '🔔'} · **${title}**`,+ lines,+ };+}++/** The post body: a separator, the headline (`@here` on open pages), and the rest. */+export function renderPost(post: CompactPost, resolved = false): RichMessage {+ const rest = post.lines.join('\n');+ if (resolved) {+ // The mention is dropped so the edit does not notify everyone again.+ return toBody(`✅ Resolved · ${post.headline}\n${rest}`);+ }+ if (post.page) {+ return msg`${post.headline} ${here}\n${rest}`;+ }+ return toBody(`${post.headline}\n${rest}`);+}++/** What an `AlertStatus` log entry did to the alert. */+export type StatusChange = 'acknowledged' | 'closed' | 'escalated' | 'other';++export function statusChange(logEntry: string): StatusChange {+ const entry = logEntry.toLowerCase();+ if (entry.startsWith('acknowledged')) return 'acknowledged';+ if (entry.startsWith('closed')) return 'closed';+ if (entry.startsWith('escalated')) return 'escalated';+ return 'other';+}++const STATUS_EMOJI: Record<StatusChange, string> = {+ acknowledged: '👀',+ closed: '✅',+ escalated: '⏫',+ other: 'ℹ️',+};++/** One thread line for a status change: `👀 Acknowledged by Sam`. */+export function statusReply(logEntry: string): RichMessage {+ return toBody(`${STATUS_EMOJI[statusChange(logEntry)]} ${logEntry}`);+}++/** GoAlert re-sent an open alert (hourly re-escalation of an unacknowledged page). */+export function stillOpenReply(+ payload: Pick<AlertPayload, 'ServiceName' | 'GoAlertURL'>+): RichMessage {+ const line = `⏰ Still open · ${markdownLinks([['Acknowledge', payload.GoAlertURL]])}`;+ return isPage(payload.ServiceName) ? msg`${line} ${here}` : toBody(line);+}++/** A status change for an alert this bot never posted, as its own one-line post. */+export function statusPost(payload: AlertStatusPayload): RichMessage {+ const parts = [+ payload.LogEntry,+ alertTitle(payload.Summary),+ markdownLinks([['Open', payload.GoAlertURL]]),+ ].filter(Boolean);+ return toBody(`${parts.join(' · ')}`);+}++export function alertBundlePost(payload: {+ ServiceName: string;+ Count: number;+ GoAlertURL: string;+}): RichMessage {+ const page = isPage(payload.ServiceName);+ const line = `${page ? '🚨 **PAGE**' : '🔔'} · ${payload.Count} alerts on ${payload.ServiceName} · ${markdownLinks([['Acknowledge', payload.GoAlertURL]])}`;+ return page ? msg`${line} ${here}` : toBody(`${line}`);+}++export function alertStatusBundlePost(payload: {+ LogEntry: string;+ Count: number;+ GoAlertURL: string;+}): RichMessage {+ return toBody(+ `${payload.LogEntry} (${payload.Count} alerts) · ${markdownLinks([['Open', payload.GoAlertURL]])}`+ );+}++export function onCallPost(payload: {+ ScheduleName: string;+ ScheduleURL: string;+ Users: { Name: string }[];+}): RichMessage {+ const names = payload.Users.map((user) => user.Name).join(', ');+ return toBody(+ `On call for ${payload.ScheduleName}: ${names || 'nobody'}\n${payload.ScheduleURL}`+ );+}diff --git a/services/bots/goalert-bot/src/goalert.ts b/services/bots/goalert-bot/src/goalert.tsnew file mode 100644index 0000000000..f002ad875d--- /dev/null+++ b/services/bots/goalert-bot/src/goalert.ts@@ -0,0 +1,83 @@+import { z } from 'zod';++/**+ * GoAlert outgoing webhook payloads (`notification/webhook` in GoAlert).+ *+ * Every payload carries `AppName` and a `Type` naming its kind. Schemas are+ * loose and text fields default to empty, because GoAlert omits fields that do+ * not apply and may add new ones.+ */++const text = z.string().default('');++const alertSchema = z.looseObject({+ Type: z.literal('Alert'),+ AlertID: z.number(),+ Summary: text,+ Details: text,+ ServiceID: text,+ ServiceName: text,+ Meta: z.record(z.string(), z.string()).nullish(),+ GoAlertURL: text,+});++const alertStatusSchema = z.looseObject({+ Type: z.literal('AlertStatus'),+ AlertID: z.number(),+ Summary: text,+ ServiceID: text,+ ServiceName: text,+ LogEntry: text,+ GoAlertURL: text,+});++const alertBundleSchema = z.looseObject({+ Type: z.literal('AlertBundle'),+ ServiceID: text,+ ServiceName: text,+ Count: z.number(),+ GoAlertURL: text,+});++const alertStatusBundleSchema = z.looseObject({+ Type: z.literal('AlertStatusBundle'),+ ServiceID: text,+ ServiceName: text,+ LogEntry: text,+ Count: z.number(),+ GoAlertURL: text,+});++const scheduleOnCallUsersSchema = z.looseObject({+ Type: z.literal('ScheduleOnCallUsers'),+ ScheduleName: text,+ ScheduleURL: text,+ Users: z.array(z.looseObject({ Name: text })).default([]),+});++const testSchema = z.looseObject({ Type: z.literal('Test') });++const verificationSchema = z.looseObject({+ Type: z.literal('Verification'),+ Code: z.union([z.string(), z.number()]).optional(),+});++const goAlertPayloadSchema = z.discriminatedUnion('Type', [+ alertSchema,+ alertStatusSchema,+ alertBundleSchema,+ alertStatusBundleSchema,+ scheduleOnCallUsersSchema,+ testSchema,+ verificationSchema,+]);++export type AlertPayload = z.infer<typeof alertSchema>;+export type AlertStatusPayload = z.infer<typeof alertStatusSchema>;+export type GoAlertPayload = z.infer<typeof goAlertPayloadSchema>;++/** Parse a webhook body, or `undefined` for anything this bot does not know. */+export function parseGoAlertPayload(body: unknown): GoAlertPayload | undefined {+ const result = goAlertPayloadSchema.safeParse(body);+ return result.success ? result.data : undefined;+}diff --git a/services/bots/goalert-bot/src/index.ts b/services/bots/goalert-bot/src/index.tsnew file mode 100644index 0000000000..402712c24a--- /dev/null+++ b/services/bots/goalert-bot/src/index.ts@@ -0,0 +1,192 @@+import { createHash, timingSafeEqual } from 'node:crypto';+import { Telemetry } from '@macro-inc/observability';+import { Hono } from 'hono';+import { toBody } from '../../../../packages/sdk/src/mentions';+import type { AlertChannel } from './channel';+import {+ alertBundlePost,+ alertStatusBundlePost,+ compactAlert,+ onCallPost,+ renderPost,+ statusChange,+ statusPost,+ statusReply,+ stillOpenReply,+} from './format';+import {+ type AlertPayload,+ type AlertStatusPayload,+ type GoAlertPayload,+ parseGoAlertPayload,+} from './goalert';+import type { AlertStore } from './store';+import { tracedChannel, tracedStore } from './traced';++const MAX_BODY_BYTES = 256 * 1024;++export type AppConfig = {+ /** Shared secret embedded in the webhook URL path. */+ webhookSecret: string;+ channel: AlertChannel;+ store: AlertStore;+};++/** What the bot did with a webhook, for logs and the response body. */+type Outcome =+ | 'posted'+ | 'still_open_replied'+ | 'status_replied'+ | 'status_posted'+ | 'bundle_posted'+ | 'notice_posted';++function sameSecret(given: string, expected: string): boolean {+ // Hashing first gives equal lengths, which timingSafeEqual requires.+ const digest = (value: string) => createHash('sha256').update(value).digest();+ return timingSafeEqual(digest(given), digest(expected));+}++async function handleAlert(+ config: AppConfig,+ payload: AlertPayload+): Promise<Outcome> {+ const posted = await config.store.get(payload.AlertID);+ if (posted) {+ await config.channel.reply(posted.messageId, stillOpenReply(payload));+ return 'still_open_replied';+ }+ const post = compactAlert(payload);+ const messageId = await config.channel.post(renderPost(post));+ try {+ await config.store.put(payload.AlertID, { messageId, post });+ } catch {+ // The post is up and the kv.put span records the failure; later status+ // changes fall back to top-level posts.+ }+ return 'posted';+}++async function handleAlertStatus(+ config: AppConfig,+ payload: AlertStatusPayload+): Promise<Outcome> {+ const posted = await config.store.get(payload.AlertID);+ if (!posted) {+ await config.channel.post(statusPost(payload));+ return 'status_posted';+ }+ const { channel } = config;+ const change = statusChange(payload.LogEntry);+ const updates = [+ channel.reply(posted.messageId, statusReply(payload.LogEntry)),+ ];+ if (change === 'acknowledged') {+ updates.push(channel.react(posted.messageId, '👀'));+ } else if (change === 'closed') {+ updates.push(+ channel.react(posted.messageId, '✅'),+ channel.edit(posted.messageId, renderPost(posted.post, true))+ );+ }+ await Promise.all(updates);+ return 'status_replied';+}++async function handle(+ config: AppConfig,+ payload: GoAlertPayload+): Promise<Outcome> {+ const { channel } = config;+ if (payload.Type === 'Alert') return handleAlert(config, payload);+ if (payload.Type === 'AlertStatus') return handleAlertStatus(config, payload);+ if (payload.Type === 'AlertBundle') {+ await channel.post(alertBundlePost(payload));+ return 'bundle_posted';+ }+ if (payload.Type === 'AlertStatusBundle') {+ await channel.post(alertStatusBundlePost(payload));+ return 'bundle_posted';+ }+ if (payload.Type === 'ScheduleOnCallUsers') {+ await channel.post(onCallPost(payload));+ return 'notice_posted';+ }+ if (payload.Type === 'Verification') {+ await channel.post(toBody(`GoAlert verification code: ${payload.Code}`));+ return 'notice_posted';+ }+ await channel.post(toBody('GoAlert test message for the Macro Alerts bot.'));+ return 'notice_posted';+}++/**+ * GoAlert → Macro Alerts channel bot.+ *+ * GoAlert's outgoing webhooks cannot set headers, so the URL carries a shared+ * secret: POST /goalert/{WEBHOOK_SECRET}. GoAlert abandons a webhook after+ * about three seconds, so each request does at most a store read, the channel+ * calls (in parallel), and a store write.+ *+ * Unrecognized payloads are acknowledged and ignored; a failed channel call+ * returns 502 so it shows in GoAlert's logs.+ */+export function createApp(config: AppConfig) {+ const app = new Hono();+ const tracedConfig: AppConfig = {+ ...config,+ channel: tracedChannel(config.channel),+ store: tracedStore(config.store),+ };++ app.get('/health', (c) => c.json({ ok: true }));++ app.post('/goalert/:secret', async (c) => {+ if (!sameSecret(c.req.param('secret'), config.webhookSecret)) {+ return c.json({ error: 'unauthorized' }, 401);+ }++ const raw = await c.req.text();+ if (raw.length > MAX_BODY_BYTES) {+ return c.json({ error: 'body too large' }, 413);+ }+ let body: unknown;+ try {+ body = JSON.parse(raw);+ } catch {+ return c.json({ error: 'invalid JSON body' }, 400);+ }++ const payload = parseGoAlertPayload(body);+ if (!payload) {+ console.warn('Ignoring unrecognized GoAlert webhook payload');+ return c.json({ ok: true, ignored: true });+ }++ return Telemetry.span('goalert_bot.webhook', async (span) => {+ span.setAttr('goalert.payload_type', payload.Type);+ if (payload.Type === 'Alert' || payload.Type === 'AlertStatus') {+ span.setAttr('goalert.alert_id', payload.AlertID);+ }+ if (+ payload.Type === 'Alert' ||+ payload.Type === 'AlertStatus' ||+ payload.Type === 'AlertBundle' ||+ payload.Type === 'AlertStatusBundle'+ ) {+ span.setAttr('goalert.service_name', payload.ServiceName);+ }+ try {+ const outcome = await handle(tracedConfig, payload);+ span.setAttr('goalert.outcome', outcome);+ return c.json({ ok: true, outcome });+ } catch (error) {+ span.setAttr('goalert.outcome', 'failed');+ span.error(error);+ return c.json({ error: 'failed to deliver message' }, 502);+ }+ });+ });++ return app;+}diff --git a/services/bots/goalert-bot/src/server.ts b/services/bots/goalert-bot/src/server.tsnew file mode 100644index 0000000000..fa770def68--- /dev/null+++ b/services/bots/goalert-bot/src/server.ts@@ -0,0 +1,36 @@+import { createAlertChannel } from './channel';+import { createApp } from './index';+import { createMemoryAlertStore } from './store';++const PORT = parseInt(process.env.PORT || '8089', 10);++const requiredVars = [+ 'BOT_TOKEN',+ 'CHANNEL_ID',+ 'ACTING_USER_ID',+ 'WEBHOOK_SECRET',+] as const;+for (const varName of requiredVars) {+ if (!process.env[varName]) {+ console.error(`Missing required environment variable: ${varName}`);+ process.exit(1);+ }+}++const app = createApp({+ webhookSecret: process.env.WEBHOOK_SECRET!,+ channel: createAlertChannel({+ channelId: process.env.CHANNEL_ID!,+ botToken: process.env.BOT_TOKEN!,+ actingUserId: process.env.ACTING_USER_ID!,+ env: (process.env.MACRO_ENV as 'prod' | 'dev' | 'local') || 'prod',+ }),+ store: createMemoryAlertStore(),+});++const server = Bun.serve({+ port: PORT,+ fetch: app.fetch,+});++console.log(`goalert-bot listening on http://localhost:${server.port}`);diff --git a/services/bots/goalert-bot/src/store.ts b/services/bots/goalert-bot/src/store.tsnew file mode 100644index 0000000000..d0d836638b--- /dev/null+++ b/services/bots/goalert-bot/src/store.ts@@ -0,0 +1,65 @@+import { z } from 'zod';++/** How long a posted alert's thread stays reachable by later status changes. */+const ALERT_TTL_SECONDS = 30 * 24 * 60 * 60;++const postedAlertSchema = z.object({+ messageId: z.string(),+ // Kept so a close can rewrite the headline without fetching the message.+ post: z.object({+ page: z.boolean(),+ headline: z.string(),+ lines: z.array(z.string()),+ }),+});++/** The channel message an alert was posted as. */+export type PostedAlert = z.infer<typeof postedAlertSchema>;++/** GoAlert alert id → the message it was posted as. */+export type AlertStore = {+ get(alertId: number): Promise<PostedAlert | undefined>;+ put(alertId: number, posted: PostedAlert): Promise<void>;+};++/** The subset of a Workers KV namespace binding this store uses. */+export type KeyValueNamespace = {+ get(key: string, type: 'json'): Promise<unknown>;+ put(+ key: string,+ value: string,+ options: { expirationTtl: number }+ ): Promise<void>;+};++function key(alertId: number): string {+ return `alert:${alertId}`;+}++/** Store backed by a Workers KV namespace; entries expire after 30 days. */+export function createKvAlertStore(kv: KeyValueNamespace): AlertStore {+ return {+ async get(alertId) {+ const value = await kv.get(key(alertId), 'json');+ if (value === null) return undefined;+ const parsed = postedAlertSchema.safeParse(value);+ return parsed.success ? parsed.data : undefined;+ },+ async put(alertId, posted) {+ await kv.put(key(alertId), JSON.stringify(posted), {+ expirationTtl: ALERT_TTL_SECONDS,+ });+ },+ };+}++/** In-process store for the local Bun server and tests. */+export function createMemoryAlertStore(): AlertStore {+ const entries = new Map<number, PostedAlert>();+ return {+ get: async (alertId) => entries.get(alertId),+ put: async (alertId, posted) => {+ entries.set(alertId, posted);+ },+ };+}diff --git a/services/bots/goalert-bot/src/traced.ts b/services/bots/goalert-bot/src/traced.tsnew file mode 100644index 0000000000..db65e12f80--- /dev/null+++ b/services/bots/goalert-bot/src/traced.ts@@ -0,0 +1,72 @@+import { type Span, Telemetry } from '@macro-inc/observability';+import type { AlertChannel } from './channel';+import type { AlertStore } from './store';++/** Run `operation` in a child span named `name`, marking it failed on error. */+async function traced<T>(+ name: string,+ attributes: Record<string, string | number>,+ operation: (span: Span) => Promise<T>+): Promise<T> {+ return Telemetry.span(name, async (span) => {+ for (const [key, value] of Object.entries(attributes)) {+ span.setAttr(key, value);+ }+ try {+ return await operation(span);+ } catch (error) {+ span.error(error);+ throw error;+ }+ });+}++/** The channel with one span per call. Bodies are not recorded. */+export function tracedChannel(channel: AlertChannel): AlertChannel {+ return {+ post: (body) =>+ traced('goalert_bot.channel.post', {}, async (span) => {+ const messageId = await channel.post(body);+ span.setAttr('goalert.message_id', messageId);+ return messageId;+ }),+ reply: (messageId, body) =>+ traced(+ 'goalert_bot.channel.reply',+ { 'goalert.message_id': messageId },+ () => channel.reply(messageId, body)+ ),+ react: (messageId, emoji) =>+ traced(+ 'goalert_bot.channel.react',+ { 'goalert.message_id': messageId },+ () => channel.react(messageId, emoji)+ ),+ edit: (messageId, body) =>+ traced(+ 'goalert_bot.channel.edit',+ { 'goalert.message_id': messageId },+ () => channel.edit(messageId, body)+ ),+ };+}++/** The alert store with one span per KV call, recording hit or miss. */+export function tracedStore(store: AlertStore): AlertStore {+ return {+ get: (alertId) =>+ traced(+ 'goalert_bot.kv.get',+ { 'goalert.alert_id': alertId },+ async (span) => {+ const posted = await store.get(alertId);+ span.setAttr('goalert.found', posted !== undefined);+ return posted;+ }+ ),+ put: (alertId, posted) =>+ traced('goalert_bot.kv.put', { 'goalert.alert_id': alertId }, () =>+ store.put(alertId, posted)+ ),+ };+}diff --git a/services/bots/goalert-bot/src/worker.ts b/services/bots/goalert-bot/src/worker.tsnew file mode 100644index 0000000000..2626934260--- /dev/null+++ b/services/bots/goalert-bot/src/worker.ts@@ -0,0 +1,75 @@+import {+ createWorkerTraceConfig,+ instrument,+} from '@macro-inc/observability/worker';+import { createAlertChannel } from './channel';+import { createApp } from './index';+import { createKvAlertStore, type KeyValueNamespace } from './store';++/**+ * Cloudflare Worker entrypoint. Config comes from worker bindings+ * (vars + secrets + the KV namespace); see wrangler.jsonc. For local+ * development outside wrangler, src/server.ts provides the same app under Bun.+ */+type Env = {+ BOT_TOKEN: string;+ CHANNEL_ID: string;+ ACTING_USER_ID: string;+ WEBHOOK_SECRET: string;+ ALERT_MESSAGES: KeyValueNamespace;+ /** Tags exported spans with deployment.environment (matches DD_ENV). */+ ENVIRONMENT: string | undefined;+ /** OTLP/HTTP traces endpoint; unset disables trace export. */+ OTEL_EXPORTER_OTLP_ENDPOINT: string | undefined;+ /** Datadog API key attached to trace exports when the endpoint is a+ * Datadog intake; secret, set via `wrangler secret put DD_API_KEY`. */+ DD_API_KEY: string | undefined;+};++const REQUIRED = [+ 'BOT_TOKEN',+ 'CHANNEL_ID',+ 'ACTING_USER_ID',+ 'WEBHOOK_SECRET',+ 'ALERT_MESSAGES',+] as const satisfies readonly (keyof Env)[];++// One app per isolate, so the SDK's cached bot lookup survives across requests.+let app: ReturnType<typeof createApp> | undefined;++function fetch(request: Request, env: Env): Response | Promise<Response> {+ const missing = REQUIRED.filter((key) => !env[key]);+ if (missing.length > 0) {+ return Response.json(+ { error: `missing worker bindings: ${missing.join(', ')}` },+ { status: 500 }+ );+ }++ app ??= createApp({+ webhookSecret: env.WEBHOOK_SECRET,+ channel: createAlertChannel({+ channelId: env.CHANNEL_ID,+ botToken: env.BOT_TOKEN,+ actingUserId: env.ACTING_USER_ID,+ env: 'prod',+ }),+ store: createKvAlertStore(env.ALERT_MESSAGES),+ });++ return app.fetch(request, env);+}++// instrument() wraps fetch with a request root span, registers the global+// tracer provider/context manager (which Telemetry.span resolves through),+// propagates traceparent, and flushes spans via waitUntil after the response.+export default instrument({ fetch }, (env: Env) =>+ createWorkerTraceConfig({+ serviceName: 'goalert-bot',+ environment: env.ENVIRONMENT ?? 'local',+ tracesUrl: env.OTEL_EXPORTER_OTLP_ENDPOINT,+ tracesHeaders: env.DD_API_KEY+ ? { 'dd-api-key': env.DD_API_KEY }+ : undefined,+ })+);diff --git a/services/bots/goalert-bot/test/format.test.ts b/services/bots/goalert-bot/test/format.test.tsnew file mode 100644index 0000000000..df15881c6d--- /dev/null+++ b/services/bots/goalert-bot/test/format.test.ts@@ -0,0 +1,120 @@+import { describe, expect, test } from 'bun:test';+import {+ compactAlert,+ firstSentences,+ renderPost,+ statusPost,+} from '../src/format';++// A real Datadog monitor message and Grafana alert details, as GoAlert+// forwards them in `Details`.+const DATADOG_DETAILS = `https://us5.datadoghq.com/event/event?id=8876996053944573810++🔴 CRITICAL: 2631.578 (threshold 1800.0)++The oldest open transaction on macro-db-prod.ctkwyjgndnfr.us-east-1.rds.amazonaws.com is 2631.578 seconds old. Long transactions hold locks and stop vacuum.++Runbook: https://github.com/macro-inc/macro/blob/main/docs/runbooks/macrodb.md#locks+Dashboard: https://us5.datadoghq.com/dashboard/42x-f55-h6z/macrodb-alerts++@webhook-goalert-notify++\`avg(last_10m):max:postgresql.activity.xact_start_age{host:macro-db-prod*} by {host} > 1800\`++Metric value: 2631.578++Metric Graph: https://us5.datadoghq.com/monitors/23404621?group=host · Monitor Status: https://us5.datadoghq.com/monitors/23404621`;++const GRAFANA_DETAILS = `| Label | Value |+| ----- | ----- |+| DBInstanceIdentifier | macro-db-prod-read-replica |+| alertname | [PROD] macro-db-prod database load above vCPU count (warning) |+| pages | false |+| severity | warning |+| Annotation | Value |+| ---------- | ----- |+| description | macro-db-prod-read-replica has 3.98x as many active sessions as vCPUs. Normal is under 1.3. |+| panel_url | https://grafana.macro-internal.com/d/prod-alerts?viewPanel=4 |+| runbook_url | https://github.com/macro-inc/macro/blob/main/docs/runbooks/macrodb.md#load |+| summary | macro-db-prod database load above vCPU count |++Source: https://grafana.macro-internal.com/alerting/grafana/macrodb-load-warn/view?orgId=1++\`\`\`+[ var='B' labels={} type='reduce' value=3.98 ]+\`\`\``;++const HERE_TAG = '<m-group-mention>{"groupAlias":"here"}</m-group-mention>';++describe('compact alert posts', () => {+ test('Datadog notify alert', () => {+ const post = compactAlert({+ ServiceName: 'Notify',+ Summary:+ '[P3] [Triggered on {host:macro-db-prod.ctkwyjgndnfr.us-east-1.rds.amazonaws.com}] [PROD] macro-db-prod long-running transaction',+ Details: DATADOG_DETAILS,+ GoAlertURL: 'https://oncall.macro-internal.com/alerts/130',+ });+ expect(renderPost(post)).toEqual({+ content: `🔔 · **macro-db-prod long-running transaction — macro-db-prod**+🔴 critical: 2631.578 (threshold 1800.0)+The oldest open transaction on macro-db-prod is 2631.578 seconds old. Long transactions hold locks and stop vacuum.+[Acknowledge](https://oncall.macro-internal.com/alerts/130) · [Graph](https://us5.datadoghq.com/dashboard/42x-f55-h6z/macrodb-alerts) · [Runbook](https://github.com/macro-inc/macro/blob/main/docs/runbooks/macrodb.md#locks)`,+ mentions: [],+ });+ });++ test('Grafana page mentions here', () => {+ const post = compactAlert({+ ServiceName: 'Grafana critical',+ Summary: 'macro-db-prod database load above vCPU count',+ Details: GRAFANA_DETAILS,+ GoAlertURL: 'https://oncall.macro-internal.com/alerts/127',+ });+ expect(renderPost(post)).toEqual({+ content: `🚨 **PAGE** · **macro-db-prod database load above vCPU count — macro-db-prod-read-replica** ${HERE_TAG}+🟠 warning+macro-db-prod-read-replica has 3.98x as many active sessions as vCPUs. Normal is under 1.3.+[Acknowledge](https://oncall.macro-internal.com/alerts/127) · [Graph](https://grafana.macro-internal.com/d/prod-alerts?viewPanel=4) · [Runbook](https://github.com/macro-inc/macro/blob/main/docs/runbooks/macrodb.md#load)`,+ mentions: [{ entity_type: 'group', entity_id: 'here' }],+ });+ });++ test('resolved page keeps the post and drops the mention', () => {+ const post = compactAlert({+ ServiceName: 'Grafana critical',+ Summary: 'macro-db-prod database load above vCPU count',+ Details: GRAFANA_DETAILS,+ GoAlertURL: 'https://oncall.macro-internal.com/alerts/127',+ });+ expect(renderPost(post, true)).toEqual({+ content: `✅ Resolved · 🚨 **PAGE** · **macro-db-prod database load above vCPU count — macro-db-prod-read-replica**+🟠 warning+macro-db-prod-read-replica has 3.98x as many active sessions as vCPUs. Normal is under 1.3.+[Acknowledge](https://oncall.macro-internal.com/alerts/127) · [Graph](https://grafana.macro-internal.com/d/prod-alerts?viewPanel=4) · [Runbook](https://github.com/macro-inc/macro/blob/main/docs/runbooks/macrodb.md#load)`,+ mentions: [],+ });+ });++ test('status change without a posted alert is one line', () => {+ expect(+ statusPost({+ Type: 'AlertStatus',+ AlertID: 48,+ ServiceID: '',+ ServiceName: 'Notify',+ Summary: '[P3] [Recovered on {tablename:events}] DynamoDB reads high',+ LogEntry: 'Closed by Datadog',+ GoAlertURL: 'https://oncall.macro-internal.com/alerts/48',+ })+ ).toEqual({+ content: `Closed by Datadog · DynamoDB reads high — events · [Open](https://oncall.macro-internal.com/alerts/48)`,+ mentions: [],+ });+ });++ test('long description is cut at a sentence', () => {+ const sentence = `${'A'.repeat(200)}. `;+ expect(firstSentences(sentence.repeat(3))).toBe(`${'A'.repeat(200)}.`);+ });+});diff --git a/services/bots/goalert-bot/test/webhook.test.ts b/services/bots/goalert-bot/test/webhook.test.tsnew file mode 100644index 0000000000..0a43e279b1--- /dev/null+++ b/services/bots/goalert-bot/test/webhook.test.ts@@ -0,0 +1,261 @@+import { describe, expect, test } from 'bun:test';+import type { RichMessage } from '../../../../packages/sdk/src/mentions';+import type { AlertChannel } from '../src/channel';+import { createApp } from '../src/index';+import { type AlertStore, createMemoryAlertStore } from '../src/store';++type ChannelCall =+ | { kind: 'post'; body: RichMessage }+ | { kind: 'reply'; messageId: string; body: RichMessage }+ | { kind: 'react'; messageId: string; emoji: string }+ | { kind: 'edit'; messageId: string; body: RichMessage };++/** Records every channel call; posts get ids `message-1`, `message-2`, … */+function fakeChannel(): AlertChannel & { calls: ChannelCall[] } {+ const calls: ChannelCall[] = [];+ return {+ calls,+ post: async (body) => {+ calls.push({ kind: 'post', body });+ return `message-${calls.filter((call) => call.kind === 'post').length}`;+ },+ reply: async (messageId, body) => {+ calls.push({ kind: 'reply', messageId, body });+ },+ react: async (messageId, emoji) => {+ calls.push({ kind: 'react', messageId, emoji });+ },+ edit: async (messageId, body) => {+ calls.push({ kind: 'edit', messageId, body });+ },+ };+}++function makeApp(channel: AlertChannel, store: AlertStore) {+ return createApp({ webhookSecret: 'test-secret', channel, store });+}++function postWebhook(+ app: ReturnType<typeof makeApp>,+ body: unknown,+ secret = 'test-secret'+) {+ return app.request(`/goalert/${secret}`, {+ method: 'POST',+ headers: { 'content-type': 'application/json' },+ body: typeof body === 'string' ? body : JSON.stringify(body),+ });+}++const HERE_TAG = '<m-group-mention>{"groupAlias":"here"}</m-group-mention>';++const PAGE_ALERT = {+ AppName: 'GoAlert',+ Type: 'Alert',+ AlertID: 127,+ Summary: 'macro-db-prod CPU high',+ Details: '| severity | critical |\n| description | CPU at 92%. |',+ ServiceID: 'b4b2c1a6-0000-4000-8000-000000000001',+ ServiceName: 'Grafana critical',+ Meta: {},+ GoAlertURL: 'https://oncall.macro-internal.com/alerts/127',+};++const PAGE_POST = {+ page: true,+ headline: '🚨 **PAGE** · **macro-db-prod CPU high**',+ lines: [+ '🔴 critical',+ 'CPU at 92%.',+ '[Acknowledge](https://oncall.macro-internal.com/alerts/127)',+ ],+};++function statusFor(alertId: number, logEntry: string) {+ return {+ AppName: 'GoAlert',+ Type: 'AlertStatus',+ AlertID: alertId,+ Summary: 'macro-db-prod CPU high',+ ServiceID: 'b4b2c1a6-0000-4000-8000-000000000001',+ ServiceName: 'Grafana critical',+ LogEntry: logEntry,+ GoAlertURL: `https://oncall.macro-internal.com/alerts/${alertId}`,+ };+}++describe('goalert-bot webhook', () => {+ test('health check', async () => {+ const app = makeApp(fakeChannel(), createMemoryAlertStore());+ const res = await app.request('/health');+ expect(res.status).toBe(200);+ expect(await res.json()).toEqual({ ok: true });+ });++ test('rejects a wrong secret without posting', async () => {+ const channel = fakeChannel();+ const app = makeApp(channel, createMemoryAlertStore());+ const res = await postWebhook(app, PAGE_ALERT, 'wrong-secret');+ expect(res.status).toBe(401);+ expect(channel.calls).toEqual([]);+ });++ test('rejects invalid JSON', async () => {+ const channel = fakeChannel();+ const app = makeApp(channel, createMemoryAlertStore());+ const res = await postWebhook(app, 'not json{');+ expect(res.status).toBe(400);+ expect(channel.calls).toEqual([]);+ });++ test('acknowledges and ignores unrecognized payloads', async () => {+ const channel = fakeChannel();+ const app = makeApp(channel, createMemoryAlertStore());+ const res = await postWebhook(app, { Type: 'SomethingNew' });+ expect(res.status).toBe(200);+ expect(await res.json()).toEqual({ ok: true, ignored: true });+ expect(channel.calls).toEqual([]);+ });++ test('new alert is posted and stored', async () => {+ const channel = fakeChannel();+ const store = createMemoryAlertStore();+ const res = await postWebhook(makeApp(channel, store), PAGE_ALERT);+ expect(res.status).toBe(200);+ expect(await res.json()).toEqual({ ok: true, outcome: 'posted' });+ expect(channel.calls).toEqual([+ {+ kind: 'post',+ body: {+ content: `🚨 **PAGE** · **macro-db-prod CPU high** ${HERE_TAG}\n🔴 critical\nCPU at 92%.\n[Acknowledge](https://oncall.macro-internal.com/alerts/127)`,+ mentions: [{ entity_type: 'group', entity_id: 'here' }],+ },+ },+ ]);+ expect(await store.get(127)).toEqual({+ messageId: 'message-1',+ post: PAGE_POST,+ });+ });++ test('repeated alert replies in its thread', async () => {+ const channel = fakeChannel();+ const store = createMemoryAlertStore();+ await store.put(127, { messageId: 'message-9', post: PAGE_POST });+ const res = await postWebhook(makeApp(channel, store), PAGE_ALERT);+ expect(await res.json()).toEqual({+ ok: true,+ outcome: 'still_open_replied',+ });+ expect(channel.calls).toEqual([+ {+ kind: 'reply',+ messageId: 'message-9',+ body: {+ content: `⏰ Still open · [Acknowledge](https://oncall.macro-internal.com/alerts/127) ${HERE_TAG}`,+ mentions: [{ entity_type: 'group', entity_id: 'here' }],+ },+ },+ ]);+ });++ test('acknowledgement replies and reacts', async () => {+ const channel = fakeChannel();+ const store = createMemoryAlertStore();+ await store.put(127, { messageId: 'message-9', post: PAGE_POST });+ const res = await postWebhook(+ makeApp(channel, store),+ statusFor(127, 'Acknowledged by Sam')+ );+ expect(await res.json()).toEqual({ ok: true, outcome: 'status_replied' });+ expect(channel.calls).toEqual([+ {+ kind: 'reply',+ messageId: 'message-9',+ body: { content: '👀 Acknowledged by Sam', mentions: [] },+ },+ { kind: 'react', messageId: 'message-9', emoji: '👀' },+ ]);+ });++ test('close replies, reacts, and marks the post resolved', async () => {+ const channel = fakeChannel();+ const store = createMemoryAlertStore();+ await store.put(127, { messageId: 'message-9', post: PAGE_POST });+ const res = await postWebhook(+ makeApp(channel, store),+ statusFor(127, 'Closed by Datadog')+ );+ expect(await res.json()).toEqual({ ok: true, outcome: 'status_replied' });+ expect(channel.calls).toEqual([+ {+ kind: 'reply',+ messageId: 'message-9',+ body: { content: '✅ Closed by Datadog', mentions: [] },+ },+ { kind: 'react', messageId: 'message-9', emoji: '✅' },+ {+ kind: 'edit',+ messageId: 'message-9',+ body: {+ content:+ '✅ Resolved · 🚨 **PAGE** · **macro-db-prod CPU high**\n🔴 critical\nCPU at 92%.\n[Acknowledge](https://oncall.macro-internal.com/alerts/127)',+ mentions: [],+ },+ },+ ]);+ });++ test('status change for an unknown alert is posted top-level', async () => {+ const channel = fakeChannel();+ const res = await postWebhook(+ makeApp(channel, createMemoryAlertStore()),+ statusFor(48, 'Closed by Datadog')+ );+ expect(await res.json()).toEqual({ ok: true, outcome: 'status_posted' });+ expect(channel.calls).toEqual([+ {+ kind: 'post',+ body: {+ content:+ 'Closed by Datadog · macro-db-prod CPU high · [Open](https://oncall.macro-internal.com/alerts/48)',+ mentions: [],+ },+ },+ ]);+ });++ test('bundle is one top-level line', async () => {+ const channel = fakeChannel();+ const res = await postWebhook(makeApp(channel, createMemoryAlertStore()), {+ AppName: 'GoAlert',+ Type: 'AlertBundle',+ ServiceID: 'b4b2c1a6-0000-4000-8000-000000000002',+ ServiceName: 'Notify',+ Count: 5,+ GoAlertURL: 'https://oncall.macro-internal.com/services/notify/alerts',+ });+ expect(await res.json()).toEqual({ ok: true, outcome: 'bundle_posted' });+ expect(channel.calls).toEqual([+ {+ kind: 'post',+ body: {+ content:+ '🔔 · 5 alerts on Notify · [Acknowledge](https://oncall.macro-internal.com/services/notify/alerts)',+ mentions: [],+ },+ },+ ]);+ });++ test('a failed channel call returns 502', async () => {+ const channel = fakeChannel();+ channel.post = async () => {+ throw new Error('channel unavailable');+ };+ const store = createMemoryAlertStore();+ const res = await postWebhook(makeApp(channel, store), PAGE_ALERT);+ expect(res.status).toBe(502);+ expect(await store.get(127)).toBeUndefined();+ });+});diff --git a/services/bots/goalert-bot/tsconfig.json b/services/bots/goalert-bot/tsconfig.jsonnew file mode 100644index 0000000000..d73c3dfc9a--- /dev/null+++ b/services/bots/goalert-bot/tsconfig.json@@ -0,0 +1,18 @@+{+ "compilerOptions": {+ "esModuleInterop": true,+ "skipLibCheck": true,+ "forceConsistentCasingInFileNames": true,+ "target": "es2022",+ "module": "es2022",+ "moduleResolution": "Bundler",+ "resolveJsonModule": true,+ "moduleDetection": "force",+ "strict": true,+ "noEmit": true,+ "lib": ["es2022", "dom"],+ "types": ["@types/node", "bun-types"]+ },+ "exclude": ["node_modules"],+ "include": ["src", "test"]+}diff --git a/services/bots/goalert-bot/wrangler.jsonc b/services/bots/goalert-bot/wrangler.jsoncnew file mode 100644index 0000000000..de191b943b--- /dev/null+++ b/services/bots/goalert-bot/wrangler.jsonc@@ -0,0 +1,39 @@+/**+ * For more details on how to configure Wrangler, refer to:+ * https://developers.cloudflare.com/workers/wrangler/configuration/+ *+ * Single prod deployment only — there is no dev environment.+ *+ * Secrets (set with `wrangler secret put <NAME>`):+ * BOT_TOKEN - user-owned bot token (x-macro-bot-token)+ * ACTING_USER_ID - the bot owner's user id; the bot acts as this user+ * WEBHOOK_SECRET - shared secret embedded in the webhook URL path+ * DD_API_KEY - optional; sent with trace exports to a Datadog intake+ */+{+ "$schema": "node_modules/wrangler/config-schema.json",+ "name": "goalert-bot",+ "main": "src/worker.ts",+ "compatibility_date": "2026-01-01",+ // node:crypto for the constant-time webhook secret check, and+ // AsyncLocalStorage for otel-cf-workers' span context.+ "compatibility_flags": ["nodejs_compat"],+ "observability": {+ "enabled": true+ },+ "vars": {+ // Macro Alerts.+ "CHANNEL_ID": "019fb34d-fcfd-7548-9a0f-7084ea16416c",+ // Tags exported spans with deployment.environment (matches DD_ENV).+ "ENVIRONMENT": "production",+ "OTEL_EXPORTER_OTLP_ENDPOINT": "https://macro-prox-prod.macroverse.workers.dev/i/otlp/v1/traces"+ },+ "kv_namespaces": [+ {+ // GoAlert alert id → posted message id (`alert:<AlertID>`).+ "binding": "ALERT_MESSAGES",+ // From `wrangler kv namespace create ALERT_MESSAGES` — see README.+ "id": "3cfc0519b2254d6b9e64434658156d52"+ }+ ]+}