Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(observability): post compact, separated alerts to the channel (#9214)
Wolf Mermelsteind16659e1eeed
diff --git a/infra/stacks/observability/AGENTS.md b/infra/stacks/observability/AGENTS.mdindex adff157400..6e9f5de4bb 100644--- a/infra/stacks/observability/AGENTS.md+++ b/infra/stacks/observability/AGENTS.md@@ -29,6 +29,8 @@ us-east-1. `README.md` is the full operations guide.From this directory:- `nix build --dry-run .#nixosConfigurations.observability.config.system.build.toplevel`- `nix fmt -- <changed .nix files>`; `python3 -m py_compile` on changed Python.+- `python3 -m unittest test_macro_alerts_relay` from `nixos/goalert/` when the+ relay's channel post format changes.From `infra/`: `bun run check`, then Biome on changed `.ts`.diff --git a/infra/stacks/observability/Pulumi.prod.yaml b/infra/stacks/observability/Pulumi.prod.yamlindex 47aa9b3e11..bfe5cb8108 100644--- a/infra/stacks/observability/Pulumi.prod.yaml+++ b/infra/stacks/observability/Pulumi.prod.yaml@@ -2,5 +2,5 @@ config:aws:region: us-east-2observability:ciIngestSecretArn: arn:aws:secretsmanager:us-east-1:569036502058:secret:observability/dev-ingest-7UhJbLobservability:alarmTopicArn: arn:aws:sns:us-east-2:569036502058:observability-prod-alerts- observability:amiId: ami-02bbbfa6d82d3cff2+ observability:amiId: ami-0339a8c2151661daaobservability:secretArn: arn:aws:secretsmanager:us-east-2:569036502058:secret:observability/prod-TwQpBJdiff --git a/infra/stacks/observability/README.md b/infra/stacks/observability/README.mdindex d4240c984d..391a44bdf9 100644--- a/infra/stacks/observability/README.md+++ b/infra/stacks/observability/README.md@@ -512,12 +512,12 @@ server; see [MCP servers](../../../docs/development/mcp-servers.md#goalert).- A nightly timer dumps the database to `s3://<logs bucket>/goalert/`, one objectper weekday. The dump goes to a file first, so a failed dump never replaces agood backup.-- `macro-alerts-relay` on `127.0.0.1:8090` turns GoAlert webhooks (`/goalert`)- into Macro channel posts and adds the channel's bot token. GoAlert webhooks- cannot set headers, and the token stays in one service. A new alert posts as- `🚨 PAGE [<service>]` (paging services) or `🔔 [Notify]`, its summary, the first- 1500 characters of its details and an `Acknowledge:` link; status changes post- one line with the GoAlert link.+- `macro-alerts-relay` on `127.0.0.1:8090` turns GoAlert webhooks into Macro+ channel posts and adds the channel's bot token; GoAlert webhooks cannot set+ headers. Each post starts with `---`, then the page or notify marker and the+ title without Datadog's bracketed tags, the state, a one-sentence description,+ and `Acknowledge · Graph · Runbook` links. Status changes are one line.+ `goalert/test_macro_alerts_relay.py` pins the format.- nginx serves `oncall.macro-internal.com` from a runtime server block. Only`/health`, `/api/v2/generic/incoming` and `/api/v2/heartbeat/` are reachablewithout a session (GoAlert checks the integration key); other inbounddiff --git a/infra/stacks/observability/nixos/goalert/macro-alerts-relay.py b/infra/stacks/observability/nixos/goalert/macro-alerts-relay.pyindex d806225f9d..2a9f2048e1 100644--- a/infra/stacks/observability/nixos/goalert/macro-alerts-relay.py+++ b/infra/stacks/observability/nixos/goalert/macro-alerts-relay.py@@ -8,47 +8,130 @@ forwarded once; GoAlert owns retries."""import jsonimport logging+import reimport osimport timefrom http.server import BaseHTTPRequestHandler, ThreadingHTTPServerfrom pathlib import Pathimport urllib.request-CREDENTIALS = Path(os.environ['CREDENTIALS_DIRECTORY'])-WEBHOOK_URL = (CREDENTIALS / 'macro_alerts_webhook_url').read_text()-BOT_TOKEN = (CREDENTIALS / 'macro_alerts_bot_token').read_text()# GoAlert abandons a webhook after three seconds.FORWARD_TIMEOUT_SECONDS = 2.5MAX_BODY_BYTES = 256 * 1024MAX_CONTENT_CHARS = 4000-MAX_DETAILS_CHARS = 1500+MAX_DESCRIPTION_CHARS = 300+# Datadog titles start with bracketed tags: `[P3] [Re-Triggered on {host:x}] [PROD] Title`.+DATADOG_TAG = re.compile(r'^\s*\[([^\]]*)\]\s*')+SCOPE = re.compile(r'on \{(.+)\}')+# Datadog's state line, from `alertMessage()` in infra/stacks/monitoring/notify.ts.+DATADOG_STATE = re.compile(r'^(🔴|🟠|✅)\s*(CRITICAL|WARNING|RECOVERED):\s*(.*)$')+GRAFANA_ROW = re.compile(r'^\|\s*([^|]+?)\s*\|\s*(.*?)\s*\|$')+# `macro-db-prod.ctkwyjgndnfr.us-east-1.rds.amazonaws.com` -> `macro-db-prod`.+AWS_HOSTNAME = re.compile(r'\b([a-z0-9-]+)\.[a-z0-9.-]+\.amazonaws\.com\b')+LINK_LINE = re.compile(r'^(Runbook|Dashboard|Metric Graph|Source):\s*(https?://\S+)')-def service_prefix(service: str) -> str:- """`Notify` only posts to the channel; every other service pages."""- marker = '🔔' if 'Notify' in service else '🚨 PAGE'- return f"{marker} [{service}]" if service else marker++def short_scope(scope: str) -> str:+ """`host:macro-db-prod.abc.us-east-1.rds.amazonaws.com` -> `macro-db-prod`."""+ values = [part.split(':', 1)[-1] for part in scope.split(',')]+ return ', '.join(v.split('.', 1)[0] if '.amazonaws.com' in v else v for v in values)+++def datadog_title(summary: str) -> tuple[str, str]:+ """The title without Datadog's bracketed tags, and the group it fired on."""+ scope = ''+ while (match := DATADOG_TAG.match(summary)):+ if (on := SCOPE.search(match.group(1))):+ scope = short_scope(on.group(1))+ summary = summary[match.end():]+ return summary.strip(), scope+++def first_sentences(text: str) -> str:+ text = AWS_HOSTNAME.sub(r'\1', ' '.join(text.split()))+ if len(text) <= MAX_DESCRIPTION_CHARS:+ return text+ cut = text[:MAX_DESCRIPTION_CHARS]+ return (cut[: cut.rfind('. ') + 1] if '. ' in cut else cut.rstrip()) or cut+++def parse_details(details: str) -> dict:+ """State, description and links from a Datadog message or Grafana's details table."""+ parsed: dict = {'links': {}}+ rows = {}+ for line in details.splitlines():+ line = line.strip()+ if (row := GRAFANA_ROW.match(line)):+ rows[row.group(1)] = row.group(2)+ elif (state := DATADOG_STATE.match(line)):+ parsed['state'] = f"{state.group(1)} {state.group(2).lower()}: {state.group(3)}"+ elif (link := LINK_LINE.match(line)):+ parsed['links'].setdefault(link.group(1), link.group(2))+ elif (+ line and 'description' not in parsed and 'state' in parsed+ and not line.startswith(('@', '`', 'http', 'Metric value', 'Paging on-call'))+ ):+ parsed['description'] = line+ if rows:+ severity = rows.get('severity', '')+ parsed['state'] = {'critical': '🔴 critical', 'warning': '🟠 warning'}.get(severity, severity)+ parsed['description'] = rows.get('description', '')+ parsed['scope'] = rows.get('DBInstanceIdentifier') or rows.get('CacheClusterId') or rows.get('QueueName', '')+ for name, key in (('Graph', 'panel_url'), ('Runbook', 'runbook_url')):+ if rows.get(key):+ parsed['links'][name] = rows[key]+ return parsed+++def markdown_links(links: dict[str, str]) -> str:+ return ' · '.join(f"[{name}]({url})" for name, url in links.items() if url)+++def headline(payload: dict) -> tuple[str, str]:+ """The page or notify marker, and the alert's cleaned-up title with its scope."""+ marker = '🔔' if 'Notify' in payload.get('ServiceName', '') else '🚨 **PAGE**'+ title, scope = datadog_title(payload.get('Summary', ''))+ return marker, f"{title} — {scope}" if scope else title+++def alert_post(payload: dict) -> str:+ """+ ---+ 🚨 **PAGE** · macro-db-prod CPU high — macro-db-prod+ 🔴 critical: 92 (threshold 85)+ macro-db-prod CPU averaged 92% over 15 minutes.+ [Acknowledge](…) · [Graph](…) · [Runbook](…)+ """+ marker, title = headline(payload)+ parsed = parse_details(payload.get('Details', ''))+ if parsed.get('scope') and parsed['scope'] not in title:+ title = f"{title} — {parsed['scope']}"+ links = {'Acknowledge': payload.get('GoAlertURL', '')}+ links['Graph'] = parsed['links'].get('Graph') or parsed['links'].get('Dashboard') or parsed['links'].get('Metric Graph', '')+ links['Runbook'] = parsed['links'].get('Runbook', '')+ lines = ['---', f"{marker} · **{title}**"]+ if parsed.get('state'):+ lines.append(parsed['state'])+ if parsed.get('description'):+ lines.append(first_sentences(parsed['description']))+ lines.append(markdown_links(links))+ return '\n'.join(lines)def goalert_content(payload: dict) -> str | None:kind = payload.get('Type')link = payload.get('GoAlertURL', '')- service = payload.get('ServiceName', '')if kind == 'Alert':- details = payload.get('Details', '').strip()- if len(details) > MAX_DETAILS_CHARS:- details = details[:MAX_DETAILS_CHARS].rstrip() + '\n…'- lines = [f"{service_prefix(service)} {payload.get('Summary', '')}"]- if details:- lines.append(details)- lines.append(f"Acknowledge: {link}")- return '\n\n'.join(lines)+ return alert_post(payload)if kind == 'AlertStatus':- return f"{payload.get('LogEntry', '')}: {payload.get('Summary', '')}\n{link}"+ _, title = headline(payload)+ return f"---\n{payload.get('LogEntry', '')} · {title} · [Open]({link})"if kind == 'AlertBundle':- return f"{service_prefix(service)} {payload.get('Count')} alerts\nAcknowledge: {link}"+ marker, _ = headline(payload)+ return f"---\n{marker} · {payload.get('Count')} alerts on {payload.get('ServiceName', '')} · [Acknowledge]({link})"if kind == 'AlertStatusBundle':- return f"{payload.get('LogEntry', '')} ({payload.get('Count')} alerts)\n{link}"+ return f"---\n{payload.get('LogEntry', '')} ({payload.get('Count')} alerts) · [Open]({link})"if kind == 'ScheduleOnCallUsers':names = ', '.join(user.get('Name', '') for user in payload.get('Users', []))return f"On call for {payload.get('ScheduleName', '')}: {names or 'nobody'}\n{payload.get('ScheduleURL', '')}"@@ -77,9 +160,9 @@ class Relay(BaseHTTPRequestHandler):self.send_error(422)returnrequest = urllib.request.Request(- WEBHOOK_URL,+ self.server.webhook_url,data=json.dumps({'content': content[:MAX_CONTENT_CHARS]}).encode(),- headers={'Content-Type': 'application/json', 'x-macro-channel-bot-token': BOT_TOKEN},+ headers={'Content-Type': 'application/json', 'x-macro-channel-bot-token': self.server.bot_token},method='POST',)started = time.monotonic()@@ -107,4 +190,8 @@ class Relay(BaseHTTPRequestHandler):if __name__ == '__main__':logging.basicConfig(level=logging.INFO, format='%(levelname)s %(message)s')- ThreadingHTTPServer(('127.0.0.1', 8090), Relay).serve_forever()+ credentials = Path(os.environ['CREDENTIALS_DIRECTORY'])+ server = ThreadingHTTPServer(('127.0.0.1', 8090), Relay)+ server.webhook_url = (credentials / 'macro_alerts_webhook_url').read_text()+ server.bot_token = (credentials / 'macro_alerts_bot_token').read_text()+ server.serve_forever()diff --git a/infra/stacks/observability/nixos/goalert/test_macro_alerts_relay.py b/infra/stacks/observability/nixos/goalert/test_macro_alerts_relay.pynew file mode 100644index 0000000000..715dbf9625--- /dev/null+++ b/infra/stacks/observability/nixos/goalert/test_macro_alerts_relay.py@@ -0,0 +1,96 @@+"""Run with `python3 -m unittest` from this directory."""+import importlib.util+from pathlib import Path+import unittest++spec = importlib.util.spec_from_file_location(+ 'relay', Path(__file__).with_name('macro-alerts-relay.py')+)+relay = importlib.util.module_from_spec(spec)+spec.loader.exec_module(relay)++DATADOG_DETAILS = """https://us5.datadoghq.com/event/event?id=8876996053944573810++🔴 CRITICAL: 2631.578 (threshold 1800.0)++The oldest open transaction on macro-db-prod.ctkwyjgndnfr.us-east-1.rds.amazonaws.com is 2631.578 seconds old. Long transactions hold locks and stop vacuum.++Runbook: https://github.com/macro-inc/macro/blob/main/docs/runbooks/macrodb.md#locks+Dashboard: https://us5.datadoghq.com/dashboard/42x-f55-h6z/macrodb-alerts++@webhook-goalert-notify++`avg(last_10m):max:postgresql.activity.xact_start_age{host:macro-db-prod*} by {host} > 1800`++Metric value: 2631.578++Metric Graph: https://us5.datadoghq.com/monitors/23404621?group=host · Monitor Status: https://us5.datadoghq.com/monitors/23404621"""++GRAFANA_DETAILS = """| Label | Value |+| ----- | ----- |+| DBInstanceIdentifier | macro-db-prod-read-replica |+| alertname | [PROD] macro-db-prod database load above vCPU count (warning) |+| pages | false |+| severity | warning |+| Annotation | Value |+| ---------- | ----- |+| description | macro-db-prod-read-replica has 3.98x as many active sessions as vCPUs. Normal is under 1.3. |+| panel_url | https://grafana.macro-internal.com/d/prod-alerts?viewPanel=4 |+| runbook_url | https://github.com/macro-inc/macro/blob/main/docs/runbooks/macrodb.md#load |+| summary | macro-db-prod database load above vCPU count |++Source: https://grafana.macro-internal.com/alerting/grafana/macrodb-load-warn/view?orgId=1++```+[ var='B' labels={} type='reduce' value=3.98 ]+```"""+++class AlertPostTest(unittest.TestCase):+ def test_datadog_alert(self):+ post = relay.goalert_content({+ 'Type': 'Alert',+ 'ServiceName': 'Notify',+ 'Summary': '[P3] [Triggered on {host:macro-db-prod.ctkwyjgndnfr.us-east-1.rds.amazonaws.com}] [PROD] macro-db-prod long-running transaction',+ 'Details': DATADOG_DETAILS,+ 'GoAlertURL': 'https://oncall.macro-internal.com/alerts/130',+ })+ self.assertEqual(post, """---+🔔 · **macro-db-prod long-running transaction — macro-db-prod**+🔴 critical: 2631.578 (threshold 1800.0)+The oldest open transaction on macro-db-prod is 2631.578 seconds old. Long transactions hold locks and stop vacuum.+[Acknowledge](https://oncall.macro-internal.com/alerts/130) · [Graph](https://us5.datadoghq.com/dashboard/42x-f55-h6z/macrodb-alerts) · [Runbook](https://github.com/macro-inc/macro/blob/main/docs/runbooks/macrodb.md#locks)""")++ def test_grafana_page(self):+ post = relay.goalert_content({+ 'Type': 'Alert',+ 'ServiceName': 'Grafana critical',+ 'Summary': 'macro-db-prod database load above vCPU count',+ 'Details': GRAFANA_DETAILS,+ 'GoAlertURL': 'https://oncall.macro-internal.com/alerts/127',+ })+ self.assertEqual(post, """---+🚨 **PAGE** · **macro-db-prod database load above vCPU count — macro-db-prod-read-replica**+🟠 warning+macro-db-prod-read-replica has 3.98x as many active sessions as vCPUs. Normal is under 1.3.+[Acknowledge](https://oncall.macro-internal.com/alerts/127) · [Graph](https://grafana.macro-internal.com/d/prod-alerts?viewPanel=4) · [Runbook](https://github.com/macro-inc/macro/blob/main/docs/runbooks/macrodb.md#load)""")++ def test_status_update_is_one_line(self):+ post = relay.goalert_content({+ 'Type': 'AlertStatus',+ 'ServiceName': 'Notify',+ 'Summary': '[P3] [Recovered on {tablename:events}] DynamoDB reads high',+ 'LogEntry': 'Closed by Datadog',+ 'GoAlertURL': 'https://oncall.macro-internal.com/alerts/48',+ })+ self.assertEqual(post, """---+Closed by Datadog · DynamoDB reads high — events · [Open](https://oncall.macro-internal.com/alerts/48)""")++ def test_long_description_is_cut_at_a_sentence(self):+ sentence = 'A' * 200 + '. '+ cut = relay.first_sentences(sentence * 3)+ self.assertEqual(cut, 'A' * 200 + '.')+++if __name__ == '__main__':+ unittest.main()diff --git a/infra/stacks/observability/nixos/prod-alerts.nix b/infra/stacks/observability/nixos/prod-alerts.nixindex 68810ffe24..ef61f95d0d 100644--- a/infra/stacks/observability/nixos/prod-alerts.nix+++ b/infra/stacks/observability/nixos/prod-alerts.nix@@ -94,7 +94,7 @@ infiresWhen = "below";warning = 2000000000;critical = 1000000000;- description = "Freeable memory is {{ $values.B }} bytes. Normal is 7-18 GB; it fell to about 0.45 GB during the October overload. Each connection holds about 14 MB of private memory, so check connection count and swap first.";+ description = "Freeable memory is {{ humanize $values.B.Value }} bytes. Normal is 7-18 GB; it fell to about 0.45 GB during the October overload. Each connection holds about 14 MB of private memory, so check connection count and swap first.";runbook = "macrodb.md#memory";}{@@ -110,7 +110,7 @@ infiresWhen = "below";warning = 1500000000;critical = 750000000;- description = "Replica freeable memory is {{ $values.B }} bytes. Normal minimum is about 3 GB. The replica serves GraphQL reads, so memory pressure shows up as slow lists.";+ description = "Replica freeable memory is {{ humanize $values.B.Value }} bytes. Normal minimum is about 3 GB. The replica serves GraphQL reads, so memory pressure shows up as slow lists.";runbook = "macrodb.md#memory";}{@@ -126,7 +126,7 @@ infiresWhen = "above";warning = 2000000000;critical = 5000000000;- description = "{{ $labels.DBInstanceIdentifier }} is using {{ $values.B }} bytes of swap. Normal is under 0.6 GB. Swap above 5 GB preceded both October overloads by hours.";+ description = "{{ $labels.DBInstanceIdentifier }} is using {{ humanize $values.B.Value }} bytes of swap. Normal is under 0.6 GB. Swap above 5 GB preceded both October overloads by hours.";runbook = "macrodb.md#memory";}{@@ -142,7 +142,7 @@ infiresWhen = "above";warning = 2;critical = 4;- description = "{{ $labels.DBInstanceIdentifier }} has {{ $values.B }}x as many active sessions as vCPUs. Normal is under 1.3; the October overload reached 8. Look at Performance Insights top waits (locks, IO) and the services holding the most connections.";+ description = "{{ $labels.DBInstanceIdentifier }} has {{ humanize $values.B.Value }}x as many active sessions as vCPUs. Normal is under 1.3; the October overload reached 8. Look at Performance Insights top waits (locks, IO) and the services holding the most connections.";runbook = "macrodb.md#load";}{@@ -158,7 +158,7 @@ infiresWhen = "above";warning = 70;critical = 85;- description = "{{ $labels.DBInstanceIdentifier }} CPU averaged {{ $values.B }}% over 15 minutes. Normal peaks are about 30%.";+ description = "{{ $labels.DBInstanceIdentifier }} CPU averaged {{ humanize $values.B.Value }}% over 15 minutes. Normal peaks are about 30%.";runbook = "macrodb.md#load";}{@@ -174,7 +174,7 @@ infiresWhen = "above";warning = 1700;critical = 2200;- description = "macro-db-prod has {{ $values.B }} connections. Normal 15-minute peaks are about 1,100; max_connections is 3,439. Each connection costs about 14 MB, so this also drains freeable memory.";+ description = "macro-db-prod has {{ humanize $values.B.Value }} connections. Normal 15-minute peaks are about 1,100; max_connections is 3,439. Each connection costs about 14 MB, so this also drains freeable memory.";runbook = "macrodb.md#connections";}{@@ -189,7 +189,7 @@ infiresWhen = "above";warning = 550;critical = 700;- description = "The read replica has {{ $values.B }} connections. Normal peaks are about 460.";+ description = "The read replica has {{ humanize $values.B.Value }} connections. Normal peaks are about 460.";runbook = "macrodb.md#connections";}{@@ -205,7 +205,7 @@ infiresWhen = "above";warning = 60;critical = 300;- description = "The read replica is {{ $values.B }} seconds behind the primary. Normal is under 6 seconds. GraphQL reads use the replica, so users see stale data and missing new items.";+ description = "The read replica is {{ humanize $values.B.Value }} seconds behind the primary. Normal is under 6 seconds. GraphQL reads use the replica, so users see stale data and missing new items.";runbook = "macrodb.md#replica-lag";}{@@ -220,7 +220,7 @@ infiresWhen = "above";warning = 0.005;critical = 0.01;- description = "{{ $labels.DBInstanceIdentifier }} disk reads take {{ $values.B }} seconds on average. Normal is about 0.6 ms.";+ description = "{{ $labels.DBInstanceIdentifier }} disk reads take {{ humanize $values.B.Value }} seconds on average. Normal is about 0.6 ms.";runbook = "macrodb.md#load";}{@@ -236,7 +236,7 @@ infiresWhen = "below";warning = 1500000000000;critical = 1000000000000;- description = "{{ $labels.DBInstanceIdentifier }} has {{ $values.B }} bytes of free storage out of 5.37 TB.";+ description = "{{ $labels.DBInstanceIdentifier }} has {{ humanize $values.B.Value }} bytes of free storage out of 5.37 TB.";runbook = "macrodb.md#storage";}];@@ -258,7 +258,7 @@ infiresWhen = "above";warning = 75;critical = 90;- description = "{{ $labels.CacheClusterId }} is using {{ $values.B }}% of maxmemory. Keys without a TTL are never evicted under volatile-lru, so at 100% writes fail.";+ description = "{{ $labels.CacheClusterId }} is using {{ humanize $values.B.Value }}% of maxmemory. Keys without a TTL are never evicted under volatile-lru, so at 100% writes fail.";runbook = "redis.md#memory";}{@@ -274,7 +274,7 @@ infiresWhen = "above";warning = 0;critical = 1000;- description = "{{ $labels.CacheClusterId }} evicted {{ $values.B }} keys in 10 minutes. Prod caches normally evict nothing.";+ description = "{{ $labels.CacheClusterId }} evicted {{ humanize $values.B.Value }} keys in 10 minutes. Prod caches normally evict nothing.";runbook = "redis.md#memory";}];@@ -295,7 +295,7 @@ infiresWhen = "above";warning = 3600;critical = 10800;- description = "The oldest message in email-service-backfill-queue-prod is {{ $values.B }} seconds old. Normal peaks are about 50 minutes; new users wait this long for their mailbox to fill.";+ description = "The oldest message in email-service-backfill-queue-prod is {{ humanize $values.B.Value }} seconds old. Normal peaks are about 50 minutes; new users wait this long for their mailbox to fill.";runbook = "queues.md#backlog";}{@@ -311,7 +311,7 @@ infiresWhen = "above";warning = 1800;critical = 7200;- description = "The oldest message in {{ $labels.QueueName }} is {{ $values.B }} seconds old. These queues normally drain within minutes.";+ description = "The oldest message in {{ $labels.QueueName }} is {{ humanize $values.B.Value }} seconds old. These queues normally drain within minutes.";runbook = "queues.md#backlog";}];