Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(observability): post compact, separated alerts to the channel (#9214)

Wolf Mermelsteind16659e1eeed
diff --git a/infra/stacks/observability/AGENTS.md b/infra/stacks/observability/AGENTS.md
index adff157400..6e9f5de4bb 100644
--- a/infra/stacks/observability/AGENTS.md
+++ b/infra/stacks/observability/AGENTS.md
@@ -29,6 +29,8 @@ us-east-1. `README.md` is the full operations guide.
From this directory:
- `nix build --dry-run .#nixosConfigurations.observability.config.system.build.toplevel`
- `nix fmt -- <changed .nix files>`; `python3 -m py_compile` on changed Python.
+- `python3 -m unittest test_macro_alerts_relay` from `nixos/goalert/` when the
+ relay's channel post format changes.
From `infra/`: `bun run check`, then Biome on changed `.ts`.
diff --git a/infra/stacks/observability/Pulumi.prod.yaml b/infra/stacks/observability/Pulumi.prod.yaml
index 47aa9b3e11..bfe5cb8108 100644
--- a/infra/stacks/observability/Pulumi.prod.yaml
+++ b/infra/stacks/observability/Pulumi.prod.yaml
@@ -2,5 +2,5 @@ config:
aws:region: us-east-2
observability:ciIngestSecretArn: arn:aws:secretsmanager:us-east-1:569036502058:secret:observability/dev-ingest-7UhJbL
observability:alarmTopicArn: arn:aws:sns:us-east-2:569036502058:observability-prod-alerts
- observability:amiId: ami-02bbbfa6d82d3cff2
+ observability:amiId: ami-0339a8c2151661daa
observability:secretArn: arn:aws:secretsmanager:us-east-2:569036502058:secret:observability/prod-TwQpBJ
diff --git a/infra/stacks/observability/README.md b/infra/stacks/observability/README.md
index d4240c984d..391a44bdf9 100644
--- a/infra/stacks/observability/README.md
+++ b/infra/stacks/observability/README.md
@@ -512,12 +512,12 @@ server; see [MCP servers](../../../docs/development/mcp-servers.md#goalert).
- A nightly timer dumps the database to `s3://<logs bucket>/goalert/`, one object
per weekday. The dump goes to a file first, so a failed dump never replaces a
good backup.
-- `macro-alerts-relay` on `127.0.0.1:8090` turns GoAlert webhooks (`/goalert`)
- into Macro channel posts and adds the channel's bot token. GoAlert webhooks
- cannot set headers, and the token stays in one service. A new alert posts as
- `🚨 PAGE [<service>]` (paging services) or `🔔 [Notify]`, its summary, the first
- 1500 characters of its details and an `Acknowledge:` link; status changes post
- one line with the GoAlert link.
+- `macro-alerts-relay` on `127.0.0.1:8090` turns GoAlert webhooks into Macro
+ channel posts and adds the channel's bot token; GoAlert webhooks cannot set
+ headers. Each post starts with `---`, then the page or notify marker and the
+ title without Datadog's bracketed tags, the state, a one-sentence description,
+ and `Acknowledge · Graph · Runbook` links. Status changes are one line.
+ `goalert/test_macro_alerts_relay.py` pins the format.
- nginx serves `oncall.macro-internal.com` from a runtime server block. Only
`/health`, `/api/v2/generic/incoming` and `/api/v2/heartbeat/` are reachable
without a session (GoAlert checks the integration key); other inbound
diff --git a/infra/stacks/observability/nixos/goalert/macro-alerts-relay.py b/infra/stacks/observability/nixos/goalert/macro-alerts-relay.py
index d806225f9d..2a9f2048e1 100644
--- a/infra/stacks/observability/nixos/goalert/macro-alerts-relay.py
+++ b/infra/stacks/observability/nixos/goalert/macro-alerts-relay.py
@@ -8,47 +8,130 @@ forwarded once; GoAlert owns retries.
"""
import json
import logging
+import re
import os
import time
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
import urllib.request
-CREDENTIALS = Path(os.environ['CREDENTIALS_DIRECTORY'])
-WEBHOOK_URL = (CREDENTIALS / 'macro_alerts_webhook_url').read_text()
-BOT_TOKEN = (CREDENTIALS / 'macro_alerts_bot_token').read_text()
# GoAlert abandons a webhook after three seconds.
FORWARD_TIMEOUT_SECONDS = 2.5
MAX_BODY_BYTES = 256 * 1024
MAX_CONTENT_CHARS = 4000
-MAX_DETAILS_CHARS = 1500
+MAX_DESCRIPTION_CHARS = 300
+# Datadog titles start with bracketed tags: `[P3] [Re-Triggered on {host:x}] [PROD] Title`.
+DATADOG_TAG = re.compile(r'^\s*\[([^\]]*)\]\s*')
+SCOPE = re.compile(r'on \{(.+)\}')
+# Datadog's state line, from `alertMessage()` in infra/stacks/monitoring/notify.ts.
+DATADOG_STATE = re.compile(r'^(🔴|🟠|✅)\s*(CRITICAL|WARNING|RECOVERED):\s*(.*)$')
+GRAFANA_ROW = re.compile(r'^\|\s*([^|]+?)\s*\|\s*(.*?)\s*\|$')
+# `macro-db-prod.ctkwyjgndnfr.us-east-1.rds.amazonaws.com` -> `macro-db-prod`.
+AWS_HOSTNAME = re.compile(r'\b([a-z0-9-]+)\.[a-z0-9.-]+\.amazonaws\.com\b')
+LINK_LINE = re.compile(r'^(Runbook|Dashboard|Metric Graph|Source):\s*(https?://\S+)')
-def service_prefix(service: str) -> str:
- """`Notify` only posts to the channel; every other service pages."""
- marker = '🔔' if 'Notify' in service else '🚨 PAGE'
- return f"{marker} [{service}]" if service else marker
+
+def short_scope(scope: str) -> str:
+ """`host:macro-db-prod.abc.us-east-1.rds.amazonaws.com` -> `macro-db-prod`."""
+ values = [part.split(':', 1)[-1] for part in scope.split(',')]
+ return ', '.join(v.split('.', 1)[0] if '.amazonaws.com' in v else v for v in values)
+
+
+def datadog_title(summary: str) -> tuple[str, str]:
+ """The title without Datadog's bracketed tags, and the group it fired on."""
+ scope = ''
+ while (match := DATADOG_TAG.match(summary)):
+ if (on := SCOPE.search(match.group(1))):
+ scope = short_scope(on.group(1))
+ summary = summary[match.end():]
+ return summary.strip(), scope
+
+
+def first_sentences(text: str) -> str:
+ text = AWS_HOSTNAME.sub(r'\1', ' '.join(text.split()))
+ if len(text) <= MAX_DESCRIPTION_CHARS:
+ return text
+ cut = text[:MAX_DESCRIPTION_CHARS]
+ return (cut[: cut.rfind('. ') + 1] if '. ' in cut else cut.rstrip()) or cut
+
+
+def parse_details(details: str) -> dict:
+ """State, description and links from a Datadog message or Grafana's details table."""
+ parsed: dict = {'links': {}}
+ rows = {}
+ for line in details.splitlines():
+ line = line.strip()
+ if (row := GRAFANA_ROW.match(line)):
+ rows[row.group(1)] = row.group(2)
+ elif (state := DATADOG_STATE.match(line)):
+ parsed['state'] = f"{state.group(1)} {state.group(2).lower()}: {state.group(3)}"
+ elif (link := LINK_LINE.match(line)):
+ parsed['links'].setdefault(link.group(1), link.group(2))
+ elif (
+ line and 'description' not in parsed and 'state' in parsed
+ and not line.startswith(('@', '`', 'http', 'Metric value', 'Paging on-call'))
+ ):
+ parsed['description'] = line
+ if rows:
+ severity = rows.get('severity', '')
+ parsed['state'] = {'critical': '🔴 critical', 'warning': '🟠 warning'}.get(severity, severity)
+ parsed['description'] = rows.get('description', '')
+ parsed['scope'] = rows.get('DBInstanceIdentifier') or rows.get('CacheClusterId') or rows.get('QueueName', '')
+ for name, key in (('Graph', 'panel_url'), ('Runbook', 'runbook_url')):
+ if rows.get(key):
+ parsed['links'][name] = rows[key]
+ return parsed
+
+
+def markdown_links(links: dict[str, str]) -> str:
+ return ' · '.join(f"[{name}]({url})" for name, url in links.items() if url)
+
+
+def headline(payload: dict) -> tuple[str, str]:
+ """The page or notify marker, and the alert's cleaned-up title with its scope."""
+ marker = '🔔' if 'Notify' in payload.get('ServiceName', '') else '🚨 **PAGE**'
+ title, scope = datadog_title(payload.get('Summary', ''))
+ return marker, f"{title} — {scope}" if scope else title
+
+
+def alert_post(payload: dict) -> str:
+ """
+ ---
+ 🚨 **PAGE** · macro-db-prod CPU high — macro-db-prod
+ 🔴 critical: 92 (threshold 85)
+ macro-db-prod CPU averaged 92% over 15 minutes.
+ [Acknowledge](…) · [Graph](…) · [Runbook](…)
+ """
+ marker, title = headline(payload)
+ parsed = parse_details(payload.get('Details', ''))
+ if parsed.get('scope') and parsed['scope'] not in title:
+ title = f"{title} — {parsed['scope']}"
+ links = {'Acknowledge': payload.get('GoAlertURL', '')}
+ links['Graph'] = parsed['links'].get('Graph') or parsed['links'].get('Dashboard') or parsed['links'].get('Metric Graph', '')
+ links['Runbook'] = parsed['links'].get('Runbook', '')
+ lines = ['---', f"{marker} · **{title}**"]
+ if parsed.get('state'):
+ lines.append(parsed['state'])
+ if parsed.get('description'):
+ lines.append(first_sentences(parsed['description']))
+ lines.append(markdown_links(links))
+ return '\n'.join(lines)
def goalert_content(payload: dict) -> str | None:
kind = payload.get('Type')
link = payload.get('GoAlertURL', '')
- service = payload.get('ServiceName', '')
if kind == 'Alert':
- details = payload.get('Details', '').strip()
- if len(details) > MAX_DETAILS_CHARS:
- details = details[:MAX_DETAILS_CHARS].rstrip() + '\n…'
- lines = [f"{service_prefix(service)} {payload.get('Summary', '')}"]
- if details:
- lines.append(details)
- lines.append(f"Acknowledge: {link}")
- return '\n\n'.join(lines)
+ return alert_post(payload)
if kind == 'AlertStatus':
- return f"{payload.get('LogEntry', '')}: {payload.get('Summary', '')}\n{link}"
+ _, title = headline(payload)
+ return f"---\n{payload.get('LogEntry', '')} · {title} · [Open]({link})"
if kind == 'AlertBundle':
- return f"{service_prefix(service)} {payload.get('Count')} alerts\nAcknowledge: {link}"
+ marker, _ = headline(payload)
+ return f"---\n{marker} · {payload.get('Count')} alerts on {payload.get('ServiceName', '')} · [Acknowledge]({link})"
if kind == 'AlertStatusBundle':
- return f"{payload.get('LogEntry', '')} ({payload.get('Count')} alerts)\n{link}"
+ return f"---\n{payload.get('LogEntry', '')} ({payload.get('Count')} alerts) · [Open]({link})"
if kind == 'ScheduleOnCallUsers':
names = ', '.join(user.get('Name', '') for user in payload.get('Users', []))
return f"On call for {payload.get('ScheduleName', '')}: {names or 'nobody'}\n{payload.get('ScheduleURL', '')}"
@@ -77,9 +160,9 @@ class Relay(BaseHTTPRequestHandler):
self.send_error(422)
return
request = urllib.request.Request(
- WEBHOOK_URL,
+ self.server.webhook_url,
data=json.dumps({'content': content[:MAX_CONTENT_CHARS]}).encode(),
- headers={'Content-Type': 'application/json', 'x-macro-channel-bot-token': BOT_TOKEN},
+ headers={'Content-Type': 'application/json', 'x-macro-channel-bot-token': self.server.bot_token},
method='POST',
)
started = time.monotonic()
@@ -107,4 +190,8 @@ class Relay(BaseHTTPRequestHandler):
if __name__ == '__main__':
logging.basicConfig(level=logging.INFO, format='%(levelname)s %(message)s')
- ThreadingHTTPServer(('127.0.0.1', 8090), Relay).serve_forever()
+ credentials = Path(os.environ['CREDENTIALS_DIRECTORY'])
+ server = ThreadingHTTPServer(('127.0.0.1', 8090), Relay)
+ server.webhook_url = (credentials / 'macro_alerts_webhook_url').read_text()
+ server.bot_token = (credentials / 'macro_alerts_bot_token').read_text()
+ server.serve_forever()
diff --git a/infra/stacks/observability/nixos/goalert/test_macro_alerts_relay.py b/infra/stacks/observability/nixos/goalert/test_macro_alerts_relay.py
new file mode 100644
index 0000000000..715dbf9625
--- /dev/null
+++ b/infra/stacks/observability/nixos/goalert/test_macro_alerts_relay.py
@@ -0,0 +1,96 @@
+"""Run with `python3 -m unittest` from this directory."""
+import importlib.util
+from pathlib import Path
+import unittest
+
+spec = importlib.util.spec_from_file_location(
+ 'relay', Path(__file__).with_name('macro-alerts-relay.py')
+)
+relay = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(relay)
+
+DATADOG_DETAILS = """https://us5.datadoghq.com/event/event?id=8876996053944573810
+
+🔴 CRITICAL: 2631.578 (threshold 1800.0)
+
+The oldest open transaction on macro-db-prod.ctkwyjgndnfr.us-east-1.rds.amazonaws.com is 2631.578 seconds old. Long transactions hold locks and stop vacuum.
+
+Runbook: https://github.com/macro-inc/macro/blob/main/docs/runbooks/macrodb.md#locks
+Dashboard: https://us5.datadoghq.com/dashboard/42x-f55-h6z/macrodb-alerts
+
+@webhook-goalert-notify
+
+`avg(last_10m):max:postgresql.activity.xact_start_age{host:macro-db-prod*} by {host} > 1800`
+
+Metric value: 2631.578
+
+Metric Graph: https://us5.datadoghq.com/monitors/23404621?group=host · Monitor Status: https://us5.datadoghq.com/monitors/23404621"""
+
+GRAFANA_DETAILS = """| Label | Value |
+| ----- | ----- |
+| DBInstanceIdentifier | macro-db-prod-read-replica |
+| alertname | [PROD] macro-db-prod database load above vCPU count (warning) |
+| pages | false |
+| severity | warning |
+| Annotation | Value |
+| ---------- | ----- |
+| description | macro-db-prod-read-replica has 3.98x as many active sessions as vCPUs. Normal is under 1.3. |
+| panel_url | https://grafana.macro-internal.com/d/prod-alerts?viewPanel=4 |
+| runbook_url | https://github.com/macro-inc/macro/blob/main/docs/runbooks/macrodb.md#load |
+| summary | macro-db-prod database load above vCPU count |
+
+Source: https://grafana.macro-internal.com/alerting/grafana/macrodb-load-warn/view?orgId=1
+
+```
+[ var='B' labels={} type='reduce' value=3.98 ]
+```"""
+
+
+class AlertPostTest(unittest.TestCase):
+ def test_datadog_alert(self):
+ post = relay.goalert_content({
+ 'Type': 'Alert',
+ 'ServiceName': 'Notify',
+ 'Summary': '[P3] [Triggered on {host:macro-db-prod.ctkwyjgndnfr.us-east-1.rds.amazonaws.com}] [PROD] macro-db-prod long-running transaction',
+ 'Details': DATADOG_DETAILS,
+ 'GoAlertURL': 'https://oncall.macro-internal.com/alerts/130',
+ })
+ self.assertEqual(post, """---
+🔔 · **macro-db-prod long-running transaction — macro-db-prod**
+🔴 critical: 2631.578 (threshold 1800.0)
+The oldest open transaction on macro-db-prod is 2631.578 seconds old. Long transactions hold locks and stop vacuum.
+[Acknowledge](https://oncall.macro-internal.com/alerts/130) · [Graph](https://us5.datadoghq.com/dashboard/42x-f55-h6z/macrodb-alerts) · [Runbook](https://github.com/macro-inc/macro/blob/main/docs/runbooks/macrodb.md#locks)""")
+
+ def test_grafana_page(self):
+ post = relay.goalert_content({
+ 'Type': 'Alert',
+ 'ServiceName': 'Grafana critical',
+ 'Summary': 'macro-db-prod database load above vCPU count',
+ 'Details': GRAFANA_DETAILS,
+ 'GoAlertURL': 'https://oncall.macro-internal.com/alerts/127',
+ })
+ self.assertEqual(post, """---
+🚨 **PAGE** · **macro-db-prod database load above vCPU count — macro-db-prod-read-replica**
+🟠 warning
+macro-db-prod-read-replica has 3.98x as many active sessions as vCPUs. Normal is under 1.3.
+[Acknowledge](https://oncall.macro-internal.com/alerts/127) · [Graph](https://grafana.macro-internal.com/d/prod-alerts?viewPanel=4) · [Runbook](https://github.com/macro-inc/macro/blob/main/docs/runbooks/macrodb.md#load)""")
+
+ def test_status_update_is_one_line(self):
+ post = relay.goalert_content({
+ 'Type': 'AlertStatus',
+ 'ServiceName': 'Notify',
+ 'Summary': '[P3] [Recovered on {tablename:events}] DynamoDB reads high',
+ 'LogEntry': 'Closed by Datadog',
+ 'GoAlertURL': 'https://oncall.macro-internal.com/alerts/48',
+ })
+ self.assertEqual(post, """---
+Closed by Datadog · DynamoDB reads high — events · [Open](https://oncall.macro-internal.com/alerts/48)""")
+
+ def test_long_description_is_cut_at_a_sentence(self):
+ sentence = 'A' * 200 + '. '
+ cut = relay.first_sentences(sentence * 3)
+ self.assertEqual(cut, 'A' * 200 + '.')
+
+
+if __name__ == '__main__':
+ unittest.main()
diff --git a/infra/stacks/observability/nixos/prod-alerts.nix b/infra/stacks/observability/nixos/prod-alerts.nix
index 68810ffe24..ef61f95d0d 100644
--- a/infra/stacks/observability/nixos/prod-alerts.nix
+++ b/infra/stacks/observability/nixos/prod-alerts.nix
@@ -94,7 +94,7 @@ in
firesWhen = "below";
warning = 2000000000;
critical = 1000000000;
- description = "Freeable memory is {{ $values.B }} bytes. Normal is 7-18 GB; it fell to about 0.45 GB during the October overload. Each connection holds about 14 MB of private memory, so check connection count and swap first.";
+ description = "Freeable memory is {{ humanize $values.B.Value }} bytes. Normal is 7-18 GB; it fell to about 0.45 GB during the October overload. Each connection holds about 14 MB of private memory, so check connection count and swap first.";
runbook = "macrodb.md#memory";
}
{
@@ -110,7 +110,7 @@ in
firesWhen = "below";
warning = 1500000000;
critical = 750000000;
- description = "Replica freeable memory is {{ $values.B }} bytes. Normal minimum is about 3 GB. The replica serves GraphQL reads, so memory pressure shows up as slow lists.";
+ description = "Replica freeable memory is {{ humanize $values.B.Value }} bytes. Normal minimum is about 3 GB. The replica serves GraphQL reads, so memory pressure shows up as slow lists.";
runbook = "macrodb.md#memory";
}
{
@@ -126,7 +126,7 @@ in
firesWhen = "above";
warning = 2000000000;
critical = 5000000000;
- description = "{{ $labels.DBInstanceIdentifier }} is using {{ $values.B }} bytes of swap. Normal is under 0.6 GB. Swap above 5 GB preceded both October overloads by hours.";
+ description = "{{ $labels.DBInstanceIdentifier }} is using {{ humanize $values.B.Value }} bytes of swap. Normal is under 0.6 GB. Swap above 5 GB preceded both October overloads by hours.";
runbook = "macrodb.md#memory";
}
{
@@ -142,7 +142,7 @@ in
firesWhen = "above";
warning = 2;
critical = 4;
- description = "{{ $labels.DBInstanceIdentifier }} has {{ $values.B }}x as many active sessions as vCPUs. Normal is under 1.3; the October overload reached 8. Look at Performance Insights top waits (locks, IO) and the services holding the most connections.";
+ description = "{{ $labels.DBInstanceIdentifier }} has {{ humanize $values.B.Value }}x as many active sessions as vCPUs. Normal is under 1.3; the October overload reached 8. Look at Performance Insights top waits (locks, IO) and the services holding the most connections.";
runbook = "macrodb.md#load";
}
{
@@ -158,7 +158,7 @@ in
firesWhen = "above";
warning = 70;
critical = 85;
- description = "{{ $labels.DBInstanceIdentifier }} CPU averaged {{ $values.B }}% over 15 minutes. Normal peaks are about 30%.";
+ description = "{{ $labels.DBInstanceIdentifier }} CPU averaged {{ humanize $values.B.Value }}% over 15 minutes. Normal peaks are about 30%.";
runbook = "macrodb.md#load";
}
{
@@ -174,7 +174,7 @@ in
firesWhen = "above";
warning = 1700;
critical = 2200;
- description = "macro-db-prod has {{ $values.B }} connections. Normal 15-minute peaks are about 1,100; max_connections is 3,439. Each connection costs about 14 MB, so this also drains freeable memory.";
+ description = "macro-db-prod has {{ humanize $values.B.Value }} connections. Normal 15-minute peaks are about 1,100; max_connections is 3,439. Each connection costs about 14 MB, so this also drains freeable memory.";
runbook = "macrodb.md#connections";
}
{
@@ -189,7 +189,7 @@ in
firesWhen = "above";
warning = 550;
critical = 700;
- description = "The read replica has {{ $values.B }} connections. Normal peaks are about 460.";
+ description = "The read replica has {{ humanize $values.B.Value }} connections. Normal peaks are about 460.";
runbook = "macrodb.md#connections";
}
{
@@ -205,7 +205,7 @@ in
firesWhen = "above";
warning = 60;
critical = 300;
- description = "The read replica is {{ $values.B }} seconds behind the primary. Normal is under 6 seconds. GraphQL reads use the replica, so users see stale data and missing new items.";
+ description = "The read replica is {{ humanize $values.B.Value }} seconds behind the primary. Normal is under 6 seconds. GraphQL reads use the replica, so users see stale data and missing new items.";
runbook = "macrodb.md#replica-lag";
}
{
@@ -220,7 +220,7 @@ in
firesWhen = "above";
warning = 0.005;
critical = 0.01;
- description = "{{ $labels.DBInstanceIdentifier }} disk reads take {{ $values.B }} seconds on average. Normal is about 0.6 ms.";
+ description = "{{ $labels.DBInstanceIdentifier }} disk reads take {{ humanize $values.B.Value }} seconds on average. Normal is about 0.6 ms.";
runbook = "macrodb.md#load";
}
{
@@ -236,7 +236,7 @@ in
firesWhen = "below";
warning = 1500000000000;
critical = 1000000000000;
- description = "{{ $labels.DBInstanceIdentifier }} has {{ $values.B }} bytes of free storage out of 5.37 TB.";
+ description = "{{ $labels.DBInstanceIdentifier }} has {{ humanize $values.B.Value }} bytes of free storage out of 5.37 TB.";
runbook = "macrodb.md#storage";
}
];
@@ -258,7 +258,7 @@ in
firesWhen = "above";
warning = 75;
critical = 90;
- description = "{{ $labels.CacheClusterId }} is using {{ $values.B }}% of maxmemory. Keys without a TTL are never evicted under volatile-lru, so at 100% writes fail.";
+ description = "{{ $labels.CacheClusterId }} is using {{ humanize $values.B.Value }}% of maxmemory. Keys without a TTL are never evicted under volatile-lru, so at 100% writes fail.";
runbook = "redis.md#memory";
}
{
@@ -274,7 +274,7 @@ in
firesWhen = "above";
warning = 0;
critical = 1000;
- description = "{{ $labels.CacheClusterId }} evicted {{ $values.B }} keys in 10 minutes. Prod caches normally evict nothing.";
+ description = "{{ $labels.CacheClusterId }} evicted {{ humanize $values.B.Value }} keys in 10 minutes. Prod caches normally evict nothing.";
runbook = "redis.md#memory";
}
];
@@ -295,7 +295,7 @@ in
firesWhen = "above";
warning = 3600;
critical = 10800;
- description = "The oldest message in email-service-backfill-queue-prod is {{ $values.B }} seconds old. Normal peaks are about 50 minutes; new users wait this long for their mailbox to fill.";
+ description = "The oldest message in email-service-backfill-queue-prod is {{ humanize $values.B.Value }} seconds old. Normal peaks are about 50 minutes; new users wait this long for their mailbox to fill.";
runbook = "queues.md#backlog";
}
{
@@ -311,7 +311,7 @@ in
firesWhen = "above";
warning = 1800;
critical = 7200;
- description = "The oldest message in {{ $labels.QueueName }} is {{ $values.B }} seconds old. These queues normally drain within minutes.";
+ description = "The oldest message in {{ $labels.QueueName }} is {{ humanize $values.B.Value }} seconds old. These queues normally drain within minutes.";
runbook = "queues.md#backlog";
}
];