Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Improve Datadog observability (#1606)

Abel Dantas46e404e00958
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7e24241e1..f25966de8 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -45,6 +45,19 @@ version of their own to them without also giving them a release of their own.
### [Unreleased]
#### Added
+- **A one-screen Datadog board and alerts that say what to do.**
+ `deploy/observability/signal-board.json` shows the four golden signals
+ (latency, traffic, errors, saturation) plus whether agents are connected and
+ messaging apps are up, grey while normal and red only past a limit. The
+ monitors are rewritten to what a user would notice: each says what users see
+ and what to check, posts when it fires, every hour while it stays firing and
+ once on recovery, and error
+ alerts count failures instead of taking a percentage, so low traffic doesn't
+ page on noise. Causes such as slow queries stay on the dashboards.
+ `deploy/observability/apply.sh` creates or updates them in Datadog by name.
+- **switch-core reports which bridge crashed.** `switch.bridges.crashed` is 1
+ per crashed collaboration bridge, tagged with its platform and display name,
+ and 0 once it runs again, so the bridge alert can name it.
- **An agent is told when Switch Trust changed or flagged what it sent.** The
result of `post_message` and `send_targeted_message`, the agent HTTP message
endpoint, the status endpoint, and the media upload endpoint (for a caption
diff --git a/core/switch_core/bridges/collaboration/lifecycle_service.py b/core/switch_core/bridges/collaboration/lifecycle_service.py
index bfb9a9236..e0e61453c 100644
--- a/core/switch_core/bridges/collaboration/lifecycle_service.py
+++ b/core/switch_core/bridges/collaboration/lifecycle_service.py
@@ -324,6 +324,8 @@ class CollaborationBridgeLifecycleService:
# Read off the row at start, so reporting never depends on what the
# CollaborationCore exposes.
self._bridge_facts: dict[str, tuple[str, object]] = {}
+ # Each started bridge's display name, so an alert can say which one died.
+ self._bridge_names: dict[str, str] = {}
# Bridges the deployment's setup step registered rather than a person,
# read off the row at start like `_bridge_facts`. Onboarding telemetry
# leaves these out: a bundled connector coming up is a deployment
@@ -1280,6 +1282,7 @@ class CollaborationBridgeLifecycleService:
# Stashed rather than passed: `_run_bridge`'s signature is what
# the tenant-binding tests patch.
self._bridge_facts[bridge_id] = (bridge.type, bridge.created_at)
+ self._bridge_names[bridge_id] = bridge.display_name
if bridge.preconfigured:
self._preconfigured.add(bridge_id)
else:
@@ -2098,6 +2101,27 @@ class CollaborationBridgeLifecycleService:
counts[name] = counts.get(name, 0) + (1 if alive else 0)
return counts
+ def crashed(self) -> list[tuple[str, str]]:
+ """(platform, display name) of each started bridge whose task has died.
+
+ `running_by_platform` says how many are down on a platform; this says
+ which, so the alert can name the bridge instead of sending someone to
+ the logs.
+ """
+ down = []
+ for bridge_id in sorted(self._started):
+ task = self._tasks.get(bridge_id)
+ if bridge_id in self._bridges and task is not None and not task.done():
+ continue
+ platform, _ = self._bridge_facts.get(bridge_id, ("none", None))
+ down.append(
+ (
+ normalise_platform(platform),
+ self._bridge_names.get(bridge_id, bridge_id),
+ )
+ )
+ return down
+
def bridges_for_tenant(self, tenant_id: str) -> list[CollaborationCore]:
"""Running bridges belonging to `tenant_id`, and none other.
diff --git a/core/switch_core/main.py b/core/switch_core/main.py
index 37f0f4903..cd1ddc609 100644
--- a/core/switch_core/main.py
+++ b/core/switch_core/main.py
@@ -1493,6 +1493,7 @@ async def run(config: SwitchConfig, process_id: str) -> None:
bridges_running=collab_lifecycle.running_count,
bridges_running_by_platform=collab_lifecycle.running_by_platform,
bridges_configured=collab_lifecycle.expected_count,
+ bridges_crashed=collab_lifecycle.crashed,
consumers_running=client_lifecycle.running_count,
agents_connected=connections.live_agents_by_transport,
pool_stats=lambda: pool_stats(engine, pool_watermark),
diff --git a/core/switch_core/observability/bootstrap.py b/core/switch_core/observability/bootstrap.py
index 59c156fb1..903ccdc87 100644
--- a/core/switch_core/observability/bootstrap.py
+++ b/core/switch_core/observability/bootstrap.py
@@ -11,7 +11,7 @@ from __future__ import annotations
import asyncio
import logging
import socket
-from collections.abc import Callable, Iterator, Mapping
+from collections.abc import Callable, Iterator, Mapping, Sequence
from dataclasses import dataclass
import httpx
@@ -21,6 +21,7 @@ from switch_core.config import SwitchConfig
from switch_core.logging_context import LogContextFilter
from switch_core.observability.catalogue import (
AGENTS_CONNECTED,
+ BRIDGES_CRASHED,
BRIDGES_RUNNING,
CONSUMERS_RUNNING,
DB_POOL_IN_USE,
@@ -91,6 +92,8 @@ class RuntimeProbes:
bridges_running: Callable[[], int]
bridges_running_by_platform: Callable[[], Mapping[str, int]]
bridges_configured: Callable[[], int]
+ # (platform, display name) per bridge whose task has died.
+ bridges_crashed: Callable[[], Sequence[tuple[str, str]]]
consumers_running: Callable[[], int]
# Agents per (transport, client); see
# `AgentConnectionRegistry.live_agents_by_transport`.
@@ -137,6 +140,9 @@ def _state_readings(probes: RuntimeProbes) -> Callable[[], Iterator[GaugeReading
# Every (transport, client) pair reported so far: one that empties out is
# reported as 0, or the dashboard would keep showing its last count.
seen: set[tuple[str, str]] = {("websocket", "unknown")}
+ # Every bridge reported down so far: once it runs again it reads 0, so its
+ # alert recovers instead of going quiet.
+ crashed_seen: set[tuple[str, str]] = set()
def readings() -> Iterator[GaugeReading]:
connected = probes.agents_connected()
@@ -154,6 +160,14 @@ def _state_readings(probes: RuntimeProbes) -> Callable[[], Iterator[GaugeReading
float(running),
{"bridge": "collaboration", "platform": platform},
)
+ crashed = set(probes.bridges_crashed())
+ crashed_seen.update(crashed)
+ for platform, name in sorted(crashed_seen):
+ yield GaugeReading(
+ BRIDGES_CRASHED,
+ 1.0 if (platform, name) in crashed else 0.0,
+ {"platform": platform, "bridge_name": name},
+ )
stats = probes.pool_stats()
if stats is not None:
diff --git a/core/switch_core/observability/catalogue.py b/core/switch_core/observability/catalogue.py
index 5032e7180..b332d254a 100644
--- a/core/switch_core/observability/catalogue.py
+++ b/core/switch_core/observability/catalogue.py
@@ -387,6 +387,16 @@ BRIDGES_RUNNING = _spec(
"bridge",
"platform",
)
+BRIDGES_CRASHED = _spec(
+ "switch.bridges.crashed",
+ "gauge",
+ "{bridge}",
+ "1 for each configured collaboration bridge whose task has died, by "
+ "platform and the bridge's display name; 0 once it runs again. The bridges "
+ "health check says that one is down, this says which.",
+ "platform",
+ "bridge_name",
+)
BRIDGE_CALL_DURATION = _spec(
"switch.bridge.call.duration",
"histogram",
@@ -667,6 +677,7 @@ CATALOGUE: dict[str, MetricSpec] = {
BRIDGE_EVENTS_OUT,
BRIDGE_ERRORS,
BRIDGES_RUNNING,
+ BRIDGES_CRASHED,
BRIDGE_CALL_DURATION,
BRIDGE_THROTTLE_HELD,
MESSAGING_EVENTS_IGNORED,
diff --git a/core/tests/switch_core/observability/test_bootstrap.py b/core/tests/switch_core/observability/test_bootstrap.py
index bcaac15e4..3b3170c96 100644
--- a/core/tests/switch_core/observability/test_bootstrap.py
+++ b/core/tests/switch_core/observability/test_bootstrap.py
@@ -76,6 +76,7 @@ def _probes(**overrides) -> RuntimeProbes:
bridges_running=lambda: 2,
bridges_running_by_platform=lambda: {"slack": 1, "mattermost": 1},
bridges_configured=lambda: 2,
+ bridges_crashed=lambda: [],
consumers_running=lambda: 5,
agents_connected=lambda: {
("websocket", "agent-runtime"): 2,
diff --git a/core/tests/switch_core/observability/test_bridge_timing.py b/core/tests/switch_core/observability/test_bridge_timing.py
index 9ec9ff5d1..fab7c1c11 100644
--- a/core/tests/switch_core/observability/test_bridge_timing.py
+++ b/core/tests/switch_core/observability/test_bridge_timing.py
@@ -186,6 +186,41 @@ class TestTheRunningGaugeSaysWhichPlatform:
assert sum(service.running_by_platform().values()) == service.running_count()
+class TestTheCrashedListNamesTheBridge:
+ def _service(self, alive: set[str]) -> CollaborationBridgeLifecycleService:
+ service = object.__new__(CollaborationBridgeLifecycleService)
+ service._started = {"a", "b", "c"} # type: ignore[attr-defined]
+ service._bridges = {b: object() for b in alive} # type: ignore[attr-defined]
+ service._tasks = {b: _DoneTask(b not in alive) for b in "abc"} # type: ignore[attr-defined]
+ service._bridge_facts = { # type: ignore[attr-defined]
+ "a": ("slack", None),
+ "b": ("teams", None),
+ "c": ("discord", None),
+ }
+ service._bridge_names = { # type: ignore[attr-defined]
+ "a": "SandboxAQ Dev",
+ "b": "SAQ Teams",
+ "c": "Design Discord",
+ }
+ return service
+
+ def test_it_lists_each_dead_bridge_by_platform_and_name(self) -> None:
+ assert self._service(alive={"a"}).crashed() == [
+ ("teams", "SAQ Teams"),
+ ("discord", "Design Discord"),
+ ]
+
+ def test_it_is_empty_while_every_bridge_runs(self) -> None:
+ assert self._service(alive={"a", "b", "c"}).crashed() == []
+
+ def test_it_agrees_with_the_running_count(self) -> None:
+ """Two signals from the same fact must not disagree on how many are down."""
+ service = self._service(alive={"b"})
+ assert (
+ len(service.crashed()) == service.expected_count() - service.running_count()
+ )
+
+
class _DoneTask:
def __init__(self, done: bool) -> None:
self._done = done
diff --git a/core/tests/switch_core/observability/test_connection_metrics.py b/core/tests/switch_core/observability/test_connection_metrics.py
index 094caaa21..39f212c01 100644
--- a/core/tests/switch_core/observability/test_connection_metrics.py
+++ b/core/tests/switch_core/observability/test_connection_metrics.py
@@ -8,6 +8,7 @@ metric.
from __future__ import annotations
+import dataclasses
from collections.abc import Iterator
from typing import Any
@@ -162,6 +163,7 @@ def _probes(agents_connected: Any) -> RuntimeProbes:
bridges_running=lambda: 0,
bridges_running_by_platform=lambda: {},
bridges_configured=lambda: 0,
+ bridges_crashed=lambda: [],
consumers_running=lambda: 0,
agents_connected=agents_connected,
pool_stats=lambda: None,
@@ -191,6 +193,27 @@ def test_the_gauge_carries_transport_and_client_but_never_the_version() -> None:
}
+def test_a_crashed_bridge_reads_one_and_zero_once_it_runs_again() -> None:
+ """Named, so the alert says which bridge; and back to 0 rather than gone,
+ so the alert recovers instead of going quiet."""
+ crashed = [("teams", "SAQ Teams")]
+ probes = dataclasses.replace(
+ _probes(lambda: {}), bridges_crashed=lambda: list(crashed)
+ )
+ readings = _state_readings(probes)
+
+ def now() -> dict[tuple, float]:
+ return {
+ tuple(sorted(r.attributes.items())): r.value
+ for r in readings()
+ if r.spec.name == "switch.bridges.crashed"
+ }
+
+ assert now() == {(("bridge_name", "SAQ Teams"), ("platform", "teams")): 1.0}
+ crashed.clear()
+ assert now() == {(("bridge_name", "SAQ Teams"), ("platform", "teams")): 0.0}
+
+
def test_the_gauge_still_reports_zero_with_nobody_connected() -> None:
"""A panel over no series reads "no data", which is not the same as none."""
readings = _connected_readings(_probes(lambda: {}))
diff --git a/core/tests/switch_core/observability/test_datadog_artifacts.py b/core/tests/switch_core/observability/test_datadog_artifacts.py
index 263801ed6..07d8716bf 100644
--- a/core/tests/switch_core/observability/test_datadog_artifacts.py
+++ b/core/tests/switch_core/observability/test_datadog_artifacts.py
@@ -32,10 +32,19 @@ def _load(name: str) -> object:
return json.loads((_ARTIFACTS / name).read_text())
[email protected]("filename", ["dashboard.json", "monitors.json"])
+ "filename", ["dashboard.json", "monitors.json", "signal-board.json"]
+)
def test_every_metric_referenced_exists(filename: str) -> None:
referenced = _referenced_metrics(_load(filename))
- unknown = sorted(referenced - set(CATALOGUE))
+ # The morning report sends its own "posted" mark; that one is not core's.
+ from_report = set(
+ re.findall(
+ r'"metric": "(switch\.[a-z0-9_.]+)"',
+ (_ARTIFACTS / "daily_report.py").read_text(),
+ )
+ )
+ unknown = sorted(referenced - set(CATALOGUE) - from_report)
assert not unknown, (
f"{filename} references {unknown}, which no longer exist in the metric "
"catalogue. A Datadog monitor over an absent series does not fire, and "
@@ -72,3 +81,28 @@ def test_monitors_are_individually_importable() -> None:
assert "@REPLACE-WITH-NOTIFICATION-HANDLE" in monitor["message"], monitor[
"name"
]
+
+
+def _charts(widgets: list[dict]) -> list[dict]:
+ """Every widget definition on a dashboard, groups opened up."""
+ found = []
+ for widget in widgets:
+ definition = widget["definition"]
+ if definition["type"] == "group":
+ found += _charts(definition["widgets"])
+ else:
+ found.append(definition)
+ return found
+
+
+def test_the_signal_board_stays_readable_at_a_glance() -> None:
+ """Working memory holds three to seven things, so a chart that draws more
+ lines than that is read by going back and forth to the legend. The board
+ is for seeing, so it keeps to five."""
+ board = _load("signal-board.json")
+ assert isinstance(board, dict)
+ for chart in _charts(board["widgets"]):
+ series = sum(
+ len(request.get("formulas", [])) for request in chart.get("requests", [])
+ )
+ assert series <= 5, f"{chart.get('title')} draws {series} series"
diff --git a/deploy/observability/README.md b/deploy/observability/README.md
index d974a8060..bf29693ba 100644
--- a/deploy/observability/README.md
+++ b/deploy/observability/README.md
@@ -6,7 +6,7 @@ the panel reading it land in the same review.
Nothing here contains an account, an API key, a team handle or a URL. The
monitors carry `@REPLACE-WITH-NOTIFICATION-HANDLE` where a destination belongs;
-fill it in when importing, not here.
+`apply.sh` fills it in from `NOTIFY` when importing.
**Do not skip that.** Datadog does not validate notification handles: a monitor
carrying the placeholder imports cleanly, shows as healthy in the UI, triggers
@@ -25,23 +25,83 @@ Every metric this server can emit is declared in
`core/switch_core/observability/catalogue.py`, with the attributes each may
carry. That file is the reference; this directory is one view of it.
+## The Signal board
+
+`signal-board.json` is the one screen to open first: the four golden signals
+(latency, traffic, errors, saturation) plus two questions only Switch has, "are
+agents connected?" and "are the messaging apps up?". Everything is grey while it
+is normal. A value turns red, and a dashed red line marks the limit an alert
+watches, so the exception is the only thing in colour. Tiles always show the
+last hour, whatever the time picker says, so a red tile means "now"; charts
+follow the time picker. The other dashboards are where you go next, for causes.
+
+Keep it that way when you edit it: one question per tile, at most five series
+per chart, a limit drawn on any chart an alert reads, and colour only for state.
+
+## Alerts
+
+Every monitor in `monitors.json` is something a user would notice and someone
+should act on now: no data, the database unreachable or refusing connections,
+the pool full for 10 minutes, messages not delivered, failing or late, a
+messaging app bridge down, server errors, a slow API, a stalled server, or
+agents dropping off together. Each says what users see and what to check, posts
+when it fires, again every hour while it stays firing (`renotify_interval: 60`),
+and once more when it recovers. A crashed bridge also gets an alert that names
+it (`switch.bridges.crashed`), and "the morning report didn't post" fires when
+the 09:00 report has been silent for 25 hours, so a quiet channel can be
+trusted.
+
+Priority follows what a user loses: P1 when Switch is down or messages are
+lost, P2 when some people or one platform are affected, P3 when it is slow but
+nothing is lost, P4 when only our own visibility is.
+
+What isn't urgent stays out of them on purpose. Causes (slow queries, CPU,
+garbage collection) belong on the dashboards, and a monitor that fires without
+anyone needing to act teaches people to skim the channel. If one fires and the
+answer is "that's fine", tune it or delete it that week.
+
+Error alerts count failures rather than taking a percentage. At low traffic one
+failure among a handful of requests reads as a high error rate, so a percentage
+would page on noise.
+
+## The morning report
+
+`daily_report.py` posts once a day into a Switch room, as an agent in that room:
+a first line that says "all good" or how many things to look at, one line per
+signal for the last 24 hours, and one graph of the day. The graph comes with a
+short lesson (how to read it, what to look for) and a comment on the day: what
+it did, with numbers, and what that means, down to a likely cause and the
+detail-board chart to open next when something is off. The graph is the one
+behind the first line that needs a look; on a good day, the one that moved most
+against the same day last week; on a quiet day, the next in turn, so the team
+meets every signal over a week or two. It covers what can wait for the morning, the
+things the monitors deliberately don't post: a pool that came close to full,
+latency creeping up, agents still on the old connection, events dropped for
+agents that were away.
+
+```bash
+DD_API_KEY=... DD_APP_KEY=... SWITCH_URL=... SWITCH_AGENT_ID=... \
+ SWITCH_ROOM_ID=... SWITCH_TOKEN=... uv run daily_report.py
+```
+
+Without the `SWITCH_*` settings it only prints the report. The application key
+needs read access to metrics, and to logs for the reason bridges fail. Its verdict, the choice of graph and the comments are tested in
+`test_daily_report.py`.
+
## Importing
+`apply.sh` creates or updates the monitors and the Signal board, matching by
+name and title so a second run updates in place:
+
```bash
-# Dashboard
-curl -X POST "https://api.<your-datadog-site>/api/v1/dashboard" \
- -H "DD-API-KEY: $DD_API_KEY" -H "DD-APPLICATION-KEY: $DD_APP_KEY" \
- -H "Content-Type: application/json" \
- --data @dashboard.json
-
-# Monitors, one at a time — the API takes a single monitor per call
-jq -c '.[]' monitors.json | while read -r monitor; do
- curl -X POST "https://api.<your-datadog-site>/api/v1/monitor" \
- -H "DD-API-KEY: $DD_API_KEY" -H "DD-APPLICATION-KEY: $DD_APP_KEY" \
- -H "Content-Type: application/json" --data "$monitor"
-done
+DD_API_KEY=... DD_APP_KEY=... NOTIFY=@slack-<workspace>-<channel> ./apply.sh
+DRY_RUN=1 ... # print what would change
```
+The application key needs monitor and dashboard write access. `dashboard.json`
+and `multi-pod-dashboard.json` are not touched by it; import those by hand with
+`POST /api/v1/dashboard` if a deployment doesn't have them yet.
+
## Check these before you trust the dashboard
**Histogram panels need two things, not one.** Request latency, delivery lag,
@@ -93,9 +153,11 @@ quiet night and a heartbeat that can legitimately be absent is not a heartbeat.
## What is deliberately not alerted on
-**Latency.** Switch holds long-poll connections open on purpose, so a slow
-request is its normal mode and a duration threshold would page on healthy
-traffic. The dashboard shows it; nothing wakes anyone for it.
+**Latency, below a level users feel.** The two latency monitors fire only when
+it is plainly bad (API p95 over 2 s, delivery p95 over 10 s, sustained). On
+pilot in early November the API p95 stayed under 1 s even while the database
+pool was full, so 2 s is a symptom, not noise. Anything slower than usual but
+not yet that bad is for the dashboard.
**Bridge event volume.** It follows whatever the humans in the channels are
doing. A quiet Sunday is not an incident, and an alert that fires every
diff --git a/deploy/observability/apply.sh b/deploy/observability/apply.sh
new file mode 100755
index 000000000..a9cb53fd4
--- /dev/null
+++ b/deploy/observability/apply.sh
@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+# Create or update this directory's monitors and dashboards in Datadog.
+#
+# DD_API_KEY=... DD_APP_KEY=... NOTIFY=@slack-<workspace>-<channel> ./apply.sh
+#
+# Optional: DD_SITE (default datadoghq.com), DRY_RUN=1 to print what would change.
+#
+# Monitors are matched by name and dashboards by title, so running it again
+# updates them in place instead of making copies. NOTIFY replaces the
+# notification placeholder in every monitor; the script refuses to run without
+# it, because a monitor left with the placeholder imports cleanly and notifies
+# nobody.
+set -euo pipefail
+cd "$(dirname "$0")"
+
+: "${DD_API_KEY:?set DD_API_KEY}"
+: "${DD_APP_KEY:?set DD_APP_KEY}"
+: "${NOTIFY:?set NOTIFY to the notification handle, e.g. @slack-workspace-channel}"
+case "$NOTIFY" in @*) ;; *) echo "NOTIFY must start with @" >&2; exit 1 ;; esac
+API="https://api.${DD_SITE:-datadoghq.com}/api/v1"
+
+dd() {
+ local method=$1 path=$2 body=${3:-}
+ if [[ -n "${DRY_RUN:-}" && "$method" != GET ]]; then
+ echo "would $method $path" >&2
+ echo '{}'
+ return
+ fi
+ curl -sf -X "$method" "$API$path" \
+ -H "DD-API-KEY: $DD_API_KEY" -H "DD-APPLICATION-KEY: $DD_APP_KEY" \
+ -H "Content-Type: application/json" ${body:+--data "$body"}
+}
+
+existing_monitors=$(dd GET "/monitor?monitor_tags=service:switch-core")
+jq -c '.[]' monitors.json | while read -r monitor; do
+ monitor=${monitor//@REPLACE-WITH-NOTIFICATION-HANDLE/$NOTIFY}
+ name=$(jq -r .name <<<"$monitor")
+ id=$(jq -r --arg n "$name" '[.[] | select(.name == $n) | .id][0] // empty' <<<"$existing_monitors")
+ if [[ -n "$id" ]]; then
+ dd PUT "/monitor/$id" "$monitor" >/dev/null
+ echo "updated monitor $id: $name"
+ else
+ dd POST "/monitor" "$monitor" >/dev/null
+ echo "created monitor: $name"
+ fi
+done
+
+existing_dashboards=$(dd GET "/dashboard")
+# Only the board this script was written for; the older dashboards were
+# imported by hand and may have drifted from their files.
+file=signal-board.json
+title=$(jq -r .title "$file")
+id=$(jq -r --arg t "$title" '[.dashboards[] | select(.title == $t) | .id][0] // empty' <<<"$existing_dashboards")
+if [[ -n "$id" ]]; then
+ dd PUT "/dashboard/$id" "@$file" >/dev/null
+ echo "updated dashboard $id: $title"
+else
+ dd POST "/dashboard" "@$file" >/dev/null
+ echo "created dashboard: $title"
+fi
diff --git a/deploy/observability/daily_report.py b/deploy/observability/daily_report.py
new file mode 100644
index 000000000..44dbe500d
--- /dev/null
+++ b/deploy/observability/daily_report.py
@@ -0,0 +1,821 @@
+# /// script
+# requires-python = ">=3.12"
+# dependencies = ["httpx"]
+# ///
+"""The morning report: one post that says whether switch-core was fine over the
+last day, with the graphs that show it.
+
+ uv run deploy/observability/daily_report.py
+
+It reads the same metrics as the Signal board and the monitors, from Datadog,
+and posts into a Switch room as an agent in that room. The first line is the
+verdict, so on a good day there is nothing else to read.
+
+Each report also carries one graph of the day, with two sentences on what it
+measures and why it matters, so the team learns the signals one at a time.
+The graph is the one behind the first line that needs a look; on a good day,
+the one that moved most against the same day last week; on a quiet day, the
+next one in turn.
+
+The alerts cover what needs someone now; this covers what can wait for the
+morning: a pool that came close to full, latency creeping up, agents still on
+the old connection, events dropped for agents that were away.
+
+Environment:
+ DD_API_KEY, DD_APP_KEY read access to the metrics and graph snapshots
+ DD_SITE default datadoghq.com
+ SERVICE the switch-core service to report on, default switch-core
+ SWITCH_URL, SWITCH_AGENT_ID, SWITCH_ROOM_ID, SWITCH_TOKEN
+ the room and the agent posting into it; without
+ them the report is only printed
+ DASHBOARD_URL optional, the Signal board's link for the last line
+
+After a post it sends one point on switch.daily_report.posted, which the
+"morning report didn't post" alert watches.
+"""
+
+from __future__ import annotations
+
+import os
+import sys
+import time
+from dataclasses import dataclass, field
+
+import httpx
+
+DAY = 24 * 3600
+
+GOOD, LOOK, BAD = "🟢", "🟡", "🔴"
+
+
+@dataclass(frozen=True)
+class Line:
+ state: str
+ text: str
+ graph: str = field(default="", compare=False)
+
+
+def check(value: float | None, *, look: float, bad: float) -> str:
+ """A value at or over `bad` is red, at or over `look` yellow, otherwise
+ green. No value means nothing was reported, which is itself worth a look."""
+ if value is None:
+ return LOOK
+ if value >= bad:
+ return BAD
+ if value >= look:
+ return LOOK
+ return GOOD
+
+
+def _ms(value: float | None) -> str:
+ if value is None:
+ return "no data"
+ return f"{value / 1000:.1f} s" if value >= 1000 else f"{value:.0f} ms"
+
+
+def _n(value: float | None) -> str:
+ return "no data" if value is None else f"{value:.0f}"
+
+
+def lines_for(m: dict[str, float | None]) -> list[Line]:
+ """The report's lines from the day's numbers, in the Signal board's order.
+ Thresholds sit below the monitors', which already posted anything worse."""
+ pool = m["pool_ratio"]
+ lines = [
+ Line(
+ BAD if not m["agents_now"] else GOOD,
+ f"Agent connections now: {_n(m['agents_now'])}",
+ graph="agents",
+ ),
+ Line(
+ GOOD if m["bridges_min"] == 1 else BAD,
+ "Messaging apps: all bridges running"
+ if m["bridges_min"] == 1
+ else "Messaging apps: a bridge was down at some point",
+ graph="bridges",
+ ),
+ Line(
+ check(m["api_p95_max"], look=1000, bad=2000),
+ f"API: worst p95 {_ms(m['api_p95_max'])}",
+ graph="api",
+ ),
+ Line(
+ check(m["delivery_p95_max"], look=3000, bad=10000),
+ f"Messages: worst delivery p95 {_ms(m['delivery_p95_max'])}",
+ graph="delivery",
+ ),
+ Line(
+ LOOK
+ if (m["server_errors"] or 0) >= 20
+ or m["send_failures"]
+ or m["delivery_failures"]
+ else GOOD,
+ f"Errors: {_n(m['server_errors'])} server errors, "
+ f"{_n(m['send_failures'])} failed sends, "
+ f"{_n(m['delivery_failures'])} failed deliveries",
+ graph="errors",
+ ),
+ Line(
+ check(pool, look=0.9, bad=1.0),
+ "Database: peak pool use " + ("no data" if pool is None else f"{pool:.0%}"),
+ graph="pool",
+ ),
+ Line(
+ check(m["loop_lag_max"], look=500, bad=1000),
+ f"Server: worst event-loop lag {_ms(m['loop_lag_max'])}",
+ graph="stalls",
+ ),
+ ]
+ if m["old_connection"]:
+ lines.append(
+ Line(LOOK, f"Agents on the old connection: {_n(m['old_connection'])}")
+ )
+ if m["events_dropped"]:
+ lines.append(
+ Line(
+ GOOD,
+ f"Events dropped for agents that were away: {_n(m['events_dropped'])}",
+ )
+ )
+ return lines
+
+
+@dataclass(frozen=True)
+class Graph:
+ """One graph the report can feature, and how to teach it to the team."""
+
+ key: str
+ title: str
+ queries: tuple[str, ...]
+ # The alert's limit, drawn as a dashed red line; None when there is none.
+ limit: float | None
+ how_to_read: str
+ what_matters: str
+ # Grouped by pod or app where it helps, so the comment can say which one.
+ detail: str
+ unit: str = "ms"
+
+
+@dataclass(frozen=True)
+class Day:
+ """What one graph did over the report's 24 hours."""
+
+ peak: float | None = None
+ peak_where: str = ""
+ peak_at: str = ""
+ mean: float | None = None
+ last_week: float | None = None
+ minutes_over: int = 0
+ # Extra findings some graphs need: the top error route, why bridges fail.
+ notes: dict[str, str] = field(default_factory=dict)
+
+ @property
+ def change(self) -> float | None:
+ if self.mean is None or not self.last_week:
+ return None
+ return (self.mean - self.last_week) / self.last_week
+
+
+def featured_graphs(service: str) -> list[Graph]:
+ s = f"service:{service}"
+ streams = "!route:/gateway/chats/events,!route:/gateway/chats/activity/events"
+ pool = (
+ f"100 * max:switch.db.pool.in_use{{{s}}} by {{pod_name}}"
+ f" / max:switch.db.pool.size{{{s}}} by {{pod_name}}"
+ )
+ return [
+ Graph(
+ "delivery",
+ "Time for a message to reach everyone in the room (p95, ms)",
+ (f"p95:switch.messages.delivery_lag{{{s}}}",),
+ 10_000,
+ "Each point is p95: line up every message from that minute by how "
+ "long it took to reach the people and agents in the room, and p95 "
+ "is the one 95 in from the fast end. So only the slowest 5 in 100 "
+ "took longer than the line.",
+ "Stretches above the dashed red line at 10 s. A single spike is one "
+ "bad minute; the alert needs 15 minutes above it.",
+ f"p95:switch.messages.delivery_lag{{{s}}}",
+ ),
+ Graph(
+ "api",
+ "API response time (p95, ms)",
+ (f"p95:switch.http.request.duration{{{s},{streams}}}",),
+ 2_000,
+ "Each point is how long the slowest 5 in 100 requests to the server "
+ "took that minute. The two live streams are left out: they stay "
+ "open for about 10 s on purpose.",
+ "Time above the dashed red line at 2 s. Everything the Console and "
+ "the agents do goes through the API, so this is how fast Switch "
+ "feels to them.",
+ f"p95:switch.http.request.duration{{{s},{streams}}}",
+ ),
+ Graph(
+ "agents",
+ "Agent connections, all pods together",
+ (f"sum:switch.agents.connected{{{s}}}",),
+ None,
+ "How many agent connections the server holds. One agent can hold "
+ "several, one per room it serves, so it counts connections, not "
+ "agents.",
+ "Cliffs: a sudden drop that stays down means agents were cut off. "
+ "Short spikes are agents reconnecting after a deploy.",
+ f"sum:switch.agents.connected{{{s}}} by {{pod_name}}",
+ unit="",
+ ),
+ Graph(
+ "pool",
+ "Database pool in use, per pod (% of its size)",
+ (pool,),
+ 100,
+ "Each pod keeps a fixed set of database connections, its pool. One "
+ "line per pod shows how many were busy, as a share of the set.",
+ "The dashed red line is 100%: every connection busy. Above it, "
+ "requests queue for a connection and everything slows down at "
+ "once. That's what happened from 30 Oct to 2 Nov.",
+ pool,
+ unit="%",
+ ),
+ Graph(
+ "stalls",
+ "Server stalls: longest time each pod was too busy (ms)",
+ (f"max:switch.runtime.event_loop_lag{{{s}}} by {{pod_name}}",),
+ 1_000,
+ "switch-core does one thing at a time on each pod. This is the "
+ "longest each pod made new work wait, minute by minute.",
+ "Spikes towards the dashed red line at 1 s. During a stall every "
+ "message on that pod waits, so stalls show up as late messages.",
+ f"max:switch.runtime.event_loop_lag{{{s}}} by {{pod_name}}",
+ ),
+ Graph(
+ "errors",
+ "Server errors and lost messages, per hour",
+ (
+ f"sum:switch.http.requests{{{s},status_class:5xx}}.as_count()"
+ ".rollup(sum, 3600)",
+ f"sum:switch.messages.send_failures{{{s}}}.as_count()"
+ ".rollup(sum, 3600)",
+ ),
+ None,
+ "Bars per hour: requests the server failed to answer (5xx), and "
+ "messages it failed to save.",
+ "Clusters of bars, and any lost message at all. A few scattered "
+ "errors a day are background; a lost message never is.",
+ f"sum:switch.http.requests{{{s},status_class:5xx}} by {{route}}.as_count()",
+ unit="",
+ ),
+ Graph(
+ "traffic",
+ "Requests per second, against the same time last week",
+ (f"sum:switch.http.requests{{{s}}}.as_rate()",),
+ None,
+ "The solid line is today, the dashed one the same hours a week "
+ "earlier, so the daily rhythm cancels out.",
+ "A gap opening between the two lines. Traffic isn't good or bad by "
+ "itself, but it explains the rest: errors with a traffic spike are "
+ "load, without one they're a bug.",
+ f"sum:switch.http.requests{{{s}}}.as_rate()",
+ unit=" requests/s",
+ ),
+ Graph(
+ "bridges",
+ "Messaging-app bridges running, by app",
+ (f"sum:switch.bridges.running{{{s}}} by {{platform}}",),
+ None,
+ "One line per messaging app: how many bridges to it are running, "
+ "over both pods.",
+ "A line falling to zero. When a bridge stops, the people in that "
+ "app's rooms stop hearing from the agents.",
+ f"sum:switch.bridges.running{{{s}}} by {{platform}}",
+ unit="",
+ ),
+ ]
+
+
+def pick_graph(
+ lines: list[Line],
+ graphs: list[Graph],
+ change: dict[str, float | None],
+ day_number: int,
+) -> tuple[Graph, str]:
+ """The graph of the day, and why it was picked.
+
+ The graph behind the first line that needs a look wins, since that's what
+ the reader will ask about. Otherwise the one that moved most against the
+ same day last week, if it moved by a quarter or more. Otherwise the next
+ one in turn, so a quiet week still walks the team through every signal.
+ """
+ by_key = {graph.key: graph for graph in graphs}
+ for line in lines:
+ if line.state != GOOD and line.graph in by_key:
+ return by_key[line.graph], "it's behind a line above that needs a look"
+ moved = [(abs(c), k) for k, c in change.items() if c is not None and k in by_key]
+ if moved:
+ size, key = max(moved)
+ if size >= 0.25:
+ direction = "up" if (change[key] or 0) > 0 else "down"
+ return by_key[key], (
+ f"it moved most against the same day last week ({direction} {size:.0%})"
+ )
+ return graphs[day_number % len(graphs)], (
+ "nothing stood out today, so it's this one's turn"
+ )
+
+
+def _v(value: float | None, unit: str) -> str:
+ if value is None:
+ return "no data"
+ if unit == "ms":
+ return _ms(value)
+ if unit == "%":
+ return f"{value:.0f}%"
+ return f"{value:,.0f}{unit}" if value >= 10 else f"{value:.1f}{unit}"
+
+
+def _limit(value: float, unit: str) -> str:
+ return f"{value / 1000:g} s" if unit == "ms" and value >= 1000 else _v(value, unit)
+
+
+def _for(minutes: int) -> str:
+ if minutes >= 23 * 60:
+ return "the whole day"
+ if minutes >= 90:
+ return f"about {minutes / 60:.0f} h"
+ return f"about {minutes} min"
+
+
+def _where(day: Day) -> str:
+ where = f" on {day.peak_where}" if day.peak_where else ""
+ at = f" around {day.peak_at} UTC" if day.peak_at else ""
+ return where + at
+
+
+def commentary(graph: Graph, day: Day) -> tuple[str, str]:
+ """What the graph says about the last 24 hours, and what to do about it.
+
+ The first part reads the numbers back in plain words; the second is a
+ diagnosis and a next step when something is off, or says there's nothing
+ to do. The advice points at the detail board's charts by name.
+ """
+ peak = _v(day.peak, graph.unit)
+ if day.peak is None:
+ return (
+ "No data in the last 24 hours.",
+ "No data usually means switch-core stopped reporting, not that all "
+ "is well: check that the pods are up and that the 'has stopped "
+ "reporting' alert hasn't fired.",
+ )
+ over = f", above the line for {_for(day.minutes_over)}" if day.minutes_over else ""
+
+ if graph.key in ("delivery", "api", "stalls"):
+ limit = graph.limit or 0
+ today = f"The worst was {peak}{_where(day)}{over}."
+ if day.minutes_over:
+ advice = {
+ "delivery": "Late messages usually come from a stalled pod or a "
+ "full database pool. On the detail board, check 'Server stalls' "
+ "and 'Database strain' for the same time. If neither moved, "
+ "check how long Slack and Discord took to answer: the delay may "
+ "be on their side.",
+ "api": "The detail board's 'Five slowest API routes' names the "
+ "route. If every route slowed together, the cause is shared: "
+ "look at 'Database query time' and 'Server stalls' for the same "
+ "time.",
+ "stalls": "A stall means one piece of work held the pod. Check "
+ "that pod's CPU on the detail board; if it was near 100%, the "
+ "pod ran out of room and another pod would help. If CPU was "
+ "low, the pod was waiting on something slow, usually the "
+ "database.",
+ }[graph.key]
+ elif day.peak >= limit / 2:
+ advice = (
+ f"Nothing fired, but {peak} is over half the {_limit(limit, graph.unit)} "
+ "limit. Not a problem yet; worth a glance if it happens again "
+ "tomorrow."
+ )
+ else:
+ advice = f"Nothing to do: well under the {_limit(limit, graph.unit)} line."
+ return today, advice
+
+ if graph.key == "pool":
+ today = f"Peak use was {peak}{_where(day)}{over}."
+ if day.minutes_over:
+ advice = (
+ "Requests queued for a database connection, so everything on "
+ "that pod slowed down at once. On the detail board, 'Database "
+ "query time' shows whether a slow query held connections, and "
+ "'Database strain' how long requests waited. If both pods "
+ "filled up, it's load: add a pod or raise the pool size."
+ )
+ elif day.peak >= 70:
+ advice = (
+ "Getting close to full. If it stays this high, it's time to "
+ "plan another pod before it tips over."
+ )
+ else:
+ advice = "Nothing to do: plenty of room left."
+ return today, advice
+
+ if graph.key in ("agents", "traffic"):
+ change = day.change
+ compared = (
+ f", against about {_v(day.last_week, graph.unit)} on the same day last week "
+ f"({'up' if change > 0 else 'down'} {abs(change):.0%})"
+ if change is not None
+ else ""
+ )
+ today = f"On average {_v(day.mean, graph.unit)}{compared}."
+ if graph.key == "agents" and change is not None and change <= -0.3:
+ advice = (
+ "Fewer agent connections than last week. If 'Agents that went "
+ "silent' on the detail board spiked at the same time, they were "
+ "cut off (network or a stalled pod); if not, people are simply "
+ "running fewer agents."
+ )
+ elif graph.key == "traffic" and change is not None and abs(change) >= 0.5:
+ advice = (
+ "A big change in traffic. Fine on its own; if errors or latency "
+ "moved too, the two are likely connected."
+ )
+ else:
+ advice = "Nothing to do: in line with last week."
+ return today, advice
+
+ if graph.key == "errors":
+ total = day.notes.get("server_errors", _v(day.mean, ""))
+ lost = day.notes.get("lost", "0")
+ today = f"{total} server errors and {lost} lost messages in 24 hours."
+ if lost != "0":
+ advice = (
+ "Messages were lost: someone sent something nobody received. "
+ "That's worth a look today. The switch-core logs for the same "
+ "time say which room and why."
+ )
+ elif float(total) >= 20 and day.notes.get("top_route"):
+ advice = (
+ f"More than the usual background. Most came from "
+ f"{day.notes['top_route']}: start there; 'Routes returning "
+ "server errors' on the detail board has the rest."
+ )
+ else:
+ top = day.notes.get("top_route")
+ advice = "Nothing to do: a handful a day is background" + (
+ f" (the most came from {top})." if top else "."
+ )
+ return today, advice
+
+ # bridges
+ if day.notes.get("check") == "failing":
+ today = "The bridges health check failed for most of the day."
+ reason = day.notes.get("top_failure")
+ advice = (f"Most failed connections were {reason}. " if reason else "") + (
+ "A 'network' failure means the app's server didn't answer; if that "
+ "bridge isn't used any more, deactivating it clears the check. "
+ "'Why connecting to a messaging app failed' on the detail board "
+ "has the full list."
+ )
+ else:
+ today = (
+ f"{peak} bridges running at the busiest point, and the health check passed."
+ )
+ advice = "Nothing to do."
+ return today, advice
+
+
+def graph_caption(graph: Graph, reason: str, day: Day) -> str:
+ today, advice = commentary(graph, day)
+ return "\n".join(
+ [
+ f"📊 Graph of the day: {graph.title}",
+ f"Picked because {reason}.",
+ "",
+ f"How to read it: {graph.how_to_read}",
+ f"What to look for: {graph.what_matters}",
+ "",
+ f"Today: {today}",
+ f"What it means: {advice}",
+ ]
+ )
+
+
+def report_text(lines: list[Line], service: str, dashboard_url: str = "") -> str:
+ to_look = [line for line in lines if line.state != GOOD]
+ if not to_look:
+ head = f"{service}, last 24 h: all good."
+ elif any(line.state == BAD for line in to_look):
+ head = f"{service}, last 24 h: {len(to_look)} to look at, some of it bad."
+ else:
+ head = f"{service}, last 24 h: {len(to_look)} to look at."
+ body = [head, *(f"{line.state} {line.text}" for line in lines)]
+ if dashboard_url:
+ body.append(dashboard_url)
+ return "\n".join(body)
+
+
+class Datadog:
+ def __init__(self, api_key: str, app_key: str, site: str) -> None:
+ self.http = httpx.Client(
+ base_url=f"https://api.{site}/api/v1",
+ headers={"DD-API-KEY": api_key, "DD-APPLICATION-KEY": app_key},
+ timeout=60,
+ )
+
+ def mark_posted(self, service: str, at: int) -> None:
+ """One point on `switch.daily_report.posted`, so the "the morning
+ report didn't post" alert can tell a quiet day from a broken job."""
+ response = self.http.post(
+ "/series",
+ json={
+ "series": [
+ {
+ "metric": "switch.daily_report.posted",
+ "type": "gauge",
+ "points": [[at, 1]],
+ "tags": [f"service:{service}"],
+ }
+ ]
+ },
+ )
+ response.raise_for_status()
+
+ def points(self, query: str, start: int, end: int) -> list[float]:
+ response = self.http.get(
+ "/query", params={"from": start, "to": end, "query": query}
+ )
+ response.raise_for_status()
+ return [
+ value
+ for series in response.json().get("series", [])
+ for _, value in series.get("pointlist", [])
+ if value is not None
+ ]
+
+ def series(
+ self, query: str, start: int, end: int
+ ) -> dict[str, list[tuple[float, float]]]:
+ """Points per series, keyed by a short name for what the series is
+ split by (a pod, an app, a route); "" when it isn't split."""
+ response = self.http.get(
+ "/query", params={"from": start, "to": end, "query": query}
+ )
+ response.raise_for_status()
+ out: dict[str, list[tuple[float, float]]] = {}
+ for series in response.json().get("series", []):
+ name = series_name(series.get("scope", ""))
+ out[name] = [
+ (ts / 1000, value)
+ for ts, value in series.get("pointlist", [])
+ if value is not None
+ ]
+ return out
+
+ def top_log_group(
+ self, search: str, facets: tuple[str, ...], start: int, end: int
+ ) -> tuple[str, int] | None:
+ """The most common combination of facets among matching logs, and how
+ many there were. None when there are none or logs can't be read."""
+ body = {
+ "compute": [{"aggregation": "count"}],
+ "filter": {
+ "query": search,
+ "from": str(start * 1000),
+ "to": str(end * 1000),
+ },
+ "group_by": [
+ {
+ "facet": f,
+ "limit": 1,
+ "sort": {"aggregation": "count", "order": "desc"},
+ }
+ for f in facets
+ ],
+ }
+ response = self.http.post(
+ str(self.http.base_url).replace("/v1", "/v2") + "/logs/analytics/aggregate",
+ json=body,
+ )
+ response.raise_for_status()
+ buckets = response.json().get("data", {}).get("buckets", [])
+ if not buckets:
+ return None
+ top = buckets[0]
+ return ", ".join(str(v) for v in top.get("by", {}).values()), int(
+ top.get("computes", {}).get("c0", 0)
+ )
+
+ def snapshot(self, graph_def: str, title: str, start: int, end: int) -> bytes:
+ """A PNG of one graph. Datadog renders it after answering, so the
+ image is fetched until it is ready."""
+ response = self.http.get(
+ "/graph/snapshot",
+ params={"graph_def": graph_def, "title": title, "start": start, "end": end},
+ )
+ response.raise_for_status()
+ url = response.json()["snapshot_url"]
+ for _ in range(20):
+ image = httpx.get(url, timeout=30)
+ if image.status_code == 200 and image.headers.get(
+ "content-type", ""
+ ).startswith("image/"):
+ return image.content
+ time.sleep(3)
+ raise TimeoutError(f"snapshot {title!r} never rendered")
+
+
+def measure(dd: Datadog, service: str, start: int, end: int) -> dict[str, float | None]:
+ s = f"service:{service}"
+
+ def agg(query: str, how) -> float | None:
+ values = dd.points(query, start, end)
+ return how(values) if values else None
+
+ last = lambda xs: xs[-1] # noqa: E731
+ in_use = dd.points(f"max:switch.db.pool.in_use{{{s}}}", start, end)
+ size = dd.points(f"max:switch.db.pool.size{{{s}}}", start, end)
+ return {
+ "agents_now": agg(f"sum:switch.agents.connected{{{s}}}", last),
+ "bridges_min": agg(f"min:switch.health.check{{{s},check:bridges}}", min),
+ "api_p95_max": agg(
+ f"p95:switch.http.request.duration{{{s},"
+ "!route:/gateway/chats/events,!route:/gateway/chats/activity/events}",
+ max,
+ ),
+ "delivery_p95_max": agg(f"p95:switch.messages.delivery_lag{{{s}}}", max),
+ "server_errors": agg(
+ f"sum:switch.http.requests{{{s},status_class:5xx}}.as_count()", sum
+ )
+ or 0,
+ "send_failures": agg(
+ f"sum:switch.messages.send_failures{{{s}}}.as_count()", sum
+ )
+ or 0,
+ "delivery_failures": agg(
+ f"sum:switch.messages.delivery_failures{{{s}}}.as_count()", sum
+ )
+ or 0,
+ "pool_ratio": max(in_use) / max(size) if in_use and size else None,
+ "loop_lag_max": agg(f"max:switch.runtime.event_loop_lag{{{s}}}", max),
+ "old_connection": agg(f"sum:switch.agents.connected{{{s},transport:sse}}", max)
+ or 0,
+ "events_dropped": agg(f"sum:switch.agent.events_dropped{{{s}}}.as_count()", sum)
+ or 0,
+ }
+
+
+def graph_def(graph: Graph) -> str:
+ """A Datadog graph definition: grey lines, the alert's limit as a dashed
+ red marker, and last week as a second line for traffic."""
+ requests = [
+ f'{{"q":"{q}","type":"line","style":{{"palette":"grey"}}}}'
+ for q in graph.queries
+ ]
+ if graph.key == "traffic":
+ requests.append(
+ f'{{"q":"week_before({graph.queries[0]})","type":"line",'
+ '"style":{"palette":"grey","type":"dashed"}}'
+ )
+ markers = (
+ f',"markers":[{{"value":"y = {graph.limit:g}","type":"error dashed"}}]'
+ if graph.limit is not None
+ else ""
+ )
+ return f'{{"viz":"timeseries","requests":[{",".join(requests)}]{markers}}}'
+
+
+def series_name(scope: str) -> str:
+ """'pod_name:switch-switch-core-58f65f47cf-zjttv,service:switch-core'
+ becomes 'pod zjttv'; other splits keep their value."""
+ for tag in scope.split(","):
+ key, _, value = tag.partition(":")
+ if key == "pod_name":
+ return f"pod {value.rsplit('-', 1)[-1]}"
+ if key in ("platform", "route", "check"):
+ return value
+ return ""
+
+
+def mean_of(dd: Datadog, query: str, start: int, end: int) -> float | None:
+ values = dd.points(query, start, end)
+ return sum(values) / len(values) if values else None
+
+
+def read_day(dd: Datadog, graph: Graph, service: str, start: int, end: int) -> Day:
+ """What the graph did over the day: its peak and where, time over the
+ limit, and its mean against the same 24 hours a week earlier."""
+ split = dd.series(graph.detail, start, end)
+ points = [(v, t, name) for name, pts in split.items() for t, v in pts]
+ if not points:
+ return Day()
+ peak, at, where = max(points)
+ minutes_over = 0
+ if graph.limit is not None:
+ for pts in split.values():
+ over = sum(b[0] - a[0] for a, b in zip(pts, pts[1:]) if a[1] > graph.limit)
+ minutes_over = max(minutes_over, round(over / 60))
+ week = 7 * DAY
+ notes: dict[str, str] = {}
+ s = f"service:{service}"
+ if graph.key == "errors":
+ totals = {name: sum(v for _, v in pts) for name, pts in split.items()}
+ notes["server_errors"] = f"{sum(totals.values()):.0f}"
+ if totals:
+ route, count = max(totals.items(), key=lambda kv: kv[1])
+ if count:
+ notes["top_route"] = f"{route} ({count:.0f})"
+ lost = sum(
+ sum(dd.points(f"sum:switch.messages.{m}{{{s}}}.as_count()", start, end))
+ for m in ("send_failures", "delivery_failures")
+ )
+ notes["lost"] = f"{lost:.0f}"
+ if graph.key == "bridges":
+ checks = dd.points(f"min:switch.health.check{{{s},check:bridges}}", start, end)
+ if checks and sum(1 for v in checks if v < 1) > len(checks) / 2:
+ notes["check"] = "failing"
+ try:
+ top = dd.top_log_group(
+ f"{s} @event.name:switch_core.bridge_connected @outcome:failure",
+ ("@bridge_platform", "@failure_reason"),
+ start,
+ end,
+ )
+ except httpx.HTTPError:
+ top = None
+ if top:
+ notes["top_failure"] = f"{top[0]} ({top[1]} times)"
+ return Day(
+ peak=peak,
+ peak_where=where if len(split) > 1 else "",
+ peak_at=time.strftime("%H:%M", time.gmtime(at)),
+ mean=mean_of(dd, graph.queries[0], start, end),
+ last_week=mean_of(dd, graph.queries[0], start - week, end - week),
+ minutes_over=minutes_over,
+ notes=notes,
+ )
+
+
+def post(text: str, images: list[tuple[str, bytes]]) -> bool:
+ url = os.environ.get("SWITCH_URL", "")
+ agent = os.environ.get("SWITCH_AGENT_ID", "")
+ room = os.environ.get("SWITCH_ROOM_ID", "")
+ token = os.environ.get("SWITCH_TOKEN", "")
+ if not (url and agent and room and token):
+ print("no Switch room set; report only printed", flush=True)
+ return False
+ headers = {"Authorization": f"Bearer {token}"}
+ base = f"{url.rstrip('/')}/agents/{agent}"
+ if images:
+ response = httpx.post(
+ f"{base}/rooms/{room}/media",
+ headers=headers,
+ data={"caption": text},
+ files=[("files", (name, png, "image/png")) for name, png in images],
+ timeout=120,
+ )
+ else:
+ response = httpx.post(
+ f"{base}/message",
+ headers=headers,
+ json={"room_id": room, "content": text},
+ timeout=30,
+ )
+ response.raise_for_status()
+ return True
+
+
+def main() -> int:
+ service = os.environ.get("SERVICE", "switch-core")
+ dd = Datadog(
+ os.environ["DD_API_KEY"],
+ os.environ["DD_APP_KEY"],
+ os.environ.get("DD_SITE", "datadoghq.com"),
+ )
+ end = int(time.time())
+ start = end - DAY
+ lines = lines_for(measure(dd, service, start, end))
+ text = report_text(lines, service, os.environ.get("DASHBOARD_URL", ""))
+ graphs = featured_graphs(service)
+ week = 7 * DAY
+ change: dict[str, float | None] = {}
+ for g in graphs:
+ now = mean_of(dd, g.queries[0], start, end)
+ before = mean_of(dd, g.queries[0], start - week, end - week)
+ change[g.key] = (now - before) / before if now is not None and before else None
+ graph, reason = pick_graph(lines, graphs, change, end // DAY)
+ day = read_day(dd, graph, service, start, end)
+ text = f"{text}\n\n{graph_caption(graph, reason, day)}"
+ print(text, flush=True)
+ images = []
+ try:
+ images.append(
+ ("graph.png", dd.snapshot(graph_def(graph), graph.title, start, end))
+ )
+ except (httpx.HTTPError, TimeoutError) as error:
+ # The verdict matters more than the picture: post without it.
+ print(f"graph {graph.title!r} skipped: {error}", file=sys.stderr, flush=True)
+ if post(text, images):
+ dd.mark_posted(service, int(time.time()))
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/deploy/observability/dashboard.json b/deploy/observability/dashboard.json
index f892d48e9..7ce526ece 100644
--- a/deploy/observability/dashboard.json
+++ b/deploy/observability/dashboard.json
@@ -745,6 +745,24 @@
"height": 4
}
},
+ {
+ "definition": {
+ "type": "toplist",
+ "title": "Crashed bridges, by name (empty while all run)",
+ "requests": [
+ {
+ "q": "max:switch.bridges.crashed{$env,$service} by {platform,bridge_name}",
+ "aggregator": "last"
+ }
+ ]
+ },
+ "layout": {
+ "x": 0,
+ "y": 101,
+ "width": 6,
+ "height": 4
+ }
+ },
{
"definition": {
"type": "timeseries",
diff --git a/deploy/observability/monitors.json b/deploy/observability/monitors.json
index 779c56c0a..b26810366 100644
--- a/deploy/observability/monitors.json
+++ b/deploy/observability/monitors.json
@@ -1,282 +1,334 @@
[
{
"name": "[Switch] switch-core has stopped reporting",
- "type": "metric alert",
+ "type": "query alert",
"query": "max(last_15m):avg:switch.runtime.event_loop_lag{service:switch-core} < 0",
- "message": "No metrics have arrived from switch-core for 15 minutes.\n\nThis is a no-data alert, and it cannot tell you which of three things happened: the server is down, it cannot reach the collector, or someone turned reporting off. Check the pod first \u2014 if it is serving, the other two are the ones left.\n\nNote that every other monitor in this set goes quiet at the same time, so treat this one as the reason the rest went silent rather than as an outage on its own.\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",
+ "message": "{{#is_alert}}No metrics have arrived from switch-core for 15 minutes. Either the server is down, it can't reach the collector, or reporting was turned off.\n\n**Check:** whether the pods are running and serving. If they are, the collector or the telemetry relay is the problem.\n\nEvery other alert goes quiet at the same time, so this one explains their silence.\n\nSignal board: open \"Switch signal\" in Datadog's dashboard list.{{/is_alert}}{{#is_recovery}}Recovered: switch-core has stopped reporting.{{/is_recovery}}\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",
"tags": [
"service:switch-core",
- "team:switch"
+ "team:switch",
+ "tier:now"
],
"options": {
"notify_no_data": true,
- "no_data_timeframe": 30,
"renotify_interval": 60,
+ "notify_audit": false,
+ "require_full_window": false,
+ "include_tags": true,
"thresholds": {
"critical": 0
},
- "include_tags": true
- }
+ "no_data_timeframe": 15
+ },
+ "priority": 1
},
{
"name": "[Switch] the database is unreachable",
- "type": "metric alert",
- "query": "min(last_5m):min:switch.health.check{service:switch-core,check:database} < 1",
- "message": "switch-core's readiness check cannot reach the database.\n\nThis is the one dependency that takes the server out of service, so Kubernetes has already emptied the Service \u2014 and because switch-core runs as a single replica there is nothing else serving. Every request is failing.\n\nCheck the database before the pod: the pod is reporting the fault, not causing it.\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",
+ "type": "query alert",
+ "query": "max(last_5m):min:switch.health.check{service:switch-core,check:database} < 1",
+ "message": "{{#is_alert}}switch-core can't reach its database. Users can't do anything that reads or writes data.\n\n**Check:** the database instance's status, and the pods' readiness.\n\nSignal board: open \"Switch signal\" in Datadog's dashboard list.{{/is_alert}}{{#is_recovery}}Recovered: the database is unreachable.{{/is_recovery}}\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",
"tags": [
"service:switch-core",
- "team:switch"
+ "team:switch",
+ "tier:now"
],
"options": {
"notify_no_data": false,
- "renotify_interval": 30,
+ "renotify_interval": 60,
+ "notify_audit": false,
+ "require_full_window": true,
+ "include_tags": true,
"thresholds": {
"critical": 1
- },
- "include_tags": true
- }
+ }
+ },
+ "priority": 1
},
{
- "name": "[Switch] no room messages are being delivered",
- "type": "metric alert",
- "query": "min(last_10m):min:switch.health.check{service:switch-core,check:message_listener} < 1",
- "message": "The Postgres LISTEN connection that wakes every room's delivery is down.\n\nThe API still answers and rooms still accept writes, so from outside Switch looks healthy \u2014 and no message reaches anybody. That combination is why this is alerted separately from the database check.\n\nIt reconnects itself with backoff, which is why it does not fail readiness: restarting the pod would drop every live session without fixing it any faster. Ten minutes of this means the backoff is not winning.\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",
+ "name": "[Switch] the database is refusing connections",
+ "type": "query alert",
+ "query": "sum(last_5m):default_zero(sum:switch.db.pool.timeouts{service:switch-core}.as_count()) > 3",
+ "message": "{{#is_alert}}Requests timed out waiting for a database connection. Users see slow or failed actions.\n\n**Check:** pool in use on the Signal board; a long query or a transaction left open usually holds the pool.\n\nSignal board: open \"Switch signal\" in Datadog's dashboard list.{{/is_alert}}{{#is_recovery}}Recovered: the database is refusing connections.{{/is_recovery}}\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",
"tags": [
"service:switch-core",
- "team:switch"
+ "team:switch",
+ "tier:now"
],
"options": {
"notify_no_data": false,
"renotify_interval": 60,
+ "notify_audit": false,
+ "require_full_window": false,
+ "include_tags": true,
"thresholds": {
- "critical": 1
- },
- "include_tags": true
- }
+ "critical": 3
+ }
+ },
+ "priority": 2
},
{
- "name": "[Switch] a collaboration bridge has crashed",
- "type": "metric alert",
- "query": "min(last_10m):min:switch.health.check{service:switch-core,check:bridges} < 1",
- "message": "A configured collaboration bridge is no longer running.\n\nThe platform it serves is cut off \u2014 messages from that Slack, Mattermost, Discord, Teams or Telegram channel reach nobody, and nothing Switch sends reaches it. The rest of Switch is unaffected, deliberately: a dead adapter must not take the whole server off the air.\n\nA bridge that crashes removes itself and does not retry, so this needs a restart of that bridge rather than waiting.\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",
+ "name": "[Switch] the database pool keeps hitting its limit",
+ "type": "query alert",
+ "query": "min(last_10m):max:switch.db.pool.in_use{service:switch-core} by {pod_name} / max:switch.db.pool.size{service:switch-core} by {pod_name} >= 1",
+ "message": "{{#is_alert}}On pod {{pod_name.name}}, every reporting interval for the last 10 minutes peaked with the whole database pool in use, so it keeps running into overflow. Requests slow down next, then start timing out.\n\n**Check:** what holds connections: a slow query, a transaction left open, or a burst of work. Raise the pool size only once you know.\n\nSignal board: open \"Switch signal\" in Datadog's dashboard list.{{/is_alert}}{{#is_recovery}}Recovered: the database pool keeps hitting its limit.{{/is_recovery}}\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",
"tags": [
"service:switch-core",
- "team:switch"
+ "team:switch",
+ "tier:now"
],
"options": {
"notify_no_data": false,
- "renotify_interval": 120,
+ "renotify_interval": 60,
+ "notify_audit": false,
+ "require_full_window": true,
+ "include_tags": true,
"thresholds": {
"critical": 1
- },
- "include_tags": true
- }
+ }
+ },
+ "priority": 2
},
{
- "name": "[Switch] messages are being dropped on delivery",
- "type": "metric alert",
- "query": "sum(last_10m):sum:switch.messages.delivery_failures{service:switch-core}.as_count() > 5",
- "message": "The delivery loop has failed to deliver {{value}} time(s) in ten minutes.\n\nEach one is a message written to a room that a client never received. The loop swallows these on purpose so one bad room cannot stop the others, so nothing else will surface them \u2014 the counter is the only evidence.\n\nThe logs carry the room and the client for each: search for \"Delivery failed for client\".\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",
+ "name": "[Switch] room messages are not being delivered",
+ "type": "query alert",
+ "query": "max(last_10m):min:switch.health.check{service:switch-core,check:message_listener} < 1",
+ "message": "{{#is_alert}}The listener that delivers room messages is down. Messages are saved but don't reach anyone until it comes back.\n\n**Check:** the pods' logs for the message listener; a restart brings it back while you look for why.\n\nSignal board: open \"Switch signal\" in Datadog's dashboard list.{{/is_alert}}{{#is_recovery}}Recovered: room messages are not being delivered.{{/is_recovery}}\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",
"tags": [
"service:switch-core",
- "team:switch"
+ "team:switch",
+ "tier:now"
],
"options": {
"notify_no_data": false,
"renotify_interval": 60,
+ "notify_audit": false,
+ "require_full_window": true,
+ "include_tags": true,
"thresholds": {
- "critical": 5,
- "warning": 1
- },
- "include_tags": true
- }
+ "critical": 1
+ }
+ },
+ "priority": 1
},
{
- "name": "[Switch] the event loop is stalling",
- "type": "metric alert",
- "query": "avg(last_10m):max:switch.runtime.event_loop_lag{service:switch-core} > 1000",
- "message": "A fixed-interval task is waking more than a second late.\n\nswitch-core is single-threaded and cooperative, so this is not one slow thing \u2014 it is everything at once. Room delivery, bridge dispatch and agent heartbeats are all waiting on the same loop, and heartbeats arriving late make live agent connections look lapsed through no fault of their own.\n\nA synchronous call on the event loop is the usual cause. The stall itself is the bug; whatever timed out because of it is a symptom.\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",
+ "name": "[Switch] messages are failing to deliver",
+ "type": "query alert",
+ "query": "sum(last_10m):default_zero(sum:switch.messages.delivery_failures{service:switch-core}.as_count()) > 5",
+ "message": "{{#is_alert}}More than 5 deliveries failed in 10 minutes. Some people or agents are missing messages.\n\n**Check:** the delivery errors in the logs; one bad room can't stop the others, so look for which rooms.\n\nSignal board: open \"Switch signal\" in Datadog's dashboard list.{{/is_alert}}{{#is_recovery}}Recovered: messages are failing to deliver.{{/is_recovery}}\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",
"tags": [
"service:switch-core",
- "team:switch"
+ "team:switch",
+ "tier:now"
],
"options": {
"notify_no_data": false,
"renotify_interval": 60,
+ "notify_audit": false,
+ "require_full_window": false,
+ "include_tags": true,
"thresholds": {
- "critical": 1000,
- "warning": 250
- },
- "include_tags": true
- }
+ "critical": 5
+ }
+ },
+ "priority": 2
},
{
- "name": "[Switch] health checks have stopped running",
- "type": "metric alert",
- "query": "min(last_10m):min:switch.health.check{service:switch-core,check:health_monitor} < 1",
- "message": "The loop that refreshes switch-core's dependency checks has stopped, or the event loop is blocked for long enough that it cannot run.\n\nThis matters more than it sounds. While it is true, every other health check on this list is reporting its last known answer \u2014 so the database, listener and bridge monitors can all sit at a comfortable 1 while nothing has actually been checked for minutes. Treat a firing here as invalidating the others, not as one more fault beside them.\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",
+ "name": "[Switch] messages are failing to send",
+ "type": "query alert",
+ "query": "sum(last_10m):default_zero(sum:switch.messages.send_failures{service:switch-core}.as_count()) > 5",
+ "message": "{{#is_alert}}More than 5 messages failed to save in 10 minutes. They are lost: the sender sees nothing, the room stays quiet.\n\n**Check:** database health first, then the send errors in the logs; `kind` there says which sends failed.\n\nSignal board: open \"Switch signal\" in Datadog's dashboard list.{{/is_alert}}{{#is_recovery}}Recovered: messages are failing to send.{{/is_recovery}}\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",
"tags": [
"service:switch-core",
- "team:switch"
+ "team:switch",
+ "tier:now"
],
"options": {
"notify_no_data": false,
"renotify_interval": 60,
+ "notify_audit": false,
+ "require_full_window": false,
+ "include_tags": true,
"thresholds": {
- "critical": 1
- },
- "include_tags": true
- }
+ "critical": 5
+ }
+ },
+ "priority": 1
},
{
- "name": "[Switch] the database pool is nearly exhausted",
- "type": "metric alert",
- "query": "avg(last_10m):avg:switch.db.pool.in_use{service:switch-core} by {pod_name} / avg:switch.db.pool.size{service:switch-core} by {pod_name} > 0.9",
- "message": "The connection pool of {{pod_name.name}} has been at more than 90% of its nominal size for ten minutes. Each pod has its own pool, so this alerts per pod.\n\nNominal, not total: the pool may open a few connections beyond its size before anything blocks, so this is an early warning rather than the moment things break. What happens if it keeps climbing is that requests start waiting for a connection rather than failing, so the symptom is latency everywhere and then a readiness check that times out \u2014 which reads as \"the database is down\" when the database is fine and this server is holding all of it.\n\nLook for a query holding a connection longer than it should before raising the pool size.\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",
+ "name": "[Switch] a messaging app bridge is down",
+ "type": "query alert",
+ "query": "max(last_15m):min:switch.health.check{service:switch-core,check:bridges} < 1",
+ "message": "{{#is_alert}}At least one configured messaging app bridge (Slack, Discord, Teams...) has stopped. People on that platform are cut off from their rooms; the rest of Switch works.\n\n**Check:** switch.bridges.running by platform on the Signal board, and \"Bridge ... crashed\" in the logs.\n\nSignal board: open \"Switch signal\" in Datadog's dashboard list.{{/is_alert}}{{#is_recovery}}Recovered: a messaging app bridge is down.{{/is_recovery}}\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",
"tags": [
"service:switch-core",
- "team:switch"
+ "team:switch",
+ "tier:now"
],
"options": {
"notify_no_data": false,
"renotify_interval": 60,
+ "notify_audit": false,
+ "require_full_window": true,
+ "include_tags": true,
"thresholds": {
- "critical": 0.9,
- "warning": 0.75
- },
- "include_tags": true
- }
+ "critical": 1
+ }
+ },
+ "priority": 2
},
{
- "name": "[Switch] the database is slow",
- "type": "metric alert",
- "query": "avg(last_10m):p95:switch.db.query.duration{service:switch-core} > 500",
- "message": "The 95th percentile statement took more than half a second over ten minutes.\n\nDistinct from the pool alert beside it, and they fail in opposite directions: the pool one says this server is holding too many connections, this one says the database is taking too long to answer on the connections it has. A slow database exhausts the pool eventually, so this should fire first \u2014 which is the point of having it.\n\nBreak it down by operation on the dashboard. Statements that failed are not timed, so this is latency and never an error rate.\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",
+ "name": "[Switch] a messaging app bridge has crashed, by name",
+ "type": "query alert",
+ "query": "max(last_5m):max:switch.bridges.crashed{service:switch-core} by {platform,bridge_name} >= 1",
+ "message": "{{#is_alert}}The {{platform.name}} bridge \"{{bridge_name.name}}\" has crashed. People in that workspace are cut off from their rooms; the rest of Switch works.\n\n**Check:** \"Bridge ... crashed\" in the logs for that bridge, then restart it or, if nobody uses it any more, deactivate it.\n\nSignal board: open \"Switch signal\" in Datadog's dashboard list.{{/is_alert}}{{#is_recovery}}Recovered: the {{platform.name}} bridge \"{{bridge_name.name}}\" runs again.{{/is_recovery}}\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",
"tags": [
"service:switch-core",
- "team:switch"
+ "team:switch",
+ "tier:now"
],
"options": {
"notify_no_data": false,
"renotify_interval": 60,
+ "notify_audit": false,
+ "require_full_window": false,
+ "include_tags": true,
"thresholds": {
- "critical": 500,
- "warning": 200
- },
- "include_tags": true
- }
+ "critical": 1
+ }
+ },
+ "priority": 2
},
{
- "name": "[Switch] the API is slow",
- "type": "metric alert",
- "query": "avg(last_10m):p95:switch.http.request.duration{service:switch-core} > 2000",
- "message": "The 95th percentile request took more than two seconds over ten minutes.\n\nThe one alert for the half of \"is it healthy\" that error rates cannot see: an endpoint answering 200 slowly is, from a user's side, an endpoint that is broken, and nothing else here fires on it.\n\np95 rather than an average, which one very slow request drags and a thousand fast ones hide. Across all routes rather than by route, because a per-route alert is thirteen alerts that each need their own threshold; break it down on the dashboard once this fires.\n\nThe agent event streams are excluded at the source \u2014 they are long polls held open for a duration the caller chose \u2014 so this is the routes where the number means something. Check the database query duration panel first: on this server a slow request is usually a slow query.\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",
+ "name": "[Switch] server errors are elevated",
+ "type": "query alert",
+ "query": "sum(last_10m):default_zero(sum:switch.http.requests{service:switch-core,status_class:5xx}.as_count()) >= 10",
+ "message": "{{#is_alert}}At least 10 requests failed with a server error in 10 minutes. Users see errors in the Console, Gateway or agents.\n\n**Check:** server errors by route on the core service health dashboard, then that route's logs.\n\nA count, not a percentage: at pilot's traffic one failure among a few requests would read as a high error rate.\n\nSignal board: open \"Switch signal\" in Datadog's dashboard list.{{/is_alert}}{{#is_recovery}}Recovered: server errors are elevated.{{/is_recovery}}\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",
"tags": [
"service:switch-core",
- "team:switch"
+ "team:switch",
+ "tier:now"
],
"options": {
"notify_no_data": false,
"renotify_interval": 60,
+ "notify_audit": false,
+ "require_full_window": false,
+ "include_tags": true,
"thresholds": {
- "critical": 2000,
- "warning": 1000
- },
- "include_tags": true
- }
+ "critical": 10
+ }
+ },
+ "priority": 2
},
{
- "name": "[Switch] server errors are elevated",
- "type": "metric alert",
- "query": "sum(last_10m):sum:switch.http.requests{service:switch-core,status_class:5xx}.as_count() / sum:switch.http.requests{service:switch-core}.as_count() > 0.05",
- "message": "More than 5% of requests answered 5xx over ten minutes.\n\nA ratio rather than a count, so a quiet period does not page and a busy one is not held to the same absolute number. Break it down by route on the dashboard: one failing endpoint and everything failing want different responses.\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",
+ "name": "[Switch] the API is slow",
+ "type": "query alert",
+ "query": "avg(last_10m):p95:switch.http.request.duration{service:switch-core,!route:/gateway/chats/events,!route:/gateway/chats/activity/events} > 2000",
+ "message": "{{#is_alert}}The slowest 5% of requests took over 2 seconds, for 10 minutes. Everything feels sluggish.\n\n**Check:** the database pool and event-loop lag on the Signal board; slowness usually starts there.\n\nSignal board: open \"Switch signal\" in Datadog's dashboard list.{{/is_alert}}{{#is_recovery}}Recovered: the API is slow.{{/is_recovery}}\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",
"tags": [
"service:switch-core",
- "team:switch"
+ "team:switch",
+ "tier:now"
],
"options": {
"notify_no_data": false,
"renotify_interval": 60,
+ "notify_audit": false,
+ "require_full_window": true,
+ "include_tags": true,
"thresholds": {
- "critical": 0.05,
- "warning": 0.01
- },
- "include_tags": true
- }
+ "critical": 2000
+ }
+ },
+ "priority": 3
},
{
- "name": "[Switch] a bridge is failing to relay",
- "type": "metric alert",
- "query": "sum(last_15m):sum:switch.bridge.errors{service:switch-core,bridge:collaboration} by {platform,direction}.as_count() > 10",
- "message": "{{platform.name}} has failed {{value}} time(s) {{direction.name}} in fifteen minutes.\n\nInbound failures are messages a person sent that nobody received; from the platform's side that is indistinguishable from being ignored. Outbound failures are Switch's replies not arriving.\n\nThe bridge is still running \u2014 a crashed one is a different alert. This is usually the platform rate-limiting, an expired token, or a channel the app was removed from.\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",
+ "name": "[Switch] messages are arriving late",
+ "type": "query alert",
+ "query": "avg(last_15m):p95:switch.messages.delivery_lag{service:switch-core} > 10000",
+ "message": "{{#is_alert}}The slowest 5% of messages took over 10 seconds to reach their recipients, for 15 minutes. Conversations stall.\n\n**Check:** event-loop lag and the database pool on the Signal board.\n\nSignal board: open \"Switch signal\" in Datadog's dashboard list.{{/is_alert}}{{#is_recovery}}Recovered: messages are arriving late.{{/is_recovery}}\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",
"tags": [
"service:switch-core",
- "team:switch"
+ "team:switch",
+ "tier:now"
],
"options": {
"notify_no_data": false,
- "renotify_interval": 120,
+ "renotify_interval": 60,
+ "notify_audit": false,
+ "require_full_window": true,
+ "include_tags": true,
"thresholds": {
- "critical": 10,
- "warning": 3
- },
- "include_tags": true
- }
+ "critical": 10000
+ }
+ },
+ "priority": 3
},
{
- "name": "[Switch] agents are losing events",
- "type": "metric alert",
- "query": "sum(last_10m):sum:switch.agent.events_dropped{service:switch-core} by {reason}.as_count() > 20",
- "message": "{{value}} buffered event(s) were discarded before an agent read them in ten minutes, reason {{reason.name}}.\n\n`overflow` means an agent is not keeping up with what is being sent to it \u2014 its buffer filled and the oldest went. `retention` means it was away longer than the window and its backlog aged out; a handful after a restart is ordinary, a steady rate is an agent that is not really connected.\n\nThe agent is told it missed events when it next reads, so nothing is pretending this did not happen \u2014 but only this counter says how often.\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",
+ "name": "[Switch] the server is stalling",
+ "type": "query alert",
+ "query": "avg(last_10m):max:switch.runtime.event_loop_lag{service:switch-core} > 1000",
+ "message": "{{#is_alert}}A switch-core process has stalled for over a second at a time, for 10 minutes. Every room and bridge on that pod waits while it does.\n\n**Check:** CPU on the pods and the slowest database queries; one blocking call stalls everything.\n\nSignal board: open \"Switch signal\" in Datadog's dashboard list.{{/is_alert}}{{#is_recovery}}Recovered: the server is stalling.{{/is_recovery}}\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",
"tags": [
"service:switch-core",
- "team:switch"
+ "team:switch",
+ "tier:now"
],
"options": {
"notify_no_data": false,
"renotify_interval": 60,
+ "notify_audit": false,
+ "require_full_window": true,
+ "include_tags": true,
"thresholds": {
- "critical": 20,
- "warning": 5
- },
- "include_tags": true
- }
+ "critical": 1000
+ }
+ },
+ "priority": 3
},
{
- "name": "[Switch] messages are failing to send",
- "type": "metric alert",
- "query": "sum(last_10m):sum:switch.messages.send_failures{service:switch-core}.as_count() > 5",
- "message": "{{value}} send(s) raised before the row was committed in ten minutes.\n\nThese are messages that were never written, so they will not be delivered and will not appear in any room. `switch.messages.sent` counts only what persisted, which means without this the symptom is an absence \u2014 and an absence looks exactly like a quiet room.\n\nUsually the database. Check the database health monitor first; if that is green, look for a constraint or a tenant binding failing on one particular path.\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",
+ "name": "[Switch] agent connections are dropping en masse",
+ "type": "query alert",
+ "query": "pct_change(avg(last_10m),last_30m):sum:switch.agents.connected{service:switch-core} < -50",
+ "message": "{{#is_alert}}Agent connections fell by more than half in 10 minutes. Agents that lost theirs stop answering in their rooms.\n\n**Check:** pod restarts and agent connection logs; a pod loss makes its agents reconnect elsewhere.\n\nSignal board: open \"Switch signal\" in Datadog's dashboard list.{{/is_alert}}{{#is_recovery}}Recovered: agent connections are dropping en masse.{{/is_recovery}}\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",
"tags": [
"service:switch-core",
- "team:switch"
+ "team:switch",
+ "tier:now"
],
"options": {
"notify_no_data": false,
"renotify_interval": 60,
+ "notify_audit": false,
+ "require_full_window": false,
+ "include_tags": true,
"thresholds": {
- "critical": 5,
- "warning": 1
- },
- "include_tags": true
- }
+ "critical": -50
+ }
+ },
+ "priority": 2
},
{
- "name": "[Switch] the database pool is refusing connections",
- "type": "metric alert",
- "query": "sum(last_5m):sum:switch.db.pool.timeouts{service:switch-core}.as_count() > 0",
- "message": "{{value}} request(s) got no pooled connection within the timeout in five minutes.\n\nThis is the pool past nearly-exhausted and into failing: it queues rather than sheds when full, so a timeout means a request waited out `db_pool_timeout` and still got nothing. Because auth resolves a token against the database before the handler runs, the first thing to fail is authentication on otherwise-fine requests — including agent heartbeats, which then look lapsed and reconnect, which loads the pool further.\n\nLook for a connection held across non-database work (an external call inside an open session) or a synchronized reconnect burst. Raising the pool size is a cushion, not the cause.\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",
+ "name": "[Switch] the morning report didn't post",
+ "type": "query alert",
+ "query": "max(last_1d):max:switch.daily_report.posted{service:switch-core} < 1",
+ "message": "{{#is_no_data}}The 09:00 report of switch-core hasn't posted to this channel for over 25 hours, so a quiet morning here doesn't mean all is well.\n\n**Check:** the switch-daily-report CronJob's last job in switch-staging (its log says why), and that its Parameter Store settings under /switch/staging/daily-report/ are in place.{{/is_no_data}}{{#is_recovery}}Recovered: the morning report posted again.{{/is_recovery}}\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",
"tags": [
"service:switch-core",
- "team:switch"
+ "team:switch",
+ "tier:now"
],
"options": {
- "notify_no_data": false,
+ "notify_no_data": true,
+ "no_data_timeframe": 1500,
"renotify_interval": 60,
+ "notify_audit": false,
+ "require_full_window": false,
+ "include_tags": true,
"thresholds": {
- "critical": 0
- },
- "include_tags": true
- }
+ "critical": 1
+ }
+ },
+ "priority": 4
}
]
diff --git a/deploy/observability/signal-board.json b/deploy/observability/signal-board.json
new file mode 100644
index 000000000..6d88f1789
--- /dev/null
+++ b/deploy/observability/signal-board.json
@@ -0,0 +1,1001 @@
+{
+ "title": "Switch signal",
+ "description": "Pilot at a glance. Tiles show the last hour; charts follow the time picker. Grey while normal; red, and a dashed red line, mark the limit an alert watches. Detail behind each section: Switch core: detail. Alerts post to Slack #switch-alert-hub. Kept in deploy/observability/signal-board.json; edit it there.",
+ "layout_type": "ordered",
+ "reflow_type": "fixed",
+ "widgets": [
+ {
+ "definition": {
+ "type": "manage_status",
+ "title": "Alerts: is anything firing right now?",
+ "title_align": "left",
+ "summary_type": "monitors",
+ "query": "tag:\"service:switch-core\" tag:\"tier:now\"",
+ "sort": "status,asc",
+ "display_format": "countsAndList",
+ "color_preference": "text",
+ "hide_zero_counts": false,
+ "show_status": true,
+ "show_last_triggered": true,
+ "show_priority": false
+ },
+ "layout": {
+ "x": 0,
+ "y": 0,
+ "width": 12,
+ "height": 3
+ }
+ },
+ {
+ "definition": {
+ "type": "group",
+ "title": "Are agents connected?",
+ "show_title": true,
+ "layout_type": "ordered",
+ "widgets": [
+ {
+ "definition": {
+ "type": "query_value",
+ "title": "connections now",
+ "title_size": "13",
+ "autoscale": true,
+ "precision": 0,
+ "requests": [
+ {
+ "formulas": [
+ {
+ "formula": "q"
+ }
+ ],
+ "queries": [
+ {
+ "data_source": "metrics",
+ "name": "q",
+ "query": "sum:switch.agents.connected{service:switch-core}",
+ "aggregator": "last"
+ }
+ ],
+ "response_format": "scalar",
+ "conditional_formats": [
+ {
+ "comparator": "<=",
+ "value": 0,
+ "palette": "white_on_red"
+ },
+ {
+ "comparator": ">",
+ "value": 0,
+ "palette": "custom_text",
+ "custom_fg_color": "#5b6672"
+ }
+ ]
+ }
+ ],
+ "time": {
+ "live_span": "1h"
+ }
+ },
+ "layout": {
+ "x": 0,
+ "y": 0,
+ "width": 2,
+ "height": 2
+ }
+ },
+ {
+ "definition": {
+ "type": "query_value",
+ "title": "pods running",
+ "title_size": "13",
+ "precision": 0,
+ "autoscale": true,
+ "time": {
+ "live_span": "1h"
+ },
+ "requests": [
+ {
+ "formulas": [
+ {
+ "formula": "count_not_null(q)"
+ }
+ ],
+ "queries": [
+ {
+ "data_source": "metrics",
+ "name": "q",
+ "query": "sum:switch.runtime.cpu_seconds{service:switch-core} by {pod_name}.as_rate()",
+ "aggregator": "last"
+ }
+ ],
+ "response_format": "scalar",
+ "conditional_formats": [
+ {
+ "comparator": "<=",
+ "value": 0,
+ "palette": "white_on_red"
+ },
+ {
+ "comparator": ">",
+ "value": 0,
+ "palette": "custom_text",
+ "custom_fg_color": "#5b6672"
+ }
+ ]
+ }
+ ]
+ },
+ "layout": {
+ "x": 2,
+ "y": 0,
+ "width": 2,
+ "height": 2
+ }
+ },
+ {
+ "definition": {
+ "type": "timeseries",
+ "title": "agent connections, all pods together",
+ "title_size": "13",
+ "show_legend": false,
+ "legend_layout": "horizontal",
+ "legend_columns": [
+ "avg",
+ "max",
+ "value"
+ ],
+ "requests": [
+ {
+ "formulas": [
+ {
+ "formula": "q",
+ "alias": "connected"
+ }
+ ],
+ "queries": [
+ {
+ "data_source": "metrics",
+ "name": "q",
+ "query": "sum:switch.agents.connected{service:switch-core}"
+ }
+ ],
+ "response_format": "timeseries",
+ "display_type": "line",
+ "style": {
+ "palette": "grey",
+ "line_type": "solid",
+ "line_width": "normal"
+ }
+ }
+ ],
+ "markers": [],
+ "yaxis": {
+ "min": "0",
+ "include_zero": true
+ }
+ },
+ "layout": {
+ "x": 0,
+ "y": 2,
+ "width": 4,
+ "height": 3
+ }
+ }
+ ]
+ },
+ "layout": {
+ "x": 0,
+ "y": 3,
+ "width": 4,
+ "height": 7
+ }
+ },
+ {
+ "definition": {
+ "type": "group",
+ "title": "Are the messaging apps up?",
+ "show_title": true,
+ "layout_type": "ordered",
+ "widgets": [
+ {
+ "definition": {
+ "type": "query_value",
+ "title": "bridges health check, last hour",
+ "title_size": "13",
+ "autoscale": true,
+ "precision": 0,
+ "requests": [
+ {
+ "formulas": [
+ {
+ "formula": "1 - q",
+ "number_format": {
+ "unit": {
+ "type": "custom_unit_label",
+ "label": "failing"
+ }
+ }
+ }
+ ],
+ "queries": [
+ {
+ "data_source": "metrics",
+ "name": "q",
+ "query": "min:switch.health.check{service:switch-core,check:bridges}",
+ "aggregator": "last"
+ }
+ ],
+ "response_format": "scalar",
+ "conditional_formats": [
+ {
+ "comparator": ">",
+ "value": 0,
+ "palette": "white_on_red"
+ },
+ {
+ "comparator": "<=",
+ "value": 0,
+ "palette": "white_on_green"
+ }
+ ]
+ }
+ ],
+ "time": {
+ "live_span": "1h"
+ }
+ },
+ "layout": {
+ "x": 0,
+ "y": 0,
+ "width": 4,
+ "height": 2
+ }
+ },
+ {
+ "definition": {
+ "type": "timeseries",
+ "title": "bridges running, by platform",
+ "title_size": "13",
+ "show_legend": true,
+ "legend_layout": "horizontal",
+ "legend_columns": [
+ "value"
+ ],
+ "requests": [
+ {
+ "formulas": [
+ {
+ "formula": "q",
+ "alias": "running"
+ }
+ ],
+ "queries": [
+ {
+ "data_source": "metrics",
+ "name": "q",
+ "query": "sum:switch.bridges.running{service:switch-core} by {platform}"
+ }
+ ],
+ "response_format": "timeseries",
+ "display_type": "line",
+ "style": {
+ "palette": "grey",
+ "line_type": "solid",
+ "line_width": "normal"
+ }
+ }
+ ],
+ "markers": [],
+ "yaxis": {
+ "min": "0",
+ "include_zero": true
+ }
+ },
+ "layout": {
+ "x": 0,
+ "y": 2,
+ "width": 4,
+ "height": 3
+ }
+ }
+ ]
+ },
+ "layout": {
+ "x": 4,
+ "y": 3,
+ "width": 4,
+ "height": 7
+ }
+ },
+ {
+ "definition": {
+ "type": "group",
+ "title": "Latency: is it fast?",
+ "show_title": true,
+ "layout_type": "ordered",
+ "widgets": [
+ {
+ "definition": {
+ "type": "query_value",
+ "title": "API p95, ms",
+ "title_size": "13",
+ "autoscale": true,
+ "precision": 0,
+ "time": {
+ "live_span": "1h"
+ },
+ "requests": [
+ {
+ "formulas": [
+ {
+ "formula": "q"
+ }
+ ],
+ "queries": [
+ {
+ "data_source": "metrics",
+ "name": "q",
+ "query": "p95:switch.http.request.duration{service:switch-core,!route:/gateway/chats/events,!route:/gateway/chats/activity/events}",
+ "aggregator": "avg"
+ }
+ ],
+ "response_format": "scalar",
+ "conditional_formats": [
+ {
+ "comparator": ">",
+ "value": 2000,
+ "palette": "white_on_red"
+ },
+ {
+ "comparator": "<=",
+ "value": 2000,
+ "palette": "white_on_green"
+ }
+ ]
+ }
+ ]
+ },
+ "layout": {
+ "x": 0,
+ "y": 0,
+ "width": 2,
+ "height": 2
+ }
+ },
+ {
+ "definition": {
+ "type": "query_value",
+ "title": "delivery p95, ms",
+ "title_size": "13",
+ "autoscale": true,
+ "precision": 0,
+ "time": {
+ "live_span": "1h"
+ },
+ "requests": [
+ {
+ "formulas": [
+ {
+ "formula": "q"
+ }
+ ],
+ "queries": [
+ {
+ "data_source": "metrics",
+ "name": "q",
+ "query": "p95:switch.messages.delivery_lag{service:switch-core}",
+ "aggregator": "avg"
+ }
+ ],
+ "response_format": "scalar",
+ "conditional_formats": [
+ {
+ "comparator": ">",
+ "value": 10000,
+ "palette": "white_on_red"
+ },
+ {
+ "comparator": "<=",
+ "value": 10000,
+ "palette": "white_on_green"
+ }
+ ]
+ }
+ ]
+ },
+ "layout": {
+ "x": 2,
+ "y": 0,
+ "width": 2,
+ "height": 2
+ }
+ },
+ {
+ "definition": {
+ "type": "timeseries",
+ "title": "API p95 ms (live streams left out)",
+ "title_size": "13",
+ "show_legend": true,
+ "legend_layout": "horizontal",
+ "legend_columns": [
+ "avg",
+ "max",
+ "value"
+ ],
+ "requests": [
+ {
+ "formulas": [
+ {
+ "formula": "q",
+ "alias": "API p95"
+ }
+ ],
+ "queries": [
+ {
+ "data_source": "metrics",
+ "name": "q",
+ "query": "p95:switch.http.request.duration{service:switch-core,!route:/gateway/chats/events,!route:/gateway/chats/activity/events}"
+ }
+ ],
+ "response_format": "timeseries",
+ "display_type": "line",
+ "style": {
+ "palette": "grey",
+ "line_type": "solid",
+ "line_width": "normal"
+ }
+ }
+ ],
+ "markers": [
+ {
+ "value": "y = 2000",
+ "display_type": "error dashed",
+ "label": "alert: 2 s"
+ }
+ ],
+ "yaxis": {
+ "min": "0",
+ "include_zero": true
+ }
+ },
+ "layout": {
+ "x": 0,
+ "y": 2,
+ "width": 4,
+ "height": 2
+ }
+ },
+ {
+ "definition": {
+ "type": "timeseries",
+ "title": "message delivery p95 ms",
+ "title_size": "13",
+ "show_legend": true,
+ "legend_layout": "horizontal",
+ "legend_columns": [
+ "avg",
+ "max",
+ "value"
+ ],
+ "requests": [
+ {
+ "formulas": [
+ {
+ "formula": "q",
+ "alias": "delivery p95"
+ }
+ ],
+ "queries": [
+ {
+ "data_source": "metrics",
+ "name": "q",
+ "query": "p95:switch.messages.delivery_lag{service:switch-core}"
+ }
+ ],
+ "response_format": "timeseries",
+ "display_type": "line",
+ "style": {
+ "palette": "grey",
+ "line_type": "solid",
+ "line_width": "normal"
+ }
+ }
+ ],
+ "markers": [
+ {
+ "value": "y = 10000",
+ "display_type": "error dashed",
+ "label": "alert: 10 s"
+ }
+ ],
+ "yaxis": {
+ "min": "0",
+ "include_zero": true
+ }
+ },
+ "layout": {
+ "x": 0,
+ "y": 4,
+ "width": 4,
+ "height": 2
+ }
+ }
+ ]
+ },
+ "layout": {
+ "x": 8,
+ "y": 3,
+ "width": 4,
+ "height": 7
+ }
+ },
+ {
+ "definition": {
+ "type": "group",
+ "title": "Traffic: how busy?",
+ "show_title": true,
+ "layout_type": "ordered",
+ "widgets": [
+ {
+ "definition": {
+ "type": "query_value",
+ "title": "requests per second, last hour",
+ "title_size": "13",
+ "autoscale": true,
+ "precision": 1,
+ "requests": [
+ {
+ "formulas": [
+ {
+ "formula": "q"
+ }
+ ],
+ "queries": [
+ {
+ "data_source": "metrics",
+ "name": "q",
+ "query": "sum:switch.http.requests{service:switch-core}.as_rate()",
+ "aggregator": "avg"
+ }
+ ],
+ "response_format": "scalar",
+ "conditional_formats": [
+ {
+ "comparator": ">=",
+ "value": 0,
+ "palette": "custom_text",
+ "custom_fg_color": "#5b6672"
+ }
+ ]
+ }
+ ],
+ "time": {
+ "live_span": "1h"
+ }
+ },
+ "layout": {
+ "x": 0,
+ "y": 0,
+ "width": 4,
+ "height": 2
+ }
+ },
+ {
+ "definition": {
+ "type": "timeseries",
+ "title": "requests per second, against the same time last week",
+ "title_size": "13",
+ "show_legend": true,
+ "legend_layout": "horizontal",
+ "legend_columns": [
+ "avg",
+ "value"
+ ],
+ "requests": [
+ {
+ "formulas": [
+ {
+ "formula": "q",
+ "alias": "now"
+ },
+ {
+ "formula": "week_before(q)",
+ "alias": "last week"
+ }
+ ],
+ "queries": [
+ {
+ "data_source": "metrics",
+ "name": "q",
+ "query": "sum:switch.http.requests{service:switch-core}.as_rate()"
+ }
+ ],
+ "response_format": "timeseries",
+ "display_type": "line",
+ "style": {
+ "palette": "grey",
+ "line_type": "solid",
+ "line_width": "normal"
+ }
+ }
+ ],
+ "yaxis": {
+ "min": "0",
+ "include_zero": true
+ }
+ },
+ "layout": {
+ "x": 0,
+ "y": 2,
+ "width": 4,
+ "height": 3
+ }
+ }
+ ]
+ },
+ "layout": {
+ "x": 0,
+ "y": 10,
+ "width": 4,
+ "height": 8
+ }
+ },
+ {
+ "definition": {
+ "type": "group",
+ "title": "Errors: is it failing?",
+ "show_title": true,
+ "layout_type": "ordered",
+ "widgets": [
+ {
+ "definition": {
+ "type": "query_value",
+ "title": "server errors, last hour",
+ "title_size": "13",
+ "autoscale": true,
+ "precision": 0,
+ "requests": [
+ {
+ "formulas": [
+ {
+ "formula": "default_zero(q)"
+ }
+ ],
+ "queries": [
+ {
+ "data_source": "metrics",
+ "name": "q",
+ "query": "sum:switch.http.requests{service:switch-core,status_class:5xx}.as_count()",
+ "aggregator": "sum"
+ }
+ ],
+ "response_format": "scalar",
+ "conditional_formats": [
+ {
+ "comparator": ">=",
+ "value": 10,
+ "palette": "white_on_red"
+ },
+ {
+ "comparator": "<",
+ "value": 10,
+ "palette": "white_on_green"
+ }
+ ]
+ }
+ ],
+ "time": {
+ "live_span": "1h"
+ }
+ },
+ "layout": {
+ "x": 0,
+ "y": 0,
+ "width": 4,
+ "height": 2
+ }
+ },
+ {
+ "definition": {
+ "type": "timeseries",
+ "title": "failures per minute",
+ "title_size": "13",
+ "show_legend": true,
+ "legend_layout": "horizontal",
+ "legend_columns": [
+ "avg",
+ "max",
+ "value"
+ ],
+ "requests": [
+ {
+ "formulas": [
+ {
+ "formula": "q",
+ "alias": "server errors"
+ }
+ ],
+ "queries": [
+ {
+ "data_source": "metrics",
+ "name": "q",
+ "query": "sum:switch.http.requests{service:switch-core,status_class:5xx}.as_count().rollup(sum, 60)"
+ }
+ ],
+ "response_format": "timeseries",
+ "display_type": "bars",
+ "style": {
+ "palette": "grey",
+ "line_type": "solid",
+ "line_width": "normal"
+ }
+ },
+ {
+ "formulas": [
+ {
+ "formula": "q",
+ "alias": "failed deliveries"
+ }
+ ],
+ "queries": [
+ {
+ "data_source": "metrics",
+ "name": "q",
+ "query": "sum:switch.messages.delivery_failures{service:switch-core}.as_count().rollup(sum, 60)"
+ }
+ ],
+ "response_format": "timeseries",
+ "display_type": "bars",
+ "style": {
+ "palette": "grey",
+ "line_type": "solid",
+ "line_width": "normal"
+ }
+ },
+ {
+ "formulas": [
+ {
+ "formula": "q",
+ "alias": "failed sends"
+ }
+ ],
+ "queries": [
+ {
+ "data_source": "metrics",
+ "name": "q",
+ "query": "sum:switch.messages.send_failures{service:switch-core}.as_count().rollup(sum, 60)"
+ }
+ ],
+ "response_format": "timeseries",
+ "display_type": "bars",
+ "style": {
+ "palette": "grey",
+ "line_type": "solid",
+ "line_width": "normal"
+ }
+ }
+ ],
+ "markers": [],
+ "yaxis": {
+ "min": "0",
+ "include_zero": true
+ }
+ },
+ "layout": {
+ "x": 0,
+ "y": 2,
+ "width": 4,
+ "height": 3
+ }
+ }
+ ]
+ },
+ "layout": {
+ "x": 4,
+ "y": 10,
+ "width": 4,
+ "height": 8
+ }
+ },
+ {
+ "definition": {
+ "type": "group",
+ "title": "Saturation: how full?",
+ "show_title": true,
+ "layout_type": "ordered",
+ "widgets": [
+ {
+ "definition": {
+ "type": "timeseries",
+ "title": "database pool in use, per pod, % of its size",
+ "title_size": "13",
+ "show_legend": false,
+ "legend_layout": "horizontal",
+ "legend_columns": [
+ "avg",
+ "max",
+ "value"
+ ],
+ "requests": [
+ {
+ "formulas": [
+ {
+ "formula": "100 * a / b",
+ "alias": "pool used %"
+ }
+ ],
+ "queries": [
+ {
+ "data_source": "metrics",
+ "name": "a",
+ "query": "max:switch.db.pool.in_use{service:switch-core} by {pod_name}"
+ },
+ {
+ "data_source": "metrics",
+ "name": "b",
+ "query": "max:switch.db.pool.size{service:switch-core} by {pod_name}"
+ }
+ ],
+ "response_format": "timeseries",
+ "display_type": "line",
+ "style": {
+ "palette": "grey",
+ "line_type": "solid",
+ "line_width": "normal"
+ }
+ }
+ ],
+ "markers": [
+ {
+ "value": "y = 100",
+ "display_type": "error dashed",
+ "label": "pool size"
+ }
+ ],
+ "yaxis": {
+ "min": "0",
+ "include_zero": true
+ }
+ },
+ "layout": {
+ "x": 0,
+ "y": 0,
+ "width": 4,
+ "height": 3
+ }
+ },
+ {
+ "definition": {
+ "type": "timeseries",
+ "title": "event-loop lag, worst, ms",
+ "title_size": "13",
+ "show_legend": false,
+ "legend_layout": "horizontal",
+ "legend_columns": [
+ "avg",
+ "max",
+ "value"
+ ],
+ "requests": [
+ {
+ "formulas": [
+ {
+ "formula": "q",
+ "alias": "lag"
+ }
+ ],
+ "queries": [
+ {
+ "data_source": "metrics",
+ "name": "q",
+ "query": "max:switch.runtime.event_loop_lag{service:switch-core}"
+ }
+ ],
+ "response_format": "timeseries",
+ "display_type": "line",
+ "style": {
+ "palette": "grey",
+ "line_type": "solid",
+ "line_width": "normal"
+ }
+ }
+ ],
+ "markers": [
+ {
+ "value": "y = 1000",
+ "display_type": "error dashed",
+ "label": "stall alert"
+ }
+ ],
+ "yaxis": {
+ "min": "0",
+ "include_zero": true
+ }
+ },
+ "layout": {
+ "x": 0,
+ "y": 3,
+ "width": 4,
+ "height": 2
+ }
+ },
+ {
+ "definition": {
+ "type": "timeseries",
+ "title": "capacity: CPU cores used against pods running",
+ "title_size": "13",
+ "show_legend": true,
+ "legend_layout": "horizontal",
+ "legend_columns": [
+ "value",
+ "max"
+ ],
+ "requests": [
+ {
+ "formulas": [
+ {
+ "formula": "a",
+ "alias": "cores used, all pods"
+ }
+ ],
+ "queries": [
+ {
+ "data_source": "metrics",
+ "name": "a",
+ "query": "sum:switch.runtime.cpu_seconds{service:switch-core}.as_rate()"
+ }
+ ],
+ "response_format": "timeseries",
+ "display_type": "area",
+ "style": {
+ "palette": "grey",
+ "line_type": "solid",
+ "line_width": "normal"
+ }
+ },
+ {
+ "formulas": [
+ {
+ "formula": "count_not_null(p)",
+ "alias": "pods running (one core each)"
+ }
+ ],
+ "queries": [
+ {
+ "data_source": "metrics",
+ "name": "p",
+ "query": "sum:switch.runtime.cpu_seconds{service:switch-core} by {pod_name}.as_rate()"
+ }
+ ],
+ "response_format": "timeseries",
+ "display_type": "line",
+ "style": {
+ "palette": "red",
+ "line_type": "dashed",
+ "line_width": "normal"
+ }
+ }
+ ],
+ "yaxis": {
+ "min": "0",
+ "include_zero": true
+ }
+ },
+ "layout": {
+ "x": 0,
+ "y": 5,
+ "width": 4,
+ "height": 2
+ }
+ }
+ ]
+ },
+ "layout": {
+ "x": 8,
+ "y": 10,
+ "width": 4,
+ "height": 8
+ }
+ }
+ ],
+ "template_variables": []
+}
diff --git a/deploy/observability/test_daily_report.py b/deploy/observability/test_daily_report.py
new file mode 100644
index 000000000..f9e4b872f
--- /dev/null
+++ b/deploy/observability/test_daily_report.py
@@ -0,0 +1,184 @@
+"""The morning report's verdict, from the day's numbers."""
+
+from __future__ import annotations
+
+import json
+
+import daily_report as r
+import httpx
+
+FINE = {
+ "agents_now": 9,
+ "bridges_min": 1,
+ "api_p95_max": 300,
+ "delivery_p95_max": 800,
+ "server_errors": 3,
+ "send_failures": 0,
+ "delivery_failures": 0,
+ "pool_ratio": 0.3,
+ "loop_lag_max": 120,
+ "old_connection": 0,
+ "events_dropped": 0,
+}
+
+
+def test_a_good_day_is_one_line_to_read():
+ text = r.report_text(r.lines_for(FINE), "switch-core")
+ assert text.splitlines()[0] == "switch-core, last 24 h: all good."
+ assert r.BAD not in text and r.LOOK not in text
+
+
+def test_a_full_pool_and_a_down_bridge_are_called_out():
+ day = {**FINE, "pool_ratio": 1.33, "bridges_min": 0}
+ text = r.report_text(r.lines_for(day), "switch-core")
+ assert (
+ text.splitlines()[0] == "switch-core, last 24 h: 2 to look at, some of it bad."
+ )
+ assert "🔴 Database: peak pool use 133%" in text
+ assert "🔴 Messaging apps: a bridge was down at some point" in text
+
+
+def test_old_connections_and_missing_data_ask_for_a_look():
+ day = {**FINE, "old_connection": 4, "api_p95_max": None}
+ lines = r.lines_for(day)
+ assert r.Line(r.LOOK, "Agents on the old connection: 4") in lines
+ assert r.Line(r.LOOK, "API: worst p95 no data") in lines
+
+
+def test_lost_messages_are_never_good():
+ day = {**FINE, "send_failures": 1}
+ assert any(
+ line.state == r.LOOK and line.text.startswith("Errors:")
+ for line in r.lines_for(day)
+ )
+
+
+GRAPHS = r.featured_graphs("switch-core")
+NO_CHANGE = {graph.key: 0.0 for graph in GRAPHS}
+
+
+def test_the_graph_of_the_day_explains_the_first_line_to_look_at():
+ lines = r.lines_for({**FINE, "pool_ratio": 0.95, "loop_lag_max": 1500})
+ graph, reason = r.pick_graph(lines, GRAPHS, NO_CHANGE, day_number=0)
+ assert graph.key == "pool"
+ assert reason == "it's behind a line above that needs a look"
+
+
+def test_on_a_good_day_the_graph_that_moved_most_is_featured():
+ change = {**NO_CHANGE, "traffic": 0.4, "api": -0.6}
+ graph, reason = r.pick_graph(r.lines_for(FINE), GRAPHS, change, day_number=0)
+ assert graph.key == "api"
+ assert reason == "it moved most against the same day last week (down 60%)"
+
+
+def test_a_quiet_week_walks_through_every_graph():
+ picked = {
+ r.pick_graph(r.lines_for(FINE), GRAPHS, NO_CHANGE, day_number=day)[0].key
+ for day in range(len(GRAPHS))
+ }
+ assert picked == {graph.key for graph in GRAPHS}
+
+
+def test_every_graph_definition_is_valid_json():
+ import json
+
+ for graph in GRAPHS:
+ assert json.loads(r.graph_def(graph))["requests"]
+
+
+BY_KEY = {graph.key: graph for graph in GRAPHS}
+
+
+def test_a_full_pool_is_explained_and_comes_with_a_next_step():
+ day = r.Day(peak=133, peak_where="pod zjttv", peak_at="14:05", minutes_over=40)
+ text = r.graph_caption(BY_KEY["pool"], "it's behind a line above", day)
+ assert "How to read it: Each pod keeps a fixed set" in text
+ assert (
+ "Today: Peak use was 133% on pod zjttv around 14:05 UTC, above the line for about 40 min."
+ in text
+ )
+ assert "add a pod or raise the pool size" in text
+
+
+def test_a_quiet_graph_says_there_is_nothing_to_do():
+ day = r.Day(peak=900, peak_at="03:00")
+ today, advice = r.commentary(BY_KEY["delivery"], day)
+ assert today == "The worst was 900 ms around 03:00 UTC."
+ assert advice.startswith("Nothing to do")
+
+
+def test_close_to_the_limit_is_flagged_before_it_fires():
+ _, advice = r.commentary(BY_KEY["api"], r.Day(peak=1500))
+ assert "over half the 2 s limit" in advice
+
+
+def test_failing_bridges_name_the_likely_cause():
+ day = r.Day(
+ peak=20,
+ notes={"check": "failing", "top_failure": "mattermost, network (1073 times)"},
+ )
+ today, advice = r.commentary(BY_KEY["bridges"], day)
+ assert today == "The bridges health check failed for most of the day."
+ assert advice.startswith(
+ "Most failed connections were mattermost, network (1073 times)."
+ )
+
+
+def test_missing_data_is_not_read_as_good_news():
+ today, advice = r.commentary(BY_KEY["stalls"], r.Day())
+ assert today == "No data in the last 24 hours."
+ assert "stopped reporting" in advice
+
+
+def test_series_names_are_short():
+ assert (
+ r.series_name(
+ "pod_name:switch-switch-core-58f65f47cf-zjttv,service:switch-core"
+ )
+ == "pod zjttv"
+ )
+ assert r.series_name("platform:slack,service:switch-core") == "slack"
+
+
+def test_a_whole_day_over_the_line_says_so():
+ today, _ = r.commentary(BY_KEY["pool"], r.Day(peak=133, minutes_over=1435))
+ assert today.endswith("above the line for the whole day.")
+
+
+def test_a_handful_of_errors_is_background():
+ notes = {"server_errors": "6", "lost": "0", "top_route": "/gateway/rooms (5)"}
+ _, advice = r.commentary(BY_KEY["errors"], r.Day(peak=5, notes=notes))
+ assert advice == (
+ "Nothing to do: a handful a day is background "
+ "(the most came from /gateway/rooms (5))."
+ )
+
+
+def test_a_printed_report_does_not_count_as_posted(monkeypatch):
+ """Only a real post may feed the "didn't post" alert, or a job that lost
+ its room settings would look healthy."""
+ for name in ("SWITCH_URL", "SWITCH_AGENT_ID", "SWITCH_ROOM_ID", "SWITCH_TOKEN"):
+ monkeypatch.delenv(name, raising=False)
+ assert r.post("text", []) is False
+
+
+def test_the_posted_mark_is_one_point_tagged_with_the_service():
+ sent = []
+
+ def handler(request: httpx.Request) -> httpx.Response:
+ sent.append((request.url.path, request.content))
+ return httpx.Response(202, json={"status": "ok"})
+
+ dd = r.Datadog("api", "app", "datadoghq.com")
+ dd.http = httpx.Client(
+ base_url="https://api.datadoghq.com/api/v1",
+ transport=httpx.MockTransport(handler),
+ )
+ dd.mark_posted("switch-core", 1_800_000_000)
+
+ path, body = sent[0]
+ assert path == "/api/v1/series"
+ series = json.loads(body)["series"][0]
+ assert series["metric"] == "switch.daily_report.posted"
+ assert series["points"] == [[1_800_000_000, 1]]
+ assert series["tags"] == ["service:switch-core"]