Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Measure event-loop stalls with a 100 ms probe (CHOO-3013) (#713)

Abel Dantas5263d5102fed
diff --git a/core/switch_core/main.py b/core/switch_core/main.py
index f1cfb2eca..934afc605 100644
--- a/core/switch_core/main.py
+++ b/core/switch_core/main.py
@@ -172,7 +172,6 @@ from switch_core.observability.pool import (
pool_stats,
)
from switch_core.observability.query import instrument_queries
-from switch_core.observability.runtime import EventLoopLag
from switch_core.outbound import guarded_async_client
from switch_core.provisioning import Provisioning
from switch_core.provisioning.postgres import PostgresProvisioning
@@ -226,7 +225,7 @@ async def _runtime_state_sweep_loop(protocol: AgentCore) -> None:
logger.exception("Runtime-state sweep failed")
-async def _connection_sweep_loop(protocol: AgentCore, lag: EventLoopLag) -> None:
+async def _connection_sweep_loop(protocol: AgentCore) -> None:
"""Expire connections whose client has stopped beating.
Skips a round after the event loop has been blocked. A stall stops us
@@ -235,16 +234,11 @@ async def _connection_sweep_loop(protocol: AgentCore, lag: EventLoopLag) -> None
leases, then every client reconnects together, which is a worse stall. The
clients were never given the chance to beat, so the honest reading is "we
were not listening", not "they went away".
-
- Its short fixed interval also makes it the most sensitive witness to the
- loop being blocked, so every round's oversleep is reported — not only the
- ones large enough to skip a sweep.
"""
while True:
started = time.monotonic()
await asyncio.sleep(_CONNECTION_SWEEP_INTERVAL)
overslept = (time.monotonic() - started) - _CONNECTION_SWEEP_INTERVAL
- lag.record(overslept)
if overslept > HEARTBEAT_TTL_SECONDS / 2:
logger.warning(
"AgentConnection sweep skipped: the event loop was blocked for %.1fs, "
@@ -1019,7 +1013,7 @@ async def run(config: SwitchConfig) -> None:
session_activity_maintenance_loop(session_factory)
)
connection_sweep_task = asyncio.create_task(
- _connection_sweep_loop(protocol, observability.lag)
+ _connection_sweep_loop(protocol)
)
service_token_task = asyncio.create_task(
service_token_maintenance_loop(session_factory, service_broker)
diff --git a/core/switch_core/observability/bootstrap.py b/core/switch_core/observability/bootstrap.py
index 0f0274a2d..043f4f3fe 100644
--- a/core/switch_core/observability/bootstrap.py
+++ b/core/switch_core/observability/bootstrap.py
@@ -59,6 +59,7 @@ from switch_core.observability.runtime import (
GcPauses,
RuntimeMetrics,
log_unreadable_sources,
+ watch_event_loop,
)
from switch_core.transport.room_cache import RoomCacheStats
from switch_core.version import switch_core_commit, switch_core_repository_url
@@ -217,6 +218,7 @@ def start_observability(
gc_pauses = GcPauses()
gc_pauses.install()
RuntimeMetrics(lag, gc_pauses).install(registry)
+ tasks.append(asyncio.create_task(watch_event_loop(lag), name="event-loop-probe"))
registry.register_observer(_state_readings(probes))
log_unreadable_sources()
diff --git a/core/switch_core/observability/catalogue.py b/core/switch_core/observability/catalogue.py
index b69462372..08aa3f2ed 100644
--- a/core/switch_core/observability/catalogue.py
+++ b/core/switch_core/observability/catalogue.py
@@ -551,9 +551,20 @@ RUNTIME_EVENT_LOOP_LAG = _spec(
"switch.runtime.event_loop_lag",
"gauge",
"ms",
- "How far past its deadline a fixed-interval task woke. The server is "
- "single-threaded, so this is the one number that says whether anything is "
- "being starved.",
+ "The longest the event loop was blocked since the last reading, measured "
+ "by a probe that wakes every 100 ms. The server is single-threaded, so "
+ "this is the one number that says whether anything is being starved.",
+)
+# Every stall the probe sees, not only the worst per interval: how often the
+# loop blocks and for how long. Below the threshold is ordinary scheduling
+# jitter and would only drown the stalls that matter.
+RUNTIME_EVENT_LOOP_STALLS = _spec(
+ "switch.runtime.event_loop_stalls",
+ "histogram",
+ "ms",
+ "Event-loop stalls of 50 ms or more, by how long the loop was blocked. "
+ "The count says how often, the buckets say how bad.",
+ bounds=(50.0, 100.0, 200.0, 300.0, 500.0, 750.0, 1000.0, 2000.0, 5000.0),
)
# One per process start, so a restart lines up against everything else on a
# dashboard. The version is already on every series' resource.
@@ -643,6 +654,7 @@ CATALOGUE: dict[str, MetricSpec] = {
RUNTIME_CPU_SECONDS,
RUNTIME_OPEN_FDS,
RUNTIME_EVENT_LOOP_LAG,
+ RUNTIME_EVENT_LOOP_STALLS,
RUNTIME_STARTS,
RUNTIME_GC_COLLECTIONS,
RUNTIME_GC_PAUSE,
diff --git a/core/switch_core/observability/runtime.py b/core/switch_core/observability/runtime.py
index ce411401e..aeebb6aa1 100644
--- a/core/switch_core/observability/runtime.py
+++ b/core/switch_core/observability/runtime.py
@@ -13,6 +13,7 @@ fabricated memory figure on a developer's laptop is worse than a missing one.
from __future__ import annotations
+import asyncio
import gc
import logging
import os
@@ -26,6 +27,7 @@ from collections.abc import Iterator
from switch_core.observability.catalogue import (
RUNTIME_CPU_SECONDS,
RUNTIME_EVENT_LOOP_LAG,
+ RUNTIME_EVENT_LOOP_STALLS,
RUNTIME_GC_COLLECTIONS,
RUNTIME_GC_PAUSE,
RUNTIME_MEMORY_RSS,
@@ -117,6 +119,35 @@ class GcPauses:
return drained
+# How often the probe wakes. A stall is seen only while it overlaps a wake-up,
+# so the interval bounds how much of a stall the reading can miss: at 100 ms a
+# 300 ms freeze is always caught, and read to within 100 ms. The cost is ten
+# timer callbacks a second.
+LOOP_PROBE_INTERVAL_SECONDS = 0.1
+
+# Oversleep below this is scheduling jitter, not a stall.
+LOOP_STALL_THRESHOLD_SECONDS = 0.05
+
+
+async def watch_event_loop(
+ lag: EventLoopLag, interval_seconds: float = LOOP_PROBE_INTERVAL_SECONDS
+) -> None:
+ """Sleep in short steps and record how late each wake-up was.
+
+ Lateness is time the loop could not run anything, which is the thing to
+ measure. A long sleep would only see the part of a stall that overlaps its
+ own deadline: a 420 ms freeze during a 2 s sleep is usually missed and, when
+ seen, read as anything from 0 to 420 ms.
+ """
+ while True:
+ started = time.monotonic()
+ await asyncio.sleep(interval_seconds)
+ overslept = (time.monotonic() - started) - interval_seconds
+ lag.record(overslept)
+ if overslept >= LOOP_STALL_THRESHOLD_SECONDS:
+ metrics().observe(RUNTIME_EVENT_LOOP_STALLS, {}, overslept * 1000.0)
+
+
def _resident_bytes() -> int | None:
"""Current resident set size, or None where it cannot be read.
diff --git a/core/tests/switch_core/observability/test_runtime.py b/core/tests/switch_core/observability/test_runtime.py
new file mode 100644
index 000000000..835713ec9
--- /dev/null
+++ b/core/tests/switch_core/observability/test_runtime.py
@@ -0,0 +1,58 @@
+import asyncio
+import contextlib
+import time
+from collections.abc import Iterator
+
+import pytest
+
+from switch_core.observability.catalogue import RUNTIME_EVENT_LOOP_STALLS
+from switch_core.observability.metrics import MetricsRegistry, install, uninstall
+from switch_core.observability.runtime import EventLoopLag, watch_event_loop
+
+
+def registry() -> Iterator[MetricsRegistry]:
+ registry = MetricsRegistry()
+ install(registry)
+ yield registry
+ uninstall()
+
+
+async def _probe_while(lag: EventLoopLag, block_seconds: float) -> None:
+ probe = asyncio.create_task(watch_event_loop(lag, interval_seconds=0.02))
+ # Let the probe start its first sleep, then hold the loop the way a
+ # synchronous call or a garbage collection would.
+ await asyncio.sleep(0.05)
+ time.sleep(block_seconds)
+ await asyncio.sleep(0.05)
+ probe.cancel()
+ with contextlib.suppress(asyncio.CancelledError):
+ await probe
+
+
+async def test_a_stall_is_read_at_its_full_length(registry: MetricsRegistry):
+ lag = EventLoopLag()
+ await _probe_while(lag, block_seconds=0.3)
+
+ # The probe's own interval is the most it can under-read by.
+ assert lag.take() >= 300 - 20
+
+
+async def test_a_stall_is_counted_with_its_duration(registry: MetricsRegistry):
+ lag = EventLoopLag()
+ await _probe_while(lag, block_seconds=0.3)
+
+ stalls = {payload.name: payload for payload in registry.collect()}[
+ RUNTIME_EVENT_LOOP_STALLS.name
+ ]
+ point = stalls.histograms[0]
+ assert point.count == 1
+ assert point.total >= 300 - 20
+
+
+async def test_scheduling_jitter_is_not_a_stall(registry: MetricsRegistry):
+ lag = EventLoopLag()
+ await _probe_while(lag, block_seconds=0.0)
+
+ names = {payload.name for payload in registry.collect()}
+ assert RUNTIME_EVENT_LOOP_STALLS.name not in names
diff --git a/deploy/observability/dashboard.json b/deploy/observability/dashboard.json
index 8cbd9989b..1316b538f 100644
--- a/deploy/observability/dashboard.json
+++ b/deploy/observability/dashboard.json
@@ -467,12 +467,17 @@
{
"definition": {
"type": "timeseries",
- "title": "Event-loop lag (ms)",
- "show_legend": false,
+ "title": "Event-loop lag (ms) and stalls of 50 ms or more",
+ "show_legend": true,
"requests": [
{
"q": "max:switch.runtime.event_loop_lag{$env,$service}",
"display_type": "bars"
+ },
+ {
+ "q": "count:switch.runtime.event_loop_stalls{$env,$service}.as_count()",
+ "display_type": "line",
+ "on_right_yaxis": true
}
]
},
diff --git a/docs/old/observability.md b/docs/old/observability.md
index 65702a9f5..bbcfc0b24 100644
--- a/docs/old/observability.md
+++ b/docs/old/observability.md
@@ -113,7 +113,10 @@ readiness never gated on it.
A steady `switch.runtime.event_loop_lag` of one or two milliseconds is the
floor, not a stall: it is how far past its deadline a timer normally wakes.
-The alert on it fires at a thousand times that.
+The alert on it fires at a thousand times that. The reading comes from a probe
+that wakes every 100 ms, so a stall longer than that is always caught and read
+to within 100 ms. `switch.runtime.event_loop_stalls` counts each stall of 50 ms
+or more by duration, which says how often the loop blocks, not only how badly.
**What this does not exercise.** A bare server has no rooms, no agents and no
bridges, so nine of the twenty-five metrics never appear — everything under