Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Measure event-loop stalls with a 100 ms probe (CHOO-3013) (#713)
Abel Dantas5263d5102fed
diff --git a/core/switch_core/main.py b/core/switch_core/main.pyindex f1cfb2eca..934afc605 100644--- a/core/switch_core/main.py+++ b/core/switch_core/main.py@@ -172,7 +172,6 @@ from switch_core.observability.pool import (pool_stats,)from switch_core.observability.query import instrument_queries-from switch_core.observability.runtime import EventLoopLagfrom switch_core.outbound import guarded_async_clientfrom switch_core.provisioning import Provisioningfrom switch_core.provisioning.postgres import PostgresProvisioning@@ -226,7 +225,7 @@ async def _runtime_state_sweep_loop(protocol: AgentCore) -> None:logger.exception("Runtime-state sweep failed")-async def _connection_sweep_loop(protocol: AgentCore, lag: EventLoopLag) -> None:+async def _connection_sweep_loop(protocol: AgentCore) -> None:"""Expire connections whose client has stopped beating.Skips a round after the event loop has been blocked. A stall stops us@@ -235,16 +234,11 @@ async def _connection_sweep_loop(protocol: AgentCore, lag: EventLoopLag) -> Noneleases, then every client reconnects together, which is a worse stall. Theclients were never given the chance to beat, so the honest reading is "wewere not listening", not "they went away".-- Its short fixed interval also makes it the most sensitive witness to the- loop being blocked, so every round's oversleep is reported — not only the- ones large enough to skip a sweep."""while True:started = time.monotonic()await asyncio.sleep(_CONNECTION_SWEEP_INTERVAL)overslept = (time.monotonic() - started) - _CONNECTION_SWEEP_INTERVAL- lag.record(overslept)if overslept > HEARTBEAT_TTL_SECONDS / 2:logger.warning("AgentConnection sweep skipped: the event loop was blocked for %.1fs, "@@ -1019,7 +1013,7 @@ async def run(config: SwitchConfig) -> None:session_activity_maintenance_loop(session_factory))connection_sweep_task = asyncio.create_task(- _connection_sweep_loop(protocol, observability.lag)+ _connection_sweep_loop(protocol))service_token_task = asyncio.create_task(service_token_maintenance_loop(session_factory, service_broker)diff --git a/core/switch_core/observability/bootstrap.py b/core/switch_core/observability/bootstrap.pyindex 0f0274a2d..043f4f3fe 100644--- a/core/switch_core/observability/bootstrap.py+++ b/core/switch_core/observability/bootstrap.py@@ -59,6 +59,7 @@ from switch_core.observability.runtime import (GcPauses,RuntimeMetrics,log_unreadable_sources,+ watch_event_loop,)from switch_core.transport.room_cache import RoomCacheStatsfrom switch_core.version import switch_core_commit, switch_core_repository_url@@ -217,6 +218,7 @@ def start_observability(gc_pauses = GcPauses()gc_pauses.install()RuntimeMetrics(lag, gc_pauses).install(registry)+ tasks.append(asyncio.create_task(watch_event_loop(lag), name="event-loop-probe"))registry.register_observer(_state_readings(probes))log_unreadable_sources()diff --git a/core/switch_core/observability/catalogue.py b/core/switch_core/observability/catalogue.pyindex b69462372..08aa3f2ed 100644--- a/core/switch_core/observability/catalogue.py+++ b/core/switch_core/observability/catalogue.py@@ -551,9 +551,20 @@ RUNTIME_EVENT_LOOP_LAG = _spec("switch.runtime.event_loop_lag","gauge","ms",- "How far past its deadline a fixed-interval task woke. The server is "- "single-threaded, so this is the one number that says whether anything is "- "being starved.",+ "The longest the event loop was blocked since the last reading, measured "+ "by a probe that wakes every 100 ms. The server is single-threaded, so "+ "this is the one number that says whether anything is being starved.",+)+# Every stall the probe sees, not only the worst per interval: how often the+# loop blocks and for how long. Below the threshold is ordinary scheduling+# jitter and would only drown the stalls that matter.+RUNTIME_EVENT_LOOP_STALLS = _spec(+ "switch.runtime.event_loop_stalls",+ "histogram",+ "ms",+ "Event-loop stalls of 50 ms or more, by how long the loop was blocked. "+ "The count says how often, the buckets say how bad.",+ bounds=(50.0, 100.0, 200.0, 300.0, 500.0, 750.0, 1000.0, 2000.0, 5000.0),)# One per process start, so a restart lines up against everything else on a# dashboard. The version is already on every series' resource.@@ -643,6 +654,7 @@ CATALOGUE: dict[str, MetricSpec] = {RUNTIME_CPU_SECONDS,RUNTIME_OPEN_FDS,RUNTIME_EVENT_LOOP_LAG,+ RUNTIME_EVENT_LOOP_STALLS,RUNTIME_STARTS,RUNTIME_GC_COLLECTIONS,RUNTIME_GC_PAUSE,diff --git a/core/switch_core/observability/runtime.py b/core/switch_core/observability/runtime.pyindex ce411401e..aeebb6aa1 100644--- a/core/switch_core/observability/runtime.py+++ b/core/switch_core/observability/runtime.py@@ -13,6 +13,7 @@ fabricated memory figure on a developer's laptop is worse than a missing one.from __future__ import annotations+import asyncioimport gcimport loggingimport os@@ -26,6 +27,7 @@ from collections.abc import Iteratorfrom switch_core.observability.catalogue import (RUNTIME_CPU_SECONDS,RUNTIME_EVENT_LOOP_LAG,+ RUNTIME_EVENT_LOOP_STALLS,RUNTIME_GC_COLLECTIONS,RUNTIME_GC_PAUSE,RUNTIME_MEMORY_RSS,@@ -117,6 +119,35 @@ class GcPauses:return drained+# How often the probe wakes. A stall is seen only while it overlaps a wake-up,+# so the interval bounds how much of a stall the reading can miss: at 100 ms a+# 300 ms freeze is always caught, and read to within 100 ms. The cost is ten+# timer callbacks a second.+LOOP_PROBE_INTERVAL_SECONDS = 0.1++# Oversleep below this is scheduling jitter, not a stall.+LOOP_STALL_THRESHOLD_SECONDS = 0.05+++async def watch_event_loop(+ lag: EventLoopLag, interval_seconds: float = LOOP_PROBE_INTERVAL_SECONDS+) -> None:+ """Sleep in short steps and record how late each wake-up was.++ Lateness is time the loop could not run anything, which is the thing to+ measure. A long sleep would only see the part of a stall that overlaps its+ own deadline: a 420 ms freeze during a 2 s sleep is usually missed and, when+ seen, read as anything from 0 to 420 ms.+ """+ while True:+ started = time.monotonic()+ await asyncio.sleep(interval_seconds)+ overslept = (time.monotonic() - started) - interval_seconds+ lag.record(overslept)+ if overslept >= LOOP_STALL_THRESHOLD_SECONDS:+ metrics().observe(RUNTIME_EVENT_LOOP_STALLS, {}, overslept * 1000.0)++def _resident_bytes() -> int | None:"""Current resident set size, or None where it cannot be read.diff --git a/core/tests/switch_core/observability/test_runtime.py b/core/tests/switch_core/observability/test_runtime.pynew file mode 100644index 000000000..835713ec9--- /dev/null+++ b/core/tests/switch_core/observability/test_runtime.py@@ -0,0 +1,58 @@+import asyncio+import contextlib+import time+from collections.abc import Iterator++import pytest++from switch_core.observability.catalogue import RUNTIME_EVENT_LOOP_STALLS+from switch_core.observability.metrics import MetricsRegistry, install, uninstall+from switch_core.observability.runtime import EventLoopLag, watch_event_loop+++def registry() -> Iterator[MetricsRegistry]:+ registry = MetricsRegistry()+ install(registry)+ yield registry+ uninstall()+++async def _probe_while(lag: EventLoopLag, block_seconds: float) -> None:+ probe = asyncio.create_task(watch_event_loop(lag, interval_seconds=0.02))+ # Let the probe start its first sleep, then hold the loop the way a+ # synchronous call or a garbage collection would.+ await asyncio.sleep(0.05)+ time.sleep(block_seconds)+ await asyncio.sleep(0.05)+ probe.cancel()+ with contextlib.suppress(asyncio.CancelledError):+ await probe+++async def test_a_stall_is_read_at_its_full_length(registry: MetricsRegistry):+ lag = EventLoopLag()+ await _probe_while(lag, block_seconds=0.3)++ # The probe's own interval is the most it can under-read by.+ assert lag.take() >= 300 - 20+++async def test_a_stall_is_counted_with_its_duration(registry: MetricsRegistry):+ lag = EventLoopLag()+ await _probe_while(lag, block_seconds=0.3)++ stalls = {payload.name: payload for payload in registry.collect()}[+ RUNTIME_EVENT_LOOP_STALLS.name+ ]+ point = stalls.histograms[0]+ assert point.count == 1+ assert point.total >= 300 - 20+++async def test_scheduling_jitter_is_not_a_stall(registry: MetricsRegistry):+ lag = EventLoopLag()+ await _probe_while(lag, block_seconds=0.0)++ names = {payload.name for payload in registry.collect()}+ assert RUNTIME_EVENT_LOOP_STALLS.name not in namesdiff --git a/deploy/observability/dashboard.json b/deploy/observability/dashboard.jsonindex 8cbd9989b..1316b538f 100644--- a/deploy/observability/dashboard.json+++ b/deploy/observability/dashboard.json@@ -467,12 +467,17 @@{"definition": {"type": "timeseries",- "title": "Event-loop lag (ms)",- "show_legend": false,+ "title": "Event-loop lag (ms) and stalls of 50 ms or more",+ "show_legend": true,"requests": [{"q": "max:switch.runtime.event_loop_lag{$env,$service}","display_type": "bars"+ },+ {+ "q": "count:switch.runtime.event_loop_stalls{$env,$service}.as_count()",+ "display_type": "line",+ "on_right_yaxis": true}]},diff --git a/docs/old/observability.md b/docs/old/observability.mdindex 65702a9f5..bbcfc0b24 100644--- a/docs/old/observability.md+++ b/docs/old/observability.md@@ -113,7 +113,10 @@ readiness never gated on it.A steady `switch.runtime.event_loop_lag` of one or two milliseconds is thefloor, not a stall: it is how far past its deadline a timer normally wakes.-The alert on it fires at a thousand times that.+The alert on it fires at a thousand times that. The reading comes from a probe+that wakes every 100 ms, so a stall longer than that is always caught and read+to within 100 ms. `switch.runtime.event_loop_stalls` counts each stall of 50 ms+or more by duration, which says how often the loop blocks, not only how badly.**What this does not exercise.** A bare server has no rooms, no agents and nobridges, so nine of the twenty-five metrics never appear — everything under