Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Report how long garbage collection stops the process, by generation (#711)
Abel Dantas02bab127aa3f
diff --git a/core/switch_core/observability/bootstrap.py b/core/switch_core/observability/bootstrap.pyindex 8507ed39f..0f0274a2d 100644--- a/core/switch_core/observability/bootstrap.py+++ b/core/switch_core/observability/bootstrap.py@@ -56,6 +56,7 @@ from switch_core.observability.otlp import OtlpClient, OtlpResourcefrom switch_core.observability.pool import PoolStatsfrom switch_core.observability.runtime import (EventLoopLag,+ GcPauses,RuntimeMetrics,log_unreadable_sources,)@@ -107,12 +108,15 @@ class Observability:_tasks: list[asyncio.Task[None]]_http_client: httpx.AsyncClient | None_log_handler: OtlpLogHandler | None+ _gc_pauses: GcPauses | Noneasync def aclose(self) -> None:# Detached first, so shutdown logging is not queued for an exporter# about to stop draining it.if self._log_handler is not None:logging.getLogger().removeHandler(self._log_handler)+ if self._gc_pauses is not None:+ self._gc_pauses.uninstall()for task in self._tasks:task.cancel()for task in self._tasks:@@ -202,6 +206,7 @@ def start_observability(_tasks=tasks,_http_client=None,_log_handler=None,+ _gc_pauses=None,)registry = MetricsRegistry()@@ -209,7 +214,9 @@ def start_observability(registry.increment(RUNTIME_STARTS, {})monitor.install(registry)- RuntimeMetrics(lag).install(registry)+ gc_pauses = GcPauses()+ gc_pauses.install()+ RuntimeMetrics(lag, gc_pauses).install(registry)registry.register_observer(_state_readings(probes))log_unreadable_sources()@@ -288,4 +295,5 @@ def start_observability(_tasks=tasks,_http_client=http_client,_log_handler=log_handler,+ _gc_pauses=gc_pauses,)diff --git a/core/switch_core/observability/catalogue.py b/core/switch_core/observability/catalogue.pyindex 48941ec03..6230b0ffe 100644--- a/core/switch_core/observability/catalogue.py+++ b/core/switch_core/observability/catalogue.py@@ -508,6 +508,19 @@ RUNTIME_GC_COLLECTIONS = _spec("Garbage collections, by generation.","generation",)+# Measured from inside the collector, so unlike the event-loop lag above it+# catches every pause, not only one that happens to overlap the sweep timer's+# wake. A full collection walks every tracked object with the interpreter lock+# held: on a large heap that freezes the event loop as surely as blocking code.+RUNTIME_GC_PAUSE = _spec(+ "switch.runtime.gc_pause",+ "histogram",+ "ms",+ "How long each garbage collection stopped the process, by generation. "+ "Generation 2 is the full collection, the one that grows with the heap.",+ "generation",+ bounds=(0.1, 0.5, 1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0),+)# ── Health ───────────────────────────────────────────────────────────────────# One series per dependency, so a dashboard shows which one broke.@@ -564,6 +577,7 @@ CATALOGUE: dict[str, MetricSpec] = {RUNTIME_EVENT_LOOP_LAG,RUNTIME_STARTS,RUNTIME_GC_COLLECTIONS,+ RUNTIME_GC_PAUSE,HEALTH_CHECK,)}diff --git a/core/switch_core/observability/runtime.py b/core/switch_core/observability/runtime.pyindex fa9753696..ce411401e 100644--- a/core/switch_core/observability/runtime.py+++ b/core/switch_core/observability/runtime.py@@ -19,12 +19,15 @@ import osimport resourceimport sysimport threading+import time+from collections import dequefrom collections.abc import Iteratorfrom switch_core.observability.catalogue import (RUNTIME_CPU_SECONDS,RUNTIME_EVENT_LOOP_LAG,RUNTIME_GC_COLLECTIONS,+ RUNTIME_GC_PAUSE,RUNTIME_MEMORY_RSS,RUNTIME_OPEN_FDS,)@@ -35,6 +38,12 @@ logger = logging.getLogger(__name__)_PROC_STATM = "/proc/self/statm"_PROC_FD = "/proc/self/fd"+# Pauses waiting to be drained into the registry. Young collections can run+# many times a second under load, so the buffer is bounded; past this many in+# one export interval the oldest are dropped, which costs resolution on the+# cheap pauses and never hides the rare long one that arrives last.+_MAX_PENDING_PAUSES = 50_000+class EventLoopLag:"""The worst oversleep seen since the last reading.@@ -68,6 +77,46 @@ class EventLoopLag:return worst+class GcPauses:+ """How long each garbage collection stopped the process.++ Fed from ``gc.callbacks``, which the collector calls on whatever thread+ triggered the collection. Nothing in that callback may take a lock: a+ collection can start while the same thread holds the metrics registry's+ lock, and waiting on it there would deadlock. So the callback only appends+ to a deque, which needs no lock, and :class:`RuntimeMetrics` drains it into+ the registry when it collects.+ """++ def __init__(self) -> None:+ self._started = 0.0+ self._pauses: deque[tuple[int, float]] = deque(maxlen=_MAX_PENDING_PAUSES)++ def install(self) -> None:+ gc.callbacks.append(self._on_collection)++ def uninstall(self) -> None:+ if self._on_collection in gc.callbacks:+ gc.callbacks.remove(self._on_collection)++ def _on_collection(self, phase: str, info: dict[str, int]) -> None:+ # Collections never overlap: the collector refuses to start while one+ # is running, so a single start time is enough.+ if phase == "start":+ self._started = time.perf_counter()+ return+ pause_ms = (time.perf_counter() - self._started) * 1000.0+ self._pauses.append((info["generation"], pause_ms))++ def drain(self) -> list[tuple[int, float]]:+ drained: list[tuple[int, float]] = []+ while True:+ try:+ drained.append(self._pauses.popleft())+ except IndexError:+ return drained++def _resident_bytes() -> int | None:"""Current resident set size, or None where it cannot be read.@@ -105,8 +154,9 @@ class RuntimeMetrics:delta through a pre-collect hook. The rest are read at collection time."""- def __init__(self, lag: EventLoopLag) -> None:+ def __init__(self, lag: EventLoopLag, gc_pauses: GcPauses) -> None:self._lag = lag+ self._gc_pauses = gc_pausesusage = resource.getrusage(resource.RUSAGE_SELF)self._last_user = usage.ru_utimeself._last_system = usage.ru_stime@@ -148,6 +198,11 @@ class RuntimeMetrics:)self._last_collections = counts+ for generation, pause_ms in self._gc_pauses.drain():+ registry.observe(+ RUNTIME_GC_PAUSE, {"generation": str(generation)}, pause_ms+ )+def _readings(self) -> Iterator[GaugeReading]:resident = _resident_bytes()if resident is not None:diff --git a/core/tests/switch_core/observability/test_gc_pauses.py b/core/tests/switch_core/observability/test_gc_pauses.pynew file mode 100644index 000000000..cc7836e04--- /dev/null+++ b/core/tests/switch_core/observability/test_gc_pauses.py@@ -0,0 +1,75 @@+"""How long garbage collection stops the process.++The event-loop lag gauge only sees a stall that overlaps the sweep timer's+wake, so a full collection freezing the server for hundreds of milliseconds+can pass unrecorded. These pauses are measured from inside the collector.+"""++from __future__ import annotations++import gc+from collections.abc import Iterator++import pytest++from switch_core.observability.metrics import MetricsRegistry, install, uninstall+from switch_core.observability.runtime import EventLoopLag, GcPauses, RuntimeMetrics+++def registry() -> Iterator[MetricsRegistry]:+ registry = MetricsRegistry()+ install(registry)+ yield registry+ uninstall()+++def gc_pauses() -> Iterator[GcPauses]:+ pauses = GcPauses()+ pauses.install()+ yield pauses+ pauses.uninstall()+++def _pause_counts(registry: MetricsRegistry) -> dict[str, int]:+ payload = next(+ (p for p in registry.collect() if p.name == "switch.runtime.gc_pause"), None+ )+ if payload is None:+ return {}+ return {+ str(point.attributes["generation"]): point.count for point in payload.histograms+ }+++def test_each_collection_is_timed_under_its_generation(gc_pauses: GcPauses) -> None:+ gc_pauses.drain()++ gc.collect(2)++ pauses = gc_pauses.drain()+ full = [pause_ms for generation, pause_ms in pauses if generation == 2]+ assert len(full) == 1+ assert full[0] >= 0.0+ assert gc_pauses.drain() == []+++def test_uninstalling_stops_the_timing(gc_pauses: GcPauses) -> None:+ gc_pauses.uninstall()+ gc_pauses.drain()++ gc.collect(2)++ assert gc_pauses.drain() == []+++def test_pauses_reach_the_registry_on_collection(+ registry: MetricsRegistry, gc_pauses: GcPauses+) -> None:+ RuntimeMetrics(EventLoopLag(), gc_pauses).install(registry)+ registry.collect()++ gc.collect(2)++ assert _pause_counts(registry).get("2") == 1diff --git a/deploy/observability/README.md b/deploy/observability/README.mdindex 4d6adf18e..79c09f0be 100644--- a/deploy/observability/README.md+++ b/deploy/observability/README.md@@ -46,8 +46,9 @@ done**Histogram panels need two things, not one.** Request latency, delivery lag,database query duration, pool hold and wait time (`switch.db.pool.hold.duration`,-`switch.db.pool.wait.duration`), platform call duration and the agent buffer's-scan length and duration are OTLP histograms, and the+`switch.db.pool.wait.duration`), platform call duration, the agent buffer's+scan length and duration, and garbage-collection pauses+(`switch.runtime.gc_pause`) are OTLP histograms, and the`p95:`/`p99:` queries here assume the collector exports them to Datadog as*distributions*. Check the collector'shistogram mode — in the older `histograms` mode they arrive as separate@@ -58,8 +59,8 @@ side rather than the collector's: **percentile aggregations are off by defaulton a distribution metric and are billed separately.** Enable them per metric inMetrics Summary, and add the tag each panel groups by to that metric'sconfigured tag set (`route` for requests, `operation` for database queries and-the agent buffer, `caller` for pool hold time but not wait time, `bridge` and-`platform` for bridge calls), or+the agent buffer, `caller` for pool hold time but not wait time, `generation` for+garbage-collection pauses, `bridge` and `platform` for bridge calls), or`p95: … by {…}` returns nothing on a fresh account. Empty is the honest outcome either way; a panel is never silentlyswitched to a different statistic.diff --git a/deploy/observability/dashboard.json b/deploy/observability/dashboard.jsonindex 2a07d7448..6793ba358 100644--- a/deploy/observability/dashboard.json+++ b/deploy/observability/dashboard.json@@ -491,7 +491,7 @@"layout": {"x": 0,"y": 38,- "width": 4,+ "width": 3,"height": 3}},@@ -508,9 +508,9 @@]},"layout": {- "x": 4,+ "x": 3,"y": 38,- "width": 4,+ "width": 3,"height": 3}},@@ -527,9 +527,32 @@]},"layout": {- "x": 8,+ "x": 6,"y": 38,- "width": 4,+ "width": 3,+ "height": 3+ }+ },+ {+ "definition": {+ "type": "timeseries",+ "title": "Garbage-collection pause p99 / max (ms) by generation. Needs distribution mapping, see README",+ "show_legend": true,+ "requests": [+ {+ "q": "p99:switch.runtime.gc_pause{$env,$service} by {generation}",+ "display_type": "line"+ },+ {+ "q": "max:switch.runtime.gc_pause{$env,$service} by {generation}",+ "display_type": "bars"+ }+ ]+ },+ "layout": {+ "x": 9,+ "y": 38,+ "width": 3,"height": 3}},