Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Report how long garbage collection stops the process, by generation (#711)

Abel Dantas02bab127aa3f
diff --git a/core/switch_core/observability/bootstrap.py b/core/switch_core/observability/bootstrap.py
index 8507ed39f..0f0274a2d 100644
--- a/core/switch_core/observability/bootstrap.py
+++ b/core/switch_core/observability/bootstrap.py
@@ -56,6 +56,7 @@ from switch_core.observability.otlp import OtlpClient, OtlpResource
from switch_core.observability.pool import PoolStats
from switch_core.observability.runtime import (
EventLoopLag,
+ GcPauses,
RuntimeMetrics,
log_unreadable_sources,
)
@@ -107,12 +108,15 @@ class Observability:
_tasks: list[asyncio.Task[None]]
_http_client: httpx.AsyncClient | None
_log_handler: OtlpLogHandler | None
+ _gc_pauses: GcPauses | None
async def aclose(self) -> None:
# Detached first, so shutdown logging is not queued for an exporter
# about to stop draining it.
if self._log_handler is not None:
logging.getLogger().removeHandler(self._log_handler)
+ if self._gc_pauses is not None:
+ self._gc_pauses.uninstall()
for task in self._tasks:
task.cancel()
for task in self._tasks:
@@ -202,6 +206,7 @@ def start_observability(
_tasks=tasks,
_http_client=None,
_log_handler=None,
+ _gc_pauses=None,
)
registry = MetricsRegistry()
@@ -209,7 +214,9 @@ def start_observability(
registry.increment(RUNTIME_STARTS, {})
monitor.install(registry)
- RuntimeMetrics(lag).install(registry)
+ gc_pauses = GcPauses()
+ gc_pauses.install()
+ RuntimeMetrics(lag, gc_pauses).install(registry)
registry.register_observer(_state_readings(probes))
log_unreadable_sources()
@@ -288,4 +295,5 @@ def start_observability(
_tasks=tasks,
_http_client=http_client,
_log_handler=log_handler,
+ _gc_pauses=gc_pauses,
)
diff --git a/core/switch_core/observability/catalogue.py b/core/switch_core/observability/catalogue.py
index 48941ec03..6230b0ffe 100644
--- a/core/switch_core/observability/catalogue.py
+++ b/core/switch_core/observability/catalogue.py
@@ -508,6 +508,19 @@ RUNTIME_GC_COLLECTIONS = _spec(
"Garbage collections, by generation.",
"generation",
)
+# Measured from inside the collector, so unlike the event-loop lag above it
+# catches every pause, not only one that happens to overlap the sweep timer's
+# wake. A full collection walks every tracked object with the interpreter lock
+# held: on a large heap that freezes the event loop as surely as blocking code.
+RUNTIME_GC_PAUSE = _spec(
+ "switch.runtime.gc_pause",
+ "histogram",
+ "ms",
+ "How long each garbage collection stopped the process, by generation. "
+ "Generation 2 is the full collection, the one that grows with the heap.",
+ "generation",
+ bounds=(0.1, 0.5, 1.0, 5.0, 10.0, 25.0, 50.0, 100.0, 250.0, 500.0, 1000.0),
+)
# ── Health ───────────────────────────────────────────────────────────────────
# One series per dependency, so a dashboard shows which one broke.
@@ -564,6 +577,7 @@ CATALOGUE: dict[str, MetricSpec] = {
RUNTIME_EVENT_LOOP_LAG,
RUNTIME_STARTS,
RUNTIME_GC_COLLECTIONS,
+ RUNTIME_GC_PAUSE,
HEALTH_CHECK,
)
}
diff --git a/core/switch_core/observability/runtime.py b/core/switch_core/observability/runtime.py
index fa9753696..ce411401e 100644
--- a/core/switch_core/observability/runtime.py
+++ b/core/switch_core/observability/runtime.py
@@ -19,12 +19,15 @@ import os
import resource
import sys
import threading
+import time
+from collections import deque
from collections.abc import Iterator
from switch_core.observability.catalogue import (
RUNTIME_CPU_SECONDS,
RUNTIME_EVENT_LOOP_LAG,
RUNTIME_GC_COLLECTIONS,
+ RUNTIME_GC_PAUSE,
RUNTIME_MEMORY_RSS,
RUNTIME_OPEN_FDS,
)
@@ -35,6 +38,12 @@ logger = logging.getLogger(__name__)
_PROC_STATM = "/proc/self/statm"
_PROC_FD = "/proc/self/fd"
+# Pauses waiting to be drained into the registry. Young collections can run
+# many times a second under load, so the buffer is bounded; past this many in
+# one export interval the oldest are dropped, which costs resolution on the
+# cheap pauses and never hides the rare long one that arrives last.
+_MAX_PENDING_PAUSES = 50_000
+
class EventLoopLag:
"""The worst oversleep seen since the last reading.
@@ -68,6 +77,46 @@ class EventLoopLag:
return worst
+class GcPauses:
+ """How long each garbage collection stopped the process.
+
+ Fed from ``gc.callbacks``, which the collector calls on whatever thread
+ triggered the collection. Nothing in that callback may take a lock: a
+ collection can start while the same thread holds the metrics registry's
+ lock, and waiting on it there would deadlock. So the callback only appends
+ to a deque, which needs no lock, and :class:`RuntimeMetrics` drains it into
+ the registry when it collects.
+ """
+
+ def __init__(self) -> None:
+ self._started = 0.0
+ self._pauses: deque[tuple[int, float]] = deque(maxlen=_MAX_PENDING_PAUSES)
+
+ def install(self) -> None:
+ gc.callbacks.append(self._on_collection)
+
+ def uninstall(self) -> None:
+ if self._on_collection in gc.callbacks:
+ gc.callbacks.remove(self._on_collection)
+
+ def _on_collection(self, phase: str, info: dict[str, int]) -> None:
+ # Collections never overlap: the collector refuses to start while one
+ # is running, so a single start time is enough.
+ if phase == "start":
+ self._started = time.perf_counter()
+ return
+ pause_ms = (time.perf_counter() - self._started) * 1000.0
+ self._pauses.append((info["generation"], pause_ms))
+
+ def drain(self) -> list[tuple[int, float]]:
+ drained: list[tuple[int, float]] = []
+ while True:
+ try:
+ drained.append(self._pauses.popleft())
+ except IndexError:
+ return drained
+
+
def _resident_bytes() -> int | None:
"""Current resident set size, or None where it cannot be read.
@@ -105,8 +154,9 @@ class RuntimeMetrics:
delta through a pre-collect hook. The rest are read at collection time.
"""
- def __init__(self, lag: EventLoopLag) -> None:
+ def __init__(self, lag: EventLoopLag, gc_pauses: GcPauses) -> None:
self._lag = lag
+ self._gc_pauses = gc_pauses
usage = resource.getrusage(resource.RUSAGE_SELF)
self._last_user = usage.ru_utime
self._last_system = usage.ru_stime
@@ -148,6 +198,11 @@ class RuntimeMetrics:
)
self._last_collections = counts
+ for generation, pause_ms in self._gc_pauses.drain():
+ registry.observe(
+ RUNTIME_GC_PAUSE, {"generation": str(generation)}, pause_ms
+ )
+
def _readings(self) -> Iterator[GaugeReading]:
resident = _resident_bytes()
if resident is not None:
diff --git a/core/tests/switch_core/observability/test_gc_pauses.py b/core/tests/switch_core/observability/test_gc_pauses.py
new file mode 100644
index 000000000..cc7836e04
--- /dev/null
+++ b/core/tests/switch_core/observability/test_gc_pauses.py
@@ -0,0 +1,75 @@
+"""How long garbage collection stops the process.
+
+The event-loop lag gauge only sees a stall that overlaps the sweep timer's
+wake, so a full collection freezing the server for hundreds of milliseconds
+can pass unrecorded. These pauses are measured from inside the collector.
+"""
+
+from __future__ import annotations
+
+import gc
+from collections.abc import Iterator
+
+import pytest
+
+from switch_core.observability.metrics import MetricsRegistry, install, uninstall
+from switch_core.observability.runtime import EventLoopLag, GcPauses, RuntimeMetrics
+
+
+def registry() -> Iterator[MetricsRegistry]:
+ registry = MetricsRegistry()
+ install(registry)
+ yield registry
+ uninstall()
+
+
+def gc_pauses() -> Iterator[GcPauses]:
+ pauses = GcPauses()
+ pauses.install()
+ yield pauses
+ pauses.uninstall()
+
+
+def _pause_counts(registry: MetricsRegistry) -> dict[str, int]:
+ payload = next(
+ (p for p in registry.collect() if p.name == "switch.runtime.gc_pause"), None
+ )
+ if payload is None:
+ return {}
+ return {
+ str(point.attributes["generation"]): point.count for point in payload.histograms
+ }
+
+
+def test_each_collection_is_timed_under_its_generation(gc_pauses: GcPauses) -> None:
+ gc_pauses.drain()
+
+ gc.collect(2)
+
+ pauses = gc_pauses.drain()
+ full = [pause_ms for generation, pause_ms in pauses if generation == 2]
+ assert len(full) == 1
+ assert full[0] >= 0.0
+ assert gc_pauses.drain() == []
+
+
+def test_uninstalling_stops_the_timing(gc_pauses: GcPauses) -> None:
+ gc_pauses.uninstall()
+ gc_pauses.drain()
+
+ gc.collect(2)
+
+ assert gc_pauses.drain() == []
+
+
+def test_pauses_reach_the_registry_on_collection(
+ registry: MetricsRegistry, gc_pauses: GcPauses
+) -> None:
+ RuntimeMetrics(EventLoopLag(), gc_pauses).install(registry)
+ registry.collect()
+
+ gc.collect(2)
+
+ assert _pause_counts(registry).get("2") == 1
diff --git a/deploy/observability/README.md b/deploy/observability/README.md
index 4d6adf18e..79c09f0be 100644
--- a/deploy/observability/README.md
+++ b/deploy/observability/README.md
@@ -46,8 +46,9 @@ done
**Histogram panels need two things, not one.** Request latency, delivery lag,
database query duration, pool hold and wait time (`switch.db.pool.hold.duration`,
-`switch.db.pool.wait.duration`), platform call duration and the agent buffer's
-scan length and duration are OTLP histograms, and the
+`switch.db.pool.wait.duration`), platform call duration, the agent buffer's
+scan length and duration, and garbage-collection pauses
+(`switch.runtime.gc_pause`) are OTLP histograms, and the
`p95:`/`p99:` queries here assume the collector exports them to Datadog as
*distributions*. Check the collector's
histogram mode — in the older `histograms` mode they arrive as separate
@@ -58,8 +59,8 @@ side rather than the collector's: **percentile aggregations are off by default
on a distribution metric and are billed separately.** Enable them per metric in
Metrics Summary, and add the tag each panel groups by to that metric's
configured tag set (`route` for requests, `operation` for database queries and
-the agent buffer, `caller` for pool hold time but not wait time, `bridge` and
-`platform` for bridge calls), or
+the agent buffer, `caller` for pool hold time but not wait time, `generation` for
+garbage-collection pauses, `bridge` and `platform` for bridge calls), or
`p95: … by {…}` returns nothing on a fresh account. Empty is the honest outcome either way; a panel is never silently
switched to a different statistic.
diff --git a/deploy/observability/dashboard.json b/deploy/observability/dashboard.json
index 2a07d7448..6793ba358 100644
--- a/deploy/observability/dashboard.json
+++ b/deploy/observability/dashboard.json
@@ -491,7 +491,7 @@
"layout": {
"x": 0,
"y": 38,
- "width": 4,
+ "width": 3,
"height": 3
}
},
@@ -508,9 +508,9 @@
]
},
"layout": {
- "x": 4,
+ "x": 3,
"y": 38,
- "width": 4,
+ "width": 3,
"height": 3
}
},
@@ -527,9 +527,32 @@
]
},
"layout": {
- "x": 8,
+ "x": 6,
"y": 38,
- "width": 4,
+ "width": 3,
+ "height": 3
+ }
+ },
+ {
+ "definition": {
+ "type": "timeseries",
+ "title": "Garbage-collection pause p99 / max (ms) by generation. Needs distribution mapping, see README",
+ "show_legend": true,
+ "requests": [
+ {
+ "q": "p99:switch.runtime.gc_pause{$env,$service} by {generation}",
+ "display_type": "line"
+ },
+ {
+ "q": "max:switch.runtime.gc_pause{$env,$service} by {generation}",
+ "display_type": "bars"
+ }
+ ]
+ },
+ "layout": {
+ "x": 9,
+ "y": 38,
+ "width": 3,
"height": 3
}
},