Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
feat(observability): peak db.pool.in_use + count pool checkout timeouts (#560)
Louis Amaudruzc454c94ee7be
diff --git a/core/switch_core/main.py b/core/switch_core/main.pyindex 401f6749e..f9a7785f3 100644--- a/core/switch_core/main.py+++ b/core/switch_core/main.py@@ -133,7 +133,7 @@ from switch_core.observability.bootstrap import (RuntimeProbes,start_observability,)-from switch_core.observability.pool import pool_stats+from switch_core.observability.pool import install_pool_watermark, pool_statsfrom switch_core.observability.query import instrument_queriesfrom switch_core.observability.runtime import EventLoopLagfrom switch_core.provisioning import Provisioning@@ -347,6 +347,7 @@ async def run(config: SwitchConfig) -> None:# reports nothing pays nothing worth counting, and turning export on does# not change how queries execute.instrument_queries(engine)+ pool_watermark = install_pool_watermark(engine)# Its connection is held rather than borrowed, so it builds its own outside# the pool. Nothing subscribes yet; it starts with the server so that the@@ -713,7 +714,7 @@ async def run(config: SwitchConfig) -> None:connectors_running=connector_lifecycle.running_count,connectors_configured=connector_lifecycle.expected_count,agents_connected=lambda: len(connections.live_agent_ids()),- pool_stats=lambda: pool_stats(engine),+ pool_stats=lambda: pool_stats(engine, pool_watermark),)# ── Lifespan: start server-side connectors once HTTP is serving ────────diff --git a/core/switch_core/observability/catalogue.py b/core/switch_core/observability/catalogue.pyindex 55bcd67b5..e3470f493 100644--- a/core/switch_core/observability/catalogue.py+++ b/core/switch_core/observability/catalogue.py@@ -92,7 +92,10 @@ DB_POOL_IN_USE = _spec("switch.db.pool.in_use","gauge","{connection}",- "Connections checked out of the pool right now.",+ "The most connections checked out of the pool at once since the last "+ "reading — a peak, not a snapshot. Exhaustion here is a burst that fills and "+ "drains the pool inside a few hundred milliseconds, which a value sampled "+ "once a minute would step straight over.",)DB_POOL_SIZE = _spec("switch.db.pool.size",@@ -106,6 +109,21 @@ DB_POOL_OVERFLOW = _spec("{connection}","Connections open beyond the pool's nominal size.",)+# The pool gauges say how full the pool got; this says when full stopped being+# enough. The pool queues rather than sheds when saturated, so a request that+# waited out `db_pool_timeout` without a connection is the failure a peak at the+# ceiling only implies. Counted on the request path, which is where every+# observed timeout has originated — auth resolves a token against the database+# before the handler runs, so a saturated pool surfaces there first. A+# background loop that times out is logged, not counted here.+DB_POOL_TIMEOUTS = _spec(+ "switch.db.pool.timeouts",+ "sum",+ "{timeout}",+ "Requests that gave up waiting for a pooled connection. Loud where a pool "+ "at its ceiling is only suggestive: this is a request that got no connection "+ "at all within the timeout.",+)# The pool gauges say whether connections are scarce; this says whether the# database is slow, which is the other half and the one a slow request is# usually about. `operation` is the statement's leading keyword mapped through@@ -311,6 +329,7 @@ CATALOGUE: dict[str, MetricSpec] = {DB_POOL_IN_USE,DB_POOL_SIZE,DB_POOL_OVERFLOW,+ DB_POOL_TIMEOUTS,DB_QUERY_DURATION,MESSAGES_SENT,MESSAGES_DELIVERED,diff --git a/core/switch_core/observability/http.py b/core/switch_core/observability/http.pyindex 8f6c81f9c..f071efc05 100644--- a/core/switch_core/observability/http.py+++ b/core/switch_core/observability/http.py@@ -11,9 +11,14 @@ from __future__ import annotationsimport time+from sqlalchemy.exc import TimeoutError as PoolCheckoutTimeoutfrom starlette.types import ASGIApp, Message, Receive, Scope, Send-from switch_core.observability.catalogue import HTTP_REQUEST_DURATION, HTTP_REQUESTS+from switch_core.observability.catalogue import (+ DB_POOL_TIMEOUTS,+ HTTP_REQUEST_DURATION,+ HTTP_REQUESTS,+)from switch_core.observability.metrics import metrics# Every request the router could not place. The path is discarded: on a public@@ -99,6 +104,14 @@ class MetricsMiddleware:await self.app(scope, receive, send_wrapper)if seen_status:outcome = status_class(seen_status[0])+ except PoolCheckoutTimeout:+ # The pool queued this request and no connection came free within+ # the timeout. It surfaces as a 5xx like anything else, but the+ # cause is scarcity rather than a handler fault, so it is counted on+ # its own — the signal a peak at the pool ceiling only implies.+ outcome = "5xx"+ registry.increment(DB_POOL_TIMEOUTS, {})+ raiseexcept Exception:outcome = "5xx"raisediff --git a/core/switch_core/observability/pool.py b/core/switch_core/observability/pool.pyindex 9f6c76952..537c075ce 100644--- a/core/switch_core/observability/pool.py+++ b/core/switch_core/observability/pool.py@@ -12,11 +12,63 @@ absent one.from __future__ import annotations+import threadingfrom dataclasses import dataclass+from sqlalchemy import eventfrom sqlalchemy.ext.asyncio import AsyncEngine+class PoolInUseWatermark:+ """The most connections checked out at once since the last reading.++ Read point-in-time on the export interval, ``in_use`` misses the failure it+ exists to show: a reconnect burst fills the pool and drains it inside a few+ hundred milliseconds — between two one-minute samples — so the panel stays+ flat through an exhaustion. Fed from the pool's ``checkout`` event, which+ fires on every rise, the peak is caught however brief it was. Reading it+ resets it, so each interval reports its own worst case rather than the worst+ ever seen — the same shape as :class:`~switch_core.observability.runtime.EventLoopLag`.+ """++ def __init__(self) -> None:+ self._lock = threading.Lock()+ self._peak = 0++ def record(self, in_use: int) -> None:+ with self._lock:+ self._peak = max(self._peak, in_use)++ def take(self, current: int) -> int:+ # Reset to what is checked out right now, not to zero: the next+ # interval's floor is the pool as it actually stands, and a pool that+ # sits at thirty between bursts should not read as empty for the minute+ # after each collection.+ with self._lock:+ peak = max(self._peak, current)+ self._peak = current+ return peak+++def install_pool_watermark(engine: AsyncEngine) -> PoolInUseWatermark:+ """Track the pool's high-water checkout count off its ``checkout`` event.++ Registered on the engine rather than sampled, so a spike that lives and dies+ between two exporter reads is still seen. Harmless on a pool that never+ fires the event: the watermark stays at whatever ``take`` last set it to,+ and :func:`pool_stats` still returns ``None`` for a pool that cannot count.+ """+ watermark = PoolInUseWatermark()++ @event.listens_for(engine.sync_engine, "checkout")+ def _record_checkout(*_: object) -> None:+ checkedout = getattr(engine.pool, "checkedout", None)+ if checkedout is not None:+ watermark.record(checkedout())++ return watermark++@dataclass(frozen=True)class PoolStats:in_use: int@@ -24,11 +76,12 @@ class PoolStats:overflow: int-def pool_stats(engine: AsyncEngine) -> PoolStats | None:+def pool_stats(engine: AsyncEngine, watermark: PoolInUseWatermark) -> PoolStats | None:pool = engine.pooltry:+ current = pool.checkedout() # type: ignore[attr-defined]return PoolStats(- in_use=pool.checkedout(), # type: ignore[attr-defined]+ in_use=watermark.take(current),size=pool.size(), # type: ignore[attr-defined]overflow=_overflow_beyond_nominal(pool.overflow()), # type: ignore[attr-defined])diff --git a/core/tests/switch_core/observability/test_http_metrics.py b/core/tests/switch_core/observability/test_http_metrics.pyindex f1ce9b77c..d7521c848 100644--- a/core/tests/switch_core/observability/test_http_metrics.py+++ b/core/tests/switch_core/observability/test_http_metrics.py@@ -1,10 +1,16 @@import pytestfrom fastapi import FastAPI+from sqlalchemy.exc import TimeoutError as PoolCheckoutTimeoutfrom starlette.testclient import TestClient-from switch_core.observability.catalogue import HTTP_REQUEST_DURATION, HTTP_REQUESTS+from switch_core.observability.catalogue import (+ DB_POOL_TIMEOUTS,+ HTTP_REQUEST_DURATION,+ HTTP_REQUESTS,+)from switch_core.observability.http import MetricsMiddlewarefrom switch_core.observability.metrics import MetricsRegistry, install, uninstall+from switch_core.observability.otlp import MetricPayload@pytest.fixture@@ -27,6 +33,12 @@ def _app() -> FastAPI:async def boom() -> None:raise RuntimeError("boom")+ @app.get("/pool-timeout")+ async def pool_timeout() -> None:+ # What a saturated pool raises when auth (or a handler) asks for a+ # connection and none comes free within `db_pool_timeout`.+ raise PoolCheckoutTimeout("QueuePool limit reached")+@inner.get("/rooms")async def gateway_rooms() -> list[str]:return []@@ -36,8 +48,8 @@ def _app() -> FastAPI:return app-def _counts(registry: MetricsRegistry) -> dict[tuple, float]:- payload = next(p for p in registry.collect() if p.name == HTTP_REQUESTS.name)+def _counts(payloads: list[MetricPayload]) -> dict[tuple, float]:+ payload = next(p for p in payloads if p.name == HTTP_REQUESTS.name)return {tuple(sorted(point.attributes.items())): point.valuefor point in payload.numbers@@ -49,7 +61,7 @@ def test_a_path_parameter_does_not_become_a_series(registry):for room_id in ("a", "b", "c"):client.get(f"/rooms/{room_id}")- counts = _counts(registry)+ counts = _counts(registry.collect())# Three requests, one series: keyed by the template, not the id.assert len(counts) == 1assert next(iter(counts.values())) == 3.0@@ -65,7 +77,7 @@ def test_an_unmatched_path_is_folded_into_one_bucket(registry):for index in range(5):client.get(f"/nope/{index}")- counts = _counts(registry)+ counts = _counts(registry.collect())# Otherwise an unauthenticated 404 loop mints a series per request.assert len(counts) == 1attributes = dict(next(iter(counts)))@@ -77,7 +89,7 @@ def test_a_mounted_route_keeps_its_prefix(registry):with TestClient(_app()) as client:client.get("/gateway/rooms")- routes = {dict(key)["route"] for key in _counts(registry)}+ routes = {dict(key)["route"] for key in _counts(registry.collect())}# The gateway's /rooms and the bridge's /rooms are different routes and# must not be counted as one.assert routes == {"/gateway/rooms"}@@ -87,12 +99,42 @@ def test_an_unhandled_exception_is_still_counted(registry):client = TestClient(_app(), raise_server_exceptions=False)client.get("/boom")- counts = _counts(registry)+ counts = _counts(registry.collect())attributes = dict(next(iter(counts)))assert attributes["route"] == "/boom"assert attributes["status_class"] == "5xx"+def _timeout_count(payloads: list[MetricPayload]) -> float:+ matching = [p for p in payloads if p.name == DB_POOL_TIMEOUTS.name]+ if not matching:+ return 0.0+ return sum(point.value for point in matching[0].numbers)+++def test_a_pool_checkout_timeout_is_counted(registry):+ client = TestClient(_app(), raise_server_exceptions=False)+ client.get("/pool-timeout")++ # Counted as a 5xx like any failure, and separately as the pool timeout it+ # was — the signal a peak at the pool ceiling only implies. Both readings+ # come from one collection: `collect()` drains and resets, so a second call+ # would find the counters already taken.+ payloads = registry.collect()+ counts = _counts(payloads)+ attributes = dict(next(iter(counts)))+ assert attributes["route"] == "/pool-timeout"+ assert attributes["status_class"] == "5xx"+ assert _timeout_count(payloads) == 1.0+++def test_an_ordinary_error_is_not_counted_as_a_pool_timeout(registry):+ client = TestClient(_app(), raise_server_exceptions=False)+ client.get("/boom")++ assert _timeout_count(registry.collect()) == 0.0++def test_duration_is_recorded_per_route(registry):with TestClient(_app()) as client:client.get("/rooms/a")@@ -133,14 +175,18 @@ def test_an_inner_route_sharing_the_mounts_name_keeps_its_prefix(registry):with TestClient(app) as client:client.get("/gateway/gatewayish")- assert {dict(key)["route"] for key in _counts(registry)} == {"/gateway/gatewayish"}+ assert {dict(key)["route"] for key in _counts(registry.collect())} == {+ "/gateway/gatewayish"+ }def test_an_unmounted_route_is_not_given_a_prefix(registry):with TestClient(_app()) as client:client.get("/rooms/a")- assert {dict(key)["route"] for key in _counts(registry)} == {"/rooms/{room_id}"}+ assert {dict(key)["route"] for key in _counts(registry.collect())} == {+ "/rooms/{room_id}"+ }def test_a_long_poll_is_counted_but_not_timed(registry):@@ -222,7 +268,7 @@ def test_mcp_traffic_is_labelled_rather_than_unmatched(registry):with TestClient(app) as client:client.post("/mcp/")- counted = {dict(key)["route"] for key in _counts(registry)}+ counted = {dict(key)["route"] for key in _counts(registry.collect())}assert counted == {"/mcp"}diff --git a/core/tests/switch_core/observability/test_pool.py b/core/tests/switch_core/observability/test_pool.pyindex 7ff256107..25a2ac36b 100644--- a/core/tests/switch_core/observability/test_pool.py+++ b/core/tests/switch_core/observability/test_pool.py@@ -4,7 +4,11 @@ from sqlalchemy import create_enginefrom sqlalchemy.ext.asyncio import create_async_enginefrom sqlalchemy.pool import NullPool, QueuePool-from switch_core.observability.pool import _overflow_beyond_nominal, pool_stats+from switch_core.observability.pool import (+ PoolInUseWatermark,+ _overflow_beyond_nominal,+ pool_stats,+)def test_overflow_is_zero_on_an_idle_pool():@@ -34,7 +38,7 @@ def test_a_queue_pool_reports_its_numbers():engine = create_async_engine("postgresql+asyncpg://u:p@localhost/db", pool_size=5, max_overflow=2)- stats = pool_stats(engine)+ stats = pool_stats(engine, PoolInUseWatermark())assert stats is not Noneassert stats.size == 5@@ -47,4 +51,34 @@ def test_a_pool_with_nothing_to_report_reports_nothing():engine = create_async_engine("postgresql+asyncpg://u:p@localhost/db", poolclass=NullPool)- assert pool_stats(engine) is None+ assert pool_stats(engine, PoolInUseWatermark()) is None+++def test_in_use_reports_the_peak_since_the_last_reading():+ """A burst that fills and drains the pool between reads must still show.++ `in_use` is the high-water mark, not the level at collection time — the two+ diverge exactly when it matters, on a spike that is gone by the next sample.+ """+ engine = create_async_engine(+ "postgresql+asyncpg://u:p@localhost/db", pool_size=5, max_overflow=2+ )+ watermark = PoolInUseWatermark()+ # A burst peaked at seven while nothing is checked out now (checkedout()==0).+ watermark.record(7)++ stats = pool_stats(engine, watermark)+ assert stats is not None+ assert stats.in_use == 7+++def test_the_watermark_resets_to_the_current_level_each_reading():+ """Each interval reports its own worst case, floored at the live level."""+ watermark = PoolInUseWatermark()+ watermark.record(9)+ # First read sees the peak, then resets to what is checked out right now (3).+ assert watermark.take(3) == 9+ # The spike is spent; a quiet interval reads the live level, not a stale 9.+ assert watermark.take(3) == 3+ # And never drops below what is actually checked out.+ assert watermark.take(6) == 6diff --git a/deploy/observability/dashboard.json b/deploy/observability/dashboard.jsonindex 5bd78d1b5..b1f93633c 100644--- a/deploy/observability/dashboard.json+++ b/deploy/observability/dashboard.json@@ -433,11 +433,11 @@{"definition": {"type": "timeseries",- "title": "Database pool: in use vs size",+ "title": "Database pool: peak in use vs size","show_legend": true,"requests": [{- "q": "avg:switch.db.pool.in_use{$env,$service}",+ "q": "max:switch.db.pool.in_use{$env,$service}","display_type": "area"},{@@ -695,6 +695,25 @@"width": 12,"height": 4}+ },+ {+ "definition": {+ "type": "timeseries",+ "title": "Database pool: checkout timeouts",+ "show_legend": false,+ "requests": [+ {+ "q": "sum:switch.db.pool.timeouts{$env,$service}.as_count()",+ "display_type": "bars"+ }+ ]+ },+ "layout": {+ "x": 0,+ "y": 63,+ "width": 12,+ "height": 4+ }}]}diff --git a/deploy/observability/monitors.json b/deploy/observability/monitors.jsonindex 5540d1308..86c172cab 100644--- a/deploy/observability/monitors.json+++ b/deploy/observability/monitors.json@@ -278,5 +278,23 @@},"include_tags": true}+ },+ {+ "name": "[Switch] the database pool is refusing connections",+ "type": "metric alert",+ "query": "sum(last_5m):sum:switch.db.pool.timeouts{service:switch-core}.as_count() > 0",+ "message": "{{value}} request(s) got no pooled connection within the timeout in five minutes.\n\nThis is the pool past nearly-exhausted and into failing: it queues rather than sheds when full, so a timeout means a request waited out `db_pool_timeout` and still got nothing. Because auth resolves a token against the database before the handler runs, the first thing to fail is authentication on otherwise-fine requests — including agent heartbeats, which then look lapsed and reconnect, which loads the pool further.\n\nLook for a connection held across non-database work (an external call inside an open session) or a synchronized reconnect burst. Raising the pool size is a cushion, not the cause.\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",+ "tags": [+ "service:switch-core",+ "team:switch"+ ],+ "options": {+ "notify_no_data": false,+ "renotify_interval": 60,+ "thresholds": {+ "critical": 0+ },+ "include_tags": true+ }}]