Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

feat(observability): peak db.pool.in_use + count pool checkout timeouts (#560)

Louis Amaudruzc454c94ee7be
diff --git a/core/switch_core/main.py b/core/switch_core/main.py
index 401f6749e..f9a7785f3 100644
--- a/core/switch_core/main.py
+++ b/core/switch_core/main.py
@@ -133,7 +133,7 @@ from switch_core.observability.bootstrap import (
RuntimeProbes,
start_observability,
)
-from switch_core.observability.pool import pool_stats
+from switch_core.observability.pool import install_pool_watermark, pool_stats
from switch_core.observability.query import instrument_queries
from switch_core.observability.runtime import EventLoopLag
from switch_core.provisioning import Provisioning
@@ -347,6 +347,7 @@ async def run(config: SwitchConfig) -> None:
# reports nothing pays nothing worth counting, and turning export on does
# not change how queries execute.
instrument_queries(engine)
+ pool_watermark = install_pool_watermark(engine)
# Its connection is held rather than borrowed, so it builds its own outside
# the pool. Nothing subscribes yet; it starts with the server so that the
@@ -713,7 +714,7 @@ async def run(config: SwitchConfig) -> None:
connectors_running=connector_lifecycle.running_count,
connectors_configured=connector_lifecycle.expected_count,
agents_connected=lambda: len(connections.live_agent_ids()),
- pool_stats=lambda: pool_stats(engine),
+ pool_stats=lambda: pool_stats(engine, pool_watermark),
)
# ── Lifespan: start server-side connectors once HTTP is serving ────────
diff --git a/core/switch_core/observability/catalogue.py b/core/switch_core/observability/catalogue.py
index 55bcd67b5..e3470f493 100644
--- a/core/switch_core/observability/catalogue.py
+++ b/core/switch_core/observability/catalogue.py
@@ -92,7 +92,10 @@ DB_POOL_IN_USE = _spec(
"switch.db.pool.in_use",
"gauge",
"{connection}",
- "Connections checked out of the pool right now.",
+ "The most connections checked out of the pool at once since the last "
+ "reading — a peak, not a snapshot. Exhaustion here is a burst that fills and "
+ "drains the pool inside a few hundred milliseconds, which a value sampled "
+ "once a minute would step straight over.",
)
DB_POOL_SIZE = _spec(
"switch.db.pool.size",
@@ -106,6 +109,21 @@ DB_POOL_OVERFLOW = _spec(
"{connection}",
"Connections open beyond the pool's nominal size.",
)
+# The pool gauges say how full the pool got; this says when full stopped being
+# enough. The pool queues rather than sheds when saturated, so a request that
+# waited out `db_pool_timeout` without a connection is the failure a peak at the
+# ceiling only implies. Counted on the request path, which is where every
+# observed timeout has originated — auth resolves a token against the database
+# before the handler runs, so a saturated pool surfaces there first. A
+# background loop that times out is logged, not counted here.
+DB_POOL_TIMEOUTS = _spec(
+ "switch.db.pool.timeouts",
+ "sum",
+ "{timeout}",
+ "Requests that gave up waiting for a pooled connection. Loud where a pool "
+ "at its ceiling is only suggestive: this is a request that got no connection "
+ "at all within the timeout.",
+)
# The pool gauges say whether connections are scarce; this says whether the
# database is slow, which is the other half and the one a slow request is
# usually about. `operation` is the statement's leading keyword mapped through
@@ -311,6 +329,7 @@ CATALOGUE: dict[str, MetricSpec] = {
DB_POOL_IN_USE,
DB_POOL_SIZE,
DB_POOL_OVERFLOW,
+ DB_POOL_TIMEOUTS,
DB_QUERY_DURATION,
MESSAGES_SENT,
MESSAGES_DELIVERED,
diff --git a/core/switch_core/observability/http.py b/core/switch_core/observability/http.py
index 8f6c81f9c..f071efc05 100644
--- a/core/switch_core/observability/http.py
+++ b/core/switch_core/observability/http.py
@@ -11,9 +11,14 @@ from __future__ import annotations
import time
+from sqlalchemy.exc import TimeoutError as PoolCheckoutTimeout
from starlette.types import ASGIApp, Message, Receive, Scope, Send
-from switch_core.observability.catalogue import HTTP_REQUEST_DURATION, HTTP_REQUESTS
+from switch_core.observability.catalogue import (
+ DB_POOL_TIMEOUTS,
+ HTTP_REQUEST_DURATION,
+ HTTP_REQUESTS,
+)
from switch_core.observability.metrics import metrics
# Every request the router could not place. The path is discarded: on a public
@@ -99,6 +104,14 @@ class MetricsMiddleware:
await self.app(scope, receive, send_wrapper)
if seen_status:
outcome = status_class(seen_status[0])
+ except PoolCheckoutTimeout:
+ # The pool queued this request and no connection came free within
+ # the timeout. It surfaces as a 5xx like anything else, but the
+ # cause is scarcity rather than a handler fault, so it is counted on
+ # its own — the signal a peak at the pool ceiling only implies.
+ outcome = "5xx"
+ registry.increment(DB_POOL_TIMEOUTS, {})
+ raise
except Exception:
outcome = "5xx"
raise
diff --git a/core/switch_core/observability/pool.py b/core/switch_core/observability/pool.py
index 9f6c76952..537c075ce 100644
--- a/core/switch_core/observability/pool.py
+++ b/core/switch_core/observability/pool.py
@@ -12,11 +12,63 @@ absent one.
from __future__ import annotations
+import threading
from dataclasses import dataclass
+from sqlalchemy import event
from sqlalchemy.ext.asyncio import AsyncEngine
+class PoolInUseWatermark:
+ """The most connections checked out at once since the last reading.
+
+ Read point-in-time on the export interval, ``in_use`` misses the failure it
+ exists to show: a reconnect burst fills the pool and drains it inside a few
+ hundred milliseconds — between two one-minute samples — so the panel stays
+ flat through an exhaustion. Fed from the pool's ``checkout`` event, which
+ fires on every rise, the peak is caught however brief it was. Reading it
+ resets it, so each interval reports its own worst case rather than the worst
+ ever seen — the same shape as :class:`~switch_core.observability.runtime.EventLoopLag`.
+ """
+
+ def __init__(self) -> None:
+ self._lock = threading.Lock()
+ self._peak = 0
+
+ def record(self, in_use: int) -> None:
+ with self._lock:
+ self._peak = max(self._peak, in_use)
+
+ def take(self, current: int) -> int:
+ # Reset to what is checked out right now, not to zero: the next
+ # interval's floor is the pool as it actually stands, and a pool that
+ # sits at thirty between bursts should not read as empty for the minute
+ # after each collection.
+ with self._lock:
+ peak = max(self._peak, current)
+ self._peak = current
+ return peak
+
+
+def install_pool_watermark(engine: AsyncEngine) -> PoolInUseWatermark:
+ """Track the pool's high-water checkout count off its ``checkout`` event.
+
+ Registered on the engine rather than sampled, so a spike that lives and dies
+ between two exporter reads is still seen. Harmless on a pool that never
+ fires the event: the watermark stays at whatever ``take`` last set it to,
+ and :func:`pool_stats` still returns ``None`` for a pool that cannot count.
+ """
+ watermark = PoolInUseWatermark()
+
+ @event.listens_for(engine.sync_engine, "checkout")
+ def _record_checkout(*_: object) -> None:
+ checkedout = getattr(engine.pool, "checkedout", None)
+ if checkedout is not None:
+ watermark.record(checkedout())
+
+ return watermark
+
+
@dataclass(frozen=True)
class PoolStats:
in_use: int
@@ -24,11 +76,12 @@ class PoolStats:
overflow: int
-def pool_stats(engine: AsyncEngine) -> PoolStats | None:
+def pool_stats(engine: AsyncEngine, watermark: PoolInUseWatermark) -> PoolStats | None:
pool = engine.pool
try:
+ current = pool.checkedout() # type: ignore[attr-defined]
return PoolStats(
- in_use=pool.checkedout(), # type: ignore[attr-defined]
+ in_use=watermark.take(current),
size=pool.size(), # type: ignore[attr-defined]
overflow=_overflow_beyond_nominal(pool.overflow()), # type: ignore[attr-defined]
)
diff --git a/core/tests/switch_core/observability/test_http_metrics.py b/core/tests/switch_core/observability/test_http_metrics.py
index f1ce9b77c..d7521c848 100644
--- a/core/tests/switch_core/observability/test_http_metrics.py
+++ b/core/tests/switch_core/observability/test_http_metrics.py
@@ -1,10 +1,16 @@
import pytest
from fastapi import FastAPI
+from sqlalchemy.exc import TimeoutError as PoolCheckoutTimeout
from starlette.testclient import TestClient
-from switch_core.observability.catalogue import HTTP_REQUEST_DURATION, HTTP_REQUESTS
+from switch_core.observability.catalogue import (
+ DB_POOL_TIMEOUTS,
+ HTTP_REQUEST_DURATION,
+ HTTP_REQUESTS,
+)
from switch_core.observability.http import MetricsMiddleware
from switch_core.observability.metrics import MetricsRegistry, install, uninstall
+from switch_core.observability.otlp import MetricPayload
@pytest.fixture
@@ -27,6 +33,12 @@ def _app() -> FastAPI:
async def boom() -> None:
raise RuntimeError("boom")
+ @app.get("/pool-timeout")
+ async def pool_timeout() -> None:
+ # What a saturated pool raises when auth (or a handler) asks for a
+ # connection and none comes free within `db_pool_timeout`.
+ raise PoolCheckoutTimeout("QueuePool limit reached")
+
@inner.get("/rooms")
async def gateway_rooms() -> list[str]:
return []
@@ -36,8 +48,8 @@ def _app() -> FastAPI:
return app
-def _counts(registry: MetricsRegistry) -> dict[tuple, float]:
- payload = next(p for p in registry.collect() if p.name == HTTP_REQUESTS.name)
+def _counts(payloads: list[MetricPayload]) -> dict[tuple, float]:
+ payload = next(p for p in payloads if p.name == HTTP_REQUESTS.name)
return {
tuple(sorted(point.attributes.items())): point.value
for point in payload.numbers
@@ -49,7 +61,7 @@ def test_a_path_parameter_does_not_become_a_series(registry):
for room_id in ("a", "b", "c"):
client.get(f"/rooms/{room_id}")
- counts = _counts(registry)
+ counts = _counts(registry.collect())
# Three requests, one series: keyed by the template, not the id.
assert len(counts) == 1
assert next(iter(counts.values())) == 3.0
@@ -65,7 +77,7 @@ def test_an_unmatched_path_is_folded_into_one_bucket(registry):
for index in range(5):
client.get(f"/nope/{index}")
- counts = _counts(registry)
+ counts = _counts(registry.collect())
# Otherwise an unauthenticated 404 loop mints a series per request.
assert len(counts) == 1
attributes = dict(next(iter(counts)))
@@ -77,7 +89,7 @@ def test_a_mounted_route_keeps_its_prefix(registry):
with TestClient(_app()) as client:
client.get("/gateway/rooms")
- routes = {dict(key)["route"] for key in _counts(registry)}
+ routes = {dict(key)["route"] for key in _counts(registry.collect())}
# The gateway's /rooms and the bridge's /rooms are different routes and
# must not be counted as one.
assert routes == {"/gateway/rooms"}
@@ -87,12 +99,42 @@ def test_an_unhandled_exception_is_still_counted(registry):
client = TestClient(_app(), raise_server_exceptions=False)
client.get("/boom")
- counts = _counts(registry)
+ counts = _counts(registry.collect())
attributes = dict(next(iter(counts)))
assert attributes["route"] == "/boom"
assert attributes["status_class"] == "5xx"
+def _timeout_count(payloads: list[MetricPayload]) -> float:
+ matching = [p for p in payloads if p.name == DB_POOL_TIMEOUTS.name]
+ if not matching:
+ return 0.0
+ return sum(point.value for point in matching[0].numbers)
+
+
+def test_a_pool_checkout_timeout_is_counted(registry):
+ client = TestClient(_app(), raise_server_exceptions=False)
+ client.get("/pool-timeout")
+
+ # Counted as a 5xx like any failure, and separately as the pool timeout it
+ # was — the signal a peak at the pool ceiling only implies. Both readings
+ # come from one collection: `collect()` drains and resets, so a second call
+ # would find the counters already taken.
+ payloads = registry.collect()
+ counts = _counts(payloads)
+ attributes = dict(next(iter(counts)))
+ assert attributes["route"] == "/pool-timeout"
+ assert attributes["status_class"] == "5xx"
+ assert _timeout_count(payloads) == 1.0
+
+
+def test_an_ordinary_error_is_not_counted_as_a_pool_timeout(registry):
+ client = TestClient(_app(), raise_server_exceptions=False)
+ client.get("/boom")
+
+ assert _timeout_count(registry.collect()) == 0.0
+
+
def test_duration_is_recorded_per_route(registry):
with TestClient(_app()) as client:
client.get("/rooms/a")
@@ -133,14 +175,18 @@ def test_an_inner_route_sharing_the_mounts_name_keeps_its_prefix(registry):
with TestClient(app) as client:
client.get("/gateway/gatewayish")
- assert {dict(key)["route"] for key in _counts(registry)} == {"/gateway/gatewayish"}
+ assert {dict(key)["route"] for key in _counts(registry.collect())} == {
+ "/gateway/gatewayish"
+ }
def test_an_unmounted_route_is_not_given_a_prefix(registry):
with TestClient(_app()) as client:
client.get("/rooms/a")
- assert {dict(key)["route"] for key in _counts(registry)} == {"/rooms/{room_id}"}
+ assert {dict(key)["route"] for key in _counts(registry.collect())} == {
+ "/rooms/{room_id}"
+ }
def test_a_long_poll_is_counted_but_not_timed(registry):
@@ -222,7 +268,7 @@ def test_mcp_traffic_is_labelled_rather_than_unmatched(registry):
with TestClient(app) as client:
client.post("/mcp/")
- counted = {dict(key)["route"] for key in _counts(registry)}
+ counted = {dict(key)["route"] for key in _counts(registry.collect())}
assert counted == {"/mcp"}
diff --git a/core/tests/switch_core/observability/test_pool.py b/core/tests/switch_core/observability/test_pool.py
index 7ff256107..25a2ac36b 100644
--- a/core/tests/switch_core/observability/test_pool.py
+++ b/core/tests/switch_core/observability/test_pool.py
@@ -4,7 +4,11 @@ from sqlalchemy import create_engine
from sqlalchemy.ext.asyncio import create_async_engine
from sqlalchemy.pool import NullPool, QueuePool
-from switch_core.observability.pool import _overflow_beyond_nominal, pool_stats
+from switch_core.observability.pool import (
+ PoolInUseWatermark,
+ _overflow_beyond_nominal,
+ pool_stats,
+)
def test_overflow_is_zero_on_an_idle_pool():
@@ -34,7 +38,7 @@ def test_a_queue_pool_reports_its_numbers():
engine = create_async_engine(
"postgresql+asyncpg://u:p@localhost/db", pool_size=5, max_overflow=2
)
- stats = pool_stats(engine)
+ stats = pool_stats(engine, PoolInUseWatermark())
assert stats is not None
assert stats.size == 5
@@ -47,4 +51,34 @@ def test_a_pool_with_nothing_to_report_reports_nothing():
engine = create_async_engine(
"postgresql+asyncpg://u:p@localhost/db", poolclass=NullPool
)
- assert pool_stats(engine) is None
+ assert pool_stats(engine, PoolInUseWatermark()) is None
+
+
+def test_in_use_reports_the_peak_since_the_last_reading():
+ """A burst that fills and drains the pool between reads must still show.
+
+ `in_use` is the high-water mark, not the level at collection time — the two
+ diverge exactly when it matters, on a spike that is gone by the next sample.
+ """
+ engine = create_async_engine(
+ "postgresql+asyncpg://u:p@localhost/db", pool_size=5, max_overflow=2
+ )
+ watermark = PoolInUseWatermark()
+ # A burst peaked at seven while nothing is checked out now (checkedout()==0).
+ watermark.record(7)
+
+ stats = pool_stats(engine, watermark)
+ assert stats is not None
+ assert stats.in_use == 7
+
+
+def test_the_watermark_resets_to_the_current_level_each_reading():
+ """Each interval reports its own worst case, floored at the live level."""
+ watermark = PoolInUseWatermark()
+ watermark.record(9)
+ # First read sees the peak, then resets to what is checked out right now (3).
+ assert watermark.take(3) == 9
+ # The spike is spent; a quiet interval reads the live level, not a stale 9.
+ assert watermark.take(3) == 3
+ # And never drops below what is actually checked out.
+ assert watermark.take(6) == 6
diff --git a/deploy/observability/dashboard.json b/deploy/observability/dashboard.json
index 5bd78d1b5..b1f93633c 100644
--- a/deploy/observability/dashboard.json
+++ b/deploy/observability/dashboard.json
@@ -433,11 +433,11 @@
{
"definition": {
"type": "timeseries",
- "title": "Database pool: in use vs size",
+ "title": "Database pool: peak in use vs size",
"show_legend": true,
"requests": [
{
- "q": "avg:switch.db.pool.in_use{$env,$service}",
+ "q": "max:switch.db.pool.in_use{$env,$service}",
"display_type": "area"
},
{
@@ -695,6 +695,25 @@
"width": 12,
"height": 4
}
+ },
+ {
+ "definition": {
+ "type": "timeseries",
+ "title": "Database pool: checkout timeouts",
+ "show_legend": false,
+ "requests": [
+ {
+ "q": "sum:switch.db.pool.timeouts{$env,$service}.as_count()",
+ "display_type": "bars"
+ }
+ ]
+ },
+ "layout": {
+ "x": 0,
+ "y": 63,
+ "width": 12,
+ "height": 4
+ }
}
]
}
diff --git a/deploy/observability/monitors.json b/deploy/observability/monitors.json
index 5540d1308..86c172cab 100644
--- a/deploy/observability/monitors.json
+++ b/deploy/observability/monitors.json
@@ -278,5 +278,23 @@
},
"include_tags": true
}
+ },
+ {
+ "name": "[Switch] the database pool is refusing connections",
+ "type": "metric alert",
+ "query": "sum(last_5m):sum:switch.db.pool.timeouts{service:switch-core}.as_count() > 0",
+ "message": "{{value}} request(s) got no pooled connection within the timeout in five minutes.\n\nThis is the pool past nearly-exhausted and into failing: it queues rather than sheds when full, so a timeout means a request waited out `db_pool_timeout` and still got nothing. Because auth resolves a token against the database before the handler runs, the first thing to fail is authentication on otherwise-fine requests — including agent heartbeats, which then look lapsed and reconnect, which loads the pool further.\n\nLook for a connection held across non-database work (an external call inside an open session) or a synchronized reconnect burst. Raising the pool size is a cushion, not the cause.\n\n@REPLACE-WITH-NOTIFICATION-HANDLE",
+ "tags": [
+ "service:switch-core",
+ "team:switch"
+ ],
+ "options": {
+ "notify_no_data": false,
+ "renotify_interval": 60,
+ "thresholds": {
+ "critical": 0
+ },
+ "include_tags": true
+ }
}
]