Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
fix(db): notice a Postgres server that fails over without closing (CHOO-2622)
Petr Bauchae32060fda17
diff --git a/core/switch_core/config.py b/core/switch_core/config.pyindex 60b051769..899b49d1b 100644--- a/core/switch_core/config.py+++ b/core/switch_core/config.py@@ -160,6 +160,22 @@ class SwitchConfig(BaseSettings):# never killed mid-transaction.db_idle_in_transaction_session_timeout: str | None = None+ # A Postgres server that goes away without closing its sockets — a managed+ # instance failing over to its standby — leaves every connection open and+ # apparently healthy. Nothing above the socket can tell the difference:+ # reads block, the listener's heartbeat never returns, and the process goes+ # on reporting itself connected while delivering nothing. Only the kernel+ # finds out, and left to its own defaults it takes around fifteen minutes.+ #+ # These two mechanisms bound that, and both are needed because they cover+ # different sockets. Keepalive probes fail a connection that was idle when+ # the server vanished; the user timeout fails one that had already sent+ # something, which keepalives never look at. Seconds; 0 disables either.+ db_tcp_keepalive_idle: int = 10+ db_tcp_keepalive_interval: int = 5+ db_tcp_keepalive_count: int = 3+ db_tcp_user_timeout: int = 30+# libpq-style TLS mode for the Postgres connection, forwarded to asyncpg.# "disable" (the default) keeps in-cluster / local-dev connections plain,# matching current behaviour. Managed Postgres (RDS / Cloud SQL / Azure)diff --git a/core/switch_core/db/engine.py b/core/switch_core/db/engine.pyindex 053c4a9bc..8a51edcbd 100644--- a/core/switch_core/db/engine.py+++ b/core/switch_core/db/engine.py@@ -1,3 +1,8 @@+import logging+import socket+from typing import Any++from sqlalchemy import eventfrom sqlalchemy.ext.asyncio import (AsyncEngine,AsyncSession,@@ -8,6 +13,87 @@ from sqlalchemy.pool import NullPoolfrom switch_core.config import SwitchConfig+logger = logging.getLogger(__name__)+++def dead_peer_socket_options(+ config: SwitchConfig,+) -> tuple[list[tuple[int, int, int]], list[str]]:+ """Socket options bounding how long a vanished server looks alive.++ Returns the `(level, option, value)` triples to set, and the names of the+ options this platform does not have. Linux has all of them. macOS spells+ the keepalive idle option differently and has no user timeout at all, so a+ developer's laptop applies what it can and says what it could not — the+ deployed system is the one that has to survive a failover.+ """+ options: list[tuple[int, int, int]] = []+ unavailable: list[str] = []++ def add(name: str, value: int) -> None:+ option = getattr(socket, name, None)+ if option is None:+ unavailable.append(name)+ return+ options.append((socket.IPPROTO_TCP, option, value))++ if config.db_tcp_keepalive_idle > 0:+ options.append((socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1))+ add("TCP_KEEPIDLE", config.db_tcp_keepalive_idle)+ add("TCP_KEEPINTVL", config.db_tcp_keepalive_interval)+ add("TCP_KEEPCNT", config.db_tcp_keepalive_count)+ if config.db_tcp_user_timeout > 0:+ add("TCP_USER_TIMEOUT", config.db_tcp_user_timeout * 1000)+ return options, unavailable+++def _socket_behind(dbapi_connection: Any) -> Any:+ """The socket asyncpg is talking over, or None if it cannot be reached.++ Duck-typed on `setsockopt` rather than checked against `socket.socket`:+ asyncio hands out an `asyncio.trsock.TransportSocket`, which wraps a real+ socket and forwards the call but is not an instance of one.+ """+ driver_connection = getattr(dbapi_connection, "driver_connection", None)+ transport = getattr(driver_connection, "_transport", None)+ if transport is None:+ return None+ sock = transport.get_extra_info("socket")+ return sock if hasattr(sock, "setsockopt") else None+++def _with_dead_peer_detection(engine: AsyncEngine, config: SwitchConfig) -> AsyncEngine:+ options, unavailable = dead_peer_socket_options(config)+ if unavailable:+ logger.warning(+ "This platform has no %s, so a Postgres server that stops "+ "answering without closing the connection will take the kernel's "+ "own timeout to be noticed rather than about %ds.",+ ", ".join(unavailable),+ max(+ config.db_tcp_user_timeout,+ config.db_tcp_keepalive_idle+ + config.db_tcp_keepalive_interval * config.db_tcp_keepalive_count,+ ),+ )+ if not options:+ return engine++ @event.listens_for(engine.sync_engine, "connect")+ def _set_socket_options(dbapi_connection: Any, _record: Any) -> None:+ sock = _socket_behind(dbapi_connection)+ if sock is None:+ logger.warning(+ "No socket behind this Postgres connection, so dead-peer "+ "detection is not in force on it: a failover will leave it "+ "looking healthy until the kernel's own timeout expires."+ )+ return+ for level, option, value in options:+ sock.setsockopt(level, option, value)++ return engine+def app_connect_args(config: SwitchConfig) -> dict[str, object]:"""asyncpg connect args for the application engine.@@ -44,7 +130,9 @@ def create_engine_from_config("connect_args": connect_args,}defaults.update(engine_kwargs)- return create_async_engine(config.database_url, **defaults)+ return _with_dead_peer_detection(+ create_async_engine(config.database_url, **defaults), config+ )def create_unpooled_engine(config: SwitchConfig) -> AsyncEngine:@@ -55,11 +143,20 @@ def create_unpooled_engine(config: SwitchConfig) -> AsyncEngine:would drop the subscription, and the silence afterwards is indistinguishablefrom a quiet room — so it takes the connection arguments and none of thepooling.++ It does take the dead-peer socket options, and needs them more than the+ pool does. A pooled connection is checked out, used and pre-pinged, so a+ dead one is found the next time someone wants it; this one is only ever+ read from, and a server that vanishes without saying so leaves it silent+ and indistinguishable from a room where nobody is talking."""- return create_async_engine(- config.database_url,- poolclass=NullPool,- connect_args=app_connect_args(config),+ return _with_dead_peer_detection(+ create_async_engine(+ config.database_url,+ poolclass=NullPool,+ connect_args=app_connect_args(config),+ ),+ config,)diff --git a/core/switch_core/messages/notify.py b/core/switch_core/messages/notify.pyindex f72fd5a68..44e0c4a87 100644--- a/core/switch_core/messages/notify.py+++ b/core/switch_core/messages/notify.py@@ -52,6 +52,14 @@ logger = logging.getLogger(__name__)# catch: without it the process looks connected and silently stops receiving.HEARTBEAT_SECONDS = 20.0+# And a bound on the heartbeat itself, because proving the connection alive is+# worth nothing if the proof can hang. The socket options set in+# `db/engine.py` are what should fail a vanished server, and they act sooner+# than this — this is the backstop for a platform that has none of them, or a+# connection they could not be applied to. Reaching it means the socket-level+# detection did not work.+HEARTBEAT_TIMEOUT_SECONDS = 60.0+# Backoff for a listener that cannot connect. Capped low: the whole system's# delivery latency is behind this, so a long sleep is a long outage.RECONNECT_BACKOFF_BASE = 1.0@@ -210,7 +218,7 @@ class MessageListener:"""Keep the connection until it fails, proving it alive as we go."""while self._running:await asyncio.sleep(HEARTBEAT_SECONDS)- await driver.execute("SELECT 1")+ await driver.execute("SELECT 1", timeout=HEARTBEAT_TIMEOUT_SECONDS)def _on_notify(self, _connection: Any, _pid: int, _channel: str, payload: strdiff --git a/core/tests/switch_core/db/test_dead_peer_detection.py b/core/tests/switch_core/db/test_dead_peer_detection.pynew file mode 100644index 000000000..4958f9e66--- /dev/null+++ b/core/tests/switch_core/db/test_dead_peer_detection.py@@ -0,0 +1,214 @@+"""Noticing a Postgres server that stopped answering without saying so.++A managed instance failing over to its standby does not close its sockets. The+connections stay open and look perfectly healthy: nothing is refused, nothing+is reset, reads simply never return. Left to the kernel's defaults that lasts+about fifteen minutes, and for all of it the process reports itself connected+while delivering nothing — the worst failure mode there is.++These tests cover what we ask the kernel for. Whether the kernel then honours+it is a failover test, not a unit test.+"""++from __future__ import annotations++import socket+from typing import TYPE_CHECKING++import pytest++from switch_core.config import SwitchConfig+from switch_core.db.engine import (+ create_engine_from_config,+ create_unpooled_engine,+ dead_peer_socket_options,+)++if TYPE_CHECKING:+ from collections.abc import Callable++ from sqlalchemy.ext.asyncio import AsyncEngine++_BASE_KWARGS = dict(+ db_host="db",+ db_port="5432",+ db_user="postgres",+ db_password="pw",+ db_name="switch",+ matrix_server_name="switch.local",+ agent_registration_token="token",+ jwt_secret_key="jwt",+ gateway_admin_email="[email protected]",+ gateway_admin_password="pw",+)+++def _config(**overrides: object) -> SwitchConfig:+ return SwitchConfig(**{**_BASE_KWARGS, **overrides}) # type: ignore[arg-type]+++def _config_for(postgres_url: str, **overrides: object) -> SwitchConfig:+ """A config pointing at the throwaway container behind `postgres_url`."""+ credentials, _, location = postgres_url.split("://", 1)[1].rpartition("@")+ user, _, password = credentials.partition(":")+ hostport, _, name = location.partition("/")+ host, _, port = hostport.partition(":")+ return _config(+ db_host=host,+ db_port=port,+ db_user=user,+ db_password=password,+ db_name=name,+ **overrides,+ )+++def _named(name: str) -> int | None:+ return getattr(socket, name, None)+++class TestWhatWeAskFor:+ def test_keepalive_is_on_by_default(self) -> None:+ options, _ = dead_peer_socket_options(_config())++ assert (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) in options++ def test_the_defaults_notice_within_a_minute(self) -> None:+ config = _config()+ idle = config.db_tcp_keepalive_idle+ probes = config.db_tcp_keepalive_interval * config.db_tcp_keepalive_count++ assert idle + probes < 60+ assert config.db_tcp_user_timeout < 60++ def test_the_user_timeout_is_asked_for_in_milliseconds(self) -> None:+ # The setting is seconds, because every other timeout in the config is.+ # TCP_USER_TIMEOUT is milliseconds, and getting that wrong by a factor+ # of a thousand is silent in both directions.+ option = _named("TCP_USER_TIMEOUT")+ if option is None:+ pytest.skip("this platform has no TCP_USER_TIMEOUT")+ options, _ = dead_peer_socket_options(_config(db_tcp_user_timeout=30))++ assert (socket.IPPROTO_TCP, option, 30_000) in options++ def test_zero_keepalive_idle_asks_for_no_keepalive_at_all(self) -> None:+ options, _ = dead_peer_socket_options(_config(db_tcp_keepalive_idle=0))++ assert not any(option == socket.SO_KEEPALIVE for _, option, _ in options)+ assert not any(name in str(options) for name in ("KEEPINTVL", "KEEPCNT"))++ def test_zero_user_timeout_asks_for_none(self) -> None:+ option = _named("TCP_USER_TIMEOUT")+ if option is None:+ pytest.skip("this platform has no TCP_USER_TIMEOUT")+ options, _ = dead_peer_socket_options(_config(db_tcp_user_timeout=0))++ assert not any(candidate == option for _, candidate, _ in options)++ def test_disabling_both_asks_for_nothing(self) -> None:+ options, _ = dead_peer_socket_options(+ _config(db_tcp_keepalive_idle=0, db_tcp_user_timeout=0)+ )++ assert options == []++ def test_an_option_this_platform_lacks_is_reported_not_dropped(self) -> None:+ # Silence here would mean a laptop and a cluster quietly disagreeing+ # about whether a failover is survivable.+ _, unavailable = dead_peer_socket_options(_config())+ expected = [+ name+ for name in (+ "TCP_KEEPIDLE",+ "TCP_KEEPINTVL",+ "TCP_KEEPCNT",+ "TCP_USER_TIMEOUT",+ )+ if _named(name) is None+ ]++ assert unavailable == expected+++class TestOnARealConnection:+ """The options have to reach the socket, not just the option list."""++ @pytest.fixture(params=["pooled", "held"])+ def make_engine(+ self, request: pytest.FixtureRequest+ ) -> Callable[[SwitchConfig], AsyncEngine]:+ if request.param == "pooled":+ return create_engine_from_config+ return create_unpooled_engine++ async def _read(self, engine: AsyncEngine, level: int, option: int) -> int:+ """One socket option, read while the connection is still open.++ The unpooled engine closes its connection on the way out of the block,+ so this cannot hand the socket back to the caller and read it there.+ """+ async with engine.connect() as connection:+ raw = await connection.get_raw_connection()+ transport = raw.driver_connection._transport # type: ignore[union-attr]+ sock = transport.get_extra_info("socket")+ return int(sock.getsockopt(level, option))++ async def test_keepalive_is_set_on_the_socket(+ self,+ postgres_url: str,+ make_engine: Callable[[SwitchConfig], AsyncEngine],+ ) -> None:+ engine = make_engine(_config_for(postgres_url))+ try:+ keepalive = await self._read(engine, socket.SOL_SOCKET, socket.SO_KEEPALIVE)++ # Non-zero rather than 1: BSD reads a boolean socket option back as+ # the option's own bit, so macOS answers 8 where Linux answers 1.+ assert keepalive != 0+ finally:+ await engine.dispose()++ @pytest.mark.parametrize(+ ("name", "setting", "scale"),+ [+ ("TCP_KEEPIDLE", "db_tcp_keepalive_idle", 1),+ ("TCP_KEEPINTVL", "db_tcp_keepalive_interval", 1),+ ("TCP_KEEPCNT", "db_tcp_keepalive_count", 1),+ ("TCP_USER_TIMEOUT", "db_tcp_user_timeout", 1000),+ ],+ )+ async def test_each_configured_value_reaches_the_socket(+ self,+ postgres_url: str,+ make_engine: Callable[[SwitchConfig], AsyncEngine],+ name: str,+ setting: str,+ scale: int,+ ) -> None:+ option = _named(name)+ if option is None:+ pytest.skip(f"this platform has no {name}")+ engine = make_engine(_config_for(postgres_url, **{setting: 7}))+ try:+ value = await self._read(engine, socket.IPPROTO_TCP, option)++ assert value == 7 * scale+ finally:+ await engine.dispose()++ async def test_nothing_is_set_when_it_is_all_turned_off(+ self,+ postgres_url: str,+ make_engine: Callable[[SwitchConfig], AsyncEngine],+ ) -> None:+ config = _config_for(+ postgres_url, db_tcp_keepalive_idle=0, db_tcp_user_timeout=0+ )+ engine = make_engine(config)+ try:+ keepalive = await self._read(engine, socket.SOL_SOCKET, socket.SO_KEEPALIVE)++ assert keepalive == 0+ finally:+ await engine.dispose()diff --git a/deploy/remote/helm/switch/templates/_helpers.tpl b/deploy/remote/helm/switch/templates/_helpers.tplindex 02311811e..4d1145f16 100644--- a/deploy/remote/helm/switch/templates/_helpers.tpl+++ b/deploy/remote/helm/switch/templates/_helpers.tpl@@ -333,6 +333,14 @@ Include with `nindent 12`.value: {{ .Values.postgresql.pool.maxOverflow | quote }}- name: DB_POOL_TIMEOUTvalue: {{ .Values.postgresql.pool.timeout | quote }}+- name: DB_TCP_KEEPALIVE_IDLE+ value: {{ .Values.postgresql.tcp.keepaliveIdle | quote }}+- name: DB_TCP_KEEPALIVE_INTERVAL+ value: {{ .Values.postgresql.tcp.keepaliveInterval | quote }}+- name: DB_TCP_KEEPALIVE_COUNT+ value: {{ .Values.postgresql.tcp.keepaliveCount | quote }}+- name: DB_TCP_USER_TIMEOUT+ value: {{ .Values.postgresql.tcp.userTimeout | quote }}{{- with .Values.postgresql.idleInTransactionSessionTimeout }}- name: DB_IDLE_IN_TRANSACTION_SESSION_TIMEOUTvalue: {{ . | quote }}diff --git a/deploy/remote/helm/switch/values.yaml b/deploy/remote/helm/switch/values.yamlindex 6bde69d9d..1e2528a92 100644--- a/deploy/remote/helm/switch/values.yaml+++ b/deploy/remote/helm/switch/values.yaml@@ -366,6 +366,25 @@ postgresql:# SQLAlchemy's 30s default hides starvation as latency.timeout: 5+ # How quickly switch-core notices a database server that stopped answering+ # without closing its connections — what a managed instance does when it+ # fails over to its standby. Nothing above the socket can tell: the+ # connections stay open, reads never return, and the process goes on+ # reporting itself healthy while delivering nothing. Left to the kernel's own+ # defaults that lasts about fifteen minutes, all of it silent.+ #+ # Both mechanisms are needed; they cover different sockets. Keepalive probes+ # fail a connection that was idle when the server went. userTimeout fails one+ # that had already sent something, which keepalives never look at. Raise them+ # on a lossy network, where a brief stall would otherwise cost good+ # connections. Seconds; 0 disables either. Linux only — a developer's macOS+ # applies what it can and says at startup what it could not.+ tcp:+ keepaliveIdle: 10+ keepaliveInterval: 5+ keepaliveCount: 3+ userTimeout: 30+# Postgres kills a switch-core connection that sits inside an open# transaction without executing anything for longer than this (e.g. "15s"),# so a leaked checkout becomes a loud, attributable error instead of a slot