Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

fix(db): notice a Postgres server that fails over without closing (CHOO-2622)

Petr Bauchae32060fda17
diff --git a/core/switch_core/config.py b/core/switch_core/config.py
index 60b051769..899b49d1b 100644
--- a/core/switch_core/config.py
+++ b/core/switch_core/config.py
@@ -160,6 +160,22 @@ class SwitchConfig(BaseSettings):
# never killed mid-transaction.
db_idle_in_transaction_session_timeout: str | None = None
+ # A Postgres server that goes away without closing its sockets — a managed
+ # instance failing over to its standby — leaves every connection open and
+ # apparently healthy. Nothing above the socket can tell the difference:
+ # reads block, the listener's heartbeat never returns, and the process goes
+ # on reporting itself connected while delivering nothing. Only the kernel
+ # finds out, and left to its own defaults it takes around fifteen minutes.
+ #
+ # These two mechanisms bound that, and both are needed because they cover
+ # different sockets. Keepalive probes fail a connection that was idle when
+ # the server vanished; the user timeout fails one that had already sent
+ # something, which keepalives never look at. Seconds; 0 disables either.
+ db_tcp_keepalive_idle: int = 10
+ db_tcp_keepalive_interval: int = 5
+ db_tcp_keepalive_count: int = 3
+ db_tcp_user_timeout: int = 30
+
# libpq-style TLS mode for the Postgres connection, forwarded to asyncpg.
# "disable" (the default) keeps in-cluster / local-dev connections plain,
# matching current behaviour. Managed Postgres (RDS / Cloud SQL / Azure)
diff --git a/core/switch_core/db/engine.py b/core/switch_core/db/engine.py
index 053c4a9bc..8a51edcbd 100644
--- a/core/switch_core/db/engine.py
+++ b/core/switch_core/db/engine.py
@@ -1,3 +1,8 @@
+import logging
+import socket
+from typing import Any
+
+from sqlalchemy import event
from sqlalchemy.ext.asyncio import (
AsyncEngine,
AsyncSession,
@@ -8,6 +13,87 @@ from sqlalchemy.pool import NullPool
from switch_core.config import SwitchConfig
+logger = logging.getLogger(__name__)
+
+
+def dead_peer_socket_options(
+ config: SwitchConfig,
+) -> tuple[list[tuple[int, int, int]], list[str]]:
+ """Socket options bounding how long a vanished server looks alive.
+
+ Returns the `(level, option, value)` triples to set, and the names of the
+ options this platform does not have. Linux has all of them. macOS spells
+ the keepalive idle option differently and has no user timeout at all, so a
+ developer's laptop applies what it can and says what it could not — the
+ deployed system is the one that has to survive a failover.
+ """
+ options: list[tuple[int, int, int]] = []
+ unavailable: list[str] = []
+
+ def add(name: str, value: int) -> None:
+ option = getattr(socket, name, None)
+ if option is None:
+ unavailable.append(name)
+ return
+ options.append((socket.IPPROTO_TCP, option, value))
+
+ if config.db_tcp_keepalive_idle > 0:
+ options.append((socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1))
+ add("TCP_KEEPIDLE", config.db_tcp_keepalive_idle)
+ add("TCP_KEEPINTVL", config.db_tcp_keepalive_interval)
+ add("TCP_KEEPCNT", config.db_tcp_keepalive_count)
+ if config.db_tcp_user_timeout > 0:
+ add("TCP_USER_TIMEOUT", config.db_tcp_user_timeout * 1000)
+ return options, unavailable
+
+
+def _socket_behind(dbapi_connection: Any) -> Any:
+ """The socket asyncpg is talking over, or None if it cannot be reached.
+
+ Duck-typed on `setsockopt` rather than checked against `socket.socket`:
+ asyncio hands out an `asyncio.trsock.TransportSocket`, which wraps a real
+ socket and forwards the call but is not an instance of one.
+ """
+ driver_connection = getattr(dbapi_connection, "driver_connection", None)
+ transport = getattr(driver_connection, "_transport", None)
+ if transport is None:
+ return None
+ sock = transport.get_extra_info("socket")
+ return sock if hasattr(sock, "setsockopt") else None
+
+
+def _with_dead_peer_detection(engine: AsyncEngine, config: SwitchConfig) -> AsyncEngine:
+ options, unavailable = dead_peer_socket_options(config)
+ if unavailable:
+ logger.warning(
+ "This platform has no %s, so a Postgres server that stops "
+ "answering without closing the connection will take the kernel's "
+ "own timeout to be noticed rather than about %ds.",
+ ", ".join(unavailable),
+ max(
+ config.db_tcp_user_timeout,
+ config.db_tcp_keepalive_idle
+ + config.db_tcp_keepalive_interval * config.db_tcp_keepalive_count,
+ ),
+ )
+ if not options:
+ return engine
+
+ @event.listens_for(engine.sync_engine, "connect")
+ def _set_socket_options(dbapi_connection: Any, _record: Any) -> None:
+ sock = _socket_behind(dbapi_connection)
+ if sock is None:
+ logger.warning(
+ "No socket behind this Postgres connection, so dead-peer "
+ "detection is not in force on it: a failover will leave it "
+ "looking healthy until the kernel's own timeout expires."
+ )
+ return
+ for level, option, value in options:
+ sock.setsockopt(level, option, value)
+
+ return engine
+
def app_connect_args(config: SwitchConfig) -> dict[str, object]:
"""asyncpg connect args for the application engine.
@@ -44,7 +130,9 @@ def create_engine_from_config(
"connect_args": connect_args,
}
defaults.update(engine_kwargs)
- return create_async_engine(config.database_url, **defaults)
+ return _with_dead_peer_detection(
+ create_async_engine(config.database_url, **defaults), config
+ )
def create_unpooled_engine(config: SwitchConfig) -> AsyncEngine:
@@ -55,11 +143,20 @@ def create_unpooled_engine(config: SwitchConfig) -> AsyncEngine:
would drop the subscription, and the silence afterwards is indistinguishable
from a quiet room — so it takes the connection arguments and none of the
pooling.
+
+ It does take the dead-peer socket options, and needs them more than the
+ pool does. A pooled connection is checked out, used and pre-pinged, so a
+ dead one is found the next time someone wants it; this one is only ever
+ read from, and a server that vanishes without saying so leaves it silent
+ and indistinguishable from a room where nobody is talking.
"""
- return create_async_engine(
- config.database_url,
- poolclass=NullPool,
- connect_args=app_connect_args(config),
+ return _with_dead_peer_detection(
+ create_async_engine(
+ config.database_url,
+ poolclass=NullPool,
+ connect_args=app_connect_args(config),
+ ),
+ config,
)
diff --git a/core/switch_core/messages/notify.py b/core/switch_core/messages/notify.py
index f72fd5a68..44e0c4a87 100644
--- a/core/switch_core/messages/notify.py
+++ b/core/switch_core/messages/notify.py
@@ -52,6 +52,14 @@ logger = logging.getLogger(__name__)
# catch: without it the process looks connected and silently stops receiving.
HEARTBEAT_SECONDS = 20.0
+# And a bound on the heartbeat itself, because proving the connection alive is
+# worth nothing if the proof can hang. The socket options set in
+# `db/engine.py` are what should fail a vanished server, and they act sooner
+# than this — this is the backstop for a platform that has none of them, or a
+# connection they could not be applied to. Reaching it means the socket-level
+# detection did not work.
+HEARTBEAT_TIMEOUT_SECONDS = 60.0
+
# Backoff for a listener that cannot connect. Capped low: the whole system's
# delivery latency is behind this, so a long sleep is a long outage.
RECONNECT_BACKOFF_BASE = 1.0
@@ -210,7 +218,7 @@ class MessageListener:
"""Keep the connection until it fails, proving it alive as we go."""
while self._running:
await asyncio.sleep(HEARTBEAT_SECONDS)
- await driver.execute("SELECT 1")
+ await driver.execute("SELECT 1", timeout=HEARTBEAT_TIMEOUT_SECONDS)
def _on_notify(
self, _connection: Any, _pid: int, _channel: str, payload: str
diff --git a/core/tests/switch_core/db/test_dead_peer_detection.py b/core/tests/switch_core/db/test_dead_peer_detection.py
new file mode 100644
index 000000000..4958f9e66
--- /dev/null
+++ b/core/tests/switch_core/db/test_dead_peer_detection.py
@@ -0,0 +1,214 @@
+"""Noticing a Postgres server that stopped answering without saying so.
+
+A managed instance failing over to its standby does not close its sockets. The
+connections stay open and look perfectly healthy: nothing is refused, nothing
+is reset, reads simply never return. Left to the kernel's defaults that lasts
+about fifteen minutes, and for all of it the process reports itself connected
+while delivering nothing — the worst failure mode there is.
+
+These tests cover what we ask the kernel for. Whether the kernel then honours
+it is a failover test, not a unit test.
+"""
+
+from __future__ import annotations
+
+import socket
+from typing import TYPE_CHECKING
+
+import pytest
+
+from switch_core.config import SwitchConfig
+from switch_core.db.engine import (
+ create_engine_from_config,
+ create_unpooled_engine,
+ dead_peer_socket_options,
+)
+
+if TYPE_CHECKING:
+ from collections.abc import Callable
+
+ from sqlalchemy.ext.asyncio import AsyncEngine
+
+_BASE_KWARGS = dict(
+ db_host="db",
+ db_port="5432",
+ db_user="postgres",
+ db_password="pw",
+ db_name="switch",
+ matrix_server_name="switch.local",
+ agent_registration_token="token",
+ jwt_secret_key="jwt",
+ gateway_admin_email="[email protected]",
+ gateway_admin_password="pw",
+)
+
+
+def _config(**overrides: object) -> SwitchConfig:
+ return SwitchConfig(**{**_BASE_KWARGS, **overrides}) # type: ignore[arg-type]
+
+
+def _config_for(postgres_url: str, **overrides: object) -> SwitchConfig:
+ """A config pointing at the throwaway container behind `postgres_url`."""
+ credentials, _, location = postgres_url.split("://", 1)[1].rpartition("@")
+ user, _, password = credentials.partition(":")
+ hostport, _, name = location.partition("/")
+ host, _, port = hostport.partition(":")
+ return _config(
+ db_host=host,
+ db_port=port,
+ db_user=user,
+ db_password=password,
+ db_name=name,
+ **overrides,
+ )
+
+
+def _named(name: str) -> int | None:
+ return getattr(socket, name, None)
+
+
+class TestWhatWeAskFor:
+ def test_keepalive_is_on_by_default(self) -> None:
+ options, _ = dead_peer_socket_options(_config())
+
+ assert (socket.SOL_SOCKET, socket.SO_KEEPALIVE, 1) in options
+
+ def test_the_defaults_notice_within_a_minute(self) -> None:
+ config = _config()
+ idle = config.db_tcp_keepalive_idle
+ probes = config.db_tcp_keepalive_interval * config.db_tcp_keepalive_count
+
+ assert idle + probes < 60
+ assert config.db_tcp_user_timeout < 60
+
+ def test_the_user_timeout_is_asked_for_in_milliseconds(self) -> None:
+ # The setting is seconds, because every other timeout in the config is.
+ # TCP_USER_TIMEOUT is milliseconds, and getting that wrong by a factor
+ # of a thousand is silent in both directions.
+ option = _named("TCP_USER_TIMEOUT")
+ if option is None:
+ pytest.skip("this platform has no TCP_USER_TIMEOUT")
+ options, _ = dead_peer_socket_options(_config(db_tcp_user_timeout=30))
+
+ assert (socket.IPPROTO_TCP, option, 30_000) in options
+
+ def test_zero_keepalive_idle_asks_for_no_keepalive_at_all(self) -> None:
+ options, _ = dead_peer_socket_options(_config(db_tcp_keepalive_idle=0))
+
+ assert not any(option == socket.SO_KEEPALIVE for _, option, _ in options)
+ assert not any(name in str(options) for name in ("KEEPINTVL", "KEEPCNT"))
+
+ def test_zero_user_timeout_asks_for_none(self) -> None:
+ option = _named("TCP_USER_TIMEOUT")
+ if option is None:
+ pytest.skip("this platform has no TCP_USER_TIMEOUT")
+ options, _ = dead_peer_socket_options(_config(db_tcp_user_timeout=0))
+
+ assert not any(candidate == option for _, candidate, _ in options)
+
+ def test_disabling_both_asks_for_nothing(self) -> None:
+ options, _ = dead_peer_socket_options(
+ _config(db_tcp_keepalive_idle=0, db_tcp_user_timeout=0)
+ )
+
+ assert options == []
+
+ def test_an_option_this_platform_lacks_is_reported_not_dropped(self) -> None:
+ # Silence here would mean a laptop and a cluster quietly disagreeing
+ # about whether a failover is survivable.
+ _, unavailable = dead_peer_socket_options(_config())
+ expected = [
+ name
+ for name in (
+ "TCP_KEEPIDLE",
+ "TCP_KEEPINTVL",
+ "TCP_KEEPCNT",
+ "TCP_USER_TIMEOUT",
+ )
+ if _named(name) is None
+ ]
+
+ assert unavailable == expected
+
+
+class TestOnARealConnection:
+ """The options have to reach the socket, not just the option list."""
+
+ @pytest.fixture(params=["pooled", "held"])
+ def make_engine(
+ self, request: pytest.FixtureRequest
+ ) -> Callable[[SwitchConfig], AsyncEngine]:
+ if request.param == "pooled":
+ return create_engine_from_config
+ return create_unpooled_engine
+
+ async def _read(self, engine: AsyncEngine, level: int, option: int) -> int:
+ """One socket option, read while the connection is still open.
+
+ The unpooled engine closes its connection on the way out of the block,
+ so this cannot hand the socket back to the caller and read it there.
+ """
+ async with engine.connect() as connection:
+ raw = await connection.get_raw_connection()
+ transport = raw.driver_connection._transport # type: ignore[union-attr]
+ sock = transport.get_extra_info("socket")
+ return int(sock.getsockopt(level, option))
+
+ async def test_keepalive_is_set_on_the_socket(
+ self,
+ postgres_url: str,
+ make_engine: Callable[[SwitchConfig], AsyncEngine],
+ ) -> None:
+ engine = make_engine(_config_for(postgres_url))
+ try:
+ keepalive = await self._read(engine, socket.SOL_SOCKET, socket.SO_KEEPALIVE)
+
+ # Non-zero rather than 1: BSD reads a boolean socket option back as
+ # the option's own bit, so macOS answers 8 where Linux answers 1.
+ assert keepalive != 0
+ finally:
+ await engine.dispose()
+
+ @pytest.mark.parametrize(
+ ("name", "setting", "scale"),
+ [
+ ("TCP_KEEPIDLE", "db_tcp_keepalive_idle", 1),
+ ("TCP_KEEPINTVL", "db_tcp_keepalive_interval", 1),
+ ("TCP_KEEPCNT", "db_tcp_keepalive_count", 1),
+ ("TCP_USER_TIMEOUT", "db_tcp_user_timeout", 1000),
+ ],
+ )
+ async def test_each_configured_value_reaches_the_socket(
+ self,
+ postgres_url: str,
+ make_engine: Callable[[SwitchConfig], AsyncEngine],
+ name: str,
+ setting: str,
+ scale: int,
+ ) -> None:
+ option = _named(name)
+ if option is None:
+ pytest.skip(f"this platform has no {name}")
+ engine = make_engine(_config_for(postgres_url, **{setting: 7}))
+ try:
+ value = await self._read(engine, socket.IPPROTO_TCP, option)
+
+ assert value == 7 * scale
+ finally:
+ await engine.dispose()
+
+ async def test_nothing_is_set_when_it_is_all_turned_off(
+ self,
+ postgres_url: str,
+ make_engine: Callable[[SwitchConfig], AsyncEngine],
+ ) -> None:
+ config = _config_for(
+ postgres_url, db_tcp_keepalive_idle=0, db_tcp_user_timeout=0
+ )
+ engine = make_engine(config)
+ try:
+ keepalive = await self._read(engine, socket.SOL_SOCKET, socket.SO_KEEPALIVE)
+
+ assert keepalive == 0
+ finally:
+ await engine.dispose()
diff --git a/deploy/remote/helm/switch/templates/_helpers.tpl b/deploy/remote/helm/switch/templates/_helpers.tpl
index 02311811e..4d1145f16 100644
--- a/deploy/remote/helm/switch/templates/_helpers.tpl
+++ b/deploy/remote/helm/switch/templates/_helpers.tpl
@@ -333,6 +333,14 @@ Include with `nindent 12`.
value: {{ .Values.postgresql.pool.maxOverflow | quote }}
- name: DB_POOL_TIMEOUT
value: {{ .Values.postgresql.pool.timeout | quote }}
+- name: DB_TCP_KEEPALIVE_IDLE
+ value: {{ .Values.postgresql.tcp.keepaliveIdle | quote }}
+- name: DB_TCP_KEEPALIVE_INTERVAL
+ value: {{ .Values.postgresql.tcp.keepaliveInterval | quote }}
+- name: DB_TCP_KEEPALIVE_COUNT
+ value: {{ .Values.postgresql.tcp.keepaliveCount | quote }}
+- name: DB_TCP_USER_TIMEOUT
+ value: {{ .Values.postgresql.tcp.userTimeout | quote }}
{{- with .Values.postgresql.idleInTransactionSessionTimeout }}
- name: DB_IDLE_IN_TRANSACTION_SESSION_TIMEOUT
value: {{ . | quote }}
diff --git a/deploy/remote/helm/switch/values.yaml b/deploy/remote/helm/switch/values.yaml
index 6bde69d9d..1e2528a92 100644
--- a/deploy/remote/helm/switch/values.yaml
+++ b/deploy/remote/helm/switch/values.yaml
@@ -366,6 +366,25 @@ postgresql:
# SQLAlchemy's 30s default hides starvation as latency.
timeout: 5
+ # How quickly switch-core notices a database server that stopped answering
+ # without closing its connections — what a managed instance does when it
+ # fails over to its standby. Nothing above the socket can tell: the
+ # connections stay open, reads never return, and the process goes on
+ # reporting itself healthy while delivering nothing. Left to the kernel's own
+ # defaults that lasts about fifteen minutes, all of it silent.
+ #
+ # Both mechanisms are needed; they cover different sockets. Keepalive probes
+ # fail a connection that was idle when the server went. userTimeout fails one
+ # that had already sent something, which keepalives never look at. Raise them
+ # on a lossy network, where a brief stall would otherwise cost good
+ # connections. Seconds; 0 disables either. Linux only — a developer's macOS
+ # applies what it can and says at startup what it could not.
+ tcp:
+ keepaliveIdle: 10
+ keepaliveInterval: 5
+ keepaliveCount: 3
+ userTimeout: 30
+
# Postgres kills a switch-core connection that sits inside an open
# transaction without executing anything for longer than this (e.g. "15s"),
# so a leaked checkout becomes a loud, attributable error instead of a slot