Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

fix(core): refuse a closed pooled connection at checkout

raullopez-sandboxd7c34e6712e1
diff --git a/core/switch_core/config.py b/core/switch_core/config.py
index f13c527af..8fcb890b8 100644
--- a/core/switch_core/config.py
+++ b/core/switch_core/config.py
@@ -740,11 +740,11 @@ class SwitchConfig(BaseSettings):
# three round trips and a transaction before any work, which under agent
# load was over a third of all the transactions core ran. What it catches
# is a connection that died while it sat in the pool, as every pooled
- # connection does when the database restarts or fails over. Without it the
- # first query on such a connection fails, and SQLAlchemy then replaces
- # every connection the pool held. So on each pod the first query fails
- # (`test_pool_pre_ping.py`), along with any query that was running when the
- # database went, which fails with the ping too. How soon a connection left
+ # connection does when the database restarts or fails over. A restart
+ # closes the socket, which asyncpg notices, and the engine's checkout check
+ # already replaces a closed connection without a round trip
+ # (`test_pool_pre_ping.py`); a query running when the database went fails
+ # with the ping or without it. How soon a connection left
# open by a failover is known to be dead is up to `db_tcp_keepalive_*` and
# `db_tcp_user_timeout` below, with the ping or without it.
db_pool_pre_ping: bool = False
diff --git a/core/switch_core/db/engine.py b/core/switch_core/db/engine.py
index 3e760eaf8..ee7d987b5 100644
--- a/core/switch_core/db/engine.py
+++ b/core/switch_core/db/engine.py
@@ -3,6 +3,7 @@ import socket
from typing import Any
from sqlalchemy import event
+from sqlalchemy.exc import DisconnectionError
from sqlalchemy.ext.asyncio import (
AsyncEngine,
AsyncSession,
@@ -159,9 +160,24 @@ def create_engine_from_config(
"connect_args": connect_args,
}
defaults.update(engine_kwargs)
- return _with_dead_peer_detection(
- create_async_engine(config.database_url, **defaults), config
- )
+ engine = create_async_engine(config.database_url, **defaults)
+ event.listen(engine.sync_engine, "checkout", _refuse_closed_connection)
+ return _with_dead_peer_detection(engine, config)
+
+
+def _refuse_closed_connection(dbapi_connection: Any, _record: Any, _proxy: Any) -> None:
+ """Swap a pooled connection that is already closed for a fresh one.
+
+ An anyio cancel scope re-delivers its cancellation at every await, so a
+ request cancelled mid-query (its client went away) is cancelled again
+ inside SQLAlchemy's invalidation of the connection. That unwinds before the
+ pool record lets go of the connection, and the record is returned to the
+ pool still holding it, closed. Raising `DisconnectionError` here makes the
+ pool discard it and connect afresh rather than fail the next borrower.
+ Free: `is_closed` reads local state, unlike the pre-ping's round trip.
+ """
+ if dbapi_connection.driver_connection.is_closed():
+ raise DisconnectionError("pooled connection was already closed")
def create_unpooled_engine(config: SwitchConfig) -> AsyncEngine:
diff --git a/core/tests/switch_core/db/test_cancelled_checkout.py b/core/tests/switch_core/db/test_cancelled_checkout.py
new file mode 100644
index 000000000..e7378aef3
--- /dev/null
+++ b/core/tests/switch_core/db/test_cancelled_checkout.py
@@ -0,0 +1,97 @@
+"""A request cancelled mid-query must not hand its dead connection to the next.
+
+Starlette cancels a request whose client went away through an anyio cancel
+scope, which re-delivers the cancellation at every await. SQLAlchemy reacts to
+the first by invalidating the connection, and the second then lands inside
+that invalidation's terminate, which unwinds before the pool record forgets
+the connection. The record goes back to the pool still holding the closed
+asyncpg connection, and whoever checks it out next failed with "connection is
+closed" until the engine learned to refuse a closed connection at checkout.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from typing import Annotated
+
+import anyio
+from fastapi import Depends
+from sqlalchemy import text
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from switch_core.db.engine import create_engine_from_config, create_session_factory
+from switch_core.gateway import dependencies as gw_deps
+from switch_core.gateway.auth import get_current_user
+from tests.switch_core.db.test_dead_peer_detection import _config_for
+from tests.switch_core.management.harness import add_member, build_harness, cookies_for
+
+
+async def test_a_cancelled_query_does_not_poison_the_next_checkout(
+ postgres_url: str,
+) -> None:
+ engine = create_engine_from_config(
+ _config_for(postgres_url, db_pool_size=1, db_max_overflow=0)
+ )
+ try:
+ with anyio.move_on_after(0.2) as scope:
+ async with engine.connect() as conn:
+ await conn.execute(text("select pg_sleep(5)"))
+ assert scope.cancelled_caught
+
+ async with engine.connect() as conn:
+ assert (await conn.execute(text("select 1"))).scalar_one() == 1
+ finally:
+ await engine.dispose()
+
+
+async def test_cancelling_one_request_does_not_fail_a_concurrent_one(
+ postgres_url: str, session_factory: async_sessionmaker[AsyncSession]
+) -> None:
+ # One pooled connection, so the concurrent request is handed exactly the
+ # one the cancelled request was using.
+ engine = create_engine_from_config(
+ _config_for(postgres_url, db_pool_size=1, db_max_overflow=0)
+ )
+ try:
+ harness = build_harness(create_session_factory(engine))
+ sleeping = asyncio.Event()
+
+ @harness.gateway_app.get("/test/slow")
+ async def slow(
+ _user: Annotated[object, Depends(get_current_user)],
+ session: Annotated[AsyncSession, Depends(gw_deps.get_session)],
+ ) -> None:
+ sleeping.set()
+ await session.execute(text("select pg_sleep(5)"))
+
+ @harness.gateway_app.get("/test/fast")
+ async def fast(
+ _user: Annotated[object, Depends(get_current_user)],
+ session: Annotated[AsyncSession, Depends(gw_deps.get_session)],
+ ) -> int:
+ return int((await session.execute(text("select 1"))).scalar_one())
+
+ ada = await add_member(session_factory, "ada")
+ cookies = cookies_for(ada)
+ async with harness.client() as client:
+ async with anyio.create_task_group() as group:
+ slow_scope = anyio.CancelScope()
+
+ async def run_slow() -> None:
+ with slow_scope:
+ await client.get("/gateway/test/slow", cookies=cookies)
+ assert slow_scope.cancelled_caught
+
+ group.start_soon(run_slow)
+ await sleeping.wait()
+ concurrent = asyncio.ensure_future(
+ client.get("/gateway/test/fast", cookies=cookies)
+ )
+ await asyncio.sleep(0.1)
+ slow_scope.cancel()
+
+ assert (await concurrent).status_code == 200
+ after = await client.get("/gateway/test/fast", cookies=cookies)
+ assert after.status_code == 200
+ finally:
+ await engine.dispose()
diff --git a/core/tests/switch_core/db/test_pool_pre_ping.py b/core/tests/switch_core/db/test_pool_pre_ping.py
index a7b695248..140b545f4 100644
--- a/core/tests/switch_core/db/test_pool_pre_ping.py
+++ b/core/tests/switch_core/db/test_pool_pre_ping.py
@@ -2,19 +2,18 @@
A pooled connection the server closed while it sat in the pool is the case
the pre-ping exists for: a database restart closes every connection the pool
-holds. With the pre-ping off, the first query on one of them fails, and
-SQLAlchemy then treats every connection the pool held at that moment as
-stale and replaces it on its next checkout. So a restart costs one failed
-query, not one per pooled connection, which is the trade the default makes.
+holds. When the server says so as it goes, as a restart does, asyncpg marks
+the connection closed and the engine's checkout check swaps it for a fresh one
+with no round trip, so no query fails with the pre-ping off either. What the
+ping still catches is a peer that vanished without closing the socket, which
+is `db_tcp_keepalive_*`'s job as much as the ping's.
"""
from __future__ import annotations
import asyncio
-import pytest
from sqlalchemy import text
-from sqlalchemy.exc import DBAPIError
from sqlalchemy.ext.asyncio import AsyncConnection, AsyncEngine
from switch_core.config import SwitchConfig
@@ -76,18 +75,11 @@ def test_the_pre_ping_is_off_by_default() -> None:
assert SwitchConfig.model_fields["db_pool_pre_ping"].default is False
-async def test_without_the_pre_ping_a_restart_costs_one_failed_query(
+async def test_without_the_pre_ping_a_restart_costs_no_failed_query(
postgres_url: str,
) -> None:
engine, closed = await _fill_and_close_pool(postgres_url, pre_ping=False)
try:
- with pytest.raises(DBAPIError) as failed:
- async with engine.connect() as conn:
- await _pid(conn)
- assert failed.value.connection_invalidated
-
- # Every connection after it is a new one, though two more of the
- # closed ones were still in the pool when the first query failed.
for _ in range(POOLED):
async with engine.connect() as conn:
assert await _pid(conn) not in closed