Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
fix(core): refuse a closed pooled connection at checkout
raullopez-sandboxd7c34e6712e1
diff --git a/core/switch_core/config.py b/core/switch_core/config.pyindex f13c527af..8fcb890b8 100644--- a/core/switch_core/config.py+++ b/core/switch_core/config.py@@ -740,11 +740,11 @@ class SwitchConfig(BaseSettings):# three round trips and a transaction before any work, which under agent# load was over a third of all the transactions core ran. What it catches# is a connection that died while it sat in the pool, as every pooled- # connection does when the database restarts or fails over. Without it the- # first query on such a connection fails, and SQLAlchemy then replaces- # every connection the pool held. So on each pod the first query fails- # (`test_pool_pre_ping.py`), along with any query that was running when the- # database went, which fails with the ping too. How soon a connection left+ # connection does when the database restarts or fails over. A restart+ # closes the socket, which asyncpg notices, and the engine's checkout check+ # already replaces a closed connection without a round trip+ # (`test_pool_pre_ping.py`); a query running when the database went fails+ # with the ping or without it. How soon a connection left# open by a failover is known to be dead is up to `db_tcp_keepalive_*` and# `db_tcp_user_timeout` below, with the ping or without it.db_pool_pre_ping: bool = Falsediff --git a/core/switch_core/db/engine.py b/core/switch_core/db/engine.pyindex 3e760eaf8..ee7d987b5 100644--- a/core/switch_core/db/engine.py+++ b/core/switch_core/db/engine.py@@ -3,6 +3,7 @@ import socketfrom typing import Anyfrom sqlalchemy import event+from sqlalchemy.exc import DisconnectionErrorfrom sqlalchemy.ext.asyncio import (AsyncEngine,AsyncSession,@@ -159,9 +160,24 @@ def create_engine_from_config("connect_args": connect_args,}defaults.update(engine_kwargs)- return _with_dead_peer_detection(- create_async_engine(config.database_url, **defaults), config- )+ engine = create_async_engine(config.database_url, **defaults)+ event.listen(engine.sync_engine, "checkout", _refuse_closed_connection)+ return _with_dead_peer_detection(engine, config)+++def _refuse_closed_connection(dbapi_connection: Any, _record: Any, _proxy: Any) -> None:+ """Swap a pooled connection that is already closed for a fresh one.++ An anyio cancel scope re-delivers its cancellation at every await, so a+ request cancelled mid-query (its client went away) is cancelled again+ inside SQLAlchemy's invalidation of the connection. That unwinds before the+ pool record lets go of the connection, and the record is returned to the+ pool still holding it, closed. Raising `DisconnectionError` here makes the+ pool discard it and connect afresh rather than fail the next borrower.+ Free: `is_closed` reads local state, unlike the pre-ping's round trip.+ """+ if dbapi_connection.driver_connection.is_closed():+ raise DisconnectionError("pooled connection was already closed")def create_unpooled_engine(config: SwitchConfig) -> AsyncEngine:diff --git a/core/tests/switch_core/db/test_cancelled_checkout.py b/core/tests/switch_core/db/test_cancelled_checkout.pynew file mode 100644index 000000000..e7378aef3--- /dev/null+++ b/core/tests/switch_core/db/test_cancelled_checkout.py@@ -0,0 +1,97 @@+"""A request cancelled mid-query must not hand its dead connection to the next.++Starlette cancels a request whose client went away through an anyio cancel+scope, which re-delivers the cancellation at every await. SQLAlchemy reacts to+the first by invalidating the connection, and the second then lands inside+that invalidation's terminate, which unwinds before the pool record forgets+the connection. The record goes back to the pool still holding the closed+asyncpg connection, and whoever checks it out next failed with "connection is+closed" until the engine learned to refuse a closed connection at checkout.+"""++from __future__ import annotations++import asyncio+from typing import Annotated++import anyio+from fastapi import Depends+from sqlalchemy import text+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker++from switch_core.db.engine import create_engine_from_config, create_session_factory+from switch_core.gateway import dependencies as gw_deps+from switch_core.gateway.auth import get_current_user+from tests.switch_core.db.test_dead_peer_detection import _config_for+from tests.switch_core.management.harness import add_member, build_harness, cookies_for+++async def test_a_cancelled_query_does_not_poison_the_next_checkout(+ postgres_url: str,+) -> None:+ engine = create_engine_from_config(+ _config_for(postgres_url, db_pool_size=1, db_max_overflow=0)+ )+ try:+ with anyio.move_on_after(0.2) as scope:+ async with engine.connect() as conn:+ await conn.execute(text("select pg_sleep(5)"))+ assert scope.cancelled_caught++ async with engine.connect() as conn:+ assert (await conn.execute(text("select 1"))).scalar_one() == 1+ finally:+ await engine.dispose()+++async def test_cancelling_one_request_does_not_fail_a_concurrent_one(+ postgres_url: str, session_factory: async_sessionmaker[AsyncSession]+) -> None:+ # One pooled connection, so the concurrent request is handed exactly the+ # one the cancelled request was using.+ engine = create_engine_from_config(+ _config_for(postgres_url, db_pool_size=1, db_max_overflow=0)+ )+ try:+ harness = build_harness(create_session_factory(engine))+ sleeping = asyncio.Event()++ @harness.gateway_app.get("/test/slow")+ async def slow(+ _user: Annotated[object, Depends(get_current_user)],+ session: Annotated[AsyncSession, Depends(gw_deps.get_session)],+ ) -> None:+ sleeping.set()+ await session.execute(text("select pg_sleep(5)"))++ @harness.gateway_app.get("/test/fast")+ async def fast(+ _user: Annotated[object, Depends(get_current_user)],+ session: Annotated[AsyncSession, Depends(gw_deps.get_session)],+ ) -> int:+ return int((await session.execute(text("select 1"))).scalar_one())++ ada = await add_member(session_factory, "ada")+ cookies = cookies_for(ada)+ async with harness.client() as client:+ async with anyio.create_task_group() as group:+ slow_scope = anyio.CancelScope()++ async def run_slow() -> None:+ with slow_scope:+ await client.get("/gateway/test/slow", cookies=cookies)+ assert slow_scope.cancelled_caught++ group.start_soon(run_slow)+ await sleeping.wait()+ concurrent = asyncio.ensure_future(+ client.get("/gateway/test/fast", cookies=cookies)+ )+ await asyncio.sleep(0.1)+ slow_scope.cancel()++ assert (await concurrent).status_code == 200+ after = await client.get("/gateway/test/fast", cookies=cookies)+ assert after.status_code == 200+ finally:+ await engine.dispose()diff --git a/core/tests/switch_core/db/test_pool_pre_ping.py b/core/tests/switch_core/db/test_pool_pre_ping.pyindex a7b695248..140b545f4 100644--- a/core/tests/switch_core/db/test_pool_pre_ping.py+++ b/core/tests/switch_core/db/test_pool_pre_ping.py@@ -2,19 +2,18 @@A pooled connection the server closed while it sat in the pool is the casethe pre-ping exists for: a database restart closes every connection the pool-holds. With the pre-ping off, the first query on one of them fails, and-SQLAlchemy then treats every connection the pool held at that moment as-stale and replaces it on its next checkout. So a restart costs one failed-query, not one per pooled connection, which is the trade the default makes.+holds. When the server says so as it goes, as a restart does, asyncpg marks+the connection closed and the engine's checkout check swaps it for a fresh one+with no round trip, so no query fails with the pre-ping off either. What the+ping still catches is a peer that vanished without closing the socket, which+is `db_tcp_keepalive_*`'s job as much as the ping's."""from __future__ import annotationsimport asyncio-import pytestfrom sqlalchemy import text-from sqlalchemy.exc import DBAPIErrorfrom sqlalchemy.ext.asyncio import AsyncConnection, AsyncEnginefrom switch_core.config import SwitchConfig@@ -76,18 +75,11 @@ def test_the_pre_ping_is_off_by_default() -> None:assert SwitchConfig.model_fields["db_pool_pre_ping"].default is False-async def test_without_the_pre_ping_a_restart_costs_one_failed_query(+async def test_without_the_pre_ping_a_restart_costs_no_failed_query(postgres_url: str,) -> None:engine, closed = await _fill_and_close_pool(postgres_url, pre_ping=False)try:- with pytest.raises(DBAPIError) as failed:- async with engine.connect() as conn:- await _pid(conn)- assert failed.value.connection_invalidated-- # Every connection after it is a new one, though two more of the- # closed ones were still in the pool when the first query failed.for _ in range(POOLED):async with engine.connect() as conn:assert await _pid(conn) not in closed