Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Try a controller's delivery pass again when the pool has no connection, instead of closing its socket (CHOO-3667)
Abel Dantas6d38dbf73fcd
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 36f64bc9b..6376e560e 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -160,6 +160,13 @@ version of their own to them without also giving them a release of their own.all its agents to find the work, again after delivering, and again beforewaiting. It now reads them once a pass, and an event that arrives afterthat read wakes the stream for the next pass.+- **A controller keeps its socket when a delivery pass cannot get a+ database connection.** When a pod was too busy to hand its streams a+ connection within the pool timeout, the controller's stream ended and its+ socket closed, and every agent on it reconnected, which needs more of+ what was short. The pass is now tried again after a short wait, doubling+ to 5 seconds, and the socket closes only if it is still failing after a+ minute.- **An agent's stream reads in one database session per pass, and pooledconnections are no longer pinged before use.** A pass that delivered amessage opened a session for each of its reads, six pool checkouts and sixdiff --git a/core/switch_core/bridges/agent/protocol/controller_stream.py b/core/switch_core/bridges/agent/protocol/controller_stream.pyindex 57d7e4d73..c51849558 100644--- a/core/switch_core/bridges/agent/protocol/controller_stream.py+++ b/core/switch_core/bridges/agent/protocol/controller_stream.py@@ -41,8 +41,9 @@ from __future__ import annotationsimport asyncioimport logging+import randomimport time-from collections.abc import AsyncGenerator, Awaitable, Callable+from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callablefrom dataclasses import dataclass, fieldfrom typing import TYPE_CHECKING, Any, Protocol@@ -54,6 +55,7 @@ from switch_core.bridges.agent.protocol.controller_presence import (from switch_core.bridges.agent.protocol.event_buffer import (CursorExpiredError,EventBuffer,+ transient_failure,)from switch_core.bridges.agent.protocol.liveness import TAKEN_OVER, Closurefrom switch_core.tenant_context import current_tenant_id@@ -66,6 +68,15 @@ logger = logging.getLogger(__name__)IDLE_INTERVAL_SECONDS = 15.0CATCH_UP_BATCH = 200+# A pass whose database work fails because the connection or the pool did is+# tried again after a wait that doubles from the first to the most, jittered,+# rather than ending the stream: ending it closes the controller's socket, and+# every agent on it reconnects, which costs more of what was short. A pass+# still failing after the last is given up on, and the socket closes.+PASS_RETRY_FIRST_SECONDS = 0.5+PASS_RETRY_MOST_SECONDS = 5.0+PASS_RETRY_GIVE_UP_SECONDS = 60.0+ControllerFrame = dict[str, Any]# Yielded when nothing happened for the idle interval; never sent.IDLE: ControllerFrame = {"event": "idle", "data": {}}@@ -107,6 +118,16 @@ class _Attached:counted: set[str] = field(default_factory=set)+def _retry_wait(failures: int) -> float:+ """Seconds to wait before trying a pass again after its `failures`-th+ failure in a row. Jittered, so the streams that failed together do not+ all try again together."""+ most = min(+ PASS_RETRY_FIRST_SECONDS * 2.0 ** (failures - 1), PASS_RETRY_MOST_SECONDS+ )+ return most * (0.5 + random.random() / 2)++def _eviction(closure: Closure) -> dict[str, Any]:return {"code": closure.code, "reason": closure.message}@@ -170,6 +191,9 @@ class _ControllerStream:conn = self._conntry:yield frame("connection_state", opening)+ # Since when, and how many times in a row, the pass has failed.+ failing_since: float | None = None+ failures = 0while True:for event, data in self._nudges.drain():yield frame(event, data)@@ -189,49 +213,69 @@ class _ControllerStream:for raw in self._detach_departed():yield raw- for raw in await self._attach_arrivals():- yield raw+ try:+ async for raw in self._attach_arrivals():+ yield raw- for agent_id in sorted(self._presence.take_rooms_changed(conn)):- if agent_id in self._attached:+ for agent_id in sorted(self._presence.take_rooms_changed(conn)):+ if agent_id in self._attached:+ yield frame(+ "agent.rooms",+ {+ "agent_id": agent_id,+ "rooms": sorted(self._presence.rooms(agent_id)),+ },+ )++ for agent_id, command in self._presence.take_session_commands(conn):+ origin = command.get("origin")+ room_id = (+ origin.get("roomId") if isinstance(origin, dict) else None+ )yield frame(- "agent.rooms",+ "agent.session_command",{"agent_id": agent_id,- "rooms": sorted(self._presence.rooms(agent_id)),+ "room_id": room_id,+ "command": command,},)- for agent_id, command in self._presence.take_session_commands(conn):- origin = command.get("origin")- room_id = origin.get("roomId") if isinstance(origin, dict) else None- yield frame(- "agent.session_command",- {"agent_id": agent_id, "room_id": room_id, "command": command},- )-- for agent_id, room_id in self._presence.take_warms(conn):- yield frame(- "agent.warm", {"agent_id": agent_id, "room_id": room_id}- )-- for raw in await self._owed_outcomes():- yield raw+ for agent_id, room_id in self._presence.take_warms(conn):+ yield frame(+ "agent.warm", {"agent_id": agent_id, "room_id": room_id}+ )- # Cleared before the pass reads the heads, so an event added- # after that read rings its agent's doorbell again and the- # wait below returns at once. The pass is then the only read- # of the heads, rather than the heads being read again after- # delivering and again before waiting.- for agent_id in self._attached:- self._buffer.doorbell(agent_id).clear()- behind = False- for attached, head in await self._due():- for raw in await self._deliver(attached, head):+ for raw in await self._owed_outcomes():yield raw- # A delivery stops at `CATCH_UP_BATCH` events or at a gap,- # and no doorbell rings for events already there.- behind = behind or attached.cursor < head++ # Cleared before the pass reads the heads, so an event+ # added after that read rings its agent's doorbell again+ # and the wait below returns at once. The pass is then the+ # only read of the heads, rather than the heads being read+ # again after delivering and again before waiting.+ for agent_id in self._attached:+ self._buffer.doorbell(agent_id).clear()+ behind = False+ for attached, head in await self._due():+ for raw in await self._deliver(attached, head):+ yield raw+ # A delivery stops at `CATCH_UP_BATCH` events or at a+ # gap, and no doorbell rings for events already there.+ behind = behind or attached.cursor < head+ except Exception as exc:+ if not transient_failure(exc):+ raise+ now = time.monotonic()+ if failing_since is None:+ failing_since = now+ failures += 1+ if now - failing_since >= PASS_RETRY_GIVE_UP_SECONDS:+ raise+ self._note_failed_pass(exc, failures)+ await asyncio.sleep(_retry_wait(failures))+ continue+ failing_since, failures = None, 0if behind:continue@@ -244,6 +288,20 @@ class _ControllerStream:self._nudges.close()self._presence.detach_stream(conn, self._token)+ def _note_failed_pass(self, exc: Exception, failures: int) -> None:+ # The first failure in a row with its traceback, then every tenth.+ if failures == 1 or failures % 10 == 0:+ logger.warning(+ "[CONTROLLER] controller=%s connection=%s: a delivery pass "+ "failed (%s, %d in a row); trying it again rather than closing "+ "the socket",+ self._conn.controller_id,+ self._conn.id,+ type(exc).__name__,+ failures,+ exc_info=failures == 1,+ )+def _ending(self) -> Closure | None:conn = self._connif conn.closure is not None:@@ -288,9 +346,12 @@ class _ControllerStream:)return frames- async def _attach_arrivals(self) -> list[ControllerFrame]:+ async def _attach_arrivals(self) -> AsyncIterator[ControllerFrame]:+ """Attach each agent newly bound to the controller, yielding its+ frames once it is. Nothing about an agent is kept until its reads and+ writes are done, so an agent whose pass fails part way is attached+ from the start by the next pass, its gap told then."""conn = self._conn- frames: list[ControllerFrame] = []for agent_id in sorted(self._presence.agents_of(conn.controller_id) - set(self._attached)):@@ -340,30 +401,29 @@ class _ControllerStream:})cursor = exc.resume_after+ reader_id = self._presence.holder_id(binding)+ # Give each of the agent's rooms that has no unread-count baseline+ # yet a baseline at the room's newest message.+ for room_id in sorted(rooms):+ await self._buffer.ensure_counting(+ self._tenant, agent_id, reader_id, room_id+ )conn.resume_cursors[agent_id] = cursorconn.sent_cursors[agent_id] = cursorattached = _Attached(agent_id=agent_id,- reader_id=self._presence.holder_id(binding),+ reader_id=reader_id,cursor=cursor,+ counted=set(rooms),)self._subscribe_outcomes(attached)self._attached[agent_id] = attached- # Give each of the agent's rooms that has no unread-count baseline- # yet a baseline at the room's newest message.- for room_id in rooms - attached.counted:- await self._buffer.ensure_counting(- self._tenant, agent_id, attached.reader_id, room_id- )- attached.counted.add(room_id)- frames.append(- frame(- "agent.attached",- {"agent_id": agent_id, "from_seq": cursor, "rooms": sorted(rooms)},- )+ yield frame(+ "agent.attached",+ {"agent_id": agent_id, "from_seq": cursor, "rooms": sorted(rooms)},)- frames.extend(frame("agent.gap", gap) for gap in gaps)- return frames+ for gap in gaps:+ yield frame("agent.gap", gap)def _subscribe_outcomes(self, attached: _Attached) -> None:if self._approvals is None or self._tenant_id is None:@@ -386,8 +446,9 @@ class _ControllerStream:frames: list[ControllerFrame] = []for attached in list(self._attached.values()):if attached.resync and self._approvals is not None:+ undelivered = await self._approvals.undelivered(attached.agent_id)attached.resync = False- for outcome in await self._approvals.undelivered(attached.agent_id):+ for outcome in undelivered:attached.outcomes[(outcome["session_id"], outcome["request_id"])] = outcomediff --git a/core/switch_core/bridges/agent/protocol/event_buffer.py b/core/switch_core/bridges/agent/protocol/event_buffer.pyindex b3340d088..d1a80a1c5 100644--- a/core/switch_core/bridges/agent/protocol/event_buffer.py+++ b/core/switch_core/bridges/agent/protocol/event_buffer.py@@ -667,7 +667,7 @@ class EventBuffer:)breakexcept Exception as exc:- if delay is None or not _transient(exc):+ if delay is None or not transient_failure(exc):raiselogger.warning("[EVENT-BUF] storing an own event for agent %s in room %s "@@ -2210,7 +2210,7 @@ class EventBuffer:return max(dropped_through or 0, (oldest or 0) - 1, 0)-def _transient(exc: BaseException) -> bool:+def transient_failure(exc: BaseException) -> bool:"""Whether a failed database call is worth retrying: the connection or theconnection pool failed, not the statement itself."""if isinstance(exc, OperationalError | InterfaceError | PoolTimeoutError):diff --git a/core/tests/switch_core/management/test_stream.py b/core/tests/switch_core/management/test_stream.pyindex b7b8337f2..495a1df63 100644--- a/core/tests/switch_core/management/test_stream.py+++ b/core/tests/switch_core/management/test_stream.py@@ -12,6 +12,7 @@ import contextlibimport timeimport pytest+from sqlalchemy.exc import TimeoutError as PoolTimeoutErrorfrom sqlalchemy.ext.asyncio import AsyncSession, async_sessionmakerfrom switch_core.bridges.agent.commands import room_control_frame@@ -898,3 +899,122 @@ async def test_a_backlog_past_one_batch_is_sent_without_waiting(assert [data["event"]["payload"]["body"] for _, data in frames] == [f"#{n}" for n in range(5)]+++async def test_a_pass_that_cannot_get_a_connection_is_tried_again(+ harness: Harness, monkeypatch: pytest.MonkeyPatch+) -> None:+ """A pass that fails because the pool had no connection for it is tried+ again, and its event is delivered, rather than the stream ending and the+ controller's socket closing with every agent on it."""+ monkeypatch.setattr(controller_stream, "PASS_RETRY_FIRST_SECONDS", 0.01)+ owner = await add_member(harness.session_factory, "ada")+ async with harness.client() as client:+ controller = await enroll_console(harness, client, owner)+ agent_id = await place_agent(client, controller, name="reviewer")+ room_id = await add_room(harness.session_factory, agent_id)+ opened = await open_connection(client, controller)+ stream = await open_stream(harness, controller, opened)+ await take(stream, 2)++ buffer = harness.protocol.event_buffer+ read_from = buffer.read_from+ failed: list[bool] = []++ async def no_connection_the_first_time(*args: object, **kwargs: object) -> object:+ if not failed:+ failed.append(True)+ raise PoolTimeoutError("QueuePool limit reached, connection timed out")+ return await read_from(*args, **kwargs) # type: ignore[arg-type]++ monkeypatch.setattr(buffer, "read_from", no_connection_the_first_time)+ await _ask(harness, agent_id, room_id, "@you")+ [(name, data)] = await take(stream, 1)+ await stream.aclose()++ assert failed == [True]+ assert name == "agent.event"+ assert data["event"]["payload"]["body"] == "@you"+++async def test_a_pass_that_fails_for_another_reason_ends_the_stream(+ harness: Harness, monkeypatch: pytest.MonkeyPatch+) -> None:+ owner = await add_member(harness.session_factory, "ada")+ async with harness.client() as client:+ controller = await enroll_console(harness, client, owner)+ agent_id = await place_agent(client, controller, name="reviewer")+ room_id = await add_room(harness.session_factory, agent_id)+ opened = await open_connection(client, controller)+ stream = await open_stream(harness, controller, opened)+ await take(stream, 2)++ async def broken(*args: object, **kwargs: object) -> object:+ raise ValueError("a bug, not the database")++ monkeypatch.setattr(harness.protocol.event_buffer, "read_from", broken)+ await _ask(harness, agent_id, room_id, "@you")+ with pytest.raises(ValueError, match="a bug"):+ await take(stream, 1)+++async def test_a_pass_still_failing_after_the_last_wait_ends_the_stream(+ harness: Harness, monkeypatch: pytest.MonkeyPatch+) -> None:+ monkeypatch.setattr(controller_stream, "PASS_RETRY_FIRST_SECONDS", 0.01)+ monkeypatch.setattr(controller_stream, "PASS_RETRY_GIVE_UP_SECONDS", 0.2)+ owner = await add_member(harness.session_factory, "ada")+ async with harness.client() as client:+ controller = await enroll_console(harness, client, owner)+ agent_id = await place_agent(client, controller, name="reviewer")+ room_id = await add_room(harness.session_factory, agent_id)+ opened = await open_connection(client, controller)+ stream = await open_stream(harness, controller, opened)+ await take(stream, 2)++ async def never_a_connection(*args: object, **kwargs: object) -> object:+ raise PoolTimeoutError("QueuePool limit reached, connection timed out")++ monkeypatch.setattr(harness.protocol.event_buffer, "read_from", never_a_connection)+ await _ask(harness, agent_id, room_id, "@you")+ with pytest.raises(PoolTimeoutError):+ await take(stream, 1, timeout=5)+++async def test_an_attach_that_fails_part_way_is_done_again_and_told_once(+ harness: Harness, monkeypatch: pytest.MonkeyPatch+) -> None:+ """Nothing about an agent is kept until its attach is done, so one whose+ attach fails is attached again from the start, and the controller hears+ it attached once."""+ monkeypatch.setattr(controller_stream, "PASS_RETRY_FIRST_SECONDS", 0.01)+ owner = await add_member(harness.session_factory, "ada")+ async with harness.client() as client:+ controller = await enroll_console(harness, client, owner)+ agent_id = await place_agent(client, controller, name="reviewer")+ room_id = await add_room(harness.session_factory, agent_id)+ opened = await open_connection(client, controller)++ buffer = harness.protocol.event_buffer+ ensure_counting = buffer.ensure_counting+ failed: list[bool] = []++ async def no_connection_the_first_time(*args: object, **kwargs: object) -> object:+ if not failed:+ failed.append(True)+ raise PoolTimeoutError("QueuePool limit reached, connection timed out")+ return await ensure_counting(*args, **kwargs) # type: ignore[arg-type]++ monkeypatch.setattr(buffer, "ensure_counting", no_connection_the_first_time)+ stream = await open_stream(harness, controller, opened)+ opening = await take(stream, 2)+ await _ask(harness, agent_id, room_id, "@you")+ [(name, _)] = await take(stream, 1)+ await stream.aclose()++ assert failed == [True]+ assert [frame_name for frame_name, _ in opening] == [+ "connection_state",+ "agent.attached",+ ]+ assert name == "agent.event"