Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Try a controller's delivery pass again when the pool has no connection, instead of closing its socket (CHOO-3667)

Abel Dantas6d38dbf73fcd
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 36f64bc9b..6376e560e 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -160,6 +160,13 @@ version of their own to them without also giving them a release of their own.
all its agents to find the work, again after delivering, and again before
waiting. It now reads them once a pass, and an event that arrives after
that read wakes the stream for the next pass.
+- **A controller keeps its socket when a delivery pass cannot get a
+ database connection.** When a pod was too busy to hand its streams a
+ connection within the pool timeout, the controller's stream ended and its
+ socket closed, and every agent on it reconnected, which needs more of
+ what was short. The pass is now tried again after a short wait, doubling
+ to 5 seconds, and the socket closes only if it is still failing after a
+ minute.
- **An agent's stream reads in one database session per pass, and pooled
connections are no longer pinged before use.** A pass that delivered a
message opened a session for each of its reads, six pool checkouts and six
diff --git a/core/switch_core/bridges/agent/protocol/controller_stream.py b/core/switch_core/bridges/agent/protocol/controller_stream.py
index 57d7e4d73..c51849558 100644
--- a/core/switch_core/bridges/agent/protocol/controller_stream.py
+++ b/core/switch_core/bridges/agent/protocol/controller_stream.py
@@ -41,8 +41,9 @@ from __future__ import annotations
import asyncio
import logging
+import random
import time
-from collections.abc import AsyncGenerator, Awaitable, Callable
+from collections.abc import AsyncGenerator, AsyncIterator, Awaitable, Callable
from dataclasses import dataclass, field
from typing import TYPE_CHECKING, Any, Protocol
@@ -54,6 +55,7 @@ from switch_core.bridges.agent.protocol.controller_presence import (
from switch_core.bridges.agent.protocol.event_buffer import (
CursorExpiredError,
EventBuffer,
+ transient_failure,
)
from switch_core.bridges.agent.protocol.liveness import TAKEN_OVER, Closure
from switch_core.tenant_context import current_tenant_id
@@ -66,6 +68,15 @@ logger = logging.getLogger(__name__)
IDLE_INTERVAL_SECONDS = 15.0
CATCH_UP_BATCH = 200
+# A pass whose database work fails because the connection or the pool did is
+# tried again after a wait that doubles from the first to the most, jittered,
+# rather than ending the stream: ending it closes the controller's socket, and
+# every agent on it reconnects, which costs more of what was short. A pass
+# still failing after the last is given up on, and the socket closes.
+PASS_RETRY_FIRST_SECONDS = 0.5
+PASS_RETRY_MOST_SECONDS = 5.0
+PASS_RETRY_GIVE_UP_SECONDS = 60.0
+
ControllerFrame = dict[str, Any]
# Yielded when nothing happened for the idle interval; never sent.
IDLE: ControllerFrame = {"event": "idle", "data": {}}
@@ -107,6 +118,16 @@ class _Attached:
counted: set[str] = field(default_factory=set)
+def _retry_wait(failures: int) -> float:
+ """Seconds to wait before trying a pass again after its `failures`-th
+ failure in a row. Jittered, so the streams that failed together do not
+ all try again together."""
+ most = min(
+ PASS_RETRY_FIRST_SECONDS * 2.0 ** (failures - 1), PASS_RETRY_MOST_SECONDS
+ )
+ return most * (0.5 + random.random() / 2)
+
+
def _eviction(closure: Closure) -> dict[str, Any]:
return {"code": closure.code, "reason": closure.message}
@@ -170,6 +191,9 @@ class _ControllerStream:
conn = self._conn
try:
yield frame("connection_state", opening)
+ # Since when, and how many times in a row, the pass has failed.
+ failing_since: float | None = None
+ failures = 0
while True:
for event, data in self._nudges.drain():
yield frame(event, data)
@@ -189,49 +213,69 @@ class _ControllerStream:
for raw in self._detach_departed():
yield raw
- for raw in await self._attach_arrivals():
- yield raw
+ try:
+ async for raw in self._attach_arrivals():
+ yield raw
- for agent_id in sorted(self._presence.take_rooms_changed(conn)):
- if agent_id in self._attached:
+ for agent_id in sorted(self._presence.take_rooms_changed(conn)):
+ if agent_id in self._attached:
+ yield frame(
+ "agent.rooms",
+ {
+ "agent_id": agent_id,
+ "rooms": sorted(self._presence.rooms(agent_id)),
+ },
+ )
+
+ for agent_id, command in self._presence.take_session_commands(conn):
+ origin = command.get("origin")
+ room_id = (
+ origin.get("roomId") if isinstance(origin, dict) else None
+ )
yield frame(
- "agent.rooms",
+ "agent.session_command",
{
"agent_id": agent_id,
- "rooms": sorted(self._presence.rooms(agent_id)),
+ "room_id": room_id,
+ "command": command,
},
)
- for agent_id, command in self._presence.take_session_commands(conn):
- origin = command.get("origin")
- room_id = origin.get("roomId") if isinstance(origin, dict) else None
- yield frame(
- "agent.session_command",
- {"agent_id": agent_id, "room_id": room_id, "command": command},
- )
-
- for agent_id, room_id in self._presence.take_warms(conn):
- yield frame(
- "agent.warm", {"agent_id": agent_id, "room_id": room_id}
- )
-
- for raw in await self._owed_outcomes():
- yield raw
+ for agent_id, room_id in self._presence.take_warms(conn):
+ yield frame(
+ "agent.warm", {"agent_id": agent_id, "room_id": room_id}
+ )
- # Cleared before the pass reads the heads, so an event added
- # after that read rings its agent's doorbell again and the
- # wait below returns at once. The pass is then the only read
- # of the heads, rather than the heads being read again after
- # delivering and again before waiting.
- for agent_id in self._attached:
- self._buffer.doorbell(agent_id).clear()
- behind = False
- for attached, head in await self._due():
- for raw in await self._deliver(attached, head):
+ for raw in await self._owed_outcomes():
yield raw
- # A delivery stops at `CATCH_UP_BATCH` events or at a gap,
- # and no doorbell rings for events already there.
- behind = behind or attached.cursor < head
+
+ # Cleared before the pass reads the heads, so an event
+ # added after that read rings its agent's doorbell again
+ # and the wait below returns at once. The pass is then the
+ # only read of the heads, rather than the heads being read
+ # again after delivering and again before waiting.
+ for agent_id in self._attached:
+ self._buffer.doorbell(agent_id).clear()
+ behind = False
+ for attached, head in await self._due():
+ for raw in await self._deliver(attached, head):
+ yield raw
+ # A delivery stops at `CATCH_UP_BATCH` events or at a
+ # gap, and no doorbell rings for events already there.
+ behind = behind or attached.cursor < head
+ except Exception as exc:
+ if not transient_failure(exc):
+ raise
+ now = time.monotonic()
+ if failing_since is None:
+ failing_since = now
+ failures += 1
+ if now - failing_since >= PASS_RETRY_GIVE_UP_SECONDS:
+ raise
+ self._note_failed_pass(exc, failures)
+ await asyncio.sleep(_retry_wait(failures))
+ continue
+ failing_since, failures = None, 0
if behind:
continue
@@ -244,6 +288,20 @@ class _ControllerStream:
self._nudges.close()
self._presence.detach_stream(conn, self._token)
+ def _note_failed_pass(self, exc: Exception, failures: int) -> None:
+ # The first failure in a row with its traceback, then every tenth.
+ if failures == 1 or failures % 10 == 0:
+ logger.warning(
+ "[CONTROLLER] controller=%s connection=%s: a delivery pass "
+ "failed (%s, %d in a row); trying it again rather than closing "
+ "the socket",
+ self._conn.controller_id,
+ self._conn.id,
+ type(exc).__name__,
+ failures,
+ exc_info=failures == 1,
+ )
+
def _ending(self) -> Closure | None:
conn = self._conn
if conn.closure is not None:
@@ -288,9 +346,12 @@ class _ControllerStream:
)
return frames
- async def _attach_arrivals(self) -> list[ControllerFrame]:
+ async def _attach_arrivals(self) -> AsyncIterator[ControllerFrame]:
+ """Attach each agent newly bound to the controller, yielding its
+ frames once it is. Nothing about an agent is kept until its reads and
+ writes are done, so an agent whose pass fails part way is attached
+ from the start by the next pass, its gap told then."""
conn = self._conn
- frames: list[ControllerFrame] = []
for agent_id in sorted(
self._presence.agents_of(conn.controller_id) - set(self._attached)
):
@@ -340,30 +401,29 @@ class _ControllerStream:
}
)
cursor = exc.resume_after
+ reader_id = self._presence.holder_id(binding)
+ # Give each of the agent's rooms that has no unread-count baseline
+ # yet a baseline at the room's newest message.
+ for room_id in sorted(rooms):
+ await self._buffer.ensure_counting(
+ self._tenant, agent_id, reader_id, room_id
+ )
conn.resume_cursors[agent_id] = cursor
conn.sent_cursors[agent_id] = cursor
attached = _Attached(
agent_id=agent_id,
- reader_id=self._presence.holder_id(binding),
+ reader_id=reader_id,
cursor=cursor,
+ counted=set(rooms),
)
self._subscribe_outcomes(attached)
self._attached[agent_id] = attached
- # Give each of the agent's rooms that has no unread-count baseline
- # yet a baseline at the room's newest message.
- for room_id in rooms - attached.counted:
- await self._buffer.ensure_counting(
- self._tenant, agent_id, attached.reader_id, room_id
- )
- attached.counted.add(room_id)
- frames.append(
- frame(
- "agent.attached",
- {"agent_id": agent_id, "from_seq": cursor, "rooms": sorted(rooms)},
- )
+ yield frame(
+ "agent.attached",
+ {"agent_id": agent_id, "from_seq": cursor, "rooms": sorted(rooms)},
)
- frames.extend(frame("agent.gap", gap) for gap in gaps)
- return frames
+ for gap in gaps:
+ yield frame("agent.gap", gap)
def _subscribe_outcomes(self, attached: _Attached) -> None:
if self._approvals is None or self._tenant_id is None:
@@ -386,8 +446,9 @@ class _ControllerStream:
frames: list[ControllerFrame] = []
for attached in list(self._attached.values()):
if attached.resync and self._approvals is not None:
+ undelivered = await self._approvals.undelivered(attached.agent_id)
attached.resync = False
- for outcome in await self._approvals.undelivered(attached.agent_id):
+ for outcome in undelivered:
attached.outcomes[
(outcome["session_id"], outcome["request_id"])
] = outcome
diff --git a/core/switch_core/bridges/agent/protocol/event_buffer.py b/core/switch_core/bridges/agent/protocol/event_buffer.py
index b3340d088..d1a80a1c5 100644
--- a/core/switch_core/bridges/agent/protocol/event_buffer.py
+++ b/core/switch_core/bridges/agent/protocol/event_buffer.py
@@ -667,7 +667,7 @@ class EventBuffer:
)
break
except Exception as exc:
- if delay is None or not _transient(exc):
+ if delay is None or not transient_failure(exc):
raise
logger.warning(
"[EVENT-BUF] storing an own event for agent %s in room %s "
@@ -2210,7 +2210,7 @@ class EventBuffer:
return max(dropped_through or 0, (oldest or 0) - 1, 0)
-def _transient(exc: BaseException) -> bool:
+def transient_failure(exc: BaseException) -> bool:
"""Whether a failed database call is worth retrying: the connection or the
connection pool failed, not the statement itself."""
if isinstance(exc, OperationalError | InterfaceError | PoolTimeoutError):
diff --git a/core/tests/switch_core/management/test_stream.py b/core/tests/switch_core/management/test_stream.py
index b7b8337f2..495a1df63 100644
--- a/core/tests/switch_core/management/test_stream.py
+++ b/core/tests/switch_core/management/test_stream.py
@@ -12,6 +12,7 @@ import contextlib
import time
import pytest
+from sqlalchemy.exc import TimeoutError as PoolTimeoutError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from switch_core.bridges.agent.commands import room_control_frame
@@ -898,3 +899,122 @@ async def test_a_backlog_past_one_batch_is_sent_without_waiting(
assert [data["event"]["payload"]["body"] for _, data in frames] == [
f"#{n}" for n in range(5)
]
+
+
+async def test_a_pass_that_cannot_get_a_connection_is_tried_again(
+ harness: Harness, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """A pass that fails because the pool had no connection for it is tried
+ again, and its event is delivered, rather than the stream ending and the
+ controller's socket closing with every agent on it."""
+ monkeypatch.setattr(controller_stream, "PASS_RETRY_FIRST_SECONDS", 0.01)
+ owner = await add_member(harness.session_factory, "ada")
+ async with harness.client() as client:
+ controller = await enroll_console(harness, client, owner)
+ agent_id = await place_agent(client, controller, name="reviewer")
+ room_id = await add_room(harness.session_factory, agent_id)
+ opened = await open_connection(client, controller)
+ stream = await open_stream(harness, controller, opened)
+ await take(stream, 2)
+
+ buffer = harness.protocol.event_buffer
+ read_from = buffer.read_from
+ failed: list[bool] = []
+
+ async def no_connection_the_first_time(*args: object, **kwargs: object) -> object:
+ if not failed:
+ failed.append(True)
+ raise PoolTimeoutError("QueuePool limit reached, connection timed out")
+ return await read_from(*args, **kwargs) # type: ignore[arg-type]
+
+ monkeypatch.setattr(buffer, "read_from", no_connection_the_first_time)
+ await _ask(harness, agent_id, room_id, "@you")
+ [(name, data)] = await take(stream, 1)
+ await stream.aclose()
+
+ assert failed == [True]
+ assert name == "agent.event"
+ assert data["event"]["payload"]["body"] == "@you"
+
+
+async def test_a_pass_that_fails_for_another_reason_ends_the_stream(
+ harness: Harness, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ owner = await add_member(harness.session_factory, "ada")
+ async with harness.client() as client:
+ controller = await enroll_console(harness, client, owner)
+ agent_id = await place_agent(client, controller, name="reviewer")
+ room_id = await add_room(harness.session_factory, agent_id)
+ opened = await open_connection(client, controller)
+ stream = await open_stream(harness, controller, opened)
+ await take(stream, 2)
+
+ async def broken(*args: object, **kwargs: object) -> object:
+ raise ValueError("a bug, not the database")
+
+ monkeypatch.setattr(harness.protocol.event_buffer, "read_from", broken)
+ await _ask(harness, agent_id, room_id, "@you")
+ with pytest.raises(ValueError, match="a bug"):
+ await take(stream, 1)
+
+
+async def test_a_pass_still_failing_after_the_last_wait_ends_the_stream(
+ harness: Harness, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ monkeypatch.setattr(controller_stream, "PASS_RETRY_FIRST_SECONDS", 0.01)
+ monkeypatch.setattr(controller_stream, "PASS_RETRY_GIVE_UP_SECONDS", 0.2)
+ owner = await add_member(harness.session_factory, "ada")
+ async with harness.client() as client:
+ controller = await enroll_console(harness, client, owner)
+ agent_id = await place_agent(client, controller, name="reviewer")
+ room_id = await add_room(harness.session_factory, agent_id)
+ opened = await open_connection(client, controller)
+ stream = await open_stream(harness, controller, opened)
+ await take(stream, 2)
+
+ async def never_a_connection(*args: object, **kwargs: object) -> object:
+ raise PoolTimeoutError("QueuePool limit reached, connection timed out")
+
+ monkeypatch.setattr(harness.protocol.event_buffer, "read_from", never_a_connection)
+ await _ask(harness, agent_id, room_id, "@you")
+ with pytest.raises(PoolTimeoutError):
+ await take(stream, 1, timeout=5)
+
+
+async def test_an_attach_that_fails_part_way_is_done_again_and_told_once(
+ harness: Harness, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ """Nothing about an agent is kept until its attach is done, so one whose
+ attach fails is attached again from the start, and the controller hears
+ it attached once."""
+ monkeypatch.setattr(controller_stream, "PASS_RETRY_FIRST_SECONDS", 0.01)
+ owner = await add_member(harness.session_factory, "ada")
+ async with harness.client() as client:
+ controller = await enroll_console(harness, client, owner)
+ agent_id = await place_agent(client, controller, name="reviewer")
+ room_id = await add_room(harness.session_factory, agent_id)
+ opened = await open_connection(client, controller)
+
+ buffer = harness.protocol.event_buffer
+ ensure_counting = buffer.ensure_counting
+ failed: list[bool] = []
+
+ async def no_connection_the_first_time(*args: object, **kwargs: object) -> object:
+ if not failed:
+ failed.append(True)
+ raise PoolTimeoutError("QueuePool limit reached, connection timed out")
+ return await ensure_counting(*args, **kwargs) # type: ignore[arg-type]
+
+ monkeypatch.setattr(buffer, "ensure_counting", no_connection_the_first_time)
+ stream = await open_stream(harness, controller, opened)
+ opening = await take(stream, 2)
+ await _ask(harness, agent_id, room_id, "@you")
+ [(name, _)] = await take(stream, 1)
+ await stream.aclose()
+
+ assert failed == [True]
+ assert [frame_name for frame_name, _ in opening] == [
+ "connection_state",
+ "agent.attached",
+ ]
+ assert name == "agent.event"