Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Retry a client that fails on the database at start, instead of dropping it (#742)
Abel Dantas3363cdc37c9f
diff --git a/core/switch_core/clients/client_lifecycle_service.py b/core/switch_core/clients/client_lifecycle_service.pyindex 2e7cb6685..feae02da4 100644--- a/core/switch_core/clients/client_lifecycle_service.py+++ b/core/switch_core/clients/client_lifecycle_service.py@@ -2,9 +2,12 @@ from __future__ import annotationsimport asyncioimport logging+import randomimport uuidfrom typing import Any, Protocol, runtime_checkable+from sqlalchemy.exc import InterfaceError, OperationalError+from sqlalchemy.exc import TimeoutError as PoolTimeoutErrorfrom sqlalchemy.ext.asyncio import AsyncSession, async_sessionmakerfrom switch_core.clients.actor import Actor, AgentActor, ClientConfig@@ -23,6 +26,26 @@ from switch_core.tenant_context import no_tenant, tenant_scopelogger = logging.getLogger(__name__)+# A client that fails on the database or the network is retried rather than+# dropped. At boot every client starts at once and asks for a connection in the+# same second; on 9 Oct that drained the pool, 13 clients timed out+# waiting for one, and each was dropped for good: its agent connected, but+# nothing on the server could post for it until the next restart. Such a+# failure says nothing about the client, so it waits and tries again, as long+# as it takes. Anything else is a bug in the client and still ends it.+# The wait is jittered: the clients that drained the pool failed together, and+# a deterministic backoff would keep them retrying together, re-draining it in+# synchronized waves. Each draws its own delay instead.+_TRANSIENT_START_ERRORS: tuple[type[BaseException], ...] = (+ PoolTimeoutError,+ OperationalError,+ InterfaceError,+ OSError,+ TimeoutError,+)+CLIENT_RETRY_BASE_SECONDS = 1.0+CLIENT_RETRY_CAP_SECONDS = 60.0+@runtime_checkableclass _PreloadsAgent(Protocol):@@ -452,10 +475,33 @@ class ClientLifecycleService:"""with no_tenant(), log_context(room_id=None):try:- if consumer is not None:- await consumer.start()- else:- await actor.connect()+ attempt = 0+ while True:+ try:+ if consumer is not None:+ await consumer.start()+ else:+ await actor.connect()+ return+ except _TRANSIENT_START_ERRORS:+ attempt += 1+ delay = random.uniform(+ 0,+ min(+ CLIENT_RETRY_BASE_SECONDS * 2 ** (attempt - 1),+ CLIENT_RETRY_CAP_SECONDS,+ ),+ )+ logger.warning(+ "Client %s (%s) failed on the database or network "+ "(attempt %d); retrying in %.0fs",+ actor.display_name,+ actor.transport_user_id,+ attempt,+ delay,+ exc_info=True,+ )+ await asyncio.sleep(delay)except Exception:logger.exception("Client %s (%s) crashed",diff --git a/core/tests/switch_core/clients/test_client_start_retry.py b/core/tests/switch_core/clients/test_client_start_retry.pynew file mode 100644index 000000000..8103d58c5--- /dev/null+++ b/core/tests/switch_core/clients/test_client_start_retry.py@@ -0,0 +1,84 @@+"""A client that fails on the database at start is retried, not dropped.++On 9 Oct a pilot restart started every client at once, the pool ran dry, and+13 clients timed out waiting for a connection. Each was dropped for good, so+its agent could not post until the next restart.+"""++from __future__ import annotations++import asyncio+from unittest.mock import MagicMock++import pytest+from sqlalchemy.exc import TimeoutError as PoolTimeoutError++from switch_core.clients import client_lifecycle_service as lifecycle+from switch_core.clients.client_lifecycle_service import ClientLifecycleService+++class _FlakyClient:+ """Fails its first `failures` starts with `error`, then runs."""++ def __init__(self, failures: int, error: Exception) -> None:+ self.display_name = "flaky"+ self.transport_user_id = "@flaky:test"+ self.starts = 0+ self._failures = failures+ self._error = error+ self.running = asyncio.Event()++ async def start(self) -> None:+ self.starts += 1+ if self.starts <= self._failures:+ raise self._error+ self.running.set()+ await asyncio.Event().wait()++ async def stop(self) -> None:+ return None+++def _service() -> ClientLifecycleService:+ return ClientLifecycleService(+ provisioning=MagicMock(),+ client_store=MagicMock(),+ tenant_store=MagicMock(),+ client_factory=MagicMock(),+ session_factory=MagicMock(),+ config=MagicMock(),+ tenants_isolated=True,+ )++[email protected](autouse=True)+def _no_wait(monkeypatch: pytest.MonkeyPatch) -> None:+ monkeypatch.setattr(lifecycle, "CLIENT_RETRY_BASE_SECONDS", 0.0)+++async def test_a_pool_timeout_at_start_is_retried_until_the_client_runs() -> None:+ service = _service()+ client = _FlakyClient(failures=2, error=PoolTimeoutError("QueuePool limit reached"))+ service._clients["c1"] = client # type: ignore[assignment]++ service._start_task("c1", client, client) # type: ignore[arg-type]+ await asyncio.wait_for(client.running.wait(), timeout=2)++ assert client.starts == 3+ assert service.get("c1") is client+ service._cancel_task("c1")+++async def test_any_other_error_still_ends_the_client() -> None:+ service = _service()+ client = _FlakyClient(failures=1, error=ValueError("a bug in the client"))+ service._clients["c1"] = client # type: ignore[assignment]++ service._start_task("c1", client, client) # type: ignore[arg-type]+ for _ in range(50):+ if service.get("c1") is None:+ break+ await asyncio.sleep(0.01)++ assert client.starts == 1+ assert service.get("c1") is None