Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Retry a client that fails on the database at start, instead of dropping it (#742)

Abel Dantas3363cdc37c9f
diff --git a/core/switch_core/clients/client_lifecycle_service.py b/core/switch_core/clients/client_lifecycle_service.py
index 2e7cb6685..feae02da4 100644
--- a/core/switch_core/clients/client_lifecycle_service.py
+++ b/core/switch_core/clients/client_lifecycle_service.py
@@ -2,9 +2,12 @@ from __future__ import annotations
import asyncio
import logging
+import random
import uuid
from typing import Any, Protocol, runtime_checkable
+from sqlalchemy.exc import InterfaceError, OperationalError
+from sqlalchemy.exc import TimeoutError as PoolTimeoutError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from switch_core.clients.actor import Actor, AgentActor, ClientConfig
@@ -23,6 +26,26 @@ from switch_core.tenant_context import no_tenant, tenant_scope
logger = logging.getLogger(__name__)
+# A client that fails on the database or the network is retried rather than
+# dropped. At boot every client starts at once and asks for a connection in the
+# same second; on 9 Oct that drained the pool, 13 clients timed out
+# waiting for one, and each was dropped for good: its agent connected, but
+# nothing on the server could post for it until the next restart. Such a
+# failure says nothing about the client, so it waits and tries again, as long
+# as it takes. Anything else is a bug in the client and still ends it.
+# The wait is jittered: the clients that drained the pool failed together, and
+# a deterministic backoff would keep them retrying together, re-draining it in
+# synchronized waves. Each draws its own delay instead.
+_TRANSIENT_START_ERRORS: tuple[type[BaseException], ...] = (
+ PoolTimeoutError,
+ OperationalError,
+ InterfaceError,
+ OSError,
+ TimeoutError,
+)
+CLIENT_RETRY_BASE_SECONDS = 1.0
+CLIENT_RETRY_CAP_SECONDS = 60.0
+
@runtime_checkable
class _PreloadsAgent(Protocol):
@@ -452,10 +475,33 @@ class ClientLifecycleService:
"""
with no_tenant(), log_context(room_id=None):
try:
- if consumer is not None:
- await consumer.start()
- else:
- await actor.connect()
+ attempt = 0
+ while True:
+ try:
+ if consumer is not None:
+ await consumer.start()
+ else:
+ await actor.connect()
+ return
+ except _TRANSIENT_START_ERRORS:
+ attempt += 1
+ delay = random.uniform(
+ 0,
+ min(
+ CLIENT_RETRY_BASE_SECONDS * 2 ** (attempt - 1),
+ CLIENT_RETRY_CAP_SECONDS,
+ ),
+ )
+ logger.warning(
+ "Client %s (%s) failed on the database or network "
+ "(attempt %d); retrying in %.0fs",
+ actor.display_name,
+ actor.transport_user_id,
+ attempt,
+ delay,
+ exc_info=True,
+ )
+ await asyncio.sleep(delay)
except Exception:
logger.exception(
"Client %s (%s) crashed",
diff --git a/core/tests/switch_core/clients/test_client_start_retry.py b/core/tests/switch_core/clients/test_client_start_retry.py
new file mode 100644
index 000000000..8103d58c5
--- /dev/null
+++ b/core/tests/switch_core/clients/test_client_start_retry.py
@@ -0,0 +1,84 @@
+"""A client that fails on the database at start is retried, not dropped.
+
+On 9 Oct a pilot restart started every client at once, the pool ran dry, and
+13 clients timed out waiting for a connection. Each was dropped for good, so
+its agent could not post until the next restart.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from unittest.mock import MagicMock
+
+import pytest
+from sqlalchemy.exc import TimeoutError as PoolTimeoutError
+
+from switch_core.clients import client_lifecycle_service as lifecycle
+from switch_core.clients.client_lifecycle_service import ClientLifecycleService
+
+
+class _FlakyClient:
+ """Fails its first `failures` starts with `error`, then runs."""
+
+ def __init__(self, failures: int, error: Exception) -> None:
+ self.display_name = "flaky"
+ self.transport_user_id = "@flaky:test"
+ self.starts = 0
+ self._failures = failures
+ self._error = error
+ self.running = asyncio.Event()
+
+ async def start(self) -> None:
+ self.starts += 1
+ if self.starts <= self._failures:
+ raise self._error
+ self.running.set()
+ await asyncio.Event().wait()
+
+ async def stop(self) -> None:
+ return None
+
+
+def _service() -> ClientLifecycleService:
+ return ClientLifecycleService(
+ provisioning=MagicMock(),
+ client_store=MagicMock(),
+ tenant_store=MagicMock(),
+ client_factory=MagicMock(),
+ session_factory=MagicMock(),
+ config=MagicMock(),
+ tenants_isolated=True,
+ )
+
+
[email protected](autouse=True)
+def _no_wait(monkeypatch: pytest.MonkeyPatch) -> None:
+ monkeypatch.setattr(lifecycle, "CLIENT_RETRY_BASE_SECONDS", 0.0)
+
+
+async def test_a_pool_timeout_at_start_is_retried_until_the_client_runs() -> None:
+ service = _service()
+ client = _FlakyClient(failures=2, error=PoolTimeoutError("QueuePool limit reached"))
+ service._clients["c1"] = client # type: ignore[assignment]
+
+ service._start_task("c1", client, client) # type: ignore[arg-type]
+ await asyncio.wait_for(client.running.wait(), timeout=2)
+
+ assert client.starts == 3
+ assert service.get("c1") is client
+ service._cancel_task("c1")
+
+
+async def test_any_other_error_still_ends_the_client() -> None:
+ service = _service()
+ client = _FlakyClient(failures=1, error=ValueError("a bug in the client"))
+ service._clients["c1"] = client # type: ignore[assignment]
+
+ service._start_task("c1", client, client) # type: ignore[arg-type]
+ for _ in range(50):
+ if service.get("c1") is None:
+ break
+ await asyncio.sleep(0.01)
+
+ assert client.starts == 1
+ assert service.get("c1") is None