Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
fix(clients): stop a dying client deafening the live one (CHOO-2623)
Wojciech Wojtyniaka2ff323553c8
diff --git a/core/switch_core/clients/client_lifecycle_service.py b/core/switch_core/clients/client_lifecycle_service.pyindex 42e6d3bd3..2c254ea91 100644--- a/core/switch_core/clients/client_lifecycle_service.py+++ b/core/switch_core/clients/client_lifecycle_service.py@@ -168,7 +168,9 @@ class ClientLifecycleService:logger.warning("Cannot stop unknown client %s", client_id)returnawait client.stop()- self._cancel_task(client_id)+ task = self._cancel_task(client_id)+ if task is not None:+ await asyncio.gather(task, return_exceptions=True)del self._clients[client_id]self._client_types.pop(client_id, None)self._client_tenants.pop(client_id, None)@@ -178,8 +180,14 @@ class ClientLifecycleService:logger.info("Stopping all %d clients", len(self._clients))for client in self._clients.values():await client.stop()- for task in self._tasks.values():+ tasks = list(self._tasks.values())+ for task in tasks:task.cancel()+ # Awaited, not merely cancelled. A cancelled task unwinds when the+ # loop next runs it, or never — and a receive loop that is never run+ # again is collected instead, which runs its `finally` under the+ # garbage collector, in a context that is not the task's.+ await asyncio.gather(*tasks, return_exceptions=True)self._clients.clear()self._client_types.clear()self._client_tenants.clear()@@ -226,13 +234,23 @@ class ClientLifecycleService:]def _start_task(self, client_id: str, client: ClientBase[ClientConfig]) -> None:+ # A client row is one running task. Anything that starts a second for+ # the same row — a boot sweep reaching a client that registration+ # already started, a bridge restarting its own — replaces the first,+ # and the first has to be told to stop rather than dropped on the+ # floor. An abandoned receive loop is still subscribed to its rooms and+ # still holds the invite slot for its user, and when the collector+ # finally reaches it, it runs the teardown for both: the live client+ # for that user goes deaf to invitations it never saw arrive.+ self._cancel_task(client_id)task = asyncio.create_task(self._run_client(client_id, client))self._tasks[client_id] = task- def _cancel_task(self, client_id: str) -> None:+ def _cancel_task(self, client_id: str) -> asyncio.Task[None] | None:task = self._tasks.pop(client_id, None)if task and not task.done():task.cancel()+ return taskasync def _run_client(self, client_id: str, client: ClientBase[ClientConfig]diff --git a/core/switch_core/transport/invites.py b/core/switch_core/transport/invites.pyindex 14a94c601..60969f0cb 100644--- a/core/switch_core/transport/invites.py+++ b/core/switch_core/transport/invites.py@@ -38,10 +38,27 @@ class InviteBus:self._handlers: dict[str, InviteHandler] = {}def register(self, user_id: str, handler: InviteHandler) -> None:+ """Make `handler` the live listener for `user_id`, displacing any other.++ One slot per user, so a client that starts while an older instance of+ itself is still unwinding takes the slot from it. That is the right+ answer — the newer one is the client with a live receive loop — and it+ is why `unregister` wants the handler back.+ """self._handlers[user_id] = handler- def unregister(self, user_id: str) -> None:- self._handlers.pop(user_id, None)+ def unregister(self, user_id: str, handler: InviteHandler) -> None:+ """Give the slot up, but only if `handler` still holds it.++ A transport unregisters from the `finally` of its receive loop, which+ runs whenever that loop ends — including long after a newer client for+ the same user took the slot. Clearing by user id alone would deafen+ that newer client: `invite` would find nobody listening, the caller+ would write the membership row itself, and the client would sit in a+ room it was never told it was in until something restarted it.+ """+ if self._handlers.get(user_id) == handler:+ del self._handlers[user_id]async def invite(self, user_id: str, transport_room_id: str) -> bool:"""Tell `user_id` it is in `transport_room_id`.diff --git a/core/switch_core/transport/postgres.py b/core/switch_core/transport/postgres.pyindex d98da420b..828b98493 100644--- a/core/switch_core/transport/postgres.py+++ b/core/switch_core/transport/postgres.py@@ -189,7 +189,7 @@ class PostgresTransport:await self._closed.wait()finally:self._receiving = False- self._invites.unregister(self.user_id)+ self._invites.unregister(self.user_id, self._on_invited)self._unwatch_all()delivery.cancel()diff --git a/core/tests/switch_core/clients/test_client_task_replacement.py b/core/tests/switch_core/clients/test_client_task_replacement.pynew file mode 100644index 000000000..725b3d4d6--- /dev/null+++ b/core/tests/switch_core/clients/test_client_task_replacement.py@@ -0,0 +1,81 @@+"""One client row, one running task (CHOO-2623).++Starting a client that is already running used to overwrite the registry entry+and drop the running task on the floor. The task stayed alive — subscribed to+its rooms, holding the invite slot for its user — until the garbage collector+reached it, at which point it ran its own teardown and took the *live* client's+invite registration with it. The live client then heard no invitation for any+room it was added to afterwards, with nothing in the log to say why.+"""++from __future__ import annotations++import asyncio+from unittest.mock import MagicMock++from switch_core.clients.client_lifecycle_service import ClientLifecycleService+from switch_core.db.stores.client_store import ClientStore+from switch_core.db.stores.tenant_store import TenantStore+++class _BlockingClient:+ """A client whose task never finishes on its own, like a receive loop."""++ def __init__(self) -> None:+ self.display_name = "blocking"+ self.matrix_user_id = "@blocking:test"+ self.stopped = False++ async def start(self) -> None:+ await asyncio.Event().wait()++ async def stop(self) -> None:+ self.stopped = True+++def _service() -> ClientLifecycleService:+ return ClientLifecycleService(+ matrix_admin=MagicMock(),+ client_store=ClientStore(),+ tenant_store=TenantStore(),+ client_factory=MagicMock(),+ session_factory=MagicMock(),+ config=MagicMock(),+ )+++async def test_starting_a_client_twice_cancels_the_first_task() -> None:+ service = _service()+ service._start_task("client-1", _BlockingClient()) # type: ignore[arg-type]+ await asyncio.sleep(0)+ first = service._tasks["client-1"]++ service._start_task("client-1", _BlockingClient()) # type: ignore[arg-type]+ second = service._tasks["client-1"]+ await asyncio.gather(first, return_exceptions=True)++ assert first.cancelled(), "the superseded task was dropped, not stopped"+ assert second is not first+ assert not second.done()++ second.cancel()+ await asyncio.gather(second, return_exceptions=True)+++async def test_stop_all_waits_for_the_tasks_it_cancelled() -> None:+ """Cancelling without awaiting leaves the unwinding to the loop, or to the+ collector if the loop never runs the task again."""+ service = _service()+ client = _BlockingClient()+ service._clients["client-1"] = client # type: ignore[assignment]+ service._client_types["client-1"] = "agent"+ service._client_tenants["client-1"] = "tenant-1"+ service._start_task("client-1", client) # type: ignore[arg-type]+ await asyncio.sleep(0)+ task = service._tasks["client-1"]++ await service.stop_all()++ assert task.done(), "stop_all returned while a client task was still running"+ assert client.stopped+ assert service._tasks == {}diff --git a/core/tests/switch_core/transport/test_invites.py b/core/tests/switch_core/transport/test_invites.pynew file mode 100644index 000000000..8ae7cc6cd--- /dev/null+++ b/core/tests/switch_core/transport/test_invites.py@@ -0,0 +1,66 @@+"""switch_core.transport.invites: who hears an invitation when two transports+for one user overlap (CHOO-2623).++The overlap is real rather than theoretical: a client row can have a second+receive loop started for it — a boot sweep reaching a client registration+already started, a bridge restarting its own — and the first loop then unwinds+whenever it is finally cancelled or collected. If that teardown could clear the+slot by user id alone it would deafen the live client, and nothing would say+so: `invite` would report nobody listening, the caller would write the+membership row itself, and the client would sit in a room it was never told it+had joined.+"""++from __future__ import annotations++from switch_core.transport.invites import InviteBus+++class _Listener:+ def __init__(self) -> None:+ self.rooms: list[str] = []++ async def on_invited(self, transport_room_id: str) -> None:+ self.rooms.append(transport_room_id)+++async def test_the_newest_registration_is_the_one_that_hears() -> None:+ bus = InviteBus()+ old, new = _Listener(), _Listener()++ bus.register("@agent:test", old.on_invited)+ bus.register("@agent:test", new.on_invited)++ assert await bus.invite("@agent:test", "!room:test") is True+ assert (old.rooms, new.rooms) == ([], ["!room:test"])+++async def test_a_superseded_listener_cannot_unregister_the_live_one() -> None:+ bus = InviteBus()+ old, new = _Listener(), _Listener()+ bus.register("@agent:test", old.on_invited)+ bus.register("@agent:test", new.on_invited)++ bus.unregister("@agent:test", old.on_invited)++ assert await bus.invite("@agent:test", "!room:test") is True, (+ "the superseded transport's teardown deafened the live client"+ )+ assert new.rooms == ["!room:test"]+++async def test_the_live_listener_can_still_unregister_itself() -> None:+ bus = InviteBus()+ listener = _Listener()+ bus.register("@agent:test", listener.on_invited)++ bus.unregister("@agent:test", listener.on_invited)++ assert await bus.invite("@agent:test", "!room:test") is False+ assert listener.rooms == []+++async def test_unregistering_a_user_nobody_registered_is_a_no_op() -> None:+ bus = InviteBus()+ bus.unregister("@agent:test", _Listener().on_invited)+ assert await bus.invite("@agent:test", "!room:test") is False