Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

fix(clients): stop a dying client deafening the live one (CHOO-2623)

Wojciech Wojtyniaka2ff323553c8
diff --git a/core/switch_core/clients/client_lifecycle_service.py b/core/switch_core/clients/client_lifecycle_service.py
index 42e6d3bd3..2c254ea91 100644
--- a/core/switch_core/clients/client_lifecycle_service.py
+++ b/core/switch_core/clients/client_lifecycle_service.py
@@ -168,7 +168,9 @@ class ClientLifecycleService:
logger.warning("Cannot stop unknown client %s", client_id)
return
await client.stop()
- self._cancel_task(client_id)
+ task = self._cancel_task(client_id)
+ if task is not None:
+ await asyncio.gather(task, return_exceptions=True)
del self._clients[client_id]
self._client_types.pop(client_id, None)
self._client_tenants.pop(client_id, None)
@@ -178,8 +180,14 @@ class ClientLifecycleService:
logger.info("Stopping all %d clients", len(self._clients))
for client in self._clients.values():
await client.stop()
- for task in self._tasks.values():
+ tasks = list(self._tasks.values())
+ for task in tasks:
task.cancel()
+ # Awaited, not merely cancelled. A cancelled task unwinds when the
+ # loop next runs it, or never — and a receive loop that is never run
+ # again is collected instead, which runs its `finally` under the
+ # garbage collector, in a context that is not the task's.
+ await asyncio.gather(*tasks, return_exceptions=True)
self._clients.clear()
self._client_types.clear()
self._client_tenants.clear()
@@ -226,13 +234,23 @@ class ClientLifecycleService:
]
def _start_task(self, client_id: str, client: ClientBase[ClientConfig]) -> None:
+ # A client row is one running task. Anything that starts a second for
+ # the same row — a boot sweep reaching a client that registration
+ # already started, a bridge restarting its own — replaces the first,
+ # and the first has to be told to stop rather than dropped on the
+ # floor. An abandoned receive loop is still subscribed to its rooms and
+ # still holds the invite slot for its user, and when the collector
+ # finally reaches it, it runs the teardown for both: the live client
+ # for that user goes deaf to invitations it never saw arrive.
+ self._cancel_task(client_id)
task = asyncio.create_task(self._run_client(client_id, client))
self._tasks[client_id] = task
- def _cancel_task(self, client_id: str) -> None:
+ def _cancel_task(self, client_id: str) -> asyncio.Task[None] | None:
task = self._tasks.pop(client_id, None)
if task and not task.done():
task.cancel()
+ return task
async def _run_client(
self, client_id: str, client: ClientBase[ClientConfig]
diff --git a/core/switch_core/transport/invites.py b/core/switch_core/transport/invites.py
index 14a94c601..60969f0cb 100644
--- a/core/switch_core/transport/invites.py
+++ b/core/switch_core/transport/invites.py
@@ -38,10 +38,27 @@ class InviteBus:
self._handlers: dict[str, InviteHandler] = {}
def register(self, user_id: str, handler: InviteHandler) -> None:
+ """Make `handler` the live listener for `user_id`, displacing any other.
+
+ One slot per user, so a client that starts while an older instance of
+ itself is still unwinding takes the slot from it. That is the right
+ answer — the newer one is the client with a live receive loop — and it
+ is why `unregister` wants the handler back.
+ """
self._handlers[user_id] = handler
- def unregister(self, user_id: str) -> None:
- self._handlers.pop(user_id, None)
+ def unregister(self, user_id: str, handler: InviteHandler) -> None:
+ """Give the slot up, but only if `handler` still holds it.
+
+ A transport unregisters from the `finally` of its receive loop, which
+ runs whenever that loop ends — including long after a newer client for
+ the same user took the slot. Clearing by user id alone would deafen
+ that newer client: `invite` would find nobody listening, the caller
+ would write the membership row itself, and the client would sit in a
+ room it was never told it was in until something restarted it.
+ """
+ if self._handlers.get(user_id) == handler:
+ del self._handlers[user_id]
async def invite(self, user_id: str, transport_room_id: str) -> bool:
"""Tell `user_id` it is in `transport_room_id`.
diff --git a/core/switch_core/transport/postgres.py b/core/switch_core/transport/postgres.py
index d98da420b..828b98493 100644
--- a/core/switch_core/transport/postgres.py
+++ b/core/switch_core/transport/postgres.py
@@ -189,7 +189,7 @@ class PostgresTransport:
await self._closed.wait()
finally:
self._receiving = False
- self._invites.unregister(self.user_id)
+ self._invites.unregister(self.user_id, self._on_invited)
self._unwatch_all()
delivery.cancel()
diff --git a/core/tests/switch_core/clients/test_client_task_replacement.py b/core/tests/switch_core/clients/test_client_task_replacement.py
new file mode 100644
index 000000000..725b3d4d6
--- /dev/null
+++ b/core/tests/switch_core/clients/test_client_task_replacement.py
@@ -0,0 +1,81 @@
+"""One client row, one running task (CHOO-2623).
+
+Starting a client that is already running used to overwrite the registry entry
+and drop the running task on the floor. The task stayed alive — subscribed to
+its rooms, holding the invite slot for its user — until the garbage collector
+reached it, at which point it ran its own teardown and took the *live* client's
+invite registration with it. The live client then heard no invitation for any
+room it was added to afterwards, with nothing in the log to say why.
+"""
+
+from __future__ import annotations
+
+import asyncio
+from unittest.mock import MagicMock
+
+from switch_core.clients.client_lifecycle_service import ClientLifecycleService
+from switch_core.db.stores.client_store import ClientStore
+from switch_core.db.stores.tenant_store import TenantStore
+
+
+class _BlockingClient:
+ """A client whose task never finishes on its own, like a receive loop."""
+
+ def __init__(self) -> None:
+ self.display_name = "blocking"
+ self.matrix_user_id = "@blocking:test"
+ self.stopped = False
+
+ async def start(self) -> None:
+ await asyncio.Event().wait()
+
+ async def stop(self) -> None:
+ self.stopped = True
+
+
+def _service() -> ClientLifecycleService:
+ return ClientLifecycleService(
+ matrix_admin=MagicMock(),
+ client_store=ClientStore(),
+ tenant_store=TenantStore(),
+ client_factory=MagicMock(),
+ session_factory=MagicMock(),
+ config=MagicMock(),
+ )
+
+
+async def test_starting_a_client_twice_cancels_the_first_task() -> None:
+ service = _service()
+ service._start_task("client-1", _BlockingClient()) # type: ignore[arg-type]
+ await asyncio.sleep(0)
+ first = service._tasks["client-1"]
+
+ service._start_task("client-1", _BlockingClient()) # type: ignore[arg-type]
+ second = service._tasks["client-1"]
+ await asyncio.gather(first, return_exceptions=True)
+
+ assert first.cancelled(), "the superseded task was dropped, not stopped"
+ assert second is not first
+ assert not second.done()
+
+ second.cancel()
+ await asyncio.gather(second, return_exceptions=True)
+
+
+async def test_stop_all_waits_for_the_tasks_it_cancelled() -> None:
+ """Cancelling without awaiting leaves the unwinding to the loop, or to the
+ collector if the loop never runs the task again."""
+ service = _service()
+ client = _BlockingClient()
+ service._clients["client-1"] = client # type: ignore[assignment]
+ service._client_types["client-1"] = "agent"
+ service._client_tenants["client-1"] = "tenant-1"
+ service._start_task("client-1", client) # type: ignore[arg-type]
+ await asyncio.sleep(0)
+ task = service._tasks["client-1"]
+
+ await service.stop_all()
+
+ assert task.done(), "stop_all returned while a client task was still running"
+ assert client.stopped
+ assert service._tasks == {}
diff --git a/core/tests/switch_core/transport/test_invites.py b/core/tests/switch_core/transport/test_invites.py
new file mode 100644
index 000000000..8ae7cc6cd
--- /dev/null
+++ b/core/tests/switch_core/transport/test_invites.py
@@ -0,0 +1,66 @@
+"""switch_core.transport.invites: who hears an invitation when two transports
+for one user overlap (CHOO-2623).
+
+The overlap is real rather than theoretical: a client row can have a second
+receive loop started for it — a boot sweep reaching a client registration
+already started, a bridge restarting its own — and the first loop then unwinds
+whenever it is finally cancelled or collected. If that teardown could clear the
+slot by user id alone it would deafen the live client, and nothing would say
+so: `invite` would report nobody listening, the caller would write the
+membership row itself, and the client would sit in a room it was never told it
+had joined.
+"""
+
+from __future__ import annotations
+
+from switch_core.transport.invites import InviteBus
+
+
+class _Listener:
+ def __init__(self) -> None:
+ self.rooms: list[str] = []
+
+ async def on_invited(self, transport_room_id: str) -> None:
+ self.rooms.append(transport_room_id)
+
+
+async def test_the_newest_registration_is_the_one_that_hears() -> None:
+ bus = InviteBus()
+ old, new = _Listener(), _Listener()
+
+ bus.register("@agent:test", old.on_invited)
+ bus.register("@agent:test", new.on_invited)
+
+ assert await bus.invite("@agent:test", "!room:test") is True
+ assert (old.rooms, new.rooms) == ([], ["!room:test"])
+
+
+async def test_a_superseded_listener_cannot_unregister_the_live_one() -> None:
+ bus = InviteBus()
+ old, new = _Listener(), _Listener()
+ bus.register("@agent:test", old.on_invited)
+ bus.register("@agent:test", new.on_invited)
+
+ bus.unregister("@agent:test", old.on_invited)
+
+ assert await bus.invite("@agent:test", "!room:test") is True, (
+ "the superseded transport's teardown deafened the live client"
+ )
+ assert new.rooms == ["!room:test"]
+
+
+async def test_the_live_listener_can_still_unregister_itself() -> None:
+ bus = InviteBus()
+ listener = _Listener()
+ bus.register("@agent:test", listener.on_invited)
+
+ bus.unregister("@agent:test", listener.on_invited)
+
+ assert await bus.invite("@agent:test", "!room:test") is False
+ assert listener.rooms == []
+
+
+async def test_unregistering_a_user_nobody_registered_is_a_no_op() -> None:
+ bus = InviteBus()
+ bus.unregister("@agent:test", _Listener().on_invited)
+ assert await bus.invite("@agent:test", "!room:test") is False