Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Removing an agent from a room does not stop message delivery (#412)

James Howe8d82319e5594
diff --git a/core/switch_core/bridges/agent/protocol/connections.py b/core/switch_core/bridges/agent/protocol/connections.py
index fcf09ca3e..8e454a2b9 100644
--- a/core/switch_core/bridges/agent/protocol/connections.py
+++ b/core/switch_core/bridges/agent/protocol/connections.py
@@ -496,6 +496,22 @@ class ConnectionRegistry:
conn.rooms.discard(room_id)
conn.wake.set()
+ def release_room_everywhere(self, agent_id: str, room_id: str) -> None:
+ """Take the room off every one of this agent's connections.
+
+ A claim outlives the membership it was checked against:
+ `require_room_member` runs when the room is claimed and never again.
+
+ Every connection, not only the live ones — a lapsed connection still
+ holds its claim, and a client reconnecting to it resumes covering the
+ room.
+ """
+ for cid in self._by_agent.get(agent_id, set()):
+ conn = self._by_id.get(cid)
+ if conn is not None and room_id in conn.rooms:
+ conn.rooms.discard(room_id)
+ conn.wake.set()
+
def claimant_of(self, agent_id: str, room_id: str) -> Connection | None:
"""The connection that has explicitly claimed this room, if any.
diff --git a/core/switch_core/bridges/agent/protocol/event_buffer.py b/core/switch_core/bridges/agent/protocol/event_buffer.py
index c4a4da027..d37c8fdef 100644
--- a/core/switch_core/bridges/agent/protocol/event_buffer.py
+++ b/core/switch_core/bridges/agent/protocol/event_buffer.py
@@ -243,6 +243,34 @@ class EventBuffer:
self._cursors.pop(agent_id, None)
self._dropped_through.pop(agent_id, None)
+ def drop_room(self, agent_id: str, room_id: str) -> None:
+ """Forget everything retained for this agent in one room.
+
+ Called when the agent is removed from it. The buffer is keyed by agent
+ and knows nothing about who is in what, so an event queued while it was
+ a member stays readable after it is not.
+
+ Sequence numbers are untouched and no gap is recorded: a reader that
+ skips these has missed nothing it was entitled to, and saying otherwise
+ would send it to re-read the context of a room it is not in.
+ """
+ events = self._events.get(agent_id)
+ if not events:
+ return
+ kept = [item for item in events if item.room_id != room_id]
+ dropped = len(events) - len(kept)
+ if not dropped:
+ return
+ events.clear()
+ events.extend(kept)
+ logger.info(
+ "[EVENT-BUF] dropped %s retained event(s) for agent=%s room=%s: "
+ "no longer a member",
+ dropped,
+ agent_id,
+ room_id,
+ )
+
# ------------------------------------------------------------------
# Legacy long-poll compatibility
#
@@ -257,21 +285,53 @@ class EventBuffer:
# Confirming clients get at-least-once instead.
# ------------------------------------------------------------------
- async def poll(self, agent_id: str, timeout: float = 30) -> list[AgentEvent]:
- return await self._legacy_poll(agent_id, "legacy:all", timeout=timeout)
+ async def poll(
+ self, agent_id: str, *, rooms: set[str], timeout: float = 30
+ ) -> list[AgentEvent]:
+ """Everything queued for this agent in the rooms it is in.
+
+ `rooms` is how the caller applies membership, and it is required: the
+ buffer is keyed by agent and knows nothing about who is in what, so an
+ event queued while the agent was a member stays queued after it is
+ removed. A default would make the leak the thing a new caller gets for
+ free.
+
+ Read once, before the wait. A room the agent is added to while this is
+ parked has its first events held back to the next poll, which the
+ caller makes immediately; asking the caller to re-read membership
+ mid-poll would buy that one round trip at the cost of a callback
+ reaching back out of the buffer, and this stays behind the narrow
+ surface described at the top of the module.
+ """
+ return await self._legacy_poll(
+ agent_id, "legacy:all", timeout=timeout, rooms=rooms
+ )
async def poll_room(
self, agent_id: str, room_id: str, timeout: float = 30
) -> list[AgentEvent]:
return await self._legacy_poll(
- agent_id, f"legacy:room:{room_id}", timeout=timeout, rooms={room_id}
+ agent_id,
+ f"legacy:room:{room_id}",
+ timeout=timeout,
+ rooms={room_id},
+ rooms_are_membership=False,
)
async def poll_notifications(
- self, agent_id: str, timeout: float = 30
+ self, agent_id: str, *, rooms: set[str], timeout: float = 30
) -> list[AgentEvent]:
+ """The notifiable half of `poll`, under the same membership rule.
+
+ This is the stream carrying messages addressed at the agent, so it is
+ the one a removal matters most on.
+ """
return await self._legacy_poll(
- agent_id, "legacy:notifications", timeout=timeout, notifiable_only=True
+ agent_id,
+ "legacy:notifications",
+ timeout=timeout,
+ rooms=rooms,
+ notifiable_only=True,
)
async def _legacy_poll(
@@ -280,20 +340,66 @@ class EventBuffer:
reader_id: str,
*,
timeout: float,
- rooms: set[str] | None = None,
+ rooms: set[str],
notifiable_only: bool = False,
+ rooms_are_membership: bool = True,
) -> list[AgentEvent]:
- cursor = self._legacy_cursor(agent_id, reader_id)
-
- events = self._legacy_read(agent_id, reader_id, cursor, rooms, notifiable_only)
+ args = (agent_id, reader_id, rooms, notifiable_only, rooms_are_membership)
+ events = self._legacy_take(*args)
if events:
- return self._advance_legacy(agent_id, reader_id, events)
+ return events
await self.wait(agent_id, timeout)
+ return self._legacy_take(*args)
+
+ def _legacy_take(
+ self,
+ agent_id: str,
+ reader_id: str,
+ rooms: set[str],
+ notifiable_only: bool,
+ rooms_are_membership: bool,
+ ) -> list[AgentEvent]:
+ """One read, moving the cursor past what this reader can never want.
+
+ Past what it can never want, and no further. A legacy read has no
+ limit, so it always reaches the end of the buffer, and leaving the
+ cursor behind an event the filter excluded parks it below the head
+ indefinitely — until retention trims it and the next read reports a
+ gap that never happened.
+
+ Which exclusions are permanent depends on the filter.
+ `notifiable_only` tests a property of the event, which is fixed, so
+ those can be skipped for good. A room set taken from membership is not
+ fixed: an agent added to a room the buffer already holds events for
+ would find the cursor already past them, and they would never be
+ delivered. `poll_room`'s filter is the reader's own scope rather than a
+ membership snapshot, so there the exclusions are permanent too — hence
+ `rooms_are_membership`.
+ """
cursor = self._legacy_cursor(agent_id, reader_id)
events = self._legacy_read(agent_id, reader_id, cursor, rooms, notifiable_only)
- return self._advance_legacy(agent_id, reader_id, events)
+ if rooms_are_membership:
+ self.confirm(
+ agent_id, reader_id, self._last_seq_in_rooms(agent_id, cursor, rooms)
+ )
+ else:
+ self.confirm(agent_id, reader_id, self.head(agent_id))
+ return [item.event for item in events]
+
+ def _last_seq_in_rooms(self, agent_id: str, after_seq: int, rooms: set[str]) -> int:
+ """The newest retained event after `after_seq` in one of `rooms`.
+
+ `after_seq` itself when there is none, so a caller confirming this
+ never moves a cursor over an event held for a room the agent is not in
+ yet.
+ """
+ last = after_seq
+ for item in self._events.get(agent_id, ()):
+ if item.seq > after_seq and item.room_id in rooms:
+ last = item.seq
+ return last
def _legacy_cursor(self, agent_id: str, reader_id: str) -> int:
readers = self._cursors.setdefault(agent_id, {})
@@ -311,7 +417,7 @@ class EventBuffer:
agent_id: str,
reader_id: str,
cursor: int,
- rooms: set[str] | None,
+ rooms: set[str],
notifiable_only: bool,
) -> list[BufferedEvent]:
try:
@@ -336,13 +442,6 @@ class EventBuffer:
agent_id, resumed, rooms=rooms, notifiable_only=notifiable_only
)
- def _advance_legacy(
- self, agent_id: str, reader_id: str, events: list[BufferedEvent]
- ) -> list[AgentEvent]:
- if events:
- self.confirm(agent_id, reader_id, events[-1].seq)
- return [item.event for item in events]
-
# ------------------------------------------------------------------
# Internals
# ------------------------------------------------------------------
diff --git a/core/switch_core/bridges/agent/protocol/service.py b/core/switch_core/bridges/agent/protocol/service.py
index ea79dcc42..c3c60c99a 100644
--- a/core/switch_core/bridges/agent/protocol/service.py
+++ b/core/switch_core/bridges/agent/protocol/service.py
@@ -1997,12 +1997,40 @@ class ProtocolService:
# ── Events ───────────────────────────────────────────────────────────────
+ async def _rooms_agent_is_in(self, agent_id: str) -> set[str]:
+ """The rooms the agent is a member of, read fresh.
+
+ Archived rooms included: what this is applied as is membership, and an
+ agent still in an archived room was not removed from it.
+ """
+ async with self.session_factory() as session:
+ rooms = await self.room_store.get_rooms_for_agent(
+ session, agent_id, include_archived=True
+ )
+ return {room.id for room in rooms}
+
async def poll_events(self, agent_id: str, timeout: float = 10) -> list[AgentEvent]:
- """Poll for events across all rooms the agent is in."""
+ """Poll for events across all rooms the agent is in.
+
+ Membership is applied here rather than trusted from the buffer, which
+ is keyed by agent and knows nothing about who is in what. Its
+ room-scoped sibling `poll_room_events` calls `require_room_member`;
+ without the same check this call would hand over events from a room
+ the agent has since been removed from.
+
+ `AgentClient.on_removed` already empties the buffer of a removed
+ room's events, so this is the second of two answers. It is the
+ authoritative one: that signal is in-process, and this reads the table
+ the removal wrote.
+ """
async with self.session_factory() as session:
await self.agent_session_store.touch_heartbeat(session, agent_id, None)
await session.commit()
- return await self.event_buffer.poll(agent_id, timeout=timeout)
+ return await self.event_buffer.poll(
+ agent_id,
+ timeout=timeout,
+ rooms=await self._rooms_agent_is_in(agent_id),
+ )
async def poll_notifications(
self, agent_id: str, timeout: float = 10
@@ -2016,8 +2044,16 @@ class ProtocolService:
`touch_watch_heartbeat` path, decoupled from this long-poll. Consuming
this stream never drains the per-room queues, so live session pollers
are unaffected.
+
+ Membership is applied exactly as it is for `poll_events`, and matters
+ more here: this is the stream carrying the messages addressed at the
+ agent.
"""
- return await self.event_buffer.poll_notifications(agent_id, timeout=timeout)
+ return await self.event_buffer.poll_notifications(
+ agent_id,
+ timeout=timeout,
+ rooms=await self._rooms_agent_is_in(agent_id),
+ )
async def touch_watch_heartbeat(self, agent_id: str) -> None:
"""Refresh an auto_session connector's global "watching" heartbeat.
diff --git a/core/switch_core/clients/agent_client.py b/core/switch_core/clients/agent_client.py
index 0d2c855ea..0ea1551c5 100644
--- a/core/switch_core/clients/agent_client.py
+++ b/core/switch_core/clients/agent_client.py
@@ -343,6 +343,21 @@ class AgentClient(ClientBase[ClientConfig]):
greeting = random.choice(AGENT_GREETINGS).format(name=name)
await self.send_message(room.room_id, greeting, format="markdown")
+ async def on_removed(self, room: RoomRef, event: InboundMembership) -> None:
+ """Forget the room's events, everywhere this agent could still read them.
+
+ Dropping the subscription only stops what has not been read yet. What
+ has is in the event buffer for the whole retention window, and is
+ served from there to a long poll, to the notification stream, and to an
+ SSE reader resuming from an old cursor — the last of which has no
+ membership of its own to apply.
+ """
+ meta = await self._resolve_room_meta(room.room_id)
+ if meta is None:
+ return
+ self._event_buffer.drop_room(self.agent.id, meta.room_id)
+ self._connections.release_room_everywhere(self.agent.id, meta.room_id)
+
async def _member_name(
self, session: AsyncSession, event: InboundMembership
) -> str:
diff --git a/core/switch_core/clients/client_base.py b/core/switch_core/clients/client_base.py
index 3c4df688a..c21634e87 100644
--- a/core/switch_core/clients/client_base.py
+++ b/core/switch_core/clients/client_base.py
@@ -227,6 +227,7 @@ class ClientBase[ConfigT: ClientConfig]:
on_member_event=self._handle_member_event,
on_custom_event=self._handle_custom_event,
on_invite=self._handle_invite,
+ on_removed=self._handle_removed,
)
)
@@ -318,6 +319,17 @@ class ClientBase[ConfigT: ClientConfig]:
"Error in on_invite for %s in %s", self.matrix_user_id, room.room_id
)
+ async def _handle_removed(self, room: RoomRef, event: InboundMembership) -> None:
+ # Departing ends the visit, the same way a leave delivered as a member
+ # event does: being added back is a fresh arrival.
+ self._self_join_dispatched.discard(room.room_id)
+ try:
+ await self.on_removed(room, event)
+ except Exception:
+ logger.exception(
+ "Error in on_removed for %s in %s", self.matrix_user_id, room.room_id
+ )
+
_EVENT_DISPATCH: dict[str, tuple[type[SwitchEvent], str]] = {
"com.switch.command": (CommandEvent, "on_command"),
"com.switch.report.tool_call": (ToolCallReport, "on_tool_call_report"),
@@ -420,6 +432,14 @@ class ClientBase[ConfigT: ClientConfig]:
)
await self.join_room(room.room_id)
+ async def on_removed(self, room: RoomRef, event: InboundMembership) -> None:
+ """This client has been taken out of a room while it was running.
+
+ Nothing to do for a client that holds no events of its own — the
+ transport has already stopped reading the room. A client that hands
+ events to something with a memory overrides this and empties it.
+ """
+
async def on_command(self, room: RoomRef, event: CommandEvent) -> None:
pass
diff --git a/core/switch_core/provisioning/postgres.py b/core/switch_core/provisioning/postgres.py
index dbbb34f45..c640ab8b1 100644
--- a/core/switch_core/provisioning/postgres.py
+++ b/core/switch_core/provisioning/postgres.py
@@ -8,9 +8,10 @@
itself, which is what keeps its transport watching the room. When nobody is
live the membership is written directly, because a client that is not
running has nothing to wake and will find the room when it starts.
-- **Removal** deletes the membership. No leave event is written: a departure
- is not something a reader of the room needs explained, and the timeline the
- log serves is what was said.
+- **Removal** deletes the membership and wakes a live client so it drops the
+ room, mirroring the invitation. No leave event is written: a departure is
+ not something a reader of the room needs explained, and the timeline the log
+ serves is what was said.
- **Discarding a room** is left to the caller's own delete, which cascades.
Nothing here has a second copy to clean up.
"""
@@ -127,13 +128,30 @@ class PostgresProvisioning:
await session.commit()
async def kick_user(self, room_id: str, user_id: str) -> None:
- """Remove a membership. Already out is success, per the port."""
+ """Remove a membership, stopping a live client's delivery with it.
+
+ Already out is success, per the port.
+
+ The wake-up is the other half of the removal, not a nicety. A running
+ client holds its own subscription to the room, so deleting the row on
+ its own leaves it reading a room it is no longer in. It is rung after
+ the commit so a client that reacts by re-reading its rooms cannot see
+ the membership it was just removed from.
+
+ By client id, not `user_id`, for the reason `invite_to_room` spells
+ out: `clients.matrix_user_id` is unique per tenant, so "the client for
+ @switch-admin" names one client per tenant and waking by handle would
+ wake whichever tenant's transport last claimed the slot — here, telling
+ the wrong tenant's client to stop reading a room it is still in while
+ the right one carried on reading a room it is not.
+ """
async with self._session_factory() as session:
switch_room_id, client_id, _ = await self._resolve(
session, room_id, user_id
)
await self._room_store.remove_client(session, client_id, switch_room_id)
await session.commit()
+ await self._invites.remove(client_id, room_id)
async def delete_room(self, room_id: str) -> None:
"""Nothing of its own to discard.
diff --git a/core/switch_core/room_service.py b/core/switch_core/room_service.py
index ebd75053a..11219bc3b 100644
--- a/core/switch_core/room_service.py
+++ b/core/switch_core/room_service.py
@@ -877,15 +877,19 @@ class RoomService:
agent_clients = self._resolve_agent_clients(agent_ids)
with tenant_scope(room.tenant_id):
- for matrix_user_id in agent_clients.values():
- await self._matrix_admin.kick_user(room.matrix_room_id, matrix_user_id)
-
+ # Both records of the membership before the wake-up. `kick_user`
+ # takes `client_rooms`; `room_agents` is the one
+ # `get_rooms_for_agent` reads, so a client reacting to the kick by
+ # re-reading its rooms must not still find this one.
async with self._session_factory() as session:
for client_id in agent_clients:
await self._room_store.remove_client(session, client_id, room_id)
await self._room_store.remove_agents(session, room_id, agent_ids)
await session.commit()
+ for matrix_user_id in agent_clients.values():
+ await self._matrix_admin.kick_user(room.matrix_room_id, matrix_user_id)
+
logger.info("Removed %d agents from room %s", len(agent_ids), room_id)
emit_safely(
diff --git a/core/switch_core/transport/invites.py b/core/switch_core/transport/invites.py
index 22a9dee99..74f99fc33 100644
--- a/core/switch_core/transport/invites.py
+++ b/core/switch_core/transport/invites.py
@@ -1,4 +1,4 @@
-"""How a client is told it has been added to a room.
+"""How a client is told its membership of a room changed.
Over Matrix this was an event: the admin invited a user, the user's sync loop
saw the invitation and `ClientBase.on_invite` joined. Nothing else had to know
@@ -12,6 +12,12 @@ refusing to ship. This is the missing signal, in the shape the clients already
handle: an invitation, delivered to a live client, auto-accepted by the same
code path as before.
+Removal is the same signal in reverse, and it is not optional. The homeserver
+used to enforce a kick: a removed client's sync simply stopped returning the
+room. Here the subscription is the client's own, so deleting the membership row
+leaves it reading a room it is no longer in. A removal must therefore be rung
+as well as written.
+
**It is in-process, and that is a real limitation.** A client running in
another replica of switch-core would not hear it. That is the same constraint
Matrix sync sessions imposed and the reason switch-core is single-replica
@@ -44,33 +50,54 @@ InviteHandler = Callable[[str], Awaitable[None]]
class InviteBus:
- """Routes an invitation to the live transport for a client, if there is one."""
+ """Routes a membership change to the live transport for a client, if any.
+
+ Named for the signal it carried first. It now carries removals too, since
+ both are the same thing: telling a running client that the set of rooms it
+ should be reading has changed underneath it.
+ """
def __init__(self) -> None:
self._handlers: dict[str, InviteHandler] = {}
+ self._removal_handlers: dict[str, InviteHandler] = {}
- def register(self, client_id: str, handler: InviteHandler) -> None:
- """Make `handler` the live listener for `client_id`, displacing any other.
+ def register(
+ self, client_id: str, handler: InviteHandler, removal_handler: InviteHandler
+ ) -> None:
+ """Make this pair the live listeners for `client_id`, displacing any other.
One slot per client, so a client that starts while an older instance of
itself is still unwinding takes the slot from it. That is the right
answer — the newer one is the client with a live receive loop — and it
- is why `unregister` wants the handler back.
+ is why `unregister` wants the handlers back.
+
+ Both arms are taken together and never separately. A client registered
+ for invitations but not removals is the shape of a leak — it would be
+ told about every room it gains and none it loses — so the state is made
+ unrepresentable here rather than guarded against at every use.
"""
self._handlers[client_id] = handler
+ self._removal_handlers[client_id] = removal_handler
- def unregister(self, client_id: str, handler: InviteHandler) -> None:
- """Give the slot up, but only if `handler` still holds it.
+ def unregister(
+ self, client_id: str, handler: InviteHandler, removal_handler: InviteHandler
+ ) -> None:
+ """Give the slot up, but only if these handlers still hold it.
A transport unregisters from the `finally` of its receive loop, which
runs whenever that loop ends — including long after a newer instance of
the same client took the slot. Clearing by id alone would deafen that
newer one: `invite` would find nobody listening, the caller would write
the membership row itself, and the client would sit in a room it was
- never told it was in until something restarted it.
+ never told it was in until something restarted it. For a removal the
+ same mistake is worse, because there is no membership row for a caller
+ to fall back to — the newer client would simply keep delivering a room
+ it had been taken out of.
"""
if self._handlers.get(client_id) == handler:
del self._handlers[client_id]
+ if self._removal_handlers.get(client_id) == removal_handler:
+ del self._removal_handlers[client_id]
async def invite(self, client_id: str, transport_room_id: str) -> bool:
"""Tell `client_id` it is in `transport_room_id`.
@@ -84,3 +111,27 @@ class InviteBus:
return False
await handler(transport_room_id)
return True
+
+ async def remove(self, client_id: str, transport_room_id: str) -> None:
+ """Tell `client_id` it is no longer in `transport_room_id`.
+
+ Nothing is returned, unlike `invite`: the caller has nothing to fall
+ back to. A client that is not running holds no subscription to drop,
+ and one that starts later reads its rooms from the table the removal
+ already updated, so "nobody was listening" is the expected answer.
+
+ A client listening for invitations but not removals is not expected —
+ `register` makes it unrepresentable — and is reported rather than left
+ to surface as a room quietly delivered to someone no longer in it.
+ """
+ handler = self._removal_handlers.get(client_id)
+ if handler is None:
+ if client_id in self._handlers:
+ logger.error(
+ "Client %s is listening for invitations but not for removals; "
+ "it will keep delivering %s after being taken out of it",
+ client_id,
+ transport_room_id,
+ )
+ return
+ await handler(transport_room_id)
diff --git a/core/switch_core/transport/port.py b/core/switch_core/transport/port.py
index 70c5e5de9..d2f5cb825 100644
--- a/core/switch_core/transport/port.py
+++ b/core/switch_core/transport/port.py
@@ -41,6 +41,10 @@ class TransportHandlers:
on_member_event: Handler | None = None
on_custom_event: Handler | None = None
on_invite: Handler | None = None
+ # The other half of `on_invite`. The transport has already stopped reading
+ # the room by the time this fires; it is for whatever downstream still
+ # holds the room's events, which the transport knows nothing about.
+ on_removed: Handler | None = None
@runtime_checkable
diff --git a/core/switch_core/transport/postgres.py b/core/switch_core/transport/postgres.py
index e7f822142..4faaae16c 100644
--- a/core/switch_core/transport/postgres.py
+++ b/core/switch_core/transport/postgres.py
@@ -227,28 +227,33 @@ class PostgresTransport:
was away is not delivered here. That is a delivery-cursor question,
and delivery cursors are a layer above this one.
"""
- rooms = await self.joined_rooms()
- if not rooms:
- logger.error(
- "Client %s is receiving but is a member of no room: it will "
- "hear nothing until something adds it to one. Under Matrix "
- "membership lived on the homeserver; here it is the "
- "client_rooms table, so a client whose rows were never "
- "written is silent rather than broken",
- self.user_id,
- )
- delivery = asyncio.create_task(self._deliver_forever())
- for transport_room_id in rooms:
- await self._watch(transport_room_id)
+ # Before the rooms are read, not after they are subscribed: reading
+ # them and watching each is several round trips, and a removal rung
+ # during those has no membership row for the caller to fall back to.
+ self._invites.register(self.client_id, self._on_invited, self._on_removed)
self._receiving = True
- self._invites.register(self.client_id, self._on_invited)
try:
- await self._closed.wait()
+ rooms = await self.joined_rooms()
+ if not rooms:
+ logger.error(
+ "Client %s is receiving but is a member of no room: it will "
+ "hear nothing until something adds it to one. Under Matrix "
+ "membership lived on the homeserver; here it is the "
+ "client_rooms table, so a client whose rows were never "
+ "written is silent rather than broken",
+ self.user_id,
+ )
+ delivery = asyncio.create_task(self._deliver_forever())
+ try:
+ for transport_room_id in rooms:
+ await self._watch(transport_room_id)
+ await self._closed.wait()
+ finally:
+ delivery.cancel()
finally:
self._receiving = False
- self._invites.unregister(self.client_id, self._on_invited)
+ self._invites.unregister(self.client_id, self._on_invited, self._on_removed)
self._unwatch_all()
- delivery.cancel()
async def _deliver_forever(self) -> None:
"""This client's own delivery loop: its rooms, one at a time.
@@ -324,6 +329,39 @@ class PostgresTransport:
),
)
+ async def _on_removed(self, transport_room_id: str) -> None:
+ """This client was taken out of a room while it was running.
+
+ The subscription is dropped here rather than left to the next restart.
+ Under Matrix a kick ended the room's delivery at the homeserver, so
+ nothing downstream had to check membership again; here the
+ subscription is this client's own and outlives the row unless it is
+ taken back.
+
+ The client is told as well, because dropping the subscription only
+ stops what has not been read yet. Anything already handed over is the
+ client's now — an agent's event buffer holds it for fifteen minutes and
+ will serve it to a poll or a resumed stream — so a removal has to reach
+ whoever is holding it, not just the reader that would have fetched
+ more.
+ """
+ self._unwatch(transport_room_id)
+ handler = self._handlers.on_removed
+ if handler is None:
+ return
+ await handler(
+ RoomRef(room_id=transport_room_id),
+ InboundMembership(
+ room_id=transport_room_id,
+ event_id=new_event_id(),
+ sender=self.user_id,
+ timestamp=_now_ms(),
+ state_key=self.user_id,
+ membership="leave",
+ display_name=self.display_name,
+ ),
+ )
+
async def _watch(
self, transport_room_id: str, *, from_seq: int | None = None
) -> None:
@@ -340,18 +378,43 @@ class PostgresTransport:
Restoring the loop-back rather than dispatching the one handler that
noticed keeps every guard downstream in play, and covers whatever else
depended on hearing its own join.
+
+ The slot in `_watching` is claimed before the head is read, so a
+ removal landing on that await has something to take back; finding the
+ claim gone afterwards is the signal to subscribe to nothing.
"""
room_id, tenant_id = await self._resolve_room_and_tenant(transport_room_id)
- if room_id in self._cursors:
+ if room_id in self._watching:
return
+ self._watching[room_id] = transport_room_id
if from_seq is not None:
- self._cursors[room_id] = from_seq
+ # The caller wrote the membership row and committed it a moment
+ # ago, so there is nothing to confirm.
+ seq = from_seq
else:
- async with tenant_session(self._session_factory, tenant_id) as session:
- self._cursors[room_id] = await self._message_store.head_seq(
- session, room_id
- )
- self._watching[room_id] = transport_room_id
+ try:
+ async with tenant_session(self._session_factory, tenant_id) as session:
+ member = await session.get(
+ ClientRoom,
+ {"client_id": self.client_id, "room_id": room_id},
+ )
+ if member is None:
+ # Removed between the room list this came from and
+ # here. The wake-up for it found nothing to unwatch, so
+ # the row is the only thing that still says so.
+ self._release_claim(room_id, transport_room_id)
+ return
+ seq = await self._message_store.head_seq(session, room_id)
+ except Exception:
+ # A claim left behind would make every later `_watch` return
+ # early on a room nothing is subscribed to.
+ self._release_claim(room_id, transport_room_id)
+ raise
+ if self._watching.get(room_id) != transport_room_id:
+ # Removed while the row and head were being read. `_unwatch` has
+ # already undone the claim and there is nothing subscribed to undo.
+ return
+ self._cursors[room_id] = seq
self._listener.subscribe(room_id, self._on_room_advanced)
self._ephemeral.subscribe(transport_room_id, self._on_ephemeral)
if from_seq is not None:
@@ -360,12 +423,37 @@ class PostgresTransport:
self._pending.add(room_id)
self._wake.set()
+ def _unwatch(self, transport_room_id: str) -> None:
+ """Stop delivering one room. Not watching it is success.
+
+ The cursor goes with the subscription. Keeping it would mean a client
+ added back to the room resumed from where it left off and was handed
+ everything said while it was out.
+ """
+ room_id = self._room_ids.get(transport_room_id)
+ if room_id is None or room_id not in self._watching:
+ return
+ self._release_claim(room_id, transport_room_id)
+
+ def _release_claim(self, room_id: str, transport_room_id: str) -> None:
+ """Undo everything `_watch` records for a room, subscribed or not.
+
+ Safe on a room `_watch` has claimed but not yet subscribed: both buses
+ treat unsubscribing something that never subscribed as success, which
+ is what lets `_watch` claim its slot before it awaits.
+ """
+ self._listener.unsubscribe(room_id, self._on_room_advanced)
+ self._ephemeral.unsubscribe(transport_room_id, self._on_ephemeral)
+ self._watching.pop(room_id, None)
+ self._cursors.pop(room_id, None)
+ # A wake-up already queued for this room would otherwise be drained
+ # after the subscription was dropped.
+ self._pending.discard(room_id)
+
def _unwatch_all(self) -> None:
- for room_id, transport_room_id in self._watching.items():
- self._listener.unsubscribe(room_id, self._on_room_advanced)
- self._ephemeral.unsubscribe(transport_room_id, self._on_ephemeral)
- self._watching.clear()
- self._cursors.clear()
+ """Drop every room, through the same path a single removal takes."""
+ for transport_room_id in list(self._watching.values()):
+ self._unwatch(transport_room_id)
async def _on_ephemeral(self, event: InboundCustomEvent) -> None:
"""An unstored event in a room this client watches.
@@ -404,6 +492,10 @@ class PostgresTransport:
raises does not cost the rows already delivered — the failure belongs
to one event, and redelivering its neighbours would be worse than
dropping it.
+
+ Every delivery is a suspension point, so the subscription is re-read
+ per row. A removal landing mid-page has to stop the rows it has not
+ reached, and must not be undone by a cursor write after it.
"""
transport_room_id = self._watching.get(room_id)
if transport_room_id is None:
@@ -422,11 +514,14 @@ class PostgresTransport:
# share a session with the read.
with tenant_scope(tenant_id):
while True:
+ cursor = self._cursors.get(room_id)
+ if cursor is None or room_id not in self._watching:
+ return
async with tenant_session(self._session_factory, tenant_id) as session:
rows = await self._message_store.list_for_room(
session,
room_id,
- after_seq=self._cursors[room_id],
+ after_seq=cursor,
limit=_DELIVERY_PAGE,
)
attachments = await self._message_store.attachments_for(
@@ -435,6 +530,8 @@ class PostgresTransport:
if not rows:
return
for row in rows:
+ if room_id not in self._watching:
+ return
self._cursors[room_id] = row.seq
await self._deliver(
transport_room_id, row, attachments.get(row.id, [])
diff --git a/core/tests/integration/test_removal_stops_delivery_e2e.py b/core/tests/integration/test_removal_stops_delivery_e2e.py
new file mode 100644
index 000000000..08915c1ce
--- /dev/null
+++ b/core/tests/integration/test_removal_stops_delivery_e2e.py
@@ -0,0 +1,177 @@
+"""End-to-end: removing an agent from a room stops delivering that room to it.
+
+Drives the genuine path, no mocks and no stubs. `RoomService` removes the agent
+the way the gateway's `DELETE /rooms/{id}/agents/{id}` does → the kick is rung
+over the invite bus → the transport drops the subscription and tells its client
+→ the client empties the event buffer of that room → every reader stops serving
+it.
+
+The unit tests cover each of those links. What only this can show is that they
+are joined: severing the one line in `ClientBase.setup` that wires the removal
+handler to the transport left the whole unit suite green while a kick reached
+nothing.
+
+Two readers rather than one, because they fail differently. `poll_events`
+applies membership from the database on every poll, so it is right even if the
+signal never arrives. The notification stream and the SSE reader have only the
+eviction — an SSE stream holds no session to re-derive membership with — so
+they are the ones that show whether the removal was really acted on.
+"""
+
+from __future__ import annotations
+
+import asyncio
+
+import pytest
+
+from switch_core.room_service import RoomCreateConfig
+from tests.integration.conftest import Harness
+
+pytestmark = [pytest.mark.integration, pytest.mark.asyncio(loop_scope="session")]
+
+
+async def _wait_joined(
+ client: object, matrix_room_id: str, timeout: float = 30
+) -> None:
+ deadline = asyncio.get_event_loop().time() + timeout
+ while asyncio.get_event_loop().time() < deadline:
+ if matrix_room_id in client.room_join_times: # type: ignore[attr-defined]
+ return
+ await asyncio.sleep(0.25)
+ raise AssertionError(f"client never joined {matrix_room_id} within {timeout}s")
+
+
+async def _wait_buffered(
+ harness: Harness, agent_id: str, room_id: str, body: str, timeout: float = 30
+) -> None:
+ """Block until the message is in the agent's buffer, addressed to it.
+
+ Addressed matters: an unaddressed message is not notifiable, and the
+ notification half of this test would then pass whatever the code did.
+ """
+ deadline = asyncio.get_event_loop().time() + timeout
+ while asyncio.get_event_loop().time() < deadline:
+ for item in harness.event_buffer.read_from(agent_id, 0):
+ if (
+ item.room_id == room_id
+ and getattr(item.event.payload, "body", None) == body
+ ):
+ assert item.notifiable, (
+ f"{body!r} was buffered but not as an addressed message, so "
+ "the notification assertions below would be vacuous"
+ )
+ return
+ await asyncio.sleep(0.25)
+ raise AssertionError(f"{body!r} never reached the buffer for room {room_id}")
+
+
+def _rooms_in_buffer(harness: Harness, agent_id: str) -> set[str]:
+ return {item.room_id for item in harness.event_buffer.read_from(agent_id, 0)}
+
+
+async def test_removal_stops_every_reader_serving_the_room(harness: Harness) -> None:
+ watcher = await harness.register_agent("e2e-removed-watcher")
+ talker = await harness.register_agent("e2e-removed-talker")
+ await harness.start_clients()
+ watcher_client = harness.client_for(watcher.agent_id)
+ await watcher_client.wait_ready()
+ await harness.client_for(talker.agent_id).wait_ready()
+
+ rooms = {}
+ for label in ("left", "kept"):
+ result = await harness.room_service.create_room(
+ RoomCreateConfig(
+ name=f"e2e-removal-{label}",
+ description=f"integration removal test ({label})",
+ agent_ids=[watcher.agent_id, talker.agent_id],
+ )
+ )
+ rooms[label] = result.room
+ await _wait_joined(watcher_client, result.room.matrix_room_id)
+
+ talker_client = harness.client_for(talker.agent_id)
+ bodies = {}
+ for label, room in rooms.items():
+ bodies[label] = f"@e2e-removed-watcher please look at the {label} room"
+ await talker_client.send_message(room.matrix_room_id, bodies[label])
+ await _wait_buffered(harness, watcher.agent_id, room.id, bodies[label])
+
+ # Both rooms are in the buffer and addressed, so every assertion after the
+ # removal is about the removal rather than about nothing being there.
+ assert _rooms_in_buffer(harness, watcher.agent_id) == {
+ rooms["left"].id,
+ rooms["kept"].id,
+ }
+
+ await harness.room_service.remove_agents_from_room(
+ rooms["left"].id, [watcher.agent_id]
+ )
+
+ # The buffer is what the SSE reader reads, and it has no membership of its
+ # own to apply — this is the whole of its defence.
+ deadline = asyncio.get_event_loop().time() + 15
+ while asyncio.get_event_loop().time() < deadline:
+ if rooms["left"].id not in _rooms_in_buffer(harness, watcher.agent_id):
+ break
+ await asyncio.sleep(0.25)
+ assert _rooms_in_buffer(harness, watcher.agent_id) == {rooms["kept"].id}, (
+ "the removed room's events are still retained for the agent, so an SSE "
+ "reader resuming from an older cursor would be handed them"
+ )
+
+ notifications = await harness.protocol.poll_notifications(
+ watcher.agent_id, timeout=0
+ )
+ assert [e.room_id for e in notifications] == [rooms["kept"].id]
+ assert [getattr(e.payload, "body", None) for e in notifications] == [bodies["kept"]]
+
+ events = await harness.protocol.poll_events(watcher.agent_id, timeout=0)
+ assert rooms["left"].id not in {e.room_id for e in events}
+ assert rooms["kept"].id in {e.room_id for e in events}
+
+
+async def test_a_room_the_agent_is_still_in_is_untouched(harness: Harness) -> None:
+ """The removal empties one room, not the agent.
+
+ `EventBuffer.remove` exists and drops everything; reaching for it on a
+ removal would take the agent's other rooms with it, which is the same
+ silence arrived at from the opposite direction.
+ """
+ watcher = await harness.register_agent("e2e-still-in-watcher")
+ talker = await harness.register_agent("e2e-still-in-talker")
+ await harness.start_clients()
+ watcher_client = harness.client_for(watcher.agent_id)
+ await watcher_client.wait_ready()
+ await harness.client_for(talker.agent_id).wait_ready()
+
+ kept = await harness.room_service.create_room(
+ RoomCreateConfig(
+ name="e2e-removal-untouched",
+ description="integration removal test (untouched)",
+ agent_ids=[watcher.agent_id, talker.agent_id],
+ )
+ )
+ left = await harness.room_service.create_room(
+ RoomCreateConfig(
+ name="e2e-removal-departed",
+ description="integration removal test (departed)",
+ agent_ids=[watcher.agent_id, talker.agent_id],
+ )
+ )
+ await _wait_joined(watcher_client, kept.room.matrix_room_id)
+ await _wait_joined(watcher_client, left.room.matrix_room_id)
+
+ body = "@e2e-still-in-watcher this one still applies"
+ await harness.client_for(talker.agent_id).send_message(
+ kept.room.matrix_room_id, body
+ )
+ await _wait_buffered(harness, watcher.agent_id, kept.room.id, body)
+
+ await harness.room_service.remove_agents_from_room(left.room.id, [watcher.agent_id])
+
+ notifications = await harness.protocol.poll_notifications(
+ watcher.agent_id, timeout=0
+ )
+ assert [getattr(e.payload, "body", None) for e in notifications] == [body], (
+ "being removed from one room took the agent's other room with it"
+ )
diff --git a/core/tests/switch_core/bridges/agent/protocol/test_connections.py b/core/tests/switch_core/bridges/agent/protocol/test_connections.py
index 53b2a8428..9ae5fe159 100644
--- a/core/tests/switch_core/bridges/agent/protocol/test_connections.py
+++ b/core/tests/switch_core/bridges/agent/protocol/test_connections.py
@@ -311,6 +311,52 @@ def test_coverage_returns_to_the_daemon_when_the_session_goes() -> None:
assert registry.holder_of(AGENT, ROOM_A) is daemon
+def test_a_removal_takes_the_room_off_every_connection_of_that_agent() -> None:
+ """A claim outlives the membership it was checked against.
+
+ `require_room_member` runs when a room is claimed and never again, so a
+ session that claimed a room the agent has since been removed from keeps
+ covering it for the life of its stream.
+ """
+ registry = ConnectionRegistry()
+ session = _open(registry, "session", scope="single")
+ other = _open(registry, "other", scope="single")
+ registry.claim_room(session, ROOM_A)
+ registry.claim_room(other, ROOM_B)
+
+ registry.release_room_everywhere(AGENT, ROOM_A)
+
+ assert session.rooms == set()
+ assert other.rooms == {ROOM_B}
+ assert registry.claimant_of(AGENT, ROOM_A) is None
+
+
+def test_a_removal_does_not_touch_another_agents_claim() -> None:
+ registry = ConnectionRegistry()
+ mine = _open(registry, "c1")
+ theirs = _open(registry, "c2", agent_id=OTHER_AGENT)
+ registry.claim_room(mine, ROOM_A)
+ registry.claim_room(theirs, ROOM_A)
+
+ registry.release_room_everywhere(AGENT, ROOM_A)
+
+ assert mine.rooms == set()
+ assert theirs.rooms == {ROOM_A}
+
+
+def test_a_removal_reaches_a_connection_whose_heartbeat_has_lapsed() -> None:
+ """A lapsed connection still holds its claim, and can be reconnected to."""
+ registry = ConnectionRegistry()
+ conn = _open(registry, "c1")
+ registry.claim_room(conn, ROOM_A)
+ conn.last_beat = time.monotonic() - HEARTBEAT_TTL_SECONDS - 1
+ assert registry.for_agent(AGENT) == []
+
+ registry.release_room_everywhere(AGENT, ROOM_A)
+
+ assert conn.rooms == set()
+
+
def test_rooms_of_one_agent_do_not_block_another() -> None:
registry = ConnectionRegistry()
mine = _open(registry, "c1")
diff --git a/core/tests/switch_core/bridges/agent/protocol/test_event_buffer.py b/core/tests/switch_core/bridges/agent/protocol/test_event_buffer.py
index c68fd10ea..f1c04b959 100644
--- a/core/tests/switch_core/bridges/agent/protocol/test_event_buffer.py
+++ b/core/tests/switch_core/bridges/agent/protocol/test_event_buffer.py
@@ -20,10 +20,10 @@ AGENT = "agent-1"
ROOM = "room-1"
-def _message(addressed: bool) -> AgentEvent:
+def _message(addressed: bool, room_id: str = ROOM) -> AgentEvent:
return AgentEvent(
type="message",
- room_id=ROOM,
+ room_id=room_id,
payload=MessagePayload(
addressed=addressed,
sender="@u:s",
@@ -64,7 +64,7 @@ async def test_addressed_message_fans_out_without_draining_room_queue() -> None:
q.enqueue(AGENT, ROOM, _message(addressed=True))
# The notification stream sees it...
- notifs = await q.poll_notifications(AGENT, timeout=0)
+ notifs = await q.poll_notifications(AGENT, timeout=0, rooms={ROOM})
assert len(notifs) == 1
assert notifs[0].type == "message"
@@ -77,7 +77,7 @@ async def test_unaddressed_message_does_not_fan_out() -> None:
q = EventBuffer()
q.enqueue(AGENT, ROOM, _message(addressed=False))
- assert await q.poll_notifications(AGENT, timeout=0) == []
+ assert await q.poll_notifications(AGENT, timeout=0, rooms={ROOM}) == []
# Still queued per-room (unaddressed chatter is delivered there, just not
# surfaced as a notification).
assert len(await q.poll_room(AGENT, ROOM, timeout=0)) == 1
@@ -86,7 +86,7 @@ async def test_unaddressed_message_does_not_fan_out() -> None:
async def test_task_event_fans_out() -> None:
q = EventBuffer()
q.enqueue(AGENT, ROOM, _task_delegate())
- notifs = await q.poll_notifications(AGENT, timeout=0)
+ notifs = await q.poll_notifications(AGENT, timeout=0, rooms={ROOM})
assert len(notifs) == 1
assert notifs[0].type == "task_delegate"
@@ -94,16 +94,151 @@ async def test_task_event_fans_out() -> None:
async def test_room_join_fans_out_only_when_listening() -> None:
q = EventBuffer()
q.enqueue(AGENT, ROOM, _room_join(listening=False))
- assert await q.poll_notifications(AGENT, timeout=0) == []
+ assert await q.poll_notifications(AGENT, timeout=0, rooms={ROOM}) == []
q.enqueue(AGENT, ROOM, _room_join(listening=True))
- notifs = await q.poll_notifications(AGENT, timeout=0)
+ notifs = await q.poll_notifications(AGENT, timeout=0, rooms={ROOM})
assert len(notifs) == 1
assert notifs[0].type == "room_join"
+async def test_polling_everything_can_be_limited_to_the_rooms_given() -> None:
+ """The buffer is keyed by agent and knows nothing about who is in what.
+
+ An event queued while the agent was a member stays queued after it is
+ removed, so the caller passes the rooms it is in now and that is what
+ keeps the event from being handed over.
+ """
+ q = EventBuffer()
+ q.enqueue(AGENT, ROOM, _message(addressed=False))
+ q.enqueue(AGENT, "room-2", _message(addressed=False, room_id="room-2"))
+
+ polled = await q.poll(AGENT, timeout=0, rooms={"room-2"})
+
+ assert [event.room_id for event in polled] == ["room-2"]
+
+
+async def test_polling_with_no_rooms_at_all_returns_nothing() -> None:
+ """An agent in no rooms is not a caller asking for every room."""
+ q = EventBuffer()
+ q.enqueue(AGENT, ROOM, _message(addressed=False))
+
+ assert await q.poll(AGENT, timeout=0, rooms=set()) == []
+
+
+async def test_notifications_are_limited_to_the_rooms_given_too() -> None:
+ """The notification stream is the one carrying addressed messages.
+
+ Filtering the all-rooms poll and not this one would leave the leak open on
+ the more sensitive of the two: an agent removed from a room would stop
+ seeing its chatter and go on being handed everything said *to* it there.
+ """
+ q = EventBuffer()
+ q.enqueue(AGENT, ROOM, _message(addressed=True))
+ q.enqueue(AGENT, "room-2", _message(addressed=True, room_id="room-2"))
+
+ polled = await q.poll_notifications(AGENT, timeout=0, rooms={"room-2"})
+
+ assert [event.room_id for event in polled] == ["room-2"]
+
+
+async def test_dropping_a_room_forgets_what_it_still_held() -> None:
+ """What the transport stops reading, the buffer has to stop holding.
+
+ Dropping the subscription only governs what has not been read yet. An
+ event already in here is served to a long poll, to the notification stream
+ and to an SSE reader resuming from an old cursor for the whole retention
+ window.
+ """
+ q = EventBuffer()
+ q.enqueue(AGENT, ROOM, _message(addressed=True))
+ q.enqueue(AGENT, "room-2", _message(addressed=True, room_id="room-2"))
+
+ q.drop_room(AGENT, ROOM)
+
+ # Gone from the low-level read every reader is built on, filter or no
+ # filter — which is what closes it for the stream, the one reader with no
+ # membership of its own to apply.
+ assert [item.room_id for item in q.read_from(AGENT, 0)] == ["room-2"]
+
+
+async def test_dropping_a_room_does_not_report_a_gap() -> None:
+ """A reader that skips the dropped events has missed nothing it was owed.
+
+ Saying otherwise would send an agent off to re-read the context of a room
+ it is no longer in.
+ """
+ q = EventBuffer()
+ q.enqueue(AGENT, ROOM, _message(addressed=True))
+ q.enqueue(AGENT, "room-2", _message(addressed=True, room_id="room-2"))
+
+ q.drop_room(AGENT, ROOM)
+
+ assert not q.has_gap_before(AGENT, 0)
+ assert [item.seq for item in q.read_from(AGENT, 0)] == [2]
+
+
+async def test_the_cursor_advances_past_what_the_reader_can_never_want() -> None:
+ """Unaddressed chatter is excluded on a property of the event, not on who
+ the agent is, so the notification reader will never want it and the cursor
+ is free to move past it. Left behind it, the cursor parks below the head
+ until retention trims it and the next read reports a gap that never was.
+ """
+ q = EventBuffer()
+ q.enqueue(AGENT, ROOM, _message(addressed=False))
+ q.enqueue(AGENT, ROOM, _message(addressed=False))
+
+ assert await q.poll_notifications(AGENT, timeout=0, rooms={ROOM}) == []
+ assert q._cursors[AGENT]["legacy:notifications"] == 2
+
+
+async def test_the_cursor_does_not_advance_past_a_room_joined_later() -> None:
+ """Membership is not a property of the event, so it can change.
+
+ A poll reads the agent's rooms once; anything arriving for a room it joins
+ while that poll is parked is filtered against the older set. Confirming
+ past it would put the cursor beyond an event the agent is entitled to, and
+ no later poll would ever reach it again — it would sit in the buffer,
+ retained and unreachable, until it aged out.
+ """
+ q = EventBuffer()
+ q.enqueue(AGENT, "room-2", _message(addressed=True, room_id="room-2"))
+
+ assert await q.poll(AGENT, timeout=0, rooms={ROOM}) == []
+ assert q._cursors[AGENT]["legacy:all"] == 0
+
+ polled = await q.poll(AGENT, timeout=0, rooms={ROOM, "room-2"})
+ assert [event.room_id for event in polled] == ["room-2"]
+
+
+async def test_the_notification_stream_also_survives_a_room_joined_later() -> None:
+ """The same rule on the stream carrying what was said *to* the agent."""
+ q = EventBuffer()
+ q.enqueue(AGENT, "room-2", _message(addressed=True, room_id="room-2"))
+
+ assert await q.poll_notifications(AGENT, timeout=0, rooms={ROOM}) == []
+ polled = await q.poll_notifications(AGENT, timeout=0, rooms={ROOM, "room-2"})
+ assert [event.room_id for event in polled] == ["room-2"]
+
+
+async def test_a_room_reader_still_advances_past_other_rooms() -> None:
+ """`poll_room`'s filter is the reader's own scope, not a membership snapshot.
+
+ `legacy:room:X` will never want room Y whatever the agent joins later, so
+ parking its cursor behind Y's events would strand it for no gain.
+ """
+ q = EventBuffer()
+ q.enqueue(AGENT, "room-2", _message(addressed=True, room_id="room-2"))
+ q.enqueue(AGENT, ROOM, _message(addressed=True))
+
+ polled = await q.poll_room(AGENT, ROOM, timeout=0)
+
+ assert [event.room_id for event in polled] == [ROOM]
+ assert q._cursors[AGENT][f"legacy:room:{ROOM}"] == 2
+
+
async def test_remove_clears_notification_queue() -> None:
q = EventBuffer()
q.enqueue(AGENT, ROOM, _message(addressed=True))
q.remove(AGENT)
- assert await q.poll_notifications(AGENT, timeout=0) == []
+ assert await q.poll_notifications(AGENT, timeout=0, rooms={ROOM}) == []
diff --git a/core/tests/switch_core/bridges/agent/protocol/test_event_buffer_resume.py b/core/tests/switch_core/bridges/agent/protocol/test_event_buffer_resume.py
index d68328e63..6dab53d4f 100644
--- a/core/tests/switch_core/bridges/agent/protocol/test_event_buffer_resume.py
+++ b/core/tests/switch_core/bridges/agent/protocol/test_event_buffer_resume.py
@@ -167,8 +167,8 @@ async def test_legacy_pollers_no_longer_steal_from_each_other() -> None:
buf.enqueue(AGENT, ROOM_A, _message("hello", addressed=True))
room = await buf.poll_room(AGENT, ROOM_A, timeout=0)
- notif = await buf.poll_notifications(AGENT, timeout=0)
- every = await buf.poll(AGENT, timeout=0)
+ notif = await buf.poll_notifications(AGENT, timeout=0, rooms={ROOM_A})
+ every = await buf.poll(AGENT, timeout=0, rooms={ROOM_A})
assert [e.payload.body for e in room] == ["hello"]
assert [e.payload.body for e in notif] == ["hello"]
diff --git a/core/tests/switch_core/bridges/agent/protocol/test_poll_events_membership.py b/core/tests/switch_core/bridges/agent/protocol/test_poll_events_membership.py
new file mode 100644
index 000000000..0f36feebd
--- /dev/null
+++ b/core/tests/switch_core/bridges/agent/protocol/test_poll_events_membership.py
@@ -0,0 +1,109 @@
+"""Polling every room applies membership, as polling one room does.
+
+The buffer is keyed by agent, so an event queued while the agent was a member
+is still there after it is removed. `poll_room_events` asks
+`require_room_member` first; this is the same question asked by the two calls
+that have no room in them — the all-rooms poll and the notification stream.
+"""
+
+from __future__ import annotations
+
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from switch_core.bridges.agent.protocol.event_buffer import EventBuffer
+from switch_core.bridges.agent.protocol.types import AgentEvent, MessagePayload
+from switch_core.db.models import Room
+from switch_core.db.stores.agent_session_store import AgentSessionStore
+from switch_core.db.stores.room_store import RoomStore
+from tests.switch_core.bridges.agent.protocol.registration_harness import (
+ make_owner,
+ make_service,
+ register,
+)
+
+
+def _message(room_id: str, body: str, *, addressed: bool = False) -> AgentEvent:
+ return AgentEvent(
+ type="message",
+ room_id=room_id,
+ payload=MessagePayload(
+ addressed=addressed,
+ sender="@someone:test",
+ sender_name="someone",
+ message_id="$m",
+ body=body,
+ timestamp=0,
+ ),
+ )
+
+
+async def _room(
+ session_factory: async_sessionmaker[AsyncSession], name: str, agent_id: str
+) -> str:
+ async with session_factory() as session:
+ room = Room(matrix_room_id=f"!{name}:test", name=name, description="")
+ session.add(room)
+ await session.flush()
+ await RoomStore().add_agents(session, room.id, [agent_id])
+ await session.commit()
+ return room.id
+
+
+async def test_events_from_a_room_the_agent_was_removed_from_are_not_returned(
+ session_factory: async_sessionmaker[AsyncSession],
+) -> None:
+ svc = make_service(session_factory)
+ svc.room_store = RoomStore() # type: ignore[attr-defined]
+ svc.agent_session_store = AgentSessionStore() # type: ignore[attr-defined]
+ svc.event_buffer = EventBuffer() # type: ignore[attr-defined]
+
+ owner = await make_owner(session_factory)
+ agent_id = await register(svc, "poller", owner)
+ kept = await _room(session_factory, "kept", agent_id)
+ left = await _room(session_factory, "left", agent_id)
+
+ svc.event_buffer.enqueue(agent_id, left, _message(left, "said in the old room"))
+ svc.event_buffer.enqueue(agent_id, kept, _message(kept, "said in this one"))
+
+ async with session_factory() as session:
+ await RoomStore().remove_agents(session, left, [agent_id])
+ await session.commit()
+
+ events = await svc.poll_events(agent_id, timeout=0)
+
+ assert [event.payload.body for event in events] == ["said in this one"]
+
+
+async def test_notifications_from_a_room_the_agent_was_removed_from_are_not_returned(
+ session_factory: async_sessionmaker[AsyncSession],
+) -> None:
+ """The stream that carries what was said *to* the agent, under the same rule.
+
+ Filtering the all-rooms poll and leaving this one open would close the less
+ sensitive half of the leak: the connector reads this to decide what the
+ agent has been asked to do.
+ """
+ svc = make_service(session_factory)
+ svc.room_store = RoomStore() # type: ignore[attr-defined]
+ svc.agent_session_store = AgentSessionStore() # type: ignore[attr-defined]
+ svc.event_buffer = EventBuffer() # type: ignore[attr-defined]
+
+ owner = await make_owner(session_factory)
+ agent_id = await register(svc, "watcher", owner)
+ kept = await _room(session_factory, "kept-notif", agent_id)
+ left = await _room(session_factory, "left-notif", agent_id)
+
+ svc.event_buffer.enqueue(
+ agent_id, left, _message(left, "asked in the old room", addressed=True)
+ )
+ svc.event_buffer.enqueue(
+ agent_id, kept, _message(kept, "asked in this one", addressed=True)
+ )
+
+ async with session_factory() as session:
+ await RoomStore().remove_agents(session, left, [agent_id])
+ await session.commit()
+
+ events = await svc.poll_notifications(agent_id, timeout=0)
+
+ assert [event.payload.body for event in events] == ["asked in this one"]
diff --git a/core/tests/switch_core/bridges/agent/protocol/test_stream.py b/core/tests/switch_core/bridges/agent/protocol/test_stream.py
index 4251180a1..0fabeaf7b 100644
--- a/core/tests/switch_core/bridges/agent/protocol/test_stream.py
+++ b/core/tests/switch_core/bridges/agent/protocol/test_stream.py
@@ -647,3 +647,69 @@ async def test_closed_stream_cannot_detach_a_recreated_connection() -> None:
assert replacement is not conn
await old.aclose()
assert registry.beat(AGENT, replacement.id, 0).stream_attached
+
+
+async def test_a_resumed_stream_does_not_replay_a_room_the_agent_was_removed_from() -> (
+ None
+):
+ """The leak this stream had no way of its own to close.
+
+ Membership is checked when a room is claimed and never again, and an
+ `all`-scope connection covers every room no sibling has claimed, so the
+ stream cannot re-derive who is in what — it holds no session to ask with.
+ A supervisor reconnecting with a resumed cursor therefore replayed the
+ whole retained tail of a room the agent had been removed from.
+
+ What closes it is acting on the removal once, where it is known: the
+ room's events leave the buffer and its claim is released, so there is
+ nothing left for any reader to be handed. This asserts the property at the
+ reader that has no other defence.
+ """
+ registry = ConnectionRegistry()
+ buffer = EventBuffer()
+
+ buffer.enqueue(AGENT, ROOM_A, _message("said before the removal"))
+ buffer.enqueue(AGENT, ROOM_B, _message("still a member here", room=ROOM_B))
+
+ # What a kick does, through the two calls `AgentClient.on_removed` makes.
+ buffer.drop_room(AGENT, ROOM_A)
+ registry.release_room_everywhere(AGENT, ROOM_A)
+
+ # The supervisor comes back and resumes from before both events.
+ conn = _open(registry, connection_id="c-resumed", scope="all", cursor=0)
+ stream = event_stream(conn=conn, registry=registry, buffer=buffer)
+ frames = await _take(stream, 2)
+
+ assert frames[1][0] == "message"
+ assert frames[1][1]["payload"]["body"] == "still a member here"
+
+
+async def test_a_last_event_id_reconnect_does_not_replay_a_removed_room() -> None:
+ """The resume a real client actually performs.
+
+ An SSE client reconnects by sending back the `Last-Event-ID` it last saw;
+ `_resolve_start_cursor` prefers it over `start_from` and the stream resumes
+ from there. That is the path the removed room's retained tail would come
+ back down, and it is the one a browser or an agent runtime takes without
+ being asked — so it is worth pinning separately from an explicitly
+ requested `start_from`.
+ """
+ registry = ConnectionRegistry()
+ buffer = EventBuffer()
+
+ seen = buffer.enqueue(AGENT, ROOM_A, _message("before the drop"))
+ buffer.enqueue(AGENT, ROOM_A, _message("while it was still a member"))
+ buffer.enqueue(AGENT, ROOM_B, _message("in the room it kept", room=ROOM_B))
+
+ buffer.drop_room(AGENT, ROOM_A)
+ registry.release_room_everywhere(AGENT, ROOM_A)
+
+ # `seen` is what the client would send back as Last-Event-ID: a cursor from
+ # while it was still in room A, resolved to exactly this by
+ # `_resolve_start_cursor`.
+ conn = _open(registry, connection_id="c-reconnect", scope="all", cursor=seen)
+ stream = event_stream(conn=conn, registry=registry, buffer=buffer)
+ frames = await _take(stream, 2)
+
+ assert frames[1][0] == "message"
+ assert frames[1][1]["payload"]["body"] == "in the room it kept"
diff --git a/core/tests/switch_core/clients/test_removed_from_room.py b/core/tests/switch_core/clients/test_removed_from_room.py
new file mode 100644
index 000000000..208e7f151
--- /dev/null
+++ b/core/tests/switch_core/clients/test_removed_from_room.py
@@ -0,0 +1,191 @@
+"""What an agent client does when it is taken out of a room while running.
+
+Dropping the transport's subscription governs only what has not been read yet.
+Everything already read is in the event buffer, which is keyed by agent and
+knows nothing about who is in what: it holds each event for the retention
+window and serves it to a long poll, to the notification stream, and to an SSE
+reader resuming from an old cursor. So the removal has to reach the things
+holding the events, not only the reader that would have fetched more.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+from switch_core.bridges.agent.protocol.connections import (
+ PROTOCOL_VERSION,
+ ClientDeclaration,
+ ConnectionRegistry,
+)
+from switch_core.bridges.agent.protocol.event_buffer import EventBuffer
+from switch_core.bridges.agent.protocol.types import AgentEvent, MessagePayload
+from switch_core.clients.agent_client import AgentClient, RoomMeta
+from switch_core.clients.client_base import ClientBase
+from switch_core.transport import InboundMembership, RoomRef
+
+AGENT = "agent-1"
+LEFT = "room-left"
+KEPT = "room-kept"
+
+
+def _message(room_id: str, body: str) -> AgentEvent:
+ return AgentEvent(
+ type="message",
+ room_id=room_id,
+ payload=MessagePayload(
+ addressed=True,
+ sender="@someone:test",
+ sender_name="someone",
+ message_id=f"$m-{body}",
+ body=body,
+ timestamp=0,
+ ),
+ )
+
+
+def _leave(transport_room_id: str) -> InboundMembership:
+ return InboundMembership(
+ room_id=transport_room_id,
+ event_id="$leave",
+ sender="@agent:test",
+ timestamp=0,
+ state_key="@agent:test",
+ membership="leave",
+ display_name="agent",
+ )
+
+
+def _client(buffer: EventBuffer, connections: ConnectionRegistry) -> SimpleNamespace:
+ """A minimal fake `self` for the unbound `AgentClient.on_removed`."""
+ meta = {
+ "!left:test": RoomMeta(
+ room_id=LEFT,
+ name="left",
+ bridge_id=None,
+ channel_type="channel_public",
+ ),
+ }
+
+ async def _resolve_room_meta(matrix_room_id: str) -> RoomMeta | None:
+ return meta.get(matrix_room_id)
+
+ return SimpleNamespace(
+ _event_buffer=buffer,
+ _connections=connections,
+ agent=SimpleNamespace(id=AGENT),
+ _resolve_room_meta=_resolve_room_meta,
+ )
+
+
+async def _removed(stub: SimpleNamespace, transport_room_id: str) -> None:
+ await AgentClient.on_removed(
+ stub, # type: ignore[arg-type]
+ RoomRef(room_id=transport_room_id),
+ _leave(transport_room_id),
+ )
+
+
+async def test_the_rooms_retained_events_are_forgotten() -> None:
+ buffer = EventBuffer()
+ buffer.enqueue(AGENT, LEFT, _message(LEFT, "said in the old room"))
+ buffer.enqueue(AGENT, KEPT, _message(KEPT, "said in this one"))
+
+ await _removed(_client(buffer, ConnectionRegistry()), "!left:test")
+
+ # Read at the level every reader is built on, filter or no filter: this is
+ # what closes it for the stream, which has no membership of its own to
+ # apply and would otherwise replay the room on every resume.
+ assert [item.room_id for item in buffer.read_from(AGENT, 0)] == [KEPT]
+
+
+async def test_the_rooms_claim_is_released() -> None:
+ connections = ConnectionRegistry()
+ conn = connections.open(
+ agent_id=AGENT,
+ connection_id="c1",
+ scope="single",
+ delivery_filter="all",
+ spawn_capable=False,
+ cursor=0,
+ declaration=ClientDeclaration(speaks=PROTOCOL_VERSION),
+ )
+ connections.claim_room(conn, LEFT)
+
+ await _removed(_client(EventBuffer(), connections), "!left:test")
+
+ assert conn.rooms == set()
+
+
+async def test_a_room_that_cannot_be_resolved_drops_nothing() -> None:
+ """Rather than guessing which room was meant and emptying the wrong one."""
+ buffer = EventBuffer()
+ buffer.enqueue(AGENT, LEFT, _message(LEFT, "still here"))
+
+ await _removed(_client(buffer, ConnectionRegistry()), "!unknown:test")
+
+ assert [item.room_id for item in buffer.read_from(AGENT, 0)] == [LEFT]
+
+
+class _CapturingTransport:
+ def __init__(self) -> None:
+ self.handlers: object | None = None
+
+ def register_handlers(self, handlers: object) -> None:
+ self.handlers = handlers
+
+
+class _BareClient(ClientBase):
+ """Enough of a client for `setup` to run and a hook to be observed."""
+
+ def __init__(self, transport: _CapturingTransport) -> None:
+ self.matrix_user_id = "@agent:test"
+ # `_transport` is a property over this, and raises until it is set.
+ self.transport = transport
+ self._self_join_dispatched: set[str] = set()
+ self.removed: list[str] = []
+
+ async def on_removed(self, room: RoomRef, event: InboundMembership) -> None:
+ self.removed.append(room.room_id)
+
+
+async def test_setup_wires_the_removal_hook_to_the_transport() -> None:
+ """The seam the whole eviction path hangs off.
+
+ Everything else here calls `on_removed` directly, and the transport tests
+ prove it dispatches `on_removed` — but nothing joined the two. Severing
+ this one line left every other test in the suite green while a kick
+ reached nobody, which is the leak with an extra step.
+ """
+ transport = _CapturingTransport()
+ client = _BareClient(transport)
+
+ client.setup()
+
+ handlers = transport.handlers
+ assert handlers is not None
+ assert handlers.on_removed is not None, ( # type: ignore[attr-defined]
+ "the transport was given no removal handler, so a kick reaches nothing"
+ )
+
+ await handlers.on_removed( # type: ignore[attr-defined]
+ RoomRef(room_id="!left:test"), _leave("!left:test")
+ )
+ assert client.removed == ["!left:test"]
+
+
+async def test_being_removed_ends_the_visit_so_a_return_is_a_fresh_arrival() -> None:
+ """The same bookkeeping a `leave` delivered as a member event does.
+
+ Otherwise an agent added back to a room it had been removed from would
+ rejoin in silence, because the join it already announced is remembered.
+ """
+ transport = _CapturingTransport()
+ client = _BareClient(transport)
+ client._self_join_dispatched.add("!left:test")
+ client.setup()
+
+ await transport.handlers.on_removed( # type: ignore[attr-defined, union-attr]
+ RoomRef(room_id="!left:test"), _leave("!left:test")
+ )
+
+ assert client._self_join_dispatched == set()
diff --git a/core/tests/switch_core/provisioning/test_postgres_provisioning.py b/core/tests/switch_core/provisioning/test_postgres_provisioning.py
index fa992e330..4b8102cac 100644
--- a/core/tests/switch_core/provisioning/test_postgres_provisioning.py
+++ b/core/tests/switch_core/provisioning/test_postgres_provisioning.py
@@ -46,6 +46,15 @@ async def _room(session: AsyncSession) -> Room:
return room
+async def _never(_room_id: str) -> None:
+ """The arm of a registration a test is not exercising.
+
+ `register` takes both arms together, so the one not under test has to be
+ something — and something that fails the test if it is rung.
+ """
+ raise AssertionError("the other half of the registration was rung")
+
+
def _provisioning(
session_factory: async_sessionmaker[AsyncSession],
invites: InviteBus | None = None,
@@ -126,7 +135,7 @@ class TestMembership:
# because `matrix_user_id` is unique per tenant and every tenant's
# admin client carries the same one.
invites = InviteBus()
- invites.register(client_id, _handler)
+ invites.register(client_id, _handler, _never)
await _provisioning(session_factory, invites).invite_to_room(
transport_room_id, user_id
)
@@ -184,6 +193,86 @@ class TestMembership:
)
assert membership is None
+ async def test_removing_a_running_member_tells_it_to_stop_reading(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ """Deleting the row is only half of a removal.
+
+ A running client holds its own subscription to the room, so the row
+ going away leaves it reading a room it is no longer in. The homeserver
+ used to end the delivery itself; here it has to be rung.
+
+ The handler reads the membership rather than only recording that it was
+ called, because the ordering is the other half of the promise: a client
+ that responds to the wake-up by re-reading its rooms must not still see
+ the one it was removed from. Asserting on the row after `kick_user` has
+ returned would pass whichever side of the commit the ring happened on.
+ """
+ async with session_factory() as session:
+ room = await _room(session)
+ client = await _client(session)
+ await session.commit()
+ transport_room_id, user_id = room.matrix_room_id, client.matrix_user_id
+ room_id, client_id = room.id, client.id
+
+ told: list[str] = []
+ seen_when_told: list[object] = []
+
+ async def _handler(removed_from: str) -> None:
+ told.append(removed_from)
+ async with session_factory() as session:
+ seen_when_told.append(
+ await session.get(
+ ClientRoom, {"client_id": client_id, "room_id": room_id}
+ )
+ )
+
+ invites = InviteBus()
+ provisioning = _provisioning(session_factory, invites)
+ await provisioning.invite_to_room(transport_room_id, user_id)
+ invites.register(client_id, _never, _handler)
+ await provisioning.kick_user(transport_room_id, user_id)
+
+ assert told == [transport_room_id]
+ # Rung after the row is gone, as seen from inside the handler itself.
+ assert seen_when_told == [None]
+
+ async def test_a_removal_names_the_client_row_not_the_handle(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ """`clients.matrix_user_id` is unique per tenant, not globally.
+
+ Every tenant's admin client carries the same handle, so a bus keyed on
+ it would tell whichever tenant's transport last claimed the slot to
+ stop reading a room it is still in — and leave the one actually removed
+ reading on. `invite_to_room` learned this; a removal is the same fact
+ in reverse, and getting it wrong here is the worse half because there
+ is no membership row for the caller to fall back to.
+ """
+ async with session_factory() as session:
+ room = await _room(session)
+ client = await _client(session)
+ await session.commit()
+ transport_room_id, user_id = room.matrix_room_id, client.matrix_user_id
+ client_id = client.id
+
+ told: list[str] = []
+
+ async def _handler(removed_from: str) -> None:
+ told.append(removed_from)
+
+ invites = InviteBus()
+ provisioning = _provisioning(session_factory, invites)
+
+ # Registered under the handle, which is what the bus is *not* keyed on.
+ invites.register(user_id, _never, _handler)
+ await provisioning.kick_user(transport_room_id, user_id)
+ assert told == []
+
+ invites.register(client_id, _never, _handler)
+ await provisioning.kick_user(transport_room_id, user_id)
+ assert told == [transport_room_id]
+
async def test_removing_a_member_who_is_already_out_is_success(
self, session_factory: async_sessionmaker[AsyncSession]
) -> None:
@@ -234,7 +323,7 @@ class TestMembership:
await provisioning.invite_to_room(transport_room_id, user_id)
invites = InviteBus()
- invites.register(client.id, _handler)
+ invites.register(client.id, _handler, _never)
await _provisioning(session_factory, invites).invite_to_room(
transport_room_id, user_id
)
@@ -310,7 +399,7 @@ class TestTwoTenantsSharingAHandle:
invites = InviteBus()
for tenant_id in tenants:
- invites.register(clients[tenant_id], _handler_for(tenant_id))
+ invites.register(clients[tenant_id], _handler_for(tenant_id), _never)
provisioning = _provisioning(rls_harness.restricted, invites)
for tenant_id in tenants:
diff --git a/core/tests/switch_core/test_room_service_pool_holds.py b/core/tests/switch_core/test_room_service_pool_holds.py
index bd6f60377..8b46e1f43 100644
--- a/core/tests/switch_core/test_room_service_pool_holds.py
+++ b/core/tests/switch_core/test_room_service_pool_holds.py
@@ -6,9 +6,12 @@ for ten seconds, and enough of them at once exhaust the pool and start failing
heartbeats fleet-wide. So the contract these tests pin is not a count: it is
that the number of *live* sessions is zero at the moment the Matrix call runs.
-They also pin the ordering the split depends on — Matrix membership must never
-exceed what the database records — because that is what makes the resulting
-inconsistency recoverable rather than a silent over-permission.
+They also pin the ordering the split depends on — transport membership must
+never exceed what the database records — because that is what makes the
+resulting inconsistency recoverable rather than a silent over-permission. Both
+ends of that read the same way: grant last, revoke first. Every row that says
+someone is in a room goes before the kick that tells them so, and none is
+written until the transport side is in place.
"""
from __future__ import annotations
@@ -238,11 +241,26 @@ class TestRemoveAgentsFromRoom:
assert [kind for kind, _who, _live in matrix.calls] == ["kick"] * 3
assert [live for _kind, _who, live in matrix.calls] == [0, 0, 0]
- async def test_the_kick_lands_before_the_row_is_dropped(
+ async def test_the_rows_are_gone_before_the_kick_lands(
self, session_factory: async_sessionmaker[AsyncSession]
) -> None:
- # Removal is the mirror of addition: revoke on Matrix first, so the
- # window is never "joined on Matrix, no membership row".
+ """Both records of the membership go first, then the kick.
+
+ Addition revokes nothing, so it can write the row last; removal has to
+ put every record of the membership beyond reach before it tells anyone,
+ because a client's answer to being kicked is to re-read its rooms.
+ `kick_user` promises exactly that — and keeps it for `client_rooms`
+ alone. `room_agents` is the second record of the same fact, and it is
+ the one `get_rooms_for_agent` reads, which is what the agent bridge
+ applies as membership on every poll. Kicking first left a window where
+ an agent already told to stop still passed that check for the room it
+ had just been removed from.
+
+ The window this opens instead is the harmless one: the rows are gone
+ and the in-memory subscription has not been dropped yet, so every read
+ of membership fails closed, and anything delivered in between is
+ evicted by the removal it is racing.
+ """
room_id, agent_ids, running = await _seed(session_factory, agents=1)
tracker = _TrackingSessionFactory(session_factory)
svc, matrix = _service(tracker, running)
@@ -257,10 +275,60 @@ class TestRemoveAgentsFromRoom:
await svc.remove_agents_from_room(room_id, agent_ids)
- assert seen_at_kick == [agent_ids]
+ assert seen_at_kick == [[]]
async with session_factory() as session:
assert await RoomStore().get_agent_ids(session, room_id) == []
+ async def test_both_records_of_the_membership_go_in_one_transaction(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ """What makes the rows-first order safe against a crash, not just a race.
+
+ The order this reverses was given its direction by CHOO's transaction
+ split (af41b8e9): "Matrix membership must never exceed what the database
+ records", so that a crash between the two halves leaves an agent
+ under-privileged rather than reading a room Switch has no record of it
+ being in. That reasoning rests on Matrix being a second, independent
+ store of membership that survives the crash.
+
+ There is no second store any more. `PostgresProvisioning` is the only
+ implementation of the port, so what `kick_user` revokes is the
+ `client_rooms` row and an in-memory subscription that dies with the
+ process. The dangerous state the invariant protected against is
+ therefore unreachable from this order — provided the two rows that
+ record the membership go together, which is what this pins. Split
+ across two commits, a crash between them would leave `room_agents`
+ saying the agent is a member after `client_rooms` says it is not, and
+ that is the stale-membership read the poll paths apply.
+ """
+ room_id, agent_ids, running = await _seed(session_factory, agents=2)
+ tracker = _TrackingSessionFactory(session_factory)
+ svc, matrix = _service(tracker, running)
+ await svc.add_agents_to_room(room_id, agent_ids=agent_ids)
+
+ # Every state the rows are seen in, sampled from a separate connection
+ # on each commit the service makes.
+ seen: list[tuple[int, int]] = []
+
+ async def _sample() -> None:
+ async with session_factory() as probe:
+ agents = await RoomStore().get_agent_ids(probe, room_id)
+ clients = await RoomStore().get_client_ids(probe, room_id)
+ seen.append((len(agents), len(clients)))
+
+ async def _kick(matrix_room_id: str, matrix_user_id: str) -> None:
+ await _sample()
+
+ matrix.kick_user = _kick # type: ignore[method-assign]
+ await svc.remove_agents_from_room(room_id, agent_ids)
+ await _sample()
+
+ # Never observed with one record of the membership gone and the other
+ # still present — the state a crash between two commits would strand.
+ assert all(a == 0 and c == 0 for a, c in seen), (
+ f"the two membership records were observed out of step: {seen}"
+ )
+
class TestDeleteRoom:
async def test_no_session_is_open_across_the_teardown(
diff --git a/core/tests/switch_core/transport/test_invites.py b/core/tests/switch_core/transport/test_invites.py
index 8ae7cc6cd..dc13bbe6e 100644
--- a/core/tests/switch_core/transport/test_invites.py
+++ b/core/tests/switch_core/transport/test_invites.py
@@ -1,66 +1,129 @@
-"""switch_core.transport.invites: who hears an invitation when two transports
-for one user overlap (CHOO-2623).
+"""switch_core.transport.invites: who hears a membership change when two
+transports for one client overlap (CHOO-2623).
The overlap is real rather than theoretical: a client row can have a second
receive loop started for it — a boot sweep reaching a client registration
already started, a bridge restarting its own — and the first loop then unwinds
whenever it is finally cancelled or collected. If that teardown could clear the
-slot by user id alone it would deafen the live client, and nothing would say
-so: `invite` would report nobody listening, the caller would write the
-membership row itself, and the client would sit in a room it was never told it
-had joined.
+slot by id alone it would deafen the live client, and nothing would say so:
+`invite` would report nobody listening, the caller would write the membership
+row itself, and the client would sit in a room it was never told it had joined.
+
+A removal is the same slot and the worse failure. There is no membership row
+for the caller to write instead, so a removal that reaches nobody is a live
+transport left delivering a room it has been taken out of — which is why both
+arms are registered and given up together, and why the bus says so out loud if
+it is ever asked to remove for a client listening only for invitations.
"""
from __future__ import annotations
+import logging
+
+import pytest
+
from switch_core.transport.invites import InviteBus
class _Listener:
def __init__(self) -> None:
self.rooms: list[str] = []
+ self.removed: list[str] = []
async def on_invited(self, transport_room_id: str) -> None:
self.rooms.append(transport_room_id)
+ async def on_removed(self, transport_room_id: str) -> None:
+ self.removed.append(transport_room_id)
+
async def test_the_newest_registration_is_the_one_that_hears() -> None:
bus = InviteBus()
old, new = _Listener(), _Listener()
- bus.register("@agent:test", old.on_invited)
- bus.register("@agent:test", new.on_invited)
+ bus.register("client-1", old.on_invited, old.on_removed)
+ bus.register("client-1", new.on_invited, new.on_removed)
- assert await bus.invite("@agent:test", "!room:test") is True
+ assert await bus.invite("client-1", "!room:test") is True
assert (old.rooms, new.rooms) == ([], ["!room:test"])
async def test_a_superseded_listener_cannot_unregister_the_live_one() -> None:
bus = InviteBus()
old, new = _Listener(), _Listener()
- bus.register("@agent:test", old.on_invited)
- bus.register("@agent:test", new.on_invited)
+ bus.register("client-1", old.on_invited, old.on_removed)
+ bus.register("client-1", new.on_invited, new.on_removed)
- bus.unregister("@agent:test", old.on_invited)
+ bus.unregister("client-1", old.on_invited, old.on_removed)
- assert await bus.invite("@agent:test", "!room:test") is True, (
+ assert await bus.invite("client-1", "!room:test") is True, (
"the superseded transport's teardown deafened the live client"
)
assert new.rooms == ["!room:test"]
+async def test_a_superseded_listener_cannot_unregister_the_live_removal() -> None:
+ """The half with no fallback, so the one the same mistake costs most.
+
+ A restarting client registers before the old loop's `finally` runs. If that
+ teardown cleared the removal slot by id, every later kick would reach
+ nobody and the live transport would keep delivering rooms it was no longer
+ in — with no membership row for the caller to write instead, and nothing
+ logged.
+ """
+ bus = InviteBus()
+ old, new = _Listener(), _Listener()
+ bus.register("client-1", old.on_invited, old.on_removed)
+ bus.register("client-1", new.on_invited, new.on_removed)
+
+ bus.unregister("client-1", old.on_invited, old.on_removed)
+
+ await bus.remove("client-1", "!room:test")
+ assert (old.removed, new.removed) == ([], ["!room:test"])
+
+
async def test_the_live_listener_can_still_unregister_itself() -> None:
bus = InviteBus()
listener = _Listener()
- bus.register("@agent:test", listener.on_invited)
+ bus.register("client-1", listener.on_invited, listener.on_removed)
+
+ bus.unregister("client-1", listener.on_invited, listener.on_removed)
+
+ assert await bus.invite("client-1", "!room:test") is False
+ await bus.remove("client-1", "!room:test")
+ assert (listener.rooms, listener.removed) == ([], [])
+
+
+async def test_unregistering_a_client_nobody_registered_is_a_no_op() -> None:
+ bus = InviteBus()
+ listener = _Listener()
+ bus.unregister("client-1", listener.on_invited, listener.on_removed)
+ assert await bus.invite("client-1", "!room:test") is False
- bus.unregister("@agent:test", listener.on_invited)
- assert await bus.invite("@agent:test", "!room:test") is False
- assert listener.rooms == []
+async def test_removing_for_a_client_that_is_not_running_is_quiet() -> None:
+ """The expected answer, not a decision point.
+ A client that is not running holds no subscription to drop, and one that
+ starts later reads its rooms from the table the removal already updated.
+ """
+ bus = InviteBus()
+ await bus.remove("client-1", "!room:test")
-async def test_unregistering_a_user_nobody_registered_is_a_no_op() -> None:
+
+async def test_a_client_listening_only_for_invitations_is_reported(
+ caplog: pytest.LogCaptureFixture,
+) -> None:
+ """The state `register` makes unrepresentable, said out loud if it happens.
+
+ On its own the symptom is a room going quietly on being delivered to
+ someone no longer in it, which is not a thing anyone goes looking for.
+ """
bus = InviteBus()
- bus.unregister("@agent:test", _Listener().on_invited)
- assert await bus.invite("@agent:test", "!room:test") is False
+ listener = _Listener()
+ bus._handlers["client-1"] = listener.on_invited
+
+ with caplog.at_level(logging.ERROR):
+ await bus.remove("client-1", "!room:test")
+
+ assert "not for removals" in caplog.text
diff --git a/core/tests/switch_core/transport/test_postgres_transport.py b/core/tests/switch_core/transport/test_postgres_transport.py
index 60e0fda33..a69c36a5d 100644
--- a/core/tests/switch_core/transport/test_postgres_transport.py
+++ b/core/tests/switch_core/transport/test_postgres_transport.py
@@ -22,10 +22,12 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from switch_core.db.models import TENANT_ZERO_ID, Client, ClientRoom, Room, Tenant
from switch_core.db.session_scope import tenant_session
+from switch_core.db.stores.client_store import ClientStore
from switch_core.db.stores.media_store import MediaStore
from switch_core.db.stores.message_store import MessageStore
from switch_core.db.stores.room_store import RoomStore
from switch_core.observability.metrics import MetricsRegistry, install, uninstall
+from switch_core.provisioning.postgres import PostgresProvisioning
from switch_core.tenant_context import current_tenant_id, tenant_scope
from switch_core.transport import (
InboundCustomEvent,
@@ -38,7 +40,7 @@ from switch_core.transport import (
)
from switch_core.transport.ephemeral import EphemeralBus
from switch_core.transport.invites import InviteBus
-from switch_core.transport.postgres import PostgresTransport
+from switch_core.transport.postgres import _DELIVERY_PAGE, PostgresTransport
from tests.conftest import RLSHarness
@@ -61,6 +63,18 @@ async def _make_room(session: AsyncSession) -> tuple[str, str, str, str]:
return room.id, room.matrix_room_id, client.id, client.matrix_user_id
+async def _make_client(session: AsyncSession, label: str) -> tuple[str, str]:
+ """Insert one more Client. Returns (client id, client mxid)."""
+ client = Client(
+ matrix_user_id=f"@{label}-{uuid.uuid4().hex[:8]}:test",
+ display_name=label,
+ type="agent",
+ )
+ session.add(client)
+ await session.flush()
+ return client.id, client.matrix_user_id
+
+
async def _watched_room(transport: PostgresTransport) -> str:
"""Wait until the transport is watching a room, and name it.
@@ -123,12 +137,20 @@ class _Received:
def __init__(self) -> None:
self.events: list[object] = []
- def handlers(self) -> TransportHandlers:
+ def handlers(
+ self,
+ *,
+ on_invite: object | None = None,
+ on_message: object | None = None,
+ on_removed: object | None = None,
+ ) -> TransportHandlers:
return TransportHandlers(
- on_message=self._take,
+ on_message=on_message or self._take, # type: ignore[arg-type]
on_media=self._take,
on_member_event=self._take,
on_custom_event=self._take,
+ on_invite=on_invite, # type: ignore[arg-type]
+ on_removed=on_removed, # type: ignore[arg-type]
)
async def _take(self, _room, event) -> None:
@@ -143,6 +165,7 @@ def _transport(
tenant_id: str = TENANT_ZERO_ID,
listener: _FakeListener | None = None,
ephemeral: EphemeralBus | None = None,
+ invites: InviteBus | None = None,
) -> PostgresTransport:
"""A transport for `client_id`, acting in `tenant_id`.
@@ -164,7 +187,7 @@ def _transport(
message_store=MessageStore(),
media_store=MediaStore(),
listener=listener or _FakeListener(),
- invites=InviteBus(),
+ invites=invites or InviteBus(),
ephemeral=ephemeral or EphemeralBus(),
)
@@ -1481,3 +1504,450 @@ class TestWhatIsMeasured:
assert {
str(point.attributes["kind"]): point.value for point in payload.numbers
} == {"message": 1.0}
+
+
+class TestBeingRemovedFromARoom:
+ """A removal has to reach the client, not only the table.
+
+ The homeserver enforced a kick: a removed client's sync stopped returning
+ the room, so nothing downstream had to check membership a second time.
+ Here the subscription is the client's own and outlives the row it was
+ resolved from, so a removal is rung over the same bus an invitation is and
+ the transport drops the room.
+ """
+
+ @pytest.fixture(autouse=True)
+ def _cleanup(self) -> Iterator[None]:
+ self._tasks: list[asyncio.Task] = []
+ yield
+ for task in self._tasks:
+ task.cancel()
+
+ def _provisioning(
+ self,
+ session_factory: async_sessionmaker[AsyncSession],
+ invites: InviteBus,
+ ) -> PostgresProvisioning:
+ return PostgresProvisioning(
+ session_factory=session_factory,
+ room_store=RoomStore(),
+ client_store=ClientStore(),
+ message_store=MessageStore(),
+ invites=invites,
+ )
+
+ async def _two_in_a_room(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> tuple[str, str, tuple[str, str], tuple[str, str]]:
+ """A room with two members already recorded.
+
+ Written directly rather than joined so that no arrival event is in the
+ way of what the test is reading.
+ """
+ async with session_factory() as session:
+ room_id, room, leaving_id, leaving_user = await _make_room(session)
+ staying_id, staying_user = await _make_client(session, "staying")
+ await RoomStore().add_client(session, leaving_id, room_id)
+ await RoomStore().add_client(session, staying_id, room_id)
+ await session.commit()
+ return room_id, room, (leaving_id, leaving_user), (staying_id, staying_user)
+
+ async def test_it_stops_being_delivered_the_rooms_messages(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ room_id, room, leaving, staying = await self._two_in_a_room(session_factory)
+ listener = _FakeListener()
+ invites = InviteBus()
+ received = _Received()
+
+ removed = _transport(
+ session_factory,
+ client_id=leaving[0],
+ user_id=leaving[1],
+ listener=listener,
+ invites=invites,
+ )
+ removed.register_handlers(received.handlers())
+ member = _transport(
+ session_factory,
+ client_id=staying[0],
+ user_id=staying[1],
+ listener=listener,
+ invites=invites,
+ )
+ member.register_handlers(_Received().handlers())
+ self._tasks.append(asyncio.create_task(removed.receive_forever()))
+ self._tasks.append(asyncio.create_task(member.receive_forever()))
+ await _watched_room(removed)
+ await _watched_room(member)
+
+ # While a member, it hears the room.
+ await member.send_message(room, "while a member", sender_name="agent one")
+ await listener.announce(room_id)
+ assert [event.body for event in received.events] == ["while a member"]
+
+ await self._provisioning(session_factory, invites).kick_user(room, leaving[1])
+
+ async with session_factory() as session:
+ members = await RoomStore().get_client_ids(session, room_id)
+ assert leaving[0] not in members
+ assert staying[0] in members
+
+ await member.send_message(room, "after removal", sender_name="agent one")
+ await listener.announce(room_id)
+
+ assert [event.body for event in received.events] == ["while a member"]
+ assert room_id not in removed._watching
+
+ async def test_being_put_back_does_not_hand_over_what_was_said_while_out(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ """The cursor goes with the subscription.
+
+ Keeping it would mean a client added back resumed from where it left
+ off and was handed everything said while it was out, which is the same
+ leak one restart later.
+ """
+ room_id, room, leaving, staying = await self._two_in_a_room(session_factory)
+ listener = _FakeListener()
+ invites = InviteBus()
+ received = _Received()
+
+ removed = _transport(
+ session_factory,
+ client_id=leaving[0],
+ user_id=leaving[1],
+ listener=listener,
+ invites=invites,
+ )
+
+ async def _accept(room_ref, _event) -> None:
+ """What the client does with an invitation, so a put-back is one."""
+ await removed.join_room(room_ref.room_id)
+
+ removed.register_handlers(received.handlers(on_invite=_accept))
+ member = _transport(
+ session_factory,
+ client_id=staying[0],
+ user_id=staying[1],
+ listener=listener,
+ invites=invites,
+ )
+ member.register_handlers(_Received().handlers())
+ self._tasks.append(asyncio.create_task(removed.receive_forever()))
+ self._tasks.append(asyncio.create_task(member.receive_forever()))
+ await _watched_room(removed)
+ await _watched_room(member)
+
+ provisioning = self._provisioning(session_factory, invites)
+ await provisioning.kick_user(room, leaving[1])
+ await member.send_message(
+ room, "said while it was out", sender_name="agent one"
+ )
+ await listener.announce(room_id)
+
+ await provisioning.invite_to_room(room, leaving[1])
+ await _settled(removed)
+
+ bodies = [
+ event.body for event in received.events if isinstance(event, InboundMessage)
+ ]
+ assert bodies == []
+ # It is back in, and told so, which is how it was told the first time.
+ assert room_id in removed._watching
+ assert [type(event) for event in received.events] == [InboundMembership]
+
+ async def test_the_client_is_told_as_well_as_unsubscribed(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ """Dropping the subscription only stops what has not been read yet.
+
+ What has already been handed over belongs to the client — an agent's
+ event buffer holds it for the retention window and serves it to a poll
+ or a resumed stream — so the removal has to reach whoever is holding
+ it, not only the reader that would have fetched more.
+ """
+ _room_id, room, leaving, _staying = await self._two_in_a_room(session_factory)
+ invites = InviteBus()
+ received = _Received()
+ told: list[object] = []
+
+ async def _on_removed(room_ref, event) -> None:
+ told.append((room_ref.room_id, event.membership))
+
+ removed = _transport(
+ session_factory,
+ client_id=leaving[0],
+ user_id=leaving[1],
+ listener=_FakeListener(),
+ invites=invites,
+ )
+ removed.register_handlers(received.handlers(on_removed=_on_removed))
+ self._tasks.append(asyncio.create_task(removed.receive_forever()))
+ await _watched_room(removed)
+
+ await self._provisioning(session_factory, invites).kick_user(room, leaving[1])
+
+ assert told == [(room, "leave")]
+
+ async def test_a_removal_mid_page_stops_the_rest_of_the_page(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ """Every delivery is a suspension point, so a page is not atomic.
+
+ A removal landing on one of them has to stop the rows it has not
+ reached — and must not be undone by them. The cursor is written per
+ row, so a row delivered after the unwatch would put the room back in
+ `_cursors` with nothing in `_watching`, and `_watch` would then decline
+ to re-subscribe for good: the client would be a member of the room and
+ permanently silent in it.
+ """
+ room_id, room, leaving, staying = await self._two_in_a_room(session_factory)
+ listener = _FakeListener()
+ invites = InviteBus()
+ provisioning = self._provisioning(session_factory, invites)
+ delivered: list[str] = []
+
+ member = _transport(
+ session_factory,
+ client_id=staying[0],
+ user_id=staying[1],
+ listener=listener,
+ invites=invites,
+ )
+ member.register_handlers(_Received().handlers())
+
+ async def _kick_on_the_first(_room_ref, event) -> None:
+ delivered.append(event.body)
+ if len(delivered) == 1:
+ await provisioning.kick_user(room, leaving[1])
+
+ removed = _transport(
+ session_factory,
+ client_id=leaving[0],
+ user_id=leaving[1],
+ listener=listener,
+ invites=invites,
+ )
+ removed.register_handlers(_Received().handlers(on_message=_kick_on_the_first))
+ self._tasks.append(asyncio.create_task(removed.receive_forever()))
+ self._tasks.append(asyncio.create_task(member.receive_forever()))
+ await _watched_room(removed)
+ await _watched_room(member)
+
+ for i in range(4):
+ await member.send_message(room, f"line {i}", sender_name="agent one")
+ await listener.announce(room_id)
+
+ # The removal landed on the first delivery; the rest of the page is
+ # not the removed client's to read.
+ assert delivered == ["line 0"]
+ assert room_id not in removed._watching
+ # And nothing was left behind to make the room unsubscribable.
+ assert room_id not in removed._cursors
+
+ await provisioning.invite_to_room(room, leaving[1])
+ await removed.join_room(room)
+ assert room_id in removed._watching
+
+ async def test_a_removal_during_startup_is_not_overtaken_by_the_subscription(
+ self,
+ session_factory: async_sessionmaker[AsyncSession],
+ monkeypatch: pytest.MonkeyPatch,
+ ) -> None:
+ """Reading the room list and subscribing to it is several round trips.
+
+ A removal rung during them used to find nothing to unwatch and then be
+ stepped over by the subscription that followed, leaving the client
+ reading a room it is not in until the process restarted — and unlike an
+ invitation, there is no membership row for the caller to write instead.
+ The listener goes up before the list is read, and the row is confirmed
+ before the subscription is taken.
+ """
+ room_id, room, leaving, _staying = await self._two_in_a_room(session_factory)
+ invites = InviteBus()
+ provisioning = self._provisioning(session_factory, invites)
+ original = PostgresTransport.joined_rooms
+ kicked = asyncio.Event()
+
+ async def _kicked_while_reading(self) -> list[str]:
+ rooms = await original(self)
+ await provisioning.kick_user(room, leaving[1])
+ kicked.set()
+ return rooms
+
+ monkeypatch.setattr(PostgresTransport, "joined_rooms", _kicked_while_reading)
+
+ removed = _transport(
+ session_factory,
+ client_id=leaving[0],
+ user_id=leaving[1],
+ listener=_FakeListener(),
+ invites=invites,
+ )
+ removed.register_handlers(_Received().handlers())
+ self._tasks.append(asyncio.create_task(removed.receive_forever()))
+
+ await asyncio.wait_for(kicked.wait(), timeout=5)
+ # Let the rest of the startup path run and decide what to subscribe
+ # to. Bailing out the moment it subscribes keeps a regression fast to
+ # see rather than making every run wait out the window.
+ for _ in range(100):
+ await asyncio.sleep(0.01)
+ if room_id in removed._watching:
+ break
+ assert room_id not in removed._watching
+ assert room_id not in removed._cursors
+
+ async def test_a_stray_cursor_does_not_make_a_room_unsubscribable(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ """The "already watching" guard is a repair, not just an idempotence check.
+
+ `_cursors` and `_watching` are written together and should never
+ disagree — the removal races that used to part them are closed above.
+ But the cost of the guard being on the wrong one is not symmetric: a
+ cursor left behind with no subscription would make every later `_watch`
+ return early, from a re-invite, a `join_room` on a recorded membership,
+ or a reconnect alike, and the client would be a member of the room and
+ permanently silent in it. Keyed on `_watching` — the fact the rest of
+ the class tests — the same divergence repairs itself on the next watch.
+
+ The divergence is forced here rather than provoked, because there is no
+ longer a way to provoke it. That is the point: this pins which of the
+ two maps the guard reads, so a later change cannot quietly move it back
+ to the one that cannot recover.
+ """
+ room_id, room, leaving, _staying = await self._two_in_a_room(session_factory)
+ removed = _transport(
+ session_factory,
+ client_id=leaving[0],
+ user_id=leaving[1],
+ listener=_FakeListener(),
+ invites=InviteBus(),
+ )
+ removed.register_handlers(_Received().handlers())
+ self._tasks.append(asyncio.create_task(removed.receive_forever()))
+ await _watched_room(removed)
+
+ # A subscription dropped without its cursor: the shape the two maps
+ # take when anything writes one and not the other.
+ del removed._watching[room_id]
+
+ await removed._watch(room)
+
+ assert room_id in removed._watching
+
+ async def test_a_removal_on_a_page_boundary_stops_the_next_page(
+ self,
+ session_factory: async_sessionmaker[AsyncSession],
+ caplog: pytest.LogCaptureFixture,
+ ) -> None:
+ """The between-pages guard, which a removal inside a page never reaches.
+
+ A room that moved a long way while a handler was busy is read in
+ bounded steps. A removal landing anywhere but the last row of a page is
+ caught by the per-row check and the loop never comes back — so to reach
+ the top of the loop at all, the kick has to land on the page boundary
+ exactly, with a full page read and the loop about to ask for another.
+
+ Reading the cursor with `.get` is what makes that arrival safe:
+ `_unwatch` pops it, so indexing would raise KeyError, and the drain
+ would be recorded as a delivery failure rather than a removal being
+ honoured — the room stops either way, but one of them pages someone.
+ """
+ room_id, room, leaving, staying = await self._two_in_a_room(session_factory)
+ listener = _FakeListener()
+ invites = InviteBus()
+ provisioning = self._provisioning(session_factory, invites)
+ delivered: list[str] = []
+
+ member = _transport(
+ session_factory,
+ client_id=staying[0],
+ user_id=staying[1],
+ listener=listener,
+ invites=invites,
+ )
+ member.register_handlers(_Received().handlers())
+
+ async def _kick_on_the_page_boundary(_room_ref, event) -> None:
+ delivered.append(event.body)
+ if len(delivered) == _DELIVERY_PAGE:
+ await provisioning.kick_user(room, leaving[1])
+
+ removed = _transport(
+ session_factory,
+ client_id=leaving[0],
+ user_id=leaving[1],
+ listener=listener,
+ invites=invites,
+ )
+ removed.register_handlers(
+ _Received().handlers(on_message=_kick_on_the_page_boundary)
+ )
+ self._tasks.append(asyncio.create_task(removed.receive_forever()))
+ self._tasks.append(asyncio.create_task(member.receive_forever()))
+ await _watched_room(removed)
+ await _watched_room(member)
+
+ # More than one page, so the loop would come back for another.
+ for i in range(_DELIVERY_PAGE + 25):
+ await member.send_message(room, f"line {i}", sender_name="agent one")
+ await listener.announce(room_id)
+
+ # The whole of page one, and none of page two.
+ assert delivered == [f"line {i}" for i in range(_DELIVERY_PAGE)]
+ assert room_id not in removed._watching
+ assert room_id not in removed._cursors
+ assert "Delivery failed" not in caplog.text, (
+ "the drain came back for another page and fell over the popped "
+ "cursor instead of noticing the removal"
+ )
+
+ async def test_a_restart_does_not_deafen_the_new_loop_to_removals(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ """A client row can have two receive loops briefly overlapping.
+
+ The older one unregisters from its `finally`, which runs whenever that
+ loop ends — including after a newer instance took the slot. Giving the
+ slot up by id alone would take the live loop's removal listener with
+ it, and a kick would then reach nobody: no membership row for the
+ caller to write instead, and a transport still delivering a room it is
+ not in.
+ """
+ room_id, room, leaving, _staying = await self._two_in_a_room(session_factory)
+ invites = InviteBus()
+
+ def _make() -> PostgresTransport:
+ t = _transport(
+ session_factory,
+ client_id=leaving[0],
+ user_id=leaving[1],
+ listener=_FakeListener(),
+ invites=invites,
+ )
+ t.register_handlers(_Received().handlers())
+ return t
+
+ old, new = _make(), _make()
+ self._tasks.append(asyncio.create_task(old.receive_forever()))
+ await _watched_room(old)
+ self._tasks.append(asyncio.create_task(new.receive_forever()))
+ await _watched_room(new)
+
+ # The older loop finishes unwinding after the newer one took the slot.
+ await old.close()
+ for _ in range(200):
+ await asyncio.sleep(0.01)
+ if not old._receiving:
+ break
+ assert not old._receiving, "the superseded loop never finished unwinding"
+
+ await self._provisioning(session_factory, invites).kick_user(room, leaving[1])
+
+ assert room_id not in new._watching, (
+ "the superseded transport's teardown deafened the live one: the kick "
+ "reached nobody and it is still delivering a room it is not in"
+ )