Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Read room memberships in batches at startup, so pods boot in seconds again (CHOO-3667)
Abel Dantas8edb09ec8212
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 6f7062c15..27f7ced1d 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -142,6 +142,12 @@ version of their own to them without also giving them a release of their own.compares it only when someone signs in.#### Changed+- **Pods start in seconds again with many rooms.** Every process checks at+ startup that each room has its clients, under a lock that lets one+ process at a time through, and read each room on its own to do it. With+ tens of thousands of rooms that took about 100 seconds, so processes+ starting together were restarted while they waited. Rooms are now read a+ thousand at a time.- **Idle agents behind a controller cost the database far less.** Acontroller stream's pass, including the one after every idle interval,read each of its agents' events in a session of its own; it now reads alldiff --git a/core/switch_core/db/stores/room_store.py b/core/switch_core/db/stores/room_store.pyindex f0f9c22de..580dbc200 100644--- a/core/switch_core/db/stores/room_store.py+++ b/core/switch_core/db/stores/room_store.py@@ -711,6 +711,24 @@ class RoomStore:for client_id, transport_user_id in result.all()}+ async def member_agent_clients_by_room(+ self, session: AsyncSession, room_ids: Collection[str]+ ) -> dict[str, dict[str, str]]:+ """`get_member_agent_clients` for many rooms in one query, keyed by+ room id. A room with no agent members has no key."""+ if not room_ids:+ return {}+ result = await session.execute(+ select(room_agents.c.room_id, Client.id, Client.transport_user_id)+ .join(Agent, Agent.id == room_agents.c.agent_id)+ .join(Client, Client.id == Agent.client_id)+ .where(room_agents.c.room_id.in_(room_ids))+ )+ by_room: dict[str, dict[str, str]] = {}+ for room_id, client_id, transport_user_id in result.all():+ by_room.setdefault(room_id, {})[client_id] = transport_user_id+ return by_room+async def get_by_bridge(self, session: AsyncSession, bridge_id: str) -> list[Room]:result = await session.execute(select(Room).where(Room.bridge_id == bridge_id))return list(result.scalars().all())diff --git a/core/switch_core/room_service.py b/core/switch_core/room_service.pyindex 7c70a4dc5..30f296b1c 100644--- a/core/switch_core/room_service.py+++ b/core/switch_core/room_service.py@@ -16,7 +16,7 @@ room Switch has no record of it being in.from __future__ import annotationsimport logging-from collections.abc import Awaitable, Callable+from collections.abc import Awaitable, Callable, Collectionfrom typing import TYPE_CHECKING, Any, Literalfrom pydantic import BaseModel@@ -64,6 +64,10 @@ logger = logging.getLogger(__name__)SYSTEM_CLIENT_TYPES = ("observe", "admin")+# Rooms whose memberships `reconcile_room_clients` reads in one query, kept+# well under Postgres' limit on bound parameters.+RECONCILE_BATCH_SIZE = 1000+class LinkedRoomSpec(BaseModel):target_room_id: str@@ -1660,30 +1664,56 @@ class RoomService:tenant, because of one room belonging to one of them. The failures arecounted and reported at `error`, so a contained one is still anoperator's problem rather than a silent one.++ The memberships are read for `RECONCILE_BATCH_SIZE` rooms at a time,+ and only a room missing a client is written to. This runs under the+ boot lock, so every process that starts waits for it: read room by+ room, a deployment with tens of thousands of rooms took longer than+ the liveness probe allows, and processes starting together were+ restarted while they waited."""# Which tenants exist is the one read that spans them (the exemption,# `db/tenant_lookup.py`); each tenant's rooms are then an ordinary# scoped read, and each room's own tenant is bound again around- # reconciling that one room.- rooms: list[Room] = []+ # repairing that one room.+ missing_by_room: list[tuple[Room, dict[str, str]]] = []+ room_count = 0for tenant_id in await all_tenant_ids(self._session_factory):+ system_clients = self._resolve_system_clients(tenant_id)async with tenant_session(self._session_factory, tenant_id) as session:# Filtered on the row's own tenant, not left to the policy: on an# owner connection no policy narrows this read, and the fan-out# would act on every tenant's rows once per tenant. See# `db/tenant_lookup.py`, "a fan-out ... filters what it reads back".- rooms.extend(+ rooms = [roomfor room in await self._room_store.get_all(session, include_archived=True)if room.tenant_id == tenant_id- )+ ]+ room_count += len(rooms)+ for start in range(0, len(rooms), RECONCILE_BATCH_SIZE):+ batch = rooms[start : start + RECONCILE_BATCH_SIZE]+ room_ids = [room.id for room in batch]+ existing = await self._room_store.client_ids_by_room(+ session, room_ids+ )+ agents = await self._room_store.member_agent_clients_by_room(+ session, room_ids+ )+ for room in batch:+ missing = _missing_clients(+ existing.get(room.id, []),+ {**agents.get(room.id, {}), **system_clients},+ )+ if missing:+ missing_by_room.append((room, missing))failures: list[str] = []- for room in rooms:+ for room, missing in missing_by_room:try:- await self._reconcile_one_room(room)+ await self._add_missing_clients(room, missing)except Exception:logger.exception("Failed to reconcile clients into room %s", room.id)failures.append(room.id)@@ -1693,7 +1723,7 @@ class RoomService:"rooms are missing system or agent clients until this is fixed ""and the service restarted.",len(failures),- len(rooms),+ room_count,", ".join(failures),)@@ -1708,23 +1738,29 @@ class RoomService:async def _reconcile_one_room(self, room: Room) -> None:with tenant_scope(room.tenant_id):async with self._session_factory() as session:- existing = set(await self._room_store.get_client_ids(session, room.id))- expected = await self._room_store.get_member_agent_clients(+ existing = await self._room_store.get_client_ids(session, room.id)+ agents = await self._room_store.get_member_agent_clients(session, room.id)- # Resolved per room rather than once for the fan-out: the system- # clients belong to a tenant, and which ones are this room's- # depends on the room.- expected.update(self._resolve_system_clients(room.tenant_id))- missing = {cid: uid for cid, uid in expected.items() if cid not in existing}- if not missing:- return+ # Resolved per room rather than once for the fan-out: the system+ # clients belong to a tenant, and which ones are this room's depends+ # on the room.+ missing = _missing_clients(+ existing, {**agents, **self._resolve_system_clients(room.tenant_id)}+ )+ if missing:+ await self._add_missing_clients(room, missing)++ async def _add_missing_clients(self, room: Room, missing: dict[str, str]) -> None:+ """Invite `missing` (`{client_id: transport_user_id}`) into the room and+ record their memberships, as the room's own tenant."""+ with tenant_scope(room.tenant_id):await self._invite_clients(room.transport_room_id, missing)async with self._session_factory() as session:for client_id in missing:await self._room_store.add_client(session, client_id, room.id)await session.commit()- logger.info("Reconciled %d client(s) into room %s", len(missing), room.id)+ logger.info("Reconciled %d client(s) into room %s", len(missing), room.id)async def resolve_bridge_users(self, bridge_id: str, names: list[str]@@ -1770,3 +1806,12 @@ class RoomService:async with self._session_factory() as session:await self._room_store.add_client(session, client_id, room_id)await session.commit()+++def _missing_clients(+ existing: Collection[str], expected: dict[str, str]+) -> dict[str, str]:+ """The entries of `expected` (`{client_id: transport_user_id}`) whose+ client is not among the room's `existing` client ids."""+ present = set(existing)+ return {cid: uid for cid, uid in expected.items() if cid not in present}diff --git a/core/tests/switch_core/db/stores/test_room_archive.py b/core/tests/switch_core/db/stores/test_room_archive.pyindex d3ea6e0e2..96c5142e7 100644--- a/core/tests/switch_core/db/stores/test_room_archive.py+++ b/core/tests/switch_core/db/stores/test_room_archive.py@@ -175,3 +175,28 @@ class TestArchivedFilteringInListing:roomless.id: set(),}assert await store.room_ids_by_agent(session, []) == {}++ async def test_member_agent_clients_by_room_reads_many_rooms(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ store = RoomStore()+ async with session_factory() as session:+ first = await _make_agent(session, "agent-1")+ second = await _make_agent(session, "agent-2")+ shared = await _make_room(store, session, "shared")+ solo = await _make_room(store, session, "solo")+ empty = await _make_room(store, session, "empty")+ await store.add_agents(session, shared.id, [first.id, second.id])+ await store.add_agents(session, solo.id, [first.id])+ await session.commit()++ assert await store.member_agent_clients_by_room(+ session, [shared.id, solo.id, empty.id]+ ) == {+ shared.id: {+ first.client_id: "@agent-1:test",+ second.client_id: "@agent-2:test",+ },+ solo.id: {first.client_id: "@agent-1:test"},+ }+ assert await store.member_agent_clients_by_room(session, []) == {}diff --git a/core/tests/switch_core/test_room_service_reconcile.py b/core/tests/switch_core/test_room_service_reconcile.pyindex e698efddf..61df9d940 100644--- a/core/tests/switch_core/test_room_service_reconcile.py+++ b/core/tests/switch_core/test_room_service_reconcile.py@@ -7,7 +7,7 @@ from typing import Anyimport pytestfrom switch_core import room_service as room_service_module-from switch_core.room_service import RoomService+from switch_core.room_service import RECONCILE_BATCH_SIZE, RoomServiceclass _FakeSessionCM:@@ -32,6 +32,7 @@ class _FakeRoomStore:self._client_ids_by_room = client_ids_by_roomself._agent_clients_by_room = agent_clients_by_roomself.added: list[tuple[str, str]] = []+ self.batches: list[int] = []async def get_all(self, session: Any, *, include_archived: bool = False@@ -46,6 +47,25 @@ class _FakeRoomStore:) -> dict[str, str]:return dict(self._agent_clients_by_room.get(room_id, {}))+ async def client_ids_by_room(+ self, session: Any, room_ids: list[str]+ ) -> dict[str, list[str]]:+ self.batches.append(len(room_ids))+ return {+ room_id: list(self._client_ids_by_room[room_id])+ for room_id in room_ids+ if room_id in self._client_ids_by_room+ }++ async def member_agent_clients_by_room(+ self, session: Any, room_ids: list[str]+ ) -> dict[str, dict[str, str]]:+ return {+ room_id: dict(self._agent_clients_by_room[room_id])+ for room_id in room_ids+ if room_id in self._agent_clients_by_room+ }+async def add_client(self, session: Any, client_id: str, room_id: str) -> None:self.added.append((client_id, room_id))@@ -268,3 +288,28 @@ class TestReconcileRoomClients:assert matrix.invited == []assert room_store.added == []++ async def test_reads_memberships_a_batch_of_rooms_at_a_time(+ self, monkeypatch: pytest.MonkeyPatch+ ) -> None:+ # Startup waits on this under the boot lock, so its reads must not+ # grow with one round trip per room.+ count = RECONCILE_BATCH_SIZE * 2 + 1+ rooms = [+ SimpleNamespace(+ id=f"room-{i}", tenant_id="tenant-1", transport_room_id=f"!{i}:s"+ )+ for i in range(count)+ ]+ svc, room_store, matrix = _build_service(+ monkeypatch,+ rooms=rooms,+ client_ids_by_room={room.id: ["admin-client"] for room in rooms[1:]},+ by_type={"admin": [_admin()]},+ )++ await svc.reconcile_room_clients()++ assert room_store.batches == [RECONCILE_BATCH_SIZE, RECONCILE_BATCH_SIZE, 1]+ assert matrix.invited == [("!0:s", "@switch-admin:switch.local")]+ assert room_store.added == [("admin-client", "room-0")]diff --git a/core/tests/switch_core/test_room_service_tenant_bindings.py b/core/tests/switch_core/test_room_service_tenant_bindings.pyindex c41ada544..4ce88b50e 100644--- a/core/tests/switch_core/test_room_service_tenant_bindings.py+++ b/core/tests/switch_core/test_room_service_tenant_bindings.py@@ -38,7 +38,7 @@ from __future__ import annotationsimport asyncioimport inspectimport uuid-from collections.abc import Callable+from collections.abc import Callable, Collectionfrom types import SimpleNamespacefrom typing import Anyfrom unittest.mock import MagicMock@@ -111,6 +111,12 @@ class _TenantRecordingRoomStore(RoomStore):await self._note("get_client_ids", session)return await super().get_client_ids(session, room_id)+ async def client_ids_by_room(+ self, session: AsyncSession, room_ids: Collection[str]+ ) -> dict[str, list[str]]:+ await self._note("client_ids_by_room", session)+ return await super().client_ids_by_room(session, room_ids)+async def add_client(self, session: AsyncSession, client_id: str, room_id: str) -> None:@@ -448,18 +454,8 @@ class TestReconcileRoomClients:assert sorted(store.tenants_for("add_client")) == sorted([TENANT_ZERO_ID, TENANT_B])- # `all_tenant_ids` names two tenants, so `get_all` runs once per- # tenant — and this fixture connects as the tables' owner, the same- # connection Phase 1 deploys as (see the module docstring), which- # exempts it from the policy `get_all` otherwise relies on to narrow- # its result. So each of those two calls sees every room, and- # `_reconcile_one_room` — bound to each room's own `tenant_id`- # regardless of which pass found it — runs twice per room rather than- # once. Harmless (the second pass finds nothing missing and returns),- # but it means every tenant appears at least once here rather than- # exactly once; the real one-pass-per-room count arrives with the- # restricted runtime role.- assert set(store.tenants_for("get_client_ids")) == {TENANT_ZERO_ID, TENANT_B}+ # Each tenant's memberships were read under that tenant, once.+ assert store.tenants_for("client_ids_by_room") == [TENANT_ZERO_ID, TENANT_B]async def test_the_room_enumeration_refuses_the_callers_tenant(self, session_factory: async_sessionmaker[AsyncSession]