Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Read room memberships in batches at startup, so pods boot in seconds again (CHOO-3667)

Abel Dantas8edb09ec8212
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 6f7062c15..27f7ced1d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -142,6 +142,12 @@ version of their own to them without also giving them a release of their own.
compares it only when someone signs in.
#### Changed
+- **Pods start in seconds again with many rooms.** Every process checks at
+ startup that each room has its clients, under a lock that lets one
+ process at a time through, and read each room on its own to do it. With
+ tens of thousands of rooms that took about 100 seconds, so processes
+ starting together were restarted while they waited. Rooms are now read a
+ thousand at a time.
- **Idle agents behind a controller cost the database far less.** A
controller stream's pass, including the one after every idle interval,
read each of its agents' events in a session of its own; it now reads all
diff --git a/core/switch_core/db/stores/room_store.py b/core/switch_core/db/stores/room_store.py
index f0f9c22de..580dbc200 100644
--- a/core/switch_core/db/stores/room_store.py
+++ b/core/switch_core/db/stores/room_store.py
@@ -711,6 +711,24 @@ class RoomStore:
for client_id, transport_user_id in result.all()
}
+ async def member_agent_clients_by_room(
+ self, session: AsyncSession, room_ids: Collection[str]
+ ) -> dict[str, dict[str, str]]:
+ """`get_member_agent_clients` for many rooms in one query, keyed by
+ room id. A room with no agent members has no key."""
+ if not room_ids:
+ return {}
+ result = await session.execute(
+ select(room_agents.c.room_id, Client.id, Client.transport_user_id)
+ .join(Agent, Agent.id == room_agents.c.agent_id)
+ .join(Client, Client.id == Agent.client_id)
+ .where(room_agents.c.room_id.in_(room_ids))
+ )
+ by_room: dict[str, dict[str, str]] = {}
+ for room_id, client_id, transport_user_id in result.all():
+ by_room.setdefault(room_id, {})[client_id] = transport_user_id
+ return by_room
+
async def get_by_bridge(self, session: AsyncSession, bridge_id: str) -> list[Room]:
result = await session.execute(select(Room).where(Room.bridge_id == bridge_id))
return list(result.scalars().all())
diff --git a/core/switch_core/room_service.py b/core/switch_core/room_service.py
index 7c70a4dc5..30f296b1c 100644
--- a/core/switch_core/room_service.py
+++ b/core/switch_core/room_service.py
@@ -16,7 +16,7 @@ room Switch has no record of it being in.
from __future__ import annotations
import logging
-from collections.abc import Awaitable, Callable
+from collections.abc import Awaitable, Callable, Collection
from typing import TYPE_CHECKING, Any, Literal
from pydantic import BaseModel
@@ -64,6 +64,10 @@ logger = logging.getLogger(__name__)
SYSTEM_CLIENT_TYPES = ("observe", "admin")
+# Rooms whose memberships `reconcile_room_clients` reads in one query, kept
+# well under Postgres' limit on bound parameters.
+RECONCILE_BATCH_SIZE = 1000
+
class LinkedRoomSpec(BaseModel):
target_room_id: str
@@ -1660,30 +1664,56 @@ class RoomService:
tenant, because of one room belonging to one of them. The failures are
counted and reported at `error`, so a contained one is still an
operator's problem rather than a silent one.
+
+ The memberships are read for `RECONCILE_BATCH_SIZE` rooms at a time,
+ and only a room missing a client is written to. This runs under the
+ boot lock, so every process that starts waits for it: read room by
+ room, a deployment with tens of thousands of rooms took longer than
+ the liveness probe allows, and processes starting together were
+ restarted while they waited.
"""
# Which tenants exist is the one read that spans them (the exemption,
# `db/tenant_lookup.py`); each tenant's rooms are then an ordinary
# scoped read, and each room's own tenant is bound again around
- # reconciling that one room.
- rooms: list[Room] = []
+ # repairing that one room.
+ missing_by_room: list[tuple[Room, dict[str, str]]] = []
+ room_count = 0
for tenant_id in await all_tenant_ids(self._session_factory):
+ system_clients = self._resolve_system_clients(tenant_id)
async with tenant_session(self._session_factory, tenant_id) as session:
# Filtered on the row's own tenant, not left to the policy: on an
# owner connection no policy narrows this read, and the fan-out
# would act on every tenant's rows once per tenant. See
# `db/tenant_lookup.py`, "a fan-out ... filters what it reads back".
- rooms.extend(
+ rooms = [
room
for room in await self._room_store.get_all(
session, include_archived=True
)
if room.tenant_id == tenant_id
- )
+ ]
+ room_count += len(rooms)
+ for start in range(0, len(rooms), RECONCILE_BATCH_SIZE):
+ batch = rooms[start : start + RECONCILE_BATCH_SIZE]
+ room_ids = [room.id for room in batch]
+ existing = await self._room_store.client_ids_by_room(
+ session, room_ids
+ )
+ agents = await self._room_store.member_agent_clients_by_room(
+ session, room_ids
+ )
+ for room in batch:
+ missing = _missing_clients(
+ existing.get(room.id, []),
+ {**agents.get(room.id, {}), **system_clients},
+ )
+ if missing:
+ missing_by_room.append((room, missing))
failures: list[str] = []
- for room in rooms:
+ for room, missing in missing_by_room:
try:
- await self._reconcile_one_room(room)
+ await self._add_missing_clients(room, missing)
except Exception:
logger.exception("Failed to reconcile clients into room %s", room.id)
failures.append(room.id)
@@ -1693,7 +1723,7 @@ class RoomService:
"rooms are missing system or agent clients until this is fixed "
"and the service restarted.",
len(failures),
- len(rooms),
+ room_count,
", ".join(failures),
)
@@ -1708,23 +1738,29 @@ class RoomService:
async def _reconcile_one_room(self, room: Room) -> None:
with tenant_scope(room.tenant_id):
async with self._session_factory() as session:
- existing = set(await self._room_store.get_client_ids(session, room.id))
- expected = await self._room_store.get_member_agent_clients(
+ existing = await self._room_store.get_client_ids(session, room.id)
+ agents = await self._room_store.get_member_agent_clients(
session, room.id
)
- # Resolved per room rather than once for the fan-out: the system
- # clients belong to a tenant, and which ones are this room's
- # depends on the room.
- expected.update(self._resolve_system_clients(room.tenant_id))
- missing = {cid: uid for cid, uid in expected.items() if cid not in existing}
- if not missing:
- return
+ # Resolved per room rather than once for the fan-out: the system
+ # clients belong to a tenant, and which ones are this room's depends
+ # on the room.
+ missing = _missing_clients(
+ existing, {**agents, **self._resolve_system_clients(room.tenant_id)}
+ )
+ if missing:
+ await self._add_missing_clients(room, missing)
+
+ async def _add_missing_clients(self, room: Room, missing: dict[str, str]) -> None:
+ """Invite `missing` (`{client_id: transport_user_id}`) into the room and
+ record their memberships, as the room's own tenant."""
+ with tenant_scope(room.tenant_id):
await self._invite_clients(room.transport_room_id, missing)
async with self._session_factory() as session:
for client_id in missing:
await self._room_store.add_client(session, client_id, room.id)
await session.commit()
- logger.info("Reconciled %d client(s) into room %s", len(missing), room.id)
+ logger.info("Reconciled %d client(s) into room %s", len(missing), room.id)
async def resolve_bridge_users(
self, bridge_id: str, names: list[str]
@@ -1770,3 +1806,12 @@ class RoomService:
async with self._session_factory() as session:
await self._room_store.add_client(session, client_id, room_id)
await session.commit()
+
+
+def _missing_clients(
+ existing: Collection[str], expected: dict[str, str]
+) -> dict[str, str]:
+ """The entries of `expected` (`{client_id: transport_user_id}`) whose
+ client is not among the room's `existing` client ids."""
+ present = set(existing)
+ return {cid: uid for cid, uid in expected.items() if cid not in present}
diff --git a/core/tests/switch_core/db/stores/test_room_archive.py b/core/tests/switch_core/db/stores/test_room_archive.py
index d3ea6e0e2..96c5142e7 100644
--- a/core/tests/switch_core/db/stores/test_room_archive.py
+++ b/core/tests/switch_core/db/stores/test_room_archive.py
@@ -175,3 +175,28 @@ class TestArchivedFilteringInListing:
roomless.id: set(),
}
assert await store.room_ids_by_agent(session, []) == {}
+
+ async def test_member_agent_clients_by_room_reads_many_rooms(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ store = RoomStore()
+ async with session_factory() as session:
+ first = await _make_agent(session, "agent-1")
+ second = await _make_agent(session, "agent-2")
+ shared = await _make_room(store, session, "shared")
+ solo = await _make_room(store, session, "solo")
+ empty = await _make_room(store, session, "empty")
+ await store.add_agents(session, shared.id, [first.id, second.id])
+ await store.add_agents(session, solo.id, [first.id])
+ await session.commit()
+
+ assert await store.member_agent_clients_by_room(
+ session, [shared.id, solo.id, empty.id]
+ ) == {
+ shared.id: {
+ first.client_id: "@agent-1:test",
+ second.client_id: "@agent-2:test",
+ },
+ solo.id: {first.client_id: "@agent-1:test"},
+ }
+ assert await store.member_agent_clients_by_room(session, []) == {}
diff --git a/core/tests/switch_core/test_room_service_reconcile.py b/core/tests/switch_core/test_room_service_reconcile.py
index e698efddf..61df9d940 100644
--- a/core/tests/switch_core/test_room_service_reconcile.py
+++ b/core/tests/switch_core/test_room_service_reconcile.py
@@ -7,7 +7,7 @@ from typing import Any
import pytest
from switch_core import room_service as room_service_module
-from switch_core.room_service import RoomService
+from switch_core.room_service import RECONCILE_BATCH_SIZE, RoomService
class _FakeSessionCM:
@@ -32,6 +32,7 @@ class _FakeRoomStore:
self._client_ids_by_room = client_ids_by_room
self._agent_clients_by_room = agent_clients_by_room
self.added: list[tuple[str, str]] = []
+ self.batches: list[int] = []
async def get_all(
self, session: Any, *, include_archived: bool = False
@@ -46,6 +47,25 @@ class _FakeRoomStore:
) -> dict[str, str]:
return dict(self._agent_clients_by_room.get(room_id, {}))
+ async def client_ids_by_room(
+ self, session: Any, room_ids: list[str]
+ ) -> dict[str, list[str]]:
+ self.batches.append(len(room_ids))
+ return {
+ room_id: list(self._client_ids_by_room[room_id])
+ for room_id in room_ids
+ if room_id in self._client_ids_by_room
+ }
+
+ async def member_agent_clients_by_room(
+ self, session: Any, room_ids: list[str]
+ ) -> dict[str, dict[str, str]]:
+ return {
+ room_id: dict(self._agent_clients_by_room[room_id])
+ for room_id in room_ids
+ if room_id in self._agent_clients_by_room
+ }
+
async def add_client(self, session: Any, client_id: str, room_id: str) -> None:
self.added.append((client_id, room_id))
@@ -268,3 +288,28 @@ class TestReconcileRoomClients:
assert matrix.invited == []
assert room_store.added == []
+
+ async def test_reads_memberships_a_batch_of_rooms_at_a_time(
+ self, monkeypatch: pytest.MonkeyPatch
+ ) -> None:
+ # Startup waits on this under the boot lock, so its reads must not
+ # grow with one round trip per room.
+ count = RECONCILE_BATCH_SIZE * 2 + 1
+ rooms = [
+ SimpleNamespace(
+ id=f"room-{i}", tenant_id="tenant-1", transport_room_id=f"!{i}:s"
+ )
+ for i in range(count)
+ ]
+ svc, room_store, matrix = _build_service(
+ monkeypatch,
+ rooms=rooms,
+ client_ids_by_room={room.id: ["admin-client"] for room in rooms[1:]},
+ by_type={"admin": [_admin()]},
+ )
+
+ await svc.reconcile_room_clients()
+
+ assert room_store.batches == [RECONCILE_BATCH_SIZE, RECONCILE_BATCH_SIZE, 1]
+ assert matrix.invited == [("!0:s", "@switch-admin:switch.local")]
+ assert room_store.added == [("admin-client", "room-0")]
diff --git a/core/tests/switch_core/test_room_service_tenant_bindings.py b/core/tests/switch_core/test_room_service_tenant_bindings.py
index c41ada544..4ce88b50e 100644
--- a/core/tests/switch_core/test_room_service_tenant_bindings.py
+++ b/core/tests/switch_core/test_room_service_tenant_bindings.py
@@ -38,7 +38,7 @@ from __future__ import annotations
import asyncio
import inspect
import uuid
-from collections.abc import Callable
+from collections.abc import Callable, Collection
from types import SimpleNamespace
from typing import Any
from unittest.mock import MagicMock
@@ -111,6 +111,12 @@ class _TenantRecordingRoomStore(RoomStore):
await self._note("get_client_ids", session)
return await super().get_client_ids(session, room_id)
+ async def client_ids_by_room(
+ self, session: AsyncSession, room_ids: Collection[str]
+ ) -> dict[str, list[str]]:
+ await self._note("client_ids_by_room", session)
+ return await super().client_ids_by_room(session, room_ids)
+
async def add_client(
self, session: AsyncSession, client_id: str, room_id: str
) -> None:
@@ -448,18 +454,8 @@ class TestReconcileRoomClients:
assert sorted(store.tenants_for("add_client")) == sorted(
[TENANT_ZERO_ID, TENANT_B]
)
- # `all_tenant_ids` names two tenants, so `get_all` runs once per
- # tenant — and this fixture connects as the tables' owner, the same
- # connection Phase 1 deploys as (see the module docstring), which
- # exempts it from the policy `get_all` otherwise relies on to narrow
- # its result. So each of those two calls sees every room, and
- # `_reconcile_one_room` — bound to each room's own `tenant_id`
- # regardless of which pass found it — runs twice per room rather than
- # once. Harmless (the second pass finds nothing missing and returns),
- # but it means every tenant appears at least once here rather than
- # exactly once; the real one-pass-per-room count arrives with the
- # restricted runtime role.
- assert set(store.tenants_for("get_client_ids")) == {TENANT_ZERO_ID, TENANT_B}
+ # Each tenant's memberships were read under that tenant, once.
+ assert store.tenants_for("client_ids_by_room") == [TENANT_ZERO_ID, TENANT_B]
async def test_the_room_enumeration_refuses_the_callers_tenant(
self, session_factory: async_sessionmaker[AsyncSession]