Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
perf(clients): stop every Matrix client writing its sync cursor at the same instant (#357)
Louis Amaudruz5b329ca15d60
diff --git a/core/switch_core/clients/client_base.py b/core/switch_core/clients/client_base.pyindex fbb5a7032..f17569b0c 100644--- a/core/switch_core/clients/client_base.py+++ b/core/switch_core/clients/client_base.py@@ -3,6 +3,7 @@ from __future__ import annotationsimport asyncioimport ioimport logging+import randomimport timefrom typing import Literal@@ -61,12 +62,38 @@ from switch_core.events import (logger = logging.getLogger(__name__)-SYNC_STATE_INTERVAL = 30+SYNC_STATE_INTERVAL = 30.0+# Every client in this process starts when the process does, and its sync+# long-poll is SYNC_STATE_INTERVAL as well, so a fixed interval has all of them+# crossing the threshold in the same instant — hundreds of one-row transactions+# competing for the connection pool at once. A per-client interval keeps them+# spread out and stops them re-converging after a restart.+SYNC_STATE_JITTER = 15.0+# A sync that carried nothing durable still advances the cursor, and persisting+# that is wasted work: resuming from the older cursor replays the same nothing.+# The token should still not be left arbitrarily far behind the server, so an+# unwritten one is flushed once it is this stale.+SYNC_STATE_MAX_STALENESS = 900.0SYNC_MAX_RETRIES = 5SYNC_BACKOFF_BASE = 1.0SYNC_BACKOFF_CAP = 60.0+def _carries_durable_events(response: SyncResponse) -> bool:+ """Whether a sync response holds anything a restart would need to replay.++ Timeline events, room state and invites are dispatched to handlers, so the+ cursor must move past them durably. Presence, typing, receipts and account+ data are not — they only advance the token.+ """+ if response.rooms.invite or response.to_device_events:+ return True+ return any(+ room.timeline.events or room.state+ for room in (*response.rooms.join.values(), *response.rooms.leave.values())+ )++class ClientConfig(BaseModel):pass@@ -113,7 +140,12 @@ class ClientBase[ConfigT: ClientConfig]:self._self_join_dispatched: set[str] = set()self._ready = asyncio.Event()self._startup_ts: int = 0- self._last_sync_persist: float = 0.0+ self._sync_persist_interval = SYNC_STATE_INTERVAL + random.uniform(+ 0.0, SYNC_STATE_JITTER+ )+ self._last_sync_persist: float = time.monotonic() - random.uniform(+ 0.0, SYNC_STATE_INTERVAL+ )self._sync_state_dirty: bool = Falseself._running: bool = False@@ -471,9 +503,15 @@ class ClientBase[ConfigT: ClientConfig]:logger.exception("Error in handler for %s in %s", event.type, room.room_id)async def _handle_sync(self, response: SyncResponse) -> None:- if response.next_batch and response.next_batch != self.next_batch_token:- self.next_batch_token = response.next_batch+ if not response.next_batch or response.next_batch == self.next_batch_token:+ return+ self.next_batch_token = response.next_batch+ if _carries_durable_events(response):await self._persist_state()+ return+ self._sync_state_dirty = True+ if (time.monotonic() - self._last_sync_persist) >= SYNC_STATE_MAX_STALENESS:+ await self._persist_state(force=True)async def _handle_sync_error(self, response: SyncError) -> None:logger.error("Sync error for %s: %s", self.matrix_user_id, response.message)@@ -771,7 +809,7 @@ class ClientBase[ConfigT: ClientConfig]:async def _persist_state(self, force: bool = False) -> None:now = time.monotonic()- if not force and (now - self._last_sync_persist) < SYNC_STATE_INTERVAL:+ if not force and (now - self._last_sync_persist) < self._sync_persist_interval:self._sync_state_dirty = Truereturndiff --git a/core/tests/switch_core/clients/test_sync_cursor_persistence.py b/core/tests/switch_core/clients/test_sync_cursor_persistence.pynew file mode 100644index 000000000..4691e5ec5--- /dev/null+++ b/core/tests/switch_core/clients/test_sync_cursor_persistence.py@@ -0,0 +1,258 @@+"""Cursor writes must not stampede the connection pool.++Every Matrix client in the process persists its `next_batch` cursor, and both+the sync long-poll and the write throttle were 30s. All clients start when the+process does, so hundreds of them crossed the threshold in the same instant and+each opened its own transaction for a one-row update — enough to exhaust the+pool on an otherwise idle instance.++Two properties fix that and are asserted here: an idle sync does not earn a+write at all, and the clients that do write are spread out rather than aligned.+"""++from __future__ import annotations++import time++import pytest+from nio import (+ DeviceList,+ DeviceOneTimeKeyCount,+ InviteInfo,+ RoomInfo,+ Rooms,+ SyncResponse,+ Timeline,+)++from switch_core.clients.client_base import (+ SYNC_STATE_INTERVAL,+ SYNC_STATE_JITTER,+ SYNC_STATE_MAX_STALENESS,+ ClientBase,+)++ROOM = "!room:switch.local"+++class _Session:+ async def __aenter__(self) -> _Session:+ return self++ async def __aexit__(self, *_: object) -> None:+ return None++ async def commit(self) -> None:+ return None+++class _Store:+ """Records what the real `_persist_state` would have written."""++ def __init__(self) -> None:+ self.writes: list[str | None] = []++ async def update_state(+ self,+ session: object,+ client_id: str,+ *,+ access_token: str | None,+ device_id: str | None,+ next_batch_token: str | None,+ ) -> None:+ self.writes.append(next_batch_token)+++class _Client(ClientBase):+ """A real client with the database swapped out.++ `_persist_state` itself is deliberately not overridden — the throttle it+ applies is half of what these tests are about.+ """++ def __init__(self) -> None:+ super().__init__(+ client_id="c1",+ matrix_user_id="@switch-agent-1:switch.local",+ display_name="agent-1",+ password="",+ server_url="http://matrix.invalid",+ session_factory=_Session, # type: ignore[arg-type]+ client_store=_Store(), # type: ignore[arg-type]+ config=ClientBase.config_class(),+ next_batch_token="s0",+ )+ # Pin the jittered interval so the throttle assertions are deterministic.+ self._sync_persist_interval = SYNC_STATE_INTERVAL++ @property+ def writes(self) -> list[str | None]:+ store: _Store = self.client_store # type: ignore[assignment]+ return store.writes+++def _sync(+ next_batch: str,+ *,+ timeline: list[object] | None = None,+ state: list[object] | None = None,+ ephemeral: list[object] | None = None,+ invite: bool = False,+ to_device: list[object] | None = None,+) -> SyncResponse:+ join = {+ ROOM: RoomInfo(+ timeline=Timeline(+ events=list(timeline or []), limited=False, prev_batch=None+ ),+ state=list(state or []),+ ephemeral=list(ephemeral or []),+ account_data=[],+ )+ }+ return SyncResponse(+ next_batch=next_batch,+ rooms=Rooms(+ invite={ROOM: InviteInfo(invite_state=[])} if invite else {},+ join=join,+ leave={},+ ),+ device_key_count=DeviceOneTimeKeyCount(curve25519=None, signed_curve25519=None),+ device_list=DeviceList(changed=[], left=[]),+ to_device_events=list(to_device or []),+ presence_events=[],+ )+++async def test_idle_sync_advances_the_cursor_without_writing() -> None:+ """The overnight case: nothing happened, so nothing is worth persisting.++ Resuming from the older cursor replays the same nothing, so the write buys+ us no safety — and it was the whole burst.+ """+ client = _Client()+ client._last_sync_persist = time.monotonic() - 10 * SYNC_STATE_INTERVAL++ for i in range(1, 6):+ await client._handle_sync(_sync(f"s{i}"))++ assert client.writes == []+ # The cursor still moved, so the next sync does not refetch.+ assert client.next_batch_token == "s5"+ assert client._sync_state_dirty is True+++async def test_ephemeral_only_sync_does_not_earn_a_write() -> None:+ """Typing notices and read receipts advance the token and mean nothing."""+ client = _Client()+ client._last_sync_persist = time.monotonic() - 10 * SYNC_STATE_INTERVAL++ await client._handle_sync(_sync("s1", ephemeral=[object()]))++ assert client.writes == []+ assert client.next_batch_token == "s1"+++ "kwargs",+ [+ {"timeline": [object()]},+ {"state": [object()]},+ {"invite": True},+ {"to_device": [object()]},+ ],+ ids=["timeline", "state", "invite", "to_device"],+)+async def test_durable_events_are_persisted(kwargs: dict[str, object]) -> None:+ """Anything a restart would replay to a handler must move the cursor."""+ client = _Client()+ client._last_sync_persist = time.monotonic() - 10 * SYNC_STATE_INTERVAL++ await client._handle_sync(_sync("s1", **kwargs)) # type: ignore[arg-type]++ assert client.writes == ["s1"]+++async def test_a_stale_cursor_is_flushed_even_while_idle() -> None:+ """Skipping idle writes must not let the token fall arbitrarily behind."""+ client = _Client()+ client._last_sync_persist = time.monotonic() - (SYNC_STATE_MAX_STALENESS - 1)++ await client._handle_sync(_sync("s1"))+ assert client.writes == []++ client._last_sync_persist = time.monotonic() - (SYNC_STATE_MAX_STALENESS + 1)+ await client._handle_sync(_sync("s2"))++ assert client.writes == ["s2"]+ assert client._sync_state_dirty is False+++async def test_busy_clients_are_still_throttled() -> None:+ """A room under load must not write a row per message."""+ client = _Client()+ client._last_sync_persist = time.monotonic() - 10 * SYNC_STATE_INTERVAL++ await client._handle_sync(_sync("s1", timeline=[object()]))+ await client._handle_sync(_sync("s2", timeline=[object()]))+ await client._handle_sync(_sync("s3", timeline=[object()]))++ assert client.writes == ["s1"]+ assert client.next_batch_token == "s3"+++async def test_an_unchanged_cursor_is_ignored() -> None:+ client = _Client()+ client._last_sync_persist = time.monotonic() - 10 * SYNC_STATE_INTERVAL++ await client._handle_sync(_sync("s0", timeline=[object()]))++ assert client.writes == []+++def _real_clients(count: int) -> list[ClientBase]:+ """Construct through `ClientBase.__init__` — the jitter is set there."""+ return [+ ClientBase(+ client_id=f"c{i}",+ matrix_user_id=f"@switch-agent-{i}:switch.local",+ display_name=f"agent-{i}",+ password="",+ server_url="http://matrix.invalid",+ session_factory=None, # type: ignore[arg-type]+ client_store=None, # type: ignore[arg-type]+ config=ClientBase.config_class(),+ )+ for i in range(count)+ ]+++def test_clients_do_not_share_a_write_deadline() -> None:+ """The stampede is the alignment, so the intervals must differ per client."""+ clients = _real_clients(200)++ intervals = {c._sync_persist_interval for c in clients}+ assert len(intervals) > 190, "intervals are effectively identical"+ assert all(+ SYNC_STATE_INTERVAL <= i <= SYNC_STATE_INTERVAL + SYNC_STATE_JITTER+ for i in intervals+ )+++def test_the_first_write_after_boot_is_spread_across_the_window() -> None:+ """Clients all start together, so seeding the clock identically would put+ their first write in the same instant too."""+ now = time.monotonic()+ elapsed = sorted(now - c._last_sync_persist for c in _real_clients(200))++ assert elapsed[0] >= 0.0+ assert elapsed[-1] <= SYNC_STATE_INTERVAL+ # Spread over the window rather than bunched at one end of it.+ assert elapsed[-1] - elapsed[0] > SYNC_STATE_INTERVAL / 2