Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

perf(clients): stop every Matrix client writing its sync cursor at the same instant (#357)

Louis Amaudruz5b329ca15d60
diff --git a/core/switch_core/clients/client_base.py b/core/switch_core/clients/client_base.py
index fbb5a7032..f17569b0c 100644
--- a/core/switch_core/clients/client_base.py
+++ b/core/switch_core/clients/client_base.py
@@ -3,6 +3,7 @@ from __future__ import annotations
import asyncio
import io
import logging
+import random
import time
from typing import Literal
@@ -61,12 +62,38 @@ from switch_core.events import (
logger = logging.getLogger(__name__)
-SYNC_STATE_INTERVAL = 30
+SYNC_STATE_INTERVAL = 30.0
+# Every client in this process starts when the process does, and its sync
+# long-poll is SYNC_STATE_INTERVAL as well, so a fixed interval has all of them
+# crossing the threshold in the same instant — hundreds of one-row transactions
+# competing for the connection pool at once. A per-client interval keeps them
+# spread out and stops them re-converging after a restart.
+SYNC_STATE_JITTER = 15.0
+# A sync that carried nothing durable still advances the cursor, and persisting
+# that is wasted work: resuming from the older cursor replays the same nothing.
+# The token should still not be left arbitrarily far behind the server, so an
+# unwritten one is flushed once it is this stale.
+SYNC_STATE_MAX_STALENESS = 900.0
SYNC_MAX_RETRIES = 5
SYNC_BACKOFF_BASE = 1.0
SYNC_BACKOFF_CAP = 60.0
+def _carries_durable_events(response: SyncResponse) -> bool:
+ """Whether a sync response holds anything a restart would need to replay.
+
+ Timeline events, room state and invites are dispatched to handlers, so the
+ cursor must move past them durably. Presence, typing, receipts and account
+ data are not — they only advance the token.
+ """
+ if response.rooms.invite or response.to_device_events:
+ return True
+ return any(
+ room.timeline.events or room.state
+ for room in (*response.rooms.join.values(), *response.rooms.leave.values())
+ )
+
+
class ClientConfig(BaseModel):
pass
@@ -113,7 +140,12 @@ class ClientBase[ConfigT: ClientConfig]:
self._self_join_dispatched: set[str] = set()
self._ready = asyncio.Event()
self._startup_ts: int = 0
- self._last_sync_persist: float = 0.0
+ self._sync_persist_interval = SYNC_STATE_INTERVAL + random.uniform(
+ 0.0, SYNC_STATE_JITTER
+ )
+ self._last_sync_persist: float = time.monotonic() - random.uniform(
+ 0.0, SYNC_STATE_INTERVAL
+ )
self._sync_state_dirty: bool = False
self._running: bool = False
@@ -471,9 +503,15 @@ class ClientBase[ConfigT: ClientConfig]:
logger.exception("Error in handler for %s in %s", event.type, room.room_id)
async def _handle_sync(self, response: SyncResponse) -> None:
- if response.next_batch and response.next_batch != self.next_batch_token:
- self.next_batch_token = response.next_batch
+ if not response.next_batch or response.next_batch == self.next_batch_token:
+ return
+ self.next_batch_token = response.next_batch
+ if _carries_durable_events(response):
await self._persist_state()
+ return
+ self._sync_state_dirty = True
+ if (time.monotonic() - self._last_sync_persist) >= SYNC_STATE_MAX_STALENESS:
+ await self._persist_state(force=True)
async def _handle_sync_error(self, response: SyncError) -> None:
logger.error("Sync error for %s: %s", self.matrix_user_id, response.message)
@@ -771,7 +809,7 @@ class ClientBase[ConfigT: ClientConfig]:
async def _persist_state(self, force: bool = False) -> None:
now = time.monotonic()
- if not force and (now - self._last_sync_persist) < SYNC_STATE_INTERVAL:
+ if not force and (now - self._last_sync_persist) < self._sync_persist_interval:
self._sync_state_dirty = True
return
diff --git a/core/tests/switch_core/clients/test_sync_cursor_persistence.py b/core/tests/switch_core/clients/test_sync_cursor_persistence.py
new file mode 100644
index 000000000..4691e5ec5
--- /dev/null
+++ b/core/tests/switch_core/clients/test_sync_cursor_persistence.py
@@ -0,0 +1,258 @@
+"""Cursor writes must not stampede the connection pool.
+
+Every Matrix client in the process persists its `next_batch` cursor, and both
+the sync long-poll and the write throttle were 30s. All clients start when the
+process does, so hundreds of them crossed the threshold in the same instant and
+each opened its own transaction for a one-row update — enough to exhaust the
+pool on an otherwise idle instance.
+
+Two properties fix that and are asserted here: an idle sync does not earn a
+write at all, and the clients that do write are spread out rather than aligned.
+"""
+
+from __future__ import annotations
+
+import time
+
+import pytest
+from nio import (
+ DeviceList,
+ DeviceOneTimeKeyCount,
+ InviteInfo,
+ RoomInfo,
+ Rooms,
+ SyncResponse,
+ Timeline,
+)
+
+from switch_core.clients.client_base import (
+ SYNC_STATE_INTERVAL,
+ SYNC_STATE_JITTER,
+ SYNC_STATE_MAX_STALENESS,
+ ClientBase,
+)
+
+ROOM = "!room:switch.local"
+
+
+class _Session:
+ async def __aenter__(self) -> _Session:
+ return self
+
+ async def __aexit__(self, *_: object) -> None:
+ return None
+
+ async def commit(self) -> None:
+ return None
+
+
+class _Store:
+ """Records what the real `_persist_state` would have written."""
+
+ def __init__(self) -> None:
+ self.writes: list[str | None] = []
+
+ async def update_state(
+ self,
+ session: object,
+ client_id: str,
+ *,
+ access_token: str | None,
+ device_id: str | None,
+ next_batch_token: str | None,
+ ) -> None:
+ self.writes.append(next_batch_token)
+
+
+class _Client(ClientBase):
+ """A real client with the database swapped out.
+
+ `_persist_state` itself is deliberately not overridden — the throttle it
+ applies is half of what these tests are about.
+ """
+
+ def __init__(self) -> None:
+ super().__init__(
+ client_id="c1",
+ matrix_user_id="@switch-agent-1:switch.local",
+ display_name="agent-1",
+ password="",
+ server_url="http://matrix.invalid",
+ session_factory=_Session, # type: ignore[arg-type]
+ client_store=_Store(), # type: ignore[arg-type]
+ config=ClientBase.config_class(),
+ next_batch_token="s0",
+ )
+ # Pin the jittered interval so the throttle assertions are deterministic.
+ self._sync_persist_interval = SYNC_STATE_INTERVAL
+
+ @property
+ def writes(self) -> list[str | None]:
+ store: _Store = self.client_store # type: ignore[assignment]
+ return store.writes
+
+
+def _sync(
+ next_batch: str,
+ *,
+ timeline: list[object] | None = None,
+ state: list[object] | None = None,
+ ephemeral: list[object] | None = None,
+ invite: bool = False,
+ to_device: list[object] | None = None,
+) -> SyncResponse:
+ join = {
+ ROOM: RoomInfo(
+ timeline=Timeline(
+ events=list(timeline or []), limited=False, prev_batch=None
+ ),
+ state=list(state or []),
+ ephemeral=list(ephemeral or []),
+ account_data=[],
+ )
+ }
+ return SyncResponse(
+ next_batch=next_batch,
+ rooms=Rooms(
+ invite={ROOM: InviteInfo(invite_state=[])} if invite else {},
+ join=join,
+ leave={},
+ ),
+ device_key_count=DeviceOneTimeKeyCount(curve25519=None, signed_curve25519=None),
+ device_list=DeviceList(changed=[], left=[]),
+ to_device_events=list(to_device or []),
+ presence_events=[],
+ )
+
+
+async def test_idle_sync_advances_the_cursor_without_writing() -> None:
+ """The overnight case: nothing happened, so nothing is worth persisting.
+
+ Resuming from the older cursor replays the same nothing, so the write buys
+ us no safety — and it was the whole burst.
+ """
+ client = _Client()
+ client._last_sync_persist = time.monotonic() - 10 * SYNC_STATE_INTERVAL
+
+ for i in range(1, 6):
+ await client._handle_sync(_sync(f"s{i}"))
+
+ assert client.writes == []
+ # The cursor still moved, so the next sync does not refetch.
+ assert client.next_batch_token == "s5"
+ assert client._sync_state_dirty is True
+
+
+async def test_ephemeral_only_sync_does_not_earn_a_write() -> None:
+ """Typing notices and read receipts advance the token and mean nothing."""
+ client = _Client()
+ client._last_sync_persist = time.monotonic() - 10 * SYNC_STATE_INTERVAL
+
+ await client._handle_sync(_sync("s1", ephemeral=[object()]))
+
+ assert client.writes == []
+ assert client.next_batch_token == "s1"
+
+
+ "kwargs",
+ [
+ {"timeline": [object()]},
+ {"state": [object()]},
+ {"invite": True},
+ {"to_device": [object()]},
+ ],
+ ids=["timeline", "state", "invite", "to_device"],
+)
+async def test_durable_events_are_persisted(kwargs: dict[str, object]) -> None:
+ """Anything a restart would replay to a handler must move the cursor."""
+ client = _Client()
+ client._last_sync_persist = time.monotonic() - 10 * SYNC_STATE_INTERVAL
+
+ await client._handle_sync(_sync("s1", **kwargs)) # type: ignore[arg-type]
+
+ assert client.writes == ["s1"]
+
+
+async def test_a_stale_cursor_is_flushed_even_while_idle() -> None:
+ """Skipping idle writes must not let the token fall arbitrarily behind."""
+ client = _Client()
+ client._last_sync_persist = time.monotonic() - (SYNC_STATE_MAX_STALENESS - 1)
+
+ await client._handle_sync(_sync("s1"))
+ assert client.writes == []
+
+ client._last_sync_persist = time.monotonic() - (SYNC_STATE_MAX_STALENESS + 1)
+ await client._handle_sync(_sync("s2"))
+
+ assert client.writes == ["s2"]
+ assert client._sync_state_dirty is False
+
+
+async def test_busy_clients_are_still_throttled() -> None:
+ """A room under load must not write a row per message."""
+ client = _Client()
+ client._last_sync_persist = time.monotonic() - 10 * SYNC_STATE_INTERVAL
+
+ await client._handle_sync(_sync("s1", timeline=[object()]))
+ await client._handle_sync(_sync("s2", timeline=[object()]))
+ await client._handle_sync(_sync("s3", timeline=[object()]))
+
+ assert client.writes == ["s1"]
+ assert client.next_batch_token == "s3"
+
+
+async def test_an_unchanged_cursor_is_ignored() -> None:
+ client = _Client()
+ client._last_sync_persist = time.monotonic() - 10 * SYNC_STATE_INTERVAL
+
+ await client._handle_sync(_sync("s0", timeline=[object()]))
+
+ assert client.writes == []
+
+
+def _real_clients(count: int) -> list[ClientBase]:
+ """Construct through `ClientBase.__init__` — the jitter is set there."""
+ return [
+ ClientBase(
+ client_id=f"c{i}",
+ matrix_user_id=f"@switch-agent-{i}:switch.local",
+ display_name=f"agent-{i}",
+ password="",
+ server_url="http://matrix.invalid",
+ session_factory=None, # type: ignore[arg-type]
+ client_store=None, # type: ignore[arg-type]
+ config=ClientBase.config_class(),
+ )
+ for i in range(count)
+ ]
+
+
+def test_clients_do_not_share_a_write_deadline() -> None:
+ """The stampede is the alignment, so the intervals must differ per client."""
+ clients = _real_clients(200)
+
+ intervals = {c._sync_persist_interval for c in clients}
+ assert len(intervals) > 190, "intervals are effectively identical"
+ assert all(
+ SYNC_STATE_INTERVAL <= i <= SYNC_STATE_INTERVAL + SYNC_STATE_JITTER
+ for i in intervals
+ )
+
+
+def test_the_first_write_after_boot_is_spread_across_the_window() -> None:
+ """Clients all start together, so seeding the clock identically would put
+ their first write in the same instant too."""
+ now = time.monotonic()
+ elapsed = sorted(now - c._last_sync_persist for c in _real_clients(200))
+
+ assert elapsed[0] >= 0.0
+ assert elapsed[-1] <= SYNC_STATE_INTERVAL
+ # Spread over the window rather than bunched at one end of it.
+ assert elapsed[-1] - elapsed[0] > SYNC_STATE_INTERVAL / 2