Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Tell Consoles on every pod when a user's chats change, and warm a controller held by another pod (#885)
Abel Dantas685e924279de
diff --git a/core/switch_core/bridges/agent/protocol/agent_connections.py b/core/switch_core/bridges/agent/protocol/agent_connections.pyindex c42741c9b..c6b46fe26 100644--- a/core/switch_core/bridges/agent/protocol/agent_connections.py+++ b/core/switch_core/bridges/agent/protocol/agent_connections.py@@ -565,6 +565,11 @@ class ConnectionRows(Protocol):block."""...+ def relay_warm(self, agent_id: str, process_id: str, room_id: str) -> None:+ """A warm for a controller whose stream another process holds. Must+ not block."""+ ...+class AgentConnectionRegistry:"""The live set of agent connections: those this process holds,@@ -1247,6 +1252,20 @@ class AgentConnectionRegistry:delivered = Truereturn delivered+ def relay_warm(self, agent_id: str, room_id: str) -> bool:+ """Ask the controller running the agent to start the session a message+ to `room_id` would start. False when no process holds that controller's+ live socket or the agent is stopped. A controller another process+ holds is asked through that process (`ConnectionRows.relay_warm`),+ which hands the warm to its `ControllerPresence.relay_warm`."""+ if self.controllers.relay_warm(agent_id, room_id):+ return True+ holder = self.controllers.process_holding(agent_id)+ if holder is None or self._rows is None:+ return False+ self._rows.relay_warm(agent_id, holder, room_id)+ return True+def _watchers(self, agent_id: str) -> list[AgentConnection]:return [conndiff --git a/core/switch_core/bridges/agent/protocol/connection_rows.py b/core/switch_core/bridges/agent/protocol/connection_rows.pyindex a7d2af971..96201f529 100644--- a/core/switch_core/bridges/agent/protocol/connection_rows.py+++ b/core/switch_core/bridges/agent/protocol/connection_rows.py@@ -37,18 +37,20 @@ lease check (`WATCH_SECONDS`) drops the copies of a process whose leaselapsed. What a signal asks this process to read goes through the bus's`PendingWork`, so a read that fails is kept and tried again.-Two requests are handed to the holder rather than served where they land: a+Three requests are handed to the holder rather than served where they land: a`POST /connection/beat` from an old event-stream client (forwarded in batches,-`agent_connections.beats`) and a session command for a stream another process+`agent_connections.beats`), a session command for a stream another processholds, the agent's watcher or its controller's (stored as a relay row for thatprocess and signalled, `agent_connections.session_command`, since its content-may not travel in a `NOTIFY`).+may not travel in a `NOTIFY`), and a warm for a controller another process+holds (`agent_connections.warm`). A warm is nothing but ids, so it travels in+the signal itself; it is a hint, and one lost costs only the head start it+would have given the agent's session."""from __future__ import annotationsimport asyncio-import jsonimport loggingfrom collections.abc import AsyncIteratorfrom contextlib import asynccontextmanager@@ -76,7 +78,7 @@ from switch_core.process_lease import (RENEW_INTERVAL_SECONDS,ProcessLease,)-from switch_core.signals import MAX_PAYLOAD_BYTES, RETRY_SECONDS+from switch_core.signals import RETRY_SECONDS, payload_chunksfrom switch_core.tenant_context import current_tenant_id, no_tenantif TYPE_CHECKING:@@ -89,6 +91,7 @@ logger = logging.getLogger(__name__)CHANGED_SIGNAL = "agent_connections.changed"BEATS_SIGNAL = "agent_connections.beats"SESSION_COMMAND_SIGNAL = "agent_connections.session_command"+WARM_SIGNAL = "agent_connections.warm"# How many incarnations are drawn from the sequence at a time. One draw per# this many opens, and a process restart wastes at most this many numbers out@@ -178,6 +181,7 @@ class SharedConnections:bus.handle(CHANGED_SIGNAL, self._on_changed)bus.handle(BEATS_SIGNAL, self._on_beats)bus.handle(SESSION_COMMAND_SIGNAL, self._on_session_command)+ bus.handle(WARM_SIGNAL, self._on_warm)bus.on_resync(self.reload_all)bus.on_recheck(self.reload_all)registry.use_rows(self)@@ -342,6 +346,11 @@ class SharedConnections:)self._writes.set()+ def relay_warm(self, agent_id: str, process_id: str, room_id: str) -> None:+ self._bus.publish(+ WARM_SIGNAL, agent_id=agent_id, room_id=room_id, process_id=process_id+ )+# ── Reading the rows into the registry ────────────────────────────────async def _read_agent(self, session: AsyncSession, agent_id: str) -> AgentSlotRows:@@ -477,12 +486,24 @@ class SharedConnections:await session.commit()self._deliver(relayed)+ def _on_warm(self, signal: dict[str, Any]) -> None:+ if signal["process_id"] != self.process_id:+ return+ if not self._registry.controllers.relay_warm(+ signal["agent_id"], signal["room_id"]+ ):+ logger.warning(+ "A warm for agent %s arrived after the controller stream it was "+ "relayed to went, or the agent was stopped; dropped",+ signal["agent_id"],+ )+def _publish_changed(self, tenant_id: str, agent_id: str, released: list[Released]) -> None:entries = [[r.connection_id, r.room_id, r.session_id] for r in released]# Sent with no rooms released too: the change is what is announced.- for chunk in _chunks(entries) or [[]]:+ for chunk in payload_chunks(entries, _SIGNAL_ENVELOPE_BYTES) or [[]]:self._bus.publish(CHANGED_SIGNAL, tenant_id=tenant_id, agent_id=agent_id, released=chunk)@@ -585,7 +606,7 @@ class SharedConnections:await self._beats_waiting.wait()self._beats_waiting.clear()beats, self._beats = list(self._beats.values()), {}- for chunk in _chunks(beats):+ for chunk in payload_chunks(beats, _SIGNAL_ENVELOPE_BYTES):try:self._bus.publish(BEATS_SIGNAL, beats=chunk)except ValueError as error:@@ -628,24 +649,3 @@ class SharedConnections:raiseexcept Exception:logger.exception("Checking the other processes' connections failed")---def _chunks(entries: list[list[Any]]) -> list[list[list[Any]]]:- """Split a signal's entries, forwarded beats or released rooms, into lists- that each fit in one payload under the `NOTIFY` limit. Measured as JSON,- which writes a character outside ASCII in up to twelve bytes. An entry- too large to fit even alone gets a list of its own, which the bus then- refuses."""- chunks: list[list[list[Any]]] = []- current: list[list[Any]] = []- size = _SIGNAL_ENVELOPE_BYTES- for entry in entries:- cost = len(json.dumps(entry)) + 1- if current and size + cost > MAX_PAYLOAD_BYTES:- chunks.append(current)- current, size = [], _SIGNAL_ENVELOPE_BYTES- current.append(entry)- size += cost- if current:- chunks.append(current)- return chunksdiff --git a/core/switch_core/bridges/agent/protocol/controller_presence.py b/core/switch_core/bridges/agent/protocol/controller_presence.pyindex 372e17d6f..f69b9a938 100644--- a/core/switch_core/bridges/agent/protocol/controller_presence.py+++ b/core/switch_core/bridges/agent/protocol/controller_presence.py@@ -591,8 +591,10 @@ class ControllerPresence:def relay_warm(self, agent_id: str, room_id: str) -> bool:"""Ask the agent's controller to start the session a message to- `room_id` would start. False when its controller is not live or the- agent is stopped. A warm already owed for the room is not repeated."""+ `room_id` would start, when this process holds its stream. False when+ its controller is not live here or the agent is stopped. A warm+ already owed for the room is not repeated. A controller held by+ another process is reached through the registry (`process_holding`)."""binding = self._bindings.get(agent_id)if binding is None or not binding.running:return Falsediff --git a/core/switch_core/chats/service.py b/core/switch_core/chats/service.pyindex 3ab8a75bd..be3b75bf1 100644--- a/core/switch_core/chats/service.py+++ b/core/switch_core/chats/service.py@@ -45,7 +45,7 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmakerfrom switch_core.addressing import allows_on_behalf_of, parse_policyfrom switch_core.authz import administers_tenant-from switch_core.bridges.agent.protocol.controller_presence import ControllerPresence+from switch_core.bridges.agent.protocol.agent_connections import AgentConnectionRegistryfrom switch_core.chats import MEMBER_CLIENT_TYPEfrom switch_core.chats.views import (COMMAND_EVENT_TYPE,@@ -100,6 +100,7 @@ from switch_core.transport.content import media_content, message_contentfrom switch_core.transport.types import MessageFormatif TYPE_CHECKING:+ from switch_core.chats.signals import ChatSignalsfrom switch_core.delivery.router import MessageRouter, Routedlogger = logging.getLogger(__name__)@@ -162,10 +163,10 @@ class ChatService:id_server_name: str,media_max_bytes: int,agent_chats_enabled: bool,- controllers: ControllerPresence,+ connections: AgentConnectionRegistry,) -> None:self.session_factory = session_factory- self._controllers = controllers+ self._connections = connectionsself.agent_chats_enabled = agent_chats_enabledself.listener = listenerself.activity_listener = activity_listener@@ -186,6 +187,9 @@ class ChatService:weakref.WeakValueDictionary())self._membership_watchers: dict[tuple[str, str], set[Callable[[], None]]] = {}+ # Where a change to people's chats made here is announced to the+ # other processes, once wired (`use_signals`).+ self._signals: ChatSignals | None = Noneself._addressing = AddressingResolver(room_store=room_store,room_role_store=RoomRoleStore(),@@ -198,10 +202,16 @@ class ChatService:# ── Membership signals ───────────────────────────────────────────────────+ def use_signals(self, signals: ChatSignals) -> None:+ """Where a change to people's chats made here is announced, so every+ other process wakes the streams it holds for them (`signals.py`)."""+ self._signals = signals+def watch_memberships(self, tenant_id: str, user_id: str, callback: Callable[[], None]) -> None:- """Call `callback` whenever this process changes the user's chats."""+ """Call `callback` whenever the user's chats change: at once when this+ process changed them, and once told when another process did."""self._membership_watchers.setdefault((tenant_id, user_id), set()).add(callback)def unwatch_memberships(@@ -214,13 +224,28 @@ class ChatService:if not watchers:del self._membership_watchers[(tenant_id, user_id)]- def _memberships_changed(self, tenant_id: str, user_ids: list[str]) -> None:+ def wake_membership_watchers(self, tenant_id: str, user_ids: list[str]) -> None:+ """Call every callback watching these users' chats in this process."""for user_id in user_ids:for callback in list(self._membership_watchers.get((tenant_id, user_id), ())):callback()+ def wake_all_membership_watchers(self) -> None:+ """Call every callback watching anyone's chats in this process, once."""+ for watchers in list(self._membership_watchers.values()):+ for callback in list(watchers):+ callback()++ def _memberships_changed(self, tenant_id: str, user_ids: list[str]) -> None:+ """The users' chats changed through this process, and the change has+ committed: the streams here are woken at once, and the other+ processes are told to wake theirs."""+ self.wake_membership_watchers(tenant_id, user_ids)+ if self._signals is not None:+ self._signals.memberships_changed(tenant_id, user_ids)+def _lock(self, *key: str) -> asyncio.Lock:lock = self._locks.get(key)if lock is None:@@ -1269,7 +1294,9 @@ class ChatService:) -> None:"""Ask the controller of the user's own agent to start its session forthis chat before the first message. A hint: nothing waits for it, and- an agent no controller hosts is not warmed."""+ an agent no controller hosts is not warmed. The controller is asked+ through whichever process holds its socket+ (`AgentConnectionRegistry.relay_warm`)."""async with tenant_session(self.session_factory, tenant_id) as session:room, client = await self.require_member(session, tenant_id, user, room_id)if agent_id not in await self._room_store.get_agent_ids(session, room.id):@@ -1287,9 +1314,9 @@ class ChatService:)if not decision.allowed:raise ChatError(403, "AGENT_NOT_ALLOWED", decision.refusal)- if not self._controllers.is_bound(agent_id):+ if not self._connections.controllers.is_bound(agent_id):return- if not self._controllers.relay_warm(agent_id, room.id):+ if not self._connections.relay_warm(agent_id, room.id):logger.warning("Could not warm agent %s for room %s: its controller is not connected or it is stopped",agent_id,diff --git a/core/switch_core/chats/signals.py b/core/switch_core/chats/signals.pynew file mode 100644index 000000000..2f5aa9ee4--- /dev/null+++ b/core/switch_core/chats/signals.py@@ -0,0 +1,63 @@+"""Changes to people's chats, reaching their Console streams on every process.++Switch Console follows a person's chats over one stream (`chat_events` in+`gateway/chats.py`), held by whichever switch-core process its request+reached. What changes which chats a person has (a chat created or archived, a+member added or removed, a room gaining or losing its agents) is served by+whichever process that request reached, and `ChatService` wakes the streams+held there at once. A stream held by another process saw the change only at+its next periodic re-read.++So each such change, once committed, announces `chats.memberships_changed`+naming the tenant and the people whose chats changed. Every other process+wakes the streams it holds for them, and each re-reads what its person may+see. A change concerning more people than one signal can name is split over+as many signals as it takes.++Signals sent while this process's signal connection was down were never+queued, so when it comes back every stream held here is woken once: a wake-up+costs that stream one re-read.+"""++from __future__ import annotations++from typing import TYPE_CHECKING, Any++from switch_core.signals import payload_chunks++if TYPE_CHECKING:+ from switch_core.chats.service import ChatService+ from switch_core.signals import SignalBus++MEMBERSHIPS_CHANGED_SIGNAL = "chats.memberships_changed"++# Room kept in every signal for what it carries beside its user ids: its kind,+# the sending process's id and the tenant id, about 200 bytes at most.+_SIGNAL_ENVELOPE_BYTES = 400+++class ChatSignals:+ def __init__(self, *, service: ChatService, bus: SignalBus) -> None:+ self._service = service+ self._bus = bus+ bus.handle(MEMBERSHIPS_CHANGED_SIGNAL, self._on_changed)+ bus.on_resync(self._wake_all)+ # No periodic re-check, unlike the other families: every stream+ # already re-reads everything on its own every `RECHECK_SECONDS`+ # (`gateway/chats.py`), which is all a missed signal can cost it.+ service.use_signals(self)++ def memberships_changed(self, tenant_id: str, user_ids: list[str]) -> None:+ """The users' chats changed and the change has committed: every other+ process wakes the streams it holds for them. This one has already+ woken its own."""+ for chunk in payload_chunks(user_ids, _SIGNAL_ENVELOPE_BYTES):+ self._bus.publish(+ MEMBERSHIPS_CHANGED_SIGNAL, tenant_id=tenant_id, user_ids=chunk+ )++ def _on_changed(self, signal: dict[str, Any]) -> None:+ self._service.wake_membership_watchers(signal["tenant_id"], signal["user_ids"])++ async def _wake_all(self) -> None:+ self._service.wake_all_membership_watchers()diff --git a/core/switch_core/gateway/chats.py b/core/switch_core/gateway/chats.pyindex 7d0edf4a4..0237686ea 100644--- a/core/switch_core/gateway/chats.py+++ b/core/switch_core/gateway/chats.py@@ -644,9 +644,10 @@ async def chat_events(Catches every member room up from its cursor in `after`, says `ready`,then pushes as rooms move. A room the caller joined since is announced as- `chat` and followed from its head; one that leaves the list is `chat.removed`. A- re-read every `recheck_seconds` recovers anything a missed notification- or another process's membership change left behind.+ `chat` and followed from its head; one that leaves the list is `chat.removed`.+ A change to the caller's chats wakes the stream at once, whichever process+ made it (`chats/signals.py`). A re-read every `recheck_seconds` recovers+ anything a missed notification or a missed signal left behind."""stream = _ChatStream(service, tenant_id, user, after)service.watch_memberships(tenant_id, user.id, stream.on_memberships)diff --git a/core/switch_core/main.py b/core/switch_core/main.pyindex 9ab2280fa..dc97618d4 100644--- a/core/switch_core/main.py+++ b/core/switch_core/main.py@@ -94,6 +94,7 @@ from switch_core.bridges.collaboration.webhooks import ()from switch_core.bridges.resource.service import ResourceServicefrom switch_core.chats.service import ChatService+from switch_core.chats.signals import ChatSignalsfrom switch_core.clients.actor import Actor, AgentActor, HumanActor, SystemActorfrom switch_core.clients.client_factory import ClientFactoryfrom switch_core.clients.client_lifecycle_service import ClientLifecycleService@@ -1107,8 +1108,11 @@ async def run(config: SwitchConfig, process_id: str) -> None:id_server_name=config.id_server_name,media_max_bytes=config.agent_media_max_bytes,agent_chats_enabled=config.agent_chats_enabled,- controllers=connections.controllers,+ connections=connections,)+ # Changes to people's chats made in another process wake the Console+ # streams held here too (`chats/signals.py`).+ ChatSignals(service=chat_service, bus=signal_bus)room_service.on_room_agents_changed(chat_service.sync_room)protocol.set_room_agents_listener(room_service.room_agents_changed)diff --git a/core/switch_core/signals.py b/core/switch_core/signals.pyindex c86be35f0..5eda31b80 100644--- a/core/switch_core/signals.py+++ b/core/switch_core/signals.py@@ -416,3 +416,24 @@ class SignalBus:handler(signal)except Exception:logger.error("Handling signal %r failed", signal["kind"], exc_info=True)+++def payload_chunks[T](entries: list[T], envelope_bytes: int) -> list[list[T]]:+ """Split the list a signal carries into lists that each fit in one payload+ under the `NOTIFY` limit, beside the `envelope_bytes` kept for the+ signal's other fields. Measured as JSON, which writes a character outside+ ASCII in up to twelve bytes. An entry too large to fit even alone gets a+ list of its own, which `SignalBus.publish` then refuses."""+ chunks: list[list[T]] = []+ current: list[T] = []+ size = envelope_bytes+ for entry in entries:+ cost = len(json.dumps(entry)) + 1+ if current and size + cost > MAX_PAYLOAD_BYTES:+ chunks.append(current)+ current, size = [], envelope_bytes+ current.append(entry)+ size += cost+ if current:+ chunks.append(current)+ return chunksdiff --git a/core/tests/switch_core/gateway/conftest.py b/core/tests/switch_core/gateway/conftest.pyindex bcc5f0342..3c984d5a3 100644--- a/core/tests/switch_core/gateway/conftest.py+++ b/core/tests/switch_core/gateway/conftest.py@@ -1,9 +1,12 @@"""The `/chats` harness: real routes, `ChatService`, `RoomService` and-Postgres provisioning, shared by the chats test modules."""+Postgres provisioning, shared by the chats test modules, and a second+switch-core process beside it for the tests across processes."""from __future__ import annotations+import asynciofrom collections.abc import AsyncIterator, Awaitable, Callable+from dataclasses import dataclassfrom types import SimpleNamespacefrom typing import Anyfrom unittest.mock import MagicMock@@ -12,21 +15,27 @@ import httpximport pytestfrom fastapi import FastAPI, Requestfrom sqlalchemy import select-from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine+from sqlalchemy.pool import NullPool+from switch_core.bridges.agent.protocol.agent_connections import AgentConnectionRegistry+from switch_core.bridges.agent.protocol.connection_rows import SharedConnectionsfrom switch_core.chats.service import ChatError, ChatService+from switch_core.chats.signals import ChatSignalsfrom switch_core.db.models import (Agent,Client,Message,User,)+from switch_core.db.stores.agent_connection_store import AgentConnectionStorefrom switch_core.db.stores.agent_store import AgentStorefrom switch_core.db.stores.client_store import ClientStorefrom switch_core.db.stores.collaboration_bridge_store import CollaborationBridgeStorefrom switch_core.db.stores.media_store import MediaStorefrom switch_core.db.stores.message_store import MessageStorefrom switch_core.db.stores.room_store import RoomStore+from switch_core.db.stores.switch_core_process_store import SwitchCoreProcessStorefrom switch_core.db.stores.usage_store import UsageStorefrom switch_core.db.stores.user_store import UserStorefrom switch_core.gateway.auth import get_current_user@@ -34,15 +43,22 @@ from switch_core.gateway.chats import (chat_error_response,router,)+from switch_core.process_identity import new_process_id+from switch_core.process_lease import ProcessLeasefrom switch_core.provisioning.postgres import PostgresProvisioningfrom switch_core.room_service import RoomServicefrom switch_core.session_activity.listener import Change+from switch_core.signals import SignalBusfrom switch_core.transport.invites import InviteBus+from tests.conftest import TEST_KEYRINGfrom tests.routing import router_onfrom tests.switch_core.gateway.agent_route_harness import add_agent_MESSAGE_STORE = MessageStore()+# How long `two_processes` waits for each process's signal bus to connect.+_CONNECT_SECONDS = 10.0+class _FakeLifecycle:"""The running-client registry, for agents that have no consumer here."""@@ -96,24 +112,6 @@ class _FakeActivityListener:await on_resync()-class _FakeControllers:- """The controller presence, as far as warming a chat reads it."""-- def __init__(self) -> None:- self.bound: set[str] = set()- self.live = True- self.warms: list[tuple[str, str]] = []-- def is_bound(self, agent_id: str) -> bool:- return agent_id in self.bound-- def relay_warm(self, agent_id: str, room_id: str) -> bool:- if not self.live:- return False- self.warms.append((agent_id, room_id))- return True--class _Harness(SimpleNamespace):client: httpx.AsyncClientfactory: async_sessionmaker[AsyncSession]@@ -122,7 +120,7 @@ class _Harness(SimpleNamespace):listener: _FakeListeneractivity: _FakeActivityListenerlifecycle: _FakeLifecycle- controllers: _FakeControllers+ connections: AgentConnectionRegistryalice: Userbob: Usercarol: User@@ -211,19 +209,17 @@ async def _user(session: AsyncSession, name: str, role: str | None) -> User:return user-async def chats(+def _chat_service(session_factory: async_sessionmaker[AsyncSession],-) -> AsyncIterator[_Harness]:- async with session_factory() as session:- alice = await _user(session, "alice", "member")- bob = await _user(session, "bob", "member")- carol = await _user(session, "carol", None)- admin = await _user(session, "admin", "admin")- await session.commit()-+ *,+ lifecycle: _FakeLifecycle,+ listener: _FakeListener,+ activity: _FakeActivityListener,+ connections: AgentConnectionRegistry,+) -> tuple[ChatService, RoomService]:+ """A chat service and the room service it works through, built as one+ switch-core process builds them."""room_store = RoomStore()- lifecycle = _FakeLifecycle()message_router = router_on(session_factory)provisioning = PostgresProvisioning(session_factory=session_factory,@@ -243,9 +239,6 @@ async def chats(session_factory=session_factory,room_cache=MagicMock(),)- listener = _FakeListener()- activity = _FakeActivityListener()- controllers = _FakeControllers()service = ChatService(session_factory=session_factory,room_service=room_service,@@ -262,9 +255,34 @@ async def chats(id_server_name="switch.test",media_max_bytes=1024,agent_chats_enabled=True,- controllers=controllers, # type: ignore[arg-type]+ connections=connections,)room_service.on_room_agents_changed(service.sync_room)+ return service, room_service+++async def chats(+ session_factory: async_sessionmaker[AsyncSession],+) -> AsyncIterator[_Harness]:+ async with session_factory() as session:+ alice = await _user(session, "alice", "member")+ bob = await _user(session, "bob", "member")+ carol = await _user(session, "carol", None)+ admin = await _user(session, "admin", "admin")+ await session.commit()++ lifecycle = _FakeLifecycle()+ listener = _FakeListener()+ activity = _FakeActivityListener()+ connections = AgentConnectionRegistry()+ service, room_service = _chat_service(+ session_factory,+ lifecycle=lifecycle,+ listener=listener,+ activity=activity,+ connections=connections,+ )app = FastAPI()app.state.chat_service = service@@ -289,9 +307,102 @@ async def chats(listener=listener,activity=activity,lifecycle=lifecycle,- controllers=controllers,+ connections=connections,alice=alice,bob=bob,carol=carol,admin=admin,)+++@dataclass+class _Process:+ """One switch-core process's side of chats on the shared database: its+ chat service and agent connections, and the signal bus over which it+ tells the other process what changed."""++ service: ChatService+ connections: AgentConnectionRegistry+ signals: ChatSignals+ shared: SharedConnections+ bus: SignalBus+ lease: ProcessLease+++def _process(+ session_factory: async_sessionmaker[AsyncSession],+ postgres_url: str,+ service: ChatService,+ connections: AgentConnectionRegistry,+) -> _Process:+ process_id = new_process_id("test-pod")+ bus = SignalBus(+ session_factory,+ lambda: create_async_engine(postgres_url, poolclass=NullPool),+ process_id=process_id,+ )+ lease = ProcessLease(+ process_id=process_id,+ session_factory=session_factory,+ processes=SwitchCoreProcessStore(),+ key_ids=[key.id for key in TEST_KEYRING.keys],+ )+ return _Process(+ service=service,+ connections=connections,+ signals=ChatSignals(service=service, bus=bus),+ shared=SharedConnections(+ registry=connections,+ session_factory=session_factory,+ store=AgentConnectionStore(),+ bus=bus,+ lease=lease,+ ),+ bus=bus,+ lease=lease,+ )+++async def two_processes(+ chats: _Harness, postgres_url: str+) -> AsyncIterator[tuple[_Process, _Process]]:+ """The harness's chat service as one switch-core process, and a second on+ the same database. Each tells the other what changed over its own signal+ bus on the real `NOTIFY` channel: people's chats (`ChatSignals`) and its+ agent connections (`SharedConnections`)."""+ connections = AgentConnectionRegistry()+ service, _ = _chat_service(+ chats.factory,+ lifecycle=chats.lifecycle,+ listener=_FakeListener(),+ activity=_FakeActivityListener(),+ connections=connections,+ )+ processes = [+ _process(chats.factory, postgres_url, chats.service, chats.connections),+ _process(chats.factory, postgres_url, service, connections),+ ]+ for process in processes:+ await process.lease.renew()+ await process.shared.start()+ await process.bus.start()+ # Each lease is renewed while the process runs, as in production, so the+ # other reads what it holds as held however long the test takes.+ runs = [+ asyncio.create_task(run())+ for process in processes+ for run in (process.lease.run, process.shared.run)+ ]+ try:+ for process in processes:+ await asyncio.wait_for(process.bus.connected.wait(), _CONNECT_SECONDS)+ yield processes[0], processes[1]+ finally:+ # The buses first, so no signal starts work after the processes+ # stopped running; then the runs, which wait for the work they started.+ for process in processes:+ await process.bus.stop()+ for run in runs:+ run.cancel()+ await asyncio.gather(*runs, return_exceptions=True)diff --git a/core/tests/switch_core/gateway/test_chats_routes.py b/core/tests/switch_core/gateway/test_chats_routes.pyindex 3c78dc03e..c3a559394 100644--- a/core/tests/switch_core/gateway/test_chats_routes.py+++ b/core/tests/switch_core/gateway/test_chats_routes.py@@ -4,7 +4,9 @@ sends, and the live event stream — against real Postgres.The routes run behind a real `ChatService`, `RoomService` and Postgresprovisioning. Only the running-client registry is faked (no agent consumerruns here) and the message listener, whose wake-ups the stream tests ring by-hand so a missed notification can be staged.+hand so a missed notification can be staged. The stream tests across+processes add a second chat service on the same database, each process with+its own signal bus on the real `NOTIFY` channel."""from __future__ import annotations@@ -12,7 +14,7 @@ from __future__ import annotationsimport asyncioimport jsonimport uuid-from collections.abc import AsyncIterator+from collections.abc import AsyncIterator, Callablefrom datetime import UTC, datetime, timedeltafrom typing import Any@@ -46,7 +48,7 @@ from switch_core.gateway.chats import (chat_events,)from switch_core.session_activity.listener import Change-from tests.switch_core.gateway.conftest import _Harness+from tests.switch_core.gateway.conftest import _Harness, _Processdef _code(response: httpx.Response) -> str:@@ -981,7 +983,8 @@ async def test_the_periodic_recheck_announces_a_chat_that_regains_an_agent()assert (await events.next())[0] == "ready"- # Written straight to the table, as another process would: no hook runs.+ # Written straight to the table: no hook runs and no signal is sent, as+ # when another process's signal is missed.async with chats.factory() as session:await session.execute(room_agents.insert().values(room_id=room_id, agent_id=agent.id)@@ -1051,6 +1054,95 @@ async def test_the_periodic_recheck_recovers_a_missed_notification(await events.stream.aclose() # type: ignore[attr-defined]+def _stream_on(process: _Process, user: User) -> _Events:+ """A stream held by `process` that re-reads on its own only after an hour,+ so whatever it says before then, something woke it to say."""+ return _Events(+ chat_events(+ process.service,+ TENANT,+ user,+ {},+ ping_seconds=60.0,+ recheck_seconds=3600.0,+ )+ )+++async def _eventually(check: Callable[[], bool]) -> None:+ async with asyncio.timeout(10.0):+ while not check():+ await asyncio.sleep(0.02)+++async def _join_unannounced(chats: _Harness, user: User, room_id: str) -> None:+ """Put the user in the room without waking any stream or sending any+ signal, as a change whose signal was lost."""+ async with chats.factory() as session:+ room = await session.get(Room, room_id)+ assert room is not None+ await chats.service._join(+ TENANT, room, await chats.service.member_actor(TENANT, user)+ )+++async def test_a_change_through_another_process_reaches_the_stream_at_once(+ chats: _Harness, two_processes: tuple[_Process, _Process]+) -> None:+ """Bob's Console streams from one process while Alice invites him through+ the other: his stream announces the chat at once, not at its re-read."""+ _, there = two_processes+ agent = await chats.agent("helper", owner=chats.alice)+ room_id = await chats.new_chat(chats.alice, agent, "c1")+ events = _stream_on(there, chats.bob)+ assert (await events.next())[0] == "ready"++ assert (await chats.invite(chats.alice, room_id, chats.bob)).status_code == 200++ name, summary = await events.next()+ assert name == "chat" and summary["roomId"] == room_id+ await events.stream.aclose() # type: ignore[attr-defined]+++async def test_every_stream_re_reads_when_its_signal_connection_comes_back(+ chats: _Harness, two_processes: tuple[_Process, _Process]+) -> None:+ """A signal sent while a process's signal connection is down never reaches+ it, so once the connection is back every stream it holds re-reads."""+ _, there = two_processes+ agent = await chats.agent("helper", owner=chats.alice)+ room_id = await chats.new_chat(chats.alice, agent, "c1")+ events = _stream_on(there, chats.bob)+ assert (await events.next())[0] == "ready"++ await there.bus.stop()+ await _join_unannounced(chats, chats.bob, room_id)+ await there.bus.start()++ name, summary = await events.next()+ assert name == "chat" and summary["roomId"] == room_id+ await events.stream.aclose() # type: ignore[attr-defined]+++async def test_a_change_concerning_more_people_than_one_signal_holds_reaches_them_all(+ two_processes: tuple[_Process, _Process],+) -> None:+ """The people a change concerns are split over as many signals as it takes+ to stay under the `NOTIFY` limit, and each one's streams are woken."""+ here, there = two_processes+ user_ids = [str(uuid.uuid4()) for _ in range(500)]+ woken: list[str] = []+ for user_id in user_ids:+ there.service.watch_memberships(+ TENANT, user_id, lambda user_id=user_id: woken.append(user_id)+ )++ here.signals.memberships_changed(TENANT, user_ids)++ await _eventually(lambda: len(woken) == len(user_ids))+ assert sorted(woken) == sorted(user_ids)++# ── Agent owners ─────────────────────────────────────────────────────────────TENANT = "00000000-0000-0000-0000-000000000000"@@ -1323,7 +1415,8 @@ async def test_the_periodic_recheck_finds_a_chat_the_agent_joined_elsewhere(events = _stream(chats, chats.alice, {}, recheck=0.2)assert (await events.next())[0] == "ready"- # Written straight to the table, as another process would: no hook runs.+ # Written straight to the table: no hook runs and no signal is sent, as+ # when another process's signal is missed.async with chats.factory() as session:await session.execute(room_agents.insert().values(room_id=room_id, agent_id=agent.id)diff --git a/core/tests/switch_core/gateway/test_chats_warm.py b/core/tests/switch_core/gateway/test_chats_warm.pyindex c90c28741..383ee3c6b 100644--- a/core/tests/switch_core/gateway/test_chats_warm.py+++ b/core/tests/switch_core/gateway/test_chats_warm.py@@ -3,12 +3,22 @@ user's own agent to start its session while the user types."""from __future__ import annotations+import asyncio+from collections.abc import Callable+import httpx-from switch_core.db.models import Agent, User-from tests.switch_core.gateway.conftest import _Harness+from switch_core.bridges.agent.protocol.agent_connections import AgentConnectionRegistry+from switch_core.bridges.agent.protocol.controller_presence import (+ Binding,+ ControllerConnection,+)+from switch_core.db.models import TENANT_ZERO_ID, Agent, User+from tests.switch_core.gateway.conftest import _Harness, _Processfrom tests.switch_core.gateway.test_chats_routes import _code+CONTROLLER = "controller-1"+async def _warm(chats: _Harness, user: User, room_id: str, agent: Agent@@ -20,15 +30,48 @@ async def _warm()+def _bind(connections: AgentConnectionRegistry, agent: Agent) -> None:+ """Bind the agent to the controller, as Management does in every process."""+ connections.controllers.bind(+ Binding(+ agent_id=agent.id,+ controller_id=CONTROLLER,+ tenant_id=TENANT_ZERO_ID,+ controller_name="the machine",+ running=True,+ )+ )+++def _on_controller(+ connections: AgentConnectionRegistry, agent: Agent+) -> ControllerConnection:+ """Put the agent on a controller whose live stream this registry's process+ holds, and return the controller's connection."""+ _bind(connections, agent)+ controllers = connections.controllers+ conn = controllers.open(+ controller_id=CONTROLLER, tenant_id=TENANT_ZERO_ID, resume_cursors={}+ )+ controllers.attach_stream(conn)+ return conn+++async def _eventually(check: Callable[[], bool]) -> None:+ async with asyncio.timeout(10.0):+ while not check():+ await asyncio.sleep(0.02)++async def test_a_warm_reaches_the_agents_controller(chats: _Harness) -> None:agent = await chats.agent("helper", owner=chats.alice)- chats.controllers.bound.add(agent.id)+ controller = _on_controller(chats.connections, agent)room_id = await chats.new_chat(chats.alice, agent, "c1")response = await _warm(chats, chats.alice, room_id, agent)assert response.status_code == 202, response.text- assert chats.controllers.warms == [(agent.id, room_id)]+ assert controller.warms == {(agent.id, room_id)}async def test_a_warm_is_accepted_when_no_controller_takes_it(@@ -38,19 +81,19 @@ async def test_a_warm_is_accepted_when_no_controller_takes_it(room_id = await chats.new_chat(chats.alice, agent, "c1")unbound = await _warm(chats, chats.alice, room_id, agent)- chats.controllers.bound.add(agent.id)- chats.controllers.live = False+ controller = _on_controller(chats.connections, agent)+ chats.connections.controllers.detach_stream(controller, controller.stream_token)offline = await _warm(chats, chats.alice, room_id, agent)assert unbound.status_code == offline.status_code == 202- assert chats.controllers.warms == []+ assert controller.warms == set()async def test_a_warm_refuses_non_members_others_agents_and_absent_agents(chats: _Harness,) -> None:agent = await chats.agent("helper", owner=chats.alice)- chats.controllers.bound.add(agent.id)+ controller = _on_controller(chats.connections, agent)room_id = await chats.new_chat(chats.alice, agent, "c1")outsider = await _warm(chats, chats.bob, room_id, agent)@@ -67,14 +110,14 @@ async def test_a_warm_refuses_non_members_others_agents_and_absent_agents(assert absent.status_code == 422assert _code(absent) == "AGENT_NOT_IN_CHAT"- assert chats.controllers.warms == []+ assert controller.warms == set()async def test_a_warm_refuses_an_owner_the_agents_policy_refuses(chats: _Harness,) -> None:agent = await chats.agent("helper", owner=chats.alice)- chats.controllers.bound.add(agent.id)+ controller = _on_controller(chats.connections, agent)room_id = await chats.new_chat(chats.alice, agent, "c1")async with chats.factory() as session:row = await session.get(Agent, agent.id)@@ -86,4 +129,24 @@ async def test_a_warm_refuses_an_owner_the_agents_policy_refuses(assert refused.status_code == 403assert _code(refused) == "AGENT_NOT_ALLOWED"- assert chats.controllers.warms == []+ assert controller.warms == set()+++async def test_a_warm_reaches_a_controller_another_process_holds(+ chats: _Harness, two_processes: tuple[_Process, _Process]+) -> None:+ """The Console's request reaches a process that does not hold the+ controller's socket, which hands the warm to the process that does."""+ here, there = two_processes+ agent = await chats.agent("helper", owner=chats.alice)+ room_id = await chats.new_chat(chats.alice, agent, "c1")+ controller = _on_controller(there.connections, agent)+ # What Management keeps in every process from the controller's rows: the+ # agent's binding, and which process holds the controller's socket.+ _bind(here.connections, agent)+ here.connections.controllers.held_elsewhere(CONTROLLER, there.bus.process_id)++ response = await _warm(chats, chats.alice, room_id, agent)++ assert response.status_code == 202, response.text+ await _eventually(lambda: controller.warms == {(agent.id, room_id)})diff --git a/core/tests/switch_core/management/test_controllers_across_processes.py b/core/tests/switch_core/management/test_controllers_across_processes.pyindex 3b0a9d375..6a940c318 100644--- a/core/tests/switch_core/management/test_controllers_across_processes.py+++ b/core/tests/switch_core/management/test_controllers_across_processes.py@@ -277,6 +277,27 @@ class TestPresence:assert conn is not Noneawait _eventually(lambda: conn.session_commands == [(agent_id, frame)])+ async def test_a_warm_reaches_the_controller_on_the_other_process(+ self, processes: tuple[Harness, Harness]+ ) -> None:+ here, there = processes+ owner = await add_member(here.session_factory, "ada")+ async with here.client() as client:+ controller = await enroll_console(here, client, owner)+ agent_id = await place_agent(client, controller, name="reviewer")+ await connect(client, controller)+ await _eventually(+ lambda: there.protocol.connections.controllers.is_live(agent_id)+ )++ assert there.protocol.connections.relay_warm(agent_id, "room")++ conn = here.protocol.connections.controllers.current_connection(+ controller.controller_id+ )+ assert conn is not None+ await _eventually(lambda: conn.warms == {(agent_id, "room")})+class TestTheConnectionOnAnyProcess:async def test_the_socket_attaches_through_the_process_that_did_not_open_it(