Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Tell Consoles on every pod when a user's chats change, and warm a controller held by another pod (#885)

Abel Dantas685e924279de
diff --git a/core/switch_core/bridges/agent/protocol/agent_connections.py b/core/switch_core/bridges/agent/protocol/agent_connections.py
index c42741c9b..c6b46fe26 100644
--- a/core/switch_core/bridges/agent/protocol/agent_connections.py
+++ b/core/switch_core/bridges/agent/protocol/agent_connections.py
@@ -565,6 +565,11 @@ class ConnectionRows(Protocol):
block."""
...
+ def relay_warm(self, agent_id: str, process_id: str, room_id: str) -> None:
+ """A warm for a controller whose stream another process holds. Must
+ not block."""
+ ...
+
class AgentConnectionRegistry:
"""The live set of agent connections: those this process holds,
@@ -1247,6 +1252,20 @@ class AgentConnectionRegistry:
delivered = True
return delivered
+ def relay_warm(self, agent_id: str, room_id: str) -> bool:
+ """Ask the controller running the agent to start the session a message
+ to `room_id` would start. False when no process holds that controller's
+ live socket or the agent is stopped. A controller another process
+ holds is asked through that process (`ConnectionRows.relay_warm`),
+ which hands the warm to its `ControllerPresence.relay_warm`."""
+ if self.controllers.relay_warm(agent_id, room_id):
+ return True
+ holder = self.controllers.process_holding(agent_id)
+ if holder is None or self._rows is None:
+ return False
+ self._rows.relay_warm(agent_id, holder, room_id)
+ return True
+
def _watchers(self, agent_id: str) -> list[AgentConnection]:
return [
conn
diff --git a/core/switch_core/bridges/agent/protocol/connection_rows.py b/core/switch_core/bridges/agent/protocol/connection_rows.py
index a7d2af971..96201f529 100644
--- a/core/switch_core/bridges/agent/protocol/connection_rows.py
+++ b/core/switch_core/bridges/agent/protocol/connection_rows.py
@@ -37,18 +37,20 @@ lease check (`WATCH_SECONDS`) drops the copies of a process whose lease
lapsed. What a signal asks this process to read goes through the bus's
`PendingWork`, so a read that fails is kept and tried again.
-Two requests are handed to the holder rather than served where they land: a
+Three requests are handed to the holder rather than served where they land: a
`POST /connection/beat` from an old event-stream client (forwarded in batches,
-`agent_connections.beats`) and a session command for a stream another process
+`agent_connections.beats`), a session command for a stream another process
holds, the agent's watcher or its controller's (stored as a relay row for that
process and signalled, `agent_connections.session_command`, since its content
-may not travel in a `NOTIFY`).
+may not travel in a `NOTIFY`), and a warm for a controller another process
+holds (`agent_connections.warm`). A warm is nothing but ids, so it travels in
+the signal itself; it is a hint, and one lost costs only the head start it
+would have given the agent's session.
"""
from __future__ import annotations
import asyncio
-import json
import logging
from collections.abc import AsyncIterator
from contextlib import asynccontextmanager
@@ -76,7 +78,7 @@ from switch_core.process_lease import (
RENEW_INTERVAL_SECONDS,
ProcessLease,
)
-from switch_core.signals import MAX_PAYLOAD_BYTES, RETRY_SECONDS
+from switch_core.signals import RETRY_SECONDS, payload_chunks
from switch_core.tenant_context import current_tenant_id, no_tenant
if TYPE_CHECKING:
@@ -89,6 +91,7 @@ logger = logging.getLogger(__name__)
CHANGED_SIGNAL = "agent_connections.changed"
BEATS_SIGNAL = "agent_connections.beats"
SESSION_COMMAND_SIGNAL = "agent_connections.session_command"
+WARM_SIGNAL = "agent_connections.warm"
# How many incarnations are drawn from the sequence at a time. One draw per
# this many opens, and a process restart wastes at most this many numbers out
@@ -178,6 +181,7 @@ class SharedConnections:
bus.handle(CHANGED_SIGNAL, self._on_changed)
bus.handle(BEATS_SIGNAL, self._on_beats)
bus.handle(SESSION_COMMAND_SIGNAL, self._on_session_command)
+ bus.handle(WARM_SIGNAL, self._on_warm)
bus.on_resync(self.reload_all)
bus.on_recheck(self.reload_all)
registry.use_rows(self)
@@ -342,6 +346,11 @@ class SharedConnections:
)
self._writes.set()
+ def relay_warm(self, agent_id: str, process_id: str, room_id: str) -> None:
+ self._bus.publish(
+ WARM_SIGNAL, agent_id=agent_id, room_id=room_id, process_id=process_id
+ )
+
# ── Reading the rows into the registry ────────────────────────────────
async def _read_agent(self, session: AsyncSession, agent_id: str) -> AgentSlotRows:
@@ -477,12 +486,24 @@ class SharedConnections:
await session.commit()
self._deliver(relayed)
+ def _on_warm(self, signal: dict[str, Any]) -> None:
+ if signal["process_id"] != self.process_id:
+ return
+ if not self._registry.controllers.relay_warm(
+ signal["agent_id"], signal["room_id"]
+ ):
+ logger.warning(
+ "A warm for agent %s arrived after the controller stream it was "
+ "relayed to went, or the agent was stopped; dropped",
+ signal["agent_id"],
+ )
+
def _publish_changed(
self, tenant_id: str, agent_id: str, released: list[Released]
) -> None:
entries = [[r.connection_id, r.room_id, r.session_id] for r in released]
# Sent with no rooms released too: the change is what is announced.
- for chunk in _chunks(entries) or [[]]:
+ for chunk in payload_chunks(entries, _SIGNAL_ENVELOPE_BYTES) or [[]]:
self._bus.publish(
CHANGED_SIGNAL, tenant_id=tenant_id, agent_id=agent_id, released=chunk
)
@@ -585,7 +606,7 @@ class SharedConnections:
await self._beats_waiting.wait()
self._beats_waiting.clear()
beats, self._beats = list(self._beats.values()), {}
- for chunk in _chunks(beats):
+ for chunk in payload_chunks(beats, _SIGNAL_ENVELOPE_BYTES):
try:
self._bus.publish(BEATS_SIGNAL, beats=chunk)
except ValueError as error:
@@ -628,24 +649,3 @@ class SharedConnections:
raise
except Exception:
logger.exception("Checking the other processes' connections failed")
-
-
-def _chunks(entries: list[list[Any]]) -> list[list[list[Any]]]:
- """Split a signal's entries, forwarded beats or released rooms, into lists
- that each fit in one payload under the `NOTIFY` limit. Measured as JSON,
- which writes a character outside ASCII in up to twelve bytes. An entry
- too large to fit even alone gets a list of its own, which the bus then
- refuses."""
- chunks: list[list[list[Any]]] = []
- current: list[list[Any]] = []
- size = _SIGNAL_ENVELOPE_BYTES
- for entry in entries:
- cost = len(json.dumps(entry)) + 1
- if current and size + cost > MAX_PAYLOAD_BYTES:
- chunks.append(current)
- current, size = [], _SIGNAL_ENVELOPE_BYTES
- current.append(entry)
- size += cost
- if current:
- chunks.append(current)
- return chunks
diff --git a/core/switch_core/bridges/agent/protocol/controller_presence.py b/core/switch_core/bridges/agent/protocol/controller_presence.py
index 372e17d6f..f69b9a938 100644
--- a/core/switch_core/bridges/agent/protocol/controller_presence.py
+++ b/core/switch_core/bridges/agent/protocol/controller_presence.py
@@ -591,8 +591,10 @@ class ControllerPresence:
def relay_warm(self, agent_id: str, room_id: str) -> bool:
"""Ask the agent's controller to start the session a message to
- `room_id` would start. False when its controller is not live or the
- agent is stopped. A warm already owed for the room is not repeated."""
+ `room_id` would start, when this process holds its stream. False when
+ its controller is not live here or the agent is stopped. A warm
+ already owed for the room is not repeated. A controller held by
+ another process is reached through the registry (`process_holding`)."""
binding = self._bindings.get(agent_id)
if binding is None or not binding.running:
return False
diff --git a/core/switch_core/chats/service.py b/core/switch_core/chats/service.py
index 3ab8a75bd..be3b75bf1 100644
--- a/core/switch_core/chats/service.py
+++ b/core/switch_core/chats/service.py
@@ -45,7 +45,7 @@ from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from switch_core.addressing import allows_on_behalf_of, parse_policy
from switch_core.authz import administers_tenant
-from switch_core.bridges.agent.protocol.controller_presence import ControllerPresence
+from switch_core.bridges.agent.protocol.agent_connections import AgentConnectionRegistry
from switch_core.chats import MEMBER_CLIENT_TYPE
from switch_core.chats.views import (
COMMAND_EVENT_TYPE,
@@ -100,6 +100,7 @@ from switch_core.transport.content import media_content, message_content
from switch_core.transport.types import MessageFormat
if TYPE_CHECKING:
+ from switch_core.chats.signals import ChatSignals
from switch_core.delivery.router import MessageRouter, Routed
logger = logging.getLogger(__name__)
@@ -162,10 +163,10 @@ class ChatService:
id_server_name: str,
media_max_bytes: int,
agent_chats_enabled: bool,
- controllers: ControllerPresence,
+ connections: AgentConnectionRegistry,
) -> None:
self.session_factory = session_factory
- self._controllers = controllers
+ self._connections = connections
self.agent_chats_enabled = agent_chats_enabled
self.listener = listener
self.activity_listener = activity_listener
@@ -186,6 +187,9 @@ class ChatService:
weakref.WeakValueDictionary()
)
self._membership_watchers: dict[tuple[str, str], set[Callable[[], None]]] = {}
+ # Where a change to people's chats made here is announced to the
+ # other processes, once wired (`use_signals`).
+ self._signals: ChatSignals | None = None
self._addressing = AddressingResolver(
room_store=room_store,
room_role_store=RoomRoleStore(),
@@ -198,10 +202,16 @@ class ChatService:
# ── Membership signals ───────────────────────────────────────────────────
+ def use_signals(self, signals: ChatSignals) -> None:
+ """Where a change to people's chats made here is announced, so every
+ other process wakes the streams it holds for them (`signals.py`)."""
+ self._signals = signals
+
def watch_memberships(
self, tenant_id: str, user_id: str, callback: Callable[[], None]
) -> None:
- """Call `callback` whenever this process changes the user's chats."""
+ """Call `callback` whenever the user's chats change: at once when this
+ process changed them, and once told when another process did."""
self._membership_watchers.setdefault((tenant_id, user_id), set()).add(callback)
def unwatch_memberships(
@@ -214,13 +224,28 @@ class ChatService:
if not watchers:
del self._membership_watchers[(tenant_id, user_id)]
- def _memberships_changed(self, tenant_id: str, user_ids: list[str]) -> None:
+ def wake_membership_watchers(self, tenant_id: str, user_ids: list[str]) -> None:
+ """Call every callback watching these users' chats in this process."""
for user_id in user_ids:
for callback in list(
self._membership_watchers.get((tenant_id, user_id), ())
):
callback()
+ def wake_all_membership_watchers(self) -> None:
+ """Call every callback watching anyone's chats in this process, once."""
+ for watchers in list(self._membership_watchers.values()):
+ for callback in list(watchers):
+ callback()
+
+ def _memberships_changed(self, tenant_id: str, user_ids: list[str]) -> None:
+ """The users' chats changed through this process, and the change has
+ committed: the streams here are woken at once, and the other
+ processes are told to wake theirs."""
+ self.wake_membership_watchers(tenant_id, user_ids)
+ if self._signals is not None:
+ self._signals.memberships_changed(tenant_id, user_ids)
+
def _lock(self, *key: str) -> asyncio.Lock:
lock = self._locks.get(key)
if lock is None:
@@ -1269,7 +1294,9 @@ class ChatService:
) -> None:
"""Ask the controller of the user's own agent to start its session for
this chat before the first message. A hint: nothing waits for it, and
- an agent no controller hosts is not warmed."""
+ an agent no controller hosts is not warmed. The controller is asked
+ through whichever process holds its socket
+ (`AgentConnectionRegistry.relay_warm`)."""
async with tenant_session(self.session_factory, tenant_id) as session:
room, client = await self.require_member(session, tenant_id, user, room_id)
if agent_id not in await self._room_store.get_agent_ids(session, room.id):
@@ -1287,9 +1314,9 @@ class ChatService:
)
if not decision.allowed:
raise ChatError(403, "AGENT_NOT_ALLOWED", decision.refusal)
- if not self._controllers.is_bound(agent_id):
+ if not self._connections.controllers.is_bound(agent_id):
return
- if not self._controllers.relay_warm(agent_id, room.id):
+ if not self._connections.relay_warm(agent_id, room.id):
logger.warning(
"Could not warm agent %s for room %s: its controller is not connected or it is stopped",
agent_id,
diff --git a/core/switch_core/chats/signals.py b/core/switch_core/chats/signals.py
new file mode 100644
index 000000000..2f5aa9ee4
--- /dev/null
+++ b/core/switch_core/chats/signals.py
@@ -0,0 +1,63 @@
+"""Changes to people's chats, reaching their Console streams on every process.
+
+Switch Console follows a person's chats over one stream (`chat_events` in
+`gateway/chats.py`), held by whichever switch-core process its request
+reached. What changes which chats a person has (a chat created or archived, a
+member added or removed, a room gaining or losing its agents) is served by
+whichever process that request reached, and `ChatService` wakes the streams
+held there at once. A stream held by another process saw the change only at
+its next periodic re-read.
+
+So each such change, once committed, announces `chats.memberships_changed`
+naming the tenant and the people whose chats changed. Every other process
+wakes the streams it holds for them, and each re-reads what its person may
+see. A change concerning more people than one signal can name is split over
+as many signals as it takes.
+
+Signals sent while this process's signal connection was down were never
+queued, so when it comes back every stream held here is woken once: a wake-up
+costs that stream one re-read.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+from switch_core.signals import payload_chunks
+
+if TYPE_CHECKING:
+ from switch_core.chats.service import ChatService
+ from switch_core.signals import SignalBus
+
+MEMBERSHIPS_CHANGED_SIGNAL = "chats.memberships_changed"
+
+# Room kept in every signal for what it carries beside its user ids: its kind,
+# the sending process's id and the tenant id, about 200 bytes at most.
+_SIGNAL_ENVELOPE_BYTES = 400
+
+
+class ChatSignals:
+ def __init__(self, *, service: ChatService, bus: SignalBus) -> None:
+ self._service = service
+ self._bus = bus
+ bus.handle(MEMBERSHIPS_CHANGED_SIGNAL, self._on_changed)
+ bus.on_resync(self._wake_all)
+ # No periodic re-check, unlike the other families: every stream
+ # already re-reads everything on its own every `RECHECK_SECONDS`
+ # (`gateway/chats.py`), which is all a missed signal can cost it.
+ service.use_signals(self)
+
+ def memberships_changed(self, tenant_id: str, user_ids: list[str]) -> None:
+ """The users' chats changed and the change has committed: every other
+ process wakes the streams it holds for them. This one has already
+ woken its own."""
+ for chunk in payload_chunks(user_ids, _SIGNAL_ENVELOPE_BYTES):
+ self._bus.publish(
+ MEMBERSHIPS_CHANGED_SIGNAL, tenant_id=tenant_id, user_ids=chunk
+ )
+
+ def _on_changed(self, signal: dict[str, Any]) -> None:
+ self._service.wake_membership_watchers(signal["tenant_id"], signal["user_ids"])
+
+ async def _wake_all(self) -> None:
+ self._service.wake_all_membership_watchers()
diff --git a/core/switch_core/gateway/chats.py b/core/switch_core/gateway/chats.py
index 7d0edf4a4..0237686ea 100644
--- a/core/switch_core/gateway/chats.py
+++ b/core/switch_core/gateway/chats.py
@@ -644,9 +644,10 @@ async def chat_events(
Catches every member room up from its cursor in `after`, says `ready`,
then pushes as rooms move. A room the caller joined since is announced as
- `chat` and followed from its head; one that leaves the list is `chat.removed`. A
- re-read every `recheck_seconds` recovers anything a missed notification
- or another process's membership change left behind.
+ `chat` and followed from its head; one that leaves the list is `chat.removed`.
+ A change to the caller's chats wakes the stream at once, whichever process
+ made it (`chats/signals.py`). A re-read every `recheck_seconds` recovers
+ anything a missed notification or a missed signal left behind.
"""
stream = _ChatStream(service, tenant_id, user, after)
service.watch_memberships(tenant_id, user.id, stream.on_memberships)
diff --git a/core/switch_core/main.py b/core/switch_core/main.py
index 9ab2280fa..dc97618d4 100644
--- a/core/switch_core/main.py
+++ b/core/switch_core/main.py
@@ -94,6 +94,7 @@ from switch_core.bridges.collaboration.webhooks import (
)
from switch_core.bridges.resource.service import ResourceService
from switch_core.chats.service import ChatService
+from switch_core.chats.signals import ChatSignals
from switch_core.clients.actor import Actor, AgentActor, HumanActor, SystemActor
from switch_core.clients.client_factory import ClientFactory
from switch_core.clients.client_lifecycle_service import ClientLifecycleService
@@ -1107,8 +1108,11 @@ async def run(config: SwitchConfig, process_id: str) -> None:
id_server_name=config.id_server_name,
media_max_bytes=config.agent_media_max_bytes,
agent_chats_enabled=config.agent_chats_enabled,
- controllers=connections.controllers,
+ connections=connections,
)
+ # Changes to people's chats made in another process wake the Console
+ # streams held here too (`chats/signals.py`).
+ ChatSignals(service=chat_service, bus=signal_bus)
room_service.on_room_agents_changed(chat_service.sync_room)
protocol.set_room_agents_listener(room_service.room_agents_changed)
diff --git a/core/switch_core/signals.py b/core/switch_core/signals.py
index c86be35f0..5eda31b80 100644
--- a/core/switch_core/signals.py
+++ b/core/switch_core/signals.py
@@ -416,3 +416,24 @@ class SignalBus:
handler(signal)
except Exception:
logger.error("Handling signal %r failed", signal["kind"], exc_info=True)
+
+
+def payload_chunks[T](entries: list[T], envelope_bytes: int) -> list[list[T]]:
+ """Split the list a signal carries into lists that each fit in one payload
+ under the `NOTIFY` limit, beside the `envelope_bytes` kept for the
+ signal's other fields. Measured as JSON, which writes a character outside
+ ASCII in up to twelve bytes. An entry too large to fit even alone gets a
+ list of its own, which `SignalBus.publish` then refuses."""
+ chunks: list[list[T]] = []
+ current: list[T] = []
+ size = envelope_bytes
+ for entry in entries:
+ cost = len(json.dumps(entry)) + 1
+ if current and size + cost > MAX_PAYLOAD_BYTES:
+ chunks.append(current)
+ current, size = [], envelope_bytes
+ current.append(entry)
+ size += cost
+ if current:
+ chunks.append(current)
+ return chunks
diff --git a/core/tests/switch_core/gateway/conftest.py b/core/tests/switch_core/gateway/conftest.py
index bcc5f0342..3c984d5a3 100644
--- a/core/tests/switch_core/gateway/conftest.py
+++ b/core/tests/switch_core/gateway/conftest.py
@@ -1,9 +1,12 @@
"""The `/chats` harness: real routes, `ChatService`, `RoomService` and
-Postgres provisioning, shared by the chats test modules."""
+Postgres provisioning, shared by the chats test modules, and a second
+switch-core process beside it for the tests across processes."""
from __future__ import annotations
+import asyncio
from collections.abc import AsyncIterator, Awaitable, Callable
+from dataclasses import dataclass
from types import SimpleNamespace
from typing import Any
from unittest.mock import MagicMock
@@ -12,21 +15,27 @@ import httpx
import pytest
from fastapi import FastAPI, Request
from sqlalchemy import select
-from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker, create_async_engine
+from sqlalchemy.pool import NullPool
+from switch_core.bridges.agent.protocol.agent_connections import AgentConnectionRegistry
+from switch_core.bridges.agent.protocol.connection_rows import SharedConnections
from switch_core.chats.service import ChatError, ChatService
+from switch_core.chats.signals import ChatSignals
from switch_core.db.models import (
Agent,
Client,
Message,
User,
)
+from switch_core.db.stores.agent_connection_store import AgentConnectionStore
from switch_core.db.stores.agent_store import AgentStore
from switch_core.db.stores.client_store import ClientStore
from switch_core.db.stores.collaboration_bridge_store import CollaborationBridgeStore
from switch_core.db.stores.media_store import MediaStore
from switch_core.db.stores.message_store import MessageStore
from switch_core.db.stores.room_store import RoomStore
+from switch_core.db.stores.switch_core_process_store import SwitchCoreProcessStore
from switch_core.db.stores.usage_store import UsageStore
from switch_core.db.stores.user_store import UserStore
from switch_core.gateway.auth import get_current_user
@@ -34,15 +43,22 @@ from switch_core.gateway.chats import (
chat_error_response,
router,
)
+from switch_core.process_identity import new_process_id
+from switch_core.process_lease import ProcessLease
from switch_core.provisioning.postgres import PostgresProvisioning
from switch_core.room_service import RoomService
from switch_core.session_activity.listener import Change
+from switch_core.signals import SignalBus
from switch_core.transport.invites import InviteBus
+from tests.conftest import TEST_KEYRING
from tests.routing import router_on
from tests.switch_core.gateway.agent_route_harness import add_agent
_MESSAGE_STORE = MessageStore()
+# How long `two_processes` waits for each process's signal bus to connect.
+_CONNECT_SECONDS = 10.0
+
class _FakeLifecycle:
"""The running-client registry, for agents that have no consumer here."""
@@ -96,24 +112,6 @@ class _FakeActivityListener:
await on_resync()
-class _FakeControllers:
- """The controller presence, as far as warming a chat reads it."""
-
- def __init__(self) -> None:
- self.bound: set[str] = set()
- self.live = True
- self.warms: list[tuple[str, str]] = []
-
- def is_bound(self, agent_id: str) -> bool:
- return agent_id in self.bound
-
- def relay_warm(self, agent_id: str, room_id: str) -> bool:
- if not self.live:
- return False
- self.warms.append((agent_id, room_id))
- return True
-
-
class _Harness(SimpleNamespace):
client: httpx.AsyncClient
factory: async_sessionmaker[AsyncSession]
@@ -122,7 +120,7 @@ class _Harness(SimpleNamespace):
listener: _FakeListener
activity: _FakeActivityListener
lifecycle: _FakeLifecycle
- controllers: _FakeControllers
+ connections: AgentConnectionRegistry
alice: User
bob: User
carol: User
@@ -211,19 +209,17 @@ async def _user(session: AsyncSession, name: str, role: str | None) -> User:
return user
-async def chats(
+def _chat_service(
session_factory: async_sessionmaker[AsyncSession],
-) -> AsyncIterator[_Harness]:
- async with session_factory() as session:
- alice = await _user(session, "alice", "member")
- bob = await _user(session, "bob", "member")
- carol = await _user(session, "carol", None)
- admin = await _user(session, "admin", "admin")
- await session.commit()
-
+ *,
+ lifecycle: _FakeLifecycle,
+ listener: _FakeListener,
+ activity: _FakeActivityListener,
+ connections: AgentConnectionRegistry,
+) -> tuple[ChatService, RoomService]:
+ """A chat service and the room service it works through, built as one
+ switch-core process builds them."""
room_store = RoomStore()
- lifecycle = _FakeLifecycle()
message_router = router_on(session_factory)
provisioning = PostgresProvisioning(
session_factory=session_factory,
@@ -243,9 +239,6 @@ async def chats(
session_factory=session_factory,
room_cache=MagicMock(),
)
- listener = _FakeListener()
- activity = _FakeActivityListener()
- controllers = _FakeControllers()
service = ChatService(
session_factory=session_factory,
room_service=room_service,
@@ -262,9 +255,34 @@ async def chats(
id_server_name="switch.test",
media_max_bytes=1024,
agent_chats_enabled=True,
- controllers=controllers, # type: ignore[arg-type]
+ connections=connections,
)
room_service.on_room_agents_changed(service.sync_room)
+ return service, room_service
+
+
+async def chats(
+ session_factory: async_sessionmaker[AsyncSession],
+) -> AsyncIterator[_Harness]:
+ async with session_factory() as session:
+ alice = await _user(session, "alice", "member")
+ bob = await _user(session, "bob", "member")
+ carol = await _user(session, "carol", None)
+ admin = await _user(session, "admin", "admin")
+ await session.commit()
+
+ lifecycle = _FakeLifecycle()
+ listener = _FakeListener()
+ activity = _FakeActivityListener()
+ connections = AgentConnectionRegistry()
+ service, room_service = _chat_service(
+ session_factory,
+ lifecycle=lifecycle,
+ listener=listener,
+ activity=activity,
+ connections=connections,
+ )
app = FastAPI()
app.state.chat_service = service
@@ -289,9 +307,102 @@ async def chats(
listener=listener,
activity=activity,
lifecycle=lifecycle,
- controllers=controllers,
+ connections=connections,
alice=alice,
bob=bob,
carol=carol,
admin=admin,
)
+
+
+@dataclass
+class _Process:
+ """One switch-core process's side of chats on the shared database: its
+ chat service and agent connections, and the signal bus over which it
+ tells the other process what changed."""
+
+ service: ChatService
+ connections: AgentConnectionRegistry
+ signals: ChatSignals
+ shared: SharedConnections
+ bus: SignalBus
+ lease: ProcessLease
+
+
+def _process(
+ session_factory: async_sessionmaker[AsyncSession],
+ postgres_url: str,
+ service: ChatService,
+ connections: AgentConnectionRegistry,
+) -> _Process:
+ process_id = new_process_id("test-pod")
+ bus = SignalBus(
+ session_factory,
+ lambda: create_async_engine(postgres_url, poolclass=NullPool),
+ process_id=process_id,
+ )
+ lease = ProcessLease(
+ process_id=process_id,
+ session_factory=session_factory,
+ processes=SwitchCoreProcessStore(),
+ key_ids=[key.id for key in TEST_KEYRING.keys],
+ )
+ return _Process(
+ service=service,
+ connections=connections,
+ signals=ChatSignals(service=service, bus=bus),
+ shared=SharedConnections(
+ registry=connections,
+ session_factory=session_factory,
+ store=AgentConnectionStore(),
+ bus=bus,
+ lease=lease,
+ ),
+ bus=bus,
+ lease=lease,
+ )
+
+
+async def two_processes(
+ chats: _Harness, postgres_url: str
+) -> AsyncIterator[tuple[_Process, _Process]]:
+ """The harness's chat service as one switch-core process, and a second on
+ the same database. Each tells the other what changed over its own signal
+ bus on the real `NOTIFY` channel: people's chats (`ChatSignals`) and its
+ agent connections (`SharedConnections`)."""
+ connections = AgentConnectionRegistry()
+ service, _ = _chat_service(
+ chats.factory,
+ lifecycle=chats.lifecycle,
+ listener=_FakeListener(),
+ activity=_FakeActivityListener(),
+ connections=connections,
+ )
+ processes = [
+ _process(chats.factory, postgres_url, chats.service, chats.connections),
+ _process(chats.factory, postgres_url, service, connections),
+ ]
+ for process in processes:
+ await process.lease.renew()
+ await process.shared.start()
+ await process.bus.start()
+ # Each lease is renewed while the process runs, as in production, so the
+ # other reads what it holds as held however long the test takes.
+ runs = [
+ asyncio.create_task(run())
+ for process in processes
+ for run in (process.lease.run, process.shared.run)
+ ]
+ try:
+ for process in processes:
+ await asyncio.wait_for(process.bus.connected.wait(), _CONNECT_SECONDS)
+ yield processes[0], processes[1]
+ finally:
+ # The buses first, so no signal starts work after the processes
+ # stopped running; then the runs, which wait for the work they started.
+ for process in processes:
+ await process.bus.stop()
+ for run in runs:
+ run.cancel()
+ await asyncio.gather(*runs, return_exceptions=True)
diff --git a/core/tests/switch_core/gateway/test_chats_routes.py b/core/tests/switch_core/gateway/test_chats_routes.py
index 3c78dc03e..c3a559394 100644
--- a/core/tests/switch_core/gateway/test_chats_routes.py
+++ b/core/tests/switch_core/gateway/test_chats_routes.py
@@ -4,7 +4,9 @@ sends, and the live event stream — against real Postgres.
The routes run behind a real `ChatService`, `RoomService` and Postgres
provisioning. Only the running-client registry is faked (no agent consumer
runs here) and the message listener, whose wake-ups the stream tests ring by
-hand so a missed notification can be staged.
+hand so a missed notification can be staged. The stream tests across
+processes add a second chat service on the same database, each process with
+its own signal bus on the real `NOTIFY` channel.
"""
from __future__ import annotations
@@ -12,7 +14,7 @@ from __future__ import annotations
import asyncio
import json
import uuid
-from collections.abc import AsyncIterator
+from collections.abc import AsyncIterator, Callable
from datetime import UTC, datetime, timedelta
from typing import Any
@@ -46,7 +48,7 @@ from switch_core.gateway.chats import (
chat_events,
)
from switch_core.session_activity.listener import Change
-from tests.switch_core.gateway.conftest import _Harness
+from tests.switch_core.gateway.conftest import _Harness, _Process
def _code(response: httpx.Response) -> str:
@@ -981,7 +983,8 @@ async def test_the_periodic_recheck_announces_a_chat_that_regains_an_agent(
)
assert (await events.next())[0] == "ready"
- # Written straight to the table, as another process would: no hook runs.
+ # Written straight to the table: no hook runs and no signal is sent, as
+ # when another process's signal is missed.
async with chats.factory() as session:
await session.execute(
room_agents.insert().values(room_id=room_id, agent_id=agent.id)
@@ -1051,6 +1054,95 @@ async def test_the_periodic_recheck_recovers_a_missed_notification(
await events.stream.aclose() # type: ignore[attr-defined]
+def _stream_on(process: _Process, user: User) -> _Events:
+ """A stream held by `process` that re-reads on its own only after an hour,
+ so whatever it says before then, something woke it to say."""
+ return _Events(
+ chat_events(
+ process.service,
+ TENANT,
+ user,
+ {},
+ ping_seconds=60.0,
+ recheck_seconds=3600.0,
+ )
+ )
+
+
+async def _eventually(check: Callable[[], bool]) -> None:
+ async with asyncio.timeout(10.0):
+ while not check():
+ await asyncio.sleep(0.02)
+
+
+async def _join_unannounced(chats: _Harness, user: User, room_id: str) -> None:
+ """Put the user in the room without waking any stream or sending any
+ signal, as a change whose signal was lost."""
+ async with chats.factory() as session:
+ room = await session.get(Room, room_id)
+ assert room is not None
+ await chats.service._join(
+ TENANT, room, await chats.service.member_actor(TENANT, user)
+ )
+
+
+async def test_a_change_through_another_process_reaches_the_stream_at_once(
+ chats: _Harness, two_processes: tuple[_Process, _Process]
+) -> None:
+ """Bob's Console streams from one process while Alice invites him through
+ the other: his stream announces the chat at once, not at its re-read."""
+ _, there = two_processes
+ agent = await chats.agent("helper", owner=chats.alice)
+ room_id = await chats.new_chat(chats.alice, agent, "c1")
+ events = _stream_on(there, chats.bob)
+ assert (await events.next())[0] == "ready"
+
+ assert (await chats.invite(chats.alice, room_id, chats.bob)).status_code == 200
+
+ name, summary = await events.next()
+ assert name == "chat" and summary["roomId"] == room_id
+ await events.stream.aclose() # type: ignore[attr-defined]
+
+
+async def test_every_stream_re_reads_when_its_signal_connection_comes_back(
+ chats: _Harness, two_processes: tuple[_Process, _Process]
+) -> None:
+ """A signal sent while a process's signal connection is down never reaches
+ it, so once the connection is back every stream it holds re-reads."""
+ _, there = two_processes
+ agent = await chats.agent("helper", owner=chats.alice)
+ room_id = await chats.new_chat(chats.alice, agent, "c1")
+ events = _stream_on(there, chats.bob)
+ assert (await events.next())[0] == "ready"
+
+ await there.bus.stop()
+ await _join_unannounced(chats, chats.bob, room_id)
+ await there.bus.start()
+
+ name, summary = await events.next()
+ assert name == "chat" and summary["roomId"] == room_id
+ await events.stream.aclose() # type: ignore[attr-defined]
+
+
+async def test_a_change_concerning_more_people_than_one_signal_holds_reaches_them_all(
+ two_processes: tuple[_Process, _Process],
+) -> None:
+ """The people a change concerns are split over as many signals as it takes
+ to stay under the `NOTIFY` limit, and each one's streams are woken."""
+ here, there = two_processes
+ user_ids = [str(uuid.uuid4()) for _ in range(500)]
+ woken: list[str] = []
+ for user_id in user_ids:
+ there.service.watch_memberships(
+ TENANT, user_id, lambda user_id=user_id: woken.append(user_id)
+ )
+
+ here.signals.memberships_changed(TENANT, user_ids)
+
+ await _eventually(lambda: len(woken) == len(user_ids))
+ assert sorted(woken) == sorted(user_ids)
+
+
# ── Agent owners ─────────────────────────────────────────────────────────────
TENANT = "00000000-0000-0000-0000-000000000000"
@@ -1323,7 +1415,8 @@ async def test_the_periodic_recheck_finds_a_chat_the_agent_joined_elsewhere(
events = _stream(chats, chats.alice, {}, recheck=0.2)
assert (await events.next())[0] == "ready"
- # Written straight to the table, as another process would: no hook runs.
+ # Written straight to the table: no hook runs and no signal is sent, as
+ # when another process's signal is missed.
async with chats.factory() as session:
await session.execute(
room_agents.insert().values(room_id=room_id, agent_id=agent.id)
diff --git a/core/tests/switch_core/gateway/test_chats_warm.py b/core/tests/switch_core/gateway/test_chats_warm.py
index c90c28741..383ee3c6b 100644
--- a/core/tests/switch_core/gateway/test_chats_warm.py
+++ b/core/tests/switch_core/gateway/test_chats_warm.py
@@ -3,12 +3,22 @@ user's own agent to start its session while the user types."""
from __future__ import annotations
+import asyncio
+from collections.abc import Callable
+
import httpx
-from switch_core.db.models import Agent, User
-from tests.switch_core.gateway.conftest import _Harness
+from switch_core.bridges.agent.protocol.agent_connections import AgentConnectionRegistry
+from switch_core.bridges.agent.protocol.controller_presence import (
+ Binding,
+ ControllerConnection,
+)
+from switch_core.db.models import TENANT_ZERO_ID, Agent, User
+from tests.switch_core.gateway.conftest import _Harness, _Process
from tests.switch_core.gateway.test_chats_routes import _code
+CONTROLLER = "controller-1"
+
async def _warm(
chats: _Harness, user: User, room_id: str, agent: Agent
@@ -20,15 +30,48 @@ async def _warm(
)
+def _bind(connections: AgentConnectionRegistry, agent: Agent) -> None:
+ """Bind the agent to the controller, as Management does in every process."""
+ connections.controllers.bind(
+ Binding(
+ agent_id=agent.id,
+ controller_id=CONTROLLER,
+ tenant_id=TENANT_ZERO_ID,
+ controller_name="the machine",
+ running=True,
+ )
+ )
+
+
+def _on_controller(
+ connections: AgentConnectionRegistry, agent: Agent
+) -> ControllerConnection:
+ """Put the agent on a controller whose live stream this registry's process
+ holds, and return the controller's connection."""
+ _bind(connections, agent)
+ controllers = connections.controllers
+ conn = controllers.open(
+ controller_id=CONTROLLER, tenant_id=TENANT_ZERO_ID, resume_cursors={}
+ )
+ controllers.attach_stream(conn)
+ return conn
+
+
+async def _eventually(check: Callable[[], bool]) -> None:
+ async with asyncio.timeout(10.0):
+ while not check():
+ await asyncio.sleep(0.02)
+
+
async def test_a_warm_reaches_the_agents_controller(chats: _Harness) -> None:
agent = await chats.agent("helper", owner=chats.alice)
- chats.controllers.bound.add(agent.id)
+ controller = _on_controller(chats.connections, agent)
room_id = await chats.new_chat(chats.alice, agent, "c1")
response = await _warm(chats, chats.alice, room_id, agent)
assert response.status_code == 202, response.text
- assert chats.controllers.warms == [(agent.id, room_id)]
+ assert controller.warms == {(agent.id, room_id)}
async def test_a_warm_is_accepted_when_no_controller_takes_it(
@@ -38,19 +81,19 @@ async def test_a_warm_is_accepted_when_no_controller_takes_it(
room_id = await chats.new_chat(chats.alice, agent, "c1")
unbound = await _warm(chats, chats.alice, room_id, agent)
- chats.controllers.bound.add(agent.id)
- chats.controllers.live = False
+ controller = _on_controller(chats.connections, agent)
+ chats.connections.controllers.detach_stream(controller, controller.stream_token)
offline = await _warm(chats, chats.alice, room_id, agent)
assert unbound.status_code == offline.status_code == 202
- assert chats.controllers.warms == []
+ assert controller.warms == set()
async def test_a_warm_refuses_non_members_others_agents_and_absent_agents(
chats: _Harness,
) -> None:
agent = await chats.agent("helper", owner=chats.alice)
- chats.controllers.bound.add(agent.id)
+ controller = _on_controller(chats.connections, agent)
room_id = await chats.new_chat(chats.alice, agent, "c1")
outsider = await _warm(chats, chats.bob, room_id, agent)
@@ -67,14 +110,14 @@ async def test_a_warm_refuses_non_members_others_agents_and_absent_agents(
assert absent.status_code == 422
assert _code(absent) == "AGENT_NOT_IN_CHAT"
- assert chats.controllers.warms == []
+ assert controller.warms == set()
async def test_a_warm_refuses_an_owner_the_agents_policy_refuses(
chats: _Harness,
) -> None:
agent = await chats.agent("helper", owner=chats.alice)
- chats.controllers.bound.add(agent.id)
+ controller = _on_controller(chats.connections, agent)
room_id = await chats.new_chat(chats.alice, agent, "c1")
async with chats.factory() as session:
row = await session.get(Agent, agent.id)
@@ -86,4 +129,24 @@ async def test_a_warm_refuses_an_owner_the_agents_policy_refuses(
assert refused.status_code == 403
assert _code(refused) == "AGENT_NOT_ALLOWED"
- assert chats.controllers.warms == []
+ assert controller.warms == set()
+
+
+async def test_a_warm_reaches_a_controller_another_process_holds(
+ chats: _Harness, two_processes: tuple[_Process, _Process]
+) -> None:
+ """The Console's request reaches a process that does not hold the
+ controller's socket, which hands the warm to the process that does."""
+ here, there = two_processes
+ agent = await chats.agent("helper", owner=chats.alice)
+ room_id = await chats.new_chat(chats.alice, agent, "c1")
+ controller = _on_controller(there.connections, agent)
+ # What Management keeps in every process from the controller's rows: the
+ # agent's binding, and which process holds the controller's socket.
+ _bind(here.connections, agent)
+ here.connections.controllers.held_elsewhere(CONTROLLER, there.bus.process_id)
+
+ response = await _warm(chats, chats.alice, room_id, agent)
+
+ assert response.status_code == 202, response.text
+ await _eventually(lambda: controller.warms == {(agent.id, room_id)})
diff --git a/core/tests/switch_core/management/test_controllers_across_processes.py b/core/tests/switch_core/management/test_controllers_across_processes.py
index 3b0a9d375..6a940c318 100644
--- a/core/tests/switch_core/management/test_controllers_across_processes.py
+++ b/core/tests/switch_core/management/test_controllers_across_processes.py
@@ -277,6 +277,27 @@ class TestPresence:
assert conn is not None
await _eventually(lambda: conn.session_commands == [(agent_id, frame)])
+ async def test_a_warm_reaches_the_controller_on_the_other_process(
+ self, processes: tuple[Harness, Harness]
+ ) -> None:
+ here, there = processes
+ owner = await add_member(here.session_factory, "ada")
+ async with here.client() as client:
+ controller = await enroll_console(here, client, owner)
+ agent_id = await place_agent(client, controller, name="reviewer")
+ await connect(client, controller)
+ await _eventually(
+ lambda: there.protocol.connections.controllers.is_live(agent_id)
+ )
+
+ assert there.protocol.connections.relay_warm(agent_id, "room")
+
+ conn = here.protocol.connections.controllers.current_connection(
+ controller.controller_id
+ )
+ assert conn is not None
+ await _eventually(lambda: conn.warms == {(agent_id, "room")})
+
class TestTheConnectionOnAnyProcess:
async def test_the_socket_attaches_through_the_process_that_did_not_open_it(