Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Read only the clients and agents a change names, so creating agents stops pinning every process (CHOO-3667)

Abel Dantasc6c247def7f4
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8b9791612..76013f638 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -224,6 +224,13 @@ version of their own to them without also giving them a release of their own.
group whose subgroup would move up next to a namesake is refused too, and
the dashboard now shows why. Names can no longer contain `/` or be blank.
Existing duplicates are left as they are and can be reached by id.
+- **Registering, renaming or deleting an agent no longer makes every process
+ read every client and agent.** The change now names the clients it touched,
+ and each process reads only those, where it read every client of the
+ workspace and, with a messaging app connected, every agent as well. The
+ 30-second re-check reads every client only when a fingerprint of the
+ workspace's clients and agents has changed since its last full read, where
+ it read them all every time.
#### Fixed
- **On Slack, the 👀 and the activity card no longer arrive after the agent's
diff --git a/core/switch_core/bridges/agent/protocol/agent_core.py b/core/switch_core/bridges/agent/protocol/agent_core.py
index 4ca938067..cb5979264 100644
--- a/core/switch_core/bridges/agent/protocol/agent_core.py
+++ b/core/switch_core/bridges/agent/protocol/agent_core.py
@@ -700,7 +700,7 @@ class AgentCore:
await session.commit()
self.client_lifecycle.start_client(client_record)
- self.client_lifecycle.announce(client_record.tenant_id)
+ self.client_lifecycle.announce(client_record.tenant_id, [client_record.id])
return agent.id
async def _reregister_agent(
@@ -1100,9 +1100,9 @@ class AgentCore:
# the next message that already matched its old name.
if isinstance(actor, AgentActor):
actor.set_agent(agent)
- # Every other process re-reads the tenant's clients, so its copy of the
+ # Every other process re-reads the agent's client, so its copy of the
# agent and the bridges it holds take the new name too.
- self.client_lifecycle.announce(tenant_id)
+ self.client_lifecycle.announce(tenant_id, [agent.client_id])
if self._agent_rename_listener is not None:
await self._agent_rename_listener(tenant_id, agent_id, old_name)
await self._agent_profile_changed(agent_id)
@@ -1404,7 +1404,7 @@ class AgentCore:
# its client must not stay stopped here.
self.client_lifecycle.deletion_failed(client_id)
raise
- self.client_lifecycle.announce(tenant_id)
+ self.client_lifecycle.announce(tenant_id, [client_id])
self.api_key_cache.invalidate_agent(resolved_id)
# Its connections end with it, here and in whichever process holds
# one: their rows went with its row. After its key is dropped, so a
diff --git a/core/switch_core/bridges/collaboration/collaboration_core.py b/core/switch_core/bridges/collaboration/collaboration_core.py
index 8093ef325..7d33e16bd 100644
--- a/core/switch_core/bridges/collaboration/collaboration_core.py
+++ b/core/switch_core/bridges/collaboration/collaboration_core.py
@@ -7,7 +7,7 @@ import logging
import re
import time
import uuid
-from collections.abc import Awaitable, Callable, Iterator, Mapping
+from collections.abc import Awaitable, Callable, Collection, Iterator, Mapping
from contextlib import contextmanager
from dataclasses import dataclass, field
from datetime import timedelta
@@ -320,10 +320,18 @@ class CollaborationCore:
# Identity provisioning runs in the background — see _create_agent_identities.
self._identity_task: asyncio.Task[None] | None = None
# Agents given an identity by this copy of the bridge, so one deleted
- # by another process is let go of here too; and whether the agents
- # changed while a provisioning run was going.
+ # by another process is let go of here too.
self._provisioned_agents: dict[str, AgentIdentity] = {}
+ # The tenant's agents as this copy last read them, by agent id, and
+ # which agent each client is, kept up to date by reading only the
+ # agents a change names (`_update_agent_identities`). None until a
+ # run has read them all.
+ self._agent_view: dict[str, AgentIdentity] | None = None
+ self._agent_of_client: dict[str, str] = {}
+ # Whether every agent has to be read again, and the clients whose
+ # agents do, for the provisioning run going or the next one.
self._identities_stale = False
+ self._identities_owed: set[str] = set()
self._channel_type_refresh_task: asyncio.Task[None] | None = None
# Turns and request cards from the tables the host reports to, pushed
# as they change. Only where the platform draws session activity.
@@ -552,14 +560,18 @@ class CollaborationCore:
"""Whether this process holds the bridge and listens for it."""
return self._active
- def agents_changed(self) -> None:
- """Agents were registered or deleted in this bridge's tenant, maybe by
- another process: give the new ones their platform identities here and
- let go of the deleted ones'. A run already going reads the agents
- again once it is done, in case it read them before the change."""
- self._identities_stale = True
+ def agents_changed(self, client_ids: Collection[str] | None = None) -> None:
+ """Agents were registered, renamed or deleted in this bridge's tenant,
+ maybe by another process: give the new ones their platform identities
+ here and let go of the deleted ones'. Only the agents of `client_ids`
+ are read again, or every agent when None. A run already going reads
+ them once it is done, in case it read them before the change."""
+ if client_ids is None or self._agent_view is None:
+ self._identities_stale = True
+ else:
+ self._identities_owed.update(client_ids)
if not self._awaiting_shared_connection():
- self._provision_identities_on_attach()
+ self._start_identity_run()
def _awaiting_shared_connection(self) -> bool:
"""Whether the bridge waits to be attached to its platform's shared
@@ -682,6 +694,10 @@ class CollaborationCore:
against the connection as it is by then; one that has finished is
replaced.
"""
+ self._identities_stale = True
+ self._start_identity_run()
+
+ def _start_identity_run(self) -> None:
if self._identity_task is not None and not self._identity_task.done():
return
self._identity_task = asyncio.create_task(self._run_agent_identities())
@@ -702,9 +718,19 @@ class CollaborationCore:
async def _run_agent_identities_unbound(self) -> None:
try:
while True:
- self._identities_stale = False
- await self._create_agent_identities()
- if not self._identities_stale:
+ if self._identities_stale:
+ self._identities_stale = False
+ self._identities_owed = set()
+ await self._create_agent_identities()
+ elif self._identities_owed:
+ owed, self._identities_owed = self._identities_owed, set()
+ try:
+ await self._update_agent_identities(owed)
+ except BaseException:
+ # Read again by the next run, whatever starts it.
+ self._identities_owed |= owed
+ raise
+ else:
break
except asyncio.CancelledError:
logger.info(
@@ -722,12 +748,54 @@ class CollaborationCore:
self._session_factory, self._bridge_tenant_id
) as session:
agents = await self._agent_store.get_all(session)
+ current = {agent.id: agent_identity(agent) for agent in agents}
+ self._agent_view = current
+ self._agent_of_client = {agent.client_id: agent.id for agent in agents}
+ await self._sync_agent_identities(current, sorted(self._provisioned_agents))
+
+ async def _update_agent_identities(self, client_ids: set[str]) -> None:
+ """`_create_agent_identities` for the agents of `client_ids` only:
+ their rows are read and the rest of the tenant's agents are taken as
+ this copy last read them, which with thousands of agents is the
+ difference between reading one row per change and every row."""
+ if self._agent_view is None:
+ await self._create_agent_identities()
+ return
+ async with tenant_session(
+ self._session_factory, self._bridge_tenant_id
+ ) as session:
+ agents = await self._agent_store.get_by_client_ids(
+ session, sorted(client_ids)
+ )
+ found = {agent.client_id: agent for agent in agents}
+ view = self._agent_view
+ changed: set[str] = set()
+ for client_id in client_ids:
+ agent = found.get(client_id)
+ known = self._agent_of_client.pop(client_id, None)
+ if known is not None:
+ view.pop(known, None)
+ changed.add(known)
+ if agent is not None:
+ self._agent_of_client[client_id] = agent.id
+ view[agent.id] = agent_identity(agent)
+ changed.add(agent.id)
+ await self._sync_agent_identities(
+ view, sorted(changed & self._provisioned_agents.keys())
+ )
+ async def _sync_agent_identities(
+ self, current: dict[str, AgentIdentity], compared: list[str]
+ ) -> None:
+ """Bring the platform in line with `current`, the tenant's agents:
+ the agents of `compared` this copy gave an identity to are renamed,
+ relabelled or let go of to match it, then every agent's identity is
+ provisioned."""
# Deleted, renamed or relabelled since the last run here, wherever that
# was served: a change served by another process reaches this copy of
- # the bridge only through this re-read.
- current = {agent.id: agent_identity(agent) for agent in agents}
- for agent_id, before in sorted(self._provisioned_agents.items()):
+ # the bridge only through a re-read.
+ for agent_id in compared:
+ before = self._provisioned_agents[agent_id]
after = current.get(agent_id)
try:
if after is None:
@@ -767,14 +835,14 @@ class CollaborationCore:
logger.warning(
"Provisioned %s identities for %d agents; %d failed",
self._bridge_type,
- len(agents),
+ len(current),
len(failures),
)
else:
logger.info(
"Provisioned %s identities for %d agents",
self._bridge_type,
- len(agents),
+ len(current),
)
async def _ensure_channel_captures(self) -> None:
diff --git a/core/switch_core/bridges/collaboration/lifecycle_service.py b/core/switch_core/bridges/collaboration/lifecycle_service.py
index 85858daaf..366827b44 100644
--- a/core/switch_core/bridges/collaboration/lifecycle_service.py
+++ b/core/switch_core/bridges/collaboration/lifecycle_service.py
@@ -1890,12 +1890,15 @@ class CollaborationBridgeLifecycleService:
exc_info=True,
)
- def agents_changed(self, tenant_id: str) -> None:
- """Agents were registered or deleted in `tenant_id`, maybe by another
- process: every bridge of the tenant running here provisions their
- platform identities again."""
+ def agents_changed(
+ self, tenant_id: str, client_ids: frozenset[str] | None = None
+ ) -> None:
+ """Agents were registered, renamed or deleted in `tenant_id`, maybe by
+ another process: every bridge of the tenant running here provisions
+ the platform identities of the agents of `client_ids` again, or of
+ every agent when None."""
for bridge in self.bridges_for_tenant(tenant_id):
- bridge.agents_changed()
+ bridge.agents_changed(client_ids)
async def remove(self, bridge_id: str) -> None:
"""Disconnect a messaging app and take its identities with it.
diff --git a/core/switch_core/clients/client_lifecycle_service.py b/core/switch_core/clients/client_lifecycle_service.py
index ae8ff56c9..405bbad1b 100644
--- a/core/switch_core/clients/client_lifecycle_service.py
+++ b/core/switch_core/clients/client_lifecycle_service.py
@@ -15,7 +15,7 @@ import asyncio
import logging
import random
import uuid
-from collections.abc import Callable
+from collections.abc import Callable, Collection
from typing import TYPE_CHECKING, Any, Protocol, runtime_checkable
from sqlalchemy.exc import InterfaceError, OperationalError
@@ -30,7 +30,7 @@ from switch_core.config import SwitchConfig
from switch_core.db.models import Agent, Client, Tenant
from switch_core.db.session_scope import tenant_session
from switch_core.db.stores.agent_store import AgentStore
-from switch_core.db.stores.client_store import ClientStore
+from switch_core.db.stores.client_store import ClientStore, Fingerprint
from switch_core.db.stores.delivery_cursor_store import DeliveryCursorStore
from switch_core.db.stores.tenant_store import TenantStore
from switch_core.db.tenant_lookup import all_tenant_ids
@@ -47,6 +47,10 @@ if TYPE_CHECKING:
# run it (`ownership.py`), keyed by client id.
CLIENT_CONSUMERS = "client"
+# Told a tenant and the clients whose agents may have changed, or None when
+# any of the tenant's agents may have.
+AgentsChangedListener = Callable[[str, frozenset[str] | None], None]
+
logger = logging.getLogger(__name__)
# A client that fails on the database or the network is retried rather than
@@ -135,19 +139,26 @@ class ClientLifecycleService:
start=self._start_owned_consumer,
stop=self._stop_owned_consumer,
)
- # Told the tenant whenever a re-read starts or stops an agent client
- # here, which is how an agent registered or deleted in another process
+ # Told the tenant, and the clients whose agents to read again or None
+ # for all of them, whenever a re-read may have changed an agent here,
+ # which is how an agent registered or deleted in another process
# reaches what keeps per-agent state of its own (a bridge's platform
# identities, `CollaborationBridgeLifecycleService.agents_changed`).
- self._agents_changed_listeners: list[Callable[[str], None]] = []
+ self._agents_changed_listeners: list[AgentsChangedListener] = []
+ # Each tenant's fingerprint as its last full read found it
+ # (`reconcile_tenant`), which the periodic re-check compares with the
+ # table's before reading every client again (`recheck_all`).
+ self._fingerprints: dict[str, Fingerprint] = {}
- def add_agents_changed_listener(self, listener: Callable[[str], None]) -> None:
+ def add_agents_changed_listener(self, listener: AgentsChangedListener) -> None:
self._agents_changed_listeners.append(listener)
- def _agents_changed(self, tenant_id: str) -> None:
+ def _agents_changed(
+ self, tenant_id: str, client_ids: frozenset[str] | None
+ ) -> None:
for listener in self._agents_changed_listeners:
try:
- listener(tenant_id)
+ listener(tenant_id, client_ids)
except Exception:
logger.exception(
"Telling a listener that tenant %s's agents changed failed",
@@ -159,12 +170,15 @@ class ClientLifecycleService:
other process starts or stops its own copy (`client_signals.py`)."""
self._signals = signals
- def announce(self, tenant_id: str) -> None:
- """A client of `tenant_id` was registered or deleted, and has
- committed: every process reconciles its copies of the tenant's
- clients with the table."""
+ def announce(
+ self, tenant_id: str, client_ids: Collection[str] | None = None
+ ) -> None:
+ """Clients of `tenant_id` were registered, renamed or deleted, and
+ that has committed: every process reconciles its copies of
+ `client_ids` with the table, or of all the tenant's clients when no
+ ids are given."""
if self._signals is not None:
- self._signals.changed(tenant_id)
+ self._signals.changed(tenant_id, client_ids)
@property
def tenants_isolated(self) -> bool:
@@ -175,6 +189,10 @@ class ClientLifecycleService:
return f"@{localpart}:{self._config.id_server_name}"
COLLAB_CLIENT_TYPES = ("bridge", "user")
+ # Collaboration clients (bridges and the people they speak for) are the
+ # bridges' to start and stop, and the chats routes write a member client's
+ # messages themselves, so neither runs as a client here.
+ _NOT_RUN_HERE = (*COLLAB_CLIENT_TYPES, MEMBER_CLIENT_TYPE)
async def ensure_system_client(self, client_type: str) -> None:
"""One system client of `client_type` per tenant, created where missing.
@@ -336,6 +354,23 @@ class ClientLifecycleService:
for tenant_id in await all_tenant_ids(self._session_factory):
await self.reconcile_tenant(tenant_id)
+ async def recheck_all(self) -> None:
+ """The periodic re-check: `reconcile_tenant` for every tenant whose
+ clients have changed since its last full read here.
+
+ Whether they have is one aggregate query (`ClientStore.fingerprint`),
+ where the full read loads every client of the tenant, so a re-check
+ with nothing to repair no longer costs a read of every client on every
+ process. A change already applied from its signal still counts as a
+ change, so the next re-check after it reads in full once."""
+ for tenant_id in await all_tenant_ids(self._session_factory):
+ async with tenant_session(self._session_factory, tenant_id) as session:
+ fingerprint = await self._client_store.fingerprint(
+ session, tenant_id, self._NOT_RUN_HERE
+ )
+ if fingerprint != self._fingerprints.get(tenant_id):
+ await self.reconcile_tenant(tenant_id)
+
async def reconcile_tenant(self, tenant_id: str) -> None:
"""Run here exactly the tenant's clients that `start_all` would run.
@@ -350,36 +385,91 @@ class ClientLifecycleService:
The registration's own signal comes after both have committed.
Collaboration clients (bridges and the people they speak for) are the
bridges' to start and stop, and a client that crashed stays stopped.
+
+ The read's fingerprint is kept for `recheck_all`. Where it differs
+ from the last one, something may have changed that no signal named
+ (an agent's description edited elsewhere sends none), so whatever
+ keeps per-agent state reads every agent again.
"""
- # Only a client already running before the read can be judged deleted
- # by it. One registered in this process while the read awaits has a
- # row the read may not have seen, and stopping it would put it in
- # `_left_stopped`, where later re-checks leave it.
- running_before = [
+ running_before = self._running_in(tenant_id, scope=None)
+ async with tenant_session(self._session_factory, tenant_id) as session:
+ records, fingerprint = await self._client_store.get_running(
+ session, tenant_id, self._NOT_RUN_HERE
+ )
+ changed = await self._apply(tenant_id, records, running_before, scope=None)
+ last = self._fingerprints.get(tenant_id)
+ self._fingerprints[tenant_id] = fingerprint
+ if last is not None and last != fingerprint:
+ self._agents_changed(tenant_id, None)
+ elif changed:
+ self._agents_changed(tenant_id, frozenset(changed))
+
+ async def reconcile_clients(
+ self, tenant_id: str, client_ids: Collection[str]
+ ) -> None:
+ """`reconcile_tenant` for the named clients only: their rows are read,
+ and each is started, stopped or renamed here to match, where the full
+ read loads every client of the tenant.
+
+ This is what a signal naming its clients does, and with thousands of
+ agents it is the difference between reading one row per change and
+ every row per change on every process. Whatever keeps per-agent state
+ is told the named clients either way, so the process that made the
+ change, which started or stopped its own copy already, reads them too.
+ """
+ scope = frozenset(client_ids)
+ if not scope:
+ return
+ running_before = self._running_in(tenant_id, scope=scope)
+ async with tenant_session(self._session_factory, tenant_id) as session:
+ records = [
+ record
+ for record in await self._client_store.get_many(
+ session, tenant_id, scope
+ )
+ if record.type not in self._NOT_RUN_HERE
+ ]
+ await self._apply(tenant_id, records, running_before, scope=scope)
+ self._agents_changed(tenant_id, scope)
+
+ def _running_in(self, tenant_id: str, *, scope: frozenset[str] | None) -> list[str]:
+ """The clients of `tenant_id` running here that a read of `scope`
+ (every client when None) can judge deleted.
+
+ Taken before the read: only a client already running before it can be
+ judged deleted by it. One registered in this process while the read
+ awaits has a row the read may not have seen, and stopping it would put
+ it in `_left_stopped`, where later re-checks leave it."""
+ return [
client_id
for client_id, client_tenant in self._client_tenants.items()
if client_tenant == tenant_id
+ and (scope is None or client_id in scope)
and self._client_types.get(client_id) not in self.COLLAB_CLIENT_TYPES
]
- async with tenant_session(self._session_factory, tenant_id) as session:
- records = [
- record
- for record in await self._client_store.get_all(session)
- if record.tenant_id == tenant_id
- and record.type not in self.COLLAB_CLIENT_TYPES
- # As in `start_all`: the chats routes write a member client's
- # messages themselves, so it has nothing to run.
- and record.type != MEMBER_CLIENT_TYPE
- ]
+
+ async def _apply(
+ self,
+ tenant_id: str,
+ records: list[Client],
+ running_before: list[str],
+ *,
+ scope: frozenset[str] | None,
+ ) -> set[str]:
+ """Start, stop and rename the copies here of what a read of `scope`
+ (every client of the tenant when None) found in `records`. Returns
+ the agent clients it started, stopped or renamed."""
present = {record.id for record in records}
# A client whose row is gone will not be started again: nothing is
# left to hold back.
self._left_stopped = {
client_id: client_tenant
for client_id, client_tenant in self._left_stopped.items()
- if client_tenant != tenant_id or client_id in present
+ if client_tenant != tenant_id
+ or client_id in present
+ or (scope is not None and client_id not in scope)
}
- agents_changed = False
+ changed: set[str] = set()
# A rename served elsewhere changes the name a copy here sends under
# (bridges recognise an agent's own messages by it) and the identities
# a bridge here holds for the agent, so both follow it now.
@@ -387,11 +477,13 @@ class ClientLifecycleService:
actor = self._clients.get(record.id)
if actor is not None and actor.display_name != record.display_name:
await actor.set_display_name(record.display_name)
- agents_changed |= record.type == "agent"
+ if record.type == "agent":
+ changed.add(record.id)
for client_id in running_before:
if client_id in self._client_tenants and client_id not in present:
logger.info("Client %s was deleted elsewhere; stopping it", client_id)
- agents_changed |= self._client_types.get(client_id) == "agent"
+ if self._client_types.get(client_id) == "agent":
+ changed.add(client_id)
await self.stop(client_id)
missing = [
record
@@ -399,9 +491,7 @@ class ClientLifecycleService:
if record.id not in self._clients and record.id not in self._left_stopped
]
if not missing:
- if agents_changed:
- self._agents_changed(tenant_id)
- return
+ return changed
agents = await self._agents_by_client_id(missing)
for record in missing:
agent = agents.get(record.id)
@@ -419,9 +509,9 @@ class ClientLifecycleService:
actor = self._register(record)
if agent is not None and isinstance(actor, _PreloadsAgent):
actor.preload_agent(agent)
- agents_changed |= agent is not None
- if agents_changed:
- self._agents_changed(tenant_id)
+ if agent is not None:
+ changed.add(record.id)
+ return changed
async def _agents_by_client_id(self, records: list[Client]) -> dict[str, Agent]:
"""The agent row behind each client, one query per tenant.
@@ -588,7 +678,7 @@ class ClientLifecycleService:
"""
tenant_id = self._left_stopped.pop(client_id, None)
if tenant_id is not None:
- self.announce(tenant_id)
+ self.announce(tenant_id, [client_id])
def get(self, client_id: str) -> Actor[ClientConfig] | None:
return self._clients.get(client_id)
diff --git a/core/switch_core/clients/client_signals.py b/core/switch_core/clients/client_signals.py
index 7ae6452e7..93507936c 100644
--- a/core/switch_core/clients/client_signals.py
+++ b/core/switch_core/clients/client_signals.py
@@ -7,40 +7,94 @@ any other process answered that agent's requests with "not running" and read
nothing for it; a client deleted kept its copies running everywhere else, on a
row that no longer exists.
-So a registration or deletion, once committed, announces `client.changed` for
-its tenant. Every process, the announcing one included, then re-reads the
-tenant's clients and starts or stops its copies to match
-(`ClientLifecycleService.reconcile_tenant`). The announcing process has
-already started or stopped its own, so for it the re-read is a check that
-changes nothing. A re-read that fails is kept and tried again, and every
-tenant is re-read when the signal connection comes back and on each periodic
-re-check, so a missed signal costs a delay rather than a client missing from
-a process until it restarts.
+So a registration, rename or deletion, once committed, announces
+`client.changed` for its tenant, naming the clients it changed. Every process,
+the announcing one included, then re-reads those clients' rows and starts,
+stops or renames its copies to match (`ClientLifecycleService.
+reconcile_clients`). The announcing process has already done so for its own,
+so for it the re-read is a check that changes nothing. A signal naming no
+clients, which a process from before they were named sends, re-reads every
+client of the tenant (`reconcile_tenant`).
+
+A re-read that fails is kept and tried again. Every tenant is re-read when the
+signal connection comes back, and each periodic re-check reads in full any
+tenant whose clients have changed since its last full read
+(`ClientLifecycleService.recheck_all`), so a missed signal costs a delay rather
+than a client missing from a process until it restarts.
"""
from __future__ import annotations
+from collections.abc import Collection, Iterable
from typing import TYPE_CHECKING, Any
+from switch_core.signals import payload_chunks
+
if TYPE_CHECKING:
from switch_core.clients.client_lifecycle_service import ClientLifecycleService
from switch_core.signals import SignalBus
CLIENT_CHANGED_SIGNAL = "client.changed"
+# Room kept in every signal for what it carries beside its list of clients:
+# its kind, the sending process's id and the tenant, about 200 bytes at most.
+_SIGNAL_ENVELOPE_BYTES = 400
+
class ClientSignals:
def __init__(self, *, lifecycle: ClientLifecycleService, bus: SignalBus) -> None:
self._bus = bus
- self._owed = bus.pending_work("clients", lifecycle.reconcile_tenant)
+ self._lifecycle = lifecycle
+ # The clients each tenant with work pending has to read again, or None
+ # where every client of the tenant has to be.
+ self._owed_clients: dict[str, set[str] | None] = {}
+ self._owed = bus.pending_work("clients", self._reconcile)
bus.handle(CLIENT_CHANGED_SIGNAL, self._on_changed)
bus.on_resync(lifecycle.reconcile_all)
- bus.on_recheck(lifecycle.reconcile_all)
+ bus.on_recheck(lifecycle.recheck_all)
lifecycle.use_signals(self)
- def changed(self, tenant_id: str) -> None:
- self._owed.add(tenant_id)
- self._bus.publish(CLIENT_CHANGED_SIGNAL, tenant_id=tenant_id)
+ def changed(self, tenant_id: str, client_ids: Collection[str] | None) -> None:
+ if client_ids is None:
+ self._owe(tenant_id, None)
+ self._bus.publish(CLIENT_CHANGED_SIGNAL, tenant_id=tenant_id)
+ return
+ named = sorted(set(client_ids))
+ self._owe(tenant_id, named)
+ for chunk in payload_chunks(named, _SIGNAL_ENVELOPE_BYTES):
+ self._bus.publish(
+ CLIENT_CHANGED_SIGNAL, tenant_id=tenant_id, client_ids=chunk
+ )
def _on_changed(self, signal: dict[str, Any]) -> None:
- self._owed.add(signal["tenant_id"])
+ client_ids = signal.get("client_ids")
+ self._owe(
+ signal["tenant_id"], client_ids if isinstance(client_ids, list) else None
+ )
+
+ def _owe(self, tenant_id: str, client_ids: Iterable[str] | None) -> None:
+ self._keep(tenant_id, client_ids)
+ self._owed.add(tenant_id)
+
+ def _keep(self, tenant_id: str, client_ids: Iterable[str] | None) -> None:
+ if client_ids is None:
+ self._owed_clients[tenant_id] = None
+ return
+ owed = self._owed_clients.setdefault(tenant_id, set())
+ if owed is not None:
+ owed.update(client_ids)
+
+ async def _reconcile(self, tenant_id: str) -> None:
+ # No entry reads every client: nothing is lost by reading too much.
+ client_ids = self._owed_clients.pop(tenant_id, None)
+ try:
+ if client_ids is None:
+ await self._lifecycle.reconcile_tenant(tenant_id)
+ else:
+ await self._lifecycle.reconcile_clients(tenant_id, client_ids)
+ except BaseException:
+ # Kept for the retry `PendingWork` makes of the tenant, which it
+ # schedules itself: adding the tenant again here would retry at
+ # once instead.
+ self._keep(tenant_id, client_ids)
+ raise
diff --git a/core/switch_core/db/stores/client_store.py b/core/switch_core/db/stores/client_store.py
index 0dc1044f3..774b99506 100644
--- a/core/switch_core/db/stores/client_store.py
+++ b/core/switch_core/db/stores/client_store.py
@@ -1,7 +1,38 @@
-from sqlalchemy import delete, select
+from collections.abc import Collection
+
+from sqlalchemy import BigInteger, ColumnElement, cast, delete, func, literal, select
+from sqlalchemy.dialects.postgresql import BIT
from sqlalchemy.ext.asyncio import AsyncSession
-from switch_core.db.models import Client, ClientRoom, require_tenant_id
+from switch_core.db.models import Agent, Client, ClientRoom, require_tenant_id
+
+# How many clients a tenant runs and what they are, as one pair of numbers:
+# the count, and the sum of a 60-bit hash of each client's id, type and name
+# and of its agent's id, name, display name and description. Equal pairs mean
+# no client was added, removed or renamed and no agent was added, removed or
+# re-described in between. Computed by the database, so reading it costs one
+# row however many clients the tenant has.
+Fingerprint = tuple[int, int]
+
+_FIELD_SEPARATOR = "\x1f"
+
+
+def _row_hash() -> ColumnElement[int]:
+ fields = func.concat_ws(
+ _FIELD_SEPARATOR,
+ Client.id,
+ Client.type,
+ Client.display_name,
+ func.coalesce(Agent.id, ""),
+ func.coalesce(Agent.name, ""),
+ func.coalesce(Agent.display_name, ""),
+ func.coalesce(Agent.description, ""),
+ )
+ # The first 15 hex digits of the md5, read as a 60-bit number.
+ return cast(
+ cast(literal("x") + func.substr(func.md5(fields), 1, 15), BIT(60)),
+ BigInteger,
+ )
class ClientStore:
@@ -12,6 +43,49 @@ class ClientStore:
async def get(self, session: AsyncSession, client_id: str) -> Client | None:
return await session.get(Client, client_id)
+ async def get_many(
+ self, session: AsyncSession, tenant_id: str, client_ids: Collection[str]
+ ) -> list[Client]:
+ """The tenant's clients among `client_ids`, in one query. An id with
+ no row in the tenant is left out."""
+ if not client_ids:
+ return []
+ result = await session.execute(
+ select(Client).where(
+ Client.tenant_id == tenant_id, Client.id.in_(list(client_ids))
+ )
+ )
+ return list(result.scalars().all())
+
+ async def get_running(
+ self, session: AsyncSession, tenant_id: str, excluded_types: Collection[str]
+ ) -> tuple[list[Client], Fingerprint]:
+ """The tenant's clients of every type but `excluded_types`, and the
+ fingerprint of those rows, from one statement and so from one
+ snapshot of the table."""
+ result = await session.execute(
+ select(Client, func.count().over(), func.sum(_row_hash()).over())
+ .outerjoin(Agent, Agent.client_id == Client.id)
+ .where(Client.tenant_id == tenant_id, Client.type.not_in(excluded_types))
+ )
+ rows = result.all()
+ if not rows:
+ return [], (0, 0)
+ return [row[0] for row in rows], (int(rows[0][1]), int(rows[0][2]))
+
+ async def fingerprint(
+ self, session: AsyncSession, tenant_id: str, excluded_types: Collection[str]
+ ) -> Fingerprint:
+ """The fingerprint `get_running` returns, without the rows."""
+ result = await session.execute(
+ select(func.count(), func.coalesce(func.sum(_row_hash()), 0))
+ .select_from(Client)
+ .outerjoin(Agent, Agent.client_id == Client.id)
+ .where(Client.tenant_id == tenant_id, Client.type.not_in(excluded_types))
+ )
+ count, total = result.one()
+ return int(count), int(total)
+
async def get_by_transport_user_id(
self, session: AsyncSession, transport_user_id: str
) -> Client | None:
diff --git a/core/tests/switch_core/bridges/agent/protocol/registration_harness.py b/core/tests/switch_core/bridges/agent/protocol/registration_harness.py
index 7007fe19c..e755917c3 100644
--- a/core/tests/switch_core/bridges/agent/protocol/registration_harness.py
+++ b/core/tests/switch_core/bridges/agent/protocol/registration_harness.py
@@ -59,7 +59,7 @@ class FakeClientLifecycle:
def start_client(self, client: Client) -> None:
self.started.append(client.id)
- def announce(self, tenant_id: str) -> None:
+ def announce(self, tenant_id: str, client_ids: object = None) -> None:
return None
diff --git a/core/tests/switch_core/bridges/agent/protocol/test_agent_tenant_scoped_admin.py b/core/tests/switch_core/bridges/agent/protocol/test_agent_tenant_scoped_admin.py
index fe243868e..a370a5503 100644
--- a/core/tests/switch_core/bridges/agent/protocol/test_agent_tenant_scoped_admin.py
+++ b/core/tests/switch_core/bridges/agent/protocol/test_agent_tenant_scoped_admin.py
@@ -75,7 +75,7 @@ class _FakeClientLifecycle:
def start_client(self, client: Client) -> None:
pass
- def announce(self, tenant_id: str) -> None:
+ def announce(self, tenant_id: str, client_ids: object = None) -> None:
return None
diff --git a/core/tests/switch_core/bridges/agent/protocol/test_bridge_identity_tenant_scope.py b/core/tests/switch_core/bridges/agent/protocol/test_bridge_identity_tenant_scope.py
index 0fbe9113a..d65f8ff04 100644
--- a/core/tests/switch_core/bridges/agent/protocol/test_bridge_identity_tenant_scope.py
+++ b/core/tests/switch_core/bridges/agent/protocol/test_bridge_identity_tenant_scope.py
@@ -144,7 +144,7 @@ class _StoppableClientLifecycle:
def start_client(self, client: Client) -> None:
pass
- def announce(self, tenant_id: str) -> None:
+ def announce(self, tenant_id: str, client_ids: object = None) -> None:
return None
async def stop(self, client_id: str) -> None:
diff --git a/core/tests/switch_core/bridges/agent/protocol/test_register_overwrite_owner_guard.py b/core/tests/switch_core/bridges/agent/protocol/test_register_overwrite_owner_guard.py
index fa95f0f64..8ea8a2660 100644
--- a/core/tests/switch_core/bridges/agent/protocol/test_register_overwrite_owner_guard.py
+++ b/core/tests/switch_core/bridges/agent/protocol/test_register_overwrite_owner_guard.py
@@ -55,7 +55,7 @@ class _FakeClientLifecycle:
def start_client(self, client: Client) -> None:
pass
- def announce(self, tenant_id: str) -> None:
+ def announce(self, tenant_id: str, client_ids: object = None) -> None:
return None
diff --git a/core/tests/switch_core/bridges/agent/protocol/test_room_roster_authz.py b/core/tests/switch_core/bridges/agent/protocol/test_room_roster_authz.py
index e6a88b5d0..2ee603a33 100644
--- a/core/tests/switch_core/bridges/agent/protocol/test_room_roster_authz.py
+++ b/core/tests/switch_core/bridges/agent/protocol/test_room_roster_authz.py
@@ -53,7 +53,7 @@ class _FakeClientLifecycle:
def start_client(self, client: Client) -> None:
pass
- def announce(self, tenant_id: str) -> None:
+ def announce(self, tenant_id: str, client_ids: object = None) -> None:
return None
diff --git a/core/tests/switch_core/bridges/agent/test_api_key_cache.py b/core/tests/switch_core/bridges/agent/test_api_key_cache.py
index 302abb14e..dfd9fcd96 100644
--- a/core/tests/switch_core/bridges/agent/test_api_key_cache.py
+++ b/core/tests/switch_core/bridges/agent/test_api_key_cache.py
@@ -60,7 +60,7 @@ class _FakeClientLifecycle:
def start_client(self, client: Client) -> None:
self.started.append(client.id)
- def announce(self, tenant_id: str) -> None:
+ def announce(self, tenant_id: str, client_ids: object = None) -> None:
return None
async def stop(self, client_id: str) -> None:
diff --git a/core/tests/switch_core/bridges/collaboration/test_bridge_identity_backfill.py b/core/tests/switch_core/bridges/collaboration/test_bridge_identity_backfill.py
index de071f676..4c93b4112 100644
--- a/core/tests/switch_core/bridges/collaboration/test_bridge_identity_backfill.py
+++ b/core/tests/switch_core/bridges/collaboration/test_bridge_identity_backfill.py
@@ -49,6 +49,9 @@ def _core(provision: Any) -> tuple[CollaborationCore, _FakeAdapter]:
core._bridge_type = "slack" # type: ignore[attr-defined]
core._adapter = adapter # type: ignore[attr-defined]
core._identity_task = None # type: ignore[attr-defined]
+ core._identities_stale = True # type: ignore[attr-defined]
+ core._identities_owed = set() # type: ignore[attr-defined]
+ core._agent_view = None # type: ignore[attr-defined]
core._channel_type_refresh_task = None # type: ignore[attr-defined]
async def _noop() -> None:
diff --git a/core/tests/switch_core/bridges/collaboration/test_bridge_identity_changes.py b/core/tests/switch_core/bridges/collaboration/test_bridge_identity_changes.py
new file mode 100644
index 000000000..8b2fbb8e3
--- /dev/null
+++ b/core/tests/switch_core/bridges/collaboration/test_bridge_identity_changes.py
@@ -0,0 +1,221 @@
+"""A bridge told which agents changed reads only those.
+
+Every process runs a copy of each bridge, and every copy is told when an
+agent is registered, renamed or deleted anywhere in its tenant. Each used to
+read every agent of the tenant then; with thousands of agents being created,
+that kept every process busy for as long as the creation lasted. The copy now
+keeps the tenant's agents as it last read them and reads again only the
+agents a change names (`CollaborationCore._update_agent_identities`).
+"""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import AsyncIterator, Sequence
+from contextlib import asynccontextmanager
+from types import SimpleNamespace
+from typing import Any
+
+from switch_core.bridges.collaboration.adapter import AgentIdentity
+from switch_core.bridges.collaboration.collaboration_core import CollaborationCore
+
+_TENANT = "tenant-a"
+
+
+def _agent(name: str, *, description: str = "helps") -> SimpleNamespace:
+ return SimpleNamespace(
+ id=f"agent-{name}",
+ client_id=f"client-{name}",
+ name=name,
+ description=description,
+ display_name=None,
+ )
+
+
+class _Store:
+ """The tenant's agent rows, and what was read of them."""
+
+ def __init__(self, *agents: SimpleNamespace) -> None:
+ self.rows = {agent.client_id: agent for agent in agents}
+ self.full_reads = 0
+ self.named_reads: list[list[str]] = []
+ self.hold_full_read: asyncio.Event | None = None
+
+ async def get_all(self, session: Any) -> list[SimpleNamespace]:
+ self.full_reads += 1
+ if self.hold_full_read is not None:
+ await self.hold_full_read.wait()
+ return list(self.rows.values())
+
+ async def get_by_client_ids(
+ self, session: Any, client_ids: list[str]
+ ) -> list[SimpleNamespace]:
+ self.named_reads.append(client_ids)
+ return [self.rows[c] for c in client_ids if c in self.rows]
+
+
+class _Adapter:
+ """What the platform was asked to do."""
+
+ def __init__(self) -> None:
+ self.calls: list[tuple[str, Any]] = []
+ self.provisioned: list[list[str]] = []
+
+ async def remove_agent_identity(self, agent: AgentIdentity) -> None:
+ self.calls.append(("remove", agent.name))
+
+ async def rename_agent_identity(self, old_name: str, agent: AgentIdentity) -> None:
+ self.calls.append(("rename", (old_name, agent.name)))
+
+ async def update_agent_identity(self, agent: AgentIdentity) -> None:
+ self.calls.append(("update", agent.name))
+
+ async def sync_agent_identities(
+ self, agents: Sequence[AgentIdentity]
+ ) -> list[tuple[str, BaseException]]:
+ self.provisioned.append(sorted(agent.name for agent in agents))
+ return []
+
+
+@asynccontextmanager
+async def _session() -> AsyncIterator[object]:
+ yield object()
+
+
+def _core(store: _Store) -> tuple[CollaborationCore, _Adapter]:
+ """A CollaborationCore with only what identity provisioning touches."""
+ core = object.__new__(CollaborationCore)
+ adapter = _Adapter()
+ core._bridge_type = "slack" # type: ignore[attr-defined]
+ core._bridge_tenant_id = _TENANT # type: ignore[attr-defined]
+ core._session_factory = _session # type: ignore[attr-defined]
+ core._agent_store = store # type: ignore[attr-defined]
+ core._adapter = adapter # type: ignore[attr-defined]
+ core._identity_task = None # type: ignore[attr-defined]
+ core._provisioned_agents = {} # type: ignore[attr-defined]
+ core._agent_view = None # type: ignore[attr-defined]
+ core._agent_of_client = {} # type: ignore[attr-defined]
+ core._identities_stale = False # type: ignore[attr-defined]
+ core._identities_owed = set() # type: ignore[attr-defined]
+ return core, adapter
+
+
+async def _settled(core: CollaborationCore) -> None:
+ task = core._identity_task
+ assert task is not None
+ await asyncio.wait_for(task, timeout=1)
+
+
+async def _started(store: _Store) -> tuple[CollaborationCore, _Adapter]:
+ """A bridge copy that has read and provisioned every agent, as at start."""
+ core, adapter = _core(store)
+ core._provision_identities_on_attach()
+ await _settled(core)
+ adapter.calls.clear()
+ adapter.provisioned.clear()
+ return core, adapter
+
+
+async def test_a_new_agent_is_read_alone_and_provisioned() -> None:
+ store = _Store(_agent("scout"), _agent("scribe"))
+ core, adapter = await _started(store)
+ store.rows["client-tracker"] = _agent("tracker")
+
+ core.agents_changed({"client-tracker"})
+ await _settled(core)
+
+ assert store.full_reads == 1
+ assert store.named_reads == [["client-tracker"]]
+ assert adapter.provisioned == [["scout", "scribe", "tracker"]]
+
+
+async def test_an_agent_deleted_elsewhere_loses_its_identity_here() -> None:
+ store = _Store(_agent("scout"), _agent("scribe"))
+ core, adapter = await _started(store)
+ del store.rows["client-scribe"]
+
+ core.agents_changed({"client-scribe"})
+ await _settled(core)
+
+ assert adapter.calls == [("remove", "scribe")]
+ assert adapter.provisioned == [["scout"]]
+ assert "agent-scribe" not in core._provisioned_agents
+ assert store.full_reads == 1
+
+
+async def test_an_agent_renamed_elsewhere_is_renamed_here() -> None:
+ store = _Store(_agent("scout"))
+ core, adapter = await _started(store)
+ renamed = _agent("scout")
+ renamed.name = "pathfinder"
+ store.rows["client-scout"] = renamed
+
+ core.agents_changed({"client-scout"})
+ await _settled(core)
+
+ assert adapter.calls == [("rename", ("scout", "pathfinder"))]
+ assert adapter.provisioned == [["pathfinder"]]
+
+
+async def test_agents_not_named_are_left_as_last_read() -> None:
+ """A description edited elsewhere sends no signal of its own; the client
+ re-check finds it and reads every agent again (`None`)."""
+ store = _Store(_agent("scout"), _agent("scribe"))
+ core, adapter = await _started(store)
+ store.rows["client-scribe"] = _agent("scribe", description="writes things")
+
+ core.agents_changed({"client-scout"})
+ await _settled(core)
+ assert adapter.calls == []
+
+ core.agents_changed(None)
+ await _settled(core)
+ assert adapter.calls == [("update", "scribe")]
+ assert store.full_reads == 2
+
+
+async def test_a_change_before_any_full_read_reads_every_agent() -> None:
+ store = _Store(_agent("scout"))
+ core, adapter = _core(store)
+
+ core.agents_changed({"client-scout"})
+ await _settled(core)
+
+ assert store.full_reads == 1
+ assert store.named_reads == []
+ assert adapter.provisioned == [["scout"]]
+
+
+async def test_a_change_during_a_full_read_is_read_after_it() -> None:
+ """The full read may have seen the tenant before the change committed."""
+ store = _Store(_agent("scout"))
+ core, adapter = await _started(store)
+ store.hold_full_read = asyncio.Event()
+ core.agents_changed(None)
+ await asyncio.sleep(0)
+ assert store.full_reads == 2
+
+ store.rows["client-tracker"] = _agent("tracker")
+ core.agents_changed({"client-tracker"})
+ store.hold_full_read.set()
+ await _settled(core)
+
+ assert store.named_reads == [["client-tracker"]]
+ assert adapter.provisioned[-1] == ["scout", "tracker"]
+
+
+async def test_a_change_during_the_first_read_reads_every_agent_again() -> None:
+ """With nothing read yet there is nothing to read the named agents into."""
+ store = _Store(_agent("scout"))
+ core, adapter = _core(store)
+ store.hold_full_read = asyncio.Event()
+ core._provision_identities_on_attach()
+ await asyncio.sleep(0)
+
+ store.rows["client-tracker"] = _agent("tracker")
+ core.agents_changed({"client-tracker"})
+ store.hold_full_read.set()
+ await _settled(core)
+
+ assert store.full_reads == 2
+ assert adapter.provisioned[-1] == ["scout", "tracker"]
diff --git a/core/tests/switch_core/bridges/collaboration/test_bridge_inbound_tenant_binding.py b/core/tests/switch_core/bridges/collaboration/test_bridge_inbound_tenant_binding.py
index 77a3f4b6f..27af71d52 100644
--- a/core/tests/switch_core/bridges/collaboration/test_bridge_inbound_tenant_binding.py
+++ b/core/tests/switch_core/bridges/collaboration/test_bridge_inbound_tenant_binding.py
@@ -299,6 +299,8 @@ class TestBridgeLevelWorkBindsTheBridgesTenant:
await original()
bridge._create_agent_identities = _watched # type: ignore[method-assign]
+ # Every agent is to be read, as `start` asks.
+ bridge._identities_stale = True
with tenant_scope("tenant-whoever-started-the-bridge"):
await bridge._run_agent_identities()
diff --git a/core/tests/switch_core/clients/test_client_signals.py b/core/tests/switch_core/clients/test_client_signals.py
index 7fe65b1c9..3dfca8cd8 100644
--- a/core/tests/switch_core/clients/test_client_signals.py
+++ b/core/tests/switch_core/clients/test_client_signals.py
@@ -21,7 +21,7 @@ from sqlalchemy.pool import NullPool
from switch_core.chats import MEMBER_CLIENT_TYPE
from switch_core.clients.client_lifecycle_service import ClientLifecycleService
-from switch_core.clients.client_signals import ClientSignals
+from switch_core.clients.client_signals import CLIENT_CHANGED_SIGNAL, ClientSignals
from switch_core.db.models import TENANT_ZERO_ID, Agent, Client
from switch_core.db.stores.client_store import ClientStore
from switch_core.db.stores.tenant_store import TenantStore
@@ -140,7 +140,7 @@ class TestAcrossProcesses:
agent = await _register(session_factory)
assert agent.client_id is not None
- here.announce(TENANT_ZERO_ID)
+ here.announce(TENANT_ZERO_ID, [agent.client_id])
await _eventually(lambda: there.get(agent.client_id) is not None) # type: ignore[arg-type]
# The announcing process re-reads too, and starts what it lacks.
@@ -158,10 +158,12 @@ class TestAcrossProcesses:
agent = await _register(session_factory)
client_id = agent.client_id
assert client_id is not None
- here.announce(TENANT_ZERO_ID)
+ here.announce(TENANT_ZERO_ID, [client_id])
await _eventually(lambda: there.get(client_id) is not None)
- told: list[str] = []
- there.add_agents_changed_listener(told.append)
+ told: list[tuple[str, frozenset[str] | None]] = []
+ there.add_agents_changed_listener(
+ lambda tenant_id, client_ids: told.append((tenant_id, client_ids))
+ )
# As `AgentCore.rename_agent` does: rename the client row, announce.
async with session_factory() as session:
@@ -169,14 +171,14 @@ class TestAcrossProcesses:
assert client is not None
client.display_name = "renamed-agent"
await session.commit()
- here.announce(TENANT_ZERO_ID)
+ here.announce(TENANT_ZERO_ID, [client_id])
await _eventually(
lambda: (
getattr(there.get(client_id), "display_name", None) == "renamed-agent"
)
)
- await _eventually(lambda: TENANT_ZERO_ID in told)
+ await _eventually(lambda: (TENANT_ZERO_ID, frozenset({client_id})) in told)
async def test_a_client_deleted_in_one_process_stops_in_the_other(
self,
@@ -187,18 +189,63 @@ class TestAcrossProcesses:
agent = await _register(session_factory)
client_id = agent.client_id
assert client_id is not None
- here.announce(TENANT_ZERO_ID)
+ here.announce(TENANT_ZERO_ID, [client_id])
await _eventually(lambda: there.get(client_id) is not None)
copy = there.get(client_id)
# As `AgentCore.delete_agent` does: stop here, delete, announce.
await here.stop(client_id)
await _delete(session_factory, agent)
- here.announce(TENANT_ZERO_ID)
+ here.announce(TENANT_ZERO_ID, [client_id])
await _eventually(lambda: there.get(client_id) is None)
assert isinstance(copy, _Consumer) and copy.stopped
+ async def test_a_change_reads_only_the_clients_it_names(
+ self,
+ session_factory: async_sessionmaker[AsyncSession],
+ two_processes: tuple[ClientLifecycleService, ClientLifecycleService],
+ ) -> None:
+ """With thousands of agents, a change that made every process read
+ every client of the tenant kept every process busy for as long as
+ agents were being created."""
+ here, there = two_processes
+ # The read every process makes once its signal connection is up.
+ await _eventually(lambda: TENANT_ZERO_ID in there._fingerprints)
+ full_reads = _count_full_reads(there)
+ agent = await _register(session_factory)
+ client_id = agent.client_id
+ assert client_id is not None
+ told: list[tuple[str, frozenset[str] | None]] = []
+ there.add_agents_changed_listener(
+ lambda tenant_id, client_ids: told.append((tenant_id, client_ids))
+ )
+
+ here.announce(TENANT_ZERO_ID, [client_id])
+
+ await _eventually(lambda: there.get(client_id) is not None)
+ await _eventually(lambda: (TENANT_ZERO_ID, frozenset({client_id})) in told)
+ assert full_reads == []
+
+ async def test_a_signal_naming_no_clients_reads_the_whole_tenant(
+ self,
+ postgres_url: str,
+ session_factory: async_sessionmaker[AsyncSession],
+ two_processes: tuple[ClientLifecycleService, ClientLifecycleService],
+ ) -> None:
+ """What a process from before signals named their clients sends,
+ during a rolling deploy."""
+ _here, there = two_processes
+ agent = await _register(session_factory)
+ older = _bus(postgres_url, session_factory)
+ await older.start()
+ try:
+ older.publish(CLIENT_CHANGED_SIGNAL, tenant_id=TENANT_ZERO_ID)
+
+ await _eventually(lambda: there.get(agent.client_id) is not None) # type: ignore[arg-type]
+ finally:
+ await older.stop()
+
class TestReconcile:
async def test_the_recheck_starts_a_client_whose_signal_was_missed(
@@ -278,11 +325,11 @@ class TestReconcile:
lifecycle = _lifecycle(session_factory)
await lifecycle.start_all()
store = lifecycle._client_store
- real_get_all = store.get_all
+ real_get_running = store.get_running
registered: list[str] = []
- async def read_then_register(session: Any) -> Any:
- records = await real_get_all(session)
+ async def read_then_register(session: Any, *args: Any) -> Any:
+ records = await real_get_running(session, *args)
if not registered:
# As `AgentCore._create_agent` does: commit, then start.
agent = await _register(session_factory)
@@ -293,11 +340,96 @@ class TestReconcile:
registered.append(record.id)
return records
- store.get_all = read_then_register # type: ignore[method-assign]
+ store.get_running = read_then_register # type: ignore[method-assign]
try:
await lifecycle.reconcile_tenant(TENANT_ZERO_ID)
finally:
- store.get_all = real_get_all # type: ignore[method-assign]
+ store.get_running = real_get_running # type: ignore[method-assign]
assert lifecycle.get(registered[0]) is not None
await lifecycle.stop_all()
+
+ async def test_a_named_client_stopped_on_purpose_stays_stopped(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ """As above, for a re-read of the named clients only."""
+ agent = await _register(session_factory)
+ assert agent.client_id is not None
+ lifecycle = _lifecycle(session_factory)
+ await lifecycle.start_all()
+ await lifecycle.stop(agent.client_id)
+
+ await lifecycle.reconcile_clients(TENANT_ZERO_ID, [agent.client_id])
+
+ assert lifecycle.get(agent.client_id) is None
+
+
+class TestRecheck:
+ """The periodic re-check, which every process runs every 30 seconds:
+ a read of every client of every tenant, until it compared fingerprints
+ first."""
+
+ async def test_with_nothing_changed_it_reads_no_client(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ lifecycle = _lifecycle(session_factory)
+ await lifecycle.start_all()
+ await lifecycle.reconcile_all()
+ full_reads = _count_full_reads(lifecycle)
+
+ await lifecycle.recheck_all()
+
+ assert full_reads == []
+ await lifecycle.stop_all()
+
+ async def test_it_starts_a_client_whose_signal_was_missed(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ lifecycle = _lifecycle(session_factory)
+ await lifecycle.start_all()
+ await lifecycle.reconcile_all()
+ agent = await _register(session_factory)
+
+ await lifecycle.recheck_all()
+
+ assert lifecycle.get(agent.client_id) is not None # type: ignore[arg-type]
+ await lifecycle.stop_all()
+
+ async def test_an_agent_edited_with_no_signal_is_read_again_by_the_bridges(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ """An agent's description edited in another process sends no signal,
+ and a bridge here keeps the description with the agent's platform
+ handle."""
+ agent = await _register(session_factory)
+ lifecycle = _lifecycle(session_factory)
+ await lifecycle.start_all()
+ await lifecycle.reconcile_all()
+ told: list[tuple[str, frozenset[str] | None]] = []
+ lifecycle.add_agents_changed_listener(
+ lambda tenant_id, client_ids: told.append((tenant_id, client_ids))
+ )
+ async with session_factory() as session:
+ row = await session.get(Agent, agent.id)
+ assert row is not None
+ row.description = "does something else now"
+ await session.commit()
+
+ await lifecycle.recheck_all()
+
+ assert told == [(TENANT_ZERO_ID, None)]
+ await lifecycle.stop_all()
+
+
+def _count_full_reads(lifecycle: ClientLifecycleService) -> list[str]:
+ """The tenants `lifecycle` reads every client of from now on."""
+ reads: list[str] = []
+ store = lifecycle._client_store
+ real_get_running = store.get_running
+
+ async def counted(session: Any, tenant_id: str, *args: Any) -> Any:
+ reads.append(tenant_id)
+ return await real_get_running(session, tenant_id, *args)
+
+ store.get_running = counted # type: ignore[method-assign]
+ return reads
diff --git a/core/tests/switch_core/db/stores/test_client_store.py b/core/tests/switch_core/db/stores/test_client_store.py
index 1d3bd760f..292009fa9 100644
--- a/core/tests/switch_core/db/stores/test_client_store.py
+++ b/core/tests/switch_core/db/stores/test_client_store.py
@@ -3,13 +3,15 @@ from __future__ import annotations
import uuid
import pytest
+from sqlalchemy import delete, inspect
from sqlalchemy.exc import IntegrityError
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
-from switch_core.db.models import Client, Room, Tenant
+from switch_core.db.models import TENANT_ZERO_ID, Agent, Client, Room, Tenant
from switch_core.db.session_scope import tenant_session
from switch_core.db.stores.client_store import ClientStore
from switch_core.db.stores.room_store import RoomStore
+from tests.switch_core.test_room_wide_mention_wakes_no_agent import _make_agent
async def _make_client(session: AsyncSession, transport_user_id: str) -> Client:
@@ -216,3 +218,128 @@ class TestGetByTransportUserIdMultiTenant:
)
assert result is not None
assert result.id == other_id
+
+
+class TestFingerprint:
+ """What the periodic re-check compares before reading every client
+ (`ClientLifecycleService.recheck_all`): equal fingerprints have to mean
+ nothing a process runs from the rows changed, and each of these changes
+ has to show."""
+
+ _NOT_RUN = ("bridge", "user", "member")
+
+ async def _fingerprint(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> tuple[int, int]:
+ async with tenant_session(session_factory, TENANT_ZERO_ID) as session:
+ return await ClientStore().fingerprint(
+ session, TENANT_ZERO_ID, self._NOT_RUN
+ )
+
+ async def test_the_rows_and_their_fingerprint_come_back_together(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ async with session_factory() as session:
+ await _make_agent(session, f"fp-{uuid.uuid4().hex[:8]}")
+ await _make_client(session, f"@plain-{uuid.uuid4().hex[:8]}:test")
+ await session.commit()
+
+ async with tenant_session(session_factory, TENANT_ZERO_ID) as session:
+ records, fingerprint = await ClientStore().get_running(
+ session, TENANT_ZERO_ID, self._NOT_RUN
+ )
+
+ assert fingerprint == await self._fingerprint(session_factory)
+ assert fingerprint[0] == len(records) >= 2
+
+ async def test_a_rename_shows(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ async with session_factory() as session:
+ client = await _make_client(session, f"@named-{uuid.uuid4().hex[:8]}:test")
+ await session.commit()
+ before = await self._fingerprint(session_factory)
+
+ async with session_factory() as session:
+ row = await session.get(Client, client.id)
+ assert row is not None
+ row.display_name = "renamed"
+ await session.commit()
+
+ after = await self._fingerprint(session_factory)
+ assert after[0] == before[0]
+ assert after != before
+
+ async def test_an_agent_row_arriving_after_its_client_shows(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ """Registration commits the client row first. A full read between the
+ two skips the client, so the agent row's arrival has to count as a
+ change for the next re-check to start it."""
+ async with session_factory() as session:
+ agent = await _make_agent(session, f"late-{uuid.uuid4().hex[:8]}")
+ await session.commit()
+ columns = {
+ attr.key: getattr(agent, attr.key)
+ for attr in inspect(Agent).column_attrs
+ }
+ async with session_factory() as session:
+ await session.execute(delete(Agent).where(Agent.id == agent.id))
+ await session.commit()
+ client_only = await self._fingerprint(session_factory)
+
+ async with session_factory() as session:
+ session.add(Agent(**columns))
+ await session.commit()
+
+ assert await self._fingerprint(session_factory) != client_only
+
+ async def test_an_agents_description_shows(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ """No signal names a description edited in another process, and a
+ bridge there keeps it with the agent's platform handle."""
+ async with session_factory() as session:
+ agent = await _make_agent(session, f"desc-{uuid.uuid4().hex[:8]}")
+ await session.commit()
+ before = await self._fingerprint(session_factory)
+
+ async with session_factory() as session:
+ row = await session.get(Agent, agent.id)
+ assert row is not None
+ row.description = "does something else now"
+ await session.commit()
+
+ assert await self._fingerprint(session_factory) != before
+
+ async def test_clients_not_run_as_clients_do_not_count(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ before = await self._fingerprint(session_factory)
+
+ async with session_factory() as session:
+ session.add(
+ Client(
+ transport_user_id=f"@person-{uuid.uuid4().hex[:8]}:test",
+ display_name="a person a bridge speaks for",
+ type="user",
+ )
+ )
+ await session.commit()
+
+ assert await self._fingerprint(session_factory) == before
+
+ async def test_another_tenants_clients_do_not_count(
+ self, session_factory: async_sessionmaker[AsyncSession]
+ ) -> None:
+ other_tenant = f"tenant-{uuid.uuid4().hex[:8]}"
+ async with session_factory() as session:
+ session.add(Tenant(id=other_tenant, slug=other_tenant, name=other_tenant))
+ await session.commit()
+ before = await self._fingerprint(session_factory)
+
+ async with tenant_session(session_factory, other_tenant) as other:
+ await _make_client(other, f"@theirs-{uuid.uuid4().hex[:8]}:test")
+ await other.commit()
+
+ assert await self._fingerprint(session_factory) == before
diff --git a/core/tests/switch_core/management/harness.py b/core/tests/switch_core/management/harness.py
index c3eaa0adc..b3c30557c 100644
--- a/core/tests/switch_core/management/harness.py
+++ b/core/tests/switch_core/management/harness.py
@@ -131,7 +131,7 @@ class _FakeClientLifecycle:
def start_client(self, client: Client) -> None:
return None
- def announce(self, tenant_id: str) -> None:
+ def announce(self, tenant_id: str, client_ids: object = None) -> None:
return None
def get_by_agent_id(self, agent_id: str) -> None: