Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Read only the clients and agents a change names, so creating agents stops pinning every process (CHOO-3667)
Abel Dantasc6c247def7f4
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 8b9791612..76013f638 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -224,6 +224,13 @@ version of their own to them without also giving them a release of their own.group whose subgroup would move up next to a namesake is refused too, andthe dashboard now shows why. Names can no longer contain `/` or be blank.Existing duplicates are left as they are and can be reached by id.+- **Registering, renaming or deleting an agent no longer makes every process+ read every client and agent.** The change now names the clients it touched,+ and each process reads only those, where it read every client of the+ workspace and, with a messaging app connected, every agent as well. The+ 30-second re-check reads every client only when a fingerprint of the+ workspace's clients and agents has changed since its last full read, where+ it read them all every time.#### Fixed- **On Slack, the 👀 and the activity card no longer arrive after the agent'sdiff --git a/core/switch_core/bridges/agent/protocol/agent_core.py b/core/switch_core/bridges/agent/protocol/agent_core.pyindex 4ca938067..cb5979264 100644--- a/core/switch_core/bridges/agent/protocol/agent_core.py+++ b/core/switch_core/bridges/agent/protocol/agent_core.py@@ -700,7 +700,7 @@ class AgentCore:await session.commit()self.client_lifecycle.start_client(client_record)- self.client_lifecycle.announce(client_record.tenant_id)+ self.client_lifecycle.announce(client_record.tenant_id, [client_record.id])return agent.idasync def _reregister_agent(@@ -1100,9 +1100,9 @@ class AgentCore:# the next message that already matched its old name.if isinstance(actor, AgentActor):actor.set_agent(agent)- # Every other process re-reads the tenant's clients, so its copy of the+ # Every other process re-reads the agent's client, so its copy of the# agent and the bridges it holds take the new name too.- self.client_lifecycle.announce(tenant_id)+ self.client_lifecycle.announce(tenant_id, [agent.client_id])if self._agent_rename_listener is not None:await self._agent_rename_listener(tenant_id, agent_id, old_name)await self._agent_profile_changed(agent_id)@@ -1404,7 +1404,7 @@ class AgentCore:# its client must not stay stopped here.self.client_lifecycle.deletion_failed(client_id)raise- self.client_lifecycle.announce(tenant_id)+ self.client_lifecycle.announce(tenant_id, [client_id])self.api_key_cache.invalidate_agent(resolved_id)# Its connections end with it, here and in whichever process holds# one: their rows went with its row. After its key is dropped, so adiff --git a/core/switch_core/bridges/collaboration/collaboration_core.py b/core/switch_core/bridges/collaboration/collaboration_core.pyindex 8093ef325..7d33e16bd 100644--- a/core/switch_core/bridges/collaboration/collaboration_core.py+++ b/core/switch_core/bridges/collaboration/collaboration_core.py@@ -7,7 +7,7 @@ import loggingimport reimport timeimport uuid-from collections.abc import Awaitable, Callable, Iterator, Mapping+from collections.abc import Awaitable, Callable, Collection, Iterator, Mappingfrom contextlib import contextmanagerfrom dataclasses import dataclass, fieldfrom datetime import timedelta@@ -320,10 +320,18 @@ class CollaborationCore:# Identity provisioning runs in the background — see _create_agent_identities.self._identity_task: asyncio.Task[None] | None = None# Agents given an identity by this copy of the bridge, so one deleted- # by another process is let go of here too; and whether the agents- # changed while a provisioning run was going.+ # by another process is let go of here too.self._provisioned_agents: dict[str, AgentIdentity] = {}+ # The tenant's agents as this copy last read them, by agent id, and+ # which agent each client is, kept up to date by reading only the+ # agents a change names (`_update_agent_identities`). None until a+ # run has read them all.+ self._agent_view: dict[str, AgentIdentity] | None = None+ self._agent_of_client: dict[str, str] = {}+ # Whether every agent has to be read again, and the clients whose+ # agents do, for the provisioning run going or the next one.self._identities_stale = False+ self._identities_owed: set[str] = set()self._channel_type_refresh_task: asyncio.Task[None] | None = None# Turns and request cards from the tables the host reports to, pushed# as they change. Only where the platform draws session activity.@@ -552,14 +560,18 @@ class CollaborationCore:"""Whether this process holds the bridge and listens for it."""return self._active- def agents_changed(self) -> None:- """Agents were registered or deleted in this bridge's tenant, maybe by- another process: give the new ones their platform identities here and- let go of the deleted ones'. A run already going reads the agents- again once it is done, in case it read them before the change."""- self._identities_stale = True+ def agents_changed(self, client_ids: Collection[str] | None = None) -> None:+ """Agents were registered, renamed or deleted in this bridge's tenant,+ maybe by another process: give the new ones their platform identities+ here and let go of the deleted ones'. Only the agents of `client_ids`+ are read again, or every agent when None. A run already going reads+ them once it is done, in case it read them before the change."""+ if client_ids is None or self._agent_view is None:+ self._identities_stale = True+ else:+ self._identities_owed.update(client_ids)if not self._awaiting_shared_connection():- self._provision_identities_on_attach()+ self._start_identity_run()def _awaiting_shared_connection(self) -> bool:"""Whether the bridge waits to be attached to its platform's shared@@ -682,6 +694,10 @@ class CollaborationCore:against the connection as it is by then; one that has finished isreplaced."""+ self._identities_stale = True+ self._start_identity_run()++ def _start_identity_run(self) -> None:if self._identity_task is not None and not self._identity_task.done():returnself._identity_task = asyncio.create_task(self._run_agent_identities())@@ -702,9 +718,19 @@ class CollaborationCore:async def _run_agent_identities_unbound(self) -> None:try:while True:- self._identities_stale = False- await self._create_agent_identities()- if not self._identities_stale:+ if self._identities_stale:+ self._identities_stale = False+ self._identities_owed = set()+ await self._create_agent_identities()+ elif self._identities_owed:+ owed, self._identities_owed = self._identities_owed, set()+ try:+ await self._update_agent_identities(owed)+ except BaseException:+ # Read again by the next run, whatever starts it.+ self._identities_owed |= owed+ raise+ else:breakexcept asyncio.CancelledError:logger.info(@@ -722,12 +748,54 @@ class CollaborationCore:self._session_factory, self._bridge_tenant_id) as session:agents = await self._agent_store.get_all(session)+ current = {agent.id: agent_identity(agent) for agent in agents}+ self._agent_view = current+ self._agent_of_client = {agent.client_id: agent.id for agent in agents}+ await self._sync_agent_identities(current, sorted(self._provisioned_agents))++ async def _update_agent_identities(self, client_ids: set[str]) -> None:+ """`_create_agent_identities` for the agents of `client_ids` only:+ their rows are read and the rest of the tenant's agents are taken as+ this copy last read them, which with thousands of agents is the+ difference between reading one row per change and every row."""+ if self._agent_view is None:+ await self._create_agent_identities()+ return+ async with tenant_session(+ self._session_factory, self._bridge_tenant_id+ ) as session:+ agents = await self._agent_store.get_by_client_ids(+ session, sorted(client_ids)+ )+ found = {agent.client_id: agent for agent in agents}+ view = self._agent_view+ changed: set[str] = set()+ for client_id in client_ids:+ agent = found.get(client_id)+ known = self._agent_of_client.pop(client_id, None)+ if known is not None:+ view.pop(known, None)+ changed.add(known)+ if agent is not None:+ self._agent_of_client[client_id] = agent.id+ view[agent.id] = agent_identity(agent)+ changed.add(agent.id)+ await self._sync_agent_identities(+ view, sorted(changed & self._provisioned_agents.keys())+ )+ async def _sync_agent_identities(+ self, current: dict[str, AgentIdentity], compared: list[str]+ ) -> None:+ """Bring the platform in line with `current`, the tenant's agents:+ the agents of `compared` this copy gave an identity to are renamed,+ relabelled or let go of to match it, then every agent's identity is+ provisioned."""# Deleted, renamed or relabelled since the last run here, wherever that# was served: a change served by another process reaches this copy of- # the bridge only through this re-read.- current = {agent.id: agent_identity(agent) for agent in agents}- for agent_id, before in sorted(self._provisioned_agents.items()):+ # the bridge only through a re-read.+ for agent_id in compared:+ before = self._provisioned_agents[agent_id]after = current.get(agent_id)try:if after is None:@@ -767,14 +835,14 @@ class CollaborationCore:logger.warning("Provisioned %s identities for %d agents; %d failed",self._bridge_type,- len(agents),+ len(current),len(failures),)else:logger.info("Provisioned %s identities for %d agents",self._bridge_type,- len(agents),+ len(current),)async def _ensure_channel_captures(self) -> None:diff --git a/core/switch_core/bridges/collaboration/lifecycle_service.py b/core/switch_core/bridges/collaboration/lifecycle_service.pyindex 85858daaf..366827b44 100644--- a/core/switch_core/bridges/collaboration/lifecycle_service.py+++ b/core/switch_core/bridges/collaboration/lifecycle_service.py@@ -1890,12 +1890,15 @@ class CollaborationBridgeLifecycleService:exc_info=True,)- def agents_changed(self, tenant_id: str) -> None:- """Agents were registered or deleted in `tenant_id`, maybe by another- process: every bridge of the tenant running here provisions their- platform identities again."""+ def agents_changed(+ self, tenant_id: str, client_ids: frozenset[str] | None = None+ ) -> None:+ """Agents were registered, renamed or deleted in `tenant_id`, maybe by+ another process: every bridge of the tenant running here provisions+ the platform identities of the agents of `client_ids` again, or of+ every agent when None."""for bridge in self.bridges_for_tenant(tenant_id):- bridge.agents_changed()+ bridge.agents_changed(client_ids)async def remove(self, bridge_id: str) -> None:"""Disconnect a messaging app and take its identities with it.diff --git a/core/switch_core/clients/client_lifecycle_service.py b/core/switch_core/clients/client_lifecycle_service.pyindex ae8ff56c9..405bbad1b 100644--- a/core/switch_core/clients/client_lifecycle_service.py+++ b/core/switch_core/clients/client_lifecycle_service.py@@ -15,7 +15,7 @@ import asyncioimport loggingimport randomimport uuid-from collections.abc import Callable+from collections.abc import Callable, Collectionfrom typing import TYPE_CHECKING, Any, Protocol, runtime_checkablefrom sqlalchemy.exc import InterfaceError, OperationalError@@ -30,7 +30,7 @@ from switch_core.config import SwitchConfigfrom switch_core.db.models import Agent, Client, Tenantfrom switch_core.db.session_scope import tenant_sessionfrom switch_core.db.stores.agent_store import AgentStore-from switch_core.db.stores.client_store import ClientStore+from switch_core.db.stores.client_store import ClientStore, Fingerprintfrom switch_core.db.stores.delivery_cursor_store import DeliveryCursorStorefrom switch_core.db.stores.tenant_store import TenantStorefrom switch_core.db.tenant_lookup import all_tenant_ids@@ -47,6 +47,10 @@ if TYPE_CHECKING:# run it (`ownership.py`), keyed by client id.CLIENT_CONSUMERS = "client"+# Told a tenant and the clients whose agents may have changed, or None when+# any of the tenant's agents may have.+AgentsChangedListener = Callable[[str, frozenset[str] | None], None]+logger = logging.getLogger(__name__)# A client that fails on the database or the network is retried rather than@@ -135,19 +139,26 @@ class ClientLifecycleService:start=self._start_owned_consumer,stop=self._stop_owned_consumer,)- # Told the tenant whenever a re-read starts or stops an agent client- # here, which is how an agent registered or deleted in another process+ # Told the tenant, and the clients whose agents to read again or None+ # for all of them, whenever a re-read may have changed an agent here,+ # which is how an agent registered or deleted in another process# reaches what keeps per-agent state of its own (a bridge's platform# identities, `CollaborationBridgeLifecycleService.agents_changed`).- self._agents_changed_listeners: list[Callable[[str], None]] = []+ self._agents_changed_listeners: list[AgentsChangedListener] = []+ # Each tenant's fingerprint as its last full read found it+ # (`reconcile_tenant`), which the periodic re-check compares with the+ # table's before reading every client again (`recheck_all`).+ self._fingerprints: dict[str, Fingerprint] = {}- def add_agents_changed_listener(self, listener: Callable[[str], None]) -> None:+ def add_agents_changed_listener(self, listener: AgentsChangedListener) -> None:self._agents_changed_listeners.append(listener)- def _agents_changed(self, tenant_id: str) -> None:+ def _agents_changed(+ self, tenant_id: str, client_ids: frozenset[str] | None+ ) -> None:for listener in self._agents_changed_listeners:try:- listener(tenant_id)+ listener(tenant_id, client_ids)except Exception:logger.exception("Telling a listener that tenant %s's agents changed failed",@@ -159,12 +170,15 @@ class ClientLifecycleService:other process starts or stops its own copy (`client_signals.py`)."""self._signals = signals- def announce(self, tenant_id: str) -> None:- """A client of `tenant_id` was registered or deleted, and has- committed: every process reconciles its copies of the tenant's- clients with the table."""+ def announce(+ self, tenant_id: str, client_ids: Collection[str] | None = None+ ) -> None:+ """Clients of `tenant_id` were registered, renamed or deleted, and+ that has committed: every process reconciles its copies of+ `client_ids` with the table, or of all the tenant's clients when no+ ids are given."""if self._signals is not None:- self._signals.changed(tenant_id)+ self._signals.changed(tenant_id, client_ids)@propertydef tenants_isolated(self) -> bool:@@ -175,6 +189,10 @@ class ClientLifecycleService:return f"@{localpart}:{self._config.id_server_name}"COLLAB_CLIENT_TYPES = ("bridge", "user")+ # Collaboration clients (bridges and the people they speak for) are the+ # bridges' to start and stop, and the chats routes write a member client's+ # messages themselves, so neither runs as a client here.+ _NOT_RUN_HERE = (*COLLAB_CLIENT_TYPES, MEMBER_CLIENT_TYPE)async def ensure_system_client(self, client_type: str) -> None:"""One system client of `client_type` per tenant, created where missing.@@ -336,6 +354,23 @@ class ClientLifecycleService:for tenant_id in await all_tenant_ids(self._session_factory):await self.reconcile_tenant(tenant_id)+ async def recheck_all(self) -> None:+ """The periodic re-check: `reconcile_tenant` for every tenant whose+ clients have changed since its last full read here.++ Whether they have is one aggregate query (`ClientStore.fingerprint`),+ where the full read loads every client of the tenant, so a re-check+ with nothing to repair no longer costs a read of every client on every+ process. A change already applied from its signal still counts as a+ change, so the next re-check after it reads in full once."""+ for tenant_id in await all_tenant_ids(self._session_factory):+ async with tenant_session(self._session_factory, tenant_id) as session:+ fingerprint = await self._client_store.fingerprint(+ session, tenant_id, self._NOT_RUN_HERE+ )+ if fingerprint != self._fingerprints.get(tenant_id):+ await self.reconcile_tenant(tenant_id)+async def reconcile_tenant(self, tenant_id: str) -> None:"""Run here exactly the tenant's clients that `start_all` would run.@@ -350,36 +385,91 @@ class ClientLifecycleService:The registration's own signal comes after both have committed.Collaboration clients (bridges and the people they speak for) are thebridges' to start and stop, and a client that crashed stays stopped.++ The read's fingerprint is kept for `recheck_all`. Where it differs+ from the last one, something may have changed that no signal named+ (an agent's description edited elsewhere sends none), so whatever+ keeps per-agent state reads every agent again."""- # Only a client already running before the read can be judged deleted- # by it. One registered in this process while the read awaits has a- # row the read may not have seen, and stopping it would put it in- # `_left_stopped`, where later re-checks leave it.- running_before = [+ running_before = self._running_in(tenant_id, scope=None)+ async with tenant_session(self._session_factory, tenant_id) as session:+ records, fingerprint = await self._client_store.get_running(+ session, tenant_id, self._NOT_RUN_HERE+ )+ changed = await self._apply(tenant_id, records, running_before, scope=None)+ last = self._fingerprints.get(tenant_id)+ self._fingerprints[tenant_id] = fingerprint+ if last is not None and last != fingerprint:+ self._agents_changed(tenant_id, None)+ elif changed:+ self._agents_changed(tenant_id, frozenset(changed))++ async def reconcile_clients(+ self, tenant_id: str, client_ids: Collection[str]+ ) -> None:+ """`reconcile_tenant` for the named clients only: their rows are read,+ and each is started, stopped or renamed here to match, where the full+ read loads every client of the tenant.++ This is what a signal naming its clients does, and with thousands of+ agents it is the difference between reading one row per change and+ every row per change on every process. Whatever keeps per-agent state+ is told the named clients either way, so the process that made the+ change, which started or stopped its own copy already, reads them too.+ """+ scope = frozenset(client_ids)+ if not scope:+ return+ running_before = self._running_in(tenant_id, scope=scope)+ async with tenant_session(self._session_factory, tenant_id) as session:+ records = [+ record+ for record in await self._client_store.get_many(+ session, tenant_id, scope+ )+ if record.type not in self._NOT_RUN_HERE+ ]+ await self._apply(tenant_id, records, running_before, scope=scope)+ self._agents_changed(tenant_id, scope)++ def _running_in(self, tenant_id: str, *, scope: frozenset[str] | None) -> list[str]:+ """The clients of `tenant_id` running here that a read of `scope`+ (every client when None) can judge deleted.++ Taken before the read: only a client already running before it can be+ judged deleted by it. One registered in this process while the read+ awaits has a row the read may not have seen, and stopping it would put+ it in `_left_stopped`, where later re-checks leave it."""+ return [client_idfor client_id, client_tenant in self._client_tenants.items()if client_tenant == tenant_id+ and (scope is None or client_id in scope)and self._client_types.get(client_id) not in self.COLLAB_CLIENT_TYPES]- async with tenant_session(self._session_factory, tenant_id) as session:- records = [- record- for record in await self._client_store.get_all(session)- if record.tenant_id == tenant_id- and record.type not in self.COLLAB_CLIENT_TYPES- # As in `start_all`: the chats routes write a member client's- # messages themselves, so it has nothing to run.- and record.type != MEMBER_CLIENT_TYPE- ]++ async def _apply(+ self,+ tenant_id: str,+ records: list[Client],+ running_before: list[str],+ *,+ scope: frozenset[str] | None,+ ) -> set[str]:+ """Start, stop and rename the copies here of what a read of `scope`+ (every client of the tenant when None) found in `records`. Returns+ the agent clients it started, stopped or renamed."""present = {record.id for record in records}# A client whose row is gone will not be started again: nothing is# left to hold back.self._left_stopped = {client_id: client_tenantfor client_id, client_tenant in self._left_stopped.items()- if client_tenant != tenant_id or client_id in present+ if client_tenant != tenant_id+ or client_id in present+ or (scope is not None and client_id not in scope)}- agents_changed = False+ changed: set[str] = set()# A rename served elsewhere changes the name a copy here sends under# (bridges recognise an agent's own messages by it) and the identities# a bridge here holds for the agent, so both follow it now.@@ -387,11 +477,13 @@ class ClientLifecycleService:actor = self._clients.get(record.id)if actor is not None and actor.display_name != record.display_name:await actor.set_display_name(record.display_name)- agents_changed |= record.type == "agent"+ if record.type == "agent":+ changed.add(record.id)for client_id in running_before:if client_id in self._client_tenants and client_id not in present:logger.info("Client %s was deleted elsewhere; stopping it", client_id)- agents_changed |= self._client_types.get(client_id) == "agent"+ if self._client_types.get(client_id) == "agent":+ changed.add(client_id)await self.stop(client_id)missing = [record@@ -399,9 +491,7 @@ class ClientLifecycleService:if record.id not in self._clients and record.id not in self._left_stopped]if not missing:- if agents_changed:- self._agents_changed(tenant_id)- return+ return changedagents = await self._agents_by_client_id(missing)for record in missing:agent = agents.get(record.id)@@ -419,9 +509,9 @@ class ClientLifecycleService:actor = self._register(record)if agent is not None and isinstance(actor, _PreloadsAgent):actor.preload_agent(agent)- agents_changed |= agent is not None- if agents_changed:- self._agents_changed(tenant_id)+ if agent is not None:+ changed.add(record.id)+ return changedasync def _agents_by_client_id(self, records: list[Client]) -> dict[str, Agent]:"""The agent row behind each client, one query per tenant.@@ -588,7 +678,7 @@ class ClientLifecycleService:"""tenant_id = self._left_stopped.pop(client_id, None)if tenant_id is not None:- self.announce(tenant_id)+ self.announce(tenant_id, [client_id])def get(self, client_id: str) -> Actor[ClientConfig] | None:return self._clients.get(client_id)diff --git a/core/switch_core/clients/client_signals.py b/core/switch_core/clients/client_signals.pyindex 7ae6452e7..93507936c 100644--- a/core/switch_core/clients/client_signals.py+++ b/core/switch_core/clients/client_signals.py@@ -7,40 +7,94 @@ any other process answered that agent's requests with "not running" and readnothing for it; a client deleted kept its copies running everywhere else, on arow that no longer exists.-So a registration or deletion, once committed, announces `client.changed` for-its tenant. Every process, the announcing one included, then re-reads the-tenant's clients and starts or stops its copies to match-(`ClientLifecycleService.reconcile_tenant`). The announcing process has-already started or stopped its own, so for it the re-read is a check that-changes nothing. A re-read that fails is kept and tried again, and every-tenant is re-read when the signal connection comes back and on each periodic-re-check, so a missed signal costs a delay rather than a client missing from-a process until it restarts.+So a registration, rename or deletion, once committed, announces+`client.changed` for its tenant, naming the clients it changed. Every process,+the announcing one included, then re-reads those clients' rows and starts,+stops or renames its copies to match (`ClientLifecycleService.+reconcile_clients`). The announcing process has already done so for its own,+so for it the re-read is a check that changes nothing. A signal naming no+clients, which a process from before they were named sends, re-reads every+client of the tenant (`reconcile_tenant`).++A re-read that fails is kept and tried again. Every tenant is re-read when the+signal connection comes back, and each periodic re-check reads in full any+tenant whose clients have changed since its last full read+(`ClientLifecycleService.recheck_all`), so a missed signal costs a delay rather+than a client missing from a process until it restarts."""from __future__ import annotations+from collections.abc import Collection, Iterablefrom typing import TYPE_CHECKING, Any+from switch_core.signals import payload_chunks+if TYPE_CHECKING:from switch_core.clients.client_lifecycle_service import ClientLifecycleServicefrom switch_core.signals import SignalBusCLIENT_CHANGED_SIGNAL = "client.changed"+# Room kept in every signal for what it carries beside its list of clients:+# its kind, the sending process's id and the tenant, about 200 bytes at most.+_SIGNAL_ENVELOPE_BYTES = 400+class ClientSignals:def __init__(self, *, lifecycle: ClientLifecycleService, bus: SignalBus) -> None:self._bus = bus- self._owed = bus.pending_work("clients", lifecycle.reconcile_tenant)+ self._lifecycle = lifecycle+ # The clients each tenant with work pending has to read again, or None+ # where every client of the tenant has to be.+ self._owed_clients: dict[str, set[str] | None] = {}+ self._owed = bus.pending_work("clients", self._reconcile)bus.handle(CLIENT_CHANGED_SIGNAL, self._on_changed)bus.on_resync(lifecycle.reconcile_all)- bus.on_recheck(lifecycle.reconcile_all)+ bus.on_recheck(lifecycle.recheck_all)lifecycle.use_signals(self)- def changed(self, tenant_id: str) -> None:- self._owed.add(tenant_id)- self._bus.publish(CLIENT_CHANGED_SIGNAL, tenant_id=tenant_id)+ def changed(self, tenant_id: str, client_ids: Collection[str] | None) -> None:+ if client_ids is None:+ self._owe(tenant_id, None)+ self._bus.publish(CLIENT_CHANGED_SIGNAL, tenant_id=tenant_id)+ return+ named = sorted(set(client_ids))+ self._owe(tenant_id, named)+ for chunk in payload_chunks(named, _SIGNAL_ENVELOPE_BYTES):+ self._bus.publish(+ CLIENT_CHANGED_SIGNAL, tenant_id=tenant_id, client_ids=chunk+ )def _on_changed(self, signal: dict[str, Any]) -> None:- self._owed.add(signal["tenant_id"])+ client_ids = signal.get("client_ids")+ self._owe(+ signal["tenant_id"], client_ids if isinstance(client_ids, list) else None+ )++ def _owe(self, tenant_id: str, client_ids: Iterable[str] | None) -> None:+ self._keep(tenant_id, client_ids)+ self._owed.add(tenant_id)++ def _keep(self, tenant_id: str, client_ids: Iterable[str] | None) -> None:+ if client_ids is None:+ self._owed_clients[tenant_id] = None+ return+ owed = self._owed_clients.setdefault(tenant_id, set())+ if owed is not None:+ owed.update(client_ids)++ async def _reconcile(self, tenant_id: str) -> None:+ # No entry reads every client: nothing is lost by reading too much.+ client_ids = self._owed_clients.pop(tenant_id, None)+ try:+ if client_ids is None:+ await self._lifecycle.reconcile_tenant(tenant_id)+ else:+ await self._lifecycle.reconcile_clients(tenant_id, client_ids)+ except BaseException:+ # Kept for the retry `PendingWork` makes of the tenant, which it+ # schedules itself: adding the tenant again here would retry at+ # once instead.+ self._keep(tenant_id, client_ids)+ raisediff --git a/core/switch_core/db/stores/client_store.py b/core/switch_core/db/stores/client_store.pyindex 0dc1044f3..774b99506 100644--- a/core/switch_core/db/stores/client_store.py+++ b/core/switch_core/db/stores/client_store.py@@ -1,7 +1,38 @@-from sqlalchemy import delete, select+from collections.abc import Collection++from sqlalchemy import BigInteger, ColumnElement, cast, delete, func, literal, select+from sqlalchemy.dialects.postgresql import BITfrom sqlalchemy.ext.asyncio import AsyncSession-from switch_core.db.models import Client, ClientRoom, require_tenant_id+from switch_core.db.models import Agent, Client, ClientRoom, require_tenant_id++# How many clients a tenant runs and what they are, as one pair of numbers:+# the count, and the sum of a 60-bit hash of each client's id, type and name+# and of its agent's id, name, display name and description. Equal pairs mean+# no client was added, removed or renamed and no agent was added, removed or+# re-described in between. Computed by the database, so reading it costs one+# row however many clients the tenant has.+Fingerprint = tuple[int, int]++_FIELD_SEPARATOR = "\x1f"+++def _row_hash() -> ColumnElement[int]:+ fields = func.concat_ws(+ _FIELD_SEPARATOR,+ Client.id,+ Client.type,+ Client.display_name,+ func.coalesce(Agent.id, ""),+ func.coalesce(Agent.name, ""),+ func.coalesce(Agent.display_name, ""),+ func.coalesce(Agent.description, ""),+ )+ # The first 15 hex digits of the md5, read as a 60-bit number.+ return cast(+ cast(literal("x") + func.substr(func.md5(fields), 1, 15), BIT(60)),+ BigInteger,+ )class ClientStore:@@ -12,6 +43,49 @@ class ClientStore:async def get(self, session: AsyncSession, client_id: str) -> Client | None:return await session.get(Client, client_id)+ async def get_many(+ self, session: AsyncSession, tenant_id: str, client_ids: Collection[str]+ ) -> list[Client]:+ """The tenant's clients among `client_ids`, in one query. An id with+ no row in the tenant is left out."""+ if not client_ids:+ return []+ result = await session.execute(+ select(Client).where(+ Client.tenant_id == tenant_id, Client.id.in_(list(client_ids))+ )+ )+ return list(result.scalars().all())++ async def get_running(+ self, session: AsyncSession, tenant_id: str, excluded_types: Collection[str]+ ) -> tuple[list[Client], Fingerprint]:+ """The tenant's clients of every type but `excluded_types`, and the+ fingerprint of those rows, from one statement and so from one+ snapshot of the table."""+ result = await session.execute(+ select(Client, func.count().over(), func.sum(_row_hash()).over())+ .outerjoin(Agent, Agent.client_id == Client.id)+ .where(Client.tenant_id == tenant_id, Client.type.not_in(excluded_types))+ )+ rows = result.all()+ if not rows:+ return [], (0, 0)+ return [row[0] for row in rows], (int(rows[0][1]), int(rows[0][2]))++ async def fingerprint(+ self, session: AsyncSession, tenant_id: str, excluded_types: Collection[str]+ ) -> Fingerprint:+ """The fingerprint `get_running` returns, without the rows."""+ result = await session.execute(+ select(func.count(), func.coalesce(func.sum(_row_hash()), 0))+ .select_from(Client)+ .outerjoin(Agent, Agent.client_id == Client.id)+ .where(Client.tenant_id == tenant_id, Client.type.not_in(excluded_types))+ )+ count, total = result.one()+ return int(count), int(total)+async def get_by_transport_user_id(self, session: AsyncSession, transport_user_id: str) -> Client | None:diff --git a/core/tests/switch_core/bridges/agent/protocol/registration_harness.py b/core/tests/switch_core/bridges/agent/protocol/registration_harness.pyindex 7007fe19c..e755917c3 100644--- a/core/tests/switch_core/bridges/agent/protocol/registration_harness.py+++ b/core/tests/switch_core/bridges/agent/protocol/registration_harness.py@@ -59,7 +59,7 @@ class FakeClientLifecycle:def start_client(self, client: Client) -> None:self.started.append(client.id)- def announce(self, tenant_id: str) -> None:+ def announce(self, tenant_id: str, client_ids: object = None) -> None:return Nonediff --git a/core/tests/switch_core/bridges/agent/protocol/test_agent_tenant_scoped_admin.py b/core/tests/switch_core/bridges/agent/protocol/test_agent_tenant_scoped_admin.pyindex fe243868e..a370a5503 100644--- a/core/tests/switch_core/bridges/agent/protocol/test_agent_tenant_scoped_admin.py+++ b/core/tests/switch_core/bridges/agent/protocol/test_agent_tenant_scoped_admin.py@@ -75,7 +75,7 @@ class _FakeClientLifecycle:def start_client(self, client: Client) -> None:pass- def announce(self, tenant_id: str) -> None:+ def announce(self, tenant_id: str, client_ids: object = None) -> None:return Nonediff --git a/core/tests/switch_core/bridges/agent/protocol/test_bridge_identity_tenant_scope.py b/core/tests/switch_core/bridges/agent/protocol/test_bridge_identity_tenant_scope.pyindex 0fbe9113a..d65f8ff04 100644--- a/core/tests/switch_core/bridges/agent/protocol/test_bridge_identity_tenant_scope.py+++ b/core/tests/switch_core/bridges/agent/protocol/test_bridge_identity_tenant_scope.py@@ -144,7 +144,7 @@ class _StoppableClientLifecycle:def start_client(self, client: Client) -> None:pass- def announce(self, tenant_id: str) -> None:+ def announce(self, tenant_id: str, client_ids: object = None) -> None:return Noneasync def stop(self, client_id: str) -> None:diff --git a/core/tests/switch_core/bridges/agent/protocol/test_register_overwrite_owner_guard.py b/core/tests/switch_core/bridges/agent/protocol/test_register_overwrite_owner_guard.pyindex fa95f0f64..8ea8a2660 100644--- a/core/tests/switch_core/bridges/agent/protocol/test_register_overwrite_owner_guard.py+++ b/core/tests/switch_core/bridges/agent/protocol/test_register_overwrite_owner_guard.py@@ -55,7 +55,7 @@ class _FakeClientLifecycle:def start_client(self, client: Client) -> None:pass- def announce(self, tenant_id: str) -> None:+ def announce(self, tenant_id: str, client_ids: object = None) -> None:return Nonediff --git a/core/tests/switch_core/bridges/agent/protocol/test_room_roster_authz.py b/core/tests/switch_core/bridges/agent/protocol/test_room_roster_authz.pyindex e6a88b5d0..2ee603a33 100644--- a/core/tests/switch_core/bridges/agent/protocol/test_room_roster_authz.py+++ b/core/tests/switch_core/bridges/agent/protocol/test_room_roster_authz.py@@ -53,7 +53,7 @@ class _FakeClientLifecycle:def start_client(self, client: Client) -> None:pass- def announce(self, tenant_id: str) -> None:+ def announce(self, tenant_id: str, client_ids: object = None) -> None:return Nonediff --git a/core/tests/switch_core/bridges/agent/test_api_key_cache.py b/core/tests/switch_core/bridges/agent/test_api_key_cache.pyindex 302abb14e..dfd9fcd96 100644--- a/core/tests/switch_core/bridges/agent/test_api_key_cache.py+++ b/core/tests/switch_core/bridges/agent/test_api_key_cache.py@@ -60,7 +60,7 @@ class _FakeClientLifecycle:def start_client(self, client: Client) -> None:self.started.append(client.id)- def announce(self, tenant_id: str) -> None:+ def announce(self, tenant_id: str, client_ids: object = None) -> None:return Noneasync def stop(self, client_id: str) -> None:diff --git a/core/tests/switch_core/bridges/collaboration/test_bridge_identity_backfill.py b/core/tests/switch_core/bridges/collaboration/test_bridge_identity_backfill.pyindex de071f676..4c93b4112 100644--- a/core/tests/switch_core/bridges/collaboration/test_bridge_identity_backfill.py+++ b/core/tests/switch_core/bridges/collaboration/test_bridge_identity_backfill.py@@ -49,6 +49,9 @@ def _core(provision: Any) -> tuple[CollaborationCore, _FakeAdapter]:core._bridge_type = "slack" # type: ignore[attr-defined]core._adapter = adapter # type: ignore[attr-defined]core._identity_task = None # type: ignore[attr-defined]+ core._identities_stale = True # type: ignore[attr-defined]+ core._identities_owed = set() # type: ignore[attr-defined]+ core._agent_view = None # type: ignore[attr-defined]core._channel_type_refresh_task = None # type: ignore[attr-defined]async def _noop() -> None:diff --git a/core/tests/switch_core/bridges/collaboration/test_bridge_identity_changes.py b/core/tests/switch_core/bridges/collaboration/test_bridge_identity_changes.pynew file mode 100644index 000000000..8b2fbb8e3--- /dev/null+++ b/core/tests/switch_core/bridges/collaboration/test_bridge_identity_changes.py@@ -0,0 +1,221 @@+"""A bridge told which agents changed reads only those.++Every process runs a copy of each bridge, and every copy is told when an+agent is registered, renamed or deleted anywhere in its tenant. Each used to+read every agent of the tenant then; with thousands of agents being created,+that kept every process busy for as long as the creation lasted. The copy now+keeps the tenant's agents as it last read them and reads again only the+agents a change names (`CollaborationCore._update_agent_identities`).+"""++from __future__ import annotations++import asyncio+from collections.abc import AsyncIterator, Sequence+from contextlib import asynccontextmanager+from types import SimpleNamespace+from typing import Any++from switch_core.bridges.collaboration.adapter import AgentIdentity+from switch_core.bridges.collaboration.collaboration_core import CollaborationCore++_TENANT = "tenant-a"+++def _agent(name: str, *, description: str = "helps") -> SimpleNamespace:+ return SimpleNamespace(+ id=f"agent-{name}",+ client_id=f"client-{name}",+ name=name,+ description=description,+ display_name=None,+ )+++class _Store:+ """The tenant's agent rows, and what was read of them."""++ def __init__(self, *agents: SimpleNamespace) -> None:+ self.rows = {agent.client_id: agent for agent in agents}+ self.full_reads = 0+ self.named_reads: list[list[str]] = []+ self.hold_full_read: asyncio.Event | None = None++ async def get_all(self, session: Any) -> list[SimpleNamespace]:+ self.full_reads += 1+ if self.hold_full_read is not None:+ await self.hold_full_read.wait()+ return list(self.rows.values())++ async def get_by_client_ids(+ self, session: Any, client_ids: list[str]+ ) -> list[SimpleNamespace]:+ self.named_reads.append(client_ids)+ return [self.rows[c] for c in client_ids if c in self.rows]+++class _Adapter:+ """What the platform was asked to do."""++ def __init__(self) -> None:+ self.calls: list[tuple[str, Any]] = []+ self.provisioned: list[list[str]] = []++ async def remove_agent_identity(self, agent: AgentIdentity) -> None:+ self.calls.append(("remove", agent.name))++ async def rename_agent_identity(self, old_name: str, agent: AgentIdentity) -> None:+ self.calls.append(("rename", (old_name, agent.name)))++ async def update_agent_identity(self, agent: AgentIdentity) -> None:+ self.calls.append(("update", agent.name))++ async def sync_agent_identities(+ self, agents: Sequence[AgentIdentity]+ ) -> list[tuple[str, BaseException]]:+ self.provisioned.append(sorted(agent.name for agent in agents))+ return []+++@asynccontextmanager+async def _session() -> AsyncIterator[object]:+ yield object()+++def _core(store: _Store) -> tuple[CollaborationCore, _Adapter]:+ """A CollaborationCore with only what identity provisioning touches."""+ core = object.__new__(CollaborationCore)+ adapter = _Adapter()+ core._bridge_type = "slack" # type: ignore[attr-defined]+ core._bridge_tenant_id = _TENANT # type: ignore[attr-defined]+ core._session_factory = _session # type: ignore[attr-defined]+ core._agent_store = store # type: ignore[attr-defined]+ core._adapter = adapter # type: ignore[attr-defined]+ core._identity_task = None # type: ignore[attr-defined]+ core._provisioned_agents = {} # type: ignore[attr-defined]+ core._agent_view = None # type: ignore[attr-defined]+ core._agent_of_client = {} # type: ignore[attr-defined]+ core._identities_stale = False # type: ignore[attr-defined]+ core._identities_owed = set() # type: ignore[attr-defined]+ return core, adapter+++async def _settled(core: CollaborationCore) -> None:+ task = core._identity_task+ assert task is not None+ await asyncio.wait_for(task, timeout=1)+++async def _started(store: _Store) -> tuple[CollaborationCore, _Adapter]:+ """A bridge copy that has read and provisioned every agent, as at start."""+ core, adapter = _core(store)+ core._provision_identities_on_attach()+ await _settled(core)+ adapter.calls.clear()+ adapter.provisioned.clear()+ return core, adapter+++async def test_a_new_agent_is_read_alone_and_provisioned() -> None:+ store = _Store(_agent("scout"), _agent("scribe"))+ core, adapter = await _started(store)+ store.rows["client-tracker"] = _agent("tracker")++ core.agents_changed({"client-tracker"})+ await _settled(core)++ assert store.full_reads == 1+ assert store.named_reads == [["client-tracker"]]+ assert adapter.provisioned == [["scout", "scribe", "tracker"]]+++async def test_an_agent_deleted_elsewhere_loses_its_identity_here() -> None:+ store = _Store(_agent("scout"), _agent("scribe"))+ core, adapter = await _started(store)+ del store.rows["client-scribe"]++ core.agents_changed({"client-scribe"})+ await _settled(core)++ assert adapter.calls == [("remove", "scribe")]+ assert adapter.provisioned == [["scout"]]+ assert "agent-scribe" not in core._provisioned_agents+ assert store.full_reads == 1+++async def test_an_agent_renamed_elsewhere_is_renamed_here() -> None:+ store = _Store(_agent("scout"))+ core, adapter = await _started(store)+ renamed = _agent("scout")+ renamed.name = "pathfinder"+ store.rows["client-scout"] = renamed++ core.agents_changed({"client-scout"})+ await _settled(core)++ assert adapter.calls == [("rename", ("scout", "pathfinder"))]+ assert adapter.provisioned == [["pathfinder"]]+++async def test_agents_not_named_are_left_as_last_read() -> None:+ """A description edited elsewhere sends no signal of its own; the client+ re-check finds it and reads every agent again (`None`)."""+ store = _Store(_agent("scout"), _agent("scribe"))+ core, adapter = await _started(store)+ store.rows["client-scribe"] = _agent("scribe", description="writes things")++ core.agents_changed({"client-scout"})+ await _settled(core)+ assert adapter.calls == []++ core.agents_changed(None)+ await _settled(core)+ assert adapter.calls == [("update", "scribe")]+ assert store.full_reads == 2+++async def test_a_change_before_any_full_read_reads_every_agent() -> None:+ store = _Store(_agent("scout"))+ core, adapter = _core(store)++ core.agents_changed({"client-scout"})+ await _settled(core)++ assert store.full_reads == 1+ assert store.named_reads == []+ assert adapter.provisioned == [["scout"]]+++async def test_a_change_during_a_full_read_is_read_after_it() -> None:+ """The full read may have seen the tenant before the change committed."""+ store = _Store(_agent("scout"))+ core, adapter = await _started(store)+ store.hold_full_read = asyncio.Event()+ core.agents_changed(None)+ await asyncio.sleep(0)+ assert store.full_reads == 2++ store.rows["client-tracker"] = _agent("tracker")+ core.agents_changed({"client-tracker"})+ store.hold_full_read.set()+ await _settled(core)++ assert store.named_reads == [["client-tracker"]]+ assert adapter.provisioned[-1] == ["scout", "tracker"]+++async def test_a_change_during_the_first_read_reads_every_agent_again() -> None:+ """With nothing read yet there is nothing to read the named agents into."""+ store = _Store(_agent("scout"))+ core, adapter = _core(store)+ store.hold_full_read = asyncio.Event()+ core._provision_identities_on_attach()+ await asyncio.sleep(0)++ store.rows["client-tracker"] = _agent("tracker")+ core.agents_changed({"client-tracker"})+ store.hold_full_read.set()+ await _settled(core)++ assert store.full_reads == 2+ assert adapter.provisioned[-1] == ["scout", "tracker"]diff --git a/core/tests/switch_core/bridges/collaboration/test_bridge_inbound_tenant_binding.py b/core/tests/switch_core/bridges/collaboration/test_bridge_inbound_tenant_binding.pyindex 77a3f4b6f..27af71d52 100644--- a/core/tests/switch_core/bridges/collaboration/test_bridge_inbound_tenant_binding.py+++ b/core/tests/switch_core/bridges/collaboration/test_bridge_inbound_tenant_binding.py@@ -299,6 +299,8 @@ class TestBridgeLevelWorkBindsTheBridgesTenant:await original()bridge._create_agent_identities = _watched # type: ignore[method-assign]+ # Every agent is to be read, as `start` asks.+ bridge._identities_stale = Truewith tenant_scope("tenant-whoever-started-the-bridge"):await bridge._run_agent_identities()diff --git a/core/tests/switch_core/clients/test_client_signals.py b/core/tests/switch_core/clients/test_client_signals.pyindex 7fe65b1c9..3dfca8cd8 100644--- a/core/tests/switch_core/clients/test_client_signals.py+++ b/core/tests/switch_core/clients/test_client_signals.py@@ -21,7 +21,7 @@ from sqlalchemy.pool import NullPoolfrom switch_core.chats import MEMBER_CLIENT_TYPEfrom switch_core.clients.client_lifecycle_service import ClientLifecycleService-from switch_core.clients.client_signals import ClientSignals+from switch_core.clients.client_signals import CLIENT_CHANGED_SIGNAL, ClientSignalsfrom switch_core.db.models import TENANT_ZERO_ID, Agent, Clientfrom switch_core.db.stores.client_store import ClientStorefrom switch_core.db.stores.tenant_store import TenantStore@@ -140,7 +140,7 @@ class TestAcrossProcesses:agent = await _register(session_factory)assert agent.client_id is not None- here.announce(TENANT_ZERO_ID)+ here.announce(TENANT_ZERO_ID, [agent.client_id])await _eventually(lambda: there.get(agent.client_id) is not None) # type: ignore[arg-type]# The announcing process re-reads too, and starts what it lacks.@@ -158,10 +158,12 @@ class TestAcrossProcesses:agent = await _register(session_factory)client_id = agent.client_idassert client_id is not None- here.announce(TENANT_ZERO_ID)+ here.announce(TENANT_ZERO_ID, [client_id])await _eventually(lambda: there.get(client_id) is not None)- told: list[str] = []- there.add_agents_changed_listener(told.append)+ told: list[tuple[str, frozenset[str] | None]] = []+ there.add_agents_changed_listener(+ lambda tenant_id, client_ids: told.append((tenant_id, client_ids))+ )# As `AgentCore.rename_agent` does: rename the client row, announce.async with session_factory() as session:@@ -169,14 +171,14 @@ class TestAcrossProcesses:assert client is not Noneclient.display_name = "renamed-agent"await session.commit()- here.announce(TENANT_ZERO_ID)+ here.announce(TENANT_ZERO_ID, [client_id])await _eventually(lambda: (getattr(there.get(client_id), "display_name", None) == "renamed-agent"))- await _eventually(lambda: TENANT_ZERO_ID in told)+ await _eventually(lambda: (TENANT_ZERO_ID, frozenset({client_id})) in told)async def test_a_client_deleted_in_one_process_stops_in_the_other(self,@@ -187,18 +189,63 @@ class TestAcrossProcesses:agent = await _register(session_factory)client_id = agent.client_idassert client_id is not None- here.announce(TENANT_ZERO_ID)+ here.announce(TENANT_ZERO_ID, [client_id])await _eventually(lambda: there.get(client_id) is not None)copy = there.get(client_id)# As `AgentCore.delete_agent` does: stop here, delete, announce.await here.stop(client_id)await _delete(session_factory, agent)- here.announce(TENANT_ZERO_ID)+ here.announce(TENANT_ZERO_ID, [client_id])await _eventually(lambda: there.get(client_id) is None)assert isinstance(copy, _Consumer) and copy.stopped+ async def test_a_change_reads_only_the_clients_it_names(+ self,+ session_factory: async_sessionmaker[AsyncSession],+ two_processes: tuple[ClientLifecycleService, ClientLifecycleService],+ ) -> None:+ """With thousands of agents, a change that made every process read+ every client of the tenant kept every process busy for as long as+ agents were being created."""+ here, there = two_processes+ # The read every process makes once its signal connection is up.+ await _eventually(lambda: TENANT_ZERO_ID in there._fingerprints)+ full_reads = _count_full_reads(there)+ agent = await _register(session_factory)+ client_id = agent.client_id+ assert client_id is not None+ told: list[tuple[str, frozenset[str] | None]] = []+ there.add_agents_changed_listener(+ lambda tenant_id, client_ids: told.append((tenant_id, client_ids))+ )++ here.announce(TENANT_ZERO_ID, [client_id])++ await _eventually(lambda: there.get(client_id) is not None)+ await _eventually(lambda: (TENANT_ZERO_ID, frozenset({client_id})) in told)+ assert full_reads == []++ async def test_a_signal_naming_no_clients_reads_the_whole_tenant(+ self,+ postgres_url: str,+ session_factory: async_sessionmaker[AsyncSession],+ two_processes: tuple[ClientLifecycleService, ClientLifecycleService],+ ) -> None:+ """What a process from before signals named their clients sends,+ during a rolling deploy."""+ _here, there = two_processes+ agent = await _register(session_factory)+ older = _bus(postgres_url, session_factory)+ await older.start()+ try:+ older.publish(CLIENT_CHANGED_SIGNAL, tenant_id=TENANT_ZERO_ID)++ await _eventually(lambda: there.get(agent.client_id) is not None) # type: ignore[arg-type]+ finally:+ await older.stop()+class TestReconcile:async def test_the_recheck_starts_a_client_whose_signal_was_missed(@@ -278,11 +325,11 @@ class TestReconcile:lifecycle = _lifecycle(session_factory)await lifecycle.start_all()store = lifecycle._client_store- real_get_all = store.get_all+ real_get_running = store.get_runningregistered: list[str] = []- async def read_then_register(session: Any) -> Any:- records = await real_get_all(session)+ async def read_then_register(session: Any, *args: Any) -> Any:+ records = await real_get_running(session, *args)if not registered:# As `AgentCore._create_agent` does: commit, then start.agent = await _register(session_factory)@@ -293,11 +340,96 @@ class TestReconcile:registered.append(record.id)return records- store.get_all = read_then_register # type: ignore[method-assign]+ store.get_running = read_then_register # type: ignore[method-assign]try:await lifecycle.reconcile_tenant(TENANT_ZERO_ID)finally:- store.get_all = real_get_all # type: ignore[method-assign]+ store.get_running = real_get_running # type: ignore[method-assign]assert lifecycle.get(registered[0]) is not Noneawait lifecycle.stop_all()++ async def test_a_named_client_stopped_on_purpose_stays_stopped(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ """As above, for a re-read of the named clients only."""+ agent = await _register(session_factory)+ assert agent.client_id is not None+ lifecycle = _lifecycle(session_factory)+ await lifecycle.start_all()+ await lifecycle.stop(agent.client_id)++ await lifecycle.reconcile_clients(TENANT_ZERO_ID, [agent.client_id])++ assert lifecycle.get(agent.client_id) is None+++class TestRecheck:+ """The periodic re-check, which every process runs every 30 seconds:+ a read of every client of every tenant, until it compared fingerprints+ first."""++ async def test_with_nothing_changed_it_reads_no_client(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ lifecycle = _lifecycle(session_factory)+ await lifecycle.start_all()+ await lifecycle.reconcile_all()+ full_reads = _count_full_reads(lifecycle)++ await lifecycle.recheck_all()++ assert full_reads == []+ await lifecycle.stop_all()++ async def test_it_starts_a_client_whose_signal_was_missed(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ lifecycle = _lifecycle(session_factory)+ await lifecycle.start_all()+ await lifecycle.reconcile_all()+ agent = await _register(session_factory)++ await lifecycle.recheck_all()++ assert lifecycle.get(agent.client_id) is not None # type: ignore[arg-type]+ await lifecycle.stop_all()++ async def test_an_agent_edited_with_no_signal_is_read_again_by_the_bridges(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ """An agent's description edited in another process sends no signal,+ and a bridge here keeps the description with the agent's platform+ handle."""+ agent = await _register(session_factory)+ lifecycle = _lifecycle(session_factory)+ await lifecycle.start_all()+ await lifecycle.reconcile_all()+ told: list[tuple[str, frozenset[str] | None]] = []+ lifecycle.add_agents_changed_listener(+ lambda tenant_id, client_ids: told.append((tenant_id, client_ids))+ )+ async with session_factory() as session:+ row = await session.get(Agent, agent.id)+ assert row is not None+ row.description = "does something else now"+ await session.commit()++ await lifecycle.recheck_all()++ assert told == [(TENANT_ZERO_ID, None)]+ await lifecycle.stop_all()+++def _count_full_reads(lifecycle: ClientLifecycleService) -> list[str]:+ """The tenants `lifecycle` reads every client of from now on."""+ reads: list[str] = []+ store = lifecycle._client_store+ real_get_running = store.get_running++ async def counted(session: Any, tenant_id: str, *args: Any) -> Any:+ reads.append(tenant_id)+ return await real_get_running(session, tenant_id, *args)++ store.get_running = counted # type: ignore[method-assign]+ return readsdiff --git a/core/tests/switch_core/db/stores/test_client_store.py b/core/tests/switch_core/db/stores/test_client_store.pyindex 1d3bd760f..292009fa9 100644--- a/core/tests/switch_core/db/stores/test_client_store.py+++ b/core/tests/switch_core/db/stores/test_client_store.py@@ -3,13 +3,15 @@ from __future__ import annotationsimport uuidimport pytest+from sqlalchemy import delete, inspectfrom sqlalchemy.exc import IntegrityErrorfrom sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker-from switch_core.db.models import Client, Room, Tenant+from switch_core.db.models import TENANT_ZERO_ID, Agent, Client, Room, Tenantfrom switch_core.db.session_scope import tenant_sessionfrom switch_core.db.stores.client_store import ClientStorefrom switch_core.db.stores.room_store import RoomStore+from tests.switch_core.test_room_wide_mention_wakes_no_agent import _make_agentasync def _make_client(session: AsyncSession, transport_user_id: str) -> Client:@@ -216,3 +218,128 @@ class TestGetByTransportUserIdMultiTenant:)assert result is not Noneassert result.id == other_id+++class TestFingerprint:+ """What the periodic re-check compares before reading every client+ (`ClientLifecycleService.recheck_all`): equal fingerprints have to mean+ nothing a process runs from the rows changed, and each of these changes+ has to show."""++ _NOT_RUN = ("bridge", "user", "member")++ async def _fingerprint(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> tuple[int, int]:+ async with tenant_session(session_factory, TENANT_ZERO_ID) as session:+ return await ClientStore().fingerprint(+ session, TENANT_ZERO_ID, self._NOT_RUN+ )++ async def test_the_rows_and_their_fingerprint_come_back_together(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ async with session_factory() as session:+ await _make_agent(session, f"fp-{uuid.uuid4().hex[:8]}")+ await _make_client(session, f"@plain-{uuid.uuid4().hex[:8]}:test")+ await session.commit()++ async with tenant_session(session_factory, TENANT_ZERO_ID) as session:+ records, fingerprint = await ClientStore().get_running(+ session, TENANT_ZERO_ID, self._NOT_RUN+ )++ assert fingerprint == await self._fingerprint(session_factory)+ assert fingerprint[0] == len(records) >= 2++ async def test_a_rename_shows(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ async with session_factory() as session:+ client = await _make_client(session, f"@named-{uuid.uuid4().hex[:8]}:test")+ await session.commit()+ before = await self._fingerprint(session_factory)++ async with session_factory() as session:+ row = await session.get(Client, client.id)+ assert row is not None+ row.display_name = "renamed"+ await session.commit()++ after = await self._fingerprint(session_factory)+ assert after[0] == before[0]+ assert after != before++ async def test_an_agent_row_arriving_after_its_client_shows(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ """Registration commits the client row first. A full read between the+ two skips the client, so the agent row's arrival has to count as a+ change for the next re-check to start it."""+ async with session_factory() as session:+ agent = await _make_agent(session, f"late-{uuid.uuid4().hex[:8]}")+ await session.commit()+ columns = {+ attr.key: getattr(agent, attr.key)+ for attr in inspect(Agent).column_attrs+ }+ async with session_factory() as session:+ await session.execute(delete(Agent).where(Agent.id == agent.id))+ await session.commit()+ client_only = await self._fingerprint(session_factory)++ async with session_factory() as session:+ session.add(Agent(**columns))+ await session.commit()++ assert await self._fingerprint(session_factory) != client_only++ async def test_an_agents_description_shows(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ """No signal names a description edited in another process, and a+ bridge there keeps it with the agent's platform handle."""+ async with session_factory() as session:+ agent = await _make_agent(session, f"desc-{uuid.uuid4().hex[:8]}")+ await session.commit()+ before = await self._fingerprint(session_factory)++ async with session_factory() as session:+ row = await session.get(Agent, agent.id)+ assert row is not None+ row.description = "does something else now"+ await session.commit()++ assert await self._fingerprint(session_factory) != before++ async def test_clients_not_run_as_clients_do_not_count(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ before = await self._fingerprint(session_factory)++ async with session_factory() as session:+ session.add(+ Client(+ transport_user_id=f"@person-{uuid.uuid4().hex[:8]}:test",+ display_name="a person a bridge speaks for",+ type="user",+ )+ )+ await session.commit()++ assert await self._fingerprint(session_factory) == before++ async def test_another_tenants_clients_do_not_count(+ self, session_factory: async_sessionmaker[AsyncSession]+ ) -> None:+ other_tenant = f"tenant-{uuid.uuid4().hex[:8]}"+ async with session_factory() as session:+ session.add(Tenant(id=other_tenant, slug=other_tenant, name=other_tenant))+ await session.commit()+ before = await self._fingerprint(session_factory)++ async with tenant_session(session_factory, other_tenant) as other:+ await _make_client(other, f"@theirs-{uuid.uuid4().hex[:8]}:test")+ await other.commit()++ assert await self._fingerprint(session_factory) == beforediff --git a/core/tests/switch_core/management/harness.py b/core/tests/switch_core/management/harness.pyindex c3eaa0adc..b3c30557c 100644--- a/core/tests/switch_core/management/harness.py+++ b/core/tests/switch_core/management/harness.py@@ -131,7 +131,7 @@ class _FakeClientLifecycle:def start_client(self, client: Client) -> None:return None- def announce(self, tenant_id: str) -> None:+ def announce(self, tenant_id: str, client_ids: object = None) -> None:return Nonedef get_by_agent_id(self, agent_id: str) -> None: