Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Find a running client by type or by agent without looking at every client (CHOO-3667)

Abel Dantas788abfdc94c2
diff --git a/CHANGELOG.md b/CHANGELOG.md
index b7fc419f6..5bf60b8da 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -166,6 +166,12 @@ version of their own to them without also giving them a release of their own.
compiled mention patterns are kept per name. In a CPU profile under
message load the event loop's own work was about 15% of a pod's CPU and
the actor lookup about 3%.
+- **Creating a room or an agent no longer looks through every running
+ client.** Every process keeps an actor for every client, and finding a
+ workspace's admin, or the actor of an agent registered in another process
+ a moment before, read all of them. Each is now found from an index of the
+ running clients by type and by agent. While 10,000 agents were being
+ created, these lookups took about 11% of a process's CPU in a profile.
- **An agent's stream reads in one database session per pass, and pooled
connections are no longer pinged before use.** A pass that delivered a
message opened a session for each of its reads, six pool checkouts and six
diff --git a/core/switch_core/clients/client_lifecycle_service.py b/core/switch_core/clients/client_lifecycle_service.py
index 309a11e2e..bc9eac494 100644
--- a/core/switch_core/clients/client_lifecycle_service.py
+++ b/core/switch_core/clients/client_lifecycle_service.py
@@ -108,8 +108,13 @@ class ClientLifecycleService:
# Every running row is an actor; the ones that read rooms also have a
# consumer, keyed by the same client id.
self._clients: dict[str, Actor[ClientConfig]] = {}
- # Agent id to the id of its client, for `get_by_agent_id`.
+ # Agent id to the id of its client, for `get_by_agent_id`; and the
+ # agents' clients whose agent this has not seen yet, the only ones a
+ # lookup the index cannot answer has to look at.
self._client_of_agent: dict[str, str] = {}
+ self._agent_unseen: dict[str, None] = {}
+ # Each tenant's running clients of each type, for `get_by_type`.
+ self._clients_of_type: dict[tuple[str, str], dict[str, None]] = {}
self._consumers: dict[str, Consumer[Any]] = {}
self._client_types: dict[str, str] = {}
# Which tenant each running client's row belongs to. The actor
@@ -332,6 +337,7 @@ class ClientLifecycleService:
agent = agents.get(record.id)
if agent is not None and isinstance(actor, _PreloadsAgent):
actor.preload_agent(agent)
+ self._agent_seen(record.id, agent.id)
async def reconcile_all(self) -> None:
"""`reconcile_tenant` for every tenant."""
@@ -421,6 +427,7 @@ class ClientLifecycleService:
actor = self._register(record)
if agent is not None and isinstance(actor, _PreloadsAgent):
actor.preload_agent(agent)
+ self._agent_seen(record.id, agent.id)
agents_changed |= agent is not None
if agents_changed:
self._agents_changed(tenant_id)
@@ -484,9 +491,7 @@ class ClientLifecycleService:
def _register(self, record: Client) -> Actor[ClientConfig]:
self._left_stopped.pop(record.id, None)
actor = self._client_factory.create_actor(record)
- self._clients[record.id] = actor
- self._client_types[record.id] = record.type
- self._client_tenants[record.id] = record.tenant_id
+ self._track(record.id, actor, record.type, record.tenant_id)
# The actor here, and the consumer, if the type has one, where this
# process owns the client.
self._consumers.pop(record.id, None)
@@ -495,6 +500,47 @@ class ClientLifecycleService:
self._owned.want(record.id)
return actor
+ def _track(
+ self,
+ client_id: str,
+ actor: Actor[ClientConfig],
+ client_type: str,
+ tenant_id: str,
+ ) -> None:
+ """Hold `actor` as the client's running copy, where every lookup of
+ the running clients finds it."""
+ self._forget(client_id)
+ self._clients[client_id] = actor
+ self._client_types[client_id] = client_type
+ self._client_tenants[client_id] = tenant_id
+ self._clients_of_type.setdefault((client_type, tenant_id), {})[client_id] = None
+ if isinstance(actor, AgentActor):
+ known = actor.agent_id
+ if known is None:
+ self._agent_unseen[client_id] = None
+ else:
+ self._agent_seen(client_id, known)
+
+ def _forget(self, client_id: str) -> None:
+ """Let go of the client's running copy and every lookup's note of it."""
+ actor = self._clients.pop(client_id, None)
+ client_type = self._client_types.pop(client_id, None)
+ tenant_id = self._client_tenants.pop(client_id, None)
+ if client_type is not None and tenant_id is not None:
+ of_type = self._clients_of_type.get((client_type, tenant_id))
+ if of_type is not None:
+ of_type.pop(client_id, None)
+ if not of_type:
+ del self._clients_of_type[(client_type, tenant_id)]
+ self._agent_unseen.pop(client_id, None)
+ if isinstance(actor, AgentActor) and actor.agent_id is not None:
+ if self._client_of_agent.get(actor.agent_id) == client_id:
+ del self._client_of_agent[actor.agent_id]
+
+ def _agent_seen(self, client_id: str, agent_id: str) -> None:
+ self._client_of_agent[agent_id] = client_id
+ self._agent_unseen.pop(client_id, None)
+
async def create_and_start(
self,
*,
@@ -529,9 +575,7 @@ class ClientLifecycleService:
task = self._cancel_task(client_id)
if task is not None:
await asyncio.gather(task, return_exceptions=True)
- del self._clients[client_id]
- self._client_types.pop(client_id, None)
- self._client_tenants.pop(client_id, None)
+ self._forget(client_id)
logger.info("Stopped client %s", client_id)
async def stop_all(self) -> None:
@@ -558,6 +602,9 @@ class ClientLifecycleService:
self._consumers.clear()
self._client_types.clear()
self._client_tenants.clear()
+ self._clients_of_type.clear()
+ self._client_of_agent.clear()
+ self._agent_unseen.clear()
self._tasks.clear()
async def delete_record(self, session: AsyncSession, client_id: str) -> None:
@@ -621,9 +668,18 @@ class ClientLifecycleService:
actor = self._clients.get(client_id)
if isinstance(actor, AgentActor) and actor.agent_id == agent_id:
return actor
- for client_id, actor in self._clients.items():
- if isinstance(actor, AgentActor) and actor.agent_id == agent_id:
- self._client_of_agent[agent_id] = client_id
+ # Every other agent's client is in the index under its own agent, so
+ # only the ones whose agent this has not seen yet can be this one.
+ for client_id in list(self._agent_unseen):
+ actor = self._clients.get(client_id)
+ if not isinstance(actor, AgentActor):
+ del self._agent_unseen[client_id]
+ continue
+ seen = actor.agent_id
+ if seen is None:
+ continue
+ self._agent_seen(client_id, seen)
+ if seen == agent_id:
return actor
return None
@@ -644,10 +700,9 @@ class ClientLifecycleService:
own a room takes the whole process down with it.
"""
return [
- client
- for client_id, client in self._clients.items()
- if self._client_types.get(client_id) == client_type
- and self._client_tenants.get(client_id) == tenant_id
+ self._clients[client_id]
+ for client_id in self._clients_of_type.get((client_type, tenant_id), ())
+ if client_id in self._clients
]
async def _start_owned_consumer(self, client_id: str, fence: Fence) -> None:
@@ -823,8 +878,6 @@ class ClientLifecycleService:
actor.transport_user_id,
)
self._leave_stopped(client_id)
- self._clients.pop(client_id, None)
+ self._forget(client_id)
self._consumers.pop(client_id, None)
- self._client_types.pop(client_id, None)
- self._client_tenants.pop(client_id, None)
self._tasks.pop(client_id, None)
diff --git a/core/tests/switch_core/clients/test_client_lifecycle_agent_lookup.py b/core/tests/switch_core/clients/test_client_lifecycle_agent_lookup.py
index 60cf9bf01..8db3e11d9 100644
--- a/core/tests/switch_core/clients/test_client_lifecycle_agent_lookup.py
+++ b/core/tests/switch_core/clients/test_client_lifecycle_agent_lookup.py
@@ -1,5 +1,6 @@
-"""An agent's actor is found without scanning every client's actor each time,
-and a lookup still finds the actor running now after it was replaced."""
+"""A running client is found by its agent or its type without looking at
+every client's actor, and a lookup still finds the actor running now after it
+was replaced."""
from __future__ import annotations
@@ -7,10 +8,12 @@ from types import SimpleNamespace
from typing import Any
from unittest.mock import MagicMock
-from switch_core.clients.actor import AgentActor
+from switch_core.clients.actor import AgentActor, SystemActor
from switch_core.clients.client_lifecycle_service import ClientLifecycleService
from tests.ownership import NeverOwned
+_TENANT = "tenant-a"
+
class _CountingScans(dict[str, Any]):
scans = 0
@@ -19,14 +22,29 @@ class _CountingScans(dict[str, Any]):
self.scans += 1
return super().items()
+ def __iter__(self): # type: ignore[no-untyped-def]
+ self.scans += 1
+ return super().__iter__()
+
+
+class _Agent(AgentActor):
+ """An agent's actor that counts how often its agent is asked for."""
+
+ asked = 0
-def _actor(agent_id: str) -> AgentActor:
- actor = AgentActor.__new__(AgentActor)
- actor._agent = SimpleNamespace(id=agent_id) # type: ignore[assignment]
+ @property
+ def agent_id(self) -> str | None:
+ _Agent.asked += 1
+ return self._agent.id if self._agent is not None else None
+
+
+def _actor(agent_id: str | None) -> _Agent:
+ actor = _Agent.__new__(_Agent)
+ actor._agent = None if agent_id is None else SimpleNamespace(id=agent_id) # type: ignore[assignment]
return actor
-def _service(clients: dict[str, Any]) -> ClientLifecycleService:
+def _service() -> ClientLifecycleService:
service = ClientLifecycleService(
provisioning=MagicMock(),
client_store=MagicMock(),
@@ -38,39 +56,84 @@ def _service(clients: dict[str, Any]) -> ClientLifecycleService:
ownership=NeverOwned(), # type: ignore[arg-type]
cursor_writer=MagicMock(),
)
- service._clients = clients
+ service._clients = _CountingScans()
+ return service
+
+
+def _running(count: int) -> ClientLifecycleService:
+ service = _service()
+ for n in range(count):
+ service._track(f"client-{n}", _actor(f"agent-{n}"), "agent", _TENANT)
return service
-def test_an_agent_found_once_is_found_again_without_a_scan() -> None:
- clients = _CountingScans({f"client-{n}": _actor(f"agent-{n}") for n in range(1000)})
- service = _service(clients)
+def test_an_agent_is_found_without_looking_at_any_other_actor() -> None:
+ service = _running(1000)
+ _Agent.asked = 0
- first = service.get_by_agent_id("agent-500")
- again = service.get_by_agent_id("agent-500")
+ found = service.get_by_agent_id("agent-500")
- assert first is clients["client-500"]
- assert again is first
- assert clients.scans == 1
+ assert found is service._clients["client-500"]
+ assert _Agent.asked == 1
+ assert service._clients.scans == 0 # type: ignore[attr-defined]
+
+
+def test_an_agent_not_running_here_is_answered_without_looking() -> None:
+ """At 10,000 agents, every lookup of an agent another process had just
+ registered read every actor here, and creating agents made that one
+ lookup per agent."""
+ service = _running(1000)
+ _Agent.asked = 0
+
+ assert service.get_by_agent_id("agent-elsewhere") is None
+ assert _Agent.asked == 0
+
+
+def test_an_agent_whose_row_comes_later_is_found_and_kept() -> None:
+ """A client registered here reads its agent row as it connects."""
+ service = _running(10)
+ late = _actor(None)
+ service._track("client-late", late, "agent", _TENANT)
+ late._agent = SimpleNamespace(id="agent-late") # type: ignore[assignment]
+
+ assert service.get_by_agent_id("agent-late") is late
+ _Agent.asked = 0
+ assert service.get_by_agent_id("agent-late") is late
+ assert _Agent.asked == 1
def test_a_replaced_actor_is_the_one_found() -> None:
- clients: dict[str, Any] = {"client-1": _actor("agent-1")}
- service = _service(clients)
- assert service.get_by_agent_id("agent-1") is clients["client-1"]
+ service = _running(1)
+ assert service.get_by_agent_id("agent-0") is service._clients["client-0"]
- clients["client-1"] = replacement = _actor("agent-1")
+ replacement = _actor("agent-0")
+ service._track("client-0", replacement, "agent", _TENANT)
- assert service.get_by_agent_id("agent-1") is replacement
+ assert service.get_by_agent_id("agent-0") is replacement
def test_a_stopped_actor_is_not_found_and_a_moved_one_is() -> None:
- clients: dict[str, Any] = {"client-1": _actor("agent-1")}
- service = _service(clients)
- assert service.get_by_agent_id("agent-1") is not None
+ service = _running(1)
+ assert service.get_by_agent_id("agent-0") is not None
+
+ service._forget("client-0")
+ assert service.get_by_agent_id("agent-0") is None
+
+ moved = _actor("agent-0")
+ service._track("client-1", moved, "agent", _TENANT)
+ assert service.get_by_agent_id("agent-0") is moved
+
+
+def test_a_type_is_found_without_looking_at_every_client() -> None:
+ """A room created asks for its tenant's admin, and with thousands of
+ agents running that read every one of them."""
+ service = _running(1000)
+ admin = SystemActor.__new__(SystemActor)
+ service._track("client-admin", admin, "admin", _TENANT)
+ service._track("client-other-admin", admin, "admin", "tenant-b")
- del clients["client-1"]
- assert service.get_by_agent_id("agent-1") is None
+ assert service.get_by_type("admin", _TENANT) == [admin]
+ assert service._clients.scans == 0 # type: ignore[attr-defined]
- clients["client-2"] = moved = _actor("agent-1")
- assert service.get_by_agent_id("agent-1") is moved
+ service._forget("client-admin")
+ assert service.get_by_type("admin", _TENANT) == []