Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Re-check controller connections only when they changed (CHOO-3667)
Abel Dantas92eb2d750855
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex e245d2d5d..b29a14d98 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -237,6 +237,11 @@ version of their own to them without also giving them a release of their own.query of its own. It now reads the columns a binding is made of in onequery per workspace, and only when a fingerprint of the placements haschanged since its last full read.+- **The 30-second re-check of who holds each controller's socket no longer+ reads every controller each time.** It compares a fingerprint of their+ connections and of the live processes first, and when it reads, reads only+ the connection columns, where it read every row with its status and+ platform, and the rooms of every agent on a controller another process held.#### Fixed- **On Slack, the 👀 and the activity card no longer arrive after the agent'sdiff --git a/core/switch_core/db/stores/agent_controller_store.py b/core/switch_core/db/stores/agent_controller_store.pyindex 6df694d67..1d88a2ddb 100644--- a/core/switch_core/db/stores/agent_controller_store.py+++ b/core/switch_core/db/stores/agent_controller_store.py@@ -1,12 +1,24 @@from __future__ import annotations+from dataclasses import dataclassfrom datetime import datetimefrom typing import Any-from sqlalchemy import and_, delete, func, or_, select, update+from sqlalchemy import (+ ColumnElement,+ Text,+ and_,+ cast,+ delete,+ func,+ or_,+ select,+ update,+)from sqlalchemy.dialects.postgresql import insertfrom sqlalchemy.ext.asyncio import AsyncSession+from switch_core.db.fingerprint import Fingerprint, row_hashfrom switch_core.db.models import (AgentController,AgentControllerEnrollmentCode,@@ -14,6 +26,49 @@ from switch_core.db.models import ()+@dataclass(frozen=True)+class ControllerConnection:+ """What a process acts on of a controller's row to know who holds its+ socket (`AgentControllerStore.connections`)."""++ id: str+ tenant_id: str+ connection_id: str | None+ connection_process_id: str | None+ disconnected_at: datetime | None+ open_connection_id: str | None+ open_process_id: str | None+++_CONNECTION_COLUMNS = (+ AgentController.id,+ AgentController.tenant_id,+ AgentController.connection_id,+ AgentController.connection_process_id,+ AgentController.disconnected_at,+ AgentController.open_connection_id,+ AgentController.open_process_id,+)++_HAS_CONNECTION = or_(+ AgentController.connection_id.is_not(None),+ AgentController.open_connection_id.is_not(None),+)+++def _connection_hash() -> ColumnElement[int]:+ """A controller's connection columns, the ones `ControllerConnection`+ holds. NULL hashes as the empty string, which no id or time is."""+ return row_hash(+ AgentController.id,+ func.coalesce(AgentController.connection_id, ""),+ func.coalesce(AgentController.connection_process_id, ""),+ func.coalesce(cast(AgentController.disconnected_at, Text), ""),+ func.coalesce(AgentController.open_connection_id, ""),+ func.coalesce(AgentController.open_process_id, ""),+ )++class AgentControllerStore:"""Agent controllers and the one-time codes they enroll with.@@ -420,20 +475,40 @@ class AgentControllerStore:)return result.scalar_one_or_none() is not None- async def list_with_connections(+ async def connections(self, session: AsyncSession, tenant_id: str- ) -> list[AgentController]:- """Every controller of the tenant that has opened a connection."""+ ) -> tuple[list[ControllerConnection], Fingerprint]:+ """The connection columns of every controller of the tenant that has+ opened a connection, and the fingerprint of those rows, from one+ statement and so one snapshot. The rest of each row, its status and+ platform among them, is not read: a process reading who holds each+ socket acts on none of it."""result = await session.execute(- select(AgentController).where(- AgentController.tenant_id == tenant_id,- or_(- AgentController.connection_id.is_not(None),- AgentController.open_connection_id.is_not(None),- ),+ select(+ *_CONNECTION_COLUMNS,+ func.count().over(),+ func.sum(_connection_hash()).over(),+ ).where(AgentController.tenant_id == tenant_id, _HAS_CONNECTION)+ )+ rows = result.all()+ if not rows:+ return [], (0, 0)+ return (+ [ControllerConnection(*row[: len(_CONNECTION_COLUMNS)]) for row in rows],+ (int(rows[0][-2]), int(rows[0][-1])),+ )++ async def connections_fingerprint(+ self, session: AsyncSession, tenant_id: str+ ) -> Fingerprint:+ """The fingerprint `connections` returns, without the rows."""+ result = await session.execute(+ select(func.count(), func.coalesce(func.sum(_connection_hash()), 0)).where(+ AgentController.tenant_id == tenant_id, _HAS_CONNECTION))- return list(result.scalars().all())+ count, total = result.one()+ return int(count), int(total)async def set_credential(self,diff --git a/core/switch_core/management/signals.py b/core/switch_core/management/signals.pyindex d9aef429b..c1850800a 100644--- a/core/switch_core/management/signals.py+++ b/core/switch_core/management/signals.py@@ -66,7 +66,10 @@ if TYPE_CHECKING:from switch_core.db.stores.agent_controller_operation_store import (AgentControllerOperationStore,)- from switch_core.db.stores.agent_controller_store import AgentControllerStore+ from switch_core.db.stores.agent_controller_store import (+ AgentControllerStore,+ ControllerConnection,+ )from switch_core.db.stores.agent_definition_store import AgentDefinitionStorefrom switch_core.db.stores.room_store import RoomStorefrom switch_core.db.stores.switch_core_process_store import (@@ -126,6 +129,14 @@ class ControllerSignals:# Each tenant's placements as the last full read of the bindings here# found them, which the periodic re-check compares first.self._binding_fingerprints: dict[str, Fingerprint] = {}+ # Each tenant's controller connections as the last full read of them+ # here found them, with the processes holding a lease then, which the+ # periodic re-check compares first; and the controllers that read+ # found, which a re-check that skips the read still counts as seen.+ self._connection_fingerprints: dict[+ str, tuple[Fingerprint, frozenset[str]]+ ] = {}+ self._connections_seen: dict[str, set[str]] = {}# Controllers, by (tenant id, controller id), whose bindings another# process changed and this one has still to read.self._bindings_owed = bus.pending_work(@@ -274,10 +285,18 @@ class ControllerSignals:self._apply_connection(controller, leases, rooms)return- async def read_connections(self) -> None:+ async def read_connections(self, *, check_first: bool = False) -> None:"""Read every controller's connection row again, in every tenant, andmake this process's view of who holds each socket agree. Run at- startup, before the bridge serves, and on resync and each re-check."""+ startup, before the bridge serves, and on resync and each re-check.++ With `check_first`, as the periodic re-check runs it, a tenant whose+ connection rows and live processes are what its last full read here+ found is not read again: comparing costs one row, and reading them,+ with the rooms of every agent of each controller another process+ holds, cost each process a few seconds of database time every 30+ seconds at thousands of controllers. Those rooms are kept up to date+ by the membership signals and `MessageRouter.recheck_agent_rooms`."""seen: set[str] = set()elsewhere = self._presence.controllers_elsewhere()with no_tenant():@@ -286,11 +305,31 @@ class ControllerSignals:mark = self._presence.connection_mark()async with tenant_session(self._session_factory, tenant_id) as session:leases = await read_leases(session, self._processes)+ holding = frozenset(+ process_id+ for process_id in leases.leases+ if leases.holds(process_id)+ )+ last = self._connection_fingerprints.get(tenant_id)+ if (+ check_first+ and last is not None+ and last+ == (+ await self._controllers.connections_fingerprint(+ session, tenant_id+ ),+ holding,+ )+ ):+ seen |= self._connections_seen.get(tenant_id, set())+ continue+ read, fingerprint = await self._controllers.connections(+ session, tenant_id+ )found = [controller- for controller in await self._controllers.list_with_connections(- session, tenant_id- )+ for controller in readif controller.tenant_id == tenant_id]rooms = {@@ -299,6 +338,8 @@ class ControllerSignals:)for controller in found}+ self._connection_fingerprints[tenant_id] = (fingerprint, holding)+ self._connections_seen[tenant_id] = {controller.id for controller in found}for controller in found:seen.add(controller.id)if self._presence.connection_changed_since(controller.id, mark):@@ -312,7 +353,7 @@ class ControllerSignals:async def _rooms_if_held_elsewhere(self,session: AsyncSession,- controller: AgentController,+ controller: AgentController | ControllerConnection,leases: ProcessLeases,) -> dict[str, set[str]]:"""The rooms of each agent the controller runs, when another process@@ -331,7 +372,7 @@ class ControllerSignals:}def _held_elsewhere(- self, controller: AgentController, leases: ProcessLeases+ self, controller: AgentController | ControllerConnection, leases: ProcessLeases) -> str | None:"""The other process holding the controller's live socket, if any."""process_id = controller.connection_process_id@@ -347,7 +388,7 @@ class ControllerSignals:def _apply_connection(self,- controller: AgentController,+ controller: AgentController | ControllerConnection,leases: ProcessLeases,rooms: dict[str, set[str]],) -> None:@@ -401,16 +442,16 @@ class ControllerSignals:async def recheck(self) -> None:"""What `resync` does, but the bindings are read only when a tenant's- placements have changed since the last full read, and a held- controller is nudged only with what this process has not told it- yet."""+ placements have changed since the last full read, the connections only+ when they or the processes holding a lease have, and a held controller+ is nudged only with what this process has not told it yet."""await self._reconcile_all_bindings(check_first=True)- await self._read_all_connections()+ await self._read_all_connections(check_first=True)await self._renudge_held(only_new=True)- async def _read_all_connections(self) -> None:+ async def _read_all_connections(self, *, check_first: bool = False) -> None:try:- await self.read_connections()+ await self.read_connections(check_first=check_first)except Exception:logger.error("Could not re-read controller connections", exc_info=True)diff --git a/core/tests/switch_core/management/test_controller_signals.py b/core/tests/switch_core/management/test_controller_signals.pyindex 6e6d2df23..e3a384aa3 100644--- a/core/tests/switch_core/management/test_controller_signals.py+++ b/core/tests/switch_core/management/test_controller_signals.py@@ -4,16 +4,17 @@ from __future__ import annotationsimport asyncioimport uuid-from datetime import UTC, datetime+from datetime import UTC, datetime, timedeltafrom typing import TYPE_CHECKING, Anyimport pytest+from sqlalchemy import func, updatefrom sqlalchemy.ext.asyncio import create_async_enginefrom sqlalchemy.pool import NullPoolfrom switch_core.bridges.agent.protocol.agent_connections import AgentConnectionRegistryfrom switch_core.bridges.agent.protocol.controller_presence import Binding-from switch_core.db.models import TENANT_ZERO_ID+from switch_core.db.models import TENANT_ZERO_ID, AgentController, SwitchCoreProcessfrom switch_core.db.session_scope import tenant_sessionfrom switch_core.db.stores.agent_controller_operation_store import (AgentControllerOperationStore,@@ -29,12 +30,14 @@ from switch_core.management.notifier import (ControllerNotifier,)from switch_core.management.signals import ControllerSignals+from switch_core.process_lease import LEASE_TTLfrom switch_core.signals import SignalBusfrom switch_core.user_changes import LocalUserChanges, SignalUserChangesfrom tests.switch_core.management.harness import (Harness,add_member,build_harness,+ connect,cookies_for,enroll_console,open_connection,@@ -394,6 +397,80 @@ class TestRecheck:binding = registry.controllers.binding(agent_id)assert binding is not None and binding.controller_name == "workstation"+ async def test_with_nothing_changed_it_reads_no_connection(+ self, harness: Harness, postgres_url: str+ ) -> None:+ """Every process re-checks who holds each controller's socket every+ 30 seconds, and with thousands of controllers reading every row, and+ the rooms of every agent on a controller another process holds, was+ a few seconds of database time each time."""+ other, registry, _ = _process(postgres_url, harness.session_factory)+ owner = await add_member(harness.session_factory, "ada")+ async with harness.client() as client:+ controller = await enroll_console(harness, client, owner)+ await place_agent(client, controller, name="reviewer")+ await connect(client, controller)+ await other.recheck()+ assert registry.controllers.holder_elsewhere(controller.controller_id)+ reads = _count_connection_reads(other)++ await other.recheck()++ assert reads == []+ assert registry.controllers.holder_elsewhere(controller.controller_id)++ async def test_a_connection_ended_with_no_signal_is_read(+ self, harness: Harness, postgres_url: str+ ) -> None:+ other, registry, _ = _process(postgres_url, harness.session_factory)+ owner = await add_member(harness.session_factory, "ada")+ async with harness.client() as client:+ controller = await enroll_console(harness, client, owner)+ await place_agent(client, controller, name="reviewer")+ await connect(client, controller)+ await other.recheck()+ assert registry.controllers.holder_elsewhere(controller.controller_id)+ reads = _count_connection_reads(other)+ async with tenant_session(harness.session_factory, TENANT_ZERO_ID) as session:+ await session.execute(+ update(AgentController)+ .where(AgentController.id == controller.controller_id)+ .values(disconnected_at=func.now())+ )+ await session.commit()++ await other.recheck()++ assert reads == [TENANT_ZERO_ID]+ assert registry.controllers.holder_elsewhere(controller.controller_id) is None++ async def test_a_holder_whose_lease_lapsed_is_read_again(+ self, harness: Harness, postgres_url: str+ ) -> None:+ """No connection row changes when the process holding it dies, so+ which processes hold a lease is compared too."""+ other, registry, _ = _process(postgres_url, harness.session_factory)+ owner = await add_member(harness.session_factory, "ada")+ async with harness.client() as client:+ controller = await enroll_console(harness, client, owner)+ await place_agent(client, controller, name="reviewer")+ await connect(client, controller)+ await other.recheck()+ assert registry.controllers.holder_elsewhere(controller.controller_id)+ async with harness.session_factory() as session:+ await session.execute(+ update(SwitchCoreProcess)+ .where(SwitchCoreProcess.id == harness.management.lease.process_id)+ .values(+ beat_at=SwitchCoreProcess.beat_at - LEASE_TTL - timedelta(seconds=1)+ )+ )+ await session.commit()++ await other.recheck()++ assert registry.controllers.holder_elsewhere(controller.controller_id) is None+async def test_a_held_controller_is_nudged_only_with_what_it_was_not_told(self, harness: Harness) -> None:@@ -446,6 +523,21 @@ class TestReconcileKeepsNewerLocalChanges:assert not presence.is_bound("a1")+def _count_connection_reads(signals: ControllerSignals) -> list[str]:+ """The tenants whose controller connections `signals` reads in full from+ now on."""+ reads: list[str] = []+ store = signals._controllers+ real = store.connections++ async def counted(session: Any, tenant_id: str) -> Any:+ reads.append(tenant_id)+ return await real(session, tenant_id)++ store.connections = counted # type: ignore[method-assign]+ return reads++def _count_placement_reads(signals: ControllerSignals) -> list[str]:"""The tenants whose placements `signals` reads in full from now on."""reads: list[str] = []