Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Re-check controller connections only when they changed (CHOO-3667)

Abel Dantas92eb2d750855
diff --git a/CHANGELOG.md b/CHANGELOG.md
index e245d2d5d..b29a14d98 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -237,6 +237,11 @@ version of their own to them without also giving them a release of their own.
query of its own. It now reads the columns a binding is made of in one
query per workspace, and only when a fingerprint of the placements has
changed since its last full read.
+- **The 30-second re-check of who holds each controller's socket no longer
+ reads every controller each time.** It compares a fingerprint of their
+ connections and of the live processes first, and when it reads, reads only
+ the connection columns, where it read every row with its status and
+ platform, and the rooms of every agent on a controller another process held.
#### Fixed
- **On Slack, the 👀 and the activity card no longer arrive after the agent's
diff --git a/core/switch_core/db/stores/agent_controller_store.py b/core/switch_core/db/stores/agent_controller_store.py
index 6df694d67..1d88a2ddb 100644
--- a/core/switch_core/db/stores/agent_controller_store.py
+++ b/core/switch_core/db/stores/agent_controller_store.py
@@ -1,12 +1,24 @@
from __future__ import annotations
+from dataclasses import dataclass
from datetime import datetime
from typing import Any
-from sqlalchemy import and_, delete, func, or_, select, update
+from sqlalchemy import (
+ ColumnElement,
+ Text,
+ and_,
+ cast,
+ delete,
+ func,
+ or_,
+ select,
+ update,
+)
from sqlalchemy.dialects.postgresql import insert
from sqlalchemy.ext.asyncio import AsyncSession
+from switch_core.db.fingerprint import Fingerprint, row_hash
from switch_core.db.models import (
AgentController,
AgentControllerEnrollmentCode,
@@ -14,6 +26,49 @@ from switch_core.db.models import (
)
+@dataclass(frozen=True)
+class ControllerConnection:
+ """What a process acts on of a controller's row to know who holds its
+ socket (`AgentControllerStore.connections`)."""
+
+ id: str
+ tenant_id: str
+ connection_id: str | None
+ connection_process_id: str | None
+ disconnected_at: datetime | None
+ open_connection_id: str | None
+ open_process_id: str | None
+
+
+_CONNECTION_COLUMNS = (
+ AgentController.id,
+ AgentController.tenant_id,
+ AgentController.connection_id,
+ AgentController.connection_process_id,
+ AgentController.disconnected_at,
+ AgentController.open_connection_id,
+ AgentController.open_process_id,
+)
+
+_HAS_CONNECTION = or_(
+ AgentController.connection_id.is_not(None),
+ AgentController.open_connection_id.is_not(None),
+)
+
+
+def _connection_hash() -> ColumnElement[int]:
+ """A controller's connection columns, the ones `ControllerConnection`
+ holds. NULL hashes as the empty string, which no id or time is."""
+ return row_hash(
+ AgentController.id,
+ func.coalesce(AgentController.connection_id, ""),
+ func.coalesce(AgentController.connection_process_id, ""),
+ func.coalesce(cast(AgentController.disconnected_at, Text), ""),
+ func.coalesce(AgentController.open_connection_id, ""),
+ func.coalesce(AgentController.open_process_id, ""),
+ )
+
+
class AgentControllerStore:
"""Agent controllers and the one-time codes they enroll with.
@@ -420,20 +475,40 @@ class AgentControllerStore:
)
return result.scalar_one_or_none() is not None
- async def list_with_connections(
+ async def connections(
self, session: AsyncSession, tenant_id: str
- ) -> list[AgentController]:
- """Every controller of the tenant that has opened a connection."""
+ ) -> tuple[list[ControllerConnection], Fingerprint]:
+ """The connection columns of every controller of the tenant that has
+ opened a connection, and the fingerprint of those rows, from one
+ statement and so one snapshot. The rest of each row, its status and
+ platform among them, is not read: a process reading who holds each
+ socket acts on none of it."""
result = await session.execute(
- select(AgentController).where(
- AgentController.tenant_id == tenant_id,
- or_(
- AgentController.connection_id.is_not(None),
- AgentController.open_connection_id.is_not(None),
- ),
+ select(
+ *_CONNECTION_COLUMNS,
+ func.count().over(),
+ func.sum(_connection_hash()).over(),
+ ).where(AgentController.tenant_id == tenant_id, _HAS_CONNECTION)
+ )
+ rows = result.all()
+ if not rows:
+ return [], (0, 0)
+ return (
+ [ControllerConnection(*row[: len(_CONNECTION_COLUMNS)]) for row in rows],
+ (int(rows[0][-2]), int(rows[0][-1])),
+ )
+
+ async def connections_fingerprint(
+ self, session: AsyncSession, tenant_id: str
+ ) -> Fingerprint:
+ """The fingerprint `connections` returns, without the rows."""
+ result = await session.execute(
+ select(func.count(), func.coalesce(func.sum(_connection_hash()), 0)).where(
+ AgentController.tenant_id == tenant_id, _HAS_CONNECTION
)
)
- return list(result.scalars().all())
+ count, total = result.one()
+ return int(count), int(total)
async def set_credential(
self,
diff --git a/core/switch_core/management/signals.py b/core/switch_core/management/signals.py
index d9aef429b..c1850800a 100644
--- a/core/switch_core/management/signals.py
+++ b/core/switch_core/management/signals.py
@@ -66,7 +66,10 @@ if TYPE_CHECKING:
from switch_core.db.stores.agent_controller_operation_store import (
AgentControllerOperationStore,
)
- from switch_core.db.stores.agent_controller_store import AgentControllerStore
+ from switch_core.db.stores.agent_controller_store import (
+ AgentControllerStore,
+ ControllerConnection,
+ )
from switch_core.db.stores.agent_definition_store import AgentDefinitionStore
from switch_core.db.stores.room_store import RoomStore
from switch_core.db.stores.switch_core_process_store import (
@@ -126,6 +129,14 @@ class ControllerSignals:
# Each tenant's placements as the last full read of the bindings here
# found them, which the periodic re-check compares first.
self._binding_fingerprints: dict[str, Fingerprint] = {}
+ # Each tenant's controller connections as the last full read of them
+ # here found them, with the processes holding a lease then, which the
+ # periodic re-check compares first; and the controllers that read
+ # found, which a re-check that skips the read still counts as seen.
+ self._connection_fingerprints: dict[
+ str, tuple[Fingerprint, frozenset[str]]
+ ] = {}
+ self._connections_seen: dict[str, set[str]] = {}
# Controllers, by (tenant id, controller id), whose bindings another
# process changed and this one has still to read.
self._bindings_owed = bus.pending_work(
@@ -274,10 +285,18 @@ class ControllerSignals:
self._apply_connection(controller, leases, rooms)
return
- async def read_connections(self) -> None:
+ async def read_connections(self, *, check_first: bool = False) -> None:
"""Read every controller's connection row again, in every tenant, and
make this process's view of who holds each socket agree. Run at
- startup, before the bridge serves, and on resync and each re-check."""
+ startup, before the bridge serves, and on resync and each re-check.
+
+ With `check_first`, as the periodic re-check runs it, a tenant whose
+ connection rows and live processes are what its last full read here
+ found is not read again: comparing costs one row, and reading them,
+ with the rooms of every agent of each controller another process
+ holds, cost each process a few seconds of database time every 30
+ seconds at thousands of controllers. Those rooms are kept up to date
+ by the membership signals and `MessageRouter.recheck_agent_rooms`."""
seen: set[str] = set()
elsewhere = self._presence.controllers_elsewhere()
with no_tenant():
@@ -286,11 +305,31 @@ class ControllerSignals:
mark = self._presence.connection_mark()
async with tenant_session(self._session_factory, tenant_id) as session:
leases = await read_leases(session, self._processes)
+ holding = frozenset(
+ process_id
+ for process_id in leases.leases
+ if leases.holds(process_id)
+ )
+ last = self._connection_fingerprints.get(tenant_id)
+ if (
+ check_first
+ and last is not None
+ and last
+ == (
+ await self._controllers.connections_fingerprint(
+ session, tenant_id
+ ),
+ holding,
+ )
+ ):
+ seen |= self._connections_seen.get(tenant_id, set())
+ continue
+ read, fingerprint = await self._controllers.connections(
+ session, tenant_id
+ )
found = [
controller
- for controller in await self._controllers.list_with_connections(
- session, tenant_id
- )
+ for controller in read
if controller.tenant_id == tenant_id
]
rooms = {
@@ -299,6 +338,8 @@ class ControllerSignals:
)
for controller in found
}
+ self._connection_fingerprints[tenant_id] = (fingerprint, holding)
+ self._connections_seen[tenant_id] = {controller.id for controller in found}
for controller in found:
seen.add(controller.id)
if self._presence.connection_changed_since(controller.id, mark):
@@ -312,7 +353,7 @@ class ControllerSignals:
async def _rooms_if_held_elsewhere(
self,
session: AsyncSession,
- controller: AgentController,
+ controller: AgentController | ControllerConnection,
leases: ProcessLeases,
) -> dict[str, set[str]]:
"""The rooms of each agent the controller runs, when another process
@@ -331,7 +372,7 @@ class ControllerSignals:
}
def _held_elsewhere(
- self, controller: AgentController, leases: ProcessLeases
+ self, controller: AgentController | ControllerConnection, leases: ProcessLeases
) -> str | None:
"""The other process holding the controller's live socket, if any."""
process_id = controller.connection_process_id
@@ -347,7 +388,7 @@ class ControllerSignals:
def _apply_connection(
self,
- controller: AgentController,
+ controller: AgentController | ControllerConnection,
leases: ProcessLeases,
rooms: dict[str, set[str]],
) -> None:
@@ -401,16 +442,16 @@ class ControllerSignals:
async def recheck(self) -> None:
"""What `resync` does, but the bindings are read only when a tenant's
- placements have changed since the last full read, and a held
- controller is nudged only with what this process has not told it
- yet."""
+ placements have changed since the last full read, the connections only
+ when they or the processes holding a lease have, and a held controller
+ is nudged only with what this process has not told it yet."""
await self._reconcile_all_bindings(check_first=True)
- await self._read_all_connections()
+ await self._read_all_connections(check_first=True)
await self._renudge_held(only_new=True)
- async def _read_all_connections(self) -> None:
+ async def _read_all_connections(self, *, check_first: bool = False) -> None:
try:
- await self.read_connections()
+ await self.read_connections(check_first=check_first)
except Exception:
logger.error("Could not re-read controller connections", exc_info=True)
diff --git a/core/tests/switch_core/management/test_controller_signals.py b/core/tests/switch_core/management/test_controller_signals.py
index 6e6d2df23..e3a384aa3 100644
--- a/core/tests/switch_core/management/test_controller_signals.py
+++ b/core/tests/switch_core/management/test_controller_signals.py
@@ -4,16 +4,17 @@ from __future__ import annotations
import asyncio
import uuid
-from datetime import UTC, datetime
+from datetime import UTC, datetime, timedelta
from typing import TYPE_CHECKING, Any
import pytest
+from sqlalchemy import func, update
from sqlalchemy.ext.asyncio import create_async_engine
from sqlalchemy.pool import NullPool
from switch_core.bridges.agent.protocol.agent_connections import AgentConnectionRegistry
from switch_core.bridges.agent.protocol.controller_presence import Binding
-from switch_core.db.models import TENANT_ZERO_ID
+from switch_core.db.models import TENANT_ZERO_ID, AgentController, SwitchCoreProcess
from switch_core.db.session_scope import tenant_session
from switch_core.db.stores.agent_controller_operation_store import (
AgentControllerOperationStore,
@@ -29,12 +30,14 @@ from switch_core.management.notifier import (
ControllerNotifier,
)
from switch_core.management.signals import ControllerSignals
+from switch_core.process_lease import LEASE_TTL
from switch_core.signals import SignalBus
from switch_core.user_changes import LocalUserChanges, SignalUserChanges
from tests.switch_core.management.harness import (
Harness,
add_member,
build_harness,
+ connect,
cookies_for,
enroll_console,
open_connection,
@@ -394,6 +397,80 @@ class TestRecheck:
binding = registry.controllers.binding(agent_id)
assert binding is not None and binding.controller_name == "workstation"
+ async def test_with_nothing_changed_it_reads_no_connection(
+ self, harness: Harness, postgres_url: str
+ ) -> None:
+ """Every process re-checks who holds each controller's socket every
+ 30 seconds, and with thousands of controllers reading every row, and
+ the rooms of every agent on a controller another process holds, was
+ a few seconds of database time each time."""
+ other, registry, _ = _process(postgres_url, harness.session_factory)
+ owner = await add_member(harness.session_factory, "ada")
+ async with harness.client() as client:
+ controller = await enroll_console(harness, client, owner)
+ await place_agent(client, controller, name="reviewer")
+ await connect(client, controller)
+ await other.recheck()
+ assert registry.controllers.holder_elsewhere(controller.controller_id)
+ reads = _count_connection_reads(other)
+
+ await other.recheck()
+
+ assert reads == []
+ assert registry.controllers.holder_elsewhere(controller.controller_id)
+
+ async def test_a_connection_ended_with_no_signal_is_read(
+ self, harness: Harness, postgres_url: str
+ ) -> None:
+ other, registry, _ = _process(postgres_url, harness.session_factory)
+ owner = await add_member(harness.session_factory, "ada")
+ async with harness.client() as client:
+ controller = await enroll_console(harness, client, owner)
+ await place_agent(client, controller, name="reviewer")
+ await connect(client, controller)
+ await other.recheck()
+ assert registry.controllers.holder_elsewhere(controller.controller_id)
+ reads = _count_connection_reads(other)
+ async with tenant_session(harness.session_factory, TENANT_ZERO_ID) as session:
+ await session.execute(
+ update(AgentController)
+ .where(AgentController.id == controller.controller_id)
+ .values(disconnected_at=func.now())
+ )
+ await session.commit()
+
+ await other.recheck()
+
+ assert reads == [TENANT_ZERO_ID]
+ assert registry.controllers.holder_elsewhere(controller.controller_id) is None
+
+ async def test_a_holder_whose_lease_lapsed_is_read_again(
+ self, harness: Harness, postgres_url: str
+ ) -> None:
+ """No connection row changes when the process holding it dies, so
+ which processes hold a lease is compared too."""
+ other, registry, _ = _process(postgres_url, harness.session_factory)
+ owner = await add_member(harness.session_factory, "ada")
+ async with harness.client() as client:
+ controller = await enroll_console(harness, client, owner)
+ await place_agent(client, controller, name="reviewer")
+ await connect(client, controller)
+ await other.recheck()
+ assert registry.controllers.holder_elsewhere(controller.controller_id)
+ async with harness.session_factory() as session:
+ await session.execute(
+ update(SwitchCoreProcess)
+ .where(SwitchCoreProcess.id == harness.management.lease.process_id)
+ .values(
+ beat_at=SwitchCoreProcess.beat_at - LEASE_TTL - timedelta(seconds=1)
+ )
+ )
+ await session.commit()
+
+ await other.recheck()
+
+ assert registry.controllers.holder_elsewhere(controller.controller_id) is None
+
async def test_a_held_controller_is_nudged_only_with_what_it_was_not_told(
self, harness: Harness
) -> None:
@@ -446,6 +523,21 @@ class TestReconcileKeepsNewerLocalChanges:
assert not presence.is_bound("a1")
+def _count_connection_reads(signals: ControllerSignals) -> list[str]:
+ """The tenants whose controller connections `signals` reads in full from
+ now on."""
+ reads: list[str] = []
+ store = signals._controllers
+ real = store.connections
+
+ async def counted(session: Any, tenant_id: str) -> Any:
+ reads.append(tenant_id)
+ return await real(session, tenant_id)
+
+ store.connections = counted # type: ignore[method-assign]
+ return reads
+
+
def _count_placement_reads(signals: ControllerSignals) -> list[str]:
"""The tenants whose placements `signals` reads in full from now on."""
reads: list[str] = []