Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Re-check controller bindings in one narrow query, and only when placements changed (CHOO-3667)
Abel Dantas2c801eb6b83a
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 76013f638..e245d2d5d 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -231,6 +231,12 @@ version of their own to them without also giving them a release of their own.30-second re-check reads every client only when a fingerprint of theworkspace's clients and agents has changed since its last full read, whereit read them all every time.+- **The 30-second re-check of which controller runs each agent reads one+ query, and nothing when no placement changed.** Every process read every+ placed agent's definition, document included, then each controller with a+ query of its own. It now reads the columns a binding is made of in one+ query per workspace, and only when a fingerprint of the placements has+ changed since its last full read.#### Fixed- **On Slack, the 👀 and the activity card no longer arrive after the agent'sdiff --git a/core/switch_core/db/stores/agent_definition_store.py b/core/switch_core/db/stores/agent_definition_store.pyindex cabac2a77..48d8b4c80 100644--- a/core/switch_core/db/stores/agent_definition_store.py+++ b/core/switch_core/db/stores/agent_definition_store.py@@ -1,11 +1,40 @@from __future__ import annotations+from dataclasses import dataclass+from datetime import datetimefrom typing import Any-from sqlalchemy import delete, select, update+from sqlalchemy import ColumnElement, cast, delete, func, select, updatefrom sqlalchemy.ext.asyncio import AsyncSession+from sqlalchemy.types import Text-from switch_core.db.models import Agent, AgentDefinition+from switch_core.db.fingerprint import Fingerprint, row_hash+from switch_core.db.models import Agent, AgentController, AgentDefinition+++@dataclass(frozen=True)+class Placement:+ """A definition placed on a controller, with what a binding needs of the+ controller: its name, and whether it is revoked. `controller_name` is None+ when the controller's row is gone."""++ agent_id: str+ controller_id: str+ desired_state: str+ controller_name: str | None+ controller_revoked_at: datetime | None+++def _placement_hash() -> ColumnElement[int]:+ """What a binding is made of (`management/bindings.py`), so a fingerprint+ of it changes exactly when some binding would."""+ return row_hash(+ AgentDefinition.agent_id,+ func.coalesce(AgentDefinition.controller_id, ""),+ AgentDefinition.desired_state,+ func.coalesce(AgentController.name, ""),+ cast(AgentController.revoked_at.is_not(None), Text),+ )class DefinitionChanged(Exception):@@ -101,6 +130,54 @@ class AgentDefinitionStore:)return list(result.scalars().all())+ async def placements(+ self, session: AsyncSession, tenant_id: str+ ) -> tuple[list[Placement], Fingerprint]:+ """Every definition placed on a controller, with its controller's name+ and revocation, and the fingerprint of them, in one statement: the+ columns a binding is made of and nothing else, so neither a+ definition's document nor a controller's status is read."""+ result = await session.execute(+ select(+ AgentDefinition.agent_id,+ AgentDefinition.controller_id,+ AgentDefinition.desired_state,+ AgentController.name,+ AgentController.revoked_at,+ func.count().over(),+ func.sum(_placement_hash()).over(),+ )+ .outerjoin(AgentController, _placed_on())+ .where(*_placed_in(tenant_id))+ )+ rows = result.all()+ if not rows:+ return [], (0, 0)+ placements = [+ Placement(+ agent_id=agent_id,+ controller_id=controller_id,+ desired_state=desired_state,+ controller_name=name,+ controller_revoked_at=revoked_at,+ )+ for agent_id, controller_id, desired_state, name, revoked_at, _, _ in rows+ ]+ return placements, (int(rows[0][5]), int(rows[0][6]))++ async def placements_fingerprint(+ self, session: AsyncSession, tenant_id: str+ ) -> Fingerprint:+ """The fingerprint `placements` returns, without the rows."""+ result = await session.execute(+ select(func.count(), func.coalesce(func.sum(_placement_hash()), 0))+ .select_from(AgentDefinition)+ .outerjoin(AgentController, _placed_on())+ .where(*_placed_in(tenant_id))+ )+ count, total = result.one()+ return int(count), int(total)+async def update(self,session: AsyncSession,@@ -154,3 +231,16 @@ class AgentDefinitionStore:).execution_options(synchronize_session=False))+++def _placed_on() -> ColumnElement[bool]:+ return (AgentController.tenant_id == AgentDefinition.tenant_id) & (+ AgentController.id == AgentDefinition.controller_id+ )+++def _placed_in(tenant_id: str) -> tuple[ColumnElement[bool], ...]:+ return (+ AgentDefinition.tenant_id == tenant_id,+ AgentDefinition.controller_id.is_not(None),+ )diff --git a/core/switch_core/management/bindings.py b/core/switch_core/management/bindings.pyindex b5f82cba7..303155cef 100644--- a/core/switch_core/management/bindings.py+++ b/core/switch_core/management/bindings.py@@ -27,6 +27,7 @@ from switch_core.bridges.agent.protocol.controller_presence import (Binding,ControllerPresence,)+from switch_core.db.fingerprint import Fingerprintfrom switch_core.db.models import AgentControllerfrom switch_core.db.models import AgentDefinition as AgentDefinitionRowfrom switch_core.db.session_scope import tenant_session@@ -55,15 +56,12 @@ async def load_bindings(*,session_factory: async_sessionmaker[AsyncSession],definitions: AgentDefinitionStore,- controllers: AgentControllerStore,presence: ControllerPresence,) -> int:"""Load every placed definition into `presence`, and mark the controllersalready revoked that agents are still placed on. Returns how many."""- bindings, revoked = await _read_bindings(- session_factory=session_factory,- definitions=definitions,- controllers=controllers,+ bindings, revoked, _ = await _read_bindings(+ session_factory=session_factory, definitions=definitions)presence.load(bindings)for controller_id in revoked:@@ -76,17 +74,29 @@ async def reconcile_bindings(*,session_factory: async_sessionmaker[AsyncSession],definitions: AgentDefinitionStore,- controllers: AgentControllerStore,presence: ControllerPresence,+ fingerprints: dict[str, Fingerprint] | None = None,) -> None:"""Bring every binding in `presence` in line with the tables, changing- only what differs, and close the controllers revoked meanwhile."""+ only what differs, and close the controllers revoked meanwhile.++ With `fingerprints`, each tenant's placements as the last full read here+ found them, nothing is read when no tenant's placements have changed+ since (`AgentDefinitionStore.placements_fingerprint`), and `fingerprints`+ is brought up to date when they have. That is the periodic re-check,+ which every process runs and which on a quiet deployment finds nothing to+ repair."""+ if fingerprints is not None and await _unchanged(+ session_factory, definitions, fingerprints+ ):+ returnsince = presence.change_mark()- bindings, revoked = await _read_bindings(- session_factory=session_factory,- definitions=definitions,- controllers=controllers,+ bindings, revoked, read = await _read_bindings(+ session_factory=session_factory, definitions=definitions)+ if fingerprints is not None:+ fingerprints.clear()+ fingerprints.update(read)presence.reconcile(bindings,agent_ids=presence.bound_agent_ids() | {b.agent_id for b in bindings},@@ -138,36 +148,55 @@ async def reconcile_controller_bindings()+async def _unchanged(+ session_factory: async_sessionmaker[AsyncSession],+ definitions: AgentDefinitionStore,+ fingerprints: dict[str, Fingerprint],+) -> bool:+ """Whether every tenant's placements read as they did when+ `fingerprints` were taken, and there are no tenants beyond those."""+ tenant_ids = await all_tenant_ids(session_factory)+ if set(tenant_ids) != set(fingerprints):+ return False+ for tenant_id in tenant_ids:+ async with tenant_session(session_factory, tenant_id) as session:+ now = await definitions.placements_fingerprint(session, tenant_id)+ if now != fingerprints[tenant_id]:+ return False+ return True++async def _read_bindings(*,session_factory: async_sessionmaker[AsyncSession],definitions: AgentDefinitionStore,- controllers: AgentControllerStore,-) -> tuple[list[Binding], set[str]]:- """Every placed definition's binding, and the revoked controllers that- agents are still placed on."""+) -> tuple[list[Binding], set[str], dict[str, Fingerprint]]:+ """Every placed definition's binding, the revoked controllers that agents+ are still placed on, and each tenant's fingerprint of its placements, one+ query per tenant."""bindings: list[Binding] = []revoked: set[str] = set()+ fingerprints: dict[str, Fingerprint] = {}for tenant_id in await all_tenant_ids(session_factory):async with tenant_session(session_factory, tenant_id) as session:- placed_on: dict[str, AgentController] = {}- # Filtered on the row's own tenant as well as by the store: see- # `db/tenant_lookup.py`, "a fan-out ... filters what it reads back".- for row in await definitions.list_placed(session, tenant_id):- if row.tenant_id != tenant_id or row.controller_id is None:- continue- controller = placed_on.get(row.controller_id)- if controller is None:- controller = await controllers.get(- session, tenant_id, row.controller_id- )- if controller is None:- raise RuntimeError(- f"agent {row.agent_id} is placed on controller "- f"{row.controller_id}, which does not exist"- )- placed_on[controller.id] = controller- if controller.revoked_at is not None:- revoked.add(controller.id)- bindings.append(binding_of(tenant_id, row, controller))- return bindings, revoked+ placements, fingerprints[tenant_id] = await definitions.placements(+ session, tenant_id+ )+ for placement in placements:+ if placement.controller_name is None:+ raise RuntimeError(+ f"agent {placement.agent_id} is placed on controller "+ f"{placement.controller_id}, which does not exist"+ )+ if placement.controller_revoked_at is not None:+ revoked.add(placement.controller_id)+ bindings.append(+ Binding(+ agent_id=placement.agent_id,+ controller_id=placement.controller_id,+ tenant_id=tenant_id,+ controller_name=placement.controller_name,+ running=placement.desired_state == "running",+ )+ )+ return bindings, revoked, fingerprintsdiff --git a/core/switch_core/management/signals.py b/core/switch_core/management/signals.pyindex 5f2a1200c..d9aef429b 100644--- a/core/switch_core/management/signals.py+++ b/core/switch_core/management/signals.py@@ -45,6 +45,7 @@ from collections.abc import Callablefrom datetime import datetimefrom typing import TYPE_CHECKING, Any+from switch_core.db.fingerprint import Fingerprintfrom switch_core.db.session_scope import tenant_sessionfrom switch_core.db.tenant_lookup import all_tenant_idsfrom switch_core.management.bindings import (@@ -122,6 +123,9 @@ class ControllerSignals:# periodic re-check tells a held controller only what is new to it.self._nudged_revision: dict[str, int] = {}self._nudged_operations: dict[str, set[str]] = {}+ # Each tenant's placements as the last full read of the bindings here+ # found them, which the periodic re-check compares first.+ self._binding_fingerprints: dict[str, Fingerprint] = {}# Controllers, by (tenant id, controller id), whose bindings another# process changed and this one has still to read.self._bindings_owed = bus.pending_work(@@ -391,14 +395,16 @@ class ControllerSignals:re-nudge every controller this process holds from what the databasesays now: its assignment revision and its open operations, or itsrevocation."""- await self._reconcile_all_bindings()+ await self._reconcile_all_bindings(check_first=False)await self._read_all_connections()await self._renudge_held(only_new=False)async def recheck(self) -> None:- """What `resync` does, but a held controller is nudged only with what- this process has not told it yet."""- await self._reconcile_all_bindings()+ """What `resync` does, but the bindings are read only when a tenant's+ placements have changed since the last full read, and a held+ controller is nudged only with what this process has not told it+ yet."""+ await self._reconcile_all_bindings(check_first=True)await self._read_all_connections()await self._renudge_held(only_new=True)@@ -408,13 +414,13 @@ class ControllerSignals:except Exception:logger.error("Could not re-read controller connections", exc_info=True)- async def _reconcile_all_bindings(self) -> None:+ async def _reconcile_all_bindings(self, *, check_first: bool) -> None:try:await reconcile_bindings(session_factory=self._session_factory,definitions=self._definitions,- controllers=self._controllers,presence=self._presence,+ fingerprints=self._binding_fingerprints if check_first else None,)except Exception:logger.error("Could not re-read controller bindings", exc_info=True)diff --git a/core/switch_core/management/wiring.py b/core/switch_core/management/wiring.pyindex c9d93e497..c209b1ea8 100644--- a/core/switch_core/management/wiring.py+++ b/core/switch_core/management/wiring.py@@ -124,7 +124,6 @@ class Management:return await load_bindings(session_factory=self.session_factory,definitions=self.service.definitions,- controllers=self.service.controllers,presence=self.service.presence,)diff --git a/core/tests/switch_core/management/test_controller_routes.py b/core/tests/switch_core/management/test_controller_routes.pyindex e0cc50daf..a20c0e55c 100644--- a/core/tests/switch_core/management/test_controller_routes.py+++ b/core/tests/switch_core/management/test_controller_routes.py@@ -916,7 +916,6 @@ class TestPlacementBindsTheAgentInCore:await load_bindings(session_factory=harness.session_factory,definitions=harness.management.service.definitions,- controllers=harness.management.service.controllers,presence=restarted,)diff --git a/core/tests/switch_core/management/test_controller_signals.py b/core/tests/switch_core/management/test_controller_signals.pyindex 6c6bca27a..6e6d2df23 100644--- a/core/tests/switch_core/management/test_controller_signals.py+++ b/core/tests/switch_core/management/test_controller_signals.py@@ -5,7 +5,7 @@ from __future__ import annotationsimport asyncioimport uuidfrom datetime import UTC, datetime-from typing import TYPE_CHECKING+from typing import TYPE_CHECKING, Anyimport pytestfrom sqlalchemy.ext.asyncio import create_async_engine@@ -352,6 +352,48 @@ class TestRecheck:await other.recheck()assert not registry.controllers.is_bound(agent_id)+ async def test_with_nothing_changed_it_reads_no_placement(+ self, harness: Harness, postgres_url: str+ ) -> None:+ """Every process re-checks every 30 seconds, and with 10,000 managed+ agents a read of every placement on each was a large share of what the+ processes did while nothing happened."""+ other, registry, _ = _process(postgres_url, harness.session_factory)+ owner = await add_member(harness.session_factory, "ada")+ async with harness.client() as client:+ controller = await enroll_console(harness, client, owner)+ agent_id = await place_agent(client, controller, name="reviewer")+ await other.recheck()+ assert registry.controllers.is_bound(agent_id)+ reads = _count_placement_reads(other)++ await other.recheck()++ assert reads == []++ async def test_a_controller_renamed_with_no_signal_is_read(+ self, harness: Harness, postgres_url: str+ ) -> None:+ other, registry, _ = _process(postgres_url, harness.session_factory)+ owner = await add_member(harness.session_factory, "ada")+ async with harness.client() as client:+ controller = await enroll_console(harness, client, owner)+ agent_id = await place_agent(client, controller, name="reviewer")+ await other.recheck()++ async with tenant_session(harness.session_factory, TENANT_ZERO_ID) as session:+ await AgentControllerStore().update_details(+ session,+ TENANT_ZERO_ID,+ controller.controller_id,+ {"name": "workstation"},+ )+ await session.commit()+ await other.recheck()++ binding = registry.controllers.binding(agent_id)+ assert binding is not None and binding.controller_name == "workstation"+async def test_a_held_controller_is_nudged_only_with_what_it_was_not_told(self, harness: Harness) -> None:@@ -402,3 +444,17 @@ class TestReconcileKeepsNewerLocalChanges:assert presence.is_bound("a1")presence.reconcile([], agent_ids={"a1"}, since=presence.change_mark())assert not presence.is_bound("a1")+++def _count_placement_reads(signals: ControllerSignals) -> list[str]:+ """The tenants whose placements `signals` reads in full from now on."""+ reads: list[str] = []+ store = signals._definitions+ real = store.placements++ async def counted(session: Any, tenant_id: str) -> Any:+ reads.append(tenant_id)+ return await real(session, tenant_id)++ store.placements = counted # type: ignore[method-assign]+ return reads