Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Re-check controller bindings in one narrow query, and only when placements changed (CHOO-3667)

Abel Dantas2c801eb6b83a
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 76013f638..e245d2d5d 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -231,6 +231,12 @@ version of their own to them without also giving them a release of their own.
30-second re-check reads every client only when a fingerprint of the
workspace's clients and agents has changed since its last full read, where
it read them all every time.
+- **The 30-second re-check of which controller runs each agent reads one
+ query, and nothing when no placement changed.** Every process read every
+ placed agent's definition, document included, then each controller with a
+ query of its own. It now reads the columns a binding is made of in one
+ query per workspace, and only when a fingerprint of the placements has
+ changed since its last full read.
#### Fixed
- **On Slack, the 👀 and the activity card no longer arrive after the agent's
diff --git a/core/switch_core/db/stores/agent_definition_store.py b/core/switch_core/db/stores/agent_definition_store.py
index cabac2a77..48d8b4c80 100644
--- a/core/switch_core/db/stores/agent_definition_store.py
+++ b/core/switch_core/db/stores/agent_definition_store.py
@@ -1,11 +1,40 @@
from __future__ import annotations
+from dataclasses import dataclass
+from datetime import datetime
from typing import Any
-from sqlalchemy import delete, select, update
+from sqlalchemy import ColumnElement, cast, delete, func, select, update
from sqlalchemy.ext.asyncio import AsyncSession
+from sqlalchemy.types import Text
-from switch_core.db.models import Agent, AgentDefinition
+from switch_core.db.fingerprint import Fingerprint, row_hash
+from switch_core.db.models import Agent, AgentController, AgentDefinition
+
+
+@dataclass(frozen=True)
+class Placement:
+ """A definition placed on a controller, with what a binding needs of the
+ controller: its name, and whether it is revoked. `controller_name` is None
+ when the controller's row is gone."""
+
+ agent_id: str
+ controller_id: str
+ desired_state: str
+ controller_name: str | None
+ controller_revoked_at: datetime | None
+
+
+def _placement_hash() -> ColumnElement[int]:
+ """What a binding is made of (`management/bindings.py`), so a fingerprint
+ of it changes exactly when some binding would."""
+ return row_hash(
+ AgentDefinition.agent_id,
+ func.coalesce(AgentDefinition.controller_id, ""),
+ AgentDefinition.desired_state,
+ func.coalesce(AgentController.name, ""),
+ cast(AgentController.revoked_at.is_not(None), Text),
+ )
class DefinitionChanged(Exception):
@@ -101,6 +130,54 @@ class AgentDefinitionStore:
)
return list(result.scalars().all())
+ async def placements(
+ self, session: AsyncSession, tenant_id: str
+ ) -> tuple[list[Placement], Fingerprint]:
+ """Every definition placed on a controller, with its controller's name
+ and revocation, and the fingerprint of them, in one statement: the
+ columns a binding is made of and nothing else, so neither a
+ definition's document nor a controller's status is read."""
+ result = await session.execute(
+ select(
+ AgentDefinition.agent_id,
+ AgentDefinition.controller_id,
+ AgentDefinition.desired_state,
+ AgentController.name,
+ AgentController.revoked_at,
+ func.count().over(),
+ func.sum(_placement_hash()).over(),
+ )
+ .outerjoin(AgentController, _placed_on())
+ .where(*_placed_in(tenant_id))
+ )
+ rows = result.all()
+ if not rows:
+ return [], (0, 0)
+ placements = [
+ Placement(
+ agent_id=agent_id,
+ controller_id=controller_id,
+ desired_state=desired_state,
+ controller_name=name,
+ controller_revoked_at=revoked_at,
+ )
+ for agent_id, controller_id, desired_state, name, revoked_at, _, _ in rows
+ ]
+ return placements, (int(rows[0][5]), int(rows[0][6]))
+
+ async def placements_fingerprint(
+ self, session: AsyncSession, tenant_id: str
+ ) -> Fingerprint:
+ """The fingerprint `placements` returns, without the rows."""
+ result = await session.execute(
+ select(func.count(), func.coalesce(func.sum(_placement_hash()), 0))
+ .select_from(AgentDefinition)
+ .outerjoin(AgentController, _placed_on())
+ .where(*_placed_in(tenant_id))
+ )
+ count, total = result.one()
+ return int(count), int(total)
+
async def update(
self,
session: AsyncSession,
@@ -154,3 +231,16 @@ class AgentDefinitionStore:
)
.execution_options(synchronize_session=False)
)
+
+
+def _placed_on() -> ColumnElement[bool]:
+ return (AgentController.tenant_id == AgentDefinition.tenant_id) & (
+ AgentController.id == AgentDefinition.controller_id
+ )
+
+
+def _placed_in(tenant_id: str) -> tuple[ColumnElement[bool], ...]:
+ return (
+ AgentDefinition.tenant_id == tenant_id,
+ AgentDefinition.controller_id.is_not(None),
+ )
diff --git a/core/switch_core/management/bindings.py b/core/switch_core/management/bindings.py
index b5f82cba7..303155cef 100644
--- a/core/switch_core/management/bindings.py
+++ b/core/switch_core/management/bindings.py
@@ -27,6 +27,7 @@ from switch_core.bridges.agent.protocol.controller_presence import (
Binding,
ControllerPresence,
)
+from switch_core.db.fingerprint import Fingerprint
from switch_core.db.models import AgentController
from switch_core.db.models import AgentDefinition as AgentDefinitionRow
from switch_core.db.session_scope import tenant_session
@@ -55,15 +56,12 @@ async def load_bindings(
*,
session_factory: async_sessionmaker[AsyncSession],
definitions: AgentDefinitionStore,
- controllers: AgentControllerStore,
presence: ControllerPresence,
) -> int:
"""Load every placed definition into `presence`, and mark the controllers
already revoked that agents are still placed on. Returns how many."""
- bindings, revoked = await _read_bindings(
- session_factory=session_factory,
- definitions=definitions,
- controllers=controllers,
+ bindings, revoked, _ = await _read_bindings(
+ session_factory=session_factory, definitions=definitions
)
presence.load(bindings)
for controller_id in revoked:
@@ -76,17 +74,29 @@ async def reconcile_bindings(
*,
session_factory: async_sessionmaker[AsyncSession],
definitions: AgentDefinitionStore,
- controllers: AgentControllerStore,
presence: ControllerPresence,
+ fingerprints: dict[str, Fingerprint] | None = None,
) -> None:
"""Bring every binding in `presence` in line with the tables, changing
- only what differs, and close the controllers revoked meanwhile."""
+ only what differs, and close the controllers revoked meanwhile.
+
+ With `fingerprints`, each tenant's placements as the last full read here
+ found them, nothing is read when no tenant's placements have changed
+ since (`AgentDefinitionStore.placements_fingerprint`), and `fingerprints`
+ is brought up to date when they have. That is the periodic re-check,
+ which every process runs and which on a quiet deployment finds nothing to
+ repair."""
+ if fingerprints is not None and await _unchanged(
+ session_factory, definitions, fingerprints
+ ):
+ return
since = presence.change_mark()
- bindings, revoked = await _read_bindings(
- session_factory=session_factory,
- definitions=definitions,
- controllers=controllers,
+ bindings, revoked, read = await _read_bindings(
+ session_factory=session_factory, definitions=definitions
)
+ if fingerprints is not None:
+ fingerprints.clear()
+ fingerprints.update(read)
presence.reconcile(
bindings,
agent_ids=presence.bound_agent_ids() | {b.agent_id for b in bindings},
@@ -138,36 +148,55 @@ async def reconcile_controller_bindings(
)
+async def _unchanged(
+ session_factory: async_sessionmaker[AsyncSession],
+ definitions: AgentDefinitionStore,
+ fingerprints: dict[str, Fingerprint],
+) -> bool:
+ """Whether every tenant's placements read as they did when
+ `fingerprints` were taken, and there are no tenants beyond those."""
+ tenant_ids = await all_tenant_ids(session_factory)
+ if set(tenant_ids) != set(fingerprints):
+ return False
+ for tenant_id in tenant_ids:
+ async with tenant_session(session_factory, tenant_id) as session:
+ now = await definitions.placements_fingerprint(session, tenant_id)
+ if now != fingerprints[tenant_id]:
+ return False
+ return True
+
+
async def _read_bindings(
*,
session_factory: async_sessionmaker[AsyncSession],
definitions: AgentDefinitionStore,
- controllers: AgentControllerStore,
-) -> tuple[list[Binding], set[str]]:
- """Every placed definition's binding, and the revoked controllers that
- agents are still placed on."""
+) -> tuple[list[Binding], set[str], dict[str, Fingerprint]]:
+ """Every placed definition's binding, the revoked controllers that agents
+ are still placed on, and each tenant's fingerprint of its placements, one
+ query per tenant."""
bindings: list[Binding] = []
revoked: set[str] = set()
+ fingerprints: dict[str, Fingerprint] = {}
for tenant_id in await all_tenant_ids(session_factory):
async with tenant_session(session_factory, tenant_id) as session:
- placed_on: dict[str, AgentController] = {}
- # Filtered on the row's own tenant as well as by the store: see
- # `db/tenant_lookup.py`, "a fan-out ... filters what it reads back".
- for row in await definitions.list_placed(session, tenant_id):
- if row.tenant_id != tenant_id or row.controller_id is None:
- continue
- controller = placed_on.get(row.controller_id)
- if controller is None:
- controller = await controllers.get(
- session, tenant_id, row.controller_id
- )
- if controller is None:
- raise RuntimeError(
- f"agent {row.agent_id} is placed on controller "
- f"{row.controller_id}, which does not exist"
- )
- placed_on[controller.id] = controller
- if controller.revoked_at is not None:
- revoked.add(controller.id)
- bindings.append(binding_of(tenant_id, row, controller))
- return bindings, revoked
+ placements, fingerprints[tenant_id] = await definitions.placements(
+ session, tenant_id
+ )
+ for placement in placements:
+ if placement.controller_name is None:
+ raise RuntimeError(
+ f"agent {placement.agent_id} is placed on controller "
+ f"{placement.controller_id}, which does not exist"
+ )
+ if placement.controller_revoked_at is not None:
+ revoked.add(placement.controller_id)
+ bindings.append(
+ Binding(
+ agent_id=placement.agent_id,
+ controller_id=placement.controller_id,
+ tenant_id=tenant_id,
+ controller_name=placement.controller_name,
+ running=placement.desired_state == "running",
+ )
+ )
+ return bindings, revoked, fingerprints
diff --git a/core/switch_core/management/signals.py b/core/switch_core/management/signals.py
index 5f2a1200c..d9aef429b 100644
--- a/core/switch_core/management/signals.py
+++ b/core/switch_core/management/signals.py
@@ -45,6 +45,7 @@ from collections.abc import Callable
from datetime import datetime
from typing import TYPE_CHECKING, Any
+from switch_core.db.fingerprint import Fingerprint
from switch_core.db.session_scope import tenant_session
from switch_core.db.tenant_lookup import all_tenant_ids
from switch_core.management.bindings import (
@@ -122,6 +123,9 @@ class ControllerSignals:
# periodic re-check tells a held controller only what is new to it.
self._nudged_revision: dict[str, int] = {}
self._nudged_operations: dict[str, set[str]] = {}
+ # Each tenant's placements as the last full read of the bindings here
+ # found them, which the periodic re-check compares first.
+ self._binding_fingerprints: dict[str, Fingerprint] = {}
# Controllers, by (tenant id, controller id), whose bindings another
# process changed and this one has still to read.
self._bindings_owed = bus.pending_work(
@@ -391,14 +395,16 @@ class ControllerSignals:
re-nudge every controller this process holds from what the database
says now: its assignment revision and its open operations, or its
revocation."""
- await self._reconcile_all_bindings()
+ await self._reconcile_all_bindings(check_first=False)
await self._read_all_connections()
await self._renudge_held(only_new=False)
async def recheck(self) -> None:
- """What `resync` does, but a held controller is nudged only with what
- this process has not told it yet."""
- await self._reconcile_all_bindings()
+ """What `resync` does, but the bindings are read only when a tenant's
+ placements have changed since the last full read, and a held
+ controller is nudged only with what this process has not told it
+ yet."""
+ await self._reconcile_all_bindings(check_first=True)
await self._read_all_connections()
await self._renudge_held(only_new=True)
@@ -408,13 +414,13 @@ class ControllerSignals:
except Exception:
logger.error("Could not re-read controller connections", exc_info=True)
- async def _reconcile_all_bindings(self) -> None:
+ async def _reconcile_all_bindings(self, *, check_first: bool) -> None:
try:
await reconcile_bindings(
session_factory=self._session_factory,
definitions=self._definitions,
- controllers=self._controllers,
presence=self._presence,
+ fingerprints=self._binding_fingerprints if check_first else None,
)
except Exception:
logger.error("Could not re-read controller bindings", exc_info=True)
diff --git a/core/switch_core/management/wiring.py b/core/switch_core/management/wiring.py
index c9d93e497..c209b1ea8 100644
--- a/core/switch_core/management/wiring.py
+++ b/core/switch_core/management/wiring.py
@@ -124,7 +124,6 @@ class Management:
return await load_bindings(
session_factory=self.session_factory,
definitions=self.service.definitions,
- controllers=self.service.controllers,
presence=self.service.presence,
)
diff --git a/core/tests/switch_core/management/test_controller_routes.py b/core/tests/switch_core/management/test_controller_routes.py
index e0cc50daf..a20c0e55c 100644
--- a/core/tests/switch_core/management/test_controller_routes.py
+++ b/core/tests/switch_core/management/test_controller_routes.py
@@ -916,7 +916,6 @@ class TestPlacementBindsTheAgentInCore:
await load_bindings(
session_factory=harness.session_factory,
definitions=harness.management.service.definitions,
- controllers=harness.management.service.controllers,
presence=restarted,
)
diff --git a/core/tests/switch_core/management/test_controller_signals.py b/core/tests/switch_core/management/test_controller_signals.py
index 6c6bca27a..6e6d2df23 100644
--- a/core/tests/switch_core/management/test_controller_signals.py
+++ b/core/tests/switch_core/management/test_controller_signals.py
@@ -5,7 +5,7 @@ from __future__ import annotations
import asyncio
import uuid
from datetime import UTC, datetime
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Any
import pytest
from sqlalchemy.ext.asyncio import create_async_engine
@@ -352,6 +352,48 @@ class TestRecheck:
await other.recheck()
assert not registry.controllers.is_bound(agent_id)
+ async def test_with_nothing_changed_it_reads_no_placement(
+ self, harness: Harness, postgres_url: str
+ ) -> None:
+ """Every process re-checks every 30 seconds, and with 10,000 managed
+ agents a read of every placement on each was a large share of what the
+ processes did while nothing happened."""
+ other, registry, _ = _process(postgres_url, harness.session_factory)
+ owner = await add_member(harness.session_factory, "ada")
+ async with harness.client() as client:
+ controller = await enroll_console(harness, client, owner)
+ agent_id = await place_agent(client, controller, name="reviewer")
+ await other.recheck()
+ assert registry.controllers.is_bound(agent_id)
+ reads = _count_placement_reads(other)
+
+ await other.recheck()
+
+ assert reads == []
+
+ async def test_a_controller_renamed_with_no_signal_is_read(
+ self, harness: Harness, postgres_url: str
+ ) -> None:
+ other, registry, _ = _process(postgres_url, harness.session_factory)
+ owner = await add_member(harness.session_factory, "ada")
+ async with harness.client() as client:
+ controller = await enroll_console(harness, client, owner)
+ agent_id = await place_agent(client, controller, name="reviewer")
+ await other.recheck()
+
+ async with tenant_session(harness.session_factory, TENANT_ZERO_ID) as session:
+ await AgentControllerStore().update_details(
+ session,
+ TENANT_ZERO_ID,
+ controller.controller_id,
+ {"name": "workstation"},
+ )
+ await session.commit()
+ await other.recheck()
+
+ binding = registry.controllers.binding(agent_id)
+ assert binding is not None and binding.controller_name == "workstation"
+
async def test_a_held_controller_is_nudged_only_with_what_it_was_not_told(
self, harness: Harness
) -> None:
@@ -402,3 +444,17 @@ class TestReconcileKeepsNewerLocalChanges:
assert presence.is_bound("a1")
presence.reconcile([], agent_ids={"a1"}, since=presence.change_mark())
assert not presence.is_bound("a1")
+
+
+def _count_placement_reads(signals: ControllerSignals) -> list[str]:
+ """The tenants whose placements `signals` reads in full from now on."""
+ reads: list[str] = []
+ store = signals._definitions
+ real = store.placements
+
+ async def counted(session: Any, tenant_id: str) -> Any:
+ reads.append(tenant_id)
+ return await real(session, tenant_id)
+
+ store.placements = counted # type: ignore[method-assign]
+ return reads