Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
"""Management's nudges and bindings, kept current in whichever process holds
the controller.
A change commits in the process that handled the request; the controller it
affects may be streaming from any process, and every process answers that
controller's agents' requests. So every change is acted on here at once (the
local subscriptions and bindings, and on revocation the local presence) and
published on the `SignalBus` for the others, which do the same on receipt.
Nudges were already wake-ups, never the record: a controller woken pulls its
assignment or its operations. A binding is a copy of the record, which is why
a process that hears `controller.assignment_changed` or `controller.renamed`
reads the controller's placed definitions again rather than trusting the
signal (`bindings.reconcile_controller_bindings`). A move bumps the revision of
both controllers, so both are re-read.
Which process holds each controller's socket is kept the same way. A
controller opening a connection, another process taking one up, and a socket
attaching or going are announced as `controller.connection_changed`, and each
process told reads the controller's row again (`_read_connection`): an agent
run by a controller connected to any process then reads as live in every one,
and a connection this process holds that another process has since replaced
or taken up is closed `taken_over`.
Each change to a controller's connection is also told to the controller's
owner, whose Console shows the machine as connected or not
(`switch_core.user_changes`). The process that made the change reads the
owner from the controller's row once it has committed (`_tell_owner`), and
publishes the notice for every process to hand to the owner's sockets.
Nothing here depends on a signal arriving. When the signal connection comes
back, every binding and every controller's connection is read again and every
controller held here is re-nudged from the database, so one that missed a
signal while the connection was down pulls again, and one revoked meanwhile is
closed. The periodic re-check does the same reads, but nudges a held
controller only with what it has not been told yet (a newer revision, an
operation it has not heard of), so a quiet controller is not made to pull
every `RECHECK_SECONDS`.
"""
from __future__ import annotations
import logging
from collections.abc import Callable
from datetime import datetime
from typing import TYPE_CHECKING, Any
from switch_core.db.fingerprint import Fingerprint
from switch_core.db.session_scope import tenant_session
from switch_core.db.tenant_lookup import all_tenant_ids
from switch_core.management.bindings import (
reconcile_bindings,
reconcile_controller_bindings,
)
from switch_core.process_lease import ProcessLeases, read_leases
from switch_core.tenant_context import no_tenant
from switch_core.user_changes import MACHINE
if TYPE_CHECKING:
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
from switch_core.bridges.agent.protocol.controller_presence import (
ControllerPresence,
)
from switch_core.db.models import AgentController
from switch_core.db.stores.agent_controller_operation_store import (
AgentControllerOperationStore,
)
from switch_core.db.stores.agent_controller_store import (
AgentControllerStore,
ControllerConnection,
)
from switch_core.db.stores.agent_definition_store import AgentDefinitionStore
from switch_core.db.stores.room_store import RoomStore
from switch_core.db.stores.switch_core_process_store import (
SwitchCoreProcessStore,
)
from switch_core.management.notifier import ControllerNotifier
from switch_core.signals import SignalBus
from switch_core.user_changes import UserChangePublisher
logger = logging.getLogger(__name__)
ASSIGNMENT_CHANGED_SIGNAL = "controller.assignment_changed"
OPERATION_PENDING_SIGNAL = "controller.operation_pending"
REVOKED_SIGNAL = "controller.revoked"
RENAMED_SIGNAL = "controller.renamed"
CONNECTION_CHANGED_SIGNAL = "controller.connection_changed"
# How many times a controller's row is read again when this process changed
# that controller's connection while it was being read, before it is left to
# the next re-check.
_READ_ATTEMPTS = 3
class ControllerSignals:
def __init__(
self,
*,
notifier: ControllerNotifier,
presence: ControllerPresence,
bus: SignalBus,
session_factory: async_sessionmaker[AsyncSession],
controllers: AgentControllerStore,
definitions: AgentDefinitionStore,
operations: AgentControllerOperationStore,
rooms: RoomStore,
processes: SwitchCoreProcessStore,
clock: Callable[[], datetime],
user_changes: UserChangePublisher,
) -> None:
self.notifier = notifier
# The process this is, as signals and the rows naming a holder call it.
self.process_id = bus.process_id
self._presence = presence
self._bus = bus
self._session_factory = session_factory
self._controllers = controllers
self._definitions = definitions
self._operations = operations
self._rooms = rooms
self._processes = processes
self._clock = clock
self._user_changes = user_changes
# What each controller has been nudged with from this process, so the
# periodic re-check tells a held controller only what is new to it.
self._nudged_revision: dict[str, int] = {}
self._nudged_operations: dict[str, set[str]] = {}
# Each tenant's placements as the last full read of the bindings here
# found them, which the periodic re-check compares first.
self._binding_fingerprints: dict[str, Fingerprint] = {}
# Each tenant's controller connections as the last full read of them
# here found them, with the processes holding a lease then, which the
# periodic re-check compares first; and the controllers that read
# found, which a re-check that skips the read still counts as seen.
self._connection_fingerprints: dict[
str, tuple[Fingerprint, frozenset[str]]
] = {}
self._connections_seen: dict[str, set[str]] = {}
# Controllers, by (tenant id, controller id), whose bindings another
# process changed and this one has still to read.
self._bindings_owed = bus.pending_work(
"controller bindings", self._reconcile_controller
)
# Controllers, by (tenant id, controller id), whose connection another
# process changed and this one has still to read.
self._connections_owed = bus.pending_work(
"controller connections", self._read_connection_of
)
# Controllers, by (tenant id, controller id), whose connection this
# process changed and whose owner has still to be told.
self._owners_owed = bus.pending_work("controller owners", self._tell_owner)
bus.handle(ASSIGNMENT_CHANGED_SIGNAL, self._on_assignment_changed)
bus.handle(OPERATION_PENDING_SIGNAL, self._on_operation_pending)
bus.handle(REVOKED_SIGNAL, self._on_revoked)
bus.handle(RENAMED_SIGNAL, self._on_renamed)
bus.handle(CONNECTION_CHANGED_SIGNAL, self._on_connection_changed)
bus.on_resync(self.resync)
bus.on_recheck(self.recheck)
# ── Announcing, from the process that made the change ─────────────────────
def assignment_changed(
self, tenant_id: str, controller_id: str, revision: int
) -> None:
"""The process that committed the change has already bound its agents."""
self._nudge_assignment(controller_id, revision)
self._bus.publish(
ASSIGNMENT_CHANGED_SIGNAL,
tenant_id=tenant_id,
controller_id=controller_id,
revision=revision,
)
def operation_pending(
self, controller_id: str, *, operation_id: str, kind: str, agent_id: str | None
) -> None:
self._nudge_operation(
controller_id, operation_id=operation_id, kind=kind, agent_id=agent_id
)
self._bus.publish(
OPERATION_PENDING_SIGNAL,
controller_id=controller_id,
operation_id=operation_id,
operation_kind=kind,
agent_id=agent_id,
)
def revoked(self, controller_id: str) -> None:
self._revoke_here(controller_id)
self._bus.publish(REVOKED_SIGNAL, controller_id=controller_id)
def renamed(self, tenant_id: str, controller_id: str) -> None:
"""The controller's owner renamed it. A rename bumps no assignment
revision, so it needs a signal of its own for the other processes to
re-read the name their bindings carry."""
self._bus.publish(
RENAMED_SIGNAL, tenant_id=tenant_id, controller_id=controller_id
)
def connection_changed(self, tenant_id: str, controller_id: str) -> None:
"""The controller's connection row changed (an open, a connection
taken up, a socket attaching or going), and this process already acts
on it."""
self._bus.publish(
CONNECTION_CHANGED_SIGNAL, tenant_id=tenant_id, controller_id=controller_id
)
self._owners_owed.add((tenant_id, controller_id))
async def _tell_owner(self, key: tuple[str, str]) -> None:
"""Tell the controller's owner that its machine changed. Their
Console reads the machine again, so the notice carries nothing else."""
tenant_id, controller_id = key
async with tenant_session(self._session_factory, tenant_id) as session:
controller = await self._controllers.get(session, tenant_id, controller_id)
if controller is None:
return
self._user_changes.publish(
tenant_id, controller.owner_id, MACHINE, controller_id
)
# ── Receiving, in every other process ─────────────────────────────────────
def _on_assignment_changed(self, signal: dict[str, Any]) -> None:
# Nudged at once, as before bindings were re-read here: the
# controller's pull may land on any process, so waiting for this
# one's read would only delay it.
self._nudge_assignment(signal["controller_id"], signal["revision"])
self._bindings_owed.add((signal["tenant_id"], signal["controller_id"]))
def _on_operation_pending(self, signal: dict[str, Any]) -> None:
self._nudge_operation(
signal["controller_id"],
operation_id=signal["operation_id"],
kind=signal["operation_kind"],
agent_id=signal["agent_id"],
)
def _on_revoked(self, signal: dict[str, Any]) -> None:
self._revoke_here(signal["controller_id"])
def _on_renamed(self, signal: dict[str, Any]) -> None:
self._bindings_owed.add((signal["tenant_id"], signal["controller_id"]))
def _on_connection_changed(self, signal: dict[str, Any]) -> None:
self._connections_owed.add((signal["tenant_id"], signal["controller_id"]))
async def _reconcile_controller(self, key: tuple[str, str]) -> None:
tenant_id, controller_id = key
await reconcile_controller_bindings(
session_factory=self._session_factory,
definitions=self._definitions,
controllers=self._controllers,
presence=self._presence,
tenant_id=tenant_id,
controller_id=controller_id,
)
# An agent just bound to a controller another process holds needs its
# rooms here, as the controller's other agents have them, to be
# present in them at once: reading the connection again loads them.
if self._presence.holder_elsewhere(controller_id) is not None:
await self._read_connection(tenant_id, controller_id)
# ── Which process holds each controller's socket ──────────────────────────
async def _read_connection_of(self, key: tuple[str, str]) -> None:
tenant_id, controller_id = key
await self._read_connection(tenant_id, controller_id)
async def _read_connection(self, tenant_id: str, controller_id: str) -> None:
"""Read the controller's row again and act on it here."""
for _ in range(_READ_ATTEMPTS):
mark = self._presence.connection_mark()
async with tenant_session(self._session_factory, tenant_id) as session:
controller = await self._controllers.get(
session, tenant_id, controller_id
)
if controller is None:
return
leases = await read_leases(session, self._processes)
rooms = await self._rooms_if_held_elsewhere(session, controller, leases)
if self._presence.connection_changed_since(controller_id, mark):
# Changed here while it was read: what was read may be older.
continue
self._apply_connection(controller, leases, rooms)
return
async def read_connections(self, *, check_first: bool = False) -> None:
"""Read every controller's connection row again, in every tenant, and
make this process's view of who holds each socket agree. Run at
startup, before the bridge serves, and on resync and each re-check.
With `check_first`, as the periodic re-check runs it, a tenant whose
connection rows and live processes are what its last full read here
found is not read again: comparing costs one row, and reading them,
with the rooms of every agent of each controller another process
holds, cost each process a few seconds of database time every 30
seconds at thousands of controllers. Those rooms are kept up to date
by the membership signals and `MessageRouter.recheck_agent_rooms`."""
seen: set[str] = set()
elsewhere = self._presence.controllers_elsewhere()
with no_tenant():
tenant_ids = await all_tenant_ids(self._session_factory)
for tenant_id in tenant_ids:
mark = self._presence.connection_mark()
async with tenant_session(self._session_factory, tenant_id) as session:
leases = await read_leases(session, self._processes)
holding = frozenset(
process_id
for process_id in leases.leases
if leases.holds(process_id)
)
last = self._connection_fingerprints.get(tenant_id)
if (
check_first
and last is not None
and last
== (
await self._controllers.connections_fingerprint(
session, tenant_id
),
holding,
)
):
seen |= self._connections_seen.get(tenant_id, set())
continue
read, fingerprint = await self._controllers.connections(
session, tenant_id
)
found = [
controller
for controller in read
if controller.tenant_id == tenant_id
]
rooms = {
controller.id: await self._rooms_if_held_elsewhere(
session, controller, leases
)
for controller in found
}
self._connection_fingerprints[tenant_id] = (fingerprint, holding)
self._connections_seen[tenant_id] = {controller.id for controller in found}
for controller in found:
seen.add(controller.id)
if self._presence.connection_changed_since(controller.id, mark):
# Changed here while it was read; read it again on its own.
await self._read_connection(tenant_id, controller.id)
continue
self._apply_connection(controller, leases, rooms[controller.id])
for controller_id in elsewhere - seen:
self._presence.held_elsewhere(controller_id, None)
async def _rooms_if_held_elsewhere(
self,
session: AsyncSession,
controller: AgentController | ControllerConnection,
leases: ProcessLeases,
) -> dict[str, set[str]]:
"""The rooms of each agent the controller runs, when another process
holds its socket: they are what makes those agents present in a room
here, and the stream that loads them runs over there."""
if self._held_elsewhere(controller, leases) is None:
return {}
# One query for all of them: every process re-reads this for every
# controller another process holds at each re-check, so a query per
# agent would grow with processes times agents.
return await self._rooms.room_ids_by_agent(
session, self._presence.agents_of(controller.id)
)
def _held_elsewhere(
self, controller: AgentController | ControllerConnection, leases: ProcessLeases
) -> str | None:
"""The other process holding the controller's live socket, if any."""
process_id = controller.connection_process_id
if (
controller.connection_id is None
or controller.disconnected_at is not None
or process_id is None
or process_id == self.process_id
or not leases.holds(process_id)
):
return None
return process_id
def _apply_connection(
self,
controller: AgentController | ControllerConnection,
leases: ProcessLeases,
rooms: dict[str, set[str]],
) -> None:
local = self._presence.current_connection(controller.id)
if local is not None and controller.open_connection_id is not None:
# The row names the connection the controller opened last and the
# process holding it. A connection held here that is not that one,
# or is that one but taken up by another process, is over here.
if (
controller.open_connection_id != local.id
or controller.open_process_id != self.process_id
):
self._presence.supersede(controller.id)
self._presence.held_elsewhere(
controller.id, self._held_elsewhere(controller, leases)
)
for agent_id, agent_rooms in rooms.items():
self._presence.set_rooms(agent_id, agent_rooms)
# ── Acting here ───────────────────────────────────────────────────────────
def _nudge_assignment(self, controller_id: str, revision: int) -> None:
if revision > self._nudged_revision.get(controller_id, -1):
self._nudged_revision[controller_id] = revision
self.notifier.assignment_changed(controller_id, revision)
def _nudge_operation(
self, controller_id: str, *, operation_id: str, kind: str, agent_id: str | None
) -> None:
self._nudged_operations.setdefault(controller_id, set()).add(operation_id)
self.notifier.operation_pending(
controller_id, operation_id=operation_id, kind=kind, agent_id=agent_id
)
def _revoke_here(self, controller_id: str) -> None:
"""The stream ends, the controller cannot open another here, and this
process's token cache forgets it."""
self.notifier.credential_revoked(controller_id)
self._presence.revoke_controller(controller_id)
# ── Resync and re-check ───────────────────────────────────────────────────
async def resync(self) -> None:
"""Re-read every binding and every controller's connection, then
re-nudge every controller this process holds from what the database
says now: its assignment revision and its open operations, or its
revocation."""
await self._reconcile_all_bindings(check_first=False)
await self._read_all_connections()
await self._renudge_held(only_new=False)
async def recheck(self) -> None:
"""What `resync` does, but the bindings are read only when a tenant's
placements have changed since the last full read, the connections only
when they or the processes holding a lease have, and a held controller
is nudged only with what this process has not told it yet."""
await self._reconcile_all_bindings(check_first=True)
await self._read_all_connections(check_first=True)
await self._renudge_held(only_new=True)
async def _read_all_connections(self, *, check_first: bool = False) -> None:
try:
await self.read_connections(check_first=check_first)
except Exception:
logger.error("Could not re-read controller connections", exc_info=True)
async def _reconcile_all_bindings(self, *, check_first: bool) -> None:
try:
await reconcile_bindings(
session_factory=self._session_factory,
definitions=self._definitions,
presence=self._presence,
fingerprints=self._binding_fingerprints if check_first else None,
)
except Exception:
logger.error("Could not re-read controller bindings", exc_info=True)
async def _renudge_held(self, *, only_new: bool) -> None:
held = self._presence.held_connections()
# What a controller held elsewhere was told is no use here: if it
# connects to this process later, its opening frame carries the state.
held_ids = {conn.controller_id for conn in held}
for remembered in (self._nudged_revision, self._nudged_operations):
for controller_id in set(remembered) - held_ids:
del remembered[controller_id]
for conn in held:
try:
async with tenant_session(
self._session_factory, conn.tenant_id
) as session:
controller = await self._controllers.get(
session, conn.tenant_id, conn.controller_id
)
if controller is None or controller.revoked_at is not None:
self._revoke_here(conn.controller_id)
continue
offered = await self._operations.list_offered(
session, conn.tenant_id, conn.controller_id, self._clock()
)
revision = controller.assignment_revision
if not only_new or revision > self._nudged_revision.get(
conn.controller_id, -1
):
self._nudge_assignment(conn.controller_id, revision)
told = self._nudged_operations.get(conn.controller_id, set())
for operation in offered:
if only_new and operation.id in told:
continue
self._nudge_operation(
conn.controller_id,
operation_id=operation.id,
kind=operation.kind,
agent_id=operation.agent_id,
)
# Only the operations still offered need remembering.
self._nudged_operations[conn.controller_id] = {
operation.id for operation in offered
}
except Exception:
logger.error(
"Could not resync controller %s", conn.controller_id, exc_info=True
)