Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Keep a controller token's rows until a signal says they changed (CHOO-3667)
Abel Dantas0a06092d1612
diff --git a/CHANGELOG.md b/CHANGELOG.mdindex 8b9791612..6bba22851 100644--- a/CHANGELOG.md+++ b/CHANGELOG.md@@ -224,6 +224,13 @@ version of their own to them without also giving them a release of their own.group whose subgroup would move up next to a namesake is refused too, andthe dashboard now shows why. Names can no longer contain `/` or be blank.Existing duplicates are left as they are and can be reached by id.+- **A controller's requests no longer read its row and its agent's row each+ time.** They were kept for 5 seconds, and a controller's requests, spread+ over several processes, rarely came back to one within that. They are now+ kept until a signal says they changed, which triggers on `agents` and+ `agent_controllers` send, and a miss reads all of the controller's agents+ at once. `CONTROLLER_AUTH_CACHE_MAX_AGENTS` (20,000) bounds the agent rows+ kept.#### Fixed- **On Slack, the π and the activity card no longer arrive after the agent'sdiff --git a/core/switch_core/bridges/agent/auth.py b/core/switch_core/bridges/agent/auth.pyindex 7c8d4e514..92428eb5f 100644--- a/core/switch_core/bridges/agent/auth.py+++ b/core/switch_core/bridges/agent/auth.py@@ -438,18 +438,9 @@ class BearerAuthMiddleware:)(scope, receive, send)return- auth_cache = controller_auth.auth_cache- agent = auth_cache.agent(principal.tenant_id, agent_id)+ agent = controller_auth.auth_cache.agent(principal.tenant_id, agent_id)if agent is None:- generation = auth_cache.generation- async with tenant_session(- self._session_factory, principal.tenant_id- ) as session:- agent = await self._agent_store.get(session, agent_id)- if agent is not None:- session.expunge(agent)- if agent is not None:- auth_cache.put_agent(principal.tenant_id, agent, generation)+ agent = await self._read_agents_of(controller_auth, principal, agent_id)if agent is None:await _controller_refusal(NOT_ASSIGNED, f"Agent {agent_id} does not exist.", 403@@ -465,6 +456,30 @@ class BearerAuthMiddleware:):await self.app(scope, receive, send)+ async def _read_agents_of(+ self,+ controller_auth: ControllerAuthenticator,+ principal: ControllerPrincipal,+ agent_id: str,+ ) -> Agent | None:+ """The row of the agent a controller acts as, read together with the+ rows of every other agent bound to the controller, which the cache+ keeps. A controller relays all of its agents' requests, and they are+ spread over every process, so one read per agent would miss on most+ of them."""+ auth_cache = controller_auth.auth_cache+ wanted = controller_auth.presence.agents_of(principal.controller_id)+ wanted.add(agent_id)+ ticket = auth_cache.ticket()+ async with tenant_session(+ self._session_factory, principal.tenant_id+ ) as session:+ agents = await self._agent_store.get_many(session, wanted)+ for found in agents:+ session.expunge(found)+ auth_cache.put_agents(principal.tenant_id, agents, ticket)+ return next((found for found in agents if found.id == agent_id), None)+async def _resolve_api_key(self, token: str) -> tuple[ApiKey | None, Agent | None]:"""Look up the token in api_keys once; return (api_key_row, agent_or_None).diff --git a/core/switch_core/bridges/agent/controller_auth_cache.py b/core/switch_core/bridges/agent/controller_auth_cache.pyindex 00960ce59..7a74b35b0 100644--- a/core/switch_core/bridges/agent/controller_auth_cache.py+++ b/core/switch_core/bridges/agent/controller_auth_cache.py@@ -2,17 +2,47 @@ from __future__ import annotationsimport timefrom collections import OrderedDict-from typing import TYPE_CHECKING+from collections.abc import Iterable+from dataclasses import dataclass+from typing import TYPE_CHECKING, Anyfrom switch_core.bridges.agent.protocol.liveness import HEARTBEAT_TTL_SECONDS+from switch_core.db.auth_rows_notify_ddl import (+ AGENT_CHANGED_SIGNAL,+ CONTROLLER_AUTH_CHANGED_SIGNAL,+)from switch_core.db.models import Agentif TYPE_CHECKING:from switch_core.bridges.agent.auth import ControllerPrincipal+ from switch_core.signals import SignalBus+++@dataclass(frozen=True)+class ReadTicket:+ """Taken before a read and handed back with what it read, so the cache+ can tell whether anything it was told while the read was in flight makes+ the read stale."""++ generation: int+ # The signal bus's listening connection when the read began, or None if+ # it was not listening then.+ listening: int | None+++@dataclass+class _Entry[V]:+ tenant_id: str+ value: V+ expires_at: float+ # The listening connection the entry was read under, while it is the one+ # listening, keeps the entry past `expires_at`. None for a read made while+ # the bus did not listen.+ held_under: int | Noneclass ControllerAuthCache:- """A short-lived memo of the two reads a controller access token costs.+ """A memo of the two reads a controller access token costs.A controller token is a signed JWT, so verifying it needs no database. Butevery request it authenticates also re-reads its controller row, to refuse@@ -26,6 +56,21 @@ class ControllerAuthCache:the binding Core holds in memory (``ControllerPresence``), which is checkedon every request and never cached here.+ How long an entry is kept:++ - **Until a signal says it changed, while one could.** A cache that+ follows the signal bus (``follow``) keeps an entry read while the bus+ listened for as long as that same connection listens. Triggers on+ ``agents`` and ``agent_controllers`` send ``agent.changed`` and+ ``controller.auth_changed`` (``db/auth_rows_notify_ddl.py``), and every+ process drops the entry it names. A controller's requests are spread+ over every process, so each process sees one controller or agent only+ every so often, and an entry kept for seconds was gone by the next+ request for it.+ - **Otherwise for ``ttl_seconds``.** An entry read while the bus did not+ listen, or kept past the connection it was read under, lasts the TTL: a+ signal sent between two connections reached neither.+What it deliberately does not do:- **Cache a refusal.** Only a controller that authenticated and an agent@@ -33,15 +78,17 @@ class ControllerAuthCache:always reaches the database.- **Outlive a revocation or a binding change.** ``ControllerPresence``calls ``invalidate_controller`` when a controller is revoked and- ``invalidate_agent`` when an agent is bound, moved, unbound or deleted;- expiry is the backstop, not the mechanism. A read that was in flight+ ``invalidate_agent`` when an agent is bound, moved, unbound or deleted,+ and the signals above reach every process. A read that was in flightwhen an entry was invalidated does not store what it read.A ``ttl_seconds`` of 0 disables it. The TTL is refused at or above theheartbeat TTL, for the reason :class:`ApiKeyCache` gives."""- def __init__(self, *, ttl_seconds: float, max_entries: int) -> None:+ def __init__(+ self, *, ttl_seconds: float, max_entries: int, max_agents: int | None = None+ ) -> None:if ttl_seconds < 0:raise ValueError(f"ttl_seconds must not be negative, got {ttl_seconds!r}")if ttl_seconds >= HEARTBEAT_TTL_SECONDS:@@ -51,81 +98,134 @@ class ControllerAuthCache:)if max_entries < 1:raise ValueError(f"max_entries must be at least 1, got {max_entries!r}")+ if max_agents is not None and max_agents < 1:+ raise ValueError(f"max_agents must be at least 1, got {max_agents!r}")self._ttl = ttl_seconds- self._max_entries = max_entries- self._controllers: OrderedDict[- tuple[str, str], tuple[float, ControllerPrincipal]- ] = OrderedDict()- self._agents: OrderedDict[tuple[str, str], tuple[float, Agent]] = OrderedDict()- # Bumped by every invalidation. A reader takes it before going to the- # database and hands it back with what it read; a read that straddled- # an invalidation is not stored.+ self._max_controllers = max_entries+ self._max_agents = max_agents if max_agents is not None else max_entries+ # Keyed by id alone: controller and agent ids are unique across+ # tenants, so a signal naming one finds its entry without a scan.+ self._controllers: OrderedDict[str, _Entry[ControllerPrincipal]] = OrderedDict()+ self._agents: OrderedDict[str, _Entry[Agent]] = OrderedDict()+ # Bumped by every invalidation. A reader takes a ticket before going+ # to the database and hands it back with what it read; a read that+ # straddled an invalidation is not stored.self._generation = 0+ self._bus: SignalBus | None = None++ def follow(self, bus: SignalBus) -> None:+ """Keep entries while `bus` listens, and drop the ones its signals+ name."""+ self._bus = bus+ bus.handle(AGENT_CHANGED_SIGNAL, self._on_agent_changed)+ bus.handle(CONTROLLER_AUTH_CHANGED_SIGNAL, self._on_controller_changed)@propertydef enabled(self) -> bool:return self._ttl > 0- @property- def generation(self) -> int:- return self._generation+ def ticket(self) -> ReadTicket:+ return ReadTicket(generation=self._generation, listening=self._listening())def controller(self, tenant_id: str, controller_id: str) -> ControllerPrincipal | None:- return _get(self._controllers, (tenant_id, controller_id))+ return self._get(self._controllers, tenant_id, controller_id)- def put_controller(self, principal: ControllerPrincipal, generation: int) -> None:- if not self.enabled or generation != self._generation:+ def put_controller(+ self, principal: ControllerPrincipal, ticket: ReadTicket+ ) -> None:+ if not self._fresh(ticket):returnself._put(- self._controllers, (principal.tenant_id, principal.controller_id), principal+ self._controllers,+ self._max_controllers,+ principal.tenant_id,+ principal.controller_id,+ principal,+ self._held_under(ticket),)def agent(self, tenant_id: str, agent_id: str) -> Agent | None:- return _get(self._agents, (tenant_id, agent_id))+ return self._get(self._agents, tenant_id, agent_id)- def put_agent(self, tenant_id: str, agent: Agent, generation: int) -> None:- if not self.enabled or generation != self._generation:+ def put_agents(+ self, tenant_id: str, agents: Iterable[Agent], ticket: ReadTicket+ ) -> None:+ if not self._fresh(ticket):return- self._put(self._agents, (tenant_id, agent.id), agent)+ held_under = self._held_under(ticket)+ for agent in agents:+ self._put(+ self._agents, self._max_agents, tenant_id, agent.id, agent, held_under+ )def invalidate_controller(self, controller_id: str) -> None:self._generation += 1- for key in [k for k in self._controllers if k[1] == controller_id]:- del self._controllers[key]+ self._controllers.pop(controller_id, None)def invalidate_agent(self, agent_id: str) -> None:self._generation += 1- for key in [k for k in self._agents if k[1] == agent_id]:- del self._agents[key]+ self._agents.pop(agent_id, None)def clear(self) -> None:self._generation += 1self._controllers.clear()self._agents.clear()+ def _on_agent_changed(self, signal: dict[str, Any]) -> None:+ self.invalidate_agent(signal["agent_id"])++ def _on_controller_changed(self, signal: dict[str, Any]) -> None:+ self.invalidate_controller(signal["controller_id"])++ def _listening(self) -> int | None:+ bus = self._bus+ if bus is None or not bus.connected.is_set():+ return None+ return bus.listens++ def _fresh(self, ticket: ReadTicket) -> bool:+ """Whether what the ticket's read found may be stored: nothing it+ could have read was invalidated while it was in flight."""+ return self.enabled and ticket.generation == self._generation++ def _held_under(self, ticket: ReadTicket) -> int | None:+ """The listening connection to keep the read's entries under, if the+ same one listened through the whole read; None keeps them for the TTL+ only."""+ listening = self._listening()+ return listening if ticket.listening == listening else None++ def _get[V](+ self, entries: OrderedDict[str, _Entry[V]], tenant_id: str, key: str+ ) -> V | None:+ entry = entries.get(key)+ if entry is None or entry.tenant_id != tenant_id:+ return None+ if entry.expires_at <= time.monotonic() and (+ entry.held_under is None or entry.held_under != self._listening()+ ):+ del entries[key]+ return None+ entries.move_to_end(key)+ return entry.value+def _put[V](self,- entries: OrderedDict[tuple[str, str], tuple[float, V]],- key: tuple[str, str],+ entries: OrderedDict[str, _Entry[V]],+ bound: int,+ tenant_id: str,+ key: str,value: V,+ held_under: int | None,) -> None:- entries[key] = (time.monotonic() + self._ttl, value)+ entries[key] = _Entry(+ tenant_id=tenant_id,+ value=value,+ expires_at=time.monotonic() + self._ttl,+ held_under=held_under,+ )entries.move_to_end(key)- while len(entries) > self._max_entries:+ while len(entries) > bound:entries.popitem(last=False)---def _get[V](- entries: OrderedDict[tuple[str, str], tuple[float, V]], key: tuple[str, str]-) -> V | None:- entry = entries.get(key)- if entry is None:- return None- expires_at, value = entry- if expires_at <= time.monotonic():- del entries[key]- return None- entries.move_to_end(key)- return valuediff --git a/core/switch_core/config.py b/core/switch_core/config.pyindex dee21b287..e0ede2d17 100644--- a/core/switch_core/config.py+++ b/core/switch_core/config.py@@ -775,6 +775,9 @@ class SwitchConfig(BaseSettings):# management on, the same TTL and bound apply to the separate cache of a# controller access token's reads (its controller row, and the agent row# it acts as), which a revocation or binding change also drops at once.+ # That cache keeps an entry past the TTL while the signal connection it+ # was read under still listens, since every change to those rows is then+ # signalled to every process; 0 disables it too.agent_auth_cache_ttl_seconds: float = 5.0# How long a row in `agent_events` (one of an agent's own events) is kept# before the retention loop deletes it. An agent that reconnects within@@ -784,6 +787,10 @@ class SwitchConfig(BaseSettings):# Bound on the memo. One entry per distinct live token; the oldest is# evicted past this, so a flood of tokens cannot grow the process.agent_auth_cache_max_entries: int = 4096+ # Bound on the agent rows the controller-token cache keeps, the oldest+ # evicted past it. Every process sees requests for every controller's+ # agents, so it needs room for all the agents controllers run.+ controller_auth_cache_max_agents: int = Field(default=20_000, ge=1)# AGENT_MANAGEMENT_ENABLED, replaced by the `agent_management` feature flag.# Read only to refuse a deployment that still turns it on, which woulddiff --git a/core/switch_core/db/auth_rows_notify_ddl.py b/core/switch_core/db/auth_rows_notify_ddl.pynew file mode 100644index 000000000..ab7a35604--- /dev/null+++ b/core/switch_core/db/auth_rows_notify_ddl.py@@ -0,0 +1,112 @@+"""The triggers that announce changes to the rows a controller access token+is authorized with, as signals.++Every process keeps the rows a controller token's requests read+(`ControllerAuthCache`): the controller's, to refuse one revoked while its+token is unexpired, and the row of the agent it acts as. It keeps them for as+long as its signal connection listens and no signal says they changed. An+agent's row is written from several places (a rename, its addressing policy,+a template update, a deletion with its owner or workspace), and a controller's+revocation commits in one process and is announced by it after the commit, so+neither can depend on every writer remembering to say so. A trigger is the one+place none of them can skip, as `membership_notify_ddl.py` argues for+memberships.++`agent.changed` names the tenant and the agent, for any update that changes+the row and for a deletion. `controller.auth_changed` names the tenant and the+controller, when its owner, its credential or its revocation changes and when+it is deleted; the status, connection and assignment columns a controller+writes every few seconds send nothing. Neither carries a sender process, so+every process handles them, the writer's included. `NOTIFY` is delivered on+commit, so a process never hears of a change it could not then read.++The migration that installs these carries its own copy, frozen as it was+written; this copy is what `create_all` builds for the tests.+"""++from __future__ import annotations++from switch_core.signals import SIGNAL_CHANNEL++AGENT_CHANGED_SIGNAL = "agent.changed"+CONTROLLER_AUTH_CHANGED_SIGNAL = "controller.auth_changed"++AGENT_NOTIFY_FUNCTION_NAME = "switch_notify_agent_changed"+AGENT_NOTIFY_TRIGGER_NAME = "agents_notify_changed"+CONTROLLER_NOTIFY_FUNCTION_NAME = "switch_notify_controller_auth_changed"+CONTROLLER_NOTIFY_TRIGGER_NAME = "agent_controllers_notify_auth_changed"++CREATE_AGENT_NOTIFY_FUNCTION = f"""+CREATE OR REPLACE FUNCTION {AGENT_NOTIFY_FUNCTION_NAME}() RETURNS trigger AS $$+DECLARE+ changed agents;+BEGIN+ IF TG_OP = 'DELETE' THEN+ changed := OLD;+ ELSIF NEW IS NOT DISTINCT FROM OLD THEN+ RETURN NULL;+ ELSE+ changed := NEW;+ END IF;+ PERFORM pg_notify(+ '{SIGNAL_CHANNEL}',+ json_build_object(+ 'kind', '{AGENT_CHANGED_SIGNAL}',+ 'tenant_id', changed.tenant_id,+ 'agent_id', changed.id+ )::text+ );+ RETURN NULL;+END;+$$ LANGUAGE plpgsql+"""++CREATE_AGENT_NOTIFY_TRIGGER = f"""+CREATE TRIGGER {AGENT_NOTIFY_TRIGGER_NAME}+ AFTER UPDATE OR DELETE ON agents+ FOR EACH ROW EXECUTE FUNCTION {AGENT_NOTIFY_FUNCTION_NAME}()+"""++CREATE_CONTROLLER_NOTIFY_FUNCTION = f"""+CREATE OR REPLACE FUNCTION {CONTROLLER_NOTIFY_FUNCTION_NAME}() RETURNS trigger AS $$+DECLARE+ changed agent_controllers;+BEGIN+ IF TG_OP = 'DELETE' THEN+ changed := OLD;+ ELSIF NEW.owner_id IS NOT DISTINCT FROM OLD.owner_id+ AND NEW.revoked_at IS NOT DISTINCT FROM OLD.revoked_at+ AND NEW.api_key_id IS NOT DISTINCT FROM OLD.api_key_id THEN+ RETURN NULL;+ ELSE+ changed := NEW;+ END IF;+ PERFORM pg_notify(+ '{SIGNAL_CHANNEL}',+ json_build_object(+ 'kind', '{CONTROLLER_AUTH_CHANGED_SIGNAL}',+ 'tenant_id', changed.tenant_id,+ 'controller_id', changed.id+ )::text+ );+ RETURN NULL;+END;+$$ LANGUAGE plpgsql+"""++CREATE_CONTROLLER_NOTIFY_TRIGGER = f"""+CREATE TRIGGER {CONTROLLER_NOTIFY_TRIGGER_NAME}+ AFTER UPDATE OF owner_id, revoked_at, api_key_id OR DELETE ON agent_controllers+ FOR EACH ROW EXECUTE FUNCTION {CONTROLLER_NOTIFY_FUNCTION_NAME}()+"""++DROP_AGENT_NOTIFY_TRIGGER = (+ f"DROP TRIGGER IF EXISTS {AGENT_NOTIFY_TRIGGER_NAME} ON agents"+)+DROP_AGENT_NOTIFY_FUNCTION = f"DROP FUNCTION IF EXISTS {AGENT_NOTIFY_FUNCTION_NAME}()"+DROP_CONTROLLER_NOTIFY_TRIGGER = (+ f"DROP TRIGGER IF EXISTS {CONTROLLER_NOTIFY_TRIGGER_NAME} ON agent_controllers"+)+DROP_CONTROLLER_NOTIFY_FUNCTION = (+ f"DROP FUNCTION IF EXISTS {CONTROLLER_NOTIFY_FUNCTION_NAME}()"+)diff --git a/core/switch_core/db/models.py b/core/switch_core/db/models.pyindex 70dff1e2c..5cb372cb9 100644--- a/core/switch_core/db/models.py+++ b/core/switch_core/db/models.py@@ -32,6 +32,14 @@ from switch_core.db.agent_event_notify_ddl import (CREATE_AGENT_EVENT_NOTIFY_TRIGGER,DROP_AGENT_EVENT_NOTIFY_TRIGGER,)+from switch_core.db.auth_rows_notify_ddl import (+ CREATE_AGENT_NOTIFY_FUNCTION,+ CREATE_AGENT_NOTIFY_TRIGGER,+ CREATE_CONTROLLER_NOTIFY_FUNCTION,+ CREATE_CONTROLLER_NOTIFY_TRIGGER,+ DROP_AGENT_NOTIFY_TRIGGER,+ DROP_CONTROLLER_NOTIFY_TRIGGER,+)from switch_core.db.base import Basefrom switch_core.db.encrypted_json import EncryptedJSONBfrom switch_core.db.membership_notify_ddl import (@@ -3880,6 +3888,28 @@ class AgentController(TenantScoped, Base):)+# The triggers that signal every process when an agent's row changes, or a+# controller's owner, credential or revocation does, so each process drops+# what it kept of them for authorizing controller tokens. As for the triggers+# above, `create_all` builds them for the tests and real databases get them+# from a migration. The SQL lives in `db/auth_rows_notify_ddl.py`.+for _table, _ddls, _drop in (+ (+ Agent.__table__,+ (CREATE_AGENT_NOTIFY_FUNCTION, CREATE_AGENT_NOTIFY_TRIGGER),+ DROP_AGENT_NOTIFY_TRIGGER,+ ),+ (+ AgentController.__table__,+ (CREATE_CONTROLLER_NOTIFY_FUNCTION, CREATE_CONTROLLER_NOTIFY_TRIGGER),+ DROP_CONTROLLER_NOTIFY_TRIGGER,+ ),+):+ for _ddl in _ddls:+ event.listen(_table, "after_create", DDL(_ddl).execute_if(dialect="postgresql"))+ event.listen(_table, "before_drop", DDL(_drop).execute_if(dialect="postgresql"))++class AgentControllerEnrollmentCode(TenantScoped, Base):"""A one-time code a headless controller enrolls with.diff --git a/core/switch_core/db/stores/agent_store.py b/core/switch_core/db/stores/agent_store.pyindex d9b647faa..94522d19d 100644--- a/core/switch_core/db/stores/agent_store.py+++ b/core/switch_core/db/stores/agent_store.py@@ -31,6 +31,15 @@ class AgentStore:async def get(self, session: AsyncSession, agent_id: str) -> Agent | None:return await session.get(Agent, agent_id)+ async def get_many(+ self, session: AsyncSession, agent_ids: Collection[str]+ ) -> list[Agent]:+ """`get` for many agents in one query; an id with no agent is left out."""+ if not agent_ids:+ return []+ result = await session.execute(select(Agent).where(Agent.id.in_(agent_ids)))+ return list(result.scalars().all())+async def get_by_name(self, session: AsyncSession, name: str) -> Agent | None:"""Resolve an agent by name within the bound tenant.diff --git a/core/switch_core/management/auth.py b/core/switch_core/management/auth.pyindex bf960e5ec..27c925dd6 100644--- a/core/switch_core/management/auth.py+++ b/core/switch_core/management/auth.py@@ -92,7 +92,7 @@ class ManagementAuthenticator:if cached is not None and cached.owner_id == claims.owner_id:return cached- generation = self._auth_cache.generation+ ticket = self._auth_cache.ticket()async with tenant_session(self._session_factory, claims.tenant_id) as session:controller = await self._controllers.get(session, claims.tenant_id, claims.controller_id@@ -111,7 +111,7 @@ class ManagementAuthenticator:owner_id=controller.owner_id,tenant_id=claims.tenant_id,)- self._auth_cache.put_controller(principal, generation)+ self._auth_cache.put_controller(principal, ticket)return principalasync def tenant_of_secret(self, secret: str) -> str | None:diff --git a/core/switch_core/management/wiring.py b/core/switch_core/management/wiring.pyindex c9d93e497..da821585b 100644--- a/core/switch_core/management/wiring.py+++ b/core/switch_core/management/wiring.py@@ -155,6 +155,7 @@ def build_management(operations = AgentControllerOperationStore()processes = SwitchCoreProcessStore()presence.use_auth_cache(auth_cache)+ auth_cache.follow(signal_bus)signals = ControllerSignals(notifier=ControllerNotifier(),presence=presence,@@ -255,6 +256,7 @@ def create_management(auth_cache=ControllerAuthCache(ttl_seconds=config.agent_auth_cache_ttl_seconds,max_entries=config.agent_auth_cache_max_entries,+ max_agents=config.controller_auth_cache_max_agents,),clock=utc_now,signal_bus=signal_bus,diff --git a/core/switch_core/migrations/versions/bad5099ef32b_auth_rows_notify.py b/core/switch_core/migrations/versions/bad5099ef32b_auth_rows_notify.pynew file mode 100644index 000000000..ec2976e36--- /dev/null+++ b/core/switch_core/migrations/versions/bad5099ef32b_auth_rows_notify.py@@ -0,0 +1,107 @@+"""auth_rows_notify: signal every switch-core process when an agent's row or a+controller's authority changes++Every process keeps the controller and agent rows a controller access token's+requests are authorized with. These triggers send `agent.changed` on the+`switch_signal` channel for every update that changes an agent's row and for+its deletion, and `controller.auth_changed` when a controller's owner,+credential or revocation changes and when it is deleted, so every process+drops what it kept of them.++Revision ID: bad5099ef32b+Revises: b7e1d4c9a2f6+Create Date: 2026-11-01 00:00:00.000000++"""++from collections.abc import Sequence++from alembic import op++revision: str = "bad5099ef32b"+down_revision: str | None = "b7e1d4c9a2f6"+branch_labels: str | Sequence[str] | None = None+depends_on: str | Sequence[str] | None = None++# Copies of the DDL in `db/auth_rows_notify_ddl.py`, written out here so this+# migration keeps creating the same objects if that module changes.+CREATE_AGENT_NOTIFY_FUNCTION = """+CREATE OR REPLACE FUNCTION switch_notify_agent_changed() RETURNS trigger AS $$+DECLARE+ changed agents;+BEGIN+ IF TG_OP = 'DELETE' THEN+ changed := OLD;+ ELSIF NEW IS NOT DISTINCT FROM OLD THEN+ RETURN NULL;+ ELSE+ changed := NEW;+ END IF;+ PERFORM pg_notify(+ 'switch_signal',+ json_build_object(+ 'kind', 'agent.changed',+ 'tenant_id', changed.tenant_id,+ 'agent_id', changed.id+ )::text+ );+ RETURN NULL;+END;+$$ LANGUAGE plpgsql+"""++CREATE_AGENT_NOTIFY_TRIGGER = """+CREATE TRIGGER agents_notify_changed+ AFTER UPDATE OR DELETE ON agents+ FOR EACH ROW EXECUTE FUNCTION switch_notify_agent_changed()+"""++CREATE_CONTROLLER_NOTIFY_FUNCTION = """+CREATE OR REPLACE FUNCTION switch_notify_controller_auth_changed() RETURNS trigger AS $$+DECLARE+ changed agent_controllers;+BEGIN+ IF TG_OP = 'DELETE' THEN+ changed := OLD;+ ELSIF NEW.owner_id IS NOT DISTINCT FROM OLD.owner_id+ AND NEW.revoked_at IS NOT DISTINCT FROM OLD.revoked_at+ AND NEW.api_key_id IS NOT DISTINCT FROM OLD.api_key_id THEN+ RETURN NULL;+ ELSE+ changed := NEW;+ END IF;+ PERFORM pg_notify(+ 'switch_signal',+ json_build_object(+ 'kind', 'controller.auth_changed',+ 'tenant_id', changed.tenant_id,+ 'controller_id', changed.id+ )::text+ );+ RETURN NULL;+END;+$$ LANGUAGE plpgsql+"""++CREATE_CONTROLLER_NOTIFY_TRIGGER = """+CREATE TRIGGER agent_controllers_notify_auth_changed+ AFTER UPDATE OF owner_id, revoked_at, api_key_id OR DELETE ON agent_controllers+ FOR EACH ROW EXECUTE FUNCTION switch_notify_controller_auth_changed()+"""+++def upgrade() -> None:+ op.execute(CREATE_AGENT_NOTIFY_FUNCTION)+ op.execute(CREATE_AGENT_NOTIFY_TRIGGER)+ op.execute(CREATE_CONTROLLER_NOTIFY_FUNCTION)+ op.execute(CREATE_CONTROLLER_NOTIFY_TRIGGER)+++def downgrade() -> None:+ op.execute(+ "DROP TRIGGER IF EXISTS agent_controllers_notify_auth_changed "+ "ON agent_controllers"+ )+ op.execute("DROP FUNCTION IF EXISTS switch_notify_controller_auth_changed()")+ op.execute("DROP TRIGGER IF EXISTS agents_notify_changed ON agents")+ op.execute("DROP FUNCTION IF EXISTS switch_notify_agent_changed()")diff --git a/core/switch_core/signals.py b/core/switch_core/signals.pyindex 5eda31b80..ea827c7d0 100644--- a/core/switch_core/signals.py+++ b/core/switch_core/signals.py@@ -172,6 +172,11 @@ class SignalBus:self._tasks: list[asyncio.Task[None]] = []self._running = Falseself.connected = asyncio.Event()+ # How many listening connections have been established. What a process+ # keeps current by signals is current only while this stays the same+ # and `connected` stays set: a signal sent between two connections+ # reached neither.+ self.listens = 0# ββ Registration ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ@@ -346,6 +351,7 @@ class SignalBus:)await driver.add_listener(SIGNAL_CHANNEL, self._on_notify)try:+ self.listens += 1self.connected.set()await self._resync_all()await self._hold_open(driver)diff --git a/core/tests/switch_core/bridges/agent/test_controller_auth_cache.py b/core/tests/switch_core/bridges/agent/test_controller_auth_cache.pyindex 186ab43b3..cad8f18f8 100644--- a/core/tests/switch_core/bridges/agent/test_controller_auth_cache.py+++ b/core/tests/switch_core/bridges/agent/test_controller_auth_cache.py@@ -8,7 +8,10 @@ ones that need a hand on the clock or on the ordering.from __future__ import annotations+import asyncio+from collections.abc import Callablefrom types import SimpleNamespace+from typing import Anyimport pytest@@ -21,6 +24,10 @@ from switch_core.bridges.agent.protocol.controller_presence import (ControllerPresence,)from switch_core.bridges.agent.protocol.liveness import HEARTBEAT_TTL_SECONDS+from switch_core.db.auth_rows_notify_ddl import (+ AGENT_CHANGED_SIGNAL,+ CONTROLLER_AUTH_CHANGED_SIGNAL,+)from switch_core.db.models import Agent@@ -51,8 +58,8 @@ def _agent(agent_id: str = "a1") -> Agent:def test_entries_expire_after_the_ttl(clock: _Clock) -> None:cache = ControllerAuthCache(ttl_seconds=5, max_entries=8)- cache.put_controller(_principal(), cache.generation)- cache.put_agent("t1", _agent(), cache.generation)+ cache.put_controller(_principal(), cache.ticket())+ cache.put_agents("t1", [_agent()], cache.ticket())clock.now += 4.9assert cache.controller("t1", "c1") == _principal()@@ -64,8 +71,8 @@ def test_entries_expire_after_the_ttl(clock: _Clock) -> None:def test_an_entry_is_keyed_by_its_tenant(clock: _Clock) -> None:cache = ControllerAuthCache(ttl_seconds=5, max_entries=8)- cache.put_controller(_principal(), cache.generation)- cache.put_agent("t1", _agent(), cache.generation)+ cache.put_controller(_principal(), cache.ticket())+ cache.put_agents("t1", [_agent()], cache.ticket())assert cache.controller("t2", "c1") is Noneassert cache.agent("t2", "a1") is None@@ -73,23 +80,23 @@ def test_an_entry_is_keyed_by_its_tenant(clock: _Clock) -> None:def test_a_read_that_straddled_an_invalidation_is_not_stored(clock: _Clock) -> None:cache = ControllerAuthCache(ttl_seconds=5, max_entries=8)- before_revoke = cache.generation+ before_revoke = cache.ticket()cache.invalidate_controller("c1")cache.put_controller(_principal(), before_revoke)assert cache.controller("t1", "c1") is None- before_unbind = cache.generation+ before_unbind = cache.ticket()cache.invalidate_agent("a1")- cache.put_agent("t1", _agent(), before_unbind)+ cache.put_agents("t1", [_agent()], before_unbind)assert cache.agent("t1", "a1") is Nonedef test_invalidation_drops_only_what_it_names(clock: _Clock) -> None:cache = ControllerAuthCache(ttl_seconds=5, max_entries=8)for principal in (_principal("c1"), _principal("c2"), _principal("c1", "t2")):- cache.put_controller(principal, cache.generation)+ cache.put_controller(principal, cache.ticket())for agent_id in ("a1", "a2"):- cache.put_agent("t1", _agent(agent_id), cache.generation)+ cache.put_agents("t1", [_agent(agent_id)], cache.ticket())cache.invalidate_controller("c1")cache.invalidate_agent("a1")@@ -104,20 +111,156 @@ def test_invalidation_drops_only_what_it_names(clock: _Clock) -> None:def test_it_holds_no_more_than_its_bound(clock: _Clock) -> None:cache = ControllerAuthCache(ttl_seconds=5, max_entries=2)for agent_id in ("a1", "a2", "a3"):- cache.put_agent("t1", _agent(agent_id), cache.generation)+ cache.put_agents("t1", [_agent(agent_id)], cache.ticket())assert cache.agent("t1", "a1") is Noneassert cache.agent("t1", "a3") is not None+def test_agents_have_a_bound_of_their_own(clock: _Clock) -> None:+ cache = ControllerAuthCache(ttl_seconds=5, max_entries=1, max_agents=3)+ cache.put_agents("t1", [_agent(a) for a in ("a1", "a2", "a3")], cache.ticket())+ for principal in (_principal("c1"), _principal("c2")):+ cache.put_controller(principal, cache.ticket())++ assert [cache.agent("t1", a) is not None for a in ("a1", "a2", "a3")] == [+ True,+ True,+ True,+ ]+ assert cache.controller("t1", "c1") is None+ assert cache.controller("t1", "c2") is not None++def test_a_ttl_of_zero_stores_nothing(clock: _Clock) -> None:cache = ControllerAuthCache(ttl_seconds=0, max_entries=8)- cache.put_controller(_principal(), cache.generation)- cache.put_agent("t1", _agent(), cache.generation)+ cache.put_controller(_principal(), cache.ticket())+ cache.put_agents("t1", [_agent()], cache.ticket())assert not cache.enabledassert cache.controller("t1", "c1") is Noneassert cache.agent("t1", "a1") is None+class _Bus:+ """Just enough of `SignalBus`: whether it listens, on which connection,+ and the handlers registered with it."""++ def __init__(self) -> None:+ self.connected = asyncio.Event()+ self.listens = 0+ self.handlers: dict[str, Callable[[dict[str, Any]], None]] = {}++ def handle(self, kind: str, handler: Callable[[dict[str, Any]], None]) -> None:+ self.handlers[kind] = handler++ def listen(self) -> None:+ self.listens += 1+ self.connected.set()++ def drop(self) -> None:+ self.connected.clear()+++def _following(bus: _Bus) -> ControllerAuthCache:+ cache = ControllerAuthCache(ttl_seconds=5, max_entries=8)+ cache.follow(bus) # type: ignore[arg-type]+ return cache+++def _warm(cache: ControllerAuthCache) -> None:+ cache.put_controller(_principal(), cache.ticket())+ cache.put_agents("t1", [_agent()], cache.ticket())+++class TestWhileSignalsListen:+ def test_an_entry_outlives_the_ttl(self, clock: _Clock) -> None:+ bus = _Bus()+ bus.listen()+ cache = _following(bus)+ _warm(cache)++ clock.now += 3600++ assert cache.controller("t1", "c1") == _principal()+ assert cache.agent("t1", "a1") is not None++ def test_the_signals_drop_what_they_name(self, clock: _Clock) -> None:+ bus = _Bus()+ bus.listen()+ cache = _following(bus)+ _warm(cache)+ cache.put_agents("t1", [_agent("a2")], cache.ticket())++ bus.handlers[CONTROLLER_AUTH_CHANGED_SIGNAL](+ {"kind": CONTROLLER_AUTH_CHANGED_SIGNAL, "controller_id": "c1"}+ )+ bus.handlers[AGENT_CHANGED_SIGNAL](+ {"kind": AGENT_CHANGED_SIGNAL, "agent_id": "a1"}+ )++ assert cache.controller("t1", "c1") is None+ assert cache.agent("t1", "a1") is None+ assert cache.agent("t1", "a2") is not None++ def test_an_entry_read_while_the_bus_was_down_lasts_the_ttl(+ self, clock: _Clock+ ) -> None:+ bus = _Bus()+ cache = _following(bus)+ _warm(cache)+ bus.listen()++ clock.now += 5.1++ assert cache.controller("t1", "c1") is None+ assert cache.agent("t1", "a1") is None++ def test_losing_the_connection_ends_the_hold(self, clock: _Clock) -> None:+ """A signal sent while the bus does not listen reaches nobody, so what+ was kept on the strength of signals goes back to lasting the TTL."""+ bus = _Bus()+ bus.listen()+ cache = _following(bus)+ _warm(cache)++ bus.drop()+ clock.now += 5.1+ assert cache.agent("t1", "a1") is None++ _warm(cache)+ bus.listen()+ clock.now += 5.1+ assert cache.controller("t1", "c1") is None++ def test_a_new_connection_does_not_revive_the_old_one_s_entries(+ self, clock: _Clock+ ) -> None:+ bus = _Bus()+ bus.listen()+ cache = _following(bus)+ _warm(cache)++ bus.drop()+ bus.listen()+ clock.now += 5.1++ assert cache.controller("t1", "c1") is None+ assert cache.agent("t1", "a1") is None++ def test_a_read_across_a_reconnect_lasts_the_ttl(self, clock: _Clock) -> None:+ bus = _Bus()+ bus.listen()+ cache = _following(bus)+ ticket = cache.ticket()++ bus.drop()+ bus.listen()+ cache.put_agents("t1", [_agent()], ticket)++ assert cache.agent("t1", "a1") is not None+ clock.now += 5.1+ assert cache.agent("t1", "a1") is None++def test_the_ttl_must_stay_below_the_heartbeat_ttl() -> None:with pytest.raises(ValueError, match="heartbeat"):ControllerAuthCache(ttl_seconds=HEARTBEAT_TTL_SECONDS, max_entries=8)@@ -125,6 +268,8 @@ def test_the_ttl_must_stay_below_the_heartbeat_ttl() -> None:ControllerAuthCache(ttl_seconds=-1, max_entries=8)with pytest.raises(ValueError, match="max_entries"):ControllerAuthCache(ttl_seconds=1, max_entries=0)+ with pytest.raises(ValueError, match="max_agents"):+ ControllerAuthCache(ttl_seconds=1, max_entries=1, max_agents=0)def test_presence_keeps_it_in_step_with_bindings_and_revocations(@@ -142,8 +287,8 @@ def test_presence_keeps_it_in_step_with_bindings_and_revocations()def warm() -> None:- cache.put_controller(_principal(), cache.generation)- cache.put_agent("t1", _agent(), cache.generation)+ cache.put_controller(_principal(), cache.ticket())+ cache.put_agents("t1", [_agent()], cache.ticket())warm()presence.bind(binding)diff --git a/core/tests/switch_core/management/test_controller_auth_cache.py b/core/tests/switch_core/management/test_controller_auth_cache.pyindex 7ecca468a..f24feb409 100644--- a/core/tests/switch_core/management/test_controller_auth_cache.py+++ b/core/tests/switch_core/management/test_controller_auth_cache.py@@ -7,19 +7,35 @@ database" means no statement reached Postgres.from __future__ import annotations+import asyncioimport json-from collections.abc import Iterator-from contextlib import contextmanager+import time+from collections.abc import AsyncIterator, Callable, Iterator+from contextlib import asynccontextmanager, contextmanager+from types import SimpleNamespacefrom typing import Anyimport pytest-from sqlalchemy import event-from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker+from sqlalchemy import delete, event, func, update+from sqlalchemy.ext.asyncio import (+ AsyncEngine,+ AsyncSession,+ async_sessionmaker,+ create_async_engine,+)+from sqlalchemy.pool import NullPool+from switch_core.bridges.agent import controller_auth_cachefrom switch_core.bridges.agent.auth import BearerAuthMiddleware-from switch_core.db.models import TENANT_ZERO_ID+from switch_core.db.auth_rows_notify_ddl import (+ AGENT_CHANGED_SIGNAL,+ CONTROLLER_AUTH_CHANGED_SIGNAL,+)+from switch_core.db.models import TENANT_ZERO_ID, Agent, AgentController+from switch_core.db.session_scope import tenant_sessionfrom switch_core.db.stores.agent_store import AgentStorefrom switch_core.db.stores.api_key_store import ApiKeyStore+from switch_core.signals import SIGNAL_CHANNELfrom tests.switch_core.management.harness import (EnrolledController,Harness,@@ -256,3 +272,205 @@ async def test_a_ttl_of_zero_reads_the_database_every_time(assert reads.of("agent_controllers") == 3assert reads.of("agents") == 3assert not harness.controller_auth_cache.enabled+++async def test_a_controller_s_agents_are_read_together(harness: Harness) -> None:+ """A miss reads every agent bound to the controller, so a request for+ another of them reads none."""+ controller, first = await placed(harness)+ async with harness.client() as client:+ second = await place_agent(client, controller, name="writer")+ mw = middleware(harness)+ harness.controller_auth_cache.clear()++ with counting(harness.session_factory) as reads:+ assert await act_as(mw, controller, first) == (200, None)+ assert await act_as(mw, controller, second) == (200, None)++ assert reads.of("agents") == 1+++# ββ While the signal bus listens ββββββββββββββββββββββββββββββββββββββββββββββ+++async def listening(+ postgres_url: str, session_factory: async_sessionmaker[AsyncSession]+) -> AsyncIterator[Harness]:+ harness = build_harness(+ session_factory,+ signal_engine=lambda: create_async_engine(postgres_url, poolclass=NullPool),+ )+ await harness.management.lease.renew()+ await harness.bus.start()+ await asyncio.wait_for(harness.bus.connected.wait(), 10)+ try:+ yield harness+ finally:+ await harness.bus.stop()+++async def _until(condition: Callable[[], bool]) -> None:+ for _ in range(500):+ if condition():+ return+ await asyncio.sleep(0.01)+ raise AssertionError("never happened")+++async def test_while_the_bus_listens_a_hit_outlives_the_ttl(+ listening: Harness, monkeypatch: pytest.MonkeyPatch+) -> None:+ controller, agent_id = await placed(listening)+ mw = middleware(listening)+ assert await act_as(mw, controller, agent_id) == (200, None)+ an_hour_on = time.monotonic() + 3600+ monkeypatch.setattr(+ controller_auth_cache, "time", SimpleNamespace(monotonic=lambda: an_hour_on)+ )++ with counting(listening.session_factory) as reads:+ assert await act_as(mw, controller, agent_id) == (200, None)++ assert reads.of("agent_controllers") == 0+ assert reads.of("agents") == 0+++async def test_an_agent_changed_in_the_database_is_dropped(listening: Harness) -> None:+ controller, agent_id = await placed(listening)+ mw = middleware(listening)+ assert await act_as(mw, controller, agent_id) == (200, None)+ cache = listening.controller_auth_cache+ assert cache.agent(TENANT_ZERO_ID, agent_id) is not None++ async with tenant_session(listening.session_factory, TENANT_ZERO_ID) as session:+ await session.execute(+ update(Agent).where(Agent.id == agent_id).values(description="changed")+ )+ await session.commit()++ await _until(lambda: cache.agent(TENANT_ZERO_ID, agent_id) is None)+++async def test_a_controller_revoked_where_nothing_announces_it_is_refused(+ listening: Harness,+) -> None:+ """Revoked by a path that tells no process, as an operator in psql would:+ the trigger tells every one."""+ controller, agent_id = await placed(listening)+ mw = middleware(listening)+ assert await act_as(mw, controller, agent_id) == (200, None)+ cache = listening.controller_auth_cache++ async with tenant_session(listening.session_factory, TENANT_ZERO_ID) as session:+ await session.execute(+ update(AgentController)+ .where(AgentController.id == controller.controller_id)+ .values(revoked_at=func.now(), api_key_id=None)+ )+ await session.commit()++ await _until(+ lambda: cache.controller(TENANT_ZERO_ID, controller.controller_id) is None+ )+ assert await act_as(mw, controller, agent_id) == (401, "controller_revoked")+++# ββ The triggers ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ++_SENTINEL = "test.sentinel"+++@asynccontextmanager+async def heard(postgres_url: str) -> AsyncIterator[list[dict[str, Any]]]:+ """The signals sent while the block runs. A sentinel is sent after the+ block, and signals arrive in commit order, so once it is heard every+ signal the block's commits sent has been too."""+ engine = create_async_engine(postgres_url, poolclass=NullPool)+ signals: list[dict[str, Any]] = []+ done = asyncio.Event()++ def on_notify(_conn: Any, _pid: int, _channel: str, payload: str) -> None:+ signal = json.loads(payload)+ if signal.get("kind") == _SENTINEL:+ done.set()+ else:+ signals.append(signal)++ try:+ async with engine.connect() as connection:+ raw = await connection.get_raw_connection()+ driver = raw.driver_connection+ assert driver is not None+ await driver.add_listener(SIGNAL_CHANNEL, on_notify)+ yield signals+ await driver.execute(+ f"NOTIFY {SIGNAL_CHANNEL}, '{json.dumps({'kind': _SENTINEL})}'"+ )+ await asyncio.wait_for(done.wait(), 10)+ finally:+ await engine.dispose()+++async def test_an_agent_s_row_changing_or_going_is_signalled(+ harness: Harness, postgres_url: str+) -> None:+ _, agent_id = await placed(harness)++ async with heard(postgres_url) as signals:+ async with tenant_session(harness.session_factory, TENANT_ZERO_ID) as session:+ # Written but unchanged: nothing to drop.+ await session.execute(+ update(Agent)+ .where(Agent.id == agent_id)+ .values(description=Agent.description)+ )+ await session.execute(+ update(Agent).where(Agent.id == agent_id).values(description="new")+ )+ await session.commit()++ assert signals == [+ {+ "kind": AGENT_CHANGED_SIGNAL,+ "tenant_id": TENANT_ZERO_ID,+ "agent_id": agent_id,+ }+ ]+++async def test_only_a_controller_s_authority_changing_is_signalled(+ harness: Harness, postgres_url: str+) -> None:+ owner = await add_member(harness.session_factory, "ada")+ async with harness.client() as client:+ controller = await enroll_console(harness, client, owner)+ row = AgentController.id == controller.controller_id++ async with heard(postgres_url) as signals:+ async with tenant_session(harness.session_factory, TENANT_ZERO_ID) as session:+ # What a controller writes as it runs.+ await session.execute(+ update(AgentController).where(row).values(updated_at=func.now())+ )+ await session.execute(+ update(AgentController)+ .where(row)+ .values(revoked_at=func.now(), api_key_id=None)+ )+ await session.commit()+ async with tenant_session(harness.session_factory, TENANT_ZERO_ID) as session:+ await session.execute(delete(AgentController).where(row))+ await session.commit()++ assert (+ signals+ == [+ {+ "kind": CONTROLLER_AUTH_CHANGED_SIGNAL,+ "tenant_id": TENANT_ZERO_ID,+ "controller_id": controller.controller_id,+ }+ ]+ * 2+ )