Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Keep a controller token's rows until a signal says they changed (CHOO-3667)

Abel Dantas0a06092d1612
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 8b9791612..6bba22851 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -224,6 +224,13 @@ version of their own to them without also giving them a release of their own.
group whose subgroup would move up next to a namesake is refused too, and
the dashboard now shows why. Names can no longer contain `/` or be blank.
Existing duplicates are left as they are and can be reached by id.
+- **A controller's requests no longer read its row and its agent's row each
+ time.** They were kept for 5 seconds, and a controller's requests, spread
+ over several processes, rarely came back to one within that. They are now
+ kept until a signal says they changed, which triggers on `agents` and
+ `agent_controllers` send, and a miss reads all of the controller's agents
+ at once. `CONTROLLER_AUTH_CACHE_MAX_AGENTS` (20,000) bounds the agent rows
+ kept.
#### Fixed
- **On Slack, the πŸ‘€ and the activity card no longer arrive after the agent's
diff --git a/core/switch_core/bridges/agent/auth.py b/core/switch_core/bridges/agent/auth.py
index 7c8d4e514..92428eb5f 100644
--- a/core/switch_core/bridges/agent/auth.py
+++ b/core/switch_core/bridges/agent/auth.py
@@ -438,18 +438,9 @@ class BearerAuthMiddleware:
)(scope, receive, send)
return
- auth_cache = controller_auth.auth_cache
- agent = auth_cache.agent(principal.tenant_id, agent_id)
+ agent = controller_auth.auth_cache.agent(principal.tenant_id, agent_id)
if agent is None:
- generation = auth_cache.generation
- async with tenant_session(
- self._session_factory, principal.tenant_id
- ) as session:
- agent = await self._agent_store.get(session, agent_id)
- if agent is not None:
- session.expunge(agent)
- if agent is not None:
- auth_cache.put_agent(principal.tenant_id, agent, generation)
+ agent = await self._read_agents_of(controller_auth, principal, agent_id)
if agent is None:
await _controller_refusal(
NOT_ASSIGNED, f"Agent {agent_id} does not exist.", 403
@@ -465,6 +456,30 @@ class BearerAuthMiddleware:
):
await self.app(scope, receive, send)
+ async def _read_agents_of(
+ self,
+ controller_auth: ControllerAuthenticator,
+ principal: ControllerPrincipal,
+ agent_id: str,
+ ) -> Agent | None:
+ """The row of the agent a controller acts as, read together with the
+ rows of every other agent bound to the controller, which the cache
+ keeps. A controller relays all of its agents' requests, and they are
+ spread over every process, so one read per agent would miss on most
+ of them."""
+ auth_cache = controller_auth.auth_cache
+ wanted = controller_auth.presence.agents_of(principal.controller_id)
+ wanted.add(agent_id)
+ ticket = auth_cache.ticket()
+ async with tenant_session(
+ self._session_factory, principal.tenant_id
+ ) as session:
+ agents = await self._agent_store.get_many(session, wanted)
+ for found in agents:
+ session.expunge(found)
+ auth_cache.put_agents(principal.tenant_id, agents, ticket)
+ return next((found for found in agents if found.id == agent_id), None)
+
async def _resolve_api_key(self, token: str) -> tuple[ApiKey | None, Agent | None]:
"""Look up the token in api_keys once; return (api_key_row, agent_or_None).
diff --git a/core/switch_core/bridges/agent/controller_auth_cache.py b/core/switch_core/bridges/agent/controller_auth_cache.py
index 00960ce59..7a74b35b0 100644
--- a/core/switch_core/bridges/agent/controller_auth_cache.py
+++ b/core/switch_core/bridges/agent/controller_auth_cache.py
@@ -2,17 +2,47 @@ from __future__ import annotations
import time
from collections import OrderedDict
-from typing import TYPE_CHECKING
+from collections.abc import Iterable
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Any
from switch_core.bridges.agent.protocol.liveness import HEARTBEAT_TTL_SECONDS
+from switch_core.db.auth_rows_notify_ddl import (
+ AGENT_CHANGED_SIGNAL,
+ CONTROLLER_AUTH_CHANGED_SIGNAL,
+)
from switch_core.db.models import Agent
if TYPE_CHECKING:
from switch_core.bridges.agent.auth import ControllerPrincipal
+ from switch_core.signals import SignalBus
+
+
+@dataclass(frozen=True)
+class ReadTicket:
+ """Taken before a read and handed back with what it read, so the cache
+ can tell whether anything it was told while the read was in flight makes
+ the read stale."""
+
+ generation: int
+ # The signal bus's listening connection when the read began, or None if
+ # it was not listening then.
+ listening: int | None
+
+
+@dataclass
+class _Entry[V]:
+ tenant_id: str
+ value: V
+ expires_at: float
+ # The listening connection the entry was read under, while it is the one
+ # listening, keeps the entry past `expires_at`. None for a read made while
+ # the bus did not listen.
+ held_under: int | None
class ControllerAuthCache:
- """A short-lived memo of the two reads a controller access token costs.
+ """A memo of the two reads a controller access token costs.
A controller token is a signed JWT, so verifying it needs no database. But
every request it authenticates also re-reads its controller row, to refuse
@@ -26,6 +56,21 @@ class ControllerAuthCache:
the binding Core holds in memory (``ControllerPresence``), which is checked
on every request and never cached here.
+ How long an entry is kept:
+
+ - **Until a signal says it changed, while one could.** A cache that
+ follows the signal bus (``follow``) keeps an entry read while the bus
+ listened for as long as that same connection listens. Triggers on
+ ``agents`` and ``agent_controllers`` send ``agent.changed`` and
+ ``controller.auth_changed`` (``db/auth_rows_notify_ddl.py``), and every
+ process drops the entry it names. A controller's requests are spread
+ over every process, so each process sees one controller or agent only
+ every so often, and an entry kept for seconds was gone by the next
+ request for it.
+ - **Otherwise for ``ttl_seconds``.** An entry read while the bus did not
+ listen, or kept past the connection it was read under, lasts the TTL: a
+ signal sent between two connections reached neither.
+
What it deliberately does not do:
- **Cache a refusal.** Only a controller that authenticated and an agent
@@ -33,15 +78,17 @@ class ControllerAuthCache:
always reaches the database.
- **Outlive a revocation or a binding change.** ``ControllerPresence``
calls ``invalidate_controller`` when a controller is revoked and
- ``invalidate_agent`` when an agent is bound, moved, unbound or deleted;
- expiry is the backstop, not the mechanism. A read that was in flight
+ ``invalidate_agent`` when an agent is bound, moved, unbound or deleted,
+ and the signals above reach every process. A read that was in flight
when an entry was invalidated does not store what it read.
A ``ttl_seconds`` of 0 disables it. The TTL is refused at or above the
heartbeat TTL, for the reason :class:`ApiKeyCache` gives.
"""
- def __init__(self, *, ttl_seconds: float, max_entries: int) -> None:
+ def __init__(
+ self, *, ttl_seconds: float, max_entries: int, max_agents: int | None = None
+ ) -> None:
if ttl_seconds < 0:
raise ValueError(f"ttl_seconds must not be negative, got {ttl_seconds!r}")
if ttl_seconds >= HEARTBEAT_TTL_SECONDS:
@@ -51,81 +98,134 @@ class ControllerAuthCache:
)
if max_entries < 1:
raise ValueError(f"max_entries must be at least 1, got {max_entries!r}")
+ if max_agents is not None and max_agents < 1:
+ raise ValueError(f"max_agents must be at least 1, got {max_agents!r}")
self._ttl = ttl_seconds
- self._max_entries = max_entries
- self._controllers: OrderedDict[
- tuple[str, str], tuple[float, ControllerPrincipal]
- ] = OrderedDict()
- self._agents: OrderedDict[tuple[str, str], tuple[float, Agent]] = OrderedDict()
- # Bumped by every invalidation. A reader takes it before going to the
- # database and hands it back with what it read; a read that straddled
- # an invalidation is not stored.
+ self._max_controllers = max_entries
+ self._max_agents = max_agents if max_agents is not None else max_entries
+ # Keyed by id alone: controller and agent ids are unique across
+ # tenants, so a signal naming one finds its entry without a scan.
+ self._controllers: OrderedDict[str, _Entry[ControllerPrincipal]] = OrderedDict()
+ self._agents: OrderedDict[str, _Entry[Agent]] = OrderedDict()
+ # Bumped by every invalidation. A reader takes a ticket before going
+ # to the database and hands it back with what it read; a read that
+ # straddled an invalidation is not stored.
self._generation = 0
+ self._bus: SignalBus | None = None
+
+ def follow(self, bus: SignalBus) -> None:
+ """Keep entries while `bus` listens, and drop the ones its signals
+ name."""
+ self._bus = bus
+ bus.handle(AGENT_CHANGED_SIGNAL, self._on_agent_changed)
+ bus.handle(CONTROLLER_AUTH_CHANGED_SIGNAL, self._on_controller_changed)
@property
def enabled(self) -> bool:
return self._ttl > 0
- @property
- def generation(self) -> int:
- return self._generation
+ def ticket(self) -> ReadTicket:
+ return ReadTicket(generation=self._generation, listening=self._listening())
def controller(
self, tenant_id: str, controller_id: str
) -> ControllerPrincipal | None:
- return _get(self._controllers, (tenant_id, controller_id))
+ return self._get(self._controllers, tenant_id, controller_id)
- def put_controller(self, principal: ControllerPrincipal, generation: int) -> None:
- if not self.enabled or generation != self._generation:
+ def put_controller(
+ self, principal: ControllerPrincipal, ticket: ReadTicket
+ ) -> None:
+ if not self._fresh(ticket):
return
self._put(
- self._controllers, (principal.tenant_id, principal.controller_id), principal
+ self._controllers,
+ self._max_controllers,
+ principal.tenant_id,
+ principal.controller_id,
+ principal,
+ self._held_under(ticket),
)
def agent(self, tenant_id: str, agent_id: str) -> Agent | None:
- return _get(self._agents, (tenant_id, agent_id))
+ return self._get(self._agents, tenant_id, agent_id)
- def put_agent(self, tenant_id: str, agent: Agent, generation: int) -> None:
- if not self.enabled or generation != self._generation:
+ def put_agents(
+ self, tenant_id: str, agents: Iterable[Agent], ticket: ReadTicket
+ ) -> None:
+ if not self._fresh(ticket):
return
- self._put(self._agents, (tenant_id, agent.id), agent)
+ held_under = self._held_under(ticket)
+ for agent in agents:
+ self._put(
+ self._agents, self._max_agents, tenant_id, agent.id, agent, held_under
+ )
def invalidate_controller(self, controller_id: str) -> None:
self._generation += 1
- for key in [k for k in self._controllers if k[1] == controller_id]:
- del self._controllers[key]
+ self._controllers.pop(controller_id, None)
def invalidate_agent(self, agent_id: str) -> None:
self._generation += 1
- for key in [k for k in self._agents if k[1] == agent_id]:
- del self._agents[key]
+ self._agents.pop(agent_id, None)
def clear(self) -> None:
self._generation += 1
self._controllers.clear()
self._agents.clear()
+ def _on_agent_changed(self, signal: dict[str, Any]) -> None:
+ self.invalidate_agent(signal["agent_id"])
+
+ def _on_controller_changed(self, signal: dict[str, Any]) -> None:
+ self.invalidate_controller(signal["controller_id"])
+
+ def _listening(self) -> int | None:
+ bus = self._bus
+ if bus is None or not bus.connected.is_set():
+ return None
+ return bus.listens
+
+ def _fresh(self, ticket: ReadTicket) -> bool:
+ """Whether what the ticket's read found may be stored: nothing it
+ could have read was invalidated while it was in flight."""
+ return self.enabled and ticket.generation == self._generation
+
+ def _held_under(self, ticket: ReadTicket) -> int | None:
+ """The listening connection to keep the read's entries under, if the
+ same one listened through the whole read; None keeps them for the TTL
+ only."""
+ listening = self._listening()
+ return listening if ticket.listening == listening else None
+
+ def _get[V](
+ self, entries: OrderedDict[str, _Entry[V]], tenant_id: str, key: str
+ ) -> V | None:
+ entry = entries.get(key)
+ if entry is None or entry.tenant_id != tenant_id:
+ return None
+ if entry.expires_at <= time.monotonic() and (
+ entry.held_under is None or entry.held_under != self._listening()
+ ):
+ del entries[key]
+ return None
+ entries.move_to_end(key)
+ return entry.value
+
def _put[V](
self,
- entries: OrderedDict[tuple[str, str], tuple[float, V]],
- key: tuple[str, str],
+ entries: OrderedDict[str, _Entry[V]],
+ bound: int,
+ tenant_id: str,
+ key: str,
value: V,
+ held_under: int | None,
) -> None:
- entries[key] = (time.monotonic() + self._ttl, value)
+ entries[key] = _Entry(
+ tenant_id=tenant_id,
+ value=value,
+ expires_at=time.monotonic() + self._ttl,
+ held_under=held_under,
+ )
entries.move_to_end(key)
- while len(entries) > self._max_entries:
+ while len(entries) > bound:
entries.popitem(last=False)
-
-
-def _get[V](
- entries: OrderedDict[tuple[str, str], tuple[float, V]], key: tuple[str, str]
-) -> V | None:
- entry = entries.get(key)
- if entry is None:
- return None
- expires_at, value = entry
- if expires_at <= time.monotonic():
- del entries[key]
- return None
- entries.move_to_end(key)
- return value
diff --git a/core/switch_core/config.py b/core/switch_core/config.py
index dee21b287..e0ede2d17 100644
--- a/core/switch_core/config.py
+++ b/core/switch_core/config.py
@@ -775,6 +775,9 @@ class SwitchConfig(BaseSettings):
# management on, the same TTL and bound apply to the separate cache of a
# controller access token's reads (its controller row, and the agent row
# it acts as), which a revocation or binding change also drops at once.
+ # That cache keeps an entry past the TTL while the signal connection it
+ # was read under still listens, since every change to those rows is then
+ # signalled to every process; 0 disables it too.
agent_auth_cache_ttl_seconds: float = 5.0
# How long a row in `agent_events` (one of an agent's own events) is kept
# before the retention loop deletes it. An agent that reconnects within
@@ -784,6 +787,10 @@ class SwitchConfig(BaseSettings):
# Bound on the memo. One entry per distinct live token; the oldest is
# evicted past this, so a flood of tokens cannot grow the process.
agent_auth_cache_max_entries: int = 4096
+ # Bound on the agent rows the controller-token cache keeps, the oldest
+ # evicted past it. Every process sees requests for every controller's
+ # agents, so it needs room for all the agents controllers run.
+ controller_auth_cache_max_agents: int = Field(default=20_000, ge=1)
# AGENT_MANAGEMENT_ENABLED, replaced by the `agent_management` feature flag.
# Read only to refuse a deployment that still turns it on, which would
diff --git a/core/switch_core/db/auth_rows_notify_ddl.py b/core/switch_core/db/auth_rows_notify_ddl.py
new file mode 100644
index 000000000..ab7a35604
--- /dev/null
+++ b/core/switch_core/db/auth_rows_notify_ddl.py
@@ -0,0 +1,112 @@
+"""The triggers that announce changes to the rows a controller access token
+is authorized with, as signals.
+
+Every process keeps the rows a controller token's requests read
+(`ControllerAuthCache`): the controller's, to refuse one revoked while its
+token is unexpired, and the row of the agent it acts as. It keeps them for as
+long as its signal connection listens and no signal says they changed. An
+agent's row is written from several places (a rename, its addressing policy,
+a template update, a deletion with its owner or workspace), and a controller's
+revocation commits in one process and is announced by it after the commit, so
+neither can depend on every writer remembering to say so. A trigger is the one
+place none of them can skip, as `membership_notify_ddl.py` argues for
+memberships.
+
+`agent.changed` names the tenant and the agent, for any update that changes
+the row and for a deletion. `controller.auth_changed` names the tenant and the
+controller, when its owner, its credential or its revocation changes and when
+it is deleted; the status, connection and assignment columns a controller
+writes every few seconds send nothing. Neither carries a sender process, so
+every process handles them, the writer's included. `NOTIFY` is delivered on
+commit, so a process never hears of a change it could not then read.
+
+The migration that installs these carries its own copy, frozen as it was
+written; this copy is what `create_all` builds for the tests.
+"""
+
+from __future__ import annotations
+
+from switch_core.signals import SIGNAL_CHANNEL
+
+AGENT_CHANGED_SIGNAL = "agent.changed"
+CONTROLLER_AUTH_CHANGED_SIGNAL = "controller.auth_changed"
+
+AGENT_NOTIFY_FUNCTION_NAME = "switch_notify_agent_changed"
+AGENT_NOTIFY_TRIGGER_NAME = "agents_notify_changed"
+CONTROLLER_NOTIFY_FUNCTION_NAME = "switch_notify_controller_auth_changed"
+CONTROLLER_NOTIFY_TRIGGER_NAME = "agent_controllers_notify_auth_changed"
+
+CREATE_AGENT_NOTIFY_FUNCTION = f"""
+CREATE OR REPLACE FUNCTION {AGENT_NOTIFY_FUNCTION_NAME}() RETURNS trigger AS $$
+DECLARE
+ changed agents;
+BEGIN
+ IF TG_OP = 'DELETE' THEN
+ changed := OLD;
+ ELSIF NEW IS NOT DISTINCT FROM OLD THEN
+ RETURN NULL;
+ ELSE
+ changed := NEW;
+ END IF;
+ PERFORM pg_notify(
+ '{SIGNAL_CHANNEL}',
+ json_build_object(
+ 'kind', '{AGENT_CHANGED_SIGNAL}',
+ 'tenant_id', changed.tenant_id,
+ 'agent_id', changed.id
+ )::text
+ );
+ RETURN NULL;
+END;
+$$ LANGUAGE plpgsql
+"""
+
+CREATE_AGENT_NOTIFY_TRIGGER = f"""
+CREATE TRIGGER {AGENT_NOTIFY_TRIGGER_NAME}
+ AFTER UPDATE OR DELETE ON agents
+ FOR EACH ROW EXECUTE FUNCTION {AGENT_NOTIFY_FUNCTION_NAME}()
+"""
+
+CREATE_CONTROLLER_NOTIFY_FUNCTION = f"""
+CREATE OR REPLACE FUNCTION {CONTROLLER_NOTIFY_FUNCTION_NAME}() RETURNS trigger AS $$
+DECLARE
+ changed agent_controllers;
+BEGIN
+ IF TG_OP = 'DELETE' THEN
+ changed := OLD;
+ ELSIF NEW.owner_id IS NOT DISTINCT FROM OLD.owner_id
+ AND NEW.revoked_at IS NOT DISTINCT FROM OLD.revoked_at
+ AND NEW.api_key_id IS NOT DISTINCT FROM OLD.api_key_id THEN
+ RETURN NULL;
+ ELSE
+ changed := NEW;
+ END IF;
+ PERFORM pg_notify(
+ '{SIGNAL_CHANNEL}',
+ json_build_object(
+ 'kind', '{CONTROLLER_AUTH_CHANGED_SIGNAL}',
+ 'tenant_id', changed.tenant_id,
+ 'controller_id', changed.id
+ )::text
+ );
+ RETURN NULL;
+END;
+$$ LANGUAGE plpgsql
+"""
+
+CREATE_CONTROLLER_NOTIFY_TRIGGER = f"""
+CREATE TRIGGER {CONTROLLER_NOTIFY_TRIGGER_NAME}
+ AFTER UPDATE OF owner_id, revoked_at, api_key_id OR DELETE ON agent_controllers
+ FOR EACH ROW EXECUTE FUNCTION {CONTROLLER_NOTIFY_FUNCTION_NAME}()
+"""
+
+DROP_AGENT_NOTIFY_TRIGGER = (
+ f"DROP TRIGGER IF EXISTS {AGENT_NOTIFY_TRIGGER_NAME} ON agents"
+)
+DROP_AGENT_NOTIFY_FUNCTION = f"DROP FUNCTION IF EXISTS {AGENT_NOTIFY_FUNCTION_NAME}()"
+DROP_CONTROLLER_NOTIFY_TRIGGER = (
+ f"DROP TRIGGER IF EXISTS {CONTROLLER_NOTIFY_TRIGGER_NAME} ON agent_controllers"
+)
+DROP_CONTROLLER_NOTIFY_FUNCTION = (
+ f"DROP FUNCTION IF EXISTS {CONTROLLER_NOTIFY_FUNCTION_NAME}()"
+)
diff --git a/core/switch_core/db/models.py b/core/switch_core/db/models.py
index 70dff1e2c..5cb372cb9 100644
--- a/core/switch_core/db/models.py
+++ b/core/switch_core/db/models.py
@@ -32,6 +32,14 @@ from switch_core.db.agent_event_notify_ddl import (
CREATE_AGENT_EVENT_NOTIFY_TRIGGER,
DROP_AGENT_EVENT_NOTIFY_TRIGGER,
)
+from switch_core.db.auth_rows_notify_ddl import (
+ CREATE_AGENT_NOTIFY_FUNCTION,
+ CREATE_AGENT_NOTIFY_TRIGGER,
+ CREATE_CONTROLLER_NOTIFY_FUNCTION,
+ CREATE_CONTROLLER_NOTIFY_TRIGGER,
+ DROP_AGENT_NOTIFY_TRIGGER,
+ DROP_CONTROLLER_NOTIFY_TRIGGER,
+)
from switch_core.db.base import Base
from switch_core.db.encrypted_json import EncryptedJSONB
from switch_core.db.membership_notify_ddl import (
@@ -3880,6 +3888,28 @@ class AgentController(TenantScoped, Base):
)
+# The triggers that signal every process when an agent's row changes, or a
+# controller's owner, credential or revocation does, so each process drops
+# what it kept of them for authorizing controller tokens. As for the triggers
+# above, `create_all` builds them for the tests and real databases get them
+# from a migration. The SQL lives in `db/auth_rows_notify_ddl.py`.
+for _table, _ddls, _drop in (
+ (
+ Agent.__table__,
+ (CREATE_AGENT_NOTIFY_FUNCTION, CREATE_AGENT_NOTIFY_TRIGGER),
+ DROP_AGENT_NOTIFY_TRIGGER,
+ ),
+ (
+ AgentController.__table__,
+ (CREATE_CONTROLLER_NOTIFY_FUNCTION, CREATE_CONTROLLER_NOTIFY_TRIGGER),
+ DROP_CONTROLLER_NOTIFY_TRIGGER,
+ ),
+):
+ for _ddl in _ddls:
+ event.listen(_table, "after_create", DDL(_ddl).execute_if(dialect="postgresql"))
+ event.listen(_table, "before_drop", DDL(_drop).execute_if(dialect="postgresql"))
+
+
class AgentControllerEnrollmentCode(TenantScoped, Base):
"""A one-time code a headless controller enrolls with.
diff --git a/core/switch_core/db/stores/agent_store.py b/core/switch_core/db/stores/agent_store.py
index d9b647faa..94522d19d 100644
--- a/core/switch_core/db/stores/agent_store.py
+++ b/core/switch_core/db/stores/agent_store.py
@@ -31,6 +31,15 @@ class AgentStore:
async def get(self, session: AsyncSession, agent_id: str) -> Agent | None:
return await session.get(Agent, agent_id)
+ async def get_many(
+ self, session: AsyncSession, agent_ids: Collection[str]
+ ) -> list[Agent]:
+ """`get` for many agents in one query; an id with no agent is left out."""
+ if not agent_ids:
+ return []
+ result = await session.execute(select(Agent).where(Agent.id.in_(agent_ids)))
+ return list(result.scalars().all())
+
async def get_by_name(self, session: AsyncSession, name: str) -> Agent | None:
"""Resolve an agent by name within the bound tenant.
diff --git a/core/switch_core/management/auth.py b/core/switch_core/management/auth.py
index bf960e5ec..27c925dd6 100644
--- a/core/switch_core/management/auth.py
+++ b/core/switch_core/management/auth.py
@@ -92,7 +92,7 @@ class ManagementAuthenticator:
if cached is not None and cached.owner_id == claims.owner_id:
return cached
- generation = self._auth_cache.generation
+ ticket = self._auth_cache.ticket()
async with tenant_session(self._session_factory, claims.tenant_id) as session:
controller = await self._controllers.get(
session, claims.tenant_id, claims.controller_id
@@ -111,7 +111,7 @@ class ManagementAuthenticator:
owner_id=controller.owner_id,
tenant_id=claims.tenant_id,
)
- self._auth_cache.put_controller(principal, generation)
+ self._auth_cache.put_controller(principal, ticket)
return principal
async def tenant_of_secret(self, secret: str) -> str | None:
diff --git a/core/switch_core/management/wiring.py b/core/switch_core/management/wiring.py
index c9d93e497..da821585b 100644
--- a/core/switch_core/management/wiring.py
+++ b/core/switch_core/management/wiring.py
@@ -155,6 +155,7 @@ def build_management(
operations = AgentControllerOperationStore()
processes = SwitchCoreProcessStore()
presence.use_auth_cache(auth_cache)
+ auth_cache.follow(signal_bus)
signals = ControllerSignals(
notifier=ControllerNotifier(),
presence=presence,
@@ -255,6 +256,7 @@ def create_management(
auth_cache=ControllerAuthCache(
ttl_seconds=config.agent_auth_cache_ttl_seconds,
max_entries=config.agent_auth_cache_max_entries,
+ max_agents=config.controller_auth_cache_max_agents,
),
clock=utc_now,
signal_bus=signal_bus,
diff --git a/core/switch_core/migrations/versions/bad5099ef32b_auth_rows_notify.py b/core/switch_core/migrations/versions/bad5099ef32b_auth_rows_notify.py
new file mode 100644
index 000000000..ec2976e36
--- /dev/null
+++ b/core/switch_core/migrations/versions/bad5099ef32b_auth_rows_notify.py
@@ -0,0 +1,107 @@
+"""auth_rows_notify: signal every switch-core process when an agent's row or a
+controller's authority changes
+
+Every process keeps the controller and agent rows a controller access token's
+requests are authorized with. These triggers send `agent.changed` on the
+`switch_signal` channel for every update that changes an agent's row and for
+its deletion, and `controller.auth_changed` when a controller's owner,
+credential or revocation changes and when it is deleted, so every process
+drops what it kept of them.
+
+Revision ID: bad5099ef32b
+Revises: b7e1d4c9a2f6
+Create Date: 2026-11-01 00:00:00.000000
+
+"""
+
+from collections.abc import Sequence
+
+from alembic import op
+
+revision: str = "bad5099ef32b"
+down_revision: str | None = "b7e1d4c9a2f6"
+branch_labels: str | Sequence[str] | None = None
+depends_on: str | Sequence[str] | None = None
+
+# Copies of the DDL in `db/auth_rows_notify_ddl.py`, written out here so this
+# migration keeps creating the same objects if that module changes.
+CREATE_AGENT_NOTIFY_FUNCTION = """
+CREATE OR REPLACE FUNCTION switch_notify_agent_changed() RETURNS trigger AS $$
+DECLARE
+ changed agents;
+BEGIN
+ IF TG_OP = 'DELETE' THEN
+ changed := OLD;
+ ELSIF NEW IS NOT DISTINCT FROM OLD THEN
+ RETURN NULL;
+ ELSE
+ changed := NEW;
+ END IF;
+ PERFORM pg_notify(
+ 'switch_signal',
+ json_build_object(
+ 'kind', 'agent.changed',
+ 'tenant_id', changed.tenant_id,
+ 'agent_id', changed.id
+ )::text
+ );
+ RETURN NULL;
+END;
+$$ LANGUAGE plpgsql
+"""
+
+CREATE_AGENT_NOTIFY_TRIGGER = """
+CREATE TRIGGER agents_notify_changed
+ AFTER UPDATE OR DELETE ON agents
+ FOR EACH ROW EXECUTE FUNCTION switch_notify_agent_changed()
+"""
+
+CREATE_CONTROLLER_NOTIFY_FUNCTION = """
+CREATE OR REPLACE FUNCTION switch_notify_controller_auth_changed() RETURNS trigger AS $$
+DECLARE
+ changed agent_controllers;
+BEGIN
+ IF TG_OP = 'DELETE' THEN
+ changed := OLD;
+ ELSIF NEW.owner_id IS NOT DISTINCT FROM OLD.owner_id
+ AND NEW.revoked_at IS NOT DISTINCT FROM OLD.revoked_at
+ AND NEW.api_key_id IS NOT DISTINCT FROM OLD.api_key_id THEN
+ RETURN NULL;
+ ELSE
+ changed := NEW;
+ END IF;
+ PERFORM pg_notify(
+ 'switch_signal',
+ json_build_object(
+ 'kind', 'controller.auth_changed',
+ 'tenant_id', changed.tenant_id,
+ 'controller_id', changed.id
+ )::text
+ );
+ RETURN NULL;
+END;
+$$ LANGUAGE plpgsql
+"""
+
+CREATE_CONTROLLER_NOTIFY_TRIGGER = """
+CREATE TRIGGER agent_controllers_notify_auth_changed
+ AFTER UPDATE OF owner_id, revoked_at, api_key_id OR DELETE ON agent_controllers
+ FOR EACH ROW EXECUTE FUNCTION switch_notify_controller_auth_changed()
+"""
+
+
+def upgrade() -> None:
+ op.execute(CREATE_AGENT_NOTIFY_FUNCTION)
+ op.execute(CREATE_AGENT_NOTIFY_TRIGGER)
+ op.execute(CREATE_CONTROLLER_NOTIFY_FUNCTION)
+ op.execute(CREATE_CONTROLLER_NOTIFY_TRIGGER)
+
+
+def downgrade() -> None:
+ op.execute(
+ "DROP TRIGGER IF EXISTS agent_controllers_notify_auth_changed "
+ "ON agent_controllers"
+ )
+ op.execute("DROP FUNCTION IF EXISTS switch_notify_controller_auth_changed()")
+ op.execute("DROP TRIGGER IF EXISTS agents_notify_changed ON agents")
+ op.execute("DROP FUNCTION IF EXISTS switch_notify_agent_changed()")
diff --git a/core/switch_core/signals.py b/core/switch_core/signals.py
index 5eda31b80..ea827c7d0 100644
--- a/core/switch_core/signals.py
+++ b/core/switch_core/signals.py
@@ -172,6 +172,11 @@ class SignalBus:
self._tasks: list[asyncio.Task[None]] = []
self._running = False
self.connected = asyncio.Event()
+ # How many listening connections have been established. What a process
+ # keeps current by signals is current only while this stays the same
+ # and `connected` stays set: a signal sent between two connections
+ # reached neither.
+ self.listens = 0
# ── Registration ──────────────────────────────────────────────────────────
@@ -346,6 +351,7 @@ class SignalBus:
)
await driver.add_listener(SIGNAL_CHANNEL, self._on_notify)
try:
+ self.listens += 1
self.connected.set()
await self._resync_all()
await self._hold_open(driver)
diff --git a/core/tests/switch_core/bridges/agent/test_controller_auth_cache.py b/core/tests/switch_core/bridges/agent/test_controller_auth_cache.py
index 186ab43b3..cad8f18f8 100644
--- a/core/tests/switch_core/bridges/agent/test_controller_auth_cache.py
+++ b/core/tests/switch_core/bridges/agent/test_controller_auth_cache.py
@@ -8,7 +8,10 @@ ones that need a hand on the clock or on the ordering.
from __future__ import annotations
+import asyncio
+from collections.abc import Callable
from types import SimpleNamespace
+from typing import Any
import pytest
@@ -21,6 +24,10 @@ from switch_core.bridges.agent.protocol.controller_presence import (
ControllerPresence,
)
from switch_core.bridges.agent.protocol.liveness import HEARTBEAT_TTL_SECONDS
+from switch_core.db.auth_rows_notify_ddl import (
+ AGENT_CHANGED_SIGNAL,
+ CONTROLLER_AUTH_CHANGED_SIGNAL,
+)
from switch_core.db.models import Agent
@@ -51,8 +58,8 @@ def _agent(agent_id: str = "a1") -> Agent:
def test_entries_expire_after_the_ttl(clock: _Clock) -> None:
cache = ControllerAuthCache(ttl_seconds=5, max_entries=8)
- cache.put_controller(_principal(), cache.generation)
- cache.put_agent("t1", _agent(), cache.generation)
+ cache.put_controller(_principal(), cache.ticket())
+ cache.put_agents("t1", [_agent()], cache.ticket())
clock.now += 4.9
assert cache.controller("t1", "c1") == _principal()
@@ -64,8 +71,8 @@ def test_entries_expire_after_the_ttl(clock: _Clock) -> None:
def test_an_entry_is_keyed_by_its_tenant(clock: _Clock) -> None:
cache = ControllerAuthCache(ttl_seconds=5, max_entries=8)
- cache.put_controller(_principal(), cache.generation)
- cache.put_agent("t1", _agent(), cache.generation)
+ cache.put_controller(_principal(), cache.ticket())
+ cache.put_agents("t1", [_agent()], cache.ticket())
assert cache.controller("t2", "c1") is None
assert cache.agent("t2", "a1") is None
@@ -73,23 +80,23 @@ def test_an_entry_is_keyed_by_its_tenant(clock: _Clock) -> None:
def test_a_read_that_straddled_an_invalidation_is_not_stored(clock: _Clock) -> None:
cache = ControllerAuthCache(ttl_seconds=5, max_entries=8)
- before_revoke = cache.generation
+ before_revoke = cache.ticket()
cache.invalidate_controller("c1")
cache.put_controller(_principal(), before_revoke)
assert cache.controller("t1", "c1") is None
- before_unbind = cache.generation
+ before_unbind = cache.ticket()
cache.invalidate_agent("a1")
- cache.put_agent("t1", _agent(), before_unbind)
+ cache.put_agents("t1", [_agent()], before_unbind)
assert cache.agent("t1", "a1") is None
def test_invalidation_drops_only_what_it_names(clock: _Clock) -> None:
cache = ControllerAuthCache(ttl_seconds=5, max_entries=8)
for principal in (_principal("c1"), _principal("c2"), _principal("c1", "t2")):
- cache.put_controller(principal, cache.generation)
+ cache.put_controller(principal, cache.ticket())
for agent_id in ("a1", "a2"):
- cache.put_agent("t1", _agent(agent_id), cache.generation)
+ cache.put_agents("t1", [_agent(agent_id)], cache.ticket())
cache.invalidate_controller("c1")
cache.invalidate_agent("a1")
@@ -104,20 +111,156 @@ def test_invalidation_drops_only_what_it_names(clock: _Clock) -> None:
def test_it_holds_no_more_than_its_bound(clock: _Clock) -> None:
cache = ControllerAuthCache(ttl_seconds=5, max_entries=2)
for agent_id in ("a1", "a2", "a3"):
- cache.put_agent("t1", _agent(agent_id), cache.generation)
+ cache.put_agents("t1", [_agent(agent_id)], cache.ticket())
assert cache.agent("t1", "a1") is None
assert cache.agent("t1", "a3") is not None
+def test_agents_have_a_bound_of_their_own(clock: _Clock) -> None:
+ cache = ControllerAuthCache(ttl_seconds=5, max_entries=1, max_agents=3)
+ cache.put_agents("t1", [_agent(a) for a in ("a1", "a2", "a3")], cache.ticket())
+ for principal in (_principal("c1"), _principal("c2")):
+ cache.put_controller(principal, cache.ticket())
+
+ assert [cache.agent("t1", a) is not None for a in ("a1", "a2", "a3")] == [
+ True,
+ True,
+ True,
+ ]
+ assert cache.controller("t1", "c1") is None
+ assert cache.controller("t1", "c2") is not None
+
+
def test_a_ttl_of_zero_stores_nothing(clock: _Clock) -> None:
cache = ControllerAuthCache(ttl_seconds=0, max_entries=8)
- cache.put_controller(_principal(), cache.generation)
- cache.put_agent("t1", _agent(), cache.generation)
+ cache.put_controller(_principal(), cache.ticket())
+ cache.put_agents("t1", [_agent()], cache.ticket())
assert not cache.enabled
assert cache.controller("t1", "c1") is None
assert cache.agent("t1", "a1") is None
+class _Bus:
+ """Just enough of `SignalBus`: whether it listens, on which connection,
+ and the handlers registered with it."""
+
+ def __init__(self) -> None:
+ self.connected = asyncio.Event()
+ self.listens = 0
+ self.handlers: dict[str, Callable[[dict[str, Any]], None]] = {}
+
+ def handle(self, kind: str, handler: Callable[[dict[str, Any]], None]) -> None:
+ self.handlers[kind] = handler
+
+ def listen(self) -> None:
+ self.listens += 1
+ self.connected.set()
+
+ def drop(self) -> None:
+ self.connected.clear()
+
+
+def _following(bus: _Bus) -> ControllerAuthCache:
+ cache = ControllerAuthCache(ttl_seconds=5, max_entries=8)
+ cache.follow(bus) # type: ignore[arg-type]
+ return cache
+
+
+def _warm(cache: ControllerAuthCache) -> None:
+ cache.put_controller(_principal(), cache.ticket())
+ cache.put_agents("t1", [_agent()], cache.ticket())
+
+
+class TestWhileSignalsListen:
+ def test_an_entry_outlives_the_ttl(self, clock: _Clock) -> None:
+ bus = _Bus()
+ bus.listen()
+ cache = _following(bus)
+ _warm(cache)
+
+ clock.now += 3600
+
+ assert cache.controller("t1", "c1") == _principal()
+ assert cache.agent("t1", "a1") is not None
+
+ def test_the_signals_drop_what_they_name(self, clock: _Clock) -> None:
+ bus = _Bus()
+ bus.listen()
+ cache = _following(bus)
+ _warm(cache)
+ cache.put_agents("t1", [_agent("a2")], cache.ticket())
+
+ bus.handlers[CONTROLLER_AUTH_CHANGED_SIGNAL](
+ {"kind": CONTROLLER_AUTH_CHANGED_SIGNAL, "controller_id": "c1"}
+ )
+ bus.handlers[AGENT_CHANGED_SIGNAL](
+ {"kind": AGENT_CHANGED_SIGNAL, "agent_id": "a1"}
+ )
+
+ assert cache.controller("t1", "c1") is None
+ assert cache.agent("t1", "a1") is None
+ assert cache.agent("t1", "a2") is not None
+
+ def test_an_entry_read_while_the_bus_was_down_lasts_the_ttl(
+ self, clock: _Clock
+ ) -> None:
+ bus = _Bus()
+ cache = _following(bus)
+ _warm(cache)
+ bus.listen()
+
+ clock.now += 5.1
+
+ assert cache.controller("t1", "c1") is None
+ assert cache.agent("t1", "a1") is None
+
+ def test_losing_the_connection_ends_the_hold(self, clock: _Clock) -> None:
+ """A signal sent while the bus does not listen reaches nobody, so what
+ was kept on the strength of signals goes back to lasting the TTL."""
+ bus = _Bus()
+ bus.listen()
+ cache = _following(bus)
+ _warm(cache)
+
+ bus.drop()
+ clock.now += 5.1
+ assert cache.agent("t1", "a1") is None
+
+ _warm(cache)
+ bus.listen()
+ clock.now += 5.1
+ assert cache.controller("t1", "c1") is None
+
+ def test_a_new_connection_does_not_revive_the_old_one_s_entries(
+ self, clock: _Clock
+ ) -> None:
+ bus = _Bus()
+ bus.listen()
+ cache = _following(bus)
+ _warm(cache)
+
+ bus.drop()
+ bus.listen()
+ clock.now += 5.1
+
+ assert cache.controller("t1", "c1") is None
+ assert cache.agent("t1", "a1") is None
+
+ def test_a_read_across_a_reconnect_lasts_the_ttl(self, clock: _Clock) -> None:
+ bus = _Bus()
+ bus.listen()
+ cache = _following(bus)
+ ticket = cache.ticket()
+
+ bus.drop()
+ bus.listen()
+ cache.put_agents("t1", [_agent()], ticket)
+
+ assert cache.agent("t1", "a1") is not None
+ clock.now += 5.1
+ assert cache.agent("t1", "a1") is None
+
+
def test_the_ttl_must_stay_below_the_heartbeat_ttl() -> None:
with pytest.raises(ValueError, match="heartbeat"):
ControllerAuthCache(ttl_seconds=HEARTBEAT_TTL_SECONDS, max_entries=8)
@@ -125,6 +268,8 @@ def test_the_ttl_must_stay_below_the_heartbeat_ttl() -> None:
ControllerAuthCache(ttl_seconds=-1, max_entries=8)
with pytest.raises(ValueError, match="max_entries"):
ControllerAuthCache(ttl_seconds=1, max_entries=0)
+ with pytest.raises(ValueError, match="max_agents"):
+ ControllerAuthCache(ttl_seconds=1, max_entries=1, max_agents=0)
def test_presence_keeps_it_in_step_with_bindings_and_revocations(
@@ -142,8 +287,8 @@ def test_presence_keeps_it_in_step_with_bindings_and_revocations(
)
def warm() -> None:
- cache.put_controller(_principal(), cache.generation)
- cache.put_agent("t1", _agent(), cache.generation)
+ cache.put_controller(_principal(), cache.ticket())
+ cache.put_agents("t1", [_agent()], cache.ticket())
warm()
presence.bind(binding)
diff --git a/core/tests/switch_core/management/test_controller_auth_cache.py b/core/tests/switch_core/management/test_controller_auth_cache.py
index 7ecca468a..f24feb409 100644
--- a/core/tests/switch_core/management/test_controller_auth_cache.py
+++ b/core/tests/switch_core/management/test_controller_auth_cache.py
@@ -7,19 +7,35 @@ database" means no statement reached Postgres.
from __future__ import annotations
+import asyncio
import json
-from collections.abc import Iterator
-from contextlib import contextmanager
+import time
+from collections.abc import AsyncIterator, Callable, Iterator
+from contextlib import asynccontextmanager, contextmanager
+from types import SimpleNamespace
from typing import Any
import pytest
-from sqlalchemy import event
-from sqlalchemy.ext.asyncio import AsyncEngine, AsyncSession, async_sessionmaker
+from sqlalchemy import delete, event, func, update
+from sqlalchemy.ext.asyncio import (
+ AsyncEngine,
+ AsyncSession,
+ async_sessionmaker,
+ create_async_engine,
+)
+from sqlalchemy.pool import NullPool
+from switch_core.bridges.agent import controller_auth_cache
from switch_core.bridges.agent.auth import BearerAuthMiddleware
-from switch_core.db.models import TENANT_ZERO_ID
+from switch_core.db.auth_rows_notify_ddl import (
+ AGENT_CHANGED_SIGNAL,
+ CONTROLLER_AUTH_CHANGED_SIGNAL,
+)
+from switch_core.db.models import TENANT_ZERO_ID, Agent, AgentController
+from switch_core.db.session_scope import tenant_session
from switch_core.db.stores.agent_store import AgentStore
from switch_core.db.stores.api_key_store import ApiKeyStore
+from switch_core.signals import SIGNAL_CHANNEL
from tests.switch_core.management.harness import (
EnrolledController,
Harness,
@@ -256,3 +272,205 @@ async def test_a_ttl_of_zero_reads_the_database_every_time(
assert reads.of("agent_controllers") == 3
assert reads.of("agents") == 3
assert not harness.controller_auth_cache.enabled
+
+
+async def test_a_controller_s_agents_are_read_together(harness: Harness) -> None:
+ """A miss reads every agent bound to the controller, so a request for
+ another of them reads none."""
+ controller, first = await placed(harness)
+ async with harness.client() as client:
+ second = await place_agent(client, controller, name="writer")
+ mw = middleware(harness)
+ harness.controller_auth_cache.clear()
+
+ with counting(harness.session_factory) as reads:
+ assert await act_as(mw, controller, first) == (200, None)
+ assert await act_as(mw, controller, second) == (200, None)
+
+ assert reads.of("agents") == 1
+
+
+# ── While the signal bus listens ──────────────────────────────────────────────
+
+
+async def listening(
+ postgres_url: str, session_factory: async_sessionmaker[AsyncSession]
+) -> AsyncIterator[Harness]:
+ harness = build_harness(
+ session_factory,
+ signal_engine=lambda: create_async_engine(postgres_url, poolclass=NullPool),
+ )
+ await harness.management.lease.renew()
+ await harness.bus.start()
+ await asyncio.wait_for(harness.bus.connected.wait(), 10)
+ try:
+ yield harness
+ finally:
+ await harness.bus.stop()
+
+
+async def _until(condition: Callable[[], bool]) -> None:
+ for _ in range(500):
+ if condition():
+ return
+ await asyncio.sleep(0.01)
+ raise AssertionError("never happened")
+
+
+async def test_while_the_bus_listens_a_hit_outlives_the_ttl(
+ listening: Harness, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ controller, agent_id = await placed(listening)
+ mw = middleware(listening)
+ assert await act_as(mw, controller, agent_id) == (200, None)
+ an_hour_on = time.monotonic() + 3600
+ monkeypatch.setattr(
+ controller_auth_cache, "time", SimpleNamespace(monotonic=lambda: an_hour_on)
+ )
+
+ with counting(listening.session_factory) as reads:
+ assert await act_as(mw, controller, agent_id) == (200, None)
+
+ assert reads.of("agent_controllers") == 0
+ assert reads.of("agents") == 0
+
+
+async def test_an_agent_changed_in_the_database_is_dropped(listening: Harness) -> None:
+ controller, agent_id = await placed(listening)
+ mw = middleware(listening)
+ assert await act_as(mw, controller, agent_id) == (200, None)
+ cache = listening.controller_auth_cache
+ assert cache.agent(TENANT_ZERO_ID, agent_id) is not None
+
+ async with tenant_session(listening.session_factory, TENANT_ZERO_ID) as session:
+ await session.execute(
+ update(Agent).where(Agent.id == agent_id).values(description="changed")
+ )
+ await session.commit()
+
+ await _until(lambda: cache.agent(TENANT_ZERO_ID, agent_id) is None)
+
+
+async def test_a_controller_revoked_where_nothing_announces_it_is_refused(
+ listening: Harness,
+) -> None:
+ """Revoked by a path that tells no process, as an operator in psql would:
+ the trigger tells every one."""
+ controller, agent_id = await placed(listening)
+ mw = middleware(listening)
+ assert await act_as(mw, controller, agent_id) == (200, None)
+ cache = listening.controller_auth_cache
+
+ async with tenant_session(listening.session_factory, TENANT_ZERO_ID) as session:
+ await session.execute(
+ update(AgentController)
+ .where(AgentController.id == controller.controller_id)
+ .values(revoked_at=func.now(), api_key_id=None)
+ )
+ await session.commit()
+
+ await _until(
+ lambda: cache.controller(TENANT_ZERO_ID, controller.controller_id) is None
+ )
+ assert await act_as(mw, controller, agent_id) == (401, "controller_revoked")
+
+
+# ── The triggers ──────────────────────────────────────────────────────────────
+
+_SENTINEL = "test.sentinel"
+
+
+@asynccontextmanager
+async def heard(postgres_url: str) -> AsyncIterator[list[dict[str, Any]]]:
+ """The signals sent while the block runs. A sentinel is sent after the
+ block, and signals arrive in commit order, so once it is heard every
+ signal the block's commits sent has been too."""
+ engine = create_async_engine(postgres_url, poolclass=NullPool)
+ signals: list[dict[str, Any]] = []
+ done = asyncio.Event()
+
+ def on_notify(_conn: Any, _pid: int, _channel: str, payload: str) -> None:
+ signal = json.loads(payload)
+ if signal.get("kind") == _SENTINEL:
+ done.set()
+ else:
+ signals.append(signal)
+
+ try:
+ async with engine.connect() as connection:
+ raw = await connection.get_raw_connection()
+ driver = raw.driver_connection
+ assert driver is not None
+ await driver.add_listener(SIGNAL_CHANNEL, on_notify)
+ yield signals
+ await driver.execute(
+ f"NOTIFY {SIGNAL_CHANNEL}, '{json.dumps({'kind': _SENTINEL})}'"
+ )
+ await asyncio.wait_for(done.wait(), 10)
+ finally:
+ await engine.dispose()
+
+
+async def test_an_agent_s_row_changing_or_going_is_signalled(
+ harness: Harness, postgres_url: str
+) -> None:
+ _, agent_id = await placed(harness)
+
+ async with heard(postgres_url) as signals:
+ async with tenant_session(harness.session_factory, TENANT_ZERO_ID) as session:
+ # Written but unchanged: nothing to drop.
+ await session.execute(
+ update(Agent)
+ .where(Agent.id == agent_id)
+ .values(description=Agent.description)
+ )
+ await session.execute(
+ update(Agent).where(Agent.id == agent_id).values(description="new")
+ )
+ await session.commit()
+
+ assert signals == [
+ {
+ "kind": AGENT_CHANGED_SIGNAL,
+ "tenant_id": TENANT_ZERO_ID,
+ "agent_id": agent_id,
+ }
+ ]
+
+
+async def test_only_a_controller_s_authority_changing_is_signalled(
+ harness: Harness, postgres_url: str
+) -> None:
+ owner = await add_member(harness.session_factory, "ada")
+ async with harness.client() as client:
+ controller = await enroll_console(harness, client, owner)
+ row = AgentController.id == controller.controller_id
+
+ async with heard(postgres_url) as signals:
+ async with tenant_session(harness.session_factory, TENANT_ZERO_ID) as session:
+ # What a controller writes as it runs.
+ await session.execute(
+ update(AgentController).where(row).values(updated_at=func.now())
+ )
+ await session.execute(
+ update(AgentController)
+ .where(row)
+ .values(revoked_at=func.now(), api_key_id=None)
+ )
+ await session.commit()
+ async with tenant_session(harness.session_factory, TENANT_ZERO_ID) as session:
+ await session.execute(delete(AgentController).where(row))
+ await session.commit()
+
+ assert (
+ signals
+ == [
+ {
+ "kind": CONTROLLER_AUTH_CHANGED_SIGNAL,
+ "tenant_id": TENANT_ZERO_ID,
+ "controller_id": controller.controller_id,
+ }
+ ]
+ * 2
+ )