Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

Let a controller send its agents' operations on its socket, not over HTTP (CHOO-3667)

Abel Dantas7ede3123d783
diff --git a/core/switch_core/bridges/agent/api/operations.py b/core/switch_core/bridges/agent/api/operations.py
index 6155b1907..3e8265f37 100644
--- a/core/switch_core/bridges/agent/api/operations.py
+++ b/core/switch_core/bridges/agent/api/operations.py
@@ -63,6 +63,26 @@ class BadArgumentsError(Exception):
"""The body did not match the operation's parameters."""
+def refusal_status(exc: Exception) -> int | None:
+ """The status an operation's refusal is answered with, over HTTP or on a
+ controller's socket; None for an exception that is a failure instead."""
+ if isinstance(exc, UnknownOperationError):
+ return 404
+ if isinstance(exc, BadArgumentsError):
+ return 400
+ if isinstance(exc, BudgetExceeded):
+ return 429
+ if isinstance(exc, GuardrailBlockedError):
+ return 422
+ if isinstance(exc, PermissionError):
+ return 403
+ if isinstance(exc, ValueError):
+ # Operations raise ValueError for "you asked for something that is not
+ # there or not allowed yet" - surfaced rather than swallowed.
+ return 400
+ return None
+
+
def list_operations() -> dict[str, dict[str, Any]]:
"""Every operation, with its parameters.
@@ -327,19 +347,10 @@ async def post_operation(
connection_id=session_key,
session=caller,
)
- except UnknownOperationError as exc:
- raise HTTPException(status_code=404, detail=str(exc)) from exc
- except BadArgumentsError as exc:
- raise HTTPException(status_code=400, detail=str(exc)) from exc
- except BudgetExceeded as exc:
- raise HTTPException(status_code=429, detail=str(exc)) from exc
- except GuardrailBlockedError as exc:
- raise HTTPException(status_code=422, detail=str(exc)) from exc
- except PermissionError as exc:
- raise HTTPException(status_code=403, detail=str(exc)) from exc
- except ValueError as exc:
- # Operations raise ValueError for "you asked for something that is not
- # there or not allowed yet" - surfaced rather than swallowed.
- raise HTTPException(status_code=400, detail=str(exc)) from exc
+ except Exception as exc:
+ status = refusal_status(exc)
+ if status is None:
+ raise
+ raise HTTPException(status_code=status, detail=str(exc)) from exc
return {"result": result}
diff --git a/core/switch_core/management/auth.py b/core/switch_core/management/auth.py
index bf960e5ec..41498defb 100644
--- a/core/switch_core/management/auth.py
+++ b/core/switch_core/management/auth.py
@@ -87,17 +87,33 @@ class ManagementAuthenticator:
reason_codes.INVALID_CREDENTIAL,
"The access token is not a valid controller token.",
) from exc
+ return await self.current(
+ ControllerPrincipal(
+ controller_id=claims.controller_id,
+ owner_id=claims.owner_id,
+ tenant_id=claims.tenant_id,
+ )
+ )
- cached = self._auth_cache.controller(claims.tenant_id, claims.controller_id)
- if cached is not None and cached.owner_id == claims.owner_id:
+ async def current(self, principal: ControllerPrincipal) -> ControllerPrincipal:
+ """The controller `principal` names, while it keeps the authority its
+ token gave it: not revoked, and owned by the same person. Every request
+ made with a controller access token is checked here, and every call on
+ a controller's socket, which authenticated once when it opened."""
+ cached = self._auth_cache.controller(
+ principal.tenant_id, principal.controller_id
+ )
+ if cached is not None and cached.owner_id == principal.owner_id:
return cached
generation = self._auth_cache.generation
- async with tenant_session(self._session_factory, claims.tenant_id) as session:
+ async with tenant_session(
+ self._session_factory, principal.tenant_id
+ ) as session:
controller = await self._controllers.get(
- session, claims.tenant_id, claims.controller_id
+ session, principal.tenant_id, principal.controller_id
)
- if controller is None or controller.owner_id != claims.owner_id:
+ if controller is None or controller.owner_id != principal.owner_id:
raise ControllerAuthError(
reason_codes.INVALID_CREDENTIAL,
"The access token names no controller.",
@@ -106,13 +122,13 @@ class ManagementAuthenticator:
raise ControllerAuthError(
reason_codes.CONTROLLER_REVOKED, "The controller has been revoked."
)
- principal = ControllerPrincipal(
+ current = ControllerPrincipal(
controller_id=controller.id,
owner_id=controller.owner_id,
- tenant_id=claims.tenant_id,
+ tenant_id=principal.tenant_id,
)
- self._auth_cache.put_controller(principal, generation)
- return principal
+ self._auth_cache.put_controller(current, generation)
+ return current
async def tenant_of_secret(self, secret: str) -> str | None:
"""The tenant an enrollment code or controller credential belongs to,
diff --git a/core/switch_core/management/controller_routes.py b/core/switch_core/management/controller_routes.py
index aa5da5d8e..030ce385e 100644
--- a/core/switch_core/management/controller_routes.py
+++ b/core/switch_core/management/controller_routes.py
@@ -23,6 +23,7 @@ import asyncio
import hashlib
import json
import logging
+from collections import deque
from collections.abc import AsyncGenerator, Awaitable, Callable, Coroutine
from typing import Annotated, Any
@@ -53,7 +54,7 @@ from switch_core.bridges.agent.protocol.controller_stream import (
)
from switch_core.bridges.agent.protocol.liveness import HEARTBEAT_INTERVAL_SECONDS
from switch_core.db.session_scope import tenant_session
-from switch_core.management import reason_codes
+from switch_core.management import reason_codes, socket_calls
from switch_core.management.auth import ManagementAuthenticator
from switch_core.management.dependencies import (
get_authenticator,
@@ -444,6 +445,8 @@ async def attach_stream(
"connection_id": conn.id,
"generation": conn.generation,
"heartbeat_interval_s": HEARTBEAT_INTERVAL_SECONDS,
+ # The socket takes its agents' operations as calls (`socket_calls`).
+ "calls": socket_calls.calls_offered(),
}
return controller_event_stream(
conn=conn,
@@ -527,6 +530,7 @@ async def connection_socket(
controller_id: str,
management: Management,
protocol: Protocol,
+ authenticator: Annotated[ManagementAuthenticator, Depends(get_authenticator)],
session_factory: Annotated[
async_sessionmaker[AsyncSession], Depends(get_management_session_factory)
],
@@ -534,14 +538,16 @@ async def connection_socket(
generation: int,
) -> None:
"""The controller's one connection: its agents' events and the management
- nudges down, its heartbeat up, on one WebSocket.
+ nudges down, its heartbeat and its agents' calls up, on one WebSocket.
Attaches to a connection opened with `POST .../connection`. Frames are
JSON `{"event", "data"}`, the events the stream carries. The server sends
`{"event": "ping"}` every heartbeat interval and the controller answers
- `{"type": "pong", "cursors": {agent_id: n}}`, which is its beat. A refused
- attach is a `refused` frame, then a close with 4000 plus the status; a
- refused beat is an `evicted` frame naming why, then a close.
+ `{"type": "pong", "cursors": {agent_id: n}}`, which is its beat. A
+ `{"type": "call", ...}` runs one of its agents' operations and is answered
+ with a `call.result` frame (`socket_calls`). A refused attach is a
+ `refused` frame, then a close with 4000 plus the status; a refused beat is
+ an `evicted` frame naming why, then a close.
"""
await websocket.accept()
try:
@@ -559,18 +565,26 @@ async def connection_socket(
await websocket.close(code=4000 + error.status_code)
return
- outbox: asyncio.Queue[ControllerFrame] = asyncio.Queue(maxsize=_SOCKET_OUTBOX)
+ outbox = _Outbox(_SOCKET_OUTBOX)
pump = asyncio.create_task(_pump(frames, outbox))
- pongs = asyncio.create_task(
- _receive_pongs(websocket, principal, protocol, connection_id, generation)
+ receiver = asyncio.create_task(
+ _receive(
+ websocket,
+ principal=principal,
+ protocol=protocol,
+ authenticator=authenticator,
+ connection_id=connection_id,
+ generation=generation,
+ outbox=outbox,
+ )
)
loop = asyncio.get_running_loop()
next_ping = loop.time() + HEARTBEAT_INTERVAL_SECONDS
try:
- # One writer: frames and pings both go out from this loop.
+ # One writer: frames, call answers and pings all go out from this loop.
while not pump.done() or not outbox.empty():
- if pongs.done():
- refusal = pongs.result()
+ if receiver.done():
+ refusal = receiver.result()
if refusal is not None:
await websocket.send_json(
{
@@ -579,12 +593,7 @@ async def connection_socket(
}
)
return
- try:
- frame = await asyncio.wait_for(
- outbox.get(), timeout=max(0.0, next_ping - loop.time())
- )
- except TimeoutError:
- frame = None
+ frame = await outbox.get(timeout=max(0.0, next_ping - loop.time()))
if frame is not None and frame is not IDLE:
await websocket.send_json(frame)
if loop.time() >= next_ping:
@@ -597,7 +606,7 @@ async def connection_socket(
return
finally:
pump.cancel()
- pongs.cancel()
+ receiver.cancel()
failure = pump.exception() if pump.done() and not pump.cancelled() else None
if failure is not None:
logger.error(
@@ -616,6 +625,53 @@ async def connection_socket(
_SOCKET_OUTBOX = 64
+class _Outbox:
+ """What the socket's one writer sends next.
+
+ The stream's frames wait here, at most `size` of them, so the stream is
+ read only as fast as the socket takes them. A call's answer never waits
+ for room, as there are no more of them than calls in flight, and goes out
+ before any frame still waiting, so a backlog of events does not hold a
+ call up.
+ """
+
+ def __init__(self, size: int) -> None:
+ self._size = size
+ self._frames: deque[ControllerFrame] = deque()
+ self._answers: deque[ControllerFrame] = deque()
+ self._waiting = asyncio.Event()
+ self._room = asyncio.Event()
+ self._room.set()
+
+ def empty(self) -> bool:
+ return not self._frames and not self._answers
+
+ async def put_frame(self, frame: ControllerFrame) -> None:
+ while len(self._frames) >= self._size:
+ self._room.clear()
+ await self._room.wait()
+ self._frames.append(frame)
+ self._waiting.set()
+
+ def put_answer(self, answer: ControllerFrame) -> None:
+ self._answers.append(answer)
+ self._waiting.set()
+
+ async def get(self, timeout: float) -> ControllerFrame | None:
+ """The next frame to send, or None once `timeout` passes with none."""
+ if self.empty():
+ self._waiting.clear()
+ try:
+ await asyncio.wait_for(self._waiting.wait(), timeout)
+ except TimeoutError:
+ return None
+ if self._answers:
+ return self._answers.popleft()
+ frame = self._frames.popleft()
+ self._room.set()
+ return frame
+
+
def _socket_principal(websocket: WebSocket, controller_id: str) -> ControllerPrincipal:
"""The controller the bearer middleware authenticated, checked against the
path and the protocol it declares, as the HTTP routes check theirs."""
@@ -630,34 +686,50 @@ def _socket_principal(websocket: WebSocket, controller_id: str) -> ControllerPri
return require_controller(controller_id, get_controller_principal(websocket))
-async def _pump(
- frames: AsyncGenerator[ControllerFrame], outbox: asyncio.Queue[ControllerFrame]
-) -> None:
+async def _pump(frames: AsyncGenerator[ControllerFrame], outbox: _Outbox) -> None:
# Closed here: cancelled while waiting on the outbox, the stream would
# otherwise detach only when the generator is garbage collected.
try:
async for frame in frames:
- await outbox.put(frame)
+ await outbox.put_frame(frame)
finally:
await frames.aclose()
-async def _receive_pongs(
+async def _receive(
websocket: WebSocket,
+ *,
principal: ControllerPrincipal,
protocol: AgentCore,
+ authenticator: ManagementAuthenticator,
connection_id: str,
generation: int,
+ outbox: _Outbox,
) -> ControllerConnectionError | None:
- """Turn each pong into a beat, until the controller goes or a beat is
- refused. Returns the refusal, so the socket can say which before it
- closes; None when the controller simply went away."""
+ """Turn each pong into a beat and start answering each call, until the
+ controller goes or a beat is refused. Returns the refusal, so the socket
+ can say which before it closes; None when the controller simply went
+ away."""
+ in_flight: set[asyncio.Task[None]] = set()
while True:
try:
message = await websocket.receive_json()
except (WebSocketDisconnect, ValueError, RuntimeError):
return None
- if not isinstance(message, dict) or message.get("type") != "pong":
+ if not isinstance(message, dict):
+ continue
+ kind = message.get("type")
+ if kind == socket_calls.CALL:
+ _start_call(
+ message,
+ in_flight=in_flight,
+ outbox=outbox,
+ principal=principal,
+ protocol=protocol,
+ authenticator=authenticator,
+ )
+ continue
+ if kind != "pong":
continue
try:
await record_beat(
@@ -669,3 +741,48 @@ async def _receive_pongs(
)
except ControllerConnectionError as refusal:
return refusal
+
+
+# Every call still running, whatever socket it came on. The event loop keeps
+# only a weak reference to a task, and a call outlives its socket when the
+# controller goes before it is answered.
+_running_calls: set[asyncio.Task[None]] = set()
+
+
+def _start_call(
+ message: dict[str, Any],
+ *,
+ in_flight: set[asyncio.Task[None]],
+ outbox: _Outbox,
+ principal: ControllerPrincipal,
+ protocol: AgentCore,
+ authenticator: ManagementAuthenticator,
+) -> None:
+ """Answer a call from a task of its own, so that a slow call holds up
+ neither the others nor the beat."""
+ id_ = socket_calls.call_id(message)
+ if id_ is None:
+ logger.warning(
+ "Controller %s sent a call with no id to answer it by; dropped it",
+ principal.controller_id,
+ )
+ return
+ if len(in_flight) >= socket_calls.MAX_CALLS_IN_FLIGHT:
+ outbox.put_answer(socket_calls.capacity_exceeded(id_))
+ return
+
+ async def run() -> None:
+ outbox.put_answer(
+ await socket_calls.answer(
+ message,
+ id_,
+ principal=principal,
+ protocol=protocol,
+ authenticator=authenticator,
+ )
+ )
+
+ task = asyncio.create_task(run())
+ for tasks in (in_flight, _running_calls):
+ tasks.add(task)
+ task.add_done_callback(tasks.discard)
diff --git a/core/switch_core/management/socket_calls.py b/core/switch_core/management/socket_calls.py
new file mode 100644
index 000000000..a48d49c14
--- /dev/null
+++ b/core/switch_core/management/socket_calls.py
@@ -0,0 +1,217 @@
+"""Agent operations a controller calls over its socket.
+
+A controller relays every call its agents' sessions make. Over HTTP each of
+those is a request of its own: parsed, routed, and authenticated with the
+controller's access token. The controller's connection already holds one
+authenticated WebSocket to this server, so it can send the operations on it
+instead (`POST /agents/{id}/ops/{operation}` is the HTTP door to the same
+code):
+
+ {"type": "call", "id": "c-17", "agent_id": "...", "operation": "post_message",
+ "arguments": {"body": "..."}, "room_id": "...", "request_id": "..."}
+
+and each is answered on the socket with the status and the body the HTTP
+route would have answered with:
+
+ {"event": "call.result", "data": {"id": "c-17", "status": 200,
+ "body": {"result": {...}}}}
+
+A call is authorized exactly as the same operation over HTTP with the
+controller's token: its controller is checked against its row (from the
+controller-token cache when it holds it), so a revoked controller is refused,
+and the agent must be bound to that controller now. The calling session's
+room is `room_id`, where an HTTP call sends `X-Switch-Room-Id`.
+
+The server says it takes calls in the socket's first frame (`calls`, with how
+many may be in flight at once). A call over that number is answered with 429
+and `capacity_exceeded` without running, so the controller can send it over
+HTTP instead. Calls run concurrently, and one still running when its socket
+closes runs to its end, as an HTTP request does when its client goes away;
+only its answer is lost.
+"""
+
+from __future__ import annotations
+
+import logging
+from dataclasses import dataclass
+from typing import Any
+
+from fastapi import HTTPException
+from pydantic import TypeAdapter
+
+from switch_core.bridges.agent.api.operations import (
+ call_operation,
+ refusal_status,
+ resolve_controller_caller,
+)
+from switch_core.bridges.agent.auth import ControllerAuthError, ControllerPrincipal
+from switch_core.bridges.agent.protocol.agent_connections import MAX_ID_LENGTH
+from switch_core.bridges.agent.protocol.agent_core import AgentCore
+from switch_core.logging_context import log_context
+from switch_core.management import reason_codes
+from switch_core.management.auth import ManagementAuthenticator
+from switch_core.management.errors import error_body
+from switch_core.request_context import sanitised_request_id
+from switch_core.tenant_context import tenant_scope
+
+logger = logging.getLogger(__name__)
+
+CALL = "call"
+CALL_RESULT = "call.result"
+
+# Calls one socket may have running at once. Told to the controller in the
+# socket's first frame; one more is refused without running.
+MAX_CALLS_IN_FLIGHT = 64
+
+_MAX_CALL_ID_LENGTH = 64
+
+# How the HTTP route encodes its answer: FastAPI serializes a route's return
+# value with Pydantic for its declared type, `dict[str, Any]` there.
+_BODY: TypeAdapter[dict[str, Any]] = TypeAdapter(dict[str, Any])
+
+
+@dataclass(frozen=True)
+class Call:
+ id: str | int
+ agent_id: str
+ operation: str
+ arguments: dict[str, Any]
+ room_id: str | None
+ request_id: str
+
+
+class _Malformed(Exception):
+ """The call cannot be run as sent; answered with 400."""
+
+
+def call_id(message: dict[str, Any]) -> str | int | None:
+ """The id to answer a call frame with, or None when it has none it can be
+ answered by."""
+ raw = message.get("id")
+ if isinstance(raw, bool):
+ return None
+ if isinstance(raw, int):
+ return raw
+ if isinstance(raw, str) and 0 < len(raw) <= _MAX_CALL_ID_LENGTH:
+ return raw
+ return None
+
+
+def calls_offered() -> dict[str, Any]:
+ """What the socket's first frame says about calls."""
+ return {"max_in_flight": MAX_CALLS_IN_FLIGHT}
+
+
+def capacity_exceeded(id_: str | int) -> dict[str, Any]:
+ """The answer to a call over `MAX_CALLS_IN_FLIGHT`, which did not run."""
+ return _result(
+ id_,
+ 429,
+ error_body(
+ reason_codes.CAPACITY_EXCEEDED,
+ f"This connection already has {MAX_CALLS_IN_FLIGHT} calls in flight; "
+ "send this one over HTTP or once one is answered.",
+ retryable=True,
+ ),
+ )
+
+
+async def answer(
+ message: dict[str, Any],
+ id_: str | int,
+ *,
+ principal: ControllerPrincipal,
+ protocol: AgentCore,
+ authenticator: ManagementAuthenticator,
+) -> dict[str, Any]:
+ """Run one call a controller sent on its socket, and return the frame
+ that answers it."""
+ try:
+ call = _parse(message, id_)
+ except _Malformed as exc:
+ return _result(id_, 400, {"detail": str(exc)})
+
+ try:
+ controller = await authenticator.current(principal)
+ except ControllerAuthError as exc:
+ return _result(id_, 401, error_body(exc.code, exc.message, retryable=False))
+ binding = authenticator.presence.binding(call.agent_id)
+ if (
+ binding is None
+ or binding.controller_id != controller.controller_id
+ or binding.tenant_id != controller.tenant_id
+ ):
+ return _result(
+ id_,
+ 403,
+ error_body(
+ reason_codes.NOT_ASSIGNED,
+ f"Agent {call.agent_id} is not assigned to this controller.",
+ retryable=False,
+ ),
+ )
+
+ with (
+ tenant_scope(controller.tenant_id),
+ log_context(
+ request_id=call.request_id,
+ agent_id=call.agent_id,
+ tenant_id=controller.tenant_id,
+ ),
+ ):
+ try:
+ session_key, caller = await resolve_controller_caller(
+ agent_id=call.agent_id, protocol=protocol, room_id=call.room_id
+ )
+ result = await call_operation(
+ operation=call.operation,
+ arguments=call.arguments,
+ agent_id=call.agent_id,
+ connection_id=session_key,
+ session=caller,
+ )
+ except HTTPException as exc:
+ return _result(id_, exc.status_code, {"detail": exc.detail})
+ except Exception as exc:
+ status = refusal_status(exc)
+ if status is None:
+ logger.exception(
+ "Operation %s failed on a controller's socket", call.operation
+ )
+ return _result(id_, 500, {"detail": "Internal Server Error"})
+ return _result(id_, status, {"detail": str(exc)})
+ return _result(id_, 200, _BODY.dump_python({"result": result}, mode="json"))
+
+
+def _parse(message: dict[str, Any], id_: str | int) -> Call:
+ agent_id = message.get("agent_id")
+ if not isinstance(agent_id, str) or not 0 < len(agent_id) <= MAX_ID_LENGTH:
+ raise _Malformed("A call names the agent it is made as in agent_id.")
+ operation = message.get("operation")
+ if not isinstance(operation, str) or not operation:
+ raise _Malformed("A call names its operation in operation.")
+ arguments = message.get("arguments", {})
+ if arguments is None:
+ arguments = {}
+ if not isinstance(arguments, dict):
+ raise _Malformed("A call's arguments are an object.")
+ room_id = message.get("room_id")
+ if room_id is not None and (
+ not isinstance(room_id, str) or not 0 < len(room_id) <= MAX_ID_LENGTH
+ ):
+ raise _Malformed("A call's room_id is a room id, or absent.")
+ request_id = message.get("request_id")
+ return Call(
+ id=id_,
+ agent_id=agent_id,
+ operation=operation,
+ arguments=arguments,
+ room_id=room_id,
+ request_id=sanitised_request_id(
+ request_id if isinstance(request_id, str) else ""
+ ),
+ )
+
+
+def _result(id_: str | int, status: int, body: Any) -> dict[str, Any]:
+ return {"event": CALL_RESULT, "data": {"id": id_, "status": status, "body": body}}
diff --git a/core/switch_core/request_context.py b/core/switch_core/request_context.py
index b393b71ee..579fb80bf 100644
--- a/core/switch_core/request_context.py
+++ b/core/switch_core/request_context.py
@@ -76,7 +76,12 @@ def _header(headers: dict[bytes, bytes], name: bytes) -> str:
def _request_id(headers: dict[bytes, bytes]) -> str:
- raw = _header(headers, _REQUEST_ID_HEADER)
+ return sanitised_request_id(_header(headers, _REQUEST_ID_HEADER))
+
+
+def sanitised_request_id(raw: str) -> str:
+ """A request id a caller sent, made safe to log, or a new one when nothing
+ is left of it."""
sanitised = "".join(c for c in raw if c.isprintable() and c not in " \t")
return sanitised[:_MAX_REQUEST_ID_LENGTH] or uuid4().hex
diff --git a/core/tests/switch_core/fixtures/agent_controllers/stream_frames.json b/core/tests/switch_core/fixtures/agent_controllers/stream_frames.json
index c04cefa06..165f1e16a 100644
--- a/core/tests/switch_core/fixtures/agent_controllers/stream_frames.json
+++ b/core/tests/switch_core/fixtures/agent_controllers/stream_frames.json
@@ -7,7 +7,10 @@
"report_within_s": 60,
"connection_id": "00000000-0000-4000-8000-0000000000e1",
"generation": 3141592653,
- "heartbeat_interval_s": 2.0
+ "heartbeat_interval_s": 2.0,
+ "calls": {
+ "max_in_flight": 64
+ }
}
},
{
diff --git a/core/tests/switch_core/management/test_connection_socket.py b/core/tests/switch_core/management/test_connection_socket.py
index dc31807ff..8bc61526e 100644
--- a/core/tests/switch_core/management/test_connection_socket.py
+++ b/core/tests/switch_core/management/test_connection_socket.py
@@ -1,10 +1,11 @@
"""The controller's connection over one WebSocket: the stream's frames down,
-pongs up as beats.
+pongs up as beats, and its agents' calls up and their answers down.
-What the stream carries, and what a beat does, are tested against Postgres in
-`test_stream.py` through the same `attach_stream` and `record_beat` the socket
-calls. These cover what is particular to the socket: framing, the ping the
-server drives, and how refusals and evictions are told.
+What the stream carries, what a beat does and what a call answers are tested
+against Postgres in `test_stream.py` and `test_socket_calls.py`, through the
+same `attach_stream`, `record_beat` and `socket_calls.answer` the socket uses.
+These cover what is particular to the socket: framing, the ping the server
+drives, how refusals and evictions are told, and how calls share the socket.
"""
from __future__ import annotations
@@ -25,8 +26,9 @@ from switch_core.bridges.agent.protocol.controller_presence import (
ControllerTakenOverError,
UnknownControllerConnectionError,
)
-from switch_core.management import controller_routes
+from switch_core.management import controller_routes, socket_calls
from switch_core.management.dependencies import (
+ get_authenticator,
get_management,
get_management_session_factory,
)
@@ -57,6 +59,9 @@ class _Calls:
self.attach_refusal: Exception | None = None
self.beat_refusal: Exception | None = None
self.beats: list[dict[str, Any]] = []
+ self.answered: list[dict[str, Any]] = []
+ # Released by a test to let the calls to `slow` finish.
+ self.slow = asyncio.Event()
@pytest.fixture
@@ -88,11 +93,28 @@ def client(calls: _Calls, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClien
raise calls.beat_refusal
return ["a1"]
+ async def answer(
+ message: dict[str, Any], id_: str | int, **kwargs: Any
+ ) -> dict[str, Any]:
+ calls.answered.append(message)
+ if message["operation"] == "slow":
+ await calls.slow.wait()
+ return {
+ "event": "call.result",
+ "data": {
+ "id": id_,
+ "status": 200,
+ "body": {"result": message["operation"]},
+ },
+ }
+
monkeypatch.setattr(controller_routes, "attach_stream", attach_stream)
monkeypatch.setattr(controller_routes, "record_beat", record_beat)
+ monkeypatch.setattr(socket_calls, "answer", answer)
app = FastAPI()
app.include_router(controller_routes.router)
app.dependency_overrides[get_protocol] = lambda: None
+ app.dependency_overrides[get_authenticator] = lambda: None
app.dependency_overrides[get_management] = lambda: None
app.dependency_overrides[get_management_session_factory] = lambda: None
with TestClient(_AuthenticatedAs(app)) as test_client:
@@ -190,3 +212,102 @@ def test_another_controllers_socket_is_forbidden(client: TestClient) -> None:
assert refused["data"]["detail"]["code"] == "forbidden"
assert closed.value.code == 4403
+
+
+def _call(id_: Any, operation: str = "post_message") -> dict[str, Any]:
+ return {
+ "type": "call",
+ "id": id_,
+ "agent_id": "a1",
+ "operation": operation,
+ "arguments": {"body": "hi"},
+ "room_id": "r1",
+ }
+
+
+def test_a_call_is_answered_on_the_socket_by_its_id(
+ client: TestClient, calls: _Calls
+) -> None:
+ with client.websocket_connect(PATH) as socket:
+ socket.send_json(_call("c-1"))
+ answer = _until(socket, "call.result")
+
+ assert answer == {
+ "event": "call.result",
+ "data": {"id": "c-1", "status": 200, "body": {"result": "post_message"}},
+ }
+ assert calls.answered == [_call("c-1")]
+
+
+def test_a_slow_call_holds_up_neither_another_call_nor_the_beat(
+ client: TestClient, calls: _Calls
+) -> None:
+ with client.websocket_connect(PATH) as socket:
+ socket.send_json(_call(1, "slow"))
+ socket.send_json(_call(2))
+ first = _until(socket, "call.result")
+ socket.send_json({"type": "pong", "cursors": {"a1": 3}})
+ _until(socket, "ping")
+ _until(socket, "ping")
+ beats = len(calls.beats)
+ calls.slow.set()
+ second = _until(socket, "call.result")
+
+ assert first["data"]["id"] == 2
+ assert beats == 1
+ assert second["data"]["id"] == 1
+
+
+def test_a_call_over_the_limit_is_refused_without_running(
+ client: TestClient, calls: _Calls, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ monkeypatch.setattr(socket_calls, "MAX_CALLS_IN_FLIGHT", 1)
+ with client.websocket_connect(PATH) as socket:
+ socket.send_json(_call(1, "slow"))
+ socket.send_json(_call(2))
+ refused = _until(socket, "call.result")
+ calls.slow.set()
+ answered = _until(socket, "call.result")
+
+ assert refused["data"]["id"] == 2
+ assert refused["data"]["status"] == 429
+ assert refused["data"]["body"]["error"]["code"] == "capacity_exceeded"
+ assert refused["data"]["body"]["error"]["retryable"] is True
+ assert answered["data"] == {"id": 1, "status": 200, "body": {"result": "slow"}}
+ assert [call["id"] for call in calls.answered] == [1]
+
+
[email protected]("id_", [None, True, "", "x" * 65, {"id": 1}])
+def test_a_call_with_no_id_to_answer_by_is_dropped(
+ client: TestClient, calls: _Calls, id_: Any
+) -> None:
+ with client.websocket_connect(PATH) as socket:
+ socket.send_json(_call(id_))
+ socket.send_json(_call("after"))
+ answer = _until(socket, "call.result")
+
+ assert answer["data"]["id"] == "after"
+ assert [call["id"] for call in calls.answered] == ["after"]
+
+
+async def test_an_answer_goes_out_before_the_frames_waiting() -> None:
+ outbox = controller_routes._Outbox(2)
+ await outbox.put_frame({"event": "agent.event", "data": {"n": 1}})
+ await outbox.put_frame({"event": "agent.event", "data": {"n": 2}})
+ third = asyncio.create_task(
+ outbox.put_frame({"event": "agent.event", "data": {"n": 3}})
+ )
+ await asyncio.sleep(0)
+ outbox.put_answer({"event": "call.result", "data": {"id": 1}})
+
+ sent = [await outbox.get(timeout=1) for _ in range(4)]
+ await third
+
+ assert not third.cancelled()
+ assert [frame["event"] if frame else None for frame in sent] == [
+ "call.result",
+ "agent.event",
+ "agent.event",
+ "agent.event",
+ ]
+ assert await outbox.get(timeout=0.01) is None
diff --git a/core/tests/switch_core/management/test_socket_calls.py b/core/tests/switch_core/management/test_socket_calls.py
new file mode 100644
index 000000000..b3af9473c
--- /dev/null
+++ b/core/tests/switch_core/management/test_socket_calls.py
@@ -0,0 +1,220 @@
+"""An operation a controller calls on its socket answers what the same call
+over HTTP with the controller's token answers, and is authorized the same way.
+
+Each case is sent through both doors, the HTTP one through the real bearer
+middleware, and the two answers compared. Against Postgres.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterator
+from datetime import UTC, datetime
+from typing import Any
+
+import pytest
+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker
+
+from switch_core.bridges.agent.auth import ControllerPrincipal
+from switch_core.bridges.agent.operations import context as op_context
+from switch_core.bridges.agent.operations import registry
+from switch_core.db.models import TENANT_ZERO_ID
+from switch_core.management import socket_calls
+from tests.switch_core.management.harness import (
+ EnrolledController,
+ Harness,
+ add_member,
+ add_room,
+ build_harness,
+ cookies_for,
+ enroll_console,
+ place_agent,
+)
+
+
+def harness(session_factory: async_sessionmaker[AsyncSession]) -> Harness:
+ return build_harness(session_factory)
+
+
+def where_am_i(harness: Harness) -> Iterator[None]:
+ """An operation that answers with the caller's room, or refuses."""
+
+ async def where_am_i(refuse: bool = False, stamped: bool = False) -> dict[str, Any]:
+ if refuse:
+ raise PermissionError("not here")
+ answer: dict[str, Any] = {"room_id": await op_context.require_connected_room()}
+ if stamped:
+ # Not JSON as it is: encoded for the socket as the route encodes it.
+ answer["at"] = datetime(2026, 11, 1, 12, 30, tzinfo=UTC)
+ return answer
+
+ registry._REGISTRY["where_am_i"] = registry.Operation(
+ name="where_am_i",
+ fn=where_am_i,
+ description="test",
+ input_schema=registry._input_schema(where_am_i),
+ )
+ previous = op_context._protocol
+ op_context.init_operations_protocol(harness.protocol)
+ try:
+ yield
+ finally:
+ registry._REGISTRY.pop("where_am_i", None)
+ op_context._protocol = previous
+
+
+def _principal(controller: EnrolledController) -> ControllerPrincipal:
+ return ControllerPrincipal(
+ controller_id=controller.controller_id,
+ owner_id=controller.owner.id,
+ tenant_id=TENANT_ZERO_ID,
+ )
+
+
+async def _both(
+ harness: Harness,
+ controller: EnrolledController,
+ agent_id: str,
+ operation: str,
+ arguments: dict[str, Any],
+ room_id: str | None,
+) -> tuple[tuple[int, Any], tuple[int, Any]]:
+ """The answer over HTTP and the answer on the socket, each as (status, body)."""
+ headers = dict(controller.headers)
+ if room_id is not None:
+ headers["X-Switch-Room-Id"] = room_id
+ async with harness.client() as client:
+ response = await client.post(
+ f"/agents/{agent_id}/ops/{operation}", json=arguments, headers=headers
+ )
+ answer = await socket_calls.answer(
+ {
+ "type": "call",
+ "id": "c-1",
+ "agent_id": agent_id,
+ "operation": operation,
+ "arguments": arguments,
+ "room_id": room_id,
+ },
+ "c-1",
+ principal=_principal(controller),
+ protocol=harness.protocol,
+ authenticator=harness.management.authenticator,
+ )
+ assert answer["event"] == "call.result"
+ assert answer["data"]["id"] == "c-1"
+ return (response.status_code, response.json()), (
+ answer["data"]["status"],
+ answer["data"]["body"],
+ )
+
+
+class TestACallAnswersWhatHttpAnswers:
+ async def test_for_each_outcome_of_an_operation(
+ self, harness: Harness, where_am_i: None
+ ) -> None:
+ owner = await add_member(harness.session_factory, "ada")
+ async with harness.client() as client:
+ controller = await enroll_console(harness, client, owner)
+ agent_id = await place_agent(client, controller, name="reviewer")
+ member_room = await add_room(harness.session_factory, agent_id)
+ other_room = await add_room(harness.session_factory, name="elsewhere")
+
+ cases: list[tuple[str, dict[str, Any], str | None]] = [
+ ("where_am_i", {}, member_room),
+ ("where_am_i", {"stamped": True}, member_room),
+ ("where_am_i", {}, other_room),
+ ("where_am_i", {}, "no-such-room"),
+ ("where_am_i", {}, None),
+ ("where_am_i", {"refuse": True}, member_room),
+ ("where_am_i", {"unexpected": 1}, member_room),
+ ("no_such_operation", {}, member_room),
+ ]
+ statuses = []
+ for operation, arguments, room_id in cases:
+ over_http, on_socket = await _both(
+ harness, controller, agent_id, operation, arguments, room_id
+ )
+ assert on_socket == over_http, (operation, arguments, room_id)
+ statuses.append(on_socket[0])
+
+ assert statuses == [200, 200, 403, 404, 400, 403, 400, 404]
+
+
+class TestACallIsAuthorizedAsTheControllersRequestIs:
+ async def test_an_agent_bound_to_another_controller_is_not_assigned(
+ self, harness: Harness, where_am_i: None
+ ) -> None:
+ owner = await add_member(harness.session_factory, "ada")
+ async with harness.client() as client:
+ mine = await enroll_console(harness, client, owner, "mine")
+ theirs = await enroll_console(harness, client, owner, "theirs")
+ agent_id = await place_agent(client, theirs, name="theirs-agent")
+
+ over_http, on_socket = await _both(
+ harness, mine, agent_id, "where_am_i", {}, None
+ )
+
+ assert on_socket == over_http
+ assert on_socket[0] == 403
+ assert on_socket[1]["error"]["code"] == "not_assigned"
+
+ async def test_a_revoked_controller_is_refused(
+ self, harness: Harness, where_am_i: None
+ ) -> None:
+ owner = await add_member(harness.session_factory, "ada")
+ async with harness.client() as client:
+ controller = await enroll_console(harness, client, owner)
+ agent_id = await place_agent(client, controller, name="reviewer")
+ room_id = await add_room(harness.session_factory, agent_id)
+ await client.delete(
+ f"/gateway/management/controllers/{controller.controller_id}",
+ cookies=cookies_for(owner),
+ )
+
+ over_http, on_socket = await _both(
+ harness, controller, agent_id, "where_am_i", {}, room_id
+ )
+
+ assert on_socket == over_http
+ assert on_socket[0] == 401
+ assert on_socket[1]["error"]["code"] == "controller_revoked"
+
+
+ ("change", "detail"),
+ [
+ ({"agent_id": None}, "agent_id"),
+ ({"agent_id": ""}, "agent_id"),
+ ({"operation": 7}, "operation"),
+ ({"arguments": ["body"]}, "arguments"),
+ ({"room_id": 3}, "room_id"),
+ ],
+)
+async def test_a_call_that_cannot_be_run_as_sent_is_a_400(
+ harness: Harness, change: dict[str, Any], detail: str
+) -> None:
+ owner = await add_member(harness.session_factory, "ada")
+ async with harness.client() as client:
+ controller = await enroll_console(harness, client, owner)
+ agent_id = await place_agent(client, controller, name="reviewer")
+ call = {
+ "type": "call",
+ "id": "c-1",
+ "agent_id": agent_id,
+ "operation": "where_am_i",
+ "arguments": {},
+ **change,
+ }
+
+ answer = await socket_calls.answer(
+ call,
+ "c-1",
+ principal=_principal(controller),
+ protocol=harness.protocol,
+ authenticator=harness.management.authenticator,
+ )
+
+ assert answer["data"]["status"] == 400
+ assert detail in answer["data"]["body"]["detail"]
diff --git a/core/tests/switch_core/management/test_stream.py b/core/tests/switch_core/management/test_stream.py
index 2e4e474cb..155e63e8c 100644
--- a/core/tests/switch_core/management/test_stream.py
+++ b/core/tests/switch_core/management/test_stream.py
@@ -30,7 +30,7 @@ from switch_core.db.stores.agent_store import AgentStore
from switch_core.db.stores.room_role_store import RoomRoleStore
from switch_core.db.stores.room_store import RoomStore
from switch_core.db.stores.user_store import UserStore
-from switch_core.management import controller_routes
+from switch_core.management import controller_routes, socket_calls
from switch_core.management.notifier import (
ASSIGNMENT_CHANGED,
CREDENTIAL_REVOKED,
@@ -159,6 +159,7 @@ class TestOpening:
"connection_id": opened["connection_id"],
"generation": opened["generation"],
"heartbeat_interval_s": HEARTBEAT_INTERVAL_SECONDS,
+ "calls": {"max_in_flight": socket_calls.MAX_CALLS_IN_FLIGHT},
},
)
assert attached == (