Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
Let a controller send its agents' operations on its socket, not over HTTP (CHOO-3667)
Abel Dantas7ede3123d783
diff --git a/core/switch_core/bridges/agent/api/operations.py b/core/switch_core/bridges/agent/api/operations.pyindex 6155b1907..3e8265f37 100644--- a/core/switch_core/bridges/agent/api/operations.py+++ b/core/switch_core/bridges/agent/api/operations.py@@ -63,6 +63,26 @@ class BadArgumentsError(Exception):"""The body did not match the operation's parameters."""+def refusal_status(exc: Exception) -> int | None:+ """The status an operation's refusal is answered with, over HTTP or on a+ controller's socket; None for an exception that is a failure instead."""+ if isinstance(exc, UnknownOperationError):+ return 404+ if isinstance(exc, BadArgumentsError):+ return 400+ if isinstance(exc, BudgetExceeded):+ return 429+ if isinstance(exc, GuardrailBlockedError):+ return 422+ if isinstance(exc, PermissionError):+ return 403+ if isinstance(exc, ValueError):+ # Operations raise ValueError for "you asked for something that is not+ # there or not allowed yet" - surfaced rather than swallowed.+ return 400+ return None++def list_operations() -> dict[str, dict[str, Any]]:"""Every operation, with its parameters.@@ -327,19 +347,10 @@ async def post_operation(connection_id=session_key,session=caller,)- except UnknownOperationError as exc:- raise HTTPException(status_code=404, detail=str(exc)) from exc- except BadArgumentsError as exc:- raise HTTPException(status_code=400, detail=str(exc)) from exc- except BudgetExceeded as exc:- raise HTTPException(status_code=429, detail=str(exc)) from exc- except GuardrailBlockedError as exc:- raise HTTPException(status_code=422, detail=str(exc)) from exc- except PermissionError as exc:- raise HTTPException(status_code=403, detail=str(exc)) from exc- except ValueError as exc:- # Operations raise ValueError for "you asked for something that is not- # there or not allowed yet" - surfaced rather than swallowed.- raise HTTPException(status_code=400, detail=str(exc)) from exc+ except Exception as exc:+ status = refusal_status(exc)+ if status is None:+ raise+ raise HTTPException(status_code=status, detail=str(exc)) from excreturn {"result": result}diff --git a/core/switch_core/management/auth.py b/core/switch_core/management/auth.pyindex bf960e5ec..41498defb 100644--- a/core/switch_core/management/auth.py+++ b/core/switch_core/management/auth.py@@ -87,17 +87,33 @@ class ManagementAuthenticator:reason_codes.INVALID_CREDENTIAL,"The access token is not a valid controller token.",) from exc+ return await self.current(+ ControllerPrincipal(+ controller_id=claims.controller_id,+ owner_id=claims.owner_id,+ tenant_id=claims.tenant_id,+ )+ )- cached = self._auth_cache.controller(claims.tenant_id, claims.controller_id)- if cached is not None and cached.owner_id == claims.owner_id:+ async def current(self, principal: ControllerPrincipal) -> ControllerPrincipal:+ """The controller `principal` names, while it keeps the authority its+ token gave it: not revoked, and owned by the same person. Every request+ made with a controller access token is checked here, and every call on+ a controller's socket, which authenticated once when it opened."""+ cached = self._auth_cache.controller(+ principal.tenant_id, principal.controller_id+ )+ if cached is not None and cached.owner_id == principal.owner_id:return cachedgeneration = self._auth_cache.generation- async with tenant_session(self._session_factory, claims.tenant_id) as session:+ async with tenant_session(+ self._session_factory, principal.tenant_id+ ) as session:controller = await self._controllers.get(- session, claims.tenant_id, claims.controller_id+ session, principal.tenant_id, principal.controller_id)- if controller is None or controller.owner_id != claims.owner_id:+ if controller is None or controller.owner_id != principal.owner_id:raise ControllerAuthError(reason_codes.INVALID_CREDENTIAL,"The access token names no controller.",@@ -106,13 +122,13 @@ class ManagementAuthenticator:raise ControllerAuthError(reason_codes.CONTROLLER_REVOKED, "The controller has been revoked.")- principal = ControllerPrincipal(+ current = ControllerPrincipal(controller_id=controller.id,owner_id=controller.owner_id,- tenant_id=claims.tenant_id,+ tenant_id=principal.tenant_id,)- self._auth_cache.put_controller(principal, generation)- return principal+ self._auth_cache.put_controller(current, generation)+ return currentasync def tenant_of_secret(self, secret: str) -> str | None:"""The tenant an enrollment code or controller credential belongs to,diff --git a/core/switch_core/management/controller_routes.py b/core/switch_core/management/controller_routes.pyindex aa5da5d8e..030ce385e 100644--- a/core/switch_core/management/controller_routes.py+++ b/core/switch_core/management/controller_routes.py@@ -23,6 +23,7 @@ import asyncioimport hashlibimport jsonimport logging+from collections import dequefrom collections.abc import AsyncGenerator, Awaitable, Callable, Coroutinefrom typing import Annotated, Any@@ -53,7 +54,7 @@ from switch_core.bridges.agent.protocol.controller_stream import ()from switch_core.bridges.agent.protocol.liveness import HEARTBEAT_INTERVAL_SECONDSfrom switch_core.db.session_scope import tenant_session-from switch_core.management import reason_codes+from switch_core.management import reason_codes, socket_callsfrom switch_core.management.auth import ManagementAuthenticatorfrom switch_core.management.dependencies import (get_authenticator,@@ -444,6 +445,8 @@ async def attach_stream("connection_id": conn.id,"generation": conn.generation,"heartbeat_interval_s": HEARTBEAT_INTERVAL_SECONDS,+ # The socket takes its agents' operations as calls (`socket_calls`).+ "calls": socket_calls.calls_offered(),}return controller_event_stream(conn=conn,@@ -527,6 +530,7 @@ async def connection_socket(controller_id: str,management: Management,protocol: Protocol,+ authenticator: Annotated[ManagementAuthenticator, Depends(get_authenticator)],session_factory: Annotated[async_sessionmaker[AsyncSession], Depends(get_management_session_factory)],@@ -534,14 +538,16 @@ async def connection_socket(generation: int,) -> None:"""The controller's one connection: its agents' events and the management- nudges down, its heartbeat up, on one WebSocket.+ nudges down, its heartbeat and its agents' calls up, on one WebSocket.Attaches to a connection opened with `POST .../connection`. Frames areJSON `{"event", "data"}`, the events the stream carries. The server sends`{"event": "ping"}` every heartbeat interval and the controller answers- `{"type": "pong", "cursors": {agent_id: n}}`, which is its beat. A refused- attach is a `refused` frame, then a close with 4000 plus the status; a- refused beat is an `evicted` frame naming why, then a close.+ `{"type": "pong", "cursors": {agent_id: n}}`, which is its beat. A+ `{"type": "call", ...}` runs one of its agents' operations and is answered+ with a `call.result` frame (`socket_calls`). A refused attach is a+ `refused` frame, then a close with 4000 plus the status; a refused beat is+ an `evicted` frame naming why, then a close."""await websocket.accept()try:@@ -559,18 +565,26 @@ async def connection_socket(await websocket.close(code=4000 + error.status_code)return- outbox: asyncio.Queue[ControllerFrame] = asyncio.Queue(maxsize=_SOCKET_OUTBOX)+ outbox = _Outbox(_SOCKET_OUTBOX)pump = asyncio.create_task(_pump(frames, outbox))- pongs = asyncio.create_task(- _receive_pongs(websocket, principal, protocol, connection_id, generation)+ receiver = asyncio.create_task(+ _receive(+ websocket,+ principal=principal,+ protocol=protocol,+ authenticator=authenticator,+ connection_id=connection_id,+ generation=generation,+ outbox=outbox,+ ))loop = asyncio.get_running_loop()next_ping = loop.time() + HEARTBEAT_INTERVAL_SECONDStry:- # One writer: frames and pings both go out from this loop.+ # One writer: frames, call answers and pings all go out from this loop.while not pump.done() or not outbox.empty():- if pongs.done():- refusal = pongs.result()+ if receiver.done():+ refusal = receiver.result()if refusal is not None:await websocket.send_json({@@ -579,12 +593,7 @@ async def connection_socket(})return- try:- frame = await asyncio.wait_for(- outbox.get(), timeout=max(0.0, next_ping - loop.time())- )- except TimeoutError:- frame = None+ frame = await outbox.get(timeout=max(0.0, next_ping - loop.time()))if frame is not None and frame is not IDLE:await websocket.send_json(frame)if loop.time() >= next_ping:@@ -597,7 +606,7 @@ async def connection_socket(returnfinally:pump.cancel()- pongs.cancel()+ receiver.cancel()failure = pump.exception() if pump.done() and not pump.cancelled() else Noneif failure is not None:logger.error(@@ -616,6 +625,53 @@ async def connection_socket(_SOCKET_OUTBOX = 64+class _Outbox:+ """What the socket's one writer sends next.++ The stream's frames wait here, at most `size` of them, so the stream is+ read only as fast as the socket takes them. A call's answer never waits+ for room, as there are no more of them than calls in flight, and goes out+ before any frame still waiting, so a backlog of events does not hold a+ call up.+ """++ def __init__(self, size: int) -> None:+ self._size = size+ self._frames: deque[ControllerFrame] = deque()+ self._answers: deque[ControllerFrame] = deque()+ self._waiting = asyncio.Event()+ self._room = asyncio.Event()+ self._room.set()++ def empty(self) -> bool:+ return not self._frames and not self._answers++ async def put_frame(self, frame: ControllerFrame) -> None:+ while len(self._frames) >= self._size:+ self._room.clear()+ await self._room.wait()+ self._frames.append(frame)+ self._waiting.set()++ def put_answer(self, answer: ControllerFrame) -> None:+ self._answers.append(answer)+ self._waiting.set()++ async def get(self, timeout: float) -> ControllerFrame | None:+ """The next frame to send, or None once `timeout` passes with none."""+ if self.empty():+ self._waiting.clear()+ try:+ await asyncio.wait_for(self._waiting.wait(), timeout)+ except TimeoutError:+ return None+ if self._answers:+ return self._answers.popleft()+ frame = self._frames.popleft()+ self._room.set()+ return frame++def _socket_principal(websocket: WebSocket, controller_id: str) -> ControllerPrincipal:"""The controller the bearer middleware authenticated, checked against thepath and the protocol it declares, as the HTTP routes check theirs."""@@ -630,34 +686,50 @@ def _socket_principal(websocket: WebSocket, controller_id: str) -> ControllerPrireturn require_controller(controller_id, get_controller_principal(websocket))-async def _pump(- frames: AsyncGenerator[ControllerFrame], outbox: asyncio.Queue[ControllerFrame]-) -> None:+async def _pump(frames: AsyncGenerator[ControllerFrame], outbox: _Outbox) -> None:# Closed here: cancelled while waiting on the outbox, the stream would# otherwise detach only when the generator is garbage collected.try:async for frame in frames:- await outbox.put(frame)+ await outbox.put_frame(frame)finally:await frames.aclose()-async def _receive_pongs(+async def _receive(websocket: WebSocket,+ *,principal: ControllerPrincipal,protocol: AgentCore,+ authenticator: ManagementAuthenticator,connection_id: str,generation: int,+ outbox: _Outbox,) -> ControllerConnectionError | None:- """Turn each pong into a beat, until the controller goes or a beat is- refused. Returns the refusal, so the socket can say which before it- closes; None when the controller simply went away."""+ """Turn each pong into a beat and start answering each call, until the+ controller goes or a beat is refused. Returns the refusal, so the socket+ can say which before it closes; None when the controller simply went+ away."""+ in_flight: set[asyncio.Task[None]] = set()while True:try:message = await websocket.receive_json()except (WebSocketDisconnect, ValueError, RuntimeError):return None- if not isinstance(message, dict) or message.get("type") != "pong":+ if not isinstance(message, dict):+ continue+ kind = message.get("type")+ if kind == socket_calls.CALL:+ _start_call(+ message,+ in_flight=in_flight,+ outbox=outbox,+ principal=principal,+ protocol=protocol,+ authenticator=authenticator,+ )+ continue+ if kind != "pong":continuetry:await record_beat(@@ -669,3 +741,48 @@ async def _receive_pongs()except ControllerConnectionError as refusal:return refusal+++# Every call still running, whatever socket it came on. The event loop keeps+# only a weak reference to a task, and a call outlives its socket when the+# controller goes before it is answered.+_running_calls: set[asyncio.Task[None]] = set()+++def _start_call(+ message: dict[str, Any],+ *,+ in_flight: set[asyncio.Task[None]],+ outbox: _Outbox,+ principal: ControllerPrincipal,+ protocol: AgentCore,+ authenticator: ManagementAuthenticator,+) -> None:+ """Answer a call from a task of its own, so that a slow call holds up+ neither the others nor the beat."""+ id_ = socket_calls.call_id(message)+ if id_ is None:+ logger.warning(+ "Controller %s sent a call with no id to answer it by; dropped it",+ principal.controller_id,+ )+ return+ if len(in_flight) >= socket_calls.MAX_CALLS_IN_FLIGHT:+ outbox.put_answer(socket_calls.capacity_exceeded(id_))+ return++ async def run() -> None:+ outbox.put_answer(+ await socket_calls.answer(+ message,+ id_,+ principal=principal,+ protocol=protocol,+ authenticator=authenticator,+ )+ )++ task = asyncio.create_task(run())+ for tasks in (in_flight, _running_calls):+ tasks.add(task)+ task.add_done_callback(tasks.discard)diff --git a/core/switch_core/management/socket_calls.py b/core/switch_core/management/socket_calls.pynew file mode 100644index 000000000..a48d49c14--- /dev/null+++ b/core/switch_core/management/socket_calls.py@@ -0,0 +1,217 @@+"""Agent operations a controller calls over its socket.++A controller relays every call its agents' sessions make. Over HTTP each of+those is a request of its own: parsed, routed, and authenticated with the+controller's access token. The controller's connection already holds one+authenticated WebSocket to this server, so it can send the operations on it+instead (`POST /agents/{id}/ops/{operation}` is the HTTP door to the same+code):++ {"type": "call", "id": "c-17", "agent_id": "...", "operation": "post_message",+ "arguments": {"body": "..."}, "room_id": "...", "request_id": "..."}++and each is answered on the socket with the status and the body the HTTP+route would have answered with:++ {"event": "call.result", "data": {"id": "c-17", "status": 200,+ "body": {"result": {...}}}}++A call is authorized exactly as the same operation over HTTP with the+controller's token: its controller is checked against its row (from the+controller-token cache when it holds it), so a revoked controller is refused,+and the agent must be bound to that controller now. The calling session's+room is `room_id`, where an HTTP call sends `X-Switch-Room-Id`.++The server says it takes calls in the socket's first frame (`calls`, with how+many may be in flight at once). A call over that number is answered with 429+and `capacity_exceeded` without running, so the controller can send it over+HTTP instead. Calls run concurrently, and one still running when its socket+closes runs to its end, as an HTTP request does when its client goes away;+only its answer is lost.+"""++from __future__ import annotations++import logging+from dataclasses import dataclass+from typing import Any++from fastapi import HTTPException+from pydantic import TypeAdapter++from switch_core.bridges.agent.api.operations import (+ call_operation,+ refusal_status,+ resolve_controller_caller,+)+from switch_core.bridges.agent.auth import ControllerAuthError, ControllerPrincipal+from switch_core.bridges.agent.protocol.agent_connections import MAX_ID_LENGTH+from switch_core.bridges.agent.protocol.agent_core import AgentCore+from switch_core.logging_context import log_context+from switch_core.management import reason_codes+from switch_core.management.auth import ManagementAuthenticator+from switch_core.management.errors import error_body+from switch_core.request_context import sanitised_request_id+from switch_core.tenant_context import tenant_scope++logger = logging.getLogger(__name__)++CALL = "call"+CALL_RESULT = "call.result"++# Calls one socket may have running at once. Told to the controller in the+# socket's first frame; one more is refused without running.+MAX_CALLS_IN_FLIGHT = 64++_MAX_CALL_ID_LENGTH = 64++# How the HTTP route encodes its answer: FastAPI serializes a route's return+# value with Pydantic for its declared type, `dict[str, Any]` there.+_BODY: TypeAdapter[dict[str, Any]] = TypeAdapter(dict[str, Any])+++@dataclass(frozen=True)+class Call:+ id: str | int+ agent_id: str+ operation: str+ arguments: dict[str, Any]+ room_id: str | None+ request_id: str+++class _Malformed(Exception):+ """The call cannot be run as sent; answered with 400."""+++def call_id(message: dict[str, Any]) -> str | int | None:+ """The id to answer a call frame with, or None when it has none it can be+ answered by."""+ raw = message.get("id")+ if isinstance(raw, bool):+ return None+ if isinstance(raw, int):+ return raw+ if isinstance(raw, str) and 0 < len(raw) <= _MAX_CALL_ID_LENGTH:+ return raw+ return None+++def calls_offered() -> dict[str, Any]:+ """What the socket's first frame says about calls."""+ return {"max_in_flight": MAX_CALLS_IN_FLIGHT}+++def capacity_exceeded(id_: str | int) -> dict[str, Any]:+ """The answer to a call over `MAX_CALLS_IN_FLIGHT`, which did not run."""+ return _result(+ id_,+ 429,+ error_body(+ reason_codes.CAPACITY_EXCEEDED,+ f"This connection already has {MAX_CALLS_IN_FLIGHT} calls in flight; "+ "send this one over HTTP or once one is answered.",+ retryable=True,+ ),+ )+++async def answer(+ message: dict[str, Any],+ id_: str | int,+ *,+ principal: ControllerPrincipal,+ protocol: AgentCore,+ authenticator: ManagementAuthenticator,+) -> dict[str, Any]:+ """Run one call a controller sent on its socket, and return the frame+ that answers it."""+ try:+ call = _parse(message, id_)+ except _Malformed as exc:+ return _result(id_, 400, {"detail": str(exc)})++ try:+ controller = await authenticator.current(principal)+ except ControllerAuthError as exc:+ return _result(id_, 401, error_body(exc.code, exc.message, retryable=False))+ binding = authenticator.presence.binding(call.agent_id)+ if (+ binding is None+ or binding.controller_id != controller.controller_id+ or binding.tenant_id != controller.tenant_id+ ):+ return _result(+ id_,+ 403,+ error_body(+ reason_codes.NOT_ASSIGNED,+ f"Agent {call.agent_id} is not assigned to this controller.",+ retryable=False,+ ),+ )++ with (+ tenant_scope(controller.tenant_id),+ log_context(+ request_id=call.request_id,+ agent_id=call.agent_id,+ tenant_id=controller.tenant_id,+ ),+ ):+ try:+ session_key, caller = await resolve_controller_caller(+ agent_id=call.agent_id, protocol=protocol, room_id=call.room_id+ )+ result = await call_operation(+ operation=call.operation,+ arguments=call.arguments,+ agent_id=call.agent_id,+ connection_id=session_key,+ session=caller,+ )+ except HTTPException as exc:+ return _result(id_, exc.status_code, {"detail": exc.detail})+ except Exception as exc:+ status = refusal_status(exc)+ if status is None:+ logger.exception(+ "Operation %s failed on a controller's socket", call.operation+ )+ return _result(id_, 500, {"detail": "Internal Server Error"})+ return _result(id_, status, {"detail": str(exc)})+ return _result(id_, 200, _BODY.dump_python({"result": result}, mode="json"))+++def _parse(message: dict[str, Any], id_: str | int) -> Call:+ agent_id = message.get("agent_id")+ if not isinstance(agent_id, str) or not 0 < len(agent_id) <= MAX_ID_LENGTH:+ raise _Malformed("A call names the agent it is made as in agent_id.")+ operation = message.get("operation")+ if not isinstance(operation, str) or not operation:+ raise _Malformed("A call names its operation in operation.")+ arguments = message.get("arguments", {})+ if arguments is None:+ arguments = {}+ if not isinstance(arguments, dict):+ raise _Malformed("A call's arguments are an object.")+ room_id = message.get("room_id")+ if room_id is not None and (+ not isinstance(room_id, str) or not 0 < len(room_id) <= MAX_ID_LENGTH+ ):+ raise _Malformed("A call's room_id is a room id, or absent.")+ request_id = message.get("request_id")+ return Call(+ id=id_,+ agent_id=agent_id,+ operation=operation,+ arguments=arguments,+ room_id=room_id,+ request_id=sanitised_request_id(+ request_id if isinstance(request_id, str) else ""+ ),+ )+++def _result(id_: str | int, status: int, body: Any) -> dict[str, Any]:+ return {"event": CALL_RESULT, "data": {"id": id_, "status": status, "body": body}}diff --git a/core/switch_core/request_context.py b/core/switch_core/request_context.pyindex b393b71ee..579fb80bf 100644--- a/core/switch_core/request_context.py+++ b/core/switch_core/request_context.py@@ -76,7 +76,12 @@ def _header(headers: dict[bytes, bytes], name: bytes) -> str:def _request_id(headers: dict[bytes, bytes]) -> str:- raw = _header(headers, _REQUEST_ID_HEADER)+ return sanitised_request_id(_header(headers, _REQUEST_ID_HEADER))+++def sanitised_request_id(raw: str) -> str:+ """A request id a caller sent, made safe to log, or a new one when nothing+ is left of it."""sanitised = "".join(c for c in raw if c.isprintable() and c not in " \t")return sanitised[:_MAX_REQUEST_ID_LENGTH] or uuid4().hexdiff --git a/core/tests/switch_core/fixtures/agent_controllers/stream_frames.json b/core/tests/switch_core/fixtures/agent_controllers/stream_frames.jsonindex c04cefa06..165f1e16a 100644--- a/core/tests/switch_core/fixtures/agent_controllers/stream_frames.json+++ b/core/tests/switch_core/fixtures/agent_controllers/stream_frames.json@@ -7,7 +7,10 @@"report_within_s": 60,"connection_id": "00000000-0000-4000-8000-0000000000e1","generation": 3141592653,- "heartbeat_interval_s": 2.0+ "heartbeat_interval_s": 2.0,+ "calls": {+ "max_in_flight": 64+ }}},{diff --git a/core/tests/switch_core/management/test_connection_socket.py b/core/tests/switch_core/management/test_connection_socket.pyindex dc31807ff..8bc61526e 100644--- a/core/tests/switch_core/management/test_connection_socket.py+++ b/core/tests/switch_core/management/test_connection_socket.py@@ -1,10 +1,11 @@"""The controller's connection over one WebSocket: the stream's frames down,-pongs up as beats.+pongs up as beats, and its agents' calls up and their answers down.-What the stream carries, and what a beat does, are tested against Postgres in-`test_stream.py` through the same `attach_stream` and `record_beat` the socket-calls. These cover what is particular to the socket: framing, the ping the-server drives, and how refusals and evictions are told.+What the stream carries, what a beat does and what a call answers are tested+against Postgres in `test_stream.py` and `test_socket_calls.py`, through the+same `attach_stream`, `record_beat` and `socket_calls.answer` the socket uses.+These cover what is particular to the socket: framing, the ping the server+drives, how refusals and evictions are told, and how calls share the socket."""from __future__ import annotations@@ -25,8 +26,9 @@ from switch_core.bridges.agent.protocol.controller_presence import (ControllerTakenOverError,UnknownControllerConnectionError,)-from switch_core.management import controller_routes+from switch_core.management import controller_routes, socket_callsfrom switch_core.management.dependencies import (+ get_authenticator,get_management,get_management_session_factory,)@@ -57,6 +59,9 @@ class _Calls:self.attach_refusal: Exception | None = Noneself.beat_refusal: Exception | None = Noneself.beats: list[dict[str, Any]] = []+ self.answered: list[dict[str, Any]] = []+ # Released by a test to let the calls to `slow` finish.+ self.slow = asyncio.Event()@pytest.fixture@@ -88,11 +93,28 @@ def client(calls: _Calls, monkeypatch: pytest.MonkeyPatch) -> Iterator[TestClienraise calls.beat_refusalreturn ["a1"]+ async def answer(+ message: dict[str, Any], id_: str | int, **kwargs: Any+ ) -> dict[str, Any]:+ calls.answered.append(message)+ if message["operation"] == "slow":+ await calls.slow.wait()+ return {+ "event": "call.result",+ "data": {+ "id": id_,+ "status": 200,+ "body": {"result": message["operation"]},+ },+ }+monkeypatch.setattr(controller_routes, "attach_stream", attach_stream)monkeypatch.setattr(controller_routes, "record_beat", record_beat)+ monkeypatch.setattr(socket_calls, "answer", answer)app = FastAPI()app.include_router(controller_routes.router)app.dependency_overrides[get_protocol] = lambda: None+ app.dependency_overrides[get_authenticator] = lambda: Noneapp.dependency_overrides[get_management] = lambda: Noneapp.dependency_overrides[get_management_session_factory] = lambda: Nonewith TestClient(_AuthenticatedAs(app)) as test_client:@@ -190,3 +212,102 @@ def test_another_controllers_socket_is_forbidden(client: TestClient) -> None:assert refused["data"]["detail"]["code"] == "forbidden"assert closed.value.code == 4403+++def _call(id_: Any, operation: str = "post_message") -> dict[str, Any]:+ return {+ "type": "call",+ "id": id_,+ "agent_id": "a1",+ "operation": operation,+ "arguments": {"body": "hi"},+ "room_id": "r1",+ }+++def test_a_call_is_answered_on_the_socket_by_its_id(+ client: TestClient, calls: _Calls+) -> None:+ with client.websocket_connect(PATH) as socket:+ socket.send_json(_call("c-1"))+ answer = _until(socket, "call.result")++ assert answer == {+ "event": "call.result",+ "data": {"id": "c-1", "status": 200, "body": {"result": "post_message"}},+ }+ assert calls.answered == [_call("c-1")]+++def test_a_slow_call_holds_up_neither_another_call_nor_the_beat(+ client: TestClient, calls: _Calls+) -> None:+ with client.websocket_connect(PATH) as socket:+ socket.send_json(_call(1, "slow"))+ socket.send_json(_call(2))+ first = _until(socket, "call.result")+ socket.send_json({"type": "pong", "cursors": {"a1": 3}})+ _until(socket, "ping")+ _until(socket, "ping")+ beats = len(calls.beats)+ calls.slow.set()+ second = _until(socket, "call.result")++ assert first["data"]["id"] == 2+ assert beats == 1+ assert second["data"]["id"] == 1+++def test_a_call_over_the_limit_is_refused_without_running(+ client: TestClient, calls: _Calls, monkeypatch: pytest.MonkeyPatch+) -> None:+ monkeypatch.setattr(socket_calls, "MAX_CALLS_IN_FLIGHT", 1)+ with client.websocket_connect(PATH) as socket:+ socket.send_json(_call(1, "slow"))+ socket.send_json(_call(2))+ refused = _until(socket, "call.result")+ calls.slow.set()+ answered = _until(socket, "call.result")++ assert refused["data"]["id"] == 2+ assert refused["data"]["status"] == 429+ assert refused["data"]["body"]["error"]["code"] == "capacity_exceeded"+ assert refused["data"]["body"]["error"]["retryable"] is True+ assert answered["data"] == {"id": 1, "status": 200, "body": {"result": "slow"}}+ assert [call["id"] for call in calls.answered] == [1]++[email protected]("id_", [None, True, "", "x" * 65, {"id": 1}])+def test_a_call_with_no_id_to_answer_by_is_dropped(+ client: TestClient, calls: _Calls, id_: Any+) -> None:+ with client.websocket_connect(PATH) as socket:+ socket.send_json(_call(id_))+ socket.send_json(_call("after"))+ answer = _until(socket, "call.result")++ assert answer["data"]["id"] == "after"+ assert [call["id"] for call in calls.answered] == ["after"]+++async def test_an_answer_goes_out_before_the_frames_waiting() -> None:+ outbox = controller_routes._Outbox(2)+ await outbox.put_frame({"event": "agent.event", "data": {"n": 1}})+ await outbox.put_frame({"event": "agent.event", "data": {"n": 2}})+ third = asyncio.create_task(+ outbox.put_frame({"event": "agent.event", "data": {"n": 3}})+ )+ await asyncio.sleep(0)+ outbox.put_answer({"event": "call.result", "data": {"id": 1}})++ sent = [await outbox.get(timeout=1) for _ in range(4)]+ await third++ assert not third.cancelled()+ assert [frame["event"] if frame else None for frame in sent] == [+ "call.result",+ "agent.event",+ "agent.event",+ "agent.event",+ ]+ assert await outbox.get(timeout=0.01) is Nonediff --git a/core/tests/switch_core/management/test_socket_calls.py b/core/tests/switch_core/management/test_socket_calls.pynew file mode 100644index 000000000..b3af9473c--- /dev/null+++ b/core/tests/switch_core/management/test_socket_calls.py@@ -0,0 +1,220 @@+"""An operation a controller calls on its socket answers what the same call+over HTTP with the controller's token answers, and is authorized the same way.++Each case is sent through both doors, the HTTP one through the real bearer+middleware, and the two answers compared. Against Postgres.+"""++from __future__ import annotations++from collections.abc import Iterator+from datetime import UTC, datetime+from typing import Any++import pytest+from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker++from switch_core.bridges.agent.auth import ControllerPrincipal+from switch_core.bridges.agent.operations import context as op_context+from switch_core.bridges.agent.operations import registry+from switch_core.db.models import TENANT_ZERO_ID+from switch_core.management import socket_calls+from tests.switch_core.management.harness import (+ EnrolledController,+ Harness,+ add_member,+ add_room,+ build_harness,+ cookies_for,+ enroll_console,+ place_agent,+)+++def harness(session_factory: async_sessionmaker[AsyncSession]) -> Harness:+ return build_harness(session_factory)+++def where_am_i(harness: Harness) -> Iterator[None]:+ """An operation that answers with the caller's room, or refuses."""++ async def where_am_i(refuse: bool = False, stamped: bool = False) -> dict[str, Any]:+ if refuse:+ raise PermissionError("not here")+ answer: dict[str, Any] = {"room_id": await op_context.require_connected_room()}+ if stamped:+ # Not JSON as it is: encoded for the socket as the route encodes it.+ answer["at"] = datetime(2026, 11, 1, 12, 30, tzinfo=UTC)+ return answer++ registry._REGISTRY["where_am_i"] = registry.Operation(+ name="where_am_i",+ fn=where_am_i,+ description="test",+ input_schema=registry._input_schema(where_am_i),+ )+ previous = op_context._protocol+ op_context.init_operations_protocol(harness.protocol)+ try:+ yield+ finally:+ registry._REGISTRY.pop("where_am_i", None)+ op_context._protocol = previous+++def _principal(controller: EnrolledController) -> ControllerPrincipal:+ return ControllerPrincipal(+ controller_id=controller.controller_id,+ owner_id=controller.owner.id,+ tenant_id=TENANT_ZERO_ID,+ )+++async def _both(+ harness: Harness,+ controller: EnrolledController,+ agent_id: str,+ operation: str,+ arguments: dict[str, Any],+ room_id: str | None,+) -> tuple[tuple[int, Any], tuple[int, Any]]:+ """The answer over HTTP and the answer on the socket, each as (status, body)."""+ headers = dict(controller.headers)+ if room_id is not None:+ headers["X-Switch-Room-Id"] = room_id+ async with harness.client() as client:+ response = await client.post(+ f"/agents/{agent_id}/ops/{operation}", json=arguments, headers=headers+ )+ answer = await socket_calls.answer(+ {+ "type": "call",+ "id": "c-1",+ "agent_id": agent_id,+ "operation": operation,+ "arguments": arguments,+ "room_id": room_id,+ },+ "c-1",+ principal=_principal(controller),+ protocol=harness.protocol,+ authenticator=harness.management.authenticator,+ )+ assert answer["event"] == "call.result"+ assert answer["data"]["id"] == "c-1"+ return (response.status_code, response.json()), (+ answer["data"]["status"],+ answer["data"]["body"],+ )+++class TestACallAnswersWhatHttpAnswers:+ async def test_for_each_outcome_of_an_operation(+ self, harness: Harness, where_am_i: None+ ) -> None:+ owner = await add_member(harness.session_factory, "ada")+ async with harness.client() as client:+ controller = await enroll_console(harness, client, owner)+ agent_id = await place_agent(client, controller, name="reviewer")+ member_room = await add_room(harness.session_factory, agent_id)+ other_room = await add_room(harness.session_factory, name="elsewhere")++ cases: list[tuple[str, dict[str, Any], str | None]] = [+ ("where_am_i", {}, member_room),+ ("where_am_i", {"stamped": True}, member_room),+ ("where_am_i", {}, other_room),+ ("where_am_i", {}, "no-such-room"),+ ("where_am_i", {}, None),+ ("where_am_i", {"refuse": True}, member_room),+ ("where_am_i", {"unexpected": 1}, member_room),+ ("no_such_operation", {}, member_room),+ ]+ statuses = []+ for operation, arguments, room_id in cases:+ over_http, on_socket = await _both(+ harness, controller, agent_id, operation, arguments, room_id+ )+ assert on_socket == over_http, (operation, arguments, room_id)+ statuses.append(on_socket[0])++ assert statuses == [200, 200, 403, 404, 400, 403, 400, 404]+++class TestACallIsAuthorizedAsTheControllersRequestIs:+ async def test_an_agent_bound_to_another_controller_is_not_assigned(+ self, harness: Harness, where_am_i: None+ ) -> None:+ owner = await add_member(harness.session_factory, "ada")+ async with harness.client() as client:+ mine = await enroll_console(harness, client, owner, "mine")+ theirs = await enroll_console(harness, client, owner, "theirs")+ agent_id = await place_agent(client, theirs, name="theirs-agent")++ over_http, on_socket = await _both(+ harness, mine, agent_id, "where_am_i", {}, None+ )++ assert on_socket == over_http+ assert on_socket[0] == 403+ assert on_socket[1]["error"]["code"] == "not_assigned"++ async def test_a_revoked_controller_is_refused(+ self, harness: Harness, where_am_i: None+ ) -> None:+ owner = await add_member(harness.session_factory, "ada")+ async with harness.client() as client:+ controller = await enroll_console(harness, client, owner)+ agent_id = await place_agent(client, controller, name="reviewer")+ room_id = await add_room(harness.session_factory, agent_id)+ await client.delete(+ f"/gateway/management/controllers/{controller.controller_id}",+ cookies=cookies_for(owner),+ )++ over_http, on_socket = await _both(+ harness, controller, agent_id, "where_am_i", {}, room_id+ )++ assert on_socket == over_http+ assert on_socket[0] == 401+ assert on_socket[1]["error"]["code"] == "controller_revoked"+++ ("change", "detail"),+ [+ ({"agent_id": None}, "agent_id"),+ ({"agent_id": ""}, "agent_id"),+ ({"operation": 7}, "operation"),+ ({"arguments": ["body"]}, "arguments"),+ ({"room_id": 3}, "room_id"),+ ],+)+async def test_a_call_that_cannot_be_run_as_sent_is_a_400(+ harness: Harness, change: dict[str, Any], detail: str+) -> None:+ owner = await add_member(harness.session_factory, "ada")+ async with harness.client() as client:+ controller = await enroll_console(harness, client, owner)+ agent_id = await place_agent(client, controller, name="reviewer")+ call = {+ "type": "call",+ "id": "c-1",+ "agent_id": agent_id,+ "operation": "where_am_i",+ "arguments": {},+ **change,+ }++ answer = await socket_calls.answer(+ call,+ "c-1",+ principal=_principal(controller),+ protocol=harness.protocol,+ authenticator=harness.management.authenticator,+ )++ assert answer["data"]["status"] == 400+ assert detail in answer["data"]["body"]["detail"]diff --git a/core/tests/switch_core/management/test_stream.py b/core/tests/switch_core/management/test_stream.pyindex 2e4e474cb..155e63e8c 100644--- a/core/tests/switch_core/management/test_stream.py+++ b/core/tests/switch_core/management/test_stream.py@@ -30,7 +30,7 @@ from switch_core.db.stores.agent_store import AgentStorefrom switch_core.db.stores.room_role_store import RoomRoleStorefrom switch_core.db.stores.room_store import RoomStorefrom switch_core.db.stores.user_store import UserStore-from switch_core.management import controller_routes+from switch_core.management import controller_routes, socket_callsfrom switch_core.management.notifier import (ASSIGNMENT_CHANGED,CREDENTIAL_REVOKED,@@ -159,6 +159,7 @@ class TestOpening:"connection_id": opened["connection_id"],"generation": opened["generation"],"heartbeat_interval_s": HEARTBEAT_INTERVAL_SECONDS,+ "calls": {"max_in_flight": socket_calls.MAX_CALLS_IN_FLIGHT},},)assert attached == (