Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

28.1 KB
Raw Blame
"""The routes an agent controller calls, mounted on the agent bridge.

All but two authenticate with a controller access token, which the bearer
middleware verifies and binds (`management/auth.py`); a route whose path
names a controller then requires it to be the token's own. Enrollment and
token exchange carry their secret in the body and resolve their tenant from
it here.

The controller's connection (`/v1/controllers/{id}/connection` and its
socket) is Core's: these routes authenticate and parse, and Core's
`ControllerPresence` and controller stream do the rest. Management only adds its nudges and the first frame's
assignment revision.

Every failure is answered in the contract's error envelope (`errors.py`).
`Switch-Controller-Protocol` is checked when sent, and every response says
which protocol versions this server accepts. Protocol 2 moved the stream and
the beat onto one WebSocket; a protocol 1 controller is refused with 426.
"""

from __future__ import annotations

import asyncio
import hashlib
import json
import logging
from collections import deque
from collections.abc import AsyncGenerator, Awaitable, Callable, Coroutine
from typing import Annotated, Any

from fastapi import (
    APIRouter,
    Depends,
    Header,
    Request,
    Response,
    WebSocket,
    WebSocketDisconnect,
)
from fastapi.responses import JSONResponse
from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker

from switch_core.bridges.agent.auth import ControllerPrincipal
from switch_core.bridges.agent.dependencies import get_protocol, get_session
from switch_core.bridges.agent.protocol.agent_core import AgentCore
from switch_core.bridges.agent.protocol.controller_presence import (
    ControllerConnectionError,
    ControllerRevokedError,
)
from switch_core.bridges.agent.protocol.controller_stream import (
    IDLE,
    IDLE_INTERVAL_SECONDS,
    ControllerFrame,
    controller_event_stream,
)
from switch_core.bridges.agent.protocol.liveness import HEARTBEAT_INTERVAL_SECONDS
from switch_core.db.session_scope import tenant_session
from switch_core.management import reason_codes, socket_calls
from switch_core.management.auth import ManagementAuthenticator
from switch_core.management.dependencies import (
    get_authenticator,
    get_controller_principal,
    get_management,
    get_management_session_factory,
    require_controller,
)
from switch_core.management.errors import ManagementError, ManagementRoute, error_body
from switch_core.management.schemas import (
    MAX_STATUS_BYTES,
    ControllerConnectionRequest,
    ControllerInfoRequest,
    DefinitionV1,
    EnrollRequest,
    OperationResultRequest,
    ProgressRequest,
    StatusReport,
    TokenRequest,
    wire_time,
)
from switch_core.management.service import ManagementService

logger = logging.getLogger(__name__)

PROTOCOL_HEADER = "Switch-Controller-Protocol"
PROTOCOL_ACCEPTS_HEADER = "Switch-Controller-Protocol-Accepts"
SUPPORTED_PROTOCOLS = "2-2"
_SUPPORTED_PROTOCOL_VERSIONS = frozenset({"2"})


class ControllerRoute(ManagementRoute):
    """A management route that also speaks the controller protocol header."""

    def get_route_handler(self) -> Callable[[Request], Coroutine[Any, Any, Response]]:
        handler = super().get_route_handler()

        async def versioned(request: Request) -> Response:
            requested = request.headers.get(PROTOCOL_HEADER)
            if requested is not None and requested.strip() not in (
                _SUPPORTED_PROTOCOL_VERSIONS
            ):
                response: Response = JSONResponse(
                    error_body(
                        reason_codes.PROTOCOL_UNSUPPORTED,
                        f"Controller protocol {requested!r} is not supported; "
                        f"Switch accepts {SUPPORTED_PROTOCOLS}.",
                        retryable=False,
                    ),
                    status_code=426,
                )
            else:
                response = await handler(request)
            response.headers[PROTOCOL_ACCEPTS_HEADER] = SUPPORTED_PROTOCOLS
            return response

        return versioned


router = APIRouter(route_class=ControllerRoute, tags=["agent management"])

Principal = Annotated[ControllerPrincipal, Depends(get_controller_principal)]
Management = Annotated[ManagementService, Depends(get_management)]
Session = Annotated[AsyncSession, Depends(get_session)]
Protocol = Annotated[AgentCore, Depends(get_protocol)]


def _path_controller(controller_id: str, principal: Principal) -> ControllerPrincipal:
    return require_controller(controller_id, principal)


PathController = Annotated[ControllerPrincipal, Depends(_path_controller)]


# Part of every assignment ETag, so a controller holding an assignment saved
# under an earlier definition format pulls it again rather than being told it
# is unchanged.
ASSIGNMENT_FORMAT = hashlib.sha256(
    json.dumps(DefinitionV1.model_json_schema(), sort_keys=True).encode()
).hexdigest()[:12]


def _etag(revision: int) -> str:
    return f'"{ASSIGNMENT_FORMAT}-{revision}"'


def _etag_matches(if_none_match: str | None, revision: int) -> bool:
    if if_none_match is None:
        return False
    for candidate in if_none_match.split(","):
        tag = candidate.strip()
        if tag == "*":
            return True
        if tag.startswith("W/"):
            tag = tag[2:]
        if tag == _etag(revision):
            return True
    return False


# ── Public: authenticated by the body ─────────────────────────────────────────


@router.post("/v1/management/controllers/enroll", status_code=201)
async def enroll_controller(
    body: EnrollRequest,
    management: Management,
    authenticator: Annotated[ManagementAuthenticator, Depends(get_authenticator)],
    session_factory: Annotated[
        async_sessionmaker[AsyncSession], Depends(get_management_session_factory)
    ],
) -> dict[str, str]:
    tenant_id = await authenticator.tenant_of_secret(body.proof.code)
    if tenant_id is None:
        raise ManagementError(
            401,
            reason_codes.ENROLLMENT_CODE_INVALID,
            "The enrollment code is invalid, already used, or expired.",
        )
    async with tenant_session(session_factory, tenant_id) as session:
        controller, credential = await management.enroll(
            session,
            tenant_id,
            code=body.proof.code,
            description=body.controller,
            public_key=body.public_key,
        )
    return {"controller_id": controller.id, "credential": credential}


@router.post("/v1/management/controllers/{controller_id}/token")
async def exchange_token(
    controller_id: str,
    body: TokenRequest,
    management: Management,
    authenticator: Annotated[ManagementAuthenticator, Depends(get_authenticator)],
    session_factory: Annotated[
        async_sessionmaker[AsyncSession], Depends(get_management_session_factory)
    ],
) -> dict[str, str]:
    tenant_id = await authenticator.tenant_of_secret(body.credential)
    if tenant_id is None:
        raise ManagementError(
            401, reason_codes.INVALID_CREDENTIAL, "The credential is not valid."
        )
    async with tenant_session(session_factory, tenant_id) as session:
        access_token, expires_at = await management.exchange_token(
            session,
            tenant_id,
            controller_id=controller_id,
            credential=body.credential,
        )
    return {"access_token": access_token, "expires_at": wire_time(expires_at)}


# ── Controller access token ───────────────────────────────────────────────────


@router.patch("/v1/management/controllers/{controller_id}")
async def update_controller_info(
    body: ControllerInfoRequest,
    principal: PathController,
    management: Management,
    session: Session,
) -> dict[str, Any]:
    """Rename this machine and/or change its description, as its owner can
    in the gateway. Returns the machine as the owner's list shows it."""
    return await management.update_own_controller(
        session,
        principal,
        {key: getattr(body, key) for key in body.model_fields_set},
    )


@router.get(
    "/v1/management/controllers/{controller_id}/provider-credentials/{provider}"
)
async def get_sealed_login(
    provider: str, principal: PathController, management: Management, session: Session
) -> JSONResponse:
    """The provider login given to this machine, sealed to its own key, or
    404 `provider_login_missing`."""
    return JSONResponse(
        await management.own_sealed_login(session, principal, provider),
        headers={"Cache-Control": "no-store"},
    )


@router.post("/v1/management/controllers/{controller_id}/credential/rotate")
async def rotate_credential(
    principal: PathController, management: Management, session: Session
) -> dict[str, str]:
    return {"credential": await management.rotate_credential(session, principal)}


@router.get("/v1/management/controllers/{controller_id}/assignment")
async def get_assignment(
    principal: PathController,
    management: Management,
    session: Session,
    if_none_match: Annotated[str | None, Header()] = None,
) -> Response:
    assignment = await management.assignment(session, principal)
    revision = assignment["revision"]
    headers = {"ETag": _etag(revision)}
    if _etag_matches(if_none_match, revision):
        return Response(status_code=304, headers=headers)
    return JSONResponse(assignment, headers=headers)


@router.put("/v1/management/controllers/{controller_id}/status")
async def put_status(
    request: Request,
    report: StatusReport,
    principal: PathController,
    management: Management,
    session: Session,
) -> dict[str, Any]:
    if len(await request.body()) > MAX_STATUS_BYTES:
        raise ManagementError(
            413,
            reason_codes.VALIDATION_ERROR,
            f"A status report may be at most {MAX_STATUS_BYTES} bytes.",
        )
    return await management.record_status(session, principal, report)


@router.get("/v1/management/controllers/{controller_id}/operations")
async def list_operations(
    principal: PathController,
    management: Management,
    session: Session,
    state: str = "pending",
) -> dict[str, list[dict[str, Any]]]:
    if state != "pending":
        raise ManagementError(
            422,
            reason_codes.VALIDATION_ERROR,
            "Only state=pending can be listed by a controller.",
        )
    return {"operations": await management.offered_operations(session, principal)}


@router.post("/v1/management/operations/{operation_id}/claim")
async def claim_operation(
    operation_id: str, principal: Principal, management: Management, session: Session
) -> dict[str, Any]:
    return await management.claim_operation(session, principal, operation_id)


@router.post("/v1/management/operations/{operation_id}/progress", status_code=204)
async def operation_progress(
    operation_id: str,
    principal: Principal,
    management: Management,
    session: Session,
    body: ProgressRequest | None = None,
) -> Response:
    await management.renew_operation(session, principal, operation_id)
    return Response(status_code=204)


@router.post("/v1/management/operations/{operation_id}/result", status_code=204)
async def operation_result(
    operation_id: str,
    body: OperationResultRequest,
    principal: Principal,
    management: Management,
    session: Session,
) -> Response:
    await management.complete_operation(session, principal, operation_id, body.stored())
    return Response(status_code=204)


# ── The controller's stream ──────────────────────────────────────────────────


def _connection_refusal(exc: ControllerConnectionError) -> ManagementError:
    status = {
        reason_codes.UNKNOWN_CONNECTION: 404,
        reason_codes.CONTROLLER_REVOKED: 401,
    }.get(exc.code, 409)
    return ManagementError(status, exc.code, str(exc))


def _rooms_reader(protocol: AgentCore) -> Callable[[str], Awaitable[set[str]]]:
    async def rooms_of(agent_id: str) -> set[str]:
        rooms = await protocol.list_rooms(agent_id, include_archived=True)
        return {room.id for room in rooms}

    return rooms_of


@router.post("/v1/controllers/{controller_id}/connection", status_code=201)
async def open_connection(
    body: ControllerConnectionRequest,
    principal: PathController,
    protocol: Protocol,
    management: Management,
    session: Session,
) -> dict[str, Any]:
    """Open the controller's connection, taking over any it had.

    The agents bound to it are attached on the stream, each from its cursor
    here; the response names them. Opening again is a takeover: the earlier
    connection's socket ends with `taken_over`, and its beats are refused,
    whichever process held it. The connection is recorded before the answer,
    so its socket can attach through any process.
    """
    presence = protocol.connections.controllers
    resume_cursors = body.resume_cursors()
    if presence.is_revoked(principal.controller_id):
        raise _connection_refusal(ControllerRevokedError(principal.controller_id))
    # Recorded before it is held: a read of the row in between would find the
    # connection this one replaces still named there, and close this one.
    identity = presence.new_connection()
    connection_id, generation = identity
    await management.record_opened(
        session,
        principal,
        connection_id=connection_id,
        generation=generation,
        resume_cursors=resume_cursors,
    )
    try:
        conn = presence.open(
            controller_id=principal.controller_id,
            tenant_id=principal.tenant_id,
            resume_cursors=resume_cursors,
            identity=identity,
        )
    except ControllerConnectionError as exc:
        raise _connection_refusal(exc) from exc
    logger.info(
        "Controller %s opened connection %s (client=%s version=%s)",
        principal.controller_id,
        conn.id,
        body.client or "unknown",
        body.client_version or "unknown",
    )
    return {
        "connection_id": conn.id,
        "generation": conn.generation,
        "heartbeat_interval_s": HEARTBEAT_INTERVAL_SECONDS,
        "agents": sorted(presence.agents_of(principal.controller_id)),
    }


async def attach_stream(
    *,
    principal: ControllerPrincipal,
    management: ManagementService,
    protocol: AgentCore,
    session_factory: async_sessionmaker[AsyncSession],
    connection_id: str,
    generation: int,
) -> AsyncGenerator[ControllerFrame]:
    """Attach the stream to an open connection: its agents' events and the
    management nudges, starting with `connection_state`.

    The nudge subscription is taken before the first frame is built, so a
    change that commits while the stream is opening is still delivered.
    Raises the refusal when the connection cannot take a stream.
    """
    presence = protocol.connections.controllers
    nudges = management.notifier.subscribe(principal.controller_id)
    try:
        async with tenant_session(session_factory, principal.tenant_id) as session:
            # Wherever it was opened or last held: decided on the
            # controller's row, which says whether it is still the one the
            # controller opened last.
            conn = await management.hold_connection(
                session, principal, connection_id, generation
            )
            state = await management.connection_state(session, principal)
        # Attached once nothing more is awaited, and refused if the connection
        # stopped being the controller's current one while the reads above
        # waited.
        token = presence.attach_stream(conn)
    except ControllerConnectionError as exc:
        nudges.close()
        raise _connection_refusal(exc) from exc
    except BaseException:
        nudges.close()
        raise
    opening = {
        **state,
        "connection_id": conn.id,
        "generation": conn.generation,
        "heartbeat_interval_s": HEARTBEAT_INTERVAL_SECONDS,
        # The socket takes its agents' operations as calls (`socket_calls`).
        "calls": socket_calls.calls_offered(),
    }
    return controller_event_stream(
        conn=conn,
        stream_token=token,
        presence=presence,
        buffer=protocol.event_buffer,
        approvals=protocol.approval_outcomes,
        nudges=nudges,
        opening=opening,
        rooms_of=_rooms_reader(protocol),
        idle_seconds=IDLE_INTERVAL_SECONDS,
    )


async def record_beat(
    *,
    principal: ControllerPrincipal,
    protocol: AgentCore,
    connection_id: str,
    generation: int,
    cursors: dict[str, int],
) -> list[str]:
    """Keep the controller, and so every agent on it, live; confirm cursors.

    A pong on the socket, every heartbeat interval; `HEARTBEAT_TTL_SECONDS`
    without one and its agents are not live. The cursors confirmed here are where a
    stream reattached to this connection resumes each agent. Raises the
    presence's refusal: `taken_over` once another connection has replaced
    this one, terminal for the client, and `no_stream` or
    `unknown_connection` when the connection has to be opened again.
    """
    presence = protocol.connections.controllers
    conn = presence.beat(principal.controller_id, connection_id, generation)
    agents = presence.agents_of(principal.controller_id)
    for agent_id, cursor in cursors.items():
        binding = presence.binding(agent_id)
        if agent_id not in agents or binding is None:
            continue
        sent = conn.sent_cursors.get(agent_id)
        if sent is None:
            continue
        # The controller reports how far it has read each agent's own events.
        # It cannot have read further than this connection has sent it. A
        # larger number comes from another history, a database restored from a
        # backup or another server, and keeping it would make the next stream
        # on this connection resume past events the agent never received. The
        # bound is kept in memory, so a beat costs no database query.
        presence.resume_from(conn, agent_id, min(cursor, sent))
    return sorted(agents)


def _refused(error: ManagementError) -> dict[str, Any]:
    """The frame a refused socket gets before it closes with 4000 + status,
    as the bearer middleware sends its own refusals."""
    return {
        "event": "refused",
        "data": {
            "status": error.status_code,
            "detail": {"code": error.code, "message": error.message},
        },
    }


def _cursors_of(message: dict[str, Any]) -> dict[str, int]:
    raw = message.get("cursors")
    if not isinstance(raw, dict):
        return {}
    return {
        agent_id: cursor
        for agent_id, cursor in raw.items()
        if isinstance(agent_id, str)
        and isinstance(cursor, int)
        and not isinstance(cursor, bool)
        and cursor >= 0
    }


@router.websocket("/v1/controllers/{controller_id}/connection/ws")
async def connection_socket(
    websocket: WebSocket,
    controller_id: str,
    management: Management,
    protocol: Protocol,
    authenticator: Annotated[ManagementAuthenticator, Depends(get_authenticator)],
    session_factory: Annotated[
        async_sessionmaker[AsyncSession], Depends(get_management_session_factory)
    ],
    connection_id: str,
    generation: int,
) -> None:
    """The controller's one connection: its agents' events and the management
    nudges down, its heartbeat and its agents' calls up, on one WebSocket.

    Attaches to a connection opened with `POST .../connection`. Frames are
    JSON `{"event", "data"}`, the events the stream carries. The server sends
    `{"event": "ping"}` every heartbeat interval and the controller answers
    `{"type": "pong", "cursors": {agent_id: n}}`, which is its beat. A
    `{"type": "call", ...}` runs one of its agents' operations and is answered
    with a `call.result` frame (`socket_calls`). A refused attach is a
    `refused` frame, then a close with 4000 plus the status; a refused beat is
    an `evicted` frame naming why, then a close.
    """
    await websocket.accept()
    try:
        principal = _socket_principal(websocket, controller_id)
        frames = await attach_stream(
            principal=principal,
            management=management,
            protocol=protocol,
            session_factory=session_factory,
            connection_id=connection_id,
            generation=generation,
        )
    except ManagementError as error:
        await websocket.send_json(_refused(error))
        await websocket.close(code=4000 + error.status_code)
        return

    outbox = _Outbox(_SOCKET_OUTBOX)
    pump = asyncio.create_task(_pump(frames, outbox))
    receiver = asyncio.create_task(
        _receive(
            websocket,
            principal=principal,
            protocol=protocol,
            authenticator=authenticator,
            connection_id=connection_id,
            generation=generation,
            outbox=outbox,
        )
    )
    loop = asyncio.get_running_loop()
    next_ping = loop.time() + HEARTBEAT_INTERVAL_SECONDS
    try:
        # One writer: frames, call answers and pings all go out from this loop.
        while not pump.done() or not outbox.empty():
            if receiver.done():
                refusal = receiver.result()
                if refusal is not None:
                    await websocket.send_json(
                        {
                            "event": "evicted",
                            "data": {"code": refusal.code, "reason": str(refusal)},
                        }
                    )
                return
            frame = await outbox.get(timeout=max(0.0, next_ping - loop.time()))
            if frame is not None and frame is not IDLE:
                await websocket.send_json(frame)
            if loop.time() >= next_ping:
                await websocket.send_json({"event": "ping", "data": {}})
                next_ping = loop.time() + HEARTBEAT_INTERVAL_SECONDS
    except WebSocketDisconnect:
        return
    except RuntimeError:
        # The server closed the socket under us (uvicorn does on shutdown).
        return
    finally:
        pump.cancel()
        receiver.cancel()
        failure = pump.exception() if pump.done() and not pump.cancelled() else None
        if failure is not None:
            logger.error(
                "Controller %s connection %s: delivery failed, closing its socket",
                principal.controller_id,
                connection_id,
                exc_info=failure,
            )
        try:
            await websocket.close(code=1011 if failure is not None else 1000)
        except RuntimeError:
            pass  # the client closed it first


# Frames buffered between the stream and the socket.
_SOCKET_OUTBOX = 64


class _Outbox:
    """What the socket's one writer sends next.

    The stream's frames wait here, at most `size` of them, so the stream is
    read only as fast as the socket takes them. A call's answer never waits
    for room, as there are no more of them than calls in flight, and goes out
    before any frame still waiting, so a backlog of events does not hold a
    call up.
    """

    def __init__(self, size: int) -> None:
        self._size = size
        self._frames: deque[ControllerFrame] = deque()
        self._answers: deque[ControllerFrame] = deque()
        self._waiting = asyncio.Event()
        self._room = asyncio.Event()
        self._room.set()

    def empty(self) -> bool:
        return not self._frames and not self._answers

    async def put_frame(self, frame: ControllerFrame) -> None:
        while len(self._frames) >= self._size:
            self._room.clear()
            await self._room.wait()
        self._frames.append(frame)
        self._waiting.set()

    def put_answer(self, answer: ControllerFrame) -> None:
        self._answers.append(answer)
        self._waiting.set()

    async def get(self, timeout: float) -> ControllerFrame | None:
        """The next frame to send, or None once `timeout` passes with none."""
        if self.empty():
            self._waiting.clear()
            try:
                await asyncio.wait_for(self._waiting.wait(), timeout)
            except TimeoutError:
                return None
        if self._answers:
            return self._answers.popleft()
        frame = self._frames.popleft()
        self._room.set()
        return frame


def _socket_principal(websocket: WebSocket, controller_id: str) -> ControllerPrincipal:
    """The controller the bearer middleware authenticated, checked against the
    path and the protocol it declares, as the HTTP routes check theirs."""
    requested = websocket.headers.get(PROTOCOL_HEADER)
    if requested is not None and requested.strip() not in _SUPPORTED_PROTOCOL_VERSIONS:
        raise ManagementError(
            426,
            reason_codes.PROTOCOL_UNSUPPORTED,
            f"Controller protocol {requested!r} is not supported; "
            f"Switch accepts {SUPPORTED_PROTOCOLS}.",
        )
    return require_controller(controller_id, get_controller_principal(websocket))


async def _pump(frames: AsyncGenerator[ControllerFrame], outbox: _Outbox) -> None:
    # Closed here: cancelled while waiting on the outbox, the stream would
    # otherwise detach only when the generator is garbage collected.
    try:
        async for frame in frames:
            await outbox.put_frame(frame)
    finally:
        await frames.aclose()


async def _receive(
    websocket: WebSocket,
    *,
    principal: ControllerPrincipal,
    protocol: AgentCore,
    authenticator: ManagementAuthenticator,
    connection_id: str,
    generation: int,
    outbox: _Outbox,
) -> ControllerConnectionError | None:
    """Turn each pong into a beat and start answering each call, until the
    controller goes or a beat is refused. Returns the refusal, so the socket
    can say which before it closes; None when the controller simply went
    away."""
    in_flight: set[asyncio.Task[None]] = set()
    while True:
        try:
            message = await websocket.receive_json()
        except (WebSocketDisconnect, ValueError, RuntimeError):
            return None
        if not isinstance(message, dict):
            continue
        kind = message.get("type")
        if kind == socket_calls.CALL:
            _start_call(
                message,
                in_flight=in_flight,
                outbox=outbox,
                principal=principal,
                protocol=protocol,
                authenticator=authenticator,
            )
            continue
        if kind != "pong":
            continue
        try:
            await record_beat(
                principal=principal,
                protocol=protocol,
                connection_id=connection_id,
                generation=generation,
                cursors=_cursors_of(message),
            )
        except ControllerConnectionError as refusal:
            return refusal


# Every call still running, whatever socket it came on. The event loop keeps
# only a weak reference to a task, and a call outlives its socket when the
# controller goes before it is answered.
_running_calls: set[asyncio.Task[None]] = set()


def _start_call(
    message: dict[str, Any],
    *,
    in_flight: set[asyncio.Task[None]],
    outbox: _Outbox,
    principal: ControllerPrincipal,
    protocol: AgentCore,
    authenticator: ManagementAuthenticator,
) -> None:
    """Answer a call from a task of its own, so that a slow call holds up
    neither the others nor the beat."""
    id_ = socket_calls.call_id(message)
    if id_ is None:
        logger.warning(
            "Controller %s sent a call with no id to answer it by; dropped it",
            principal.controller_id,
        )
        return
    if len(in_flight) >= socket_calls.MAX_CALLS_IN_FLIGHT:
        outbox.put_answer(socket_calls.capacity_exceeded(id_))
        return

    async def run() -> None:
        outbox.put_answer(
            await socket_calls.answer(
                message,
                id_,
                principal=principal,
                protocol=protocol,
                authenticator=authenticator,
            )
        )

    task = asyncio.create_task(run())
    for tasks in (in_flight, _running_calls):
        tasks.add(task)
        task.add_done_callback(tasks.discard)