Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

9.9 KB
Raw Blame
"""Switch cloud machines, which run the agent controller.

A cloud machine is its owner's, whichever workspaces it serves
(`CloudMachine`). For each workspace it boots a `switch-agent-controller` of
its own, and enrolls it with a one-time code Core mints in that workspace and
hands over when it prepares the machine. The controller it enrolls as is
linked to the workspace's row on the machine (`MachineWorkspace.controller_id`);
the owner's managed agents placed on it run there, each as a Linux user of its
own, and its status reports are what make the machine ready.

Enrollment codes and controllers belong to agent management, which Core does
not import (`management/wiring.py` installs it here with
`set_cloud_enrollment`); Core keeps the machine side.
"""

from __future__ import annotations

from datetime import UTC, datetime, timedelta
from typing import Any, Literal, Protocol

from sqlalchemy import select
from sqlalchemy.ext.asyncio import AsyncSession

from switch_core.db.models import (
    AgentController,
    CloudMachine,
    MachineWorkspace,
    require_tenant_id,
)
from switch_core.db.stores.hosted_machine_store import (
    CloudMachineStore,
    await_reconnect,
    bump_revision,
    idle_sleeping,
)
from switch_core.gateway.dependencies import current_telemetry
from switch_core.keys import Keyring
from switch_core.telemetry import emit_after_commit

# How recent the machine's last status report must be for its silence to count
# as idle rather than as a machine that stopped reporting.
FRESH_HEARTBEAT = timedelta(minutes=5)


class CloudEnrollment(Protocol):
    """What agent management does for a cloud machine's controller."""

    async def machine_enrollment_code(
        self, session: AsyncSession, workspace: MachineWorkspace, now: datetime
    ) -> str:
        """A new one-time code the machine's controller for the bound workspace
        enrolls with. The caller commits."""
        ...


_enrollment: CloudEnrollment | None = None


def set_cloud_enrollment(enrollment: CloudEnrollment | None) -> None:
    global _enrollment
    _enrollment = enrollment


class CloudEnrollmentUnavailable(Exception):
    """Controller machines need agent management, and this server does not run it."""


def cloud_enrollment() -> CloudEnrollment:
    if _enrollment is None:
        raise CloudEnrollmentUnavailable(
            "Cloud machines need agent management: turn on the agent_management feature flag."
        )
    return _enrollment


async def live_controller(
    session: AsyncSession, workspace: MachineWorkspace
) -> AgentController | None:
    """The controller the machine enrolled as in the workspace, unless it was
    revoked since."""
    if workspace.controller_id is None:
        return None
    controller = await session.get(
        AgentController, workspace.controller_id, populate_existing=True
    )
    if (
        controller is None
        or controller.tenant_id != require_tenant_id()
        or controller.revoked_at is not None
    ):
        return None
    return controller


async def enrollment_code(
    session: AsyncSession,
    workspace: MachineWorkspace,
    revision: int,
    keyring: Keyring,
    now: datetime,
) -> str:
    """The code the machine's controller for the workspace enrolls with at the
    machine's `revision`: the same one for every retry of a revision, a new one
    for a new revision. The caller commits."""
    if (
        workspace.enrollment_code_revision == revision
        and workspace.enrollment_code_encrypted is not None
    ):
        return keyring.decrypt(workspace.enrollment_code_encrypted)
    code = await cloud_enrollment().machine_enrollment_code(session, workspace, now)
    workspace.enrollment_code_encrypted = keyring.encrypt(code)
    workspace.enrollment_code_revision = revision
    return code


async def machine_of_controller(
    session: AsyncSession, controller_id: str
) -> tuple[CloudMachine, MachineWorkspace] | None:
    """The live machine running `controller_id` for the bound workspace,
    locked, with the workspace's row on it; None for any other controller."""
    workspace = await session.scalar(
        select(MachineWorkspace).where(
            MachineWorkspace.tenant_id == require_tenant_id(),
            MachineWorkspace.controller_id == controller_id,
        )
    )
    if workspace is None:
        return None
    machine = await CloudMachineStore().locked(session, workspace.machine_id)
    if machine is None or machine.state == "deleted":
        return None
    return machine, workspace


async def enroll_revoked_again(
    session: AsyncSession, controller_id: str, now: datetime
) -> CloudMachine | None:
    """After the bound workspace's controller on a cloud machine is revoked,
    give the machine a new revision, so it is prepared again with a new code
    for that workspace and, running, relaunched on the same disk to enroll.
    The other workspaces keep their enrollment. A machine that is not meant to
    run enrolls at its next start. None for a controller on no cloud machine.
    The caller commits."""
    found = await machine_of_controller(session, controller_id)
    if found is None:
        return None
    machine, _workspace = found
    if machine.desired_state == "running":
        bump_revision(machine, now)
        await_reconnect(machine)
    return machine


def controller_reports(machine: CloudMachine) -> dict[str, dict[str, Any]]:
    """Per workspace on the machine (its `MachineWorkspace` id), what its
    controller last reported: `{at, sessions_running}`."""
    return dict((machine.heartbeat or {}).get("controllers") or {})


def workspace_reported(machine: CloudMachine, workspace_id: str) -> bool:
    """Whether the workspace's controller has reported since the machine was
    last seen starting: its agents can run."""
    report = controller_reports(machine).get(workspace_id)
    return (
        report is not None
        and machine.running_observed_at is not None
        and datetime.fromisoformat(report["at"]) > machine.running_observed_at
    )


def _reading(free: Any, total: Any) -> dict[str, int] | None:
    if isinstance(free, int) and isinstance(total, int) and total > 0:
        return {"total_bytes": total, "available_bytes": free}
    return None


async def record_controller_status(
    session: AsyncSession, controller_id: str, machine_reading: dict[str, Any]
) -> None:
    """A status report from a cloud machine's controller is its heartbeat: it
    records the machine's disk and memory, and makes a machine that started
    at this revision ready. The caller commits."""
    found = await machine_of_controller(session, controller_id)
    if found is None:
        return
    machine, workspace = found
    now = datetime.now(UTC)
    sessions = machine_reading.get("sessions_running")
    reports = controller_reports(machine)
    reports[workspace.id] = {
        "at": now.isoformat(),
        "sessions_running": sessions if isinstance(sessions, int) else None,
    }
    machine.heartbeat = {
        "disk": _reading(
            machine_reading.get("disk_free_bytes"),
            machine_reading.get("disk_total_bytes"),
        ),
        "memory": _reading(
            machine_reading.get("mem_free_bytes"),
            machine_reading.get("mem_total_bytes"),
        ),
        "controllers": reports,
    }
    machine.heartbeat_at = now
    if isinstance(sessions, int) and sessions > 0:
        machine.active_at = now
    if (
        machine.state == "provisioning"
        and machine.desired_state == "running"
        and machine.running_observed_at is not None
    ):
        machine.state = "ready"
        machine.error = None
        machine.error_code = None
        machine.updated_at = now
        # Revision 1 is the machine as first claimed: anything since (a wake, a
        # restart, a workspace joining) bumped it. Only the first is timed,
        # since nothing records when a later start began.
        new_machine = machine.revision == 1
        emit_after_commit(
            session,
            current_telemetry(),
            "cloud_machine_ready",
            {
                "new_machine": new_machine,
                "provisioning_seconds": (
                    max((now - machine.created_at).total_seconds(), 0.0)
                    if new_machine
                    else -1
                ),
            },
        )


async def wake_controller_machine(
    session: AsyncSession,
    controller_id: str,
    now: datetime,
    trigger: Literal["message", "agent_placed"],
) -> CloudMachine | None:
    """The cloud machine running `controller_id`, started again if it was put
    to sleep when idle, and kept awake otherwise. A machine its owner stopped
    is left stopped. None for a controller that runs on no cloud machine. The
    caller holds no other machine lock, and commits; `trigger` is what woke
    it, for telemetry."""
    found = await machine_of_controller(session, controller_id)
    if found is None:
        return None
    machine, _workspace = found
    store = CloudMachineStore()
    if idle_sleeping(machine):
        store.start(machine, now)
        emit_after_commit(
            session,
            current_telemetry(),
            "cloud_machine_woken",
            {"trigger": trigger, "stopped_by": "idle"},
        )
    elif machine.desired_state == "running":
        machine.active_at = now
    return machine


def controller_machine_idle(
    machine: CloudMachine, idle_after: timedelta, now: datetime
) -> bool:
    """Whether a controller machine has been idle for `idle_after`: the last
    status report of every controller it runs is recent and says no session
    runs, and neither a session nor a message addressed to its agents has kept
    it active since."""
    reports = controller_reports(machine).values()
    return (
        bool(reports)
        and all(
            report.get("sessions_running") == 0
            and now - datetime.fromisoformat(report["at"]) <= FRESH_HEARTBEAT
            for report in reports
        )
        and now - machine.active_at >= idle_after
    )