Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

10.1 KB
Raw Blame
from __future__ import annotations

import argparse
import json
import logging
import os
import signal
import sys
import threading
import uuid
from dataclasses import replace
from pathlib import Path
from time import time
from typing import Any

import boto3

from .cloud import Ec2Cloud
from .config import ConfigError, ControllerConfig
from .gateway import Gateway, GatewayConfig
from .health import check_health
from .lock import ControllerAlreadyRunning, ControllerLock
from .model import DesiredState, Machine
from .reconciler import Reconciler
from .store import MachineStore, StoreError, rebind_superseded_fingerprint


def parser() -> argparse.ArgumentParser:
    result = argparse.ArgumentParser(prog="switch-hosted-controller")
    result.add_argument("--config", required=True, type=Path)
    result.add_argument("--gateway-config", type=Path)
    subparsers = result.add_subparsers(dest="command", required=True)

    create = subparsers.add_parser("create", help="create a machine")
    create.add_argument("--instance-type", required=True)

    for command in ("start", "stop", "status"):
        child = subparsers.add_parser(command)
        child.add_argument("machine_id")

    delete = subparsers.add_parser("delete")
    delete.add_argument("machine_id")
    delete.add_argument("--confirm-machine-id", required=True)
    cleanup = delete.add_mutually_exclusive_group(required=True)
    cleanup.add_argument("--retain-volume", action="store_true")
    cleanup.add_argument("--delete-volume", action="store_true")

    upgrade = subparsers.add_parser(
        "upgrade",
        help="use the configured image, and the cloud gateway's instance type with"
        " --gateway-config, after the old instance is stopped and terminated",
    )
    upgrade.add_argument("machine_id")
    upgrade.add_argument("--confirm-instance-id", required=True)

    subparsers.add_parser("list")
    health = subparsers.add_parser("health")
    health.add_argument("--max-age", required=True, type=float)
    subparsers.add_parser("reconcile-once")
    subparsers.add_parser("serve")
    return result


def main(argv: list[str] | None = None) -> int:
    args = parser().parse_args(argv)
    logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s")
    try:
        config = ControllerConfig.load(args.config)
        if args.command in {"serve", "reconcile-once"}:
            return _reconcile_command(config, args.command, args.gateway_config)
        if args.command == "health":
            return _health(args.max_age)
        return _state_command(config, args)
    except (ConfigError, StoreError, ControllerAlreadyRunning) as exc:
        print(json.dumps({"error": str(exc)}), file=sys.stderr)
        return 2


def _state_command(config: ControllerConfig, args: argparse.Namespace) -> int:
    store = _open_store(config)
    try:
        if args.command == "create":
            if args.instance_type not in config.allowed_instance_types:
                raise ConfigError("instance type is not in allowed_instance_types")
            machine = store.insert(
                machine_id=str(uuid.uuid4()),
                core_revision=0,
                instance_type=args.instance_type,
                image_id=config.image_id,
                max_machines=config.max_machines,
            )
            _print_machine(machine, config)
            return 0
        if args.command == "list":
            print(
                json.dumps(
                    [_machine_output(machine, config) for machine in store.list()], sort_keys=True
                )
            )
            return 0
        try:
            machine_id = str(uuid.UUID(args.machine_id))
        except ValueError:
            raise ConfigError("machine_id must be a machine's UUID") from None
        machine = store.get(machine_id)
        if args.command == "status":
            _print_machine(machine, config)
            return 0
        if args.command == "start":
            _print_machine(
                store.set_desired(machine.machine_id, DesiredState.RUNNING, None), config
            )
            return 0
        if args.command == "stop":
            _print_machine(
                store.set_desired(machine.machine_id, DesiredState.STOPPED, None), config
            )
            return 0
        if args.command == "delete":
            if args.confirm_machine_id != machine.machine_id:
                raise StoreError("--confirm-machine-id must exactly match machine_id")
            desired = DesiredState.DELETED if args.delete_volume else DesiredState.RETAINED
            _print_machine(store.set_desired(machine.machine_id, desired, None), config)
            return 0
        if args.command == "upgrade":
            if (
                machine.desired_state is not DesiredState.STOPPED
                or machine.instance_id is None
                or machine.instance_id != args.confirm_instance_id
            ):
                raise StoreError(
                    "stop the machine and confirm its recorded instance before upgrading"
                )
            cloud = Ec2Cloud(boto3.client("ec2", region_name=config.region), config)
            instance = cloud.get_instance(machine)
            volume = cloud.get_volume(machine)
            if (
                instance is None
                or instance["State"]["Name"] != "terminated"
                or volume is None
                or volume["State"] != "available"
                or volume.get("Attachments")
            ):
                raise StoreError(
                    "the old instance must be confirmed terminated and its retained disk detached"
                )
            instance_type = (
                GatewayConfig.load(args.gateway_config).instance_type
                if args.gateway_config
                else machine.instance_type
            )
            if instance_type not in config.allowed_instance_types:
                raise ConfigError("instance type is not in allowed_instance_types")
            cloud.validate_image(replace(machine, image_id=config.image_id))
            claim = store.mark_instance_terminal_observed(machine.machine_id, machine.instance_id)
            _print_machine(store.upgrade_terminated(claim, config.image_id, instance_type), config)
            return 0
        raise AssertionError(f"unhandled command {args.command}")
    finally:
        store.close()


def _open_store(config: ControllerConfig) -> MachineStore:
    rebind_superseded_fingerprint(
        config.state_db_path, config.superseded_fingerprint(), config.fingerprint()
    )
    return MachineStore(config.state_db_path, config.fingerprint())


def _reconcile_command(config: ControllerConfig, command: str, gateway_path: Path | None) -> int:
    with ControllerLock(config.lock_path):
        store = _open_store(config)
        try:
            ec2 = boto3.client("ec2", region_name=config.region)
            settings = GatewayConfig.load(gateway_path) if gateway_path else None
            reconciler = Reconciler(
                store,
                Ec2Cloud(ec2, config),
                instance_type=settings.instance_type if settings else None,
            )
            gateway = Gateway(settings, config, store) if settings else None
            _touch_health()
            if command == "reconcile-once":
                try:
                    reconciler.reconcile_all()
                finally:
                    _touch_health()
                return 0
            stop = threading.Event()

            def request_stop(_signum: int, _frame: Any) -> None:
                stop.set()

            signal.signal(signal.SIGTERM, request_stop)
            signal.signal(signal.SIGINT, request_stop)
            while not stop.is_set():
                listed = None
                if gateway:
                    try:
                        listed = gateway.machines()
                        gateway.sync_machines(listed)
                    except Exception as error:
                        logging.error("Cloud machine polling failed: %s", type(error).__name__)
                try:
                    reconciler.reconcile_all()
                except Exception as error:
                    logging.error("Cloud machine reconciliation failed: %s", type(error).__name__)
                if gateway and listed is not None:
                    try:
                        gateway.report_observations(listed)
                    except Exception as error:
                        logging.error(
                            "Cloud observation reporting failed: %s", type(error).__name__
                        )
                _touch_health()
                stop.wait(config.poll_interval_seconds)
            return 0
        finally:
            store.close()


_HEALTH_PATH = Path("/tmp/switch-hosted-controller-health")


def _touch_health() -> None:
    temporary = _HEALTH_PATH.with_name(f"{_HEALTH_PATH.name}.{os.getpid()}")
    temporary.write_text(f"{time()}\n")
    temporary.replace(_HEALTH_PATH)


def _health(max_age: float) -> int:
    try:
        return check_health(_HEALTH_PATH, max_age)
    except ValueError as error:
        raise ConfigError(str(error)) from error


def _print_machine(machine: Machine, config: ControllerConfig) -> None:
    print(json.dumps(_machine_output(machine, config), sort_keys=True))


def _machine_output(machine: Machine, config: ControllerConfig) -> dict[str, Any]:
    return {
        "machine_id": machine.machine_id,
        "instance_type": machine.instance_type,
        # False while the machine still runs a type dropped from the list: its
        # instance keeps it through stops and starts until it is replaced.
        "instance_type_allowed": machine.instance_type in config.allowed_instance_types,
        "desired_state": machine.desired_state.value,
        "desired_revision": machine.desired_revision,
        "observed_state": machine.observed_state.value,
        "instance_id": machine.instance_id,
        "instance_seq": machine.instance_seq,
        "data_volume_id": machine.data_volume_id,
        "retain_until": machine.retain_until.isoformat() if machine.retain_until else None,
        "error": machine.error,
    }


if __name__ == "__main__":
    raise SystemExit(main())