Skip to content

Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.

21.3 KB
Raw Blame
from __future__ import annotations

import copy
from datetime import UTC, datetime, timedelta
from pathlib import Path

import pytest
from botocore.stub import Stubber
from test_controller import config, ec2_client, store_and_machine, volume, with_bundle

from switch_hosted_controller import reconciler as reconciler_module
from switch_hosted_controller.cloud import CloudResourceError, Ec2Cloud
from switch_hosted_controller.model import DesiredState, ObservedState
from switch_hosted_controller.reconciler import Reconciler
from switch_hosted_controller.store import MachineStore, StoreError


def terminal_instance(cloud: Ec2Cloud, machine) -> dict:
    return {
        "InstanceId": machine.instance_id,
        "ImageId": machine.image_id,
        "InstanceType": machine.instance_type,
        "Placement": {"AvailabilityZone": cloud._config.availability_zone},
        "State": {"Name": "terminated", "Code": 48},
        "Tags": cloud._tags(machine, "worker"),
    }


def record_compute(store: MachineStore, machine, availability_zone: str):
    machine = store.record_volume(machine.machine_id, "vol-0123456789abcdef0", availability_zone)
    return store.record_instance(machine.machine_id, "i-0123456789abcdef0", None)


def accept_delete(store: MachineStore, machine, retain_until: datetime | None):
    claim = store.set_desired(machine.machine_id, DesiredState.STOPPED, None)
    store.set_observed(claim, ObservedState.STOPPED)
    return store.set_desired(machine.machine_id, DesiredState.DELETED, retain_until)


def test_recorded_terminated_instance_accepts_sparse_aws_response(tmp_path: Path):
    cfg = config(tmp_path)
    store, machine = store_and_machine(cfg)
    machine = store.record_instance(machine.machine_id, "i-0123456789abcdef0", None)
    client = ec2_client()
    cloud = Ec2Cloud(client, cfg)
    terminated = terminal_instance(cloud, machine)
    assert not {
        "IamInstanceProfile",
        "SubnetId",
        "SecurityGroups",
        "NetworkInterfaces",
        "MetadataOptions",
    }.intersection(terminated)

    with Stubber(client) as stubber:
        stubber.add_response(
            "describe_instances",
            {"Reservations": [{"Instances": [terminated]}]},
            {"InstanceIds": [machine.instance_id]},
        )
        assert cloud.get_instance(machine) == terminated
    store.close()


@pytest.mark.parametrize("corruption", ["instance-id", "ownership-tag"])
def test_recorded_terminated_instance_still_validates_identity(tmp_path: Path, corruption: str):
    cfg = config(tmp_path)
    store, machine = store_and_machine(cfg)
    machine = store.record_instance(machine.machine_id, "i-0123456789abcdef0", None)
    client = ec2_client()
    cloud = Ec2Cloud(client, cfg)
    terminated = terminal_instance(cloud, machine)
    if corruption == "instance-id":
        terminated["InstanceId"] = "i-fffffffffffffffff"
    else:
        terminated["Tags"] = copy.deepcopy(terminated["Tags"])
        next(tag for tag in terminated["Tags"] if tag["Key"] == "switch:machine-id")["Value"] = (
            "00000000-0000-4000-8000-00000000beef"
        )

    with Stubber(client) as stubber:
        stubber.add_response(
            "describe_instances",
            {"Reservations": [{"Instances": [terminated]}]},
            {"InstanceIds": [machine.instance_id]},
        )
        with pytest.raises(CloudResourceError):
            cloud.get_instance(machine)
    store.close()


def test_terminated_then_not_found_remains_safely_stopped_and_deletable(
    tmp_path: Path,
):
    cfg = config(tmp_path)
    store, machine = store_and_machine(cfg)
    machine = store.record_instance(machine.machine_id, "i-0123456789abcdef0", None)
    claim = store.set_desired(machine.machine_id, DesiredState.STOPPED, None)
    client = ec2_client()
    cloud = Ec2Cloud(client, cfg)

    with Stubber(client) as stubber:
        stubber.add_response(
            "describe_instances",
            {"Reservations": [{"Instances": [terminal_instance(cloud, claim)]}]},
            {"InstanceIds": [machine.instance_id]},
        )
        first = Reconciler(store, cloud, instance_type=None).reconcile(machine.machine_id)
    assert first.instance_terminal_observed
    assert first.observed_state is ObservedState.STOPPED

    with Stubber(client) as stubber:
        stubber.add_client_error(
            "describe_instances",
            service_error_code="InvalidInstanceID.NotFound",
            expected_params={"InstanceIds": [machine.instance_id]},
        )
        second = Reconciler(store, cloud, instance_type=None).reconcile(machine.machine_id)
    assert second.observed_state is ObservedState.STOPPED
    assert second.observed_revision == second.desired_revision
    assert second.observed_operation_id == second.operation_id
    assert (
        store.set_desired(machine.machine_id, DesiredState.DELETED, None).desired_state
        is DesiredState.DELETED
    )
    store.close()


class StartRaceCloud:
    def __init__(self, database: Path, fingerprint: str, *, fail: bool):
        self.database = database
        self.fingerprint = fingerprint
        self.fail = fail
        self.delete_rejected = False
        self.start_calls = 0

    def get_volume(self, machine):
        return {
            "VolumeId": machine.data_volume_id,
            "State": "in-use",
            "Attachments": [
                {
                    "InstanceId": machine.instance_id,
                    "Device": "/dev/sdf",
                    "State": "attached",
                }
            ],
        }

    def get_instance(self, machine):
        return {"InstanceId": machine.instance_id, "State": {"Name": "stopped"}}

    def start_instance(self, machine):
        self.start_calls += 1
        concurrent = MachineStore(self.database, self.fingerprint)
        try:
            concurrent.set_desired(machine.machine_id, DesiredState.STOPPED, None)
            try:
                concurrent.set_desired(machine.machine_id, DesiredState.DELETED, None)
            except StoreError:
                self.delete_rejected = True
        finally:
            concurrent.close()
        if self.fail:
            raise CloudResourceError("synthetic start failure")


@pytest.mark.parametrize("start_fails", [False, True])
def test_start_callback_cannot_publish_stale_success_or_error(tmp_path: Path, start_fails: bool):
    cfg = config(tmp_path)
    store, machine = store_and_machine(cfg)
    store.record_volume(machine.machine_id, "vol-0123456789abcdef0", cfg.availability_zone)
    bundled = with_bundle(store, machine.machine_id, 1)
    machine = store.record_instance(machine.machine_id, "i-0123456789abcdef0", bundled.bundle)
    store.set_observed(machine, ObservedState.STOPPED)
    cloud = StartRaceCloud(cfg.state_db_path, cfg.fingerprint(), fail=start_fails)

    Reconciler(store, cloud, instance_type=None).reconcile_all()

    current = store.get(machine.machine_id)
    assert cloud.start_calls == 1
    assert cloud.delete_rejected
    assert current.desired_state is DesiredState.STOPPED
    assert current.observed_state is ObservedState.PENDING
    assert current.observed_revision == current.desired_revision
    assert current.observed_operation_id == current.operation_id
    assert current.error is None
    store.close()


def test_delayed_old_stopped_observation_cannot_authorize_newer_stop(
    tmp_path: Path,
):
    cfg = config(tmp_path)
    store, machine = store_and_machine(cfg)
    old_stop = store.set_desired(machine.machine_id, DesiredState.STOPPED, None)
    store.set_desired(machine.machine_id, DesiredState.RUNNING, None)
    new_stop = store.set_desired(machine.machine_id, DesiredState.STOPPED, None)

    current = store.set_observed(old_stop, ObservedState.STOPPED)

    assert current.operation_id == new_stop.operation_id
    assert current.observed_state is ObservedState.PENDING
    with pytest.raises(StoreError, match="freshly observed stopped"):
        store.set_desired(machine.machine_id, DesiredState.DELETED, None)
    store.close()


class DeletionCloud:
    def __init__(self, states: list[str | None]):
        self.states = iter(states)
        self.volume_exists = True
        self.calls: list[str] = []

    def get_instance(self, machine):
        state = next(self.states)
        if state is None:
            return None
        return {"InstanceId": machine.instance_id, "State": {"Name": state}}

    def stop_instance(self, _machine):
        self.calls.append("stop-instance")

    def terminate_instance(self, _machine):
        self.calls.append("terminate-instance")

    def get_volume(self, machine):
        self.calls.append("get-volume")
        if not self.volume_exists:
            return None
        return {"VolumeId": machine.data_volume_id, "State": "available", "Attachments": []}

    def delete_volume(self, _machine):
        self.calls.append("delete-volume")
        self.volume_exists = False


def test_retained_stops_running_before_termination_and_keeps_volume(
    tmp_path: Path,
):
    cfg = config(tmp_path)
    store, machine = store_and_machine(cfg)
    machine = record_compute(store, machine, cfg.availability_zone)
    store.set_desired(machine.machine_id, DesiredState.RETAINED, None)
    cloud = DeletionCloud(["running", "stopped", "terminated"])
    reconciler = Reconciler(store, cloud, instance_type=None)

    assert reconciler.reconcile(machine.machine_id).observed_state is ObservedState.STOPPING
    assert cloud.calls == ["stop-instance"]
    assert reconciler.reconcile(machine.machine_id).observed_state is ObservedState.STOPPING
    assert cloud.calls == ["stop-instance", "terminate-instance"]
    retained = reconciler.reconcile(machine.machine_id)

    assert retained.observed_state is ObservedState.RETAINED
    assert retained.instance_id is None
    assert retained.previous_instance_id == "i-0123456789abcdef0"
    assert retained.instance_seq == 1
    assert retained.data_volume_id == "vol-0123456789abcdef0"
    assert cloud.calls == ["stop-instance", "terminate-instance", "get-volume"]
    assert cloud.volume_exists
    assert reconciler.reconcile(machine.machine_id) == retained
    store.close()


def test_retained_with_a_missing_volume_needs_attention(tmp_path: Path):
    cfg = config(tmp_path)
    store, machine = store_and_machine(cfg)
    machine = record_compute(store, machine, cfg.availability_zone)
    store.set_desired(machine.machine_id, DesiredState.RETAINED, None)
    cloud = DeletionCloud(["terminated"])
    cloud.volume_exists = False
    result = Reconciler(store, cloud, instance_type=None).reconcile(machine.machine_id)
    assert result.observed_state is ObservedState.NEEDS_ATTENTION
    assert result.error == "recorded data volume cannot be found"
    store.close()


class LaunchCloud:
    def __init__(self, image_id: str = "ami-0123456789abcdef0"):
        self.image_id = image_id
        self.launched = []

    def get_volume(self, machine):
        return {"VolumeId": machine.data_volume_id, "State": "available", "Attachments": []}

    def get_instance(self, machine):
        return None

    def validate_image(self, machine):
        pass

    def validate_capacity(self):
        pass

    def run_instance(self, machine):
        self.launched.append(machine)
        return "i-0000000000000000b"


def test_running_after_retained_launches_the_next_instance_on_the_kept_volume(tmp_path: Path):
    cfg = config(tmp_path)
    store, machine = store_and_machine(cfg)
    first = record_compute(store, machine, cfg.availability_zone)
    store.set_desired(machine.machine_id, DesiredState.RETAINED, None)
    Reconciler(store, DeletionCloud(["terminated"]), instance_type=None).reconcile(
        machine.machine_id
    )
    bundled = with_bundle(store, machine.machine_id, 2)
    store.set_desired(machine.machine_id, DesiredState.RUNNING, None)
    cloud = LaunchCloud()

    relaunched = Reconciler(store, cloud, instance_type=None).reconcile(machine.machine_id)

    assert relaunched.instance_id == "i-0000000000000000b"
    assert relaunched.data_volume_id == first.data_volume_id
    assert relaunched.previous_instance_id == first.instance_id
    assert relaunched.instance_bundle == bundled.bundle
    [launched] = cloud.launched
    assert launched.instance_seq == 1
    assert launched.bundle == store.get(machine.machine_id).bundle
    tokens = Ec2Cloud(ec2_client(), cfg)
    assert tokens._token(launched, f"instance-{launched.instance_seq}") != tokens._token(
        first, f"instance-{first.instance_seq}"
    )
    assert tokens._token(launched, "data-volume") == tokens._token(first, "data-volume")
    store.close()


def test_a_new_instance_moves_onto_the_configured_image(tmp_path: Path):
    cfg = config(tmp_path)
    store, machine = store_and_machine(cfg)
    first = record_compute(store, machine, cfg.availability_zone)
    store.set_desired(machine.machine_id, DesiredState.RETAINED, None)
    Reconciler(store, DeletionCloud(["terminated"]), instance_type=None).reconcile(
        machine.machine_id
    )
    with_bundle(store, machine.machine_id, 2)
    store.set_desired(machine.machine_id, DesiredState.RUNNING, None)
    cloud = LaunchCloud(image_id="ami-22222222222222222")

    relaunched = Reconciler(store, cloud, instance_type=None).reconcile(machine.machine_id)

    [launched] = cloud.launched
    assert first.image_id == cfg.image_id
    assert launched.image_id == relaunched.image_id == "ami-22222222222222222"
    assert relaunched.data_volume_id == first.data_volume_id
    store.close()


@pytest.mark.parametrize(
    "configured, expected", [("r7i.2xlarge", "r7i.2xlarge"), (None, "m6i.large")]
)
def test_a_new_instance_moves_onto_the_configured_instance_type(
    tmp_path: Path, configured, expected
):
    cfg = config(tmp_path)
    store, machine = store_and_machine(cfg)
    first = record_compute(store, machine, cfg.availability_zone)
    store.set_desired(machine.machine_id, DesiredState.RETAINED, None)
    Reconciler(store, DeletionCloud(["terminated"]), instance_type=None).reconcile(
        machine.machine_id
    )
    with_bundle(store, machine.machine_id, 2)
    store.set_desired(machine.machine_id, DesiredState.RUNNING, None)
    cloud = LaunchCloud()

    relaunched = Reconciler(store, cloud, instance_type=configured).reconcile(machine.machine_id)

    [launched] = cloud.launched
    assert first.instance_type == "m6i.large"
    assert launched.instance_type == relaunched.instance_type == expected
    assert relaunched.data_volume_id == first.data_volume_id
    store.close()


def test_the_image_and_type_of_an_issued_launch_are_not_changed(tmp_path: Path):
    cfg = config(tmp_path)
    store, machine = store_and_machine(cfg)
    machine = store.record_volume(machine.machine_id, "vol-0123456789abcdef0", "us-east-1a")
    machine = store.mark_instance_launch_intent(machine)
    machine = store.mark_instance_launch_issued(machine, datetime.now(UTC))
    unchanged = store.use_launch_spec(machine, "ami-22222222222222222", "r7i.2xlarge")
    assert (unchanged.image_id, unchanged.instance_type) == (cfg.image_id, "m6i.large")
    store.close()


def test_deleted_keeps_the_volume_until_retain_until(tmp_path: Path, monkeypatch):
    cfg = config(tmp_path)
    store, machine = store_and_machine(cfg)
    machine = record_compute(store, machine, cfg.availability_zone)
    retain_until = datetime(2026, 1, 8, tzinfo=UTC)
    accept_delete(store, machine, retain_until)
    cloud = DeletionCloud(["terminated", None, None, None])
    reconciler = Reconciler(store, cloud, instance_type=None)

    monkeypatch.setattr(reconciler_module, "utcnow", lambda: retain_until - timedelta(seconds=1))
    assert reconciler.reconcile(machine.machine_id).observed_state is ObservedState.DELETING
    assert reconciler.reconcile(machine.machine_id).observed_state is ObservedState.DELETING
    assert "delete-volume" not in cloud.calls
    assert cloud.volume_exists

    monkeypatch.setattr(reconciler_module, "utcnow", lambda: retain_until)
    assert reconciler.reconcile(machine.machine_id).observed_state is ObservedState.DELETING
    assert cloud.calls[-1] == "delete-volume"
    assert not cloud.volume_exists
    assert reconciler.reconcile(machine.machine_id).observed_state is ObservedState.DELETED
    store.close()


def test_accepted_delete_waits_for_pending_and_deletes_volume_only_after_terminal(
    tmp_path: Path,
):
    cfg = config(tmp_path)
    store, machine = store_and_machine(cfg)
    machine = record_compute(store, machine, cfg.availability_zone)
    accept_delete(store, machine, None)
    cloud = DeletionCloud(["pending", "stopped", "terminated", None])
    reconciler = Reconciler(store, cloud, instance_type=None)

    assert reconciler.reconcile(machine.machine_id).observed_state is ObservedState.DELETING
    assert cloud.calls == []
    assert reconciler.reconcile(machine.machine_id).observed_state is ObservedState.DELETING
    assert cloud.calls == ["terminate-instance"]
    assert reconciler.reconcile(machine.machine_id).observed_state is ObservedState.DELETING
    assert cloud.calls == ["terminate-instance", "get-volume", "delete-volume"]
    assert not cloud.volume_exists
    deleted = reconciler.reconcile(machine.machine_id)

    assert deleted.observed_state is ObservedState.DELETED
    assert cloud.calls == ["terminate-instance", "get-volume", "delete-volume", "get-volume"]
    store.close()


def test_purged_recorded_instance_reads_as_not_found(tmp_path: Path):
    cfg = config(tmp_path)
    store, machine = store_and_machine(cfg)
    machine = store.record_instance(machine.machine_id, "i-0123456789abcdef0", None)
    client = ec2_client()
    cloud = Ec2Cloud(client, cfg)
    terminated = terminal_instance(cloud, machine)

    with Stubber(client) as stubber:
        stubber.add_response(
            "describe_instances", {"Reservations": []}, {"InstanceIds": [machine.instance_id]}
        )
        stubber.add_response(
            "describe_instances",
            {"Reservations": [{"Instances": [terminated, terminated]}]},
            {"InstanceIds": [machine.instance_id]},
        )
        assert cloud.get_instance(machine) is None
        with pytest.raises(CloudResourceError, match="exactly one instance"):
            cloud.get_instance(machine)
    store.close()


def stuck_purged_deletion(tmp_path: Path, *, terminal_observed: bool):
    cfg = config(tmp_path)
    store, machine = store_and_machine(cfg)
    machine = record_compute(store, machine, cfg.availability_zone)
    claim = accept_delete(store, machine, None)
    if terminal_observed:
        store.mark_instance_terminal_observed(machine.machine_id, machine.instance_id)
    store.set_observed(
        claim,
        ObservedState.NEEDS_ATTENTION,
        "recorded instance lookup did not return exactly one instance",
    )
    return cfg, store, store.get(machine.machine_id)


def test_deletion_completes_after_terminated_instance_is_purged(tmp_path: Path):
    cfg, store, machine = stuck_purged_deletion(tmp_path, terminal_observed=True)
    client = ec2_client()
    cloud = Ec2Cloud(client, cfg)

    with Stubber(client) as stubber:
        stubber.add_response(
            "describe_instances", {"Reservations": []}, {"InstanceIds": [machine.instance_id]}
        )
        stubber.add_response(
            "describe_volumes",
            {"Volumes": [volume(cfg, machine)]},
            {"VolumeIds": [machine.data_volume_id]},
        )
        stubber.add_response("delete_volume", {}, {"VolumeId": machine.data_volume_id})
        deleting = Reconciler(store, cloud, instance_type=None).reconcile(machine.machine_id)
        stubber.add_response(
            "describe_instances", {"Reservations": []}, {"InstanceIds": [machine.instance_id]}
        )
        stubber.add_client_error(
            "describe_volumes",
            service_error_code="InvalidVolume.NotFound",
            expected_params={"VolumeIds": [machine.data_volume_id]},
        )
        deleted = Reconciler(store, cloud, instance_type=None).reconcile(machine.machine_id)
        stubber.assert_no_pending_responses()
    assert deleting.observed_state is ObservedState.DELETING
    assert deleted.observed_state is ObservedState.DELETED
    assert deleted.error is None

    with Stubber(client):
        again = Reconciler(store, cloud, instance_type=None).reconcile(machine.machine_id)
    assert again.observed_state is ObservedState.DELETED
    store.close()


def test_purged_instance_without_terminal_evidence_needs_attention(tmp_path: Path):
    cfg, store, machine = stuck_purged_deletion(tmp_path, terminal_observed=False)
    client = ec2_client()
    cloud = Ec2Cloud(client, cfg)

    with Stubber(client) as stubber:
        stubber.add_response(
            "describe_instances", {"Reservations": []}, {"InstanceIds": [machine.instance_id]}
        )
        result = Reconciler(store, cloud, instance_type=None).reconcile(machine.machine_id)
    assert result.observed_state is ObservedState.NEEDS_ATTENTION
    assert result.error == "recorded instance state is unknown during deletion"
    store.close()


def test_completed_deletion_is_not_reconciled_again(tmp_path: Path):
    cfg = config(tmp_path)
    store, machine = store_and_machine(cfg)
    machine = record_compute(store, machine, cfg.availability_zone)
    accept_delete(store, machine, None)
    cloud = DeletionCloud(["terminated"])
    cloud.volume_exists = False
    reconciler = Reconciler(store, cloud, instance_type=None)
    assert reconciler.reconcile(machine.machine_id).observed_state is ObservedState.DELETED
    calls = list(cloud.calls)

    assert reconciler.reconcile(machine.machine_id).observed_state is ObservedState.DELETED
    assert cloud.calls == calls
    store.close()