Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
from __future__ import annotations
import json
import sqlite3
from dataclasses import replace
from datetime import UTC, datetime
from pathlib import Path
import boto3
import pytest
from botocore.stub import Stubber
from switch_hosted_controller.cloud import CloudCapacityError, CloudResourceError, Ec2Cloud
from switch_hosted_controller.config import ConfigError, ControllerConfig
from switch_hosted_controller.model import DesiredState, Machine, ObservedState
from switch_hosted_controller.reconciler import Reconciler, needs_new_user_data
from switch_hosted_controller.store import (
SLOT_ROWS_MESSAGE,
CapacityError,
MachineStore,
StoreError,
rebind_superseded_fingerprint,
)
MACHINE_ID = "3f1c2b4a-0000-4000-8000-000000000001"
INSTANCE_PROFILE_ARN = "arn:aws:iam::123456789012:instance-profile/switch-machine"
CORE_FIXTURES = (
Path(__file__).parents[4] / "core" / "tests" / "switch_core" / "fixtures" / "hosted_machines"
)
def config(tmp_path: Path, *, max_machines: int = 1) -> ControllerConfig:
return ControllerConfig.from_dict(config_dict(tmp_path, max_machines=max_machines))
def fixture_config(tmp_path: Path, instance_type: str) -> ControllerConfig:
"""The installation of the cross-stream fixtures: inst-test, with room for one machine."""
raw = config_dict(tmp_path, max_machines=1)
raw["installation_id"] = "inst-test"
raw["allowed_instance_types"] = [instance_type]
raw["instance_profile_arn"] = "arn:aws:iam::000000000000:instance-profile/switch-machine"
return ControllerConfig.from_dict(raw)
def config_dict(tmp_path: Path, *, max_machines: int) -> dict:
return {
"installation_id": "test-installation",
"region": "us-east-1",
"availability_zone": "us-east-1a",
"subnet_id": "subnet-0123456789abcdef0",
"security_group_ids": ["sg-0123456789abcdef0"],
"image_id": "ami-0123456789abcdef0",
"root_device_name": "/dev/xvda",
"allowed_instance_types": ["m6i.large"],
"max_machines": max_machines,
"root_volume_gib": 20,
"data_volume_gib": 40,
"instance_profile_arn": INSTANCE_PROFILE_ARN,
"state_db_path": str(tmp_path / "state.db"),
"lock_path": str(tmp_path / "controller.lock"),
"poll_interval_seconds": 1,
}
def insert_machine(store: MachineStore, cfg: ControllerConfig, machine_id: str) -> Machine:
return store.insert(
machine_id=machine_id,
core_revision=1,
instance_type="m6i.large",
image_id=cfg.image_id,
max_machines=cfg.max_machines,
)
SEAT_A = "5e1c2b4a-0000-4000-8000-0000000000a1"
SEAT_B = "5e1c2b4a-0000-4000-8000-0000000000b1"
def bundle_json(machine_id: str, enrollment_code: str) -> str:
return json.dumps(
{
"version": 5,
"installationId": "test-installation",
"machineId": machine_id,
"dataVolumeId": "vol-0123456789abcdef0",
"apiEndpoint": "https://switch.example.test/agent-api",
"controllers": [{"key": SEAT_A, "id": None, "enrollmentCode": enrollment_code}],
},
separators=(",", ":"),
)
def with_bundle(store: MachineStore, machine_id: str, revision: int) -> Machine:
token = f"bundle-{revision}"
store.require_bundle(machine_id, revision, token)
return store.record_bundle(
machine_id, token, bundle_json(machine_id, f"swce_SyntheticCodeRevision{revision:04d}")
)
def store_and_machine(cfg: ControllerConfig) -> tuple[MachineStore, Machine]:
store = MachineStore(cfg.state_db_path, cfg.fingerprint())
return store, insert_machine(store, cfg, MACHINE_ID)
def ec2_client():
return boto3.client(
"ec2",
region_name="us-east-1",
aws_access_key_id="test",
aws_secret_access_key="test",
aws_session_token="test",
)
def volume(cfg: ControllerConfig, machine, *, state: str = "available", attachments=None):
return {
"VolumeId": "vol-0123456789abcdef0",
"AvailabilityZone": cfg.availability_zone,
"Encrypted": True,
"KmsKeyId": "arn:aws:kms:us-east-1:123456789012:key/example",
"Size": cfg.data_volume_gib,
"VolumeType": "gp3",
"Iops": 3000,
"Throughput": 125,
"State": state,
"Attachments": attachments or [],
"Tags": Ec2Cloud(ec2_client(), cfg)._tags(machine, "data"),
}
def instance(cfg: ControllerConfig, machine, state: str):
return {
"InstanceId": "i-0123456789abcdef0",
"ImageId": cfg.image_id,
"InstanceType": machine.instance_type,
"State": {"Name": state, "Code": 16 if state == "running" else 80},
"IamInstanceProfile": {"Arn": cfg.instance_profile_arn, "Id": "AIPAEXAMPLE"},
"Placement": {"AvailabilityZone": cfg.availability_zone},
"SubnetId": cfg.subnet_id,
"SecurityGroups": [{"GroupId": cfg.security_group_ids[0], "GroupName": "worker"}],
"NetworkInterfaces": [
{
"NetworkInterfaceId": "eni-0123456789abcdef0",
"SubnetId": cfg.subnet_id,
"Groups": [{"GroupId": cfg.security_group_ids[0], "GroupName": "worker"}],
"Attachment": {"DeviceIndex": 0, "DeleteOnTermination": True},
}
],
"MetadataOptions": {
"State": "applied",
"HttpEndpoint": "enabled",
"HttpTokens": "required",
"HttpPutResponseHopLimit": 1,
"InstanceMetadataTags": "disabled",
},
"Tags": Ec2Cloud(ec2_client(), cfg)._tags(machine, "worker"),
}
def test_store_persists_intent_and_rejects_changed_deployment(tmp_path: Path):
cfg = config(tmp_path)
store, machine = store_and_machine(cfg)
machine = store.mark_volume_create_intent(machine)
store.mark_instance_launch_intent(machine)
store.close()
restarted = MachineStore(cfg.state_db_path, cfg.fingerprint())
assert restarted.get(MACHINE_ID).volume_create_intent
assert restarted.get(MACHINE_ID).instance_launch_intent
restarted.close()
changed = replace(cfg, installation_id="different-installation")
with pytest.raises(StoreError, match="immutable configuration"):
MachineStore(cfg.state_db_path, changed.fingerprint())
def test_capacity_counts_machines_not_deleted(tmp_path: Path):
cfg = config(tmp_path, max_machines=2)
store, first = store_and_machine(cfg)
second_id = "3f1c2b4a-0000-4000-8000-000000000002"
third_id = "3f1c2b4a-0000-4000-8000-000000000003"
second = insert_machine(store, cfg, second_id)
with pytest.raises(CapacityError):
insert_machine(store, cfg, third_id)
assert store.find(third_id) is None
second = store.set_desired(second.machine_id, DesiredState.STOPPED, None)
second = store.set_observed(second, ObservedState.STOPPED, None)
store.set_desired(second.machine_id, DesiredState.DELETED, None)
third = insert_machine(store, cfg, third_id)
assert [machine.machine_id for machine in store.list()] == [
first.machine_id,
second_id,
third.machine_id,
]
store.close()
def test_machines_are_keyed_by_machine_id(tmp_path: Path):
cfg = config(tmp_path, max_machines=2)
store, machine = store_and_machine(cfg)
other_id = "3f1c2b4a-0000-4000-8000-000000000002"
assert store.find(MACHINE_ID) == machine
assert store.find(other_id) is None
machine = store.set_desired(machine.machine_id, DesiredState.STOPPED, None)
machine = store.set_observed(machine, ObservedState.STOPPED, None)
machine = store.set_desired(machine.machine_id, DesiredState.DELETED, None)
store.set_observed(machine, ObservedState.DELETED, None)
with pytest.raises(StoreError, match="already exists"):
insert_machine(store, cfg, MACHINE_ID)
other = insert_machine(store, cfg, other_id)
assert store.find(other_id) == other
cloud = Ec2Cloud(ec2_client(), cfg)
assert cloud._token(machine, "data-volume") != cloud._token(other, "data-volume")
store.close()
def test_deleted_machine_keeps_the_latest_retention_deadline(tmp_path: Path):
cfg = config(tmp_path)
store, machine = store_and_machine(cfg)
machine = store.set_desired(machine.machine_id, DesiredState.RETAINED, None)
with pytest.raises(StoreError, match="freshly observed"):
store.set_desired(machine.machine_id, DesiredState.DELETED, None)
machine = store.set_observed(machine, ObservedState.RETAINED, None)
later = datetime(2026, 1, 8, tzinfo=UTC)
earlier = datetime(2026, 1, 1, tzinfo=UTC)
machine = store.set_desired(machine.machine_id, DesiredState.DELETED, later)
assert (
store.set_desired(machine.machine_id, DesiredState.DELETED, earlier).retain_until == later
)
assert store.set_desired(machine.machine_id, DesiredState.DELETED, None).retain_until == later
with pytest.raises(StoreError, match="cannot be restarted"):
store.set_desired(machine.machine_id, DesiredState.RUNNING, None)
store.close()
def test_retained_release_starts_the_next_instance_sequence(tmp_path: Path):
cfg = config(tmp_path)
store, machine = store_and_machine(cfg)
store.record_volume(machine.machine_id, "vol-0123456789abcdef0", cfg.availability_zone)
machine = store.record_instance(machine.machine_id, "i-0123456789abcdef0", "bundle-1")
machine = store.set_desired(machine.machine_id, DesiredState.RETAINED, None)
with pytest.raises(StoreError, match="terminated"):
store.release_terminated(machine)
machine = store.mark_instance_terminal_observed(machine.machine_id, machine.instance_id)
released = store.release_terminated(machine)
assert released.instance_id is None
assert released.previous_instance_id == "i-0123456789abcdef0"
assert released.data_volume_id == "vol-0123456789abcdef0"
assert released.instance_seq == machine.instance_seq + 1
assert released.recovery_count == 0
assert machine.instance_bundle == "bundle-1"
assert released.instance_bundle is None
store.close()
def test_uncertain_launch_then_stop_waits_for_late_instance(tmp_path: Path):
cfg = config(tmp_path)
store, machine = store_and_machine(cfg)
machine = store.mark_volume_create_intent(machine)
machine = store.record_volume(
machine.machine_id, "vol-0123456789abcdef0", cfg.availability_zone
)
machine = store.mark_instance_launch_intent(machine)
store.mark_instance_launch_issued(machine, datetime.now(UTC))
store.set_desired(machine.machine_id, DesiredState.STOPPED, None)
client = ec2_client()
cloud = Ec2Cloud(client, cfg)
filters = cloud._resource_filters(machine, "worker")
worker = instance(cfg, store.get(machine.machine_id), "running")
stopped_worker = {**worker, "State": {"Name": "stopped", "Code": 80}}
with Stubber(client) as stubber:
stubber.add_response("describe_instances", {"Reservations": []}, {"Filters": filters})
stubber.add_response(
"describe_instances", {"Reservations": [{"Instances": [worker]}]}, {"Filters": filters}
)
stubber.add_response(
"describe_instances",
{"Reservations": [{"Instances": [worker]}]},
{"InstanceIds": [worker["InstanceId"]]},
)
stubber.add_response(
"stop_instances",
{"StoppingInstances": []},
{"InstanceIds": [worker["InstanceId"]], "Force": False},
)
stubber.add_response(
"describe_instances",
{"Reservations": [{"Instances": [stopped_worker]}]},
{"InstanceIds": [worker["InstanceId"]]},
)
reconciler = Reconciler(store, cloud, instance_type=None)
assert (
reconciler.reconcile(machine.machine_id).observed_state is ObservedState.NEEDS_ATTENTION
)
adopted = reconciler.reconcile(machine.machine_id)
assert adopted.instance_id == worker["InstanceId"]
assert reconciler.reconcile(machine.machine_id).observed_state is ObservedState.STOPPING
assert reconciler.reconcile(machine.machine_id).observed_state is ObservedState.STOPPED
store.close()
def test_cloud_capacity_counts_all_pages(tmp_path: Path):
cfg = config(tmp_path)
store, machine = store_and_machine(cfg)
machine = store.record_volume(
machine.machine_id, "vol-0123456789abcdef0", cfg.availability_zone
)
client = ec2_client()
cloud = Ec2Cloud(client, cfg)
expected_filters = [
{"Name": "tag:switch:installation-id", "Values": [cfg.installation_id]},
{"Name": "tag:switch:managed-by", "Values": ["switch-hosted-controller"]},
{
"Name": "instance-state-name",
"Values": ["pending", "running", "stopping", "stopped", "shutting-down"],
},
]
with Stubber(client) as stubber:
stubber.add_response(
"describe_instances",
{"Reservations": [], "NextToken": "page-2"},
{"Filters": expected_filters},
)
stubber.add_response(
"describe_instances",
{"Reservations": [{"Instances": [{"InstanceId": "i-foreign"}]}]},
{"Filters": expected_filters, "NextToken": "page-2"},
)
with pytest.raises(CloudCapacityError):
cloud.validate_capacity()
store.close()
def test_foreign_recorded_instance_fails_closed(tmp_path: Path):
cfg = config(tmp_path)
store, machine = store_and_machine(cfg)
machine = store.record_instance(machine.machine_id, "i-0123456789abcdef0", None)
foreign = instance(cfg, machine, "running")
foreign["Tags"] = [
{"Key": "switch:installation-id", "Value": "other-installation"},
{"Key": "switch:machine-id", "Value": machine.machine_id},
]
client = ec2_client()
with Stubber(client) as stubber:
stubber.add_response(
"describe_instances",
{"Reservations": [{"Instances": [foreign]}]},
{"InstanceIds": [machine.instance_id]},
)
with pytest.raises(CloudResourceError, match="ownership tag"):
Ec2Cloud(client, cfg).get_instance(machine)
store.close()
def test_stop_and_start_use_only_recorded_instance(tmp_path: Path):
cfg = config(tmp_path)
store, machine = store_and_machine(cfg)
machine = store.record_instance(machine.machine_id, "i-0123456789abcdef0", None)
client = ec2_client()
with Stubber(client) as stubber:
stubber.add_response(
"stop_instances",
{"StoppingInstances": []},
{"InstanceIds": [machine.instance_id], "Force": False},
)
stubber.add_response(
"start_instances", {"StartingInstances": []}, {"InstanceIds": [machine.instance_id]}
)
cloud = Ec2Cloud(client, cfg)
cloud.stop_instance(machine)
cloud.start_instance(machine)
store.close()
REVISION_1_BUNDLE = bundle_json(MACHINE_ID, "swce_SyntheticCodeRevision0001")
REVISION_2_BUNDLE = bundle_json(MACHINE_ID, "swce_SyntheticCodeRevision0002")
def stopped_instance_with_bundle(cfg: ControllerConfig, instance_bundle: str | None):
store, machine = store_and_machine(cfg)
store.record_volume(machine.machine_id, "vol-0123456789abcdef0", cfg.availability_zone)
store.record_instance(machine.machine_id, "i-0123456789abcdef0", instance_bundle)
return store, with_bundle(store, machine.machine_id, 2)
@pytest.mark.parametrize("instance_bundle", [REVISION_1_BUNDLE, None])
def test_an_instance_with_a_stale_bundle_makes_way_for_one_launched_with_the_new_one(
tmp_path: Path, instance_bundle
):
cfg = config(tmp_path)
store, machine = stopped_instance_with_bundle(cfg, instance_bundle)
attachment = {"InstanceId": machine.instance_id, "Device": "/dev/sdf", "State": "attached"}
client = ec2_client()
with Stubber(client) as stubber:
stubber.add_response(
"describe_volumes",
{"Volumes": [volume(cfg, machine, state="in-use", attachments=[attachment])]},
{"VolumeIds": [machine.data_volume_id]},
)
stubber.add_response(
"describe_instances",
{"Reservations": [{"Instances": [instance(cfg, machine, "stopped")]}]},
{"InstanceIds": [machine.instance_id]},
)
stubber.add_response(
"terminate_instances",
{"TerminatingInstances": []},
{"InstanceIds": [machine.instance_id]},
)
result = Reconciler(store, Ec2Cloud(client, cfg), instance_type=None).reconcile(
machine.machine_id
)
stubber.assert_no_pending_responses()
assert result.observed_state is ObservedState.PROVISIONING
assert result.instance_terminate_issued
assert result.instance_id == "i-0123456789abcdef0"
assert result.data_volume_id == "vol-0123456789abcdef0"
store.close()
def test_starting_an_instance_with_the_current_bundle_only_starts_it(tmp_path: Path):
cfg = config(tmp_path)
store, machine = stopped_instance_with_bundle(cfg, REVISION_2_BUNDLE)
client = ec2_client()
with Stubber(client) as stubber:
stubber.add_response(
"describe_volumes",
{"Volumes": [volume(cfg, machine)]},
{"VolumeIds": [machine.data_volume_id]},
)
stubber.add_response(
"describe_instances",
{"Reservations": [{"Instances": [instance(cfg, machine, "stopped")]}]},
{"InstanceIds": [machine.instance_id]},
)
stubber.add_response(
"start_instances", {"StartingInstances": []}, {"InstanceIds": [machine.instance_id]}
)
result = Reconciler(store, Ec2Cloud(client, cfg), instance_type=None).reconcile(
machine.machine_id
)
stubber.assert_no_pending_responses()
assert result.observed_state is ObservedState.PROVISIONING
assert result.instance_bundle == REVISION_2_BUNDLE
store.close()
def _with_controller(
bundle: str, controller_id: str | None, enrollment_code: str | None, key: str = SEAT_A
) -> str:
value = json.loads(bundle)
entry = {"key": key, "id": controller_id, "enrollmentCode": enrollment_code}
controllers = [item for item in value["controllers"] if item["key"] != key]
if len(controllers) == len(value["controllers"]):
value["controllers"] = [*controllers, entry]
else:
value["controllers"] = [
entry if item["key"] == key else item for item in value["controllers"]
]
return json.dumps(value, separators=(",", ":"))
def _without_seat(bundle: str, key: str) -> str:
value = json.loads(bundle)
value["controllers"] = [item for item in value["controllers"] if item["key"] != key]
return json.dumps(value, separators=(",", ":"))
def _as_version_4(bundle: str) -> str:
value = json.loads(bundle)
[entry] = value.pop("controllers")
value["version"] = 4
value["controller"] = {"id": entry["id"], "enrollmentCode": entry["enrollmentCode"]}
return json.dumps(value, separators=(",", ":"))
CONTROLLER_A = "9a1c2b4a-0000-4000-8000-0000000000aa"
CONTROLLER_B = "9a1c2b4a-0000-4000-8000-0000000000bb"
@pytest.mark.parametrize(
("current", "wanted", "replaced"),
[
(None, REVISION_2_BUNDLE, True),
(REVISION_2_BUNDLE, REVISION_2_BUNDLE, False),
(REVISION_1_BUNDLE, REVISION_2_BUNDLE, True),
(REVISION_1_BUNDLE, _with_controller(REVISION_1_BUNDLE, CONTROLLER_A, None), False),
(
_with_controller(REVISION_1_BUNDLE, CONTROLLER_A, None),
_with_controller(REVISION_1_BUNDLE, CONTROLLER_A, None),
False,
),
(
_with_controller(REVISION_1_BUNDLE, CONTROLLER_A, None),
_with_controller(REVISION_1_BUNDLE, CONTROLLER_B, None),
True,
),
(_with_controller(REVISION_1_BUNDLE, CONTROLLER_A, None), REVISION_2_BUNDLE, True),
(
REVISION_1_BUNDLE,
REVISION_1_BUNDLE.replace("switch.example.test", "other.example.test"),
True,
),
(
REVISION_1_BUNDLE,
_with_controller(
REVISION_1_BUNDLE.replace("switch.example.test", "other.example.test"),
CONTROLLER_A,
None,
),
True,
),
],
)
def test_user_data_is_replaced_only_when_the_boot_would_need_it(current, wanted, replaced):
assert needs_new_user_data(current, wanted) is replaced
TWO_SEATS = _with_controller(REVISION_1_BUNDLE, None, "swce_SyntheticSeatBRevision0001", SEAT_B)
TWO_SEATS_ENROLLED_A = _with_controller(TWO_SEATS, CONTROLLER_A, None)
@pytest.mark.parametrize(
("current", "wanted", "replaced"),
[
(REVISION_1_BUNDLE, TWO_SEATS, True),
(TWO_SEATS, REVISION_1_BUNDLE, True),
(TWO_SEATS, _without_seat(TWO_SEATS, SEAT_A), True),
(TWO_SEATS, TWO_SEATS, False),
(TWO_SEATS, TWO_SEATS_ENROLLED_A, False),
(
TWO_SEATS,
_with_controller(TWO_SEATS_ENROLLED_A, CONTROLLER_B, None, SEAT_B),
False,
),
(
TWO_SEATS,
_with_controller(TWO_SEATS_ENROLLED_A, None, "swce_SyntheticSeatBRevision0002", SEAT_B),
True,
),
(TWO_SEATS_ENROLLED_A, TWO_SEATS_ENROLLED_A, False),
(TWO_SEATS_ENROLLED_A, _with_controller(TWO_SEATS, CONTROLLER_B, None), True),
(TWO_SEATS_ENROLLED_A, TWO_SEATS, True),
(
TWO_SEATS_ENROLLED_A,
_with_controller(TWO_SEATS, None, "swce_SyntheticCodeRevision0002"),
True,
),
(
_with_controller(REVISION_1_BUNDLE, CONTROLLER_A, None),
_with_controller(TWO_SEATS_ENROLLED_A, None, "swce_SyntheticSeatCRevision0001", SEAT_B),
True,
),
(
TWO_SEATS,
json.dumps(
{
**json.loads(TWO_SEATS),
"controllers": json.loads(TWO_SEATS)["controllers"][::-1],
},
separators=(",", ":"),
),
False,
),
],
)
def test_user_data_is_replaced_per_workspace_seat(current, wanted, replaced):
assert needs_new_user_data(current, wanted) is replaced
@pytest.mark.parametrize(
("current", "wanted", "replaced"),
[
(_as_version_4(REVISION_1_BUNDLE), REVISION_1_BUNDLE, True),
(_as_version_4(REVISION_1_BUNDLE), _as_version_4(REVISION_1_BUNDLE), False),
(
_as_version_4(REVISION_1_BUNDLE),
_as_version_4(_with_controller(REVISION_1_BUNDLE, CONTROLLER_A, None)),
False,
),
(_as_version_4(REVISION_1_BUNDLE), _as_version_4(REVISION_2_BUNDLE), True),
(json.dumps({"version": 3}), REVISION_1_BUNDLE, True),
],
)
def test_user_data_of_another_bundle_version_is_replaced(current, wanted, replaced):
assert needs_new_user_data(current, wanted) is replaced
def test_starting_an_enrolled_instance_keeps_the_user_data_it_enrolled_with(tmp_path: Path):
cfg = config(tmp_path)
store, machine = stopped_instance_with_bundle(cfg, REVISION_1_BUNDLE)
enrolled = _with_controller(REVISION_1_BUNDLE, CONTROLLER_A, None)
store.require_bundle(machine.machine_id, 3, "bundle-3")
machine = store.record_bundle(machine.machine_id, "bundle-3", enrolled)
client = ec2_client()
with Stubber(client) as stubber:
stubber.add_response(
"describe_volumes",
{"Volumes": [volume(cfg, machine)]},
{"VolumeIds": [machine.data_volume_id]},
)
stubber.add_response(
"describe_instances",
{"Reservations": [{"Instances": [instance(cfg, machine, "stopped")]}]},
{"InstanceIds": [machine.instance_id]},
)
stubber.add_response(
"start_instances", {"StartingInstances": []}, {"InstanceIds": [machine.instance_id]}
)
result = Reconciler(store, Ec2Cloud(client, cfg), instance_type=None).reconcile(
machine.machine_id
)
stubber.assert_no_pending_responses()
assert result.instance_bundle == REVISION_1_BUNDLE
store.close()
def config_raw(tmp_path: Path, **overrides) -> dict:
return {**config_dict(tmp_path, max_machines=1), **overrides}
@pytest.mark.parametrize(
"arn",
[
"arn:aws:iam::123456789012:role/switch-machine",
"arn:aws:secretsmanager:us-east-1:123456789012:secret:switch-machine",
"arn:aws:iam::123456789012:instance-profile",
"instance-profile/switch-machine",
None,
],
)
def test_config_requires_an_instance_profile_arn(tmp_path: Path, arn):
with pytest.raises(ConfigError, match="instance_profile_arn"):
ControllerConfig.from_dict(config_raw(tmp_path, instance_profile_arn=arn))
def test_config_refuses_machine_slots(tmp_path: Path):
raw = config_raw(tmp_path)
del raw["instance_profile_arn"]
raw["machine_slots"] = {
"slot-a": {
"instance_profile_arn": INSTANCE_PROFILE_ARN,
"assignment_secret_arn": "arn:aws:secretsmanager:us-east-1:123456789012:secret:a",
}
}
with pytest.raises(
ConfigError, match=r"missing=\['instance_profile_arn'\], unknown=\['machine_slots'\]"
):
ControllerConfig.from_dict(raw)
@pytest.mark.parametrize(
("max_machines", "valid"), [(0, False), (1, True), (100, True), (101, False)]
)
def test_capacity_is_bounded_by_configuration_alone(tmp_path: Path, max_machines, valid):
raw = config_raw(tmp_path, max_machines=max_machines)
if valid:
assert ControllerConfig.from_dict(raw).max_machines == max_machines
else:
with pytest.raises(ConfigError, match="max_machines"):
ControllerConfig.from_dict(raw)
def test_recovery_requires_terminated_predecessor_and_retains_disk(tmp_path):
cfg = config(tmp_path)
store, machine = store_and_machine(cfg)
store.record_volume(machine.machine_id, "vol-0123456789abcdef0", cfg.availability_zone)
machine = store.record_instance(machine.machine_id, "i-0123456789abcdef0", "bundle-1")
with pytest.raises(StoreError, match="terminated"):
store.replace_terminated(machine, unexpected=True)
machine = store.mark_instance_terminal_observed(machine.machine_id, machine.instance_id)
replacement = store.replace_terminated(machine, unexpected=True)
assert replacement.instance_id is None
assert replacement.previous_instance_id == "i-0123456789abcdef0"
assert replacement.data_volume_id == "vol-0123456789abcdef0"
assert replacement.recovery_count == 1
assert replacement.instance_seq == 1
assert replacement.instance_bundle is None
assert not replacement.instance_launch_issued
cloud = Ec2Cloud(ec2_client(), cfg)
assert cloud._token(machine, "instance-0") != cloud._token(replacement, "instance-1")
for index in range(2, 5):
current = store.record_instance(machine.machine_id, f"i-{index:017x}", None)
current = store.mark_instance_terminal_observed(machine.machine_id, current.instance_id)
if index == 4:
with pytest.raises(StoreError, match="limit"):
store.replace_terminated(current, unexpected=True)
else:
store.replace_terminated(current, unexpected=True)
store.close()
def legacy_database(path: Path, rows: list[tuple[str, str]]) -> None:
path.parent.mkdir(parents=True, exist_ok=True)
connection = sqlite3.connect(path)
connection.execute(
"CREATE TABLE agents (agent_id TEXT PRIMARY KEY, desired_state TEXT, observed_state TEXT)"
)
connection.executemany(
"INSERT INTO agents VALUES (?, ?, ?)",
[(f"agent-{index}", *row) for index, row in enumerate(rows)],
)
connection.commit()
connection.close()
@pytest.mark.parametrize(
"row", [("running", "running"), ("deleted", "deleting"), ("stopped", "deleted")]
)
def test_store_refuses_live_legacy_agent_rows(tmp_path, row):
cfg = config(tmp_path)
legacy_database(cfg.state_db_path, [("deleted", "deleted"), row])
with pytest.raises(StoreError) as raised:
MachineStore(cfg.state_db_path, cfg.fingerprint())
assert str(raised.value) == (
"legacy per-agent rows present; see 'Moving to one machine per user' in deploy/hosted/README.md"
)
def test_store_opens_beside_deleted_legacy_rows_and_keeps_them(tmp_path):
cfg = config(tmp_path)
legacy_database(cfg.state_db_path, [("deleted", "deleted"), ("deleted", "deleted")])
store, machine = store_and_machine(cfg)
assert store.find(MACHINE_ID) == machine
store.close()
connection = sqlite3.connect(cfg.state_db_path)
assert connection.execute("SELECT COUNT(*) FROM agents").fetchone()[0] == 2
connection.close()
def slot_era_database(cfg: ControllerConfig, rows: list[tuple[str, str]]) -> None:
cfg.state_db_path.parent.mkdir(parents=True, exist_ok=True)
connection = sqlite3.connect(cfg.state_db_path)
connection.executescript(
"""
CREATE TABLE machines (
machine_id TEXT NOT NULL UNIQUE,
slot_id TEXT NOT NULL,
generation INTEGER NOT NULL,
desired_state TEXT NOT NULL,
observed_state TEXT NOT NULL,
assignment_secret_arn TEXT NOT NULL,
instance_profile_arn TEXT NOT NULL,
PRIMARY KEY (slot_id, generation)
);
CREATE UNIQUE INDEX machines_one_live_row_per_slot
ON machines (slot_id) WHERE observed_state <> 'deleted';
CREATE TABLE controller_metadata (key TEXT PRIMARY KEY, value TEXT NOT NULL);
"""
)
connection.execute(
"INSERT INTO controller_metadata VALUES ('controller_fingerprint', ?)",
(cfg.fingerprint(),),
)
connection.executemany(
"INSERT INTO machines VALUES (?, 'slot-a', ?, ?, ?, 'secret-arn', 'profile-arn')",
[
(f"3f1c2b4a-0000-4000-8000-0000000000{generation:02d}", generation, *row)
for generation, row in enumerate(rows, start=1)
],
)
connection.commit()
connection.close()
def test_slot_era_database_with_only_deleted_machines_is_reopened_clean(tmp_path):
cfg = config(tmp_path)
slot_era_database(cfg, [("deleted", "deleted"), ("deleted", "deleted")])
store = MachineStore(cfg.state_db_path, cfg.fingerprint())
assert store.list() == []
machine = insert_machine(store, cfg, MACHINE_ID)
assert store.list() == [machine]
store.close()
connection = sqlite3.connect(cfg.state_db_path)
columns = {row[1] for row in connection.execute("PRAGMA table_info(machines)")}
indexes = connection.execute(
"SELECT name FROM sqlite_master WHERE type = 'index' AND name = 'machines_one_live_row_per_slot'"
).fetchall()
connection.close()
assert "slot_id" not in columns
assert {"bundle", "instance_bundle"} <= columns
assert "instance_bundle_token" not in columns
assert indexes == []
reopened = MachineStore(cfg.state_db_path, cfg.fingerprint())
assert reopened.list() == [machine]
reopened.close()
with pytest.raises(StoreError, match="immutable configuration"):
MachineStore(cfg.state_db_path, replace(cfg, installation_id="other").fingerprint())
@pytest.mark.parametrize(
"row",
[
("running", "running"),
("stopped", "stopped"),
("retained", "retained"),
("deleted", "deleting"),
("running", "needs_attention"),
],
)
def test_slot_era_database_with_a_live_machine_is_refused(tmp_path, row):
cfg = config(tmp_path)
slot_era_database(cfg, [("deleted", "deleted"), row])
with pytest.raises(StoreError) as raised:
MachineStore(cfg.state_db_path, cfg.fingerprint())
assert str(raised.value) == SLOT_ROWS_MESSAGE
connection = sqlite3.connect(cfg.state_db_path)
assert connection.execute("SELECT COUNT(*) FROM machines").fetchone()[0] == 2
connection.close()
def test_capacity_image_and_instance_types_can_change_but_placement_cannot(tmp_path):
cfg = config(tmp_path)
MachineStore(cfg.state_db_path, cfg.fingerprint()).close()
expanded = config(tmp_path, max_machines=2)
current = MachineStore(expanded.state_db_path, expanded.fingerprint())
current.close()
new_image = replace(expanded, image_id="ami-11111111111111111")
MachineStore(new_image.state_db_path, new_image.fingerprint()).close()
new_types = replace(new_image, allowed_instance_types=frozenset({"r7i.2xlarge"}))
MachineStore(new_types.state_db_path, new_types.fingerprint()).close()
incompatible = replace(expanded, subnet_id="subnet-11111111111111111")
with pytest.raises(StoreError, match="immutable"):
MachineStore(incompatible.state_db_path, incompatible.fingerprint())
def test_a_database_bound_to_the_superseded_fingerprint_is_rebound(tmp_path):
cfg = config(tmp_path)
MachineStore(cfg.state_db_path, cfg.superseded_fingerprint()).close()
with pytest.raises(StoreError, match="immutable"):
MachineStore(cfg.state_db_path, cfg.fingerprint())
rebind_superseded_fingerprint(
cfg.state_db_path, cfg.superseded_fingerprint(), cfg.fingerprint()
)
MachineStore(cfg.state_db_path, cfg.fingerprint()).close()
def test_rebinding_leaves_another_placement_refused(tmp_path):
cfg = config(tmp_path)
other = replace(cfg, subnet_id="subnet-11111111111111111")
MachineStore(cfg.state_db_path, other.superseded_fingerprint()).close()
rebind_superseded_fingerprint(
cfg.state_db_path, cfg.superseded_fingerprint(), cfg.fingerprint()
)
with pytest.raises(StoreError, match="immutable"):
MachineStore(cfg.state_db_path, cfg.fingerprint())
def test_rebinding_a_missing_database_creates_nothing(tmp_path):
cfg = config(tmp_path)
rebind_superseded_fingerprint(
cfg.state_db_path, cfg.superseded_fingerprint(), cfg.fingerprint()
)
assert not cfg.state_db_path.exists()
def test_an_upgrade_can_change_only_the_instance_type(tmp_path):
cfg = config(tmp_path)
store, machine = store_and_machine(cfg)
store.record_volume(machine.machine_id, "vol-0123456789abcdef0", cfg.availability_zone)
machine = store.record_instance(machine.machine_id, "i-0123456789abcdef0", "bundle-1")
machine = store.set_desired(machine.machine_id, DesiredState.STOPPED, None)
machine = store.mark_instance_terminal_observed(machine.machine_id, machine.instance_id)
with pytest.raises(StoreError, match="already uses this image and instance type"):
store.upgrade_terminated(machine, machine.image_id, machine.instance_type)
upgraded = store.upgrade_terminated(machine, machine.image_id, "r7i.2xlarge")
assert upgraded.instance_type == "r7i.2xlarge"
assert upgraded.image_id == machine.image_id
assert upgraded.instance_id is None
assert upgraded.data_volume_id == machine.data_volume_id
store.close()
def test_image_upgrade_requires_stopped_terminal_claim_and_preserves_disk(tmp_path):
cfg = config(tmp_path)
store, machine = store_and_machine(cfg)
store.record_volume(machine.machine_id, "vol-0123456789abcdef0", cfg.availability_zone)
machine = store.record_instance(machine.machine_id, "i-0123456789abcdef0", "bundle-1")
with pytest.raises(StoreError, match="stopped"):
store.upgrade_terminated(machine, "ami-11111111111111111", machine.instance_type)
machine = store.set_desired(machine.machine_id, DesiredState.STOPPED, None)
with pytest.raises(StoreError, match="terminated"):
store.upgrade_terminated(machine, "ami-11111111111111111", machine.instance_type)
machine = store.mark_instance_terminal_observed(machine.machine_id, machine.instance_id)
upgraded = store.upgrade_terminated(machine, "ami-11111111111111111", machine.instance_type)
assert upgraded.instance_id is None
assert upgraded.previous_instance_id == machine.instance_id
assert upgraded.data_volume_id == machine.data_volume_id
assert upgraded.desired_state is DesiredState.STOPPED
assert upgraded.image_id != machine.image_id
assert upgraded.instance_bundle is None
with pytest.raises(StoreError, match="changed"):
store.upgrade_terminated(machine, "ami-11111111111111111", machine.instance_type)
store.close()
@pytest.mark.parametrize("desired", [DesiredState.RUNNING, DesiredState.STOPPED])
def test_terminating_worker_waits_without_profile_or_replacement(tmp_path, desired):
cfg = config(tmp_path)
store, machine = store_and_machine(cfg)
machine = store.record_volume(
machine.machine_id, "vol-0123456789abcdef0", cfg.availability_zone
)
machine = store.record_instance(machine.machine_id, "i-0123456789abcdef0", None)
machine = store.set_desired(machine.machine_id, desired, None)
worker = instance(cfg, machine, "shutting-down")
worker.pop("IamInstanceProfile")
worker.pop("NetworkInterfaces")
client = ec2_client()
with Stubber(client) as stubber:
if desired is DesiredState.RUNNING:
stubber.add_response(
"describe_volumes",
{"Volumes": [volume(cfg, machine)]},
{"VolumeIds": [machine.data_volume_id]},
)
stubber.add_response(
"describe_instances",
{"Reservations": [{"Instances": [worker]}]},
{"InstanceIds": [machine.instance_id]},
)
result = Reconciler(store, Ec2Cloud(client, cfg), instance_type=None).reconcile(
machine.machine_id
)
expected = (
ObservedState.PROVISIONING
if desired is DesiredState.RUNNING
else ObservedState.STOPPING
)
assert result.observed_state is expected
assert result.instance_id == machine.instance_id
assert result.error is None
stubber.assert_no_pending_responses()
store.close()
@pytest.mark.parametrize("current", [True, False])
def test_a_database_that_kept_bundle_tokens_keeps_the_current_instance_bundle(
tmp_path: Path, current: bool
):
cfg = config(tmp_path)
store, machine = stopped_instance_with_bundle(cfg, None)
store.close()
connection = sqlite3.connect(cfg.state_db_path)
connection.executescript(
"""
ALTER TABLE machines ADD COLUMN instance_bundle_token TEXT;
ALTER TABLE machines DROP COLUMN instance_bundle;
"""
)
connection.execute(
"UPDATE machines SET instance_bundle_token = ?",
(machine.bundle_token if current else "bundle-1",),
)
connection.commit()
connection.close()
reopened = MachineStore(cfg.state_db_path, cfg.fingerprint())
assert reopened.get(machine.machine_id).instance_bundle == (machine.bundle if current else None)
reopened.close()
connection = sqlite3.connect(cfg.state_db_path)
columns = {row[1] for row in connection.execute("PRAGMA table_info(machines)")}
connection.close()
assert "instance_bundle_token" not in columns