Open-source AI teammates in Slack, Teams, Discord, Telegram and Mattermost. Self-host it, with any agent harness and any model.
Python agent-orchestrationaiai-agentschatopsdiscordlangchainmcpmicrosoft-teamsmulti-agentopen-sourceopenaiself-hostedslack
from __future__ import annotations
import hashlib
from typing import Any
from botocore.exceptions import ClientError
from .config import ControllerConfig
from .model import Machine
MANAGED_BY = "switch-hosted-controller"
DATA_DEVICE = "/dev/sdf"
class CloudResourceError(RuntimeError):
pass
class CloudCapacityError(CloudResourceError):
pass
class Ec2Cloud:
def __init__(self, ec2: Any, config: ControllerConfig):
self._ec2 = ec2
self._config = config
@property
def availability_zone(self) -> str:
return self._config.availability_zone
@property
def image_id(self) -> str:
"""The image a new instance launches with."""
return self._config.image_id
def validate_image(self, machine: Machine) -> None:
images = self._ec2.describe_images(ImageIds=[machine.image_id]).get("Images", [])
if len(images) != 1:
raise CloudResourceError("configured AMI lookup did not return exactly one image")
image = images[0]
if image.get("State") != "available":
raise CloudResourceError("configured AMI is not available")
if image.get("Architecture") != "x86_64" or image.get("VirtualizationType") != "hvm":
raise CloudResourceError("configured AMI must be x86_64 HVM")
if image.get("RootDeviceType") != "ebs":
raise CloudResourceError("configured AMI must use an EBS root")
if image.get("RootDeviceName") != self._config.root_device_name:
raise CloudResourceError("configured root_device_name differs from the AMI")
mappings = image.get("BlockDeviceMappings", [])
if len(mappings) != 1 or mappings[0].get("DeviceName") != self._config.root_device_name:
raise CloudResourceError("configured AMI must define exactly one root block device")
if not mappings[0].get("Ebs") or mappings[0].get("NoDevice"):
raise CloudResourceError("configured AMI root mapping is not an EBS device")
def discover_volume(self, machine: Machine) -> dict[str, Any] | None:
volumes = self._describe_volumes(Filters=self._resource_filters(machine, "data"))
if len(volumes) > 1:
raise CloudResourceError("multiple controller-owned data volumes exist for the machine")
if not volumes:
return None
self._validate_volume(volumes[0], machine)
return volumes[0]
def get_volume(self, machine: Machine) -> dict[str, Any] | None:
if machine.data_volume_id is None:
return self.discover_volume(machine)
try:
volumes = self._describe_volumes(VolumeIds=[machine.data_volume_id])
except ClientError as exc:
if _error_code(exc) == "InvalidVolume.NotFound":
return None
raise
if len(volumes) != 1:
raise CloudResourceError("recorded volume lookup did not return exactly one volume")
self._validate_volume(volumes[0], machine)
return volumes[0]
def create_volume(self, machine: Machine) -> str:
response = self._ec2.create_volume(
AvailabilityZone=self._config.availability_zone,
Encrypted=True,
VolumeType="gp3",
KmsKeyId="alias/aws/ebs",
Iops=3000,
Throughput=125,
Size=self._config.data_volume_gib,
ClientToken=self._token(machine, "data-volume"),
TagSpecifications=[{"ResourceType": "volume", "Tags": self._tags(machine, "data")}],
)
return response["VolumeId"]
def discover_instance(self, machine: Machine) -> dict[str, Any] | None:
instances = self._describe_instances(Filters=self._resource_filters(machine, "worker"))
active = [instance for instance in instances if instance["State"]["Name"] != "terminated"]
if len(active) > 1:
raise CloudResourceError("multiple controller-owned instances exist for the machine")
if not active:
return None
self._validate_instance(active[0], machine)
return active[0]
def find_launched_instance(self, machine: Machine) -> dict[str, Any] | None:
"""The instance, in any state, launched with this machine's current launch token."""
instances = self._describe_instances(
Filters=[
{"Name": "client-token", "Values": [self._launch_token(machine)]},
*self._resource_filters(machine, "worker"),
]
)
if len(instances) > 1:
raise CloudResourceError("multiple instances share the machine's launch token")
if not instances:
return None
self._validate_instance(instances[0], machine)
return instances[0]
def get_instance(self, machine: Machine) -> dict[str, Any] | None:
if machine.instance_id is None:
return self.discover_instance(machine)
try:
instances = self._describe_instances(InstanceIds=[machine.instance_id])
except ClientError as exc:
if _error_code(exc) == "InvalidInstanceID.NotFound":
return None
raise
if not instances:
return None
if len(instances) != 1:
raise CloudResourceError("recorded instance lookup did not return exactly one instance")
self._validate_instance(instances[0], machine)
return instances[0]
def validate_capacity(self) -> None:
active = self._describe_instances(
Filters=[
{"Name": "tag:switch:installation-id", "Values": [self._config.installation_id]},
{"Name": "tag:switch:managed-by", "Values": [MANAGED_BY]},
{
"Name": "instance-state-name",
"Values": ["pending", "running", "stopping", "stopped", "shutting-down"],
},
]
)
if len(active) >= self._config.max_machines:
raise CloudCapacityError("cloud instance capacity is exhausted")
def run_instance(self, machine: Machine) -> str:
if machine.data_volume_id is None:
raise CloudResourceError("cannot launch without a recorded data volume")
if machine.bundle is None:
raise CloudResourceError("cannot launch without a prepared bundle")
response = self._ec2.run_instances(
ImageId=machine.image_id,
InstanceType=machine.instance_type,
MinCount=1,
MaxCount=1,
ClientToken=self._launch_token(machine),
IamInstanceProfile={"Arn": self._config.instance_profile_arn},
Placement={"AvailabilityZone": self._config.availability_zone},
NetworkInterfaces=[
{
"DeviceIndex": 0,
"SubnetId": self._config.subnet_id,
"Groups": list(self._config.security_group_ids),
"AssociatePublicIpAddress": False,
"DeleteOnTermination": True,
}
],
MetadataOptions={
"HttpEndpoint": "enabled",
"HttpTokens": "required",
"HttpPutResponseHopLimit": 1,
"InstanceMetadataTags": "disabled",
},
BlockDeviceMappings=[
{
"DeviceName": self._config.root_device_name,
"Ebs": {
"DeleteOnTermination": True,
"Encrypted": True,
"KmsKeyId": "alias/aws/ebs",
"Iops": 3000,
"Throughput": 125,
"VolumeSize": self._config.root_volume_gib,
"VolumeType": "gp3",
},
}
],
TagSpecifications=[
{"ResourceType": "instance", "Tags": self._tags(machine, "worker")},
{"ResourceType": "volume", "Tags": self._tags(machine, "root")},
{"ResourceType": "network-interface", "Tags": self._tags(machine, "network")},
],
UserData=machine.bundle,
)
instances = response.get("Instances", [])
if len(instances) != 1:
raise CloudResourceError("RunInstances did not return exactly one instance")
return instances[0]["InstanceId"]
def attach_volume(self, machine: Machine) -> None:
if machine.instance_id is None or machine.data_volume_id is None:
raise CloudResourceError("cannot attach without recorded instance and volume")
self._ec2.attach_volume(
Device=DATA_DEVICE, InstanceId=machine.instance_id, VolumeId=machine.data_volume_id
)
def enforce_data_retention(self, machine: Machine) -> None:
if machine.instance_id is None or machine.data_volume_id is None:
raise CloudResourceError("cannot set retention without recorded instance and volume")
self._ec2.modify_instance_attribute(
InstanceId=machine.instance_id,
Attribute="blockDeviceMapping",
BlockDeviceMappings=[
{
"DeviceName": DATA_DEVICE,
"Ebs": {"DeleteOnTermination": False, "VolumeId": machine.data_volume_id},
}
],
)
def start_instance(self, machine: Machine) -> None:
if machine.instance_id is None:
raise CloudResourceError("cannot start without a recorded instance")
self._ec2.start_instances(InstanceIds=[machine.instance_id])
def stop_instance(self, machine: Machine) -> None:
if machine.instance_id is None:
raise CloudResourceError("cannot stop without a recorded instance")
self._ec2.stop_instances(InstanceIds=[machine.instance_id], Force=False)
def terminate_instance(self, machine: Machine) -> None:
if machine.instance_id is None:
raise CloudResourceError("cannot terminate without a recorded instance")
self._ec2.terminate_instances(InstanceIds=[machine.instance_id])
def delete_volume(self, machine: Machine) -> None:
if machine.data_volume_id is None:
raise CloudResourceError("cannot delete without a recorded volume")
self._ec2.delete_volume(VolumeId=machine.data_volume_id)
def _validate_volume(self, volume: dict[str, Any], machine: Machine) -> None:
self._validate_tags(volume, machine, "data")
if volume.get("AvailabilityZone") != self._config.availability_zone:
raise CloudResourceError("data volume is in the wrong availability zone")
if volume.get("Encrypted") is not True:
raise CloudResourceError("data volume is not encrypted")
if volume.get("Size") != self._config.data_volume_gib:
raise CloudResourceError("data volume size differs from immutable spec")
if volume.get("VolumeType") != "gp3":
raise CloudResourceError("data volume type differs from immutable spec")
if volume.get("Iops") != 3000 or volume.get("Throughput") != 125:
raise CloudResourceError("data volume performance differs from immutable spec")
if not volume.get("KmsKeyId"):
raise CloudResourceError("data volume has no KMS key identity")
def _validate_instance(self, instance: dict[str, Any], machine: Machine) -> None:
instance_id = instance.get("InstanceId")
if not instance_id:
raise CloudResourceError("instance has no identity")
if machine.instance_id is not None and instance_id != machine.instance_id:
raise CloudResourceError("instance identity differs from recorded instance")
self._validate_tags(instance, machine, "worker")
if instance.get("ImageId") != machine.image_id:
raise CloudResourceError("instance image differs from immutable spec")
if instance.get("InstanceType") != machine.instance_type:
raise CloudResourceError("instance type differs from immutable spec")
placement = instance.get("Placement") or {}
if placement.get("AvailabilityZone") != self._config.availability_zone:
raise CloudResourceError("instance is in the wrong availability zone")
if instance.get("State", {}).get("Name") in {"shutting-down", "terminated"}:
return
profile = instance.get("IamInstanceProfile") or {}
if profile.get("Arn") != self._config.instance_profile_arn:
raise CloudResourceError("instance profile differs from immutable spec")
if instance.get("SubnetId") != self._config.subnet_id:
raise CloudResourceError("instance is in the wrong subnet")
actual_groups = {group["GroupId"] for group in instance.get("SecurityGroups", [])}
if actual_groups != set(self._config.security_group_ids):
raise CloudResourceError("instance security groups differ from immutable spec")
if instance.get("PublicIpAddress"):
raise CloudResourceError("instance unexpectedly has a public IP address")
interfaces = instance.get("NetworkInterfaces", [])
if len(interfaces) != 1:
raise CloudResourceError("instance must have exactly one network interface")
interface = interfaces[0]
if interface.get("SubnetId") != self._config.subnet_id:
raise CloudResourceError("instance network interface is in the wrong subnet")
interface_groups = {group["GroupId"] for group in interface.get("Groups", [])}
if interface_groups != set(self._config.security_group_ids):
raise CloudResourceError("network interface security groups differ from spec")
attachment = interface.get("Attachment") or {}
if attachment.get("DeviceIndex") != 0 or attachment.get("DeleteOnTermination") is not True:
raise CloudResourceError("network interface attachment differs from spec")
if interface.get("Association", {}).get("PublicIp"):
raise CloudResourceError("instance network interface has a public IP address")
metadata = instance.get("MetadataOptions") or {}
expected_metadata = {
"HttpEndpoint": "enabled",
"HttpTokens": "required",
"HttpPutResponseHopLimit": 1,
"InstanceMetadataTags": "disabled",
}
for key, expected in expected_metadata.items():
if metadata.get(key) != expected:
raise CloudResourceError(f"instance metadata option {key} differs from spec")
def _validate_tags(self, resource: dict[str, Any], machine: Machine, purpose: str) -> None:
tags = {tag["Key"]: tag["Value"] for tag in resource.get("Tags", [])}
required = {tag["Key"]: tag["Value"] for tag in self._tags(machine, purpose)}
for key, expected in required.items():
if tags.get(key) != expected:
raise CloudResourceError(f"resource ownership tag {key!r} is missing or incorrect")
def _tags(self, machine: Machine, purpose: str) -> list[dict[str, str]]:
return [
{"Key": "switch:installation-id", "Value": self._config.installation_id},
{"Key": "switch:machine-id", "Value": machine.machine_id},
{"Key": "switch:purpose", "Value": purpose},
{"Key": "switch:managed-by", "Value": MANAGED_BY},
]
def _resource_filters(self, machine: Machine, purpose: str) -> list[dict[str, Any]]:
return [
{"Name": "tag:switch:installation-id", "Values": [self._config.installation_id]},
{"Name": "tag:switch:machine-id", "Values": [machine.machine_id]},
{"Name": "tag:switch:purpose", "Values": [purpose]},
{"Name": "tag:switch:managed-by", "Values": [MANAGED_BY]},
]
def _launch_token(self, machine: Machine) -> str:
return self._token(machine, f"instance-{machine.instance_seq}")
def _token(self, machine: Machine, resource: str) -> str:
material = f"{self._config.installation_id}:{machine.machine_id}:{resource}"
return f"switch-m-{hashlib.sha256(material.encode()).hexdigest()[:48]}"
def _describe_instances(self, **kwargs: Any) -> list[dict[str, Any]]:
instances: list[dict[str, Any]] = []
request = dict(kwargs)
while True:
response = self._ec2.describe_instances(**request)
instances.extend(
instance
for reservation in response.get("Reservations", [])
for instance in reservation["Instances"]
)
token = response.get("NextToken")
if not token:
return instances
request["NextToken"] = token
def _describe_volumes(self, **kwargs: Any) -> list[dict[str, Any]]:
volumes: list[dict[str, Any]] = []
request = dict(kwargs)
while True:
response = self._ec2.describe_volumes(**request)
volumes.extend(response.get("Volumes", []))
token = response.get("NextToken")
if not token:
return volumes
request["NextToken"] = token
def _error_code(exc: ClientError) -> str:
return exc.response.get("Error", {}).get("Code", "")
AMBIGUOUS_CLIENT_ERRORS = {
"IdempotentParameterMismatch",
"RequestLimitExceeded",
"RequestTimeout",
"RequestTimeoutException",
"Throttling",
"ThrottlingException",
}
def rejected(exc: ClientError) -> bool:
"""Whether EC2 definitely refused the request, so it created nothing."""
status = exc.response.get("ResponseMetadata", {}).get("HTTPStatusCode")
return (
isinstance(status, int)
and 400 <= status < 500
and _error_code(exc) not in AMBIGUOUS_CLIENT_ERRORS
)