Light, fluffy, and always free - The AWS Local Emulator alternative
feat(ec2): add instance metadata catalog and architecture parity (#1596)
Jason van Zylc233eb489122
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/Ec2Tests.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/Ec2Tests.javaindex 755efefbd3..93a02175e3 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/Ec2Tests.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/Ec2Tests.java@@ -158,6 +158,19 @@ class Ec2Tests {assertThat(resp.instanceTypes()).isNotEmpty();}+ @Test+ @Order(7)+ @DisplayName("DescribeInstanceTypes - processor architectures")+ void describeInstanceTypeProcessorArchitectures() {+ DescribeInstanceTypesResponse resp = ec2.describeInstanceTypes(DescribeInstanceTypesRequest.builder()+ .instanceTypes(InstanceType.fromValue("m6gd.2xlarge"))+ .build());++ assertThat(resp.instanceTypes()).hasSize(1);+ assertThat(resp.instanceTypes().get(0).processorInfo().supportedArchitecturesAsStrings())+ .containsExactly("arm64");+ }+@Test@Order(8)@DisplayName("CreateVpc - create VPC with CIDR")diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2InstanceTypeCatalog.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2InstanceTypeCatalog.javaindex f4adcf1aee..2ac89c2600 100644--- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2InstanceTypeCatalog.java+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2InstanceTypeCatalog.java@@ -120,6 +120,7 @@ public class Ec2InstanceTypeCatalog {public String instanceType;public int vcpu;public int memoryMib;+ public int localStorageGiB;public List<String> supportedArchitectures = List.of();public Boolean currentGeneration;@@ -128,6 +129,8 @@ public class Ec2InstanceTypeCatalog {type.put("instanceType", instanceType);type.put("vcpu", vcpu);type.put("memoryMib", memoryMib);+ type.put("instanceStorageSupported", localStorageGiB > 0);+ type.put("localStorageGiB", localStorageGiB);type.put("supportedArchitectures", List.copyOf(supportedArchitectures));type.put("currentGeneration", currentGeneration == null || currentGeneration);return type;diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2MetadataServer.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2MetadataServer.javaindex c4b4b89326..3cb696c290 100644--- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2MetadataServer.java+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2MetadataServer.java@@ -310,9 +310,19 @@ public class Ec2MetadataServer {if (inst == null) {return;}+ String body = instanceIdentityDocument(inst, config.defaultAccountId());+ ctx.response().setStatusCode(200)+ .putHeader("content-type", "application/json")+ .end(body);+ }++ static String instanceIdentityDocument(Instance inst, String accountId) {String az = inst.getPlacement() != null ? inst.getPlacement().getAvailabilityZone() : "us-east-1a";- String body = "{\"accountId\":\"" + config.defaultAccountId() + "\","- + "\"architecture\":\"x86_64\","+ String architecture = inst.getArchitecture() == null || inst.getArchitecture().isBlank()+ ? "x86_64"+ : inst.getArchitecture();+ String body = "{\"accountId\":\"" + accountId + "\","+ + "\"architecture\":\"" + architecture + "\","+ "\"availabilityZone\":\"" + az + "\","+ "\"imageId\":\"" + inst.getImageId() + "\","+ "\"instanceId\":\"" + inst.getInstanceId() + "\","@@ -320,9 +330,7 @@ public class Ec2MetadataServer {+ "\"privateIp\":\"" + nvl(inst.getPrivateIpAddress()) + "\","+ "\"region\":\"" + inst.getRegion() + "\","+ "\"version\":\"2017-09-30\"}";- ctx.response().setStatusCode(200)- .putHeader("content-type", "application/json")- .end(body);+ return body;}// ── Instance resolution ───────────────────────────────────────────────────diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2QueryHandler.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2QueryHandler.javaindex dc0d01032f..4bc8522a54 100644--- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2QueryHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2QueryHandler.java@@ -1601,11 +1601,18 @@ public class Ec2QueryHandler {.start("memoryInfo").elem("sizeInMiB", String.valueOf(t.get("memoryMib"))).end("memoryInfo")+ .elem("instanceStorageSupported", String.valueOf(t.get("instanceStorageSupported")));+ if (Boolean.TRUE.equals(t.get("instanceStorageSupported"))) {+ xml.start("instanceStorageInfo")+ .elem("totalSizeInGB", String.valueOf(t.get("localStorageGiB")))+ .end("instanceStorageInfo");+ }+ xml.start("processorInfo").start("supportedArchitectures");for (String arch : (List<String>) t.get("supportedArchitectures")) {- xml.start("item").elem("item", arch).end("item");+ xml.elem("item", arch);}- xml.end("supportedArchitectures").end("item");+ xml.end("supportedArchitectures").end("processorInfo").end("item");}xml.end("instanceTypeSet").end("DescribeInstanceTypesResponse");return xmlResponse(xml.build());diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.javaindex e170d01dac..55705f2a36 100644--- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java@@ -569,7 +569,10 @@ public class Ec2Service {reservation.setReservationId(reservationId);reservation.setOwnerId(accountId);+ String effectiveInstanceType = instanceType != null ? instanceType : "t2.micro";+ validateArchitectureCompatibility(imageId, effectiveInstanceType);int count = Math.min(maxCount, Math.max(minCount, 1));+ String architecture = architectureFor(imageId, effectiveInstanceType);for (int i = 0; i < count; i++) {String instanceId = "i-" + randomHex(17);String privateIp = assignPrivateIp(region, finalSubnetId);@@ -578,7 +581,7 @@ public class Ec2Service {inst.setInstanceId(instanceId);inst.setImageId(imageId);inst.setState(InstanceState.pending());- inst.setInstanceType(instanceType != null ? instanceType : "t2.micro");+ inst.setInstanceType(effectiveInstanceType);inst.setPlacement(new Placement(az));inst.setSubnetId(finalSubnetId);inst.setVpcId(vpcId);@@ -586,7 +589,7 @@ public class Ec2Service {inst.setPrivateDnsName("ip-" + privateIp.replace('.', '-') + ".ec2.internal");inst.setKeyName(keyName);inst.setSecurityGroups(new ArrayList<>(sgIdentifiers));- inst.setArchitecture("x86_64");+ inst.setArchitecture(architecture);inst.setLaunchTime(Instant.now());inst.setAmiLaunchIndex(i);inst.setClientToken(clientToken);@@ -700,6 +703,36 @@ public class Ec2Service {}}+ private void validateArchitectureCompatibility(String imageId, String instanceType) {+ Optional<String> imageArchitecture = imageCatalog.findByIdOrAlias(imageId)+ .map(image -> image.architecture)+ .filter(value -> !value.isBlank());+ if (imageArchitecture.isEmpty()) {+ return;+ }+ instanceTypeCatalog.find(instanceType)+ .filter(type -> type.supportedArchitectures.stream()+ .noneMatch(imageArchitecture.get()::equals))+ .ifPresent(type -> {+ throw new AwsException("InvalidParameterValue",+ "The architecture '" + imageArchitecture.get()+ + "' of the specified image does not match the architecture supported by instance type '"+ + instanceType + "'.",+ 400);+ });+ }++ private String architectureFor(String imageId, String instanceType) {+ Optional<Ec2ImageCatalog.CatalogImage> image = imageCatalog.findByIdOrAlias(imageId);+ return image.map(catalogImage -> catalogImage.architecture)+ .filter(value -> !value.isBlank())+ .or(() -> instanceTypeCatalog.find(instanceType)+ .flatMap(type -> type.supportedArchitectures.stream()+ .filter(value -> value != null && !value.isBlank())+ .findFirst()))+ .orElse("x86_64");+ }+public Subnet requireSubnet(String region, String subnetId) {ensureDefaultResources(region);Subnet subnet = subnets.get(key(region, subnetId)).orElse(null);diff --git a/src/main/resources/ec2/instance-type-catalog.yaml b/src/main/resources/ec2/instance-type-catalog.yamlindex 0daaf1a17f..745587a690 100644--- a/src/main/resources/ec2/instance-type-catalog.yaml+++ b/src/main/resources/ec2/instance-type-catalog.yaml@@ -58,6 +58,7 @@ instanceTypes:- instanceType: m6gd.largevcpu: 2memoryMib: 8192+ localStorageGiB: 118supportedArchitectures:- arm64currentGeneration: true@@ -65,6 +66,7 @@ instanceTypes:- instanceType: m6gd.2xlargevcpu: 8memoryMib: 32768+ localStorageGiB: 474supportedArchitectures:- arm64currentGeneration: true@@ -72,6 +74,7 @@ instanceTypes:- instanceType: m7gd.largevcpu: 2memoryMib: 8192+ localStorageGiB: 118supportedArchitectures:- arm64currentGeneration: true@@ -79,6 +82,7 @@ instanceTypes:- instanceType: m7gd.2xlargevcpu: 8memoryMib: 32768+ localStorageGiB: 474supportedArchitectures:- arm64currentGeneration: true@@ -86,6 +90,7 @@ instanceTypes:- instanceType: m8gd.mediumvcpu: 1memoryMib: 4096+ localStorageGiB: 59supportedArchitectures:- arm64currentGeneration: true@@ -93,6 +98,7 @@ instanceTypes:- instanceType: m8gd.largevcpu: 2memoryMib: 8192+ localStorageGiB: 118supportedArchitectures:- arm64currentGeneration: true@@ -100,6 +106,7 @@ instanceTypes:- instanceType: m8gd.2xlargevcpu: 8memoryMib: 32768+ localStorageGiB: 474supportedArchitectures:- arm64currentGeneration: truediff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2InstanceTypeCatalogTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2InstanceTypeCatalogTest.javaindex 2b34e461f4..d7c03a96c6 100644--- a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2InstanceTypeCatalogTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2InstanceTypeCatalogTest.java@@ -59,6 +59,7 @@ class Ec2InstanceTypeCatalogTest {assertEquals(2, instanceType.vcpu);assertEquals(8192, instanceType.memoryMib);+ assertEquals(118, instanceType.localStorageGiB);assertEquals(List.of("arm64"), instanceType.supportedArchitectures);assertTrue(instanceType.currentGeneration);}diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2IntegrationTest.javaindex 58afc69698..5e3e5b0652 100644--- a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2IntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2IntegrationTest.java@@ -247,13 +247,18 @@ class Ec2IntegrationTest {void describeInstanceTypes() {given().formParam("Action", "DescribeInstanceTypes")+ .formParam("InstanceType.1", "m6gd.2xlarge").header("Authorization", AUTH_HEADER).when().post("/").then().statusCode(200).contentType("application/xml")- .body("DescribeInstanceTypesResponse.instanceTypeSet.item.size()", greaterThan(0));+ .body("DescribeInstanceTypesResponse.instanceTypeSet.item.instanceType", equalTo("m6gd.2xlarge"))+ .body("DescribeInstanceTypesResponse.instanceTypeSet.item.instanceStorageSupported", equalTo("true"))+ .body("DescribeInstanceTypesResponse.instanceTypeSet.item.instanceStorageInfo.totalSizeInGB", equalTo("474"))+ .body("DescribeInstanceTypesResponse.instanceTypeSet.item.processorInfo.supportedArchitectures.item",+ equalTo("arm64"));}@Test@@ -277,7 +282,11 @@ class Ec2IntegrationTest {everyItem(equalTo("2"))).body("DescribeInstanceTypesResponse.instanceTypeSet.item.memoryInfo.sizeInMiB",everyItem(equalTo("8192")))- .body("DescribeInstanceTypesResponse.instanceTypeSet.item.supportedArchitectures.item.item",+ .body("DescribeInstanceTypesResponse.instanceTypeSet.item.instanceStorageSupported",+ everyItem(equalTo("true")))+ .body("DescribeInstanceTypesResponse.instanceTypeSet.item.instanceStorageInfo.totalSizeInGB",+ everyItem(equalTo("118")))+ .body("DescribeInstanceTypesResponse.instanceTypeSet.item.processorInfo.supportedArchitectures.item",everyItem(equalTo("arm64")));}@@ -1326,6 +1335,52 @@ class Ec2IntegrationTest {.extract().path("RunInstancesResponse.instancesSet.item.instanceId");}+ @Test+ @Order(80)+ void runInstancesWithArm64ImageDescribesArm64Architecture() {+ String armInstanceId = given()+ .formParam("Action", "RunInstances")+ .formParam("ImageId", "ami-ubuntu2404-cloud-arm64")+ .formParam("InstanceType", "t4g.medium")+ .formParam("MinCount", "1")+ .formParam("MaxCount", "1")+ .header("Authorization", AUTH_HEADER)+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("RunInstancesResponse.instancesSet.item.architecture", equalTo("arm64"))+ .extract().path("RunInstancesResponse.instancesSet.item.instanceId");++ given()+ .formParam("Action", "DescribeInstances")+ .formParam("InstanceId.1", armInstanceId)+ .header("Authorization", AUTH_HEADER)+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("DescribeInstancesResponse.reservationSet.item.instancesSet.item.architecture",+ equalTo("arm64"));+ }++ @Test+ @Order(80)+ void runInstancesRejectsIncompatibleImageAndInstanceTypeArchitecture() {+ given()+ .formParam("Action", "RunInstances")+ .formParam("ImageId", "ami-ubuntu2404-amd64")+ .formParam("InstanceType", "t4g.medium")+ .formParam("MinCount", "1")+ .formParam("MaxCount", "1")+ .header("Authorization", AUTH_HEADER)+ .when()+ .post("/")+ .then()+ .statusCode(400)+ .body("Response.Errors.Error.Code", equalTo("InvalidParameterValue"));+ }+@Test@Order(81)void describeInstances() {diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2MetadataServerTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2MetadataServerTest.javaindex 7afafec836..6981f67c5b 100644--- a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2MetadataServerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2MetadataServerTest.java@@ -1,6 +1,7 @@package io.github.hectorvent.floci.services.ec2;import io.github.hectorvent.floci.services.ec2.model.Instance;+import io.github.hectorvent.floci.services.ec2.model.Placement;import io.github.hectorvent.floci.services.ec2.model.Tag;import io.github.hectorvent.floci.services.iam.IamService;import io.github.hectorvent.floci.services.iam.model.InstanceProfile;@@ -52,6 +53,25 @@ class Ec2MetadataServerTest {assertTrue(Ec2MetadataServer.instanceTagValue(instance, "Missing").isEmpty());}+ @Test+ void identityDocumentUsesInstanceArchitectureWithX8664Fallback() {+ Instance instance = new Instance();+ instance.setInstanceId("i-arm");+ instance.setArchitecture("arm64");+ instance.setImageId("ami-arm");+ instance.setInstanceType("t4g.medium");+ instance.setPlacement(new Placement("us-west-2a"));+ instance.setPrivateIpAddress("10.0.0.10");+ instance.setRegion("us-west-2");++ assertTrue(Ec2MetadataServer.instanceIdentityDocument(instance, "000000000000")+ .contains("\"architecture\":\"arm64\""));++ instance.setArchitecture(null);+ assertTrue(Ec2MetadataServer.instanceIdentityDocument(instance, "000000000000")+ .contains("\"architecture\":\"x86_64\""));+ }+@Testvoid staleContainerUnregisterDoesNotRemoveCurrentRegistration() {Ec2MetadataServer server = new Ec2MetadataServer(null, null, null);diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.javaindex 652760276f..f711c387be 100644--- a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.java@@ -63,6 +63,60 @@ class Ec2ServiceTest {assertEquals(400, error.getHttpStatus());}+ @Test+ void runInstancesStoresArchitectureFromImageCatalog() {+ Ec2Service service = new Ec2Service(mockConfig(true), mock(Ec2ContainerManager.class),+ mock(Ec2PortForwardManager.class),+ mock(AmiImageResolver.class), new Ec2ImageCatalog(), new Ec2InstanceTypeCatalog(),+ new InMemoryStorageFactory());++ Reservation reservation = service.runInstances("us-east-1", "ami-ubuntu2404-cloud-arm64", "t4g.medium",+ 1, 1, null, List.of(), null, null, List.of(), null, null);++ assertEquals("arm64", reservation.getInstances().getFirst().getArchitecture());+ }++ @Test+ void runInstancesKeepsX8664FallbackForUnknownImageAndType() {+ Ec2Service service = new Ec2Service(mockConfig(true), mock(Ec2ContainerManager.class),+ mock(Ec2PortForwardManager.class),+ mock(AmiImageResolver.class), new Ec2ImageCatalog(), new Ec2InstanceTypeCatalog(),+ new InMemoryStorageFactory());++ Reservation reservation = service.runInstances("us-east-1", "ami-unknown", "unknown.type",+ 1, 1, null, List.of(), null, null, List.of(), null, null);++ assertEquals("x86_64", reservation.getInstances().getFirst().getArchitecture());+ }++ @Test+ void runInstancesFallsBackToInstanceTypeArchitectureForUnknownImage() {+ Ec2Service service = new Ec2Service(mockConfig(true), mock(Ec2ContainerManager.class),+ mock(Ec2PortForwardManager.class),+ mock(AmiImageResolver.class), new Ec2ImageCatalog(), new Ec2InstanceTypeCatalog(),+ new InMemoryStorageFactory());++ Reservation reservation = service.runInstances("us-east-1", "ami-unknown", "t4g.medium",+ 1, 1, null, List.of(), null, null, List.of(), null, null);++ assertEquals("arm64", reservation.getInstances().getFirst().getArchitecture());+ }++ @Test+ void runInstancesRejectsIncompatibleImageAndInstanceTypeArchitectures() {+ Ec2Service service = new Ec2Service(mockConfig(true), mock(Ec2ContainerManager.class),+ mock(Ec2PortForwardManager.class),+ mock(AmiImageResolver.class), new Ec2ImageCatalog(), new Ec2InstanceTypeCatalog(),+ new InMemoryStorageFactory());++ AwsException error = assertThrows(AwsException.class, () -> service.runInstances(+ "us-east-1", "ami-ubuntu2404-amd64", "t4g.medium",+ 1, 1, null, List.of(), null, null, List.of(), null, null));++ assertEquals("InvalidParameterValue", error.getErrorCode());+ assertEquals(400, error.getHttpStatus());+ }+@Testvoid launchTemplateVersionInheritsOmittedFieldsFromRequestedSourceVersion() {Ec2Service service = new Ec2Service(mockConfig(true), mock(Ec2ContainerManager.class),