Light, fluffy, and always free - The AWS Local Emulator alternative
feat(ec2): configure additional AWS-managed prefix lists (#6401)
Andre Kurait307380750ffd
diff --git a/docs/services/ec2.md b/docs/services/ec2.mdindex 7981b4bbe1..71af34e986 100644--- a/docs/services/ec2.md+++ b/docs/services/ec2.md@@ -552,6 +552,31 @@ Two AWS-managed prefix lists exist in every region without being created —read-only, and served by both `DescribePrefixLists` and `DescribeManagedPrefixLists`;modifying or deleting one returns `UnsupportedOperation`.+Additional regional AWS-managed lists can be supplied through+`FLOCI_SERVICES_EC2_MANAGED_PREFIX_LIST_CATALOG_PATH`. Set it to a readable JSON+file inside the Floci container, for example:++```json+{+ "eu-west-1": [{+ "prefixListId": "pl-0123456789abcdef0",+ "prefixListName": "example-managed-service",+ "cidrs": ["192.0.2.0/24"]+ }]+}+```++The file is read once when EC2 starts. Each entry needs a unique ID and name+within its region and at least one IPv4 CIDR. CIDRs are normalized to network+blocks; repeated or equivalent blocks within one list are rejected. Entries+cannot replace the built-in lists. Invalid or unreadable files prevent startup+when EC2 is enabled; disabled EC2 does not read the catalog. Configured lists+have the same AWS ownership, read-only behavior, security-group reference+validation and describe/entry responses as the built-in lists. Both configured+and built-in CIDRs participate in security-group policy evaluation and firewall+rules when enforcement is enabled. They are available across accounts in their+configured region. Leaving the setting unset keeps the built-in catalog.+A customer-managed list may not take a name AWS reserves for its own: `com.amazonaws.`,`com.amazon.` or `com.aws.`, each including the trailing dot. `CreateManagedPrefixList`rejects those with `InvalidParameterValue`; a name that merely resembles one, such asdiff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex 8d1f377d10..3f9cd7a212 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -3218,6 +3218,9 @@ public interface EmulatorConfig {/** Optional full EC2 catalog file for locally built guest images. */Optional<String> imageCatalogPath();+ /** Optional JSON file of additional regional AWS-managed prefix lists. */+ Optional<String> managedPrefixListCatalogPath();+@WithDefault("true")boolean enabled();diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.javaindex 62adf3df3d..86d7dc692c 100644--- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java@@ -1,6 +1,7 @@package io.github.hectorvent.floci.services.ec2;import com.fasterxml.jackson.core.type.TypeReference;+import com.fasterxml.jackson.databind.ObjectMapper;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AvailabilityZoneIds;import io.github.hectorvent.floci.core.common.AwsArnUtils;@@ -108,7 +109,11 @@ import jakarta.enterprise.context.ContextNotActiveException;import jakarta.inject.Inject;import org.jboss.logging.Logger;+import java.io.IOException;+import java.io.InputStream;import java.nio.charset.StandardCharsets;+import java.nio.file.Files;+import java.nio.file.Path;import java.time.Duration;import java.time.Instant;import java.time.ZoneOffset;@@ -292,6 +297,7 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettabprivate final StorageBackend<String, SpotInstanceRequest> spotInstanceRequests;private final StorageBackend<String, NetworkAcl> networkAcls;private final StorageBackend<String, ManagedPrefixList> managedPrefixLists;+ private final Map<String, List<PrefixList>> configuredManagedPrefixLists;private final StorageBackend<String, TransitGateway> transitGateways;private final StorageBackend<String, TransitGatewayRouteTable> transitGatewayRouteTables;private final StorageBackend<String, TransitGatewayVpcAttachment> transitGatewayVpcAttachments;@@ -849,6 +855,7 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettabthis.networkInterfaces = networkInterfaces;this.capacityReservations = capacityReservations;this.volumeModifications = volumeModifications;+ this.configuredManagedPrefixLists = loadManagedPrefixListCatalog();}@PostConstruct@@ -1655,6 +1662,66 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettab* {@code GatewayVpcEndpointAwsService}); the reversed-suffix names apply to interface endpoints only.*/private List<ManagedPrefixList> awsManagedPrefixLists(String region) {+ List<ManagedPrefixList> lists = new ArrayList<>(defaultAwsManagedPrefixLists(region));+ for (PrefixList configured : configuredManagedPrefixLists.getOrDefault(region, List.of())) {+ lists.add(awsManagedPrefixList(region, configured.getPrefixListId(),+ configured.getPrefixListName(), configured.getCidrs()));+ }+ return lists;+ }++ private Map<String, List<PrefixList>> loadManagedPrefixListCatalog() {+ if (!config.services().ec2().enabled()) {+ return Map.of();+ }+ Optional<String> path = config.services().ec2().managedPrefixListCatalogPath().filter(value -> !value.isBlank());+ if (path.isEmpty()) {+ return Map.of();+ }+ try (InputStream input = Files.newInputStream(Path.of(path.get()))) {+ Map<String, List<PrefixList>> catalog = new ObjectMapper().readValue(input, new TypeReference<>() {});+ if (catalog == null) {+ throw new IllegalArgumentException("Expected a region-to-prefix-lists object");+ }+ for (Map.Entry<String, List<PrefixList>> region : catalog.entrySet()) {+ if (region.getKey().isBlank() || region.getValue() == null) {+ throw new IllegalArgumentException("Expected a nonblank region and a list");+ }+ Set<String> ids = new HashSet<>();+ Set<String> names = new HashSet<>();+ for (ManagedPrefixList list : defaultAwsManagedPrefixLists(region.getKey())) {+ ids.add(list.getPrefixListId());+ names.add(list.getPrefixListName());+ }+ for (PrefixList list : region.getValue()) {+ if (list == null || list.getPrefixListId() == null+ || !list.getPrefixListId().matches("pl-(?:[0-9a-f]{8}|[0-9a-f]{17})")+ || list.getPrefixListName() == null || list.getPrefixListName().isBlank()+ || list.getCidrs() == null || list.getCidrs().isEmpty()+ || !ids.add(list.getPrefixListId()) || !names.add(list.getPrefixListName())) {+ throw new IllegalArgumentException("Invalid or duplicate prefix list in " + region.getKey());+ }+ Set<String> canonicalCidrs = new LinkedHashSet<>();+ for (String cidr : list.getCidrs()) {+ Optional<String> canonical = cidr == null || cidr.contains(":")+ ? Optional.empty() : CidrCanonicalizer.canonicalize(cidr);+ if (canonical.isEmpty()) {+ throw new IllegalArgumentException("Expected an IPv4 prefix list entry in " + region.getKey());+ }+ if (!canonicalCidrs.add(canonical.get())) {+ throw new IllegalArgumentException("Duplicate prefix list entry in " + region.getKey());+ }+ }+ list.setCidrs(List.copyOf(canonicalCidrs));+ }+ }+ return Map.copyOf(catalog);+ } catch (IOException | IllegalArgumentException error) {+ throw new IllegalStateException("Invalid EC2 managed prefix list catalog: " + path.get(), error);+ }+ }++ private List<ManagedPrefixList> defaultAwsManagedPrefixLists(String region) {return List.of(awsManagedPrefixList(region, "pl-63a5400a", "com.amazonaws." + region + ".s3",List.of("52.216.0.0/15", "54.231.0.0/16")),@@ -4112,15 +4179,20 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettab}private Map<String, List<String>> policyPrefixLists(String region, List<SecurityGroup> groups) {+ Map<String, ManagedPrefixList> awsLists = awsManagedPrefixLists(region).stream()+ .collect(Collectors.toMap(ManagedPrefixList::getPrefixListId, Function.identity()));Map<String, List<String>> resolved = new LinkedHashMap<>();for (SecurityGroup group : groups) {Stream.concat(group.getIpPermissions().stream(),group.getIpPermissionsEgress().stream()).flatMap(permission -> permission.getPrefixListIds().stream()).map(PrefixListId::getPrefixListId).distinct()- .forEach(id -> resolved.put(id, managedPrefixLists.get(key(region, id)).isEmpty()- ? List.of() : getManagedPrefixListEntries(region, id, null).stream()- .map(PrefixListEntry::getCidr).toList()));+ .forEach(id -> {+ ManagedPrefixList list = managedPrefixLists.get(key(region, id))+ .orElseGet(() -> awsLists.get(id));+ resolved.put(id, list == null ? List.of() : list.currentEntries().stream()+ .map(PrefixListEntry::getCidr).toList());+ });}return resolved;}diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex b214ab6dab..70c57b7802 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -676,6 +676,7 @@ floci:enforce-vcpu-quotas: false# Optional complete catalog override for locally built guest images.image-catalog-path: ${FLOCI_SERVICES_EC2_IMAGE_CATALOG_PATH:}+ managed-prefix-list-catalog-path: ${FLOCI_SERVICES_EC2_MANAGED_PREFIX_LIST_CATALOG_PATH:}enabled: trueimds-port: 9169# Sweep EC2 containers left on the daemon by a previous run that died mid-launch ordiff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.javaindex 872d883175..95bf1712ea 100644--- a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.java@@ -67,6 +67,7 @@ import org.bouncycastle.openssl.PEMKeyPair;import org.bouncycastle.openssl.PEMParser;import org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter;import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.io.TempDir;import org.junit.jupiter.params.ParameterizedTest;import org.junit.jupiter.params.provider.CsvSource;import org.junit.jupiter.params.provider.ValueSource;@@ -75,10 +76,13 @@ import org.mockito.InOrder;import org.mockito.MockMakers;import org.mockito.MockedStatic;+import java.io.IOException;import java.io.StringReader;import java.lang.reflect.Field;import java.lang.reflect.Method;import java.nio.charset.StandardCharsets;+import java.nio.file.Files;+import java.nio.file.Path;import java.security.KeyPairGenerator;import java.security.interfaces.RSAPrivateCrtKey;import java.time.Duration;@@ -123,8 +127,10 @@ import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyMap;import static org.mockito.ArgumentMatchers.anyString;import static org.mockito.ArgumentMatchers.argThat;+import static org.mockito.ArgumentMatchers.eq;import static org.mockito.Mockito.CALLS_REAL_METHODS;import static org.mockito.Mockito.RETURNS_SELF;import static org.mockito.Mockito.atLeastOnce;@@ -6006,12 +6012,176 @@ class Ec2ServiceTest {// =========================================================================private static Ec2Service prefixListService() {- return new Ec2Service(mockConfig(true), mock(Ec2ContainerManager.class),+ return prefixListService(mockConfig(true));+ }++ private static Ec2Service prefixListService(EmulatorConfig config) {+ return new Ec2Service(config, mock(Ec2ContainerManager.class),mock(Ec2PortForwardManager.class),mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class), new Ec2InstanceTypeCatalog(),new InMemoryStorageFactory());}+ @Test+ void configuredAwsManagedPrefixListUsesExistingRegionalReadOnlyApis(@TempDir Path temp) throws IOException {+ Path catalog = temp.resolve("prefix-lists.json");+ Files.writeString(catalog, """+ {"eu-west-1": [{"prefixListId": "pl-0123456789abcdef0",+ "prefixListName": "example-managed-service", "cidrs": ["192.0.2.19/24"]}]}+ """);+ EmulatorConfig config = mockConfig(true);+ when(config.services().ec2().enabled()).thenReturn(true);+ when(config.services().ec2().managedPrefixListCatalogPath()).thenReturn(Optional.of(catalog.toString()));+ Ec2Service service = prefixListService(config);+ String id = "pl-0123456789abcdef0";++ ManagedPrefixList managed = service.describeManagedPrefixLists("eu-west-1", List.of(id), Map.of()).getFirst();+ assertTrue(managed.isAwsManaged());+ assertEquals("AWS", managed.getOwnerId());+ assertEquals("IPv4", managed.getAddressFamily());+ assertEquals("arn:aws:ec2:eu-west-1:aws:prefix-list/" + id, managed.getPrefixListArn());+ assertEquals(List.of("192.0.2.0/24"), service.describePrefixLists("eu-west-1", List.of(id), Map.of())+ .getFirst().getCidrs());+ assertEquals("192.0.2.0/24", service.getManagedPrefixListEntries("eu-west-1", id, null).getFirst().getCidr());+ assertEquals(3, service.describeManagedPrefixLists("eu-west-1", List.of(), Map.of()).size());+ assertEquals("InvalidPrefixListID.NotFound", assertThrows(AwsException.class,+ () -> service.describeManagedPrefixLists("us-east-1", List.of(id), Map.of())).getErrorCode());+ assertEquals("UnsupportedOperation", assertThrows(AwsException.class,+ () -> service.deleteManagedPrefixList("eu-west-1", id)).getErrorCode());+ assertEquals("UnsupportedOperation", assertThrows(AwsException.class,+ () -> service.modifyManagedPrefixList("eu-west-1", id, null, "changed", null,+ List.of(), List.of())).getErrorCode());++ String groupId = service.createSecurityGroup("eu-west-1", "consumer", "consumer", null).getGroupId();+ IpPermission permission = tcpPermission(443);+ permission.getPrefixListIds().add(new PrefixListId(id, null));+ assertEquals(id, service.authorizeSecurityGroupIngress("eu-west-1", groupId, List.of(permission))+ .getFirst().getPrefixListId());+ }++ @ParameterizedTest+ @ValueSource(strings = {+ "not-json",+ "null",+ "{\"eu-west-1\": null}",+ "{\"eu-west-1\": [null]}",+ "{\"eu-west-1\": [{\"prefixListId\":\"pl-invalid\",\"prefixListName\":\"example\",\"cidrs\":[\"192.0.2.0/24\"]}]}",+ "{\"eu-west-1\": [{\"prefixListId\":\"pl-12345678\",\"prefixListName\":\"example\",\"cidrs\":[]}]}",+ "{\"eu-west-1\": [{\"prefixListId\":\"pl-12345678\",\"prefixListName\":\"example\",\"cidrs\":[\"2001:db8::/32\"]}]}",+ "{\"eu-west-1\": [{\"prefixListId\":\"pl-12345678\",\"prefixListName\":\"example\",\"cidrs\":[\"invalid\"]}]}",+ "{\"eu-west-1\": [{\"prefixListId\":\"pl-63a5400a\",\"prefixListName\":\"example\",\"cidrs\":[\"192.0.2.0/24\"]}]}",+ "{\"eu-west-1\": [{\"prefixListId\":\"pl-12345678\",\"prefixListName\":\"com.amazonaws.eu-west-1.s3\","+ + "\"cidrs\":[\"192.0.2.0/24\"]}]}"+ })+ void invalidManagedPrefixListCatalogFailsStartup(String json, @TempDir Path temp) throws IOException {+ Path catalog = temp.resolve("prefix-lists.json");+ Files.writeString(catalog, json);+ EmulatorConfig config = mockConfig(true);+ when(config.services().ec2().enabled()).thenReturn(true);+ when(config.services().ec2().managedPrefixListCatalogPath()).thenReturn(Optional.of(catalog.toString()));++ IllegalStateException error = assertThrows(IllegalStateException.class, () -> prefixListService(config));+ assertTrue(error.getMessage().contains(catalog.toString()));+ }++ @Test+ void missingManagedPrefixListCatalogFailsStartup(@TempDir Path temp) {+ EmulatorConfig config = mockConfig(true);+ when(config.services().ec2().enabled()).thenReturn(true);+ Path catalog = temp.resolve("missing.json");+ when(config.services().ec2().managedPrefixListCatalogPath()).thenReturn(Optional.of(catalog.toString()));++ assertThrows(IllegalStateException.class, () -> prefixListService(config));+ }++ @ParameterizedTest+ @ValueSource(strings = {"192.0.2.0/24", "192.0.2.19/24"})+ void duplicateCanonicalCatalogEntriesFailStartup(String duplicate, @TempDir Path temp) throws IOException {+ Path catalog = temp.resolve("prefix-lists.json");+ Files.writeString(catalog, """+ {"eu-west-1": [{"prefixListId": "pl-0123456789abcdef0",+ "prefixListName": "example-managed-service", "cidrs": ["192.0.2.0/24", "%s"]}]}+ """.formatted(duplicate));+ EmulatorConfig config = mockConfig(true);+ when(config.services().ec2().enabled()).thenReturn(true);+ when(config.services().ec2().managedPrefixListCatalogPath()).thenReturn(Optional.of(catalog.toString()));++ assertThrows(IllegalStateException.class, () -> prefixListService(config));+ }++ @Test+ void disabledEc2DoesNotReadManagedPrefixListCatalog(@TempDir Path temp) {+ EmulatorConfig config = mockConfig(true);+ when(config.services().ec2().enabled()).thenReturn(false);+ when(config.services().ec2().managedPrefixListCatalogPath())+ .thenReturn(Optional.of(temp.resolve("missing.json").toString()));++ assertDoesNotThrow(() -> prefixListService(config));+ }++ @ParameterizedTest+ @CsvSource({"configured,false", "configured,true", "builtin,false", "builtin,true",+ "customer,false", "customer,true"})+ void prefixListFirewallConsumersReceiveCanonicalEntries(String kind, boolean egress, @TempDir Path temp)+ throws IOException {+ String region = "eu-west-1";+ Path catalog = temp.resolve("prefix-lists.json");+ Files.writeString(catalog, """+ {"eu-west-1": [{"prefixListId": "pl-0123456789abcdef0",+ "prefixListName": "example-managed-service", "cidrs": ["192.0.2.19/24"]}]}+ """);+ EmulatorConfig config = mockConfig(false);+ when(config.services().ec2().enabled()).thenReturn(true);+ when(config.services().ec2().managedPrefixListCatalogPath()).thenReturn(Optional.of(catalog.toString()));+ EmulatorConfig.NetworkConfig network = mock(EmulatorConfig.NetworkConfig.class);+ EmulatorConfig.SecurityGroupEnforcementConfig enforcement = mock(EmulatorConfig.SecurityGroupEnforcementConfig.class);+ when(config.network()).thenReturn(network);+ when(network.securityGroupEnforcement()).thenReturn(enforcement);+ when(enforcement.enabled()).thenReturn(true);+ Ec2ContainerManager manager = mock(Ec2ContainerManager.class);+ AtomicReference<Map<String, List<String>>> policyLists = new AtomicReference<>();+ doAnswer(invocation -> {+ policyLists.set(invocation.getArgument(2));+ return null;+ }).when(manager).refreshSecurityGroups(eq(region), anyMap(), anyMap());+ Ec2Service service = new Ec2Service(config, manager, mock(Ec2PortForwardManager.class),+ mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class), new Ec2InstanceTypeCatalog(),+ new InMemoryStorageFactory());+ String id = switch (kind) {+ case "configured" -> "pl-0123456789abcdef0";+ case "builtin" -> "pl-63a5400a";+ case "customer" -> service.createManagedPrefixList(region, "customer-list", "IPv4", 1,+ List.of(new PrefixListEntry("192.0.2.0/24", "test")), List.of()).getPrefixListId();+ default -> throw new AssertionError(kind);+ };+ SecurityGroup group = service.createSecurityGroup(region, "consumer", "consumer", null);+ if (egress) {+ service.revokeSecurityGroupEgress(region, group.getGroupId(), new ArrayList<>(group.getIpPermissionsEgress()));+ }+ IpPermission permission = tcpPermission(443);+ permission.getPrefixListIds().add(new PrefixListId(id, null));+ if (egress) {+ service.authorizeSecurityGroupEgress(region, group.getGroupId(), List.of(permission));+ } else {+ service.authorizeSecurityGroupIngress(region, group.getGroupId(), List.of(permission));+ }+ String allowed = "builtin".equals(kind) ? "52.216.1.9" : "192.0.2.9";+ String cidr = "builtin".equals(kind) ? "52.216.0.0/15" : "192.0.2.0/24";+ Map<String, List<String>> resolved = policyLists.get();+ assertNotNull(resolved);+ assertTrue(resolved.get(id).contains(cidr));+ SecurityGroupPolicy.Peer peer = new SecurityGroupPolicy.Peer(allowed, group.getOwnerId(),+ group.getVpcId(), Set.of());+ assertTrue(SecurityGroupPolicy.allows(List.of(group), egress, "tcp", 443, 0, peer, resolved));+ assertFalse(SecurityGroupPolicy.allows(List.of(group), egress, "tcp", 444, 0, peer, resolved));+ assertFalse(SecurityGroupPolicy.allows(List.of(group), egress, "tcp", 443, 0,+ new SecurityGroupPolicy.Peer("203.0.113.9", group.getOwnerId(), group.getVpcId(), Set.of()), resolved));+ SecurityGroupNftCompiler.Endpoint endpoint = new SecurityGroupNftCompiler.Endpoint(group.getOwnerId(), region,+ group.getVpcId(), "eni-consumer", "10.0.0.9", "172.20.0.9", Set.of(group.getGroupId()), List.of(group));+ String rules = SecurityGroupNftCompiler.compile(endpoint, List.of(), resolved);+ assertTrue(rules.contains(cidr));+ }+@Testvoid createManagedPrefixListStoresEntriesAtVersionOne() {Ec2Service service = prefixListService();diff --git a/src/test/resources/application.yml b/src/test/resources/application.ymlindex 905120dd0a..e642926088 100644--- a/src/test/resources/application.yml+++ b/src/test/resources/application.yml@@ -321,6 +321,7 @@ floci:enforce-vcpu-quotas: false# Optional complete catalog override for locally built guest images.image-catalog-path: ${FLOCI_SERVICES_EC2_IMAGE_CATALOG_PATH:}+ managed-prefix-list-catalog-path: ${FLOCI_SERVICES_EC2_MANAGED_PREFIX_LIST_CATALOG_PATH:}enabled: trueimds-port: 9169ssh-port-range-start: 2200