Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(ec2): configure additional AWS-managed prefix lists (#6401)

Andre Kurait307380750ffd
diff --git a/docs/services/ec2.md b/docs/services/ec2.md
index 7981b4bbe1..71af34e986 100644
--- a/docs/services/ec2.md
+++ b/docs/services/ec2.md
@@ -552,6 +552,31 @@ Two AWS-managed prefix lists exist in every region without being created —
read-only, and served by both `DescribePrefixLists` and `DescribeManagedPrefixLists`;
modifying or deleting one returns `UnsupportedOperation`.
+Additional regional AWS-managed lists can be supplied through
+`FLOCI_SERVICES_EC2_MANAGED_PREFIX_LIST_CATALOG_PATH`. Set it to a readable JSON
+file inside the Floci container, for example:
+
+```json
+{
+ "eu-west-1": [{
+ "prefixListId": "pl-0123456789abcdef0",
+ "prefixListName": "example-managed-service",
+ "cidrs": ["192.0.2.0/24"]
+ }]
+}
+```
+
+The file is read once when EC2 starts. Each entry needs a unique ID and name
+within its region and at least one IPv4 CIDR. CIDRs are normalized to network
+blocks; repeated or equivalent blocks within one list are rejected. Entries
+cannot replace the built-in lists. Invalid or unreadable files prevent startup
+when EC2 is enabled; disabled EC2 does not read the catalog. Configured lists
+have the same AWS ownership, read-only behavior, security-group reference
+validation and describe/entry responses as the built-in lists. Both configured
+and built-in CIDRs participate in security-group policy evaluation and firewall
+rules when enforcement is enabled. They are available across accounts in their
+configured region. Leaving the setting unset keeps the built-in catalog.
+
A customer-managed list may not take a name AWS reserves for its own: `com.amazonaws.`,
`com.amazon.` or `com.aws.`, each including the trailing dot. `CreateManagedPrefixList`
rejects those with `InvalidParameterValue`; a name that merely resembles one, such as
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
index 8d1f377d10..3f9cd7a212 100644
--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
@@ -3218,6 +3218,9 @@ public interface EmulatorConfig {
/** Optional full EC2 catalog file for locally built guest images. */
Optional<String> imageCatalogPath();
+ /** Optional JSON file of additional regional AWS-managed prefix lists. */
+ Optional<String> managedPrefixListCatalogPath();
+
@WithDefault("true")
boolean enabled();
diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java
index 62adf3df3d..86d7dc692c 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java
@@ -1,6 +1,7 @@
package io.github.hectorvent.floci.services.ec2;
import com.fasterxml.jackson.core.type.TypeReference;
+import com.fasterxml.jackson.databind.ObjectMapper;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AvailabilityZoneIds;
import io.github.hectorvent.floci.core.common.AwsArnUtils;
@@ -108,7 +109,11 @@ import jakarta.enterprise.context.ContextNotActiveException;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
+import java.io.IOException;
+import java.io.InputStream;
import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.time.Duration;
import java.time.Instant;
import java.time.ZoneOffset;
@@ -292,6 +297,7 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettab
private final StorageBackend<String, SpotInstanceRequest> spotInstanceRequests;
private final StorageBackend<String, NetworkAcl> networkAcls;
private final StorageBackend<String, ManagedPrefixList> managedPrefixLists;
+ private final Map<String, List<PrefixList>> configuredManagedPrefixLists;
private final StorageBackend<String, TransitGateway> transitGateways;
private final StorageBackend<String, TransitGatewayRouteTable> transitGatewayRouteTables;
private final StorageBackend<String, TransitGatewayVpcAttachment> transitGatewayVpcAttachments;
@@ -849,6 +855,7 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettab
this.networkInterfaces = networkInterfaces;
this.capacityReservations = capacityReservations;
this.volumeModifications = volumeModifications;
+ this.configuredManagedPrefixLists = loadManagedPrefixListCatalog();
}
@PostConstruct
@@ -1655,6 +1662,66 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettab
* {@code GatewayVpcEndpointAwsService}); the reversed-suffix names apply to interface endpoints only.
*/
private List<ManagedPrefixList> awsManagedPrefixLists(String region) {
+ List<ManagedPrefixList> lists = new ArrayList<>(defaultAwsManagedPrefixLists(region));
+ for (PrefixList configured : configuredManagedPrefixLists.getOrDefault(region, List.of())) {
+ lists.add(awsManagedPrefixList(region, configured.getPrefixListId(),
+ configured.getPrefixListName(), configured.getCidrs()));
+ }
+ return lists;
+ }
+
+ private Map<String, List<PrefixList>> loadManagedPrefixListCatalog() {
+ if (!config.services().ec2().enabled()) {
+ return Map.of();
+ }
+ Optional<String> path = config.services().ec2().managedPrefixListCatalogPath().filter(value -> !value.isBlank());
+ if (path.isEmpty()) {
+ return Map.of();
+ }
+ try (InputStream input = Files.newInputStream(Path.of(path.get()))) {
+ Map<String, List<PrefixList>> catalog = new ObjectMapper().readValue(input, new TypeReference<>() {});
+ if (catalog == null) {
+ throw new IllegalArgumentException("Expected a region-to-prefix-lists object");
+ }
+ for (Map.Entry<String, List<PrefixList>> region : catalog.entrySet()) {
+ if (region.getKey().isBlank() || region.getValue() == null) {
+ throw new IllegalArgumentException("Expected a nonblank region and a list");
+ }
+ Set<String> ids = new HashSet<>();
+ Set<String> names = new HashSet<>();
+ for (ManagedPrefixList list : defaultAwsManagedPrefixLists(region.getKey())) {
+ ids.add(list.getPrefixListId());
+ names.add(list.getPrefixListName());
+ }
+ for (PrefixList list : region.getValue()) {
+ if (list == null || list.getPrefixListId() == null
+ || !list.getPrefixListId().matches("pl-(?:[0-9a-f]{8}|[0-9a-f]{17})")
+ || list.getPrefixListName() == null || list.getPrefixListName().isBlank()
+ || list.getCidrs() == null || list.getCidrs().isEmpty()
+ || !ids.add(list.getPrefixListId()) || !names.add(list.getPrefixListName())) {
+ throw new IllegalArgumentException("Invalid or duplicate prefix list in " + region.getKey());
+ }
+ Set<String> canonicalCidrs = new LinkedHashSet<>();
+ for (String cidr : list.getCidrs()) {
+ Optional<String> canonical = cidr == null || cidr.contains(":")
+ ? Optional.empty() : CidrCanonicalizer.canonicalize(cidr);
+ if (canonical.isEmpty()) {
+ throw new IllegalArgumentException("Expected an IPv4 prefix list entry in " + region.getKey());
+ }
+ if (!canonicalCidrs.add(canonical.get())) {
+ throw new IllegalArgumentException("Duplicate prefix list entry in " + region.getKey());
+ }
+ }
+ list.setCidrs(List.copyOf(canonicalCidrs));
+ }
+ }
+ return Map.copyOf(catalog);
+ } catch (IOException | IllegalArgumentException error) {
+ throw new IllegalStateException("Invalid EC2 managed prefix list catalog: " + path.get(), error);
+ }
+ }
+
+ private List<ManagedPrefixList> defaultAwsManagedPrefixLists(String region) {
return List.of(
awsManagedPrefixList(region, "pl-63a5400a", "com.amazonaws." + region + ".s3",
List.of("52.216.0.0/15", "54.231.0.0/16")),
@@ -4112,15 +4179,20 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider, Resettab
}
private Map<String, List<String>> policyPrefixLists(String region, List<SecurityGroup> groups) {
+ Map<String, ManagedPrefixList> awsLists = awsManagedPrefixLists(region).stream()
+ .collect(Collectors.toMap(ManagedPrefixList::getPrefixListId, Function.identity()));
Map<String, List<String>> resolved = new LinkedHashMap<>();
for (SecurityGroup group : groups) {
Stream.concat(group.getIpPermissions().stream(),
group.getIpPermissionsEgress().stream())
.flatMap(permission -> permission.getPrefixListIds().stream())
.map(PrefixListId::getPrefixListId).distinct()
- .forEach(id -> resolved.put(id, managedPrefixLists.get(key(region, id)).isEmpty()
- ? List.of() : getManagedPrefixListEntries(region, id, null).stream()
- .map(PrefixListEntry::getCidr).toList()));
+ .forEach(id -> {
+ ManagedPrefixList list = managedPrefixLists.get(key(region, id))
+ .orElseGet(() -> awsLists.get(id));
+ resolved.put(id, list == null ? List.of() : list.currentEntries().stream()
+ .map(PrefixListEntry::getCidr).toList());
+ });
}
return resolved;
}
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index b214ab6dab..70c57b7802 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -676,6 +676,7 @@ floci:
enforce-vcpu-quotas: false
# Optional complete catalog override for locally built guest images.
image-catalog-path: ${FLOCI_SERVICES_EC2_IMAGE_CATALOG_PATH:}
+ managed-prefix-list-catalog-path: ${FLOCI_SERVICES_EC2_MANAGED_PREFIX_LIST_CATALOG_PATH:}
enabled: true
imds-port: 9169
# Sweep EC2 containers left on the daemon by a previous run that died mid-launch or
diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.java
index 872d883175..95bf1712ea 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ServiceTest.java
@@ -67,6 +67,7 @@ import org.bouncycastle.openssl.PEMKeyPair;
import org.bouncycastle.openssl.PEMParser;
import org.bouncycastle.openssl.jcajce.JcaPEMKeyConverter;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
import org.junit.jupiter.params.provider.ValueSource;
@@ -75,10 +76,13 @@ import org.mockito.InOrder;
import org.mockito.MockMakers;
import org.mockito.MockedStatic;
+import java.io.IOException;
import java.io.StringReader;
import java.lang.reflect.Field;
import java.lang.reflect.Method;
import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.security.KeyPairGenerator;
import java.security.interfaces.RSAPrivateCrtKey;
import java.time.Duration;
@@ -123,8 +127,10 @@ import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyMap;
import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.argThat;
+import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.CALLS_REAL_METHODS;
import static org.mockito.Mockito.RETURNS_SELF;
import static org.mockito.Mockito.atLeastOnce;
@@ -6006,12 +6012,176 @@ class Ec2ServiceTest {
// =========================================================================
private static Ec2Service prefixListService() {
- return new Ec2Service(mockConfig(true), mock(Ec2ContainerManager.class),
+ return prefixListService(mockConfig(true));
+ }
+
+ private static Ec2Service prefixListService(EmulatorConfig config) {
+ return new Ec2Service(config, mock(Ec2ContainerManager.class),
mock(Ec2PortForwardManager.class),
mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class), new Ec2InstanceTypeCatalog(),
new InMemoryStorageFactory());
}
+ @Test
+ void configuredAwsManagedPrefixListUsesExistingRegionalReadOnlyApis(@TempDir Path temp) throws IOException {
+ Path catalog = temp.resolve("prefix-lists.json");
+ Files.writeString(catalog, """
+ {"eu-west-1": [{"prefixListId": "pl-0123456789abcdef0",
+ "prefixListName": "example-managed-service", "cidrs": ["192.0.2.19/24"]}]}
+ """);
+ EmulatorConfig config = mockConfig(true);
+ when(config.services().ec2().enabled()).thenReturn(true);
+ when(config.services().ec2().managedPrefixListCatalogPath()).thenReturn(Optional.of(catalog.toString()));
+ Ec2Service service = prefixListService(config);
+ String id = "pl-0123456789abcdef0";
+
+ ManagedPrefixList managed = service.describeManagedPrefixLists("eu-west-1", List.of(id), Map.of()).getFirst();
+ assertTrue(managed.isAwsManaged());
+ assertEquals("AWS", managed.getOwnerId());
+ assertEquals("IPv4", managed.getAddressFamily());
+ assertEquals("arn:aws:ec2:eu-west-1:aws:prefix-list/" + id, managed.getPrefixListArn());
+ assertEquals(List.of("192.0.2.0/24"), service.describePrefixLists("eu-west-1", List.of(id), Map.of())
+ .getFirst().getCidrs());
+ assertEquals("192.0.2.0/24", service.getManagedPrefixListEntries("eu-west-1", id, null).getFirst().getCidr());
+ assertEquals(3, service.describeManagedPrefixLists("eu-west-1", List.of(), Map.of()).size());
+ assertEquals("InvalidPrefixListID.NotFound", assertThrows(AwsException.class,
+ () -> service.describeManagedPrefixLists("us-east-1", List.of(id), Map.of())).getErrorCode());
+ assertEquals("UnsupportedOperation", assertThrows(AwsException.class,
+ () -> service.deleteManagedPrefixList("eu-west-1", id)).getErrorCode());
+ assertEquals("UnsupportedOperation", assertThrows(AwsException.class,
+ () -> service.modifyManagedPrefixList("eu-west-1", id, null, "changed", null,
+ List.of(), List.of())).getErrorCode());
+
+ String groupId = service.createSecurityGroup("eu-west-1", "consumer", "consumer", null).getGroupId();
+ IpPermission permission = tcpPermission(443);
+ permission.getPrefixListIds().add(new PrefixListId(id, null));
+ assertEquals(id, service.authorizeSecurityGroupIngress("eu-west-1", groupId, List.of(permission))
+ .getFirst().getPrefixListId());
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {
+ "not-json",
+ "null",
+ "{\"eu-west-1\": null}",
+ "{\"eu-west-1\": [null]}",
+ "{\"eu-west-1\": [{\"prefixListId\":\"pl-invalid\",\"prefixListName\":\"example\",\"cidrs\":[\"192.0.2.0/24\"]}]}",
+ "{\"eu-west-1\": [{\"prefixListId\":\"pl-12345678\",\"prefixListName\":\"example\",\"cidrs\":[]}]}",
+ "{\"eu-west-1\": [{\"prefixListId\":\"pl-12345678\",\"prefixListName\":\"example\",\"cidrs\":[\"2001:db8::/32\"]}]}",
+ "{\"eu-west-1\": [{\"prefixListId\":\"pl-12345678\",\"prefixListName\":\"example\",\"cidrs\":[\"invalid\"]}]}",
+ "{\"eu-west-1\": [{\"prefixListId\":\"pl-63a5400a\",\"prefixListName\":\"example\",\"cidrs\":[\"192.0.2.0/24\"]}]}",
+ "{\"eu-west-1\": [{\"prefixListId\":\"pl-12345678\",\"prefixListName\":\"com.amazonaws.eu-west-1.s3\","
+ + "\"cidrs\":[\"192.0.2.0/24\"]}]}"
+ })
+ void invalidManagedPrefixListCatalogFailsStartup(String json, @TempDir Path temp) throws IOException {
+ Path catalog = temp.resolve("prefix-lists.json");
+ Files.writeString(catalog, json);
+ EmulatorConfig config = mockConfig(true);
+ when(config.services().ec2().enabled()).thenReturn(true);
+ when(config.services().ec2().managedPrefixListCatalogPath()).thenReturn(Optional.of(catalog.toString()));
+
+ IllegalStateException error = assertThrows(IllegalStateException.class, () -> prefixListService(config));
+ assertTrue(error.getMessage().contains(catalog.toString()));
+ }
+
+ @Test
+ void missingManagedPrefixListCatalogFailsStartup(@TempDir Path temp) {
+ EmulatorConfig config = mockConfig(true);
+ when(config.services().ec2().enabled()).thenReturn(true);
+ Path catalog = temp.resolve("missing.json");
+ when(config.services().ec2().managedPrefixListCatalogPath()).thenReturn(Optional.of(catalog.toString()));
+
+ assertThrows(IllegalStateException.class, () -> prefixListService(config));
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"192.0.2.0/24", "192.0.2.19/24"})
+ void duplicateCanonicalCatalogEntriesFailStartup(String duplicate, @TempDir Path temp) throws IOException {
+ Path catalog = temp.resolve("prefix-lists.json");
+ Files.writeString(catalog, """
+ {"eu-west-1": [{"prefixListId": "pl-0123456789abcdef0",
+ "prefixListName": "example-managed-service", "cidrs": ["192.0.2.0/24", "%s"]}]}
+ """.formatted(duplicate));
+ EmulatorConfig config = mockConfig(true);
+ when(config.services().ec2().enabled()).thenReturn(true);
+ when(config.services().ec2().managedPrefixListCatalogPath()).thenReturn(Optional.of(catalog.toString()));
+
+ assertThrows(IllegalStateException.class, () -> prefixListService(config));
+ }
+
+ @Test
+ void disabledEc2DoesNotReadManagedPrefixListCatalog(@TempDir Path temp) {
+ EmulatorConfig config = mockConfig(true);
+ when(config.services().ec2().enabled()).thenReturn(false);
+ when(config.services().ec2().managedPrefixListCatalogPath())
+ .thenReturn(Optional.of(temp.resolve("missing.json").toString()));
+
+ assertDoesNotThrow(() -> prefixListService(config));
+ }
+
+ @ParameterizedTest
+ @CsvSource({"configured,false", "configured,true", "builtin,false", "builtin,true",
+ "customer,false", "customer,true"})
+ void prefixListFirewallConsumersReceiveCanonicalEntries(String kind, boolean egress, @TempDir Path temp)
+ throws IOException {
+ String region = "eu-west-1";
+ Path catalog = temp.resolve("prefix-lists.json");
+ Files.writeString(catalog, """
+ {"eu-west-1": [{"prefixListId": "pl-0123456789abcdef0",
+ "prefixListName": "example-managed-service", "cidrs": ["192.0.2.19/24"]}]}
+ """);
+ EmulatorConfig config = mockConfig(false);
+ when(config.services().ec2().enabled()).thenReturn(true);
+ when(config.services().ec2().managedPrefixListCatalogPath()).thenReturn(Optional.of(catalog.toString()));
+ EmulatorConfig.NetworkConfig network = mock(EmulatorConfig.NetworkConfig.class);
+ EmulatorConfig.SecurityGroupEnforcementConfig enforcement = mock(EmulatorConfig.SecurityGroupEnforcementConfig.class);
+ when(config.network()).thenReturn(network);
+ when(network.securityGroupEnforcement()).thenReturn(enforcement);
+ when(enforcement.enabled()).thenReturn(true);
+ Ec2ContainerManager manager = mock(Ec2ContainerManager.class);
+ AtomicReference<Map<String, List<String>>> policyLists = new AtomicReference<>();
+ doAnswer(invocation -> {
+ policyLists.set(invocation.getArgument(2));
+ return null;
+ }).when(manager).refreshSecurityGroups(eq(region), anyMap(), anyMap());
+ Ec2Service service = new Ec2Service(config, manager, mock(Ec2PortForwardManager.class),
+ mock(AmiImageResolver.class), mock(Ec2ImageCatalog.class), new Ec2InstanceTypeCatalog(),
+ new InMemoryStorageFactory());
+ String id = switch (kind) {
+ case "configured" -> "pl-0123456789abcdef0";
+ case "builtin" -> "pl-63a5400a";
+ case "customer" -> service.createManagedPrefixList(region, "customer-list", "IPv4", 1,
+ List.of(new PrefixListEntry("192.0.2.0/24", "test")), List.of()).getPrefixListId();
+ default -> throw new AssertionError(kind);
+ };
+ SecurityGroup group = service.createSecurityGroup(region, "consumer", "consumer", null);
+ if (egress) {
+ service.revokeSecurityGroupEgress(region, group.getGroupId(), new ArrayList<>(group.getIpPermissionsEgress()));
+ }
+ IpPermission permission = tcpPermission(443);
+ permission.getPrefixListIds().add(new PrefixListId(id, null));
+ if (egress) {
+ service.authorizeSecurityGroupEgress(region, group.getGroupId(), List.of(permission));
+ } else {
+ service.authorizeSecurityGroupIngress(region, group.getGroupId(), List.of(permission));
+ }
+ String allowed = "builtin".equals(kind) ? "52.216.1.9" : "192.0.2.9";
+ String cidr = "builtin".equals(kind) ? "52.216.0.0/15" : "192.0.2.0/24";
+ Map<String, List<String>> resolved = policyLists.get();
+ assertNotNull(resolved);
+ assertTrue(resolved.get(id).contains(cidr));
+ SecurityGroupPolicy.Peer peer = new SecurityGroupPolicy.Peer(allowed, group.getOwnerId(),
+ group.getVpcId(), Set.of());
+ assertTrue(SecurityGroupPolicy.allows(List.of(group), egress, "tcp", 443, 0, peer, resolved));
+ assertFalse(SecurityGroupPolicy.allows(List.of(group), egress, "tcp", 444, 0, peer, resolved));
+ assertFalse(SecurityGroupPolicy.allows(List.of(group), egress, "tcp", 443, 0,
+ new SecurityGroupPolicy.Peer("203.0.113.9", group.getOwnerId(), group.getVpcId(), Set.of()), resolved));
+ SecurityGroupNftCompiler.Endpoint endpoint = new SecurityGroupNftCompiler.Endpoint(group.getOwnerId(), region,
+ group.getVpcId(), "eni-consumer", "10.0.0.9", "172.20.0.9", Set.of(group.getGroupId()), List.of(group));
+ String rules = SecurityGroupNftCompiler.compile(endpoint, List.of(), resolved);
+ assertTrue(rules.contains(cidr));
+ }
+
@Test
void createManagedPrefixListStoresEntriesAtVersionOne() {
Ec2Service service = prefixListService();
diff --git a/src/test/resources/application.yml b/src/test/resources/application.yml
index 905120dd0a..e642926088 100644
--- a/src/test/resources/application.yml
+++ b/src/test/resources/application.yml
@@ -321,6 +321,7 @@ floci:
enforce-vcpu-quotas: false
# Optional complete catalog override for locally built guest images.
image-catalog-path: ${FLOCI_SERVICES_EC2_IMAGE_CATALOG_PATH:}
+ managed-prefix-list-catalog-path: ${FLOCI_SERVICES_EC2_MANAGED_PREFIX_LIST_CATALOG_PATH:}
enabled: true
imds-port: 9169
ssh-port-range-start: 2200