Light, fluffy, and always free - The AWS Local Emulator alternative
fix(ec2): instances under security group enforcement no longer terminate at launch (#4510)
TATSUNO “Taz” Yasuhiro4faa2b1354d4
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.javaindex ca8347f10a..4f08114414 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java@@ -1059,8 +1059,10 @@ public class ContainerLifecycleManager {}// DNS servers — used to inject Floci's embedded DNS so spawned containers- // can resolve *.localhost.floci.io to Floci's Docker network IP.- if (spec.dnsServers() != null && !spec.dnsServers().isEmpty()) {+ // can resolve *.localhost.floci.io to Floci's Docker network IP. Docker rejects them+ // together with container:<id> network mode, where the resolver comes from that container.+ if (spec.dnsServers() != null && !spec.dnsServers().isEmpty()+ && !isContainerNetworkMode(spec.networkMode())) {hostConfig.withDns(spec.dnsServers().toArray(new String[0]));}diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/SecurityGroupFirewallManager.java b/src/main/java/io/github/hectorvent/floci/services/ec2/SecurityGroupFirewallManager.javaindex 3f8776347a..e6590faad0 100644--- a/src/main/java/io/github/hectorvent/floci/services/ec2/SecurityGroupFirewallManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/SecurityGroupFirewallManager.java@@ -10,6 +10,7 @@ import com.github.dockerjava.api.model.Info;import org.apache.commons.compress.archivers.tar.TarArchiveEntry;import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;import io.github.hectorvent.floci.core.common.docker.ContainerSpec;@@ -42,15 +43,18 @@ public class SecurityGroupFirewallManager {private final ContainerBuilder containerBuilder;private final ContainerLifecycleManager lifecycleManager;private final EmulatorConfig config;+ private final EmbeddedDnsServer embeddedDnsServer;private final Map<String, ProtectedEndpoint> endpoints = new HashMap<>();@Injectpublic SecurityGroupFirewallManager(DockerClient dockerClient, ContainerBuilder containerBuilder,- ContainerLifecycleManager lifecycleManager, EmulatorConfig config) {+ ContainerLifecycleManager lifecycleManager, EmulatorConfig config,+ EmbeddedDnsServer embeddedDnsServer) {this.dockerClient = dockerClient;this.containerBuilder = containerBuilder;this.lifecycleManager = lifecycleManager;this.config = config;+ this.embeddedDnsServer = embeddedDnsServer;}public boolean enabled() {@@ -107,6 +111,9 @@ public class SecurityGroupFirewallManager {ContainerBuilder.Builder builder = containerBuilder.newContainer(config.network().securityGroupEnforcement().helperImage()).withName(name).withDockerNetwork(dockerNetwork)+ // Workloads join this network namespace and use its resolver and hosts file.+ .withEmbeddedDns()+ .withHostDockerInternalOnLinux().withEntrypoint(List.of("sh", "-c")).withCmd(List.of("exec sleep 2147483647")).withLabels(ContainerStorageHelper.resourceIdentityLabels(service, resourceId, accountId, region))@@ -206,10 +213,11 @@ public class SecurityGroupFirewallManager {}List<SecurityGroupNftCompiler.Endpoint> peers = endpoints.values().stream().map(ProtectedEndpoint::endpoint).toList();+ List<String> vpcResolvers = vpcResolvers();for (ProtectedEndpoint protectedEndpoint : new ArrayList<>(endpoints.values())) {try {String rules = SecurityGroupNftCompiler.compile(protectedEndpoint.endpoint(), peers,- protectedEndpoint.prefixLists());+ protectedEndpoint.prefixLists(), vpcResolvers);apply(protectedEndpoint.helperId(), rules);} catch (RuntimeException e) {endpoints.values().forEach(endpoint -> quarantine(endpoint.helperId()));@@ -241,6 +249,14 @@ public class SecurityGroupFirewallManager {reconcileAll();}+ /**+ * Floci's embedded DNS stands in for the VPC's Amazon DNS server, the one resolver security+ * groups never filter on AWS. The fallback resolvers are ordinary internet destinations.+ */+ List<String> vpcResolvers() {+ return embeddedDnsServer.getServerIp().map(List::of).orElse(List.of());+ }+private void quarantine(String helperId) {try {apply(helperId, "flush chain inet floci_sg ingress\nflush chain inet floci_sg egress\n");diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/SecurityGroupNftCompiler.java b/src/main/java/io/github/hectorvent/floci/services/ec2/SecurityGroupNftCompiler.javaindex 5c30d26fb8..f520684adf 100644--- a/src/main/java/io/github/hectorvent/floci/services/ec2/SecurityGroupNftCompiler.java+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/SecurityGroupNftCompiler.java@@ -33,6 +33,11 @@ public final class SecurityGroupNftCompiler {public static String compile(Endpoint target, Collection<Endpoint> peers,Map<String, List<String>> prefixLists) {+ return compile(target, peers, prefixLists, List.of());+ }++ public static String compile(Endpoint target, Collection<Endpoint> peers,+ Map<String, List<String>> prefixLists, List<String> vpcResolvers) {if (target == null || target.groups() == null || target.groups().isEmpty()) {throw new IllegalArgumentException("Protected endpoint needs at least one security group");}@@ -43,8 +48,17 @@ public final class SecurityGroupNftCompiler {.append("add rule inet floci_sg ingress ct state established,related accept\n").append("add rule inet floci_sg egress ct state established,related accept\n");- // Docker's embedded DNS is loopback and covered above. DHCP and the- // link-local metadata endpoint are the other local infrastructure exceptions.+ // Docker's embedded DNS listens on loopback but forwards from this namespace. Security+ // groups never filter the VPC resolver on AWS. Only DNS is exempt, because the resolver+ // address also serves Floci's API. DHCP and the link-local metadata endpoint are the+ // other local infrastructure exceptions.+ for (String server : vpcResolvers) {+ String family = server.contains(":") ? "ip6" : "ip";+ rules.append("add rule inet floci_sg egress ").append(family).append(" daddr ").append(server)+ .append(" udp dport 53 accept\n")+ .append("add rule inet floci_sg egress ").append(family).append(" daddr ").append(server)+ .append(" tcp dport 53 accept\n");+ }rules.append("add rule inet floci_sg egress udp sport 68 udp dport 67 accept\n").append("add rule inet floci_sg ingress udp sport 67 udp dport 68 accept\n").append("add rule inet floci_sg egress ip daddr 169.254.169.254 accept\n")diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex 49685089d5..decccea277 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -43,8 +43,9 @@ quarkus:# The Pricing service reads its bundled snapshot, EC2 reads its image# catalog (ec2/image-catalog.yaml), SageMaker reads its instance type# catalog (sagemaker/instance-type-catalog.yaml), AppSync's GraphQL- # parser reads its message bundles, and API Gateway's Velocity engine- # reads its default properties at runtime via+ # parser reads its message bundles, EC2 security group enforcement builds+ # its helper image from docker/*.Dockerfile, and API Gateway's Velocity+ # engine reads its default properties at runtime via# ClassLoader#getResourceAsStream. Quarkus's static resource analysis# cannot detect these dynamically-built paths, so they are registered# explicitly to ensure GraalVM includes every file in the native image.@@ -57,6 +58,7 @@ quarkus:- ui/*.html- appsync/*.mjs- ec2/*.yaml+ - docker/*.Dockerfile- sagemaker/*.yaml- eks/*.json- aws/*.jsondiff --git a/src/test/java/io/github/hectorvent/floci/config/NativeImageResourceIncludesTest.java b/src/test/java/io/github/hectorvent/floci/config/NativeImageResourceIncludesTest.javaindex 8be96e96a6..ed2dd06576 100644--- a/src/test/java/io/github/hectorvent/floci/config/NativeImageResourceIncludesTest.java+++ b/src/test/java/io/github/hectorvent/floci/config/NativeImageResourceIncludesTest.java@@ -44,6 +44,9 @@ class NativeImageResourceIncludesTest {private static final String JAVA_SUFFIX = ".java";+ /** A Dockerfile path literal. LocallyBuiltHelperImage reads it through a method parameter. */+ private static final Pattern DOCKERFILE_LITERAL = Pattern.compile("\"(/?[\\w.\\-]+(?:/[\\w.\\-]+)*\\.Dockerfile)\"");+/** A {@code static final String NAME = "some/path.ext";} declaration holding a resource path. */private static final Pattern RESOURCE_CONSTANT = Pattern.compile("static\\s+final\\s+String\\s+(\\w+)\\s*=\\s*\"(/?[\\w.\\-]+(?:/[\\w.\\-]+)+\\.[A-Za-z0-9]+)\"");@@ -91,6 +94,41 @@ class NativeImageResourceIncludesTest {.reduce((a, b) -> a + "\n " + b).orElse(""));}+ @Test+ void everyHelperDockerfileIsIncludedInTheNativeImage() throws IOException {+ List<PathMatcher> includes = parseIncludes();+ Map<String, String> offenders = new LinkedHashMap<>();++ try (Stream<Path> sources = Files.walk(MAIN_SOURCE_ROOT)) {+ sources.filter(p -> p.toString().endsWith(JAVA_SUFFIX)).forEach(source -> {+ Matcher literals = DOCKERFILE_LITERAL.matcher(readSource(source));+ while (literals.find()) {+ String resource = literals.group(1).startsWith("/")+ ? literals.group(1).substring(1) : literals.group(1);+ if (Files.exists(MAIN_RESOURCE_ROOT.resolve(resource))+ && includes.stream().noneMatch(m -> m.matches(Path.of(resource)))) {+ offenders.put(resource, source.toString());+ }+ }+ });+ }++ assertTrue(offenders.isEmpty(),+ "These Dockerfiles are built from the classpath at runtime, but no glob in " + APPLICATION_YML+ + " under quarkus.native.resources.includes matches them:\n "+ + offenders.entrySet().stream()+ .map(e -> e.getKey() + " (named in " + e.getValue() + ")")+ .reduce((a, b) -> a + "\n " + b).orElse(""));+ }++ private static String readSource(Path javaFile) {+ try {+ return Files.readString(javaFile);+ } catch (IOException e) {+ throw new UncheckedIOException("Unable to read source file " + javaFile, e);+ }+ }+/** Resource paths declared as a constant in {@code javaFile} and passed to getResourceAsStream there. */private List<String> resourcesReadThroughAConstant(Path javaFile) {String source;diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerLabelsTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerLabelsTest.javaindex 3aa869851d..5a94444b36 100644--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerLabelsTest.java+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerLabelsTest.java@@ -181,6 +181,21 @@ class ContainerLifecycleManagerLabelsTest {|| hostConfig.getValue().getExtraHosts().length == 0);}+ @Test+ void createOmitsDnsForContainerNetworkMode() {+ CreateContainerCmd createCmd = stubCreateContainer();+ ContainerSpec spec = new ContainerSpec(+ "busybox:stable", null, List.of(), null, null, null, Map.of(), List.of(),+ "container:router-id", List.of(), List.of(), List.of(),+ Map.of(), null, false, null, List.of("172.18.0.2"), null, null, List.of());++ manager().create(spec);++ ArgumentCaptor<HostConfig> hostConfig = ArgumentCaptor.forClass(HostConfig.class);+ verify(createCmd).withHostConfig(hostConfig.capture());+ assertTrue(hostConfig.getValue().getDns() == null || hostConfig.getValue().getDns().length == 0);+ }+@Testvoid createIncludesNamespaceLabelWhenConfigured() {when(dockerConfig.resourceNamespace()).thenReturn(Optional.of("run-one"));diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/SecurityGroupFirewallManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/SecurityGroupFirewallManagerTest.javanew file mode 100644index 0000000000..8386a196da--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/SecurityGroupFirewallManagerTest.java@@ -0,0 +1,76 @@+package io.github.hectorvent.floci.services.ec2;++import com.github.dockerjava.api.DockerClient;+import com.github.dockerjava.api.model.Info;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;+import io.github.hectorvent.floci.core.common.docker.ContainerSpec;+import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;+import org.junit.jupiter.api.Test;+import org.mockito.ArgumentCaptor;++import java.util.List;+import java.util.Map;+import java.util.Optional;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.when;++class SecurityGroupFirewallManagerTest {++ @Test+ void helperCarriesTheResolverAndHostsItsWorkloadsUse() {+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);+ when(config.network().securityGroupEnforcement().enabled()).thenReturn(true);+ when(config.network().securityGroupEnforcement().helperImage()).thenReturn("floci/network-helper:local");+ when(config.docker().resourceNamespace()).thenReturn(Optional.empty());+ when(config.docker().imageRegistryBase()).thenReturn(Optional.empty());+ DockerClient dockerClient = mock(DockerClient.class, RETURNS_DEEP_STUBS);+ Info linux = mock(Info.class);+ when(linux.getOsType()).thenReturn("linux");+ when(dockerClient.infoCmd().exec()).thenReturn(linux);+ when(dockerClient.inspectImageCmd(anyString()).exec()).thenReturn(null);+ DockerHostResolver dockerHostResolver = mock(DockerHostResolver.class);+ when(dockerHostResolver.isLinuxHost()).thenReturn(true);+ EmbeddedDnsServer embeddedDnsServer = mock(EmbeddedDnsServer.class);+ when(embeddedDnsServer.getServerIp()).thenReturn(Optional.of("172.18.0.2"));+ ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);+ when(lifecycleManager.createAndStart(any())).thenThrow(new IllegalStateException("stop after the spec"));+ SecurityGroupFirewallManager manager = new SecurityGroupFirewallManager(dockerClient,+ new ContainerBuilder(config, dockerHostResolver, embeddedDnsServer), lifecycleManager, config,+ embeddedDnsServer);++ assertThrows(IllegalStateException.class, () -> manager.createNamespace("ec2", "i-1", "000000000000",+ "us-east-1", Optional.empty(), Map.of()));++ ArgumentCaptor<ContainerSpec> helper = ArgumentCaptor.forClass(ContainerSpec.class);+ verify(lifecycleManager).createAndStart(helper.capture());+ assertTrue(helper.getValue().dnsServers().contains("172.18.0.2"), "the helper uses Floci's DNS");+ assertTrue(helper.getValue().extraHosts().contains("host.docker.internal:host-gateway"),+ "the helper resolves host.docker.internal");+ }++ @Test+ void onlyFlocisDnsIsExemptFromTheSecurityGroups() {+ EmbeddedDnsServer embeddedDnsServer = mock(EmbeddedDnsServer.class);+ when(embeddedDnsServer.getServerIp()).thenReturn(Optional.of("172.18.0.2"));+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);+ when(config.dns().containerFallbackServers()).thenReturn(List.of("8.8.8.8", "8.8.4.4"));+ SecurityGroupFirewallManager manager = new SecurityGroupFirewallManager(mock(DockerClient.class),+ mock(ContainerBuilder.class), mock(ContainerLifecycleManager.class), config, embeddedDnsServer);++ assertEquals(List.of("172.18.0.2"), manager.vpcResolvers());++ when(embeddedDnsServer.getServerIp()).thenReturn(Optional.empty());+ assertEquals(List.of(), manager.vpcResolvers());+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/SecurityGroupNftCompilerTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/SecurityGroupNftCompilerTest.javaindex 18963de583..69bbec4f8c 100644--- a/src/test/java/io/github/hectorvent/floci/services/ec2/SecurityGroupNftCompilerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/SecurityGroupNftCompilerTest.java@@ -72,6 +72,21 @@ class SecurityGroupNftCompilerTest {assertTrue(nft.contains("ip saddr 10.1.0.0/16 meta l4proto icmpv6 accept"));}+ @Test+ void vpcResolversStayReachableWhenTheGroupsAllowNoEgress() {+ SecurityGroup group = new SecurityGroup();+ group.setVpcId("vpc-1");+ group.setOwnerId("000000000000");++ String nft = SecurityGroupNftCompiler.compile(endpoint("eni-a", "10.0.0.2", "172.17.0.2", group),+ List.of(), Map.of(), List.of("172.18.0.2", "fd00::53"));++ assertTrue(nft.contains("egress ip daddr 172.18.0.2 udp dport 53 accept"));+ assertTrue(nft.contains("egress ip daddr 172.18.0.2 tcp dport 53 accept"));+ assertTrue(nft.contains("egress ip6 daddr fd00::53 udp dport 53 accept"));+ assertFalse(nft.contains("daddr 172.18.0.2 accept"), "only DNS is allowed to the resolver");+ }+private static SecurityGroupNftCompiler.Endpoint endpoint(String eni, String logical,String transport, SecurityGroup group) {return new SecurityGroupNftCompiler.Endpoint("000000000000", "us-east-1", "vpc-1",