Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(ec2): instances under security group enforcement no longer terminate at launch (#4510)

TATSUNO “Taz” Yasuhiro4faa2b1354d4
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
index ca8347f10a..4f08114414 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
@@ -1059,8 +1059,10 @@ public class ContainerLifecycleManager {
}
// DNS servers — used to inject Floci's embedded DNS so spawned containers
- // can resolve *.localhost.floci.io to Floci's Docker network IP.
- if (spec.dnsServers() != null && !spec.dnsServers().isEmpty()) {
+ // can resolve *.localhost.floci.io to Floci's Docker network IP. Docker rejects them
+ // together with container:<id> network mode, where the resolver comes from that container.
+ if (spec.dnsServers() != null && !spec.dnsServers().isEmpty()
+ && !isContainerNetworkMode(spec.networkMode())) {
hostConfig.withDns(spec.dnsServers().toArray(new String[0]));
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/SecurityGroupFirewallManager.java b/src/main/java/io/github/hectorvent/floci/services/ec2/SecurityGroupFirewallManager.java
index 3f8776347a..e6590faad0 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ec2/SecurityGroupFirewallManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/SecurityGroupFirewallManager.java
@@ -10,6 +10,7 @@ import com.github.dockerjava.api.model.Info;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;
import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
import io.github.hectorvent.floci.core.common.docker.ContainerSpec;
@@ -42,15 +43,18 @@ public class SecurityGroupFirewallManager {
private final ContainerBuilder containerBuilder;
private final ContainerLifecycleManager lifecycleManager;
private final EmulatorConfig config;
+ private final EmbeddedDnsServer embeddedDnsServer;
private final Map<String, ProtectedEndpoint> endpoints = new HashMap<>();
@Inject
public SecurityGroupFirewallManager(DockerClient dockerClient, ContainerBuilder containerBuilder,
- ContainerLifecycleManager lifecycleManager, EmulatorConfig config) {
+ ContainerLifecycleManager lifecycleManager, EmulatorConfig config,
+ EmbeddedDnsServer embeddedDnsServer) {
this.dockerClient = dockerClient;
this.containerBuilder = containerBuilder;
this.lifecycleManager = lifecycleManager;
this.config = config;
+ this.embeddedDnsServer = embeddedDnsServer;
}
public boolean enabled() {
@@ -107,6 +111,9 @@ public class SecurityGroupFirewallManager {
ContainerBuilder.Builder builder = containerBuilder.newContainer(config.network().securityGroupEnforcement().helperImage())
.withName(name)
.withDockerNetwork(dockerNetwork)
+ // Workloads join this network namespace and use its resolver and hosts file.
+ .withEmbeddedDns()
+ .withHostDockerInternalOnLinux()
.withEntrypoint(List.of("sh", "-c"))
.withCmd(List.of("exec sleep 2147483647"))
.withLabels(ContainerStorageHelper.resourceIdentityLabels(service, resourceId, accountId, region))
@@ -206,10 +213,11 @@ public class SecurityGroupFirewallManager {
}
List<SecurityGroupNftCompiler.Endpoint> peers = endpoints.values().stream()
.map(ProtectedEndpoint::endpoint).toList();
+ List<String> vpcResolvers = vpcResolvers();
for (ProtectedEndpoint protectedEndpoint : new ArrayList<>(endpoints.values())) {
try {
String rules = SecurityGroupNftCompiler.compile(protectedEndpoint.endpoint(), peers,
- protectedEndpoint.prefixLists());
+ protectedEndpoint.prefixLists(), vpcResolvers);
apply(protectedEndpoint.helperId(), rules);
} catch (RuntimeException e) {
endpoints.values().forEach(endpoint -> quarantine(endpoint.helperId()));
@@ -241,6 +249,14 @@ public class SecurityGroupFirewallManager {
reconcileAll();
}
+ /**
+ * Floci's embedded DNS stands in for the VPC's Amazon DNS server, the one resolver security
+ * groups never filter on AWS. The fallback resolvers are ordinary internet destinations.
+ */
+ List<String> vpcResolvers() {
+ return embeddedDnsServer.getServerIp().map(List::of).orElse(List.of());
+ }
+
private void quarantine(String helperId) {
try {
apply(helperId, "flush chain inet floci_sg ingress\nflush chain inet floci_sg egress\n");
diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/SecurityGroupNftCompiler.java b/src/main/java/io/github/hectorvent/floci/services/ec2/SecurityGroupNftCompiler.java
index 5c30d26fb8..f520684adf 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ec2/SecurityGroupNftCompiler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/SecurityGroupNftCompiler.java
@@ -33,6 +33,11 @@ public final class SecurityGroupNftCompiler {
public static String compile(Endpoint target, Collection<Endpoint> peers,
Map<String, List<String>> prefixLists) {
+ return compile(target, peers, prefixLists, List.of());
+ }
+
+ public static String compile(Endpoint target, Collection<Endpoint> peers,
+ Map<String, List<String>> prefixLists, List<String> vpcResolvers) {
if (target == null || target.groups() == null || target.groups().isEmpty()) {
throw new IllegalArgumentException("Protected endpoint needs at least one security group");
}
@@ -43,8 +48,17 @@ public final class SecurityGroupNftCompiler {
.append("add rule inet floci_sg ingress ct state established,related accept\n")
.append("add rule inet floci_sg egress ct state established,related accept\n");
- // Docker's embedded DNS is loopback and covered above. DHCP and the
- // link-local metadata endpoint are the other local infrastructure exceptions.
+ // Docker's embedded DNS listens on loopback but forwards from this namespace. Security
+ // groups never filter the VPC resolver on AWS. Only DNS is exempt, because the resolver
+ // address also serves Floci's API. DHCP and the link-local metadata endpoint are the
+ // other local infrastructure exceptions.
+ for (String server : vpcResolvers) {
+ String family = server.contains(":") ? "ip6" : "ip";
+ rules.append("add rule inet floci_sg egress ").append(family).append(" daddr ").append(server)
+ .append(" udp dport 53 accept\n")
+ .append("add rule inet floci_sg egress ").append(family).append(" daddr ").append(server)
+ .append(" tcp dport 53 accept\n");
+ }
rules.append("add rule inet floci_sg egress udp sport 68 udp dport 67 accept\n")
.append("add rule inet floci_sg ingress udp sport 67 udp dport 68 accept\n")
.append("add rule inet floci_sg egress ip daddr 169.254.169.254 accept\n")
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index 49685089d5..decccea277 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -43,8 +43,9 @@ quarkus:
# The Pricing service reads its bundled snapshot, EC2 reads its image
# catalog (ec2/image-catalog.yaml), SageMaker reads its instance type
# catalog (sagemaker/instance-type-catalog.yaml), AppSync's GraphQL
- # parser reads its message bundles, and API Gateway's Velocity engine
- # reads its default properties at runtime via
+ # parser reads its message bundles, EC2 security group enforcement builds
+ # its helper image from docker/*.Dockerfile, and API Gateway's Velocity
+ # engine reads its default properties at runtime via
# ClassLoader#getResourceAsStream. Quarkus's static resource analysis
# cannot detect these dynamically-built paths, so they are registered
# explicitly to ensure GraalVM includes every file in the native image.
@@ -57,6 +58,7 @@ quarkus:
- ui/*.html
- appsync/*.mjs
- ec2/*.yaml
+ - docker/*.Dockerfile
- sagemaker/*.yaml
- eks/*.json
- aws/*.json
diff --git a/src/test/java/io/github/hectorvent/floci/config/NativeImageResourceIncludesTest.java b/src/test/java/io/github/hectorvent/floci/config/NativeImageResourceIncludesTest.java
index 8be96e96a6..ed2dd06576 100644
--- a/src/test/java/io/github/hectorvent/floci/config/NativeImageResourceIncludesTest.java
+++ b/src/test/java/io/github/hectorvent/floci/config/NativeImageResourceIncludesTest.java
@@ -44,6 +44,9 @@ class NativeImageResourceIncludesTest {
private static final String JAVA_SUFFIX = ".java";
+ /** A Dockerfile path literal. LocallyBuiltHelperImage reads it through a method parameter. */
+ private static final Pattern DOCKERFILE_LITERAL = Pattern.compile("\"(/?[\\w.\\-]+(?:/[\\w.\\-]+)*\\.Dockerfile)\"");
+
/** A {@code static final String NAME = "some/path.ext";} declaration holding a resource path. */
private static final Pattern RESOURCE_CONSTANT = Pattern.compile(
"static\\s+final\\s+String\\s+(\\w+)\\s*=\\s*\"(/?[\\w.\\-]+(?:/[\\w.\\-]+)+\\.[A-Za-z0-9]+)\"");
@@ -91,6 +94,41 @@ class NativeImageResourceIncludesTest {
.reduce((a, b) -> a + "\n " + b).orElse(""));
}
+ @Test
+ void everyHelperDockerfileIsIncludedInTheNativeImage() throws IOException {
+ List<PathMatcher> includes = parseIncludes();
+ Map<String, String> offenders = new LinkedHashMap<>();
+
+ try (Stream<Path> sources = Files.walk(MAIN_SOURCE_ROOT)) {
+ sources.filter(p -> p.toString().endsWith(JAVA_SUFFIX)).forEach(source -> {
+ Matcher literals = DOCKERFILE_LITERAL.matcher(readSource(source));
+ while (literals.find()) {
+ String resource = literals.group(1).startsWith("/")
+ ? literals.group(1).substring(1) : literals.group(1);
+ if (Files.exists(MAIN_RESOURCE_ROOT.resolve(resource))
+ && includes.stream().noneMatch(m -> m.matches(Path.of(resource)))) {
+ offenders.put(resource, source.toString());
+ }
+ }
+ });
+ }
+
+ assertTrue(offenders.isEmpty(),
+ "These Dockerfiles are built from the classpath at runtime, but no glob in " + APPLICATION_YML
+ + " under quarkus.native.resources.includes matches them:\n "
+ + offenders.entrySet().stream()
+ .map(e -> e.getKey() + " (named in " + e.getValue() + ")")
+ .reduce((a, b) -> a + "\n " + b).orElse(""));
+ }
+
+ private static String readSource(Path javaFile) {
+ try {
+ return Files.readString(javaFile);
+ } catch (IOException e) {
+ throw new UncheckedIOException("Unable to read source file " + javaFile, e);
+ }
+ }
+
/** Resource paths declared as a constant in {@code javaFile} and passed to getResourceAsStream there. */
private List<String> resourcesReadThroughAConstant(Path javaFile) {
String source;
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerLabelsTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerLabelsTest.java
index 3aa869851d..5a94444b36 100644
--- a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerLabelsTest.java
+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerLabelsTest.java
@@ -181,6 +181,21 @@ class ContainerLifecycleManagerLabelsTest {
|| hostConfig.getValue().getExtraHosts().length == 0);
}
+ @Test
+ void createOmitsDnsForContainerNetworkMode() {
+ CreateContainerCmd createCmd = stubCreateContainer();
+ ContainerSpec spec = new ContainerSpec(
+ "busybox:stable", null, List.of(), null, null, null, Map.of(), List.of(),
+ "container:router-id", List.of(), List.of(), List.of(),
+ Map.of(), null, false, null, List.of("172.18.0.2"), null, null, List.of());
+
+ manager().create(spec);
+
+ ArgumentCaptor<HostConfig> hostConfig = ArgumentCaptor.forClass(HostConfig.class);
+ verify(createCmd).withHostConfig(hostConfig.capture());
+ assertTrue(hostConfig.getValue().getDns() == null || hostConfig.getValue().getDns().length == 0);
+ }
+
@Test
void createIncludesNamespaceLabelWhenConfigured() {
when(dockerConfig.resourceNamespace()).thenReturn(Optional.of("run-one"));
diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/SecurityGroupFirewallManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/SecurityGroupFirewallManagerTest.java
new file mode 100644
index 0000000000..8386a196da
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/SecurityGroupFirewallManagerTest.java
@@ -0,0 +1,76 @@
+package io.github.hectorvent.floci.services.ec2;
+
+import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.model.Info;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;
+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
+import io.github.hectorvent.floci.core.common.docker.ContainerSpec;
+import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;
+import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
+
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+class SecurityGroupFirewallManagerTest {
+
+ @Test
+ void helperCarriesTheResolverAndHostsItsWorkloadsUse() {
+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+ when(config.network().securityGroupEnforcement().enabled()).thenReturn(true);
+ when(config.network().securityGroupEnforcement().helperImage()).thenReturn("floci/network-helper:local");
+ when(config.docker().resourceNamespace()).thenReturn(Optional.empty());
+ when(config.docker().imageRegistryBase()).thenReturn(Optional.empty());
+ DockerClient dockerClient = mock(DockerClient.class, RETURNS_DEEP_STUBS);
+ Info linux = mock(Info.class);
+ when(linux.getOsType()).thenReturn("linux");
+ when(dockerClient.infoCmd().exec()).thenReturn(linux);
+ when(dockerClient.inspectImageCmd(anyString()).exec()).thenReturn(null);
+ DockerHostResolver dockerHostResolver = mock(DockerHostResolver.class);
+ when(dockerHostResolver.isLinuxHost()).thenReturn(true);
+ EmbeddedDnsServer embeddedDnsServer = mock(EmbeddedDnsServer.class);
+ when(embeddedDnsServer.getServerIp()).thenReturn(Optional.of("172.18.0.2"));
+ ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);
+ when(lifecycleManager.createAndStart(any())).thenThrow(new IllegalStateException("stop after the spec"));
+ SecurityGroupFirewallManager manager = new SecurityGroupFirewallManager(dockerClient,
+ new ContainerBuilder(config, dockerHostResolver, embeddedDnsServer), lifecycleManager, config,
+ embeddedDnsServer);
+
+ assertThrows(IllegalStateException.class, () -> manager.createNamespace("ec2", "i-1", "000000000000",
+ "us-east-1", Optional.empty(), Map.of()));
+
+ ArgumentCaptor<ContainerSpec> helper = ArgumentCaptor.forClass(ContainerSpec.class);
+ verify(lifecycleManager).createAndStart(helper.capture());
+ assertTrue(helper.getValue().dnsServers().contains("172.18.0.2"), "the helper uses Floci's DNS");
+ assertTrue(helper.getValue().extraHosts().contains("host.docker.internal:host-gateway"),
+ "the helper resolves host.docker.internal");
+ }
+
+ @Test
+ void onlyFlocisDnsIsExemptFromTheSecurityGroups() {
+ EmbeddedDnsServer embeddedDnsServer = mock(EmbeddedDnsServer.class);
+ when(embeddedDnsServer.getServerIp()).thenReturn(Optional.of("172.18.0.2"));
+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+ when(config.dns().containerFallbackServers()).thenReturn(List.of("8.8.8.8", "8.8.4.4"));
+ SecurityGroupFirewallManager manager = new SecurityGroupFirewallManager(mock(DockerClient.class),
+ mock(ContainerBuilder.class), mock(ContainerLifecycleManager.class), config, embeddedDnsServer);
+
+ assertEquals(List.of("172.18.0.2"), manager.vpcResolvers());
+
+ when(embeddedDnsServer.getServerIp()).thenReturn(Optional.empty());
+ assertEquals(List.of(), manager.vpcResolvers());
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/SecurityGroupNftCompilerTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/SecurityGroupNftCompilerTest.java
index 18963de583..69bbec4f8c 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ec2/SecurityGroupNftCompilerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/SecurityGroupNftCompilerTest.java
@@ -72,6 +72,21 @@ class SecurityGroupNftCompilerTest {
assertTrue(nft.contains("ip saddr 10.1.0.0/16 meta l4proto icmpv6 accept"));
}
+ @Test
+ void vpcResolversStayReachableWhenTheGroupsAllowNoEgress() {
+ SecurityGroup group = new SecurityGroup();
+ group.setVpcId("vpc-1");
+ group.setOwnerId("000000000000");
+
+ String nft = SecurityGroupNftCompiler.compile(endpoint("eni-a", "10.0.0.2", "172.17.0.2", group),
+ List.of(), Map.of(), List.of("172.18.0.2", "fd00::53"));
+
+ assertTrue(nft.contains("egress ip daddr 172.18.0.2 udp dport 53 accept"));
+ assertTrue(nft.contains("egress ip daddr 172.18.0.2 tcp dport 53 accept"));
+ assertTrue(nft.contains("egress ip6 daddr fd00::53 udp dport 53 accept"));
+ assertFalse(nft.contains("daddr 172.18.0.2 accept"), "only DNS is allowed to the resolver");
+ }
+
private static SecurityGroupNftCompiler.Endpoint endpoint(String eni, String logical,
String transport, SecurityGroup group) {
return new SecurityGroupNftCompiler.Endpoint("000000000000", "us-east-1", "vpc-1",