Light, fluffy, and always free - The AWS Local Emulator alternative
fix(eks): replace a surviving k3s container left on another Docker network (#6063)
Robin Baeckmanaeb798939bc5
diff --git a/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java b/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.javaindex 5de58680a1..064b5ddd34 100644--- a/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java@@ -4,6 +4,7 @@ import com.fasterxml.jackson.core.type.TypeReference;import com.github.dockerjava.api.DockerClient;import com.github.dockerjava.api.command.ExecCreateCmdResponse;import com.github.dockerjava.api.command.InspectContainerResponse;+import com.github.dockerjava.api.model.Container;import com.github.dockerjava.api.model.ContainerNetwork;import com.github.dockerjava.api.model.Info;import io.github.hectorvent.floci.config.EmulatorConfig;@@ -775,10 +776,11 @@ public class EksClusterManager* Re-latches a persisted cluster onto its k3s container after a Floci restart. A surviving* container - running, or stopped by a Docker daemon reboot - is adopted (started if needed),* keeping its published API server port and data volume, so the cluster's workloads come back- * as they were. When the container is gone, the cluster is recreated via {@link #startCluster};- * the named k3s data volume is reused if it survived. Callers should put the cluster back into- * CREATING so the readiness poller re-verifies the API server and re-extracts the certificate- * authority before marking it ACTIVE again.+ * as they were. A survivor whose node capacity changed, or that is not on the Docker network+ * Floci now uses, is replaced on the same volume. When the container is gone, the cluster is+ * recreated via {@link #startCluster}; the named k3s data volume is reused if it survived.+ * Callers should put the cluster back into CREATING so the readiness poller re-verifies the API+ * server and re-extracts the certificate authority before marking it ACTIVE again.*/public void restoreCluster(Cluster cluster) {if (cluster.getDockerName() == null) {@@ -816,11 +818,30 @@ public class EksClusterManagerstartCluster(cluster);return;}- if (capacityChanged) {+ if (capacityChanged || onAnotherNetwork(existing.get())) {replaceClusterContainer(cluster, null);}}+ // Floci in a container reaches k3s over the Docker network; on the host it uses the published port.+ private boolean onAnotherNetwork(Container survivor) {+ if (!containerDetector.isRunningInContainer()) {+ return false;+ }+ Optional<String> network = containerBuilder.resolveDockerNetwork(config.services().eks().dockerNetwork());+ if (network.isEmpty() || survivor.getNetworkSettings() == null+ || survivor.getNetworkSettings().getNetworks() == null+ || survivor.getNetworkSettings().getNetworks().isEmpty()) {+ return false;+ }+ boolean moved = !survivor.getNetworkSettings().getNetworks().containsKey(network.get());+ if (moved) {+ LOG.infov("Surviving k3s container {0} is not on Docker network {1}; replacing it (its data volume is kept)",+ survivor.getId(), network.get());+ }+ return moved;+ }+private boolean adoptSurvivingCluster(Cluster cluster, String containerId) {if (config.services().eks().irsaSigningKey() && oidcService != null) {reinjectSigningKeys(containerId, cluster);@@ -885,9 +906,9 @@ public class EksClusterManagercloseQuietly(clusterLogHandles.remove(clusterResourceName(cluster)));cluster.setContainerId(null);startCluster(cluster, oldPort);- LOG.infov("Replaced EKS cluster {0} to apply current node capacity limits", cluster.getName());+ LOG.infov("Replaced the k3s container of EKS cluster {0}", cluster.getName());} catch (RuntimeException replacement) {- LOG.warnv("Could not replace EKS cluster {0} for node capacity: {1}; restoring surviving node",+ LOG.warnv("Could not replace the k3s container of EKS cluster {0}: {1}; restoring surviving node",cluster.getName(), replacement.getMessage());restorePreviousNodeType(cluster, previousType);if (!renamed) {@@ -901,7 +922,7 @@ public class EksClusterManager}} catch (RuntimeException rollback) {throw new IllegalStateException("Could not restore EKS cluster " + cluster.getName()- + " after node capacity replacement failed", rollback);+ + " after its k3s container replacement failed", rollback);}return false;}diff --git a/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.javaindex 0590e6dce1..d24e94dd67 100644--- a/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java@@ -492,6 +492,7 @@ class EksClusterManagerTest {private PortAllocator portAllocator;private DockerClient dockerClient;private ContainerBuilder containerBuilder;+ private ContainerDetector containerDetector;private EksClusterManager manager;@BeforeEach@@ -521,8 +522,9 @@ class EksClusterManagerTest {dockerClient = Mockito.mock(DockerClient.class, Mockito.RETURNS_DEEP_STUBS);when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);+ containerDetector = Mockito.mock(ContainerDetector.class);manager = new EksClusterManager(containerBuilder, lifecycleManager,- Mockito.mock(ContainerDetector.class), portAllocator,+ containerDetector, portAllocator,Mockito.mock(DockerHostResolver.class), Mockito.mock(EcrRegistryManager.class),config, Mockito.mock(RegionResolver.class));}@@ -642,6 +644,64 @@ class EksClusterManagerTest {assertEquals(Boolean.FALSE, cluster.getDefaultStorageClass());}+ @Test+ void replacesASurvivorLeftOnANetworkFlociNoLongerUses() {+ when(containerDetector.isRunningInContainer()).thenReturn(true);+ restoreSurvivorOn("\"bridge\":{\"IPAddress\":\"172.17.0.3\"}");+ stubFreshStart("cid-new", 6440);++ Cluster cluster = cluster();+ manager.restoreCluster(cluster);++ assertEquals("cid-new", cluster.getContainerId());+ verify(dockerClient).stopContainerCmd("cid-old");+ }++ @Test+ void keepsASurvivorOnTheNetworkFlociUsesBesideTheDefaultBridge() {+ when(containerDetector.isRunningInContainer()).thenReturn(true);+ restoreSurvivorOn("\"bridge\":{\"IPAddress\":\"172.17.0.3\"},"+ + "\"floci_default\":{\"IPAddress\":\"172.18.0.4\"}");++ Cluster cluster = cluster();+ manager.restoreCluster(cluster);++ assertEquals("cid-old", cluster.getContainerId());+ verify(lifecycleManager, never()).create(any());+ }++ @Test+ void keepsASurvivorThatReportsNoNetworks() {+ when(containerDetector.isRunningInContainer()).thenReturn(true);+ restoreSurvivorOn("");++ Cluster cluster = cluster();+ manager.restoreCluster(cluster);++ assertEquals("cid-old", cluster.getContainerId());+ verify(lifecycleManager, never()).create(any());+ }++ @Test+ void keepsASurvivorOnAnotherNetworkWhenFlociRunsOnTheHost() {+ restoreSurvivorOn("\"bridge\":{\"IPAddress\":\"172.17.0.3\"}");++ Cluster cluster = cluster();+ manager.restoreCluster(cluster);++ assertEquals("cid-old", cluster.getContainerId());+ verify(lifecycleManager, never()).create(any());+ }++ private void restoreSurvivorOn(String networks) {+ when(containerBuilder.resolveDockerNetwork(Optional.empty())).thenReturn(Optional.of("floci_default"));+ when(lifecycleManager.findByName("floci-eks-demo")).thenReturn(Optional.of(containerFromJson(+ "{\"Id\":\"cid-old\",\"Labels\":{\"io.floci.eks.node-capacity\":\"m5.large:unbounded\"},"+ + "\"NetworkSettings\":{\"Networks\":{" + networks + "}}}")));+ when(lifecycleManager.adopt(eq("cid-old"), eq(List.of(6443)), any()))+ .thenReturn(new ContainerInfo("cid-old", Map.of(), Map.of(6443, 6440)));+ }+@Testvoid recreatesAnOldSurvivorWithoutCapacityLimits() {when(lifecycleManager.findByName("floci-eks-demo"))