Light, fluffy, and always free - The AWS Local Emulator alternative
fix(eks): implement cluster restoration after restart to maintain state (#2780)
Matheus Andréeaecb3642598
diff --git a/docs/services/eks.md b/docs/services/eks.mdindex e3c82a2697..e9c0ac9e5f 100644--- a/docs/services/eks.md+++ b/docs/services/eks.md@@ -73,6 +73,29 @@ services:!!! note "No port mapping needed for k3s ports"k3s containers bind their API server port (6500–6599) directly on the host via Docker — no `ports:` entry is required in `docker-compose.yml`. See [Ports Reference](../configuration/ports.md#ports-65006599-eks-real-mode) for the full explanation.+#### Clusters survive a restart++With a persistent [storage mode](../configuration/storage.md) (the default), clusters recorded in+`eks-clusters.json` are **re-latched to their k3s containers when Floci starts**:++- A surviving container (for example after a Docker Desktop / daemon reboot) is adopted and+ started in place, keeping its published API server port and data volume — deployments come+ back as they were.+- A missing container is recreated. Its named k3s data volume (`floci-eks-<name>`; for a+ [non-default account](../configuration/multi-account.md), `floci-eks-<account>.<name>`) is+ reused if it survived; volumes follow the global prune policy+ (`FLOCI_STORAGE_PRUNE_VOLUMES_ON_DELETE`, default `false`), so they are retained when the+ container is stopped or the cluster deleted, except in `memory` storage mode.+- A non-default-account cluster created before account-qualified naming keeps its historical+ `floci-eks-<name>` container and volume: restoration adopts the surviving container when its+ `io.floci.account` label matches the owning account, so pre-upgrade workloads are not+ orphaned.++A restored cluster reports `CREATING` until its API server answers again, then returns to+`ACTIVE` with a freshly extracted certificate authority. If the container cannot be brought+back (for example Docker is unavailable), the cluster is marked `FAILED` instead of appearing+`ACTIVE` while unreachable.+## Configuration| Variable | Default | Description |diff --git a/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java b/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.javaindex 285bc15728..15aeff762e 100644--- a/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java@@ -89,7 +89,10 @@ public class EksClusterManager {*/public void startCluster(Cluster cluster) {String image = config.services().eks().defaultImage();- String containerName = ContainerStorageHelper.resourceName(config, "eks", null, cluster.getName());+ if (cluster.getDockerName() == null) {+ cluster.setDockerName(accountQualifiedName(cluster));+ }+ String containerName = cluster.getDockerName();LOG.infov("Starting k3s container for EKS cluster: {0} using image {1}",cluster.getName(), image);@@ -113,10 +116,15 @@ public class EksClusterManager {// socket (kine.sock) on macOS APFS, which returns EINVAL on chmod — crashing// k3s before it can start. Named volumes live in the Docker VM's Linux// filesystem, so chmod works correctly and data persists across container restarts.- String volumeName = ContainerStorageHelper.resourceName(config, "eks", null, cluster.getName());+ String volumeName = cluster.getDockerName();List<String> serverArgs = buildServerArgs(config.services().eks().disableCni());+ // The account label comes from the cluster record when set (restore runs with no request+ // context); regionResolver is the fallback for the create path.+ String labelAccountId = cluster.getAccountId() != null+ ? cluster.getAccountId()+ : regionResolver.getAccountId();ContainerBuilder.Builder specBuilder = containerBuilder.newContainer(image).withName(containerName).withEnv("K3S_KUBECONFIG_MODE", "644")@@ -126,7 +134,7 @@ public class EksClusterManager {.withPrivileged(true).withLogRotation().withLabels(ContainerStorageHelper.resourceIdentityLabels(- "eks", cluster.getName(), regionResolver.getAccountId(), regionResolver.getDefaultRegion()));+ "eks", cluster.getName(), labelAccountId, regionResolver.getDefaultRegion()));// Wire a token-authentication webhook so `aws eks get-token` bearer tokens are validated by// Floci and mapped to cluster-admin. The k3s API server POSTs a TokenReview to Floci's@@ -169,15 +177,75 @@ public class EksClusterManager {injectEcrRegistryMirror(containerId, cluster.getName());ContainerInfo info = lifecycleManager.startCreated(containerId, spec);- // Public endpoint: see floci.services.eks.endpoint-mode. `host` (default) is the host-reachable- // published port (k3s cert carries `--tls-san=localhost`, so it verifies against the CA that- // describe-cluster returns); `network` is the container DNS name (pre-#1118 behaviour).+ applyEndpoints(cluster, containerName, hostPort, info);++ LOG.infov("k3s container {0} started for cluster {1} on port {2} (internal: {3})",+ containerId, cluster.getName(), String.valueOf(hostPort), cluster.getInternalEndpoint());+ }++ /**+ * Re-latches a persisted cluster onto its k3s container after a Floci restart. A surviving+ * container — running, or stopped by a Docker daemon reboot — is adopted (started if needed),+ * keeping its published API server port and data volume, so the cluster's workloads come back+ * as they were. When the container is gone, the cluster is recreated via {@link #startCluster};+ * the named k3s data volume is reused if it survived. Callers should put the cluster back into+ * CREATING so the readiness poller re-verifies the API server and re-extracts the certificate+ * authority before marking it ACTIVE again.+ */+ public void restoreCluster(Cluster cluster) {+ if (cluster.getDockerName() == null) {+ cluster.setDockerName(resolveRestoredDockerName(cluster));+ }+ String containerName = cluster.getDockerName();+ var existing = lifecycleManager.findByName(containerName);+ if (existing.isEmpty()) {+ LOG.infov("No surviving k3s container for EKS cluster {0}; recreating it "+ + "(a surviving data volume is reused)", cluster.getName());+ startCluster(cluster);+ return;+ }++ ContainerInfo info;+ try {+ info = lifecycleManager.adopt(existing.get().getId(), List.of(K3S_API_SERVER_PORT));+ } catch (Exception e) {+ LOG.warnv("Could not adopt surviving k3s container {0} for EKS cluster {1} ({2}); recreating it",+ containerName, cluster.getName(), e.getMessage());+ startCluster(cluster);+ return;+ }++ var publishedPort = info.publishedHostPort(K3S_API_SERVER_PORT);+ if (publishedPort.isEmpty()) {+ LOG.warnv("Surviving k3s container {0} publishes no API server port; recreating it", containerName);+ startCluster(cluster);+ return;+ }++ int hostPort = publishedPort.getAsInt();+ // Keep the allocator away from a port Docker already holds for this cluster.+ portAllocator.markReserved(hostPort);+ cluster.setContainerId(info.containerId());+ cluster.setHostPort(hostPort);+ applyEndpoints(cluster, containerName, hostPort, info);++ LOG.infov("Adopted surviving k3s container {0} for EKS cluster {1} on port {2} (internal: {3})",+ info.containerId(), cluster.getName(), String.valueOf(hostPort), cluster.getInternalEndpoint());+ }++ /**+ * Sets the cluster's public and internal endpoints for a started or adopted container.+ * Public endpoint: see floci.services.eks.endpoint-mode. `host` (default) is the host-reachable+ * published port (k3s cert carries `--tls-san=localhost`, so it verifies against the CA that+ * describe-cluster returns); `network` is the container DNS name (pre-#1118 behaviour).+ * The internal endpoint uses the resolved container IP so the readiness poller works from inside+ * the Docker network (where localhost:<hostPort> would not reach the k3s container).+ */+ private void applyEndpoints(Cluster cluster, String containerName, int hostPort, ContainerInfo info) {cluster.setEndpoint(resolvePublicEndpoint(containerDetector.isRunningInContainer(), config.services().eks().endpointMode(),containerName, hostPort));- // Internal endpoint uses the resolved container IP so the readiness poller works from inside- // the Docker network (where localhost:<hostPort> would not reach the k3s container).if (containerDetector.isRunningInContainer()) {ContainerLifecycleManager.EndpointInfo ep = info.getEndpoint(K3S_API_SERVER_PORT);cluster.setInternalEndpoint(ep != null@@ -186,9 +254,6 @@ public class EksClusterManager {} else {cluster.setInternalEndpoint("https://localhost:" + hostPort);}-- LOG.infov("k3s container {0} started for cluster {1} on port {2} (internal: {3})",- containerId, cluster.getName(), String.valueOf(hostPort), cluster.getInternalEndpoint());}/**@@ -250,7 +315,10 @@ public class EksClusterManager {}/**- * Stops and removes the k3s container for the given cluster.+ * Stops and removes the k3s container for the given cluster. The k3s data volume follows the+ * storage prune policy ({@link ContainerStorageHelper#removeNamedVolume}): it is only removed+ * in {@code memory} storage mode or when {@code prune-volumes-on-delete} is set, so a persisted+ * cluster's workloads survive a Floci restart and are re-latched by {@link #restoreCluster}.*/public void stopCluster(Cluster cluster) {if (cluster.getContainerId() == null) {@@ -261,10 +329,107 @@ public class EksClusterManager {return;}lifecycleManager.stopAndRemove(cluster.getContainerId(), null);- lifecycleManager.removeVolume(ContainerStorageHelper.resourceName(config, "eks", null, cluster.getName()));+ ContainerStorageHelper.removeNamedVolume(config, lifecycleManager, clusterResourceName(cluster));LOG.infov("Stopped k3s container for cluster {0}", cluster.getName());}+ /**+ * Docker container/volume name for a cluster: the name already resolved for this record+ * ({@link Cluster#getDockerName()}, set by startCluster/restoreCluster), or the+ * account-qualified name for a record no container operation has touched yet.+ */+ String clusterResourceName(Cluster cluster) {+ return cluster.getDockerName() != null ? cluster.getDockerName() : accountQualifiedName(cluster);+ }++ /**+ * Account-qualified Docker name for a cluster. Cluster names are unique only within an+ * account, so a non-default account's cluster is qualified with its account ID — otherwise+ * two accounts' same-named clusters would resolve to the same container and data volume,+ * letting one account reach (or, via startCluster's stale-container removal, destroy) the+ * other's workloads. The default account keeps the historical unqualified name so existing+ * containers, volumes, and {@code endpoint-mode=network} DNS names keep working.+ *+ * <p>The qualifier separator is a dot: EksService validates cluster names against the AWS+ * charset ({@code [0-9A-Za-z][A-Za-z0-9\-_]*}), which admits no dot, so no default-account+ * cluster name can spell out {@code <accountId>.<name>} and collide with another account's+ * qualified name — a dash separator would (cluster "999999999999-demo" vs account+ * 999999999999's "demo"). Dots are valid in Docker container and volume names.+ *+ * <p>The record's accountId is set before every call path reaches here (createCluster on+ * create; the startup account rehydration on restore/stop); a null falls back to the+ * default-account name.+ */+ private String accountQualifiedName(Cluster cluster) {+ String accountId = cluster.getAccountId();+ boolean defaultAccount = accountId == null || accountId.equals(config.defaultAccountId());+ return ContainerStorageHelper.resourceName(config, "eks", null,+ defaultAccount ? cluster.getName() : accountId + "." + cluster.getName());+ }++ /**+ * Resolves which Docker name a restored record's resources actually live under. Clusters+ * created before account-qualified naming used the account-independent legacy name+ * {@code floci-eks-<name>} for every account — a non-default account's cluster must keep+ * that name when its own container survived there, or the upgrade would recreate the+ * cluster under the qualified name and orphan the historical workloads. The legacy+ * container is claimed only when its {@code io.floci.account} label matches the owning+ * account; another account's container — or one with no verifiable owner — is left+ * untouched and the cluster starts fresh under the qualified name. A surviving legacy+ * volume without its container carries no ownership label and is deliberately not claimed —+ * it is reported via {@link #warnUnclaimedLegacyState} so the operator can migrate the data+ * by hand. The result is deterministic across restarts for a given Docker state.+ */+ private String resolveRestoredDockerName(Cluster cluster) {+ String qualified = accountQualifiedName(cluster);+ String legacy = ContainerStorageHelper.resourceName(config, "eks", null, cluster.getName());+ if (legacy.equals(qualified)) {+ return legacy; // default account: the names never diverged+ }+ var legacySurvivor = lifecycleManager.findByName(legacy);+ if (legacySurvivor.isPresent()) {+ var labels = legacySurvivor.get().getLabels();+ String owner = labels != null ? labels.get("io.floci.account") : null;+ if (cluster.getAccountId() != null && cluster.getAccountId().equals(owner)) {+ LOG.infov("EKS cluster {0} (account {1}) keeps its pre-upgrade Docker name {2}",+ cluster.getName(), cluster.getAccountId(), legacy);+ return legacy;+ }+ if (owner == null) {+ warnUnclaimedLegacyState(cluster, legacy, "container");+ }+ // A container labeled with another account is simply not this cluster's — no warning.+ } else if (volumeExists(legacy)) {+ warnUnclaimedLegacyState(cluster, legacy, "data volume");+ }+ return qualified;+ }++ /**+ * A legacy-named container or volume whose owning account cannot be verified is never claimed+ * for a non-default account — handing it over on a guess would expose another account's data,+ * the very cross-bind the qualified names exist to prevent. It is reported instead of being+ * silently orphaned, so an operator who knows the data belongs to this cluster can migrate it+ * into the qualified volume by hand.+ */+ private void warnUnclaimedLegacyState(Cluster cluster, String legacy, String kind) {+ LOG.warnv("EKS cluster {0} (account {1}) starts under its account-qualified Docker name; "+ + "a pre-upgrade {2} named {3} survives but carries no verifiable owning account, "+ + "so it is NOT adopted. If its data belongs to this cluster, copy it into the "+ + "cluster's qualified volume manually (docker volume inspect {3}).",+ cluster.getName(), cluster.getAccountId(), kind, legacy);+ }++ /** Whether a Docker volume with this exact name exists. Any lookup failure counts as absent. */+ private boolean volumeExists(String volumeName) {+ try {+ lifecycleManager.getDockerClient().inspectVolumeCmd(volumeName).exec();+ return true;+ } catch (Exception e) {+ return false;+ }+ }+/*** Builds the k3s {@code server} command-line args. When {@code disableCni} is true, flannel,* k3s's default network policy controller, and kube-proxy are all disabled up front — see thediff --git a/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java b/src/main/java/io/github/hectorvent/floci/services/eks/EksService.javaindex 35d7b109c1..cd05123900 100644--- a/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java+++ b/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java@@ -52,6 +52,10 @@ public class EksService implements TagHandler, ResourceProvider {private static final Logger LOG = Logger.getLogger(EksService.class);+ /** The AWS charset for EKS cluster names. It admits no dot, which the Docker-name account+ * qualifier relies on — see EksClusterManager#accountQualifiedName. */+ static final String CLUSTER_NAME_REGEX = "[0-9A-Za-z][A-Za-z0-9\\-_]*";+private final StorageBackend<String, Cluster> storage;private final StorageBackend<String, Nodegroup> nodeGroupStorage;private final StorageBackend<String, FargateProfile> fargateProfileStorage;@@ -86,10 +90,72 @@ public class EksService implements TagHandler, ResourceProvider {public void init() {backfillOidcIdentities();if (!config.services().eks().mock()) {+ restorePersistedClusters();startReadinessPoller();}}+ /**+ * Re-latches persisted clusters onto their k3s containers after a restart (#2609). Without+ * this, a cluster restored from {@code eks-clusters.json} reported ACTIVE but its container+ * was never restarted after a Docker daemon reboot, so every kubectl/deploy against it failed.+ * A surviving container is adopted (and started if stopped); a missing one is recreated+ * against the cluster's retained data volume. Restored clusters go back to CREATING so the+ * readiness poller re-verifies the API server and re-extracts the certificate authority+ * before marking them ACTIVE again.+ */+ private void restorePersistedClusters() {+ for (AccountAwareStorageBackend.AccountEntry<Cluster> entry : allClusterEntries()) {+ Cluster cluster = entry.value();+ if (cluster.getContainerId() != null+ || (cluster.getStatus() != ClusterStatus.ACTIVE+ && cluster.getStatus() != ClusterStatus.CREATING)) {+ continue;+ }+ // Cluster.accountId is @JsonIgnore, so a reloaded record carries none — the owning+ // account must come from the storage key, or a non-default account's cluster would be+ // written back under the default account (stale owner record + duplicate). Rehydrate+ // it on the record too, so the readiness poller's later put lands under the owner.+ if (cluster.getAccountId() == null) {+ cluster.setAccountId(entry.accountId());+ }+ // A persisted name that predates create-time validation may violate the AWS charset —+ // in particular contain a dot, which could spell out another account's qualified+ // Docker name and cross-bind its container. Such a record is never restored; the+ // cluster must be deleted and recreated under a valid name.+ if (cluster.getName() == null || !cluster.getName().matches(CLUSTER_NAME_REGEX)) {+ LOG.errorv("Not restoring EKS cluster \"{0}\" (account {1}): its persisted name "+ + "violates the AWS charset and could alias another account''s Docker "+ + "resources. Delete it and recreate it under a valid name.",+ cluster.getName(), entry.accountId());+ cluster.setStatus(ClusterStatus.FAILED);+ putClusterForAccount(entry.accountId(), cluster);+ continue;+ }+ try {+ LOG.infov("Restoring k3s container for persisted EKS cluster {0}", cluster.getName());+ cluster.setStatus(ClusterStatus.CREATING);+ clusterManager.restoreCluster(cluster);+ } catch (Exception e) {+ LOG.errorv("Failed to restore k3s container for EKS cluster {0}: {1}",+ cluster.getName(), e.getMessage());+ cluster.setStatus(ClusterStatus.FAILED);+ }+ putClusterForAccount(entry.accountId(), cluster);+ }+ }++ private List<AccountAwareStorageBackend.AccountEntry<Cluster>> allClusterEntries() {+ if (storage instanceof AccountAwareStorageBackend<Cluster> aware) {+ return aware.scanAllAccountEntries(k -> true);+ }+ return storage.scan(k -> true).stream()+ .map(cluster -> new AccountAwareStorageBackend.AccountEntry<>(+ cluster.getAccountId() != null ? cluster.getAccountId() : regionResolver.getAccountId(),+ cluster.getName(), cluster))+ .toList();+ }+/*** Gives clusters persisted before IRSA support an OIDC issuer and signing key. Without this,* a cluster restored from {@code eks-clusters.json} would report no@@ -97,13 +163,16 @@ public class EksService implements TagHandler, ResourceProvider {* it was recreated.*/private void backfillOidcIdentities() {- for (Cluster cluster : allClusters()) {- // Runs at startup with no request context, so the owning account must come from the- // cluster record and be passed explicitly, so the account-scoped put()/get() would- // resolve to the default account and strand a cluster owned by any other one.- String accountId = cluster.getAccountId() != null- ? cluster.getAccountId()- : regionResolver.getAccountId();+ for (AccountAwareStorageBackend.AccountEntry<Cluster> entry : allClusterEntries()) {+ Cluster cluster = entry.value();+ // Runs at startup with no request context, and Cluster.accountId is @JsonIgnore so a+ // reloaded record carries none — the owning account comes from the storage key and is+ // passed explicitly, or the account-scoped put()/get() would resolve to the default+ // account and strand a cluster (and its signing key) owned by any other one.+ String accountId = entry.accountId();+ if (cluster.getAccountId() == null) {+ cluster.setAccountId(accountId);+ }if (cluster.getIdentity() != null && cluster.getIdentity().getOidc() != null&& cluster.getIdentity().getOidc().getIssuer() != null) {@@ -143,6 +212,16 @@ public class EksService implements TagHandler, ResourceProvider {if (name == null || name.isBlank()) {throw new AwsException("InvalidParameterException", "Cluster name is required", 400);}+ // The AWS constraint on EKS cluster names. Enforcing it also guarantees no name can+ // contain the dot EksClusterManager uses to account-qualify Docker names, so a+ // default-account cluster name can never spell out another account's qualified name+ // and collide with its container or data volume.+ if (name.length() > 100 || !name.matches(CLUSTER_NAME_REGEX)) {+ throw new AwsException("InvalidParameterException",+ "Value '" + name + "' at 'name' failed to satisfy constraint: Member must "+ + "satisfy regular expression pattern: ^" + CLUSTER_NAME_REGEX + "$",+ 400);+ }if (storage.get(name).isPresent()) {throw new AwsException("ResourceInUseException","Cluster already exists: " + name, 409);diff --git a/src/main/java/io/github/hectorvent/floci/services/eks/model/Cluster.java b/src/main/java/io/github/hectorvent/floci/services/eks/model/Cluster.javaindex bbd77add06..b5825ccad1 100644--- a/src/main/java/io/github/hectorvent/floci/services/eks/model/Cluster.java+++ b/src/main/java/io/github/hectorvent/floci/services/eks/model/Cluster.java@@ -65,6 +65,14 @@ public class Cluster {@JsonIgnoreprivate int hostPort;+ /**+ * Resolved Docker container/volume name for this cluster's k3s resources. In-memory only+ * (never part of the AWS response shape): assigned when the container is started, or+ * re-resolved deterministically from surviving Docker resources on restore.+ */+ @JsonIgnore+ private String dockerName;+public Cluster() {}public String getName() { return name; }@@ -117,4 +125,7 @@ public class Cluster {public int getHostPort() { return hostPort; }public void setHostPort(int hostPort) { this.hostPort = hostPort; }++ public String getDockerName() { return dockerName; }+ public void setDockerName(String dockerName) { this.dockerName = dockerName; }}diff --git a/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.javaindex f89579e8f4..15194b6785 100644--- a/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java@@ -12,8 +12,13 @@ import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;import io.github.hectorvent.floci.core.common.docker.PortAllocator;import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager;import io.github.hectorvent.floci.services.eks.model.Cluster;++import com.fasterxml.jackson.core.JsonProcessingException;+import com.fasterxml.jackson.databind.ObjectMapper;import com.github.dockerjava.api.DockerClient;import com.github.dockerjava.api.command.CopyArchiveToContainerCmd;+import com.github.dockerjava.api.command.InspectVolumeCmd;+import com.github.dockerjava.api.model.Container;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Nested;import org.junit.jupiter.api.Test;@@ -207,6 +212,293 @@ class EksClusterManagerTest {"io.floci.region", "us-east-1"));}+ /** Re-latching persisted clusters after a Floci/Docker restart (#2609), without a Docker daemon. */+ @Nested+ class RestoreCluster {++ private EmulatorConfig config;+ private EmulatorConfig.StorageConfig storage;+ private ContainerLifecycleManager lifecycleManager;+ private PortAllocator portAllocator;+ private EksClusterManager manager;++ @BeforeEach+ void setUp() {+ config = Mockito.mock(EmulatorConfig.class);+ EmulatorConfig.ServicesConfig services = Mockito.mock(EmulatorConfig.ServicesConfig.class);+ EmulatorConfig.EksServiceConfig eks = Mockito.mock(EmulatorConfig.EksServiceConfig.class);+ storage = Mockito.mock(EmulatorConfig.StorageConfig.class);+ when(config.services()).thenReturn(services);+ when(config.storage()).thenReturn(storage);+ when(services.eks()).thenReturn(eks);+ when(eks.defaultImage()).thenReturn("rancher/k3s:v1.30.0-k3s1");+ when(eks.apiServerBasePort()).thenReturn(6440);+ when(eks.apiServerMaxPort()).thenReturn(6499);+ when(eks.dockerNetwork()).thenReturn(Optional.empty());+ when(eks.endpointMode()).thenReturn("host");+ when(eks.disableCni()).thenReturn(false);+ when(eks.iamAuthWebhook()).thenReturn(false);+ when(eks.ecrRegistryMirror()).thenReturn(false);++ lifecycleManager = Mockito.mock(ContainerLifecycleManager.class);+ ContainerBuilder containerBuilder = Mockito.mock(ContainerBuilder.class);+ ContainerBuilder.Builder builder = Mockito.mock(ContainerBuilder.Builder.class, Mockito.RETURNS_SELF);+ when(containerBuilder.newContainer(anyString())).thenReturn(builder);+ when(builder.build()).thenReturn(Mockito.mock(ContainerSpec.class));+ portAllocator = Mockito.mock(PortAllocator.class);++ manager = new EksClusterManager(containerBuilder, lifecycleManager,+ Mockito.mock(ContainerDetector.class), portAllocator,+ Mockito.mock(DockerHostResolver.class), Mockito.mock(EcrRegistryManager.class),+ config, Mockito.mock(RegionResolver.class));+ }++ // Container's getters are final, so survivors are built from JSON instead of mocked.+ private Container survivingContainer(String id) {+ return containerFromJson("{\"Id\":\"" + id + "\"}");+ }++ private Container survivingContainerOwnedBy(String id, String accountId) {+ return containerFromJson("{\"Id\":\"" + id + "\","+ + "\"Labels\":{\"io.floci.account\":\"" + accountId + "\"}}");+ }++ private Container containerFromJson(String json) {+ try {+ return new ObjectMapper().readValue(json, Container.class);+ } catch (JsonProcessingException e) {+ throw new IllegalStateException(e);+ }+ }++ private Cluster cluster() {+ Cluster cluster = new Cluster();+ cluster.setName("demo");+ return cluster;+ }++ private void stubFreshStart(String containerId, int allocatedPort) {+ when(lifecycleManager.create(any())).thenReturn(containerId);+ when(lifecycleManager.startCreated(any(), any()))+ .thenReturn(new ContainerInfo(containerId, Map.of()));+ when(portAllocator.allocate(6440, 6499)).thenReturn(allocatedPort);+ }++ @Test+ void adoptsASurvivingContainerAndKeepsItsPublishedPort() {+ when(lifecycleManager.findByName("floci-eks-demo"))+ .thenReturn(Optional.of(survivingContainer("cid-1")));+ // adopt() starts a stopped container — the Docker-reboot case from #2609.+ when(lifecycleManager.adopt("cid-1", List.of(6443)))+ .thenReturn(new ContainerInfo("cid-1", Map.of(), Map.of(6443, 6512)));++ Cluster cluster = cluster();+ manager.restoreCluster(cluster);++ assertEquals("cid-1", cluster.getContainerId());+ assertEquals(6512, cluster.getHostPort());+ assertEquals("https://localhost:6512", cluster.getEndpoint());+ assertEquals("https://localhost:6512", cluster.getInternalEndpoint());+ // The port Docker already holds must not be handed out to another cluster.+ verify(portAllocator).markReserved(6512);+ verify(lifecycleManager, never()).create(any());+ }++ @Test+ void recreatesTheContainerWhenNoneSurvives() {+ when(lifecycleManager.findByName("floci-eks-demo")).thenReturn(Optional.empty());+ stubFreshStart("cid-new", 6440);++ Cluster cluster = cluster();+ manager.restoreCluster(cluster);++ assertEquals("cid-new", cluster.getContainerId());+ assertEquals(6440, cluster.getHostPort());+ verify(lifecycleManager).create(any());+ verify(lifecycleManager, never()).adopt(anyString(), any());+ }++ @Test+ void recreatesWhenTheSurvivingContainerPublishesNoPort() {+ when(lifecycleManager.findByName("floci-eks-demo"))+ .thenReturn(Optional.of(survivingContainer("cid-1")));+ when(lifecycleManager.adopt("cid-1", List.of(6443)))+ .thenReturn(new ContainerInfo("cid-1", Map.of()));+ stubFreshStart("cid-new", 6440);++ Cluster cluster = cluster();+ manager.restoreCluster(cluster);++ assertEquals("cid-new", cluster.getContainerId());+ verify(portAllocator, never()).markReserved(6440);+ }++ @Test+ void recreatesWhenAdoptionFails() {+ when(lifecycleManager.findByName("floci-eks-demo"))+ .thenReturn(Optional.of(survivingContainer("cid-1")));+ when(lifecycleManager.adopt(anyString(), any())).thenThrow(new RuntimeException("broken"));+ stubFreshStart("cid-new", 6440);++ Cluster cluster = cluster();+ manager.restoreCluster(cluster);++ assertEquals("cid-new", cluster.getContainerId());+ // startCluster removes the broken survivor before reusing its name.+ verify(lifecycleManager).removeIfExists("floci-eks-demo");+ }++ @Test+ void stopClusterRetainsTheDataVolumeInPersistentStorageMode() {+ when(storage.mode()).thenReturn("hybrid");+ when(storage.pruneVolumesOnDelete()).thenReturn(false);++ Cluster cluster = cluster();+ cluster.setContainerId("cid-1");+ manager.stopCluster(cluster);++ verify(lifecycleManager).stopAndRemove("cid-1", null);+ // The volume must survive so restoreCluster can bring the workloads back.+ verify(lifecycleManager, never()).removeVolume(anyString());+ }++ @Test+ void stopClusterRemovesTheDataVolumeInMemoryStorageMode() {+ when(storage.mode()).thenReturn("memory");++ Cluster cluster = cluster();+ cluster.setContainerId("cid-1");+ manager.stopCluster(cluster);++ verify(lifecycleManager).stopAndRemove("cid-1", null);+ verify(lifecycleManager).removeVolume("floci-eks-demo");+ }++ @Test+ void nonDefaultAccountClustersGetAccountQualifiedDockerNames() {+ // Cluster names are unique only per account: an unqualified name would cross-bind two+ // accounts' same-named clusters to one container and data volume.+ when(config.defaultAccountId()).thenReturn("000000000000");+ when(lifecycleManager.findByName("floci-eks-demo")).thenReturn(Optional.empty());+ when(lifecycleManager.findByName("floci-eks-999999999999.demo"))+ .thenReturn(Optional.of(survivingContainer("cid-9")));+ when(lifecycleManager.adopt("cid-9", List.of(6443)))+ .thenReturn(new ContainerInfo("cid-9", Map.of(), Map.of(6443, 6520)));++ Cluster cluster = cluster();+ cluster.setAccountId("999999999999");+ manager.restoreCluster(cluster);++ assertEquals("cid-9", cluster.getContainerId());+ assertEquals("floci-eks-999999999999.demo", cluster.getDockerName());+ }++ @Test+ void defaultAccountClustersKeepTheHistoricalUnqualifiedName() {+ when(config.defaultAccountId()).thenReturn("000000000000");++ Cluster cluster = cluster();+ cluster.setAccountId("000000000000");++ assertEquals("floci-eks-demo", manager.clusterResourceName(cluster));+ }++ @Test+ void legacyNamedContainerIsKeptWhenItsAccountLabelMatchesTheOwner() {+ // A non-default-account cluster created before account-qualified naming left its+ // container (and mounted data volume) under floci-eks-<name>. Restoration must keep+ // that name — recreating under the qualified name would orphan the workloads.+ when(config.defaultAccountId()).thenReturn("000000000000");+ when(lifecycleManager.findByName("floci-eks-demo"))+ .thenReturn(Optional.of(survivingContainerOwnedBy("cid-legacy", "999999999999")));+ when(lifecycleManager.adopt("cid-legacy", List.of(6443)))+ .thenReturn(new ContainerInfo("cid-legacy", Map.of(), Map.of(6443, 6512)));++ Cluster cluster = cluster();+ cluster.setAccountId("999999999999");+ manager.restoreCluster(cluster);++ assertEquals("cid-legacy", cluster.getContainerId());+ assertEquals("floci-eks-demo", cluster.getDockerName());+ verify(lifecycleManager, never()).create(any());+ }++ @Test+ void legacyNamedContainerOfAnotherAccountIsNeverClaimed() {+ // The legacy container belongs to whoever's label it carries. A different account's+ // restored cluster must start fresh under its qualified name, leaving the survivor+ // (and its data) untouched.+ when(config.defaultAccountId()).thenReturn("000000000000");+ when(lifecycleManager.findByName("floci-eks-demo"))+ .thenReturn(Optional.of(survivingContainerOwnedBy("cid-other", "111111111111")));+ when(lifecycleManager.findByName("floci-eks-999999999999.demo")).thenReturn(Optional.empty());+ stubFreshStart("cid-new", 6440);++ Cluster cluster = cluster();+ cluster.setAccountId("999999999999");+ manager.restoreCluster(cluster);++ assertEquals("floci-eks-999999999999.demo", cluster.getDockerName());+ assertEquals("cid-new", cluster.getContainerId());+ verify(lifecycleManager, never()).adopt(anyString(), any());+ verify(lifecycleManager, never()).removeIfExists("floci-eks-demo");+ }++ @Test+ void legacyNamedContainerWithoutAnOwnerLabelIsNeverClaimed() {+ // No label means no verifiable owner — adopting on a guess could hand another+ // account's workloads over. The cluster starts fresh under its qualified name and+ // the unclaimed survivor is only reported.+ when(config.defaultAccountId()).thenReturn("000000000000");+ when(lifecycleManager.findByName("floci-eks-demo"))+ .thenReturn(Optional.of(survivingContainer("cid-unlabeled")));+ when(lifecycleManager.findByName("floci-eks-999999999999.demo")).thenReturn(Optional.empty());+ stubFreshStart("cid-new", 6440);++ Cluster cluster = cluster();+ cluster.setAccountId("999999999999");+ manager.restoreCluster(cluster);++ assertEquals("floci-eks-999999999999.demo", cluster.getDockerName());+ verify(lifecycleManager, never()).adopt(anyString(), any());+ verify(lifecycleManager, never()).removeIfExists("floci-eks-demo");+ }++ @Test+ void legacyVolumeWithoutItsContainerIsNeverClaimed() {+ // A surviving volume carries no ownership label at all, so it cannot be verified for+ // any account — the cluster starts fresh under its qualified name and the volume is+ // reported for manual migration instead of being silently mounted or orphaned.+ when(config.defaultAccountId()).thenReturn("000000000000");+ DockerClient dockerClient = Mockito.mock(DockerClient.class);+ InspectVolumeCmd inspectVolumeCmd = Mockito.mock(InspectVolumeCmd.class);+ when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);+ when(dockerClient.inspectVolumeCmd("floci-eks-demo")).thenReturn(inspectVolumeCmd);+ stubFreshStart("cid-new", 6440);++ Cluster cluster = cluster();+ cluster.setAccountId("999999999999");+ manager.restoreCluster(cluster);++ assertEquals("floci-eks-999999999999.demo", cluster.getDockerName());+ assertEquals("cid-new", cluster.getContainerId());+ verify(inspectVolumeCmd).exec();+ }++ @Test+ void startClusterAssignsTheAccountQualifiedDockerName() {+ when(config.defaultAccountId()).thenReturn("000000000000");+ stubFreshStart("cid-new", 6440);++ Cluster cluster = cluster();+ cluster.setAccountId("999999999999");+ manager.startCluster(cluster);++ assertEquals("floci-eks-999999999999.demo", cluster.getDockerName());+ verify(lifecycleManager).removeIfExists("floci-eks-999999999999.demo");+ }+ }+/** Mirror-injection guard behavior, without a Docker daemon. */@Nestedclass InjectEcrRegistryMirror {diff --git a/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.javaindex e1ad966e2e..0f26d572f5 100644--- a/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java@@ -41,7 +41,11 @@ import java.util.Map;import java.util.Optional;import static org.junit.jupiter.api.Assertions.*;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.Mockito.doThrow;import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.never;+import static org.mockito.Mockito.verify;import static org.mockito.Mockito.when;class EksServiceTest {@@ -67,9 +71,14 @@ class EksServiceTest {}private EmulatorConfig testConfig() {+ return testConfig(true);+ }++ private EmulatorConfig testConfig(boolean mock) {EmulatorConfig.EksServiceConfig eksConfig = proxy(EmulatorConfig.EksServiceConfig.class,(proxy, method, args) -> switch (method.getName()) {- case "enabled", "mock" -> true;+ case "enabled" -> true;+ case "mock" -> mock;case "apiServerBasePort" -> 6500;default -> defaultValue(method);});@@ -256,7 +265,9 @@ class EksServiceTest {void initBackfillsUnderTheOwningAccountNotTheDefault() {// Startup has no request context, so the account-scoped put() resolves to the default// account. A cluster owned by another account must still be migrated in place, or the owner- // keeps an issuer-less record while a duplicate appears under the default account.+ // keeps an issuer-less record while a duplicate appears under the default account. The+ // record itself carries no accountId (it is @JsonIgnore, dropped on reload) — the owner+ // can only come from the storage key.String otherAccount = "999999999999";StorageBackend<String, Cluster> rawClusters = new InMemoryStorage<>();StorageBackend<String, ClusterOidcKey> rawKeys = new InMemoryStorage<>();@@ -265,7 +276,6 @@ class EksServiceTest {Cluster legacy = new Cluster();legacy.setName("legacy-cluster");- legacy.setAccountId(otherAccount);legacy.setStatus(ClusterStatus.ACTIVE);clusterStore.putForAccount(otherAccount, "legacy-cluster", legacy);@@ -277,6 +287,8 @@ class EksServiceTest {Cluster migrated = clusterStore.getForAccount(otherAccount, "legacy-cluster").orElseThrow();String issuer = migrated.getIdentity().getOidc().getIssuer();assertNotNull(issuer);+ // The owner rehydrated from the storage key sticks to the record for later puts.+ assertEquals(otherAccount, migrated.getAccountId());// No duplicate stranded under the default account.assertTrue(clusterStore.getForAccount("000000000000", "legacy-cluster").isEmpty());// The signing key is stored under the owner too, and is still resolvable by issuer.@@ -284,6 +296,138 @@ class EksServiceTest {assertTrue(oidcService.findVerificationKey(issuer).isPresent());}+ @Test+ void initRestoresPersistedClustersAfterARestart() {+ // A cluster restored from eks-clusters.json after a Floci/Docker restart (#2609) has no+ // container attached — init must re-latch it and hand it back to the readiness poller.+ StorageBackend<String, Cluster> rawClusters = new InMemoryStorage<>();+ var clusterStore = new AccountAwareStorageBackend<>(rawClusters, null, "000000000000");+ Cluster persisted = new Cluster();+ persisted.setName("persisted-cluster");+ persisted.setStatus(ClusterStatus.ACTIVE);+ clusterStore.putForAccount("000000000000", "persisted-cluster", persisted);++ Cluster failed = new Cluster();+ failed.setName("failed-cluster");+ failed.setStatus(ClusterStatus.FAILED);+ clusterStore.putForAccount("000000000000", "failed-cluster", failed);++ EksClusterManager clusterManager = mock(EksClusterManager.class);+ EksService restarted = new EksService(fixedStorageFactory(clusterStore), testConfig(false),+ new RegionResolver("us-east-1", "000000000000"), clusterManager, null,+ new EksOidcService(fixedStorageFactory(new InMemoryStorage<String, ClusterOidcKey>()),+ new ObjectMapper()));+ try {+ restarted.init();++ verify(clusterManager).restoreCluster(persisted);+ // CREATING hands the cluster to the readiness poller, which re-extracts the CA and+ // flips it back to ACTIVE once the API server answers.+ assertEquals(ClusterStatus.CREATING,+ restarted.describeCluster("persisted-cluster").getStatus());+ // A FAILED record has nothing to re-latch.+ verify(clusterManager, never()).restoreCluster(failed);+ assertEquals(ClusterStatus.FAILED,+ restarted.describeCluster("failed-cluster").getStatus());+ } finally {+ restarted.shutdown();+ }+ }++ @Test+ void initRestoresUnderTheOwningAccountNotTheDefault() {+ // Cluster.accountId is @JsonIgnore: a record reloaded from eks-clusters.json carries no+ // account — only its storage key does. Restoration must derive the owner from the key,+ // or the restored state lands under the default account while the owner keeps a stale+ // record with no restored runtime fields.+ String otherAccount = "999999999999";+ StorageBackend<String, Cluster> rawClusters = new InMemoryStorage<>();+ var clusterStore = new AccountAwareStorageBackend<>(rawClusters, null, "000000000000");++ Cluster persisted = new Cluster();+ persisted.setName("persisted-cluster");+ persisted.setStatus(ClusterStatus.ACTIVE);+ // An existing identity keeps backfillOidcIdentities from writing the record itself.+ persisted.setIdentity(new ClusterIdentity(new OidcIdentity(+ "https://oidc.eks.us-east-1.amazonaws.com/id/ABCDEF0123456789ABCDEF0123456789")));+ clusterStore.putForAccount(otherAccount, "persisted-cluster", persisted);++ EksClusterManager clusterManager = mock(EksClusterManager.class);+ EksService restarted = new EksService(fixedStorageFactory(clusterStore), testConfig(false),+ new RegionResolver("us-east-1", "000000000000"), clusterManager, null,+ new EksOidcService(fixedStorageFactory(new InMemoryStorage<String, ClusterOidcKey>()),+ new ObjectMapper()));+ try {+ restarted.init();++ verify(clusterManager).restoreCluster(persisted);+ Cluster restored = clusterStore.getForAccount(otherAccount, "persisted-cluster").orElseThrow();+ assertEquals(ClusterStatus.CREATING, restored.getStatus());+ // Rehydrated from the storage key, so the readiness poller's later put also lands+ // under the owner.+ assertEquals(otherAccount, restored.getAccountId());+ // No duplicate stranded under the default account.+ assertTrue(clusterStore.getForAccount("000000000000", "persisted-cluster").isEmpty());+ } finally {+ restarted.shutdown();+ }+ }++ @Test+ void initDoesNotRestoreClustersWithNamesOutsideTheAwsCharset() {+ // A record persisted before create-time name validation can carry a name with a dot —+ // which would map to another account's qualified Docker name (999999999999.demo in the+ // default account aliases account 999999999999's "demo"). Restoration must refuse it+ // rather than adopt or remove that account's container.+ StorageBackend<String, Cluster> rawClusters = new InMemoryStorage<>();+ var clusterStore = new AccountAwareStorageBackend<>(rawClusters, null, "000000000000");+ Cluster invalid = new Cluster();+ invalid.setName("999999999999.demo");+ invalid.setStatus(ClusterStatus.ACTIVE);+ clusterStore.putForAccount("000000000000", "999999999999.demo", invalid);++ EksClusterManager clusterManager = mock(EksClusterManager.class);+ EksService restarted = new EksService(fixedStorageFactory(clusterStore), testConfig(false),+ new RegionResolver("us-east-1", "000000000000"), clusterManager, null,+ new EksOidcService(fixedStorageFactory(new InMemoryStorage<String, ClusterOidcKey>()),+ new ObjectMapper()));+ try {+ restarted.init();++ verify(clusterManager, never()).restoreCluster(any(Cluster.class));+ assertEquals(ClusterStatus.FAILED,+ restarted.describeCluster("999999999999.demo").getStatus());+ } finally {+ restarted.shutdown();+ }+ }++ @Test+ void initMarksAClusterFailedWhenItsContainerCannotBeRestored() {+ StorageBackend<String, Cluster> rawClusters = new InMemoryStorage<>();+ var clusterStore = new AccountAwareStorageBackend<>(rawClusters, null, "000000000000");+ Cluster persisted = new Cluster();+ persisted.setName("persisted-cluster");+ persisted.setStatus(ClusterStatus.ACTIVE);+ clusterStore.putForAccount("000000000000", "persisted-cluster", persisted);++ EksClusterManager clusterManager = mock(EksClusterManager.class);+ doThrow(new RuntimeException("no docker")).when(clusterManager).restoreCluster(persisted);+ EksService restarted = new EksService(fixedStorageFactory(clusterStore), testConfig(false),+ new RegionResolver("us-east-1", "000000000000"), clusterManager, null,+ new EksOidcService(fixedStorageFactory(new InMemoryStorage<String, ClusterOidcKey>()),+ new ObjectMapper()));+ try {+ restarted.init();++ // Better an honest FAILED than an ACTIVE cluster no kubectl can reach.+ assertEquals(ClusterStatus.FAILED,+ restarted.describeCluster("persisted-cluster").getStatus());+ } finally {+ restarted.shutdown();+ }+ }+@SuppressWarnings("unchecked")private StorageFactory fixedStorageFactory(StorageBackend<String, ?> backend) {return new StorageFactory(null, null) {@@ -310,6 +454,32 @@ class EksServiceTest {assertThrows(AwsException.class, () -> eksService.createCluster(req));}+ @Test+ void createClusterRejectsNamesOutsideTheAwsCharset() {+ // Matches real EKS validation. The dot matters most: EksClusterManager account-qualifies+ // Docker names as <account>.<name>, so a name containing a dot could spell out another+ // account's qualified name and collide with its container and data volume.+ for (String invalid : List.of("999999999999.demo", "has space", "-starts-with-dash",+ "_starts-with-underscore", "a".repeat(101))) {+ CreateClusterRequest req = new CreateClusterRequest();+ req.setName(invalid);+ req.setRoleArn("arn:aws:iam::000000000000:role/eks-role");++ AwsException ex = assertThrows(AwsException.class, () -> eksService.createCluster(req),+ "should reject: " + invalid);+ assertEquals("InvalidParameterException", ex.getErrorCode());+ assertEquals(400, ex.getHttpStatus());+ }+ assertTrue(eksService.listClusters().isEmpty());+ }++ @Test+ void createClusterAcceptsTheFullAwsNameCharset() {+ createTestCluster("Valid-Name_123");++ assertTrue(eksService.listClusters().contains("Valid-Name_123"));+ }+@Testvoid createClusterWithNonExistentSubnetFails() {StorageFactory storageFactory = new StorageFactory(null, null) {