Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(eks): implement cluster restoration after restart to maintain state (#2780)

Matheus Andréeaecb3642598
diff --git a/docs/services/eks.md b/docs/services/eks.md
index e3c82a2697..e9c0ac9e5f 100644
--- a/docs/services/eks.md
+++ b/docs/services/eks.md
@@ -73,6 +73,29 @@ services:
!!! note "No port mapping needed for k3s ports"
k3s containers bind their API server port (6500–6599) directly on the host via Docker — no `ports:` entry is required in `docker-compose.yml`. See [Ports Reference](../configuration/ports.md#ports-65006599-eks-real-mode) for the full explanation.
+#### Clusters survive a restart
+
+With a persistent [storage mode](../configuration/storage.md) (the default), clusters recorded in
+`eks-clusters.json` are **re-latched to their k3s containers when Floci starts**:
+
+- A surviving container (for example after a Docker Desktop / daemon reboot) is adopted and
+ started in place, keeping its published API server port and data volume — deployments come
+ back as they were.
+- A missing container is recreated. Its named k3s data volume (`floci-eks-<name>`; for a
+ [non-default account](../configuration/multi-account.md), `floci-eks-<account>.<name>`) is
+ reused if it survived; volumes follow the global prune policy
+ (`FLOCI_STORAGE_PRUNE_VOLUMES_ON_DELETE`, default `false`), so they are retained when the
+ container is stopped or the cluster deleted, except in `memory` storage mode.
+- A non-default-account cluster created before account-qualified naming keeps its historical
+ `floci-eks-<name>` container and volume: restoration adopts the surviving container when its
+ `io.floci.account` label matches the owning account, so pre-upgrade workloads are not
+ orphaned.
+
+A restored cluster reports `CREATING` until its API server answers again, then returns to
+`ACTIVE` with a freshly extracted certificate authority. If the container cannot be brought
+back (for example Docker is unavailable), the cluster is marked `FAILED` instead of appearing
+`ACTIVE` while unreachable.
+
## Configuration
| Variable | Default | Description |
diff --git a/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java b/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java
index 285bc15728..15aeff762e 100644
--- a/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java
@@ -89,7 +89,10 @@ public class EksClusterManager {
*/
public void startCluster(Cluster cluster) {
String image = config.services().eks().defaultImage();
- String containerName = ContainerStorageHelper.resourceName(config, "eks", null, cluster.getName());
+ if (cluster.getDockerName() == null) {
+ cluster.setDockerName(accountQualifiedName(cluster));
+ }
+ String containerName = cluster.getDockerName();
LOG.infov("Starting k3s container for EKS cluster: {0} using image {1}",
cluster.getName(), image);
@@ -113,10 +116,15 @@ public class EksClusterManager {
// socket (kine.sock) on macOS APFS, which returns EINVAL on chmod — crashing
// k3s before it can start. Named volumes live in the Docker VM's Linux
// filesystem, so chmod works correctly and data persists across container restarts.
- String volumeName = ContainerStorageHelper.resourceName(config, "eks", null, cluster.getName());
+ String volumeName = cluster.getDockerName();
List<String> serverArgs = buildServerArgs(config.services().eks().disableCni());
+ // The account label comes from the cluster record when set (restore runs with no request
+ // context); regionResolver is the fallback for the create path.
+ String labelAccountId = cluster.getAccountId() != null
+ ? cluster.getAccountId()
+ : regionResolver.getAccountId();
ContainerBuilder.Builder specBuilder = containerBuilder.newContainer(image)
.withName(containerName)
.withEnv("K3S_KUBECONFIG_MODE", "644")
@@ -126,7 +134,7 @@ public class EksClusterManager {
.withPrivileged(true)
.withLogRotation()
.withLabels(ContainerStorageHelper.resourceIdentityLabels(
- "eks", cluster.getName(), regionResolver.getAccountId(), regionResolver.getDefaultRegion()));
+ "eks", cluster.getName(), labelAccountId, regionResolver.getDefaultRegion()));
// Wire a token-authentication webhook so `aws eks get-token` bearer tokens are validated by
// Floci and mapped to cluster-admin. The k3s API server POSTs a TokenReview to Floci's
@@ -169,15 +177,75 @@ public class EksClusterManager {
injectEcrRegistryMirror(containerId, cluster.getName());
ContainerInfo info = lifecycleManager.startCreated(containerId, spec);
- // Public endpoint: see floci.services.eks.endpoint-mode. `host` (default) is the host-reachable
- // published port (k3s cert carries `--tls-san=localhost`, so it verifies against the CA that
- // describe-cluster returns); `network` is the container DNS name (pre-#1118 behaviour).
+ applyEndpoints(cluster, containerName, hostPort, info);
+
+ LOG.infov("k3s container {0} started for cluster {1} on port {2} (internal: {3})",
+ containerId, cluster.getName(), String.valueOf(hostPort), cluster.getInternalEndpoint());
+ }
+
+ /**
+ * Re-latches a persisted cluster onto its k3s container after a Floci restart. A surviving
+ * container — running, or stopped by a Docker daemon reboot — is adopted (started if needed),
+ * keeping its published API server port and data volume, so the cluster's workloads come back
+ * as they were. When the container is gone, the cluster is recreated via {@link #startCluster};
+ * the named k3s data volume is reused if it survived. Callers should put the cluster back into
+ * CREATING so the readiness poller re-verifies the API server and re-extracts the certificate
+ * authority before marking it ACTIVE again.
+ */
+ public void restoreCluster(Cluster cluster) {
+ if (cluster.getDockerName() == null) {
+ cluster.setDockerName(resolveRestoredDockerName(cluster));
+ }
+ String containerName = cluster.getDockerName();
+ var existing = lifecycleManager.findByName(containerName);
+ if (existing.isEmpty()) {
+ LOG.infov("No surviving k3s container for EKS cluster {0}; recreating it "
+ + "(a surviving data volume is reused)", cluster.getName());
+ startCluster(cluster);
+ return;
+ }
+
+ ContainerInfo info;
+ try {
+ info = lifecycleManager.adopt(existing.get().getId(), List.of(K3S_API_SERVER_PORT));
+ } catch (Exception e) {
+ LOG.warnv("Could not adopt surviving k3s container {0} for EKS cluster {1} ({2}); recreating it",
+ containerName, cluster.getName(), e.getMessage());
+ startCluster(cluster);
+ return;
+ }
+
+ var publishedPort = info.publishedHostPort(K3S_API_SERVER_PORT);
+ if (publishedPort.isEmpty()) {
+ LOG.warnv("Surviving k3s container {0} publishes no API server port; recreating it", containerName);
+ startCluster(cluster);
+ return;
+ }
+
+ int hostPort = publishedPort.getAsInt();
+ // Keep the allocator away from a port Docker already holds for this cluster.
+ portAllocator.markReserved(hostPort);
+ cluster.setContainerId(info.containerId());
+ cluster.setHostPort(hostPort);
+ applyEndpoints(cluster, containerName, hostPort, info);
+
+ LOG.infov("Adopted surviving k3s container {0} for EKS cluster {1} on port {2} (internal: {3})",
+ info.containerId(), cluster.getName(), String.valueOf(hostPort), cluster.getInternalEndpoint());
+ }
+
+ /**
+ * Sets the cluster's public and internal endpoints for a started or adopted container.
+ * Public endpoint: see floci.services.eks.endpoint-mode. `host` (default) is the host-reachable
+ * published port (k3s cert carries `--tls-san=localhost`, so it verifies against the CA that
+ * describe-cluster returns); `network` is the container DNS name (pre-#1118 behaviour).
+ * The internal endpoint uses the resolved container IP so the readiness poller works from inside
+ * the Docker network (where localhost:<hostPort> would not reach the k3s container).
+ */
+ private void applyEndpoints(Cluster cluster, String containerName, int hostPort, ContainerInfo info) {
cluster.setEndpoint(resolvePublicEndpoint(
containerDetector.isRunningInContainer(), config.services().eks().endpointMode(),
containerName, hostPort));
- // Internal endpoint uses the resolved container IP so the readiness poller works from inside
- // the Docker network (where localhost:<hostPort> would not reach the k3s container).
if (containerDetector.isRunningInContainer()) {
ContainerLifecycleManager.EndpointInfo ep = info.getEndpoint(K3S_API_SERVER_PORT);
cluster.setInternalEndpoint(ep != null
@@ -186,9 +254,6 @@ public class EksClusterManager {
} else {
cluster.setInternalEndpoint("https://localhost:" + hostPort);
}
-
- LOG.infov("k3s container {0} started for cluster {1} on port {2} (internal: {3})",
- containerId, cluster.getName(), String.valueOf(hostPort), cluster.getInternalEndpoint());
}
/**
@@ -250,7 +315,10 @@ public class EksClusterManager {
}
/**
- * Stops and removes the k3s container for the given cluster.
+ * Stops and removes the k3s container for the given cluster. The k3s data volume follows the
+ * storage prune policy ({@link ContainerStorageHelper#removeNamedVolume}): it is only removed
+ * in {@code memory} storage mode or when {@code prune-volumes-on-delete} is set, so a persisted
+ * cluster's workloads survive a Floci restart and are re-latched by {@link #restoreCluster}.
*/
public void stopCluster(Cluster cluster) {
if (cluster.getContainerId() == null) {
@@ -261,10 +329,107 @@ public class EksClusterManager {
return;
}
lifecycleManager.stopAndRemove(cluster.getContainerId(), null);
- lifecycleManager.removeVolume(ContainerStorageHelper.resourceName(config, "eks", null, cluster.getName()));
+ ContainerStorageHelper.removeNamedVolume(config, lifecycleManager, clusterResourceName(cluster));
LOG.infov("Stopped k3s container for cluster {0}", cluster.getName());
}
+ /**
+ * Docker container/volume name for a cluster: the name already resolved for this record
+ * ({@link Cluster#getDockerName()}, set by startCluster/restoreCluster), or the
+ * account-qualified name for a record no container operation has touched yet.
+ */
+ String clusterResourceName(Cluster cluster) {
+ return cluster.getDockerName() != null ? cluster.getDockerName() : accountQualifiedName(cluster);
+ }
+
+ /**
+ * Account-qualified Docker name for a cluster. Cluster names are unique only within an
+ * account, so a non-default account's cluster is qualified with its account ID — otherwise
+ * two accounts' same-named clusters would resolve to the same container and data volume,
+ * letting one account reach (or, via startCluster's stale-container removal, destroy) the
+ * other's workloads. The default account keeps the historical unqualified name so existing
+ * containers, volumes, and {@code endpoint-mode=network} DNS names keep working.
+ *
+ * <p>The qualifier separator is a dot: EksService validates cluster names against the AWS
+ * charset ({@code [0-9A-Za-z][A-Za-z0-9\-_]*}), which admits no dot, so no default-account
+ * cluster name can spell out {@code <accountId>.<name>} and collide with another account's
+ * qualified name — a dash separator would (cluster "999999999999-demo" vs account
+ * 999999999999's "demo"). Dots are valid in Docker container and volume names.
+ *
+ * <p>The record's accountId is set before every call path reaches here (createCluster on
+ * create; the startup account rehydration on restore/stop); a null falls back to the
+ * default-account name.
+ */
+ private String accountQualifiedName(Cluster cluster) {
+ String accountId = cluster.getAccountId();
+ boolean defaultAccount = accountId == null || accountId.equals(config.defaultAccountId());
+ return ContainerStorageHelper.resourceName(config, "eks", null,
+ defaultAccount ? cluster.getName() : accountId + "." + cluster.getName());
+ }
+
+ /**
+ * Resolves which Docker name a restored record's resources actually live under. Clusters
+ * created before account-qualified naming used the account-independent legacy name
+ * {@code floci-eks-<name>} for every account — a non-default account's cluster must keep
+ * that name when its own container survived there, or the upgrade would recreate the
+ * cluster under the qualified name and orphan the historical workloads. The legacy
+ * container is claimed only when its {@code io.floci.account} label matches the owning
+ * account; another account's container — or one with no verifiable owner — is left
+ * untouched and the cluster starts fresh under the qualified name. A surviving legacy
+ * volume without its container carries no ownership label and is deliberately not claimed —
+ * it is reported via {@link #warnUnclaimedLegacyState} so the operator can migrate the data
+ * by hand. The result is deterministic across restarts for a given Docker state.
+ */
+ private String resolveRestoredDockerName(Cluster cluster) {
+ String qualified = accountQualifiedName(cluster);
+ String legacy = ContainerStorageHelper.resourceName(config, "eks", null, cluster.getName());
+ if (legacy.equals(qualified)) {
+ return legacy; // default account: the names never diverged
+ }
+ var legacySurvivor = lifecycleManager.findByName(legacy);
+ if (legacySurvivor.isPresent()) {
+ var labels = legacySurvivor.get().getLabels();
+ String owner = labels != null ? labels.get("io.floci.account") : null;
+ if (cluster.getAccountId() != null && cluster.getAccountId().equals(owner)) {
+ LOG.infov("EKS cluster {0} (account {1}) keeps its pre-upgrade Docker name {2}",
+ cluster.getName(), cluster.getAccountId(), legacy);
+ return legacy;
+ }
+ if (owner == null) {
+ warnUnclaimedLegacyState(cluster, legacy, "container");
+ }
+ // A container labeled with another account is simply not this cluster's — no warning.
+ } else if (volumeExists(legacy)) {
+ warnUnclaimedLegacyState(cluster, legacy, "data volume");
+ }
+ return qualified;
+ }
+
+ /**
+ * A legacy-named container or volume whose owning account cannot be verified is never claimed
+ * for a non-default account — handing it over on a guess would expose another account's data,
+ * the very cross-bind the qualified names exist to prevent. It is reported instead of being
+ * silently orphaned, so an operator who knows the data belongs to this cluster can migrate it
+ * into the qualified volume by hand.
+ */
+ private void warnUnclaimedLegacyState(Cluster cluster, String legacy, String kind) {
+ LOG.warnv("EKS cluster {0} (account {1}) starts under its account-qualified Docker name; "
+ + "a pre-upgrade {2} named {3} survives but carries no verifiable owning account, "
+ + "so it is NOT adopted. If its data belongs to this cluster, copy it into the "
+ + "cluster's qualified volume manually (docker volume inspect {3}).",
+ cluster.getName(), cluster.getAccountId(), kind, legacy);
+ }
+
+ /** Whether a Docker volume with this exact name exists. Any lookup failure counts as absent. */
+ private boolean volumeExists(String volumeName) {
+ try {
+ lifecycleManager.getDockerClient().inspectVolumeCmd(volumeName).exec();
+ return true;
+ } catch (Exception e) {
+ return false;
+ }
+ }
+
/**
* Builds the k3s {@code server} command-line args. When {@code disableCni} is true, flannel,
* k3s's default network policy controller, and kube-proxy are all disabled up front — see the
diff --git a/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java b/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java
index 35d7b109c1..cd05123900 100644
--- a/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java
@@ -52,6 +52,10 @@ public class EksService implements TagHandler, ResourceProvider {
private static final Logger LOG = Logger.getLogger(EksService.class);
+ /** The AWS charset for EKS cluster names. It admits no dot, which the Docker-name account
+ * qualifier relies on — see EksClusterManager#accountQualifiedName. */
+ static final String CLUSTER_NAME_REGEX = "[0-9A-Za-z][A-Za-z0-9\\-_]*";
+
private final StorageBackend<String, Cluster> storage;
private final StorageBackend<String, Nodegroup> nodeGroupStorage;
private final StorageBackend<String, FargateProfile> fargateProfileStorage;
@@ -86,10 +90,72 @@ public class EksService implements TagHandler, ResourceProvider {
public void init() {
backfillOidcIdentities();
if (!config.services().eks().mock()) {
+ restorePersistedClusters();
startReadinessPoller();
}
}
+ /**
+ * Re-latches persisted clusters onto their k3s containers after a restart (#2609). Without
+ * this, a cluster restored from {@code eks-clusters.json} reported ACTIVE but its container
+ * was never restarted after a Docker daemon reboot, so every kubectl/deploy against it failed.
+ * A surviving container is adopted (and started if stopped); a missing one is recreated
+ * against the cluster's retained data volume. Restored clusters go back to CREATING so the
+ * readiness poller re-verifies the API server and re-extracts the certificate authority
+ * before marking them ACTIVE again.
+ */
+ private void restorePersistedClusters() {
+ for (AccountAwareStorageBackend.AccountEntry<Cluster> entry : allClusterEntries()) {
+ Cluster cluster = entry.value();
+ if (cluster.getContainerId() != null
+ || (cluster.getStatus() != ClusterStatus.ACTIVE
+ && cluster.getStatus() != ClusterStatus.CREATING)) {
+ continue;
+ }
+ // Cluster.accountId is @JsonIgnore, so a reloaded record carries none — the owning
+ // account must come from the storage key, or a non-default account's cluster would be
+ // written back under the default account (stale owner record + duplicate). Rehydrate
+ // it on the record too, so the readiness poller's later put lands under the owner.
+ if (cluster.getAccountId() == null) {
+ cluster.setAccountId(entry.accountId());
+ }
+ // A persisted name that predates create-time validation may violate the AWS charset —
+ // in particular contain a dot, which could spell out another account's qualified
+ // Docker name and cross-bind its container. Such a record is never restored; the
+ // cluster must be deleted and recreated under a valid name.
+ if (cluster.getName() == null || !cluster.getName().matches(CLUSTER_NAME_REGEX)) {
+ LOG.errorv("Not restoring EKS cluster \"{0}\" (account {1}): its persisted name "
+ + "violates the AWS charset and could alias another account''s Docker "
+ + "resources. Delete it and recreate it under a valid name.",
+ cluster.getName(), entry.accountId());
+ cluster.setStatus(ClusterStatus.FAILED);
+ putClusterForAccount(entry.accountId(), cluster);
+ continue;
+ }
+ try {
+ LOG.infov("Restoring k3s container for persisted EKS cluster {0}", cluster.getName());
+ cluster.setStatus(ClusterStatus.CREATING);
+ clusterManager.restoreCluster(cluster);
+ } catch (Exception e) {
+ LOG.errorv("Failed to restore k3s container for EKS cluster {0}: {1}",
+ cluster.getName(), e.getMessage());
+ cluster.setStatus(ClusterStatus.FAILED);
+ }
+ putClusterForAccount(entry.accountId(), cluster);
+ }
+ }
+
+ private List<AccountAwareStorageBackend.AccountEntry<Cluster>> allClusterEntries() {
+ if (storage instanceof AccountAwareStorageBackend<Cluster> aware) {
+ return aware.scanAllAccountEntries(k -> true);
+ }
+ return storage.scan(k -> true).stream()
+ .map(cluster -> new AccountAwareStorageBackend.AccountEntry<>(
+ cluster.getAccountId() != null ? cluster.getAccountId() : regionResolver.getAccountId(),
+ cluster.getName(), cluster))
+ .toList();
+ }
+
/**
* Gives clusters persisted before IRSA support an OIDC issuer and signing key. Without this,
* a cluster restored from {@code eks-clusters.json} would report no
@@ -97,13 +163,16 @@ public class EksService implements TagHandler, ResourceProvider {
* it was recreated.
*/
private void backfillOidcIdentities() {
- for (Cluster cluster : allClusters()) {
- // Runs at startup with no request context, so the owning account must come from the
- // cluster record and be passed explicitly, so the account-scoped put()/get() would
- // resolve to the default account and strand a cluster owned by any other one.
- String accountId = cluster.getAccountId() != null
- ? cluster.getAccountId()
- : regionResolver.getAccountId();
+ for (AccountAwareStorageBackend.AccountEntry<Cluster> entry : allClusterEntries()) {
+ Cluster cluster = entry.value();
+ // Runs at startup with no request context, and Cluster.accountId is @JsonIgnore so a
+ // reloaded record carries none — the owning account comes from the storage key and is
+ // passed explicitly, or the account-scoped put()/get() would resolve to the default
+ // account and strand a cluster (and its signing key) owned by any other one.
+ String accountId = entry.accountId();
+ if (cluster.getAccountId() == null) {
+ cluster.setAccountId(accountId);
+ }
if (cluster.getIdentity() != null && cluster.getIdentity().getOidc() != null
&& cluster.getIdentity().getOidc().getIssuer() != null) {
@@ -143,6 +212,16 @@ public class EksService implements TagHandler, ResourceProvider {
if (name == null || name.isBlank()) {
throw new AwsException("InvalidParameterException", "Cluster name is required", 400);
}
+ // The AWS constraint on EKS cluster names. Enforcing it also guarantees no name can
+ // contain the dot EksClusterManager uses to account-qualify Docker names, so a
+ // default-account cluster name can never spell out another account's qualified name
+ // and collide with its container or data volume.
+ if (name.length() > 100 || !name.matches(CLUSTER_NAME_REGEX)) {
+ throw new AwsException("InvalidParameterException",
+ "Value '" + name + "' at 'name' failed to satisfy constraint: Member must "
+ + "satisfy regular expression pattern: ^" + CLUSTER_NAME_REGEX + "$",
+ 400);
+ }
if (storage.get(name).isPresent()) {
throw new AwsException("ResourceInUseException",
"Cluster already exists: " + name, 409);
diff --git a/src/main/java/io/github/hectorvent/floci/services/eks/model/Cluster.java b/src/main/java/io/github/hectorvent/floci/services/eks/model/Cluster.java
index bbd77add06..b5825ccad1 100644
--- a/src/main/java/io/github/hectorvent/floci/services/eks/model/Cluster.java
+++ b/src/main/java/io/github/hectorvent/floci/services/eks/model/Cluster.java
@@ -65,6 +65,14 @@ public class Cluster {
@JsonIgnore
private int hostPort;
+ /**
+ * Resolved Docker container/volume name for this cluster's k3s resources. In-memory only
+ * (never part of the AWS response shape): assigned when the container is started, or
+ * re-resolved deterministically from surviving Docker resources on restore.
+ */
+ @JsonIgnore
+ private String dockerName;
+
public Cluster() {}
public String getName() { return name; }
@@ -117,4 +125,7 @@ public class Cluster {
public int getHostPort() { return hostPort; }
public void setHostPort(int hostPort) { this.hostPort = hostPort; }
+
+ public String getDockerName() { return dockerName; }
+ public void setDockerName(String dockerName) { this.dockerName = dockerName; }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java
index f89579e8f4..15194b6785 100644
--- a/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/eks/EksClusterManagerTest.java
@@ -12,8 +12,13 @@ import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;
import io.github.hectorvent.floci.core.common.docker.PortAllocator;
import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager;
import io.github.hectorvent.floci.services.eks.model.Cluster;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.command.CopyArchiveToContainerCmd;
+import com.github.dockerjava.api.command.InspectVolumeCmd;
+import com.github.dockerjava.api.model.Container;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Nested;
import org.junit.jupiter.api.Test;
@@ -207,6 +212,293 @@ class EksClusterManagerTest {
"io.floci.region", "us-east-1"));
}
+ /** Re-latching persisted clusters after a Floci/Docker restart (#2609), without a Docker daemon. */
+ @Nested
+ class RestoreCluster {
+
+ private EmulatorConfig config;
+ private EmulatorConfig.StorageConfig storage;
+ private ContainerLifecycleManager lifecycleManager;
+ private PortAllocator portAllocator;
+ private EksClusterManager manager;
+
+ @BeforeEach
+ void setUp() {
+ config = Mockito.mock(EmulatorConfig.class);
+ EmulatorConfig.ServicesConfig services = Mockito.mock(EmulatorConfig.ServicesConfig.class);
+ EmulatorConfig.EksServiceConfig eks = Mockito.mock(EmulatorConfig.EksServiceConfig.class);
+ storage = Mockito.mock(EmulatorConfig.StorageConfig.class);
+ when(config.services()).thenReturn(services);
+ when(config.storage()).thenReturn(storage);
+ when(services.eks()).thenReturn(eks);
+ when(eks.defaultImage()).thenReturn("rancher/k3s:v1.30.0-k3s1");
+ when(eks.apiServerBasePort()).thenReturn(6440);
+ when(eks.apiServerMaxPort()).thenReturn(6499);
+ when(eks.dockerNetwork()).thenReturn(Optional.empty());
+ when(eks.endpointMode()).thenReturn("host");
+ when(eks.disableCni()).thenReturn(false);
+ when(eks.iamAuthWebhook()).thenReturn(false);
+ when(eks.ecrRegistryMirror()).thenReturn(false);
+
+ lifecycleManager = Mockito.mock(ContainerLifecycleManager.class);
+ ContainerBuilder containerBuilder = Mockito.mock(ContainerBuilder.class);
+ ContainerBuilder.Builder builder = Mockito.mock(ContainerBuilder.Builder.class, Mockito.RETURNS_SELF);
+ when(containerBuilder.newContainer(anyString())).thenReturn(builder);
+ when(builder.build()).thenReturn(Mockito.mock(ContainerSpec.class));
+ portAllocator = Mockito.mock(PortAllocator.class);
+
+ manager = new EksClusterManager(containerBuilder, lifecycleManager,
+ Mockito.mock(ContainerDetector.class), portAllocator,
+ Mockito.mock(DockerHostResolver.class), Mockito.mock(EcrRegistryManager.class),
+ config, Mockito.mock(RegionResolver.class));
+ }
+
+ // Container's getters are final, so survivors are built from JSON instead of mocked.
+ private Container survivingContainer(String id) {
+ return containerFromJson("{\"Id\":\"" + id + "\"}");
+ }
+
+ private Container survivingContainerOwnedBy(String id, String accountId) {
+ return containerFromJson("{\"Id\":\"" + id + "\","
+ + "\"Labels\":{\"io.floci.account\":\"" + accountId + "\"}}");
+ }
+
+ private Container containerFromJson(String json) {
+ try {
+ return new ObjectMapper().readValue(json, Container.class);
+ } catch (JsonProcessingException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+
+ private Cluster cluster() {
+ Cluster cluster = new Cluster();
+ cluster.setName("demo");
+ return cluster;
+ }
+
+ private void stubFreshStart(String containerId, int allocatedPort) {
+ when(lifecycleManager.create(any())).thenReturn(containerId);
+ when(lifecycleManager.startCreated(any(), any()))
+ .thenReturn(new ContainerInfo(containerId, Map.of()));
+ when(portAllocator.allocate(6440, 6499)).thenReturn(allocatedPort);
+ }
+
+ @Test
+ void adoptsASurvivingContainerAndKeepsItsPublishedPort() {
+ when(lifecycleManager.findByName("floci-eks-demo"))
+ .thenReturn(Optional.of(survivingContainer("cid-1")));
+ // adopt() starts a stopped container — the Docker-reboot case from #2609.
+ when(lifecycleManager.adopt("cid-1", List.of(6443)))
+ .thenReturn(new ContainerInfo("cid-1", Map.of(), Map.of(6443, 6512)));
+
+ Cluster cluster = cluster();
+ manager.restoreCluster(cluster);
+
+ assertEquals("cid-1", cluster.getContainerId());
+ assertEquals(6512, cluster.getHostPort());
+ assertEquals("https://localhost:6512", cluster.getEndpoint());
+ assertEquals("https://localhost:6512", cluster.getInternalEndpoint());
+ // The port Docker already holds must not be handed out to another cluster.
+ verify(portAllocator).markReserved(6512);
+ verify(lifecycleManager, never()).create(any());
+ }
+
+ @Test
+ void recreatesTheContainerWhenNoneSurvives() {
+ when(lifecycleManager.findByName("floci-eks-demo")).thenReturn(Optional.empty());
+ stubFreshStart("cid-new", 6440);
+
+ Cluster cluster = cluster();
+ manager.restoreCluster(cluster);
+
+ assertEquals("cid-new", cluster.getContainerId());
+ assertEquals(6440, cluster.getHostPort());
+ verify(lifecycleManager).create(any());
+ verify(lifecycleManager, never()).adopt(anyString(), any());
+ }
+
+ @Test
+ void recreatesWhenTheSurvivingContainerPublishesNoPort() {
+ when(lifecycleManager.findByName("floci-eks-demo"))
+ .thenReturn(Optional.of(survivingContainer("cid-1")));
+ when(lifecycleManager.adopt("cid-1", List.of(6443)))
+ .thenReturn(new ContainerInfo("cid-1", Map.of()));
+ stubFreshStart("cid-new", 6440);
+
+ Cluster cluster = cluster();
+ manager.restoreCluster(cluster);
+
+ assertEquals("cid-new", cluster.getContainerId());
+ verify(portAllocator, never()).markReserved(6440);
+ }
+
+ @Test
+ void recreatesWhenAdoptionFails() {
+ when(lifecycleManager.findByName("floci-eks-demo"))
+ .thenReturn(Optional.of(survivingContainer("cid-1")));
+ when(lifecycleManager.adopt(anyString(), any())).thenThrow(new RuntimeException("broken"));
+ stubFreshStart("cid-new", 6440);
+
+ Cluster cluster = cluster();
+ manager.restoreCluster(cluster);
+
+ assertEquals("cid-new", cluster.getContainerId());
+ // startCluster removes the broken survivor before reusing its name.
+ verify(lifecycleManager).removeIfExists("floci-eks-demo");
+ }
+
+ @Test
+ void stopClusterRetainsTheDataVolumeInPersistentStorageMode() {
+ when(storage.mode()).thenReturn("hybrid");
+ when(storage.pruneVolumesOnDelete()).thenReturn(false);
+
+ Cluster cluster = cluster();
+ cluster.setContainerId("cid-1");
+ manager.stopCluster(cluster);
+
+ verify(lifecycleManager).stopAndRemove("cid-1", null);
+ // The volume must survive so restoreCluster can bring the workloads back.
+ verify(lifecycleManager, never()).removeVolume(anyString());
+ }
+
+ @Test
+ void stopClusterRemovesTheDataVolumeInMemoryStorageMode() {
+ when(storage.mode()).thenReturn("memory");
+
+ Cluster cluster = cluster();
+ cluster.setContainerId("cid-1");
+ manager.stopCluster(cluster);
+
+ verify(lifecycleManager).stopAndRemove("cid-1", null);
+ verify(lifecycleManager).removeVolume("floci-eks-demo");
+ }
+
+ @Test
+ void nonDefaultAccountClustersGetAccountQualifiedDockerNames() {
+ // Cluster names are unique only per account: an unqualified name would cross-bind two
+ // accounts' same-named clusters to one container and data volume.
+ when(config.defaultAccountId()).thenReturn("000000000000");
+ when(lifecycleManager.findByName("floci-eks-demo")).thenReturn(Optional.empty());
+ when(lifecycleManager.findByName("floci-eks-999999999999.demo"))
+ .thenReturn(Optional.of(survivingContainer("cid-9")));
+ when(lifecycleManager.adopt("cid-9", List.of(6443)))
+ .thenReturn(new ContainerInfo("cid-9", Map.of(), Map.of(6443, 6520)));
+
+ Cluster cluster = cluster();
+ cluster.setAccountId("999999999999");
+ manager.restoreCluster(cluster);
+
+ assertEquals("cid-9", cluster.getContainerId());
+ assertEquals("floci-eks-999999999999.demo", cluster.getDockerName());
+ }
+
+ @Test
+ void defaultAccountClustersKeepTheHistoricalUnqualifiedName() {
+ when(config.defaultAccountId()).thenReturn("000000000000");
+
+ Cluster cluster = cluster();
+ cluster.setAccountId("000000000000");
+
+ assertEquals("floci-eks-demo", manager.clusterResourceName(cluster));
+ }
+
+ @Test
+ void legacyNamedContainerIsKeptWhenItsAccountLabelMatchesTheOwner() {
+ // A non-default-account cluster created before account-qualified naming left its
+ // container (and mounted data volume) under floci-eks-<name>. Restoration must keep
+ // that name — recreating under the qualified name would orphan the workloads.
+ when(config.defaultAccountId()).thenReturn("000000000000");
+ when(lifecycleManager.findByName("floci-eks-demo"))
+ .thenReturn(Optional.of(survivingContainerOwnedBy("cid-legacy", "999999999999")));
+ when(lifecycleManager.adopt("cid-legacy", List.of(6443)))
+ .thenReturn(new ContainerInfo("cid-legacy", Map.of(), Map.of(6443, 6512)));
+
+ Cluster cluster = cluster();
+ cluster.setAccountId("999999999999");
+ manager.restoreCluster(cluster);
+
+ assertEquals("cid-legacy", cluster.getContainerId());
+ assertEquals("floci-eks-demo", cluster.getDockerName());
+ verify(lifecycleManager, never()).create(any());
+ }
+
+ @Test
+ void legacyNamedContainerOfAnotherAccountIsNeverClaimed() {
+ // The legacy container belongs to whoever's label it carries. A different account's
+ // restored cluster must start fresh under its qualified name, leaving the survivor
+ // (and its data) untouched.
+ when(config.defaultAccountId()).thenReturn("000000000000");
+ when(lifecycleManager.findByName("floci-eks-demo"))
+ .thenReturn(Optional.of(survivingContainerOwnedBy("cid-other", "111111111111")));
+ when(lifecycleManager.findByName("floci-eks-999999999999.demo")).thenReturn(Optional.empty());
+ stubFreshStart("cid-new", 6440);
+
+ Cluster cluster = cluster();
+ cluster.setAccountId("999999999999");
+ manager.restoreCluster(cluster);
+
+ assertEquals("floci-eks-999999999999.demo", cluster.getDockerName());
+ assertEquals("cid-new", cluster.getContainerId());
+ verify(lifecycleManager, never()).adopt(anyString(), any());
+ verify(lifecycleManager, never()).removeIfExists("floci-eks-demo");
+ }
+
+ @Test
+ void legacyNamedContainerWithoutAnOwnerLabelIsNeverClaimed() {
+ // No label means no verifiable owner — adopting on a guess could hand another
+ // account's workloads over. The cluster starts fresh under its qualified name and
+ // the unclaimed survivor is only reported.
+ when(config.defaultAccountId()).thenReturn("000000000000");
+ when(lifecycleManager.findByName("floci-eks-demo"))
+ .thenReturn(Optional.of(survivingContainer("cid-unlabeled")));
+ when(lifecycleManager.findByName("floci-eks-999999999999.demo")).thenReturn(Optional.empty());
+ stubFreshStart("cid-new", 6440);
+
+ Cluster cluster = cluster();
+ cluster.setAccountId("999999999999");
+ manager.restoreCluster(cluster);
+
+ assertEquals("floci-eks-999999999999.demo", cluster.getDockerName());
+ verify(lifecycleManager, never()).adopt(anyString(), any());
+ verify(lifecycleManager, never()).removeIfExists("floci-eks-demo");
+ }
+
+ @Test
+ void legacyVolumeWithoutItsContainerIsNeverClaimed() {
+ // A surviving volume carries no ownership label at all, so it cannot be verified for
+ // any account — the cluster starts fresh under its qualified name and the volume is
+ // reported for manual migration instead of being silently mounted or orphaned.
+ when(config.defaultAccountId()).thenReturn("000000000000");
+ DockerClient dockerClient = Mockito.mock(DockerClient.class);
+ InspectVolumeCmd inspectVolumeCmd = Mockito.mock(InspectVolumeCmd.class);
+ when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);
+ when(dockerClient.inspectVolumeCmd("floci-eks-demo")).thenReturn(inspectVolumeCmd);
+ stubFreshStart("cid-new", 6440);
+
+ Cluster cluster = cluster();
+ cluster.setAccountId("999999999999");
+ manager.restoreCluster(cluster);
+
+ assertEquals("floci-eks-999999999999.demo", cluster.getDockerName());
+ assertEquals("cid-new", cluster.getContainerId());
+ verify(inspectVolumeCmd).exec();
+ }
+
+ @Test
+ void startClusterAssignsTheAccountQualifiedDockerName() {
+ when(config.defaultAccountId()).thenReturn("000000000000");
+ stubFreshStart("cid-new", 6440);
+
+ Cluster cluster = cluster();
+ cluster.setAccountId("999999999999");
+ manager.startCluster(cluster);
+
+ assertEquals("floci-eks-999999999999.demo", cluster.getDockerName());
+ verify(lifecycleManager).removeIfExists("floci-eks-999999999999.demo");
+ }
+ }
+
/** Mirror-injection guard behavior, without a Docker daemon. */
@Nested
class InjectEcrRegistryMirror {
diff --git a/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java
index e1ad966e2e..0f26d572f5 100644
--- a/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java
@@ -41,7 +41,11 @@ import java.util.Map;
import java.util.Optional;
import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
class EksServiceTest {
@@ -67,9 +71,14 @@ class EksServiceTest {
}
private EmulatorConfig testConfig() {
+ return testConfig(true);
+ }
+
+ private EmulatorConfig testConfig(boolean mock) {
EmulatorConfig.EksServiceConfig eksConfig = proxy(EmulatorConfig.EksServiceConfig.class,
(proxy, method, args) -> switch (method.getName()) {
- case "enabled", "mock" -> true;
+ case "enabled" -> true;
+ case "mock" -> mock;
case "apiServerBasePort" -> 6500;
default -> defaultValue(method);
});
@@ -256,7 +265,9 @@ class EksServiceTest {
void initBackfillsUnderTheOwningAccountNotTheDefault() {
// Startup has no request context, so the account-scoped put() resolves to the default
// account. A cluster owned by another account must still be migrated in place, or the owner
- // keeps an issuer-less record while a duplicate appears under the default account.
+ // keeps an issuer-less record while a duplicate appears under the default account. The
+ // record itself carries no accountId (it is @JsonIgnore, dropped on reload) — the owner
+ // can only come from the storage key.
String otherAccount = "999999999999";
StorageBackend<String, Cluster> rawClusters = new InMemoryStorage<>();
StorageBackend<String, ClusterOidcKey> rawKeys = new InMemoryStorage<>();
@@ -265,7 +276,6 @@ class EksServiceTest {
Cluster legacy = new Cluster();
legacy.setName("legacy-cluster");
- legacy.setAccountId(otherAccount);
legacy.setStatus(ClusterStatus.ACTIVE);
clusterStore.putForAccount(otherAccount, "legacy-cluster", legacy);
@@ -277,6 +287,8 @@ class EksServiceTest {
Cluster migrated = clusterStore.getForAccount(otherAccount, "legacy-cluster").orElseThrow();
String issuer = migrated.getIdentity().getOidc().getIssuer();
assertNotNull(issuer);
+ // The owner rehydrated from the storage key sticks to the record for later puts.
+ assertEquals(otherAccount, migrated.getAccountId());
// No duplicate stranded under the default account.
assertTrue(clusterStore.getForAccount("000000000000", "legacy-cluster").isEmpty());
// The signing key is stored under the owner too, and is still resolvable by issuer.
@@ -284,6 +296,138 @@ class EksServiceTest {
assertTrue(oidcService.findVerificationKey(issuer).isPresent());
}
+ @Test
+ void initRestoresPersistedClustersAfterARestart() {
+ // A cluster restored from eks-clusters.json after a Floci/Docker restart (#2609) has no
+ // container attached — init must re-latch it and hand it back to the readiness poller.
+ StorageBackend<String, Cluster> rawClusters = new InMemoryStorage<>();
+ var clusterStore = new AccountAwareStorageBackend<>(rawClusters, null, "000000000000");
+ Cluster persisted = new Cluster();
+ persisted.setName("persisted-cluster");
+ persisted.setStatus(ClusterStatus.ACTIVE);
+ clusterStore.putForAccount("000000000000", "persisted-cluster", persisted);
+
+ Cluster failed = new Cluster();
+ failed.setName("failed-cluster");
+ failed.setStatus(ClusterStatus.FAILED);
+ clusterStore.putForAccount("000000000000", "failed-cluster", failed);
+
+ EksClusterManager clusterManager = mock(EksClusterManager.class);
+ EksService restarted = new EksService(fixedStorageFactory(clusterStore), testConfig(false),
+ new RegionResolver("us-east-1", "000000000000"), clusterManager, null,
+ new EksOidcService(fixedStorageFactory(new InMemoryStorage<String, ClusterOidcKey>()),
+ new ObjectMapper()));
+ try {
+ restarted.init();
+
+ verify(clusterManager).restoreCluster(persisted);
+ // CREATING hands the cluster to the readiness poller, which re-extracts the CA and
+ // flips it back to ACTIVE once the API server answers.
+ assertEquals(ClusterStatus.CREATING,
+ restarted.describeCluster("persisted-cluster").getStatus());
+ // A FAILED record has nothing to re-latch.
+ verify(clusterManager, never()).restoreCluster(failed);
+ assertEquals(ClusterStatus.FAILED,
+ restarted.describeCluster("failed-cluster").getStatus());
+ } finally {
+ restarted.shutdown();
+ }
+ }
+
+ @Test
+ void initRestoresUnderTheOwningAccountNotTheDefault() {
+ // Cluster.accountId is @JsonIgnore: a record reloaded from eks-clusters.json carries no
+ // account — only its storage key does. Restoration must derive the owner from the key,
+ // or the restored state lands under the default account while the owner keeps a stale
+ // record with no restored runtime fields.
+ String otherAccount = "999999999999";
+ StorageBackend<String, Cluster> rawClusters = new InMemoryStorage<>();
+ var clusterStore = new AccountAwareStorageBackend<>(rawClusters, null, "000000000000");
+
+ Cluster persisted = new Cluster();
+ persisted.setName("persisted-cluster");
+ persisted.setStatus(ClusterStatus.ACTIVE);
+ // An existing identity keeps backfillOidcIdentities from writing the record itself.
+ persisted.setIdentity(new ClusterIdentity(new OidcIdentity(
+ "https://oidc.eks.us-east-1.amazonaws.com/id/ABCDEF0123456789ABCDEF0123456789")));
+ clusterStore.putForAccount(otherAccount, "persisted-cluster", persisted);
+
+ EksClusterManager clusterManager = mock(EksClusterManager.class);
+ EksService restarted = new EksService(fixedStorageFactory(clusterStore), testConfig(false),
+ new RegionResolver("us-east-1", "000000000000"), clusterManager, null,
+ new EksOidcService(fixedStorageFactory(new InMemoryStorage<String, ClusterOidcKey>()),
+ new ObjectMapper()));
+ try {
+ restarted.init();
+
+ verify(clusterManager).restoreCluster(persisted);
+ Cluster restored = clusterStore.getForAccount(otherAccount, "persisted-cluster").orElseThrow();
+ assertEquals(ClusterStatus.CREATING, restored.getStatus());
+ // Rehydrated from the storage key, so the readiness poller's later put also lands
+ // under the owner.
+ assertEquals(otherAccount, restored.getAccountId());
+ // No duplicate stranded under the default account.
+ assertTrue(clusterStore.getForAccount("000000000000", "persisted-cluster").isEmpty());
+ } finally {
+ restarted.shutdown();
+ }
+ }
+
+ @Test
+ void initDoesNotRestoreClustersWithNamesOutsideTheAwsCharset() {
+ // A record persisted before create-time name validation can carry a name with a dot —
+ // which would map to another account's qualified Docker name (999999999999.demo in the
+ // default account aliases account 999999999999's "demo"). Restoration must refuse it
+ // rather than adopt or remove that account's container.
+ StorageBackend<String, Cluster> rawClusters = new InMemoryStorage<>();
+ var clusterStore = new AccountAwareStorageBackend<>(rawClusters, null, "000000000000");
+ Cluster invalid = new Cluster();
+ invalid.setName("999999999999.demo");
+ invalid.setStatus(ClusterStatus.ACTIVE);
+ clusterStore.putForAccount("000000000000", "999999999999.demo", invalid);
+
+ EksClusterManager clusterManager = mock(EksClusterManager.class);
+ EksService restarted = new EksService(fixedStorageFactory(clusterStore), testConfig(false),
+ new RegionResolver("us-east-1", "000000000000"), clusterManager, null,
+ new EksOidcService(fixedStorageFactory(new InMemoryStorage<String, ClusterOidcKey>()),
+ new ObjectMapper()));
+ try {
+ restarted.init();
+
+ verify(clusterManager, never()).restoreCluster(any(Cluster.class));
+ assertEquals(ClusterStatus.FAILED,
+ restarted.describeCluster("999999999999.demo").getStatus());
+ } finally {
+ restarted.shutdown();
+ }
+ }
+
+ @Test
+ void initMarksAClusterFailedWhenItsContainerCannotBeRestored() {
+ StorageBackend<String, Cluster> rawClusters = new InMemoryStorage<>();
+ var clusterStore = new AccountAwareStorageBackend<>(rawClusters, null, "000000000000");
+ Cluster persisted = new Cluster();
+ persisted.setName("persisted-cluster");
+ persisted.setStatus(ClusterStatus.ACTIVE);
+ clusterStore.putForAccount("000000000000", "persisted-cluster", persisted);
+
+ EksClusterManager clusterManager = mock(EksClusterManager.class);
+ doThrow(new RuntimeException("no docker")).when(clusterManager).restoreCluster(persisted);
+ EksService restarted = new EksService(fixedStorageFactory(clusterStore), testConfig(false),
+ new RegionResolver("us-east-1", "000000000000"), clusterManager, null,
+ new EksOidcService(fixedStorageFactory(new InMemoryStorage<String, ClusterOidcKey>()),
+ new ObjectMapper()));
+ try {
+ restarted.init();
+
+ // Better an honest FAILED than an ACTIVE cluster no kubectl can reach.
+ assertEquals(ClusterStatus.FAILED,
+ restarted.describeCluster("persisted-cluster").getStatus());
+ } finally {
+ restarted.shutdown();
+ }
+ }
+
@SuppressWarnings("unchecked")
private StorageFactory fixedStorageFactory(StorageBackend<String, ?> backend) {
return new StorageFactory(null, null) {
@@ -310,6 +454,32 @@ class EksServiceTest {
assertThrows(AwsException.class, () -> eksService.createCluster(req));
}
+ @Test
+ void createClusterRejectsNamesOutsideTheAwsCharset() {
+ // Matches real EKS validation. The dot matters most: EksClusterManager account-qualifies
+ // Docker names as <account>.<name>, so a name containing a dot could spell out another
+ // account's qualified name and collide with its container and data volume.
+ for (String invalid : List.of("999999999999.demo", "has space", "-starts-with-dash",
+ "_starts-with-underscore", "a".repeat(101))) {
+ CreateClusterRequest req = new CreateClusterRequest();
+ req.setName(invalid);
+ req.setRoleArn("arn:aws:iam::000000000000:role/eks-role");
+
+ AwsException ex = assertThrows(AwsException.class, () -> eksService.createCluster(req),
+ "should reject: " + invalid);
+ assertEquals("InvalidParameterException", ex.getErrorCode());
+ assertEquals(400, ex.getHttpStatus());
+ }
+ assertTrue(eksService.listClusters().isEmpty());
+ }
+
+ @Test
+ void createClusterAcceptsTheFullAwsNameCharset() {
+ createTestCluster("Valid-Name_123");
+
+ assertTrue(eksService.listClusters().contains("Valid-Name_123"));
+ }
+
@Test
void createClusterWithNonExistentSubnetFails() {
StorageFactory storageFactory = new StorageFactory(null, null) {