Light, fluffy, and always free - The AWS Local Emulator alternative
fix(ec2,ecr,eks,elasticache,msk,opensearch): degrade gracefully with no Docker daemon (#2975)
lex001aa981fef27a
diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.javaindex 9d0c05b634..7e891d964c 100644--- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.java@@ -106,6 +106,8 @@ public class Ec2ContainerManager {private final RegionResolver regionResolver;private final ContainerNetworkReachability containerNetworkReachability;+ private volatile boolean dockerUnavailableLogged;+private final ExecutorService executor = Executors.newCachedThreadPool(r -> {Thread t = new Thread(r, "ec2-container-launcher");t.setDaemon(true);@@ -147,7 +149,8 @@ public class Ec2ContainerManager {/*** Launches a Docker container for the given EC2 instance.* The instance starts in pending state; an async thread transitions it to running- * and handles SSH key injection and UserData execution.+ * and handles SSH key injection and UserData execution. With no Docker daemon+ * reachable the instance goes straight to running as metadata only.** @param instance the EC2 instance model (mutated in-place as state transitions occur)* @param dockerImage Docker image URI resolved from the instance's AMI ID@@ -169,6 +172,16 @@ public class Ec2ContainerManager {public void launch(Instance instance, ResolvedAmiImage image, String publicKey, String region, Set<Integer> appPorts) {instance.setState(InstanceState.pending());+ // An instance record is metadata: id, addresses, tags and lifecycle state are all+ // served without a container runtime. Only the guest itself (SSH, UserData, SSM+ // commands) needs Docker, so when no daemon is reachable the instance still runs+ // instead of dying: Floci in Docker without a mounted socket, or a stopped daemon+ // on the host, would otherwise terminate every instance the moment it launched.+ if (!isDockerAvailable()) {+ markContainerlessRunning(instance);+ return;+ }+executor.submit(() -> {try {String instanceId = instance.getInstanceId();@@ -274,7 +287,14 @@ public class Ec2ContainerManager {failLaunch(instance);} catch (Exception e) {LOG.warnv("Failed to launch EC2 instance {0}: {1}", instance.getInstanceId(), e.getMessage());- failLaunch(instance);+ // The daemon can disappear between the probe above and any of the calls in+ // this block. Losing Docker is not the instance's fault, so degrade to a+ // metadata-only instance; a genuine container failure still fails the launch.+ if (isDockerAvailable()) {+ failLaunch(instance);+ } else {+ markContainerlessRunning(instance);+ }}});}@@ -486,6 +506,40 @@ public class Ec2ContainerManager {private record StartedContainer(String containerId, int sshHostPort) {}+ /**+ * Reports whether a Docker daemon is reachable, logging the transition in each+ * direction once rather than on every launch.+ */+ public boolean isDockerAvailable() {+ try {+ dockerClient.pingCmd().exec();+ if (dockerUnavailableLogged) {+ dockerUnavailableLogged = false;+ LOG.info("Docker daemon is reachable again; new EC2 instances get a backing container.");+ }+ return true;+ } catch (Exception e) {+ if (!dockerUnavailableLogged) {+ dockerUnavailableLogged = true;+ LOG.warnv("No Docker daemon is reachable from Floci ({0}). EC2 instances are emulated as "+ + "metadata only: they reach running and honour stop, start and terminate, but have "+ + "no backing container, so SSH, UserData and SSM command execution stay unavailable "+ + "until a daemon is reachable.", e.getMessage());+ }+ return false;+ }+ }++ /**+ * Brings an instance to running with no container behind it. Stop, start, terminate and+ * reboot already handle a null container id, so the rest of the lifecycle keeps working.+ */+ private void markContainerlessRunning(Instance instance) {+ LOG.infov("EC2 instance {0} is running without a backing container (no Docker daemon reachable)",+ instance.getInstanceId());+ instance.setState(InstanceState.running());+ }+/*** Synchronous stop for emulator shutdown: tears down the port-forward sidecars and* stops the container with a short timeout so N instances cannot exhaust the SIGTERMdiff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.javaindex 48eea89167..719fb4abfd 100644--- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2Service.java@@ -3894,7 +3894,10 @@ public class Ec2Service implements ContainerTeardown, ResourceProvider {if (instance == null) {return false;}- if (config.services().ec2().mock()) {+ // Mock mode and instances launched with no Docker daemon reachable have nothing to+ // inspect, so the recorded lifecycle state is the answer. Reporting those as not+ // running would make AutoScaling replace healthy instances in a loop.+ if (config.services().ec2().mock() || instance.getDockerContainerId() == null) {String state = instance.getState() != null ? instance.getState().getName() : null;return state == null|| (!"shutting-down".equals(state) && !"terminated".equals(state) && !"stopping".equals(state));diff --git a/src/main/java/io/github/hectorvent/floci/services/ecr/EcrService.java b/src/main/java/io/github/hectorvent/floci/services/ecr/EcrService.javaindex 99586ceb7c..dafa70c114 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecr/EcrService.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecr/EcrService.java@@ -121,7 +121,12 @@ public class EcrService implements ResourceProvider {Map<String, String> tags,String region) {validateRepoName(repositoryName);- registryManager.ensureStarted();+ // Repository records are pure metadata: the ARN and the URI are derived from the+ // configured account, region and registry port, none of which need Docker. Start+ // the backing registry opportunistically so the URI reflects an already-adopted+ // container's published port, but never fail CreateRepository when no Docker+ // daemon is reachable, the registry is retried on the next image operation.+ registryManager.tryEnsureStarted();String account = effectiveAccount(registryId);String key = key(region, account, repositoryName);if (repoStore.get(key).isPresent()) {@@ -220,7 +225,7 @@ public class EcrService implements ResourceProvider {// ============================================================public AuthorizationData getAuthorizationToken() {- registryManager.ensureStarted();+ requireRegistry();String token = Base64.getEncoder().encodeToString("AWS:floci".getBytes(StandardCharsets.UTF_8));Instant expires = Instant.now().plusSeconds(12 * 60 * 60);@@ -234,7 +239,7 @@ public class EcrService implements ResourceProvider {public List<ImageIdentifier> listImages(String repositoryName, String registryId, String region) {Repository repo = requireRepo(repositoryName, registryId, region);- registryManager.ensureStarted();+ requireRegistry();String internal = registryManager.internalRepoName(repo.getRegistryId(), region, repositoryName);try {RegistryHttpClient http = registryManager.httpClient();@@ -256,7 +261,7 @@ public class EcrService implements ResourceProvider {String registryId,String region) {Repository repo = requireRepo(repositoryName, registryId, region);- registryManager.ensureStarted();+ requireRegistry();String internal = registryManager.internalRepoName(repo.getRegistryId(), region, repositoryName);RegistryHttpClient http = registryManager.httpClient();@@ -331,7 +336,7 @@ public class EcrService implements ResourceProvider {String registryId,String region) {Repository repo = requireRepo(repositoryName, registryId, region);- registryManager.ensureStarted();+ requireRegistry();String internal = registryManager.internalRepoName(repo.getRegistryId(), region, repositoryName);RegistryHttpClient http = registryManager.httpClient();@@ -371,7 +376,7 @@ public class EcrService implements ResourceProvider {String registryId,String region) {Repository repo = requireRepo(repositoryName, registryId, region);- registryManager.ensureStarted();+ requireRegistry();String internal = registryManager.internalRepoName(repo.getRegistryId(), region, repositoryName);RegistryHttpClient http = registryManager.httpClient();@@ -533,6 +538,20 @@ public class EcrService implements ResourceProvider {}+ /**+ * Gate for the ECR data plane, which cannot be emulated without the backing+ * registry container. Surfaces ECR's modelled {@code ServerException} instead of+ * letting the Docker client's {@code SocketException} escape as an InternalFailure.+ */+ private void requireRegistry() {+ if (!registryManager.tryEnsureStarted()) {+ throw new AwsException("ServerException",+ "The ECR backing registry is unavailable because no Docker daemon is reachable "+ + "from Floci. Repository metadata operations are supported; image push, "+ + "pull and image queries require Docker.", 500);+ }+ }+private List<String> listTagsBestEffort(String account, String region, String repoName) {try {return registryManager.httpClient()diff --git a/src/main/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryManager.java b/src/main/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryManager.javaindex 3f80d06a71..769e9004e8 100644--- a/src/main/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryManager.java@@ -66,6 +66,7 @@ public class EcrRegistryManager {private volatile boolean started;private volatile boolean reconciled;+ private volatile boolean unavailableLogged;private volatile int hostPort;private volatile String containerId;private volatile Closeable logStream;@@ -189,6 +190,36 @@ public class EcrRegistryManager {this.reconcileHook = hook;}+ /**+ * Attempts {@link #ensureStarted()} and reports whether the backing registry is+ * usable, instead of propagating the failure. Callers that only need repository+ * metadata (ARN, URI, tags) use this so ECR's control plane keeps working when+ * Floci itself runs inside Docker without access to a Docker daemon; the registry+ * is retried on the next call and starts as soon as a daemon becomes reachable.+ *+ * @return true when the registry container is running+ */+ public boolean tryEnsureStarted() {+ if (started) {+ return true;+ }+ try {+ ensureStarted();+ } catch (RuntimeException e) {+ if (!unavailableLogged) {+ unavailableLogged = true;+ LOG.warnv("ECR backing registry is unavailable ({0}). Repository metadata operations "+ + "continue to work; image push, pull and image queries stay disabled until a "+ + "Docker daemon is reachable.", e.getMessage());+ }+ return false;+ }+ if (started) {+ unavailableLogged = false;+ }+ return started;+ }+/*** Lazily starts (or reuses) the {@code registry:2} container. Idempotent and* thread-safe. Throws if Docker is unreachable.diff --git a/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java b/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.javaindex ce01f4b3a7..1e2ba21cce 100644--- a/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/eks/EksClusterManager.java@@ -82,6 +82,45 @@ public class EksClusterManager {this.regionResolver = regionResolver;}+ /**+ * Attempts {@link #startCluster} and reports the k3s backend as unavailable instead of+ * propagating the failure, when the cause is that no Docker daemon is reachable from Floci:+ * Floci running inside Docker without a mounted socket, or a stopped daemon on the host. A+ * failure raised while the daemon <em>is</em> reachable is a genuine provisioning error and+ * still propagates, so a cluster only reaches FAILED for a reason AWS would also fail on.+ *+ * @return true when the k3s container was created and started+ */+ public boolean tryStartCluster(Cluster cluster) {+ try {+ startCluster(cluster);+ return true;+ } catch (RuntimeException e) {+ if (isDockerReachable()) {+ throw e;+ }+ LOG.warnv("No Docker daemon is reachable from Floci ({0}). EKS cluster {1} is created as "+ + "metadata only: describe, list, tag, nodegroups, Fargate profiles and delete "+ + "work, but the cluster has no Kubernetes API server and kubectl cannot connect "+ + "to the endpoint it reports.", e.getMessage(), cluster.getName());+ return false;+ }+ }++ /**+ * Probes the configured Docker endpoint, which is how a missing daemon is told apart from a+ * k3s container that failed for its own reasons.+ */+ public boolean isDockerReachable() {+ try {+ lifecycleManager.getDockerClient().pingCmd().exec();+ return true;+ } catch (Exception e) {+ LOG.debugv("Docker daemon is not reachable: {0}", e.getMessage());+ return false;+ }+ }+/*** Starts a k3s container for the given cluster. Updates the cluster with* the container ID and host port. The cluster status remains CREATING untildiff --git a/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java b/src/main/java/io/github/hectorvent/floci/services/eks/EksService.javaindex cd05123900..7adf462712 100644--- a/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java+++ b/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java@@ -137,9 +137,17 @@ public class EksService implements TagHandler, ResourceProvider {cluster.setStatus(ClusterStatus.CREATING);clusterManager.restoreCluster(cluster);} catch (Exception e) {- LOG.errorv("Failed to restore k3s container for EKS cluster {0}: {1}",- cluster.getName(), e.getMessage());- cluster.setStatus(ClusterStatus.FAILED);+ if (!clusterManager.isDockerReachable()) {+ // Same degradation as create: a restored cluster is metadata that stands on+ // its own, and FAILED is reserved for errors AWS would also report.+ LOG.warnv("No Docker daemon is reachable from Floci; restored EKS cluster {0} "+ + "comes back as metadata only.", cluster.getName());+ markMetadataOnlyActive(cluster);+ } else {+ LOG.errorv("Failed to restore k3s container for EKS cluster {0}: {1}",+ cluster.getName(), e.getMessage());+ cluster.setStatus(ClusterStatus.FAILED);+ }}putClusterForAccount(entry.accountId(), cluster);}@@ -251,11 +259,15 @@ public class EksService implements TagHandler, ResourceProvider {oidcService.ensureKey(name, issuer);if (config.services().eks().mock()) {- cluster.setStatus(ClusterStatus.ACTIVE);- cluster.setEndpoint("https://localhost:" + config.services().eks().apiServerBasePort());+ markMetadataOnlyActive(cluster);} else {try {- clusterManager.startCluster(cluster);+ if (!clusterManager.tryStartCluster(cluster)) {+ // No Docker daemon: the k3s control plane cannot run, but the cluster record is+ // metadata that stands on its own. FAILED is reserved for provisioning errors+ // AWS would also report, and would strand every IaC apply that polls for ACTIVE.+ markMetadataOnlyActive(cluster);+ }} catch (Exception e) {LOG.errorv("Failed to start k3s container for cluster {0}: {1}", name, e.getMessage());cluster.setStatus(ClusterStatus.FAILED);@@ -266,6 +278,17 @@ public class EksService implements TagHandler, ResourceProvider {return cluster;}+ /**+ * Marks a cluster ACTIVE with no Kubernetes API server behind it, the shape used by mock mode+ * and by a Floci that cannot reach a Docker daemon. Every EKS API Floci implements is control+ * plane (clusters, nodegroups, Fargate profiles, tags) and keeps working; the empty+ * certificateAuthority is what tells a caller no real cluster is listening on the endpoint.+ */+ private void markMetadataOnlyActive(Cluster cluster) {+ cluster.setStatus(ClusterStatus.ACTIVE);+ cluster.setEndpoint("https://localhost:" + config.services().eks().apiServerBasePort());+ }+public Cluster describeCluster(String name) {return storage.get(name).orElseThrow(() -> new AwsException("ResourceNotFoundException",diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedService.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedService.javaindex 631696ca10..b7bf3d6baf 100644--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedService.java+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedService.java@@ -25,6 +25,8 @@ public class ElastiCacheMemcachedService {private static final Logger LOG = Logger.getLogger(ElastiCacheMemcachedService.class);private static final String ENGINE = "memcached";private static final String ENGINE_VERSION = "1.6.22";+ /** Memcached's well-known port, used as the endpoint port when no backing container exists. */+ private static final int BACKEND_PORT = 11211;private final StorageBackend<String, CacheCluster> clusters;private final ElastiCacheMemcachedContainerManager containerManager;@@ -49,20 +51,30 @@ public class ElastiCacheMemcachedService {String image = config.services().elasticache().defaultMemcachedImage();LOG.infov("Creating Memcached cluster {0} with image {1}", clusterId, image);- ElastiCacheContainerHandle handle = containerManager.start(clusterId, image);+ // A cache cluster record is metadata: its id and endpoint are derived from configuration,+ // so the cluster is created and reaches 'available' even when no Docker daemon is+ // reachable. Only connecting to the cache needs the container.+ ElastiCacheContainerHandle handle = containerManager.tryStart(clusterId, image);String endpointHost = resolveEndpointHost(handle);- Endpoint endpoint = new Endpoint(endpointHost, handle.getPort());+ int endpointPort = handle != null ? handle.getPort() : BACKEND_PORT;+ Endpoint endpoint = new Endpoint(endpointHost, endpointPort);CacheCluster cluster = new CacheCluster(clusterId, CacheClusterStatus.AVAILABLE, ENGINE, ENGINE_VERSION,endpoint, Instant.now());- cluster.setContainerId(handle.getContainerId());- cluster.setContainerHost(handle.getHost());- cluster.setContainerPort(handle.getPort());+ if (handle != null) {+ cluster.setContainerId(handle.getContainerId());+ cluster.setContainerHost(handle.getHost());+ cluster.setContainerPort(handle.getPort());+ } else {+ LOG.warnv("Memcached cluster {0} created without a backing container: no Docker daemon "+ + "is reachable. Metadata operations work; connections to the cache do not "+ + "until a daemon appears.", clusterId);+ }clusters.put(clusterId, cluster);- LOG.infov("Memcached cluster {0} created, endpoint={1}:{2}", clusterId, endpointHost, handle.getPort());+ LOG.infov("Memcached cluster {0} created, endpoint={1}:{2}", clusterId, endpointHost, endpointPort);return cluster;}@@ -100,6 +112,6 @@ public class ElastiCacheMemcachedService {}private String resolveEndpointHost(ElastiCacheContainerHandle handle) {- return config.hostname().orElse(handle.getHost());+ return config.hostname().orElse(handle != null ? handle.getHost() : "localhost");}}diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.javaindex a6e103dd40..e79e80ab52 100644--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java@@ -229,16 +229,17 @@ public class ElastiCacheService implements ResourceProvider {ElastiCacheContainerHandle handle = null;try {- handle = containerManager.start(groupId, image);+ // A replication group record is metadata: its id, endpoint host and proxy port are+ // derived from configuration and need no Docker, so the group is created and reaches+ // 'available' even when no daemon is reachable. Only connecting to the cache needs+ // the container.+ handle = containerManager.tryStart(groupId, image);String endpointHost = resolveEndpointHost();Endpoint endpoint = new Endpoint(endpointHost, proxyPort);ReplicationGroup group = new ReplicationGroup(groupId, request.description(), ReplicationGroupStatus.AVAILABLE,authMode, endpoint, Instant.now(), proxyPort);- group.setContainerId(handle.getContainerId());- group.setContainerHost(handle.getHost());- group.setContainerPort(handle.getPort());group.setAuthToken(request.authToken());group.setArn(regionResolver.buildArn("elasticache", request.region(),"replicationgroup:" + groupId));@@ -248,11 +249,23 @@ public class ElastiCacheService implements ResourceProvider {: 1);applyCommonAttributes(group, request, resolvedSettings);+ if (handle != null) {+ group.setContainerId(handle.getContainerId());+ group.setContainerHost(handle.getHost());+ group.setContainerPort(handle.getPort());+ }+synchronized (lockFor("rg:" + groupId)) {groups.put(groupId, group);- proxyManager.startProxy(groupId, authMode, proxyPort,- handle.getHost(), handle.getPort(),- (username, password) -> validatePassword(groupId, username, password));+ if (handle != null) {+ proxyManager.startProxy(groupId, authMode, proxyPort,+ handle.getHost(), handle.getPort(),+ (username, password) -> validatePassword(groupId, username, password));+ } else {+ LOG.warnv("Replication group {0} created without a backing cache container: no "+ + "Docker daemon is reachable. Metadata operations work; connections to "+ + "the cache do not until a daemon appears.", groupId);+ }}LOG.infov("Replication group {0} created, endpoint={1}:{2}", groupId, endpointHost, String.valueOf(proxyPort));diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManager.javaindex 8bfff344f5..bd67924d51 100644--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManager.java@@ -53,6 +53,7 @@ public class ElastiCacheContainerManager {private final EmulatorConfig config;private final RegionResolver regionResolver;private final Map<String, ElastiCacheContainerHandle> activeContainers = new ConcurrentHashMap<>();+ private volatile boolean dockerUnavailableLogged;@Injectpublic ElastiCacheContainerManager(ContainerBuilder containerBuilder,@@ -69,6 +70,55 @@ public class ElastiCacheContainerManager {this.regionResolver = regionResolver;}+ /**+ * Attempts {@link #start} and reports the backend as unavailable instead of propagating the+ * failure, when the cause is that no Docker daemon is reachable from Floci: Floci running+ * inside Docker without a mounted socket, or a stopped daemon on the host. A failure raised+ * while the daemon <em>is</em> reachable is a genuine container problem and still propagates,+ * so nothing changes for a Floci that can start Valkey containers.+ *+ * @return the container handle, or {@code null} when no Docker daemon is reachable and no+ * container was created+ */+ public ElastiCacheContainerHandle tryStart(String groupId, String image) {+ try {+ ElastiCacheContainerHandle handle = start(groupId, image);+ dockerUnavailableLogged = false;+ return handle;+ } catch (RuntimeException e) {+ if (isDockerReachable()) {+ throw e;+ }+ boolean partial = activeContainers.containsKey(groupId);+ stopByGroupId(groupId);+ if (partial) {+ throw e;+ }+ if (!dockerUnavailableLogged) {+ dockerUnavailableLogged = true;+ LOG.warnv("No Docker daemon is reachable from Floci ({0}). ElastiCache metadata "+ + "operations keep working and replication groups still reach 'available', "+ + "but they have no backing Valkey container until a daemon becomes "+ + "reachable.", e.getMessage());+ }+ return null;+ }+ }++ /**+ * Probes the configured Docker endpoint, which is how a missing daemon is told apart from a+ * container that failed for its own reasons.+ */+ public boolean isDockerReachable() {+ try {+ lifecycleManager.getDockerClient().pingCmd().exec();+ return true;+ } catch (Exception e) {+ LOG.debugv("Docker daemon is not reachable: {0}", e.getMessage());+ return false;+ }+ }+public ElastiCacheContainerHandle start(String groupId, String image) {return start(groupId, image, List.of());}diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheMemcachedContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheMemcachedContainerManager.javaindex 21110e4f2c..2f34c0b4d6 100644--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheMemcachedContainerManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheMemcachedContainerManager.java@@ -49,6 +49,7 @@ public class ElastiCacheMemcachedContainerManager {private final EmulatorConfig config;private final RegionResolver regionResolver;private final Map<String, ElastiCacheContainerHandle> activeContainers = new ConcurrentHashMap<>();+ private volatile boolean dockerUnavailableLogged;@Injectpublic ElastiCacheMemcachedContainerManager(ContainerBuilder containerBuilder,@@ -65,6 +66,55 @@ public class ElastiCacheMemcachedContainerManager {this.regionResolver = regionResolver;}+ /**+ * Attempts {@link #start} and reports the backend as unavailable instead of propagating the+ * failure, when the cause is that no Docker daemon is reachable from Floci: Floci running+ * inside Docker without a mounted socket, or a stopped daemon on the host. A failure raised+ * while the daemon <em>is</em> reachable is a genuine container problem and still propagates,+ * so nothing changes for a Floci that can start Memcached containers.+ *+ * @return the container handle, or {@code null} when no Docker daemon is reachable and no+ * container was created+ */+ public ElastiCacheContainerHandle tryStart(String clusterId, String image) {+ try {+ ElastiCacheContainerHandle handle = start(clusterId, image);+ dockerUnavailableLogged = false;+ return handle;+ } catch (RuntimeException e) {+ if (isDockerReachable()) {+ throw e;+ }+ ElastiCacheContainerHandle partial = activeContainers.get(clusterId);+ if (partial != null) {+ stop(partial);+ throw e;+ }+ if (!dockerUnavailableLogged) {+ dockerUnavailableLogged = true;+ LOG.warnv("No Docker daemon is reachable from Floci ({0}). ElastiCache metadata "+ + "operations keep working and cache clusters still reach 'available', but "+ + "they have no backing Memcached container until a daemon becomes "+ + "reachable.", e.getMessage());+ }+ return null;+ }+ }++ /**+ * Probes the configured Docker endpoint, which is how a missing daemon is told apart from a+ * container that failed for its own reasons.+ */+ public boolean isDockerReachable() {+ try {+ lifecycleManager.getDockerClient().pingCmd().exec();+ return true;+ } catch (Exception e) {+ LOG.debugv("Docker daemon is not reachable: {0}", e.getMessage());+ return false;+ }+ }+public ElastiCacheContainerHandle start(String clusterId, String image) {LOG.infov("Starting Memcached container for cluster: {0}", clusterId);diff --git a/src/main/java/io/github/hectorvent/floci/services/msk/MskService.java b/src/main/java/io/github/hectorvent/floci/services/msk/MskService.javaindex 924998d675..1c670fb068 100644--- a/src/main/java/io/github/hectorvent/floci/services/msk/MskService.java+++ b/src/main/java/io/github/hectorvent/floci/services/msk/MskService.java@@ -162,8 +162,12 @@ public class MskService implements ResourceProvider {if (config.services().msk().mock()) {cluster.setState(ClusterState.ACTIVE);cluster.setBootstrapBrokers("localhost:9092");- } else {- redpandaManager.startContainer(cluster);+ } else if (!redpandaManager.tryStartContainer(cluster)) {+ LOG.warnv("MSK cluster {0} created without a backing Kafka broker: no Docker daemon "+ + "is reachable. Metadata operations work; producing and consuming do not "+ + "until a daemon appears.", clusterName);+ cluster.setState(ClusterState.ACTIVE);+ cluster.setBootstrapBrokers("localhost:9092");}storage.put(clusterArn, cluster);diff --git a/src/main/java/io/github/hectorvent/floci/services/msk/RedpandaManager.java b/src/main/java/io/github/hectorvent/floci/services/msk/RedpandaManager.javaindex 4b41942ee4..b86e1c9248 100644--- a/src/main/java/io/github/hectorvent/floci/services/msk/RedpandaManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/msk/RedpandaManager.java@@ -43,6 +43,7 @@ public class RedpandaManager {private final RegionResolver regionResolver;private final PortAllocator portAllocator;private final Map<String, Closeable> logStreams = new ConcurrentHashMap<>();+ private volatile boolean dockerUnavailableLogged;@Injectpublic RedpandaManager(ContainerBuilder containerBuilder,@@ -61,6 +62,49 @@ public class RedpandaManager {this.portAllocator = portAllocator;}+ /**+ * Attempts {@link #startContainer} and reports the broker as unavailable instead of+ * propagating the failure, when the cause is that no Docker daemon is reachable from Floci:+ * Floci running inside Docker without a mounted socket, or a stopped daemon on the host. A+ * failure raised while the daemon <em>is</em> reachable is a genuine container problem and+ * still propagates, so nothing changes for a Floci that can start Redpanda containers.+ *+ * @return {@code true} when the container started, {@code false} when no Docker daemon is+ * reachable+ */+ public boolean tryStartContainer(MskCluster cluster) {+ try {+ startContainer(cluster);+ dockerUnavailableLogged = false;+ return true;+ } catch (RuntimeException e) {+ if (isDockerReachable()) {+ throw e;+ }+ if (!dockerUnavailableLogged) {+ dockerUnavailableLogged = true;+ LOG.warnv("No Docker daemon is reachable from Floci ({0}). MSK metadata operations "+ + "keep working and clusters still reach ACTIVE, but they have no backing "+ + "Kafka broker until a daemon becomes reachable.", e.getMessage());+ }+ return false;+ }+ }++ /**+ * Probes the configured Docker endpoint, which is how a missing daemon is told apart from a+ * container that failed for its own reasons.+ */+ public boolean isDockerReachable() {+ try {+ lifecycleManager.getDockerClient().pingCmd().exec();+ return true;+ } catch (Exception e) {+ LOG.debugv("Docker daemon is not reachable: {0}", e.getMessage());+ return false;+ }+ }+public void startContainer(MskCluster cluster) {String image = config.services().msk().defaultImage();LOG.infov("Starting Redpanda container for MSK cluster: {0} using image {1}", cluster.getClusterName(), image);diff --git a/src/main/java/io/github/hectorvent/floci/services/opensearch/OpenSearchDomainManager.java b/src/main/java/io/github/hectorvent/floci/services/opensearch/OpenSearchDomainManager.javaindex d1aff1afd9..65fb45a905 100644--- a/src/main/java/io/github/hectorvent/floci/services/opensearch/OpenSearchDomainManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/opensearch/OpenSearchDomainManager.java@@ -34,6 +34,7 @@ public class OpenSearchDomainManager {private final ContainerDetector containerDetector;private final PortAllocator portAllocator;private final EmulatorConfig config;+ private volatile boolean dockerUnavailableLogged;private final RegionResolver regionResolver;@Inject@@ -51,6 +52,50 @@ public class OpenSearchDomainManager {this.regionResolver = regionResolver;}+ /**+ * Attempts {@link #startDomain} and reports the backend as unavailable instead of propagating+ * the failure, when the cause is that no Docker daemon is reachable from Floci: Floci running+ * inside Docker without a mounted socket, or a stopped daemon on the host. A failure raised+ * while the daemon <em>is</em> reachable is a genuine container problem and still propagates,+ * so nothing changes for a Floci that can start OpenSearch containers.+ *+ * @return {@code true} when the container started, {@code false} when no Docker daemon is+ * reachable+ */+ public boolean tryStartDomain(Domain domain) {+ try {+ startDomain(domain);+ dockerUnavailableLogged = false;+ return true;+ } catch (RuntimeException e) {+ if (isDockerReachable()) {+ throw e;+ }+ if (!dockerUnavailableLogged) {+ dockerUnavailableLogged = true;+ LOG.warnv("No Docker daemon is reachable from Floci ({0}). OpenSearch metadata "+ + "operations keep working and domains still report Processing=false, but "+ + "they have no backing search container until a daemon becomes reachable.",+ e.getMessage());+ }+ return false;+ }+ }++ /**+ * Probes the configured Docker endpoint, which is how a missing daemon is told apart from a+ * container that failed for its own reasons.+ */+ public boolean isDockerReachable() {+ try {+ lifecycleManager.getDockerClient().pingCmd().exec();+ return true;+ } catch (Exception e) {+ LOG.debugv("Docker daemon is not reachable: {0}", e.getMessage());+ return false;+ }+ }+public void startDomain(Domain domain) {String image = resolveImage(domain.getEngineVersion());String containerName = containerName(domain);diff --git a/src/main/java/io/github/hectorvent/floci/services/opensearch/OpenSearchService.java b/src/main/java/io/github/hectorvent/floci/services/opensearch/OpenSearchService.javaindex d99840672c..a5f6b77cd7 100644--- a/src/main/java/io/github/hectorvent/floci/services/opensearch/OpenSearchService.java+++ b/src/main/java/io/github/hectorvent/floci/services/opensearch/OpenSearchService.java@@ -146,7 +146,9 @@ public class OpenSearchService implements ResourceProvider {domain.setProcessing(false);} else {domain.setProcessing(true);- domainManager.startDomain(domain);+ if (!domainManager.tryStartDomain(domain)) {+ domain.setProcessing(false);+ }}domainStore.put(domainName, domain);diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.javaindex 192118df87..d7e3b1d6e0 100644--- a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.java@@ -65,6 +65,10 @@ import static org.mockito.Mockito.times;import static org.mockito.Mockito.verify;import static org.mockito.Mockito.when;import static org.mockito.Mockito.withSettings;+import com.github.dockerjava.api.command.PingCmd;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.mockito.Mockito.doNothing;+import static org.mockito.Mockito.doThrow;class Ec2ContainerManagerTest {@@ -941,6 +945,7 @@ class Ec2ContainerManagerTest {when(ec2.imdsPort()).thenReturn(9169);DockerClient dockerClient = mock(DockerClient.class);+ stubDockerPing(dockerClient, true);Ec2MetadataServer metadataServer = mock(Ec2MetadataServer.class);ContainerLogStreamer logStreamer = mock(ContainerLogStreamer.class);Ec2PortForwardManager portForwardManager = mock(Ec2PortForwardManager.class);@@ -1003,6 +1008,65 @@ class Ec2ContainerManagerTest {assertTrue(condition.getAsBoolean(), "condition was not met before timeout");}+ @Test+ void launchRunsInstanceAsMetadataOnlyWhenNoDockerDaemonIsReachable() throws Exception {+ LaunchHarness harness = launchHarness();+ stubDockerPing(harness.dockerClient(), false);++ Instance instance = instance("i-nodocker");++ harness.manager().launch(instance, "ubuntu:24.04", null, "us-west-2");++ assertEquals("running", instance.getState().getName());+ assertNull(instance.getDockerContainerId());+ verify(harness.lifecycleManager(), never()).create(any(ContainerSpec.class));+ verify(harness.metadataServer(), never()).registerContainer(anyString(), anyString(), any());+ }++ @Test+ void launchDegradesToRunningWhenDockerDisappearsMidLaunch() throws Exception {+ LaunchHarness harness = launchHarness();+ PingCmd ping = mock(PingCmd.class);+ when(harness.dockerClient().pingCmd()).thenReturn(ping);+ doNothing().doThrow(new RuntimeException("No such file or directory")).when(ping).exec();+ when(harness.lifecycleManager().create(any(ContainerSpec.class)))+ .thenThrow(new RuntimeException("No such file or directory"));++ Instance instance = instance("i-dockergone");++ harness.manager().launch(instance, "ubuntu:24.04", null, "us-west-2");++ awaitUntil(() -> "running".equals(instance.getState().getName()), Duration.ofSeconds(2));+ }++ @Test+ void metadataOnlyInstanceStopsStartsAndTerminatesWithoutAContainer() throws Exception {+ LaunchHarness harness = launchHarness();+ stubDockerPing(harness.dockerClient(), false);+ Instance instance = instance("i-lifecycle");++ harness.manager().launch(instance, "ubuntu:24.04", null, "us-west-2");+ assertEquals("running", instance.getState().getName());++ harness.manager().stop(instance);+ assertEquals("stopped", instance.getState().getName());++ harness.manager().start(instance);+ assertEquals("running", instance.getState().getName());++ harness.manager().terminate(instance);+ awaitUntil(() -> "terminated".equals(instance.getState().getName()), Duration.ofSeconds(2));+ }++ /** Stubs the daemon reachability probe every launch makes before touching Docker. */+ private static void stubDockerPing(DockerClient dockerClient, boolean reachable) {+ PingCmd ping = mock(PingCmd.class);+ when(dockerClient.pingCmd()).thenReturn(ping);+ if (!reachable) {+ doThrow(new RuntimeException("No such file or directory")).when(ping).exec();+ }+ }+private record LaunchHarness(Ec2ContainerManager manager,ContainerLifecycleManager lifecycleManager,DockerClient dockerClient,diff --git a/src/test/java/io/github/hectorvent/floci/services/ecr/EcrServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/ecr/EcrServiceTest.javaindex 5ab296f869..4090342719 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecr/EcrServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecr/EcrServiceTest.java@@ -55,7 +55,9 @@ class EcrServiceTest {when(registryManager.getProxyEndpoint()).thenReturn("http://localhost:5000");when(registryManager.internalRepoName(anyString(), anyString(), anyString())).thenAnswer(inv -> inv.getArgument(0) + "/" + inv.getArgument(1) + "/" + inv.getArgument(2));- // ensureStarted() is a no-op on the mock — no Docker calls in any test below.+ // The mock never talks to Docker; report the backing registry as available so the+ // data-plane operations run. Registry-unavailable behaviour has its own tests below.+ when(registryManager.tryEnsureStarted()).thenReturn(true);EmulatorConfig config = Mockito.mock(EmulatorConfig.class);RegionResolver regionResolver = new RegionResolver(REGION, ACCOUNT);@@ -80,7 +82,6 @@ class EcrServiceTest {assertTrue(repo.getRepositoryArn().startsWith("arn:aws:ecr:us-east-1:000000000000:repository/"));assertTrue(repo.getRepositoryUri().contains("localhost:"));assertEquals("MUTABLE", repo.getImageTagMutability());- Mockito.verify(registryManager).ensureStarted();}@Test@@ -116,6 +117,66 @@ class EcrServiceTest {assertEquals("platform", repo.getTags().get("team"));}+ // ------------------------------------------------------------+ // Registry unavailable (Floci inside Docker with no Docker daemon)+ // ------------------------------------------------------------++ @Test+ void createRepository_succeedsAsMetadataWhenRegistryIsUnavailable() {+ when(registryManager.tryEnsureStarted()).thenReturn(false);++ Repository repo = service.createRepository(REPO, null, "IMMUTABLE", true, null, null,+ Map.of("Project", "choudoufu"), REGION);++ assertEquals(REPO, repo.getRepositoryName());+ assertEquals(ACCOUNT, repo.getRegistryId());+ assertEquals("arn:aws:ecr:" + REGION + ":" + ACCOUNT + ":repository/" + REPO,+ repo.getRepositoryArn());+ assertTrue(repo.getRepositoryUri().endsWith("/" + REPO), repo.getRepositoryUri());+ assertEquals("IMMUTABLE", repo.getImageTagMutability());+ assertEquals("choudoufu", repo.getTags().get("Project"));+ }++ @Test+ void repositoryMetadataCrudWorksWhenRegistryIsUnavailable() {+ when(registryManager.tryEnsureStarted()).thenReturn(false);++ service.createRepository(REPO, null, null, null, null, null, Map.of("env", "dev"), REGION);++ assertEquals(1, service.describeRepositories(List.of(REPO), null, REGION).size());+ assertEquals("dev", service.listTagsForResource(REPO, null, REGION).get("env"));++ service.tagResource(REPO, null, Map.of("team", "platform"), REGION);+ assertEquals("platform", service.listTagsForResource(REPO, null, REGION).get("team"));++ service.deleteRepository(REPO, null, false, REGION);+ assertThrows(AwsException.class,+ () -> service.describeRepositories(List.of(REPO), null, REGION));+ }++ @Test+ void dataPlaneOperationsFailWithServerExceptionWhenRegistryIsUnavailable() {+ service.createRepository(REPO, null, null, null, null, null, null, REGION);+ when(registryManager.tryEnsureStarted()).thenReturn(false);++ AwsException auth = assertThrows(AwsException.class, () -> service.getAuthorizationToken());+ assertEquals("ServerException", auth.getErrorCode());+ assertEquals(500, auth.getHttpStatus());+ assertTrue(auth.getMessage().contains("Docker"), auth.getMessage());++ assertEquals("ServerException",+ assertThrows(AwsException.class, () -> service.listImages(REPO, null, REGION)).getErrorCode());+ assertEquals("ServerException",+ assertThrows(AwsException.class,+ () -> service.describeImages(REPO, null, null, REGION)).getErrorCode());+ assertEquals("ServerException",+ assertThrows(AwsException.class,+ () -> service.batchGetImage(REPO, List.of(), null, null, REGION)).getErrorCode());+ assertEquals("ServerException",+ assertThrows(AwsException.class,+ () -> service.batchDeleteImage(REPO, List.of(), null, REGION)).getErrorCode());+ }+// ------------------------------------------------------------// DescribeRepositories// ------------------------------------------------------------@@ -227,7 +288,7 @@ class EcrServiceTest {assertNotNull(data.getExpiresAt());String decoded = new String(Base64.getDecoder().decode(data.getAuthorizationToken()));assertTrue(decoded.startsWith("AWS:"), "decoded token should start with AWS: but was: " + decoded);- Mockito.verify(registryManager).ensureStarted();+ Mockito.verify(registryManager).tryEnsureStarted();}@Testdiff --git a/src/test/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryManagerTest.javaindex 5673b03f02..899541c205 100644--- a/src/test/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryManagerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ecr/registry/EcrRegistryManagerTest.java@@ -143,6 +143,28 @@ class EcrRegistryManagerTest {}}+ @Test+ void tryEnsureStarted_reportsFailureInsteadOfThrowingWhenDockerIsUnreachable() {+ // Floci running inside Docker without a mounted daemon socket. The control plane+ // must keep working, so the failure is reported rather than propagated.+ when(lifecycleManager.createAndStart(any()))+ .thenThrow(new RuntimeException("Cannot connect to the Docker daemon"));++ for (int attempt = 0; attempt < 6; attempt++) {+ assertFalse(manager.tryEnsureStarted(), "attempt " + attempt + " should report unavailable");+ }+ assertFalse(manager.isStarted());+ }++ @Test+ void tryEnsureStarted_reportsSuccessOnceTheRegistryStarts() {+ when(lifecycleManager.createAndStart(any()))+ .thenReturn(new ContainerLifecycleManager.ContainerInfo("0123456789abcdef", Map.of()));++ assertTrue(manager.tryEnsureStarted());+ assertTrue(manager.isStarted());+ }+@Testvoid httpClient_usesRegistryContainerDnsWhenRunningInsideDocker() {when(containerDetector.isRunningInContainer()).thenReturn(true);diff --git a/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.javaindex 0f26d572f5..61df0b9a04 100644--- a/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java@@ -412,7 +412,10 @@ class EksServiceTest {clusterStore.putForAccount("000000000000", "persisted-cluster", persisted);EksClusterManager clusterManager = mock(EksClusterManager.class);- doThrow(new RuntimeException("no docker")).when(clusterManager).restoreCluster(persisted);+ // The daemon is reachable, so this is a genuine restore failure: the daemonless+ // degradation path (which marks the cluster metadata-only ACTIVE) must not kick in.+ when(clusterManager.isDockerReachable()).thenReturn(true);+ doThrow(new RuntimeException("bad container state")).when(clusterManager).restoreCluster(persisted);EksService restarted = new EksService(fixedStorageFactory(clusterStore), testConfig(false),new RegionResolver("us-east-1", "000000000000"), clusterManager, null,new EksOidcService(fixedStorageFactory(new InMemoryStorage<String, ClusterOidcKey>()),@@ -428,6 +431,33 @@ class EksServiceTest {}}+ @Test+ void initRestoresAClusterAsMetadataOnlyWhenNoDockerDaemonIsReachable() {+ StorageBackend<String, Cluster> rawClusters = new InMemoryStorage<>();+ var clusterStore = new AccountAwareStorageBackend<>(rawClusters, null, "000000000000");+ Cluster persisted = new Cluster();+ persisted.setName("persisted-cluster");+ persisted.setStatus(ClusterStatus.ACTIVE);+ clusterStore.putForAccount("000000000000", "persisted-cluster", persisted);++ EksClusterManager clusterManager = mock(EksClusterManager.class);+ when(clusterManager.isDockerReachable()).thenReturn(false);+ doThrow(new RuntimeException("no docker")).when(clusterManager).restoreCluster(persisted);+ EksService restarted = new EksService(fixedStorageFactory(clusterStore), testConfig(false),+ new RegionResolver("us-east-1", "000000000000"), clusterManager, null,+ new EksOidcService(fixedStorageFactory(new InMemoryStorage<String, ClusterOidcKey>()),+ new ObjectMapper()));+ try {+ restarted.init();++ // Same degradation as create: losing the daemon is not the cluster's fault.+ assertEquals(ClusterStatus.ACTIVE,+ restarted.describeCluster("persisted-cluster").getStatus());+ } finally {+ restarted.shutdown();+ }+ }+@SuppressWarnings("unchecked")private StorageFactory fixedStorageFactory(StorageBackend<String, ?> backend) {return new StorageFactory(null, null) {@@ -950,4 +980,91 @@ class EksServiceTest {() -> eksService.deleteFargateProfile("my-eks-cluster", "missing-profile"));assertEquals(404, delete.getHttpStatus());}++ private EksService newService(EksClusterManager clusterManager, boolean mock) {+ StorageFactory storageFactory = new StorageFactory(null, null) {+ @Override+ public <V> AccountAwareStorageBackend<V> create(String serviceName, String fileName,+ TypeReference<Map<String, V>> typeReference) {+ return AccountAwareStorageBackend.inMemory("000000000000");+ }+ };+ return new EksService(storageFactory, testConfig(mock),+ new RegionResolver("us-east-1", "000000000000"), clusterManager, null,+ new EksOidcService(storageFactory, new ObjectMapper()));+ }++ @Test+ void createClusterStaysCreatingWhileTheK3sContainerBoots() {+ EksClusterManager clusterManager = mock(EksClusterManager.class);+ when(clusterManager.tryStartCluster(any())).thenReturn(true);+ EksService service = newService(clusterManager, false);++ CreateClusterRequest request = new CreateClusterRequest();+ request.setName("real-eks");+ request.setRoleArn("arn:aws:iam::000000000000:role/eks-role");++ assertEquals(ClusterStatus.CREATING, service.createCluster(request).getStatus());+ }++ @Test+ void createClusterReachesActiveMetadataWhenNoDockerDaemonIsReachable() {+ EksClusterManager clusterManager = mock(EksClusterManager.class);+ when(clusterManager.tryStartCluster(any())).thenReturn(false);+ EksService service = newService(clusterManager, false);++ CreateClusterRequest request = new CreateClusterRequest();+ request.setName("probe-eks");+ request.setRoleArn("arn:aws:iam::000000000000:role/eks-role");+ request.setTags(Map.of("tofu-estate", "probe1"));++ Cluster cluster = service.createCluster(request);++ assertEquals(ClusterStatus.ACTIVE, cluster.getStatus());+ assertEquals("https://localhost:6500", cluster.getEndpoint());+ assertEquals("probe1", cluster.getTags().get("tofu-estate"));+ // No k3s API server exists. The empty CA is what says so.+ assertEquals("", cluster.getCertificateAuthority().getData());+ assertNull(cluster.getContainerId());+ }++ @Test+ void clusterMetadataCrudWorksWhenNoDockerDaemonIsReachable() {+ EksClusterManager clusterManager = mock(EksClusterManager.class);+ when(clusterManager.tryStartCluster(any())).thenReturn(false);+ EksService service = newService(clusterManager, false);++ CreateClusterRequest request = new CreateClusterRequest();+ request.setName("probe-eks");+ request.setRoleArn("arn:aws:iam::000000000000:role/eks-role");+ request.setTags(Map.of("tofu-estate", "probe1"));+ String arn = service.createCluster(request).getArn();++ assertEquals(ClusterStatus.ACTIVE, service.describeCluster("probe-eks").getStatus());+ assertEquals(List.of("probe-eks"), service.listClusters());+ assertEquals("probe1", service.listTagsForResource(arn).get("tofu-estate"));++ service.tagResource(arn, Map.of("Name", "probe-eks"));+ assertEquals("probe-eks", service.listTagsForResource(arn).get("Name"));++ assertEquals(NodegroupStatus.ACTIVE,+ service.createNodeGroup("probe-eks", nodeGroupRequest("ng-1")).getStatus());++ service.deleteCluster("probe-eks");+ assertThrows(AwsException.class, () -> service.describeCluster("probe-eks"));+ }++ @Test+ void createClusterStillFailsOnAGenuineProvisioningErrorWithAReachableDaemon() {+ EksClusterManager clusterManager = mock(EksClusterManager.class);+ when(clusterManager.tryStartCluster(any()))+ .thenThrow(new RuntimeException("no such image: rancher/k3s"));+ EksService service = newService(clusterManager, false);++ CreateClusterRequest request = new CreateClusterRequest();+ request.setName("broken-eks");+ request.setRoleArn("arn:aws:iam::000000000000:role/eks-role");++ assertEquals(ClusterStatus.FAILED, service.createCluster(request).getStatus());+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedServiceTest.javaindex bd10437abd..e5f84923ba 100644--- a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedServiceTest.java@@ -24,10 +24,11 @@ import static org.mockito.Mockito.when;class ElastiCacheMemcachedServiceTest {private ElastiCacheMemcachedService service;+ private ElastiCacheMemcachedContainerManager containerManager;@BeforeEachvoid setUp() {- ElastiCacheMemcachedContainerManager containerManager = mock(ElastiCacheMemcachedContainerManager.class);+ containerManager = mock(ElastiCacheMemcachedContainerManager.class);StorageFactory storageFactory = mock(StorageFactory.class);EmulatorConfig config = mock(EmulatorConfig.class);@@ -39,7 +40,7 @@ class ElastiCacheMemcachedServiceTest {when(config.hostname()).thenReturn(Optional.of("localhost"));when(storageFactory.create(anyString(), anyString(), any())).thenAnswer(inv -> AccountAwareStorageBackend.inMemory("000000000000"));- when(containerManager.start(anyString(), anyString()))+ when(containerManager.tryStart(anyString(), anyString())).thenReturn(new ElastiCacheContainerHandle("cid", "cluster", "localhost", 11211));service = new ElastiCacheMemcachedService(containerManager, storageFactory, config);@@ -111,7 +112,7 @@ class ElastiCacheMemcachedServiceTest {when(config.hostname()).thenReturn(Optional.empty());when(storageFactory.create(anyString(), anyString(), any())).thenAnswer(inv -> AccountAwareStorageBackend.inMemory("000000000000"));- when(containerManager.start(anyString(), anyString()))+ when(containerManager.tryStart(anyString(), anyString())).thenReturn(new ElastiCacheContainerHandle("cid", "cluster", "172.20.0.10", 11211));ElastiCacheMemcachedService containerModeService =@@ -121,4 +122,25 @@ class ElastiCacheMemcachedServiceTest {assertEquals("172.20.0.10", cluster.getConfigurationEndpoint().address());}++ @Test+ void createClusterWithoutDockerDaemonStillReachesAvailable() {+ // tryStart() returns null when no Docker daemon is reachable. The cache cluster record is+ // metadata, so the create still succeeds and the cluster reaches 'available' on the first+ // describe (what SDK/Terraform waiters poll), on Memcached's well-known port.+ when(containerManager.tryStart(anyString(), anyString())).thenReturn(null);++ CacheCluster cluster = service.createCacheCluster("no-docker-cluster");++ assertEquals(CacheClusterStatus.AVAILABLE, cluster.getCacheClusterStatus());+ assertEquals("localhost", cluster.getConfigurationEndpoint().address());+ assertEquals(11211, cluster.getConfigurationEndpoint().port());+ assertEquals("no-docker-cluster",+ service.getCacheCluster("no-docker-cluster").getCacheClusterId());++ // Delete must not reach for a container that was never created.+ service.deleteCacheCluster("no-docker-cluster");+ org.mockito.Mockito.verify(containerManager, org.mockito.Mockito.never())+ .stop(org.mockito.ArgumentMatchers.any());+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.javaindex 9ad4ec460d..1511987046 100644--- a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.elasticache;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.core.common.RegionResolver;@@ -75,7 +76,7 @@ class ElastiCacheServiceTest {when(config.hostname()).thenReturn(java.util.Optional.of("localhost"));when(storageFactory.create(anyString(), anyString(), any())).thenAnswer(inv -> AccountAwareStorageBackend.inMemory("000000000000"));- when(containerManager.start(anyString(), anyString()))+ when(containerManager.tryStart(anyString(), anyString())).thenReturn(new ElastiCacheContainerHandle("cid", "grp", "localhost", 6379));doNothing().when(proxyManager).startProxy(anyString(), any(), anyInt(), anyString(), anyInt(), any());Ec2Service ec2Service = org.mockito.Mockito.mock(Ec2Service.class);@@ -174,7 +175,7 @@ class ElastiCacheServiceTest {void failedProvisioningRollsBackContainerAndReleasesProxyPort() {ElastiCacheContainerHandle handle =new ElastiCacheContainerHandle("cid", "grp", "localhost", 6379);- when(containerManager.start(anyString(), anyString())).thenReturn(handle);+ when(containerManager.tryStart(anyString(), anyString())).thenReturn(handle);// Proxy startup blows up after the port is reserved and the container is started.doThrow(new RuntimeException("proxy boom"))@@ -203,7 +204,7 @@ class ElastiCacheServiceTest {void failedContainerStartupCleansUpContainerByIdAndReleasesPort() {// Models a readiness timeout: start() throws without ever returning a handle.doThrow(new RuntimeException("readiness boom"))- .when(containerManager).start(eq("grp"), anyString());+ .when(containerManager).tryStart(eq("grp"), anyString());assertThrows(RuntimeException.class,() -> service.createReplicationGroup("grp", "test", AuthMode.PASSWORD, null, "us-east-1"));@@ -212,7 +213,7 @@ class ElastiCacheServiceTest {verify(containerManager).stopByGroupId("grp");// The reserved proxy port was still released: a subsequent successful create reuses the base port.- when(containerManager.start(anyString(), anyString()))+ when(containerManager.tryStart(anyString(), anyString())).thenReturn(new ElastiCacheContainerHandle("cid", "grp2", "localhost", 6379));ReplicationGroup recovered =service.createReplicationGroup("grp2", "test", AuthMode.PASSWORD, null, "us-east-1");@@ -503,7 +504,7 @@ class ElastiCacheServiceTest {void concurrentCreateForSameGroupIdIsRejectedWhileFirstIsProvisioning() throws InterruptedException {CountDownLatch startedLatch = new CountDownLatch(1);CountDownLatch releaseLatch = new CountDownLatch(1);- when(containerManager.start(anyString(), anyString())).thenAnswer(inv -> {+ when(containerManager.tryStart(anyString(), anyString())).thenAnswer(inv -> {startedLatch.countDown();assertTrue(releaseLatch.await(5, TimeUnit.SECONDS), "test timed out waiting for release");return new ElastiCacheContainerHandle("cid", "grp", "localhost", 6379);@@ -723,7 +724,7 @@ class ElastiCacheServiceTest {service.createCacheParameterGroup("custom-pg", "redis7", "in use", Map.of());CountDownLatch startedLatch = new CountDownLatch(1);CountDownLatch releaseLatch = new CountDownLatch(1);- when(containerManager.start(anyString(), anyString())).thenAnswer(inv -> {+ when(containerManager.tryStart(anyString(), anyString())).thenAnswer(inv -> {startedLatch.countDown();assertTrue(releaseLatch.await(5, TimeUnit.SECONDS), "test timed out waiting for release");return new ElastiCacheContainerHandle("cid", "grp", "localhost", 6379);@@ -773,7 +774,7 @@ class ElastiCacheServiceTest {CountDownLatch startedLatch = new CountDownLatch(1);CountDownLatch releaseLatch = new CountDownLatch(1);- when(containerManager.start(anyString(), anyString())).thenAnswer(inv -> {+ when(containerManager.tryStart(anyString(), anyString())).thenAnswer(inv -> {startedLatch.countDown();assertTrue(releaseLatch.await(5, TimeUnit.SECONDS), "test timed out waiting for release");return new ElastiCacheContainerHandle("cid", "grp", "localhost", 6379);@@ -808,7 +809,7 @@ class ElastiCacheServiceTest {@Testvoid aFailedCreateReleasesItsClaimOnTheParameterGroup() {service.createCacheParameterGroup("custom-pg", "redis7", "in use", Map.of());- when(containerManager.start(anyString(), anyString()))+ when(containerManager.tryStart(anyString(), anyString())).thenThrow(new RuntimeException("docker is down"));assertThrows(RuntimeException.class,diff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManagerTest.javaindex 35550327ef..fe918bebea 100644--- a/src/test/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManagerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManagerTest.java@@ -17,8 +17,10 @@ import java.nio.charset.StandardCharsets;import java.util.Map;import java.util.Optional;+import static org.junit.jupiter.api.Assertions.assertThrows;import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.eq;import static org.mockito.Mockito.RETURNS_SELF;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.verify;@@ -92,4 +94,35 @@ class ElastiCacheContainerManagerTest {"io.floci.region", "us-east-1"));}}++ @Test+ void tryStartRethrowsWhenDaemonBecomesUnreachableAfterContainerWasCreated() {+ ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);+ when(lifecycleManager.createAndStart(any())).thenReturn(new ContainerLifecycleManager.ContainerInfo(+ "container-id", Map.of(6379,+ new ContainerLifecycleManager.EndpointInfo("127.0.0.1", 6379))));+ when(lifecycleManager.getDockerClient()).thenThrow(new IllegalStateException("no daemon"));+ ContainerLogStreamer logStreamer = mock(ContainerLogStreamer.class);+ when(logStreamer.attach(any(), any(), any(), any(), any()))+ .thenThrow(new IllegalStateException("log attach failed after daemon went away"));++ ContainerBuilder containerBuilder = mock(ContainerBuilder.class);+ ContainerBuilder.Builder builder = mock(ContainerBuilder.Builder.class, RETURNS_SELF);+ when(containerBuilder.newContainer(anyString())).thenReturn(builder);+ when(builder.build()).thenReturn(mock(ContainerSpec.class));++ EmulatorConfig config = mock(EmulatorConfig.class);+ EmulatorConfig.ServicesConfig services = mock(EmulatorConfig.ServicesConfig.class);+ EmulatorConfig.ElastiCacheServiceConfig elasticache = mock(EmulatorConfig.ElastiCacheServiceConfig.class);+ when(config.services()).thenReturn(services);+ when(services.elasticache()).thenReturn(elasticache);+ when(elasticache.dockerNetwork()).thenReturn(Optional.empty());++ ElastiCacheContainerManager manager = new ElastiCacheContainerManager(containerBuilder, lifecycleManager,+ logStreamer, mock(ContainerDetector.class), config, mock(RegionResolver.class));++ assertThrows(IllegalStateException.class, () -> manager.tryStart("my-group", "valkey/valkey:7.2"));++ verify(lifecycleManager).stopAndRemove(eq("container-id"), any());+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheMemcachedContainerManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheMemcachedContainerManagerTest.javaindex b14755469c..0c9d596363 100644--- a/src/test/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheMemcachedContainerManagerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheMemcachedContainerManagerTest.java@@ -17,8 +17,10 @@ import java.nio.charset.StandardCharsets;import java.util.Map;import java.util.Optional;+import static org.junit.jupiter.api.Assertions.assertThrows;import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.eq;import static org.mockito.Mockito.RETURNS_SELF;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.verify;@@ -79,4 +81,36 @@ class ElastiCacheMemcachedContainerManagerTest {"io.floci.region", "us-east-1"));}}++ @Test+ void tryStartRethrowsWhenDaemonBecomesUnreachableAfterContainerWasCreated() {+ ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);+ when(lifecycleManager.createAndStart(any())).thenReturn(new ContainerLifecycleManager.ContainerInfo(+ "container-id", Map.of(11211,+ new ContainerLifecycleManager.EndpointInfo("127.0.0.1", 11211))));+ when(lifecycleManager.getDockerClient()).thenThrow(new IllegalStateException("no daemon"));+ ContainerLogStreamer logStreamer = mock(ContainerLogStreamer.class);+ when(logStreamer.attach(any(), any(), any(), any(), any()))+ .thenThrow(new IllegalStateException("log attach failed after daemon went away"));++ ContainerBuilder containerBuilder = mock(ContainerBuilder.class);+ ContainerBuilder.Builder builder = mock(ContainerBuilder.Builder.class, RETURNS_SELF);+ when(containerBuilder.newContainer(anyString())).thenReturn(builder);+ when(builder.build()).thenReturn(mock(ContainerSpec.class));++ EmulatorConfig config = mock(EmulatorConfig.class);+ EmulatorConfig.ServicesConfig services = mock(EmulatorConfig.ServicesConfig.class);+ EmulatorConfig.ElastiCacheServiceConfig elasticache = mock(EmulatorConfig.ElastiCacheServiceConfig.class);+ when(config.services()).thenReturn(services);+ when(services.elasticache()).thenReturn(elasticache);+ when(elasticache.dockerNetwork()).thenReturn(Optional.empty());++ ElastiCacheMemcachedContainerManager manager = new ElastiCacheMemcachedContainerManager(+ containerBuilder, lifecycleManager, logStreamer, mock(ContainerDetector.class),+ config, mock(RegionResolver.class));++ assertThrows(IllegalStateException.class, () -> manager.tryStart("my-cluster", "memcached:1.6"));++ verify(lifecycleManager).stopAndRemove(eq("container-id"), any());+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/msk/MskServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/msk/MskServiceTest.javaindex 6138fe37af..0ae39a19ee 100644--- a/src/test/java/io/github/hectorvent/floci/services/msk/MskServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/msk/MskServiceTest.java@@ -136,6 +136,34 @@ class MskServiceTest {assertTrue(mskService.listClusters().isEmpty());}+ @Test+ void createClusterStaysCreatingWhileBrokerStarts() {+ when(config.services().msk().mock()).thenReturn(false);+ when(redpandaManager.tryStartContainer(Mockito.any())).thenReturn(true);++ MskCluster cluster = mskService.createCluster("docker-cluster");++ assertEquals(ClusterState.CREATING, cluster.getState());+ assertEquals(ClusterState.CREATING,+ mskService.describeCluster(cluster.getClusterArn()).getState());+ }++ @Test+ void createClusterWithoutDockerDaemonStillReachesActive() {+ // tryStartContainer() returns false when no Docker daemon is reachable. The cluster+ // record is metadata, so the create still succeeds, reaches ACTIVE, and reports a+ // placeholder bootstrap address until a daemon appears.+ when(config.services().msk().mock()).thenReturn(false);+ when(redpandaManager.tryStartContainer(Mockito.any())).thenReturn(false);++ MskCluster cluster = mskService.createCluster("no-docker-cluster");++ assertEquals(ClusterState.ACTIVE, cluster.getState());+ assertEquals("localhost:9092", cluster.getBootstrapBrokers());+ assertEquals("no-docker-cluster",+ mskService.describeCluster(cluster.getClusterArn()).getClusterName());+ }+@Testvoid createClusterPersistsRequestMetadata() {CreateClusterRequest request = new CreateClusterRequest();diff --git a/src/test/java/io/github/hectorvent/floci/services/opensearch/OpenSearchServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/opensearch/OpenSearchServiceTest.javanew file mode 100644index 0000000000..6da3c7d243--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/opensearch/OpenSearchServiceTest.java@@ -0,0 +1,82 @@+package io.github.hectorvent.floci.services.opensearch;++import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.services.opensearch.model.Domain;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.mockito.Mockito;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.when;++/**+ * A domain stays {@code Processing=true} until the readiness poller flips it. Mock mode and a+ * Floci with no reachable Docker daemon report {@code Processing=false} from the first read.+ */+class OpenSearchServiceTest {++ private OpenSearchService service;+ private OpenSearchDomainManager domainManager;+ private EmulatorConfig.OpenSearchServiceConfig osConfig;++ @BeforeEach+ void setUp() {+ domainManager = mock(OpenSearchDomainManager.class);+ EmulatorConfig config = mock(EmulatorConfig.class);+ EmulatorConfig.ServicesConfig servicesConfig = mock(EmulatorConfig.ServicesConfig.class);+ osConfig = mock(EmulatorConfig.OpenSearchServiceConfig.class);+ when(config.services()).thenReturn(servicesConfig);+ when(servicesConfig.opensearch()).thenReturn(osConfig);+ when(osConfig.mock()).thenReturn(false);++ RegionResolver regionResolver = new RegionResolver("us-east-1", "000000000000");+ service = new OpenSearchService(AccountAwareStorageBackend.inMemory("000000000000"),+ config, regionResolver, domainManager);+ }++ @Test+ void createDomainKeepsProcessingTrueWhileContainerStarts() {+ when(domainManager.tryStartDomain(any())).thenReturn(true);++ Domain domain = service.createDomain("docker-domain", "OpenSearch_2.11",+ null, null, null, "us-east-1");++ assertTrue(domain.isProcessing(),+ "the readiness poller flips Processing once the container answers");+ assertTrue(service.describeDomain("docker-domain").isProcessing());+ verify(domainManager).tryStartDomain(any());+ }++ @Test+ void createDomainWithoutDockerDaemonStillSucceedsAndReportsProcessingFalse() {+ // tryStartDomain() returns false when no Docker daemon is reachable. The domain record is+ // metadata, so the create still succeeds and the first describe already reports the+ // terminal state.+ when(domainManager.tryStartDomain(any())).thenReturn(false);++ Domain domain = service.createDomain("no-docker-domain", "OpenSearch_2.11",+ null, null, null, "us-east-1");++ assertFalse(domain.isProcessing());+ assertEquals("no-docker-domain", service.describeDomain("no-docker-domain").getDomainName());+ assertFalse(service.describeDomain("no-docker-domain").isProcessing());+ }++ @Test+ void mockModeKeepsProcessingFalse() {+ when(osConfig.mock()).thenReturn(true);++ Domain domain = service.createDomain("mock-domain", "OpenSearch_2.11",+ null, null, null, "us-east-1");++ assertFalse(domain.isProcessing());+ verify(domainManager, Mockito.never()).tryStartDomain(any());+ }+}