Light, fluffy, and always free - The AWS Local Emulator alternative
fix(elasticache): re-provision caches on restart instead of reporting a dead endpoint as available (#4094)
Abhigya14d13ee798b8
diff --git a/docs/services/elasticache.md b/docs/services/elasticache.mdindex c3785584bd..eb14c34d8e 100644--- a/docs/services/elasticache.md+++ b/docs/services/elasticache.md@@ -60,14 +60,34 @@ case — the shipped `docker-compose.yml` uses `localhost.floci.io`, which publiFloci's embedded DNS resolve it to the Floci container from inside Docker. Cluster-mode groupsthen announce that name and report it as their `ConfigurationEndpoint`.-With `persistent`, `hybrid` or `wal` storage, cluster-mode groups are re-provisioned from their-persisted topology on startup: containers are restarted, the cluster is re-formed (caches restart-empty, as on any Floci restart) and each node's proxy port is re-reserved. Ports are re-reserved-and groups marked `creating` before Floci reports ready; the container restarts and cluster-formation run in the background so a slow Docker daemon cannot delay readiness, and each group-flips to `available` once its data plane is back. A group whose data plane cannot be brought back-is reported with status `create-failed` instead of `available`, and its member clusters answer-`DescribeCacheClusters` with `restore-failed` (`CacheClusterStatus` has no `create-failed` value).+With `persistent`, `hybrid` or `wal` storage, every replication group and Memcached cluster is+re-provisioned from its persisted record on startup: containers are restarted, cluster-mode groups+are re-formed, and proxy ports are re-reserved. Caches restart empty, as on any Floci restart:+only the topology is persisted, never the keyspace.++Ports are re-reserved and records marked `creating` before Floci reports ready; the container+restarts and cluster formation run in the background so a slow Docker daemon cannot delay+readiness, and each record flips to `available` once its data plane is back. A replication group+whose data plane cannot be brought back is reported with status `create-failed` instead of+`available`, and its member clusters answer `DescribeCacheClusters` with `restore-failed`+(`CacheClusterStatus` has no `create-failed` value). A Memcached cluster that cannot be brought+back reports `restore-failed` directly.++Reporting a failed record as failed matters more than it looks. A record that is left unreconciled+keeps its `available` status with nothing behind the endpoint, so the control plane answers healthy+while every connection fails, and the proxy port it still advertises is free for the next create to+take: the old endpoint then reaches an unrelated cache rather than failing cleanly.++A Memcached cluster's endpoint follows its new container. Outside Docker, Floci publishes the+backend on a host port Docker picks per run, so a restored cluster's `ConfigurationEndpoint` port+can differ from the one it had before the restart. Replication groups keep their port: it is a+proxy port Floci owns and re-reserves.++A delete that arrives while a record is still `creating` wins. The restore takes the same+per-record monitor `DeleteReplicationGroup` and `DeleteCacheCluster` take, and skips its write-back+when the record is gone, so a group or cluster deleted in the first seconds after boot stays+deleted rather than coming back `available`. Any container the abandoned restore had already+started is stopped.`DescribeReplicationGroups` reports the topology honestly: `ClusterEnabled`, one `NodeGroup` pershard with its `Slots`, `NodeGroupMembers`, and `MemberClusters`. Each member also answersdiff --git a/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java b/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.javaindex 20670f2f55..096f379f8d 100644--- a/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java+++ b/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java@@ -14,6 +14,7 @@ import io.github.hectorvent.floci.services.floci.ui.FlociUiManager;import io.github.hectorvent.floci.services.amazonmq.container.RabbitMqManager;import io.github.hectorvent.floci.services.kinesisanalytics.container.FlinkContainerManager;import io.github.hectorvent.floci.services.iam.IamService;+import io.github.hectorvent.floci.services.elasticache.ElastiCacheMemcachedService;import io.github.hectorvent.floci.services.elasticache.ElastiCacheService;import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerManager;import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheMemcachedContainerManager;@@ -73,6 +74,7 @@ public class EmulatorLifecycle {private final EmulatorConfig config;private final IamService iamService;private final ElastiCacheService elastiCacheService;+ private final ElastiCacheMemcachedService elastiCacheMemcachedService;private final ElastiCacheContainerManager elastiCacheContainerManager;private final ElastiCacheMemcachedContainerManager elastiCacheMemcachedContainerManager;private final ElastiCacheProxyManager elastiCacheProxyManager;@@ -108,6 +110,7 @@ public class EmulatorLifecycle {EmulatorConfig config,IamService iamService,ElastiCacheService elastiCacheService,+ ElastiCacheMemcachedService elastiCacheMemcachedService,ElastiCacheContainerManager elastiCacheContainerManager,ElastiCacheMemcachedContainerManager elastiCacheMemcachedContainerManager,ElastiCacheProxyManager elastiCacheProxyManager,@@ -142,6 +145,7 @@ public class EmulatorLifecycle {this.config = config;this.iamService = iamService;this.elastiCacheService = elastiCacheService;+ this.elastiCacheMemcachedService = elastiCacheMemcachedService;this.elastiCacheContainerManager = elastiCacheContainerManager;this.elastiCacheMemcachedContainerManager = elastiCacheMemcachedContainerManager;this.elastiCacheProxyManager = elastiCacheProxyManager;@@ -222,7 +226,11 @@ public class EmulatorLifecycle {}if (config.services().elasticache().enabled()) {elastiCacheService.restorePersistedRuntime().exceptionally(ex -> {- LOG.warnv("ElastiCache cluster-mode restore failed: {0}", ex.getMessage());+ LOG.warnv("ElastiCache replication group restore failed: {0}", ex.getMessage());+ return null;+ });+ elastiCacheMemcachedService.restorePersistedRuntime().exceptionally(ex -> {+ LOG.warnv("ElastiCache Memcached restore failed: {0}", ex.getMessage());return null;});}diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedService.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedService.javaindex b7bf3d6baf..5b415827a1 100644--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedService.java+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedService.java@@ -15,9 +15,12 @@ import jakarta.inject.Inject;import org.jboss.logging.Logger;import java.time.Instant;+import java.util.ArrayList;import java.util.Collection;import java.util.List;import java.util.Map;+import java.util.concurrent.CompletableFuture;+import java.util.concurrent.ConcurrentHashMap;@ApplicationScopedpublic class ElastiCacheMemcachedService {@@ -31,6 +34,7 @@ public class ElastiCacheMemcachedService {private final StorageBackend<String, CacheCluster> clusters;private final ElastiCacheMemcachedContainerManager containerManager;private final EmulatorConfig config;+ private final ConcurrentHashMap<String, Object> clusterLocks = new ConcurrentHashMap<>();@Injectpublic ElastiCacheMemcachedService(ElastiCacheMemcachedContainerManager containerManager,@@ -78,6 +82,147 @@ public class ElastiCacheMemcachedService {return cluster;}+ /**+ * Restarts the container behind every cache cluster restored from disk. Invoked from+ * {@code EmulatorLifecycle} after {@code storageFactory.loadAll()}, for the same reason+ * {@code ElastiCacheService.restorePersistedRuntime} exists: only the record is persisted,+ * the container is process-local, and a cluster left unreconciled reports {@code available}+ * with nothing behind its endpoint. A cluster whose container cannot be brought back reports+ * {@code restore-failed} instead.+ *+ * <p>Starting a container waits for a readiness probe and may pull an image, so the work runs+ * in the background and must not delay emulator readiness. Clusters are marked+ * {@code creating} synchronously and flip to {@code available} or {@code restore-failed} as+ * each restore finishes. The cache comes back empty, as on any Floci restart.+ */+ public CompletableFuture<Void> restorePersistedRuntime() {+ List<CacheCluster> toRestore = new ArrayList<>();+ for (CacheCluster cluster : clusters.scan(k -> true)) {+ if (cluster.getCacheClusterStatus() == CacheClusterStatus.DELETING) {+ continue;+ }+ cluster.setCacheClusterStatus(CacheClusterStatus.CREATING);+ clusters.put(cluster.getCacheClusterId(), cluster);+ toRestore.add(cluster);+ }+ if (toRestore.isEmpty()) {+ return CompletableFuture.completedFuture(null);+ }+ LOG.infov("Restoring {0} Memcached cluster(s) in the background", String.valueOf(toRestore.size()));+ return CompletableFuture.runAsync(() -> toRestore.forEach(this::restoreCluster));+ }++ /**+ * The endpoint is rebuilt from the new container rather than replayed from the record: the+ * host port Docker publishes is chosen per run, so a restored cluster that kept its old+ * endpoint would advertise a port nothing listens on.+ *+ * <p>The container work runs outside the cluster's monitor, so a restore that may pull an+ * image does not hold a delete of the same cluster behind it. The write-back takes the+ * monitor and re-reads the record: a client that deleted the cluster in the first seconds+ * after boot, while it was still {@code creating}, must not see it come back+ * {@code available} with a fresh container behind it.+ */+ private void restoreCluster(CacheCluster cluster) {+ String clusterId = cluster.getCacheClusterId();+ String image = config.services().elasticache().defaultMemcachedImage();+ try {+ ElastiCacheContainerHandle handle = containerManager.tryStart(clusterId, image);+ synchronized (lockFor(clusterId)) {+ if (restoreTargetLost(clusterId)) {+ abandonRestoredContainer(clusterId, handle);+ return;+ }+ if (handle != null) {+ cluster.setContainerId(handle.getContainerId());+ cluster.setContainerHost(handle.getHost());+ cluster.setContainerPort(handle.getPort());+ cluster.setConfigurationEndpoint(+ new Endpoint(resolveEndpointHost(handle), handle.getPort()));+ } else {+ // Cleared rather than left alone: whatever the record carried describes a+ // container from the previous process, and nothing must read it as live.+ cluster.setContainerId(null);+ cluster.setContainerHost(null);+ cluster.setContainerPort(0);+ LOG.warnv("Memcached cluster {0} restored without a backing container: no Docker "+ + "daemon is reachable. Metadata operations work; connections to the cache "+ + "do not until a daemon appears.", clusterId);+ }+ cluster.setCacheClusterStatus(CacheClusterStatus.AVAILABLE);+ clusters.put(clusterId, cluster);+ LOG.infov("Restored Memcached cluster {0}, endpoint={1}:{2}", clusterId,+ cluster.getConfigurationEndpoint().address(),+ String.valueOf(cluster.getConfigurationEndpoint().port()));+ }+ } catch (RuntimeException e) {+ failRestore(cluster, e);+ }+ }++ /**+ * Reports a cluster whose container could not be brought back as {@code restore-failed}+ * without deleting the record: the cluster still exists on the control plane, it just has+ * nothing behind it. A cluster deleted while the failed attempt ran is not written back at+ * all, because that would resurrect it.+ */+ private void failRestore(CacheCluster cluster, RuntimeException cause) {+ String clusterId = cluster.getCacheClusterId();+ synchronized (lockFor(clusterId)) {+ if (restoreTargetLost(clusterId)) {+ LOG.warnv(cause, "Failed to restore Memcached cluster {0}, which was deleted while "+ + "it was being restored", clusterId);+ return;+ }+ cluster.setContainerId(null);+ cluster.setContainerHost(null);+ cluster.setContainerPort(0);+ cluster.setCacheClusterStatus(CacheClusterStatus.RESTORE_FAILED);+ cluster.setConfigurationEndpoint(null);+ try {+ clusters.put(clusterId, cluster);+ } catch (RuntimeException persistFailure) {+ cause.addSuppressed(persistFailure);+ }+ LOG.warnv(cause, "Failed to restore Memcached cluster {0}", clusterId);+ }+ }++ /**+ * Whether the record a restore is about to write back is still there to write back to.+ * Callers must hold the cluster's monitor: {@link #deleteCacheCluster} removed the record+ * under that same monitor, and putting the cluster back afterwards would resurrect one the+ * caller was told had been deleted.+ */+ private boolean restoreTargetLost(String clusterId) {+ return clusters.get(clusterId)+ .map(current -> current.getCacheClusterStatus() == CacheClusterStatus.DELETING)+ .orElse(true);+ }++ /** Drops the container a restore started for a cluster that was deleted while it ran. */+ private void abandonRestoredContainer(String clusterId, ElastiCacheContainerHandle handle) {+ if (handle != null) {+ try {+ containerManager.stop(handle);+ } catch (RuntimeException e) {+ LOG.warnv("Error stopping container for deleted Memcached cluster {0}: {1}",+ clusterId, e.getMessage());+ }+ }+ LOG.infov("Discarded the restored container for Memcached cluster {0}: it was deleted "+ + "while it was being restored", clusterId);+ }++ /**+ * One monitor per cluster id, taken by every writer that reads a record and writes it back.+ * Guarding the record itself would be too late: a restore that resolved its cluster before+ * locking would otherwise write back one a concurrent delete had already removed.+ */+ private Object lockFor(String clusterId) {+ return clusterLocks.computeIfAbsent(clusterId, key -> new Object());+ }+public CacheCluster getCacheCluster(String clusterId) {return clusters.get(clusterId).orElseThrow(() ->new AwsException("CacheClusterNotFound",@@ -95,20 +240,24 @@ public class ElastiCacheMemcachedService {}public CacheCluster deleteCacheCluster(String clusterId) {- CacheCluster cluster = getCacheCluster(clusterId);+ // one monitor per cluster for delete and restore: a restore's read-modify-write could+ // otherwise write the cluster back after this removed it+ synchronized (lockFor(clusterId)) {+ CacheCluster cluster = getCacheCluster(clusterId);- cluster.setCacheClusterStatus(CacheClusterStatus.DELETING);- clusters.put(clusterId, cluster);+ cluster.setCacheClusterStatus(CacheClusterStatus.DELETING);+ clusters.put(clusterId, cluster);- if (cluster.getContainerId() != null) {- containerManager.stop(new ElastiCacheContainerHandle(- cluster.getContainerId(), clusterId,- cluster.getContainerHost(), cluster.getContainerPort()));- }+ if (cluster.getContainerId() != null) {+ containerManager.stop(new ElastiCacheContainerHandle(+ cluster.getContainerId(), clusterId,+ cluster.getContainerHost(), cluster.getContainerPort()));+ }- clusters.delete(clusterId);- LOG.infov("Memcached cluster {0} deleted", clusterId);- return cluster;+ clusters.delete(clusterId);+ LOG.infov("Memcached cluster {0} deleted", clusterId);+ return cluster;+ }}private String resolveEndpointHost(ElastiCacheContainerHandle handle) {diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.javaindex 1a61f63e0f..be6c5fab58 100644--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.java@@ -737,7 +737,7 @@ private Response handleCreateCacheParameterGroup(MultivaluedMap<String, String>XmlBuilder xml = new XmlBuilder().start("CacheCluster").elem("CacheClusterId", c.getCacheClusterId())- .elem("CacheClusterStatus", c.getCacheClusterStatus().name().toLowerCase())+ .elem("CacheClusterStatus", c.getCacheClusterStatus().wireName()).elem("Engine", c.getEngine()).elem("EngineVersion", c.getEngineVersion());if (ep != null) {diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.javaindex 763ae168d7..3deacb3758 100644--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java@@ -506,53 +506,221 @@ public class ElastiCacheService implements ResourceProvider {}/**- * Brings the data plane back up for cluster-mode groups restored from disk. Invoked from+ * Brings the data plane back up for every group restored from disk. Invoked from* {@code EmulatorLifecycle} after {@code storageFactory.loadAll()}, alongside the other* services that restore persisted runtime. Only topology survives a restart — containers,* proxies and port reservations are process-local — so each group is re-provisioned from- * its persisted node list, and a group whose data plane cannot come back is reported- * {@code create-failed} instead of available. Cluster-mode-disabled groups are deliberately- * out of scope: they keep their pre-existing behaviour of reporting available after a- * restart without a restored runtime.+ * its persisted record, and a group whose data plane cannot come back is reported+ * {@code create-failed} instead of available.+ *+ * <p>A group that is not re-provisioned is worse than one that is reported failed. Its+ * status stays {@code available} with nothing behind the endpoint, so the control plane+ * answers healthy while every connection fails, and the proxy port it still advertises is+ * free for the next create to take — pointing the old endpoint at an unrelated cache rather+ * than failing cleanly.** <p>Container restarts and cluster formation can take a readiness timeout and a formation* timeout per group, so the data-plane work runs in the background and must not delay- * emulator readiness. Groups are marked {@code creating} synchronously — each node's proxy- * port is reserved at the same time, before any request can claim it — and flip to- * {@code available} or {@code create-failed} as their restoration finishes.+ * emulator readiness. Groups are marked {@code creating} synchronously — each proxy port is+ * reserved at the same time, before any request can claim it — and flip to {@code available}+ * or {@code create-failed} as their restoration finishes.*/public CompletableFuture<Void> restorePersistedRuntime() {- List<ReplicationGroup> toRestore = new ArrayList<>();+ List<ReplicationGroup> clusterModeToRestore = new ArrayList<>();+ List<ReplicationGroup> singleNodeToRestore = new ArrayList<>();for (ReplicationGroup group : groups.scan(k -> true)) {- if (!group.isClusterEnabled() || group.getClusterNodes().isEmpty()- || group.getStatus() == ReplicationGroupStatus.DELETING) {+ if (group.getStatus() == ReplicationGroupStatus.DELETING) {continue;}- List<Integer> reserved = new ArrayList<>();- try {- for (ClusterNode node : group.getClusterNodes()) {- node.setProxyPort(reserveOrAllocateProxyPort(node.getProxyPort()));- reserved.add(node.getProxyPort());- }- group.setStatus(ReplicationGroupStatus.CREATING);- toRestore.add(group);- } catch (RuntimeException e) {- reserved.forEach(this::releaseProxyPort);- group.setStatus(ReplicationGroupStatus.CREATE_FAILED);- group.setConfigurationEndpoint(null);- LOG.warnv(e, "Failed to restore cluster-mode replication group {0}",- group.getReplicationGroupId());+ if (group.isClusterEnabled() && !group.getClusterNodes().isEmpty()) {+ reserveClusterModeGroup(group, clusterModeToRestore);+ } else {+ reserveSingleNodeGroup(group, singleNodeToRestore);}groups.put(group.getReplicationGroupId(), group);}- if (toRestore.isEmpty()) {+ if (clusterModeToRestore.isEmpty() && singleNodeToRestore.isEmpty()) {return CompletableFuture.completedFuture(null);}- LOG.infov("Restoring {0} cluster-mode replication group(s) in the background",- String.valueOf(toRestore.size()));- return CompletableFuture.runAsync(() -> toRestore.forEach(this::restoreClusterModeGroup));+ LOG.infov("Restoring {0} cluster-mode and {1} single-node replication group(s) in the background",+ String.valueOf(clusterModeToRestore.size()), String.valueOf(singleNodeToRestore.size()));+ return CompletableFuture.runAsync(() -> {+ clusterModeToRestore.forEach(this::restoreClusterModeGroup);+ singleNodeToRestore.forEach(this::restoreSingleNodeGroup);+ });+ }++ private void reserveClusterModeGroup(ReplicationGroup group, List<ReplicationGroup> toRestore) {+ List<Integer> reserved = new ArrayList<>();+ try {+ for (ClusterNode node : group.getClusterNodes()) {+ node.setProxyPort(reserveOrAllocateProxyPort(node.getProxyPort()));+ reserved.add(node.getProxyPort());+ }+ group.setStatus(ReplicationGroupStatus.CREATING);+ toRestore.add(group);+ } catch (RuntimeException e) {+ reserved.forEach(this::releaseProxyPort);+ group.setStatus(ReplicationGroupStatus.CREATE_FAILED);+ group.setConfigurationEndpoint(null);+ LOG.warnv(e, "Failed to restore cluster-mode replication group {0}",+ group.getReplicationGroupId());+ }+ }++ /**+ * Reserves the port a cluster-mode-disabled group comes back advertising and marks it+ * {@code creating} for {@link #restoreSingleNodeGroup}. The endpoint host is re-derived from+ * configuration rather than replayed from the record, so a group restored under a changed+ * {@code FLOCI_HOSTNAME} advertises the name this process actually answers to.+ */+ private void reserveSingleNodeGroup(ReplicationGroup group, List<ReplicationGroup> toRestore) {+ try {+ group.setProxyPort(reserveOrAllocateProxyPort(group.getProxyPort()));+ group.setConfigurationEndpoint(new Endpoint(resolveEndpointHost(), group.getProxyPort()));+ group.setStatus(ReplicationGroupStatus.CREATING);+ toRestore.add(group);+ } catch (RuntimeException e) {+ group.setStatus(ReplicationGroupStatus.CREATE_FAILED);+ group.setConfigurationEndpoint(null);+ LOG.warnv(e, "Failed to restore replication group {0}", group.getReplicationGroupId());+ }+ }++ /**+ * Restarts the container and auth proxy behind a cluster-mode-disabled group, the same pair+ * {@link #provisionSingleNodeGroup} creates. The cache comes back empty, as it does for+ * cluster-mode groups: only the topology is persisted, never the keyspace.+ *+ * <p>Uses {@code tryStart} for the same reason the create path does: with no Docker daemon+ * reachable, the group keeps its metadata and reports available without a container, rather+ * than being failed for a daemon that may appear later. A failure raised while the daemon+ * <em>is</em> reachable is a genuine container problem and fails the group.+ *+ * <p>The container work runs outside the group's monitor, as it does on create: a restore+ * that may pull an image must not hold a delete of the same group behind it. The write-back+ * takes the monitor and re-reads the record, because a delete taken while the container+ * started has already removed it.+ */+ private void restoreSingleNodeGroup(ReplicationGroup group) {+ String groupId = group.getReplicationGroupId();+ String image = config.services().elasticache().defaultImage();+ try {+ ElastiCacheContainerHandle handle = containerManager.tryStart(groupId, image);+ synchronized (lockFor("rg:" + groupId)) {+ if (restoreTargetLost(groupId)) {+ abandonRestoredContainer(groupId, handle);+ return;+ }+ if (handle != null) {+ group.setContainerId(handle.getContainerId());+ group.setContainerHost(handle.getHost());+ group.setContainerPort(handle.getPort());+ proxyManager.startProxy(groupId, group.getAuthMode(), group.getProxyPort(),+ handle.getHost(), handle.getPort(),+ (username, password) -> validatePassword(groupId, username, password));+ } else {+ // Cleared rather than left alone: whatever the record carried describes a+ // container from the previous process, and nothing must read it as live.+ group.setContainerId(null);+ group.setContainerHost(null);+ group.setContainerPort(0);+ LOG.warnv("Replication group {0} restored without a backing cache container: no "+ + "Docker daemon is reachable. Metadata operations work; connections to "+ + "the cache do not until a daemon appears.", groupId);+ }+ group.setStatus(ReplicationGroupStatus.AVAILABLE);+ groups.put(groupId, group);+ LOG.infov("Restored replication group {0}, endpoint={1}:{2}", groupId,+ group.getConfigurationEndpoint().address(),+ String.valueOf(group.getProxyPort()));+ }+ } catch (RuntimeException e) {+ failSingleNodeRestore(group, e);+ }+ }++ /**+ * Whether the record a restore is about to write back is still there to write back to. Callers+ * must hold the group's monitor: a delete taken while the container work ran removed the record+ * under that same monitor, and a restore that put its group back afterwards would resurrect a+ * group the caller was told had been deleted, as {@code available}, with a fresh container+ * behind it.+ */+ private boolean restoreTargetLost(String groupId) {+ return groups.get(groupId)+ .map(current -> current.getStatus() == ReplicationGroupStatus.DELETING)+ .orElse(true);}+ /**+ * Drops the container a restore started for a group that was deleted while it ran. The proxy+ * port is deliberately not released: the delete that removed the record released it already,+ * and releasing it a second time would return a port a later create has since taken to the+ * pool, letting two groups be handed the same one.+ */+ private void abandonRestoredContainer(String groupId, ElastiCacheContainerHandle handle) {+ if (handle != null) {+ try {+ containerManager.stop(handle);+ } catch (RuntimeException e) {+ LOG.warnv("Error stopping container for deleted replication group {0}: {1}",+ groupId, e.getMessage());+ }+ }+ LOG.infov("Discarded the restored data plane for replication group {0}: it was deleted "+ + "while it was being restored", groupId);+ }++ /**+ * Reports a group whose data plane could not be brought back, without deleting the record:+ * the group still exists on the control plane, it just has nothing behind it. Tears down+ * whatever the attempt left running and frees the port, so the next create can use it rather+ * than leaking a reservation nothing serves.+ *+ * <p>A group deleted while the failed attempt ran gets the teardown but no write-back and no+ * port release: reporting {@code create-failed} would put a deleted group back, and the+ * delete released the port itself.+ */+ private void failSingleNodeRestore(ReplicationGroup group, RuntimeException cause) {+ String groupId = group.getReplicationGroupId();+ synchronized (lockFor("rg:" + groupId)) {+ try {+ proxyManager.stopProxy(groupId);+ } catch (RuntimeException e) {+ LOG.warnv("Error stopping proxy for replication group {0}: {1}", groupId, e.getMessage());+ }+ try {+ containerManager.stopByGroupId(groupId);+ } catch (RuntimeException e) {+ LOG.warnv("Error stopping container for replication group {0}: {1}", groupId, e.getMessage());+ }+ if (restoreTargetLost(groupId)) {+ LOG.warnv(cause, "Failed to restore replication group {0}, which was deleted while "+ + "it was being restored", groupId);+ return;+ }+ releaseProxyPort(group.getProxyPort());+ group.setContainerId(null);+ group.setContainerHost(null);+ group.setContainerPort(0);+ group.setStatus(ReplicationGroupStatus.CREATE_FAILED);+ group.setConfigurationEndpoint(null);+ try {+ groups.put(groupId, group);+ } catch (RuntimeException persistFailure) {+ cause.addSuppressed(persistFailure);+ }+ LOG.warnv(cause, "Failed to restore replication group {0}", groupId);+ }+ }++ /**+ * Restarts the containers, cluster formation and proxies behind a cluster-mode group. Like+ * {@link #restoreSingleNodeGroup}, the container work runs outside the group's monitor and+ * only the write-back takes it, so a delete that ran in the meantime is seen before the+ * group is put back.+ */private void restoreClusterModeGroup(ReplicationGroup group) {String groupId = group.getReplicationGroupId();String image = config.services().elasticache().defaultImage();@@ -584,24 +752,57 @@ public class ElastiCacheService implements ResourceProvider {clusterFormation.form(groupId, formationNodes, group.getNumNodeGroups());- for (int i = 0; i < nodes.size(); i++) {- ClusterNode node = nodes.get(i);- ElastiCacheContainerHandle handle = handles.get(i);- proxyManager.startProxy(node.getMemberClusterId(), group.getAuthMode(), node.getProxyPort(),- handle.getHost(), handle.getPort(),- (username, password) -> validatePassword(groupId, username, password));- startedProxyKeys.add(node.getMemberClusterId());- }+ synchronized (lockFor("rg:" + groupId)) {+ if (restoreTargetLost(groupId)) {+ // No ports: the delete that removed the record released every one of them.+ rollbackClusterModeGroup(groupId, startedProxyKeys, handles, inFlightMemberId,+ List.of());+ LOG.infov("Discarded the restored data plane for cluster-mode replication group "+ + "{0}: it was deleted while it was being restored", groupId);+ return;+ }+ for (int i = 0; i < nodes.size(); i++) {+ ClusterNode node = nodes.get(i);+ ElastiCacheContainerHandle handle = handles.get(i);+ proxyManager.startProxy(node.getMemberClusterId(), group.getAuthMode(), node.getProxyPort(),+ handle.getHost(), handle.getPort(),+ (username, password) -> validatePassword(groupId, username, password));+ startedProxyKeys.add(node.getMemberClusterId());+ }- group.setConfigurationEndpoint(new Endpoint(endpointHost, nodes.getFirst().getProxyPort()));- group.setStatus(ReplicationGroupStatus.AVAILABLE);- groups.put(groupId, group);- LOG.infov("Restored cluster-mode replication group {0}: {1} node(s), configuration endpoint={2}:{3}",- groupId, String.valueOf(nodes.size()), endpointHost,- String.valueOf(group.getConfigurationEndpoint().port()));+ group.setConfigurationEndpoint(new Endpoint(endpointHost, nodes.getFirst().getProxyPort()));+ group.setStatus(ReplicationGroupStatus.AVAILABLE);+ groups.put(groupId, group);+ LOG.infov("Restored cluster-mode replication group {0}: {1} node(s), configuration endpoint={2}:{3}",+ groupId, String.valueOf(nodes.size()), endpointHost,+ String.valueOf(group.getConfigurationEndpoint().port()));+ }} catch (RuntimeException e) {- rollbackClusterModeGroup(groupId, startedProxyKeys, handles, inFlightMemberId, reservedPorts);- for (ClusterNode node : nodes) {+ failClusterModeRestore(group, startedProxyKeys, handles, inFlightMemberId, reservedPorts, e);+ }+ }++ /**+ * Reports a cluster-mode group whose data plane could not be brought back, mirroring+ * {@link #failSingleNodeRestore}: the record stays, reporting {@code create-failed} with no+ * endpoint, unless a delete removed it while the attempt ran. In that case the rollback still+ * runs but the ports are left to the delete that released them, and nothing is written back.+ */+ private void failClusterModeRestore(ReplicationGroup group, List<String> startedProxyKeys,+ List<ElastiCacheContainerHandle> handles,+ String inFlightMemberId, Collection<Integer> reservedPorts,+ RuntimeException cause) {+ String groupId = group.getReplicationGroupId();+ synchronized (lockFor("rg:" + groupId)) {+ boolean lost = restoreTargetLost(groupId);+ rollbackClusterModeGroup(groupId, startedProxyKeys, handles, inFlightMemberId,+ lost ? List.of() : reservedPorts);+ if (lost) {+ LOG.warnv(cause, "Failed to restore cluster-mode replication group {0}, which was "+ + "deleted while it was being restored", groupId);+ return;+ }+ for (ClusterNode node : group.getClusterNodes()) {node.setContainerId(null);node.setContainerHost(null);node.setContainerPort(0);@@ -611,9 +812,9 @@ public class ElastiCacheService implements ResourceProvider {try {groups.put(groupId, group);} catch (RuntimeException persistFailure) {- e.addSuppressed(persistFailure);+ cause.addSuppressed(persistFailure);}- LOG.warnv(e, "Failed to restore cluster-mode replication group {0}", groupId);+ LOG.warnv(cause, "Failed to restore cluster-mode replication group {0}", groupId);}}diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/model/CacheClusterStatus.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/model/CacheClusterStatus.javaindex 80db4647eb..39611e8ce2 100644--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/model/CacheClusterStatus.java+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/model/CacheClusterStatus.java@@ -2,7 +2,14 @@ package io.github.hectorvent.floci.services.elasticache.model;import io.quarkus.runtime.annotations.RegisterForReflection;+import java.util.Locale;+@RegisterForReflectionpublic enum CacheClusterStatus {- AVAILABLE, CREATING, DELETING+ AVAILABLE, CREATING, DELETING, RESTORE_FAILED;++ /** The status string as AWS reports it, e.g. {@code restore-failed}. */+ public String wireName() {+ return name().toLowerCase(Locale.ROOT).replace('_', '-');+ }}diff --git a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.javaindex f72e6585c5..4a7d6ee7cf 100644--- a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java+++ b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java@@ -5,13 +5,13 @@ import io.github.hectorvent.floci.core.common.ContainerTeardown;import io.github.hectorvent.floci.core.common.ServiceRegistry;import io.github.hectorvent.floci.core.storage.PersistentPathValidator;import io.github.hectorvent.floci.core.storage.StorageFactory;-import io.github.hectorvent.floci.lifecycle.InitLifecycleState;import io.github.hectorvent.floci.lifecycle.inithook.InitializationHook;import io.github.hectorvent.floci.lifecycle.inithook.InitializationHooksRunner;import io.github.hectorvent.floci.services.appsync.graphql.SchemaCreationWorker;import io.github.hectorvent.floci.services.docdb.container.DocDbContainerManager;import io.github.hectorvent.floci.services.ec2.Ec2MetadataServer;import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager;+import io.github.hectorvent.floci.services.elasticache.ElastiCacheMemcachedService;import io.github.hectorvent.floci.services.elasticache.ElastiCacheService;import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerManager;import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheMemcachedContainerManager;@@ -39,6 +39,7 @@ import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.DisplayName;import org.junit.jupiter.api.Test;import org.junit.jupiter.api.extension.ExtendWith;+import org.mockito.InOrder;import org.mockito.Mock;import org.mockito.Mockito;import org.mockito.junit.jupiter.MockitoExtension;@@ -73,6 +74,7 @@ class EmulatorLifecycleTest {@Mock private IamService iamService;@Mock private EmulatorConfig.ElastiCacheServiceConfig elastiCacheServiceConfig;@Mock private ElastiCacheService elastiCacheService;+ @Mock private ElastiCacheMemcachedService elastiCacheMemcachedService;@Mock private ElastiCacheContainerManager elastiCacheContainerManager;@Mock private ElastiCacheMemcachedContainerManager elastiCacheMemcachedContainerManager;@Mock private ElastiCacheProxyManager elastiCacheProxyManager;@@ -131,7 +133,7 @@ class EmulatorLifecycleTest {emulatorLifecycle = new EmulatorLifecycle(storageFactory, serviceRegistry, config,- iamService, elastiCacheService,+ iamService, elastiCacheService, elastiCacheMemcachedService,elastiCacheContainerManager, elastiCacheMemcachedContainerManager,elastiCacheProxyManager, rdsContainerManager, rdsProxyManager,memoryDbContainerManager, memoryDbProxyManager,@@ -206,15 +208,18 @@ class EmulatorLifecycleTest {stubStorageConfig();when(elastiCacheServiceConfig.enabled()).thenReturn(true);when(elastiCacheService.restorePersistedRuntime())- .thenReturn(java.util.concurrent.CompletableFuture.completedFuture(null));+ .thenReturn(CompletableFuture.completedFuture(null));+ when(elastiCacheMemcachedService.restorePersistedRuntime())+ .thenReturn(CompletableFuture.completedFuture(null));when(initializationHooksRunner.hasHooks(InitializationHook.START)).thenReturn(false);when(initializationHooksRunner.hasHooks(InitializationHook.READY)).thenReturn(false);emulatorLifecycle.onStart(Mockito.mock(StartupEvent.class));- var inOrder = Mockito.inOrder(storageFactory, elastiCacheService);+ InOrder inOrder = Mockito.inOrder(storageFactory, elastiCacheService, elastiCacheMemcachedService);inOrder.verify(storageFactory).loadAll();inOrder.verify(elastiCacheService).restorePersistedRuntime();+ inOrder.verify(elastiCacheMemcachedService).restorePersistedRuntime();}@Test@@ -227,6 +232,7 @@ class EmulatorLifecycleTest {emulatorLifecycle.onStart(Mockito.mock(StartupEvent.class));Mockito.verify(elastiCacheService, Mockito.never()).restorePersistedRuntime();+ Mockito.verify(elastiCacheMemcachedService, Mockito.never()).restorePersistedRuntime();}@Testdiff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedServiceTest.javaindex 3ab63226f8..478c279a81 100644--- a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedServiceTest.java@@ -12,12 +12,16 @@ import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import java.util.Collection;+import java.util.Map;import java.util.Optional;+import java.util.concurrent.ConcurrentHashMap;import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNull;import static org.junit.jupiter.api.Assertions.assertThrows;import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.eq;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.never;import static org.mockito.Mockito.verify;@@ -144,4 +148,94 @@ class ElastiCacheMemcachedServiceTest {service.deleteCacheCluster("no-docker-cluster");verify(containerManager, never()).stop(any());}++ @Test+ void restorePersistedRuntimeRestartsTheContainerAndRepointsTheEndpoint() {+ StorageFactory storageFactory = sharedStorageFactory();+ ElastiCacheMemcachedContainerManager beforeRestart = mock(ElastiCacheMemcachedContainerManager.class);+ when(beforeRestart.tryStart(anyString(), anyString()))+ .thenReturn(new ElastiCacheContainerHandle("cid", "my-cluster", "localhost", 32770));+ serviceWith(storageFactory, beforeRestart).createCacheCluster("my-cluster");++ ElastiCacheMemcachedContainerManager restarted = mock(ElastiCacheMemcachedContainerManager.class);+ when(restarted.tryStart(anyString(), anyString()))+ .thenReturn(new ElastiCacheContainerHandle("cid2", "my-cluster", "localhost", 32771));+ ElastiCacheMemcachedService restartedService = serviceWith(storageFactory, restarted);++ restartedService.restorePersistedRuntime().join();++ verify(restarted).tryStart(eq("my-cluster"), anyString());+ CacheCluster cluster = restartedService.getCacheCluster("my-cluster");+ assertEquals(CacheClusterStatus.AVAILABLE, cluster.getCacheClusterStatus());+ assertEquals(32771, cluster.getConfigurationEndpoint().port(),+ "Docker publishes a fresh host port per run, so the endpoint must follow it");+ }++ @Test+ void memcachedRestoreFailureReportsRestoreFailed() {+ StorageFactory storageFactory = sharedStorageFactory();+ ElastiCacheMemcachedContainerManager beforeRestart = mock(ElastiCacheMemcachedContainerManager.class);+ when(beforeRestart.tryStart(anyString(), anyString()))+ .thenReturn(new ElastiCacheContainerHandle("cid", "my-cluster", "localhost", 11211));+ serviceWith(storageFactory, beforeRestart).createCacheCluster("my-cluster");++ ElastiCacheMemcachedContainerManager restarted = mock(ElastiCacheMemcachedContainerManager.class);+ when(restarted.tryStart(anyString(), anyString()))+ .thenThrow(new RuntimeException("container failed"));+ ElastiCacheMemcachedService restartedService = serviceWith(storageFactory, restarted);++ restartedService.restorePersistedRuntime().join();++ CacheCluster cluster = restartedService.getCacheCluster("my-cluster");+ assertEquals(CacheClusterStatus.RESTORE_FAILED, cluster.getCacheClusterStatus());+ assertNull(cluster.getConfigurationEndpoint(),+ "A cluster whose container is gone must not advertise an endpoint");+ }++ @Test+ void restoreDoesNotResurrectAClusterDeletedWhileItWasRestoring() {+ StorageFactory storageFactory = sharedStorageFactory();+ ElastiCacheMemcachedContainerManager beforeRestart = mock(ElastiCacheMemcachedContainerManager.class);+ when(beforeRestart.tryStart(anyString(), anyString()))+ .thenReturn(new ElastiCacheContainerHandle("cid", "my-cluster", "localhost", 32770));+ serviceWith(storageFactory, beforeRestart).createCacheCluster("my-cluster");++ ElastiCacheMemcachedContainerManager restarted = mock(ElastiCacheMemcachedContainerManager.class);+ ElastiCacheMemcachedService restartedService = serviceWith(storageFactory, restarted);+ ElastiCacheContainerHandle restoredHandle =+ new ElastiCacheContainerHandle("cid2", "my-cluster", "localhost", 32771);+ // The delete lands in the window the cluster's monitor closes: the container is up, the+ // record has not been written back yet.+ when(restarted.tryStart(anyString(), anyString())).thenAnswer(inv -> {+ restartedService.deleteCacheCluster("my-cluster");+ return restoredHandle;+ });++ restartedService.restorePersistedRuntime().join();++ assertThrows(AwsException.class, () -> restartedService.getCacheCluster("my-cluster"),+ "A cluster deleted while it was restoring must stay deleted");+ verify(restarted).stop(restoredHandle);+ }++ private static StorageFactory sharedStorageFactory() {+ StorageFactory storageFactory = mock(StorageFactory.class);+ Map<String, Object> backends = new ConcurrentHashMap<>();+ when(storageFactory.create(anyString(), anyString(), any())).thenAnswer(inv ->+ backends.computeIfAbsent(inv.getArgument(1, String.class),+ key -> AccountAwareStorageBackend.inMemory("000000000000")));+ return storageFactory;+ }++ private static ElastiCacheMemcachedService serviceWith(StorageFactory storageFactory,+ ElastiCacheMemcachedContainerManager containerManager) {+ EmulatorConfig config = mock(EmulatorConfig.class);+ EmulatorConfig.ServicesConfig servicesConfig = mock(EmulatorConfig.ServicesConfig.class);+ EmulatorConfig.ElastiCacheServiceConfig ecConfig = mock(EmulatorConfig.ElastiCacheServiceConfig.class);+ when(config.services()).thenReturn(servicesConfig);+ when(servicesConfig.elasticache()).thenReturn(ecConfig);+ when(ecConfig.defaultMemcachedImage()).thenReturn("memcached:1.6");+ when(config.hostname()).thenReturn(Optional.of("localhost"));+ return new ElastiCacheMemcachedService(containerManager, storageFactory, config);+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.javaindex a962e20b15..f1b15e8143 100644--- a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java@@ -485,6 +485,148 @@ class ElastiCacheServiceTest {.thenReturn(new ElastiCacheContainerHandle("cid", "grp", "localhost", 6379));}+ private static void stubSingleNodeContainer(ElastiCacheContainerManager containerManager) {+ when(containerManager.tryStart(anyString(), anyString())).thenAnswer(inv ->+ new ElastiCacheContainerHandle("cid-" + inv.getArgument(0, String.class),+ inv.getArgument(0, String.class), "localhost", 6379));+ }++ private static StorageFactory storageWithSingleNodeGroup(String groupId) {+ StorageFactory storageFactory = sharedStorageFactory();+ ElastiCacheContainerManager beforeRestart = mock(ElastiCacheContainerManager.class);+ stubSingleNodeContainer(beforeRestart);+ serviceWith(storageFactory, beforeRestart, mock(ElastiCacheProxyManager.class),+ mock(ValkeyClusterFormation.class))+ .createReplicationGroup(groupId, "test", AuthMode.PASSWORD, null, "us-east-1");+ return storageFactory;+ }++ @Test+ void restorePersistedRuntimeReprovisionsSingleNodeGroups() {+ StorageFactory storageFactory = storageWithSingleNodeGroup("grp");++ ElastiCacheContainerManager restartedContainers = mock(ElastiCacheContainerManager.class);+ stubSingleNodeContainer(restartedContainers);+ ElastiCacheProxyManager restartedProxies = mock(ElastiCacheProxyManager.class);+ ElastiCacheService restarted = serviceWith(storageFactory, restartedContainers,+ restartedProxies, mock(ValkeyClusterFormation.class));++ restarted.restorePersistedRuntime().join();++ verify(restartedContainers).tryStart(eq("grp"), anyString());+ verify(restartedProxies).startProxy(eq("grp"), eq(AuthMode.PASSWORD), eq(16379),+ eq("localhost"), eq(6379), any());+ ReplicationGroup restored = restarted.getReplicationGroup("grp");+ assertEquals(ReplicationGroupStatus.AVAILABLE, restored.getStatus());+ assertEquals(16379, restored.getConfigurationEndpoint().port());+ assertEquals("cid-grp", restored.getContainerId(),+ "A restored group must track the container it actually has");++ ReplicationGroup next =+ restarted.createReplicationGroup("grp2", "test", AuthMode.NO_AUTH, null, "us-east-1");+ assertEquals(16380, next.getProxyPort(),+ "A restored group's port must be reserved again so new groups cannot take it");+ }++ @Test+ void singleNodeRestoreFailureReportsCreateFailedAndReleasesThePort() {+ StorageFactory storageFactory = storageWithSingleNodeGroup("grp");++ ElastiCacheContainerManager restartedContainers = mock(ElastiCacheContainerManager.class);+ // Only the restore fails: the create that checks the port was freed must still get through.+ when(restartedContainers.tryStart(eq("grp"), anyString()))+ .thenThrow(new RuntimeException("container failed"));+ ElastiCacheProxyManager restartedProxies = mock(ElastiCacheProxyManager.class);+ ElastiCacheService restarted = serviceWith(storageFactory, restartedContainers,+ restartedProxies, mock(ValkeyClusterFormation.class));++ restarted.restorePersistedRuntime().join();++ ReplicationGroup failed = restarted.getReplicationGroup("grp");+ assertEquals(ReplicationGroupStatus.CREATE_FAILED, failed.getStatus());+ assertNull(failed.getConfigurationEndpoint(),+ "A group whose data plane is gone must not advertise an endpoint");+ verify(restartedProxies, never()).startProxy(anyString(), any(), anyInt(), anyString(), anyInt(), any());++ ReplicationGroup next =+ restarted.createReplicationGroup("grp2", "test", AuthMode.NO_AUTH, null, "us-east-1");+ assertEquals(16379, next.getProxyPort(),+ "The failed restore's port must be released for the next group");+ }++ @Test+ void singleNodeRestoreWithoutADockerDaemonKeepsTheGroupAvailable() {+ StorageFactory storageFactory = storageWithSingleNodeGroup("grp");++ ElastiCacheContainerManager restartedContainers = mock(ElastiCacheContainerManager.class);+ when(restartedContainers.tryStart(anyString(), anyString())).thenReturn(null);+ ElastiCacheProxyManager restartedProxies = mock(ElastiCacheProxyManager.class);+ ElastiCacheService restarted = serviceWith(storageFactory, restartedContainers,+ restartedProxies, mock(ValkeyClusterFormation.class));++ restarted.restorePersistedRuntime().join();++ ReplicationGroup restored = restarted.getReplicationGroup("grp");+ assertEquals(ReplicationGroupStatus.AVAILABLE, restored.getStatus(),+ "No reachable daemon is the create path's documented degraded mode, not a failure");+ assertNull(restored.getContainerId());+ verify(restartedProxies, never()).startProxy(anyString(), any(), anyInt(), anyString(), anyInt(), any());+ }++ @Test+ void restoreDoesNotResurrectAGroupDeletedWhileItWasRestoring() {+ StorageFactory storageFactory = storageWithSingleNodeGroup("grp");++ ElastiCacheContainerManager restartedContainers = mock(ElastiCacheContainerManager.class);+ ElastiCacheProxyManager restartedProxies = mock(ElastiCacheProxyManager.class);+ ElastiCacheService restarted = serviceWith(storageFactory, restartedContainers,+ restartedProxies, mock(ValkeyClusterFormation.class));+ ElastiCacheContainerHandle restoredHandle =+ new ElastiCacheContainerHandle("cid-grp-restored", "grp", "localhost", 6379);+ // The delete lands in the window the group's monitor closes: the container is up, the+ // record has not been written back yet.+ when(restartedContainers.tryStart(eq("grp"), anyString())).thenAnswer(inv -> {+ restarted.deleteReplicationGroup("grp");+ // Takes the port that delete just freed, so a restore that released it a second+ // time would hand the same port out twice.+ restarted.createReplicationGroup("grp2", "test", AuthMode.NO_AUTH, null, "us-east-1");+ return restoredHandle;+ });++ restarted.restorePersistedRuntime().join();++ assertThrows(AwsException.class, () -> restarted.getReplicationGroup("grp"),+ "A group deleted while it was restoring must stay deleted");+ verify(restartedProxies, never()).startProxy(eq("grp"), any(), anyInt(), anyString(), anyInt(), any());+ verify(restartedContainers).stop(restoredHandle);++ ReplicationGroup next =+ restarted.createReplicationGroup("grp3", "test", AuthMode.NO_AUTH, null, "us-east-1");+ assertEquals(16380, next.getProxyPort(),+ "The abandoned restore must leave grp2 holding the port the delete released");+ }++ @Test+ void restorePersistedRuntimeSkipsGroupsBeingDeleted() {+ StorageFactory storageFactory = storageWithSingleNodeGroup("grp");+ ElastiCacheContainerManager beforeRestart = mock(ElastiCacheContainerManager.class);+ stubSingleNodeContainer(beforeRestart);+ ElastiCacheService before = serviceWith(storageFactory, beforeRestart,+ mock(ElastiCacheProxyManager.class), mock(ValkeyClusterFormation.class));+ // The in-memory backend hands back the stored instance, so this is the persisted record.+ before.getReplicationGroup("grp").setStatus(ReplicationGroupStatus.DELETING);++ ElastiCacheContainerManager restartedContainers = mock(ElastiCacheContainerManager.class);+ stubSingleNodeContainer(restartedContainers);+ ElastiCacheService restarted = serviceWith(storageFactory, restartedContainers,+ mock(ElastiCacheProxyManager.class), mock(ValkeyClusterFormation.class));++ restarted.restorePersistedRuntime().join();++ verify(restartedContainers, never()).tryStart(anyString(), anyString());+ assertEquals(ReplicationGroupStatus.DELETING, restarted.getReplicationGroup("grp").getStatus());+ }+@Testvoid restorePersistedRuntimeReprovisionsClusterModeGroups() {StorageFactory storageFactory = sharedStorageFactory();