Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(elasticache): re-provision caches on restart instead of reporting a dead endpoint as available (#4094)

Abhigya14d13ee798b8
diff --git a/docs/services/elasticache.md b/docs/services/elasticache.md
index c3785584bd..eb14c34d8e 100644
--- a/docs/services/elasticache.md
+++ b/docs/services/elasticache.md
@@ -60,14 +60,34 @@ case — the shipped `docker-compose.yml` uses `localhost.floci.io`, which publi
Floci's embedded DNS resolve it to the Floci container from inside Docker. Cluster-mode groups
then announce that name and report it as their `ConfigurationEndpoint`.
-With `persistent`, `hybrid` or `wal` storage, cluster-mode groups are re-provisioned from their
-persisted topology on startup: containers are restarted, the cluster is re-formed (caches restart
-empty, as on any Floci restart) and each node's proxy port is re-reserved. Ports are re-reserved
-and groups marked `creating` before Floci reports ready; the container restarts and cluster
-formation run in the background so a slow Docker daemon cannot delay readiness, and each group
-flips to `available` once its data plane is back. A group whose data plane cannot be brought back
-is reported with status `create-failed` instead of `available`, and its member clusters answer
-`DescribeCacheClusters` with `restore-failed` (`CacheClusterStatus` has no `create-failed` value).
+With `persistent`, `hybrid` or `wal` storage, every replication group and Memcached cluster is
+re-provisioned from its persisted record on startup: containers are restarted, cluster-mode groups
+are re-formed, and proxy ports are re-reserved. Caches restart empty, as on any Floci restart:
+only the topology is persisted, never the keyspace.
+
+Ports are re-reserved and records marked `creating` before Floci reports ready; the container
+restarts and cluster formation run in the background so a slow Docker daemon cannot delay
+readiness, and each record flips to `available` once its data plane is back. A replication group
+whose data plane cannot be brought back is reported with status `create-failed` instead of
+`available`, and its member clusters answer `DescribeCacheClusters` with `restore-failed`
+(`CacheClusterStatus` has no `create-failed` value). A Memcached cluster that cannot be brought
+back reports `restore-failed` directly.
+
+Reporting a failed record as failed matters more than it looks. A record that is left unreconciled
+keeps its `available` status with nothing behind the endpoint, so the control plane answers healthy
+while every connection fails, and the proxy port it still advertises is free for the next create to
+take: the old endpoint then reaches an unrelated cache rather than failing cleanly.
+
+A Memcached cluster's endpoint follows its new container. Outside Docker, Floci publishes the
+backend on a host port Docker picks per run, so a restored cluster's `ConfigurationEndpoint` port
+can differ from the one it had before the restart. Replication groups keep their port: it is a
+proxy port Floci owns and re-reserves.
+
+A delete that arrives while a record is still `creating` wins. The restore takes the same
+per-record monitor `DeleteReplicationGroup` and `DeleteCacheCluster` take, and skips its write-back
+when the record is gone, so a group or cluster deleted in the first seconds after boot stays
+deleted rather than coming back `available`. Any container the abandoned restore had already
+started is stopped.
`DescribeReplicationGroups` reports the topology honestly: `ClusterEnabled`, one `NodeGroup` per
shard with its `Slots`, `NodeGroupMembers`, and `MemberClusters`. Each member also answers
diff --git a/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java b/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java
index 20670f2f55..096f379f8d 100644
--- a/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java
+++ b/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java
@@ -14,6 +14,7 @@ import io.github.hectorvent.floci.services.floci.ui.FlociUiManager;
import io.github.hectorvent.floci.services.amazonmq.container.RabbitMqManager;
import io.github.hectorvent.floci.services.kinesisanalytics.container.FlinkContainerManager;
import io.github.hectorvent.floci.services.iam.IamService;
+import io.github.hectorvent.floci.services.elasticache.ElastiCacheMemcachedService;
import io.github.hectorvent.floci.services.elasticache.ElastiCacheService;
import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerManager;
import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheMemcachedContainerManager;
@@ -73,6 +74,7 @@ public class EmulatorLifecycle {
private final EmulatorConfig config;
private final IamService iamService;
private final ElastiCacheService elastiCacheService;
+ private final ElastiCacheMemcachedService elastiCacheMemcachedService;
private final ElastiCacheContainerManager elastiCacheContainerManager;
private final ElastiCacheMemcachedContainerManager elastiCacheMemcachedContainerManager;
private final ElastiCacheProxyManager elastiCacheProxyManager;
@@ -108,6 +110,7 @@ public class EmulatorLifecycle {
EmulatorConfig config,
IamService iamService,
ElastiCacheService elastiCacheService,
+ ElastiCacheMemcachedService elastiCacheMemcachedService,
ElastiCacheContainerManager elastiCacheContainerManager,
ElastiCacheMemcachedContainerManager elastiCacheMemcachedContainerManager,
ElastiCacheProxyManager elastiCacheProxyManager,
@@ -142,6 +145,7 @@ public class EmulatorLifecycle {
this.config = config;
this.iamService = iamService;
this.elastiCacheService = elastiCacheService;
+ this.elastiCacheMemcachedService = elastiCacheMemcachedService;
this.elastiCacheContainerManager = elastiCacheContainerManager;
this.elastiCacheMemcachedContainerManager = elastiCacheMemcachedContainerManager;
this.elastiCacheProxyManager = elastiCacheProxyManager;
@@ -222,7 +226,11 @@ public class EmulatorLifecycle {
}
if (config.services().elasticache().enabled()) {
elastiCacheService.restorePersistedRuntime().exceptionally(ex -> {
- LOG.warnv("ElastiCache cluster-mode restore failed: {0}", ex.getMessage());
+ LOG.warnv("ElastiCache replication group restore failed: {0}", ex.getMessage());
+ return null;
+ });
+ elastiCacheMemcachedService.restorePersistedRuntime().exceptionally(ex -> {
+ LOG.warnv("ElastiCache Memcached restore failed: {0}", ex.getMessage());
return null;
});
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedService.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedService.java
index b7bf3d6baf..5b415827a1 100644
--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedService.java
@@ -15,9 +15,12 @@ import jakarta.inject.Inject;
import org.jboss.logging.Logger;
import java.time.Instant;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
@ApplicationScoped
public class ElastiCacheMemcachedService {
@@ -31,6 +34,7 @@ public class ElastiCacheMemcachedService {
private final StorageBackend<String, CacheCluster> clusters;
private final ElastiCacheMemcachedContainerManager containerManager;
private final EmulatorConfig config;
+ private final ConcurrentHashMap<String, Object> clusterLocks = new ConcurrentHashMap<>();
@Inject
public ElastiCacheMemcachedService(ElastiCacheMemcachedContainerManager containerManager,
@@ -78,6 +82,147 @@ public class ElastiCacheMemcachedService {
return cluster;
}
+ /**
+ * Restarts the container behind every cache cluster restored from disk. Invoked from
+ * {@code EmulatorLifecycle} after {@code storageFactory.loadAll()}, for the same reason
+ * {@code ElastiCacheService.restorePersistedRuntime} exists: only the record is persisted,
+ * the container is process-local, and a cluster left unreconciled reports {@code available}
+ * with nothing behind its endpoint. A cluster whose container cannot be brought back reports
+ * {@code restore-failed} instead.
+ *
+ * <p>Starting a container waits for a readiness probe and may pull an image, so the work runs
+ * in the background and must not delay emulator readiness. Clusters are marked
+ * {@code creating} synchronously and flip to {@code available} or {@code restore-failed} as
+ * each restore finishes. The cache comes back empty, as on any Floci restart.
+ */
+ public CompletableFuture<Void> restorePersistedRuntime() {
+ List<CacheCluster> toRestore = new ArrayList<>();
+ for (CacheCluster cluster : clusters.scan(k -> true)) {
+ if (cluster.getCacheClusterStatus() == CacheClusterStatus.DELETING) {
+ continue;
+ }
+ cluster.setCacheClusterStatus(CacheClusterStatus.CREATING);
+ clusters.put(cluster.getCacheClusterId(), cluster);
+ toRestore.add(cluster);
+ }
+ if (toRestore.isEmpty()) {
+ return CompletableFuture.completedFuture(null);
+ }
+ LOG.infov("Restoring {0} Memcached cluster(s) in the background", String.valueOf(toRestore.size()));
+ return CompletableFuture.runAsync(() -> toRestore.forEach(this::restoreCluster));
+ }
+
+ /**
+ * The endpoint is rebuilt from the new container rather than replayed from the record: the
+ * host port Docker publishes is chosen per run, so a restored cluster that kept its old
+ * endpoint would advertise a port nothing listens on.
+ *
+ * <p>The container work runs outside the cluster's monitor, so a restore that may pull an
+ * image does not hold a delete of the same cluster behind it. The write-back takes the
+ * monitor and re-reads the record: a client that deleted the cluster in the first seconds
+ * after boot, while it was still {@code creating}, must not see it come back
+ * {@code available} with a fresh container behind it.
+ */
+ private void restoreCluster(CacheCluster cluster) {
+ String clusterId = cluster.getCacheClusterId();
+ String image = config.services().elasticache().defaultMemcachedImage();
+ try {
+ ElastiCacheContainerHandle handle = containerManager.tryStart(clusterId, image);
+ synchronized (lockFor(clusterId)) {
+ if (restoreTargetLost(clusterId)) {
+ abandonRestoredContainer(clusterId, handle);
+ return;
+ }
+ if (handle != null) {
+ cluster.setContainerId(handle.getContainerId());
+ cluster.setContainerHost(handle.getHost());
+ cluster.setContainerPort(handle.getPort());
+ cluster.setConfigurationEndpoint(
+ new Endpoint(resolveEndpointHost(handle), handle.getPort()));
+ } else {
+ // Cleared rather than left alone: whatever the record carried describes a
+ // container from the previous process, and nothing must read it as live.
+ cluster.setContainerId(null);
+ cluster.setContainerHost(null);
+ cluster.setContainerPort(0);
+ LOG.warnv("Memcached cluster {0} restored without a backing container: no Docker "
+ + "daemon is reachable. Metadata operations work; connections to the cache "
+ + "do not until a daemon appears.", clusterId);
+ }
+ cluster.setCacheClusterStatus(CacheClusterStatus.AVAILABLE);
+ clusters.put(clusterId, cluster);
+ LOG.infov("Restored Memcached cluster {0}, endpoint={1}:{2}", clusterId,
+ cluster.getConfigurationEndpoint().address(),
+ String.valueOf(cluster.getConfigurationEndpoint().port()));
+ }
+ } catch (RuntimeException e) {
+ failRestore(cluster, e);
+ }
+ }
+
+ /**
+ * Reports a cluster whose container could not be brought back as {@code restore-failed}
+ * without deleting the record: the cluster still exists on the control plane, it just has
+ * nothing behind it. A cluster deleted while the failed attempt ran is not written back at
+ * all, because that would resurrect it.
+ */
+ private void failRestore(CacheCluster cluster, RuntimeException cause) {
+ String clusterId = cluster.getCacheClusterId();
+ synchronized (lockFor(clusterId)) {
+ if (restoreTargetLost(clusterId)) {
+ LOG.warnv(cause, "Failed to restore Memcached cluster {0}, which was deleted while "
+ + "it was being restored", clusterId);
+ return;
+ }
+ cluster.setContainerId(null);
+ cluster.setContainerHost(null);
+ cluster.setContainerPort(0);
+ cluster.setCacheClusterStatus(CacheClusterStatus.RESTORE_FAILED);
+ cluster.setConfigurationEndpoint(null);
+ try {
+ clusters.put(clusterId, cluster);
+ } catch (RuntimeException persistFailure) {
+ cause.addSuppressed(persistFailure);
+ }
+ LOG.warnv(cause, "Failed to restore Memcached cluster {0}", clusterId);
+ }
+ }
+
+ /**
+ * Whether the record a restore is about to write back is still there to write back to.
+ * Callers must hold the cluster's monitor: {@link #deleteCacheCluster} removed the record
+ * under that same monitor, and putting the cluster back afterwards would resurrect one the
+ * caller was told had been deleted.
+ */
+ private boolean restoreTargetLost(String clusterId) {
+ return clusters.get(clusterId)
+ .map(current -> current.getCacheClusterStatus() == CacheClusterStatus.DELETING)
+ .orElse(true);
+ }
+
+ /** Drops the container a restore started for a cluster that was deleted while it ran. */
+ private void abandonRestoredContainer(String clusterId, ElastiCacheContainerHandle handle) {
+ if (handle != null) {
+ try {
+ containerManager.stop(handle);
+ } catch (RuntimeException e) {
+ LOG.warnv("Error stopping container for deleted Memcached cluster {0}: {1}",
+ clusterId, e.getMessage());
+ }
+ }
+ LOG.infov("Discarded the restored container for Memcached cluster {0}: it was deleted "
+ + "while it was being restored", clusterId);
+ }
+
+ /**
+ * One monitor per cluster id, taken by every writer that reads a record and writes it back.
+ * Guarding the record itself would be too late: a restore that resolved its cluster before
+ * locking would otherwise write back one a concurrent delete had already removed.
+ */
+ private Object lockFor(String clusterId) {
+ return clusterLocks.computeIfAbsent(clusterId, key -> new Object());
+ }
+
public CacheCluster getCacheCluster(String clusterId) {
return clusters.get(clusterId).orElseThrow(() ->
new AwsException("CacheClusterNotFound",
@@ -95,20 +240,24 @@ public class ElastiCacheMemcachedService {
}
public CacheCluster deleteCacheCluster(String clusterId) {
- CacheCluster cluster = getCacheCluster(clusterId);
+ // one monitor per cluster for delete and restore: a restore's read-modify-write could
+ // otherwise write the cluster back after this removed it
+ synchronized (lockFor(clusterId)) {
+ CacheCluster cluster = getCacheCluster(clusterId);
- cluster.setCacheClusterStatus(CacheClusterStatus.DELETING);
- clusters.put(clusterId, cluster);
+ cluster.setCacheClusterStatus(CacheClusterStatus.DELETING);
+ clusters.put(clusterId, cluster);
- if (cluster.getContainerId() != null) {
- containerManager.stop(new ElastiCacheContainerHandle(
- cluster.getContainerId(), clusterId,
- cluster.getContainerHost(), cluster.getContainerPort()));
- }
+ if (cluster.getContainerId() != null) {
+ containerManager.stop(new ElastiCacheContainerHandle(
+ cluster.getContainerId(), clusterId,
+ cluster.getContainerHost(), cluster.getContainerPort()));
+ }
- clusters.delete(clusterId);
- LOG.infov("Memcached cluster {0} deleted", clusterId);
- return cluster;
+ clusters.delete(clusterId);
+ LOG.infov("Memcached cluster {0} deleted", clusterId);
+ return cluster;
+ }
}
private String resolveEndpointHost(ElastiCacheContainerHandle handle) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.java
index 1a61f63e0f..be6c5fab58 100644
--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.java
@@ -737,7 +737,7 @@ private Response handleCreateCacheParameterGroup(MultivaluedMap<String, String>
XmlBuilder xml = new XmlBuilder()
.start("CacheCluster")
.elem("CacheClusterId", c.getCacheClusterId())
- .elem("CacheClusterStatus", c.getCacheClusterStatus().name().toLowerCase())
+ .elem("CacheClusterStatus", c.getCacheClusterStatus().wireName())
.elem("Engine", c.getEngine())
.elem("EngineVersion", c.getEngineVersion());
if (ep != null) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java
index 763ae168d7..3deacb3758 100644
--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java
@@ -506,53 +506,221 @@ public class ElastiCacheService implements ResourceProvider {
}
/**
- * Brings the data plane back up for cluster-mode groups restored from disk. Invoked from
+ * Brings the data plane back up for every group restored from disk. Invoked from
* {@code EmulatorLifecycle} after {@code storageFactory.loadAll()}, alongside the other
* services that restore persisted runtime. Only topology survives a restart — containers,
* proxies and port reservations are process-local — so each group is re-provisioned from
- * its persisted node list, and a group whose data plane cannot come back is reported
- * {@code create-failed} instead of available. Cluster-mode-disabled groups are deliberately
- * out of scope: they keep their pre-existing behaviour of reporting available after a
- * restart without a restored runtime.
+ * its persisted record, and a group whose data plane cannot come back is reported
+ * {@code create-failed} instead of available.
+ *
+ * <p>A group that is not re-provisioned is worse than one that is reported failed. Its
+ * status stays {@code available} with nothing behind the endpoint, so the control plane
+ * answers healthy while every connection fails, and the proxy port it still advertises is
+ * free for the next create to take — pointing the old endpoint at an unrelated cache rather
+ * than failing cleanly.
*
* <p>Container restarts and cluster formation can take a readiness timeout and a formation
* timeout per group, so the data-plane work runs in the background and must not delay
- * emulator readiness. Groups are marked {@code creating} synchronously — each node's proxy
- * port is reserved at the same time, before any request can claim it — and flip to
- * {@code available} or {@code create-failed} as their restoration finishes.
+ * emulator readiness. Groups are marked {@code creating} synchronously — each proxy port is
+ * reserved at the same time, before any request can claim it — and flip to {@code available}
+ * or {@code create-failed} as their restoration finishes.
*/
public CompletableFuture<Void> restorePersistedRuntime() {
- List<ReplicationGroup> toRestore = new ArrayList<>();
+ List<ReplicationGroup> clusterModeToRestore = new ArrayList<>();
+ List<ReplicationGroup> singleNodeToRestore = new ArrayList<>();
for (ReplicationGroup group : groups.scan(k -> true)) {
- if (!group.isClusterEnabled() || group.getClusterNodes().isEmpty()
- || group.getStatus() == ReplicationGroupStatus.DELETING) {
+ if (group.getStatus() == ReplicationGroupStatus.DELETING) {
continue;
}
- List<Integer> reserved = new ArrayList<>();
- try {
- for (ClusterNode node : group.getClusterNodes()) {
- node.setProxyPort(reserveOrAllocateProxyPort(node.getProxyPort()));
- reserved.add(node.getProxyPort());
- }
- group.setStatus(ReplicationGroupStatus.CREATING);
- toRestore.add(group);
- } catch (RuntimeException e) {
- reserved.forEach(this::releaseProxyPort);
- group.setStatus(ReplicationGroupStatus.CREATE_FAILED);
- group.setConfigurationEndpoint(null);
- LOG.warnv(e, "Failed to restore cluster-mode replication group {0}",
- group.getReplicationGroupId());
+ if (group.isClusterEnabled() && !group.getClusterNodes().isEmpty()) {
+ reserveClusterModeGroup(group, clusterModeToRestore);
+ } else {
+ reserveSingleNodeGroup(group, singleNodeToRestore);
}
groups.put(group.getReplicationGroupId(), group);
}
- if (toRestore.isEmpty()) {
+ if (clusterModeToRestore.isEmpty() && singleNodeToRestore.isEmpty()) {
return CompletableFuture.completedFuture(null);
}
- LOG.infov("Restoring {0} cluster-mode replication group(s) in the background",
- String.valueOf(toRestore.size()));
- return CompletableFuture.runAsync(() -> toRestore.forEach(this::restoreClusterModeGroup));
+ LOG.infov("Restoring {0} cluster-mode and {1} single-node replication group(s) in the background",
+ String.valueOf(clusterModeToRestore.size()), String.valueOf(singleNodeToRestore.size()));
+ return CompletableFuture.runAsync(() -> {
+ clusterModeToRestore.forEach(this::restoreClusterModeGroup);
+ singleNodeToRestore.forEach(this::restoreSingleNodeGroup);
+ });
+ }
+
+ private void reserveClusterModeGroup(ReplicationGroup group, List<ReplicationGroup> toRestore) {
+ List<Integer> reserved = new ArrayList<>();
+ try {
+ for (ClusterNode node : group.getClusterNodes()) {
+ node.setProxyPort(reserveOrAllocateProxyPort(node.getProxyPort()));
+ reserved.add(node.getProxyPort());
+ }
+ group.setStatus(ReplicationGroupStatus.CREATING);
+ toRestore.add(group);
+ } catch (RuntimeException e) {
+ reserved.forEach(this::releaseProxyPort);
+ group.setStatus(ReplicationGroupStatus.CREATE_FAILED);
+ group.setConfigurationEndpoint(null);
+ LOG.warnv(e, "Failed to restore cluster-mode replication group {0}",
+ group.getReplicationGroupId());
+ }
+ }
+
+ /**
+ * Reserves the port a cluster-mode-disabled group comes back advertising and marks it
+ * {@code creating} for {@link #restoreSingleNodeGroup}. The endpoint host is re-derived from
+ * configuration rather than replayed from the record, so a group restored under a changed
+ * {@code FLOCI_HOSTNAME} advertises the name this process actually answers to.
+ */
+ private void reserveSingleNodeGroup(ReplicationGroup group, List<ReplicationGroup> toRestore) {
+ try {
+ group.setProxyPort(reserveOrAllocateProxyPort(group.getProxyPort()));
+ group.setConfigurationEndpoint(new Endpoint(resolveEndpointHost(), group.getProxyPort()));
+ group.setStatus(ReplicationGroupStatus.CREATING);
+ toRestore.add(group);
+ } catch (RuntimeException e) {
+ group.setStatus(ReplicationGroupStatus.CREATE_FAILED);
+ group.setConfigurationEndpoint(null);
+ LOG.warnv(e, "Failed to restore replication group {0}", group.getReplicationGroupId());
+ }
+ }
+
+ /**
+ * Restarts the container and auth proxy behind a cluster-mode-disabled group, the same pair
+ * {@link #provisionSingleNodeGroup} creates. The cache comes back empty, as it does for
+ * cluster-mode groups: only the topology is persisted, never the keyspace.
+ *
+ * <p>Uses {@code tryStart} for the same reason the create path does: with no Docker daemon
+ * reachable, the group keeps its metadata and reports available without a container, rather
+ * than being failed for a daemon that may appear later. A failure raised while the daemon
+ * <em>is</em> reachable is a genuine container problem and fails the group.
+ *
+ * <p>The container work runs outside the group's monitor, as it does on create: a restore
+ * that may pull an image must not hold a delete of the same group behind it. The write-back
+ * takes the monitor and re-reads the record, because a delete taken while the container
+ * started has already removed it.
+ */
+ private void restoreSingleNodeGroup(ReplicationGroup group) {
+ String groupId = group.getReplicationGroupId();
+ String image = config.services().elasticache().defaultImage();
+ try {
+ ElastiCacheContainerHandle handle = containerManager.tryStart(groupId, image);
+ synchronized (lockFor("rg:" + groupId)) {
+ if (restoreTargetLost(groupId)) {
+ abandonRestoredContainer(groupId, handle);
+ return;
+ }
+ if (handle != null) {
+ group.setContainerId(handle.getContainerId());
+ group.setContainerHost(handle.getHost());
+ group.setContainerPort(handle.getPort());
+ proxyManager.startProxy(groupId, group.getAuthMode(), group.getProxyPort(),
+ handle.getHost(), handle.getPort(),
+ (username, password) -> validatePassword(groupId, username, password));
+ } else {
+ // Cleared rather than left alone: whatever the record carried describes a
+ // container from the previous process, and nothing must read it as live.
+ group.setContainerId(null);
+ group.setContainerHost(null);
+ group.setContainerPort(0);
+ LOG.warnv("Replication group {0} restored without a backing cache container: no "
+ + "Docker daemon is reachable. Metadata operations work; connections to "
+ + "the cache do not until a daemon appears.", groupId);
+ }
+ group.setStatus(ReplicationGroupStatus.AVAILABLE);
+ groups.put(groupId, group);
+ LOG.infov("Restored replication group {0}, endpoint={1}:{2}", groupId,
+ group.getConfigurationEndpoint().address(),
+ String.valueOf(group.getProxyPort()));
+ }
+ } catch (RuntimeException e) {
+ failSingleNodeRestore(group, e);
+ }
+ }
+
+ /**
+ * Whether the record a restore is about to write back is still there to write back to. Callers
+ * must hold the group's monitor: a delete taken while the container work ran removed the record
+ * under that same monitor, and a restore that put its group back afterwards would resurrect a
+ * group the caller was told had been deleted, as {@code available}, with a fresh container
+ * behind it.
+ */
+ private boolean restoreTargetLost(String groupId) {
+ return groups.get(groupId)
+ .map(current -> current.getStatus() == ReplicationGroupStatus.DELETING)
+ .orElse(true);
}
+ /**
+ * Drops the container a restore started for a group that was deleted while it ran. The proxy
+ * port is deliberately not released: the delete that removed the record released it already,
+ * and releasing it a second time would return a port a later create has since taken to the
+ * pool, letting two groups be handed the same one.
+ */
+ private void abandonRestoredContainer(String groupId, ElastiCacheContainerHandle handle) {
+ if (handle != null) {
+ try {
+ containerManager.stop(handle);
+ } catch (RuntimeException e) {
+ LOG.warnv("Error stopping container for deleted replication group {0}: {1}",
+ groupId, e.getMessage());
+ }
+ }
+ LOG.infov("Discarded the restored data plane for replication group {0}: it was deleted "
+ + "while it was being restored", groupId);
+ }
+
+ /**
+ * Reports a group whose data plane could not be brought back, without deleting the record:
+ * the group still exists on the control plane, it just has nothing behind it. Tears down
+ * whatever the attempt left running and frees the port, so the next create can use it rather
+ * than leaking a reservation nothing serves.
+ *
+ * <p>A group deleted while the failed attempt ran gets the teardown but no write-back and no
+ * port release: reporting {@code create-failed} would put a deleted group back, and the
+ * delete released the port itself.
+ */
+ private void failSingleNodeRestore(ReplicationGroup group, RuntimeException cause) {
+ String groupId = group.getReplicationGroupId();
+ synchronized (lockFor("rg:" + groupId)) {
+ try {
+ proxyManager.stopProxy(groupId);
+ } catch (RuntimeException e) {
+ LOG.warnv("Error stopping proxy for replication group {0}: {1}", groupId, e.getMessage());
+ }
+ try {
+ containerManager.stopByGroupId(groupId);
+ } catch (RuntimeException e) {
+ LOG.warnv("Error stopping container for replication group {0}: {1}", groupId, e.getMessage());
+ }
+ if (restoreTargetLost(groupId)) {
+ LOG.warnv(cause, "Failed to restore replication group {0}, which was deleted while "
+ + "it was being restored", groupId);
+ return;
+ }
+ releaseProxyPort(group.getProxyPort());
+ group.setContainerId(null);
+ group.setContainerHost(null);
+ group.setContainerPort(0);
+ group.setStatus(ReplicationGroupStatus.CREATE_FAILED);
+ group.setConfigurationEndpoint(null);
+ try {
+ groups.put(groupId, group);
+ } catch (RuntimeException persistFailure) {
+ cause.addSuppressed(persistFailure);
+ }
+ LOG.warnv(cause, "Failed to restore replication group {0}", groupId);
+ }
+ }
+
+ /**
+ * Restarts the containers, cluster formation and proxies behind a cluster-mode group. Like
+ * {@link #restoreSingleNodeGroup}, the container work runs outside the group's monitor and
+ * only the write-back takes it, so a delete that ran in the meantime is seen before the
+ * group is put back.
+ */
private void restoreClusterModeGroup(ReplicationGroup group) {
String groupId = group.getReplicationGroupId();
String image = config.services().elasticache().defaultImage();
@@ -584,24 +752,57 @@ public class ElastiCacheService implements ResourceProvider {
clusterFormation.form(groupId, formationNodes, group.getNumNodeGroups());
- for (int i = 0; i < nodes.size(); i++) {
- ClusterNode node = nodes.get(i);
- ElastiCacheContainerHandle handle = handles.get(i);
- proxyManager.startProxy(node.getMemberClusterId(), group.getAuthMode(), node.getProxyPort(),
- handle.getHost(), handle.getPort(),
- (username, password) -> validatePassword(groupId, username, password));
- startedProxyKeys.add(node.getMemberClusterId());
- }
+ synchronized (lockFor("rg:" + groupId)) {
+ if (restoreTargetLost(groupId)) {
+ // No ports: the delete that removed the record released every one of them.
+ rollbackClusterModeGroup(groupId, startedProxyKeys, handles, inFlightMemberId,
+ List.of());
+ LOG.infov("Discarded the restored data plane for cluster-mode replication group "
+ + "{0}: it was deleted while it was being restored", groupId);
+ return;
+ }
+ for (int i = 0; i < nodes.size(); i++) {
+ ClusterNode node = nodes.get(i);
+ ElastiCacheContainerHandle handle = handles.get(i);
+ proxyManager.startProxy(node.getMemberClusterId(), group.getAuthMode(), node.getProxyPort(),
+ handle.getHost(), handle.getPort(),
+ (username, password) -> validatePassword(groupId, username, password));
+ startedProxyKeys.add(node.getMemberClusterId());
+ }
- group.setConfigurationEndpoint(new Endpoint(endpointHost, nodes.getFirst().getProxyPort()));
- group.setStatus(ReplicationGroupStatus.AVAILABLE);
- groups.put(groupId, group);
- LOG.infov("Restored cluster-mode replication group {0}: {1} node(s), configuration endpoint={2}:{3}",
- groupId, String.valueOf(nodes.size()), endpointHost,
- String.valueOf(group.getConfigurationEndpoint().port()));
+ group.setConfigurationEndpoint(new Endpoint(endpointHost, nodes.getFirst().getProxyPort()));
+ group.setStatus(ReplicationGroupStatus.AVAILABLE);
+ groups.put(groupId, group);
+ LOG.infov("Restored cluster-mode replication group {0}: {1} node(s), configuration endpoint={2}:{3}",
+ groupId, String.valueOf(nodes.size()), endpointHost,
+ String.valueOf(group.getConfigurationEndpoint().port()));
+ }
} catch (RuntimeException e) {
- rollbackClusterModeGroup(groupId, startedProxyKeys, handles, inFlightMemberId, reservedPorts);
- for (ClusterNode node : nodes) {
+ failClusterModeRestore(group, startedProxyKeys, handles, inFlightMemberId, reservedPorts, e);
+ }
+ }
+
+ /**
+ * Reports a cluster-mode group whose data plane could not be brought back, mirroring
+ * {@link #failSingleNodeRestore}: the record stays, reporting {@code create-failed} with no
+ * endpoint, unless a delete removed it while the attempt ran. In that case the rollback still
+ * runs but the ports are left to the delete that released them, and nothing is written back.
+ */
+ private void failClusterModeRestore(ReplicationGroup group, List<String> startedProxyKeys,
+ List<ElastiCacheContainerHandle> handles,
+ String inFlightMemberId, Collection<Integer> reservedPorts,
+ RuntimeException cause) {
+ String groupId = group.getReplicationGroupId();
+ synchronized (lockFor("rg:" + groupId)) {
+ boolean lost = restoreTargetLost(groupId);
+ rollbackClusterModeGroup(groupId, startedProxyKeys, handles, inFlightMemberId,
+ lost ? List.of() : reservedPorts);
+ if (lost) {
+ LOG.warnv(cause, "Failed to restore cluster-mode replication group {0}, which was "
+ + "deleted while it was being restored", groupId);
+ return;
+ }
+ for (ClusterNode node : group.getClusterNodes()) {
node.setContainerId(null);
node.setContainerHost(null);
node.setContainerPort(0);
@@ -611,9 +812,9 @@ public class ElastiCacheService implements ResourceProvider {
try {
groups.put(groupId, group);
} catch (RuntimeException persistFailure) {
- e.addSuppressed(persistFailure);
+ cause.addSuppressed(persistFailure);
}
- LOG.warnv(e, "Failed to restore cluster-mode replication group {0}", groupId);
+ LOG.warnv(cause, "Failed to restore cluster-mode replication group {0}", groupId);
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/model/CacheClusterStatus.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/model/CacheClusterStatus.java
index 80db4647eb..39611e8ce2 100644
--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/model/CacheClusterStatus.java
+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/model/CacheClusterStatus.java
@@ -2,7 +2,14 @@ package io.github.hectorvent.floci.services.elasticache.model;
import io.quarkus.runtime.annotations.RegisterForReflection;
+import java.util.Locale;
+
@RegisterForReflection
public enum CacheClusterStatus {
- AVAILABLE, CREATING, DELETING
+ AVAILABLE, CREATING, DELETING, RESTORE_FAILED;
+
+ /** The status string as AWS reports it, e.g. {@code restore-failed}. */
+ public String wireName() {
+ return name().toLowerCase(Locale.ROOT).replace('_', '-');
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java
index f72e6585c5..4a7d6ee7cf 100644
--- a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java
+++ b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java
@@ -5,13 +5,13 @@ import io.github.hectorvent.floci.core.common.ContainerTeardown;
import io.github.hectorvent.floci.core.common.ServiceRegistry;
import io.github.hectorvent.floci.core.storage.PersistentPathValidator;
import io.github.hectorvent.floci.core.storage.StorageFactory;
-import io.github.hectorvent.floci.lifecycle.InitLifecycleState;
import io.github.hectorvent.floci.lifecycle.inithook.InitializationHook;
import io.github.hectorvent.floci.lifecycle.inithook.InitializationHooksRunner;
import io.github.hectorvent.floci.services.appsync.graphql.SchemaCreationWorker;
import io.github.hectorvent.floci.services.docdb.container.DocDbContainerManager;
import io.github.hectorvent.floci.services.ec2.Ec2MetadataServer;
import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager;
+import io.github.hectorvent.floci.services.elasticache.ElastiCacheMemcachedService;
import io.github.hectorvent.floci.services.elasticache.ElastiCacheService;
import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerManager;
import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheMemcachedContainerManager;
@@ -39,6 +39,7 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.InOrder;
import org.mockito.Mock;
import org.mockito.Mockito;
import org.mockito.junit.jupiter.MockitoExtension;
@@ -73,6 +74,7 @@ class EmulatorLifecycleTest {
@Mock private IamService iamService;
@Mock private EmulatorConfig.ElastiCacheServiceConfig elastiCacheServiceConfig;
@Mock private ElastiCacheService elastiCacheService;
+ @Mock private ElastiCacheMemcachedService elastiCacheMemcachedService;
@Mock private ElastiCacheContainerManager elastiCacheContainerManager;
@Mock private ElastiCacheMemcachedContainerManager elastiCacheMemcachedContainerManager;
@Mock private ElastiCacheProxyManager elastiCacheProxyManager;
@@ -131,7 +133,7 @@ class EmulatorLifecycleTest {
emulatorLifecycle = new EmulatorLifecycle(
storageFactory, serviceRegistry, config,
- iamService, elastiCacheService,
+ iamService, elastiCacheService, elastiCacheMemcachedService,
elastiCacheContainerManager, elastiCacheMemcachedContainerManager,
elastiCacheProxyManager, rdsContainerManager, rdsProxyManager,
memoryDbContainerManager, memoryDbProxyManager,
@@ -206,15 +208,18 @@ class EmulatorLifecycleTest {
stubStorageConfig();
when(elastiCacheServiceConfig.enabled()).thenReturn(true);
when(elastiCacheService.restorePersistedRuntime())
- .thenReturn(java.util.concurrent.CompletableFuture.completedFuture(null));
+ .thenReturn(CompletableFuture.completedFuture(null));
+ when(elastiCacheMemcachedService.restorePersistedRuntime())
+ .thenReturn(CompletableFuture.completedFuture(null));
when(initializationHooksRunner.hasHooks(InitializationHook.START)).thenReturn(false);
when(initializationHooksRunner.hasHooks(InitializationHook.READY)).thenReturn(false);
emulatorLifecycle.onStart(Mockito.mock(StartupEvent.class));
- var inOrder = Mockito.inOrder(storageFactory, elastiCacheService);
+ InOrder inOrder = Mockito.inOrder(storageFactory, elastiCacheService, elastiCacheMemcachedService);
inOrder.verify(storageFactory).loadAll();
inOrder.verify(elastiCacheService).restorePersistedRuntime();
+ inOrder.verify(elastiCacheMemcachedService).restorePersistedRuntime();
}
@Test
@@ -227,6 +232,7 @@ class EmulatorLifecycleTest {
emulatorLifecycle.onStart(Mockito.mock(StartupEvent.class));
Mockito.verify(elastiCacheService, Mockito.never()).restorePersistedRuntime();
+ Mockito.verify(elastiCacheMemcachedService, Mockito.never()).restorePersistedRuntime();
}
@Test
diff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedServiceTest.java
index 3ab63226f8..478c279a81 100644
--- a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedServiceTest.java
@@ -12,12 +12,16 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.util.Collection;
+import java.util.Map;
import java.util.Optional;
+import java.util.concurrent.ConcurrentHashMap;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.verify;
@@ -144,4 +148,94 @@ class ElastiCacheMemcachedServiceTest {
service.deleteCacheCluster("no-docker-cluster");
verify(containerManager, never()).stop(any());
}
+
+ @Test
+ void restorePersistedRuntimeRestartsTheContainerAndRepointsTheEndpoint() {
+ StorageFactory storageFactory = sharedStorageFactory();
+ ElastiCacheMemcachedContainerManager beforeRestart = mock(ElastiCacheMemcachedContainerManager.class);
+ when(beforeRestart.tryStart(anyString(), anyString()))
+ .thenReturn(new ElastiCacheContainerHandle("cid", "my-cluster", "localhost", 32770));
+ serviceWith(storageFactory, beforeRestart).createCacheCluster("my-cluster");
+
+ ElastiCacheMemcachedContainerManager restarted = mock(ElastiCacheMemcachedContainerManager.class);
+ when(restarted.tryStart(anyString(), anyString()))
+ .thenReturn(new ElastiCacheContainerHandle("cid2", "my-cluster", "localhost", 32771));
+ ElastiCacheMemcachedService restartedService = serviceWith(storageFactory, restarted);
+
+ restartedService.restorePersistedRuntime().join();
+
+ verify(restarted).tryStart(eq("my-cluster"), anyString());
+ CacheCluster cluster = restartedService.getCacheCluster("my-cluster");
+ assertEquals(CacheClusterStatus.AVAILABLE, cluster.getCacheClusterStatus());
+ assertEquals(32771, cluster.getConfigurationEndpoint().port(),
+ "Docker publishes a fresh host port per run, so the endpoint must follow it");
+ }
+
+ @Test
+ void memcachedRestoreFailureReportsRestoreFailed() {
+ StorageFactory storageFactory = sharedStorageFactory();
+ ElastiCacheMemcachedContainerManager beforeRestart = mock(ElastiCacheMemcachedContainerManager.class);
+ when(beforeRestart.tryStart(anyString(), anyString()))
+ .thenReturn(new ElastiCacheContainerHandle("cid", "my-cluster", "localhost", 11211));
+ serviceWith(storageFactory, beforeRestart).createCacheCluster("my-cluster");
+
+ ElastiCacheMemcachedContainerManager restarted = mock(ElastiCacheMemcachedContainerManager.class);
+ when(restarted.tryStart(anyString(), anyString()))
+ .thenThrow(new RuntimeException("container failed"));
+ ElastiCacheMemcachedService restartedService = serviceWith(storageFactory, restarted);
+
+ restartedService.restorePersistedRuntime().join();
+
+ CacheCluster cluster = restartedService.getCacheCluster("my-cluster");
+ assertEquals(CacheClusterStatus.RESTORE_FAILED, cluster.getCacheClusterStatus());
+ assertNull(cluster.getConfigurationEndpoint(),
+ "A cluster whose container is gone must not advertise an endpoint");
+ }
+
+ @Test
+ void restoreDoesNotResurrectAClusterDeletedWhileItWasRestoring() {
+ StorageFactory storageFactory = sharedStorageFactory();
+ ElastiCacheMemcachedContainerManager beforeRestart = mock(ElastiCacheMemcachedContainerManager.class);
+ when(beforeRestart.tryStart(anyString(), anyString()))
+ .thenReturn(new ElastiCacheContainerHandle("cid", "my-cluster", "localhost", 32770));
+ serviceWith(storageFactory, beforeRestart).createCacheCluster("my-cluster");
+
+ ElastiCacheMemcachedContainerManager restarted = mock(ElastiCacheMemcachedContainerManager.class);
+ ElastiCacheMemcachedService restartedService = serviceWith(storageFactory, restarted);
+ ElastiCacheContainerHandle restoredHandle =
+ new ElastiCacheContainerHandle("cid2", "my-cluster", "localhost", 32771);
+ // The delete lands in the window the cluster's monitor closes: the container is up, the
+ // record has not been written back yet.
+ when(restarted.tryStart(anyString(), anyString())).thenAnswer(inv -> {
+ restartedService.deleteCacheCluster("my-cluster");
+ return restoredHandle;
+ });
+
+ restartedService.restorePersistedRuntime().join();
+
+ assertThrows(AwsException.class, () -> restartedService.getCacheCluster("my-cluster"),
+ "A cluster deleted while it was restoring must stay deleted");
+ verify(restarted).stop(restoredHandle);
+ }
+
+ private static StorageFactory sharedStorageFactory() {
+ StorageFactory storageFactory = mock(StorageFactory.class);
+ Map<String, Object> backends = new ConcurrentHashMap<>();
+ when(storageFactory.create(anyString(), anyString(), any())).thenAnswer(inv ->
+ backends.computeIfAbsent(inv.getArgument(1, String.class),
+ key -> AccountAwareStorageBackend.inMemory("000000000000")));
+ return storageFactory;
+ }
+
+ private static ElastiCacheMemcachedService serviceWith(StorageFactory storageFactory,
+ ElastiCacheMemcachedContainerManager containerManager) {
+ EmulatorConfig config = mock(EmulatorConfig.class);
+ EmulatorConfig.ServicesConfig servicesConfig = mock(EmulatorConfig.ServicesConfig.class);
+ EmulatorConfig.ElastiCacheServiceConfig ecConfig = mock(EmulatorConfig.ElastiCacheServiceConfig.class);
+ when(config.services()).thenReturn(servicesConfig);
+ when(servicesConfig.elasticache()).thenReturn(ecConfig);
+ when(ecConfig.defaultMemcachedImage()).thenReturn("memcached:1.6");
+ when(config.hostname()).thenReturn(Optional.of("localhost"));
+ return new ElastiCacheMemcachedService(containerManager, storageFactory, config);
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java
index a962e20b15..f1b15e8143 100644
--- a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java
@@ -485,6 +485,148 @@ class ElastiCacheServiceTest {
.thenReturn(new ElastiCacheContainerHandle("cid", "grp", "localhost", 6379));
}
+ private static void stubSingleNodeContainer(ElastiCacheContainerManager containerManager) {
+ when(containerManager.tryStart(anyString(), anyString())).thenAnswer(inv ->
+ new ElastiCacheContainerHandle("cid-" + inv.getArgument(0, String.class),
+ inv.getArgument(0, String.class), "localhost", 6379));
+ }
+
+ private static StorageFactory storageWithSingleNodeGroup(String groupId) {
+ StorageFactory storageFactory = sharedStorageFactory();
+ ElastiCacheContainerManager beforeRestart = mock(ElastiCacheContainerManager.class);
+ stubSingleNodeContainer(beforeRestart);
+ serviceWith(storageFactory, beforeRestart, mock(ElastiCacheProxyManager.class),
+ mock(ValkeyClusterFormation.class))
+ .createReplicationGroup(groupId, "test", AuthMode.PASSWORD, null, "us-east-1");
+ return storageFactory;
+ }
+
+ @Test
+ void restorePersistedRuntimeReprovisionsSingleNodeGroups() {
+ StorageFactory storageFactory = storageWithSingleNodeGroup("grp");
+
+ ElastiCacheContainerManager restartedContainers = mock(ElastiCacheContainerManager.class);
+ stubSingleNodeContainer(restartedContainers);
+ ElastiCacheProxyManager restartedProxies = mock(ElastiCacheProxyManager.class);
+ ElastiCacheService restarted = serviceWith(storageFactory, restartedContainers,
+ restartedProxies, mock(ValkeyClusterFormation.class));
+
+ restarted.restorePersistedRuntime().join();
+
+ verify(restartedContainers).tryStart(eq("grp"), anyString());
+ verify(restartedProxies).startProxy(eq("grp"), eq(AuthMode.PASSWORD), eq(16379),
+ eq("localhost"), eq(6379), any());
+ ReplicationGroup restored = restarted.getReplicationGroup("grp");
+ assertEquals(ReplicationGroupStatus.AVAILABLE, restored.getStatus());
+ assertEquals(16379, restored.getConfigurationEndpoint().port());
+ assertEquals("cid-grp", restored.getContainerId(),
+ "A restored group must track the container it actually has");
+
+ ReplicationGroup next =
+ restarted.createReplicationGroup("grp2", "test", AuthMode.NO_AUTH, null, "us-east-1");
+ assertEquals(16380, next.getProxyPort(),
+ "A restored group's port must be reserved again so new groups cannot take it");
+ }
+
+ @Test
+ void singleNodeRestoreFailureReportsCreateFailedAndReleasesThePort() {
+ StorageFactory storageFactory = storageWithSingleNodeGroup("grp");
+
+ ElastiCacheContainerManager restartedContainers = mock(ElastiCacheContainerManager.class);
+ // Only the restore fails: the create that checks the port was freed must still get through.
+ when(restartedContainers.tryStart(eq("grp"), anyString()))
+ .thenThrow(new RuntimeException("container failed"));
+ ElastiCacheProxyManager restartedProxies = mock(ElastiCacheProxyManager.class);
+ ElastiCacheService restarted = serviceWith(storageFactory, restartedContainers,
+ restartedProxies, mock(ValkeyClusterFormation.class));
+
+ restarted.restorePersistedRuntime().join();
+
+ ReplicationGroup failed = restarted.getReplicationGroup("grp");
+ assertEquals(ReplicationGroupStatus.CREATE_FAILED, failed.getStatus());
+ assertNull(failed.getConfigurationEndpoint(),
+ "A group whose data plane is gone must not advertise an endpoint");
+ verify(restartedProxies, never()).startProxy(anyString(), any(), anyInt(), anyString(), anyInt(), any());
+
+ ReplicationGroup next =
+ restarted.createReplicationGroup("grp2", "test", AuthMode.NO_AUTH, null, "us-east-1");
+ assertEquals(16379, next.getProxyPort(),
+ "The failed restore's port must be released for the next group");
+ }
+
+ @Test
+ void singleNodeRestoreWithoutADockerDaemonKeepsTheGroupAvailable() {
+ StorageFactory storageFactory = storageWithSingleNodeGroup("grp");
+
+ ElastiCacheContainerManager restartedContainers = mock(ElastiCacheContainerManager.class);
+ when(restartedContainers.tryStart(anyString(), anyString())).thenReturn(null);
+ ElastiCacheProxyManager restartedProxies = mock(ElastiCacheProxyManager.class);
+ ElastiCacheService restarted = serviceWith(storageFactory, restartedContainers,
+ restartedProxies, mock(ValkeyClusterFormation.class));
+
+ restarted.restorePersistedRuntime().join();
+
+ ReplicationGroup restored = restarted.getReplicationGroup("grp");
+ assertEquals(ReplicationGroupStatus.AVAILABLE, restored.getStatus(),
+ "No reachable daemon is the create path's documented degraded mode, not a failure");
+ assertNull(restored.getContainerId());
+ verify(restartedProxies, never()).startProxy(anyString(), any(), anyInt(), anyString(), anyInt(), any());
+ }
+
+ @Test
+ void restoreDoesNotResurrectAGroupDeletedWhileItWasRestoring() {
+ StorageFactory storageFactory = storageWithSingleNodeGroup("grp");
+
+ ElastiCacheContainerManager restartedContainers = mock(ElastiCacheContainerManager.class);
+ ElastiCacheProxyManager restartedProxies = mock(ElastiCacheProxyManager.class);
+ ElastiCacheService restarted = serviceWith(storageFactory, restartedContainers,
+ restartedProxies, mock(ValkeyClusterFormation.class));
+ ElastiCacheContainerHandle restoredHandle =
+ new ElastiCacheContainerHandle("cid-grp-restored", "grp", "localhost", 6379);
+ // The delete lands in the window the group's monitor closes: the container is up, the
+ // record has not been written back yet.
+ when(restartedContainers.tryStart(eq("grp"), anyString())).thenAnswer(inv -> {
+ restarted.deleteReplicationGroup("grp");
+ // Takes the port that delete just freed, so a restore that released it a second
+ // time would hand the same port out twice.
+ restarted.createReplicationGroup("grp2", "test", AuthMode.NO_AUTH, null, "us-east-1");
+ return restoredHandle;
+ });
+
+ restarted.restorePersistedRuntime().join();
+
+ assertThrows(AwsException.class, () -> restarted.getReplicationGroup("grp"),
+ "A group deleted while it was restoring must stay deleted");
+ verify(restartedProxies, never()).startProxy(eq("grp"), any(), anyInt(), anyString(), anyInt(), any());
+ verify(restartedContainers).stop(restoredHandle);
+
+ ReplicationGroup next =
+ restarted.createReplicationGroup("grp3", "test", AuthMode.NO_AUTH, null, "us-east-1");
+ assertEquals(16380, next.getProxyPort(),
+ "The abandoned restore must leave grp2 holding the port the delete released");
+ }
+
+ @Test
+ void restorePersistedRuntimeSkipsGroupsBeingDeleted() {
+ StorageFactory storageFactory = storageWithSingleNodeGroup("grp");
+ ElastiCacheContainerManager beforeRestart = mock(ElastiCacheContainerManager.class);
+ stubSingleNodeContainer(beforeRestart);
+ ElastiCacheService before = serviceWith(storageFactory, beforeRestart,
+ mock(ElastiCacheProxyManager.class), mock(ValkeyClusterFormation.class));
+ // The in-memory backend hands back the stored instance, so this is the persisted record.
+ before.getReplicationGroup("grp").setStatus(ReplicationGroupStatus.DELETING);
+
+ ElastiCacheContainerManager restartedContainers = mock(ElastiCacheContainerManager.class);
+ stubSingleNodeContainer(restartedContainers);
+ ElastiCacheService restarted = serviceWith(storageFactory, restartedContainers,
+ mock(ElastiCacheProxyManager.class), mock(ValkeyClusterFormation.class));
+
+ restarted.restorePersistedRuntime().join();
+
+ verify(restartedContainers, never()).tryStart(anyString(), anyString());
+ assertEquals(ReplicationGroupStatus.DELETING, restarted.getReplicationGroup("grp").getStatus());
+ }
+
@Test
void restorePersistedRuntimeReprovisionsClusterModeGroups() {
StorageFactory storageFactory = sharedStorageFactory();