Light, fluffy, and always free - The AWS Local Emulator alternative
feat(elasticache): support cluster-mode replication groups (#2633)
Kurt McAlpine97ff0bc3e9ff
diff --git a/compatibility-tests/compat-terraform/Dockerfile b/compatibility-tests/compat-terraform/Dockerfileindex 53cc87b4d1..779b621fca 100644--- a/compatibility-tests/compat-terraform/Dockerfile+++ b/compatibility-tests/compat-terraform/Dockerfile@@ -1,6 +1,6 @@FROM hashicorp/terraform:1.14.7 AS terraform-FROM amazon/aws-cli:2.15.52+FROM amazon/aws-cli:2.34.32# Copy terraform binary from official Terraform imageCOPY --from=terraform /bin/terraform /usr/local/bin/terraformdiff --git a/compatibility-tests/compat-terraform/test/elasticache-cluster-tf/main.tf b/compatibility-tests/compat-terraform/test/elasticache-cluster-tf/main.tfnew file mode 100644index 0000000000..8f5ada5b79--- /dev/null+++ b/compatibility-tests/compat-terraform/test/elasticache-cluster-tf/main.tf@@ -0,0 +1,26 @@+resource "aws_elasticache_replication_group" "cluster" {+ replication_group_id = "floci-tf-valkey-cluster"+ description = "floci terraform cluster-mode compatibility test"+ engine = "valkey"+ node_type = "cache.t4g.micro"+ parameter_group_name = "default.valkey8.cluster.on"+ automatic_failover_enabled = true+ num_node_groups = 2+ replicas_per_node_group = 1+}++output "cluster_enabled" {+ value = aws_elasticache_replication_group.cluster.cluster_enabled+}++output "configuration_endpoint_address" {+ value = aws_elasticache_replication_group.cluster.configuration_endpoint_address+}++output "num_node_groups" {+ value = aws_elasticache_replication_group.cluster.num_node_groups+}++output "member_clusters" {+ value = aws_elasticache_replication_group.cluster.member_clusters+}diff --git a/compatibility-tests/compat-terraform/test/elasticache-cluster-tf/provider.tf b/compatibility-tests/compat-terraform/test/elasticache-cluster-tf/provider.tfnew file mode 100644index 0000000000..b32d832003--- /dev/null+++ b/compatibility-tests/compat-terraform/test/elasticache-cluster-tf/provider.tf@@ -0,0 +1,28 @@+terraform {+ required_providers {+ aws = {+ source = "hashicorp/aws"+ version = "~> 6.0"+ }+ }+}++variable "endpoint" {+ type = string+ default = "http://localhost:4566"+}++provider "aws" {+ region = "us-east-1"+ access_key = "test"+ secret_key = "test"++ skip_credentials_validation = true+ skip_metadata_api_check = true+ skip_requesting_account_id = true+ s3_use_path_style = true++ endpoints {+ elasticache = var.endpoint+ }+}diff --git a/compatibility-tests/compat-terraform/test/elasticache_cluster.bats b/compatibility-tests/compat-terraform/test/elasticache_cluster.batsnew file mode 100644index 0000000000..72c7bf037b--- /dev/null+++ b/compatibility-tests/compat-terraform/test/elasticache_cluster.bats@@ -0,0 +1,131 @@+#!/usr/bin/env bats+# ElastiCache Cluster Mode Compatibility Test+#+# Applies an aws_elasticache_replication_group with num_node_groups /+# replicas_per_node_group (Terraform's cluster-mode shape) against floci and+# verifies both sides of the contract: the terraform read-back is honest+# (cluster_enabled, member_clusters, a clean re-plan), and the data plane is a+# genuine Valkey cluster (CLUSTER INFO over RESP through the auth proxy).++setup_file() {+ load 'test_helper/common-setup'++ EC_TF_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/elasticache-cluster-tf" && pwd)"+ cd "$EC_TF_DIR"++ echo "# === ElastiCache Cluster Mode Test ===" >&3+ echo "# Endpoint: $FLOCI_ENDPOINT" >&3+ echo "# Config: $EC_TF_DIR" >&3++ rm -rf .terraform .terraform.lock.hcl terraform.tfstate* 2>/dev/null || true++ echo "# --- terraform init ---" >&3+ run terraform init -input=false -no-color+ if [ "$status" -ne 0 ]; then+ echo "# terraform init failed: $output" >&3+ return 1+ fi++ echo "# --- terraform apply (2 shards, 1 replica each) ---" >&3+ run terraform apply -var="endpoint=${FLOCI_ENDPOINT}" -input=false -auto-approve -no-color+ if [ "$status" -ne 0 ]; then+ echo "# terraform apply failed: $output" >&3+ return 1+ fi+}++teardown_file() {+ load 'test_helper/common-setup'++ EC_TF_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/elasticache-cluster-tf" && pwd)"+ cd "$EC_TF_DIR"++ terraform destroy -var="endpoint=${FLOCI_ENDPOINT}" -input=false -auto-approve -no-color || true+ rm -rf .terraform .terraform.lock.hcl terraform.tfstate* 2>/dev/null || true+}++setup() {+ load 'test_helper/common-setup'+ EC_TF_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/elasticache-cluster-tf" && pwd)"+}++# Sends one RESP command over /dev/tcp and echoes whatever the server replies+# within the read window. No redis-cli in this image, so raw RESP it is.+valkey_command() {+ local host="$1" port="$2"+ shift 2+ local payload="*$#\r\n"+ local arg+ for arg in "$@"; do+ payload+="\$${#arg}\r\n${arg}\r\n"+ done+ exec 9<>"/dev/tcp/${host}/${port}" || return 1+ printf '%b' "$payload" >&9+ local reply="" chunk+ while IFS= read -r -t 3 -u 9 chunk; do+ reply+="${chunk}"$'\n'+ done+ exec 9<&- 9>&-+ printf '%s' "$reply"+}++@test "ElastiCache cluster mode: terraform reads back cluster_enabled=true" {+ run terraform -chdir="$EC_TF_DIR" output -raw cluster_enabled+ assert_success+ assert_output "true"++ run terraform -chdir="$EC_TF_DIR" output -raw num_node_groups+ assert_success+ assert_output "2"+}++@test "ElastiCache cluster mode: describe reports the sharded topology" {+ run aws_cmd elasticache describe-replication-groups \+ --replication-group-id floci-tf-valkey-cluster+ assert_success+ [ "$(json_get "$output" '.ReplicationGroups[0].ClusterEnabled')" = "true" ]+ [ "$(json_get "$output" '.ReplicationGroups[0].AutomaticFailover')" = "enabled" ]+ [ "$(json_get "$output" '.ReplicationGroups[0].Engine')" = "valkey" ]+ [ "$(json_get "$output" '.ReplicationGroups[0].NodeGroups | length')" = "2" ]+ [ "$(json_get "$output" '.ReplicationGroups[0].NodeGroups[0].Slots')" = "0-8191" ]+ [ "$(json_get "$output" '.ReplicationGroups[0].NodeGroups[1].Slots')" = "8192-16383" ]+ [ "$(json_get "$output" '.ReplicationGroups[0].MemberClusters | length')" = "4" ]+}++@test "ElastiCache cluster mode: member cache clusters answer DescribeCacheClusters" {+ member=$(terraform -chdir="$EC_TF_DIR" output -json member_clusters | jq -r '.[0]')+ run aws_cmd elasticache describe-cache-clusters \+ --cache-cluster-id "$member" --show-cache-node-info+ assert_success+ [ "$(json_get "$output" '.CacheClusters[0].ReplicationGroupId')" = "floci-tf-valkey-cluster" ]+ [ "$(json_get "$output" '.CacheClusters[0].Engine')" = "valkey" ]+ [ -n "$(json_get "$output" '.CacheClusters[0].CacheNodes[0].Endpoint.Port')" ]+}++# The critical assertion: the endpoint terraform hands out fronts a Valkey+# node that is actually running in cluster mode, in a formed, healthy cluster.+@test "ElastiCache cluster mode: Valkey runs in cluster mode behind the configuration endpoint" {+ host=$(terraform -chdir="$EC_TF_DIR" output -raw configuration_endpoint_address)+ describe=$(aws_cmd elasticache describe-replication-groups \+ --replication-group-id floci-tf-valkey-cluster)+ port=$(json_get "$describe" '.ReplicationGroups[0].ConfigurationEndpoint.Port')++ run valkey_command "$host" "$port" INFO cluster+ assert_success+ assert_output --partial "cluster_enabled:1"++ run valkey_command "$host" "$port" CLUSTER INFO+ assert_success+ assert_output --partial "cluster_state:ok"+ assert_output --partial "cluster_size:2"+ assert_output --partial "cluster_known_nodes:4"+}++# Guards against read-back drift (the class of bug in floci-io/floci#2481):+# a second plan straight after apply must not want to change anything.+@test "ElastiCache cluster mode: re-plan after apply is clean" {+ cd "$EC_TF_DIR"+ run terraform plan -var="endpoint=${FLOCI_ENDPOINT}" -input=false -no-color -detailed-exitcode+ assert_success+ assert_output --partial "No changes"+}diff --git a/docker-compose.yml b/docker-compose.ymlindex b882e6a07e..0046854e6b 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -20,6 +20,9 @@ services:# If you were using this env var for persistence, migrate: docker volume ls -f label=floci=trueFLOCI_HOSTNAME: flociFLOCI_BASE_URL: http://floci:4566+ # localhost.floci.io resolves everywhere: public DNS -> 127.0.0.1 on the host+ # (reaching the published proxy ports), the network alias below inside Compose.+ FLOCI_SERVICES_ELASTICACHE_CLUSTER_ANNOUNCE_HOSTNAME: localhost.floci.ioFLOCI_SERVICES_LAMBDA_HOT_RELOAD_ENABLED: "true"# Match the compatibility workflow (.github/workflows/compatibility.yml) so the# local Docker test run exercises the same Floci configuration as CI.diff --git a/docs/configuration/environment-variables.md b/docs/configuration/environment-variables.mdindex bd8378fe56..da64eddbe8 100644--- a/docs/configuration/environment-variables.md+++ b/docs/configuration/environment-variables.md@@ -336,6 +336,7 @@ These services spawn Docker containers. They require access to the Docker socket| `FLOCI_SERVICES_ELASTICACHE_PROXY_MAX_PORT` | `6399` | Last port in the ElastiCache proxy range || `FLOCI_SERVICES_ELASTICACHE_DEFAULT_IMAGE` | `valkey/valkey:8` | Default Docker image for cache clusters || `FLOCI_SERVICES_ELASTICACHE_DOCKER_NETWORK` | _(none)_ | Docker network for ElastiCache containers (overrides `FLOCI_SERVICES_DOCKER_NETWORK`) |+| `FLOCI_SERVICES_ELASTICACHE_CLUSTER_ANNOUNCE_HOSTNAME` | _(none)_ | Hostname cluster-mode nodes announce in `MOVED`/`ASK` redirects and topology responses, and report as the `ConfigurationEndpoint`. Set to a name every client resolves (e.g. `localhost.floci.io`) when `FLOCI_HOSTNAME` only resolves inside Floci's Docker network. Defaults to `FLOCI_HOSTNAME` |### RDSdiff --git a/docs/services/elasticache.md b/docs/services/elasticache.mdindex 6f15199001..38a24cd8e7 100644--- a/docs/services/elasticache.md+++ b/docs/services/elasticache.md@@ -35,6 +35,67 @@ Floci manages real Valkey/Redis Docker containers and proxies TCP connections to| `ListTagsForResource` | Tags on a parameter group ARN |<!-- floci:actions:end -->+### Cluster Mode++`CreateReplicationGroup` provisions a real sharded Valkey cluster when the request asks for one:+`NumNodeGroups` greater than 1, a `default.*.cluster.on` parameter group, a custom parameter group+with `cluster-enabled` set to `yes`, or `ClusterMode=enabled`.++Floci starts one `--cluster-enabled` container per node — `NumNodeGroups × (1 + ReplicasPerNodeGroup)`+in total — forms the cluster (config epochs, MEET, slot assignment, replica attachment), and fronts+each node with its own auth-proxy port from the proxy port range. Nodes announce Floci's configured+hostname as their preferred endpoint (`cluster-announce-hostname` with+`cluster-preferred-endpoint-type hostname`, plus `cluster-announce-client-ipv4`/+`cluster-announce-port`), so `CLUSTER SLOTS`, `CLUSTER SHARDS` and `MOVED`/`ASK` redirects hand+clients the same name the `ConfigurationEndpoint` reports, while the cluster bus keeps using the+container network. Any cluster-aware Redis/Valkey client works against the reported+`ConfigurationEndpoint`.++Because clients must resolve the announced name to follow redirects, a `FLOCI_HOSTNAME` that only+resolves inside Floci's Docker network (such as the Compose service name `floci`) breaks+cluster-aware clients connecting from outside it. Set+`FLOCI_SERVICES_ELASTICACHE_CLUSTER_ANNOUNCE_HOSTNAME` to a universally resolvable name in that+case — the shipped `docker-compose.yml` uses `localhost.floci.io`, which public DNS resolves to+`127.0.0.1` on the host (reaching the published proxy ports) while the Compose network alias and+Floci's embedded DNS resolve it to the Floci container from inside Docker. Cluster-mode groups+then announce that name and report it as their `ConfigurationEndpoint`.++With `persistent`, `hybrid` or `wal` storage, cluster-mode groups are re-provisioned from their+persisted topology on startup: containers are restarted, the cluster is re-formed (caches restart+empty, as on any Floci restart) and each node's proxy port is re-reserved. Ports are re-reserved+and groups marked `creating` before Floci reports ready; the container restarts and cluster+formation run in the background so a slow Docker daemon cannot delay readiness, and each group+flips to `available` once its data plane is back. A group whose data plane cannot be brought back+is reported with status `create-failed` instead of `available`, and its member clusters answer+`DescribeCacheClusters` with `restore-failed` (`CacheClusterStatus` has no `create-failed` value).++`DescribeReplicationGroups` reports the topology honestly: `ClusterEnabled`, one `NodeGroup` per+shard with its `Slots`, `NodeGroupMembers`, and `MemberClusters`. Each member also answers+`DescribeCacheClusters` (as on AWS), which is what terraform-provider-aws reads node type, engine+version and port from.++Cluster mode requires a Valkey 8.1+ image (the default `valkey/valkey:8` qualifies) for+`cluster-announce-client-ipv4` support. Each node consumes one port from the proxy range, so size+`FLOCI_SERVICES_ELASTICACHE_PROXY_BASE_PORT`/`_MAX_PORT` to the number of nodes you need.++```bash+aws elasticache create-replication-group \+ --replication-group-id my-sharded-cache \+ --replication-group-description "Sharded dev cache" \+ --engine valkey \+ --cache-parameter-group-name default.valkey8.cluster.on \+ --num-node-groups 2 \+ --replicas-per-node-group 1 \+ --endpoint-url $AWS_ENDPOINT_URL++aws elasticache describe-replication-groups \+ --replication-group-id my-sharded-cache \+ --query 'ReplicationGroups[0].ConfigurationEndpoint' \+ --endpoint-url $AWS_ENDPOINT_URL++redis-cli -c -h localhost -p <configuration-endpoint-port> set mykey "hello"+```+### Cache Subnet GroupsA subnet group's VPC and each subnet's availability zone are read from the subnets themselves, asdiff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex 62a5f4b6b3..8fc9f4941b 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -1063,6 +1063,14 @@ public interface EmulatorConfig {/** Docker network to attach ElastiCache containers to. Empty = default bridge. */Optional<String> dockerNetwork();++ /**+ * Hostname cluster-mode nodes announce in MOVED/ASK redirects and cluster topology,+ * and that is reported as the ConfigurationEndpoint. Must resolve to Floci from every+ * client location; set it when the main hostname only resolves inside Floci's Docker+ * network. Empty = the main hostname.+ */+ Optional<String> clusterAnnounceHostname();}interface MemoryDbServiceConfig {diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.javaindex 467d757829..f17f7a3895 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java@@ -593,6 +593,16 @@ public class ContainerLifecycleManager {return resolveEndpoint(inspect, containerPort);}+ /**+ * Resolves the container's IP on its Docker network, independent of whether Floci runs+ * natively or in a container. Used for addresses that sibling containers (not Floci itself)+ * must dial — e.g. ElastiCache cluster-bus peering between Valkey nodes.+ */+ public String resolveContainerNetworkIp(String containerId, String preferredNetwork) {+ InspectContainerResponse inspect = dockerClient.inspectContainerCmd(containerId).exec();+ return resolveContainerIp(inspect, preferredNetwork);+ }+/*** Returns the underlying DockerClient for operations not covered by this manager.* Prefer using manager methods when available.diff --git a/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java b/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.javaindex 0a3ae41a58..af1ae39e7e 100644--- a/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java+++ b/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java@@ -13,6 +13,7 @@ import io.github.hectorvent.floci.services.floci.ui.FlociUiManager;import io.github.hectorvent.floci.services.amazonmq.container.RabbitMqManager;import io.github.hectorvent.floci.services.kinesisanalytics.container.FlinkContainerManager;import io.github.hectorvent.floci.services.iam.IamService;+import io.github.hectorvent.floci.services.elasticache.ElastiCacheService;import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerManager;import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheMemcachedContainerManager;import io.github.hectorvent.floci.services.elasticache.proxy.ElastiCacheProxyManager;@@ -66,6 +67,7 @@ public class EmulatorLifecycle {private final ServiceRegistry serviceRegistry;private final EmulatorConfig config;private final IamService iamService;+ private final ElastiCacheService elastiCacheService;private final ElastiCacheContainerManager elastiCacheContainerManager;private final ElastiCacheMemcachedContainerManager elastiCacheMemcachedContainerManager;private final ElastiCacheProxyManager elastiCacheProxyManager;@@ -97,6 +99,7 @@ public class EmulatorLifecycle {public EmulatorLifecycle(StorageFactory storageFactory, ServiceRegistry serviceRegistry,EmulatorConfig config,IamService iamService,+ ElastiCacheService elastiCacheService,ElastiCacheContainerManager elastiCacheContainerManager,ElastiCacheMemcachedContainerManager elastiCacheMemcachedContainerManager,ElastiCacheProxyManager elastiCacheProxyManager,@@ -127,6 +130,7 @@ public class EmulatorLifecycle {this.serviceRegistry = serviceRegistry;this.config = config;this.iamService = iamService;+ this.elastiCacheService = elastiCacheService;this.elastiCacheContainerManager = elastiCacheContainerManager;this.elastiCacheMemcachedContainerManager = elastiCacheMemcachedContainerManager;this.elastiCacheProxyManager = elastiCacheProxyManager;@@ -189,6 +193,12 @@ public class EmulatorLifecycle {dynamodbStreamsPoller.startPersistedPollers();pipesService.startPersistedPollers();rdsService.restorePersistedRuntime();+ if (config.services().elasticache().enabled()) {+ elastiCacheService.restorePersistedRuntime().exceptionally(ex -> {+ LOG.warnv("ElastiCache cluster-mode restore failed: {0}", ex.getMessage());+ return null;+ });+ }if (config.services().elbv2().enabled()) {elbV2Service.restorePersistedRuntime();}diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.javaindex 67e8750516..7e10a17090 100644--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.java@@ -9,11 +9,13 @@ import io.github.hectorvent.floci.core.common.XmlBuilder;import io.github.hectorvent.floci.services.elasticache.model.AuthMode;import io.github.hectorvent.floci.services.elasticache.model.CacheCluster;import io.github.hectorvent.floci.services.elasticache.model.CacheParameterGroup;+import io.github.hectorvent.floci.services.elasticache.model.ClusterNode;import io.github.hectorvent.floci.services.elasticache.model.CacheSubnetGroup;import io.github.hectorvent.floci.services.elasticache.model.ElastiCacheUser;import io.github.hectorvent.floci.services.elasticache.model.Endpoint;import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroup;import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroupSettings;+import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroupStatus;import io.github.hectorvent.floci.services.elasticache.proxy.SigV4Validator;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;@@ -106,8 +108,25 @@ public class ElastiCacheQueryHandler {try {ReplicationGroup group = service.createReplicationGroup(- groupId, description != null ? description : "", authMode, authToken, region,- replicationGroupSettings(params), parseTags(params));+ new ElastiCacheService.CreateReplicationGroupRequest(+ groupId,+ description != null ? description : "",+ authMode,+ authToken,+ region,+ params.getFirst("Engine"),+ params.getFirst("EngineVersion"),+ params.getFirst("CacheNodeType"),+ params.getFirst("CacheParameterGroupName"),+ params.getFirst("CacheSubnetGroupName"),+ params.getFirst("ClusterMode"),+ intParam(params, "NumNodeGroups"),+ intParam(params, "ReplicasPerNodeGroup"),+ intParam(params, "NumCacheClusters"),+ boolParam(params, "AutomaticFailoverEnabled"),+ boolParam(params, "MultiAZEnabled"),+ replicationGroupSettings(params),+ parseTags(params)));String result = replicationGroupXml(group);return Response.ok(AwsQueryResponse.envelope("CreateReplicationGroup", AwsNamespaces.EC, result)).build();} catch (AwsException e) {@@ -137,6 +156,28 @@ public class ElastiCacheQueryHandler {}}+ private static Integer intParam(MultivaluedMap<String, String> params, String name) {+ String value = params.getFirst(name);+ if (value == null || value.isBlank()) {+ return null;+ }+ try {+ return Integer.valueOf(value.trim());+ } catch (NumberFormatException e) {+ throw new AwsException("InvalidParameterValue",+ "The parameter " + name + " must be an integer.", 400);+ }+ }++ private static Boolean boolParam(MultivaluedMap<String, String> params, String name) {+ String value = params.getFirst(name);+ if (value == null || value.isBlank()) {+ return null;+ }+ return Boolean.parseBoolean(value.trim());+ }++private Response handleDescribeReplicationGroups(MultivaluedMap<String, String> params) {String filterId = params.getFirst("ReplicationGroupId");try {@@ -295,12 +336,22 @@ public class ElastiCacheQueryHandler {private Response handleDescribeCacheClusters(MultivaluedMap<String, String> params) {String filterId = params.getFirst("CacheClusterId");+ boolean showNodeInfo = "true".equalsIgnoreCase(params.getFirst("ShowCacheNodeInfo"));try {- Collection<CacheCluster> clusterList = memcachedService.listCacheClusters(filterId);+ List<ElastiCacheService.MemberCacheCluster> members = service.listMemberCacheClusters(filterId);+ Collection<CacheCluster> clusterList;+ if (filterId != null && !filterId.isBlank() && !members.isEmpty()) {+ clusterList = List.of();+ } else {+ clusterList = memcachedService.listCacheClusters(filterId);+ }var xml = new XmlBuilder().start("CacheClusters");for (CacheCluster c : clusterList) {xml.raw(cacheClusterXml(c));}+ for (ElastiCacheService.MemberCacheCluster member : members) {+ xml.raw(memberCacheClusterXml(member, showNodeInfo));+ }xml.end("CacheClusters").start("Marker").end("Marker");return Response.ok(AwsQueryResponse.envelope("DescribeCacheClusters", AwsNamespaces.EC, xml.build())).build();} catch (AwsException e) {@@ -308,6 +359,63 @@ public class ElastiCacheQueryHandler {}}+ private String memberCacheClusterXml(ElastiCacheService.MemberCacheCluster member, boolean showNodeInfo) {+ ReplicationGroup g = member.group();+ boolean authTokenEnabled = g.getAuthMode() == AuthMode.PASSWORD;+ var xml = new XmlBuilder()+ .start("CacheCluster")+ .elem("CacheClusterId", member.cacheClusterId())+ .elem("CacheClusterStatus", memberCacheClusterStatus(g))+ .elem("NumCacheNodes", 1L)+ .elem("ReplicationGroupId", g.getReplicationGroupId())+ .elem("AutoMinorVersionUpgrade", true)+ .elem("AuthTokenEnabled", authTokenEnabled)+ .elem("TransitEncryptionEnabled", authTokenEnabled)+ .elem("AtRestEncryptionEnabled", false);+ if (g.getEngine() != null) {+ xml.elem("Engine", g.getEngine());+ }+ if (g.getEngineVersion() != null) {+ xml.elem("EngineVersion", g.getEngineVersion());+ }+ if (g.getCacheNodeType() != null) {+ xml.elem("CacheNodeType", g.getCacheNodeType());+ }+ if (g.getCacheParameterGroupName() != null) {+ xml.start("CacheParameterGroup")+ .elem("CacheParameterGroupName", g.getCacheParameterGroupName())+ .elem("ParameterApplyStatus", "in-sync")+ .end("CacheParameterGroup");+ }+ if (g.getCacheSubnetGroupName() != null) {+ xml.elem("CacheSubnetGroupName", g.getCacheSubnetGroupName());+ }+ if (showNodeInfo && g.getConfigurationEndpoint() != null) {+ xml.start("CacheNodes")+ .start("CacheNode")+ .elem("CacheNodeId", "0001")+ .elem("CacheNodeStatus", "available")+ .start("Endpoint")+ .elem("Address", g.getConfigurationEndpoint().address())+ .elem("Port", (long) member.port())+ .end("Endpoint")+ .end("CacheNode")+ .end("CacheNodes");+ }+ return xml.end("CacheCluster").build();+ }++ /**+ * Members mirror the group's status, except {@code create-failed}: that value is only legal+ * on ReplicationGroup.Status, so members report the documented {@code restore-failed}.+ */+ private static String memberCacheClusterStatus(ReplicationGroup g) {+ if (g.getStatus() == ReplicationGroupStatus.CREATE_FAILED) {+ return "restore-failed";+ }+ return g.getStatus().wireName();+ }+private Response handleDeleteCacheCluster(MultivaluedMap<String, String> params) {String clusterId = params.getFirst("CacheClusterId");if (clusterId == null || clusterId.isBlank()) {@@ -640,26 +748,46 @@ private Response handleCreateCacheParameterGroup(MultivaluedMap<String, String>private String replicationGroupXml(ReplicationGroup g) {Endpoint ep = g.getConfigurationEndpoint();boolean authTokenEnabled = g.getAuthMode() == AuthMode.PASSWORD;+ List<ElastiCacheService.MemberCacheCluster> members = service.memberCacheClusters(g);var xml = new XmlBuilder().start("ReplicationGroup").elem("ReplicationGroupId", g.getReplicationGroupId()).elem("Description", g.getDescription())- .elem("Status", g.getStatus().name().toLowerCase())+ .elem("Status", g.getStatus().wireName()).elem("AuthTokenEnabled", authTokenEnabled).elem("TransitEncryptionEnabled", authTokenEnabled).elem("AtRestEncryptionEnabled", g.isAtRestEncryptionEnabled())- .elem("ClusterEnabled", false)- .elem("MultiAZ", "disabled")- .elem("AutomaticFailover", "disabled")+ .elem("ClusterEnabled", g.isClusterEnabled())+ .elem("ClusterMode", g.isClusterEnabled() ? "enabled" : "disabled")+ .elem("MultiAZ", g.isMultiAzEnabled() ? "enabled" : "disabled")+ .elem("AutomaticFailover", g.isAutomaticFailoverEnabled() ? "enabled" : "disabled").elem("SnapshotRetentionLimit", (long) g.getSnapshotRetentionLimit());xml.elem("SnapshotWindow", g.getSnapshotWindow() != null? g.getSnapshotWindow() : ReplicationGroupSettings.DEFAULT_SNAPSHOT_WINDOW);if (g.getKmsKeyId() != null) {xml.elem("KmsKeyId", g.getKmsKeyId());}+ if (g.getEngine() != null) {+ xml.elem("Engine", g.getEngine());+ }+ if (g.getCacheNodeType() != null) {+ xml.elem("CacheNodeType", g.getCacheNodeType());+ }if (g.getArn() != null) {xml.elem("ARN", g.getArn());}+ xml.start("MemberClusters");+ for (ElastiCacheService.MemberCacheCluster member : members) {+ xml.elem("ClusterId", member.cacheClusterId());+ }+ xml.end("MemberClusters");+ xml.start("NodeGroups");+ if (g.isClusterEnabled() && !g.getClusterNodes().isEmpty()) {+ appendClusterModeNodeGroups(xml, g);+ } else {+ appendSingleNodeGroup(xml, g, members, ep);+ }+ xml.end("NodeGroups");if (ep != null) {xml.start("ConfigurationEndpoint").elem("Address", ep.address())@@ -669,6 +797,60 @@ private Response handleCreateCacheParameterGroup(MultivaluedMap<String, String>return xml.end("ReplicationGroup").build();}+ private static void appendClusterModeNodeGroups(XmlBuilder xml, ReplicationGroup g) {+ Map<String, List<ClusterNode>> byNodeGroup = new LinkedHashMap<>();+ for (ClusterNode node : g.getClusterNodes()) {+ byNodeGroup.computeIfAbsent(node.getNodeGroupId(), key -> new ArrayList<>()).add(node);+ }+ byNodeGroup.forEach((nodeGroupId, nodes) -> {+ xml.start("NodeGroup")+ .elem("NodeGroupId", nodeGroupId)+ .elem("Status", "available")+ .elem("Slots", nodes.getFirst().getSlots())+ .start("NodeGroupMembers");+ for (ClusterNode node : nodes) {+ xml.start("NodeGroupMember")+ .elem("CacheClusterId", node.getMemberClusterId())+ .elem("CacheNodeId", "0001")+ .end("NodeGroupMember");+ }+ xml.end("NodeGroupMembers").end("NodeGroup");+ });+ }++ private static void appendSingleNodeGroup(XmlBuilder xml, ReplicationGroup g,+ List<ElastiCacheService.MemberCacheCluster> members,+ Endpoint ep) {+ xml.start("NodeGroup")+ .elem("NodeGroupId", "0001")+ .elem("Status", "available");+ if (ep != null) {+ xml.start("PrimaryEndpoint")+ .elem("Address", ep.address())+ .elem("Port", (long) ep.port())+ .end("PrimaryEndpoint")+ .start("ReaderEndpoint")+ .elem("Address", ep.address())+ .elem("Port", (long) ep.port())+ .end("ReaderEndpoint");+ }+ xml.start("NodeGroupMembers");+ for (ElastiCacheService.MemberCacheCluster member : members) {+ xml.start("NodeGroupMember")+ .elem("CacheClusterId", member.cacheClusterId())+ .elem("CacheNodeId", "0001")+ .elem("CurrentRole", member.primary() ? "primary" : "replica");+ if (ep != null) {+ xml.start("ReadEndpoint")+ .elem("Address", ep.address())+ .elem("Port", (long) member.port())+ .end("ReadEndpoint");+ }+ xml.end("NodeGroupMember");+ }+ xml.end("NodeGroupMembers").end("NodeGroup");+ }+private String userXml(ElastiCacheUser u) {String authType = switch (u.getAuthMode()) {case IAM -> "iam";diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.javaindex f009664c9d..21fcb40200 100644--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java@@ -12,9 +12,11 @@ import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerHandle;import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerManager;+import io.github.hectorvent.floci.services.elasticache.container.ValkeyClusterFormation;import io.github.hectorvent.floci.services.elasticache.model.AuthMode;import io.github.hectorvent.floci.services.elasticache.model.CacheParameterGroup;import io.github.hectorvent.floci.services.elasticache.model.CacheSubnetGroup;+import io.github.hectorvent.floci.services.elasticache.model.ClusterNode;import io.github.hectorvent.floci.services.elasticache.model.Endpoint;import io.github.hectorvent.floci.services.elasticache.model.ElastiCacheUser;import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroup;@@ -26,6 +28,9 @@ import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;+import java.net.Inet4Address;+import java.net.InetAddress;+import java.net.UnknownHostException;import java.time.Instant;import java.util.ArrayList;import java.util.Collection;@@ -37,6 +42,7 @@ import java.util.List;import java.util.Locale;import java.util.Map;import java.util.Set;+import java.util.concurrent.CompletableFuture;import java.util.concurrent.ConcurrentHashMap;import java.util.regex.Pattern;@@ -55,6 +61,7 @@ public class ElastiCacheService implements ResourceProvider {private final StorageBackend<String, CacheSubnetGroup> subnetGroups;private final ElastiCacheContainerManager containerManager;private final ElastiCacheProxyManager proxyManager;+ private final ValkeyClusterFormation clusterFormation;private final EmulatorConfig config;private final KmsService kmsService;private final Ec2Service ec2Service;@@ -66,6 +73,7 @@ public class ElastiCacheService implements ResourceProvider {@Injectpublic ElastiCacheService(ElastiCacheContainerManager containerManager,ElastiCacheProxyManager proxyManager,+ ValkeyClusterFormation clusterFormation,StorageFactory storageFactory,EmulatorConfig config,Ec2Service ec2Service,@@ -74,6 +82,7 @@ public class ElastiCacheService implements ResourceProvider {this.kmsService = kmsService;this.containerManager = containerManager;this.proxyManager = proxyManager;+ this.clusterFormation = clusterFormation;this.config = config;this.ec2Service = ec2Service;this.regionResolver = regionResolver;@@ -87,6 +96,31 @@ public class ElastiCacheService implements ResourceProvider {new TypeReference<Map<String, CacheSubnetGroup>>() {});}+ /**+ * The CreateReplicationGroup parameters floci models. Anything unset falls back to the+ * defaults a bare create has always had: one non-cluster node behind one proxy port.+ */+ public record CreateReplicationGroupRequest(+ String replicationGroupId,+ String description,+ AuthMode authMode,+ String authToken,+ String region,+ String engine,+ String engineVersion,+ String cacheNodeType,+ String cacheParameterGroupName,+ String cacheSubnetGroupName,+ String clusterMode,+ Integer numNodeGroups,+ Integer replicasPerNodeGroup,+ Integer numCacheClusters,+ Boolean automaticFailoverEnabled,+ Boolean multiAzEnabled,+ ReplicationGroupSettings settings,+ Map<String, String> tags) {+ }+public ReplicationGroup createReplicationGroup(String groupId, String description,AuthMode authMode, String authToken, String region) {return createReplicationGroup(groupId, description, authMode, authToken, region,@@ -97,10 +131,19 @@ public class ElastiCacheService implements ResourceProvider {AuthMode authMode, String authToken, String region,ReplicationGroupSettings settings,Map<String, String> tags) {+ return createReplicationGroup(new CreateReplicationGroupRequest(groupId, description,+ authMode, authToken, region, null, null, null, null, null, null,+ null, null, null, null, null, settings, tags));+ }++ public ReplicationGroup createReplicationGroup(CreateReplicationGroupRequest request) {+ String groupId = request.replicationGroupId();+ ReplicationGroupSettings settings = request.settings() != null+ ? request.settings() : ReplicationGroupSettings.defaults();settings.validate();// resolved with the other validations, before a port is taken or a container startedReplicationGroupSettings resolvedSettings =- settings.withKmsKeyId(resolveKmsKeyArn(settings.kmsKeyId(), region));+ settings.withKmsKeyId(resolveKmsKeyArn(settings.kmsKeyId(), request.region()));if (groups.get(groupId).isPresent()) {throw new AwsException("ReplicationGroupAlreadyExistsFault","Replication group " + groupId + " already exists.", 400);@@ -113,47 +156,392 @@ public class ElastiCacheService implements ResourceProvider {}try {- int proxyPort = allocateProxyPort();- String image = config.services().elasticache().defaultImage();+ if (resolveClusterEnabled(request)) {+ return provisionClusterModeGroup(request, resolvedSettings);+ }+ return provisionSingleNodeGroup(request, resolvedSettings);+ } finally {+ provisioningGroupIds.remove(groupId);+ }+ }++ private ReplicationGroup provisionSingleNodeGroup(CreateReplicationGroupRequest request,+ ReplicationGroupSettings resolvedSettings) {+ String groupId = request.replicationGroupId();+ AuthMode authMode = request.authMode();+ int proxyPort = allocateProxyPort();+ String image = config.services().elasticache().defaultImage();- LOG.infov("Creating replication group {0} with authMode={1} on proxy port {2}",- groupId, authMode, String.valueOf(proxyPort));+ LOG.infov("Creating replication group {0} with authMode={1} on proxy port {2}",+ groupId, authMode, String.valueOf(proxyPort));++ ElastiCacheContainerHandle handle = null;+ try {+ handle = containerManager.start(groupId, image);++ String endpointHost = resolveEndpointHost();+ Endpoint endpoint = new Endpoint(endpointHost, proxyPort);+ ReplicationGroup group = new ReplicationGroup(+ groupId, request.description(), ReplicationGroupStatus.AVAILABLE,+ authMode, endpoint, Instant.now(), proxyPort);+ group.setContainerId(handle.getContainerId());+ group.setContainerHost(handle.getHost());+ group.setContainerPort(handle.getPort());+ group.setAuthToken(request.authToken());+ group.setArn(regionResolver.buildArn("elasticache", request.region(),+ "replicationgroup:" + groupId));+ group.setRegion(request.region());+ group.setNumCacheClusters(request.numCacheClusters() != null+ ? validateRange("NumCacheClusters", request.numCacheClusters(), 1, 6)+ : 1);+ applyCommonAttributes(group, request, resolvedSettings);++ proxyManager.startProxy(groupId, authMode, proxyPort,+ handle.getHost(), handle.getPort(),+ (username, password) -> validatePassword(groupId, username, password));- ElastiCacheContainerHandle handle = null;+ groups.put(groupId, group);+ LOG.infov("Replication group {0} created, endpoint={1}:{2}", groupId, endpointHost, String.valueOf(proxyPort));+ return group;+ } catch (RuntimeException e) {+ LOG.warnv("Replication group {0} provisioning failed, rolling back: {1}", groupId, e.getMessage());+ rollbackReplicationGroup(groupId, handle, proxyPort);+ throw e;+ }+ }++ private ReplicationGroup provisionClusterModeGroup(CreateReplicationGroupRequest request,+ ReplicationGroupSettings resolvedSettings) {+ String groupId = request.replicationGroupId();+ AuthMode authMode = request.authMode();+ int numNodeGroups = request.numNodeGroups() != null+ ? validateNumNodeGroups(request.numNodeGroups())+ : 1;+ int replicasPerNodeGroup = request.replicasPerNodeGroup() != null+ ? validateRange("ReplicasPerNodeGroup", request.replicasPerNodeGroup(), 0, 5)+ : 0;+ String image = config.services().elasticache().defaultImage();+ String endpointHost = resolveClusterAnnounceHost();+ String announceIp = resolveAnnounceIp(endpointHost);++ LOG.infov("Creating cluster-mode replication group {0} with {1} node group(s), {2} replica(s) each",+ groupId, String.valueOf(numNodeGroups), String.valueOf(replicasPerNodeGroup));++ List<ClusterNode> nodes = new ArrayList<>();+ List<ElastiCacheContainerHandle> handles = new ArrayList<>();+ List<String> startedProxyKeys = new ArrayList<>();+ String inFlightMemberId = null;+ try {+ for (int shard = 0; shard < numNodeGroups; shard++) {+ String nodeGroupId = String.format("%04d", shard + 1);+ int[] slots = ValkeyClusterFormation.slotRange(shard, numNodeGroups);+ for (int member = 0; member <= replicasPerNodeGroup; member++) {+ String memberId = groupId + "-" + nodeGroupId + "-" + String.format("%03d", member + 1);+ nodes.add(new ClusterNode(memberId, nodeGroupId, member == 0,+ allocateProxyPort(), slots[0] + "-" + slots[1]));+ }+ }++ List<ValkeyClusterFormation.Node> formationNodes = new ArrayList<>(nodes.size());+ for (ClusterNode node : nodes) {+ inFlightMemberId = node.getMemberClusterId();+ ElastiCacheContainerHandle handle = containerManager.start(+ node.getMemberClusterId(), image,+ clusterNodeFlags(endpointHost, announceIp, node.getProxyPort()));+ inFlightMemberId = null;+ handles.add(handle);+ node.setContainerId(handle.getContainerId());+ node.setContainerHost(handle.getHost());+ node.setContainerPort(handle.getPort());+ String networkIp = handle.getNetworkIp() != null ? handle.getNetworkIp() : handle.getHost();+ formationNodes.add(new ValkeyClusterFormation.Node(+ handle.getHost(), handle.getPort(), networkIp,+ Integer.parseInt(node.getNodeGroupId()) - 1, node.isPrimary()));+ }++ clusterFormation.form(groupId, formationNodes, numNodeGroups);++ for (int i = 0; i < nodes.size(); i++) {+ ClusterNode node = nodes.get(i);+ ElastiCacheContainerHandle handle = handles.get(i);+ proxyManager.startProxy(node.getMemberClusterId(), authMode, node.getProxyPort(),+ handle.getHost(), handle.getPort(),+ (username, password) -> validatePassword(groupId, username, password));+ startedProxyKeys.add(node.getMemberClusterId());+ }++ Endpoint configurationEndpoint = new Endpoint(endpointHost, nodes.getFirst().getProxyPort());+ ReplicationGroup group = new ReplicationGroup(+ groupId, request.description(), ReplicationGroupStatus.AVAILABLE,+ authMode, configurationEndpoint, Instant.now(), nodes.getFirst().getProxyPort());+ group.setAuthToken(request.authToken());+ group.setArn(regionResolver.buildArn("elasticache", request.region(),+ "replicationgroup:" + groupId));+ group.setRegion(request.region());+ group.setClusterEnabled(true);+ group.setNumNodeGroups(numNodeGroups);+ group.setReplicasPerNodeGroup(replicasPerNodeGroup);+ group.setNumCacheClusters(nodes.size());+ group.setClusterNodes(nodes);+ applyCommonAttributes(group, request, resolvedSettings);++ groups.put(groupId, group);+ LOG.infov("Cluster-mode replication group {0} created, configuration endpoint={1}:{2}",+ groupId, endpointHost, String.valueOf(configurationEndpoint.port()));+ return group;+ } catch (RuntimeException e) {+ LOG.warnv("Cluster-mode replication group {0} provisioning failed, rolling back: {1}",+ groupId, e.getMessage());+ rollbackClusterModeGroup(groupId, startedProxyKeys, handles, inFlightMemberId,+ nodes.stream().map(ClusterNode::getProxyPort).toList());+ throw e;+ }+ }++ private void applyCommonAttributes(ReplicationGroup group, CreateReplicationGroupRequest request,+ ReplicationGroupSettings resolvedSettings) {+ group.setEngine(request.engine() != null && !request.engine().isBlank()+ ? normalizeEngine(request.engine())+ : defaultEngineForImage());+ group.setEngineVersion(request.engineVersion() != null && !request.engineVersion().isBlank()+ ? request.engineVersion()+ : ("valkey".equals(group.getEngine()) ? "8.1" : "7.1"));+ group.setCacheNodeType(request.cacheNodeType() != null && !request.cacheNodeType().isBlank()+ ? request.cacheNodeType()+ : "cache.t4g.micro");+ group.setCacheParameterGroupName(request.cacheParameterGroupName());+ group.setCacheSubnetGroupName(request.cacheSubnetGroupName());+ group.setAutomaticFailoverEnabled(Boolean.TRUE.equals(request.automaticFailoverEnabled()));+ group.setMultiAzEnabled(Boolean.TRUE.equals(request.multiAzEnabled()));+ group.setSnapshotWindow(ReplicationGroupSettings.DEFAULT_SNAPSHOT_WINDOW);+ resolvedSettings.applyTo(group);+ if (request.tags() != null && !request.tags().isEmpty()) {+ group.setTags(new LinkedHashMap<>(request.tags()));+ }+ }++ private String defaultEngineForImage() {+ String image = config.services().elasticache().defaultImage();+ return image != null && image.contains("valkey") ? "valkey" : "redis";+ }++ /**+ * Cluster mode follows AWS's signal — the parameter group — plus the request shapes that+ * imply it: an explicit ClusterMode, or more than one node group. ClusterMode+ * {@code compatible} — the mid-migration state — is deliberately not modelled; it enables+ * cluster mode only when one of the other signals also does.+ */+ private boolean resolveClusterEnabled(CreateReplicationGroupRequest request) {+ if ("enabled".equalsIgnoreCase(request.clusterMode())) {+ return true;+ }+ String parameterGroupName = request.cacheParameterGroupName();+ if (parameterGroupName != null) {+ if (parameterGroupName.endsWith(".cluster.on")) {+ return true;+ }+ boolean customClusterGroup = findParameterGroup(parameterGroupName)+ .map(group -> "yes".equalsIgnoreCase(group.getParameters().get("cluster-enabled")))+ .orElse(false);+ if (customClusterGroup) {+ return true;+ }+ }+ return request.numNodeGroups() != null && request.numNodeGroups() > 1;+ }++ /**+ * Nodes announce the cluster announce hostname as their preferred endpoint, so MOVED/ASK+ * redirects and CLUSTER SLOTS hand clients the exact name floci reports as the configuration+ * endpoint — how (or whether) that name resolves inside floci's own container is irrelevant.+ * The best-effort IPv4 stays announced for consumers that read the address fields instead.+ */+ private static List<String> clusterNodeFlags(String endpointHost, String announceIp, int proxyPort) {+ return List.of(+ "--cluster-enabled", "yes",+ "--cluster-announce-hostname", endpointHost,+ "--cluster-preferred-endpoint-type", "hostname",+ "--cluster-announce-client-ipv4", announceIp,+ "--cluster-announce-port", String.valueOf(proxyPort),+ "--cluster-announce-bus-port", "16379");+ }++ private static String resolveAnnounceIp(String endpointHost) {+ try {+ for (InetAddress address : InetAddress.getAllByName(endpointHost)) {+ if (address instanceof Inet4Address) {+ return address.getHostAddress();+ }+ }+ } catch (UnknownHostException e) {+ LOG.warnv("Could not resolve {0} to an IPv4 for cluster announce, using 127.0.0.1: {1}",+ endpointHost, e.getMessage());+ }+ return "127.0.0.1";+ }++ private static int validateRange(String parameterName, int value, int min, int max) {+ if (value < min || value > max) {+ throw new AwsException("InvalidParameterValue",+ parameterName + " must be between " + min + " and " + max + ".", 400);+ }+ return value;+ }++ /** 500 is AWS's raised NumNodeGroups quota (the default is 90; 500 with a limit increase). */+ private static final int MAX_NODE_GROUPS = 500;++ private static int validateNumNodeGroups(int value) {+ if (value > MAX_NODE_GROUPS) {+ throw new AwsException("NodeGroupsPerReplicationGroupQuotaExceeded",+ "The request cannot be processed because it would exceed the maximum allowed"+ + " number of node groups (shards) in a single replication group."+ + " The maximum is " + MAX_NODE_GROUPS + ".", 400);+ }+ return validateRange("NumNodeGroups", value, 1, MAX_NODE_GROUPS);+ }++ private void rollbackClusterModeGroup(String groupId, List<String> startedProxyKeys,+ List<ElastiCacheContainerHandle> handles,+ String inFlightMemberId, Collection<Integer> proxyPorts) {+ for (String proxyKey : startedProxyKeys) {+ try {+ proxyManager.stopProxy(proxyKey);+ } catch (RuntimeException e) {+ LOG.warnv("Error stopping proxy {0} during rollback of group {1}: {2}",+ proxyKey, groupId, e.getMessage());+ }+ }+ for (ElastiCacheContainerHandle handle : handles) {+ try {+ containerManager.stop(handle);+ } catch (RuntimeException e) {+ LOG.warnv("Error stopping container {0} during rollback of group {1}: {2}",+ handle.getContainerId(), groupId, e.getMessage());+ }+ }+ if (inFlightMemberId != null) {+ try {+ containerManager.stopByGroupId(inFlightMemberId);+ } catch (RuntimeException e) {+ LOG.warnv("Error stopping in-flight container {0} during rollback of group {1}: {2}",+ inFlightMemberId, groupId, e.getMessage());+ }+ }+ for (int port : proxyPorts) {+ releaseProxyPort(port);+ }+ }++ /**+ * Brings the data plane back up for cluster-mode groups restored from disk. Invoked from+ * {@code EmulatorLifecycle} after {@code storageFactory.loadAll()}, alongside the other+ * services that restore persisted runtime. Only topology survives a restart — containers,+ * proxies and port reservations are process-local — so each group is re-provisioned from+ * its persisted node list, and a group whose data plane cannot come back is reported+ * {@code create-failed} instead of available. Cluster-mode-disabled groups are deliberately+ * out of scope: they keep their pre-existing behaviour of reporting available after a+ * restart without a restored runtime.+ *+ * <p>Container restarts and cluster formation can take a readiness timeout and a formation+ * timeout per group, so the data-plane work runs in the background and must not delay+ * emulator readiness. Groups are marked {@code creating} synchronously — each node's proxy+ * port is reserved at the same time, before any request can claim it — and flip to+ * {@code available} or {@code create-failed} as their restoration finishes.+ */+ public CompletableFuture<Void> restorePersistedRuntime() {+ List<ReplicationGroup> toRestore = new ArrayList<>();+ for (ReplicationGroup group : groups.scan(k -> true)) {+ if (!group.isClusterEnabled() || group.getClusterNodes().isEmpty()+ || group.getStatus() == ReplicationGroupStatus.DELETING) {+ continue;+ }+ List<Integer> reserved = new ArrayList<>();try {- handle = containerManager.start(groupId, image);-- String endpointHost = resolveEndpointHost();- Endpoint endpoint = new Endpoint(endpointHost, proxyPort);- ReplicationGroup group = new ReplicationGroup(- groupId, description, ReplicationGroupStatus.AVAILABLE,- authMode, endpoint, Instant.now(), proxyPort);- group.setContainerId(handle.getContainerId());- group.setContainerHost(handle.getHost());- group.setContainerPort(handle.getPort());- group.setAuthToken(authToken);- group.setArn(regionResolver.buildArn("elasticache", region, "replicationgroup:" + groupId));- group.setRegion(region);- group.setSnapshotWindow(ReplicationGroupSettings.DEFAULT_SNAPSHOT_WINDOW);- resolvedSettings.applyTo(group);- if (tags != null && !tags.isEmpty()) {- group.setTags(new LinkedHashMap<>(tags));+ for (ClusterNode node : group.getClusterNodes()) {+ node.setProxyPort(reserveOrAllocateProxyPort(node.getProxyPort()));+ reserved.add(node.getProxyPort());}+ group.setStatus(ReplicationGroupStatus.CREATING);+ toRestore.add(group);+ } catch (RuntimeException e) {+ reserved.forEach(this::releaseProxyPort);+ group.setStatus(ReplicationGroupStatus.CREATE_FAILED);+ group.setConfigurationEndpoint(null);+ LOG.warnv(e, "Failed to restore cluster-mode replication group {0}",+ group.getReplicationGroupId());+ }+ groups.put(group.getReplicationGroupId(), group);+ }+ if (toRestore.isEmpty()) {+ return CompletableFuture.completedFuture(null);+ }+ LOG.infov("Restoring {0} cluster-mode replication group(s) in the background",+ String.valueOf(toRestore.size()));+ return CompletableFuture.runAsync(() -> toRestore.forEach(this::restoreClusterModeGroup));+ }++ private void restoreClusterModeGroup(ReplicationGroup group) {+ String groupId = group.getReplicationGroupId();+ String image = config.services().elasticache().defaultImage();+ String endpointHost = resolveClusterAnnounceHost();+ String announceIp = resolveAnnounceIp(endpointHost);+ List<ClusterNode> nodes = group.getClusterNodes();++ List<ElastiCacheContainerHandle> handles = new ArrayList<>();+ List<String> startedProxyKeys = new ArrayList<>();+ List<Integer> reservedPorts = nodes.stream().map(ClusterNode::getProxyPort).toList();+ String inFlightMemberId = null;+ try {+ List<ValkeyClusterFormation.Node> formationNodes = new ArrayList<>(nodes.size());+ for (ClusterNode node : nodes) {+ inFlightMemberId = node.getMemberClusterId();+ ElastiCacheContainerHandle handle = containerManager.start(+ node.getMemberClusterId(), image,+ clusterNodeFlags(endpointHost, announceIp, node.getProxyPort()));+ inFlightMemberId = null;+ handles.add(handle);+ node.setContainerId(handle.getContainerId());+ node.setContainerHost(handle.getHost());+ node.setContainerPort(handle.getPort());+ String networkIp = handle.getNetworkIp() != null ? handle.getNetworkIp() : handle.getHost();+ formationNodes.add(new ValkeyClusterFormation.Node(+ handle.getHost(), handle.getPort(), networkIp,+ Integer.parseInt(node.getNodeGroupId()) - 1, node.isPrimary()));+ }- proxyManager.startProxy(groupId, authMode, proxyPort,+ clusterFormation.form(groupId, formationNodes, group.getNumNodeGroups());++ for (int i = 0; i < nodes.size(); i++) {+ ClusterNode node = nodes.get(i);+ ElastiCacheContainerHandle handle = handles.get(i);+ proxyManager.startProxy(node.getMemberClusterId(), group.getAuthMode(), node.getProxyPort(),handle.getHost(), handle.getPort(),(username, password) -> validatePassword(groupId, username, password));+ startedProxyKeys.add(node.getMemberClusterId());+ }+ group.setConfigurationEndpoint(new Endpoint(endpointHost, nodes.getFirst().getProxyPort()));+ group.setStatus(ReplicationGroupStatus.AVAILABLE);+ groups.put(groupId, group);+ LOG.infov("Restored cluster-mode replication group {0}: {1} node(s), configuration endpoint={2}:{3}",+ groupId, String.valueOf(nodes.size()), endpointHost,+ String.valueOf(group.getConfigurationEndpoint().port()));+ } catch (RuntimeException e) {+ rollbackClusterModeGroup(groupId, startedProxyKeys, handles, inFlightMemberId, reservedPorts);+ for (ClusterNode node : nodes) {+ node.setContainerId(null);+ node.setContainerHost(null);+ node.setContainerPort(0);+ }+ group.setStatus(ReplicationGroupStatus.CREATE_FAILED);+ group.setConfigurationEndpoint(null);+ try {groups.put(groupId, group);- LOG.infov("Replication group {0} created, endpoint={1}:{2}", groupId, endpointHost, String.valueOf(proxyPort));- return group;- } catch (RuntimeException e) {- LOG.warnv("Replication group {0} provisioning failed, rolling back: {1}", groupId, e.getMessage());- rollbackReplicationGroup(groupId, handle, proxyPort);- throw e;+ } catch (RuntimeException persistFailure) {+ e.addSuppressed(persistFailure);}- } finally {- provisioningGroupIds.remove(groupId);+ LOG.warnv(e, "Failed to restore cluster-mode replication group {0}", groupId);}}@@ -206,19 +594,73 @@ public class ElastiCacheService implements ResourceProvider {group.setStatus(ReplicationGroupStatus.DELETING);groups.put(groupId, group);- proxyManager.stopProxy(groupId);+ if (group.isClusterEnabled() && !group.getClusterNodes().isEmpty()) {+ for (ClusterNode node : group.getClusterNodes()) {+ proxyManager.stopProxy(node.getMemberClusterId());+ if (node.getContainerId() != null) {+ containerManager.stop(new ElastiCacheContainerHandle(+ node.getContainerId(), node.getMemberClusterId(),+ node.getContainerHost(), node.getContainerPort()));+ } else {+ // Transient container fields are lost across a Floci restart; the+ // deterministic container name still finds the node.+ containerManager.stopByGroupId(node.getMemberClusterId());+ }+ releaseProxyPort(node.getProxyPort());+ }+ } else {+ proxyManager.stopProxy(groupId);- if (group.getContainerId() != null) {- containerManager.stop(new ElastiCacheContainerHandle(- group.getContainerId(), groupId, group.getContainerHost(), group.getContainerPort()));- }+ if (group.getContainerId() != null) {+ containerManager.stop(new ElastiCacheContainerHandle(+ group.getContainerId(), groupId, group.getContainerHost(), group.getContainerPort()));+ }- releaseProxyPort(group.getProxyPort());+ releaseProxyPort(group.getProxyPort());+ }groups.delete(groupId);LOG.infov("Replication group {0} deleted", groupId);}}+ /**+ * A replication-group member viewed as the cache cluster AWS reports it as. The terraform+ * provider follows {@code MemberClusters} into DescribeCacheClusters, so every member name+ * a describe hands out must answer there.+ */+ public record MemberCacheCluster(ReplicationGroup group, String cacheClusterId,+ int port, boolean primary) {}++ public List<MemberCacheCluster> listMemberCacheClusters(String filterClusterId) {+ List<MemberCacheCluster> members = new ArrayList<>();+ for (ReplicationGroup group : groups.scan(k -> true)) {+ for (MemberCacheCluster member : memberCacheClusters(group)) {+ if (filterClusterId == null || filterClusterId.isBlank()+ || filterClusterId.equals(member.cacheClusterId())) {+ members.add(member);+ }+ }+ }+ return members;+ }++ public List<MemberCacheCluster> memberCacheClusters(ReplicationGroup group) {+ List<MemberCacheCluster> members = new ArrayList<>();+ if (group.isClusterEnabled() && !group.getClusterNodes().isEmpty()) {+ for (ClusterNode node : group.getClusterNodes()) {+ members.add(new MemberCacheCluster(group, node.getMemberClusterId(),+ node.getProxyPort(), node.isPrimary()));+ }+ return members;+ }+ for (int i = 1; i <= group.getNumCacheClusters(); i++) {+ members.add(new MemberCacheCluster(group,+ group.getReplicationGroupId() + "-" + String.format("%03d", i),+ group.getProxyPort(), i == 1));+ }+ return members;+ }+public ReplicationGroup modifyReplicationGroup(String groupId, List<String> userIdsToAdd,List<String> userIdsToRemove) {return modifyReplicationGroup(groupId, userIdsToAdd, userIdsToRemove,@@ -394,6 +836,18 @@ public class ElastiCacheService implements ResourceProvider {return config.hostname().orElse("localhost");}+ /**+ * Cluster-mode groups bake this name into MOVED/ASK redirects and topology responses that+ * every client must resolve, unlike the plain endpoint host that clients may override —+ * so a hostname resolvable only inside Floci's Docker network (e.g. a Compose service+ * name) needs the cluster-announce-hostname override.+ */+ private String resolveClusterAnnounceHost() {+ return config.services().elasticache().clusterAnnounceHostname()+ .filter(host -> !host.isBlank())+ .orElseGet(this::resolveEndpointHost);+ }+private int allocateProxyPort() {int base = config.services().elasticache().proxyBasePort();int max = config.services().elasticache().proxyMaxPort();@@ -416,6 +870,13 @@ public class ElastiCacheService implements ResourceProvider {usedPorts.remove(port);}+ private int reserveOrAllocateProxyPort(int persistedPort) {+ if (persistedPort > 0 && usedPorts.add(persistedPort)) {+ return persistedPort;+ }+ return allocateProxyPort();+ }+// ── Cache Subnet Groups ───────────────────────────────────────────────────/**diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerHandle.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerHandle.javaindex 264df02f8a..3b431cc7f4 100644--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerHandle.java+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerHandle.java@@ -11,6 +11,7 @@ public class ElastiCacheContainerHandle {private final String groupId;private final String host;private final int port;+ private String networkIp;private Closeable logStream;public ElastiCacheContainerHandle(String containerId, String groupId, String host, int port) {@@ -24,6 +25,11 @@ public class ElastiCacheContainerHandle {public String getGroupId() { return groupId; }public String getHost() { return host; }public int getPort() { return port; }++ /** The container's IP on its Docker network — the address sibling containers dial. */+ public String getNetworkIp() { return networkIp; }+ public void setNetworkIp(String networkIp) { this.networkIp = networkIp; }+public Closeable getLogStream() { return logStream; }public void setLogStream(Closeable logStream) { this.logStream = logStream; }}diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManager.javaindex f95e7dd283..8bfff344f5 100644--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManager.java@@ -70,6 +70,15 @@ public class ElastiCacheContainerManager {}public ElastiCacheContainerHandle start(String groupId, String image) {+ return start(groupId, image, List.of());+ }++ /**+ * Starts a backend container for the given resource id, appending {@code extraServerFlags}+ * to the Valkey server command line (via the image's {@code VALKEY_EXTRA_FLAGS} hook).+ * Cluster-mode nodes use this to pass {@code --cluster-enabled} and announce settings.+ */+ public ElastiCacheContainerHandle start(String groupId, String image, List<String> extraServerFlags) {LOG.infov("Starting ElastiCache backend container for group: {0}", groupId);String containerName = containerName(groupId);@@ -77,12 +86,17 @@ public class ElastiCacheContainerManager {// Remove any stale container with the same namelifecycleManager.removeIfExists(containerName);+ StringBuilder serverFlags = new StringBuilder("--loglevel verbose");+ for (String flag : extraServerFlags) {+ serverFlags.append(' ').append(flag);+ }+// Build container spec. Only publish the backend port to the host in// native mode — in Docker mode the JVM reaches the container via its// network IP, no host binding needed.ContainerBuilder.Builder specBuilder = containerBuilder.newContainer(image).withName(containerName)- .withEnv("VALKEY_EXTRA_FLAGS", "--loglevel verbose")+ .withEnv("VALKEY_EXTRA_FLAGS", serverFlags.toString()).withDockerNetwork(config.services().elasticache().dockerNetwork()).withLogRotation().withLabels(ContainerStorageHelper.resourceIdentityLabels(@@ -104,6 +118,13 @@ public class ElastiCacheContainerManager {ElastiCacheContainerHandle handle = new ElastiCacheContainerHandle(info.containerId(), groupId, endpoint.host(), endpoint.port());+ try {+ handle.setNetworkIp(lifecycleManager.resolveContainerNetworkIp(+ info.containerId(), config.services().elasticache().dockerNetwork().orElse(null)));+ } catch (RuntimeException e) {+ LOG.warnv("Could not resolve network IP for ElastiCache container {0}: {1}",+ info.containerId(), e.getMessage());+ }activeContainers.put(groupId, handle);// Attach log streamingdiff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ValkeyClusterFormation.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ValkeyClusterFormation.javanew file mode 100644index 0000000000..3f61c5f661--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ValkeyClusterFormation.java@@ -0,0 +1,310 @@+package io.github.hectorvent.floci.services.elasticache.container;++import jakarta.enterprise.context.ApplicationScoped;+import org.jboss.logging.Logger;++import java.io.Closeable;+import java.io.IOException;+import java.io.InputStream;+import java.io.OutputStream;+import java.net.InetSocketAddress;+import java.net.Socket;+import java.nio.charset.StandardCharsets;+import java.util.ArrayList;+import java.util.List;++/**+ * Forms a Valkey cluster out of freshly started cluster-enabled nodes, the way+ * {@code valkey-cli --cluster create} does: distinct config epochs, MEET, slot+ * assignment on the primaries, then REPLICATE for the replicas.+ *+ * <p>Commands are issued from the JVM over each node's Floci-reachable endpoint;+ * the addresses handed to MEET are the nodes' Docker-network IPs, because that is+ * where the nodes reach <em>each other</em>.+ */+@ApplicationScoped+public class ValkeyClusterFormation {++ private static final Logger LOG = Logger.getLogger(ValkeyClusterFormation.class);++ private static final int BACKEND_PORT = 6379;+ private static final int TOTAL_SLOTS = 16384;+ private static final int CONNECT_TIMEOUT_MS = 5_000;+ private static final int READ_TIMEOUT_MS = 10_000;+ private static final long FORMATION_DEADLINE_MS = 60_000;+ private static final long RETRY_SLEEP_MS = 200;++ /**+ * One node to join into the cluster.+ *+ * @param endpointHost host Floci uses to reach the node+ * @param endpointPort port Floci uses to reach the node+ * @param networkIp the node's Docker-network IP, dialled by its peers+ * @param nodeGroup zero-based shard index+ * @param primary whether this node holds the shard's slots+ */+ public record Node(String endpointHost, int endpointPort, String networkIp,+ int nodeGroup, boolean primary) {}++ public void form(String groupId, List<Node> nodes, int numNodeGroups) {+ long deadline = System.currentTimeMillis() + FORMATION_DEADLINE_MS;+ List<RespClient> clients = new ArrayList<>(nodes.size());+ try {+ for (Node node : nodes) {+ clients.add(new RespClient(node.endpointHost(), node.endpointPort()));+ }++ List<String> nodeIds = new ArrayList<>(nodes.size());+ for (RespClient client : clients) {+ nodeIds.add(client.callString("CLUSTER", "MYID"));+ }++ for (int i = 0; i < clients.size(); i++) {+ try {+ clients.get(i).callString("CLUSTER", "SET-CONFIG-EPOCH", String.valueOf(i + 1));+ } catch (RespError e) {+ LOG.debugv("SET-CONFIG-EPOCH on node {0} of group {1}: {2}",+ String.valueOf(i), groupId, e.getMessage());+ }+ }++ for (int i = 1; i < nodes.size(); i++) {+ clients.getFirst().callString("CLUSTER", "MEET",+ nodes.get(i).networkIp(), String.valueOf(BACKEND_PORT));+ }++ for (int i = 0; i < nodes.size(); i++) {+ Node node = nodes.get(i);+ if (node.primary()) {+ int[] range = slotRange(node.nodeGroup(), numNodeGroups);+ clients.get(i).callString("CLUSTER", "ADDSLOTSRANGE",+ String.valueOf(range[0]), String.valueOf(range[1]));+ }+ }++ awaitKnownNodes(groupId, clients, nodes.size(), deadline);++ for (int i = 0; i < nodes.size(); i++) {+ Node node = nodes.get(i);+ if (!node.primary()) {+ String primaryId = nodeIds.get(primaryIndex(nodes, node.nodeGroup()));+ replicateWithRetry(groupId, clients.get(i), primaryId, deadline);+ }+ }++ awaitClusterOk(groupId, clients, deadline);+ LOG.infov("Valkey cluster for group {0} formed: {1} shard(s), {2} node(s)",+ groupId, String.valueOf(numNodeGroups), String.valueOf(nodes.size()));+ } catch (IOException e) {+ throw new RuntimeException("Cluster formation for group " + groupId + " failed: " + e.getMessage(), e);+ } finally {+ clients.forEach(RespClient::closeQuietly);+ }+ }++ /** The contiguous slot range served by the given shard, covering all 16384 slots overall. */+ public static int[] slotRange(int nodeGroup, int numNodeGroups) {+ int start = nodeGroup * TOTAL_SLOTS / numNodeGroups;+ int end = (nodeGroup + 1) * TOTAL_SLOTS / numNodeGroups - 1;+ return new int[] {start, end};+ }++ private static int primaryIndex(List<Node> nodes, int nodeGroup) {+ for (int i = 0; i < nodes.size(); i++) {+ if (nodes.get(i).primary() && nodes.get(i).nodeGroup() == nodeGroup) {+ return i;+ }+ }+ throw new IllegalStateException("No primary for node group " + nodeGroup);+ }++ private static void replicateWithRetry(String groupId, RespClient replica,+ String primaryId, long deadline) throws IOException {+ while (true) {+ try {+ replica.callString("CLUSTER", "REPLICATE", primaryId);+ return;+ } catch (RespError e) {+ // The replica may not have gossiped the primary yet — retry until the deadline.+ if (System.currentTimeMillis() >= deadline) {+ throw new RuntimeException("Cluster formation for group " + groupId+ + " timed out waiting to replicate " + primaryId + ": " + e.getMessage(), e);+ }+ sleep(groupId);+ }+ }+ }++ private static void awaitKnownNodes(String groupId, List<RespClient> clients,+ int expected, long deadline) throws IOException {+ while (!allMatch(clients, info -> parseInfoField(info, "cluster_known_nodes") >= expected)) {+ if (System.currentTimeMillis() >= deadline) {+ throw new RuntimeException("Cluster formation for group " + groupId+ + " timed out waiting for all " + expected + " nodes to meet");+ }+ sleep(groupId);+ }+ }++ private static void awaitClusterOk(String groupId, List<RespClient> clients,+ long deadline) throws IOException {+ while (!allMatch(clients, info -> info.contains("cluster_state:ok"))) {+ if (System.currentTimeMillis() >= deadline) {+ throw new RuntimeException("Cluster formation for group " + groupId+ + " timed out waiting for cluster_state:ok");+ }+ sleep(groupId);+ }+ }++ private static boolean allMatch(List<RespClient> clients,+ java.util.function.Predicate<String> infoPredicate) throws IOException {+ for (RespClient client : clients) {+ if (!infoPredicate.test(client.callString("CLUSTER", "INFO"))) {+ return false;+ }+ }+ return true;+ }++ private static long parseInfoField(String info, String field) {+ for (String line : info.split("\r?\n")) {+ if (line.startsWith(field + ":")) {+ try {+ return Long.parseLong(line.substring(field.length() + 1).trim());+ } catch (NumberFormatException e) {+ return -1;+ }+ }+ }+ return -1;+ }++ private static void sleep(String groupId) {+ try {+ Thread.sleep(RETRY_SLEEP_MS);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new RuntimeException("Interrupted while forming cluster for group " + groupId, e);+ }+ }++ /** An {@code -ERR} reply from the server, distinct from transport failures. */+ static final class RespError extends IOException {+ RespError(String message) {+ super(message);+ }+ }++ /** Minimal RESP2 client: sends one command array, reads one reply. */+ static final class RespClient implements Closeable {++ private final Socket socket;+ private final InputStream in;+ private final OutputStream out;++ RespClient(String host, int port) throws IOException {+ socket = new Socket();+ socket.connect(new InetSocketAddress(host, port), CONNECT_TIMEOUT_MS);+ socket.setTcpNoDelay(true);+ socket.setSoTimeout(READ_TIMEOUT_MS);+ in = socket.getInputStream();+ out = socket.getOutputStream();+ }++ String callString(String... args) throws IOException {+ Object reply = call(args);+ return reply == null ? null : reply.toString();+ }++ Object call(String... args) throws IOException {+ StringBuilder header = new StringBuilder();+ header.append('*').append(args.length).append("\r\n");+ out.write(header.toString().getBytes(StandardCharsets.UTF_8));+ for (String arg : args) {+ byte[] bytes = arg.getBytes(StandardCharsets.UTF_8);+ out.write(("$" + bytes.length + "\r\n").getBytes(StandardCharsets.UTF_8));+ out.write(bytes);+ out.write('\r');+ out.write('\n');+ }+ out.flush();+ return readReply();+ }++ private Object readReply() throws IOException {+ int type = in.read();+ if (type == -1) {+ throw new IOException("Connection closed while awaiting reply");+ }+ String line = readLine();+ return switch (type) {+ case '+' -> line;+ case '-' -> throw new RespError(line);+ case ':' -> Long.parseLong(line);+ case '$' -> readBulk(Integer.parseInt(line));+ case '*' -> readArray(Integer.parseInt(line));+ default -> throw new IOException("Unexpected RESP type: " + (char) type);+ };+ }++ private String readBulk(int length) throws IOException {+ if (length < 0) {+ return null;+ }+ byte[] data = in.readNBytes(length);+ if (data.length != length) {+ throw new IOException("Truncated bulk reply");+ }+ expectCrLf();+ return new String(data, StandardCharsets.UTF_8);+ }++ private List<Object> readArray(int count) throws IOException {+ if (count < 0) {+ return null;+ }+ List<Object> items = new ArrayList<>(count);+ for (int i = 0; i < count; i++) {+ items.add(readReply());+ }+ return items;+ }++ private String readLine() throws IOException {+ StringBuilder sb = new StringBuilder();+ int b;+ while ((b = in.read()) != -1) {+ if (b == '\r') {+ int next = in.read();+ if (next != '\n') {+ throw new IOException("Malformed RESP line terminator");+ }+ return sb.toString();+ }+ sb.append((char) b);+ }+ throw new IOException("Connection closed mid-line");+ }++ private void expectCrLf() throws IOException {+ if (in.read() != '\r' || in.read() != '\n') {+ throw new IOException("Missing CRLF after bulk reply");+ }+ }++ @Override+ public void close() throws IOException {+ socket.close();+ }++ void closeQuietly() {+ try {+ close();+ } catch (IOException e) {+ LOG.debugv("Ignoring close failure for formation connection to {0}: {1}",+ socket.getRemoteSocketAddress(), e.getMessage());+ }+ }+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ClusterNode.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ClusterNode.javanew file mode 100644index 0000000000..535ce938b9--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ClusterNode.java@@ -0,0 +1,57 @@+package io.github.hectorvent.floci.services.elasticache.model;++import io.quarkus.runtime.annotations.RegisterForReflection;++/**+ * One node of a cluster-mode-enabled replication group: a member cache cluster+ * backed by its own Valkey container and auth-proxy port.+ */+@RegisterForReflection+public class ClusterNode {++ private String memberClusterId;+ private String nodeGroupId;+ private boolean primary;+ private int proxyPort;+ private String slots;++ // Transient fields — not persisted, restored on container restart+ private transient String containerId;+ private transient String containerHost;+ private transient int containerPort;++ public ClusterNode() {}++ public ClusterNode(String memberClusterId, String nodeGroupId, boolean primary,+ int proxyPort, String slots) {+ this.memberClusterId = memberClusterId;+ this.nodeGroupId = nodeGroupId;+ this.primary = primary;+ this.proxyPort = proxyPort;+ this.slots = slots;+ }++ public String getMemberClusterId() { return memberClusterId; }+ public void setMemberClusterId(String memberClusterId) { this.memberClusterId = memberClusterId; }++ public String getNodeGroupId() { return nodeGroupId; }+ public void setNodeGroupId(String nodeGroupId) { this.nodeGroupId = nodeGroupId; }++ public boolean isPrimary() { return primary; }+ public void setPrimary(boolean primary) { this.primary = primary; }++ public int getProxyPort() { return proxyPort; }+ public void setProxyPort(int proxyPort) { this.proxyPort = proxyPort; }++ public String getSlots() { return slots; }+ public void setSlots(String slots) { this.slots = slots; }++ public String getContainerId() { return containerId; }+ public void setContainerId(String containerId) { this.containerId = containerId; }++ public String getContainerHost() { return containerHost; }+ public void setContainerHost(String containerHost) { this.containerHost = containerHost; }++ public int getContainerPort() { return containerPort; }+ public void setContainerPort(int containerPort) { this.containerPort = containerPort; }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroup.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroup.javaindex d2e418fd0a..92875f9ff5 100644--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroup.java+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroup.java@@ -3,8 +3,10 @@ package io.github.hectorvent.floci.services.elasticache.model;import io.quarkus.runtime.annotations.RegisterForReflection;import java.time.Instant;+import java.util.ArrayList;import java.util.HashSet;import java.util.LinkedHashMap;+import java.util.List;import java.util.Map;import java.util.Set;@@ -27,6 +29,18 @@ public class ReplicationGroup {private int snapshotRetentionLimit;private String snapshotWindow;private Map<String, String> tags = new LinkedHashMap<>();+ private boolean clusterEnabled;+ private int numNodeGroups = 1;+ private int replicasPerNodeGroup;+ private int numCacheClusters = 1;+ private String engine;+ private String engineVersion;+ private String cacheNodeType;+ private String cacheParameterGroupName;+ private String cacheSubnetGroupName;+ private boolean automaticFailoverEnabled;+ private boolean multiAzEnabled;+ private List<ClusterNode> clusterNodes = new ArrayList<>();// Transient fields — not persisted, restored on container restartprivate transient String containerId;@@ -105,4 +119,42 @@ public class ReplicationGroup {public String getRegion() { return region; }public void setRegion(String region) { this.region = region; }++ public boolean isClusterEnabled() { return clusterEnabled; }+ public void setClusterEnabled(boolean clusterEnabled) { this.clusterEnabled = clusterEnabled; }++ public int getNumNodeGroups() { return numNodeGroups; }+ public void setNumNodeGroups(int numNodeGroups) { this.numNodeGroups = numNodeGroups; }++ public int getReplicasPerNodeGroup() { return replicasPerNodeGroup; }+ public void setReplicasPerNodeGroup(int replicasPerNodeGroup) { this.replicasPerNodeGroup = replicasPerNodeGroup; }++ public int getNumCacheClusters() { return numCacheClusters; }+ public void setNumCacheClusters(int numCacheClusters) { this.numCacheClusters = numCacheClusters; }++ public String getEngine() { return engine; }+ public void setEngine(String engine) { this.engine = engine; }++ public String getEngineVersion() { return engineVersion; }+ public void setEngineVersion(String engineVersion) { this.engineVersion = engineVersion; }++ public String getCacheNodeType() { return cacheNodeType; }+ public void setCacheNodeType(String cacheNodeType) { this.cacheNodeType = cacheNodeType; }++ public String getCacheParameterGroupName() { return cacheParameterGroupName; }+ public void setCacheParameterGroupName(String cacheParameterGroupName) { this.cacheParameterGroupName = cacheParameterGroupName; }++ public String getCacheSubnetGroupName() { return cacheSubnetGroupName; }+ public void setCacheSubnetGroupName(String cacheSubnetGroupName) { this.cacheSubnetGroupName = cacheSubnetGroupName; }++ public boolean isAutomaticFailoverEnabled() { return automaticFailoverEnabled; }+ public void setAutomaticFailoverEnabled(boolean automaticFailoverEnabled) { this.automaticFailoverEnabled = automaticFailoverEnabled; }++ public boolean isMultiAzEnabled() { return multiAzEnabled; }+ public void setMultiAzEnabled(boolean multiAzEnabled) { this.multiAzEnabled = multiAzEnabled; }++ public List<ClusterNode> getClusterNodes() { return clusterNodes; }+ public void setClusterNodes(List<ClusterNode> clusterNodes) {+ this.clusterNodes = clusterNodes != null ? clusterNodes : new ArrayList<>();+ }}diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroupStatus.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroupStatus.javaindex bd59857129..e5bd61f478 100644--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroupStatus.java+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroupStatus.java@@ -2,7 +2,14 @@ package io.github.hectorvent.floci.services.elasticache.model;import io.quarkus.runtime.annotations.RegisterForReflection;+import java.util.Locale;+@RegisterForReflectionpublic enum ReplicationGroupStatus {- CREATING, AVAILABLE, DELETING+ CREATING, AVAILABLE, DELETING, CREATE_FAILED;++ /** The status string as AWS reports it, e.g. {@code create-failed}. */+ public String wireName() {+ return name().toLowerCase(Locale.ROOT).replace('_', '-');+ }}diff --git a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.javaindex 6dd6c20da6..0590a5e18c 100644--- a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java+++ b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java@@ -7,6 +7,7 @@ import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.lifecycle.InitLifecycleState;import io.github.hectorvent.floci.lifecycle.inithook.InitializationHook;import io.github.hectorvent.floci.lifecycle.inithook.InitializationHooksRunner;+import io.github.hectorvent.floci.services.elasticache.ElastiCacheService;import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerManager;import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheMemcachedContainerManager;import io.github.hectorvent.floci.services.elasticache.proxy.ElastiCacheProxyManager;@@ -60,6 +61,8 @@ class EmulatorLifecycleTest {@Mock private EmulatorConfig.Ec2ServiceConfig ec2ServiceConfig;@Mock private EmulatorConfig.ElbV2ServiceConfig elbv2ServiceConfig;@Mock private IamService iamService;+ @Mock private EmulatorConfig.ElastiCacheServiceConfig elastiCacheServiceConfig;+ @Mock private ElastiCacheService elastiCacheService;@Mock private ElastiCacheContainerManager elastiCacheContainerManager;@Mock private ElastiCacheMemcachedContainerManager elastiCacheMemcachedContainerManager;@Mock private ElastiCacheProxyManager elastiCacheProxyManager;@@ -97,13 +100,15 @@ class EmulatorLifecycleTest {Mockito.lenient().when(ec2ServiceConfig.enabled()).thenReturn(false);Mockito.lenient().when(servicesConfig.elbv2()).thenReturn(elbv2ServiceConfig);Mockito.lenient().when(elbv2ServiceConfig.enabled()).thenReturn(false);+ Mockito.lenient().when(servicesConfig.elasticache()).thenReturn(elastiCacheServiceConfig);+ Mockito.lenient().when(elastiCacheServiceConfig.enabled()).thenReturn(false);Mockito.lenient().when(config.tls()).thenReturn(tlsConfig);Mockito.lenient().when(tlsConfig.enabled()).thenReturn(false);Mockito.lenient().when(config.port()).thenReturn(4566);emulatorLifecycle = new EmulatorLifecycle(storageFactory, serviceRegistry, config,- iamService,+ iamService, elastiCacheService,elastiCacheContainerManager, elastiCacheMemcachedContainerManager,elastiCacheProxyManager, rdsContainerManager, rdsProxyManager,memoryDbContainerManager, memoryDbProxyManager,@@ -170,6 +175,35 @@ class EmulatorLifecycleTest {Mockito.verify(elbV2Service, Mockito.never()).restorePersistedRuntime();}+ @Test+ @DisplayName("Should restore ElastiCache persisted runtime after loading storage when elasticache is enabled")+ void shouldRestoreElastiCachePersistedRuntimeAfterStorageLoad() {+ stubStorageConfig();+ when(elastiCacheServiceConfig.enabled()).thenReturn(true);+ when(elastiCacheService.restorePersistedRuntime())+ .thenReturn(java.util.concurrent.CompletableFuture.completedFuture(null));+ when(initializationHooksRunner.hasHooks(InitializationHook.START)).thenReturn(false);+ when(initializationHooksRunner.hasHooks(InitializationHook.READY)).thenReturn(false);++ emulatorLifecycle.onStart(Mockito.mock(StartupEvent.class));++ var inOrder = Mockito.inOrder(storageFactory, elastiCacheService);+ inOrder.verify(storageFactory).loadAll();+ inOrder.verify(elastiCacheService).restorePersistedRuntime();+ }++ @Test+ @DisplayName("Should not restore ElastiCache persisted runtime when elasticache is disabled")+ void shouldNotRestoreElastiCachePersistedRuntimeWhenDisabled() {+ stubStorageConfig();+ when(initializationHooksRunner.hasHooks(InitializationHook.START)).thenReturn(false);+ when(initializationHooksRunner.hasHooks(InitializationHook.READY)).thenReturn(false);++ emulatorLifecycle.onStart(Mockito.mock(StartupEvent.class));++ Mockito.verify(elastiCacheService, Mockito.never()).restorePersistedRuntime();+ }+@Test@DisplayName("Should rehydrate AppSync schemas after orphan recovery on startup")void shouldRehydrateAppSyncSchemasAfterOrphanRecovery() {diff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheClusterIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheClusterIntegrationTest.javanew file mode 100644index 0000000000..76ee86d839--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheClusterIntegrationTest.java@@ -0,0 +1,266 @@+package io.github.hectorvent.floci.services.elasticache;++import io.quarkus.test.junit.QuarkusTest;+import io.restassured.path.xml.XmlPath;+import org.junit.jupiter.api.AfterAll;+import org.junit.jupiter.api.Assumptions;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.MethodOrderer;+import org.junit.jupiter.api.Order;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.TestMethodOrder;++import java.io.IOException;+import java.io.InputStream;+import java.io.OutputStream;+import java.net.Socket;+import java.nio.charset.StandardCharsets;+import java.util.List;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.equalTo;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertTrue;++/**+ * End-to-end coverage for cluster-mode replication groups: real sharded Valkey+ * topology behind per-node auth proxies, honest describe output, and MOVED+ * redirects that point clients at the proxy ports.+ *+ * <p>Key slots used below are the well-known CRC16 values: {@code bar} → 5061+ * (first shard of two) and {@code foo} → 12182 (second shard of two).+ */+@QuarkusTest+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class ElastiCacheClusterIntegrationTest {++ private static final String AUTH_HEADER =+ "AWS4-HMAC-SHA256 Credential=test/20260412/us-east-1/elasticache/aws4_request";+ private static final String GROUP_ID = "it-ec-cluster";++ private static final int SOCKET_TIMEOUT_MS = 10_000;++ private static int configurationEndpointPort;+ private static int secondShardPort;++ @BeforeAll+ static void requireDocker() {+ Assumptions.assumeTrue(isDockerAvailable(), "Docker daemon must be available for ElastiCache integration tests");+ }++ @AfterAll+ static void cleanup() {+ try {+ given()+ .formParam("Action", "DeleteReplicationGroup")+ .formParam("ReplicationGroupId", GROUP_ID)+ .header("Authorization", AUTH_HEADER)+ .post("/");+ } catch (Exception e) {+ System.err.println("Cleanup of " + GROUP_ID + " failed: " + e.getMessage());+ }+ }++ @Test+ @Order(1)+ void createClusterModeReplicationGroup() {+ configurationEndpointPort =+ given()+ .formParam("Action", "CreateReplicationGroup")+ .formParam("ReplicationGroupId", GROUP_ID)+ .formParam("ReplicationGroupDescription", "Cluster mode integration test")+ .formParam("Engine", "valkey")+ .formParam("EngineVersion", "8.2")+ .formParam("CacheNodeType", "cache.t4g.micro")+ .formParam("CacheParameterGroupName", "default.valkey8.cluster.on")+ .formParam("NumNodeGroups", "2")+ .formParam("ReplicasPerNodeGroup", "1")+ .formParam("AutomaticFailoverEnabled", "true")+ .header("Authorization", AUTH_HEADER)+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("CreateReplicationGroupResponse.CreateReplicationGroupResult.ReplicationGroup.ReplicationGroupId", equalTo(GROUP_ID))+ .body("CreateReplicationGroupResponse.CreateReplicationGroupResult.ReplicationGroup.Status", equalTo("available"))+ .body("CreateReplicationGroupResponse.CreateReplicationGroupResult.ReplicationGroup.ClusterEnabled", equalTo("true"))+ .body("CreateReplicationGroupResponse.CreateReplicationGroupResult.ReplicationGroup.AutomaticFailover", equalTo("enabled"))+ .body("CreateReplicationGroupResponse.CreateReplicationGroupResult.ReplicationGroup.Engine", equalTo("valkey"))+ .extract()+ .xmlPath()+ .getInt("CreateReplicationGroupResponse.CreateReplicationGroupResult.ReplicationGroup.ConfigurationEndpoint.Port");+ }++ @Test+ @Order(2)+ void describeReportsClusterTopology() {+ XmlPath xml = given()+ .formParam("Action", "DescribeReplicationGroups")+ .formParam("ReplicationGroupId", GROUP_ID)+ .header("Authorization", AUTH_HEADER)+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .extract()+ .xmlPath();++ String prefix = "DescribeReplicationGroupsResponse.DescribeReplicationGroupsResult.ReplicationGroups.ReplicationGroup.";+ assertEquals("true", xml.getString(prefix + "ClusterEnabled"));+ List<String> memberClusters = xml.getList(prefix + "MemberClusters.ClusterId");+ assertEquals(List.of(GROUP_ID + "-0001-001", GROUP_ID + "-0001-002",+ GROUP_ID + "-0002-001", GROUP_ID + "-0002-002"), memberClusters);+ assertEquals(List.of("0001", "0002"), xml.getList(prefix + "NodeGroups.NodeGroup.NodeGroupId"));+ assertEquals(List.of("0-8191", "8192-16383"), xml.getList(prefix + "NodeGroups.NodeGroup.Slots"));+ }++ @Test+ @Order(3)+ void describeCacheClustersAnswersForMembers() {+ secondShardPort = given()+ .formParam("Action", "DescribeCacheClusters")+ .formParam("CacheClusterId", GROUP_ID + "-0002-001")+ .formParam("ShowCacheNodeInfo", "true")+ .header("Authorization", AUTH_HEADER)+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("DescribeCacheClustersResponse.DescribeCacheClustersResult.CacheClusters.CacheCluster.CacheClusterId",+ equalTo(GROUP_ID + "-0002-001"))+ .body("DescribeCacheClustersResponse.DescribeCacheClustersResult.CacheClusters.CacheCluster.ReplicationGroupId",+ equalTo(GROUP_ID))+ .body("DescribeCacheClustersResponse.DescribeCacheClustersResult.CacheClusters.CacheCluster.Engine",+ equalTo("valkey"))+ .body("DescribeCacheClustersResponse.DescribeCacheClustersResult.CacheClusters.CacheCluster.EngineVersion",+ equalTo("8.2"))+ .extract()+ .xmlPath()+ .getInt("DescribeCacheClustersResponse.DescribeCacheClustersResult.CacheClusters.CacheCluster.CacheNodes.CacheNode.Endpoint.Port");+ }++ @Test+ @Order(4)+ void clusterStateIsOkThroughTheProxy() throws Exception {+ try (Socket socket = openSocket(configurationEndpointPort)) {+ write(socket, respArray("PING"));+ assertEquals("+PONG\r\n", readLine(socket));++ write(socket, respArray("CLUSTER", "INFO"));+ String info = readBulk(socket);+ assertTrue(info.contains("cluster_state:ok"), "Expected cluster_state:ok but got: " + info);+ assertTrue(info.contains("cluster_known_nodes:4"), "Expected 4 known nodes but got: " + info);+ }+ }++ @Test+ @Order(5)+ void movedRedirectPointsAtTheOtherShardsProxyPort() throws Exception {+ try (Socket socket = openSocket(configurationEndpointPort)) {+ write(socket, respArray("SET", "bar", "first-shard-value"));+ assertEquals("+OK\r\n", readLine(socket));++ write(socket, respArray("SET", "foo", "second-shard-value"));+ String reply = readLine(socket);+ assertTrue(reply.startsWith("-MOVED 12182 "), "Expected MOVED redirect but got: " + reply);+ String target = reply.trim().substring("-MOVED 12182 ".length());+ assertEquals("localhost:" + secondShardPort, target,+ "MOVED must point at the announced hostname and the second shard primary's proxy port");+ }++ try (Socket socket = openSocket(secondShardPort)) {+ write(socket, respArray("SET", "foo", "second-shard-value"));+ assertEquals("+OK\r\n", readLine(socket));++ write(socket, respArray("GET", "foo"));+ assertEquals("second-shard-value", readBulk(socket));+ }+ }++ @Test+ @Order(6)+ void deleteClusterModeReplicationGroup() {+ given()+ .formParam("Action", "DeleteReplicationGroup")+ .formParam("ReplicationGroupId", GROUP_ID)+ .header("Authorization", AUTH_HEADER)+ .when()+ .post("/")+ .then()+ .statusCode(200);++ given()+ .formParam("Action", "DescribeReplicationGroups")+ .formParam("ReplicationGroupId", GROUP_ID)+ .header("Authorization", AUTH_HEADER)+ .when()+ .post("/")+ .then()+ .statusCode(404);+ }++ private static boolean isDockerAvailable() {+ try {+ Process process = new ProcessBuilder("docker", "info").start();+ return process.waitFor() == 0;+ } catch (Exception e) {+ return false;+ }+ }++ private static Socket openSocket(int port) throws IOException {+ Socket socket = new Socket("localhost", port);+ socket.setTcpNoDelay(true);+ socket.setSoTimeout(SOCKET_TIMEOUT_MS);+ return socket;+ }++ private static void write(Socket socket, byte[] data) throws IOException {+ OutputStream out = socket.getOutputStream();+ out.write(data);+ out.flush();+ }++ private static byte[] respArray(String... args) {+ StringBuilder sb = new StringBuilder();+ sb.append('*').append(args.length).append("\r\n");+ for (String arg : args) {+ byte[] bytes = arg.getBytes(StandardCharsets.UTF_8);+ sb.append('$').append(bytes.length).append("\r\n").append(arg).append("\r\n");+ }+ return sb.toString().getBytes(StandardCharsets.UTF_8);+ }++ private static String readLine(Socket socket) throws IOException {+ InputStream in = socket.getInputStream();+ StringBuilder sb = new StringBuilder();+ int b;+ while ((b = in.read()) != -1) {+ sb.append((char) b);+ if (sb.length() >= 2 && sb.charAt(sb.length() - 2) == '\r' && sb.charAt(sb.length() - 1) == '\n') {+ return sb.toString();+ }+ }+ throw new IOException("Connection closed before line terminator; read so far: " + sb);+ }++ private static String readBulk(Socket socket) throws IOException {+ String header = readLine(socket);+ if (!header.startsWith("$")) {+ throw new IOException("Expected bulk reply but got: " + header);+ }+ int length = Integer.parseInt(header.substring(1).trim());+ if (length < 0) {+ return null;+ }+ InputStream in = socket.getInputStream();+ byte[] data = in.readNBytes(length);+ if (data.length != length) {+ throw new IOException("Truncated bulk reply");+ }+ if (in.read() != '\r' || in.read() != '\n') {+ throw new IOException("Missing CRLF after bulk reply");+ }+ return new String(data, StandardCharsets.UTF_8);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandlerTest.javaindex dd7afcf206..3037b45ec7 100644--- a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandlerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandlerTest.java@@ -1,6 +1,11 @@package io.github.hectorvent.floci.services.elasticache;import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.services.elasticache.model.AuthMode;+import io.github.hectorvent.floci.services.elasticache.model.ClusterNode;+import io.github.hectorvent.floci.services.elasticache.model.Endpoint;+import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroup;+import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroupStatus;import io.github.hectorvent.floci.services.elasticache.proxy.SigV4Validator;import jakarta.ws.rs.core.MultivaluedHashMap;import jakarta.ws.rs.core.MultivaluedMap;@@ -19,8 +24,12 @@ import static org.mockito.Mockito.*;import org.mockito.ArgumentCaptor;import org.junit.jupiter.api.Test;+import java.time.Instant;+import java.util.List;+import static org.junit.jupiter.api.Assertions.*;import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.when;/*** Verifies the empty-list read responses for the subnet/parameter group describes, so@@ -64,6 +73,124 @@ class ElastiCacheQueryHandlerTest {assertFalse(body.contains("<Marker>"), "Empty list must omit Marker");}+ @Test+ void describeReplicationGroups_reportsClusterModeTopology() {+ ReplicationGroup group = new ReplicationGroup("grp", "d", ReplicationGroupStatus.AVAILABLE,+ AuthMode.NO_AUTH, new Endpoint("localhost", 6379), Instant.now(), 6379);+ group.setClusterEnabled(true);+ group.setEngine("valkey");+ group.setAutomaticFailoverEnabled(true);+ group.setClusterNodes(List.of(+ new ClusterNode("grp-0001-001", "0001", true, 6379, "0-8191"),+ new ClusterNode("grp-0002-001", "0002", true, 6380, "8192-16383")));+ when(service.listReplicationGroups("grp")).thenReturn(List.of(group));+ when(service.memberCacheClusters(group)).thenReturn(List.of(+ new ElastiCacheService.MemberCacheCluster(group, "grp-0001-001", 6379, true),+ new ElastiCacheService.MemberCacheCluster(group, "grp-0002-001", 6380, true)));++ MultivaluedMap<String, String> params = params();+ params.putSingle("ReplicationGroupId", "grp");+ Response response = handler.handle("DescribeReplicationGroups", params, "us-east-1");++ assertEquals(200, response.getStatus());+ String body = (String) response.getEntity();+ assertTrue(body.contains("<ClusterEnabled>true</ClusterEnabled>"));+ assertTrue(body.contains("<ClusterMode>enabled</ClusterMode>"));+ assertTrue(body.contains("<AutomaticFailover>enabled</AutomaticFailover>"));+ assertTrue(body.contains("<Engine>valkey</Engine>"));+ assertTrue(body.contains("<ClusterId>grp-0001-001</ClusterId>"));+ assertTrue(body.contains("<ClusterId>grp-0002-001</ClusterId>"));+ assertTrue(body.contains("<NodeGroupId>0002</NodeGroupId>"));+ assertTrue(body.contains("<Slots>8192-16383</Slots>"));+ assertTrue(body.contains("<CacheClusterId>grp-0002-001</CacheClusterId>"));+ assertFalse(body.contains("<PrimaryEndpoint>"),+ "Cluster-mode node groups carry Slots, not a PrimaryEndpoint");+ }++ @Test+ void describeReplicationGroups_reportsSingleNodeGroupWithPrimaryEndpoint() {+ ReplicationGroup group = new ReplicationGroup("grp", "d", ReplicationGroupStatus.AVAILABLE,+ AuthMode.NO_AUTH, new Endpoint("localhost", 6379), Instant.now(), 6379);+ group.setEngine("valkey");+ when(service.listReplicationGroups("grp")).thenReturn(List.of(group));+ when(service.memberCacheClusters(group)).thenReturn(List.of(+ new ElastiCacheService.MemberCacheCluster(group, "grp-001", 6379, true)));++ MultivaluedMap<String, String> params = params();+ params.putSingle("ReplicationGroupId", "grp");+ Response response = handler.handle("DescribeReplicationGroups", params, "us-east-1");++ assertEquals(200, response.getStatus());+ String body = (String) response.getEntity();+ assertTrue(body.contains("<ClusterEnabled>false</ClusterEnabled>"));+ assertTrue(body.contains("<ClusterId>grp-001</ClusterId>"));+ assertTrue(body.contains("<PrimaryEndpoint><Address>localhost</Address><Port>6379</Port></PrimaryEndpoint>"));+ assertTrue(body.contains("<CurrentRole>primary</CurrentRole>"));+ }++ @Test+ void describeCacheClusters_answersForReplicationGroupMembers() {+ ReplicationGroup group = new ReplicationGroup("grp", "d", ReplicationGroupStatus.AVAILABLE,+ AuthMode.NO_AUTH, new Endpoint("localhost", 6379), Instant.now(), 6379);+ group.setEngine("valkey");+ group.setEngineVersion("8.2");+ group.setCacheNodeType("cache.t4g.micro");+ when(service.listMemberCacheClusters("grp-0001-001")).thenReturn(List.of(+ new ElastiCacheService.MemberCacheCluster(group, "grp-0001-001", 6379, true)));++ MultivaluedMap<String, String> params = params();+ params.putSingle("CacheClusterId", "grp-0001-001");+ params.putSingle("ShowCacheNodeInfo", "true");+ Response response = handler.handle("DescribeCacheClusters", params, "us-east-1");++ assertEquals(200, response.getStatus());+ String body = (String) response.getEntity();+ assertTrue(body.contains("<CacheClusterId>grp-0001-001</CacheClusterId>"));+ assertTrue(body.contains("<CacheClusterStatus>available</CacheClusterStatus>"));+ assertTrue(body.contains("<ReplicationGroupId>grp</ReplicationGroupId>"));+ assertTrue(body.contains("<Engine>valkey</Engine>"));+ assertTrue(body.contains("<EngineVersion>8.2</EngineVersion>"));+ assertTrue(body.contains("<CacheNodeType>cache.t4g.micro</CacheNodeType>"));+ assertTrue(body.contains("<Endpoint><Address>localhost</Address><Port>6379</Port></Endpoint>"));+ }++ @Test+ void describeCacheClusters_membersOfFailedGroupReportRestoreFailed() {+ ReplicationGroup group = new ReplicationGroup("grp", "d", ReplicationGroupStatus.CREATE_FAILED,+ AuthMode.NO_AUTH, null, Instant.now(), 6379);+ when(service.listMemberCacheClusters("grp-0001-001")).thenReturn(List.of(+ new ElastiCacheService.MemberCacheCluster(group, "grp-0001-001", 6379, true)));++ MultivaluedMap<String, String> params = params();+ params.putSingle("CacheClusterId", "grp-0001-001");+ params.putSingle("ShowCacheNodeInfo", "true");+ Response response = handler.handle("DescribeCacheClusters", params, "us-east-1");++ assertEquals(200, response.getStatus());+ String body = (String) response.getEntity();+ assertTrue(body.contains("<CacheClusterStatus>restore-failed</CacheClusterStatus>"),+ "CacheClusterStatus has no create-failed; the documented failure value is restore-failed: " + body);+ assertFalse(body.contains("<CacheNodes>"),+ "A failed group has no endpoint, so members must not hand out node endpoints");+ }++ @Test+ void describeReplicationGroups_reportsCreateFailedStatusInAwsWireForm() {+ ReplicationGroup group = new ReplicationGroup("grp", "d", ReplicationGroupStatus.CREATE_FAILED,+ AuthMode.NO_AUTH, null, Instant.now(), 6379);+ when(service.listReplicationGroups("grp")).thenReturn(List.of(group));+ when(service.memberCacheClusters(group)).thenReturn(List.of());++ MultivaluedMap<String, String> params = params();+ params.putSingle("ReplicationGroupId", "grp");+ Response response = handler.handle("DescribeReplicationGroups", params, "us-east-1");++ assertEquals(200, response.getStatus());+ String body = (String) response.getEntity();+ assertTrue(body.contains("<Status>create-failed</Status>"),+ "AWS reports this status hyphenated, not as the enum constant");+ }+@Testvoid unsupportedOperationStillReturnsQueryError() {Response response = handler.handle("NoSuchAction", params(), "us-east-1");@@ -85,7 +212,7 @@ class ElastiCacheQueryHandlerTest {@Testvoid createReplicationGroup_passesSettingsAndTagsToService() {- when(service.createReplicationGroup(any(), any(), any(), any(), any(), any(), any()))+ when(service.createReplicationGroup(any(ElastiCacheService.CreateReplicationGroupRequest.class))).thenReturn(group("g1"));MultivaluedMap<String, String> p = params();p.add("ReplicationGroupId", "g1");@@ -99,15 +226,24 @@ class ElastiCacheQueryHandlerTest {assertEquals(200, handler.handle("CreateReplicationGroup", p, "us-east-1").getStatus());- verify(service).createReplicationGroup(eq("g1"), eq("d"), eq(AuthMode.NO_AUTH), isNull(), eq("us-east-1"),- eq(new ReplicationGroupSettings(true, "alias/cache", 7, "06:30-07:30")), eq(Map.of("Name", "g1")));+ ArgumentCaptor<ElastiCacheService.CreateReplicationGroupRequest> captor =+ ArgumentCaptor.forClass(ElastiCacheService.CreateReplicationGroupRequest.class);+ verify(service).createReplicationGroup(captor.capture());+ ElastiCacheService.CreateReplicationGroupRequest request = captor.getValue();+ assertEquals("g1", request.replicationGroupId());+ assertEquals("d", request.description());+ assertEquals(AuthMode.NO_AUTH, request.authMode());+ assertEquals("us-east-1", request.region());+ assertEquals(new ReplicationGroupSettings(true, "alias/cache", 7, "06:30-07:30"), request.settings());+ assertEquals(Map.of("Name", "g1"), request.tags());}@Testvoid createReplicationGroup_readsTheEncryptionFlagAsAwsDoes() {- when(service.createReplicationGroup(any(), any(), any(), any(), any(), any(), any()))+ when(service.createReplicationGroup(any(ElastiCacheService.CreateReplicationGroupRequest.class))).thenReturn(group("g1"));- ArgumentCaptor<ReplicationGroupSettings> captor = ArgumentCaptor.forClass(ReplicationGroupSettings.class);+ ArgumentCaptor<ElastiCacheService.CreateReplicationGroupRequest> captor =+ ArgumentCaptor.forClass(ElastiCacheService.CreateReplicationGroupRequest.class);// a live account reads anything but "false" as true — probed with banana, yes and "TRUE "for (String value : new String[] {"true", "banana", "yes", "TRUE "}) {MultivaluedMap<String, String> p = params();@@ -125,8 +261,9 @@ class ElastiCacheQueryHandlerTest {omitted.add("ReplicationGroupId", "g1");handler.handle("CreateReplicationGroup", omitted, "us-east-1");- verify(service, times(7)).createReplicationGroup(any(), any(), any(), any(), any(), captor.capture(), any());- List<Boolean> seen = captor.getAllValues().stream().map(ReplicationGroupSettings::atRestEncryptionEnabled).toList();+ verify(service, times(7)).createReplicationGroup(captor.capture());+ List<Boolean> seen = captor.getAllValues().stream()+ .map(r -> r.settings().atRestEncryptionEnabled()).toList();assertEquals(java.util.Arrays.asList(true, true, true, true, false, false, null), seen);}diff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.javaindex f1ffdc7269..2c96db9834 100644--- a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java@@ -12,14 +12,20 @@ import io.github.hectorvent.floci.services.kms.model.KmsKey;import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroupSettings;import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerHandle;import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerManager;+import io.github.hectorvent.floci.services.elasticache.container.ValkeyClusterFormation;import io.github.hectorvent.floci.services.elasticache.model.AuthMode;+import io.github.hectorvent.floci.services.elasticache.model.ClusterNode;import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroup;+import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroupStatus;import io.github.hectorvent.floci.services.elasticache.proxy.ElastiCacheProxyManager;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;+import org.mockito.ArgumentCaptor;import java.util.List;import java.util.Map;+import java.util.concurrent.CompletableFuture;+import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.CountDownLatch;import java.util.concurrent.TimeUnit;@@ -36,6 +42,7 @@ import static org.mockito.Mockito.doNothing;import static org.mockito.Mockito.doThrow;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.never;+import static org.mockito.Mockito.times;import static org.mockito.Mockito.verify;import static org.mockito.Mockito.when;@@ -46,6 +53,7 @@ class ElastiCacheServiceTest {private ElastiCacheContainerManager containerManager;private ElastiCacheProxyManager proxyManager;private EmulatorConfig config;+ private ValkeyClusterFormation clusterFormation;@BeforeEachvoid setUp() {@@ -71,8 +79,10 @@ class ElastiCacheServiceTest {kmsService = org.mockito.Mockito.mock(KmsService.class);when(kmsService.describeKey(any(), any())).thenThrow(new AwsException("NotFoundException", "Key not found", 404));- service = new ElastiCacheService(containerManager, proxyManager, storageFactory, config,- ec2Service, new RegionResolver("us-east-1", "000000000000"), kmsService);+ clusterFormation = mock(ValkeyClusterFormation.class);+ service = new ElastiCacheService(containerManager, proxyManager, clusterFormation,+ storageFactory, config, ec2Service, new RegionResolver("us-east-1", "000000000000"),+ kmsService);}@Test@@ -97,8 +107,8 @@ class ElastiCacheServiceTest {when(cm.start(anyString(), anyString())).thenReturn(new ElastiCacheContainerHandle("cid", "grp", "localhost", 6379));doNothing().when(pm).startProxy(anyString(), any(), anyInt(), anyString(), anyInt(), any());- ElastiCacheService svc = new ElastiCacheService(cm, pm, sf, cfg,- org.mockito.Mockito.mock(Ec2Service.class),+ ElastiCacheService svc = new ElastiCacheService(cm, pm, mock(ValkeyClusterFormation.class),+ sf, cfg, org.mockito.Mockito.mock(Ec2Service.class),new RegionResolver("us-east-1", "000000000000"),org.mockito.Mockito.mock(KmsService.class));@@ -207,6 +217,285 @@ class ElastiCacheServiceTest {"Port from the failed create must be released so the next group reuses it");}+ private static ElastiCacheService.CreateReplicationGroupRequest clusterRequest(+ String groupId, Integer numNodeGroups, Integer replicasPerNodeGroup) {+ return new ElastiCacheService.CreateReplicationGroupRequest(groupId, "test",+ AuthMode.NO_AUTH, null, "us-east-1", "valkey", "8.2", "cache.t4g.micro",+ "default.valkey8.cluster.on", null, null, numNodeGroups, replicasPerNodeGroup,+ null, true, null, ReplicationGroupSettings.defaults(), Map.of());+ }++ private void stubPerNodeContainers() {+ when(containerManager.start(anyString(), anyString(), any())).thenAnswer(inv ->+ new ElastiCacheContainerHandle("cid-" + inv.getArgument(0, String.class),+ inv.getArgument(0, String.class), "localhost", 6379));+ }++ @Test+ void clusterModeCreateStartsOneContainerAndProxyPerNode() {+ stubPerNodeContainers();++ ReplicationGroup group = service.createReplicationGroup(clusterRequest("grp", 2, 1));++ assertTrue(group.isClusterEnabled());+ assertEquals(2, group.getNumNodeGroups());+ assertEquals(1, group.getReplicasPerNodeGroup());+ assertEquals(4, group.getClusterNodes().size());++ List<ClusterNode> nodes = group.getClusterNodes();+ assertEquals("grp-0001-001", nodes.get(0).getMemberClusterId());+ assertEquals("grp-0001-002", nodes.get(1).getMemberClusterId());+ assertEquals("grp-0002-001", nodes.get(2).getMemberClusterId());+ assertEquals("grp-0002-002", nodes.get(3).getMemberClusterId());+ assertTrue(nodes.get(0).isPrimary());+ assertFalse(nodes.get(1).isPrimary());+ assertEquals("0-8191", nodes.get(0).getSlots());+ assertEquals("8192-16383", nodes.get(2).getSlots());+ assertEquals(4, nodes.stream().map(ClusterNode::getProxyPort).distinct().count(),+ "Each node must own its own proxy port");+ assertEquals(16379, group.getConfigurationEndpoint().port());++ verify(clusterFormation).form(eq("grp"), any(), eq(2));+ verify(containerManager, times(4)).start(anyString(), anyString(), any());+ verify(proxyManager, times(4)).startProxy(anyString(), any(), anyInt(), anyString(), anyInt(), any());+ }++ @Test+ void clusterOnParameterGroupEnablesClusterModeForSingleShard() {+ stubPerNodeContainers();++ ReplicationGroup group = service.createReplicationGroup(clusterRequest("grp", 1, 0));++ assertTrue(group.isClusterEnabled());+ assertEquals(1, group.getClusterNodes().size());+ assertEquals("0-16383", group.getClusterNodes().getFirst().getSlots());+ }++ @Test+ void numNodeGroupsBeyondQuotaSurfacesModeledQuotaFault() {+ AwsException ex = assertThrows(AwsException.class,+ () -> service.createReplicationGroup(clusterRequest("grp", 501, 0)));++ assertEquals("NodeGroupsPerReplicationGroupQuotaExceeded", ex.getErrorCode());+ assertEquals(400, ex.getHttpStatus());+ verify(containerManager, never()).start(anyString(), anyString(), any());+ }++ @Test+ void plainCreateStaysClusterModeDisabled() {+ ReplicationGroup group =+ service.createReplicationGroup("grp", "test", AuthMode.NO_AUTH, null, "us-east-1");++ assertFalse(group.isClusterEnabled());+ assertTrue(group.getClusterNodes().isEmpty());+ assertEquals(List.of("grp-001"),+ service.memberCacheClusters(group).stream()+ .map(ElastiCacheService.MemberCacheCluster::cacheClusterId).toList());+ verify(containerManager, never()).start(anyString(), anyString(), any());+ }++ @Test+ void clusterFormationFailureRollsBackAllNodesAndReleasesPorts() {+ stubPerNodeContainers();+ doThrow(new RuntimeException("formation boom"))+ .when(clusterFormation).form(anyString(), any(), anyInt());++ assertThrows(RuntimeException.class,+ () -> service.createReplicationGroup(clusterRequest("grp", 2, 1)));++ verify(containerManager, times(4)).stop(any());+ verify(proxyManager, never()).startProxy(anyString(), any(), anyInt(), anyString(), anyInt(), any());++ ReplicationGroup recovered =+ service.createReplicationGroup("grp2", "test", AuthMode.PASSWORD, null, "us-east-1");+ assertEquals(16379, recovered.getProxyPort(),+ "Ports from the failed cluster create must be released for the next group");+ }++ @Test+ void deleteClusterModeGroupStopsEveryNodeAndReleasesPorts() {+ stubPerNodeContainers();+ service.createReplicationGroup(clusterRequest("grp", 2, 0));++ service.deleteReplicationGroup("grp");++ verify(proxyManager).stopProxy("grp-0001-001");+ verify(proxyManager).stopProxy("grp-0002-001");+ verify(containerManager, times(2)).stop(any());++ ReplicationGroup recovered =+ service.createReplicationGroup("grp2", "test", AuthMode.PASSWORD, null, "us-east-1");+ assertEquals(16379, recovered.getProxyPort(),+ "Ports from the deleted cluster group must be released for the next group");+ }++ @Test+ void clusterNodesAnnounceTheConfiguredHostnameAsPreferredEndpoint() {+ stubPerNodeContainers();++ service.createReplicationGroup(clusterRequest("grp", 1, 0));++ @SuppressWarnings("unchecked")+ ArgumentCaptor<List<String>> flags = ArgumentCaptor.forClass(List.class);+ verify(containerManager).start(eq("grp-0001-001"), anyString(), flags.capture());+ List<String> captured = flags.getValue();+ assertEquals("localhost", flagValue(captured, "--cluster-announce-hostname"));+ assertEquals("hostname", flagValue(captured, "--cluster-preferred-endpoint-type"));+ assertEquals("16379", flagValue(captured, "--cluster-announce-port"));+ }++ @Test+ void clusterAnnounceHostnameOverrideIsAnnouncedAndReported() {+ when(config.services().elasticache().clusterAnnounceHostname())+ .thenReturn(java.util.Optional.of("localhost.floci.io"));+ stubPerNodeContainers();++ ReplicationGroup group = service.createReplicationGroup(clusterRequest("grp", 1, 0));++ @SuppressWarnings("unchecked")+ ArgumentCaptor<List<String>> flags = ArgumentCaptor.forClass(List.class);+ verify(containerManager).start(eq("grp-0001-001"), anyString(), flags.capture());+ assertEquals("localhost.floci.io", flagValue(flags.getValue(), "--cluster-announce-hostname"));+ assertEquals("localhost.floci.io", group.getConfigurationEndpoint().address());+ }++ private static String flagValue(List<String> flags, String flag) {+ int index = flags.indexOf(flag);+ assertTrue(index >= 0 && index + 1 < flags.size(), "Missing flag " + flag + " in " + flags);+ return flags.get(index + 1);+ }++ private static StorageFactory sharedStorageFactory() {+ StorageFactory storageFactory = mock(StorageFactory.class);+ Map<String, Object> backends = new ConcurrentHashMap<>();+ when(storageFactory.create(anyString(), anyString(), any())).thenAnswer(inv ->+ backends.computeIfAbsent(inv.getArgument(1, String.class),+ key -> AccountAwareStorageBackend.inMemory("000000000000")));+ return storageFactory;+ }++ private static ElastiCacheService serviceWith(StorageFactory storageFactory,+ ElastiCacheContainerManager containerManager,+ ElastiCacheProxyManager proxyManager,+ ValkeyClusterFormation clusterFormation) {+ EmulatorConfig config = mock(EmulatorConfig.class);+ EmulatorConfig.ServicesConfig servicesConfig = mock(EmulatorConfig.ServicesConfig.class);+ EmulatorConfig.ElastiCacheServiceConfig ecConfig = mock(EmulatorConfig.ElastiCacheServiceConfig.class);+ when(config.services()).thenReturn(servicesConfig);+ when(servicesConfig.elasticache()).thenReturn(ecConfig);+ when(ecConfig.proxyBasePort()).thenReturn(16379);+ when(ecConfig.proxyMaxPort()).thenReturn(16399);+ when(ecConfig.defaultImage()).thenReturn("valkey/valkey:8");+ when(config.hostname()).thenReturn(java.util.Optional.of("localhost"));+ return new ElastiCacheService(containerManager, proxyManager, clusterFormation,+ storageFactory, config, mock(Ec2Service.class),+ new RegionResolver("us-east-1", "000000000000"), mock(KmsService.class));+ }++ private static void stubPerNodeContainers(ElastiCacheContainerManager containerManager) {+ when(containerManager.start(anyString(), anyString(), any())).thenAnswer(inv ->+ new ElastiCacheContainerHandle("cid-" + inv.getArgument(0, String.class),+ inv.getArgument(0, String.class), "localhost", 6379));+ when(containerManager.start(anyString(), anyString()))+ .thenReturn(new ElastiCacheContainerHandle("cid", "grp", "localhost", 6379));+ }++ @Test+ void restorePersistedRuntimeReprovisionsClusterModeGroups() {+ StorageFactory storageFactory = sharedStorageFactory();+ ElastiCacheContainerManager beforeRestart = mock(ElastiCacheContainerManager.class);+ stubPerNodeContainers(beforeRestart);+ serviceWith(storageFactory, beforeRestart, mock(ElastiCacheProxyManager.class),+ mock(ValkeyClusterFormation.class))+ .createReplicationGroup(clusterRequest("grp", 2, 1));++ ElastiCacheContainerManager restartedContainers = mock(ElastiCacheContainerManager.class);+ stubPerNodeContainers(restartedContainers);+ ElastiCacheProxyManager restartedProxies = mock(ElastiCacheProxyManager.class);+ ValkeyClusterFormation restartedFormation = mock(ValkeyClusterFormation.class);+ ElastiCacheService restarted = serviceWith(storageFactory, restartedContainers,+ restartedProxies, restartedFormation);++ restarted.restorePersistedRuntime().join();++ verify(restartedContainers, times(4)).start(anyString(), anyString(), any());+ verify(restartedFormation).form(eq("grp"), any(), eq(2));+ verify(restartedProxies, times(4)).startProxy(anyString(), any(), anyInt(), anyString(), anyInt(), any());+ ReplicationGroup restored = restarted.getReplicationGroup("grp");+ assertEquals(ReplicationGroupStatus.AVAILABLE, restored.getStatus());+ assertEquals(16379, restored.getConfigurationEndpoint().port());+ assertEquals("cid-grp-0001-001", restored.getClusterNodes().getFirst().getContainerId());++ ReplicationGroup next =+ restarted.createReplicationGroup("grp2", "test", AuthMode.NO_AUTH, null, "us-east-1");+ assertEquals(16383, next.getProxyPort(),+ "Restored node ports must be reserved again so new groups cannot take them");+ }++ @Test+ void restoreFailureReportsCreateFailedAndReleasesPorts() {+ StorageFactory storageFactory = sharedStorageFactory();+ ElastiCacheContainerManager beforeRestart = mock(ElastiCacheContainerManager.class);+ stubPerNodeContainers(beforeRestart);+ serviceWith(storageFactory, beforeRestart, mock(ElastiCacheProxyManager.class),+ mock(ValkeyClusterFormation.class))+ .createReplicationGroup(clusterRequest("grp", 2, 0));++ ElastiCacheContainerManager restartedContainers = mock(ElastiCacheContainerManager.class);+ when(restartedContainers.start(anyString(), anyString(), any()))+ .thenThrow(new RuntimeException("docker down"));+ when(restartedContainers.start(anyString(), anyString()))+ .thenReturn(new ElastiCacheContainerHandle("cid", "grp", "localhost", 6379));+ ElastiCacheProxyManager restartedProxies = mock(ElastiCacheProxyManager.class);+ ElastiCacheService restarted = serviceWith(storageFactory, restartedContainers,+ restartedProxies, mock(ValkeyClusterFormation.class));++ restarted.restorePersistedRuntime().join();++ ReplicationGroup failed = restarted.getReplicationGroup("grp");+ assertEquals(ReplicationGroupStatus.CREATE_FAILED, failed.getStatus());+ assertNull(failed.getConfigurationEndpoint(),+ "A group whose data plane is gone must not advertise an endpoint");+ verify(restartedProxies, never()).startProxy(anyString(), any(), anyInt(), anyString(), anyInt(), any());++ ReplicationGroup next =+ restarted.createReplicationGroup("grp2", "test", AuthMode.NO_AUTH, null, "us-east-1");+ assertEquals(16379, next.getProxyPort(),+ "Ports from the failed restore must be released for the next group");+ }++ @Test+ void restoreRunsInBackgroundAndReportsCreatingUntilDone() throws InterruptedException {+ StorageFactory storageFactory = sharedStorageFactory();+ ElastiCacheContainerManager beforeRestart = mock(ElastiCacheContainerManager.class);+ stubPerNodeContainers(beforeRestart);+ serviceWith(storageFactory, beforeRestart, mock(ElastiCacheProxyManager.class),+ mock(ValkeyClusterFormation.class))+ .createReplicationGroup(clusterRequest("grp", 2, 0));++ ElastiCacheContainerManager restartedContainers = mock(ElastiCacheContainerManager.class);+ CountDownLatch restoreStarted = new CountDownLatch(1);+ CountDownLatch releaseRestore = new CountDownLatch(1);+ when(restartedContainers.start(anyString(), anyString(), any())).thenAnswer(inv -> {+ restoreStarted.countDown();+ releaseRestore.await(5, TimeUnit.SECONDS);+ return new ElastiCacheContainerHandle("cid-" + inv.getArgument(0, String.class),+ inv.getArgument(0, String.class), "localhost", 6379);+ });+ ElastiCacheService restarted = serviceWith(storageFactory, restartedContainers,+ mock(ElastiCacheProxyManager.class), mock(ValkeyClusterFormation.class));++ CompletableFuture<Void> restore = restarted.restorePersistedRuntime();++ assertTrue(restoreStarted.await(5, TimeUnit.SECONDS));+ assertFalse(restore.isDone(), "Restore must not block the caller while the data plane comes up");+ assertEquals(ReplicationGroupStatus.CREATING, restarted.getReplicationGroup("grp").getStatus(),+ "Groups must report creating while their restore is in flight");+ releaseRestore.countDown();+ restore.join();+ assertEquals(ReplicationGroupStatus.AVAILABLE, restarted.getReplicationGroup("grp").getStatus());+ }+@Testvoid concurrentCreateForSameGroupIdIsRejectedWhileFirstIsProvisioning() throws InterruptedException {CountDownLatch startedLatch = new CountDownLatch(1);@@ -356,8 +645,9 @@ class ElastiCacheServiceTest {.thenAnswer(inv -> new AccountAwareStorageBackend<>(pausing, null, "000000000000"));when(factory.create(anyString(), org.mockito.ArgumentMatchers.argThat(f -> !"elasticache-groups.json".equals(f)), any())).thenAnswer(inv -> AccountAwareStorageBackend.inMemory("000000000000"));- ElastiCacheService svc = new ElastiCacheService(containerManager, proxyManager, factory, config,- org.mockito.Mockito.mock(Ec2Service.class), new RegionResolver("us-east-1", "000000000000"), kmsService);+ ElastiCacheService svc = new ElastiCacheService(containerManager, proxyManager, clusterFormation,+ factory, config, org.mockito.Mockito.mock(Ec2Service.class),+ new RegionResolver("us-east-1", "000000000000"), kmsService);svc.createReplicationGroup("g1", "d", AuthMode.NO_AUTH, null, "us-east-1");pausing.pauseOn(PausingStorageBackend.Call.PUT, "g1");