Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(elasticache): support cluster-mode replication groups (#2633)

Kurt McAlpine97ff0bc3e9ff
diff --git a/compatibility-tests/compat-terraform/Dockerfile b/compatibility-tests/compat-terraform/Dockerfile
index 53cc87b4d1..779b621fca 100644
--- a/compatibility-tests/compat-terraform/Dockerfile
+++ b/compatibility-tests/compat-terraform/Dockerfile
@@ -1,6 +1,6 @@
FROM hashicorp/terraform:1.14.7 AS terraform
-FROM amazon/aws-cli:2.15.52
+FROM amazon/aws-cli:2.34.32
# Copy terraform binary from official Terraform image
COPY --from=terraform /bin/terraform /usr/local/bin/terraform
diff --git a/compatibility-tests/compat-terraform/test/elasticache-cluster-tf/main.tf b/compatibility-tests/compat-terraform/test/elasticache-cluster-tf/main.tf
new file mode 100644
index 0000000000..8f5ada5b79
--- /dev/null
+++ b/compatibility-tests/compat-terraform/test/elasticache-cluster-tf/main.tf
@@ -0,0 +1,26 @@
+resource "aws_elasticache_replication_group" "cluster" {
+ replication_group_id = "floci-tf-valkey-cluster"
+ description = "floci terraform cluster-mode compatibility test"
+ engine = "valkey"
+ node_type = "cache.t4g.micro"
+ parameter_group_name = "default.valkey8.cluster.on"
+ automatic_failover_enabled = true
+ num_node_groups = 2
+ replicas_per_node_group = 1
+}
+
+output "cluster_enabled" {
+ value = aws_elasticache_replication_group.cluster.cluster_enabled
+}
+
+output "configuration_endpoint_address" {
+ value = aws_elasticache_replication_group.cluster.configuration_endpoint_address
+}
+
+output "num_node_groups" {
+ value = aws_elasticache_replication_group.cluster.num_node_groups
+}
+
+output "member_clusters" {
+ value = aws_elasticache_replication_group.cluster.member_clusters
+}
diff --git a/compatibility-tests/compat-terraform/test/elasticache-cluster-tf/provider.tf b/compatibility-tests/compat-terraform/test/elasticache-cluster-tf/provider.tf
new file mode 100644
index 0000000000..b32d832003
--- /dev/null
+++ b/compatibility-tests/compat-terraform/test/elasticache-cluster-tf/provider.tf
@@ -0,0 +1,28 @@
+terraform {
+ required_providers {
+ aws = {
+ source = "hashicorp/aws"
+ version = "~> 6.0"
+ }
+ }
+}
+
+variable "endpoint" {
+ type = string
+ default = "http://localhost:4566"
+}
+
+provider "aws" {
+ region = "us-east-1"
+ access_key = "test"
+ secret_key = "test"
+
+ skip_credentials_validation = true
+ skip_metadata_api_check = true
+ skip_requesting_account_id = true
+ s3_use_path_style = true
+
+ endpoints {
+ elasticache = var.endpoint
+ }
+}
diff --git a/compatibility-tests/compat-terraform/test/elasticache_cluster.bats b/compatibility-tests/compat-terraform/test/elasticache_cluster.bats
new file mode 100644
index 0000000000..72c7bf037b
--- /dev/null
+++ b/compatibility-tests/compat-terraform/test/elasticache_cluster.bats
@@ -0,0 +1,131 @@
+#!/usr/bin/env bats
+# ElastiCache Cluster Mode Compatibility Test
+#
+# Applies an aws_elasticache_replication_group with num_node_groups /
+# replicas_per_node_group (Terraform's cluster-mode shape) against floci and
+# verifies both sides of the contract: the terraform read-back is honest
+# (cluster_enabled, member_clusters, a clean re-plan), and the data plane is a
+# genuine Valkey cluster (CLUSTER INFO over RESP through the auth proxy).
+
+setup_file() {
+ load 'test_helper/common-setup'
+
+ EC_TF_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/elasticache-cluster-tf" && pwd)"
+ cd "$EC_TF_DIR"
+
+ echo "# === ElastiCache Cluster Mode Test ===" >&3
+ echo "# Endpoint: $FLOCI_ENDPOINT" >&3
+ echo "# Config: $EC_TF_DIR" >&3
+
+ rm -rf .terraform .terraform.lock.hcl terraform.tfstate* 2>/dev/null || true
+
+ echo "# --- terraform init ---" >&3
+ run terraform init -input=false -no-color
+ if [ "$status" -ne 0 ]; then
+ echo "# terraform init failed: $output" >&3
+ return 1
+ fi
+
+ echo "# --- terraform apply (2 shards, 1 replica each) ---" >&3
+ run terraform apply -var="endpoint=${FLOCI_ENDPOINT}" -input=false -auto-approve -no-color
+ if [ "$status" -ne 0 ]; then
+ echo "# terraform apply failed: $output" >&3
+ return 1
+ fi
+}
+
+teardown_file() {
+ load 'test_helper/common-setup'
+
+ EC_TF_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/elasticache-cluster-tf" && pwd)"
+ cd "$EC_TF_DIR"
+
+ terraform destroy -var="endpoint=${FLOCI_ENDPOINT}" -input=false -auto-approve -no-color || true
+ rm -rf .terraform .terraform.lock.hcl terraform.tfstate* 2>/dev/null || true
+}
+
+setup() {
+ load 'test_helper/common-setup'
+ EC_TF_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/elasticache-cluster-tf" && pwd)"
+}
+
+# Sends one RESP command over /dev/tcp and echoes whatever the server replies
+# within the read window. No redis-cli in this image, so raw RESP it is.
+valkey_command() {
+ local host="$1" port="$2"
+ shift 2
+ local payload="*$#\r\n"
+ local arg
+ for arg in "$@"; do
+ payload+="\$${#arg}\r\n${arg}\r\n"
+ done
+ exec 9<>"/dev/tcp/${host}/${port}" || return 1
+ printf '%b' "$payload" >&9
+ local reply="" chunk
+ while IFS= read -r -t 3 -u 9 chunk; do
+ reply+="${chunk}"$'\n'
+ done
+ exec 9<&- 9>&-
+ printf '%s' "$reply"
+}
+
+@test "ElastiCache cluster mode: terraform reads back cluster_enabled=true" {
+ run terraform -chdir="$EC_TF_DIR" output -raw cluster_enabled
+ assert_success
+ assert_output "true"
+
+ run terraform -chdir="$EC_TF_DIR" output -raw num_node_groups
+ assert_success
+ assert_output "2"
+}
+
+@test "ElastiCache cluster mode: describe reports the sharded topology" {
+ run aws_cmd elasticache describe-replication-groups \
+ --replication-group-id floci-tf-valkey-cluster
+ assert_success
+ [ "$(json_get "$output" '.ReplicationGroups[0].ClusterEnabled')" = "true" ]
+ [ "$(json_get "$output" '.ReplicationGroups[0].AutomaticFailover')" = "enabled" ]
+ [ "$(json_get "$output" '.ReplicationGroups[0].Engine')" = "valkey" ]
+ [ "$(json_get "$output" '.ReplicationGroups[0].NodeGroups | length')" = "2" ]
+ [ "$(json_get "$output" '.ReplicationGroups[0].NodeGroups[0].Slots')" = "0-8191" ]
+ [ "$(json_get "$output" '.ReplicationGroups[0].NodeGroups[1].Slots')" = "8192-16383" ]
+ [ "$(json_get "$output" '.ReplicationGroups[0].MemberClusters | length')" = "4" ]
+}
+
+@test "ElastiCache cluster mode: member cache clusters answer DescribeCacheClusters" {
+ member=$(terraform -chdir="$EC_TF_DIR" output -json member_clusters | jq -r '.[0]')
+ run aws_cmd elasticache describe-cache-clusters \
+ --cache-cluster-id "$member" --show-cache-node-info
+ assert_success
+ [ "$(json_get "$output" '.CacheClusters[0].ReplicationGroupId')" = "floci-tf-valkey-cluster" ]
+ [ "$(json_get "$output" '.CacheClusters[0].Engine')" = "valkey" ]
+ [ -n "$(json_get "$output" '.CacheClusters[0].CacheNodes[0].Endpoint.Port')" ]
+}
+
+# The critical assertion: the endpoint terraform hands out fronts a Valkey
+# node that is actually running in cluster mode, in a formed, healthy cluster.
+@test "ElastiCache cluster mode: Valkey runs in cluster mode behind the configuration endpoint" {
+ host=$(terraform -chdir="$EC_TF_DIR" output -raw configuration_endpoint_address)
+ describe=$(aws_cmd elasticache describe-replication-groups \
+ --replication-group-id floci-tf-valkey-cluster)
+ port=$(json_get "$describe" '.ReplicationGroups[0].ConfigurationEndpoint.Port')
+
+ run valkey_command "$host" "$port" INFO cluster
+ assert_success
+ assert_output --partial "cluster_enabled:1"
+
+ run valkey_command "$host" "$port" CLUSTER INFO
+ assert_success
+ assert_output --partial "cluster_state:ok"
+ assert_output --partial "cluster_size:2"
+ assert_output --partial "cluster_known_nodes:4"
+}
+
+# Guards against read-back drift (the class of bug in floci-io/floci#2481):
+# a second plan straight after apply must not want to change anything.
+@test "ElastiCache cluster mode: re-plan after apply is clean" {
+ cd "$EC_TF_DIR"
+ run terraform plan -var="endpoint=${FLOCI_ENDPOINT}" -input=false -no-color -detailed-exitcode
+ assert_success
+ assert_output --partial "No changes"
+}
diff --git a/docker-compose.yml b/docker-compose.yml
index b882e6a07e..0046854e6b 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -20,6 +20,9 @@ services:
# If you were using this env var for persistence, migrate: docker volume ls -f label=floci=true
FLOCI_HOSTNAME: floci
FLOCI_BASE_URL: http://floci:4566
+ # localhost.floci.io resolves everywhere: public DNS -> 127.0.0.1 on the host
+ # (reaching the published proxy ports), the network alias below inside Compose.
+ FLOCI_SERVICES_ELASTICACHE_CLUSTER_ANNOUNCE_HOSTNAME: localhost.floci.io
FLOCI_SERVICES_LAMBDA_HOT_RELOAD_ENABLED: "true"
# Match the compatibility workflow (.github/workflows/compatibility.yml) so the
# local Docker test run exercises the same Floci configuration as CI.
diff --git a/docs/configuration/environment-variables.md b/docs/configuration/environment-variables.md
index bd8378fe56..da64eddbe8 100644
--- a/docs/configuration/environment-variables.md
+++ b/docs/configuration/environment-variables.md
@@ -336,6 +336,7 @@ These services spawn Docker containers. They require access to the Docker socket
| `FLOCI_SERVICES_ELASTICACHE_PROXY_MAX_PORT` | `6399` | Last port in the ElastiCache proxy range |
| `FLOCI_SERVICES_ELASTICACHE_DEFAULT_IMAGE` | `valkey/valkey:8` | Default Docker image for cache clusters |
| `FLOCI_SERVICES_ELASTICACHE_DOCKER_NETWORK` | _(none)_ | Docker network for ElastiCache containers (overrides `FLOCI_SERVICES_DOCKER_NETWORK`) |
+| `FLOCI_SERVICES_ELASTICACHE_CLUSTER_ANNOUNCE_HOSTNAME` | _(none)_ | Hostname cluster-mode nodes announce in `MOVED`/`ASK` redirects and topology responses, and report as the `ConfigurationEndpoint`. Set to a name every client resolves (e.g. `localhost.floci.io`) when `FLOCI_HOSTNAME` only resolves inside Floci's Docker network. Defaults to `FLOCI_HOSTNAME` |
### RDS
diff --git a/docs/services/elasticache.md b/docs/services/elasticache.md
index 6f15199001..38a24cd8e7 100644
--- a/docs/services/elasticache.md
+++ b/docs/services/elasticache.md
@@ -35,6 +35,67 @@ Floci manages real Valkey/Redis Docker containers and proxies TCP connections to
| `ListTagsForResource` | Tags on a parameter group ARN |
<!-- floci:actions:end -->
+### Cluster Mode
+
+`CreateReplicationGroup` provisions a real sharded Valkey cluster when the request asks for one:
+`NumNodeGroups` greater than 1, a `default.*.cluster.on` parameter group, a custom parameter group
+with `cluster-enabled` set to `yes`, or `ClusterMode=enabled`.
+
+Floci starts one `--cluster-enabled` container per node — `NumNodeGroups × (1 + ReplicasPerNodeGroup)`
+in total — forms the cluster (config epochs, MEET, slot assignment, replica attachment), and fronts
+each node with its own auth-proxy port from the proxy port range. Nodes announce Floci's configured
+hostname as their preferred endpoint (`cluster-announce-hostname` with
+`cluster-preferred-endpoint-type hostname`, plus `cluster-announce-client-ipv4`/
+`cluster-announce-port`), so `CLUSTER SLOTS`, `CLUSTER SHARDS` and `MOVED`/`ASK` redirects hand
+clients the same name the `ConfigurationEndpoint` reports, while the cluster bus keeps using the
+container network. Any cluster-aware Redis/Valkey client works against the reported
+`ConfigurationEndpoint`.
+
+Because clients must resolve the announced name to follow redirects, a `FLOCI_HOSTNAME` that only
+resolves inside Floci's Docker network (such as the Compose service name `floci`) breaks
+cluster-aware clients connecting from outside it. Set
+`FLOCI_SERVICES_ELASTICACHE_CLUSTER_ANNOUNCE_HOSTNAME` to a universally resolvable name in that
+case — the shipped `docker-compose.yml` uses `localhost.floci.io`, which public DNS resolves to
+`127.0.0.1` on the host (reaching the published proxy ports) while the Compose network alias and
+Floci's embedded DNS resolve it to the Floci container from inside Docker. Cluster-mode groups
+then announce that name and report it as their `ConfigurationEndpoint`.
+
+With `persistent`, `hybrid` or `wal` storage, cluster-mode groups are re-provisioned from their
+persisted topology on startup: containers are restarted, the cluster is re-formed (caches restart
+empty, as on any Floci restart) and each node's proxy port is re-reserved. Ports are re-reserved
+and groups marked `creating` before Floci reports ready; the container restarts and cluster
+formation run in the background so a slow Docker daemon cannot delay readiness, and each group
+flips to `available` once its data plane is back. A group whose data plane cannot be brought back
+is reported with status `create-failed` instead of `available`, and its member clusters answer
+`DescribeCacheClusters` with `restore-failed` (`CacheClusterStatus` has no `create-failed` value).
+
+`DescribeReplicationGroups` reports the topology honestly: `ClusterEnabled`, one `NodeGroup` per
+shard with its `Slots`, `NodeGroupMembers`, and `MemberClusters`. Each member also answers
+`DescribeCacheClusters` (as on AWS), which is what terraform-provider-aws reads node type, engine
+version and port from.
+
+Cluster mode requires a Valkey 8.1+ image (the default `valkey/valkey:8` qualifies) for
+`cluster-announce-client-ipv4` support. Each node consumes one port from the proxy range, so size
+`FLOCI_SERVICES_ELASTICACHE_PROXY_BASE_PORT`/`_MAX_PORT` to the number of nodes you need.
+
+```bash
+aws elasticache create-replication-group \
+ --replication-group-id my-sharded-cache \
+ --replication-group-description "Sharded dev cache" \
+ --engine valkey \
+ --cache-parameter-group-name default.valkey8.cluster.on \
+ --num-node-groups 2 \
+ --replicas-per-node-group 1 \
+ --endpoint-url $AWS_ENDPOINT_URL
+
+aws elasticache describe-replication-groups \
+ --replication-group-id my-sharded-cache \
+ --query 'ReplicationGroups[0].ConfigurationEndpoint' \
+ --endpoint-url $AWS_ENDPOINT_URL
+
+redis-cli -c -h localhost -p <configuration-endpoint-port> set mykey "hello"
+```
+
### Cache Subnet Groups
A subnet group's VPC and each subnet's availability zone are read from the subnets themselves, as
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
index 62a5f4b6b3..8fc9f4941b 100644
--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
@@ -1063,6 +1063,14 @@ public interface EmulatorConfig {
/** Docker network to attach ElastiCache containers to. Empty = default bridge. */
Optional<String> dockerNetwork();
+
+ /**
+ * Hostname cluster-mode nodes announce in MOVED/ASK redirects and cluster topology,
+ * and that is reported as the ConfigurationEndpoint. Must resolve to Floci from every
+ * client location; set it when the main hostname only resolves inside Floci's Docker
+ * network. Empty = the main hostname.
+ */
+ Optional<String> clusterAnnounceHostname();
}
interface MemoryDbServiceConfig {
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
index 467d757829..f17f7a3895 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
@@ -593,6 +593,16 @@ public class ContainerLifecycleManager {
return resolveEndpoint(inspect, containerPort);
}
+ /**
+ * Resolves the container's IP on its Docker network, independent of whether Floci runs
+ * natively or in a container. Used for addresses that sibling containers (not Floci itself)
+ * must dial — e.g. ElastiCache cluster-bus peering between Valkey nodes.
+ */
+ public String resolveContainerNetworkIp(String containerId, String preferredNetwork) {
+ InspectContainerResponse inspect = dockerClient.inspectContainerCmd(containerId).exec();
+ return resolveContainerIp(inspect, preferredNetwork);
+ }
+
/**
* Returns the underlying DockerClient for operations not covered by this manager.
* Prefer using manager methods when available.
diff --git a/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java b/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java
index 0a3ae41a58..af1ae39e7e 100644
--- a/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java
+++ b/src/main/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycle.java
@@ -13,6 +13,7 @@ import io.github.hectorvent.floci.services.floci.ui.FlociUiManager;
import io.github.hectorvent.floci.services.amazonmq.container.RabbitMqManager;
import io.github.hectorvent.floci.services.kinesisanalytics.container.FlinkContainerManager;
import io.github.hectorvent.floci.services.iam.IamService;
+import io.github.hectorvent.floci.services.elasticache.ElastiCacheService;
import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerManager;
import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheMemcachedContainerManager;
import io.github.hectorvent.floci.services.elasticache.proxy.ElastiCacheProxyManager;
@@ -66,6 +67,7 @@ public class EmulatorLifecycle {
private final ServiceRegistry serviceRegistry;
private final EmulatorConfig config;
private final IamService iamService;
+ private final ElastiCacheService elastiCacheService;
private final ElastiCacheContainerManager elastiCacheContainerManager;
private final ElastiCacheMemcachedContainerManager elastiCacheMemcachedContainerManager;
private final ElastiCacheProxyManager elastiCacheProxyManager;
@@ -97,6 +99,7 @@ public class EmulatorLifecycle {
public EmulatorLifecycle(StorageFactory storageFactory, ServiceRegistry serviceRegistry,
EmulatorConfig config,
IamService iamService,
+ ElastiCacheService elastiCacheService,
ElastiCacheContainerManager elastiCacheContainerManager,
ElastiCacheMemcachedContainerManager elastiCacheMemcachedContainerManager,
ElastiCacheProxyManager elastiCacheProxyManager,
@@ -127,6 +130,7 @@ public class EmulatorLifecycle {
this.serviceRegistry = serviceRegistry;
this.config = config;
this.iamService = iamService;
+ this.elastiCacheService = elastiCacheService;
this.elastiCacheContainerManager = elastiCacheContainerManager;
this.elastiCacheMemcachedContainerManager = elastiCacheMemcachedContainerManager;
this.elastiCacheProxyManager = elastiCacheProxyManager;
@@ -189,6 +193,12 @@ public class EmulatorLifecycle {
dynamodbStreamsPoller.startPersistedPollers();
pipesService.startPersistedPollers();
rdsService.restorePersistedRuntime();
+ if (config.services().elasticache().enabled()) {
+ elastiCacheService.restorePersistedRuntime().exceptionally(ex -> {
+ LOG.warnv("ElastiCache cluster-mode restore failed: {0}", ex.getMessage());
+ return null;
+ });
+ }
if (config.services().elbv2().enabled()) {
elbV2Service.restorePersistedRuntime();
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.java
index 67e8750516..7e10a17090 100644
--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.java
@@ -9,11 +9,13 @@ import io.github.hectorvent.floci.core.common.XmlBuilder;
import io.github.hectorvent.floci.services.elasticache.model.AuthMode;
import io.github.hectorvent.floci.services.elasticache.model.CacheCluster;
import io.github.hectorvent.floci.services.elasticache.model.CacheParameterGroup;
+import io.github.hectorvent.floci.services.elasticache.model.ClusterNode;
import io.github.hectorvent.floci.services.elasticache.model.CacheSubnetGroup;
import io.github.hectorvent.floci.services.elasticache.model.ElastiCacheUser;
import io.github.hectorvent.floci.services.elasticache.model.Endpoint;
import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroup;
import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroupSettings;
+import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroupStatus;
import io.github.hectorvent.floci.services.elasticache.proxy.SigV4Validator;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
@@ -106,8 +108,25 @@ public class ElastiCacheQueryHandler {
try {
ReplicationGroup group = service.createReplicationGroup(
- groupId, description != null ? description : "", authMode, authToken, region,
- replicationGroupSettings(params), parseTags(params));
+ new ElastiCacheService.CreateReplicationGroupRequest(
+ groupId,
+ description != null ? description : "",
+ authMode,
+ authToken,
+ region,
+ params.getFirst("Engine"),
+ params.getFirst("EngineVersion"),
+ params.getFirst("CacheNodeType"),
+ params.getFirst("CacheParameterGroupName"),
+ params.getFirst("CacheSubnetGroupName"),
+ params.getFirst("ClusterMode"),
+ intParam(params, "NumNodeGroups"),
+ intParam(params, "ReplicasPerNodeGroup"),
+ intParam(params, "NumCacheClusters"),
+ boolParam(params, "AutomaticFailoverEnabled"),
+ boolParam(params, "MultiAZEnabled"),
+ replicationGroupSettings(params),
+ parseTags(params)));
String result = replicationGroupXml(group);
return Response.ok(AwsQueryResponse.envelope("CreateReplicationGroup", AwsNamespaces.EC, result)).build();
} catch (AwsException e) {
@@ -137,6 +156,28 @@ public class ElastiCacheQueryHandler {
}
}
+ private static Integer intParam(MultivaluedMap<String, String> params, String name) {
+ String value = params.getFirst(name);
+ if (value == null || value.isBlank()) {
+ return null;
+ }
+ try {
+ return Integer.valueOf(value.trim());
+ } catch (NumberFormatException e) {
+ throw new AwsException("InvalidParameterValue",
+ "The parameter " + name + " must be an integer.", 400);
+ }
+ }
+
+ private static Boolean boolParam(MultivaluedMap<String, String> params, String name) {
+ String value = params.getFirst(name);
+ if (value == null || value.isBlank()) {
+ return null;
+ }
+ return Boolean.parseBoolean(value.trim());
+ }
+
+
private Response handleDescribeReplicationGroups(MultivaluedMap<String, String> params) {
String filterId = params.getFirst("ReplicationGroupId");
try {
@@ -295,12 +336,22 @@ public class ElastiCacheQueryHandler {
private Response handleDescribeCacheClusters(MultivaluedMap<String, String> params) {
String filterId = params.getFirst("CacheClusterId");
+ boolean showNodeInfo = "true".equalsIgnoreCase(params.getFirst("ShowCacheNodeInfo"));
try {
- Collection<CacheCluster> clusterList = memcachedService.listCacheClusters(filterId);
+ List<ElastiCacheService.MemberCacheCluster> members = service.listMemberCacheClusters(filterId);
+ Collection<CacheCluster> clusterList;
+ if (filterId != null && !filterId.isBlank() && !members.isEmpty()) {
+ clusterList = List.of();
+ } else {
+ clusterList = memcachedService.listCacheClusters(filterId);
+ }
var xml = new XmlBuilder().start("CacheClusters");
for (CacheCluster c : clusterList) {
xml.raw(cacheClusterXml(c));
}
+ for (ElastiCacheService.MemberCacheCluster member : members) {
+ xml.raw(memberCacheClusterXml(member, showNodeInfo));
+ }
xml.end("CacheClusters").start("Marker").end("Marker");
return Response.ok(AwsQueryResponse.envelope("DescribeCacheClusters", AwsNamespaces.EC, xml.build())).build();
} catch (AwsException e) {
@@ -308,6 +359,63 @@ public class ElastiCacheQueryHandler {
}
}
+ private String memberCacheClusterXml(ElastiCacheService.MemberCacheCluster member, boolean showNodeInfo) {
+ ReplicationGroup g = member.group();
+ boolean authTokenEnabled = g.getAuthMode() == AuthMode.PASSWORD;
+ var xml = new XmlBuilder()
+ .start("CacheCluster")
+ .elem("CacheClusterId", member.cacheClusterId())
+ .elem("CacheClusterStatus", memberCacheClusterStatus(g))
+ .elem("NumCacheNodes", 1L)
+ .elem("ReplicationGroupId", g.getReplicationGroupId())
+ .elem("AutoMinorVersionUpgrade", true)
+ .elem("AuthTokenEnabled", authTokenEnabled)
+ .elem("TransitEncryptionEnabled", authTokenEnabled)
+ .elem("AtRestEncryptionEnabled", false);
+ if (g.getEngine() != null) {
+ xml.elem("Engine", g.getEngine());
+ }
+ if (g.getEngineVersion() != null) {
+ xml.elem("EngineVersion", g.getEngineVersion());
+ }
+ if (g.getCacheNodeType() != null) {
+ xml.elem("CacheNodeType", g.getCacheNodeType());
+ }
+ if (g.getCacheParameterGroupName() != null) {
+ xml.start("CacheParameterGroup")
+ .elem("CacheParameterGroupName", g.getCacheParameterGroupName())
+ .elem("ParameterApplyStatus", "in-sync")
+ .end("CacheParameterGroup");
+ }
+ if (g.getCacheSubnetGroupName() != null) {
+ xml.elem("CacheSubnetGroupName", g.getCacheSubnetGroupName());
+ }
+ if (showNodeInfo && g.getConfigurationEndpoint() != null) {
+ xml.start("CacheNodes")
+ .start("CacheNode")
+ .elem("CacheNodeId", "0001")
+ .elem("CacheNodeStatus", "available")
+ .start("Endpoint")
+ .elem("Address", g.getConfigurationEndpoint().address())
+ .elem("Port", (long) member.port())
+ .end("Endpoint")
+ .end("CacheNode")
+ .end("CacheNodes");
+ }
+ return xml.end("CacheCluster").build();
+ }
+
+ /**
+ * Members mirror the group's status, except {@code create-failed}: that value is only legal
+ * on ReplicationGroup.Status, so members report the documented {@code restore-failed}.
+ */
+ private static String memberCacheClusterStatus(ReplicationGroup g) {
+ if (g.getStatus() == ReplicationGroupStatus.CREATE_FAILED) {
+ return "restore-failed";
+ }
+ return g.getStatus().wireName();
+ }
+
private Response handleDeleteCacheCluster(MultivaluedMap<String, String> params) {
String clusterId = params.getFirst("CacheClusterId");
if (clusterId == null || clusterId.isBlank()) {
@@ -640,26 +748,46 @@ private Response handleCreateCacheParameterGroup(MultivaluedMap<String, String>
private String replicationGroupXml(ReplicationGroup g) {
Endpoint ep = g.getConfigurationEndpoint();
boolean authTokenEnabled = g.getAuthMode() == AuthMode.PASSWORD;
+ List<ElastiCacheService.MemberCacheCluster> members = service.memberCacheClusters(g);
var xml = new XmlBuilder()
.start("ReplicationGroup")
.elem("ReplicationGroupId", g.getReplicationGroupId())
.elem("Description", g.getDescription())
- .elem("Status", g.getStatus().name().toLowerCase())
+ .elem("Status", g.getStatus().wireName())
.elem("AuthTokenEnabled", authTokenEnabled)
.elem("TransitEncryptionEnabled", authTokenEnabled)
.elem("AtRestEncryptionEnabled", g.isAtRestEncryptionEnabled())
- .elem("ClusterEnabled", false)
- .elem("MultiAZ", "disabled")
- .elem("AutomaticFailover", "disabled")
+ .elem("ClusterEnabled", g.isClusterEnabled())
+ .elem("ClusterMode", g.isClusterEnabled() ? "enabled" : "disabled")
+ .elem("MultiAZ", g.isMultiAzEnabled() ? "enabled" : "disabled")
+ .elem("AutomaticFailover", g.isAutomaticFailoverEnabled() ? "enabled" : "disabled")
.elem("SnapshotRetentionLimit", (long) g.getSnapshotRetentionLimit());
xml.elem("SnapshotWindow", g.getSnapshotWindow() != null
? g.getSnapshotWindow() : ReplicationGroupSettings.DEFAULT_SNAPSHOT_WINDOW);
if (g.getKmsKeyId() != null) {
xml.elem("KmsKeyId", g.getKmsKeyId());
}
+ if (g.getEngine() != null) {
+ xml.elem("Engine", g.getEngine());
+ }
+ if (g.getCacheNodeType() != null) {
+ xml.elem("CacheNodeType", g.getCacheNodeType());
+ }
if (g.getArn() != null) {
xml.elem("ARN", g.getArn());
}
+ xml.start("MemberClusters");
+ for (ElastiCacheService.MemberCacheCluster member : members) {
+ xml.elem("ClusterId", member.cacheClusterId());
+ }
+ xml.end("MemberClusters");
+ xml.start("NodeGroups");
+ if (g.isClusterEnabled() && !g.getClusterNodes().isEmpty()) {
+ appendClusterModeNodeGroups(xml, g);
+ } else {
+ appendSingleNodeGroup(xml, g, members, ep);
+ }
+ xml.end("NodeGroups");
if (ep != null) {
xml.start("ConfigurationEndpoint")
.elem("Address", ep.address())
@@ -669,6 +797,60 @@ private Response handleCreateCacheParameterGroup(MultivaluedMap<String, String>
return xml.end("ReplicationGroup").build();
}
+ private static void appendClusterModeNodeGroups(XmlBuilder xml, ReplicationGroup g) {
+ Map<String, List<ClusterNode>> byNodeGroup = new LinkedHashMap<>();
+ for (ClusterNode node : g.getClusterNodes()) {
+ byNodeGroup.computeIfAbsent(node.getNodeGroupId(), key -> new ArrayList<>()).add(node);
+ }
+ byNodeGroup.forEach((nodeGroupId, nodes) -> {
+ xml.start("NodeGroup")
+ .elem("NodeGroupId", nodeGroupId)
+ .elem("Status", "available")
+ .elem("Slots", nodes.getFirst().getSlots())
+ .start("NodeGroupMembers");
+ for (ClusterNode node : nodes) {
+ xml.start("NodeGroupMember")
+ .elem("CacheClusterId", node.getMemberClusterId())
+ .elem("CacheNodeId", "0001")
+ .end("NodeGroupMember");
+ }
+ xml.end("NodeGroupMembers").end("NodeGroup");
+ });
+ }
+
+ private static void appendSingleNodeGroup(XmlBuilder xml, ReplicationGroup g,
+ List<ElastiCacheService.MemberCacheCluster> members,
+ Endpoint ep) {
+ xml.start("NodeGroup")
+ .elem("NodeGroupId", "0001")
+ .elem("Status", "available");
+ if (ep != null) {
+ xml.start("PrimaryEndpoint")
+ .elem("Address", ep.address())
+ .elem("Port", (long) ep.port())
+ .end("PrimaryEndpoint")
+ .start("ReaderEndpoint")
+ .elem("Address", ep.address())
+ .elem("Port", (long) ep.port())
+ .end("ReaderEndpoint");
+ }
+ xml.start("NodeGroupMembers");
+ for (ElastiCacheService.MemberCacheCluster member : members) {
+ xml.start("NodeGroupMember")
+ .elem("CacheClusterId", member.cacheClusterId())
+ .elem("CacheNodeId", "0001")
+ .elem("CurrentRole", member.primary() ? "primary" : "replica");
+ if (ep != null) {
+ xml.start("ReadEndpoint")
+ .elem("Address", ep.address())
+ .elem("Port", (long) member.port())
+ .end("ReadEndpoint");
+ }
+ xml.end("NodeGroupMember");
+ }
+ xml.end("NodeGroupMembers").end("NodeGroup");
+ }
+
private String userXml(ElastiCacheUser u) {
String authType = switch (u.getAuthMode()) {
case IAM -> "iam";
diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java
index f009664c9d..21fcb40200 100644
--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java
@@ -12,9 +12,11 @@ import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerHandle;
import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerManager;
+import io.github.hectorvent.floci.services.elasticache.container.ValkeyClusterFormation;
import io.github.hectorvent.floci.services.elasticache.model.AuthMode;
import io.github.hectorvent.floci.services.elasticache.model.CacheParameterGroup;
import io.github.hectorvent.floci.services.elasticache.model.CacheSubnetGroup;
+import io.github.hectorvent.floci.services.elasticache.model.ClusterNode;
import io.github.hectorvent.floci.services.elasticache.model.Endpoint;
import io.github.hectorvent.floci.services.elasticache.model.ElastiCacheUser;
import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroup;
@@ -26,6 +28,9 @@ import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
+import java.net.Inet4Address;
+import java.net.InetAddress;
+import java.net.UnknownHostException;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
@@ -37,6 +42,7 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
import java.util.regex.Pattern;
@@ -55,6 +61,7 @@ public class ElastiCacheService implements ResourceProvider {
private final StorageBackend<String, CacheSubnetGroup> subnetGroups;
private final ElastiCacheContainerManager containerManager;
private final ElastiCacheProxyManager proxyManager;
+ private final ValkeyClusterFormation clusterFormation;
private final EmulatorConfig config;
private final KmsService kmsService;
private final Ec2Service ec2Service;
@@ -66,6 +73,7 @@ public class ElastiCacheService implements ResourceProvider {
@Inject
public ElastiCacheService(ElastiCacheContainerManager containerManager,
ElastiCacheProxyManager proxyManager,
+ ValkeyClusterFormation clusterFormation,
StorageFactory storageFactory,
EmulatorConfig config,
Ec2Service ec2Service,
@@ -74,6 +82,7 @@ public class ElastiCacheService implements ResourceProvider {
this.kmsService = kmsService;
this.containerManager = containerManager;
this.proxyManager = proxyManager;
+ this.clusterFormation = clusterFormation;
this.config = config;
this.ec2Service = ec2Service;
this.regionResolver = regionResolver;
@@ -87,6 +96,31 @@ public class ElastiCacheService implements ResourceProvider {
new TypeReference<Map<String, CacheSubnetGroup>>() {});
}
+ /**
+ * The CreateReplicationGroup parameters floci models. Anything unset falls back to the
+ * defaults a bare create has always had: one non-cluster node behind one proxy port.
+ */
+ public record CreateReplicationGroupRequest(
+ String replicationGroupId,
+ String description,
+ AuthMode authMode,
+ String authToken,
+ String region,
+ String engine,
+ String engineVersion,
+ String cacheNodeType,
+ String cacheParameterGroupName,
+ String cacheSubnetGroupName,
+ String clusterMode,
+ Integer numNodeGroups,
+ Integer replicasPerNodeGroup,
+ Integer numCacheClusters,
+ Boolean automaticFailoverEnabled,
+ Boolean multiAzEnabled,
+ ReplicationGroupSettings settings,
+ Map<String, String> tags) {
+ }
+
public ReplicationGroup createReplicationGroup(String groupId, String description,
AuthMode authMode, String authToken, String region) {
return createReplicationGroup(groupId, description, authMode, authToken, region,
@@ -97,10 +131,19 @@ public class ElastiCacheService implements ResourceProvider {
AuthMode authMode, String authToken, String region,
ReplicationGroupSettings settings,
Map<String, String> tags) {
+ return createReplicationGroup(new CreateReplicationGroupRequest(groupId, description,
+ authMode, authToken, region, null, null, null, null, null, null,
+ null, null, null, null, null, settings, tags));
+ }
+
+ public ReplicationGroup createReplicationGroup(CreateReplicationGroupRequest request) {
+ String groupId = request.replicationGroupId();
+ ReplicationGroupSettings settings = request.settings() != null
+ ? request.settings() : ReplicationGroupSettings.defaults();
settings.validate();
// resolved with the other validations, before a port is taken or a container started
ReplicationGroupSettings resolvedSettings =
- settings.withKmsKeyId(resolveKmsKeyArn(settings.kmsKeyId(), region));
+ settings.withKmsKeyId(resolveKmsKeyArn(settings.kmsKeyId(), request.region()));
if (groups.get(groupId).isPresent()) {
throw new AwsException("ReplicationGroupAlreadyExistsFault",
"Replication group " + groupId + " already exists.", 400);
@@ -113,47 +156,392 @@ public class ElastiCacheService implements ResourceProvider {
}
try {
- int proxyPort = allocateProxyPort();
- String image = config.services().elasticache().defaultImage();
+ if (resolveClusterEnabled(request)) {
+ return provisionClusterModeGroup(request, resolvedSettings);
+ }
+ return provisionSingleNodeGroup(request, resolvedSettings);
+ } finally {
+ provisioningGroupIds.remove(groupId);
+ }
+ }
+
+ private ReplicationGroup provisionSingleNodeGroup(CreateReplicationGroupRequest request,
+ ReplicationGroupSettings resolvedSettings) {
+ String groupId = request.replicationGroupId();
+ AuthMode authMode = request.authMode();
+ int proxyPort = allocateProxyPort();
+ String image = config.services().elasticache().defaultImage();
- LOG.infov("Creating replication group {0} with authMode={1} on proxy port {2}",
- groupId, authMode, String.valueOf(proxyPort));
+ LOG.infov("Creating replication group {0} with authMode={1} on proxy port {2}",
+ groupId, authMode, String.valueOf(proxyPort));
+
+ ElastiCacheContainerHandle handle = null;
+ try {
+ handle = containerManager.start(groupId, image);
+
+ String endpointHost = resolveEndpointHost();
+ Endpoint endpoint = new Endpoint(endpointHost, proxyPort);
+ ReplicationGroup group = new ReplicationGroup(
+ groupId, request.description(), ReplicationGroupStatus.AVAILABLE,
+ authMode, endpoint, Instant.now(), proxyPort);
+ group.setContainerId(handle.getContainerId());
+ group.setContainerHost(handle.getHost());
+ group.setContainerPort(handle.getPort());
+ group.setAuthToken(request.authToken());
+ group.setArn(regionResolver.buildArn("elasticache", request.region(),
+ "replicationgroup:" + groupId));
+ group.setRegion(request.region());
+ group.setNumCacheClusters(request.numCacheClusters() != null
+ ? validateRange("NumCacheClusters", request.numCacheClusters(), 1, 6)
+ : 1);
+ applyCommonAttributes(group, request, resolvedSettings);
+
+ proxyManager.startProxy(groupId, authMode, proxyPort,
+ handle.getHost(), handle.getPort(),
+ (username, password) -> validatePassword(groupId, username, password));
- ElastiCacheContainerHandle handle = null;
+ groups.put(groupId, group);
+ LOG.infov("Replication group {0} created, endpoint={1}:{2}", groupId, endpointHost, String.valueOf(proxyPort));
+ return group;
+ } catch (RuntimeException e) {
+ LOG.warnv("Replication group {0} provisioning failed, rolling back: {1}", groupId, e.getMessage());
+ rollbackReplicationGroup(groupId, handle, proxyPort);
+ throw e;
+ }
+ }
+
+ private ReplicationGroup provisionClusterModeGroup(CreateReplicationGroupRequest request,
+ ReplicationGroupSettings resolvedSettings) {
+ String groupId = request.replicationGroupId();
+ AuthMode authMode = request.authMode();
+ int numNodeGroups = request.numNodeGroups() != null
+ ? validateNumNodeGroups(request.numNodeGroups())
+ : 1;
+ int replicasPerNodeGroup = request.replicasPerNodeGroup() != null
+ ? validateRange("ReplicasPerNodeGroup", request.replicasPerNodeGroup(), 0, 5)
+ : 0;
+ String image = config.services().elasticache().defaultImage();
+ String endpointHost = resolveClusterAnnounceHost();
+ String announceIp = resolveAnnounceIp(endpointHost);
+
+ LOG.infov("Creating cluster-mode replication group {0} with {1} node group(s), {2} replica(s) each",
+ groupId, String.valueOf(numNodeGroups), String.valueOf(replicasPerNodeGroup));
+
+ List<ClusterNode> nodes = new ArrayList<>();
+ List<ElastiCacheContainerHandle> handles = new ArrayList<>();
+ List<String> startedProxyKeys = new ArrayList<>();
+ String inFlightMemberId = null;
+ try {
+ for (int shard = 0; shard < numNodeGroups; shard++) {
+ String nodeGroupId = String.format("%04d", shard + 1);
+ int[] slots = ValkeyClusterFormation.slotRange(shard, numNodeGroups);
+ for (int member = 0; member <= replicasPerNodeGroup; member++) {
+ String memberId = groupId + "-" + nodeGroupId + "-" + String.format("%03d", member + 1);
+ nodes.add(new ClusterNode(memberId, nodeGroupId, member == 0,
+ allocateProxyPort(), slots[0] + "-" + slots[1]));
+ }
+ }
+
+ List<ValkeyClusterFormation.Node> formationNodes = new ArrayList<>(nodes.size());
+ for (ClusterNode node : nodes) {
+ inFlightMemberId = node.getMemberClusterId();
+ ElastiCacheContainerHandle handle = containerManager.start(
+ node.getMemberClusterId(), image,
+ clusterNodeFlags(endpointHost, announceIp, node.getProxyPort()));
+ inFlightMemberId = null;
+ handles.add(handle);
+ node.setContainerId(handle.getContainerId());
+ node.setContainerHost(handle.getHost());
+ node.setContainerPort(handle.getPort());
+ String networkIp = handle.getNetworkIp() != null ? handle.getNetworkIp() : handle.getHost();
+ formationNodes.add(new ValkeyClusterFormation.Node(
+ handle.getHost(), handle.getPort(), networkIp,
+ Integer.parseInt(node.getNodeGroupId()) - 1, node.isPrimary()));
+ }
+
+ clusterFormation.form(groupId, formationNodes, numNodeGroups);
+
+ for (int i = 0; i < nodes.size(); i++) {
+ ClusterNode node = nodes.get(i);
+ ElastiCacheContainerHandle handle = handles.get(i);
+ proxyManager.startProxy(node.getMemberClusterId(), authMode, node.getProxyPort(),
+ handle.getHost(), handle.getPort(),
+ (username, password) -> validatePassword(groupId, username, password));
+ startedProxyKeys.add(node.getMemberClusterId());
+ }
+
+ Endpoint configurationEndpoint = new Endpoint(endpointHost, nodes.getFirst().getProxyPort());
+ ReplicationGroup group = new ReplicationGroup(
+ groupId, request.description(), ReplicationGroupStatus.AVAILABLE,
+ authMode, configurationEndpoint, Instant.now(), nodes.getFirst().getProxyPort());
+ group.setAuthToken(request.authToken());
+ group.setArn(regionResolver.buildArn("elasticache", request.region(),
+ "replicationgroup:" + groupId));
+ group.setRegion(request.region());
+ group.setClusterEnabled(true);
+ group.setNumNodeGroups(numNodeGroups);
+ group.setReplicasPerNodeGroup(replicasPerNodeGroup);
+ group.setNumCacheClusters(nodes.size());
+ group.setClusterNodes(nodes);
+ applyCommonAttributes(group, request, resolvedSettings);
+
+ groups.put(groupId, group);
+ LOG.infov("Cluster-mode replication group {0} created, configuration endpoint={1}:{2}",
+ groupId, endpointHost, String.valueOf(configurationEndpoint.port()));
+ return group;
+ } catch (RuntimeException e) {
+ LOG.warnv("Cluster-mode replication group {0} provisioning failed, rolling back: {1}",
+ groupId, e.getMessage());
+ rollbackClusterModeGroup(groupId, startedProxyKeys, handles, inFlightMemberId,
+ nodes.stream().map(ClusterNode::getProxyPort).toList());
+ throw e;
+ }
+ }
+
+ private void applyCommonAttributes(ReplicationGroup group, CreateReplicationGroupRequest request,
+ ReplicationGroupSettings resolvedSettings) {
+ group.setEngine(request.engine() != null && !request.engine().isBlank()
+ ? normalizeEngine(request.engine())
+ : defaultEngineForImage());
+ group.setEngineVersion(request.engineVersion() != null && !request.engineVersion().isBlank()
+ ? request.engineVersion()
+ : ("valkey".equals(group.getEngine()) ? "8.1" : "7.1"));
+ group.setCacheNodeType(request.cacheNodeType() != null && !request.cacheNodeType().isBlank()
+ ? request.cacheNodeType()
+ : "cache.t4g.micro");
+ group.setCacheParameterGroupName(request.cacheParameterGroupName());
+ group.setCacheSubnetGroupName(request.cacheSubnetGroupName());
+ group.setAutomaticFailoverEnabled(Boolean.TRUE.equals(request.automaticFailoverEnabled()));
+ group.setMultiAzEnabled(Boolean.TRUE.equals(request.multiAzEnabled()));
+ group.setSnapshotWindow(ReplicationGroupSettings.DEFAULT_SNAPSHOT_WINDOW);
+ resolvedSettings.applyTo(group);
+ if (request.tags() != null && !request.tags().isEmpty()) {
+ group.setTags(new LinkedHashMap<>(request.tags()));
+ }
+ }
+
+ private String defaultEngineForImage() {
+ String image = config.services().elasticache().defaultImage();
+ return image != null && image.contains("valkey") ? "valkey" : "redis";
+ }
+
+ /**
+ * Cluster mode follows AWS's signal — the parameter group — plus the request shapes that
+ * imply it: an explicit ClusterMode, or more than one node group. ClusterMode
+ * {@code compatible} — the mid-migration state — is deliberately not modelled; it enables
+ * cluster mode only when one of the other signals also does.
+ */
+ private boolean resolveClusterEnabled(CreateReplicationGroupRequest request) {
+ if ("enabled".equalsIgnoreCase(request.clusterMode())) {
+ return true;
+ }
+ String parameterGroupName = request.cacheParameterGroupName();
+ if (parameterGroupName != null) {
+ if (parameterGroupName.endsWith(".cluster.on")) {
+ return true;
+ }
+ boolean customClusterGroup = findParameterGroup(parameterGroupName)
+ .map(group -> "yes".equalsIgnoreCase(group.getParameters().get("cluster-enabled")))
+ .orElse(false);
+ if (customClusterGroup) {
+ return true;
+ }
+ }
+ return request.numNodeGroups() != null && request.numNodeGroups() > 1;
+ }
+
+ /**
+ * Nodes announce the cluster announce hostname as their preferred endpoint, so MOVED/ASK
+ * redirects and CLUSTER SLOTS hand clients the exact name floci reports as the configuration
+ * endpoint — how (or whether) that name resolves inside floci's own container is irrelevant.
+ * The best-effort IPv4 stays announced for consumers that read the address fields instead.
+ */
+ private static List<String> clusterNodeFlags(String endpointHost, String announceIp, int proxyPort) {
+ return List.of(
+ "--cluster-enabled", "yes",
+ "--cluster-announce-hostname", endpointHost,
+ "--cluster-preferred-endpoint-type", "hostname",
+ "--cluster-announce-client-ipv4", announceIp,
+ "--cluster-announce-port", String.valueOf(proxyPort),
+ "--cluster-announce-bus-port", "16379");
+ }
+
+ private static String resolveAnnounceIp(String endpointHost) {
+ try {
+ for (InetAddress address : InetAddress.getAllByName(endpointHost)) {
+ if (address instanceof Inet4Address) {
+ return address.getHostAddress();
+ }
+ }
+ } catch (UnknownHostException e) {
+ LOG.warnv("Could not resolve {0} to an IPv4 for cluster announce, using 127.0.0.1: {1}",
+ endpointHost, e.getMessage());
+ }
+ return "127.0.0.1";
+ }
+
+ private static int validateRange(String parameterName, int value, int min, int max) {
+ if (value < min || value > max) {
+ throw new AwsException("InvalidParameterValue",
+ parameterName + " must be between " + min + " and " + max + ".", 400);
+ }
+ return value;
+ }
+
+ /** 500 is AWS's raised NumNodeGroups quota (the default is 90; 500 with a limit increase). */
+ private static final int MAX_NODE_GROUPS = 500;
+
+ private static int validateNumNodeGroups(int value) {
+ if (value > MAX_NODE_GROUPS) {
+ throw new AwsException("NodeGroupsPerReplicationGroupQuotaExceeded",
+ "The request cannot be processed because it would exceed the maximum allowed"
+ + " number of node groups (shards) in a single replication group."
+ + " The maximum is " + MAX_NODE_GROUPS + ".", 400);
+ }
+ return validateRange("NumNodeGroups", value, 1, MAX_NODE_GROUPS);
+ }
+
+ private void rollbackClusterModeGroup(String groupId, List<String> startedProxyKeys,
+ List<ElastiCacheContainerHandle> handles,
+ String inFlightMemberId, Collection<Integer> proxyPorts) {
+ for (String proxyKey : startedProxyKeys) {
+ try {
+ proxyManager.stopProxy(proxyKey);
+ } catch (RuntimeException e) {
+ LOG.warnv("Error stopping proxy {0} during rollback of group {1}: {2}",
+ proxyKey, groupId, e.getMessage());
+ }
+ }
+ for (ElastiCacheContainerHandle handle : handles) {
+ try {
+ containerManager.stop(handle);
+ } catch (RuntimeException e) {
+ LOG.warnv("Error stopping container {0} during rollback of group {1}: {2}",
+ handle.getContainerId(), groupId, e.getMessage());
+ }
+ }
+ if (inFlightMemberId != null) {
+ try {
+ containerManager.stopByGroupId(inFlightMemberId);
+ } catch (RuntimeException e) {
+ LOG.warnv("Error stopping in-flight container {0} during rollback of group {1}: {2}",
+ inFlightMemberId, groupId, e.getMessage());
+ }
+ }
+ for (int port : proxyPorts) {
+ releaseProxyPort(port);
+ }
+ }
+
+ /**
+ * Brings the data plane back up for cluster-mode groups restored from disk. Invoked from
+ * {@code EmulatorLifecycle} after {@code storageFactory.loadAll()}, alongside the other
+ * services that restore persisted runtime. Only topology survives a restart — containers,
+ * proxies and port reservations are process-local — so each group is re-provisioned from
+ * its persisted node list, and a group whose data plane cannot come back is reported
+ * {@code create-failed} instead of available. Cluster-mode-disabled groups are deliberately
+ * out of scope: they keep their pre-existing behaviour of reporting available after a
+ * restart without a restored runtime.
+ *
+ * <p>Container restarts and cluster formation can take a readiness timeout and a formation
+ * timeout per group, so the data-plane work runs in the background and must not delay
+ * emulator readiness. Groups are marked {@code creating} synchronously — each node's proxy
+ * port is reserved at the same time, before any request can claim it — and flip to
+ * {@code available} or {@code create-failed} as their restoration finishes.
+ */
+ public CompletableFuture<Void> restorePersistedRuntime() {
+ List<ReplicationGroup> toRestore = new ArrayList<>();
+ for (ReplicationGroup group : groups.scan(k -> true)) {
+ if (!group.isClusterEnabled() || group.getClusterNodes().isEmpty()
+ || group.getStatus() == ReplicationGroupStatus.DELETING) {
+ continue;
+ }
+ List<Integer> reserved = new ArrayList<>();
try {
- handle = containerManager.start(groupId, image);
-
- String endpointHost = resolveEndpointHost();
- Endpoint endpoint = new Endpoint(endpointHost, proxyPort);
- ReplicationGroup group = new ReplicationGroup(
- groupId, description, ReplicationGroupStatus.AVAILABLE,
- authMode, endpoint, Instant.now(), proxyPort);
- group.setContainerId(handle.getContainerId());
- group.setContainerHost(handle.getHost());
- group.setContainerPort(handle.getPort());
- group.setAuthToken(authToken);
- group.setArn(regionResolver.buildArn("elasticache", region, "replicationgroup:" + groupId));
- group.setRegion(region);
- group.setSnapshotWindow(ReplicationGroupSettings.DEFAULT_SNAPSHOT_WINDOW);
- resolvedSettings.applyTo(group);
- if (tags != null && !tags.isEmpty()) {
- group.setTags(new LinkedHashMap<>(tags));
+ for (ClusterNode node : group.getClusterNodes()) {
+ node.setProxyPort(reserveOrAllocateProxyPort(node.getProxyPort()));
+ reserved.add(node.getProxyPort());
}
+ group.setStatus(ReplicationGroupStatus.CREATING);
+ toRestore.add(group);
+ } catch (RuntimeException e) {
+ reserved.forEach(this::releaseProxyPort);
+ group.setStatus(ReplicationGroupStatus.CREATE_FAILED);
+ group.setConfigurationEndpoint(null);
+ LOG.warnv(e, "Failed to restore cluster-mode replication group {0}",
+ group.getReplicationGroupId());
+ }
+ groups.put(group.getReplicationGroupId(), group);
+ }
+ if (toRestore.isEmpty()) {
+ return CompletableFuture.completedFuture(null);
+ }
+ LOG.infov("Restoring {0} cluster-mode replication group(s) in the background",
+ String.valueOf(toRestore.size()));
+ return CompletableFuture.runAsync(() -> toRestore.forEach(this::restoreClusterModeGroup));
+ }
+
+ private void restoreClusterModeGroup(ReplicationGroup group) {
+ String groupId = group.getReplicationGroupId();
+ String image = config.services().elasticache().defaultImage();
+ String endpointHost = resolveClusterAnnounceHost();
+ String announceIp = resolveAnnounceIp(endpointHost);
+ List<ClusterNode> nodes = group.getClusterNodes();
+
+ List<ElastiCacheContainerHandle> handles = new ArrayList<>();
+ List<String> startedProxyKeys = new ArrayList<>();
+ List<Integer> reservedPorts = nodes.stream().map(ClusterNode::getProxyPort).toList();
+ String inFlightMemberId = null;
+ try {
+ List<ValkeyClusterFormation.Node> formationNodes = new ArrayList<>(nodes.size());
+ for (ClusterNode node : nodes) {
+ inFlightMemberId = node.getMemberClusterId();
+ ElastiCacheContainerHandle handle = containerManager.start(
+ node.getMemberClusterId(), image,
+ clusterNodeFlags(endpointHost, announceIp, node.getProxyPort()));
+ inFlightMemberId = null;
+ handles.add(handle);
+ node.setContainerId(handle.getContainerId());
+ node.setContainerHost(handle.getHost());
+ node.setContainerPort(handle.getPort());
+ String networkIp = handle.getNetworkIp() != null ? handle.getNetworkIp() : handle.getHost();
+ formationNodes.add(new ValkeyClusterFormation.Node(
+ handle.getHost(), handle.getPort(), networkIp,
+ Integer.parseInt(node.getNodeGroupId()) - 1, node.isPrimary()));
+ }
- proxyManager.startProxy(groupId, authMode, proxyPort,
+ clusterFormation.form(groupId, formationNodes, group.getNumNodeGroups());
+
+ for (int i = 0; i < nodes.size(); i++) {
+ ClusterNode node = nodes.get(i);
+ ElastiCacheContainerHandle handle = handles.get(i);
+ proxyManager.startProxy(node.getMemberClusterId(), group.getAuthMode(), node.getProxyPort(),
handle.getHost(), handle.getPort(),
(username, password) -> validatePassword(groupId, username, password));
+ startedProxyKeys.add(node.getMemberClusterId());
+ }
+ group.setConfigurationEndpoint(new Endpoint(endpointHost, nodes.getFirst().getProxyPort()));
+ group.setStatus(ReplicationGroupStatus.AVAILABLE);
+ groups.put(groupId, group);
+ LOG.infov("Restored cluster-mode replication group {0}: {1} node(s), configuration endpoint={2}:{3}",
+ groupId, String.valueOf(nodes.size()), endpointHost,
+ String.valueOf(group.getConfigurationEndpoint().port()));
+ } catch (RuntimeException e) {
+ rollbackClusterModeGroup(groupId, startedProxyKeys, handles, inFlightMemberId, reservedPorts);
+ for (ClusterNode node : nodes) {
+ node.setContainerId(null);
+ node.setContainerHost(null);
+ node.setContainerPort(0);
+ }
+ group.setStatus(ReplicationGroupStatus.CREATE_FAILED);
+ group.setConfigurationEndpoint(null);
+ try {
groups.put(groupId, group);
- LOG.infov("Replication group {0} created, endpoint={1}:{2}", groupId, endpointHost, String.valueOf(proxyPort));
- return group;
- } catch (RuntimeException e) {
- LOG.warnv("Replication group {0} provisioning failed, rolling back: {1}", groupId, e.getMessage());
- rollbackReplicationGroup(groupId, handle, proxyPort);
- throw e;
+ } catch (RuntimeException persistFailure) {
+ e.addSuppressed(persistFailure);
}
- } finally {
- provisioningGroupIds.remove(groupId);
+ LOG.warnv(e, "Failed to restore cluster-mode replication group {0}", groupId);
}
}
@@ -206,19 +594,73 @@ public class ElastiCacheService implements ResourceProvider {
group.setStatus(ReplicationGroupStatus.DELETING);
groups.put(groupId, group);
- proxyManager.stopProxy(groupId);
+ if (group.isClusterEnabled() && !group.getClusterNodes().isEmpty()) {
+ for (ClusterNode node : group.getClusterNodes()) {
+ proxyManager.stopProxy(node.getMemberClusterId());
+ if (node.getContainerId() != null) {
+ containerManager.stop(new ElastiCacheContainerHandle(
+ node.getContainerId(), node.getMemberClusterId(),
+ node.getContainerHost(), node.getContainerPort()));
+ } else {
+ // Transient container fields are lost across a Floci restart; the
+ // deterministic container name still finds the node.
+ containerManager.stopByGroupId(node.getMemberClusterId());
+ }
+ releaseProxyPort(node.getProxyPort());
+ }
+ } else {
+ proxyManager.stopProxy(groupId);
- if (group.getContainerId() != null) {
- containerManager.stop(new ElastiCacheContainerHandle(
- group.getContainerId(), groupId, group.getContainerHost(), group.getContainerPort()));
- }
+ if (group.getContainerId() != null) {
+ containerManager.stop(new ElastiCacheContainerHandle(
+ group.getContainerId(), groupId, group.getContainerHost(), group.getContainerPort()));
+ }
- releaseProxyPort(group.getProxyPort());
+ releaseProxyPort(group.getProxyPort());
+ }
groups.delete(groupId);
LOG.infov("Replication group {0} deleted", groupId);
}
}
+ /**
+ * A replication-group member viewed as the cache cluster AWS reports it as. The terraform
+ * provider follows {@code MemberClusters} into DescribeCacheClusters, so every member name
+ * a describe hands out must answer there.
+ */
+ public record MemberCacheCluster(ReplicationGroup group, String cacheClusterId,
+ int port, boolean primary) {}
+
+ public List<MemberCacheCluster> listMemberCacheClusters(String filterClusterId) {
+ List<MemberCacheCluster> members = new ArrayList<>();
+ for (ReplicationGroup group : groups.scan(k -> true)) {
+ for (MemberCacheCluster member : memberCacheClusters(group)) {
+ if (filterClusterId == null || filterClusterId.isBlank()
+ || filterClusterId.equals(member.cacheClusterId())) {
+ members.add(member);
+ }
+ }
+ }
+ return members;
+ }
+
+ public List<MemberCacheCluster> memberCacheClusters(ReplicationGroup group) {
+ List<MemberCacheCluster> members = new ArrayList<>();
+ if (group.isClusterEnabled() && !group.getClusterNodes().isEmpty()) {
+ for (ClusterNode node : group.getClusterNodes()) {
+ members.add(new MemberCacheCluster(group, node.getMemberClusterId(),
+ node.getProxyPort(), node.isPrimary()));
+ }
+ return members;
+ }
+ for (int i = 1; i <= group.getNumCacheClusters(); i++) {
+ members.add(new MemberCacheCluster(group,
+ group.getReplicationGroupId() + "-" + String.format("%03d", i),
+ group.getProxyPort(), i == 1));
+ }
+ return members;
+ }
+
public ReplicationGroup modifyReplicationGroup(String groupId, List<String> userIdsToAdd,
List<String> userIdsToRemove) {
return modifyReplicationGroup(groupId, userIdsToAdd, userIdsToRemove,
@@ -394,6 +836,18 @@ public class ElastiCacheService implements ResourceProvider {
return config.hostname().orElse("localhost");
}
+ /**
+ * Cluster-mode groups bake this name into MOVED/ASK redirects and topology responses that
+ * every client must resolve, unlike the plain endpoint host that clients may override —
+ * so a hostname resolvable only inside Floci's Docker network (e.g. a Compose service
+ * name) needs the cluster-announce-hostname override.
+ */
+ private String resolveClusterAnnounceHost() {
+ return config.services().elasticache().clusterAnnounceHostname()
+ .filter(host -> !host.isBlank())
+ .orElseGet(this::resolveEndpointHost);
+ }
+
private int allocateProxyPort() {
int base = config.services().elasticache().proxyBasePort();
int max = config.services().elasticache().proxyMaxPort();
@@ -416,6 +870,13 @@ public class ElastiCacheService implements ResourceProvider {
usedPorts.remove(port);
}
+ private int reserveOrAllocateProxyPort(int persistedPort) {
+ if (persistedPort > 0 && usedPorts.add(persistedPort)) {
+ return persistedPort;
+ }
+ return allocateProxyPort();
+ }
+
// ── Cache Subnet Groups ───────────────────────────────────────────────────
/**
diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerHandle.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerHandle.java
index 264df02f8a..3b431cc7f4 100644
--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerHandle.java
+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerHandle.java
@@ -11,6 +11,7 @@ public class ElastiCacheContainerHandle {
private final String groupId;
private final String host;
private final int port;
+ private String networkIp;
private Closeable logStream;
public ElastiCacheContainerHandle(String containerId, String groupId, String host, int port) {
@@ -24,6 +25,11 @@ public class ElastiCacheContainerHandle {
public String getGroupId() { return groupId; }
public String getHost() { return host; }
public int getPort() { return port; }
+
+ /** The container's IP on its Docker network — the address sibling containers dial. */
+ public String getNetworkIp() { return networkIp; }
+ public void setNetworkIp(String networkIp) { this.networkIp = networkIp; }
+
public Closeable getLogStream() { return logStream; }
public void setLogStream(Closeable logStream) { this.logStream = logStream; }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManager.java
index f95e7dd283..8bfff344f5 100644
--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ElastiCacheContainerManager.java
@@ -70,6 +70,15 @@ public class ElastiCacheContainerManager {
}
public ElastiCacheContainerHandle start(String groupId, String image) {
+ return start(groupId, image, List.of());
+ }
+
+ /**
+ * Starts a backend container for the given resource id, appending {@code extraServerFlags}
+ * to the Valkey server command line (via the image's {@code VALKEY_EXTRA_FLAGS} hook).
+ * Cluster-mode nodes use this to pass {@code --cluster-enabled} and announce settings.
+ */
+ public ElastiCacheContainerHandle start(String groupId, String image, List<String> extraServerFlags) {
LOG.infov("Starting ElastiCache backend container for group: {0}", groupId);
String containerName = containerName(groupId);
@@ -77,12 +86,17 @@ public class ElastiCacheContainerManager {
// Remove any stale container with the same name
lifecycleManager.removeIfExists(containerName);
+ StringBuilder serverFlags = new StringBuilder("--loglevel verbose");
+ for (String flag : extraServerFlags) {
+ serverFlags.append(' ').append(flag);
+ }
+
// Build container spec. Only publish the backend port to the host in
// native mode — in Docker mode the JVM reaches the container via its
// network IP, no host binding needed.
ContainerBuilder.Builder specBuilder = containerBuilder.newContainer(image)
.withName(containerName)
- .withEnv("VALKEY_EXTRA_FLAGS", "--loglevel verbose")
+ .withEnv("VALKEY_EXTRA_FLAGS", serverFlags.toString())
.withDockerNetwork(config.services().elasticache().dockerNetwork())
.withLogRotation()
.withLabels(ContainerStorageHelper.resourceIdentityLabels(
@@ -104,6 +118,13 @@ public class ElastiCacheContainerManager {
ElastiCacheContainerHandle handle = new ElastiCacheContainerHandle(
info.containerId(), groupId, endpoint.host(), endpoint.port());
+ try {
+ handle.setNetworkIp(lifecycleManager.resolveContainerNetworkIp(
+ info.containerId(), config.services().elasticache().dockerNetwork().orElse(null)));
+ } catch (RuntimeException e) {
+ LOG.warnv("Could not resolve network IP for ElastiCache container {0}: {1}",
+ info.containerId(), e.getMessage());
+ }
activeContainers.put(groupId, handle);
// Attach log streaming
diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ValkeyClusterFormation.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ValkeyClusterFormation.java
new file mode 100644
index 0000000000..3f61c5f661
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/container/ValkeyClusterFormation.java
@@ -0,0 +1,310 @@
+package io.github.hectorvent.floci.services.elasticache.container;
+
+import jakarta.enterprise.context.ApplicationScoped;
+import org.jboss.logging.Logger;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.InetSocketAddress;
+import java.net.Socket;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Forms a Valkey cluster out of freshly started cluster-enabled nodes, the way
+ * {@code valkey-cli --cluster create} does: distinct config epochs, MEET, slot
+ * assignment on the primaries, then REPLICATE for the replicas.
+ *
+ * <p>Commands are issued from the JVM over each node's Floci-reachable endpoint;
+ * the addresses handed to MEET are the nodes' Docker-network IPs, because that is
+ * where the nodes reach <em>each other</em>.
+ */
+@ApplicationScoped
+public class ValkeyClusterFormation {
+
+ private static final Logger LOG = Logger.getLogger(ValkeyClusterFormation.class);
+
+ private static final int BACKEND_PORT = 6379;
+ private static final int TOTAL_SLOTS = 16384;
+ private static final int CONNECT_TIMEOUT_MS = 5_000;
+ private static final int READ_TIMEOUT_MS = 10_000;
+ private static final long FORMATION_DEADLINE_MS = 60_000;
+ private static final long RETRY_SLEEP_MS = 200;
+
+ /**
+ * One node to join into the cluster.
+ *
+ * @param endpointHost host Floci uses to reach the node
+ * @param endpointPort port Floci uses to reach the node
+ * @param networkIp the node's Docker-network IP, dialled by its peers
+ * @param nodeGroup zero-based shard index
+ * @param primary whether this node holds the shard's slots
+ */
+ public record Node(String endpointHost, int endpointPort, String networkIp,
+ int nodeGroup, boolean primary) {}
+
+ public void form(String groupId, List<Node> nodes, int numNodeGroups) {
+ long deadline = System.currentTimeMillis() + FORMATION_DEADLINE_MS;
+ List<RespClient> clients = new ArrayList<>(nodes.size());
+ try {
+ for (Node node : nodes) {
+ clients.add(new RespClient(node.endpointHost(), node.endpointPort()));
+ }
+
+ List<String> nodeIds = new ArrayList<>(nodes.size());
+ for (RespClient client : clients) {
+ nodeIds.add(client.callString("CLUSTER", "MYID"));
+ }
+
+ for (int i = 0; i < clients.size(); i++) {
+ try {
+ clients.get(i).callString("CLUSTER", "SET-CONFIG-EPOCH", String.valueOf(i + 1));
+ } catch (RespError e) {
+ LOG.debugv("SET-CONFIG-EPOCH on node {0} of group {1}: {2}",
+ String.valueOf(i), groupId, e.getMessage());
+ }
+ }
+
+ for (int i = 1; i < nodes.size(); i++) {
+ clients.getFirst().callString("CLUSTER", "MEET",
+ nodes.get(i).networkIp(), String.valueOf(BACKEND_PORT));
+ }
+
+ for (int i = 0; i < nodes.size(); i++) {
+ Node node = nodes.get(i);
+ if (node.primary()) {
+ int[] range = slotRange(node.nodeGroup(), numNodeGroups);
+ clients.get(i).callString("CLUSTER", "ADDSLOTSRANGE",
+ String.valueOf(range[0]), String.valueOf(range[1]));
+ }
+ }
+
+ awaitKnownNodes(groupId, clients, nodes.size(), deadline);
+
+ for (int i = 0; i < nodes.size(); i++) {
+ Node node = nodes.get(i);
+ if (!node.primary()) {
+ String primaryId = nodeIds.get(primaryIndex(nodes, node.nodeGroup()));
+ replicateWithRetry(groupId, clients.get(i), primaryId, deadline);
+ }
+ }
+
+ awaitClusterOk(groupId, clients, deadline);
+ LOG.infov("Valkey cluster for group {0} formed: {1} shard(s), {2} node(s)",
+ groupId, String.valueOf(numNodeGroups), String.valueOf(nodes.size()));
+ } catch (IOException e) {
+ throw new RuntimeException("Cluster formation for group " + groupId + " failed: " + e.getMessage(), e);
+ } finally {
+ clients.forEach(RespClient::closeQuietly);
+ }
+ }
+
+ /** The contiguous slot range served by the given shard, covering all 16384 slots overall. */
+ public static int[] slotRange(int nodeGroup, int numNodeGroups) {
+ int start = nodeGroup * TOTAL_SLOTS / numNodeGroups;
+ int end = (nodeGroup + 1) * TOTAL_SLOTS / numNodeGroups - 1;
+ return new int[] {start, end};
+ }
+
+ private static int primaryIndex(List<Node> nodes, int nodeGroup) {
+ for (int i = 0; i < nodes.size(); i++) {
+ if (nodes.get(i).primary() && nodes.get(i).nodeGroup() == nodeGroup) {
+ return i;
+ }
+ }
+ throw new IllegalStateException("No primary for node group " + nodeGroup);
+ }
+
+ private static void replicateWithRetry(String groupId, RespClient replica,
+ String primaryId, long deadline) throws IOException {
+ while (true) {
+ try {
+ replica.callString("CLUSTER", "REPLICATE", primaryId);
+ return;
+ } catch (RespError e) {
+ // The replica may not have gossiped the primary yet — retry until the deadline.
+ if (System.currentTimeMillis() >= deadline) {
+ throw new RuntimeException("Cluster formation for group " + groupId
+ + " timed out waiting to replicate " + primaryId + ": " + e.getMessage(), e);
+ }
+ sleep(groupId);
+ }
+ }
+ }
+
+ private static void awaitKnownNodes(String groupId, List<RespClient> clients,
+ int expected, long deadline) throws IOException {
+ while (!allMatch(clients, info -> parseInfoField(info, "cluster_known_nodes") >= expected)) {
+ if (System.currentTimeMillis() >= deadline) {
+ throw new RuntimeException("Cluster formation for group " + groupId
+ + " timed out waiting for all " + expected + " nodes to meet");
+ }
+ sleep(groupId);
+ }
+ }
+
+ private static void awaitClusterOk(String groupId, List<RespClient> clients,
+ long deadline) throws IOException {
+ while (!allMatch(clients, info -> info.contains("cluster_state:ok"))) {
+ if (System.currentTimeMillis() >= deadline) {
+ throw new RuntimeException("Cluster formation for group " + groupId
+ + " timed out waiting for cluster_state:ok");
+ }
+ sleep(groupId);
+ }
+ }
+
+ private static boolean allMatch(List<RespClient> clients,
+ java.util.function.Predicate<String> infoPredicate) throws IOException {
+ for (RespClient client : clients) {
+ if (!infoPredicate.test(client.callString("CLUSTER", "INFO"))) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private static long parseInfoField(String info, String field) {
+ for (String line : info.split("\r?\n")) {
+ if (line.startsWith(field + ":")) {
+ try {
+ return Long.parseLong(line.substring(field.length() + 1).trim());
+ } catch (NumberFormatException e) {
+ return -1;
+ }
+ }
+ }
+ return -1;
+ }
+
+ private static void sleep(String groupId) {
+ try {
+ Thread.sleep(RETRY_SLEEP_MS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException("Interrupted while forming cluster for group " + groupId, e);
+ }
+ }
+
+ /** An {@code -ERR} reply from the server, distinct from transport failures. */
+ static final class RespError extends IOException {
+ RespError(String message) {
+ super(message);
+ }
+ }
+
+ /** Minimal RESP2 client: sends one command array, reads one reply. */
+ static final class RespClient implements Closeable {
+
+ private final Socket socket;
+ private final InputStream in;
+ private final OutputStream out;
+
+ RespClient(String host, int port) throws IOException {
+ socket = new Socket();
+ socket.connect(new InetSocketAddress(host, port), CONNECT_TIMEOUT_MS);
+ socket.setTcpNoDelay(true);
+ socket.setSoTimeout(READ_TIMEOUT_MS);
+ in = socket.getInputStream();
+ out = socket.getOutputStream();
+ }
+
+ String callString(String... args) throws IOException {
+ Object reply = call(args);
+ return reply == null ? null : reply.toString();
+ }
+
+ Object call(String... args) throws IOException {
+ StringBuilder header = new StringBuilder();
+ header.append('*').append(args.length).append("\r\n");
+ out.write(header.toString().getBytes(StandardCharsets.UTF_8));
+ for (String arg : args) {
+ byte[] bytes = arg.getBytes(StandardCharsets.UTF_8);
+ out.write(("$" + bytes.length + "\r\n").getBytes(StandardCharsets.UTF_8));
+ out.write(bytes);
+ out.write('\r');
+ out.write('\n');
+ }
+ out.flush();
+ return readReply();
+ }
+
+ private Object readReply() throws IOException {
+ int type = in.read();
+ if (type == -1) {
+ throw new IOException("Connection closed while awaiting reply");
+ }
+ String line = readLine();
+ return switch (type) {
+ case '+' -> line;
+ case '-' -> throw new RespError(line);
+ case ':' -> Long.parseLong(line);
+ case '$' -> readBulk(Integer.parseInt(line));
+ case '*' -> readArray(Integer.parseInt(line));
+ default -> throw new IOException("Unexpected RESP type: " + (char) type);
+ };
+ }
+
+ private String readBulk(int length) throws IOException {
+ if (length < 0) {
+ return null;
+ }
+ byte[] data = in.readNBytes(length);
+ if (data.length != length) {
+ throw new IOException("Truncated bulk reply");
+ }
+ expectCrLf();
+ return new String(data, StandardCharsets.UTF_8);
+ }
+
+ private List<Object> readArray(int count) throws IOException {
+ if (count < 0) {
+ return null;
+ }
+ List<Object> items = new ArrayList<>(count);
+ for (int i = 0; i < count; i++) {
+ items.add(readReply());
+ }
+ return items;
+ }
+
+ private String readLine() throws IOException {
+ StringBuilder sb = new StringBuilder();
+ int b;
+ while ((b = in.read()) != -1) {
+ if (b == '\r') {
+ int next = in.read();
+ if (next != '\n') {
+ throw new IOException("Malformed RESP line terminator");
+ }
+ return sb.toString();
+ }
+ sb.append((char) b);
+ }
+ throw new IOException("Connection closed mid-line");
+ }
+
+ private void expectCrLf() throws IOException {
+ if (in.read() != '\r' || in.read() != '\n') {
+ throw new IOException("Missing CRLF after bulk reply");
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ socket.close();
+ }
+
+ void closeQuietly() {
+ try {
+ close();
+ } catch (IOException e) {
+ LOG.debugv("Ignoring close failure for formation connection to {0}: {1}",
+ socket.getRemoteSocketAddress(), e.getMessage());
+ }
+ }
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ClusterNode.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ClusterNode.java
new file mode 100644
index 0000000000..535ce938b9
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ClusterNode.java
@@ -0,0 +1,57 @@
+package io.github.hectorvent.floci.services.elasticache.model;
+
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+/**
+ * One node of a cluster-mode-enabled replication group: a member cache cluster
+ * backed by its own Valkey container and auth-proxy port.
+ */
+@RegisterForReflection
+public class ClusterNode {
+
+ private String memberClusterId;
+ private String nodeGroupId;
+ private boolean primary;
+ private int proxyPort;
+ private String slots;
+
+ // Transient fields — not persisted, restored on container restart
+ private transient String containerId;
+ private transient String containerHost;
+ private transient int containerPort;
+
+ public ClusterNode() {}
+
+ public ClusterNode(String memberClusterId, String nodeGroupId, boolean primary,
+ int proxyPort, String slots) {
+ this.memberClusterId = memberClusterId;
+ this.nodeGroupId = nodeGroupId;
+ this.primary = primary;
+ this.proxyPort = proxyPort;
+ this.slots = slots;
+ }
+
+ public String getMemberClusterId() { return memberClusterId; }
+ public void setMemberClusterId(String memberClusterId) { this.memberClusterId = memberClusterId; }
+
+ public String getNodeGroupId() { return nodeGroupId; }
+ public void setNodeGroupId(String nodeGroupId) { this.nodeGroupId = nodeGroupId; }
+
+ public boolean isPrimary() { return primary; }
+ public void setPrimary(boolean primary) { this.primary = primary; }
+
+ public int getProxyPort() { return proxyPort; }
+ public void setProxyPort(int proxyPort) { this.proxyPort = proxyPort; }
+
+ public String getSlots() { return slots; }
+ public void setSlots(String slots) { this.slots = slots; }
+
+ public String getContainerId() { return containerId; }
+ public void setContainerId(String containerId) { this.containerId = containerId; }
+
+ public String getContainerHost() { return containerHost; }
+ public void setContainerHost(String containerHost) { this.containerHost = containerHost; }
+
+ public int getContainerPort() { return containerPort; }
+ public void setContainerPort(int containerPort) { this.containerPort = containerPort; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroup.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroup.java
index d2e418fd0a..92875f9ff5 100644
--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroup.java
+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroup.java
@@ -3,8 +3,10 @@ package io.github.hectorvent.floci.services.elasticache.model;
import io.quarkus.runtime.annotations.RegisterForReflection;
import java.time.Instant;
+import java.util.ArrayList;
import java.util.HashSet;
import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -27,6 +29,18 @@ public class ReplicationGroup {
private int snapshotRetentionLimit;
private String snapshotWindow;
private Map<String, String> tags = new LinkedHashMap<>();
+ private boolean clusterEnabled;
+ private int numNodeGroups = 1;
+ private int replicasPerNodeGroup;
+ private int numCacheClusters = 1;
+ private String engine;
+ private String engineVersion;
+ private String cacheNodeType;
+ private String cacheParameterGroupName;
+ private String cacheSubnetGroupName;
+ private boolean automaticFailoverEnabled;
+ private boolean multiAzEnabled;
+ private List<ClusterNode> clusterNodes = new ArrayList<>();
// Transient fields — not persisted, restored on container restart
private transient String containerId;
@@ -105,4 +119,42 @@ public class ReplicationGroup {
public String getRegion() { return region; }
public void setRegion(String region) { this.region = region; }
+
+ public boolean isClusterEnabled() { return clusterEnabled; }
+ public void setClusterEnabled(boolean clusterEnabled) { this.clusterEnabled = clusterEnabled; }
+
+ public int getNumNodeGroups() { return numNodeGroups; }
+ public void setNumNodeGroups(int numNodeGroups) { this.numNodeGroups = numNodeGroups; }
+
+ public int getReplicasPerNodeGroup() { return replicasPerNodeGroup; }
+ public void setReplicasPerNodeGroup(int replicasPerNodeGroup) { this.replicasPerNodeGroup = replicasPerNodeGroup; }
+
+ public int getNumCacheClusters() { return numCacheClusters; }
+ public void setNumCacheClusters(int numCacheClusters) { this.numCacheClusters = numCacheClusters; }
+
+ public String getEngine() { return engine; }
+ public void setEngine(String engine) { this.engine = engine; }
+
+ public String getEngineVersion() { return engineVersion; }
+ public void setEngineVersion(String engineVersion) { this.engineVersion = engineVersion; }
+
+ public String getCacheNodeType() { return cacheNodeType; }
+ public void setCacheNodeType(String cacheNodeType) { this.cacheNodeType = cacheNodeType; }
+
+ public String getCacheParameterGroupName() { return cacheParameterGroupName; }
+ public void setCacheParameterGroupName(String cacheParameterGroupName) { this.cacheParameterGroupName = cacheParameterGroupName; }
+
+ public String getCacheSubnetGroupName() { return cacheSubnetGroupName; }
+ public void setCacheSubnetGroupName(String cacheSubnetGroupName) { this.cacheSubnetGroupName = cacheSubnetGroupName; }
+
+ public boolean isAutomaticFailoverEnabled() { return automaticFailoverEnabled; }
+ public void setAutomaticFailoverEnabled(boolean automaticFailoverEnabled) { this.automaticFailoverEnabled = automaticFailoverEnabled; }
+
+ public boolean isMultiAzEnabled() { return multiAzEnabled; }
+ public void setMultiAzEnabled(boolean multiAzEnabled) { this.multiAzEnabled = multiAzEnabled; }
+
+ public List<ClusterNode> getClusterNodes() { return clusterNodes; }
+ public void setClusterNodes(List<ClusterNode> clusterNodes) {
+ this.clusterNodes = clusterNodes != null ? clusterNodes : new ArrayList<>();
+ }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroupStatus.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroupStatus.java
index bd59857129..e5bd61f478 100644
--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroupStatus.java
+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroupStatus.java
@@ -2,7 +2,14 @@ package io.github.hectorvent.floci.services.elasticache.model;
import io.quarkus.runtime.annotations.RegisterForReflection;
+import java.util.Locale;
+
@RegisterForReflection
public enum ReplicationGroupStatus {
- CREATING, AVAILABLE, DELETING
+ CREATING, AVAILABLE, DELETING, CREATE_FAILED;
+
+ /** The status string as AWS reports it, e.g. {@code create-failed}. */
+ public String wireName() {
+ return name().toLowerCase(Locale.ROOT).replace('_', '-');
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java
index 6dd6c20da6..0590a5e18c 100644
--- a/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java
+++ b/src/test/java/io/github/hectorvent/floci/lifecycle/EmulatorLifecycleTest.java
@@ -7,6 +7,7 @@ import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.lifecycle.InitLifecycleState;
import io.github.hectorvent.floci.lifecycle.inithook.InitializationHook;
import io.github.hectorvent.floci.lifecycle.inithook.InitializationHooksRunner;
+import io.github.hectorvent.floci.services.elasticache.ElastiCacheService;
import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerManager;
import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheMemcachedContainerManager;
import io.github.hectorvent.floci.services.elasticache.proxy.ElastiCacheProxyManager;
@@ -60,6 +61,8 @@ class EmulatorLifecycleTest {
@Mock private EmulatorConfig.Ec2ServiceConfig ec2ServiceConfig;
@Mock private EmulatorConfig.ElbV2ServiceConfig elbv2ServiceConfig;
@Mock private IamService iamService;
+ @Mock private EmulatorConfig.ElastiCacheServiceConfig elastiCacheServiceConfig;
+ @Mock private ElastiCacheService elastiCacheService;
@Mock private ElastiCacheContainerManager elastiCacheContainerManager;
@Mock private ElastiCacheMemcachedContainerManager elastiCacheMemcachedContainerManager;
@Mock private ElastiCacheProxyManager elastiCacheProxyManager;
@@ -97,13 +100,15 @@ class EmulatorLifecycleTest {
Mockito.lenient().when(ec2ServiceConfig.enabled()).thenReturn(false);
Mockito.lenient().when(servicesConfig.elbv2()).thenReturn(elbv2ServiceConfig);
Mockito.lenient().when(elbv2ServiceConfig.enabled()).thenReturn(false);
+ Mockito.lenient().when(servicesConfig.elasticache()).thenReturn(elastiCacheServiceConfig);
+ Mockito.lenient().when(elastiCacheServiceConfig.enabled()).thenReturn(false);
Mockito.lenient().when(config.tls()).thenReturn(tlsConfig);
Mockito.lenient().when(tlsConfig.enabled()).thenReturn(false);
Mockito.lenient().when(config.port()).thenReturn(4566);
emulatorLifecycle = new EmulatorLifecycle(
storageFactory, serviceRegistry, config,
- iamService,
+ iamService, elastiCacheService,
elastiCacheContainerManager, elastiCacheMemcachedContainerManager,
elastiCacheProxyManager, rdsContainerManager, rdsProxyManager,
memoryDbContainerManager, memoryDbProxyManager,
@@ -170,6 +175,35 @@ class EmulatorLifecycleTest {
Mockito.verify(elbV2Service, Mockito.never()).restorePersistedRuntime();
}
+ @Test
+ @DisplayName("Should restore ElastiCache persisted runtime after loading storage when elasticache is enabled")
+ void shouldRestoreElastiCachePersistedRuntimeAfterStorageLoad() {
+ stubStorageConfig();
+ when(elastiCacheServiceConfig.enabled()).thenReturn(true);
+ when(elastiCacheService.restorePersistedRuntime())
+ .thenReturn(java.util.concurrent.CompletableFuture.completedFuture(null));
+ when(initializationHooksRunner.hasHooks(InitializationHook.START)).thenReturn(false);
+ when(initializationHooksRunner.hasHooks(InitializationHook.READY)).thenReturn(false);
+
+ emulatorLifecycle.onStart(Mockito.mock(StartupEvent.class));
+
+ var inOrder = Mockito.inOrder(storageFactory, elastiCacheService);
+ inOrder.verify(storageFactory).loadAll();
+ inOrder.verify(elastiCacheService).restorePersistedRuntime();
+ }
+
+ @Test
+ @DisplayName("Should not restore ElastiCache persisted runtime when elasticache is disabled")
+ void shouldNotRestoreElastiCachePersistedRuntimeWhenDisabled() {
+ stubStorageConfig();
+ when(initializationHooksRunner.hasHooks(InitializationHook.START)).thenReturn(false);
+ when(initializationHooksRunner.hasHooks(InitializationHook.READY)).thenReturn(false);
+
+ emulatorLifecycle.onStart(Mockito.mock(StartupEvent.class));
+
+ Mockito.verify(elastiCacheService, Mockito.never()).restorePersistedRuntime();
+ }
+
@Test
@DisplayName("Should rehydrate AppSync schemas after orphan recovery on startup")
void shouldRehydrateAppSyncSchemasAfterOrphanRecovery() {
diff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheClusterIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheClusterIntegrationTest.java
new file mode 100644
index 0000000000..76ee86d839
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheClusterIntegrationTest.java
@@ -0,0 +1,266 @@
+package io.github.hectorvent.floci.services.elasticache;
+
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.path.xml.XmlPath;
+import org.junit.jupiter.api.AfterAll;
+import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.Socket;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * End-to-end coverage for cluster-mode replication groups: real sharded Valkey
+ * topology behind per-node auth proxies, honest describe output, and MOVED
+ * redirects that point clients at the proxy ports.
+ *
+ * <p>Key slots used below are the well-known CRC16 values: {@code bar} → 5061
+ * (first shard of two) and {@code foo} → 12182 (second shard of two).
+ */
+@QuarkusTest
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class ElastiCacheClusterIntegrationTest {
+
+ private static final String AUTH_HEADER =
+ "AWS4-HMAC-SHA256 Credential=test/20260412/us-east-1/elasticache/aws4_request";
+ private static final String GROUP_ID = "it-ec-cluster";
+
+ private static final int SOCKET_TIMEOUT_MS = 10_000;
+
+ private static int configurationEndpointPort;
+ private static int secondShardPort;
+
+ @BeforeAll
+ static void requireDocker() {
+ Assumptions.assumeTrue(isDockerAvailable(), "Docker daemon must be available for ElastiCache integration tests");
+ }
+
+ @AfterAll
+ static void cleanup() {
+ try {
+ given()
+ .formParam("Action", "DeleteReplicationGroup")
+ .formParam("ReplicationGroupId", GROUP_ID)
+ .header("Authorization", AUTH_HEADER)
+ .post("/");
+ } catch (Exception e) {
+ System.err.println("Cleanup of " + GROUP_ID + " failed: " + e.getMessage());
+ }
+ }
+
+ @Test
+ @Order(1)
+ void createClusterModeReplicationGroup() {
+ configurationEndpointPort =
+ given()
+ .formParam("Action", "CreateReplicationGroup")
+ .formParam("ReplicationGroupId", GROUP_ID)
+ .formParam("ReplicationGroupDescription", "Cluster mode integration test")
+ .formParam("Engine", "valkey")
+ .formParam("EngineVersion", "8.2")
+ .formParam("CacheNodeType", "cache.t4g.micro")
+ .formParam("CacheParameterGroupName", "default.valkey8.cluster.on")
+ .formParam("NumNodeGroups", "2")
+ .formParam("ReplicasPerNodeGroup", "1")
+ .formParam("AutomaticFailoverEnabled", "true")
+ .header("Authorization", AUTH_HEADER)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("CreateReplicationGroupResponse.CreateReplicationGroupResult.ReplicationGroup.ReplicationGroupId", equalTo(GROUP_ID))
+ .body("CreateReplicationGroupResponse.CreateReplicationGroupResult.ReplicationGroup.Status", equalTo("available"))
+ .body("CreateReplicationGroupResponse.CreateReplicationGroupResult.ReplicationGroup.ClusterEnabled", equalTo("true"))
+ .body("CreateReplicationGroupResponse.CreateReplicationGroupResult.ReplicationGroup.AutomaticFailover", equalTo("enabled"))
+ .body("CreateReplicationGroupResponse.CreateReplicationGroupResult.ReplicationGroup.Engine", equalTo("valkey"))
+ .extract()
+ .xmlPath()
+ .getInt("CreateReplicationGroupResponse.CreateReplicationGroupResult.ReplicationGroup.ConfigurationEndpoint.Port");
+ }
+
+ @Test
+ @Order(2)
+ void describeReportsClusterTopology() {
+ XmlPath xml = given()
+ .formParam("Action", "DescribeReplicationGroups")
+ .formParam("ReplicationGroupId", GROUP_ID)
+ .header("Authorization", AUTH_HEADER)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .extract()
+ .xmlPath();
+
+ String prefix = "DescribeReplicationGroupsResponse.DescribeReplicationGroupsResult.ReplicationGroups.ReplicationGroup.";
+ assertEquals("true", xml.getString(prefix + "ClusterEnabled"));
+ List<String> memberClusters = xml.getList(prefix + "MemberClusters.ClusterId");
+ assertEquals(List.of(GROUP_ID + "-0001-001", GROUP_ID + "-0001-002",
+ GROUP_ID + "-0002-001", GROUP_ID + "-0002-002"), memberClusters);
+ assertEquals(List.of("0001", "0002"), xml.getList(prefix + "NodeGroups.NodeGroup.NodeGroupId"));
+ assertEquals(List.of("0-8191", "8192-16383"), xml.getList(prefix + "NodeGroups.NodeGroup.Slots"));
+ }
+
+ @Test
+ @Order(3)
+ void describeCacheClustersAnswersForMembers() {
+ secondShardPort = given()
+ .formParam("Action", "DescribeCacheClusters")
+ .formParam("CacheClusterId", GROUP_ID + "-0002-001")
+ .formParam("ShowCacheNodeInfo", "true")
+ .header("Authorization", AUTH_HEADER)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200)
+ .body("DescribeCacheClustersResponse.DescribeCacheClustersResult.CacheClusters.CacheCluster.CacheClusterId",
+ equalTo(GROUP_ID + "-0002-001"))
+ .body("DescribeCacheClustersResponse.DescribeCacheClustersResult.CacheClusters.CacheCluster.ReplicationGroupId",
+ equalTo(GROUP_ID))
+ .body("DescribeCacheClustersResponse.DescribeCacheClustersResult.CacheClusters.CacheCluster.Engine",
+ equalTo("valkey"))
+ .body("DescribeCacheClustersResponse.DescribeCacheClustersResult.CacheClusters.CacheCluster.EngineVersion",
+ equalTo("8.2"))
+ .extract()
+ .xmlPath()
+ .getInt("DescribeCacheClustersResponse.DescribeCacheClustersResult.CacheClusters.CacheCluster.CacheNodes.CacheNode.Endpoint.Port");
+ }
+
+ @Test
+ @Order(4)
+ void clusterStateIsOkThroughTheProxy() throws Exception {
+ try (Socket socket = openSocket(configurationEndpointPort)) {
+ write(socket, respArray("PING"));
+ assertEquals("+PONG\r\n", readLine(socket));
+
+ write(socket, respArray("CLUSTER", "INFO"));
+ String info = readBulk(socket);
+ assertTrue(info.contains("cluster_state:ok"), "Expected cluster_state:ok but got: " + info);
+ assertTrue(info.contains("cluster_known_nodes:4"), "Expected 4 known nodes but got: " + info);
+ }
+ }
+
+ @Test
+ @Order(5)
+ void movedRedirectPointsAtTheOtherShardsProxyPort() throws Exception {
+ try (Socket socket = openSocket(configurationEndpointPort)) {
+ write(socket, respArray("SET", "bar", "first-shard-value"));
+ assertEquals("+OK\r\n", readLine(socket));
+
+ write(socket, respArray("SET", "foo", "second-shard-value"));
+ String reply = readLine(socket);
+ assertTrue(reply.startsWith("-MOVED 12182 "), "Expected MOVED redirect but got: " + reply);
+ String target = reply.trim().substring("-MOVED 12182 ".length());
+ assertEquals("localhost:" + secondShardPort, target,
+ "MOVED must point at the announced hostname and the second shard primary's proxy port");
+ }
+
+ try (Socket socket = openSocket(secondShardPort)) {
+ write(socket, respArray("SET", "foo", "second-shard-value"));
+ assertEquals("+OK\r\n", readLine(socket));
+
+ write(socket, respArray("GET", "foo"));
+ assertEquals("second-shard-value", readBulk(socket));
+ }
+ }
+
+ @Test
+ @Order(6)
+ void deleteClusterModeReplicationGroup() {
+ given()
+ .formParam("Action", "DeleteReplicationGroup")
+ .formParam("ReplicationGroupId", GROUP_ID)
+ .header("Authorization", AUTH_HEADER)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(200);
+
+ given()
+ .formParam("Action", "DescribeReplicationGroups")
+ .formParam("ReplicationGroupId", GROUP_ID)
+ .header("Authorization", AUTH_HEADER)
+ .when()
+ .post("/")
+ .then()
+ .statusCode(404);
+ }
+
+ private static boolean isDockerAvailable() {
+ try {
+ Process process = new ProcessBuilder("docker", "info").start();
+ return process.waitFor() == 0;
+ } catch (Exception e) {
+ return false;
+ }
+ }
+
+ private static Socket openSocket(int port) throws IOException {
+ Socket socket = new Socket("localhost", port);
+ socket.setTcpNoDelay(true);
+ socket.setSoTimeout(SOCKET_TIMEOUT_MS);
+ return socket;
+ }
+
+ private static void write(Socket socket, byte[] data) throws IOException {
+ OutputStream out = socket.getOutputStream();
+ out.write(data);
+ out.flush();
+ }
+
+ private static byte[] respArray(String... args) {
+ StringBuilder sb = new StringBuilder();
+ sb.append('*').append(args.length).append("\r\n");
+ for (String arg : args) {
+ byte[] bytes = arg.getBytes(StandardCharsets.UTF_8);
+ sb.append('$').append(bytes.length).append("\r\n").append(arg).append("\r\n");
+ }
+ return sb.toString().getBytes(StandardCharsets.UTF_8);
+ }
+
+ private static String readLine(Socket socket) throws IOException {
+ InputStream in = socket.getInputStream();
+ StringBuilder sb = new StringBuilder();
+ int b;
+ while ((b = in.read()) != -1) {
+ sb.append((char) b);
+ if (sb.length() >= 2 && sb.charAt(sb.length() - 2) == '\r' && sb.charAt(sb.length() - 1) == '\n') {
+ return sb.toString();
+ }
+ }
+ throw new IOException("Connection closed before line terminator; read so far: " + sb);
+ }
+
+ private static String readBulk(Socket socket) throws IOException {
+ String header = readLine(socket);
+ if (!header.startsWith("$")) {
+ throw new IOException("Expected bulk reply but got: " + header);
+ }
+ int length = Integer.parseInt(header.substring(1).trim());
+ if (length < 0) {
+ return null;
+ }
+ InputStream in = socket.getInputStream();
+ byte[] data = in.readNBytes(length);
+ if (data.length != length) {
+ throw new IOException("Truncated bulk reply");
+ }
+ if (in.read() != '\r' || in.read() != '\n') {
+ throw new IOException("Missing CRLF after bulk reply");
+ }
+ return new String(data, StandardCharsets.UTF_8);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandlerTest.java
index dd7afcf206..3037b45ec7 100644
--- a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandlerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandlerTest.java
@@ -1,6 +1,11 @@
package io.github.hectorvent.floci.services.elasticache;
import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.services.elasticache.model.AuthMode;
+import io.github.hectorvent.floci.services.elasticache.model.ClusterNode;
+import io.github.hectorvent.floci.services.elasticache.model.Endpoint;
+import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroup;
+import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroupStatus;
import io.github.hectorvent.floci.services.elasticache.proxy.SigV4Validator;
import jakarta.ws.rs.core.MultivaluedHashMap;
import jakarta.ws.rs.core.MultivaluedMap;
@@ -19,8 +24,12 @@ import static org.mockito.Mockito.*;
import org.mockito.ArgumentCaptor;
import org.junit.jupiter.api.Test;
+import java.time.Instant;
+import java.util.List;
+
import static org.junit.jupiter.api.Assertions.*;
import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
/**
* Verifies the empty-list read responses for the subnet/parameter group describes, so
@@ -64,6 +73,124 @@ class ElastiCacheQueryHandlerTest {
assertFalse(body.contains("<Marker>"), "Empty list must omit Marker");
}
+ @Test
+ void describeReplicationGroups_reportsClusterModeTopology() {
+ ReplicationGroup group = new ReplicationGroup("grp", "d", ReplicationGroupStatus.AVAILABLE,
+ AuthMode.NO_AUTH, new Endpoint("localhost", 6379), Instant.now(), 6379);
+ group.setClusterEnabled(true);
+ group.setEngine("valkey");
+ group.setAutomaticFailoverEnabled(true);
+ group.setClusterNodes(List.of(
+ new ClusterNode("grp-0001-001", "0001", true, 6379, "0-8191"),
+ new ClusterNode("grp-0002-001", "0002", true, 6380, "8192-16383")));
+ when(service.listReplicationGroups("grp")).thenReturn(List.of(group));
+ when(service.memberCacheClusters(group)).thenReturn(List.of(
+ new ElastiCacheService.MemberCacheCluster(group, "grp-0001-001", 6379, true),
+ new ElastiCacheService.MemberCacheCluster(group, "grp-0002-001", 6380, true)));
+
+ MultivaluedMap<String, String> params = params();
+ params.putSingle("ReplicationGroupId", "grp");
+ Response response = handler.handle("DescribeReplicationGroups", params, "us-east-1");
+
+ assertEquals(200, response.getStatus());
+ String body = (String) response.getEntity();
+ assertTrue(body.contains("<ClusterEnabled>true</ClusterEnabled>"));
+ assertTrue(body.contains("<ClusterMode>enabled</ClusterMode>"));
+ assertTrue(body.contains("<AutomaticFailover>enabled</AutomaticFailover>"));
+ assertTrue(body.contains("<Engine>valkey</Engine>"));
+ assertTrue(body.contains("<ClusterId>grp-0001-001</ClusterId>"));
+ assertTrue(body.contains("<ClusterId>grp-0002-001</ClusterId>"));
+ assertTrue(body.contains("<NodeGroupId>0002</NodeGroupId>"));
+ assertTrue(body.contains("<Slots>8192-16383</Slots>"));
+ assertTrue(body.contains("<CacheClusterId>grp-0002-001</CacheClusterId>"));
+ assertFalse(body.contains("<PrimaryEndpoint>"),
+ "Cluster-mode node groups carry Slots, not a PrimaryEndpoint");
+ }
+
+ @Test
+ void describeReplicationGroups_reportsSingleNodeGroupWithPrimaryEndpoint() {
+ ReplicationGroup group = new ReplicationGroup("grp", "d", ReplicationGroupStatus.AVAILABLE,
+ AuthMode.NO_AUTH, new Endpoint("localhost", 6379), Instant.now(), 6379);
+ group.setEngine("valkey");
+ when(service.listReplicationGroups("grp")).thenReturn(List.of(group));
+ when(service.memberCacheClusters(group)).thenReturn(List.of(
+ new ElastiCacheService.MemberCacheCluster(group, "grp-001", 6379, true)));
+
+ MultivaluedMap<String, String> params = params();
+ params.putSingle("ReplicationGroupId", "grp");
+ Response response = handler.handle("DescribeReplicationGroups", params, "us-east-1");
+
+ assertEquals(200, response.getStatus());
+ String body = (String) response.getEntity();
+ assertTrue(body.contains("<ClusterEnabled>false</ClusterEnabled>"));
+ assertTrue(body.contains("<ClusterId>grp-001</ClusterId>"));
+ assertTrue(body.contains("<PrimaryEndpoint><Address>localhost</Address><Port>6379</Port></PrimaryEndpoint>"));
+ assertTrue(body.contains("<CurrentRole>primary</CurrentRole>"));
+ }
+
+ @Test
+ void describeCacheClusters_answersForReplicationGroupMembers() {
+ ReplicationGroup group = new ReplicationGroup("grp", "d", ReplicationGroupStatus.AVAILABLE,
+ AuthMode.NO_AUTH, new Endpoint("localhost", 6379), Instant.now(), 6379);
+ group.setEngine("valkey");
+ group.setEngineVersion("8.2");
+ group.setCacheNodeType("cache.t4g.micro");
+ when(service.listMemberCacheClusters("grp-0001-001")).thenReturn(List.of(
+ new ElastiCacheService.MemberCacheCluster(group, "grp-0001-001", 6379, true)));
+
+ MultivaluedMap<String, String> params = params();
+ params.putSingle("CacheClusterId", "grp-0001-001");
+ params.putSingle("ShowCacheNodeInfo", "true");
+ Response response = handler.handle("DescribeCacheClusters", params, "us-east-1");
+
+ assertEquals(200, response.getStatus());
+ String body = (String) response.getEntity();
+ assertTrue(body.contains("<CacheClusterId>grp-0001-001</CacheClusterId>"));
+ assertTrue(body.contains("<CacheClusterStatus>available</CacheClusterStatus>"));
+ assertTrue(body.contains("<ReplicationGroupId>grp</ReplicationGroupId>"));
+ assertTrue(body.contains("<Engine>valkey</Engine>"));
+ assertTrue(body.contains("<EngineVersion>8.2</EngineVersion>"));
+ assertTrue(body.contains("<CacheNodeType>cache.t4g.micro</CacheNodeType>"));
+ assertTrue(body.contains("<Endpoint><Address>localhost</Address><Port>6379</Port></Endpoint>"));
+ }
+
+ @Test
+ void describeCacheClusters_membersOfFailedGroupReportRestoreFailed() {
+ ReplicationGroup group = new ReplicationGroup("grp", "d", ReplicationGroupStatus.CREATE_FAILED,
+ AuthMode.NO_AUTH, null, Instant.now(), 6379);
+ when(service.listMemberCacheClusters("grp-0001-001")).thenReturn(List.of(
+ new ElastiCacheService.MemberCacheCluster(group, "grp-0001-001", 6379, true)));
+
+ MultivaluedMap<String, String> params = params();
+ params.putSingle("CacheClusterId", "grp-0001-001");
+ params.putSingle("ShowCacheNodeInfo", "true");
+ Response response = handler.handle("DescribeCacheClusters", params, "us-east-1");
+
+ assertEquals(200, response.getStatus());
+ String body = (String) response.getEntity();
+ assertTrue(body.contains("<CacheClusterStatus>restore-failed</CacheClusterStatus>"),
+ "CacheClusterStatus has no create-failed; the documented failure value is restore-failed: " + body);
+ assertFalse(body.contains("<CacheNodes>"),
+ "A failed group has no endpoint, so members must not hand out node endpoints");
+ }
+
+ @Test
+ void describeReplicationGroups_reportsCreateFailedStatusInAwsWireForm() {
+ ReplicationGroup group = new ReplicationGroup("grp", "d", ReplicationGroupStatus.CREATE_FAILED,
+ AuthMode.NO_AUTH, null, Instant.now(), 6379);
+ when(service.listReplicationGroups("grp")).thenReturn(List.of(group));
+ when(service.memberCacheClusters(group)).thenReturn(List.of());
+
+ MultivaluedMap<String, String> params = params();
+ params.putSingle("ReplicationGroupId", "grp");
+ Response response = handler.handle("DescribeReplicationGroups", params, "us-east-1");
+
+ assertEquals(200, response.getStatus());
+ String body = (String) response.getEntity();
+ assertTrue(body.contains("<Status>create-failed</Status>"),
+ "AWS reports this status hyphenated, not as the enum constant");
+ }
+
@Test
void unsupportedOperationStillReturnsQueryError() {
Response response = handler.handle("NoSuchAction", params(), "us-east-1");
@@ -85,7 +212,7 @@ class ElastiCacheQueryHandlerTest {
@Test
void createReplicationGroup_passesSettingsAndTagsToService() {
- when(service.createReplicationGroup(any(), any(), any(), any(), any(), any(), any()))
+ when(service.createReplicationGroup(any(ElastiCacheService.CreateReplicationGroupRequest.class)))
.thenReturn(group("g1"));
MultivaluedMap<String, String> p = params();
p.add("ReplicationGroupId", "g1");
@@ -99,15 +226,24 @@ class ElastiCacheQueryHandlerTest {
assertEquals(200, handler.handle("CreateReplicationGroup", p, "us-east-1").getStatus());
- verify(service).createReplicationGroup(eq("g1"), eq("d"), eq(AuthMode.NO_AUTH), isNull(), eq("us-east-1"),
- eq(new ReplicationGroupSettings(true, "alias/cache", 7, "06:30-07:30")), eq(Map.of("Name", "g1")));
+ ArgumentCaptor<ElastiCacheService.CreateReplicationGroupRequest> captor =
+ ArgumentCaptor.forClass(ElastiCacheService.CreateReplicationGroupRequest.class);
+ verify(service).createReplicationGroup(captor.capture());
+ ElastiCacheService.CreateReplicationGroupRequest request = captor.getValue();
+ assertEquals("g1", request.replicationGroupId());
+ assertEquals("d", request.description());
+ assertEquals(AuthMode.NO_AUTH, request.authMode());
+ assertEquals("us-east-1", request.region());
+ assertEquals(new ReplicationGroupSettings(true, "alias/cache", 7, "06:30-07:30"), request.settings());
+ assertEquals(Map.of("Name", "g1"), request.tags());
}
@Test
void createReplicationGroup_readsTheEncryptionFlagAsAwsDoes() {
- when(service.createReplicationGroup(any(), any(), any(), any(), any(), any(), any()))
+ when(service.createReplicationGroup(any(ElastiCacheService.CreateReplicationGroupRequest.class)))
.thenReturn(group("g1"));
- ArgumentCaptor<ReplicationGroupSettings> captor = ArgumentCaptor.forClass(ReplicationGroupSettings.class);
+ ArgumentCaptor<ElastiCacheService.CreateReplicationGroupRequest> captor =
+ ArgumentCaptor.forClass(ElastiCacheService.CreateReplicationGroupRequest.class);
// a live account reads anything but "false" as true — probed with banana, yes and "TRUE "
for (String value : new String[] {"true", "banana", "yes", "TRUE "}) {
MultivaluedMap<String, String> p = params();
@@ -125,8 +261,9 @@ class ElastiCacheQueryHandlerTest {
omitted.add("ReplicationGroupId", "g1");
handler.handle("CreateReplicationGroup", omitted, "us-east-1");
- verify(service, times(7)).createReplicationGroup(any(), any(), any(), any(), any(), captor.capture(), any());
- List<Boolean> seen = captor.getAllValues().stream().map(ReplicationGroupSettings::atRestEncryptionEnabled).toList();
+ verify(service, times(7)).createReplicationGroup(captor.capture());
+ List<Boolean> seen = captor.getAllValues().stream()
+ .map(r -> r.settings().atRestEncryptionEnabled()).toList();
assertEquals(java.util.Arrays.asList(true, true, true, true, false, false, null), seen);
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java
index f1ffdc7269..2c96db9834 100644
--- a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java
@@ -12,14 +12,20 @@ import io.github.hectorvent.floci.services.kms.model.KmsKey;
import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroupSettings;
import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerHandle;
import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerManager;
+import io.github.hectorvent.floci.services.elasticache.container.ValkeyClusterFormation;
import io.github.hectorvent.floci.services.elasticache.model.AuthMode;
+import io.github.hectorvent.floci.services.elasticache.model.ClusterNode;
import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroup;
+import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroupStatus;
import io.github.hectorvent.floci.services.elasticache.proxy.ElastiCacheProxyManager;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.mockito.ArgumentCaptor;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
@@ -36,6 +42,7 @@ import static org.mockito.Mockito.doNothing;
import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -46,6 +53,7 @@ class ElastiCacheServiceTest {
private ElastiCacheContainerManager containerManager;
private ElastiCacheProxyManager proxyManager;
private EmulatorConfig config;
+ private ValkeyClusterFormation clusterFormation;
@BeforeEach
void setUp() {
@@ -71,8 +79,10 @@ class ElastiCacheServiceTest {
kmsService = org.mockito.Mockito.mock(KmsService.class);
when(kmsService.describeKey(any(), any())).thenThrow(
new AwsException("NotFoundException", "Key not found", 404));
- service = new ElastiCacheService(containerManager, proxyManager, storageFactory, config,
- ec2Service, new RegionResolver("us-east-1", "000000000000"), kmsService);
+ clusterFormation = mock(ValkeyClusterFormation.class);
+ service = new ElastiCacheService(containerManager, proxyManager, clusterFormation,
+ storageFactory, config, ec2Service, new RegionResolver("us-east-1", "000000000000"),
+ kmsService);
}
@Test
@@ -97,8 +107,8 @@ class ElastiCacheServiceTest {
when(cm.start(anyString(), anyString()))
.thenReturn(new ElastiCacheContainerHandle("cid", "grp", "localhost", 6379));
doNothing().when(pm).startProxy(anyString(), any(), anyInt(), anyString(), anyInt(), any());
- ElastiCacheService svc = new ElastiCacheService(cm, pm, sf, cfg,
- org.mockito.Mockito.mock(Ec2Service.class),
+ ElastiCacheService svc = new ElastiCacheService(cm, pm, mock(ValkeyClusterFormation.class),
+ sf, cfg, org.mockito.Mockito.mock(Ec2Service.class),
new RegionResolver("us-east-1", "000000000000"),
org.mockito.Mockito.mock(KmsService.class));
@@ -207,6 +217,285 @@ class ElastiCacheServiceTest {
"Port from the failed create must be released so the next group reuses it");
}
+ private static ElastiCacheService.CreateReplicationGroupRequest clusterRequest(
+ String groupId, Integer numNodeGroups, Integer replicasPerNodeGroup) {
+ return new ElastiCacheService.CreateReplicationGroupRequest(groupId, "test",
+ AuthMode.NO_AUTH, null, "us-east-1", "valkey", "8.2", "cache.t4g.micro",
+ "default.valkey8.cluster.on", null, null, numNodeGroups, replicasPerNodeGroup,
+ null, true, null, ReplicationGroupSettings.defaults(), Map.of());
+ }
+
+ private void stubPerNodeContainers() {
+ when(containerManager.start(anyString(), anyString(), any())).thenAnswer(inv ->
+ new ElastiCacheContainerHandle("cid-" + inv.getArgument(0, String.class),
+ inv.getArgument(0, String.class), "localhost", 6379));
+ }
+
+ @Test
+ void clusterModeCreateStartsOneContainerAndProxyPerNode() {
+ stubPerNodeContainers();
+
+ ReplicationGroup group = service.createReplicationGroup(clusterRequest("grp", 2, 1));
+
+ assertTrue(group.isClusterEnabled());
+ assertEquals(2, group.getNumNodeGroups());
+ assertEquals(1, group.getReplicasPerNodeGroup());
+ assertEquals(4, group.getClusterNodes().size());
+
+ List<ClusterNode> nodes = group.getClusterNodes();
+ assertEquals("grp-0001-001", nodes.get(0).getMemberClusterId());
+ assertEquals("grp-0001-002", nodes.get(1).getMemberClusterId());
+ assertEquals("grp-0002-001", nodes.get(2).getMemberClusterId());
+ assertEquals("grp-0002-002", nodes.get(3).getMemberClusterId());
+ assertTrue(nodes.get(0).isPrimary());
+ assertFalse(nodes.get(1).isPrimary());
+ assertEquals("0-8191", nodes.get(0).getSlots());
+ assertEquals("8192-16383", nodes.get(2).getSlots());
+ assertEquals(4, nodes.stream().map(ClusterNode::getProxyPort).distinct().count(),
+ "Each node must own its own proxy port");
+ assertEquals(16379, group.getConfigurationEndpoint().port());
+
+ verify(clusterFormation).form(eq("grp"), any(), eq(2));
+ verify(containerManager, times(4)).start(anyString(), anyString(), any());
+ verify(proxyManager, times(4)).startProxy(anyString(), any(), anyInt(), anyString(), anyInt(), any());
+ }
+
+ @Test
+ void clusterOnParameterGroupEnablesClusterModeForSingleShard() {
+ stubPerNodeContainers();
+
+ ReplicationGroup group = service.createReplicationGroup(clusterRequest("grp", 1, 0));
+
+ assertTrue(group.isClusterEnabled());
+ assertEquals(1, group.getClusterNodes().size());
+ assertEquals("0-16383", group.getClusterNodes().getFirst().getSlots());
+ }
+
+ @Test
+ void numNodeGroupsBeyondQuotaSurfacesModeledQuotaFault() {
+ AwsException ex = assertThrows(AwsException.class,
+ () -> service.createReplicationGroup(clusterRequest("grp", 501, 0)));
+
+ assertEquals("NodeGroupsPerReplicationGroupQuotaExceeded", ex.getErrorCode());
+ assertEquals(400, ex.getHttpStatus());
+ verify(containerManager, never()).start(anyString(), anyString(), any());
+ }
+
+ @Test
+ void plainCreateStaysClusterModeDisabled() {
+ ReplicationGroup group =
+ service.createReplicationGroup("grp", "test", AuthMode.NO_AUTH, null, "us-east-1");
+
+ assertFalse(group.isClusterEnabled());
+ assertTrue(group.getClusterNodes().isEmpty());
+ assertEquals(List.of("grp-001"),
+ service.memberCacheClusters(group).stream()
+ .map(ElastiCacheService.MemberCacheCluster::cacheClusterId).toList());
+ verify(containerManager, never()).start(anyString(), anyString(), any());
+ }
+
+ @Test
+ void clusterFormationFailureRollsBackAllNodesAndReleasesPorts() {
+ stubPerNodeContainers();
+ doThrow(new RuntimeException("formation boom"))
+ .when(clusterFormation).form(anyString(), any(), anyInt());
+
+ assertThrows(RuntimeException.class,
+ () -> service.createReplicationGroup(clusterRequest("grp", 2, 1)));
+
+ verify(containerManager, times(4)).stop(any());
+ verify(proxyManager, never()).startProxy(anyString(), any(), anyInt(), anyString(), anyInt(), any());
+
+ ReplicationGroup recovered =
+ service.createReplicationGroup("grp2", "test", AuthMode.PASSWORD, null, "us-east-1");
+ assertEquals(16379, recovered.getProxyPort(),
+ "Ports from the failed cluster create must be released for the next group");
+ }
+
+ @Test
+ void deleteClusterModeGroupStopsEveryNodeAndReleasesPorts() {
+ stubPerNodeContainers();
+ service.createReplicationGroup(clusterRequest("grp", 2, 0));
+
+ service.deleteReplicationGroup("grp");
+
+ verify(proxyManager).stopProxy("grp-0001-001");
+ verify(proxyManager).stopProxy("grp-0002-001");
+ verify(containerManager, times(2)).stop(any());
+
+ ReplicationGroup recovered =
+ service.createReplicationGroup("grp2", "test", AuthMode.PASSWORD, null, "us-east-1");
+ assertEquals(16379, recovered.getProxyPort(),
+ "Ports from the deleted cluster group must be released for the next group");
+ }
+
+ @Test
+ void clusterNodesAnnounceTheConfiguredHostnameAsPreferredEndpoint() {
+ stubPerNodeContainers();
+
+ service.createReplicationGroup(clusterRequest("grp", 1, 0));
+
+ @SuppressWarnings("unchecked")
+ ArgumentCaptor<List<String>> flags = ArgumentCaptor.forClass(List.class);
+ verify(containerManager).start(eq("grp-0001-001"), anyString(), flags.capture());
+ List<String> captured = flags.getValue();
+ assertEquals("localhost", flagValue(captured, "--cluster-announce-hostname"));
+ assertEquals("hostname", flagValue(captured, "--cluster-preferred-endpoint-type"));
+ assertEquals("16379", flagValue(captured, "--cluster-announce-port"));
+ }
+
+ @Test
+ void clusterAnnounceHostnameOverrideIsAnnouncedAndReported() {
+ when(config.services().elasticache().clusterAnnounceHostname())
+ .thenReturn(java.util.Optional.of("localhost.floci.io"));
+ stubPerNodeContainers();
+
+ ReplicationGroup group = service.createReplicationGroup(clusterRequest("grp", 1, 0));
+
+ @SuppressWarnings("unchecked")
+ ArgumentCaptor<List<String>> flags = ArgumentCaptor.forClass(List.class);
+ verify(containerManager).start(eq("grp-0001-001"), anyString(), flags.capture());
+ assertEquals("localhost.floci.io", flagValue(flags.getValue(), "--cluster-announce-hostname"));
+ assertEquals("localhost.floci.io", group.getConfigurationEndpoint().address());
+ }
+
+ private static String flagValue(List<String> flags, String flag) {
+ int index = flags.indexOf(flag);
+ assertTrue(index >= 0 && index + 1 < flags.size(), "Missing flag " + flag + " in " + flags);
+ return flags.get(index + 1);
+ }
+
+ private static StorageFactory sharedStorageFactory() {
+ StorageFactory storageFactory = mock(StorageFactory.class);
+ Map<String, Object> backends = new ConcurrentHashMap<>();
+ when(storageFactory.create(anyString(), anyString(), any())).thenAnswer(inv ->
+ backends.computeIfAbsent(inv.getArgument(1, String.class),
+ key -> AccountAwareStorageBackend.inMemory("000000000000")));
+ return storageFactory;
+ }
+
+ private static ElastiCacheService serviceWith(StorageFactory storageFactory,
+ ElastiCacheContainerManager containerManager,
+ ElastiCacheProxyManager proxyManager,
+ ValkeyClusterFormation clusterFormation) {
+ EmulatorConfig config = mock(EmulatorConfig.class);
+ EmulatorConfig.ServicesConfig servicesConfig = mock(EmulatorConfig.ServicesConfig.class);
+ EmulatorConfig.ElastiCacheServiceConfig ecConfig = mock(EmulatorConfig.ElastiCacheServiceConfig.class);
+ when(config.services()).thenReturn(servicesConfig);
+ when(servicesConfig.elasticache()).thenReturn(ecConfig);
+ when(ecConfig.proxyBasePort()).thenReturn(16379);
+ when(ecConfig.proxyMaxPort()).thenReturn(16399);
+ when(ecConfig.defaultImage()).thenReturn("valkey/valkey:8");
+ when(config.hostname()).thenReturn(java.util.Optional.of("localhost"));
+ return new ElastiCacheService(containerManager, proxyManager, clusterFormation,
+ storageFactory, config, mock(Ec2Service.class),
+ new RegionResolver("us-east-1", "000000000000"), mock(KmsService.class));
+ }
+
+ private static void stubPerNodeContainers(ElastiCacheContainerManager containerManager) {
+ when(containerManager.start(anyString(), anyString(), any())).thenAnswer(inv ->
+ new ElastiCacheContainerHandle("cid-" + inv.getArgument(0, String.class),
+ inv.getArgument(0, String.class), "localhost", 6379));
+ when(containerManager.start(anyString(), anyString()))
+ .thenReturn(new ElastiCacheContainerHandle("cid", "grp", "localhost", 6379));
+ }
+
+ @Test
+ void restorePersistedRuntimeReprovisionsClusterModeGroups() {
+ StorageFactory storageFactory = sharedStorageFactory();
+ ElastiCacheContainerManager beforeRestart = mock(ElastiCacheContainerManager.class);
+ stubPerNodeContainers(beforeRestart);
+ serviceWith(storageFactory, beforeRestart, mock(ElastiCacheProxyManager.class),
+ mock(ValkeyClusterFormation.class))
+ .createReplicationGroup(clusterRequest("grp", 2, 1));
+
+ ElastiCacheContainerManager restartedContainers = mock(ElastiCacheContainerManager.class);
+ stubPerNodeContainers(restartedContainers);
+ ElastiCacheProxyManager restartedProxies = mock(ElastiCacheProxyManager.class);
+ ValkeyClusterFormation restartedFormation = mock(ValkeyClusterFormation.class);
+ ElastiCacheService restarted = serviceWith(storageFactory, restartedContainers,
+ restartedProxies, restartedFormation);
+
+ restarted.restorePersistedRuntime().join();
+
+ verify(restartedContainers, times(4)).start(anyString(), anyString(), any());
+ verify(restartedFormation).form(eq("grp"), any(), eq(2));
+ verify(restartedProxies, times(4)).startProxy(anyString(), any(), anyInt(), anyString(), anyInt(), any());
+ ReplicationGroup restored = restarted.getReplicationGroup("grp");
+ assertEquals(ReplicationGroupStatus.AVAILABLE, restored.getStatus());
+ assertEquals(16379, restored.getConfigurationEndpoint().port());
+ assertEquals("cid-grp-0001-001", restored.getClusterNodes().getFirst().getContainerId());
+
+ ReplicationGroup next =
+ restarted.createReplicationGroup("grp2", "test", AuthMode.NO_AUTH, null, "us-east-1");
+ assertEquals(16383, next.getProxyPort(),
+ "Restored node ports must be reserved again so new groups cannot take them");
+ }
+
+ @Test
+ void restoreFailureReportsCreateFailedAndReleasesPorts() {
+ StorageFactory storageFactory = sharedStorageFactory();
+ ElastiCacheContainerManager beforeRestart = mock(ElastiCacheContainerManager.class);
+ stubPerNodeContainers(beforeRestart);
+ serviceWith(storageFactory, beforeRestart, mock(ElastiCacheProxyManager.class),
+ mock(ValkeyClusterFormation.class))
+ .createReplicationGroup(clusterRequest("grp", 2, 0));
+
+ ElastiCacheContainerManager restartedContainers = mock(ElastiCacheContainerManager.class);
+ when(restartedContainers.start(anyString(), anyString(), any()))
+ .thenThrow(new RuntimeException("docker down"));
+ when(restartedContainers.start(anyString(), anyString()))
+ .thenReturn(new ElastiCacheContainerHandle("cid", "grp", "localhost", 6379));
+ ElastiCacheProxyManager restartedProxies = mock(ElastiCacheProxyManager.class);
+ ElastiCacheService restarted = serviceWith(storageFactory, restartedContainers,
+ restartedProxies, mock(ValkeyClusterFormation.class));
+
+ restarted.restorePersistedRuntime().join();
+
+ ReplicationGroup failed = restarted.getReplicationGroup("grp");
+ assertEquals(ReplicationGroupStatus.CREATE_FAILED, failed.getStatus());
+ assertNull(failed.getConfigurationEndpoint(),
+ "A group whose data plane is gone must not advertise an endpoint");
+ verify(restartedProxies, never()).startProxy(anyString(), any(), anyInt(), anyString(), anyInt(), any());
+
+ ReplicationGroup next =
+ restarted.createReplicationGroup("grp2", "test", AuthMode.NO_AUTH, null, "us-east-1");
+ assertEquals(16379, next.getProxyPort(),
+ "Ports from the failed restore must be released for the next group");
+ }
+
+ @Test
+ void restoreRunsInBackgroundAndReportsCreatingUntilDone() throws InterruptedException {
+ StorageFactory storageFactory = sharedStorageFactory();
+ ElastiCacheContainerManager beforeRestart = mock(ElastiCacheContainerManager.class);
+ stubPerNodeContainers(beforeRestart);
+ serviceWith(storageFactory, beforeRestart, mock(ElastiCacheProxyManager.class),
+ mock(ValkeyClusterFormation.class))
+ .createReplicationGroup(clusterRequest("grp", 2, 0));
+
+ ElastiCacheContainerManager restartedContainers = mock(ElastiCacheContainerManager.class);
+ CountDownLatch restoreStarted = new CountDownLatch(1);
+ CountDownLatch releaseRestore = new CountDownLatch(1);
+ when(restartedContainers.start(anyString(), anyString(), any())).thenAnswer(inv -> {
+ restoreStarted.countDown();
+ releaseRestore.await(5, TimeUnit.SECONDS);
+ return new ElastiCacheContainerHandle("cid-" + inv.getArgument(0, String.class),
+ inv.getArgument(0, String.class), "localhost", 6379);
+ });
+ ElastiCacheService restarted = serviceWith(storageFactory, restartedContainers,
+ mock(ElastiCacheProxyManager.class), mock(ValkeyClusterFormation.class));
+
+ CompletableFuture<Void> restore = restarted.restorePersistedRuntime();
+
+ assertTrue(restoreStarted.await(5, TimeUnit.SECONDS));
+ assertFalse(restore.isDone(), "Restore must not block the caller while the data plane comes up");
+ assertEquals(ReplicationGroupStatus.CREATING, restarted.getReplicationGroup("grp").getStatus(),
+ "Groups must report creating while their restore is in flight");
+ releaseRestore.countDown();
+ restore.join();
+ assertEquals(ReplicationGroupStatus.AVAILABLE, restarted.getReplicationGroup("grp").getStatus());
+ }
+
@Test
void concurrentCreateForSameGroupIdIsRejectedWhileFirstIsProvisioning() throws InterruptedException {
CountDownLatch startedLatch = new CountDownLatch(1);
@@ -356,8 +645,9 @@ class ElastiCacheServiceTest {
.thenAnswer(inv -> new AccountAwareStorageBackend<>(pausing, null, "000000000000"));
when(factory.create(anyString(), org.mockito.ArgumentMatchers.argThat(f -> !"elasticache-groups.json".equals(f)), any()))
.thenAnswer(inv -> AccountAwareStorageBackend.inMemory("000000000000"));
- ElastiCacheService svc = new ElastiCacheService(containerManager, proxyManager, factory, config,
- org.mockito.Mockito.mock(Ec2Service.class), new RegionResolver("us-east-1", "000000000000"), kmsService);
+ ElastiCacheService svc = new ElastiCacheService(containerManager, proxyManager, clusterFormation,
+ factory, config, org.mockito.Mockito.mock(Ec2Service.class),
+ new RegionResolver("us-east-1", "000000000000"), kmsService);
svc.createReplicationGroup("g1", "d", AuthMode.NO_AUTH, null, "us-east-1");
pausing.pauseOn(PausingStorageBackend.Call.PUT, "g1");