Light, fluffy, and always free - The AWS Local Emulator alternative
fix(elasticache): create single-node redis clusters and close provisioning races (#3634)
Allen Sanborn684239b0ef71
diff --git a/compatibility-tests/compat-terraform/test/elasticache-single-node-tf/main.tf b/compatibility-tests/compat-terraform/test/elasticache-single-node-tf/main.tfnew file mode 100644index 0000000000..fda97a5de4--- /dev/null+++ b/compatibility-tests/compat-terraform/test/elasticache-single-node-tf/main.tf@@ -0,0 +1,33 @@+resource "aws_elasticache_cluster" "single_node" {+ cluster_id = "floci-tf-redis-single"+ engine = "redis"+ engine_version = "7.1"+ node_type = "cache.t4g.micro"+ num_cache_nodes = 1+ port = 6396+ parameter_group_name = "default.redis7"++ snapshot_retention_limit = 5+ snapshot_window = "03:00-05:00"+ maintenance_window = "tue:04:00-tue:05:00"+}++output "cluster_id" {+ value = aws_elasticache_cluster.single_node.cluster_id+}++output "engine" {+ value = aws_elasticache_cluster.single_node.engine+}++output "port" {+ value = aws_elasticache_cluster.single_node.port+}++output "cache_node_address" {+ value = aws_elasticache_cluster.single_node.cache_nodes[0].address+}++output "cache_node_port" {+ value = aws_elasticache_cluster.single_node.cache_nodes[0].port+}diff --git a/compatibility-tests/compat-terraform/test/elasticache-single-node-tf/provider.tf b/compatibility-tests/compat-terraform/test/elasticache-single-node-tf/provider.tfnew file mode 100644index 0000000000..b32d832003--- /dev/null+++ b/compatibility-tests/compat-terraform/test/elasticache-single-node-tf/provider.tf@@ -0,0 +1,28 @@+terraform {+ required_providers {+ aws = {+ source = "hashicorp/aws"+ version = "~> 6.0"+ }+ }+}++variable "endpoint" {+ type = string+ default = "http://localhost:4566"+}++provider "aws" {+ region = "us-east-1"+ access_key = "test"+ secret_key = "test"++ skip_credentials_validation = true+ skip_metadata_api_check = true+ skip_requesting_account_id = true+ s3_use_path_style = true++ endpoints {+ elasticache = var.endpoint+ }+}diff --git a/compatibility-tests/compat-terraform/test/elasticache_single_node.bats b/compatibility-tests/compat-terraform/test/elasticache_single_node.batsnew file mode 100644index 0000000000..d21fd642b1--- /dev/null+++ b/compatibility-tests/compat-terraform/test/elasticache_single_node.bats@@ -0,0 +1,130 @@+#!/usr/bin/env bats+# ElastiCache single-node Redis Compatibility Test+#+# Applies an aws_elasticache_cluster with engine = "redis", the shape terraform+# sends to CreateCacheCluster rather than CreateReplicationGroup, and verifies+# both sides of the contract: the read-back is honest enough that a second plan+# wants nothing (the optional snapshot and maintenance arguments included), and+# the node endpoint terraform hands out is a real Redis/Valkey answering RESP.++setup_file() {+ load 'test_helper/common-setup'++ EC_TF_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/elasticache-single-node-tf" && pwd)"+ cd "$EC_TF_DIR"++ echo "# === ElastiCache Single-Node Redis Test ===" >&3+ echo "# Endpoint: $FLOCI_ENDPOINT" >&3+ echo "# Config: $EC_TF_DIR" >&3++ rm -rf .terraform .terraform.lock.hcl terraform.tfstate* 2>/dev/null || true++ echo "# --- terraform init ---" >&3+ run terraform init -input=false -no-color+ if [ "$status" -ne 0 ]; then+ echo "# terraform init failed: $output" >&3+ return 1+ fi++ echo "# --- terraform apply (single-node redis) ---" >&3+ run terraform apply -var="endpoint=${FLOCI_ENDPOINT}" -input=false -auto-approve -no-color+ if [ "$status" -ne 0 ]; then+ echo "# terraform apply failed: $output" >&3+ return 1+ fi+}++teardown_file() {+ load 'test_helper/common-setup'++ EC_TF_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/elasticache-single-node-tf" && pwd)"+ cd "$EC_TF_DIR"++ terraform destroy -var="endpoint=${FLOCI_ENDPOINT}" -input=false -auto-approve -no-color || true+ rm -rf .terraform .terraform.lock.hcl terraform.tfstate* 2>/dev/null || true+}++setup() {+ load 'test_helper/common-setup'+ EC_TF_DIR="$(cd "$(dirname "$BATS_TEST_FILENAME")/elasticache-single-node-tf" && pwd)"+}++# Sends one RESP command over /dev/tcp and echoes whatever the server replies+# within the read window. No redis-cli in this image, so raw RESP it is.+valkey_command() {+ local host="$1" port="$2"+ shift 2+ local payload="*$#\r\n"+ local arg+ for arg in "$@"; do+ payload+="\$${#arg}\r\n${arg}\r\n"+ done+ exec 9<>"/dev/tcp/${host}/${port}" || return 1+ printf '%b' "$payload" >&9+ local reply="" chunk+ while IFS= read -r -t 3 -u 9 chunk; do+ reply+="${chunk}"$'\n'+ done+ exec 9<&- 9>&-+ printf '%s' "$reply"+}++@test "ElastiCache single node: terraform reads back the engine and port" {+ run terraform -chdir="$EC_TF_DIR" output -raw engine+ assert_success+ assert_output "redis"++ run terraform -chdir="$EC_TF_DIR" output -raw port+ assert_success+ assert_output "6396"+}++@test "ElastiCache single node: describe reports one node, no replication group" {+ run aws_cmd elasticache describe-cache-clusters \+ --cache-cluster-id floci-tf-redis-single --show-cache-node-info+ assert_success+ [ "$(json_get "$output" '.CacheClusters[0].Engine')" = "redis" ]+ [ "$(json_get "$output" '.CacheClusters[0].NumCacheNodes')" = "1" ]+ [ "$(json_get "$output" '.CacheClusters[0].CacheNodes | length')" = "1" ]+ [ "$(json_get "$output" '.CacheClusters[0].ReplicationGroupId')" = "null" ]++ # a cluster created this way is not a replication group and must not answer as one+ run aws_cmd elasticache describe-replication-groups \+ --replication-group-id floci-tf-redis-single+ assert_failure+}++@test "ElastiCache single node: the optional arguments read back as sent" {+ run aws_cmd elasticache describe-cache-clusters --cache-cluster-id floci-tf-redis-single+ assert_success+ [ "$(json_get "$output" '.CacheClusters[0].SnapshotRetentionLimit')" = "5" ]+ [ "$(json_get "$output" '.CacheClusters[0].SnapshotWindow')" = "03:00-05:00" ]+ [ "$(json_get "$output" '.CacheClusters[0].PreferredMaintenanceWindow')" = "tue:04:00-tue:05:00" ]+}++# The critical assertion: the endpoint terraform hands out fronts a real cache.+@test "ElastiCache single node: the node endpoint answers RESP" {+ host=$(terraform -chdir="$EC_TF_DIR" output -raw cache_node_address)+ port=$(terraform -chdir="$EC_TF_DIR" output -raw cache_node_port)++ run valkey_command "$host" "$port" PING+ assert_success+ assert_output --partial "PONG"++ run valkey_command "$host" "$port" SET tf-single-node-key a-value+ assert_success+ assert_output --partial "OK"++ run valkey_command "$host" "$port" GET tf-single-node-key+ assert_success+ assert_output --partial "a-value"+}++# Guards against read-back drift (the class of bug in floci-io/floci#2481):+# a second plan straight after apply must not want to change anything.+@test "ElastiCache single node: re-plan after apply is clean" {+ cd "$EC_TF_DIR"+ run terraform plan -var="endpoint=${FLOCI_ENDPOINT}" -input=false -no-color -detailed-exitcode+ assert_success+ assert_output --partial "No changes"+}diff --git a/docs/services/elasticache.md b/docs/services/elasticache.mdindex eb14c34d8e..2b5d85f2c5 100644--- a/docs/services/elasticache.md+++ b/docs/services/elasticache.md@@ -20,9 +20,9 @@ Floci manages real Valkey/Redis Docker containers and proxies TCP connections to| `DescribeUsers` | List ElastiCache users || `ModifyUser` | Update user access strings || `DeleteUser` | Remove an ElastiCache user |-| `CreateCacheCluster` | - |-| `DescribeCacheClusters` | - |-| `DeleteCacheCluster` | - |+| `CreateCacheCluster` | Start a Memcached cluster, or a single-node Redis/Valkey one (`NumCacheNodes` must be 1) |+| `DescribeCacheClusters` | List cache clusters: Memcached, single-node Redis/Valkey, and replication group members |+| `DeleteCacheCluster` | Stop and remove a cache cluster || `CreateCacheSubnetGroup` | Create a cache subnet group || `DescribeCacheSubnetGroups` | List cache subnet groups || `ModifyCacheSubnetGroup` | Replace a group's description or subnets |@@ -35,6 +35,45 @@ Floci manages real Valkey/Redis Docker containers and proxies TCP connections to| `ListTagsForResource` | Tags on a parameter group ARN |<!-- floci:actions:end -->+### Single-node Redis/Valkey clusters++`CreateCacheCluster` serves three engines. `Engine=memcached` starts a Memcached container and+reports its node-discovery `ConfigurationEndpoint`. `Engine=redis` or `Engine=valkey` starts a+single-node cluster with no replication group: the same Valkey container and auth proxy a+cluster-mode-disabled replication group gets, on a port from the same proxy range, which defaults+to 6379. AWS allows only one node in that shape, so `NumCacheNodes` greater than 1 is refused with+`InvalidParameterValue`, as it is on a live account.++This is the call `aws_elasticache_cluster` with `engine = "redis"` emits, which is why it is not+interchangeable with `CreateReplicationGroup`: a single-node cluster created this way never appears+in `DescribeReplicationGroups`. Following AWS, it carries no `ConfigurationEndpoint` either, and+reports its node's address under `CacheNodes` when the request sets `ShowCacheNodeInfo`. These+clusters are re-provisioned after a Floci restart, the same as replication groups and Memcached+clusters.++An id is taken across all three at once: a standalone cache cluster, a Memcached cluster and a+replication group cannot share one. Two of them would have `DescribeCacheClusters` report the same+id twice, and for a cache cluster against a replication group it is worse, since Floci names both+their containers `valkey-<id>` and keys both their proxies by it. Whichever create arrives second+is refused, whether it is `CreateCacheCluster` or `CreateReplicationGroup`, rather than allowed to+remove the first's container.++`CacheSubnetGroupName` must name a subnet group that exists, as on AWS, on the `Engine=redis` and+`Engine=valkey` paths. `Engine=memcached` drops the parameter: it is neither checked nor echoed+back. (`CreateReplicationGroup` does not check it either, so a replication group can still be+created against a name nothing resolves.)++Each record re-reserves its proxy port as it is restored, before Floci serves anything, so a+create is never handed a port a surviving cluster or replication group still advertises.++`SnapshotRetentionLimit`, `SnapshotWindow`, `PreferredMaintenanceWindow`,+`PreferredAvailabilityZone`, `SecurityGroupIds`, `NetworkType`, `IpDiscovery` and+`AtRestEncryptionEnabled` (output-only on real AWS's CreateCacheCluster; accepted here leniently and echoed) are kept and echoed by `DescribeCacheClusters`, with the same defaults+and the same validation the replication group applies, since every one is an optional+`aws_elasticache_cluster` argument that would otherwise read back unset and leave a permanent+plan diff. `NotificationConfiguration` and `LogDeliveryConfigurations` are **not** modelled: a+request may send them, and the describe will not report them back.+### Cluster Mode`CreateReplicationGroup` provisions a real sharded Valkey cluster when the request asks for one:@@ -60,8 +99,8 @@ case — the shipped `docker-compose.yml` uses `localhost.floci.io`, which publiFloci's embedded DNS resolve it to the Floci container from inside Docker. Cluster-mode groupsthen announce that name and report it as their `ConfigurationEndpoint`.-With `persistent`, `hybrid` or `wal` storage, every replication group and Memcached cluster is-re-provisioned from its persisted record on startup: containers are restarted, cluster-mode groups+With `persistent`, `hybrid` or `wal` storage, every replication group, standalone cache cluster+and Memcached cluster is re-provisioned from its persisted record on startup: containers are restarted, cluster-mode groupsare re-formed, and proxy ports are re-reserved. Caches restart empty, as on any Floci restart:only the topology is persisted, never the keyspace.@@ -70,8 +109,8 @@ restarts and cluster formation run in the background so a slow Docker daemon canreadiness, and each record flips to `available` once its data plane is back. A replication groupwhose data plane cannot be brought back is reported with status `create-failed` instead of`available`, and its member clusters answer `DescribeCacheClusters` with `restore-failed`-(`CacheClusterStatus` has no `create-failed` value). A Memcached cluster that cannot be brought-back reports `restore-failed` directly.+(`CacheClusterStatus` has no `create-failed` value). A standalone cache cluster or Memcached+cluster that cannot be brought back reports `restore-failed` directly.Reporting a failed record as failed matters more than it looks. A record that is left unreconciledkeeps its `available` status with nothing behind the endpoint, so the control plane answers healthy@@ -80,8 +119,8 @@ take: the old endpoint then reaches an unrelated cache rather than failing cleanA Memcached cluster's endpoint follows its new container. Outside Docker, Floci publishes thebackend on a host port Docker picks per run, so a restored cluster's `ConfigurationEndpoint` port-can differ from the one it had before the restart. Replication groups keep their port: it is a-proxy port Floci owns and re-reserves.+can differ from the one it had before the restart. Replication groups and standalone cache+clusters keep their port: it is a proxy port Floci owns and re-reserves.A delete that arrives while a record is still `creating` wins. The restore takes the sameper-record monitor `DeleteReplicationGroup` and `DeleteCacheCluster` take, and skips its write-backdiff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedService.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedService.javaindex 5b415827a1..a05bc69790 100644--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedService.java+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedService.java@@ -10,6 +10,7 @@ import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheMemcimport io.github.hectorvent.floci.services.elasticache.model.CacheCluster;import io.github.hectorvent.floci.services.elasticache.model.CacheClusterStatus;import io.github.hectorvent.floci.services.elasticache.model.Endpoint;+import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroup;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;@@ -32,23 +33,60 @@ public class ElastiCacheMemcachedService {private static final int BACKEND_PORT = 11211;private final StorageBackend<String, CacheCluster> clusters;+ /**+ * The two stores {@link ElastiCacheService} writes, read here for the id check alone.+ * {@link StorageFactory#create} keys backends by file path and hands back the instance that+ * service already holds, so these are those stores rather than copies: a cache cluster id is+ * one namespace whatever engine claims it.+ */+ private final StorageBackend<String, CacheCluster> redisClusters;+ private final StorageBackend<String, ReplicationGroup> groups;private final ElastiCacheMemcachedContainerManager containerManager;private final EmulatorConfig config;+ /** Shared with {@link ElastiCacheService}: one namespace, one set of in-flight claims. */+ private final ElastiCacheProvisioningIds provisioningIds;private final ConcurrentHashMap<String, Object> clusterLocks = new ConcurrentHashMap<>();@Injectpublic ElastiCacheMemcachedService(ElastiCacheMemcachedContainerManager containerManager,StorageFactory storageFactory,- EmulatorConfig config) {+ EmulatorConfig config,+ ElastiCacheProvisioningIds provisioningIds) {this.containerManager = containerManager;this.config = config;+ this.provisioningIds = provisioningIds;this.clusters = storageFactory.create("elasticache", "elasticache-cache-clusters.json",new TypeReference<Map<String, CacheCluster>>() {});+ this.redisClusters = storageFactory.create("elasticache", "elasticache-redis-clusters.json",+ new TypeReference<Map<String, CacheCluster>>() {});+ this.groups = storageFactory.create("elasticache", "elasticache-groups.json",+ new TypeReference<Map<String, ReplicationGroup>>() {});}public CacheCluster createCacheCluster(String clusterId) {- if (clusters.get(clusterId).isPresent()) {- throw new AwsException("CacheClusterAlreadyExistsFault",+ // Claimed before the store checks rather than after, because no create here or in+ // ElastiCacheService persists its record until its container has started: a store check+ // that passes is no promise the id is still free by the time this one writes. The claim+ // is the same set the redis paths take, so of two concurrent creates for one id only the+ // one that claims it reaches the stores at all.+ if (!provisioningIds.claim(clusterId)) {+ throw new AwsException("CacheClusterAlreadyExists",+ "Cache cluster " + clusterId + " is already being created.", 400);+ }+ try {+ return provisionCacheCluster(clusterId);+ } finally {+ provisioningIds.release(clusterId);+ }+ }++ private CacheCluster provisionCacheCluster(String clusterId) {+ // Every store that answers DescribeCacheClusters, not just this one: two records sharing+ // an id would have one describe report it twice, each with a different engine.+ if (clusters.get(clusterId).isPresent()+ || redisClusters.get(clusterId).isPresent()+ || groups.get(clusterId).isPresent()) {+ throw new AwsException("CacheClusterAlreadyExists","Cache cluster " + clusterId + " already exists.", 400);}diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheProvisioningIds.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheProvisioningIds.javanew file mode 100644index 0000000000..656aaefe4d--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheProvisioningIds.java@@ -0,0 +1,36 @@+package io.github.hectorvent.floci.services.elasticache;++import jakarta.enterprise.context.ApplicationScoped;++import java.util.Set;+import java.util.concurrent.ConcurrentHashMap;++/**+ * Ids claimed by in-flight creates: replication groups, standalone cache clusters and Memcached+ * clusters alike. One set, because the three share a namespace in floci whatever AWS does with+ * them, and because none of the create paths persists its record until after container startup.+ * Without a claim held across that window two creates of the same id both read three empty stores+ * and both go on to write, leaving one id in two stores: the redis paths would remove each other's+ * container and orphan a proxy listener, and a describe would report the id twice.+ *+ * <p>A separate bean rather than a field on one service, so both {@link ElastiCacheService} and+ * {@link ElastiCacheMemcachedService} claim against the same instance.+ */+@ApplicationScoped+public class ElastiCacheProvisioningIds {++ private final Set<String> claimed = ConcurrentHashMap.newKeySet();++ /**+ * Claims the id for the calling create. Returns false when another in-flight create already+ * holds it, in which case the caller must not release it.+ */+ public boolean claim(String id) {+ return claimed.add(id);+ }++ /** Releases a claim this caller took. Safe to call on any exit path. */+ public void release(String id) {+ claimed.remove(id);+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.javaindex be6c5fab58..fd44e279d1 100644--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.java@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.elasticache;import io.github.hectorvent.floci.core.common.AwsArnUtils;import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.BackupWindows;import io.github.hectorvent.floci.core.common.AwsNamespaces;import io.github.hectorvent.floci.core.common.AwsQueryResponse;import io.github.hectorvent.floci.core.common.RegionResolver;@@ -69,7 +70,7 @@ public class ElastiCacheQueryHandler {case "DescribeUsers" -> handleDescribeUsers(params);case "ModifyUser" -> handleModifyUser(params);case "DeleteUser" -> handleDeleteUser(params);- case "CreateCacheCluster" -> handleCreateCacheCluster(params);+ case "CreateCacheCluster" -> handleCreateCacheCluster(params, region);case "DescribeCacheClusters" -> handleDescribeCacheClusters(params);case "DeleteCacheCluster" -> handleDeleteCacheCluster(params);case "CreateCacheSubnetGroup" -> handleCreateCacheSubnetGroup(params);@@ -315,9 +316,9 @@ public class ElastiCacheQueryHandler {}}- // ── Cache Clusters (Memcached) ────────────────────────────────────────────+ // ── Cache Clusters (Memcached, and single-node Redis/Valkey) ──────────────- private Response handleCreateCacheCluster(MultivaluedMap<String, String> params) {+ private Response handleCreateCacheCluster(MultivaluedMap<String, String> params, String region) {String clusterId = params.getFirst("CacheClusterId");String engine = params.getFirst("Engine");@@ -325,33 +326,70 @@ public class ElastiCacheQueryHandler {return AwsQueryResponse.error("InvalidParameterValue","CacheClusterId is required.", AwsNamespaces.EC, 400);}- if (!"memcached".equalsIgnoreCase(engine)) {+ if (!"memcached".equalsIgnoreCase(engine) && !"redis".equalsIgnoreCase(engine)+ && !"valkey".equalsIgnoreCase(engine)) {return AwsQueryResponse.error("InvalidParameterValue",- "Engine must be 'memcached'. For Redis/Valkey use CreateReplicationGroup.", AwsNamespaces.EC, 400);+ "Engine must be 'memcached', 'redis' or 'valkey'.", AwsNamespaces.EC, 400);}try {- CacheCluster cluster = memcachedService.createCacheCluster(clusterId);- return Response.ok(AwsQueryResponse.envelope("CreateCacheCluster", AwsNamespaces.EC, cacheClusterXml(cluster))).build();+ CacheCluster cluster = "memcached".equalsIgnoreCase(engine)+ ? memcachedService.createCacheCluster(clusterId)+ : service.createCacheCluster(new ElastiCacheService.CreateCacheClusterRequest(+ clusterId,+ engine,+ params.getFirst("EngineVersion"),+ params.getFirst("CacheNodeType"),+ intParam(params, "NumCacheNodes"),+ intParam(params, "Port"),+ cacheClusterAuthMode(params),+ params.getFirst("AuthToken"),+ params.getFirst("CacheParameterGroupName"),+ params.getFirst("CacheSubnetGroupName"),+ optionalInt(params.getFirst("SnapshotRetentionLimit")),+ params.getFirst("SnapshotWindow"),+ params.getFirst("PreferredMaintenanceWindow"),+ params.getFirst("PreferredAvailabilityZone"),+ parseSecurityGroupIds(params),+ params.getFirst("NetworkType"),+ params.getFirst("IpDiscovery"),+ boolParam(params, "AtRestEncryptionEnabled"),+ region,+ parseTags(params)));+ return Response.ok(AwsQueryResponse.envelope("CreateCacheCluster", AwsNamespaces.EC,+ cacheClusterXml(cluster, false))).build();} catch (AwsException e) {return AwsQueryResponse.error(e.getErrorCode(), e.getMessage(), AwsNamespaces.EC, e.getHttpStatus());}}+ /** Read exactly as CreateReplicationGroup reads it, so both actions agree about one request. */+ private static AuthMode cacheClusterAuthMode(MultivaluedMap<String, String> params) {+ String authToken = params.getFirst("AuthToken");+ if (authToken != null && !authToken.isBlank()) {+ return AuthMode.PASSWORD;+ }+ if ("true".equalsIgnoreCase(params.getFirst("TransitEncryptionEnabled"))) {+ return AuthMode.IAM;+ }+ return AuthMode.NO_AUTH;+ }+private Response handleDescribeCacheClusters(MultivaluedMap<String, String> params) {String filterId = params.getFirst("CacheClusterId");boolean showNodeInfo = "true".equalsIgnoreCase(params.getFirst("ShowCacheNodeInfo"));+ boolean filtered = filterId != null && !filterId.isBlank();try {List<ElastiCacheService.MemberCacheCluster> members = service.listMemberCacheClusters(filterId);- Collection<CacheCluster> clusterList;- if (filterId != null && !filterId.isBlank() && !members.isEmpty()) {- clusterList = List.of();- } else {- clusterList = memcachedService.listCacheClusters(filterId);+ List<CacheCluster> clusterList = new ArrayList<>(service.findCacheClusters(filterId));+ // The Memcached lookup is the one that raises CacheClusterNotFound for an unknown id,+ // which is the right answer only once no other source has answered.+ if (!filtered || (members.isEmpty() && clusterList.isEmpty())) {+ clusterList.addAll(memcachedService.listCacheClusters(filterId));}XmlBuilder xml = new XmlBuilder().start("CacheClusters");for (CacheCluster c : clusterList) {- xml.raw(cacheClusterXml(c));+ xml.raw(cacheClusterXml(c, showNodeInfo));}for (ElastiCacheService.MemberCacheCluster member : members) {xml.raw(memberCacheClusterXml(member, showNodeInfo));@@ -427,8 +465,13 @@ public class ElastiCacheQueryHandler {"CacheClusterId is required.", AwsNamespaces.EC, 400);}try {- CacheCluster cluster = memcachedService.deleteCacheCluster(clusterId);- return Response.ok(AwsQueryResponse.envelope("DeleteCacheCluster", AwsNamespaces.EC, cacheClusterXml(cluster))).build();+ // Standalone redis/valkey clusters are stored apart from Memcached ones, so the id is+ // looked for there first and the Memcached delete keeps raising the not-found fault.+ CacheCluster cluster = service.findCacheClusters(clusterId).isEmpty()+ ? memcachedService.deleteCacheCluster(clusterId)+ : service.deleteCacheCluster(clusterId);+ return Response.ok(AwsQueryResponse.envelope("DeleteCacheCluster", AwsNamespaces.EC,+ cacheClusterXml(cluster, false))).build();} catch (AwsException e) {return AwsQueryResponse.error(e.getErrorCode(), e.getMessage(), AwsNamespaces.EC, e.getHttpStatus());}@@ -521,6 +564,21 @@ public class ElastiCacheQueryHandler {return subnetIds;}+ /** Reads the SecurityGroupIds list under every spelling the Query protocol sends it in. */+ private static List<String> parseSecurityGroupIds(MultivaluedMap<String, String> params) {+ List<String> securityGroupIds = new ArrayList<>();+ for (String prefix : List.of("SecurityGroupIds.SecurityGroupId", "SecurityGroupIds.member")) {+ for (int i = 1; ; i++) {+ String securityGroupId = params.getFirst(prefix + "." + i);+ if (securityGroupId == null) {+ break;+ }+ securityGroupIds.add(securityGroupId);+ }+ }+ return securityGroupIds;+ }+private Response handleCreateCacheParameterGroup(MultivaluedMap<String, String> params) {try {CacheParameterGroup group = service.createCacheParameterGroup(@@ -626,6 +684,22 @@ private Response handleCreateCacheParameterGroup(MultivaluedMap<String, String>}tags = group.getTags();}+ if ("cluster".equals(arn[5])) {+ CacheCluster cluster = service.findCacheClusters(arn[6]).stream().findFirst().orElse(null);+ if (cluster != null) {+ // Same shape as the replicationgroup arm: the store keys clusters by id+ // alone, so the record found must also be the one this ARN names.+ if (cluster.getArn() != null && !cluster.getArn().equalsIgnoreCase(resourceName)) {+ throw new AwsException("CacheClusterNotFound",+ "Cache cluster " + arn[6] + " not found.", 404);+ }+ tags = cluster.getTags();+ } else if (service.listMemberCacheClusters(arn[6]).isEmpty()) {+ // Memcached clusters and replication group members carry no tags, but they do+ // exist. Only an id no source knows is a not-found.+ memcachedService.getCacheCluster(arn[6]);+ }+ }if ("subnetgroup".equals(arn[5])) {tags = service.describeCacheSubnetGroups(arn[6]).getFirst().getTags();}@@ -732,7 +806,15 @@ private Response handleCreateCacheParameterGroup(MultivaluedMap<String, String>// ── XML helpers ───────────────────────────────────────────────────────────- private String cacheClusterXml(CacheCluster c) {+ /**+ * A cache cluster as its engine has AWS report it. Memcached carries a ConfigurationEndpoint,+ * the address of its node-discovery endpoint; a single-node redis or valkey cluster has none,+ * and its endpoint is the node's, reported under CacheNodes when the request asks for node+ * info. terraform-provider-aws reads {@code port} from whichever of the two is present, so a+ * redis cluster answering with a ConfigurationEndpoint instead would report itself as the+ * wrong thing.+ */+ private String cacheClusterXml(CacheCluster c, boolean showNodeInfo) {Endpoint ep = c.getConfigurationEndpoint();XmlBuilder xml = new XmlBuilder().start("CacheCluster")@@ -740,11 +822,87 @@ private Response handleCreateCacheParameterGroup(MultivaluedMap<String, String>.elem("CacheClusterStatus", c.getCacheClusterStatus().wireName()).elem("Engine", c.getEngine()).elem("EngineVersion", c.getEngineVersion());- if (ep != null) {- xml.start("ConfigurationEndpoint")- .elem("Address", ep.address())- .elem("Port", (long) ep.port())- .end("ConfigurationEndpoint");+ if ("memcached".equals(c.getEngine())) {+ if (ep != null) {+ xml.start("ConfigurationEndpoint")+ .elem("Address", ep.address())+ .elem("Port", (long) ep.port())+ .end("ConfigurationEndpoint");+ }+ return xml.end("CacheCluster").build();+ }++ xml.elem("NumCacheNodes", (long) c.getNumCacheNodes())+ .elem("AutoMinorVersionUpgrade", true)+ .elem("AuthTokenEnabled", c.getAuthMode() == AuthMode.PASSWORD)+ .elem("TransitEncryptionEnabled", c.getAuthMode() != null && c.getAuthMode() != AuthMode.NO_AUTH)+ .elem("AtRestEncryptionEnabled", c.isAtRestEncryptionEnabled())+ .elem("SnapshotRetentionLimit", (long) c.getSnapshotRetentionLimit())+ .elem("SnapshotWindow", c.getSnapshotWindow() != null+ ? c.getSnapshotWindow() : ReplicationGroupSettings.DEFAULT_SNAPSHOT_WINDOW)+ .elem("PreferredMaintenanceWindow", c.getPreferredMaintenanceWindow() != null+ ? c.getPreferredMaintenanceWindow() : BackupWindows.DEFAULT_MAINTENANCE_WINDOW);+ if (c.getCacheClusterCreateTime() != null) {+ xml.elem("CacheClusterCreateTime", c.getCacheClusterCreateTime().toString());+ }+ if (c.getPreferredAvailabilityZone() != null) {+ xml.elem("PreferredAvailabilityZone", c.getPreferredAvailabilityZone());+ }+ if (c.getNetworkType() != null) {+ xml.elem("NetworkType", c.getNetworkType());+ }+ if (c.getIpDiscovery() != null) {+ xml.elem("IpDiscovery", c.getIpDiscovery());+ }+ if (!c.getSecurityGroupIds().isEmpty()) {+ xml.start("SecurityGroups");+ for (String securityGroupId : c.getSecurityGroupIds()) {+ xml.start("member")+ .elem("SecurityGroupId", securityGroupId)+ .elem("Status", "active")+ .end("member");+ }+ xml.end("SecurityGroups");+ }+ if (c.getCacheNodeType() != null) {+ xml.elem("CacheNodeType", c.getCacheNodeType());+ }+ if (c.getCacheParameterGroupName() != null) {+ xml.start("CacheParameterGroup")+ .elem("CacheParameterGroupName", c.getCacheParameterGroupName())+ .elem("ParameterApplyStatus", "in-sync")+ .end("CacheParameterGroup");+ }+ if (c.getCacheSubnetGroupName() != null) {+ xml.elem("CacheSubnetGroupName", c.getCacheSubnetGroupName());+ }+ if (c.getArn() != null) {+ xml.elem("ARN", c.getArn());+ }+ if (showNodeInfo && ep != null) {+ // terraform-provider-aws dereferences CacheNodeCreateTime and ParameterGroupStatus+ // without a nil check while adopting the resource, so a node carrying only an+ // endpoint fails the refresh right after a successful create. Both are stable+ // metadata AWS returns on every available node, and both are already known here:+ // a single-node cluster's node was created with the cluster, and its parameter+ // group has the same in-sync status reported for the cluster above.+ xml.start("CacheNodes")+ .start("CacheNode")+ .elem("CacheNodeId", "0001")+ .elem("CacheNodeStatus", "available");+ if (c.getCacheClusterCreateTime() != null) {+ xml.elem("CacheNodeCreateTime", c.getCacheClusterCreateTime().toString());+ }+ xml.start("Endpoint")+ .elem("Address", ep.address())+ .elem("Port", (long) ep.port())+ .end("Endpoint")+ .elem("ParameterGroupStatus", "in-sync");+ if (c.getPreferredAvailabilityZone() != null) {+ xml.elem("CustomerAvailabilityZone", c.getPreferredAvailabilityZone());+ }+ xml.end("CacheNode")+ .end("CacheNodes");}return xml.end("CacheCluster").build();}diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.javaindex 3deacb3758..66a6f0dc4a 100644--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java@@ -4,6 +4,7 @@ import com.fasterxml.jackson.core.type.TypeReference;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsArnUtils;import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.BackupWindows;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.resource.ExplorerResource;import io.github.hectorvent.floci.core.resource.ResourceProvider;@@ -17,6 +18,8 @@ import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContimport io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerManager;import io.github.hectorvent.floci.services.elasticache.container.ValkeyClusterFormation;import io.github.hectorvent.floci.services.elasticache.model.AuthMode;+import io.github.hectorvent.floci.services.elasticache.model.CacheCluster;+import io.github.hectorvent.floci.services.elasticache.model.CacheClusterStatus;import io.github.hectorvent.floci.services.elasticache.model.CacheParameterGroup;import io.github.hectorvent.floci.services.elasticache.model.CacheSubnetGroup;import io.github.hectorvent.floci.services.elasticache.model.ClusterNode;@@ -58,6 +61,14 @@ public class ElastiCacheService implements ResourceProvider {private static final Logger LOG = Logger.getLogger(ElastiCacheService.class);private final StorageBackend<String, ReplicationGroup> groups;+ private final StorageBackend<String, CacheCluster> cacheClusters;+ /**+ * The Memcached cluster store, the one {@link ElastiCacheMemcachedService} writes.+ * {@link StorageFactory#create} keys backends by file path and hands the second caller the+ * first caller's instance, so this is that store rather than a copy of it: a cache cluster id+ * is one namespace whatever engine claims it, and only a reader of both can say it is free.+ */+ private final StorageBackend<String, CacheCluster> memcachedClusters;private final StorageBackend<String, ElastiCacheUser> users;private final AccountAwareStorageBackend<CacheParameterGroup> parameterGroups;private final StorageBackend<String, CacheSubnetGroup> subnetGroups;@@ -69,7 +80,23 @@ public class ElastiCacheService implements ResourceProvider {private final Ec2Service ec2Service;private final RegionResolver regionResolver;private final Set<Integer> usedPorts = ConcurrentHashMap.newKeySet();- private final Set<String> provisioningGroupIds = ConcurrentHashMap.newKeySet();+ /**+ * Ids claimed by in-flight creates: replication groups, standalone cache clusters and the+ * Memcached clusters {@link ElastiCacheMemcachedService} creates, all claiming against the+ * one instance so a create on either side sees the other's claim. Both redis paths name+ * their container {@code valkey-<id>} and register their proxy under the id, so a second+ * create of a live id would remove the first's container and orphan its listener.+ */+ private final ElastiCacheProvisioningIds provisioningIds;+ /**+ * Records whose advertised port this process holds in {@link #usedPorts}: standalone cache+ * clusters, and the replication groups that advertise one port of their own rather than a+ * port per node. A record restored from disk whose port was already taken keeps advertising+ * it but does not own it, and a delete that released it anyway would hand a live record's+ * port to the next create. One set for both because an id is unique across the stores, so+ * the two kinds cannot collide here either.+ */+ private final Set<String> recordsHoldingTheirPort = ConcurrentHashMap.newKeySet();private final ConcurrentHashMap<String, Object> parameterGroupLocks = new ConcurrentHashMap<>();/*** Parameter groups claimed by in-flight creates, keyed by account and name (see@@ -88,8 +115,10 @@ public class ElastiCacheService implements ResourceProvider {EmulatorConfig config,Ec2Service ec2Service,RegionResolver regionResolver,- KmsService kmsService) {+ KmsService kmsService,+ ElastiCacheProvisioningIds provisioningIds) {this.kmsService = kmsService;+ this.provisioningIds = provisioningIds;this.containerManager = containerManager;this.proxyManager = proxyManager;this.clusterFormation = clusterFormation;@@ -98,6 +127,10 @@ public class ElastiCacheService implements ResourceProvider {this.regionResolver = regionResolver;this.groups = storageFactory.create("elasticache", "elasticache-groups.json",new TypeReference<Map<String, ReplicationGroup>>() {});+ this.cacheClusters = storageFactory.create("elasticache", "elasticache-redis-clusters.json",+ new TypeReference<Map<String, CacheCluster>>() {});+ this.memcachedClusters = storageFactory.create("elasticache", "elasticache-cache-clusters.json",+ new TypeReference<Map<String, CacheCluster>>() {});this.users = storageFactory.create("elasticache", "elasticache-users.json",new TypeReference<Map<String, ElastiCacheUser>>() {});this.parameterGroups = storageFactory.create("elasticache", "elasticache-parameter-groups.json",@@ -159,13 +192,16 @@ public class ElastiCacheService implements ResourceProvider {// DeleteCacheParameterGroup sees the dependency before the stored-groups scan can.String parameterGroupReservation = reserveParameterGroup(request.cacheParameterGroupName());try {- if (groups.get(groupId).isPresent()) {+ // A standalone cache cluster of that id counts as taken, memcached included: see+ // provisioningIds, and cacheClusterIdTaken for the same test the other way round.+ if (groups.get(groupId).isPresent() || cacheClusters.get(groupId).isPresent()+ || memcachedClusters.get(groupId).isPresent()) {throw new AwsException("ReplicationGroupAlreadyExistsFault","Replication group " + groupId + " already exists.", 400);}// Claim the id for the whole provisioning attempt so a concurrent create can't race// ahead and be stopped by this request's handle-less rollback fallback.- if (!provisioningGroupIds.add(groupId)) {+ if (!provisioningIds.claim(groupId)) {throw new AwsException("ReplicationGroupAlreadyExistsFault","Replication group " + groupId + " is already being created.", 400);}@@ -176,7 +212,7 @@ public class ElastiCacheService implements ResourceProvider {}return provisionSingleNodeGroup(request, resolvedSettings);} finally {- provisioningGroupIds.remove(groupId);+ provisioningIds.release(groupId);}} finally {releaseParameterGroup(parameterGroupReservation);@@ -268,6 +304,8 @@ public class ElastiCacheService implements ResourceProvider {+ "Docker daemon is reachable. Metadata operations work; connections to "+ "the cache do not until a daemon appears.", groupId);}+ // Under the same monitor as the put, for the reason given on recordsHoldingTheirPort.+ recordsHoldingTheirPort.add(groupId);}LOG.infov("Replication group {0} created, endpoint={1}:{2}", groupId, endpointHost, String.valueOf(proxyPort));@@ -377,7 +415,7 @@ public class ElastiCacheService implements ResourceProvider {: defaultEngineForImage());group.setEngineVersion(request.engineVersion() != null && !request.engineVersion().isBlank()? request.engineVersion()- : ("valkey".equals(group.getEngine()) ? "8.1" : "7.1"));+ : defaultEngineVersion(group.getEngine()));group.setCacheNodeType(request.cacheNodeType() != null && !request.cacheNodeType().isBlank()? request.cacheNodeType(): "cache.t4g.micro");@@ -392,6 +430,10 @@ public class ElastiCacheService implements ResourceProvider {}}+ private static String defaultEngineVersion(String engine) {+ return "valkey".equals(engine) ? "8.1" : "7.1";+ }+private String defaultEngineForImage() {String image = config.services().elasticache().defaultImage();return image != null && image.contains("valkey") ? "valkey" : "redis";@@ -519,15 +561,22 @@ public class ElastiCacheService implements ResourceProvider {* free for the next create to take — pointing the old endpoint at an unrelated cache rather* than failing cleanly.*+ * <p>Standalone Redis and Valkey cache clusters are restored the same way, and must be: they+ * are the same Valkey container behind the same auth proxy on a port from the same range as a+ * cluster-mode-disabled group, so leaving them out would reproduce exactly the state above+ * for the one shape that reaches Floci through {@code aws_elasticache_cluster}. Memcached+ * clusters are {@code ElastiCacheMemcachedService}'s to restore, from its own store.+ ** <p>Container restarts and cluster formation can take a readiness timeout and a formation* timeout per group, so the data-plane work runs in the background and must not delay- * emulator readiness. Groups are marked {@code creating} synchronously — each proxy port is- * reserved at the same time, before any request can claim it — and flip to {@code available}- * or {@code create-failed} as their restoration finishes.+ * emulator readiness. Records are marked {@code creating} synchronously, each proxy port+ * reserved at the same time, before any request can claim it, and flip to {@code available}+ * or to the failure status their store models as their restoration finishes.*/public CompletableFuture<Void> restorePersistedRuntime() {List<ReplicationGroup> clusterModeToRestore = new ArrayList<>();List<ReplicationGroup> singleNodeToRestore = new ArrayList<>();+ List<CacheCluster> cacheClustersToRestore = new ArrayList<>();for (ReplicationGroup group : groups.scan(k -> true)) {if (group.getStatus() == ReplicationGroupStatus.DELETING) {continue;@@ -539,14 +588,25 @@ public class ElastiCacheService implements ResourceProvider {}groups.put(group.getReplicationGroupId(), group);}- if (clusterModeToRestore.isEmpty() && singleNodeToRestore.isEmpty()) {+ for (CacheCluster cluster : cacheClusters.scan(k -> true)) {+ if (cluster.getCacheClusterStatus() == CacheClusterStatus.DELETING) {+ continue;+ }+ reserveCacheCluster(cluster, cacheClustersToRestore);+ cacheClusters.put(cluster.getCacheClusterId(), cluster);+ }+ if (clusterModeToRestore.isEmpty() && singleNodeToRestore.isEmpty()+ && cacheClustersToRestore.isEmpty()) {return CompletableFuture.completedFuture(null);}- LOG.infov("Restoring {0} cluster-mode and {1} single-node replication group(s) in the background",- String.valueOf(clusterModeToRestore.size()), String.valueOf(singleNodeToRestore.size()));+ LOG.infov("Restoring {0} cluster-mode and {1} single-node replication group(s) and {2} "+ + "standalone cache cluster(s) in the background",+ String.valueOf(clusterModeToRestore.size()), String.valueOf(singleNodeToRestore.size()),+ String.valueOf(cacheClustersToRestore.size()));return CompletableFuture.runAsync(() -> {clusterModeToRestore.forEach(this::restoreClusterModeGroup);singleNodeToRestore.forEach(this::restoreSingleNodeGroup);+ cacheClustersToRestore.forEach(this::restoreCacheCluster);});}@@ -579,6 +639,10 @@ public class ElastiCacheService implements ResourceProvider {group.setProxyPort(reserveOrAllocateProxyPort(group.getProxyPort()));group.setConfigurationEndpoint(new Endpoint(resolveEndpointHost(), group.getProxyPort()));group.setStatus(ReplicationGroupStatus.CREATING);+ // The reservation above succeeded, so this process owns the port whether or not it is+ // the one the record came back with. Without this the delete path would decline to+ // free a port nothing else holds, and the reservation would outlive the group.+ recordsHoldingTheirPort.add(group.getReplicationGroupId());toRestore.add(group);} catch (RuntimeException e) {group.setStatus(ReplicationGroupStatus.CREATE_FAILED);@@ -587,6 +651,161 @@ public class ElastiCacheService implements ResourceProvider {}}+ /**+ * Reserves the proxy port a standalone cache cluster comes back advertising and marks it+ * {@code creating} for {@link #restoreCacheCluster}, the same pair of steps+ * {@link #reserveSingleNodeGroup} performs for a cluster-mode-disabled group. As there, the+ * endpoint host is re-derived from configuration rather than replayed, so a cluster restored+ * under a changed {@code FLOCI_HOSTNAME} advertises the name this process answers to.+ *+ * <p>A cluster whose port cannot be reserved is reported {@code restore-failed}, the status+ * AWS models on {@code CacheCluster}; a replication group reports {@code create-failed},+ * which is the value its own model carries.+ */+ private void reserveCacheCluster(CacheCluster cluster, List<CacheCluster> toRestore) {+ String clusterId = cluster.getCacheClusterId();+ try {+ Endpoint persisted = cluster.getConfigurationEndpoint();+ int proxyPort = reserveOrAllocateProxyPort(persisted != null ? persisted.port() : 0);+ cluster.setConfigurationEndpoint(new Endpoint(resolveEndpointHost(), proxyPort));+ cluster.setCacheClusterStatus(CacheClusterStatus.CREATING);+ recordsHoldingTheirPort.add(clusterId);+ toRestore.add(cluster);+ } catch (RuntimeException e) {+ cluster.setCacheClusterStatus(CacheClusterStatus.RESTORE_FAILED);+ cluster.setConfigurationEndpoint(null);+ LOG.warnv(e, "Failed to restore cache cluster {0}", clusterId);+ }+ }++ /**+ * Restarts the container and auth proxy behind a standalone Redis or Valkey cache cluster,+ * the same pair {@link #provisionCacheCluster} creates. The cache comes back empty, as it+ * does for a replication group: only the record is persisted, never the keyspace.+ *+ * <p>Uses {@code tryStart} for the same reason the create path does: with no Docker daemon+ * reachable the cluster keeps its metadata and reports available without a container, rather+ * than being failed for a daemon that may appear later.+ *+ * <p>The container work runs outside the cluster's monitor, so a restore that may pull an+ * image does not hold a delete of the same cluster behind it. The write-back takes the+ * monitor and re-reads the record, because a delete taken while the container started has+ * already removed it.+ */+ private void restoreCacheCluster(CacheCluster cluster) {+ String clusterId = cluster.getCacheClusterId();+ String image = config.services().elasticache().defaultImage();+ try {+ ElastiCacheContainerHandle handle = containerManager.tryStart(clusterId, image);+ synchronized (lockFor("cc:" + clusterId)) {+ if (cacheClusterRestoreTargetLost(clusterId)) {+ abandonRestoredCacheClusterContainer(clusterId, handle);+ return;+ }+ if (handle != null) {+ cluster.setContainerId(handle.getContainerId());+ cluster.setContainerHost(handle.getHost());+ cluster.setContainerPort(handle.getPort());+ proxyManager.startProxy(clusterId, cluster.getAuthMode(),+ cluster.getConfigurationEndpoint().port(), handle.getHost(), handle.getPort(),+ (username, password) -> validateCacheClusterPassword(clusterId, username, password));+ } else {+ // Cleared rather than left alone: whatever the record carried describes a+ // container from the previous process, and nothing must read it as live.+ cluster.setContainerId(null);+ cluster.setContainerHost(null);+ cluster.setContainerPort(0);+ LOG.warnv("Cache cluster {0} restored without a backing cache container: no "+ + "Docker daemon is reachable. Metadata operations work; connections to "+ + "the cache do not until a daemon appears.", clusterId);+ }+ cluster.setCacheClusterStatus(CacheClusterStatus.AVAILABLE);+ cacheClusters.put(clusterId, cluster);+ LOG.infov("Restored cache cluster {0}, endpoint={1}:{2}", clusterId,+ cluster.getConfigurationEndpoint().address(),+ String.valueOf(cluster.getConfigurationEndpoint().port()));+ }+ } catch (RuntimeException e) {+ failCacheClusterRestore(cluster, e);+ }+ }++ /**+ * Whether the cache cluster a restore is about to write back is still there to write back to.+ * Callers must hold the cluster's monitor, for the reason given on+ * {@link #restoreTargetLost}.+ */+ private boolean cacheClusterRestoreTargetLost(String clusterId) {+ return cacheClusters.get(clusterId)+ .map(current -> current.getCacheClusterStatus() == CacheClusterStatus.DELETING)+ .orElse(true);+ }++ /**+ * Drops the container a restore started for a cache cluster that was deleted while it ran.+ * The proxy port is deliberately not released here: the delete that removed the record+ * released it already, and a second release would return a port a later create has since+ * taken to the pool.+ */+ private void abandonRestoredCacheClusterContainer(String clusterId, ElastiCacheContainerHandle handle) {+ if (handle != null) {+ try {+ containerManager.stop(handle);+ } catch (RuntimeException e) {+ LOG.warnv("Error stopping container for deleted cache cluster {0}: {1}",+ clusterId, e.getMessage());+ }+ }+ LOG.infov("Discarded the restored data plane for cache cluster {0}: it was deleted while "+ + "it was being restored", clusterId);+ }++ /**+ * Reports a cache cluster whose data plane could not be brought back as+ * {@code restore-failed} without deleting the record: the cluster still exists on the control+ * plane, it just has nothing behind it. Tears down whatever the attempt left running and+ * frees the port, so the next create can use it rather than leaking a reservation nothing+ * serves.+ *+ * <p>A cluster deleted while the failed attempt ran gets the teardown but no write-back and+ * no port release, for the reasons {@link #failSingleNodeRestore} gives.+ */+ private void failCacheClusterRestore(CacheCluster cluster, RuntimeException cause) {+ String clusterId = cluster.getCacheClusterId();+ synchronized (lockFor("cc:" + clusterId)) {+ try {+ proxyManager.stopProxy(clusterId);+ } catch (RuntimeException e) {+ LOG.warnv("Error stopping proxy for cache cluster {0}: {1}", clusterId, e.getMessage());+ }+ try {+ containerManager.stopByGroupId(clusterId);+ } catch (RuntimeException e) {+ LOG.warnv("Error stopping container for cache cluster {0}: {1}", clusterId, e.getMessage());+ }+ if (cacheClusterRestoreTargetLost(clusterId)) {+ LOG.warnv(cause, "Failed to restore cache cluster {0}, which was deleted while it "+ + "was being restored", clusterId);+ return;+ }+ Endpoint endpoint = cluster.getConfigurationEndpoint();+ if (endpoint != null && recordsHoldingTheirPort.remove(clusterId)) {+ releaseProxyPort(endpoint.port());+ }+ cluster.setContainerId(null);+ cluster.setContainerHost(null);+ cluster.setContainerPort(0);+ cluster.setCacheClusterStatus(CacheClusterStatus.RESTORE_FAILED);+ cluster.setConfigurationEndpoint(null);+ try {+ cacheClusters.put(clusterId, cluster);+ } catch (RuntimeException persistFailure) {+ cause.addSuppressed(persistFailure);+ }+ LOG.warnv(cause, "Failed to restore cache cluster {0}", clusterId);+ }+ }+/*** Restarts the container and auth proxy behind a cluster-mode-disabled group, the same pair* {@link #provisionSingleNodeGroup} creates. The cache comes back empty, as it does for@@ -700,7 +919,11 @@ public class ElastiCacheService implements ResourceProvider {+ "it was being restored", groupId);return;}- releaseProxyPort(group.getProxyPort());+ // Dropped from the holders as it is released, so the delete of a failed group does+ // not free the port a second time and hand a live record's port to the next create.+ if (recordsHoldingTheirPort.remove(groupId)) {+ releaseProxyPort(group.getProxyPort());+ }group.setContainerId(null);group.setContainerHost(null);group.setContainerPort(0);@@ -837,6 +1060,7 @@ public class ElastiCacheService implements ResourceProvider {LOG.warnv("Error stopping container for replication group {0}: {1}", groupId, e.getMessage());} finally {groups.delete(groupId);+ recordsHoldingTheirPort.remove(groupId);releaseProxyPort(proxyPort);}}@@ -890,13 +1114,299 @@ public class ElastiCacheService implements ResourceProvider {group.getContainerId(), groupId, group.getContainerHost(), group.getContainerPort()));}- releaseProxyPort(group.getProxyPort());+ // Only a port this process reserved for this record, as on the cluster path: a+ // restored group whose port was already taken advertises one it does not own.+ if (recordsHoldingTheirPort.remove(groupId)) {+ releaseProxyPort(group.getProxyPort());+ }}groups.delete(groupId);LOG.infov("Replication group {0} deleted", groupId);}}+ // ── Cache Clusters (single-node Redis/Valkey) ─────────────────────────────++ /**+ * The CreateCacheCluster parameters floci models for a standalone redis or valkey cluster.+ * AWS accepts that action for redis and valkey as well as memcached, as long as the cluster+ * has exactly one node, and terraform's {@code aws_elasticache_cluster} with+ * {@code engine = "redis"} emits precisely that call: no replication group is involved, so+ * these clusters are stored on their own and never appear in DescribeReplicationGroups.+ */+ public record CreateCacheClusterRequest(+ String cacheClusterId,+ String engine,+ String engineVersion,+ String cacheNodeType,+ Integer numCacheNodes,+ Integer port,+ AuthMode authMode,+ String authToken,+ String cacheParameterGroupName,+ String cacheSubnetGroupName,+ Integer snapshotRetentionLimit,+ String snapshotWindow,+ String preferredMaintenanceWindow,+ String preferredAvailabilityZone,+ List<String> securityGroupIds,+ String networkType,+ String ipDiscovery,+ Boolean atRestEncryptionEnabled,+ String region,+ Map<String, String> tags) {+ }++ public CacheCluster createCacheCluster(CreateCacheClusterRequest request) {+ String clusterId = request.cacheClusterId();+ String engine = normalizeEngine(request.engine());+ if (request.numCacheNodes() != null && request.numCacheNodes() != 1) {+ throw new AwsException("InvalidParameterValue",+ "NumCacheNodes should be 1 if engine is " + engine, 400);+ }+ // The snapshot members take the replication group's checks, so one request worded one way+ // is refused the same way whichever action carries it.+ cacheClusterSettings(request).validate();+ if (request.preferredMaintenanceWindow() != null && !request.preferredMaintenanceWindow().isBlank()) {+ BackupWindows.parseMaintenanceWindow(request.preferredMaintenanceWindow());+ }+ requireCacheSubnetGroup(request.cacheSubnetGroupName());+ String parameterGroupReservation = reserveParameterGroup(request.cacheParameterGroupName());+ try {+ if (cacheClusterIdTaken(clusterId)) {+ throw new AwsException("CacheClusterAlreadyExists",+ "Cache cluster " + clusterId + " already exists.", 400);+ }+ if (!provisioningIds.claim(clusterId)) {+ throw new AwsException("CacheClusterAlreadyExists",+ "Cache cluster " + clusterId + " is already being created.", 400);+ }+ try {+ return provisionCacheCluster(request, engine);+ } finally {+ provisioningIds.release(clusterId);+ }+ } finally {+ releaseParameterGroup(parameterGroupReservation);+ }+ }++ /**+ * Whether any of the three stores already answers for that id. A Memcached cluster of the+ * same name is only a describe that reports one id twice, but a replication group is worse:+ * see provisioningIds for what its container and proxy would lose.+ */+ private boolean cacheClusterIdTaken(String clusterId) {+ return cacheClusters.get(clusterId).isPresent()+ || groups.get(clusterId).isPresent()+ || memcachedClusters.get(clusterId).isPresent();+ }++ /**+ * A named subnet group must exist. Storing a name nothing resolves would have the describe+ * report a group a caller cannot look up, and AWS refuses the create instead.+ *+ * <p>{@code CreateReplicationGroup} does not make this check, so a replication group can+ * still be created against a subnet group that is not there.+ */+ private void requireCacheSubnetGroup(String name) {+ if (name == null || name.isBlank()) {+ return;+ }+ if (subnetGroups.get(name).isEmpty()) {+ throw new AwsException("CacheSubnetGroupNotFoundFault",+ "Cache subnet group " + name + " not found.", 400);+ }+ }++ /**+ * The request's snapshot members as the replication group models them, so both actions share+ * one set of checks and one set of defaults.+ */+ private static ReplicationGroupSettings cacheClusterSettings(CreateCacheClusterRequest request) {+ return new ReplicationGroupSettings(request.atRestEncryptionEnabled(), null,+ request.snapshotRetentionLimit(), request.snapshotWindow());+ }++ /**+ * Provisions the cluster exactly as {@link #provisionSingleNodeGroup} provisions a+ * cluster-mode-disabled replication group: the same Valkey container, fronted by the same+ * auth proxy on a port from the same range, so the endpoint a describe reports answers RESP.+ */+ private CacheCluster provisionCacheCluster(CreateCacheClusterRequest request, String engine) {+ String clusterId = request.cacheClusterId();+ AuthMode authMode = request.authMode() != null ? request.authMode() : AuthMode.NO_AUTH;+ int proxyPort = allocateProxyPort(request.port());+ String image = config.services().elasticache().defaultImage();++ LOG.infov("Creating single-node {0} cache cluster {1} with authMode={2} on proxy port {3}",+ engine, clusterId, authMode, String.valueOf(proxyPort));++ ElastiCacheContainerHandle handle = null;+ try {+ // As for a replication group, the record is metadata: it reaches 'available' even+ // when no Docker daemon is reachable, and only connecting to the cache needs the+ // container.+ handle = containerManager.tryStart(clusterId, image);++ CacheCluster cluster = new CacheCluster(clusterId, CacheClusterStatus.AVAILABLE, engine,+ request.engineVersion() != null && !request.engineVersion().isBlank()+ ? request.engineVersion()+ : defaultEngineVersion(engine),+ new Endpoint(resolveEndpointHost(), proxyPort), Instant.now());+ cluster.setNumCacheNodes(1);+ cluster.setCacheNodeType(request.cacheNodeType() != null && !request.cacheNodeType().isBlank()+ ? request.cacheNodeType()+ : "cache.t4g.micro");+ cluster.setAuthMode(authMode);+ cluster.setAuthToken(request.authToken());+ cluster.setArn(regionResolver.buildArn("elasticache", request.region(), "cluster:" + clusterId));+ cluster.setCacheParameterGroupName(request.cacheParameterGroupName());+ cluster.setCacheSubnetGroupName(request.cacheSubnetGroupName());+ applyCacheClusterSettings(cluster, request);+ if (request.tags() != null && !request.tags().isEmpty()) {+ cluster.setTags(new LinkedHashMap<>(request.tags()));+ }+ if (handle != null) {+ cluster.setContainerId(handle.getContainerId());+ cluster.setContainerHost(handle.getHost());+ cluster.setContainerPort(handle.getPort());+ }++ synchronized (lockFor("cc:" + clusterId)) {+ cacheClusters.put(clusterId, cluster);+ if (handle != null) {+ proxyManager.startProxy(clusterId, authMode, proxyPort,+ handle.getHost(), handle.getPort(),+ (username, password) -> validateCacheClusterPassword(clusterId, username, password));+ } else {+ LOG.warnv("Cache cluster {0} created without a backing cache container: no "+ + "Docker daemon is reachable. Metadata operations work; connections to "+ + "the cache do not until a daemon appears.", clusterId);+ }+ // Under the same monitor as the put: a delete slipping into the gap would find+ // no claim on the port and leave the reservation behind for good.+ recordsHoldingTheirPort.add(clusterId);+ }++ LOG.infov("Cache cluster {0} created, endpoint={1}:{2}", clusterId,+ cluster.getConfigurationEndpoint().address(), String.valueOf(proxyPort));+ return cluster;+ } catch (RuntimeException e) {+ LOG.warnv("Cache cluster {0} provisioning failed, rolling back: {1}", clusterId, e.getMessage());+ rollbackCacheCluster(clusterId, handle, proxyPort);+ throw e;+ }+ }++ /**+ * The optional members a request may carry, stored so a describe can echo them. Every one is+ * an optional {@code aws_elasticache_cluster} argument: dropping any of them would read back+ * as unset on the next plan, which is a diff terraform can never settle.+ */+ private void applyCacheClusterSettings(CacheCluster cluster, CreateCacheClusterRequest request) {+ cluster.setSnapshotRetentionLimit(request.snapshotRetentionLimit() != null+ ? request.snapshotRetentionLimit() : 0);+ cluster.setSnapshotWindow(request.snapshotWindow() != null && !request.snapshotWindow().isBlank()+ ? request.snapshotWindow() : ReplicationGroupSettings.DEFAULT_SNAPSHOT_WINDOW);+ cluster.setPreferredMaintenanceWindow(request.preferredMaintenanceWindow() != null+ && !request.preferredMaintenanceWindow().isBlank()+ ? BackupWindows.lowerCase(request.preferredMaintenanceWindow())+ : BackupWindows.DEFAULT_MAINTENANCE_WINDOW);+ cluster.setPreferredAvailabilityZone(request.preferredAvailabilityZone() != null+ && !request.preferredAvailabilityZone().isBlank()+ ? request.preferredAvailabilityZone()+ : regionResolver.getRegion() + "a");+ cluster.setNetworkType(request.networkType() != null && !request.networkType().isBlank()+ ? request.networkType() : "ipv4");+ cluster.setIpDiscovery(request.ipDiscovery() != null && !request.ipDiscovery().isBlank()+ ? request.ipDiscovery() : "ipv4");+ cluster.setAtRestEncryptionEnabled(Boolean.TRUE.equals(request.atRestEncryptionEnabled()));+ if (request.securityGroupIds() != null && !request.securityGroupIds().isEmpty()) {+ cluster.setSecurityGroupIds(new ArrayList<>(request.securityGroupIds()));+ }+ }++ private void rollbackCacheCluster(String clusterId, ElastiCacheContainerHandle handle, int proxyPort) {+ try {+ if (handle != null) {+ proxyManager.stopProxy(clusterId);+ }+ } catch (RuntimeException e) {+ LOG.warnv("Error stopping proxy for cache cluster {0}: {1}", clusterId, e.getMessage());+ }+ try {+ if (handle != null) {+ containerManager.stop(handle);+ } else {+ // No handle: a readiness timeout throws before start() can return one.+ containerManager.stopByGroupId(clusterId);+ }+ } catch (RuntimeException e) {+ LOG.warnv("Error stopping container for cache cluster {0}: {1}", clusterId, e.getMessage());+ } finally {+ cacheClusters.delete(clusterId);+ recordsHoldingTheirPort.remove(clusterId);+ releaseProxyPort(proxyPort);+ }+ }++ /**+ * The standalone cluster by that id, or all of them. Reports no match as an empty list rather+ * than a fault: DescribeCacheClusters answers from three sources, and only the last one to+ * find nothing can say the id does not exist.+ */+ public List<CacheCluster> findCacheClusters(String filterClusterId) {+ if (filterClusterId != null && !filterClusterId.isBlank()) {+ return cacheClusters.get(filterClusterId).map(List::of).orElseGet(List::of);+ }+ return cacheClusters.scan(k -> true);+ }++ public CacheCluster deleteCacheCluster(String clusterId) {+ synchronized (lockFor("cc:" + clusterId)) {+ CacheCluster cluster = cacheClusters.get(clusterId).orElseThrow(() ->+ new AwsException("CacheClusterNotFound",+ "Cache cluster " + clusterId + " not found.", 404));++ cluster.setCacheClusterStatus(CacheClusterStatus.DELETING);+ cacheClusters.put(clusterId, cluster);++ proxyManager.stopProxy(clusterId);+ if (cluster.getContainerId() != null) {+ containerManager.stop(new ElastiCacheContainerHandle(cluster.getContainerId(), clusterId,+ cluster.getContainerHost(), cluster.getContainerPort()));+ } else {+ // Transient container fields are lost across a Floci restart; the deterministic+ // container name still finds the cluster's container.+ containerManager.stopByGroupId(clusterId);+ }+ // Only a port this process reserved for this record: a restored cluster whose port+ // was already taken advertises one it does not own, and freeing it would hand the+ // holder's port to the next create.+ if (cluster.getConfigurationEndpoint() != null && recordsHoldingTheirPort.remove(clusterId)) {+ releaseProxyPort(cluster.getConfigurationEndpoint().port());+ }++ cacheClusters.delete(clusterId);+ LOG.infov("Cache cluster {0} deleted", clusterId);+ return cluster;+ }+ }++ /**+ * A standalone cluster's AUTH token, checked the same way {@link #validatePassword} checks a+ * replication group's: only the single-argument AUTH form, since a cluster created this way+ * carries no user list.+ */+ public boolean validateCacheClusterPassword(String clusterId, String username, String password) {+ CacheCluster cluster = cacheClusters.get(clusterId).orElse(null);+ if (cluster == null || cluster.getAuthToken() == null) {+ return false;+ }+ return (username == null || username.isEmpty()) && cluster.getAuthToken().equals(password);+ }+/*** A replication-group member viewed as the cache cluster AWS reports it as. The terraform* provider follows {@code MemberClusters} into DescribeCacheClusters, so every member name@@ -1465,13 +1975,15 @@ public class ElastiCacheService implements ResourceProvider {}/**- * Whether a stored replication group still references the parameter group by that name, or a- * create that named it is still provisioning and about to store one.+ * Whether a stored replication group or cache cluster still references the parameter group by+ * that name, or a create that named it is still provisioning and about to store one.*/private boolean isParameterGroupInUse(String name) {return reservedParameterGroups.containsKey(parameterGroupReservationKey(name))|| groups.scan(key -> true).stream()- .anyMatch(group -> name.equals(group.getCacheParameterGroupName()));+ .anyMatch(group -> name.equals(group.getCacheParameterGroupName()))+ || cacheClusters.scan(key -> true).stream()+ .anyMatch(cluster -> name.equals(cluster.getCacheParameterGroupName()));}@Override@@ -1488,6 +2000,18 @@ public class ElastiCacheService implements ResourceProvider {group.getCreatedAt() != null ? group.getCreatedAt() : Instant.now(),Map.of()));}+ for (CacheCluster cluster : cacheClusters.scan(k -> true)) {+ if (cluster.getArn() == null) {+ continue;+ }+ AwsArnUtils.Arn parsed = AwsArnUtils.parse(cluster.getArn());+ resources.add(new ExplorerResource(+ cluster.getArn(), "elasticache:cluster", "elasticache",+ parsed.region(), parsed.accountId(),+ cluster.getCacheClusterCreateTime() != null+ ? cluster.getCacheClusterCreateTime() : Instant.now(),+ Map.of()));+ }return resources;}diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/model/CacheCluster.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/model/CacheCluster.javaindex d0b652eaa1..38b876b9b6 100644--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/model/CacheCluster.java+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/model/CacheCluster.java@@ -3,6 +3,10 @@ package io.github.hectorvent.floci.services.elasticache.model;import io.quarkus.runtime.annotations.RegisterForReflection;import java.time.Instant;+import java.util.ArrayList;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;@RegisterForReflectionpublic class CacheCluster {@@ -13,8 +17,24 @@ public class CacheCluster {private String engineVersion;private Endpoint configurationEndpoint;private Instant cacheClusterCreateTime;-- // Transient — not persisted, restored on container restart+ private String cacheNodeType;+ private int numCacheNodes;+ private String arn;+ private String cacheParameterGroupName;+ private String cacheSubnetGroupName;+ private AuthMode authMode;+ private String authToken;+ private int snapshotRetentionLimit;+ private String snapshotWindow;+ private String preferredMaintenanceWindow;+ private String preferredAvailabilityZone;+ private String networkType;+ private String ipDiscovery;+ private boolean atRestEncryptionEnabled;+ private List<String> securityGroupIds = new ArrayList<>();+ private Map<String, String> tags = new LinkedHashMap<>();++ // Transient: not persisted, restored on container restartprivate transient String containerId;private transient String containerHost;private transient int containerPort;@@ -50,6 +70,54 @@ public class CacheCluster {public Instant getCacheClusterCreateTime() { return cacheClusterCreateTime; }public void setCacheClusterCreateTime(Instant cacheClusterCreateTime) { this.cacheClusterCreateTime = cacheClusterCreateTime; }+ public String getCacheNodeType() { return cacheNodeType; }+ public void setCacheNodeType(String cacheNodeType) { this.cacheNodeType = cacheNodeType; }++ public int getNumCacheNodes() { return numCacheNodes; }+ public void setNumCacheNodes(int numCacheNodes) { this.numCacheNodes = numCacheNodes; }++ public String getArn() { return arn; }+ public void setArn(String arn) { this.arn = arn; }++ public String getCacheParameterGroupName() { return cacheParameterGroupName; }+ public void setCacheParameterGroupName(String cacheParameterGroupName) { this.cacheParameterGroupName = cacheParameterGroupName; }++ public String getCacheSubnetGroupName() { return cacheSubnetGroupName; }+ public void setCacheSubnetGroupName(String cacheSubnetGroupName) { this.cacheSubnetGroupName = cacheSubnetGroupName; }++ public AuthMode getAuthMode() { return authMode; }+ public void setAuthMode(AuthMode authMode) { this.authMode = authMode; }++ public String getAuthToken() { return authToken; }+ public void setAuthToken(String authToken) { this.authToken = authToken; }++ public int getSnapshotRetentionLimit() { return snapshotRetentionLimit; }+ public void setSnapshotRetentionLimit(int snapshotRetentionLimit) { this.snapshotRetentionLimit = snapshotRetentionLimit; }++ public String getSnapshotWindow() { return snapshotWindow; }+ public void setSnapshotWindow(String snapshotWindow) { this.snapshotWindow = snapshotWindow; }++ public String getPreferredMaintenanceWindow() { return preferredMaintenanceWindow; }+ public void setPreferredMaintenanceWindow(String preferredMaintenanceWindow) { this.preferredMaintenanceWindow = preferredMaintenanceWindow; }++ public String getPreferredAvailabilityZone() { return preferredAvailabilityZone; }+ public void setPreferredAvailabilityZone(String preferredAvailabilityZone) { this.preferredAvailabilityZone = preferredAvailabilityZone; }++ public String getNetworkType() { return networkType; }+ public void setNetworkType(String networkType) { this.networkType = networkType; }++ public String getIpDiscovery() { return ipDiscovery; }+ public void setIpDiscovery(String ipDiscovery) { this.ipDiscovery = ipDiscovery; }++ public boolean isAtRestEncryptionEnabled() { return atRestEncryptionEnabled; }+ public void setAtRestEncryptionEnabled(boolean atRestEncryptionEnabled) { this.atRestEncryptionEnabled = atRestEncryptionEnabled; }++ public List<String> getSecurityGroupIds() { return securityGroupIds; }+ public void setSecurityGroupIds(List<String> securityGroupIds) { this.securityGroupIds = securityGroupIds == null ? new ArrayList<>() : securityGroupIds; }++ public Map<String, String> getTags() { return tags; }+ public void setTags(Map<String, String> tags) { this.tags = tags == null ? new LinkedHashMap<>() : tags; }+public String getContainerId() { return containerId; }public void setContainerId(String containerId) { this.containerId = containerId; }diff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedIntegrationTest.javaindex 97cdaa89d9..deba245b73 100644--- a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedIntegrationTest.java@@ -97,10 +97,13 @@ class ElastiCacheMemcachedIntegrationTest {@Test@Order(3)void createCacheClusterWithInvalidEngineReturnsError() {+ // Engine=redis used to be refused here. It is a valid single-node cache cluster on AWS and+ // is now accepted (see ElastiCacheRedisClusterIntegrationTest), so the refusal is asserted+ // with an engine ElastiCache really does not have.given().formParam("Action", "CreateCacheCluster")- .formParam("CacheClusterId", "redis-attempt")- .formParam("Engine", "redis")+ .formParam("CacheClusterId", "mongodb-attempt")+ .formParam("Engine", "mongodb").header("Authorization", AUTH_HEADER).when().post("/")diff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedServiceTest.javaindex 478c279a81..5a761144a4 100644--- a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemcachedServiceTest.java@@ -17,8 +17,10 @@ import java.util.Optional;import java.util.concurrent.ConcurrentHashMap;import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;import static org.junit.jupiter.api.Assertions.assertNull;import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.anyString;import static org.mockito.ArgumentMatchers.eq;@@ -31,10 +33,12 @@ class ElastiCacheMemcachedServiceTest {private ElastiCacheMemcachedService service;private ElastiCacheMemcachedContainerManager containerManager;+ private ElastiCacheProvisioningIds provisioningIds;@BeforeEachvoid setUp() {containerManager = mock(ElastiCacheMemcachedContainerManager.class);+ provisioningIds = new ElastiCacheProvisioningIds();StorageFactory storageFactory = mock(StorageFactory.class);EmulatorConfig config = mock(EmulatorConfig.class);@@ -49,7 +53,7 @@ class ElastiCacheMemcachedServiceTest {when(containerManager.tryStart(anyString(), anyString())).thenReturn(new ElastiCacheContainerHandle("cid", "cluster", "localhost", 11211));- service = new ElastiCacheMemcachedService(containerManager, storageFactory, config);+ service = new ElastiCacheMemcachedService(containerManager, storageFactory, config, provisioningIds);}@Test@@ -67,7 +71,29 @@ class ElastiCacheMemcachedServiceTest {service.createCacheCluster("my-cluster");AwsException ex = assertThrows(AwsException.class, () -> service.createCacheCluster("my-cluster"));- assertEquals("CacheClusterAlreadyExistsFault", ex.getErrorCode());+ assertEquals("CacheClusterAlreadyExists", ex.getErrorCode());+ }++ @Test+ void createIsRefusedWhileARedisCreateHoldsTheSameIdInFlight() {+ // What a concurrent CreateReplicationGroup or redis CreateCacheCluster leaves in the+ // shared set between claiming the id and persisting its record. The three stores are+ // still empty in that window, so the store checks alone would let this create through+ // and both would write the same id.+ assertTrue(provisioningIds.claim("racing-cluster"));++ AwsException ex = assertThrows(AwsException.class,+ () -> service.createCacheCluster("racing-cluster"));+ assertEquals("CacheClusterAlreadyExists", ex.getErrorCode());+ verify(containerManager, never()).tryStart(eq("racing-cluster"), anyString());++ // The refusal must not have released the claim the other create still holds.+ assertFalse(provisioningIds.claim("racing-cluster"));++ // Once that create finishes and releases, the id is free again.+ provisioningIds.release("racing-cluster");+ assertEquals("racing-cluster",+ service.createCacheCluster("racing-cluster").getCacheClusterId());}@Test@@ -122,7 +148,8 @@ class ElastiCacheMemcachedServiceTest {.thenReturn(new ElastiCacheContainerHandle("cid", "cluster", "172.20.0.10", 11211));ElastiCacheMemcachedService containerModeService =- new ElastiCacheMemcachedService(containerManager, storageFactory, config);+ new ElastiCacheMemcachedService(containerManager, storageFactory, config,+ new ElastiCacheProvisioningIds());CacheCluster cluster = containerModeService.createCacheCluster("container-cluster");@@ -236,6 +263,7 @@ class ElastiCacheMemcachedServiceTest {when(servicesConfig.elasticache()).thenReturn(ecConfig);when(ecConfig.defaultMemcachedImage()).thenReturn("memcached:1.6");when(config.hostname()).thenReturn(Optional.of("localhost"));- return new ElastiCacheMemcachedService(containerManager, storageFactory, config);+ return new ElastiCacheMemcachedService(containerManager, storageFactory, config,+ new ElastiCacheProvisioningIds());}}diff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandlerTest.javaindex 4a2a63ad7a..40e7cebc86 100644--- a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandlerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandlerTest.java@@ -3,6 +3,8 @@ package io.github.hectorvent.floci.services.elasticache;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.services.elasticache.model.AuthMode;+import io.github.hectorvent.floci.services.elasticache.model.CacheCluster;+import io.github.hectorvent.floci.services.elasticache.model.CacheClusterStatus;import io.github.hectorvent.floci.services.elasticache.model.ClusterNode;import io.github.hectorvent.floci.services.elasticache.model.Endpoint;import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroup;@@ -17,6 +19,7 @@ import org.junit.jupiter.api.Test;import org.mockito.ArgumentCaptor;import java.time.Instant;+import java.util.ArrayList;import java.util.Arrays;import java.util.LinkedHashMap;import java.util.List;@@ -34,12 +37,13 @@ class ElastiCacheQueryHandlerTest {private ElastiCacheQueryHandler handler;private ElastiCacheService service;+ private ElastiCacheMemcachedService memcachedService;@BeforeEachvoid setUp() {SigV4Validator sigV4Validator = mock(SigV4Validator.class);service = mock(ElastiCacheService.class);- ElastiCacheMemcachedService memcachedService = mock(ElastiCacheMemcachedService.class);+ memcachedService = mock(ElastiCacheMemcachedService.class);RegionResolver regionResolver = mock(RegionResolver.class);when(regionResolver.getRegion()).thenReturn("us-east-1");when(regionResolver.getAccountId()).thenReturn("000000000000");@@ -415,4 +419,274 @@ class ElastiCacheQueryHandlerTest {assertEquals(200, handler.handle("ModifyReplicationGroup", p, "us-east-1").getStatus());verify(service).modifyReplicationGroup("g1", null, null, new ReplicationGroupSettings(null, null, 3, "01:00-02:00"));}++ // ── CreateCacheCluster / DescribeCacheClusters: single-node redis ─────────++ private static CacheCluster redisCacheCluster(String id) {+ CacheCluster cluster = new CacheCluster(id, CacheClusterStatus.AVAILABLE, "redis", "7.1",+ new Endpoint("localhost", 6379), Instant.now());+ cluster.setNumCacheNodes(1);+ cluster.setCacheNodeType("cache.t4g.micro");+ cluster.setAuthMode(AuthMode.NO_AUTH);+ cluster.setArn("arn:aws:elasticache:us-east-1:000000000000:cluster:" + id);+ cluster.setSnapshotRetentionLimit(5);+ cluster.setSnapshotWindow("03:00-05:00");+ cluster.setPreferredMaintenanceWindow("tue:04:00-tue:05:00");+ cluster.setPreferredAvailabilityZone("us-east-1b");+ cluster.setNetworkType("ipv4");+ cluster.setIpDiscovery("ipv4");+ cluster.setSecurityGroupIds(new ArrayList<>(List.of("sg-123")));+ return cluster;+ }++ @Test+ void createCacheCluster_redisIsAcceptedAndProvisionedAsASingleNodeCluster() {+ // Real AWS accepts CreateCacheCluster with Engine=redis, and terraform's+ // aws_elasticache_cluster with engine "redis" emits exactly this call.+ ArgumentCaptor<ElastiCacheService.CreateCacheClusterRequest> captor =+ ArgumentCaptor.forClass(ElastiCacheService.CreateCacheClusterRequest.class);+ when(service.createCacheCluster(captor.capture())).thenReturn(redisCacheCluster("tf-redis"));++ MultivaluedMap<String, String> p = params();+ p.add("CacheClusterId", "tf-redis");+ p.add("Engine", "redis");+ p.add("NumCacheNodes", "1");+ p.add("CacheNodeType", "cache.t4g.micro");+ p.add("Port", "6379");++ Response response = handler.handle("CreateCacheCluster", p, "us-east-1");++ assertEquals(200, response.getStatus(), (String) response.getEntity());+ assertEquals("redis", captor.getValue().engine());+ assertEquals(1, captor.getValue().numCacheNodes());+ assertEquals(6379, captor.getValue().port());+ assertEquals("us-east-1", captor.getValue().region());+ verify(memcachedService, never()).createCacheCluster(anyString());++ String body = (String) response.getEntity();+ assertTrue(body.contains("<Engine>redis</Engine>"), body);+ assertTrue(body.contains("<NumCacheNodes>1</NumCacheNodes>"), body);+ assertTrue(body.contains("<CacheNodeType>cache.t4g.micro</CacheNodeType>"), body);+ // AWS reports CacheNodes only on a describe that asked for node info+ assertFalse(body.contains("<CacheNodes>"), body);+ }++ @Test+ void createCacheCluster_passesEveryOptionalMemberTheRequestCarries() {+ // Each of these is an optional aws_elasticache_cluster argument: one dropped here reads+ // back as unset on the next plan and terraform can never settle the diff.+ ArgumentCaptor<ElastiCacheService.CreateCacheClusterRequest> captor =+ ArgumentCaptor.forClass(ElastiCacheService.CreateCacheClusterRequest.class);+ when(service.createCacheCluster(captor.capture())).thenReturn(redisCacheCluster("tf-redis"));++ MultivaluedMap<String, String> p = params();+ p.add("CacheClusterId", "tf-redis");+ p.add("Engine", "redis");+ p.add("SnapshotRetentionLimit", "5");+ p.add("SnapshotWindow", "03:00-05:00");+ p.add("PreferredMaintenanceWindow", "tue:04:00-tue:05:00");+ p.add("PreferredAvailabilityZone", "us-east-1b");+ p.add("SecurityGroupIds.member.1", "sg-123");+ p.add("NetworkType", "ipv4");+ p.add("IpDiscovery", "ipv4");+ p.add("AtRestEncryptionEnabled", "true");++ assertEquals(200, handler.handle("CreateCacheCluster", p, "us-east-1").getStatus());++ ElastiCacheService.CreateCacheClusterRequest request = captor.getValue();+ assertEquals(5, request.snapshotRetentionLimit());+ assertEquals("03:00-05:00", request.snapshotWindow());+ assertEquals("tue:04:00-tue:05:00", request.preferredMaintenanceWindow());+ assertEquals("us-east-1b", request.preferredAvailabilityZone());+ assertEquals(List.of("sg-123"), request.securityGroupIds());+ assertEquals("ipv4", request.networkType());+ assertEquals("ipv4", request.ipDiscovery());+ assertEquals(Boolean.TRUE, request.atRestEncryptionEnabled());+ }++ @Test+ void createCacheCluster_unknownEngineIsStillRefused() {+ MultivaluedMap<String, String> p = params();+ p.add("CacheClusterId", "mystery");+ p.add("Engine", "mongodb");++ Response response = handler.handle("CreateCacheCluster", p, "us-east-1");++ assertEquals(400, response.getStatus());+ assertTrue(((String) response.getEntity()).contains("InvalidParameterValue"));+ verifyNoInteractions(memcachedService);+ }++ @Test+ void createCacheCluster_memcachedStillGoesToTheMemcachedService() {+ when(memcachedService.createCacheCluster("mc")).thenReturn(new CacheCluster(+ "mc", CacheClusterStatus.AVAILABLE, "memcached", "1.6.22",+ new Endpoint("localhost", 11211), Instant.now()));++ MultivaluedMap<String, String> p = params();+ p.add("CacheClusterId", "mc");+ p.add("Engine", "memcached");++ String body = (String) handler.handle("CreateCacheCluster", p, "us-east-1").getEntity();++ verify(memcachedService).createCacheCluster("mc");+ verify(service, never()).createCacheCluster(any());+ assertTrue(body.contains(+ "<ConfigurationEndpoint><Address>localhost</Address><Port>11211</Port></ConfigurationEndpoint>"), body);+ }++ @Test+ void describeCacheClusters_reportsASingleNodeRedisClusterByItsNodeEndpoint() {+ // AWS gives a redis cache cluster no ConfigurationEndpoint (that is Memcached's node+ // discovery endpoint); terraform-provider-aws reads the port off CacheNodes instead.+ when(service.findCacheClusters("tf-redis")).thenReturn(List.of(redisCacheCluster("tf-redis")));++ MultivaluedMap<String, String> p = params();+ p.add("CacheClusterId", "tf-redis");+ p.add("ShowCacheNodeInfo", "true");+ Response response = handler.handle("DescribeCacheClusters", p, "us-east-1");++ assertEquals(200, response.getStatus());+ String body = (String) response.getEntity();+ assertTrue(body.contains("<CacheClusterId>tf-redis</CacheClusterId>"), body);+ assertTrue(body.contains("<CacheClusterStatus>available</CacheClusterStatus>"), body);+ assertTrue(body.contains("<Engine>redis</Engine>"), body);+ assertTrue(body.contains("<EngineVersion>7.1</EngineVersion>"), body);+ assertTrue(body.contains("<ARN>arn:aws:elasticache:us-east-1:000000000000:cluster:tf-redis</ARN>"), body);+ assertTrue(body.contains("<CacheNodeId>0001</CacheNodeId>"), body);+ assertTrue(body.contains("<Endpoint><Address>localhost</Address><Port>6379</Port></Endpoint>"), body);+ assertFalse(body.contains("<ConfigurationEndpoint>"), body);+ // the optional members are echoed, so a terraform re-plan has nothing to change+ assertTrue(body.contains("<SnapshotRetentionLimit>5</SnapshotRetentionLimit>"), body);+ assertTrue(body.contains("<SnapshotWindow>03:00-05:00</SnapshotWindow>"), body);+ assertTrue(body.contains("<PreferredMaintenanceWindow>tue:04:00-tue:05:00</PreferredMaintenanceWindow>"), body);+ assertTrue(body.contains("<PreferredAvailabilityZone>us-east-1b</PreferredAvailabilityZone>"), body);+ assertTrue(body.contains("<SecurityGroupId>sg-123</SecurityGroupId>"), body);+ assertTrue(body.contains("<NetworkType>ipv4</NetworkType>"), body);+ assertTrue(body.contains("<IpDiscovery>ipv4</IpDiscovery>"), body);+ assertTrue(body.contains("<CacheClusterCreateTime>"), body);+ // the Memcached store is not consulted for an id another source answered for, so its+ // not-found fault cannot turn a hit into a 404+ verify(memcachedService, never()).listCacheClusters(anyString());+ }++ @Test+ void describeCacheClusters_singleNodeCarriesTheNodeMetadataTerraformDereferences() {+ // The node shape, not the endpoint, is what broke adoption: a create succeeded and the+ // very next refresh died with "Unexpected nil pointer" showing CacheNodeCreateTime:<nil>+ // and ParameterGroupStatus:<nil>. terraform-provider-aws reads both without a nil check,+ // and AWS returns both on every available node.+ when(service.findCacheClusters("tf-redis")).thenReturn(List.of(redisCacheCluster("tf-redis")));++ MultivaluedMap<String, String> p = params();+ p.add("CacheClusterId", "tf-redis");+ p.add("ShowCacheNodeInfo", "true");+ Response response = handler.handle("DescribeCacheClusters", p, "us-east-1");++ assertEquals(200, response.getStatus());+ String body = (String) response.getEntity();+ assertTrue(body.contains("<CacheNodeCreateTime>"), body);+ assertTrue(body.contains("<ParameterGroupStatus>in-sync</ParameterGroupStatus>"), body);+ // AWS reports the node's AZ as the cluster's; a mismatch is a permanent terraform diff+ assertTrue(body.contains("<CustomerAvailabilityZone>us-east-1b</CustomerAvailabilityZone>"), body);+ // the node was created with its cluster, so the two timestamps agree+ String nodeTime = between(body, "<CacheNodeCreateTime>", "</CacheNodeCreateTime>");+ String clusterTime = between(body, "<CacheClusterCreateTime>", "</CacheClusterCreateTime>");+ assertEquals(clusterTime, nodeTime, body);+ }++ private static String between(String body, String open, String close) {+ int from = body.indexOf(open);+ assertTrue(from >= 0, "missing " + open + " in " + body);+ return body.substring(from + open.length(), body.indexOf(close, from));+ }++ @Test+ void describeCacheClusters_stillRaisesNotFoundWhenNoSourceKnowsTheId() {+ when(service.findCacheClusters("absent")).thenReturn(List.of());+ when(memcachedService.listCacheClusters("absent")).thenThrow(+ new AwsException("CacheClusterNotFound", "Cache cluster absent not found.", 404));++ MultivaluedMap<String, String> p = params();+ p.add("CacheClusterId", "absent");++ assertEquals(404, handler.handle("DescribeCacheClusters", p, "us-east-1").getStatus());+ }++ @Test+ void deleteCacheCluster_routesToTheServiceThatHoldsTheCluster() {+ CacheCluster cluster = redisCacheCluster("tf-redis");+ when(service.findCacheClusters("tf-redis")).thenReturn(List.of(cluster));+ when(service.deleteCacheCluster("tf-redis")).thenReturn(cluster);++ MultivaluedMap<String, String> p = params();+ p.add("CacheClusterId", "tf-redis");++ assertEquals(200, handler.handle("DeleteCacheCluster", p, "us-east-1").getStatus());+ verify(service).deleteCacheCluster("tf-redis");+ verify(memcachedService, never()).deleteCacheCluster(anyString());+ }++ @Test+ void listTagsForResource_readsTagsOffACacheClusterArn() {+ CacheCluster cluster = redisCacheCluster("tf-redis");+ cluster.setTags(new LinkedHashMap<>(Map.of("Name", "cache")));+ when(service.findCacheClusters("tf-redis")).thenReturn(List.of(cluster));++ MultivaluedMap<String, String> p = params();+ p.add("ResourceName", "arn:aws:elasticache:us-east-1:000000000000:cluster:tf-redis");++ String body = (String) handler.handle("ListTagsForResource", p, "us-east-1").getEntity();+ assertTrue(body.contains("<Key>Name</Key>"), body);+ assertTrue(body.contains("<Value>cache</Value>"), body);+ }++ @Test+ void listTagsForResource_clusterArnOfAnUnknownIdIsNotFound() {+ when(service.findCacheClusters("absent")).thenReturn(List.of());+ when(memcachedService.getCacheCluster("absent")).thenThrow(+ new AwsException("CacheClusterNotFound", "Cache cluster absent not found.", 404));++ MultivaluedMap<String, String> p = params();+ p.add("ResourceName", "arn:aws:elasticache:us-east-1:000000000000:cluster:absent");++ Response response = handler.handle("ListTagsForResource", p, "us-east-1");+ assertEquals(404, response.getStatus());+ assertTrue(((String) response.getEntity()).contains("CacheClusterNotFound"));+ }++ @Test+ void listTagsForResource_clusterArnFromAnotherRegionIsNotThisCluster() {+ // the store keys clusters by id alone, so a same-named cluster created under another+ // region is not the one this ARN names+ CacheCluster elsewhere = redisCacheCluster("tf-redis");+ elsewhere.setArn("arn:aws:elasticache:eu-west-1:000000000000:cluster:tf-redis");+ elsewhere.setTags(new LinkedHashMap<>(Map.of("Name", "west")));+ when(service.findCacheClusters("tf-redis")).thenReturn(List.of(elsewhere));++ MultivaluedMap<String, String> p = params();+ p.add("ResourceName", "arn:aws:elasticache:us-east-1:000000000000:cluster:tf-redis");++ Response response = handler.handle("ListTagsForResource", p, "us-east-1");+ assertEquals(404, response.getStatus(), (String) response.getEntity());+ assertFalse(((String) response.getEntity()).contains("west"));+ }++ @Test+ void listTagsForResource_memcachedClusterArnStillAnswersWithNoTags() {+ // Memcached clusters carry no tags but they do exist: this must not become a 404.+ when(service.findCacheClusters("mc")).thenReturn(List.of());+ when(service.listMemberCacheClusters("mc")).thenReturn(List.of());+ when(memcachedService.getCacheCluster("mc")).thenReturn(new CacheCluster(+ "mc", CacheClusterStatus.AVAILABLE, "memcached", "1.6.22",+ new Endpoint("localhost", 11211), Instant.now()));++ MultivaluedMap<String, String> p = params();+ p.add("ResourceName", "arn:aws:elasticache:us-east-1:000000000000:cluster:mc");++ Response response = handler.handle("ListTagsForResource", p, "us-east-1");+ assertEquals(200, response.getStatus());+ assertTrue(((String) response.getEntity()).contains("<TagList></TagList>"),+ (String) response.getEntity());+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheRedisClusterIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheRedisClusterIntegrationTest.javanew file mode 100644index 0000000000..07ada5a24b--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheRedisClusterIntegrationTest.java@@ -0,0 +1,242 @@+package io.github.hectorvent.floci.services.elasticache;++import io.quarkus.test.junit.QuarkusTest;+import io.restassured.path.xml.XmlPath;+import org.junit.jupiter.api.AfterAll;+import org.junit.jupiter.api.Assumptions;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.MethodOrderer;+import org.junit.jupiter.api.Order;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.TestMethodOrder;++import java.io.IOException;+import java.io.InputStream;+import java.io.OutputStream;+import java.net.Socket;+import java.nio.charset.StandardCharsets;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.notNullValue;+import static org.junit.jupiter.api.Assertions.assertEquals;++/**+ * End-to-end coverage for a single-node redis cache cluster: the shape+ * {@code aws_elasticache_cluster} with {@code engine = "redis"} creates, which AWS serves through+ * CreateCacheCluster rather than CreateReplicationGroup. The node endpoint the describe reports+ * has to speak RESP, which is what tells this apart from a metadata-only record.+ */+@QuarkusTest+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class ElastiCacheRedisClusterIntegrationTest {++ private static final String AUTH_HEADER =+ "AWS4-HMAC-SHA256 Credential=test/20260412/us-east-1/elasticache/aws4_request";+ private static final String CLUSTER_ID = "it-redis-single";+ private static final int SOCKET_TIMEOUT_MS = 10_000;++ private static String nodeHost;+ private static int nodePort;++ @BeforeAll+ static void requireDocker() {+ Assumptions.assumeTrue(isDockerAvailable(), "Docker daemon must be available for ElastiCache integration tests");+ }++ @AfterAll+ static void cleanup() {+ try {+ given()+ .formParam("Action", "DeleteCacheCluster")+ .formParam("CacheClusterId", CLUSTER_ID)+ .header("Authorization", AUTH_HEADER)+ .post("/");+ } catch (Exception e) {+ System.err.println("Cleanup of " + CLUSTER_ID + " failed: " + e.getMessage());+ }+ }++ @Test+ @Order(1)+ void createSingleNodeRedisCacheCluster() {+ given()+ .formParam("Action", "CreateCacheCluster")+ .formParam("CacheClusterId", CLUSTER_ID)+ .formParam("Engine", "redis")+ .formParam("EngineVersion", "7.1")+ .formParam("NumCacheNodes", "1")+ .formParam("CacheNodeType", "cache.t4g.micro")+ .formParam("SnapshotRetentionLimit", "5")+ .formParam("SnapshotWindow", "03:00-05:00")+ .formParam("PreferredMaintenanceWindow", "tue:04:00-tue:05:00")+ .header("Authorization", AUTH_HEADER)+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .contentType("application/xml")+ .body("CreateCacheClusterResponse.CreateCacheClusterResult.CacheCluster.CacheClusterId", equalTo(CLUSTER_ID))+ .body("CreateCacheClusterResponse.CreateCacheClusterResult.CacheCluster.CacheClusterStatus", equalTo("available"))+ .body("CreateCacheClusterResponse.CreateCacheClusterResult.CacheCluster.Engine", equalTo("redis"))+ .body("CreateCacheClusterResponse.CreateCacheClusterResult.CacheCluster.NumCacheNodes", equalTo("1"));+ }++ @Test+ @Order(2)+ void describeReportsTheClusterWithItsNodeEndpoint() {+ XmlPath describe =+ given()+ .formParam("Action", "DescribeCacheClusters")+ .formParam("CacheClusterId", CLUSTER_ID)+ .formParam("ShowCacheNodeInfo", "true")+ .header("Authorization", AUTH_HEADER)+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("DescribeCacheClustersResponse.DescribeCacheClustersResult.CacheClusters.CacheCluster.Engine",+ equalTo("redis"))+ .body("DescribeCacheClustersResponse.DescribeCacheClustersResult.CacheClusters.CacheCluster.EngineVersion",+ equalTo("7.1"))+ .body("DescribeCacheClustersResponse.DescribeCacheClustersResult.CacheClusters.CacheCluster.CacheNodeType",+ equalTo("cache.t4g.micro"))+ .body("DescribeCacheClustersResponse.DescribeCacheClustersResult.CacheClusters.CacheCluster.CacheNodes.CacheNode.Endpoint.Address",+ notNullValue())+ // read back exactly as sent: an optional argument that comes back unset is a+ // terraform diff that never settles+ .body("DescribeCacheClustersResponse.DescribeCacheClustersResult.CacheClusters.CacheCluster.SnapshotRetentionLimit",+ equalTo("5"))+ .body("DescribeCacheClustersResponse.DescribeCacheClustersResult.CacheClusters.CacheCluster.SnapshotWindow",+ equalTo("03:00-05:00"))+ .body("DescribeCacheClustersResponse.DescribeCacheClustersResult.CacheClusters.CacheCluster.PreferredMaintenanceWindow",+ equalTo("tue:04:00-tue:05:00"))+ .extract()+ .xmlPath();++ nodeHost = describe.getString(+ "DescribeCacheClustersResponse.DescribeCacheClustersResult.CacheClusters.CacheCluster.CacheNodes.CacheNode.Endpoint.Address");+ nodePort = describe.getInt(+ "DescribeCacheClustersResponse.DescribeCacheClustersResult.CacheClusters.CacheCluster.CacheNodes.CacheNode.Endpoint.Port");+ }++ @Test+ @Order(3)+ void theReportedNodeEndpointSpeaksResp() throws Exception {+ try (Socket socket = openSocket(nodeHost, nodePort)) {+ write(socket, respArray("PING"));+ assertEquals("+PONG\r\n", readLine(socket));++ write(socket, respArray("SET", "single-node-key", "a-value"));+ assertEquals("+OK\r\n", readLine(socket));++ write(socket, respArray("GET", "single-node-key"));+ assertEquals("a-value", readBulk(socket));+ }+ }++ @Test+ @Order(4)+ void moreThanOneNodeIsRefused() {+ given()+ .formParam("Action", "CreateCacheCluster")+ .formParam("CacheClusterId", "it-redis-two-nodes")+ .formParam("Engine", "redis")+ .formParam("NumCacheNodes", "2")+ .header("Authorization", AUTH_HEADER)+ .when()+ .post("/")+ .then()+ .statusCode(400)+ .body("ErrorResponse.Error.Code", equalTo("InvalidParameterValue"))+ .body("ErrorResponse.Error.Message", equalTo("NumCacheNodes should be 1 if engine is redis"));+ }++ @Test+ @Order(5)+ void deleteCacheClusterRemovesItAndFreesTheEndpoint() {+ given()+ .formParam("Action", "DeleteCacheCluster")+ .formParam("CacheClusterId", CLUSTER_ID)+ .header("Authorization", AUTH_HEADER)+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .body("DeleteCacheClusterResponse.DeleteCacheClusterResult.CacheCluster.CacheClusterId", equalTo(CLUSTER_ID));++ given()+ .formParam("Action", "DescribeCacheClusters")+ .formParam("CacheClusterId", CLUSTER_ID)+ .header("Authorization", AUTH_HEADER)+ .when()+ .post("/")+ .then()+ .statusCode(404);+ }++ private static boolean isDockerAvailable() {+ try {+ Process process = new ProcessBuilder("docker", "info").start();+ return process.waitFor() == 0;+ } catch (Exception e) {+ return false;+ }+ }++ private static Socket openSocket(String host, int port) throws IOException {+ Socket socket = new Socket(host, port);+ socket.setTcpNoDelay(true);+ socket.setSoTimeout(SOCKET_TIMEOUT_MS);+ return socket;+ }++ private static void write(Socket socket, byte[] data) throws IOException {+ OutputStream out = socket.getOutputStream();+ out.write(data);+ out.flush();+ }++ private static byte[] respArray(String... args) {+ StringBuilder sb = new StringBuilder();+ sb.append('*').append(args.length).append("\r\n");+ for (String arg : args) {+ byte[] bytes = arg.getBytes(StandardCharsets.UTF_8);+ sb.append('$').append(bytes.length).append("\r\n").append(arg).append("\r\n");+ }+ return sb.toString().getBytes(StandardCharsets.UTF_8);+ }++ private static String readLine(Socket socket) throws IOException {+ InputStream in = socket.getInputStream();+ StringBuilder sb = new StringBuilder();+ int b;+ while ((b = in.read()) != -1) {+ sb.append((char) b);+ if (sb.length() >= 2 && sb.charAt(sb.length() - 2) == '\r' && sb.charAt(sb.length() - 1) == '\n') {+ return sb.toString();+ }+ }+ throw new IOException("Connection closed before line terminator; read so far: " + sb);+ }++ private static String readBulk(Socket socket) throws IOException {+ String header = readLine(socket);+ if (!header.startsWith("$")) {+ throw new IOException("Expected bulk reply but got: " + header);+ }+ int length = Integer.parseInt(header.substring(1).trim());+ if (length < 0) {+ return null;+ }+ InputStream in = socket.getInputStream();+ byte[] data = in.readNBytes(length);+ if (data.length != length) {+ throw new IOException("Truncated bulk reply");+ }+ if (in.read() != '\r' || in.read() != '\n') {+ throw new IOException("Missing CRLF after bulk reply");+ }+ return new String(data, StandardCharsets.UTF_8);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.javaindex f1b15e8143..c7d576d8c8 100644--- a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java@@ -1,5 +1,6 @@package io.github.hectorvent.floci.services.elasticache;+import com.fasterxml.jackson.core.type.TypeReference;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;@@ -10,9 +11,13 @@ import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.ec2.Ec2Service;import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerHandle;import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerManager;+import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheMemcachedContainerManager;import io.github.hectorvent.floci.services.elasticache.container.ValkeyClusterFormation;import io.github.hectorvent.floci.services.elasticache.model.AuthMode;+import io.github.hectorvent.floci.services.elasticache.model.CacheCluster;+import io.github.hectorvent.floci.services.elasticache.model.CacheClusterStatus;import io.github.hectorvent.floci.services.elasticache.model.ClusterNode;+import io.github.hectorvent.floci.services.elasticache.model.Endpoint;import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroup;import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroupSettings;import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroupStatus;@@ -24,6 +29,7 @@ import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import org.mockito.ArgumentCaptor;+import java.time.Instant;import java.util.List;import java.util.Map;import java.util.Optional;@@ -60,12 +66,17 @@ class ElastiCacheServiceTest {private ElastiCacheProxyManager proxyManager;private EmulatorConfig config;private ValkeyClusterFormation clusterFormation;+ private StorageFactory storageFactory;+ private ElastiCacheMemcachedContainerManager memcachedContainerManager;+ private Ec2Service ec2Service;+ private ElastiCacheProvisioningIds provisioningIds;@BeforeEachvoid setUp() {containerManager = mock(ElastiCacheContainerManager.class);proxyManager = mock(ElastiCacheProxyManager.class);- StorageFactory storageFactory = mock(StorageFactory.class);+ provisioningIds = new ElastiCacheProvisioningIds();+ storageFactory = mock(StorageFactory.class);config = mock(EmulatorConfig.class);EmulatorConfig.ServicesConfig servicesConfig = mock(EmulatorConfig.ServicesConfig.class);@@ -75,20 +86,28 @@ class ElastiCacheServiceTest {when(ecConfig.proxyBasePort()).thenReturn(16379);when(ecConfig.proxyMaxPort()).thenReturn(16399);when(ecConfig.defaultImage()).thenReturn("valkey/valkey:8");+ when(ecConfig.defaultMemcachedImage()).thenReturn("memcached:1.6");when(config.hostname()).thenReturn(Optional.of("localhost"));- when(storageFactory.create(anyString(), anyString(), any())).thenAnswer(inv -> AccountAwareStorageBackend.inMemory("000000000000"));+ // Keyed by file name, as the real StorageFactory is: it hands the second caller of a path+ // the first caller's backend, which is what lets the two services share one id namespace.+ Map<String, AccountAwareStorageBackend<?>> backendsByFile = new ConcurrentHashMap<>();+ when(storageFactory.create(anyString(), anyString(), any())).thenAnswer(inv ->+ backendsByFile.computeIfAbsent(inv.getArgument(1),+ file -> AccountAwareStorageBackend.inMemory("000000000000")));+ memcachedContainerManager = mock(ElastiCacheMemcachedContainerManager.class);+ when(memcachedContainerManager.tryStart(anyString(), anyString())).thenReturn(null);when(containerManager.tryStart(anyString(), anyString())).thenReturn(new ElastiCacheContainerHandle("cid", "grp", "localhost", 6379));doNothing().when(proxyManager).startProxy(anyString(), any(), anyInt(), anyString(), anyInt(), any());- Ec2Service ec2Service = mock(Ec2Service.class);+ ec2Service = mock(Ec2Service.class);kmsService = mock(KmsService.class);when(kmsService.describeKey(any(), any())).thenThrow(new AwsException("NotFoundException", "Key not found", 404));clusterFormation = mock(ValkeyClusterFormation.class);service = new ElastiCacheService(containerManager, proxyManager, clusterFormation,storageFactory, config, ec2Service, new RegionResolver("us-east-1", "000000000000"),- kmsService);+ kmsService, provisioningIds);}@Test@@ -116,7 +135,7 @@ class ElastiCacheServiceTest {ElastiCacheService svc = new ElastiCacheService(cm, pm, mock(ValkeyClusterFormation.class),sf, cfg, mock(Ec2Service.class),new RegionResolver("us-east-1", "000000000000"),- mock(KmsService.class));+ mock(KmsService.class), provisioningIds);svc.createReplicationGroup("g1", "d", AuthMode.PASSWORD, null, "us-east-1");@@ -474,7 +493,7 @@ class ElastiCacheServiceTest {when(config.hostname()).thenReturn(Optional.of("localhost"));return new ElastiCacheService(containerManager, proxyManager, clusterFormation,storageFactory, config, mock(Ec2Service.class),- new RegionResolver("us-east-1", "000000000000"), mock(KmsService.class));+ new RegionResolver("us-east-1", "000000000000"), mock(KmsService.class), new ElastiCacheProvisioningIds());}private static void stubPerNodeContainers(ElastiCacheContainerManager containerManager) {@@ -874,7 +893,7 @@ class ElastiCacheServiceTest {.thenAnswer(inv -> AccountAwareStorageBackend.inMemory("000000000000"));ElastiCacheService svc = new ElastiCacheService(containerManager, proxyManager, clusterFormation,factory, config, mock(Ec2Service.class),- new RegionResolver("us-east-1", "000000000000"), kmsService);+ new RegionResolver("us-east-1", "000000000000"), kmsService, provisioningIds);svc.createReplicationGroup("g1", "d", AuthMode.NO_AUTH, null, "us-east-1");pausing.pauseOn(PausingStorageBackend.Call.PUT, "g1");@@ -993,7 +1012,7 @@ class ElastiCacheServiceTest {new AccountAwareStorageBackend<>(new InMemoryStorage<>(), requestContextInstance, "000000000000"));ElastiCacheService svc = new ElastiCacheService(containerManager, proxyManager, clusterFormation,factory, config, mock(Ec2Service.class),- new RegionResolver("us-east-1", "000000000000"), kmsService);+ new RegionResolver("us-east-1", "000000000000"), kmsService, provisioningIds);CountDownLatch startedLatch = new CountDownLatch(1);CountDownLatch releaseLatch = new CountDownLatch(1);@@ -1054,4 +1073,638 @@ class ElastiCacheServiceTest {service.deleteCacheParameterGroup("custom-pg");assertTrue(service.findParameterGroup("custom-pg").isEmpty());}++ // ── CreateCacheCluster: single-node redis/valkey ──────────────────────────++ private static ElastiCacheService.CreateCacheClusterRequest cacheClusterRequest(+ String clusterId, String engine, Integer numCacheNodes) {+ return cacheClusterRequest(clusterId, engine, numCacheNodes, AuthMode.NO_AUTH, null, null);+ }++ private static ElastiCacheService.CreateCacheClusterRequest cacheClusterRequest(+ String clusterId, String engine, Integer numCacheNodes, AuthMode authMode,+ String authToken, String parameterGroupName) {+ return new ElastiCacheService.CreateCacheClusterRequest(clusterId, engine, null, null,+ numCacheNodes, null, authMode, authToken, parameterGroupName, null,+ null, null, null, null, null, null, null, null, "us-east-1", Map.of());+ }++ @Test+ void singleNodeRedisClusterIsBackedByAValkeyContainerBehindAProxy() {+ // The point of the path: terraform's aws_elasticache_cluster with engine "redis" sends+ // this call, and the endpoint it reads back has to answer. That means a container and a+ // proxy on the port the describe reports, not a metadata-only record.+ CacheCluster cluster = service.createCacheCluster(cacheClusterRequest("tf-redis", "redis", 1));++ assertEquals("redis", cluster.getEngine());+ assertEquals("7.1", cluster.getEngineVersion());+ assertEquals("cache.t4g.micro", cluster.getCacheNodeType());+ assertEquals(1, cluster.getNumCacheNodes());+ assertEquals(CacheClusterStatus.AVAILABLE, cluster.getCacheClusterStatus());+ assertEquals("localhost", cluster.getConfigurationEndpoint().address());+ assertEquals(16379, cluster.getConfigurationEndpoint().port());+ assertEquals("arn:aws:elasticache:us-east-1:000000000000:cluster:tf-redis", cluster.getArn());++ verify(containerManager).tryStart(eq("tf-redis"), eq("valkey/valkey:8"));+ verify(proxyManager).startProxy(eq("tf-redis"), eq(AuthMode.NO_AUTH), eq(16379),+ eq("localhost"), eq(6379), any());++ // read back through a separate call, not the create's own return value+ assertEquals("tf-redis", service.findCacheClusters("tf-redis").getFirst().getCacheClusterId());+ }++ @Test+ void valkeyCacheClusterTakesTheValkeyEngineVersionDefault() {+ assertEquals("8.1", service.createCacheCluster(cacheClusterRequest("tf-valkey", "valkey", null))+ .getEngineVersion());+ }++ @Test+ void redisCacheClusterWithMoreThanOneNodeIsRefusedAsOnAws() {+ AwsException ex = assertThrows(AwsException.class,+ () -> service.createCacheCluster(cacheClusterRequest("too-big", "redis", 2)));++ assertEquals("InvalidParameterValue", ex.getErrorCode());+ assertEquals("NumCacheNodes should be 1 if engine is redis", ex.getMessage());+ verify(containerManager, never()).tryStart(eq("too-big"), anyString());+ assertTrue(service.findCacheClusters("too-big").isEmpty());+ }++ @Test+ void duplicateCacheClusterIdIsRefused() {+ service.createCacheCluster(cacheClusterRequest("dupe", "redis", 1));++ AwsException ex = assertThrows(AwsException.class,+ () -> service.createCacheCluster(cacheClusterRequest("dupe", "redis", 1)));+ // botocore elasticache/2015-02-02 codes this CacheClusterAlreadyExists: the Fault suffix+ // is the shape name, not the wire code, and an SDK matches on the code.+ assertEquals("CacheClusterAlreadyExists", ex.getErrorCode());+ }++ @Test+ void deletingACacheClusterStopsItsProxyAndContainerAndFreesThePort() {+ service.createCacheCluster(cacheClusterRequest("cc", "redis", 1));++ service.deleteCacheCluster("cc");++ verify(proxyManager).stopProxy("cc");+ verify(containerManager).stop(any());+ assertTrue(service.findCacheClusters("cc").isEmpty());+ assertEquals("CacheClusterNotFound",+ assertThrows(AwsException.class, () -> service.deleteCacheCluster("cc")).getErrorCode());++ // the freed proxy port goes to the next cluster rather than being leaked+ assertEquals(16379, service.createCacheCluster(cacheClusterRequest("cc2", "redis", 1))+ .getConfigurationEndpoint().port());+ }++ @Test+ void aCacheClusterAndAReplicationGroupNeverShareAProxyPort() {+ service.createReplicationGroup("grp", "d", AuthMode.NO_AUTH, null, "us-east-1");++ assertEquals(16380, service.createCacheCluster(cacheClusterRequest("cc", "redis", 1))+ .getConfigurationEndpoint().port());+ }++ @Test+ void aCacheClusterHoldsItsParameterGroupAgainstDeletion() {+ service.createCacheParameterGroup("cc-pg", "redis7", "in use", Map.of());+ service.createCacheCluster(+ cacheClusterRequest("cc", "redis", 1, AuthMode.NO_AUTH, null, "cc-pg"));++ assertEquals("InvalidCacheParameterGroupState",+ assertThrows(AwsException.class, () -> service.deleteCacheParameterGroup("cc-pg"))+ .getErrorCode());+ }++ @Test+ void anAuthTokenOnACacheClusterIsValidatedAgainstThatClusterAlone() {+ service.createCacheCluster(+ cacheClusterRequest("auth-cc", "redis", 1, AuthMode.PASSWORD, "s3cret-token", null));++ assertTrue(service.validateCacheClusterPassword("auth-cc", null, "s3cret-token"));+ assertFalse(service.validateCacheClusterPassword("auth-cc", null, "wrong"));+ assertFalse(service.validateCacheClusterPassword("other-cc", null, "s3cret-token"));+ }++ @Test+ void aCacheClusterCannotTakeTheIdOfALiveReplicationGroup() {+ // Not a cosmetic clash. Both name their container valkey-<id> and register their proxy+ // under the id, so letting this through would have ElastiCacheContainerManager.start+ // removeIfExists the group's running container and the proxy registry overwrite its+ // entry, leaving a listener bound that nothing can stop.+ service.createReplicationGroup("shared", "d", AuthMode.NO_AUTH, null, "us-east-1");++ AwsException ex = assertThrows(AwsException.class,+ () -> service.createCacheCluster(cacheClusterRequest("shared", "redis", 1)));++ assertEquals("CacheClusterAlreadyExists", ex.getErrorCode());+ // the live group kept its container and its proxy: nothing was started or removed for the+ // refused request+ verify(containerManager, times(1)).tryStart(eq("shared"), anyString());+ verify(containerManager, never()).stopByGroupId("shared");+ verify(proxyManager, times(1)).startProxy(eq("shared"), any(), anyInt(), anyString(), anyInt(), any());+ assertEquals("shared", service.getReplicationGroup("shared").getReplicationGroupId());+ }++ @Test+ void aReplicationGroupCannotTakeTheIdOfALiveCacheCluster() {+ service.createCacheCluster(cacheClusterRequest("shared", "redis", 1));++ AwsException ex = assertThrows(AwsException.class,+ () -> service.createReplicationGroup("shared", "d", AuthMode.NO_AUTH, null, "us-east-1"));++ assertEquals("ReplicationGroupAlreadyExistsFault", ex.getErrorCode());+ verify(containerManager, times(1)).tryStart(eq("shared"), anyString());+ verify(containerManager, never()).stopByGroupId("shared");+ assertEquals("shared", service.findCacheClusters("shared").getFirst().getCacheClusterId());+ }++ @Test+ void twoConcurrentCreatesOfOneIdCannotBothProvisionIt() throws Exception {+ // The stored-record check alone cannot separate them: neither create has stored anything+ // while the other is inside tryStart, so both would pass it, and the loser's rollback+ // would stopByGroupId the winner's container out from under it.+ CountDownLatch startedLatch = new CountDownLatch(1);+ CountDownLatch releaseLatch = new CountDownLatch(1);+ when(containerManager.tryStart(eq("raced"), anyString())).thenAnswer(inv -> {+ startedLatch.countDown();+ assertTrue(releaseLatch.await(5, TimeUnit.SECONDS), "test timed out waiting for release");+ return new ElastiCacheContainerHandle("cid", "raced", "localhost", 6379);+ });++ Thread winner = new Thread(() -> service.createCacheCluster(cacheClusterRequest("raced", "redis", 1)));+ winner.start();+ assertTrue(startedLatch.await(5, TimeUnit.SECONDS), "create never reached container start");++ AwsException ex = assertThrows(AwsException.class,+ () -> service.createCacheCluster(cacheClusterRequest("raced", "redis", 1)));+ assertEquals("CacheClusterAlreadyExists", ex.getErrorCode());+ // the loser must not have reached for the container the winner is still starting+ verify(containerManager, never()).stopByGroupId("raced");++ releaseLatch.countDown();+ winner.join(5000);+ assertEquals("raced", service.findCacheClusters("raced").getFirst().getCacheClusterId());+ verify(containerManager, times(1)).tryStart(eq("raced"), anyString());+ }++ @Test+ void aReplicationGroupCreateIsBlockedByAnInFlightCacheClusterCreateOfThatId() throws Exception {+ CountDownLatch startedLatch = new CountDownLatch(1);+ CountDownLatch releaseLatch = new CountDownLatch(1);+ when(containerManager.tryStart(eq("raced"), anyString())).thenAnswer(inv -> {+ startedLatch.countDown();+ assertTrue(releaseLatch.await(5, TimeUnit.SECONDS), "test timed out waiting for release");+ return new ElastiCacheContainerHandle("cid", "raced", "localhost", 6379);+ });++ Thread cacheCluster = new Thread(() -> service.createCacheCluster(cacheClusterRequest("raced", "redis", 1)));+ cacheCluster.start();+ assertTrue(startedLatch.await(5, TimeUnit.SECONDS), "create never reached container start");++ assertEquals("ReplicationGroupAlreadyExistsFault",+ assertThrows(AwsException.class, () -> service.createReplicationGroup(+ "raced", "d", AuthMode.NO_AUTH, null, "us-east-1")).getErrorCode());+ verify(containerManager, never()).stopByGroupId("raced");++ releaseLatch.countDown();+ cacheCluster.join(5000);+ }++ @Test+ void theOptionalMembersARequestCarriesAreStoredAndTheRestDefaulted() {+ // Every one of these is an optional aws_elasticache_cluster argument. Dropping any of them+ // reads back as unset on the next plan, which is a diff terraform can never settle.+ CacheCluster cluster = service.createCacheCluster(new ElastiCacheService.CreateCacheClusterRequest(+ "settings-cc", "redis", null, null, 1, null, AuthMode.NO_AUTH, null, null, null,+ 5, "03:00-05:00", "Tue:04:00-Tue:05:00", "us-east-1b", List.of("sg-123"),+ "ipv4", "ipv4", true, "us-east-1", Map.of()));++ assertEquals(5, cluster.getSnapshotRetentionLimit());+ assertEquals("03:00-05:00", cluster.getSnapshotWindow());+ assertEquals("tue:04:00-tue:05:00", cluster.getPreferredMaintenanceWindow());+ assertEquals("us-east-1b", cluster.getPreferredAvailabilityZone());+ assertEquals(List.of("sg-123"), cluster.getSecurityGroupIds());+ assertTrue(cluster.isAtRestEncryptionEnabled());++ // and a request that carries none of them still reads back concrete values, not zeroes+ CacheCluster bare = service.createCacheCluster(cacheClusterRequest("bare-cc", "redis", 1));+ assertEquals(0, bare.getSnapshotRetentionLimit());+ assertEquals("00:00-01:00", bare.getSnapshotWindow());+ assertEquals("mon:00:00-mon:03:00", bare.getPreferredMaintenanceWindow());+ assertEquals("us-east-1a", bare.getPreferredAvailabilityZone());+ assertEquals("ipv4", bare.getNetworkType());+ assertEquals("ipv4", bare.getIpDiscovery());+ assertFalse(bare.isAtRestEncryptionEnabled());+ }++ @Test+ void theSnapshotMembersTakeTheReplicationGroupsChecks() {+ AwsException ex = assertThrows(AwsException.class,+ () -> service.createCacheCluster(new ElastiCacheService.CreateCacheClusterRequest(+ "bad-cc", "redis", null, null, 1, null, AuthMode.NO_AUTH, null, null, null,+ 99, null, null, null, null, null, null, null, "us-east-1", Map.of())));+ assertEquals("InvalidParameterValue", ex.getErrorCode());+ assertTrue(ex.getMessage().contains("Retention limit must be between 0 and 35"), ex.getMessage());++ assertEquals("InvalidParameterValue",+ assertThrows(AwsException.class,+ () -> service.createCacheCluster(new ElastiCacheService.CreateCacheClusterRequest(+ "bad-cc", "redis", null, null, 1, null, AuthMode.NO_AUTH, null, null, null,+ null, null, "notaday:04:00-notaday:05:00", null, null, null, null, null,+ "us-east-1", Map.of()))).getErrorCode());++ // a refused request provisions nothing+ verify(containerManager, never()).tryStart(eq("bad-cc"), anyString());+ assertTrue(service.findCacheClusters("bad-cc").isEmpty());+ }++ @Test+ void aStandaloneClusterIsListedAsAnExplorerResource() {+ service.createCacheCluster(cacheClusterRequest("explorer-cc", "redis", 1));++ assertTrue(service.getResources().stream().anyMatch(r ->+ "arn:aws:elasticache:us-east-1:000000000000:cluster:explorer-cc".equals(r.arn())),+ "the standalone cluster must appear in the resource explorer alongside groups");+ }++ /** A second service over the same stores: what a restart leaves behind, minus the runtime. */+ private ElastiCacheService serviceAfterRestart() {+ return new ElastiCacheService(containerManager, proxyManager, clusterFormation,+ storageFactory, config, ec2Service, new RegionResolver("us-east-1", "000000000000"),+ kmsService, provisioningIds);+ }++ private ElastiCacheMemcachedService memcachedService() {+ return new ElastiCacheMemcachedService(memcachedContainerManager, storageFactory, config, provisioningIds);+ }++ @Test+ void aPersistedClusterKeepsItsPortAcrossARestart() {+ // Before this, nothing re-added the port to usedPorts on startup: the restored cluster+ // went on advertising 16379 while the next create was handed the same 16379, and the+ // first delete then freed the second cluster's reservation.+ assertEquals(16379, service.createCacheCluster(cacheClusterRequest("persisted", "redis", 1))+ .getConfigurationEndpoint().port());++ ElastiCacheService restarted = serviceAfterRestart();+ restarted.restorePersistedRuntime().join();++ assertEquals(16380, restarted.createCacheCluster(cacheClusterRequest("after", "redis", 1))+ .getConfigurationEndpoint().port());+ assertEquals(16379, restarted.findCacheClusters("persisted").getFirst()+ .getConfigurationEndpoint().port());+ }++ private static StorageFactory storageWithStandaloneCluster(String clusterId) {+ StorageFactory storageFactory = sharedStorageFactory();+ ElastiCacheContainerManager beforeRestart = mock(ElastiCacheContainerManager.class);+ stubSingleNodeContainer(beforeRestart);+ serviceWith(storageFactory, beforeRestart, mock(ElastiCacheProxyManager.class),+ mock(ValkeyClusterFormation.class))+ .createCacheCluster(cacheClusterRequest(clusterId, "redis", 1, AuthMode.PASSWORD,+ "a-long-enough-auth-token", null));+ return storageFactory;+ }++ @Test+ void restorePersistedRuntimeReprovisionsStandaloneCacheClusters() {+ // The shape aws_elasticache_cluster actually creates. Left unrestored it is the exact+ // state #4094 fixed for every other shape: available with nothing behind the endpoint.+ StorageFactory storageFactory = storageWithStandaloneCluster("tf-redis");++ ElastiCacheContainerManager restartedContainers = mock(ElastiCacheContainerManager.class);+ stubSingleNodeContainer(restartedContainers);+ ElastiCacheProxyManager restartedProxies = mock(ElastiCacheProxyManager.class);+ ElastiCacheService restarted = serviceWith(storageFactory, restartedContainers,+ restartedProxies, mock(ValkeyClusterFormation.class));++ restarted.restorePersistedRuntime().join();++ verify(restartedContainers).tryStart(eq("tf-redis"), anyString());+ verify(restartedProxies).startProxy(eq("tf-redis"), eq(AuthMode.PASSWORD), eq(16379),+ eq("localhost"), eq(6379), any());+ CacheCluster restored = restarted.findCacheClusters("tf-redis").getFirst();+ assertEquals(CacheClusterStatus.AVAILABLE, restored.getCacheClusterStatus());+ assertEquals(16379, restored.getConfigurationEndpoint().port());+ assertEquals("cid-tf-redis", restored.getContainerId(),+ "A restored cluster must track the container it actually has");+ }++ @Test+ void standaloneClusterRestoreFailureReportsRestoreFailedAndReleasesThePort() {+ StorageFactory storageFactory = storageWithStandaloneCluster("tf-redis");++ ElastiCacheContainerManager restartedContainers = mock(ElastiCacheContainerManager.class);+ // Only the restore fails: the create that checks the port was freed must still get through.+ when(restartedContainers.tryStart(eq("tf-redis"), anyString()))+ .thenThrow(new RuntimeException("container failed"));+ ElastiCacheProxyManager restartedProxies = mock(ElastiCacheProxyManager.class);+ ElastiCacheService restarted = serviceWith(storageFactory, restartedContainers,+ restartedProxies, mock(ValkeyClusterFormation.class));++ restarted.restorePersistedRuntime().join();++ CacheCluster failed = restarted.findCacheClusters("tf-redis").getFirst();+ assertEquals(CacheClusterStatus.RESTORE_FAILED, failed.getCacheClusterStatus(),+ "restore-failed is the value CacheClusterStatus models; create-failed is not");+ assertNull(failed.getConfigurationEndpoint(),+ "A cluster whose data plane is gone must not advertise an endpoint");+ verify(restartedProxies, never()).startProxy(anyString(), any(), anyInt(), anyString(),+ anyInt(), any());++ CacheCluster next = restarted.createCacheCluster(cacheClusterRequest("next", "redis", 1));+ assertEquals(16379, next.getConfigurationEndpoint().port(),+ "The failed restore's port must be released for the next cluster");+ }++ @Test+ void standaloneClusterRestoreWithoutADockerDaemonKeepsTheClusterAvailable() {+ StorageFactory storageFactory = storageWithStandaloneCluster("tf-redis");++ ElastiCacheContainerManager restartedContainers = mock(ElastiCacheContainerManager.class);+ when(restartedContainers.tryStart(anyString(), anyString())).thenReturn(null);+ ElastiCacheProxyManager restartedProxies = mock(ElastiCacheProxyManager.class);+ ElastiCacheService restarted = serviceWith(storageFactory, restartedContainers,+ restartedProxies, mock(ValkeyClusterFormation.class));++ restarted.restorePersistedRuntime().join();++ CacheCluster restored = restarted.findCacheClusters("tf-redis").getFirst();+ assertEquals(CacheClusterStatus.AVAILABLE, restored.getCacheClusterStatus(),+ "No reachable daemon is the create path's documented degraded mode, not a failure");+ assertNull(restored.getContainerId());+ verify(restartedProxies, never()).startProxy(anyString(), any(), anyInt(), anyString(),+ anyInt(), any());+ }++ @Test+ void restorePersistedRuntimeSkipsStandaloneClustersBeingDeleted() {+ StorageFactory storageFactory = storageWithStandaloneCluster("tf-redis");+ ElastiCacheContainerManager beforeRestart = mock(ElastiCacheContainerManager.class);+ stubSingleNodeContainer(beforeRestart);+ ElastiCacheService before = serviceWith(storageFactory, beforeRestart,+ mock(ElastiCacheProxyManager.class), mock(ValkeyClusterFormation.class));+ // The in-memory backend hands back the stored instance, so this is the persisted record.+ before.findCacheClusters("tf-redis").getFirst()+ .setCacheClusterStatus(CacheClusterStatus.DELETING);++ ElastiCacheContainerManager restartedContainers = mock(ElastiCacheContainerManager.class);+ stubSingleNodeContainer(restartedContainers);+ ElastiCacheService restarted = serviceWith(storageFactory, restartedContainers,+ mock(ElastiCacheProxyManager.class), mock(ValkeyClusterFormation.class));++ restarted.restorePersistedRuntime().join();++ verify(restartedContainers, never()).tryStart(anyString(), anyString());+ assertEquals(CacheClusterStatus.DELETING,+ restarted.findCacheClusters("tf-redis").getFirst().getCacheClusterStatus());+ }++ @Test+ void restoreDoesNotResurrectAStandaloneClusterDeletedWhileItWasRestoring() {+ StorageFactory storageFactory = storageWithStandaloneCluster("tf-redis");++ ElastiCacheContainerManager restartedContainers = mock(ElastiCacheContainerManager.class);+ ElastiCacheProxyManager restartedProxies = mock(ElastiCacheProxyManager.class);+ ElastiCacheService restarted = serviceWith(storageFactory, restartedContainers,+ restartedProxies, mock(ValkeyClusterFormation.class));+ ElastiCacheContainerHandle restoredHandle =+ new ElastiCacheContainerHandle("cid-restored", "tf-redis", "localhost", 6379);+ // The delete lands in the window the cluster's monitor closes: the container is up, the+ // record has not been written back yet.+ when(restartedContainers.tryStart(eq("tf-redis"), anyString())).thenAnswer(inv -> {+ restarted.deleteCacheCluster("tf-redis");+ // Takes the port that delete just freed, so a restore that released it a second+ // time would hand the same port out twice.+ restarted.createCacheCluster(cacheClusterRequest("after", "redis", 1));+ return restoredHandle;+ });++ restarted.restorePersistedRuntime().join();++ assertTrue(restarted.findCacheClusters("tf-redis").isEmpty(),+ "A cluster deleted while it was restoring must stay deleted");+ verify(restartedProxies, never()).startProxy(eq("tf-redis"), any(), anyInt(), anyString(),+ anyInt(), any());+ verify(restartedContainers).stop(restoredHandle);++ CacheCluster next = restarted.createCacheCluster(cacheClusterRequest("third", "redis", 1));+ assertEquals(16380, next.getConfigurationEndpoint().port(),+ "The abandoned restore must leave 'after' holding the port the delete released");+ }++ @Test+ void deletingARestoredClusterFreesThePortItActuallyHolds() {+ service.createCacheCluster(cacheClusterRequest("persisted", "redis", 1));+ ElastiCacheService restarted = serviceAfterRestart();+ restarted.restorePersistedRuntime().join();+ restarted.createCacheCluster(cacheClusterRequest("after", "redis", 1));++ restarted.deleteCacheCluster("persisted");++ // 16379 is free again because that record owned it; 16380 is still the other cluster's+ assertEquals(16379, restarted.createCacheCluster(cacheClusterRequest("third", "redis", 1))+ .getConfigurationEndpoint().port());+ assertEquals(16380, restarted.findCacheClusters("after").getFirst()+ .getConfigurationEndpoint().port());+ }++ @Test+ void deletingARecordThatNeverHeldItsPortDoesNotFreeTheHoldersPort() {+ // Two records advertising one port: only one can hold it. Here the holder is the created+ // cluster and the other record never reserved anything, so its delete must leave the+ // reservation alone rather than hand a live cluster's port to the next create.+ service.createCacheCluster(cacheClusterRequest("holder", "redis", 1));+ AccountAwareStorageBackend<CacheCluster> store = storageFactory.create("elasticache",+ "elasticache-redis-clusters.json", new TypeReference<Map<String, CacheCluster>>() {});+ store.put("squatter", new CacheCluster("squatter", CacheClusterStatus.AVAILABLE, "redis",+ "7.1", new Endpoint("localhost", 16379), Instant.now()));++ service.deleteCacheCluster("squatter");++ assertEquals(16380, service.createCacheCluster(cacheClusterRequest("next", "redis", 1))+ .getConfigurationEndpoint().port(),+ "16379 is still held by the cluster that actually reserved it");+ }++ @Test+ void deletingAGroupThatNeverHeldItsPortDoesNotFreeTheHoldersPort() {+ // The same asymmetry on the replication-group side: a group written straight into the+ // store advertises 16379 without ever reserving it, so its delete must not free the+ // reservation the cluster is holding.+ service.createCacheCluster(cacheClusterRequest("holder", "redis", 1));+ AccountAwareStorageBackend<ReplicationGroup> store = storageFactory.create("elasticache",+ "elasticache-groups.json", new TypeReference<Map<String, ReplicationGroup>>() {});+ store.put("squatter", new ReplicationGroup("squatter", "d", ReplicationGroupStatus.AVAILABLE,+ AuthMode.NO_AUTH, new Endpoint("localhost", 16379), Instant.now(), 16379));++ service.deleteReplicationGroup("squatter");++ assertEquals(16380, service.createCacheCluster(cacheClusterRequest("next", "redis", 1))+ .getConfigurationEndpoint().port(),+ "16379 is still held by the cluster that actually reserved it");+ }++ @Test+ void deletingAGroupCreatedInProcessFreesItsPort() {+ // The ordinary path, which the squatter and restored-group tests both step around: a+ // group created here holds its port, so its delete has to give it back.+ service.createReplicationGroup("grp", "d", AuthMode.NO_AUTH, null, "us-east-1");++ service.deleteReplicationGroup("grp");++ assertEquals(16379, service.createCacheCluster(cacheClusterRequest("next", "redis", 1))+ .getConfigurationEndpoint().port());+ }++ @Test+ void deletingARestoredGroupFreesThePortItActuallyHolds() {+ service.createReplicationGroup("persisted", "d", AuthMode.NO_AUTH, null, "us-east-1");+ ElastiCacheService restarted = serviceAfterRestart();+ restarted.restorePersistedRuntime().join();+ restarted.createCacheCluster(cacheClusterRequest("after", "redis", 1));++ restarted.deleteReplicationGroup("persisted");++ // 16379 is free again because that group owned it; 16380 is still the cluster's.+ assertEquals(16379, restarted.createCacheCluster(cacheClusterRequest("third", "redis", 1))+ .getConfigurationEndpoint().port());+ }++ @Test+ void aReplicationGroupCannotTakeTheIdOfAMemcachedCluster() {+ // The third store counts too: CreateReplicationGroup would otherwise name a group after+ // a live memcached cluster and take over its container and proxy registration.+ memcachedService().createCacheCluster("shared-id");++ AwsException ex = assertThrows(AwsException.class, () -> service.createReplicationGroup(+ "shared-id", "d", AuthMode.NO_AUTH, null, "us-east-1"));++ assertEquals("ReplicationGroupAlreadyExistsFault", ex.getErrorCode());+ verify(containerManager, never()).tryStart(eq("shared-id"), anyString());+ }++ @Test+ void aPersistedReplicationGroupAlsoKeepsItsPortAcrossARestart() {+ service.createReplicationGroup("grp", "d", AuthMode.NO_AUTH, null, "us-east-1");++ ElastiCacheService restarted = serviceAfterRestart();+ restarted.restorePersistedRuntime().join();++ assertEquals(16380, restarted.createCacheCluster(cacheClusterRequest("after", "redis", 1))+ .getConfigurationEndpoint().port());+ }++ @Test+ void aCacheClusterCannotTakeTheIdOfAMemcachedCluster() {+ // One namespace: DescribeCacheClusters answers from every store, so two records sharing+ // an id would have it reported twice, each with a different engine.+ memcachedService().createCacheCluster("shared-id");++ AwsException ex = assertThrows(AwsException.class,+ () -> service.createCacheCluster(cacheClusterRequest("shared-id", "redis", 1)));++ assertEquals("CacheClusterAlreadyExists", ex.getErrorCode());+ verify(containerManager, never()).tryStart(eq("shared-id"), anyString());+ }++ @Test+ void aMemcachedClusterCannotTakeTheIdOfARedisClusterOrAGroup() {+ ElastiCacheMemcachedService memcached = memcachedService();+ service.createCacheCluster(cacheClusterRequest("redis-id", "redis", 1));+ service.createReplicationGroup("group-id", "d", AuthMode.NO_AUTH, null, "us-east-1");++ assertEquals("CacheClusterAlreadyExists",+ assertThrows(AwsException.class, () -> memcached.createCacheCluster("redis-id"))+ .getErrorCode());+ assertEquals("CacheClusterAlreadyExists",+ assertThrows(AwsException.class, () -> memcached.createCacheCluster("group-id"))+ .getErrorCode());+ verify(memcachedContainerManager, never()).tryStart(eq("redis-id"), anyString());+ verify(memcachedContainerManager, never()).tryStart(eq("group-id"), anyString());+ }++ @Test+ void aMemcachedCreateIsRefusedWhileAReplicationGroupCreateHoldsTheSameIdInFlight()+ throws InterruptedException {+ // The stores were the only thing the memcached path consulted, and neither path writes+ // its record until its container has started. Held in that window, a group create and a+ // memcached create for one id both saw three empty stores and both went on to write,+ // leaving the id in two stores with one describe reporting it twice.+ ElastiCacheMemcachedService memcached = memcachedService();+ CountDownLatch startedLatch = new CountDownLatch(1);+ CountDownLatch releaseLatch = new CountDownLatch(1);+ when(containerManager.tryStart(anyString(), anyString())).thenAnswer(inv -> {+ startedLatch.countDown();+ assertTrue(releaseLatch.await(5, TimeUnit.SECONDS), "test timed out waiting for release");+ return new ElastiCacheContainerHandle("cid", "grp", "localhost", 6379);+ });++ Thread groupCreate = new Thread(() ->+ service.createReplicationGroup("shared-id", "d", AuthMode.NO_AUTH, null, "us-east-1"));+ groupCreate.start();+ assertTrue(startedLatch.await(5, TimeUnit.SECONDS), "create never reached container start");++ AwsException ex = assertThrows(AwsException.class,+ () -> memcached.createCacheCluster("shared-id"));+ assertEquals("CacheClusterAlreadyExists", ex.getErrorCode());+ verify(memcachedContainerManager, never()).tryStart(eq("shared-id"), anyString());++ releaseLatch.countDown();+ groupCreate.join(5000);++ // One record for the id, in the store the winning create writes.+ assertEquals("shared-id", service.getReplicationGroup("shared-id").getReplicationGroupId());+ assertTrue(memcached.listCacheClusters(null).isEmpty(),+ "the refused memcached create must not have written a second record for the id");+ }++ @Test+ void anUnknownCacheSubnetGroupIsRefused() {+ AwsException ex = assertThrows(AwsException.class,+ () -> service.createCacheCluster(new ElastiCacheService.CreateCacheClusterRequest(+ "sng-cc", "redis", null, null, 1, null, AuthMode.NO_AUTH, null, null,+ "no-such-group", null, null, null, null, null, null, null, null,+ "us-east-1", Map.of())));++ assertEquals("CacheSubnetGroupNotFoundFault", ex.getErrorCode());+ // and nothing was provisioned against the name that does not resolve+ verify(containerManager, never()).tryStart(eq("sng-cc"), anyString());+ assertTrue(service.findCacheClusters("sng-cc").isEmpty());+ }++ @Test+ void aKnownCacheSubnetGroupIsAcceptedAndReported() {+ io.github.hectorvent.floci.services.ec2.model.Subnet subnet =+ new io.github.hectorvent.floci.services.ec2.model.Subnet();+ subnet.setSubnetId("subnet-1");+ subnet.setVpcId("vpc-1");+ subnet.setAvailabilityZone("us-east-1a");+ when(ec2Service.describeSubnets(anyString(), any(), any())).thenReturn(List.of(subnet));+ service.createCacheSubnetGroup("real-group", "d", List.of("subnet-1"), Map.of());++ CacheCluster cluster = service.createCacheCluster(new ElastiCacheService.CreateCacheClusterRequest(+ "sng-cc", "redis", null, null, 1, null, AuthMode.NO_AUTH, null, null,+ "real-group", null, null, null, null, null, null, null, null,+ "us-east-1", Map.of()));++ assertEquals("real-group", cluster.getCacheSubnetGroupName());+ }++ @Test+ void aCacheClusterCreatedWithoutDockerStillReachesAvailable() {+ when(containerManager.tryStart(anyString(), anyString())).thenReturn(null);++ CacheCluster cluster = service.createCacheCluster(cacheClusterRequest("no-docker", "redis", 1));++ assertEquals(CacheClusterStatus.AVAILABLE, cluster.getCacheClusterStatus());+ verify(proxyManager, never()).startProxy(eq("no-docker"), any(), anyInt(), anyString(), anyInt(), any());++ // delete must not reach for a container that was never created+ service.deleteCacheCluster("no-docker");+ verify(containerManager, never()).stop(any());+ verify(containerManager).stopByGroupId("no-docker");+ }}