Light, fluffy, and always free - The AWS Local Emulator alternative
fix(elasticache): preserve member security group metadata (#5937)
Aniket Shukla3a44e19e3653
diff --git a/docs/services/elasticache.md b/docs/services/elasticache.mdindex 259e407ebc..e926d8a1cd 100644--- a/docs/services/elasticache.md+++ b/docs/services/elasticache.md@@ -150,6 +150,12 @@ request may send them, and the describe will not report them back.### Cluster Mode+`SecurityGroupIds` supplied to `CreateReplicationGroup` are stored with the group and returned+on every member by `DescribeCacheClusters` as `SecurityGroups` memberships with `Status=active`.+This includes shards and replicas in cluster mode. Groups without stored IDs return no memberships.+The IDs are control-plane+metadata, not network rules applied to the backing cache containers.+`CreateReplicationGroup` provisions a real sharded Valkey cluster when the request asks for one:`NumNodeGroups` greater than 1, a `default.*.cluster.on` parameter group, a custom parameter groupwith `cluster-enabled` set to `yes`, or `ClusterMode=enabled`.diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.javaindex 8582f1b1cd..2948e47303 100644--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.java@@ -143,7 +143,8 @@ public class ElastiCacheQueryHandler {intParam(params, "Port"),replicationGroupSettings(params),parseTags(params),- extractMemberList(params, "UserGroupIds.member.")));+ extractMemberList(params, "UserGroupIds.member."),+ parseSecurityGroupIds(params)));String result = replicationGroupXml(group);return Response.ok(AwsQueryResponse.envelope("CreateReplicationGroup", AwsNamespaces.EC, result)).build();} catch (AwsException e) {@@ -525,6 +526,7 @@ public class ElastiCacheQueryHandler {if (g.getCacheSubnetGroupName() != null) {xml.elem("CacheSubnetGroupName", g.getCacheSubnetGroupName());}+ appendSecurityGroups(xml, g.getSecurityGroupIds());if (showNodeInfo && g.getConfigurationEndpoint() != null) {xml.start("CacheNodes").start("CacheNode")@@ -959,16 +961,7 @@ private Response handleCreateCacheParameterGroup(MultivaluedMap<String, String>if (c.getIpDiscovery() != null) {xml.elem("IpDiscovery", c.getIpDiscovery());}- if (!c.getSecurityGroupIds().isEmpty()) {- xml.start("SecurityGroups");- for (String securityGroupId : c.getSecurityGroupIds()) {- xml.start("member")- .elem("SecurityGroupId", securityGroupId)- .elem("Status", "active")- .end("member");- }- xml.end("SecurityGroups");- }+ appendSecurityGroups(xml, c.getSecurityGroupIds());if (c.getCacheNodeType() != null) {xml.elem("CacheNodeType", c.getCacheNodeType());}@@ -1016,6 +1009,19 @@ private Response handleCreateCacheParameterGroup(MultivaluedMap<String, String>return xml.end("CacheCluster").build();}+ private static void appendSecurityGroups(XmlBuilder xml, List<String> securityGroupIds) {+ if (!securityGroupIds.isEmpty()) {+ xml.start("SecurityGroups");+ for (String securityGroupId : securityGroupIds) {+ xml.start("member")+ .elem("SecurityGroupId", securityGroupId)+ .elem("Status", "active")+ .end("member");+ }+ xml.end("SecurityGroups");+ }+ }+/*** TransitEncryptionEnabled is not AuthTokenEnabled. An auth token forces encryption in transit,* so a PASSWORD group is always encrypted — but the reverse does not hold: a group created withdiff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.javaindex b0bd92a45f..a433f99a74 100644--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java@@ -198,7 +198,8 @@ public class ElastiCacheService implements ResourceProvider {Integer port,ReplicationGroupSettings settings,Map<String, String> tags,- List<String> userGroupIds) {+ List<String> userGroupIds,+ List<String> securityGroupIds) {}public ReplicationGroup createReplicationGroup(String groupId, String description,@@ -213,7 +214,7 @@ public class ElastiCacheService implements ResourceProvider {Map<String, String> tags) {return createReplicationGroup(new CreateReplicationGroupRequest(groupId, description,authMode, authToken, region, null, null, null, null, null, null,- null, null, null, null, null, null, settings, tags, null));+ null, null, null, null, null, null, settings, tags, null, null));}public ReplicationGroup createReplicationGroup(CreateReplicationGroupRequest request) {@@ -500,6 +501,7 @@ public class ElastiCacheService implements ResourceProvider {: "cache.t4g.micro");group.setCacheParameterGroupName(request.cacheParameterGroupName());group.setCacheSubnetGroupName(request.cacheSubnetGroupName());+ group.setSecurityGroupIds(request.securityGroupIds());group.setAutomaticFailoverEnabled(Boolean.TRUE.equals(request.automaticFailoverEnabled()));group.setMultiAzEnabled(Boolean.TRUE.equals(request.multiAzEnabled()));group.setSnapshotWindow(ReplicationGroupSettings.DEFAULT_SNAPSHOT_WINDOW);diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroup.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroup.javaindex 1d762a40f6..d435c1b9fb 100644--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroup.java+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroup.java@@ -47,6 +47,7 @@ public class ReplicationGroup {private String cacheNodeType;private String cacheParameterGroupName;private String cacheSubnetGroupName;+ private List<String> securityGroupIds = new ArrayList<>();private boolean automaticFailoverEnabled;private boolean multiAzEnabled;private List<ClusterNode> clusterNodes = new ArrayList<>();@@ -182,4 +183,9 @@ public class ReplicationGroup {public void setClusterNodes(List<ClusterNode> clusterNodes) {this.clusterNodes = clusterNodes != null ? clusterNodes : new ArrayList<>();}++ public List<String> getSecurityGroupIds() { return securityGroupIds; }+ public void setSecurityGroupIds(List<String> securityGroupIds) {+ this.securityGroupIds = securityGroupIds == null ? new ArrayList<>() : new ArrayList<>(securityGroupIds);+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemberSecurityGroupsIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemberSecurityGroupsIntegrationTest.javanew file mode 100644index 0000000000..54951c9a7d--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheMemberSecurityGroupsIntegrationTest.java@@ -0,0 +1,174 @@+package io.github.hectorvent.floci.services.elasticache;++import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerHandle;+import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerManager;+import io.github.hectorvent.floci.services.elasticache.container.ValkeyClusterFormation;+import io.quarkus.test.InjectMock;+import io.quarkus.test.common.http.TestHTTPResource;+import io.quarkus.test.junit.QuarkusTest;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;+import software.amazon.awssdk.regions.Region;+import software.amazon.awssdk.services.elasticache.ElastiCacheClient;+import software.amazon.awssdk.services.elasticache.model.CacheCluster;+import software.amazon.awssdk.services.elasticache.model.CreateReplicationGroupRequest;+import software.amazon.awssdk.services.elasticache.model.ReplicationGroupNotFoundException;+import software.amazon.awssdk.services.elasticache.model.SecurityGroupMembership;++import java.net.URI;+import java.util.List;+import java.util.UUID;++import static io.restassured.RestAssured.given;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.mockito.ArgumentMatchers.anyList;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.Mockito.when;++@QuarkusTest+class ElastiCacheMemberSecurityGroupsIntegrationTest {++ private static final String AUTH_HEADER =+ "AWS4-HMAC-SHA256 Credential=test/20261018/us-east-1/elasticache/aws4_request";+ private static final List<String> SECURITY_GROUP_IDS = List.of("sg-0123456789abcdef0", "sg-0fedcba9876543210");++ @TestHTTPResource+ URI endpoint;++ @InjectMock+ ElastiCacheContainerManager containerManager;++ // Cluster formation talks to the backing Valkey nodes, which are not started by these control-plane tests.+ @InjectMock+ ValkeyClusterFormation clusterFormation;++ @BeforeEach+ void stubOnlyBackendProvisioning() {+ when(containerManager.tryStart(anyString(), anyString(), anyString()))+ .thenAnswer(invocation -> containerHandle(invocation.getArgument(0)));+ when(containerManager.start(anyString(), anyString(), anyList(), anyString()))+ .thenAnswer(invocation -> containerHandle(invocation.getArgument(0)));+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void sdkCreatePreservesSecurityGroupsOnEveryMember(boolean clusterEnabled) {+ String groupId = groupId();+ try (ElastiCacheClient client = client()) {+ try {+ client.createReplicationGroup(createRequest(groupId, clusterEnabled)+ .securityGroupIds(SECURITY_GROUP_IDS).build());++ assertMemberSecurityGroups(client, groupId, clusterEnabled ? 4 : 1, SECURITY_GROUP_IDS);+ } finally {+ deleteGroup(client, groupId);+ }+ }+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void omittedSecurityGroupsDoNotFabricateMemberships(boolean clusterEnabled) {+ String groupId = groupId();+ try (ElastiCacheClient client = client()) {+ try {+ client.createReplicationGroup(createRequest(groupId, clusterEnabled).build());++ assertMemberSecurityGroups(client, groupId, clusterEnabled ? 4 : 1, List.of());+ } finally {+ deleteGroup(client, groupId);+ }+ }+ }++ @ParameterizedTest+ @ValueSource(strings = {"SecurityGroupIds.SecurityGroupId", "SecurityGroupIds.member"})+ void queryListSpellingsRoundtripThroughTypedSdkMembers(String prefix) {+ String groupId = groupId();+ try (ElastiCacheClient client = client()) {+ try {+ given()+ .port(endpoint.getPort())+ .header("Authorization", AUTH_HEADER)+ .formParam("Action", "CreateReplicationGroup")+ .formParam("Version", "2015-02-02")+ .formParam("ReplicationGroupId", groupId)+ .formParam("ReplicationGroupDescription", "member security groups")+ .formParam("Engine", "valkey")+ .formParam("NumCacheClusters", "2")+ .formParam(prefix + ".1", SECURITY_GROUP_IDS.getFirst())+ .formParam(prefix + ".2", SECURITY_GROUP_IDS.getLast())+ .when().post("/")+ .then().statusCode(200);++ assertMemberSecurityGroups(client, groupId, 2, SECURITY_GROUP_IDS);+ } finally {+ deleteGroup(client, groupId);+ }+ }+ }++ private ElastiCacheClient client() {+ return ElastiCacheClient.builder().endpointOverride(endpoint).region(Region.US_EAST_1)+ .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create("test", "test")))+ .build();+ }++ private static CreateReplicationGroupRequest.Builder createRequest(String groupId, boolean clusterEnabled) {+ CreateReplicationGroupRequest.Builder request = CreateReplicationGroupRequest.builder()+ .replicationGroupId(groupId).replicationGroupDescription("member security groups")+ .engine("valkey").clusterMode(clusterEnabled ? "enabled" : "disabled");+ if (clusterEnabled) {+ request.numNodeGroups(2).replicasPerNodeGroup(1).automaticFailoverEnabled(true);+ }+ return request;+ }++ private static void assertMemberSecurityGroups(ElastiCacheClient client, String groupId,+ int expectedMembers, List<String> expectedSecurityGroupIds) {+ List<String> memberIds = client.describeReplicationGroups(request -> request.replicationGroupId(groupId))+ .replicationGroups().getFirst().memberClusters();+ assertEquals(expectedMembers, memberIds.size());+ for (String memberId : memberIds) {+ List<CacheCluster> clusters = client.describeCacheClusters(request -> request+ .cacheClusterId(memberId).showCacheNodeInfo(true)).cacheClusters();+ assertEquals(1, clusters.size(), memberId);+ assertEquals(groupId, clusters.getFirst().replicationGroupId());+ assertSecurityGroups(clusters.getFirst(), expectedSecurityGroupIds);+ }++ List<CacheCluster> listedMembers = client.describeCacheClusters().cacheClusters().stream()+ .filter(cluster -> groupId.equals(cluster.replicationGroupId())).toList();+ assertEquals(expectedMembers, listedMembers.size());+ for (CacheCluster cluster : listedMembers) {+ assertSecurityGroups(cluster, expectedSecurityGroupIds);+ }+ }++ private static void assertSecurityGroups(CacheCluster cluster, List<String> expectedIds) {+ assertEquals(expectedIds, cluster.securityGroups().stream()+ .map(SecurityGroupMembership::securityGroupId).toList(), cluster.cacheClusterId());+ for (SecurityGroupMembership membership : cluster.securityGroups()) {+ assertEquals("active", membership.status(), cluster.cacheClusterId());+ }+ }++ private static ElastiCacheContainerHandle containerHandle(String memberId) {+ return new ElastiCacheContainerHandle("stub-" + memberId, memberId, "127.0.0.1", 16379);+ }++ private static String groupId() {+ return "sg-members-" + UUID.randomUUID().toString().substring(0, 8);+ }++ private static void deleteGroup(ElastiCacheClient client, String groupId) {+ try {+ client.deleteReplicationGroup(request -> request.replicationGroupId(groupId));+ } catch (ReplicationGroupNotFoundException expected) {+ // A failed create may not have stored the group; there is then nothing to clean up.+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.javaindex 67284fd710..654e7cbd9e 100644--- a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java@@ -273,7 +273,7 @@ class ElastiCacheServiceTest {ElastiCacheService.CreateReplicationGroupRequest request = new ElastiCacheService.CreateReplicationGroupRequest("grp", "test", AuthMode.IAM, null, "us-east-1", "redis", null, null, null, null, null,null, null, null, null, null, null, ReplicationGroupSettings.defaults(), Map.of(),- List.of("missing-group"));+ List.of("missing-group"), null);AwsException refused = assertThrows(AwsException.class, () -> service.createReplicationGroup(request));@@ -307,7 +307,7 @@ class ElastiCacheServiceTest {void anEmptyUserGroupAdmitsNobody() {service.createReplicationGroup(new ElastiCacheService.CreateReplicationGroupRequest("grp", "test", AuthMode.IAM, null, "us-east-1", "valkey", null, null, null, null, null,- null, null, null, null, null, null, ReplicationGroupSettings.defaults(), Map.of(), null));+ null, null, null, null, null, null, ReplicationGroupSettings.defaults(), Map.of(), null, null));service.createUserGroup("empty-users", "valkey", List.of(), "us-east-1");service.modifyReplicationGroup("grp", List.of("empty-users"), null);@@ -396,7 +396,7 @@ class ElastiCacheServiceTest {ElastiCacheService.CreateReplicationGroupRequest request = new ElastiCacheService.CreateReplicationGroupRequest("grp", "test", AuthMode.IAM, null, "us-east-1", "redis", null, null, null, null, null,null, null, null, null, null, null, ReplicationGroupSettings.defaults(), Map.of(),- List.of("reserved"));+ List.of("reserved"), null);assertThrows(RuntimeException.class, () -> service.createReplicationGroup(request));@@ -647,14 +647,14 @@ class ElastiCacheServiceTest {return new ElastiCacheService.CreateReplicationGroupRequest(groupId, "test",AuthMode.NO_AUTH, null, "us-east-1", "valkey", "8.2", "cache.t4g.micro","default.valkey8.cluster.on", null, null, numNodeGroups, replicasPerNodeGroup,- null, true, null, null, ReplicationGroupSettings.defaults(), Map.of(), null);+ null, true, null, null, ReplicationGroupSettings.defaults(), Map.of(), null, null);}private static ElastiCacheService.CreateReplicationGroupRequest singleNodeRequest(String groupId, Integer port) {return new ElastiCacheService.CreateReplicationGroupRequest(groupId, "test",AuthMode.NO_AUTH, null, "us-east-1", null, null, null, null, null, null,- null, null, null, null, null, port, ReplicationGroupSettings.defaults(), Map.of(), null);+ null, null, null, null, null, port, ReplicationGroupSettings.defaults(), Map.of(), null, null);}// botocore models Port as an optional input on CreateReplicationGroup ("the port number on@@ -744,7 +744,7 @@ class ElastiCacheServiceTest {new ElastiCacheService.CreateReplicationGroupRequest("grp", "test",AuthMode.NO_AUTH, null, "us-east-1", "valkey", "8.2", "cache.t4g.micro","default.valkey8.cluster.on", null, null, 2, 1,- null, true, null, 16390, ReplicationGroupSettings.defaults(), Map.of(), null));+ null, true, null, 16390, ReplicationGroupSettings.defaults(), Map.of(), null, null));assertEquals(16390, group.getConfigurationEndpoint().port());assertEquals(16390, group.getClusterNodes().getFirst().getProxyPort());@@ -1600,7 +1600,7 @@ class ElastiCacheServiceTest {return new ElastiCacheService.CreateReplicationGroupRequest(groupId, "test",AuthMode.NO_AUTH, null, "us-east-1", null, null, null,parameterGroupName, null, null, null, null,- null, null, null, null, ReplicationGroupSettings.defaults(), Map.of(), null);+ null, null, null, null, ReplicationGroupSettings.defaults(), Map.of(), null, null);}@Test@@ -1752,7 +1752,7 @@ class ElastiCacheServiceTest {svc.createReplicationGroup(new ElastiCacheService.CreateReplicationGroupRequest("grp", "test", AuthMode.IAM, null, "us-east-1", "redis", null, null, null, null, null,null, null, null, null, null, null, ReplicationGroupSettings.defaults(), Map.of(),- List.of("shared-ug")));+ List.of("shared-ug"), null));});accountA.start();assertTrue(startedLatch.await(5, TimeUnit.SECONDS), "create never reached container start");@@ -2450,7 +2450,7 @@ class ElastiCacheServiceTest {service.createReplicationGroup(new ElastiCacheService.CreateReplicationGroupRequest("rg-cluster", "desc", AuthMode.NO_AUTH, null, "us-east-1", null, null, null,null, "rg-used-group", null, null, null, null, null, null, null,- ReplicationGroupSettings.defaults(), Map.of(), null));+ ReplicationGroupSettings.defaults(), Map.of(), null, null));AwsException ex = assertThrows(AwsException.class, () -> service.deleteCacheSubnetGroup("rg-used-group"));assertEquals("CacheSubnetGroupInUse", ex.getErrorCode());diff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/ReplicationGroupSecurityGroupsTest.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/ReplicationGroupSecurityGroupsTest.javanew file mode 100644index 0000000000..42db69f816--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/ReplicationGroupSecurityGroupsTest.java@@ -0,0 +1,40 @@+package io.github.hectorvent.floci.services.elasticache;++import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroup;+import org.junit.jupiter.api.Test;++import static org.junit.jupiter.api.Assertions.assertEquals;++class ReplicationGroupSecurityGroupsTest {++ private final ObjectMapper mapper = new ObjectMapper();++ @Test+ void securityGroupsSurviveThePersistedJsonRoundTrip() throws Exception {+ ReplicationGroup group = mapper.readValue("""+ {"replicationGroupId":"restored","securityGroupIds":["sg-first","sg-second"]}+ """, ReplicationGroup.class);++ ReplicationGroup restored = mapper.readValue(mapper.writeValueAsString(group), ReplicationGroup.class);++ assertEquals(mapper.readTree("[\"sg-first\",\"sg-second\"]"),+ mapper.valueToTree(restored).get("securityGroupIds"));+ }++ @Test+ void legacyRecordsWithoutSecurityGroupsKeepAnEmptyList() throws Exception {+ ReplicationGroup group = mapper.readValue("{\"replicationGroupId\":\"legacy\"}", ReplicationGroup.class);++ assertEquals(mapper.readTree("[]"), mapper.valueToTree(group).get("securityGroupIds"));+ }++ @Test+ void explicitNullSecurityGroupsNormalizeToAnEmptyList() throws Exception {+ ReplicationGroup group = mapper.readValue("""+ {"replicationGroupId":"legacy-null","securityGroupIds":null}+ """, ReplicationGroup.class);++ assertEquals(mapper.readTree("[]"), mapper.valueToTree(group).get("securityGroupIds"));+ }+}