Light, fluffy, and always free - The AWS Local Emulator alternative
fix(elasticache): keep a replication group's encryption, snapshot settings, tags and ARN (#2632)
Michal Schreiber75f8f42d5fbd
diff --git a/compatibility-tests/sdk-test-awscli/test/elasticache.bats b/compatibility-tests/sdk-test-awscli/test/elasticache.batsindex a6a02ce50f..04440ce528 100644--- a/compatibility-tests/sdk-test-awscli/test/elasticache.bats+++ b/compatibility-tests/sdk-test-awscli/test/elasticache.bats@@ -137,3 +137,41 @@ teardown() {run aws_cmd elasticache delete-cache-subnet-group --cache-subnet-group-name "$SNG_EU"assert_success}++@test "ElastiCache: replication group keeps its encryption, snapshot settings and tags" {+ RG="bats-rg-$(unique_name)"+ run aws_cmd kms create-key --description "$RG"+ assert_success+ KEY_ARN=$(json_get "$output" '.KeyMetadata.Arn')+ KEY_ID=$(json_get "$output" '.KeyMetadata.KeyId')+ aws_cmd kms create-alias --alias-name "alias/$RG" --target-key-id "$KEY_ID" >/dev/null++ run aws_cmd elasticache create-replication-group --replication-group-id "$RG" --replication-group-description d \+ --engine redis --cache-node-type cache.t3.micro --num-cache-clusters 1 \+ --at-rest-encryption-enabled --kms-key-id "alias/$RG" \+ --snapshot-retention-limit 7 --snapshot-window 06:30-07:30 --tags "Key=Name,Value=$RG"+ assert_success++ run aws_cmd elasticache describe-replication-groups --replication-group-id "$RG" \+ --query 'ReplicationGroups[0].[KmsKeyId,AtRestEncryptionEnabled,SnapshotRetentionLimit,SnapshotWindow,ARN]'+ assert_success+ assert_output --partial "$KEY_ARN"+ assert_output --partial 'true'+ assert_output --partial '7'+ assert_output --partial '"06:30-07:30"'+ assert_output --partial "replicationgroup:$RG"+ ARN=$(aws_cmd elasticache describe-replication-groups --replication-group-id "$RG" --query 'ReplicationGroups[0].ARN' --output text)++ run aws_cmd elasticache list-tags-for-resource --resource-name "$ARN"+ assert_success+ assert_output --partial "\"Value\": \"$RG\""++ run aws_cmd elasticache modify-replication-group --replication-group-id "$RG" --snapshot-retention-limit 3 --snapshot-window 01:00-02:00 --apply-immediately+ assert_success+ run aws_cmd elasticache describe-replication-groups --replication-group-id "$RG" --query 'ReplicationGroups[0].[SnapshotRetentionLimit,SnapshotWindow]'+ assert_success+ assert_output --partial '3'+ assert_output --partial '"01:00-02:00"'++ aws_cmd elasticache delete-replication-group --replication-group-id "$RG" >/dev/null 2>&1 || true+}diff --git a/docs/services/elasticache.md b/docs/services/elasticache.mdindex beaee49d2a..6f15199001 100644--- a/docs/services/elasticache.md+++ b/docs/services/elasticache.md@@ -12,9 +12,9 @@ Floci manages real Valkey/Redis Docker containers and proxies TCP connections to| Action | Description || --- | --- || `ValidateIamAuthToken` | Validate an IAM auth token (data-plane auth) |-| `CreateReplicationGroup` | Start a new Redis/Valkey cluster |+| `CreateReplicationGroup` | Start a new Redis/Valkey cluster; `AtRestEncryptionEnabled`, `KmsKeyId` (resolved to the key ARN), `SnapshotRetentionLimit`, `SnapshotWindow` and `Tags` are kept and returned, with the group `ARN` || `DescribeReplicationGroups` | List clusters and their connection info |-| `ModifyReplicationGroup` | - |+| `ModifyReplicationGroup` | Modify `SnapshotRetentionLimit` and `SnapshotWindow`, and the associated user groups || `DeleteReplicationGroup` | Stop and remove a cluster || `CreateUser` | Create an ElastiCache IAM user || `DescribeUsers` | List ElastiCache users |diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.javaindex a20ae29c3d..67e8750516 100644--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.java@@ -13,6 +13,7 @@ import io.github.hectorvent.floci.services.elasticache.model.CacheSubnetGroup;import io.github.hectorvent.floci.services.elasticache.model.ElastiCacheUser;import io.github.hectorvent.floci.services.elasticache.model.Endpoint;import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroup;+import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroupSettings;import io.github.hectorvent.floci.services.elasticache.proxy.SigV4Validator;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;@@ -105,7 +106,8 @@ public class ElastiCacheQueryHandler {try {ReplicationGroup group = service.createReplicationGroup(- groupId, description != null ? description : "", authMode, authToken, region);+ groupId, description != null ? description : "", authMode, authToken, region,+ replicationGroupSettings(params), parseTags(params));String result = replicationGroupXml(group);return Response.ok(AwsQueryResponse.envelope("CreateReplicationGroup", AwsNamespaces.EC, result)).build();} catch (AwsException e) {@@ -113,6 +115,28 @@ public class ElastiCacheQueryHandler {}}+ private static ReplicationGroupSettings replicationGroupSettings(MultivaluedMap<String, String> params) {+ String atRest = params.getFirst("AtRestEncryptionEnabled");+ // A live account reads any value that is not "false" as true — "banana", "yes" and+ // "TRUE " all created an encrypted group — so the flag is read the same way here.+ return new ReplicationGroupSettings(+ atRest == null ? null : !"false".equalsIgnoreCase(atRest.trim()),+ params.getFirst("KmsKeyId"),+ optionalInt(params.getFirst("SnapshotRetentionLimit")),+ params.getFirst("SnapshotWindow"));+ }++ private static Integer optionalInt(String value) {+ if (value == null || value.isBlank()) {+ return null;+ }+ try {+ return Integer.parseInt(value.trim());+ } catch (NumberFormatException e) {+ throw new AwsException("InvalidParameterValue", "Value " + value + " is not a valid integer.", 400);+ }+ }+private Response handleDescribeReplicationGroups(MultivaluedMap<String, String> params) {String filterId = params.getFirst("ReplicationGroupId");try {@@ -153,9 +177,12 @@ public class ElastiCacheQueryHandler {List<String> userIdsToAdd = extractMemberList(params, "UserGroupIdsToAdd.member.");List<String> userIdsToRemove = extractMemberList(params, "UserGroupIdsToRemove.member.");try {+ // AtRestEncryptionEnabled and KmsKeyId are fixed at create and not in the modify shape+ ReplicationGroupSettings settings = new ReplicationGroupSettings(null, null,+ optionalInt(params.getFirst("SnapshotRetentionLimit")), params.getFirst("SnapshotWindow"));ReplicationGroup group = service.modifyReplicationGroup(groupId,userIdsToAdd.isEmpty() ? null : userIdsToAdd,- userIdsToRemove.isEmpty() ? null : userIdsToRemove);+ userIdsToRemove.isEmpty() ? null : userIdsToRemove, settings);String result = replicationGroupXml(group);return Response.ok(AwsQueryResponse.envelope("ModifyReplicationGroup", AwsNamespaces.EC, result)).build();} catch (AwsException e) {@@ -478,6 +505,15 @@ private Response handleCreateCacheParameterGroup(MultivaluedMap<String, String>}Map<String, String> tags = Map.of();+ if ("replicationgroup".equals(arn[5])) {+ // the store keys groups by id alone; the record must be the one the ARN names+ ReplicationGroup group = service.getReplicationGroup(arn[6]);+ if (group.getArn() != null && !group.getArn().equalsIgnoreCase(resourceName)) {+ throw new AwsException("ReplicationGroupNotFoundFault",+ "Replication group " + arn[6] + " not found.", 404);+ }+ tags = group.getTags();+ }if ("subnetgroup".equals(arn[5])) {tags = service.describeCacheSubnetGroups(arn[6]).getFirst().getTags();}@@ -611,11 +647,19 @@ private Response handleCreateCacheParameterGroup(MultivaluedMap<String, String>.elem("Status", g.getStatus().name().toLowerCase()).elem("AuthTokenEnabled", authTokenEnabled).elem("TransitEncryptionEnabled", authTokenEnabled)- .elem("AtRestEncryptionEnabled", false)+ .elem("AtRestEncryptionEnabled", g.isAtRestEncryptionEnabled()).elem("ClusterEnabled", false).elem("MultiAZ", "disabled").elem("AutomaticFailover", "disabled")- .elem("SnapshotRetentionLimit", 0L);+ .elem("SnapshotRetentionLimit", (long) g.getSnapshotRetentionLimit());+ xml.elem("SnapshotWindow", g.getSnapshotWindow() != null+ ? g.getSnapshotWindow() : ReplicationGroupSettings.DEFAULT_SNAPSHOT_WINDOW);+ if (g.getKmsKeyId() != null) {+ xml.elem("KmsKeyId", g.getKmsKeyId());+ }+ if (g.getArn() != null) {+ xml.elem("ARN", g.getArn());+ }if (ep != null) {xml.start("ConfigurationEndpoint").elem("Address", ep.address())diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.javaindex 2bb19d0dab..f009664c9d 100644--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java@@ -29,6 +29,9 @@ import org.jboss.logging.Logger;import java.time.Instant;import java.util.ArrayList;import java.util.Collection;+import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroupSettings;+import io.github.hectorvent.floci.services.kms.KmsService;+import io.github.hectorvent.floci.services.kms.model.KmsKey;import java.util.LinkedHashMap;import java.util.List;import java.util.Locale;@@ -53,6 +56,7 @@ public class ElastiCacheService implements ResourceProvider {private final ElastiCacheContainerManager containerManager;private final ElastiCacheProxyManager proxyManager;private final EmulatorConfig config;+ private final KmsService kmsService;private final Ec2Service ec2Service;private final RegionResolver regionResolver;private final Set<Integer> usedPorts = ConcurrentHashMap.newKeySet();@@ -65,7 +69,9 @@ public class ElastiCacheService implements ResourceProvider {StorageFactory storageFactory,EmulatorConfig config,Ec2Service ec2Service,- RegionResolver regionResolver) {+ RegionResolver regionResolver,+ KmsService kmsService) {+ this.kmsService = kmsService;this.containerManager = containerManager;this.proxyManager = proxyManager;this.config = config;@@ -83,6 +89,18 @@ public class ElastiCacheService implements ResourceProvider {public ReplicationGroup createReplicationGroup(String groupId, String description,AuthMode authMode, String authToken, String region) {+ return createReplicationGroup(groupId, description, authMode, authToken, region,+ ReplicationGroupSettings.defaults(), Map.of());+ }++ public ReplicationGroup createReplicationGroup(String groupId, String description,+ AuthMode authMode, String authToken, String region,+ ReplicationGroupSettings settings,+ Map<String, String> tags) {+ settings.validate();+ // resolved with the other validations, before a port is taken or a container started+ ReplicationGroupSettings resolvedSettings =+ settings.withKmsKeyId(resolveKmsKeyArn(settings.kmsKeyId(), region));if (groups.get(groupId).isPresent()) {throw new AwsException("ReplicationGroupAlreadyExistsFault","Replication group " + groupId + " already exists.", 400);@@ -116,6 +134,11 @@ public class ElastiCacheService implements ResourceProvider {group.setAuthToken(authToken);group.setArn(regionResolver.buildArn("elasticache", region, "replicationgroup:" + groupId));group.setRegion(region);+ group.setSnapshotWindow(ReplicationGroupSettings.DEFAULT_SNAPSHOT_WINDOW);+ resolvedSettings.applyTo(group);+ if (tags != null && !tags.isEmpty()) {+ group.setTags(new LinkedHashMap<>(tags));+ }proxyManager.startProxy(groupId, authMode, proxyPort,handle.getHost(), handle.getPort(),@@ -173,41 +196,88 @@ public class ElastiCacheService implements ResourceProvider {}public void deleteReplicationGroup(String groupId) {- ReplicationGroup group = groups.get(groupId).orElseThrow(() ->- new AwsException("ReplicationGroupNotFoundFault",- "Replication group " + groupId + " not found.", 404));+ // one monitor per group for delete and modify: a modify's read-modify-write could+ // otherwise write the group back after this removed it+ synchronized (lockFor("rg:" + groupId)) {+ ReplicationGroup group = groups.get(groupId).orElseThrow(() ->+ new AwsException("ReplicationGroupNotFoundFault",+ "Replication group " + groupId + " not found.", 404));- group.setStatus(ReplicationGroupStatus.DELETING);- groups.put(groupId, group);+ group.setStatus(ReplicationGroupStatus.DELETING);+ groups.put(groupId, group);- proxyManager.stopProxy(groupId);+ proxyManager.stopProxy(groupId);- if (group.getContainerId() != null) {- containerManager.stop(new ElastiCacheContainerHandle(- group.getContainerId(), groupId, group.getContainerHost(), group.getContainerPort()));- }+ if (group.getContainerId() != null) {+ containerManager.stop(new ElastiCacheContainerHandle(+ group.getContainerId(), groupId, group.getContainerHost(), group.getContainerPort()));+ }- releaseProxyPort(group.getProxyPort());- groups.delete(groupId);- LOG.infov("Replication group {0} deleted", groupId);+ releaseProxyPort(group.getProxyPort());+ groups.delete(groupId);+ LOG.infov("Replication group {0} deleted", groupId);+ }}public ReplicationGroup modifyReplicationGroup(String groupId, List<String> userIdsToAdd,List<String> userIdsToRemove) {- ReplicationGroup group = getReplicationGroup(groupId);+ return modifyReplicationGroup(groupId, userIdsToAdd, userIdsToRemove,+ ReplicationGroupSettings.unchanged());+ }- if (userIdsToAdd != null) {- for (String userId : userIdsToAdd) {- getUser(userId); // validate user exists- group.getAssociatedUserIds().add(userId);+ public ReplicationGroup modifyReplicationGroup(String groupId, List<String> userIdsToAdd,+ List<String> userIdsToRemove,+ ReplicationGroupSettings settings) {+ settings.validate();+ synchronized (lockFor("rg:" + groupId)) {+ ReplicationGroup group = getReplicationGroup(groupId);+ // every check before any change: the store hands out its own object, so a mutation+ // made before a later refusal would stay visible+ if (userIdsToAdd != null) {+ for (String userId : userIdsToAdd) {+ getUser(userId);+ }+ }+ settings.applyTo(group);+ if (userIdsToAdd != null) {+ group.getAssociatedUserIds().addAll(userIdsToAdd);}+ if (userIdsToRemove != null) {+ group.getAssociatedUserIds().removeAll(userIdsToRemove);+ }++ groups.put(groupId, group);+ return group;+ }+ }++ /**+ * A live account takes the key as a key id, key ARN or alias and reports the key ARN on the+ * group; a key it cannot use is one fault whatever the reason.+ */+ private String resolveKmsKeyArn(String kmsKeyId, String region) {+ if (kmsKeyId == null || kmsKeyId.isBlank()) {+ return null;+ }+ if (kmsService == null) {+ throw new IllegalStateException("ElastiCacheService was built without a KmsService; "+ + "a KmsKeyId cannot be resolved");}- if (userIdsToRemove != null) {- group.getAssociatedUserIds().removeAll(userIdsToRemove);+ KmsKey key;+ try {+ key = kmsService.describeKey(kmsKeyId, region);+ } catch (AwsException e) {+ throw kmsKeyNotAccessible(kmsKeyId);+ }+ if (!key.isEnabled() || "PendingDeletion".equals(key.getKeyState())) {+ throw kmsKeyNotAccessible(kmsKeyId);}+ return key.getArn();+ }- groups.put(groupId, group);- return group;+ private static AwsException kmsKeyNotAccessible(String kmsKeyId) {+ return new AwsException("InvalidParameterValue",+ "KMS key does not exist with key id: " + kmsKeyId, 400);}public ElastiCacheUser createUser(String userId, String userName, AuthMode authMode,diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroup.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroup.javaindex ca2197913f..d2e418fd0a 100644--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroup.java+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroup.java@@ -4,6 +4,8 @@ import io.quarkus.runtime.annotations.RegisterForReflection;import java.time.Instant;import java.util.HashSet;+import java.util.LinkedHashMap;+import java.util.Map;import java.util.Set;@RegisterForReflection@@ -20,6 +22,11 @@ public class ReplicationGroup {private Set<String> associatedUserIds = new HashSet<>();private String arn;private String region;+ private boolean atRestEncryptionEnabled;+ private String kmsKeyId;+ private int snapshotRetentionLimit;+ private String snapshotWindow;+ private Map<String, String> tags = new LinkedHashMap<>();// Transient fields — not persisted, restored on container restartprivate transient String containerId;@@ -78,6 +85,21 @@ public class ReplicationGroup {public int getContainerPort() { return containerPort; }public void setContainerPort(int containerPort) { this.containerPort = containerPort; }+ public boolean isAtRestEncryptionEnabled() { return atRestEncryptionEnabled; }+ public void setAtRestEncryptionEnabled(boolean atRestEncryptionEnabled) { this.atRestEncryptionEnabled = atRestEncryptionEnabled; }++ public String getKmsKeyId() { return kmsKeyId; }+ public void setKmsKeyId(String kmsKeyId) { this.kmsKeyId = kmsKeyId; }++ public int getSnapshotRetentionLimit() { return snapshotRetentionLimit; }+ public void setSnapshotRetentionLimit(int snapshotRetentionLimit) { this.snapshotRetentionLimit = snapshotRetentionLimit; }++ public String getSnapshotWindow() { return snapshotWindow; }+ public void setSnapshotWindow(String snapshotWindow) { this.snapshotWindow = snapshotWindow; }++ public Map<String, String> getTags() { return tags; }+ public void setTags(Map<String, String> tags) { this.tags = tags == null ? new LinkedHashMap<>() : tags; }+public String getArn() { return arn; }public void setArn(String arn) { this.arn = arn; }diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroupSettings.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroupSettings.javanew file mode 100644index 0000000000..61e7272a4a--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroupSettings.java@@ -0,0 +1,100 @@+package io.github.hectorvent.floci.services.elasticache.model;++import io.github.hectorvent.floci.core.common.AwsException;+import io.quarkus.runtime.annotations.RegisterForReflection;++import java.util.regex.Matcher;+import java.util.regex.Pattern;++/**+ * The encryption and snapshot settings of a replication group as a request carries them: a null+ * member is one the request left out — the AWS default on create, unchanged on modify.+ */+@RegisterForReflection+public record ReplicationGroupSettings(Boolean atRestEncryptionEnabled,+ String kmsKeyId,+ Integer snapshotRetentionLimit,+ String snapshotWindow) {++ private static final Pattern TIME = Pattern.compile("^([01]\\d|2[0-3]):([0-5]\\d)$");+ private static final int MINUTES_PER_DAY = 24 * 60;+ private static final int MINIMUM_WINDOW_MINUTES = 60;++ /** Where AWS picks a random one-hour window, Floci picks this. */+ public static final String DEFAULT_SNAPSHOT_WINDOW = "00:00-01:00";++ public static ReplicationGroupSettings defaults() {+ return new ReplicationGroupSettings(null, null, null, null);+ }++ public static ReplicationGroupSettings unchanged() {+ return defaults();+ }++ /** The checks a live account applies to these parameters, with its wording. */+ public void validate() {+ // an omitted AtRestEncryptionEnabled is false on a live account, and refused the same way+ if (kmsKeyId != null && !kmsKeyId.isBlank() && !Boolean.TRUE.equals(atRestEncryptionEnabled)) {+ throw new AwsException("InvalidParameterCombination",+ "Please enable encryption at rest to use Customer Managed CMK", 400);+ }+ if (snapshotRetentionLimit != null && (snapshotRetentionLimit < 0 || snapshotRetentionLimit > 35)) {+ throw new AwsException("InvalidParameterValue", "Invalid snapshot retention limit: "+ + snapshotRetentionLimit + ". Retention limit must be between 0 and 35.", 400);+ }+ if (snapshotWindow != null) {+ parseWindow(snapshotWindow);+ }+ }++ private static void parseWindow(String window) {+ String[] parts = window.split("-", -1);+ if (parts.length != 2) {+ throw invalidWindow(window);+ }+ int start = minuteOfDay(parts[0], window);+ int end = minuteOfDay(parts[1], window);+ // an end before the start wraps midnight; an end equal to the start is an empty window,+ // which a live account refuses as shorter than an hour, not a full day+ if (end < start) {+ end += MINUTES_PER_DAY;+ }+ if (end - start < MINIMUM_WINDOW_MINUTES) {+ throw new AwsException("InvalidParameterValue",+ "Snapshot window must be at least " + MINIMUM_WINDOW_MINUTES + " minutes.", 400);+ }+ }++ private static int minuteOfDay(String time, String window) {+ Matcher m = TIME.matcher(time);+ if (!m.matches()) {+ throw invalidWindow(window);+ }+ return Integer.parseInt(m.group(1)) * 60 + Integer.parseInt(m.group(2));+ }++ private static AwsException invalidWindow(String window) {+ return new AwsException("InvalidParameterValue", "Invalid backup window format. Should be "+ + "specified as a range hh24:mi-hh24:mi (24H Clock UTC). Example: 03:15-08:15", 400);+ }++ public ReplicationGroupSettings withKmsKeyId(String resolvedKmsKeyId) {+ return new ReplicationGroupSettings(atRestEncryptionEnabled, resolvedKmsKeyId,+ snapshotRetentionLimit, snapshotWindow);+ }++ public void applyTo(ReplicationGroup group) {+ if (atRestEncryptionEnabled != null) {+ group.setAtRestEncryptionEnabled(atRestEncryptionEnabled);+ }+ if (kmsKeyId != null && !kmsKeyId.isBlank()) {+ group.setKmsKeyId(kmsKeyId);+ }+ if (snapshotRetentionLimit != null) {+ group.setSnapshotRetentionLimit(snapshotRetentionLimit);+ }+ if (snapshotWindow != null && !snapshotWindow.isBlank()) {+ group.setSnapshotWindow(snapshotWindow);+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandlerTest.javaindex 9c9d06bf3d..dd7afcf206 100644--- a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandlerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandlerTest.java@@ -6,6 +6,17 @@ import jakarta.ws.rs.core.MultivaluedHashMap;import jakarta.ws.rs.core.MultivaluedMap;import jakarta.ws.rs.core.Response;import org.junit.jupiter.api.BeforeEach;+import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroup;+import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroupSettings;+import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroupStatus;+import io.github.hectorvent.floci.services.elasticache.model.AuthMode;+import io.github.hectorvent.floci.services.elasticache.model.Endpoint;+import io.github.hectorvent.floci.core.common.AwsException;+import java.util.List;+import java.util.Map;+import static org.mockito.ArgumentMatchers.*;+import static org.mockito.Mockito.*;+import org.mockito.ArgumentCaptor;import org.junit.jupiter.api.Test;import static org.junit.jupiter.api.Assertions.*;@@ -18,13 +29,16 @@ import static org.mockito.Mockito.mock;class ElastiCacheQueryHandlerTest {private ElastiCacheQueryHandler handler;+ private ElastiCacheService service;@BeforeEachvoid setUp() {SigV4Validator sigV4Validator = mock(SigV4Validator.class);- ElastiCacheService service = mock(ElastiCacheService.class);+ service = mock(ElastiCacheService.class);ElastiCacheMemcachedService memcachedService = mock(ElastiCacheMemcachedService.class);RegionResolver regionResolver = mock(RegionResolver.class);+ when(regionResolver.getRegion()).thenReturn("us-east-1");+ when(regionResolver.getAccountId()).thenReturn("000000000000");handler = new ElastiCacheQueryHandler(sigV4Validator, service, memcachedService, regionResolver);}@@ -61,4 +75,124 @@ class ElastiCacheQueryHandlerTest {private static MultivaluedMap<String, String> params() {return new MultivaluedHashMap<>();}++ private static ReplicationGroup group(String id) {+ ReplicationGroup g = new ReplicationGroup(id, "d", ReplicationGroupStatus.AVAILABLE, AuthMode.NO_AUTH,+ new Endpoint("localhost", 6379), java.time.Instant.now(), 6379);+ g.setArn("arn:aws:elasticache:us-east-1:000000000000:replicationgroup:" + id);+ return g;+ }++ @Test+ void createReplicationGroup_passesSettingsAndTagsToService() {+ when(service.createReplicationGroup(any(), any(), any(), any(), any(), any(), any()))+ .thenReturn(group("g1"));+ MultivaluedMap<String, String> p = params();+ p.add("ReplicationGroupId", "g1");+ p.add("ReplicationGroupDescription", "d");+ p.add("AtRestEncryptionEnabled", "true");+ p.add("KmsKeyId", "alias/cache");+ p.add("SnapshotRetentionLimit", "7");+ p.add("SnapshotWindow", "06:30-07:30");+ p.add("Tags.Tag.1.Key", "Name");+ p.add("Tags.Tag.1.Value", "g1");++ assertEquals(200, handler.handle("CreateReplicationGroup", p, "us-east-1").getStatus());++ verify(service).createReplicationGroup(eq("g1"), eq("d"), eq(AuthMode.NO_AUTH), isNull(), eq("us-east-1"),+ eq(new ReplicationGroupSettings(true, "alias/cache", 7, "06:30-07:30")), eq(Map.of("Name", "g1")));+ }++ @Test+ void createReplicationGroup_readsTheEncryptionFlagAsAwsDoes() {+ when(service.createReplicationGroup(any(), any(), any(), any(), any(), any(), any()))+ .thenReturn(group("g1"));+ ArgumentCaptor<ReplicationGroupSettings> captor = ArgumentCaptor.forClass(ReplicationGroupSettings.class);+ // a live account reads anything but "false" as true — probed with banana, yes and "TRUE "+ for (String value : new String[] {"true", "banana", "yes", "TRUE "}) {+ MultivaluedMap<String, String> p = params();+ p.add("ReplicationGroupId", "g1");+ p.add("AtRestEncryptionEnabled", value);+ handler.handle("CreateReplicationGroup", p, "us-east-1");+ }+ for (String value : new String[] {"false", "FALSE"}) {+ MultivaluedMap<String, String> p = params();+ p.add("ReplicationGroupId", "g1");+ p.add("AtRestEncryptionEnabled", value);+ handler.handle("CreateReplicationGroup", p, "us-east-1");+ }+ MultivaluedMap<String, String> omitted = params();+ omitted.add("ReplicationGroupId", "g1");+ handler.handle("CreateReplicationGroup", omitted, "us-east-1");++ verify(service, times(7)).createReplicationGroup(any(), any(), any(), any(), any(), captor.capture(), any());+ List<Boolean> seen = captor.getAllValues().stream().map(ReplicationGroupSettings::atRestEncryptionEnabled).toList();+ assertEquals(java.util.Arrays.asList(true, true, true, true, false, false, null), seen);+ }++ @Test+ void describeReplicationGroups_emitsStoredSettingsAndArn() {+ ReplicationGroup g = group("g1");+ g.setAtRestEncryptionEnabled(true);+ g.setKmsKeyId("arn:aws:kms:us-east-1:000000000000:key/k1");+ g.setSnapshotRetentionLimit(7);+ g.setSnapshotWindow("06:30-07:30");+ when(service.listReplicationGroups("g1")).thenReturn(List.of(g));+ MultivaluedMap<String, String> p = params();+ p.add("ReplicationGroupId", "g1");++ String body = (String) handler.handle("DescribeReplicationGroups", p, "us-east-1").getEntity();++ assertTrue(body.contains("<AtRestEncryptionEnabled>true</AtRestEncryptionEnabled>"), body);+ assertTrue(body.contains("<KmsKeyId>arn:aws:kms:us-east-1:000000000000:key/k1</KmsKeyId>"), body);+ assertTrue(body.contains("<SnapshotRetentionLimit>7</SnapshotRetentionLimit>"), body);+ assertTrue(body.contains("<SnapshotWindow>06:30-07:30</SnapshotWindow>"), body);+ assertTrue(body.contains("<ARN>arn:aws:elasticache:us-east-1:000000000000:replicationgroup:g1</ARN>"), body);+ }++ @Test+ void listTagsForResource_readsAReplicationGroupByItsArn() {+ ReplicationGroup g = group("g1");+ g.setTags(new java.util.LinkedHashMap<>(Map.of("Name", "g1")));+ when(service.getReplicationGroup("g1")).thenReturn(g);+ when(service.getReplicationGroup("absent")).thenThrow(+ new AwsException("ReplicationGroupNotFoundFault", "Replication group absent not found.", 404));++ MultivaluedMap<String, String> p = params();+ p.add("ResourceName", "arn:aws:elasticache:us-east-1:000000000000:replicationgroup:g1");+ String body = (String) handler.handle("ListTagsForResource", p, "us-east-1").getEntity();+ assertTrue(body.contains("<Key>Name</Key>"), body);++ p = params();+ p.add("ResourceName", "arn:aws:elasticache:us-east-1:000000000000:replicationgroup:absent");+ Response response = handler.handle("ListTagsForResource", p, "us-east-1");+ assertEquals(404, response.getStatus());+ assertTrue(((String) response.getEntity()).contains("ReplicationGroupNotFoundFault"));++ // the store keys groups by id alone: a same-named group created under another region is+ // not the one this ARN names+ ReplicationGroup elsewhere = group("g2");+ elsewhere.setArn("arn:aws:elasticache:eu-west-1:000000000000:replicationgroup:g2");+ elsewhere.setTags(new java.util.LinkedHashMap<>(Map.of("Name", "west")));+ when(service.getReplicationGroup("g2")).thenReturn(elsewhere);+ p = params();+ p.add("ResourceName", "arn:aws:elasticache:us-east-1:000000000000:replicationgroup:g2");+ response = handler.handle("ListTagsForResource", p, "us-east-1");+ assertEquals(404, response.getStatus(), (String) response.getEntity());+ assertFalse(((String) response.getEntity()).contains("west"));+ }++ @Test+ void modifyReplicationGroup_passesSnapshotSettingsOnly() {+ when(service.modifyReplicationGroup(eq("g1"), isNull(), isNull(), any())).thenReturn(group("g1"));+ MultivaluedMap<String, String> p = params();+ p.add("ReplicationGroupId", "g1");+ p.add("SnapshotRetentionLimit", "3");+ p.add("SnapshotWindow", "01:00-02:00");+ p.add("AtRestEncryptionEnabled", "true");+ p.add("KmsKeyId", "alias/other");++ assertEquals(200, handler.handle("ModifyReplicationGroup", p, "us-east-1").getStatus());+ verify(service).modifyReplicationGroup("g1", null, null, new ReplicationGroupSettings(null, null, 3, "01:00-02:00"));+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.javaindex ea3321c008..f1ffdc7269 100644--- a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java@@ -6,6 +6,10 @@ import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.services.ec2.Ec2Service;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.services.kms.KmsService;+import io.github.hectorvent.floci.services.kms.model.KmsKey;+import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroupSettings;import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerHandle;import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerManager;import io.github.hectorvent.floci.services.elasticache.model.AuthMode;@@ -15,11 +19,13 @@ import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import java.util.List;+import java.util.Map;import java.util.concurrent.CountDownLatch;import java.util.concurrent.TimeUnit;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNull;import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;@@ -36,15 +42,17 @@ import static org.mockito.Mockito.when;class ElastiCacheServiceTest {private ElastiCacheService service;+ private KmsService kmsService;private ElastiCacheContainerManager containerManager;private ElastiCacheProxyManager proxyManager;+ private EmulatorConfig config;@BeforeEachvoid setUp() {containerManager = mock(ElastiCacheContainerManager.class);proxyManager = mock(ElastiCacheProxyManager.class);StorageFactory storageFactory = mock(StorageFactory.class);- EmulatorConfig config = mock(EmulatorConfig.class);+ config = mock(EmulatorConfig.class);EmulatorConfig.ServicesConfig servicesConfig = mock(EmulatorConfig.ServicesConfig.class);EmulatorConfig.ElastiCacheServiceConfig ecConfig = mock(EmulatorConfig.ElastiCacheServiceConfig.class);@@ -60,8 +68,11 @@ class ElastiCacheServiceTest {.thenReturn(new ElastiCacheContainerHandle("cid", "grp", "localhost", 6379));doNothing().when(proxyManager).startProxy(anyString(), any(), anyInt(), anyString(), anyInt(), any());Ec2Service ec2Service = org.mockito.Mockito.mock(Ec2Service.class);+ kmsService = org.mockito.Mockito.mock(KmsService.class);+ when(kmsService.describeKey(any(), any())).thenThrow(+ new AwsException("NotFoundException", "Key not found", 404));service = new ElastiCacheService(containerManager, proxyManager, storageFactory, config,- ec2Service, new RegionResolver("us-east-1", "000000000000"));+ ec2Service, new RegionResolver("us-east-1", "000000000000"), kmsService);}@Test@@ -88,7 +99,8 @@ class ElastiCacheServiceTest {doNothing().when(pm).startProxy(anyString(), any(), anyInt(), anyString(), anyInt(), any());ElastiCacheService svc = new ElastiCacheService(cm, pm, sf, cfg,org.mockito.Mockito.mock(Ec2Service.class),- new RegionResolver("us-east-1", "000000000000"));+ new RegionResolver("us-east-1", "000000000000"),+ org.mockito.Mockito.mock(KmsService.class));svc.createReplicationGroup("g1", "d", AuthMode.PASSWORD, null, "us-east-1");@@ -221,4 +233,158 @@ class ElastiCacheServiceTest {assertEquals("grp", service.getReplicationGroup("grp").getReplicationGroupId());}++ private static final String KEY_ARN = "arn:aws:kms:us-east-1:000000000000:key/k1";++ private KmsKey knownKey(String... forms) {+ KmsKey key = new KmsKey();+ key.setKeyId("k1");+ key.setArn(KEY_ARN);+ key.setEnabled(true);+ key.setKeyState("Enabled");+ for (String form : forms) {+ org.mockito.Mockito.doReturn(key).when(kmsService).describeKey(form, "us-east-1");+ }+ return key;+ }++ @Test+ void createReplicationGroupStoresEncryptionSnapshotSettingsAndTags() {+ knownKey("alias/cache");+ ReplicationGroup group = service.createReplicationGroup("g1", "d", AuthMode.NO_AUTH, null, "us-east-1",+ new ReplicationGroupSettings(true, "alias/cache", 7, "06:30-07:30"),+ Map.of("Name", "g1", "env", "tst"));++ ReplicationGroup stored = service.getReplicationGroup("g1");+ assertTrue(stored.isAtRestEncryptionEnabled());+ assertEquals(KEY_ARN, stored.getKmsKeyId());+ assertEquals(7, stored.getSnapshotRetentionLimit());+ assertEquals("06:30-07:30", stored.getSnapshotWindow());+ assertEquals(Map.of("Name", "g1", "env", "tst"), stored.getTags());+ assertEquals("arn:aws:elasticache:us-east-1:000000000000:replicationgroup:g1", group.getArn());+ }++ @Test+ void createReplicationGroupWithoutSettingsKeepsAwsDefaults() {+ service.createReplicationGroup("g1", "d", AuthMode.NO_AUTH, null, "us-east-1");+ ReplicationGroup stored = service.getReplicationGroup("g1");+ assertFalse(stored.isAtRestEncryptionEnabled());+ assertNull(stored.getKmsKeyId());+ assertEquals(0, stored.getSnapshotRetentionLimit());+ assertTrue(stored.getTags().isEmpty());+ assertEquals(ReplicationGroupSettings.DEFAULT_SNAPSHOT_WINDOW, stored.getSnapshotWindow());+ }++ @Test+ void createReplicationGroupRejectsAKeyItCannotUseBeforeStartingAContainer() {+ AwsException missing = assertThrows(AwsException.class, () -> service.createReplicationGroup(+ "g1", "d", AuthMode.NO_AUTH, null, "us-east-1",+ new ReplicationGroupSettings(true, "alias/does-not-exist", null, null), Map.of()));+ assertEquals("InvalidParameterValue", missing.getErrorCode());+ assertEquals("KMS key does not exist with key id: alias/does-not-exist", missing.getMessage());+ assertThrows(AwsException.class, () -> service.getReplicationGroup("g1"));+ org.mockito.Mockito.verify(containerManager, org.mockito.Mockito.never()).start(anyString(), anyString());++ AwsException combination = assertThrows(AwsException.class, () -> service.createReplicationGroup(+ "g1", "d", AuthMode.NO_AUTH, null, "us-east-1",+ new ReplicationGroupSettings(false, KEY_ARN, null, null), Map.of()));+ assertEquals("InvalidParameterCombination", combination.getErrorCode());+ assertEquals("Please enable encryption at rest to use Customer Managed CMK", combination.getMessage());+ // leaving AtRestEncryptionEnabled out is false on a live account, refused the same way+ AwsException omitted = assertThrows(AwsException.class, () -> service.createReplicationGroup(+ "g1", "d", AuthMode.NO_AUTH, null, "us-east-1",+ new ReplicationGroupSettings(null, KEY_ARN, null, null), Map.of()));+ assertEquals("InvalidParameterCombination", omitted.getErrorCode());++ AwsException retention = assertThrows(AwsException.class, () -> service.createReplicationGroup(+ "g1", "d", AuthMode.NO_AUTH, null, "us-east-1",+ new ReplicationGroupSettings(null, null, 36, null), Map.of()));+ assertEquals("Invalid snapshot retention limit: 36. Retention limit must be between 0 and 35.", retention.getMessage());+ AwsException window = assertThrows(AwsException.class, () -> service.createReplicationGroup(+ "g1", "d", AuthMode.NO_AUTH, null, "us-east-1",+ new ReplicationGroupSettings(null, null, null, "25:00-26:00"), Map.of()));+ assertEquals("Invalid backup window format. Should be specified as a range hh24:mi-hh24:mi (24H Clock UTC). Example: 03:15-08:15", window.getMessage());+ AwsException shortWindow = assertThrows(AwsException.class, () -> service.createReplicationGroup(+ "g1", "d", AuthMode.NO_AUTH, null, "us-east-1",+ new ReplicationGroupSettings(null, null, null, "05:00-05:30"), Map.of()));+ assertEquals("Snapshot window must be at least 60 minutes.", shortWindow.getMessage());+ // equal start and end is an empty window, not a full day+ AwsException emptyWindow = assertThrows(AwsException.class, () -> service.createReplicationGroup(+ "g1", "d", AuthMode.NO_AUTH, null, "us-east-1",+ new ReplicationGroupSettings(null, null, null, "05:00-05:00"), Map.of()));+ assertEquals("Snapshot window must be at least 60 minutes.", emptyWindow.getMessage());+ // while a window wrapping midnight is measured across it+ service.createReplicationGroup("wrap", "d", AuthMode.NO_AUTH, null, "us-east-1",+ new ReplicationGroupSettings(null, null, null, "23:30-00:30"), Map.of());+ assertEquals("23:30-00:30", service.getReplicationGroup("wrap").getSnapshotWindow());+ }++ @Test+ void modifyReplicationGroupChangesSnapshotSettingsAndKeepsEncryption() {+ knownKey(KEY_ARN);+ service.createReplicationGroup("g1", "d", AuthMode.NO_AUTH, null, "us-east-1",+ new ReplicationGroupSettings(true, KEY_ARN, 7, "06:30-07:30"), Map.of());++ service.modifyReplicationGroup("g1", null, null, new ReplicationGroupSettings(null, null, 3, "01:00-02:00"));++ ReplicationGroup stored = service.getReplicationGroup("g1");+ assertEquals(3, stored.getSnapshotRetentionLimit());+ assertEquals("01:00-02:00", stored.getSnapshotWindow());+ assertTrue(stored.isAtRestEncryptionEnabled());+ assertEquals(KEY_ARN, stored.getKmsKeyId());++ assertThrows(AwsException.class, () -> service.modifyReplicationGroup("g1", null, null,+ new ReplicationGroupSettings(null, null, null, "25:00-26:00")));+ assertEquals("01:00-02:00", service.getReplicationGroup("g1").getSnapshotWindow());++ // a refusal later in the same request must not leave the earlier part applied: the store+ // hands out its own object, so settings applied before the user check would stay visible+ AwsException unknownUser = assertThrows(AwsException.class, () -> service.modifyReplicationGroup(+ "g1", List.of("no-such-user"), null, new ReplicationGroupSettings(null, null, 9, "03:00-04:00")));+ assertEquals("UserNotFoundFault", unknownUser.getErrorCode());+ assertEquals(3, service.getReplicationGroup("g1").getSnapshotRetentionLimit());+ assertEquals("01:00-02:00", service.getReplicationGroup("g1").getSnapshotWindow());+ }++ @Test+ void modifyReplicationGroupCannotWriteAGroupBackAfterDelete() throws Exception {+ // modify has read the group, delete removes it, modify writes its copy back — the store is+ // held inside modify's put so the delete can be run in exactly that window+ PausingStorageBackend<ReplicationGroup> pausing = new PausingStorageBackend<>(new InMemoryStorage<>());+ StorageFactory factory = org.mockito.Mockito.mock(StorageFactory.class);+ when(factory.create(anyString(), eq("elasticache-groups.json"), any()))+ .thenAnswer(inv -> new AccountAwareStorageBackend<>(pausing, null, "000000000000"));+ when(factory.create(anyString(), org.mockito.ArgumentMatchers.argThat(f -> !"elasticache-groups.json".equals(f)), any()))+ .thenAnswer(inv -> AccountAwareStorageBackend.inMemory("000000000000"));+ ElastiCacheService svc = new ElastiCacheService(containerManager, proxyManager, factory, config,+ org.mockito.Mockito.mock(Ec2Service.class), new RegionResolver("us-east-1", "000000000000"), kmsService);+ svc.createReplicationGroup("g1", "d", AuthMode.NO_AUTH, null, "us-east-1");++ pausing.pauseOn(PausingStorageBackend.Call.PUT, "g1");+ java.util.concurrent.atomic.AtomicReference<Throwable> modifyOutcome = new java.util.concurrent.atomic.AtomicReference<>();+ Thread modify = new Thread(() -> {+ try {+ svc.modifyReplicationGroup("g1", null, null, new ReplicationGroupSettings(null, null, 3, null));+ } catch (Throwable t) {+ modifyOutcome.set(t);+ }+ });+ modify.start();+ pausing.awaitReached();++ Thread delete = new Thread(() -> svc.deleteReplicationGroup("g1"));+ delete.start();+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);+ while (delete.getState() != Thread.State.BLOCKED && delete.getState() != Thread.State.TERMINATED) {+ assertTrue(System.nanoTime() < deadline, "delete neither ran nor queued");+ Thread.onSpinWait();+ }+ pausing.release();+ modify.join(5000);+ delete.join(5000);++ assertNull(modifyOutcome.get(), "modify completed before the delete");+ assertThrows(AwsException.class, () -> svc.getReplicationGroup("g1"),+ "the deleted group must not come back from the modify");+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/PausingStorageBackend.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/PausingStorageBackend.javanew file mode 100644index 0000000000..40e6fd7b86--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/PausingStorageBackend.java@@ -0,0 +1,123 @@+package io.github.hectorvent.floci.services.elasticache;++import io.github.hectorvent.floci.core.storage.StorageBackend;++import java.util.List;+import java.util.Optional;+import java.util.Set;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.TimeUnit;+import java.util.function.Predicate;++/**+ * A store that can be stopped in the middle of an operation, so a race can be staged rather than+ * hoped for.+ *+ * <p>A concurrency test that loops and waits for an unlucky interleaving proves nothing when the+ * window is a few instructions wide: it passes with the fix and without it. Holding one thread at a+ * chosen call while another runs to completion makes the interleaving the test claims to cover the+ * one it actually runs.+ */+final class PausingStorageBackend<V> implements StorageBackend<String, V> {++ /** Which call to hold, and on which key. */+ enum Call { PUT, DELETE, SCAN }++ private final StorageBackend<String, V> delegate;+ private volatile Call pauseOn;+ private volatile String pauseKey;+ private final java.util.concurrent.atomic.AtomicInteger skip =+ new java.util.concurrent.atomic.AtomicInteger();+ private final CountDownLatch reached = new CountDownLatch(1);+ private final CountDownLatch release = new CountDownLatch(1);++ PausingStorageBackend(StorageBackend<String, V> delegate) {+ this.delegate = delegate;+ }++ /** Hold the next matching call until {@link #release()}; {@code key} may be null for SCAN. */+ void pauseOn(Call call, String key) {+ pauseOn(call, key, 0);+ }++ /** As above, after letting {@code skipFirst} matching calls through untouched. */+ void pauseOn(Call call, String key, int skipFirst) {+ this.pauseOn = call;+ this.pauseKey = key;+ this.skip.set(skipFirst);+ }++ /** Waits until the paused call has been reached. */+ void awaitReached() throws InterruptedException {+ if (!reached.await(10, TimeUnit.SECONDS)) {+ throw new IllegalStateException("the paused call was never reached");+ }+ }++ void release() {+ release.countDown();+ }++ private void pauseIfMatching(Call call, String key) {+ // Keys arrive account-prefixed here, since this sits underneath the account-aware layer.+ if (pauseOn != call || (pauseKey != null && (key == null || !key.endsWith(pauseKey)))) {+ return;+ }+ if (skip.getAndDecrement() > 0) {+ return;+ }+ pauseOn = null;+ reached.countDown();+ try {+ if (!release.await(10, TimeUnit.SECONDS)) {+ throw new IllegalStateException("the paused call was never released");+ }+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new IllegalStateException(e);+ }+ }++ @Override+ public void put(String key, V value) {+ pauseIfMatching(Call.PUT, key);+ delegate.put(key, value);+ }++ @Override+ public Optional<V> get(String key) {+ return delegate.get(key);+ }++ @Override+ public void delete(String key) {+ pauseIfMatching(Call.DELETE, key);+ delegate.delete(key);+ }++ @Override+ public List<V> scan(Predicate<String> keyFilter) {+ pauseIfMatching(Call.SCAN, null);+ return delegate.scan(keyFilter);+ }++ @Override+ public Set<String> keys() {+ return delegate.keys();+ }++ @Override+ public void flush() {+ delegate.flush();+ }++ @Override+ public void load() {+ delegate.load();+ }++ @Override+ public void clear() {+ delegate.clear();+ }+}