Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(elasticache): keep a replication group's encryption, snapshot settings, tags and ARN (#2632)

Michal Schreiber75f8f42d5fbd
diff --git a/compatibility-tests/sdk-test-awscli/test/elasticache.bats b/compatibility-tests/sdk-test-awscli/test/elasticache.bats
index a6a02ce50f..04440ce528 100644
--- a/compatibility-tests/sdk-test-awscli/test/elasticache.bats
+++ b/compatibility-tests/sdk-test-awscli/test/elasticache.bats
@@ -137,3 +137,41 @@ teardown() {
run aws_cmd elasticache delete-cache-subnet-group --cache-subnet-group-name "$SNG_EU"
assert_success
}
+
+@test "ElastiCache: replication group keeps its encryption, snapshot settings and tags" {
+ RG="bats-rg-$(unique_name)"
+ run aws_cmd kms create-key --description "$RG"
+ assert_success
+ KEY_ARN=$(json_get "$output" '.KeyMetadata.Arn')
+ KEY_ID=$(json_get "$output" '.KeyMetadata.KeyId')
+ aws_cmd kms create-alias --alias-name "alias/$RG" --target-key-id "$KEY_ID" >/dev/null
+
+ run aws_cmd elasticache create-replication-group --replication-group-id "$RG" --replication-group-description d \
+ --engine redis --cache-node-type cache.t3.micro --num-cache-clusters 1 \
+ --at-rest-encryption-enabled --kms-key-id "alias/$RG" \
+ --snapshot-retention-limit 7 --snapshot-window 06:30-07:30 --tags "Key=Name,Value=$RG"
+ assert_success
+
+ run aws_cmd elasticache describe-replication-groups --replication-group-id "$RG" \
+ --query 'ReplicationGroups[0].[KmsKeyId,AtRestEncryptionEnabled,SnapshotRetentionLimit,SnapshotWindow,ARN]'
+ assert_success
+ assert_output --partial "$KEY_ARN"
+ assert_output --partial 'true'
+ assert_output --partial '7'
+ assert_output --partial '"06:30-07:30"'
+ assert_output --partial "replicationgroup:$RG"
+ ARN=$(aws_cmd elasticache describe-replication-groups --replication-group-id "$RG" --query 'ReplicationGroups[0].ARN' --output text)
+
+ run aws_cmd elasticache list-tags-for-resource --resource-name "$ARN"
+ assert_success
+ assert_output --partial "\"Value\": \"$RG\""
+
+ run aws_cmd elasticache modify-replication-group --replication-group-id "$RG" --snapshot-retention-limit 3 --snapshot-window 01:00-02:00 --apply-immediately
+ assert_success
+ run aws_cmd elasticache describe-replication-groups --replication-group-id "$RG" --query 'ReplicationGroups[0].[SnapshotRetentionLimit,SnapshotWindow]'
+ assert_success
+ assert_output --partial '3'
+ assert_output --partial '"01:00-02:00"'
+
+ aws_cmd elasticache delete-replication-group --replication-group-id "$RG" >/dev/null 2>&1 || true
+}
diff --git a/docs/services/elasticache.md b/docs/services/elasticache.md
index beaee49d2a..6f15199001 100644
--- a/docs/services/elasticache.md
+++ b/docs/services/elasticache.md
@@ -12,9 +12,9 @@ Floci manages real Valkey/Redis Docker containers and proxies TCP connections to
| Action | Description |
| --- | --- |
| `ValidateIamAuthToken` | Validate an IAM auth token (data-plane auth) |
-| `CreateReplicationGroup` | Start a new Redis/Valkey cluster |
+| `CreateReplicationGroup` | Start a new Redis/Valkey cluster; `AtRestEncryptionEnabled`, `KmsKeyId` (resolved to the key ARN), `SnapshotRetentionLimit`, `SnapshotWindow` and `Tags` are kept and returned, with the group `ARN` |
| `DescribeReplicationGroups` | List clusters and their connection info |
-| `ModifyReplicationGroup` | - |
+| `ModifyReplicationGroup` | Modify `SnapshotRetentionLimit` and `SnapshotWindow`, and the associated user groups |
| `DeleteReplicationGroup` | Stop and remove a cluster |
| `CreateUser` | Create an ElastiCache IAM user |
| `DescribeUsers` | List ElastiCache users |
diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.java
index a20ae29c3d..67e8750516 100644
--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandler.java
@@ -13,6 +13,7 @@ import io.github.hectorvent.floci.services.elasticache.model.CacheSubnetGroup;
import io.github.hectorvent.floci.services.elasticache.model.ElastiCacheUser;
import io.github.hectorvent.floci.services.elasticache.model.Endpoint;
import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroup;
+import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroupSettings;
import io.github.hectorvent.floci.services.elasticache.proxy.SigV4Validator;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
@@ -105,7 +106,8 @@ public class ElastiCacheQueryHandler {
try {
ReplicationGroup group = service.createReplicationGroup(
- groupId, description != null ? description : "", authMode, authToken, region);
+ groupId, description != null ? description : "", authMode, authToken, region,
+ replicationGroupSettings(params), parseTags(params));
String result = replicationGroupXml(group);
return Response.ok(AwsQueryResponse.envelope("CreateReplicationGroup", AwsNamespaces.EC, result)).build();
} catch (AwsException e) {
@@ -113,6 +115,28 @@ public class ElastiCacheQueryHandler {
}
}
+ private static ReplicationGroupSettings replicationGroupSettings(MultivaluedMap<String, String> params) {
+ String atRest = params.getFirst("AtRestEncryptionEnabled");
+ // A live account reads any value that is not "false" as true — "banana", "yes" and
+ // "TRUE " all created an encrypted group — so the flag is read the same way here.
+ return new ReplicationGroupSettings(
+ atRest == null ? null : !"false".equalsIgnoreCase(atRest.trim()),
+ params.getFirst("KmsKeyId"),
+ optionalInt(params.getFirst("SnapshotRetentionLimit")),
+ params.getFirst("SnapshotWindow"));
+ }
+
+ private static Integer optionalInt(String value) {
+ if (value == null || value.isBlank()) {
+ return null;
+ }
+ try {
+ return Integer.parseInt(value.trim());
+ } catch (NumberFormatException e) {
+ throw new AwsException("InvalidParameterValue", "Value " + value + " is not a valid integer.", 400);
+ }
+ }
+
private Response handleDescribeReplicationGroups(MultivaluedMap<String, String> params) {
String filterId = params.getFirst("ReplicationGroupId");
try {
@@ -153,9 +177,12 @@ public class ElastiCacheQueryHandler {
List<String> userIdsToAdd = extractMemberList(params, "UserGroupIdsToAdd.member.");
List<String> userIdsToRemove = extractMemberList(params, "UserGroupIdsToRemove.member.");
try {
+ // AtRestEncryptionEnabled and KmsKeyId are fixed at create and not in the modify shape
+ ReplicationGroupSettings settings = new ReplicationGroupSettings(null, null,
+ optionalInt(params.getFirst("SnapshotRetentionLimit")), params.getFirst("SnapshotWindow"));
ReplicationGroup group = service.modifyReplicationGroup(groupId,
userIdsToAdd.isEmpty() ? null : userIdsToAdd,
- userIdsToRemove.isEmpty() ? null : userIdsToRemove);
+ userIdsToRemove.isEmpty() ? null : userIdsToRemove, settings);
String result = replicationGroupXml(group);
return Response.ok(AwsQueryResponse.envelope("ModifyReplicationGroup", AwsNamespaces.EC, result)).build();
} catch (AwsException e) {
@@ -478,6 +505,15 @@ private Response handleCreateCacheParameterGroup(MultivaluedMap<String, String>
}
Map<String, String> tags = Map.of();
+ if ("replicationgroup".equals(arn[5])) {
+ // the store keys groups by id alone; the record must be the one the ARN names
+ ReplicationGroup group = service.getReplicationGroup(arn[6]);
+ if (group.getArn() != null && !group.getArn().equalsIgnoreCase(resourceName)) {
+ throw new AwsException("ReplicationGroupNotFoundFault",
+ "Replication group " + arn[6] + " not found.", 404);
+ }
+ tags = group.getTags();
+ }
if ("subnetgroup".equals(arn[5])) {
tags = service.describeCacheSubnetGroups(arn[6]).getFirst().getTags();
}
@@ -611,11 +647,19 @@ private Response handleCreateCacheParameterGroup(MultivaluedMap<String, String>
.elem("Status", g.getStatus().name().toLowerCase())
.elem("AuthTokenEnabled", authTokenEnabled)
.elem("TransitEncryptionEnabled", authTokenEnabled)
- .elem("AtRestEncryptionEnabled", false)
+ .elem("AtRestEncryptionEnabled", g.isAtRestEncryptionEnabled())
.elem("ClusterEnabled", false)
.elem("MultiAZ", "disabled")
.elem("AutomaticFailover", "disabled")
- .elem("SnapshotRetentionLimit", 0L);
+ .elem("SnapshotRetentionLimit", (long) g.getSnapshotRetentionLimit());
+ xml.elem("SnapshotWindow", g.getSnapshotWindow() != null
+ ? g.getSnapshotWindow() : ReplicationGroupSettings.DEFAULT_SNAPSHOT_WINDOW);
+ if (g.getKmsKeyId() != null) {
+ xml.elem("KmsKeyId", g.getKmsKeyId());
+ }
+ if (g.getArn() != null) {
+ xml.elem("ARN", g.getArn());
+ }
if (ep != null) {
xml.start("ConfigurationEndpoint")
.elem("Address", ep.address())
diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java
index 2bb19d0dab..f009664c9d 100644
--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheService.java
@@ -29,6 +29,9 @@ import org.jboss.logging.Logger;
import java.time.Instant;
import java.util.ArrayList;
import java.util.Collection;
+import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroupSettings;
+import io.github.hectorvent.floci.services.kms.KmsService;
+import io.github.hectorvent.floci.services.kms.model.KmsKey;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Locale;
@@ -53,6 +56,7 @@ public class ElastiCacheService implements ResourceProvider {
private final ElastiCacheContainerManager containerManager;
private final ElastiCacheProxyManager proxyManager;
private final EmulatorConfig config;
+ private final KmsService kmsService;
private final Ec2Service ec2Service;
private final RegionResolver regionResolver;
private final Set<Integer> usedPorts = ConcurrentHashMap.newKeySet();
@@ -65,7 +69,9 @@ public class ElastiCacheService implements ResourceProvider {
StorageFactory storageFactory,
EmulatorConfig config,
Ec2Service ec2Service,
- RegionResolver regionResolver) {
+ RegionResolver regionResolver,
+ KmsService kmsService) {
+ this.kmsService = kmsService;
this.containerManager = containerManager;
this.proxyManager = proxyManager;
this.config = config;
@@ -83,6 +89,18 @@ public class ElastiCacheService implements ResourceProvider {
public ReplicationGroup createReplicationGroup(String groupId, String description,
AuthMode authMode, String authToken, String region) {
+ return createReplicationGroup(groupId, description, authMode, authToken, region,
+ ReplicationGroupSettings.defaults(), Map.of());
+ }
+
+ public ReplicationGroup createReplicationGroup(String groupId, String description,
+ AuthMode authMode, String authToken, String region,
+ ReplicationGroupSettings settings,
+ Map<String, String> tags) {
+ settings.validate();
+ // resolved with the other validations, before a port is taken or a container started
+ ReplicationGroupSettings resolvedSettings =
+ settings.withKmsKeyId(resolveKmsKeyArn(settings.kmsKeyId(), region));
if (groups.get(groupId).isPresent()) {
throw new AwsException("ReplicationGroupAlreadyExistsFault",
"Replication group " + groupId + " already exists.", 400);
@@ -116,6 +134,11 @@ public class ElastiCacheService implements ResourceProvider {
group.setAuthToken(authToken);
group.setArn(regionResolver.buildArn("elasticache", region, "replicationgroup:" + groupId));
group.setRegion(region);
+ group.setSnapshotWindow(ReplicationGroupSettings.DEFAULT_SNAPSHOT_WINDOW);
+ resolvedSettings.applyTo(group);
+ if (tags != null && !tags.isEmpty()) {
+ group.setTags(new LinkedHashMap<>(tags));
+ }
proxyManager.startProxy(groupId, authMode, proxyPort,
handle.getHost(), handle.getPort(),
@@ -173,41 +196,88 @@ public class ElastiCacheService implements ResourceProvider {
}
public void deleteReplicationGroup(String groupId) {
- ReplicationGroup group = groups.get(groupId).orElseThrow(() ->
- new AwsException("ReplicationGroupNotFoundFault",
- "Replication group " + groupId + " not found.", 404));
+ // one monitor per group for delete and modify: a modify's read-modify-write could
+ // otherwise write the group back after this removed it
+ synchronized (lockFor("rg:" + groupId)) {
+ ReplicationGroup group = groups.get(groupId).orElseThrow(() ->
+ new AwsException("ReplicationGroupNotFoundFault",
+ "Replication group " + groupId + " not found.", 404));
- group.setStatus(ReplicationGroupStatus.DELETING);
- groups.put(groupId, group);
+ group.setStatus(ReplicationGroupStatus.DELETING);
+ groups.put(groupId, group);
- proxyManager.stopProxy(groupId);
+ proxyManager.stopProxy(groupId);
- if (group.getContainerId() != null) {
- containerManager.stop(new ElastiCacheContainerHandle(
- group.getContainerId(), groupId, group.getContainerHost(), group.getContainerPort()));
- }
+ if (group.getContainerId() != null) {
+ containerManager.stop(new ElastiCacheContainerHandle(
+ group.getContainerId(), groupId, group.getContainerHost(), group.getContainerPort()));
+ }
- releaseProxyPort(group.getProxyPort());
- groups.delete(groupId);
- LOG.infov("Replication group {0} deleted", groupId);
+ releaseProxyPort(group.getProxyPort());
+ groups.delete(groupId);
+ LOG.infov("Replication group {0} deleted", groupId);
+ }
}
public ReplicationGroup modifyReplicationGroup(String groupId, List<String> userIdsToAdd,
List<String> userIdsToRemove) {
- ReplicationGroup group = getReplicationGroup(groupId);
+ return modifyReplicationGroup(groupId, userIdsToAdd, userIdsToRemove,
+ ReplicationGroupSettings.unchanged());
+ }
- if (userIdsToAdd != null) {
- for (String userId : userIdsToAdd) {
- getUser(userId); // validate user exists
- group.getAssociatedUserIds().add(userId);
+ public ReplicationGroup modifyReplicationGroup(String groupId, List<String> userIdsToAdd,
+ List<String> userIdsToRemove,
+ ReplicationGroupSettings settings) {
+ settings.validate();
+ synchronized (lockFor("rg:" + groupId)) {
+ ReplicationGroup group = getReplicationGroup(groupId);
+ // every check before any change: the store hands out its own object, so a mutation
+ // made before a later refusal would stay visible
+ if (userIdsToAdd != null) {
+ for (String userId : userIdsToAdd) {
+ getUser(userId);
+ }
+ }
+ settings.applyTo(group);
+ if (userIdsToAdd != null) {
+ group.getAssociatedUserIds().addAll(userIdsToAdd);
}
+ if (userIdsToRemove != null) {
+ group.getAssociatedUserIds().removeAll(userIdsToRemove);
+ }
+
+ groups.put(groupId, group);
+ return group;
+ }
+ }
+
+ /**
+ * A live account takes the key as a key id, key ARN or alias and reports the key ARN on the
+ * group; a key it cannot use is one fault whatever the reason.
+ */
+ private String resolveKmsKeyArn(String kmsKeyId, String region) {
+ if (kmsKeyId == null || kmsKeyId.isBlank()) {
+ return null;
+ }
+ if (kmsService == null) {
+ throw new IllegalStateException("ElastiCacheService was built without a KmsService; "
+ + "a KmsKeyId cannot be resolved");
}
- if (userIdsToRemove != null) {
- group.getAssociatedUserIds().removeAll(userIdsToRemove);
+ KmsKey key;
+ try {
+ key = kmsService.describeKey(kmsKeyId, region);
+ } catch (AwsException e) {
+ throw kmsKeyNotAccessible(kmsKeyId);
+ }
+ if (!key.isEnabled() || "PendingDeletion".equals(key.getKeyState())) {
+ throw kmsKeyNotAccessible(kmsKeyId);
}
+ return key.getArn();
+ }
- groups.put(groupId, group);
- return group;
+ private static AwsException kmsKeyNotAccessible(String kmsKeyId) {
+ return new AwsException("InvalidParameterValue",
+ "KMS key does not exist with key id: " + kmsKeyId, 400);
}
public ElastiCacheUser createUser(String userId, String userName, AuthMode authMode,
diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroup.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroup.java
index ca2197913f..d2e418fd0a 100644
--- a/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroup.java
+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroup.java
@@ -4,6 +4,8 @@ import io.quarkus.runtime.annotations.RegisterForReflection;
import java.time.Instant;
import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.Map;
import java.util.Set;
@RegisterForReflection
@@ -20,6 +22,11 @@ public class ReplicationGroup {
private Set<String> associatedUserIds = new HashSet<>();
private String arn;
private String region;
+ private boolean atRestEncryptionEnabled;
+ private String kmsKeyId;
+ private int snapshotRetentionLimit;
+ private String snapshotWindow;
+ private Map<String, String> tags = new LinkedHashMap<>();
// Transient fields — not persisted, restored on container restart
private transient String containerId;
@@ -78,6 +85,21 @@ public class ReplicationGroup {
public int getContainerPort() { return containerPort; }
public void setContainerPort(int containerPort) { this.containerPort = containerPort; }
+ public boolean isAtRestEncryptionEnabled() { return atRestEncryptionEnabled; }
+ public void setAtRestEncryptionEnabled(boolean atRestEncryptionEnabled) { this.atRestEncryptionEnabled = atRestEncryptionEnabled; }
+
+ public String getKmsKeyId() { return kmsKeyId; }
+ public void setKmsKeyId(String kmsKeyId) { this.kmsKeyId = kmsKeyId; }
+
+ public int getSnapshotRetentionLimit() { return snapshotRetentionLimit; }
+ public void setSnapshotRetentionLimit(int snapshotRetentionLimit) { this.snapshotRetentionLimit = snapshotRetentionLimit; }
+
+ public String getSnapshotWindow() { return snapshotWindow; }
+ public void setSnapshotWindow(String snapshotWindow) { this.snapshotWindow = snapshotWindow; }
+
+ public Map<String, String> getTags() { return tags; }
+ public void setTags(Map<String, String> tags) { this.tags = tags == null ? new LinkedHashMap<>() : tags; }
+
public String getArn() { return arn; }
public void setArn(String arn) { this.arn = arn; }
diff --git a/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroupSettings.java b/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroupSettings.java
new file mode 100644
index 0000000000..61e7272a4a
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/elasticache/model/ReplicationGroupSettings.java
@@ -0,0 +1,100 @@
+package io.github.hectorvent.floci.services.elasticache.model;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.quarkus.runtime.annotations.RegisterForReflection;
+
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+/**
+ * The encryption and snapshot settings of a replication group as a request carries them: a null
+ * member is one the request left out — the AWS default on create, unchanged on modify.
+ */
+@RegisterForReflection
+public record ReplicationGroupSettings(Boolean atRestEncryptionEnabled,
+ String kmsKeyId,
+ Integer snapshotRetentionLimit,
+ String snapshotWindow) {
+
+ private static final Pattern TIME = Pattern.compile("^([01]\\d|2[0-3]):([0-5]\\d)$");
+ private static final int MINUTES_PER_DAY = 24 * 60;
+ private static final int MINIMUM_WINDOW_MINUTES = 60;
+
+ /** Where AWS picks a random one-hour window, Floci picks this. */
+ public static final String DEFAULT_SNAPSHOT_WINDOW = "00:00-01:00";
+
+ public static ReplicationGroupSettings defaults() {
+ return new ReplicationGroupSettings(null, null, null, null);
+ }
+
+ public static ReplicationGroupSettings unchanged() {
+ return defaults();
+ }
+
+ /** The checks a live account applies to these parameters, with its wording. */
+ public void validate() {
+ // an omitted AtRestEncryptionEnabled is false on a live account, and refused the same way
+ if (kmsKeyId != null && !kmsKeyId.isBlank() && !Boolean.TRUE.equals(atRestEncryptionEnabled)) {
+ throw new AwsException("InvalidParameterCombination",
+ "Please enable encryption at rest to use Customer Managed CMK", 400);
+ }
+ if (snapshotRetentionLimit != null && (snapshotRetentionLimit < 0 || snapshotRetentionLimit > 35)) {
+ throw new AwsException("InvalidParameterValue", "Invalid snapshot retention limit: "
+ + snapshotRetentionLimit + ". Retention limit must be between 0 and 35.", 400);
+ }
+ if (snapshotWindow != null) {
+ parseWindow(snapshotWindow);
+ }
+ }
+
+ private static void parseWindow(String window) {
+ String[] parts = window.split("-", -1);
+ if (parts.length != 2) {
+ throw invalidWindow(window);
+ }
+ int start = minuteOfDay(parts[0], window);
+ int end = minuteOfDay(parts[1], window);
+ // an end before the start wraps midnight; an end equal to the start is an empty window,
+ // which a live account refuses as shorter than an hour, not a full day
+ if (end < start) {
+ end += MINUTES_PER_DAY;
+ }
+ if (end - start < MINIMUM_WINDOW_MINUTES) {
+ throw new AwsException("InvalidParameterValue",
+ "Snapshot window must be at least " + MINIMUM_WINDOW_MINUTES + " minutes.", 400);
+ }
+ }
+
+ private static int minuteOfDay(String time, String window) {
+ Matcher m = TIME.matcher(time);
+ if (!m.matches()) {
+ throw invalidWindow(window);
+ }
+ return Integer.parseInt(m.group(1)) * 60 + Integer.parseInt(m.group(2));
+ }
+
+ private static AwsException invalidWindow(String window) {
+ return new AwsException("InvalidParameterValue", "Invalid backup window format. Should be "
+ + "specified as a range hh24:mi-hh24:mi (24H Clock UTC). Example: 03:15-08:15", 400);
+ }
+
+ public ReplicationGroupSettings withKmsKeyId(String resolvedKmsKeyId) {
+ return new ReplicationGroupSettings(atRestEncryptionEnabled, resolvedKmsKeyId,
+ snapshotRetentionLimit, snapshotWindow);
+ }
+
+ public void applyTo(ReplicationGroup group) {
+ if (atRestEncryptionEnabled != null) {
+ group.setAtRestEncryptionEnabled(atRestEncryptionEnabled);
+ }
+ if (kmsKeyId != null && !kmsKeyId.isBlank()) {
+ group.setKmsKeyId(kmsKeyId);
+ }
+ if (snapshotRetentionLimit != null) {
+ group.setSnapshotRetentionLimit(snapshotRetentionLimit);
+ }
+ if (snapshotWindow != null && !snapshotWindow.isBlank()) {
+ group.setSnapshotWindow(snapshotWindow);
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandlerTest.java
index 9c9d06bf3d..dd7afcf206 100644
--- a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandlerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheQueryHandlerTest.java
@@ -6,6 +6,17 @@ import jakarta.ws.rs.core.MultivaluedHashMap;
import jakarta.ws.rs.core.MultivaluedMap;
import jakarta.ws.rs.core.Response;
import org.junit.jupiter.api.BeforeEach;
+import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroup;
+import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroupSettings;
+import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroupStatus;
+import io.github.hectorvent.floci.services.elasticache.model.AuthMode;
+import io.github.hectorvent.floci.services.elasticache.model.Endpoint;
+import io.github.hectorvent.floci.core.common.AwsException;
+import java.util.List;
+import java.util.Map;
+import static org.mockito.ArgumentMatchers.*;
+import static org.mockito.Mockito.*;
+import org.mockito.ArgumentCaptor;
import org.junit.jupiter.api.Test;
import static org.junit.jupiter.api.Assertions.*;
@@ -18,13 +29,16 @@ import static org.mockito.Mockito.mock;
class ElastiCacheQueryHandlerTest {
private ElastiCacheQueryHandler handler;
+ private ElastiCacheService service;
@BeforeEach
void setUp() {
SigV4Validator sigV4Validator = mock(SigV4Validator.class);
- ElastiCacheService service = mock(ElastiCacheService.class);
+ service = mock(ElastiCacheService.class);
ElastiCacheMemcachedService memcachedService = mock(ElastiCacheMemcachedService.class);
RegionResolver regionResolver = mock(RegionResolver.class);
+ when(regionResolver.getRegion()).thenReturn("us-east-1");
+ when(regionResolver.getAccountId()).thenReturn("000000000000");
handler = new ElastiCacheQueryHandler(sigV4Validator, service, memcachedService, regionResolver);
}
@@ -61,4 +75,124 @@ class ElastiCacheQueryHandlerTest {
private static MultivaluedMap<String, String> params() {
return new MultivaluedHashMap<>();
}
+
+ private static ReplicationGroup group(String id) {
+ ReplicationGroup g = new ReplicationGroup(id, "d", ReplicationGroupStatus.AVAILABLE, AuthMode.NO_AUTH,
+ new Endpoint("localhost", 6379), java.time.Instant.now(), 6379);
+ g.setArn("arn:aws:elasticache:us-east-1:000000000000:replicationgroup:" + id);
+ return g;
+ }
+
+ @Test
+ void createReplicationGroup_passesSettingsAndTagsToService() {
+ when(service.createReplicationGroup(any(), any(), any(), any(), any(), any(), any()))
+ .thenReturn(group("g1"));
+ MultivaluedMap<String, String> p = params();
+ p.add("ReplicationGroupId", "g1");
+ p.add("ReplicationGroupDescription", "d");
+ p.add("AtRestEncryptionEnabled", "true");
+ p.add("KmsKeyId", "alias/cache");
+ p.add("SnapshotRetentionLimit", "7");
+ p.add("SnapshotWindow", "06:30-07:30");
+ p.add("Tags.Tag.1.Key", "Name");
+ p.add("Tags.Tag.1.Value", "g1");
+
+ assertEquals(200, handler.handle("CreateReplicationGroup", p, "us-east-1").getStatus());
+
+ verify(service).createReplicationGroup(eq("g1"), eq("d"), eq(AuthMode.NO_AUTH), isNull(), eq("us-east-1"),
+ eq(new ReplicationGroupSettings(true, "alias/cache", 7, "06:30-07:30")), eq(Map.of("Name", "g1")));
+ }
+
+ @Test
+ void createReplicationGroup_readsTheEncryptionFlagAsAwsDoes() {
+ when(service.createReplicationGroup(any(), any(), any(), any(), any(), any(), any()))
+ .thenReturn(group("g1"));
+ ArgumentCaptor<ReplicationGroupSettings> captor = ArgumentCaptor.forClass(ReplicationGroupSettings.class);
+ // a live account reads anything but "false" as true — probed with banana, yes and "TRUE "
+ for (String value : new String[] {"true", "banana", "yes", "TRUE "}) {
+ MultivaluedMap<String, String> p = params();
+ p.add("ReplicationGroupId", "g1");
+ p.add("AtRestEncryptionEnabled", value);
+ handler.handle("CreateReplicationGroup", p, "us-east-1");
+ }
+ for (String value : new String[] {"false", "FALSE"}) {
+ MultivaluedMap<String, String> p = params();
+ p.add("ReplicationGroupId", "g1");
+ p.add("AtRestEncryptionEnabled", value);
+ handler.handle("CreateReplicationGroup", p, "us-east-1");
+ }
+ MultivaluedMap<String, String> omitted = params();
+ omitted.add("ReplicationGroupId", "g1");
+ handler.handle("CreateReplicationGroup", omitted, "us-east-1");
+
+ verify(service, times(7)).createReplicationGroup(any(), any(), any(), any(), any(), captor.capture(), any());
+ List<Boolean> seen = captor.getAllValues().stream().map(ReplicationGroupSettings::atRestEncryptionEnabled).toList();
+ assertEquals(java.util.Arrays.asList(true, true, true, true, false, false, null), seen);
+ }
+
+ @Test
+ void describeReplicationGroups_emitsStoredSettingsAndArn() {
+ ReplicationGroup g = group("g1");
+ g.setAtRestEncryptionEnabled(true);
+ g.setKmsKeyId("arn:aws:kms:us-east-1:000000000000:key/k1");
+ g.setSnapshotRetentionLimit(7);
+ g.setSnapshotWindow("06:30-07:30");
+ when(service.listReplicationGroups("g1")).thenReturn(List.of(g));
+ MultivaluedMap<String, String> p = params();
+ p.add("ReplicationGroupId", "g1");
+
+ String body = (String) handler.handle("DescribeReplicationGroups", p, "us-east-1").getEntity();
+
+ assertTrue(body.contains("<AtRestEncryptionEnabled>true</AtRestEncryptionEnabled>"), body);
+ assertTrue(body.contains("<KmsKeyId>arn:aws:kms:us-east-1:000000000000:key/k1</KmsKeyId>"), body);
+ assertTrue(body.contains("<SnapshotRetentionLimit>7</SnapshotRetentionLimit>"), body);
+ assertTrue(body.contains("<SnapshotWindow>06:30-07:30</SnapshotWindow>"), body);
+ assertTrue(body.contains("<ARN>arn:aws:elasticache:us-east-1:000000000000:replicationgroup:g1</ARN>"), body);
+ }
+
+ @Test
+ void listTagsForResource_readsAReplicationGroupByItsArn() {
+ ReplicationGroup g = group("g1");
+ g.setTags(new java.util.LinkedHashMap<>(Map.of("Name", "g1")));
+ when(service.getReplicationGroup("g1")).thenReturn(g);
+ when(service.getReplicationGroup("absent")).thenThrow(
+ new AwsException("ReplicationGroupNotFoundFault", "Replication group absent not found.", 404));
+
+ MultivaluedMap<String, String> p = params();
+ p.add("ResourceName", "arn:aws:elasticache:us-east-1:000000000000:replicationgroup:g1");
+ String body = (String) handler.handle("ListTagsForResource", p, "us-east-1").getEntity();
+ assertTrue(body.contains("<Key>Name</Key>"), body);
+
+ p = params();
+ p.add("ResourceName", "arn:aws:elasticache:us-east-1:000000000000:replicationgroup:absent");
+ Response response = handler.handle("ListTagsForResource", p, "us-east-1");
+ assertEquals(404, response.getStatus());
+ assertTrue(((String) response.getEntity()).contains("ReplicationGroupNotFoundFault"));
+
+ // the store keys groups by id alone: a same-named group created under another region is
+ // not the one this ARN names
+ ReplicationGroup elsewhere = group("g2");
+ elsewhere.setArn("arn:aws:elasticache:eu-west-1:000000000000:replicationgroup:g2");
+ elsewhere.setTags(new java.util.LinkedHashMap<>(Map.of("Name", "west")));
+ when(service.getReplicationGroup("g2")).thenReturn(elsewhere);
+ p = params();
+ p.add("ResourceName", "arn:aws:elasticache:us-east-1:000000000000:replicationgroup:g2");
+ response = handler.handle("ListTagsForResource", p, "us-east-1");
+ assertEquals(404, response.getStatus(), (String) response.getEntity());
+ assertFalse(((String) response.getEntity()).contains("west"));
+ }
+
+ @Test
+ void modifyReplicationGroup_passesSnapshotSettingsOnly() {
+ when(service.modifyReplicationGroup(eq("g1"), isNull(), isNull(), any())).thenReturn(group("g1"));
+ MultivaluedMap<String, String> p = params();
+ p.add("ReplicationGroupId", "g1");
+ p.add("SnapshotRetentionLimit", "3");
+ p.add("SnapshotWindow", "01:00-02:00");
+ p.add("AtRestEncryptionEnabled", "true");
+ p.add("KmsKeyId", "alias/other");
+
+ assertEquals(200, handler.handle("ModifyReplicationGroup", p, "us-east-1").getStatus());
+ verify(service).modifyReplicationGroup("g1", null, null, new ReplicationGroupSettings(null, null, 3, "01:00-02:00"));
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java
index ea3321c008..f1ffdc7269 100644
--- a/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/ElastiCacheServiceTest.java
@@ -6,6 +6,10 @@ import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.services.ec2.Ec2Service;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.services.kms.KmsService;
+import io.github.hectorvent.floci.services.kms.model.KmsKey;
+import io.github.hectorvent.floci.services.elasticache.model.ReplicationGroupSettings;
import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerHandle;
import io.github.hectorvent.floci.services.elasticache.container.ElastiCacheContainerManager;
import io.github.hectorvent.floci.services.elasticache.model.AuthMode;
@@ -15,11 +19,13 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
@@ -36,15 +42,17 @@ import static org.mockito.Mockito.when;
class ElastiCacheServiceTest {
private ElastiCacheService service;
+ private KmsService kmsService;
private ElastiCacheContainerManager containerManager;
private ElastiCacheProxyManager proxyManager;
+ private EmulatorConfig config;
@BeforeEach
void setUp() {
containerManager = mock(ElastiCacheContainerManager.class);
proxyManager = mock(ElastiCacheProxyManager.class);
StorageFactory storageFactory = mock(StorageFactory.class);
- EmulatorConfig config = mock(EmulatorConfig.class);
+ config = mock(EmulatorConfig.class);
EmulatorConfig.ServicesConfig servicesConfig = mock(EmulatorConfig.ServicesConfig.class);
EmulatorConfig.ElastiCacheServiceConfig ecConfig = mock(EmulatorConfig.ElastiCacheServiceConfig.class);
@@ -60,8 +68,11 @@ class ElastiCacheServiceTest {
.thenReturn(new ElastiCacheContainerHandle("cid", "grp", "localhost", 6379));
doNothing().when(proxyManager).startProxy(anyString(), any(), anyInt(), anyString(), anyInt(), any());
Ec2Service ec2Service = org.mockito.Mockito.mock(Ec2Service.class);
+ kmsService = org.mockito.Mockito.mock(KmsService.class);
+ when(kmsService.describeKey(any(), any())).thenThrow(
+ new AwsException("NotFoundException", "Key not found", 404));
service = new ElastiCacheService(containerManager, proxyManager, storageFactory, config,
- ec2Service, new RegionResolver("us-east-1", "000000000000"));
+ ec2Service, new RegionResolver("us-east-1", "000000000000"), kmsService);
}
@Test
@@ -88,7 +99,8 @@ class ElastiCacheServiceTest {
doNothing().when(pm).startProxy(anyString(), any(), anyInt(), anyString(), anyInt(), any());
ElastiCacheService svc = new ElastiCacheService(cm, pm, sf, cfg,
org.mockito.Mockito.mock(Ec2Service.class),
- new RegionResolver("us-east-1", "000000000000"));
+ new RegionResolver("us-east-1", "000000000000"),
+ org.mockito.Mockito.mock(KmsService.class));
svc.createReplicationGroup("g1", "d", AuthMode.PASSWORD, null, "us-east-1");
@@ -221,4 +233,158 @@ class ElastiCacheServiceTest {
assertEquals("grp", service.getReplicationGroup("grp").getReplicationGroupId());
}
+
+ private static final String KEY_ARN = "arn:aws:kms:us-east-1:000000000000:key/k1";
+
+ private KmsKey knownKey(String... forms) {
+ KmsKey key = new KmsKey();
+ key.setKeyId("k1");
+ key.setArn(KEY_ARN);
+ key.setEnabled(true);
+ key.setKeyState("Enabled");
+ for (String form : forms) {
+ org.mockito.Mockito.doReturn(key).when(kmsService).describeKey(form, "us-east-1");
+ }
+ return key;
+ }
+
+ @Test
+ void createReplicationGroupStoresEncryptionSnapshotSettingsAndTags() {
+ knownKey("alias/cache");
+ ReplicationGroup group = service.createReplicationGroup("g1", "d", AuthMode.NO_AUTH, null, "us-east-1",
+ new ReplicationGroupSettings(true, "alias/cache", 7, "06:30-07:30"),
+ Map.of("Name", "g1", "env", "tst"));
+
+ ReplicationGroup stored = service.getReplicationGroup("g1");
+ assertTrue(stored.isAtRestEncryptionEnabled());
+ assertEquals(KEY_ARN, stored.getKmsKeyId());
+ assertEquals(7, stored.getSnapshotRetentionLimit());
+ assertEquals("06:30-07:30", stored.getSnapshotWindow());
+ assertEquals(Map.of("Name", "g1", "env", "tst"), stored.getTags());
+ assertEquals("arn:aws:elasticache:us-east-1:000000000000:replicationgroup:g1", group.getArn());
+ }
+
+ @Test
+ void createReplicationGroupWithoutSettingsKeepsAwsDefaults() {
+ service.createReplicationGroup("g1", "d", AuthMode.NO_AUTH, null, "us-east-1");
+ ReplicationGroup stored = service.getReplicationGroup("g1");
+ assertFalse(stored.isAtRestEncryptionEnabled());
+ assertNull(stored.getKmsKeyId());
+ assertEquals(0, stored.getSnapshotRetentionLimit());
+ assertTrue(stored.getTags().isEmpty());
+ assertEquals(ReplicationGroupSettings.DEFAULT_SNAPSHOT_WINDOW, stored.getSnapshotWindow());
+ }
+
+ @Test
+ void createReplicationGroupRejectsAKeyItCannotUseBeforeStartingAContainer() {
+ AwsException missing = assertThrows(AwsException.class, () -> service.createReplicationGroup(
+ "g1", "d", AuthMode.NO_AUTH, null, "us-east-1",
+ new ReplicationGroupSettings(true, "alias/does-not-exist", null, null), Map.of()));
+ assertEquals("InvalidParameterValue", missing.getErrorCode());
+ assertEquals("KMS key does not exist with key id: alias/does-not-exist", missing.getMessage());
+ assertThrows(AwsException.class, () -> service.getReplicationGroup("g1"));
+ org.mockito.Mockito.verify(containerManager, org.mockito.Mockito.never()).start(anyString(), anyString());
+
+ AwsException combination = assertThrows(AwsException.class, () -> service.createReplicationGroup(
+ "g1", "d", AuthMode.NO_AUTH, null, "us-east-1",
+ new ReplicationGroupSettings(false, KEY_ARN, null, null), Map.of()));
+ assertEquals("InvalidParameterCombination", combination.getErrorCode());
+ assertEquals("Please enable encryption at rest to use Customer Managed CMK", combination.getMessage());
+ // leaving AtRestEncryptionEnabled out is false on a live account, refused the same way
+ AwsException omitted = assertThrows(AwsException.class, () -> service.createReplicationGroup(
+ "g1", "d", AuthMode.NO_AUTH, null, "us-east-1",
+ new ReplicationGroupSettings(null, KEY_ARN, null, null), Map.of()));
+ assertEquals("InvalidParameterCombination", omitted.getErrorCode());
+
+ AwsException retention = assertThrows(AwsException.class, () -> service.createReplicationGroup(
+ "g1", "d", AuthMode.NO_AUTH, null, "us-east-1",
+ new ReplicationGroupSettings(null, null, 36, null), Map.of()));
+ assertEquals("Invalid snapshot retention limit: 36. Retention limit must be between 0 and 35.", retention.getMessage());
+ AwsException window = assertThrows(AwsException.class, () -> service.createReplicationGroup(
+ "g1", "d", AuthMode.NO_AUTH, null, "us-east-1",
+ new ReplicationGroupSettings(null, null, null, "25:00-26:00"), Map.of()));
+ assertEquals("Invalid backup window format. Should be specified as a range hh24:mi-hh24:mi (24H Clock UTC). Example: 03:15-08:15", window.getMessage());
+ AwsException shortWindow = assertThrows(AwsException.class, () -> service.createReplicationGroup(
+ "g1", "d", AuthMode.NO_AUTH, null, "us-east-1",
+ new ReplicationGroupSettings(null, null, null, "05:00-05:30"), Map.of()));
+ assertEquals("Snapshot window must be at least 60 minutes.", shortWindow.getMessage());
+ // equal start and end is an empty window, not a full day
+ AwsException emptyWindow = assertThrows(AwsException.class, () -> service.createReplicationGroup(
+ "g1", "d", AuthMode.NO_AUTH, null, "us-east-1",
+ new ReplicationGroupSettings(null, null, null, "05:00-05:00"), Map.of()));
+ assertEquals("Snapshot window must be at least 60 minutes.", emptyWindow.getMessage());
+ // while a window wrapping midnight is measured across it
+ service.createReplicationGroup("wrap", "d", AuthMode.NO_AUTH, null, "us-east-1",
+ new ReplicationGroupSettings(null, null, null, "23:30-00:30"), Map.of());
+ assertEquals("23:30-00:30", service.getReplicationGroup("wrap").getSnapshotWindow());
+ }
+
+ @Test
+ void modifyReplicationGroupChangesSnapshotSettingsAndKeepsEncryption() {
+ knownKey(KEY_ARN);
+ service.createReplicationGroup("g1", "d", AuthMode.NO_AUTH, null, "us-east-1",
+ new ReplicationGroupSettings(true, KEY_ARN, 7, "06:30-07:30"), Map.of());
+
+ service.modifyReplicationGroup("g1", null, null, new ReplicationGroupSettings(null, null, 3, "01:00-02:00"));
+
+ ReplicationGroup stored = service.getReplicationGroup("g1");
+ assertEquals(3, stored.getSnapshotRetentionLimit());
+ assertEquals("01:00-02:00", stored.getSnapshotWindow());
+ assertTrue(stored.isAtRestEncryptionEnabled());
+ assertEquals(KEY_ARN, stored.getKmsKeyId());
+
+ assertThrows(AwsException.class, () -> service.modifyReplicationGroup("g1", null, null,
+ new ReplicationGroupSettings(null, null, null, "25:00-26:00")));
+ assertEquals("01:00-02:00", service.getReplicationGroup("g1").getSnapshotWindow());
+
+ // a refusal later in the same request must not leave the earlier part applied: the store
+ // hands out its own object, so settings applied before the user check would stay visible
+ AwsException unknownUser = assertThrows(AwsException.class, () -> service.modifyReplicationGroup(
+ "g1", List.of("no-such-user"), null, new ReplicationGroupSettings(null, null, 9, "03:00-04:00")));
+ assertEquals("UserNotFoundFault", unknownUser.getErrorCode());
+ assertEquals(3, service.getReplicationGroup("g1").getSnapshotRetentionLimit());
+ assertEquals("01:00-02:00", service.getReplicationGroup("g1").getSnapshotWindow());
+ }
+
+ @Test
+ void modifyReplicationGroupCannotWriteAGroupBackAfterDelete() throws Exception {
+ // modify has read the group, delete removes it, modify writes its copy back — the store is
+ // held inside modify's put so the delete can be run in exactly that window
+ PausingStorageBackend<ReplicationGroup> pausing = new PausingStorageBackend<>(new InMemoryStorage<>());
+ StorageFactory factory = org.mockito.Mockito.mock(StorageFactory.class);
+ when(factory.create(anyString(), eq("elasticache-groups.json"), any()))
+ .thenAnswer(inv -> new AccountAwareStorageBackend<>(pausing, null, "000000000000"));
+ when(factory.create(anyString(), org.mockito.ArgumentMatchers.argThat(f -> !"elasticache-groups.json".equals(f)), any()))
+ .thenAnswer(inv -> AccountAwareStorageBackend.inMemory("000000000000"));
+ ElastiCacheService svc = new ElastiCacheService(containerManager, proxyManager, factory, config,
+ org.mockito.Mockito.mock(Ec2Service.class), new RegionResolver("us-east-1", "000000000000"), kmsService);
+ svc.createReplicationGroup("g1", "d", AuthMode.NO_AUTH, null, "us-east-1");
+
+ pausing.pauseOn(PausingStorageBackend.Call.PUT, "g1");
+ java.util.concurrent.atomic.AtomicReference<Throwable> modifyOutcome = new java.util.concurrent.atomic.AtomicReference<>();
+ Thread modify = new Thread(() -> {
+ try {
+ svc.modifyReplicationGroup("g1", null, null, new ReplicationGroupSettings(null, null, 3, null));
+ } catch (Throwable t) {
+ modifyOutcome.set(t);
+ }
+ });
+ modify.start();
+ pausing.awaitReached();
+
+ Thread delete = new Thread(() -> svc.deleteReplicationGroup("g1"));
+ delete.start();
+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
+ while (delete.getState() != Thread.State.BLOCKED && delete.getState() != Thread.State.TERMINATED) {
+ assertTrue(System.nanoTime() < deadline, "delete neither ran nor queued");
+ Thread.onSpinWait();
+ }
+ pausing.release();
+ modify.join(5000);
+ delete.join(5000);
+
+ assertNull(modifyOutcome.get(), "modify completed before the delete");
+ assertThrows(AwsException.class, () -> svc.getReplicationGroup("g1"),
+ "the deleted group must not come back from the modify");
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/elasticache/PausingStorageBackend.java b/src/test/java/io/github/hectorvent/floci/services/elasticache/PausingStorageBackend.java
new file mode 100644
index 0000000000..40e6fd7b86
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/elasticache/PausingStorageBackend.java
@@ -0,0 +1,123 @@
+package io.github.hectorvent.floci.services.elasticache;
+
+import io.github.hectorvent.floci.core.storage.StorageBackend;
+
+import java.util.List;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Predicate;
+
+/**
+ * A store that can be stopped in the middle of an operation, so a race can be staged rather than
+ * hoped for.
+ *
+ * <p>A concurrency test that loops and waits for an unlucky interleaving proves nothing when the
+ * window is a few instructions wide: it passes with the fix and without it. Holding one thread at a
+ * chosen call while another runs to completion makes the interleaving the test claims to cover the
+ * one it actually runs.
+ */
+final class PausingStorageBackend<V> implements StorageBackend<String, V> {
+
+ /** Which call to hold, and on which key. */
+ enum Call { PUT, DELETE, SCAN }
+
+ private final StorageBackend<String, V> delegate;
+ private volatile Call pauseOn;
+ private volatile String pauseKey;
+ private final java.util.concurrent.atomic.AtomicInteger skip =
+ new java.util.concurrent.atomic.AtomicInteger();
+ private final CountDownLatch reached = new CountDownLatch(1);
+ private final CountDownLatch release = new CountDownLatch(1);
+
+ PausingStorageBackend(StorageBackend<String, V> delegate) {
+ this.delegate = delegate;
+ }
+
+ /** Hold the next matching call until {@link #release()}; {@code key} may be null for SCAN. */
+ void pauseOn(Call call, String key) {
+ pauseOn(call, key, 0);
+ }
+
+ /** As above, after letting {@code skipFirst} matching calls through untouched. */
+ void pauseOn(Call call, String key, int skipFirst) {
+ this.pauseOn = call;
+ this.pauseKey = key;
+ this.skip.set(skipFirst);
+ }
+
+ /** Waits until the paused call has been reached. */
+ void awaitReached() throws InterruptedException {
+ if (!reached.await(10, TimeUnit.SECONDS)) {
+ throw new IllegalStateException("the paused call was never reached");
+ }
+ }
+
+ void release() {
+ release.countDown();
+ }
+
+ private void pauseIfMatching(Call call, String key) {
+ // Keys arrive account-prefixed here, since this sits underneath the account-aware layer.
+ if (pauseOn != call || (pauseKey != null && (key == null || !key.endsWith(pauseKey)))) {
+ return;
+ }
+ if (skip.getAndDecrement() > 0) {
+ return;
+ }
+ pauseOn = null;
+ reached.countDown();
+ try {
+ if (!release.await(10, TimeUnit.SECONDS)) {
+ throw new IllegalStateException("the paused call was never released");
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(e);
+ }
+ }
+
+ @Override
+ public void put(String key, V value) {
+ pauseIfMatching(Call.PUT, key);
+ delegate.put(key, value);
+ }
+
+ @Override
+ public Optional<V> get(String key) {
+ return delegate.get(key);
+ }
+
+ @Override
+ public void delete(String key) {
+ pauseIfMatching(Call.DELETE, key);
+ delegate.delete(key);
+ }
+
+ @Override
+ public List<V> scan(Predicate<String> keyFilter) {
+ pauseIfMatching(Call.SCAN, null);
+ return delegate.scan(keyFilter);
+ }
+
+ @Override
+ public Set<String> keys() {
+ return delegate.keys();
+ }
+
+ @Override
+ public void flush() {
+ delegate.flush();
+ }
+
+ @Override
+ public void load() {
+ delegate.load();
+ }
+
+ @Override
+ public void clear() {
+ delegate.clear();
+ }
+}