Light, fluffy, and always free - The AWS Local Emulator alternative
fix(kinesis): stop dropping shard records under concurrent producers and resharding (#3000)
Guilherme Scaranse73deb13c01db
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarder.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarder.javaindex b10b9c97de..fbd11390d6 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarder.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarder.java@@ -14,6 +14,7 @@ import org.jboss.logging.Logger;import java.time.Instant;import java.util.List;import java.util.UUID;+import java.util.concurrent.atomic.AtomicLong;@ApplicationScopedpublic class KinesisStreamingForwarder {@@ -22,6 +23,7 @@ public class KinesisStreamingForwarder {private final KinesisService kinesisService;private final ObjectMapper objectMapper;+ private final AtomicLong forwardFailures = new AtomicLong();@Injectpublic KinesisStreamingForwarder(KinesisService kinesisService, ObjectMapper objectMapper) {@@ -53,12 +55,22 @@ public class KinesisStreamingForwarder {LOG.debugv("Forwarded DynamoDB event to Kinesis stream {0}: {1} on {2}",streamName, eventName, table.getTableName());} catch (Exception e) {- LOG.warnv("Failed to forward DynamoDB event to Kinesis destination {0}: {1}",- dest.getStreamArn(), e.getMessage());+ // Do NOT rethrow: put/delete/update persist before forwarding, so a rethrow would+ // turn a committed write into a spurious 5xx; the TTL sweep forwards mid-loop and+ // persists only after the batch, so a rethrow would abort the sweep. Surface the+ // drop loudly (ERROR + stack trace) and count it for process-local diagnostics.+ forwardFailures.incrementAndGet();+ LOG.errorv(e, "Dropped DynamoDB CDC event: failed to forward {0} on table {1} to Kinesis destination {2}",+ eventName, table.getTableName(), dest.getStreamArn());}}}+ /** Process-local count of CDC events dropped because forwarding to Kinesis threw. */+ public long getForwardFailureCount() {+ return forwardFailures.get();+ }+private ObjectNode buildPayload(String eventName, JsonNode keys,JsonNode newImage, JsonNode oldImage,String tableName, String region, Instant timestamp) {diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisInspectionController.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisInspectionController.javaindex 2c6c1983e0..849e3eb2bc 100644--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisInspectionController.java+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisInspectionController.java@@ -104,7 +104,7 @@ public class KinesisInspectionController {node.put("StreamCreationTimestamp", stream.getStreamCreationTimestamp().toString());}node.put("OpenShardCount", stream.getShards().stream().filter(s -> !s.isClosed()).count());- node.put("RecordCount", stream.getShards().stream().mapToInt(s -> s.getRecords().size()).sum());+ node.put("RecordCount", stream.getShards().stream().mapToInt(KinesisShard::recordCount).sum());ArrayNode enhancedMonitoring = node.putArray("EnhancedMonitoring");Set<String> metrics = stream.getEnhancedMonitoringMetrics();if (metrics != null) {@@ -130,7 +130,7 @@ public class KinesisInspectionController {ObjectNode node = objectMapper.createObjectNode();node.put("ShardId", shard.getShardId());node.put("Closed", shard.isClosed());- node.put("RecordCount", shard.getRecords().size());+ node.put("RecordCount", shard.recordCount());if (shard.getParentShardId() != null) {node.put("ParentShardId", shard.getParentShardId());}diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.javaindex 23a54b175d..822f0debea 100644--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java@@ -638,7 +638,7 @@ public class KinesisJsonHandler {}int maxResults = request.has("MaxResults") ? request.path("MaxResults").asInt(1000) : 1000;- List<KinesisShard> page = shards.size() > maxResults ? shards.subList(0, maxResults) : shards;+ List<KinesisShard> page = paginateShards(shards, maxResults);ObjectNode response = objectMapper.createObjectNode();ArrayNode shardsArray = response.putArray("Shards");@@ -666,6 +666,18 @@ public class KinesisJsonHandler {return Response.ok(response).build();}+ /**+ * Take a ListShards page, snapshotting first. The shard list is a live+ * {@link java.util.concurrent.CopyOnWriteArrayList}: its {@code subList} view is NOT an independent+ * snapshot. It stays bound to the backing array and throws {@link java.util.ConcurrentModificationException}+ * the moment a concurrent split/merge appends a shard. Copying to an immutable list first gives a page+ * that reads consistently regardless of concurrent resharding.+ */+ static List<KinesisShard> paginateShards(List<KinesisShard> shards, int maxResults) {+ List<KinesisShard> snapshot = List.copyOf(shards);+ return snapshot.size() > maxResults ? snapshot.subList(0, maxResults) : snapshot;+ }+private Response handleEnableEnhancedMonitoring(JsonNode request, String region) {String streamName = resolveStreamName(request);diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.javaindex d4807bceb8..5dd3e24f10 100644--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java@@ -17,6 +17,7 @@ import org.jboss.logging.Logger;import java.nio.charset.StandardCharsets;import java.time.Instant;import java.util.*;+import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.atomic.AtomicLong;import io.github.hectorvent.floci.core.resource.ExplorerResource;import io.github.hectorvent.floci.core.resource.ResourceProvider;@@ -47,6 +48,18 @@ public class KinesisService implements ResourceProvider {private final StorageBackend<String, KinesisConsumer> consumerStore;private final RegionResolver regionResolver;private final AtomicLong sequenceGenerator = new AtomicLong(System.currentTimeMillis());+ // One monitor per stream key, handed out by lockFor. Serializes sequence allocation + shard+ // append + persist so the in-memory log, the assigned sequence order and the WAL write order+ // all agree, and serializes every other read-modify-write on a stream against deleteStream.+ private final ConcurrentHashMap<String, Object> streamAppendLocks = new ConcurrentHashMap<>();+ // Package-private test seam, run inside the append critical section after the sequence is+ // allocated and before the record is appended. Default no-op in production.+ Runnable putRecordAppendHook = () -> {};+ // Package-private test seam, run after the pre-lock resolve/validation and before the append+ // lock is acquired. Lets a test deterministically wedge a producer in the window where a+ // concurrent deleteStream can win the lock first, exercising the in-lock re-resolve guard.+ // Default no-op in production.+ Runnable putRecordBeforeLockHook = () -> {};@Injectpublic KinesisService(StorageFactory factory, RegionResolver regionResolver) {@@ -118,20 +131,23 @@ public class KinesisService implements ResourceProvider {throw new AwsException("InvalidArgumentException","StreamMode must be PROVISIONED or ON_DEMAND, got: " + streamMode, 400);}- KinesisStream stream = resolveStream(streamName, region);- if (!"ACTIVE".equals(stream.getStreamStatus())) {- throw new AwsException("ResourceInUseException",- "Stream " + streamName + " is not ACTIVE (current state: " + stream.getStreamStatus() + ")", 400);- }- // Same-mode is a no-op. Mirrors the same-value behaviour in- // increase/decreaseStreamRetentionPeriod (see #342). Avoids breaking- // terraform-provider-aws which calls UpdateStreamMode on every refresh.- if (streamMode.equals(stream.getStreamMode())) {- return;+ String key = regionKey(region, streamName);+ synchronized (lockFor(key)) {+ KinesisStream stream = resolveStream(streamName, region);+ if (!"ACTIVE".equals(stream.getStreamStatus())) {+ throw new AwsException("ResourceInUseException",+ "Stream " + streamName + " is not ACTIVE (current state: " + stream.getStreamStatus() + ")", 400);+ }+ // Same-mode is a no-op. Mirrors the same-value behaviour in+ // increase/decreaseStreamRetentionPeriod (see #342). Avoids breaking+ // terraform-provider-aws which calls UpdateStreamMode on every refresh.+ if (streamMode.equals(stream.getStreamMode())) {+ return;+ }+ stream.setStreamMode(streamMode);+ store.put(key, stream);+ LOG.infov("Updated stream mode for {0} to {1}", streamName, streamMode);}- stream.setStreamMode(streamMode);- store.put(regionKey(region, streamName), stream);- LOG.infov("Updated stream mode for {0} to {1}", streamName, streamMode);}public List<String> listStreams(String region) {@@ -193,20 +209,31 @@ public class KinesisService implements ResourceProvider {public void deleteStream(String streamName, String region) {String storageKey = regionKey(region, streamName);- store.delete(storageKey);+ // Delete under the per-stream monitor every writer takes, so the delete is serialized with+ // any in-flight append, split, merge or metadata write on this stream. See lockFor for why+ // the monitor is deliberately left in the map.+ synchronized (lockFor(storageKey)) {+ store.delete(storageKey);+ }LOG.infov("Deleted Kinesis stream: {0}", streamName);}public void addTagsToStream(String streamName, Map<String, String> tags, String region) {- KinesisStream stream = resolveStream(streamName, region);- stream.getTags().putAll(tags);- store.put(regionKey(region, streamName), stream);+ String key = regionKey(region, streamName);+ synchronized (lockFor(key)) {+ KinesisStream stream = resolveStream(streamName, region);+ stream.getTags().putAll(tags);+ store.put(key, stream);+ }}public void removeTagsFromStream(String streamName, List<String> tagKeys, String region) {- KinesisStream stream = resolveStream(streamName, region);- tagKeys.forEach(stream.getTags()::remove);- store.put(regionKey(region, streamName), stream);+ String key = regionKey(region, streamName);+ synchronized (lockFor(key)) {+ KinesisStream stream = resolveStream(streamName, region);+ tagKeys.forEach(stream.getTags()::remove);+ store.put(key, stream);+ }}public Map<String, String> listTagsForStream(String streamName, String region) {@@ -214,75 +241,90 @@ public class KinesisService implements ResourceProvider {}public void startStreamEncryption(String streamName, String encryptionType, String keyId, String region) {- KinesisStream stream = resolveStream(streamName, region);- stream.setEncryptionType(encryptionType);- stream.setKeyId(keyId);- store.put(regionKey(region, streamName), stream);+ String key = regionKey(region, streamName);+ synchronized (lockFor(key)) {+ KinesisStream stream = resolveStream(streamName, region);+ stream.setEncryptionType(encryptionType);+ stream.setKeyId(keyId);+ store.put(key, stream);+ }}public void increaseStreamRetentionPeriod(String streamName, int retentionPeriodHours, String region) {- KinesisStream stream = resolveStream(streamName, region);- if (retentionPeriodHours > 8760) {- throw new AwsException("InvalidArgumentException",- "Retention period must not exceed 8760 hours (365 days)", 400);- }- if (retentionPeriodHours < stream.getRetentionPeriodHours()) {- throw new AwsException("InvalidArgumentException",- "Requested retention period (" + retentionPeriodHours +- " hours) must not be less than current retention period (" +- stream.getRetentionPeriodHours() + " hours)", 400);- }- // Same value is a no-op on real AWS despite the API doc wording ("must be more than- // current"). Proof: terraform-provider-aws calls IncreaseStreamRetentionPeriod on- // stream creation unconditionally when retention_period is set (stream.go Create path),- // so every default-retention TF stream would fail if AWS rejected same-value. See #342.- if (retentionPeriodHours == stream.getRetentionPeriodHours()) {- return;+ String key = regionKey(region, streamName);+ synchronized (lockFor(key)) {+ KinesisStream stream = resolveStream(streamName, region);+ if (retentionPeriodHours > 8760) {+ throw new AwsException("InvalidArgumentException",+ "Retention period must not exceed 8760 hours (365 days)", 400);+ }+ if (retentionPeriodHours < stream.getRetentionPeriodHours()) {+ throw new AwsException("InvalidArgumentException",+ "Requested retention period (" + retentionPeriodHours ++ " hours) must not be less than current retention period (" ++ stream.getRetentionPeriodHours() + " hours)", 400);+ }+ // Same value is a no-op on real AWS despite the API doc wording ("must be more than+ // current"). Proof: terraform-provider-aws calls IncreaseStreamRetentionPeriod on+ // stream creation unconditionally when retention_period is set (stream.go Create path),+ // so every default-retention TF stream would fail if AWS rejected same-value. See #342.+ if (retentionPeriodHours == stream.getRetentionPeriodHours()) {+ return;+ }+ stream.setRetentionPeriodHours(retentionPeriodHours);+ store.put(key, stream);+ LOG.infov("Increased retention period for stream {0} to {1} hours", streamName, retentionPeriodHours);}- stream.setRetentionPeriodHours(retentionPeriodHours);- store.put(regionKey(region, streamName), stream);- LOG.infov("Increased retention period for stream {0} to {1} hours", streamName, retentionPeriodHours);}public void decreaseStreamRetentionPeriod(String streamName, int retentionPeriodHours, String region) {- KinesisStream stream = resolveStream(streamName, region);- if (retentionPeriodHours < 24) {- throw new AwsException("InvalidArgumentException",- "Retention period must not be less than 24 hours", 400);- }- if (retentionPeriodHours > stream.getRetentionPeriodHours()) {- throw new AwsException("InvalidArgumentException",- "Requested retention period (" + retentionPeriodHours +- " hours) must not be greater than current retention period (" +- stream.getRetentionPeriodHours() + " hours)", 400);- }- // Same value is a no-op on real AWS (mirrors IncreaseStreamRetentionPeriod). See #342.- if (retentionPeriodHours == stream.getRetentionPeriodHours()) {- return;+ String key = regionKey(region, streamName);+ synchronized (lockFor(key)) {+ KinesisStream stream = resolveStream(streamName, region);+ if (retentionPeriodHours < 24) {+ throw new AwsException("InvalidArgumentException",+ "Retention period must not be less than 24 hours", 400);+ }+ if (retentionPeriodHours > stream.getRetentionPeriodHours()) {+ throw new AwsException("InvalidArgumentException",+ "Requested retention period (" + retentionPeriodHours ++ " hours) must not be greater than current retention period (" ++ stream.getRetentionPeriodHours() + " hours)", 400);+ }+ // Same value is a no-op on real AWS (mirrors IncreaseStreamRetentionPeriod). See #342.+ if (retentionPeriodHours == stream.getRetentionPeriodHours()) {+ return;+ }+ stream.setRetentionPeriodHours(retentionPeriodHours);+ store.put(key, stream);+ LOG.infov("Decreased retention period for stream {0} to {1} hours", streamName, retentionPeriodHours);}- stream.setRetentionPeriodHours(retentionPeriodHours);- store.put(regionKey(region, streamName), stream);- LOG.infov("Decreased retention period for stream {0} to {1} hours", streamName, retentionPeriodHours);}public Set<String> enableEnhancedMonitoring(String streamName, List<String> metrics, String region) {- KinesisStream stream = resolveStream(streamName, region);- Set<String> current = new HashSet<>(stream.getEnhancedMonitoringMetrics());- Set<String> desired = resolveMetrics(metrics);- stream.getEnhancedMonitoringMetrics().addAll(desired);- store.put(regionKey(region, streamName), stream);- LOG.infov("Enabled enhanced monitoring for stream {0}: {1}", streamName, desired);- return current;+ String key = regionKey(region, streamName);+ synchronized (lockFor(key)) {+ KinesisStream stream = resolveStream(streamName, region);+ Set<String> current = new HashSet<>(stream.getEnhancedMonitoringMetrics());+ Set<String> desired = resolveMetrics(metrics);+ stream.getEnhancedMonitoringMetrics().addAll(desired);+ store.put(key, stream);+ LOG.infov("Enabled enhanced monitoring for stream {0}: {1}", streamName, desired);+ return current;+ }}public Set<String> disableEnhancedMonitoring(String streamName, List<String> metrics, String region) {- KinesisStream stream = resolveStream(streamName, region);- Set<String> current = new HashSet<>(stream.getEnhancedMonitoringMetrics());- Set<String> toRemove = resolveMetrics(metrics);- stream.getEnhancedMonitoringMetrics().removeAll(toRemove);- store.put(regionKey(region, streamName), stream);- LOG.infov("Disabled enhanced monitoring for stream {0}: {1}", streamName, toRemove);- return current;+ String key = regionKey(region, streamName);+ synchronized (lockFor(key)) {+ KinesisStream stream = resolveStream(streamName, region);+ Set<String> current = new HashSet<>(stream.getEnhancedMonitoringMetrics());+ Set<String> toRemove = resolveMetrics(metrics);+ stream.getEnhancedMonitoringMetrics().removeAll(toRemove);+ store.put(key, stream);+ LOG.infov("Disabled enhanced monitoring for stream {0}: {1}", streamName, toRemove);+ return current;+ }}private Set<String> resolveMetrics(List<String> metrics) {@@ -306,80 +348,101 @@ public class KinesisService implements ResourceProvider {}public void stopStreamEncryption(String streamName, String region) {- KinesisStream stream = resolveStream(streamName, region);- stream.setEncryptionType("NONE");- stream.setKeyId(null);- store.put(regionKey(region, streamName), stream);+ String key = regionKey(region, streamName);+ synchronized (lockFor(key)) {+ KinesisStream stream = resolveStream(streamName, region);+ stream.setEncryptionType("NONE");+ stream.setKeyId(null);+ store.put(key, stream);+ }}public void splitShard(String streamName, String shardId, String newStartingHashKey, String region) {- KinesisStream stream = resolveStream(streamName, region);- KinesisShard parent = stream.getShards().stream()- .filter(s -> s.getShardId().equals(shardId))- .findFirst()- .orElseThrow(() -> new AwsException("ResourceNotFoundException", "Shard " + shardId + " not found", 400));+ String key = regionKey(region, streamName);+ // Share the producer critical section so a split can't race an append: selectShard runs+ // under the same lock, so a producer never appends to a parent a completed split has closed.+ synchronized (lockFor(key)) {+ // Resolve under the lock so a concurrent deleteStream (same monitor) cannot leave us+ // mutating and re-persisting a stale, deleted instance.+ KinesisStream stream = resolveStream(streamName, region);+ KinesisShard parent = stream.getShards().stream()+ .filter(s -> s.getShardId().equals(shardId))+ .findFirst()+ .orElseThrow(() -> new AwsException("ResourceNotFoundException", "Shard " + shardId + " not found", 400));- if (parent.isClosed()) {- throw new AwsException("InvalidArgumentException", "Shard " + shardId + " is already closed", 400);- }+ if (parent.isClosed()) {+ throw new AwsException("InvalidArgumentException", "Shard " + shardId + " is already closed", 400);+ }- parent.setClosed(true);- parent.setSequenceNumberRange(new KinesisShard.SequenceNumberRange(- parent.getSequenceNumberRange().startingSequenceNumber(),- String.valueOf(sequenceGenerator.get())));+ parent.setClosed(true);+ parent.setSequenceNumberRange(new KinesisShard.SequenceNumberRange(+ parent.getSequenceNumberRange().startingSequenceNumber(),+ String.valueOf(sequenceGenerator.get())));- String start = parent.getHashKeyRange().startingHashKey();- String end = parent.getHashKeyRange().endingHashKey();+ String start = parent.getHashKeyRange().startingHashKey();+ String end = parent.getHashKeyRange().endingHashKey();- KinesisShard child1 = new KinesisShard(nextShardId(stream), start, subtractOne(newStartingHashKey), String.valueOf(sequenceGenerator.get()));- child1.setParentShardId(shardId);+ // Derive both child ids from the base size WITHOUT relying on the first add: nextShardId+ // is size-based, so two sequential adds would either collide (both ids computed pre-add)+ // or expose a half-published topology (child1 visible before child2). Compute base+0 and+ // base+1 up front so the ids stay distinct, then publish BOTH children in one atomic+ // CopyOnWriteArrayList.addAll, so a lock-free reader can never observe {parent, child1}+ // without child2.+ int base = stream.getShards().size();+ KinesisShard child1 = new KinesisShard(String.format("shardId-%012d", base), start, subtractOne(newStartingHashKey), String.valueOf(sequenceGenerator.get()));+ child1.setParentShardId(shardId);- KinesisShard child2 = new KinesisShard(nextShardId(stream), newStartingHashKey, end, String.valueOf(sequenceGenerator.get()));- child2.setParentShardId(shardId);+ KinesisShard child2 = new KinesisShard(String.format("shardId-%012d", base + 1), newStartingHashKey, end, String.valueOf(sequenceGenerator.get()));+ child2.setParentShardId(shardId);- stream.getShards().add(child1);- stream.getShards().add(child2);- store.put(regionKey(region, streamName), stream);- LOG.infov("Split shard {0} in stream {1}", shardId, streamName);+ stream.getShards().addAll(List.of(child1, child2));+ store.put(key, stream);+ LOG.infov("Split shard {0} in stream {1}", shardId, streamName);+ }}public void mergeShards(String streamName, String shardId, String adjacentShardId, String region) {- KinesisStream stream = resolveStream(streamName, region);- KinesisShard shard1 = stream.getShards().stream()- .filter(s -> s.getShardId().equals(shardId))- .findFirst()- .orElseThrow(() -> new AwsException("ResourceNotFoundException", "Shard " + shardId + " not found", 400));- KinesisShard shard2 = stream.getShards().stream()- .filter(s -> s.getShardId().equals(adjacentShardId))- .findFirst()- .orElseThrow(() -> new AwsException("ResourceNotFoundException", "Shard " + adjacentShardId + " not found", 400));+ String key = regionKey(region, streamName);+ synchronized (lockFor(key)) {+ // Resolve under the lock so a concurrent deleteStream (same monitor) cannot leave us+ // mutating and re-persisting a stale, deleted instance.+ KinesisStream stream = resolveStream(streamName, region);+ KinesisShard shard1 = stream.getShards().stream()+ .filter(s -> s.getShardId().equals(shardId))+ .findFirst()+ .orElseThrow(() -> new AwsException("ResourceNotFoundException", "Shard " + shardId + " not found", 400));+ KinesisShard shard2 = stream.getShards().stream()+ .filter(s -> s.getShardId().equals(adjacentShardId))+ .findFirst()+ .orElseThrow(() -> new AwsException("ResourceNotFoundException", "Shard " + adjacentShardId + " not found", 400));- if (shard1.isClosed() || shard2.isClosed()) {- throw new AwsException("InvalidArgumentException", "One or both shards are already closed", 400);- }+ if (shard1.isClosed() || shard2.isClosed()) {+ throw new AwsException("InvalidArgumentException", "One or both shards are already closed", 400);+ }- shard1.setClosed(true);- shard2.setClosed(true);- String seq = String.valueOf(sequenceGenerator.get());- shard1.setSequenceNumberRange(new KinesisShard.SequenceNumberRange(shard1.getSequenceNumberRange().startingSequenceNumber(), seq));- shard2.setSequenceNumberRange(new KinesisShard.SequenceNumberRange(shard2.getSequenceNumberRange().startingSequenceNumber(), seq));+ shard1.setClosed(true);+ shard2.setClosed(true);+ String seq = String.valueOf(sequenceGenerator.get());+ shard1.setSequenceNumberRange(new KinesisShard.SequenceNumberRange(shard1.getSequenceNumberRange().startingSequenceNumber(), seq));+ shard2.setSequenceNumberRange(new KinesisShard.SequenceNumberRange(shard2.getSequenceNumberRange().startingSequenceNumber(), seq));- // Combine hash ranges (assuming they are adjacent)- java.math.BigInteger s1Start = new java.math.BigInteger(shard1.getHashKeyRange().startingHashKey());- java.math.BigInteger s2Start = new java.math.BigInteger(shard2.getHashKeyRange().startingHashKey());-- String start = s1Start.min(s2Start).toString();- java.math.BigInteger s1End = new java.math.BigInteger(shard1.getHashKeyRange().endingHashKey());- java.math.BigInteger s2End = new java.math.BigInteger(shard2.getHashKeyRange().endingHashKey());- String end = s1End.max(s2End).toString();+ // Combine hash ranges (assuming they are adjacent)+ java.math.BigInteger s1Start = new java.math.BigInteger(shard1.getHashKeyRange().startingHashKey());+ java.math.BigInteger s2Start = new java.math.BigInteger(shard2.getHashKeyRange().startingHashKey());- KinesisShard child = new KinesisShard(nextShardId(stream), start, end, seq);- child.setParentShardId(shardId);- child.setAdjacentParentShardId(adjacentShardId);+ String start = s1Start.min(s2Start).toString();+ java.math.BigInteger s1End = new java.math.BigInteger(shard1.getHashKeyRange().endingHashKey());+ java.math.BigInteger s2End = new java.math.BigInteger(shard2.getHashKeyRange().endingHashKey());+ String end = s1End.max(s2End).toString();- stream.getShards().add(child);- store.put(regionKey(region, streamName), stream);- LOG.infov("Merged shards {0} and {1} in stream {2}", shardId, adjacentShardId, streamName);+ KinesisShard child = new KinesisShard(nextShardId(stream), start, end, seq);+ child.setParentShardId(shardId);+ child.setAdjacentParentShardId(adjacentShardId);++ stream.getShards().add(child);+ store.put(key, stream);+ LOG.infov("Merged shards {0} and {1} in stream {2}", shardId, adjacentShardId, streamName);+ }}private String nextShardId(KinesisStream stream) {@@ -452,35 +515,49 @@ public class KinesisService implements ResourceProvider {}public void updateMaxRecordSize(String streamName, int maxRecordSizeInKiB, String region) {- KinesisStream stream = resolveStream(streamName, region);- validateMaxRecordSize(maxRecordSizeInKiB, "ValidationException");- if ("ON_DEMAND".equals(stream.getStreamMode())) {- throw new AwsException("ValidationException",- "UpdateMaxRecordSize is only supported for data streams with the provisioned capacity mode.",- 400);- }- if (!"ACTIVE".equals(stream.getStreamStatus())) {- throw new AwsException("ResourceInUseException",- "Stream " + streamName + " is not ACTIVE (current state: "- + stream.getStreamStatus() + ")", 400);+ String key = regionKey(region, streamName);+ synchronized (lockFor(key)) {+ KinesisStream stream = resolveStream(streamName, region);+ validateMaxRecordSize(maxRecordSizeInKiB, "ValidationException");+ if ("ON_DEMAND".equals(stream.getStreamMode())) {+ throw new AwsException("ValidationException",+ "UpdateMaxRecordSize is only supported for data streams with the provisioned capacity mode.",+ 400);+ }+ if (!"ACTIVE".equals(stream.getStreamStatus())) {+ throw new AwsException("ResourceInUseException",+ "Stream " + streamName + " is not ACTIVE (current state: "+ + stream.getStreamStatus() + ")", 400);+ }+ stream.setMaxRecordSizeInKiB(maxRecordSizeInKiB);+ store.put(key, stream);+ LOG.infov("Updated max record size for {0} to {1} KiB", streamName, maxRecordSizeInKiB);}- stream.setMaxRecordSizeInKiB(maxRecordSizeInKiB);- store.put(regionKey(region, streamName), stream);- LOG.infov("Updated max record size for {0} to {1} KiB", streamName, maxRecordSizeInKiB);}public PutRecordResult putRecordWithShardId(String streamName, byte[] data, String partitionKey, String region) {+ String key = regionKey(region, streamName);KinesisStream stream = resolveStream(streamName, region);validateRecordSize(stream, data, partitionKey);- KinesisShard shard = selectShard(stream, partitionKey);-- String sequenceNumber = String.valueOf(sequenceGenerator.incrementAndGet());- KinesisRecord record = new KinesisRecord(data, partitionKey, sequenceNumber, Instant.now());-- shard.getRecords().add(record);- store.put(regionKey(region, streamName), stream);-- return new PutRecordResult(sequenceNumber, shard.getShardId());+ putRecordBeforeLockHook.run();++ // Serialize shard selection, sequence allocation, append and persistence per stream so+ // concurrent producers can neither lose an append (plain ArrayList) nor store records+ // out of sequence order (which would break AT_/AFTER_SEQUENCE_NUMBER scans and WAL replay).+ synchronized (lockFor(key)) {+ // Re-resolve under the lock. deleteStream holds this same monitor, so a stream deleted+ // between the resolve above and here is now absent from the store. Appending to and+ // persisting the stale pre-delete instance would resurrect a deleted stream, so bind to+ // the instance currently in the store and fail like any missing stream if it is gone.+ KinesisStream current = resolveStream(streamName, region);+ KinesisShard shard = selectShard(current, partitionKey);+ String sequenceNumber = String.valueOf(sequenceGenerator.incrementAndGet());+ putRecordAppendHook.run();+ KinesisRecord record = new KinesisRecord(data, partitionKey, sequenceNumber, Instant.now());+ shard.addRecord(record);+ store.put(key, current);+ return new PutRecordResult(sequenceNumber, shard.getShardId());+ }}public String getShardIterator(String streamName, String shardId, String type, String sequenceNumber, String region) {@@ -527,7 +604,7 @@ public class KinesisService implements ResourceProvider {.findFirst().orElseThrow(() -> new AwsException("ResourceNotFoundException", "Shard not found", 400));// LATEST resumes from the tip snapshot taken now; other types resolve in getRecords.- return "LATEST".equals(type) ? shard.getRecords().size() : 0;+ return "LATEST".equals(type) ? shard.recordCount() : 0;}private int parseIteratorIndex(String value) {@@ -786,6 +863,23 @@ public class KinesisService implements ResourceProvider {return region + "::" + name;}+ /**+ * The monitor guarding a stream key. Every path that mutates a stored stream takes it: record+ * appends, split and merge, the metadata writes, and {@link #deleteStream}.+ *+ * <p>Holding it is only half the contract: a writer must also resolve the stream inside the+ * monitor. A stream resolved before the monitor was acquired may already have been deleted, and+ * persisting that stale instance would resurrect it or clobber a stream since recreated under+ * the same name.+ *+ * <p>Monitors are never removed. One stable monitor per key is a bounded leak, whereas dropping+ * one on delete would let a writer that already resolved the stream create a fresh monitor and+ * persist the deleted instance under it, defeating the serialization.+ */+ private Object lockFor(String storageKey) {+ return streamAppendLocks.computeIfAbsent(storageKey, k -> new Object());+ }+// ─── Resource Explorer 2 ───────────────────────────────────────────────────@Overridediff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/model/KinesisShard.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/model/KinesisShard.javaindex 1d088ceb51..58a8aa4963 100644--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/model/KinesisShard.java+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/model/KinesisShard.java@@ -15,6 +15,7 @@ public class KinesisShard {private String adjacentParentShardId;private HashKeyRange hashKeyRange;private SequenceNumberRange sequenceNumberRange;+ private final Object recordsMonitor = new Object();private List<KinesisRecord> records = new ArrayList<>();private boolean closed = false;private Instant creationTimestamp = Instant.now();@@ -42,8 +43,38 @@ public class KinesisShard {public SequenceNumberRange getSequenceNumberRange() { return sequenceNumberRange; }public void setSequenceNumberRange(SequenceNumberRange range) { this.sequenceNumberRange = range; }- public List<KinesisRecord> getRecords() { return records; }- public void setRecords(List<KinesisRecord> records) { this.records = records; }+ /**+ * A shallow snapshot of this shard's records. Safe to index and iterate without a+ * ConcurrentModificationException even while producers append concurrently. Structural+ * mutation of the returned list does NOT affect the shard (append via {@link #addRecord});+ * element references are shared, so per-element setters still write through.+ */+ public List<KinesisRecord> getRecords() {+ synchronized (recordsMonitor) {+ return new ArrayList<>(records);+ }+ }++ /** Appends a record. The sole production mutation path for a shard's log. */+ public void addRecord(KinesisRecord record) {+ synchronized (recordsMonitor) {+ records.add(record);+ }+ }++ /** Number of records currently held, without copying the log. */+ public int recordCount() {+ synchronized (recordsMonitor) {+ return records.size();+ }+ }++ /** Jackson rehydration only; not safe for concurrent replacement during live traffic. */+ public void setRecords(List<KinesisRecord> records) {+ synchronized (recordsMonitor) {+ this.records = records == null ? new ArrayList<>() : new ArrayList<>(records);+ }+ }public boolean isClosed() { return closed; }public void setClosed(boolean closed) { this.closed = closed; }diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/model/KinesisStream.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/model/KinesisStream.javaindex ca0130da39..d8af576ae1 100644--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/model/KinesisStream.java+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/model/KinesisStream.java@@ -4,12 +4,12 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties;import io.quarkus.runtime.annotations.RegisterForReflection;import java.time.Instant;-import java.util.ArrayList;import java.util.HashMap;import java.util.HashSet;import java.util.List;import java.util.Map;import java.util.Set;+import java.util.concurrent.CopyOnWriteArrayList;@RegisterForReflection@JsonIgnoreProperties(ignoreUnknown = true)@@ -18,7 +18,7 @@ public class KinesisStream {private String streamArn;private String accountId;private String streamStatus;- private List<KinesisShard> shards = new ArrayList<>();+ private volatile List<KinesisShard> shards = new CopyOnWriteArrayList<>();private int retentionPeriodHours = 24;private Instant streamCreationTimestamp;private Map<String, String> tags = new HashMap<>();@@ -51,7 +51,9 @@ public class KinesisStream {public void setStreamStatus(String streamStatus) { this.streamStatus = streamStatus; }public List<KinesisShard> getShards() { return shards; }- public void setShards(List<KinesisShard> shards) { this.shards = shards; }+ public void setShards(List<KinesisShard> shards) {+ this.shards = shards == null ? new CopyOnWriteArrayList<>() : new CopyOnWriteArrayList<>(shards);+ }public int getRetentionPeriodHours() { return retentionPeriodHours; }public void setRetentionPeriodHours(int retentionPeriodHours) { this.retentionPeriodHours = retentionPeriodHours; }diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbConcurrencyIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbConcurrencyIntegrationTest.javaindex 3facae51a2..37d88e121c 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbConcurrencyIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbConcurrencyIntegrationTest.java@@ -1,13 +1,19 @@package io.github.hectorvent.floci.services.dynamodb;import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.InMemoryStorage;import io.github.hectorvent.floci.core.storage.StorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;import io.github.hectorvent.floci.services.dynamodb.model.ConditionalCheckFailedException;import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;+import io.github.hectorvent.floci.services.dynamodb.model.KinesisStreamingDestination;import io.github.hectorvent.floci.services.dynamodb.model.StreamDescription;import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;+import io.github.hectorvent.floci.services.kinesis.KinesisService;+import io.github.hectorvent.floci.services.kinesis.model.KinesisRecord;+import io.github.hectorvent.floci.services.kinesis.model.KinesisStream;import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;import com.fasterxml.jackson.databind.node.ObjectNode;@@ -17,6 +23,7 @@ import org.junit.jupiter.api.Test;import java.util.ArrayList;import java.util.Collections;+import java.util.HashMap;import java.util.HashSet;import java.util.List;import java.util.Map;@@ -29,6 +36,10 @@ import java.util.concurrent.TimeUnit;import java.util.concurrent.atomic.AtomicInteger;import static org.junit.jupiter.api.Assertions.*;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.when;/*** Concurrency compatibility suite for {@link DynamoDbService}.@@ -481,6 +492,89 @@ class DynamoDbConcurrencyIntegrationTest {assertEquals(ops, previous, "final event must reflect final counter value");}+ /**+ * End-to-end CDC wiring: concurrent putItem must forward EVERY record to the attached Kinesis+ * destination, with no loss and per-shard sequence order preserved. This is the gap the suite's+ * production wiring left uncovered (the shared {@code service} field wires {@code forwarder=null}).+ * See issue #571 (Kinesis append race).+ */+ @RepeatedTest(REPEATS)+ void concurrent_putItem_forwards_every_cdc_record_to_kinesis() throws Exception {+ String region = "us-east-1";+ StorageBackend<String, TableDefinition> tableStore = new InMemoryStorage<>();+ // Public KinesisService ctor via a mocked StorageFactory that hands out in-memory+ // backends keyed by file name (the FirehoseServiceTest pattern), because the+ // package-private ctor is not visible from this package.+ Map<String, AccountAwareStorageBackend<?>> backends = new HashMap<>();+ StorageFactory storageFactory = mock(StorageFactory.class);+ when(storageFactory.create(anyString(), anyString(), any()))+ .thenAnswer(inv -> backends.computeIfAbsent(+ inv.getArgument(0) + "/" + inv.getArgument(1),+ k -> AccountAwareStorageBackend.inMemory("000000000000")));+ KinesisService kinesis = new KinesisService(storageFactory,+ new RegionResolver("us-east-1", "000000000000"));+ KinesisStreamingForwarder forwarder = new KinesisStreamingForwarder(kinesis, mapper);+ DynamoDbService svc = new DynamoDbService(+ tableStore, new InMemoryStorage<>(),+ new RegionResolver("us-east-1", "000000000000"), null, forwarder);++ KinesisStream stream = kinesis.createStream("cdc-stream", 1, region);+ TableDefinition table = svc.createTable("Events",+ List.of(new KeySchemaElement("pk", "HASH"), new KeySchemaElement("sk", "RANGE")),+ List.of(new AttributeDefinition("pk", "S"), new AttributeDefinition("sk", "S")),+ 5L, 5L, region);+ table.getKinesisStreamingDestinations().add(+ new KinesisStreamingDestination(stream.getStreamArn()));++ int threads = 16;+ int opsPerThread = 25;+ int expected = threads * opsPerThread; // 400+ AtomicInteger idSource = new AtomicInteger();++ List<Throwable> errors = runConcurrently(threads, () -> {+ for (int i = 0; i < opsPerThread; i++) {+ int id = idSource.getAndIncrement();+ ObjectNode item = mapper.createObjectNode();+ item.set("pk", stringAttr("fixed"));+ item.set("sk", stringAttr("t-" + id));+ svc.putItem("Events", item, region);+ }+ });+ assertTrue(errors.isEmpty(), () -> "unexpected errors: " + errors);++ // Paginated drain of the single shard.+ String shardId = kinesis.describeStream("cdc-stream", region).getShards().getFirst().getShardId();+ String iterator = kinesis.getShardIterator("cdc-stream", shardId, "TRIM_HORIZON", null, region);+ List<KinesisRecord> drained = new ArrayList<>();+ while (true) {+ Map<String, Object> page = kinesis.getRecords(iterator, 1000, region);+ @SuppressWarnings("unchecked")+ List<KinesisRecord> records = (List<KinesisRecord>) page.get("Records");+ if (records.isEmpty()) break;+ drained.addAll(records);+ iterator = (String) page.get("NextShardIterator");+ }++ assertEquals(expected, drained.size(), "every CDC event must reach Kinesis");++ Set<String> sortKeys = new HashSet<>();+ long previousSeq = Long.MIN_VALUE;+ for (KinesisRecord record : drained) {+ JsonNode payload = mapper.readTree(record.getData());+ sortKeys.add(payload.get("dynamodb").get("Keys").get("sk").get("S").asText());+ long seq = Long.parseLong(record.getSequenceNumber());+ assertTrue(seq > previousSeq, "per-shard sequence numbers must be strictly increasing");+ previousSeq = seq;+ }++ Set<String> expectedKeys = new HashSet<>();+ for (int i = 0; i < expected; i++) {+ expectedKeys.add("t-" + i);+ }+ assertEquals(expectedKeys, sortKeys, "the forwarded sk set must match exactly, with no gaps");+ assertEquals(0L, forwarder.getForwardFailureCount(), "no CDC event may be dropped");+ }+@Testvoid baseline_single_threaded_still_works() {String region = "eu-west-1";diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlForwardingTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlForwardingTest.javanew file mode 100644index 0000000000..ea52c227b9--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlForwardingTest.java@@ -0,0 +1,178 @@+package io.github.hectorvent.floci.services.dynamodb;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ObjectNode;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.core.storage.StorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;+import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;+import io.github.hectorvent.floci.services.dynamodb.model.KinesisStreamingDestination;+import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;+import io.github.hectorvent.floci.services.kinesis.KinesisService;+import io.github.hectorvent.floci.services.kinesis.model.KinesisRecord;+import io.github.hectorvent.floci.services.kinesis.model.KinesisStream;+import org.junit.jupiter.api.Test;++import java.time.Instant;+import java.util.ArrayList;+import java.util.HashMap;+import java.util.List;+import java.util.Map;++import static org.junit.jupiter.api.Assertions.*;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.when;++/**+ * TTL sweep must forward a REMOVE per expired item to Kinesis (happy path) and must isolate+ * forwarding failures: a throwing forwarder cannot abort the sweep or block persistence.+ * See issue #571.+ */+class DynamoDbTtlForwardingTest {++ private static final String REGION = "us-east-1";++ private final ObjectMapper mapper = new ObjectMapper();++ private KinesisService realKinesis() {+ Map<String, AccountAwareStorageBackend<?>> backends = new HashMap<>();+ StorageFactory factory = mock(StorageFactory.class);+ when(factory.create(anyString(), anyString(), any()))+ .thenAnswer(inv -> backends.computeIfAbsent(+ inv.getArgument(0) + "/" + inv.getArgument(1),+ k -> AccountAwareStorageBackend.inMemory("000000000000")));+ return new KinesisService(factory, new RegionResolver("us-east-1", "000000000000"));+ }++ private ObjectNode stringAttr(String v) {+ ObjectNode n = mapper.createObjectNode();+ n.put("S", v);+ return n;+ }++ private ObjectNode numberAttr(long v) {+ ObjectNode n = mapper.createObjectNode();+ n.put("N", String.valueOf(v));+ return n;+ }++ private void seedTtlTable(DynamoDbService svc, int expiredCount) {+ svc.createTable("TtlTable",+ List.of(new KeySchemaElement("pk", "HASH")),+ List.of(new AttributeDefinition("pk", "S")),+ 5L, 5L, REGION);+ svc.updateTimeToLive("TtlTable", "expireAt", true, REGION);+ long past = Instant.now().getEpochSecond() - 3600;+ for (int i = 0; i < expiredCount; i++) {+ ObjectNode item = mapper.createObjectNode();+ item.set("pk", stringAttr("row-" + i));+ item.set("expireAt", numberAttr(past));+ svc.putItem("TtlTable", item, REGION);+ }+ }++ @SuppressWarnings("unchecked")+ private List<KinesisRecord> drain(KinesisService kinesis, String streamName) {+ String shardId = kinesis.describeStream(streamName, REGION).getShards().getFirst().getShardId();+ String iterator = kinesis.getShardIterator(streamName, shardId, "TRIM_HORIZON", null, REGION);+ List<KinesisRecord> all = new ArrayList<>();+ while (true) {+ Map<String, Object> page = kinesis.getRecords(iterator, 1000, REGION);+ List<KinesisRecord> records = (List<KinesisRecord>) page.get("Records");+ if (records.isEmpty()) break;+ all.addAll(records);+ iterator = (String) page.get("NextShardIterator");+ }+ return all;+ }++ @Test+ void ttlSweep_forwards_one_remove_per_expired_item() throws Exception {+ StorageBackend<String, TableDefinition> tableStore = new InMemoryStorage<>();+ StorageBackend<String, Map<String, JsonNode>> itemStore = new InMemoryStorage<>();+ KinesisService kinesis = realKinesis();+ KinesisStreamingForwarder forwarder = new KinesisStreamingForwarder(kinesis, mapper);+ DynamoDbService svc = new DynamoDbService(+ tableStore, itemStore, new RegionResolver("us-east-1", "000000000000"), null, forwarder);++ KinesisStream stream = kinesis.createStream("ttl-stream", 1, REGION);+ int expired = 5;+ seedTtlTable(svc, expired);+ // Attach the destination after createTable (live stored ref).+ TableDefinition table = tableStore.get("us-east-1::TtlTable").orElseThrow();+ table.getKinesisStreamingDestinations().add(new KinesisStreamingDestination(stream.getStreamArn()));++ svc.deleteExpiredItems();++ List<KinesisRecord> drained = drain(kinesis, "ttl-stream");+ assertEquals(expired, drained.size(), "one Kinesis record per expired item");+ for (KinesisRecord record : drained) {+ JsonNode payload = mapper.readTree(record.getData());+ assertEquals("REMOVE", payload.get("eventName").asText(), "TTL deletions forward as REMOVE");+ }+ assertEquals(0L, forwarder.getForwardFailureCount(), "happy path drops nothing");++ for (int i = 0; i < expired; i++) {+ ObjectNode key = mapper.createObjectNode();+ key.set("pk", stringAttr("row-" + i));+ assertNull(svc.getItem("TtlTable", key, REGION), "expired item must be gone");+ }+ }++ @Test+ void ttlSweep_isolates_forwarding_failures_and_still_persists() {+ StorageBackend<String, TableDefinition> tableStore = new InMemoryStorage<>();+ StorageBackend<String, Map<String, JsonNode>> itemStore = new InMemoryStorage<>();++ KinesisService throwingKinesis = mock(KinesisService.class);+ when(throwingKinesis.putRecord(anyString(), any(byte[].class), anyString(), anyString()))+ .thenThrow(new RuntimeException("kinesis unavailable"));+ KinesisStreamingForwarder forwarder = new KinesisStreamingForwarder(throwingKinesis, mapper);+ DynamoDbService svc = new DynamoDbService(+ tableStore, itemStore, new RegionResolver("us-east-1", "000000000000"), null, forwarder);++ int expired = 5;+ seedTtlTable(svc, expired);+ TableDefinition table = tableStore.get("us-east-1::TtlTable").orElseThrow();+ table.getKinesisStreamingDestinations().add(new KinesisStreamingDestination(+ "arn:aws:kinesis:us-east-1:000000000000:stream/ttl-stream"));++ // A forwarding failure must not propagate out of the sweep.+ assertDoesNotThrow(svc::deleteExpiredItems);++ assertEquals(expired, forwarder.getForwardFailureCount(),+ "each failed forward must be counted, not swallowed silently");++ // Removed in-memory AND persisted: a fresh service over the same stores must not see them.+ for (int i = 0; i < expired; i++) {+ ObjectNode key = mapper.createObjectNode();+ key.set("pk", stringAttr("row-" + i));+ assertNull(svc.getItem("TtlTable", key, REGION), "expired item removed from live state");+ }++ // Direct proof of persistence: the sweep must have written the trimmed map back to the+ // underlying item store. Inspecting the stored map (rather than getItem) is what makes this+ // fail if persistItemsForAccount is dropped: getItem would still return null for expired+ // rows via isExpired even though the stale rows remained physically persisted. putItem+ // during seeding persisted 'expired' rows here, so the store must now hold none of them.+ Map<String, JsonNode> persisted = itemStore.get("us-east-1::TtlTable").orElseGet(Map::of);+ assertTrue(persisted.isEmpty(),+ () -> "TTL removals must be persisted to the item store; stale rows still present: "+ + persisted.keySet());++ // Corroborate via a fresh service reload over the same stores.+ DynamoDbService reloaded = new DynamoDbService(+ tableStore, itemStore, new RegionResolver("us-east-1", "000000000000"));+ for (int i = 0; i < expired; i++) {+ ObjectNode key = mapper.createObjectNode();+ key.set("pk", stringAttr("row-" + i));+ assertNull(reloaded.getItem("TtlTable", key, REGION), "expired removal must be persisted");+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarderTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarderTest.javaindex 1b8a004626..04800d1533 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarderTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarderTest.java@@ -12,6 +12,7 @@ import org.junit.jupiter.api.Test;import java.util.List;+import static org.junit.jupiter.api.Assertions.assertEquals;import static org.mockito.ArgumentMatchers.*;import static org.mockito.Mockito.*;@@ -98,5 +99,7 @@ class KinesisStreamingForwarderTest {verify(kinesisService).putRecord(eq("stream-1"), any(byte[].class), anyString(), anyString());verify(kinesisService).putRecord(eq("stream-2"), any(byte[].class), anyString(), anyString());+ assertEquals(1L, forwarder.getForwardFailureCount(),+ "the single failed forward must be counted");}}diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisListShardsConcurrencyTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisListShardsConcurrencyTest.javanew file mode 100644index 0000000000..55b47d8c8f--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisListShardsConcurrencyTest.java@@ -0,0 +1,77 @@+package io.github.hectorvent.floci.services.kinesis;++import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.services.kinesis.model.KinesisShard;+import io.github.hectorvent.floci.services.kinesis.model.KinesisStream;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.util.List;++import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;+import static org.junit.jupiter.api.Assertions.assertEquals;++/**+ * Regression for the ListShards {@code CopyOnWriteArrayList} sub-list hazard.+ *+ * <p>Exercises {@link KinesisJsonHandler#paginateShards(List, int)} directly. That is the exact+ * production step ListShards uses to take a {@code MaxResults} page, so the test needs no JAX-RS+ * runtime (absent from the sandbox launcher) and no timing window.+ */+class KinesisListShardsConcurrencyTest {++ private static final String REGION = "us-east-1";++ private KinesisService service;++ @BeforeEach+ void setUp() {+ service = new KinesisService(+ new InMemoryStorage<>(),+ new InMemoryStorage<>(),+ new RegionResolver(REGION, "123456789012"));+ }++ /**+ * A ListShards page must survive a concurrent split of the underlying stream.+ *+ * <p>ListShards takes {@code MaxResults < shard count}, so a sub-list page is taken; then a split+ * appends two children to the live backing list. Pre-fix ({@code shards.subList(...)} over the live+ * CopyOnWriteArrayList) reading the page throws {@link java.util.ConcurrentModificationException};+ * post-fix (page taken from a {@code List.copyOf} snapshot) the page reflects the pre-split shards+ * and reads cleanly.+ */+ @Test+ void listShardsPageSurvivesConcurrentSplit() {+ service.createStream("reshard-stream", 3, REGION);+ KinesisStream stream = service.describeStream("reshard-stream", REGION);++ // The page ListShards would build for MaxResults=2 against the live shard list.+ List<KinesisShard> page = KinesisJsonHandler.paginateShards(stream.getShards(), 2);++ // A concurrent split closes the parent and atomically appends two children, mutating the+ // live backing list the pre-fix sub-list page still points at.+ String firstShardId = stream.getShards().getFirst().getShardId();+ service.splitShard("reshard-stream", firstShardId,+ "170141183460469231731687303715884105728", REGION);++ // Pre-fix: iterating/sizing the live sub-list here throws ConcurrentModificationException.+ assertDoesNotThrow(() -> {+ int seen = 0;+ for (KinesisShard shard : page) {+ shard.getShardId();+ seen++;+ }+ assertEquals(2, seen);+ });++ // A consistent page: exactly the first two shards of the pre-split snapshot, in order.+ assertEquals(2, page.size());+ assertEquals("shardId-000000000000", page.get(0).getShardId());+ assertEquals("shardId-000000000001", page.get(1).getShardId());++ // The split itself completed on the underlying stream: 3 original + 2 children.+ assertEquals(5, service.describeStream("reshard-stream", REGION).getShards().size());+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisServiceTest.javaindex 42ee0afb1b..ec20fc862d 100644--- a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisServiceTest.java@@ -3,18 +3,39 @@ package io.github.hectorvent.floci.services.kinesis;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.services.kinesis.model.KinesisConsumer;import io.github.hectorvent.floci.services.kinesis.model.KinesisRecord;import io.github.hectorvent.floci.services.kinesis.model.KinesisShard;import io.github.hectorvent.floci.services.kinesis.model.KinesisStream;import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.RepeatedTest;import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.Arguments;+import org.junit.jupiter.params.provider.MethodSource;import java.nio.charset.StandardCharsets;import java.time.Instant;+import java.util.ArrayList;+import java.util.Collections;+import java.util.HashSet;+import java.util.Iterator;import java.util.List;import java.util.Map;+import java.util.Optional;import java.util.Set;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.TimeoutException;+import java.util.concurrent.atomic.AtomicBoolean;+import java.util.concurrent.atomic.AtomicReference;+import java.util.function.Consumer;+import java.util.function.Predicate;+import java.util.stream.Stream;import static org.junit.jupiter.api.Assertions.*;@@ -288,7 +309,127 @@ class KinesisServiceTest {KinesisStream updated = kinesisService.describeStream("my-stream", REGION);assertEquals(3, updated.getShards().size());- assertTrue(updated.getShards().getFirst().isClosed());+ KinesisShard parent = updated.getShards().getFirst();+ assertTrue(parent.isClosed());++ // Both children must get distinct shard ids. nextShardId is size-based, so computing both+ // ids before adding either child assigns the same id to both (regression guard).+ List<String> shardIds = updated.getShards().stream().map(KinesisShard::getShardId).toList();+ assertEquals(shardIds.size(), new HashSet<>(shardIds).size(), "shard ids must be unique after split");++ // The single atomic addAll must publish a COMPLETE topology: both children present, each+ // pointing at the parent, together tiling the parent's whole hash range with no gap/overlap.+ List<KinesisShard> children = updated.getShards().stream()+ .filter(s -> shardId.equals(s.getParentShardId()))+ .toList();+ assertEquals(2, children.size(), "a split must publish exactly two children");+ KinesisShard child1 = children.get(0);+ KinesisShard child2 = children.get(1);+ assertEquals(parent.getHashKeyRange().startingHashKey(), child1.getHashKeyRange().startingHashKey());+ assertEquals("170141183460469231731687303715884105727", child1.getHashKeyRange().endingHashKey());+ assertEquals("170141183460469231731687303715884105728", child2.getHashKeyRange().startingHashKey());+ assertEquals(parent.getHashKeyRange().endingHashKey(), child2.getHashKeyRange().endingHashKey());+ }++ /**+ * A lock-free reader must never observe a half-published split: a closed parent with only one+ * of its two children. splitShard publishes both children in one atomic CopyOnWriteArrayList.addAll,+ * so every snapshot holds either zero or both split-children of any given parent.+ *+ * <p>Fail-without-fix: reverting the addAll to two sequential {@code add()} calls opens a window+ * (between the two adds, widened by the child2 shard construction) in which the list holds child1+ * but not child2; the hammering reader below catches it and records a violation.+ */+ @RepeatedTest(3)+ void splitShard_concurrentReaderNeverSeesHalfPublishedTopology() throws Exception {+ kinesisService.createStream("my-stream", 1, REGION);+ KinesisStream stream = kinesisService.describeStream("my-stream", REGION);++ AtomicReference<String> violation = new AtomicReference<>();+ AtomicBoolean done = new AtomicBoolean(false);++ Thread reader = new Thread(() -> {+ while (!done.get()) {+ // A fresh snapshot of the live list, exactly as a lock-free DescribeStream/ListShards sees it.+ List<KinesisShard> snapshot = new ArrayList<>(stream.getShards());+ Map<String, Integer> splitChildCount = new java.util.HashMap<>();+ for (KinesisShard s : snapshot) {+ // Split children carry a parent but no adjacent-parent (that marks a merge child).+ if (s.getParentShardId() != null && s.getAdjacentParentShardId() == null) {+ splitChildCount.merge(s.getParentShardId(), 1, Integer::sum);+ }+ }+ for (Map.Entry<String, Integer> e : splitChildCount.entrySet()) {+ if (e.getValue() != 2) {+ violation.compareAndSet(null,+ "parent " + e.getKey() + " observed with " + e.getValue()+ + " split-children (expected 0 or 2)");+ return;+ }+ }+ }+ });+ reader.start();++ // Split an open shard repeatedly; each split is one control-plane resharding with its own+ // publish window. Always split an open shard so open shards remain available.+ for (int i = 0; i < 3000 && violation.get() == null; i++) {+ // Split the widest open shard so ranges never narrow to the point splitting is impossible;+ // this keeps every one of the 3000 iterations a real split with its own publish window.+ KinesisShard open = stream.getShards().stream()+ .filter(s -> !s.isClosed())+ .max(java.util.Comparator.comparing(s ->+ new java.math.BigInteger(s.getHashKeyRange().endingHashKey())+ .subtract(new java.math.BigInteger(s.getHashKeyRange().startingHashKey()))))+ .orElseThrow();+ java.math.BigInteger start = new java.math.BigInteger(open.getHashKeyRange().startingHashKey());+ java.math.BigInteger end = new java.math.BigInteger(open.getHashKeyRange().endingHashKey());+ java.math.BigInteger mid = start.add(end).divide(java.math.BigInteger.TWO);+ if (mid.compareTo(start) <= 0 || mid.compareTo(end) >= 0) {+ break; // range too narrow to split further+ }+ kinesisService.splitShard("my-stream", open.getShardId(), mid.toString(), REGION);+ }+ done.set(true);+ reader.join(TimeUnit.SECONDS.toMillis(10));+ assertFalse(reader.isAlive(), "reader thread did not terminate");++ assertNull(violation.get(), () -> "half-published split observed: " + violation.get());+ }++ @Test+ void getShards_readerMidIteration_survivesConcurrentSplit() {+ // Two shards so the snapshot the reader opens still has an element left to read AFTER the+ // concurrent split; a single-shard snapshot would be exhausted by the first next() and the+ // second next() would throw NoSuchElementException even under COWAL.+ kinesisService.createStream("my-stream", 2, REGION);+ // describeStream returns the same KinesisStream instance the store holds, so these iterators+ // are over the exact backing list splitShard mutates.+ KinesisStream stream = kinesisService.describeStream("my-stream", REGION);+ String shardId = stream.getShards().getFirst().getShardId();++ // A second iterator opened before the split, used to assert the pre-split snapshot is fixed.+ Iterator<KinesisShard> preSplitSnapshot = stream.getShards().iterator();+ Iterator<KinesisShard> reader = stream.getShards().iterator();+ reader.next();++ kinesisService.splitShard("my-stream", shardId, "170141183460469231731687303715884105728", REGION);++ // A plain ArrayList iterator throws ConcurrentModificationException here; COWAL keeps reading+ // its fixed pre-split snapshot.+ assertDoesNotThrow(reader::next);++ // The snapshot opened before the split still reflects exactly the original two shards,+ // regardless of the two children the split appended afterwards.+ int snapshotSize = 0;+ while (preSplitSnapshot.hasNext()) {+ preSplitSnapshot.next();+ snapshotSize++;+ }+ assertEquals(2, snapshotSize);++ // A fresh read observes the completed reshard: 2 original shards + 2 children.+ assertEquals(4, kinesisService.describeStream("my-stream", REGION).getShards().size());}@Test@@ -444,4 +585,504 @@ class KinesisServiceTest {.get("Records");assertEquals(1, got.size(), "AT_TIMESTAMP boundary is >= (inclusive)");}++ // ─── Concurrency regression coverage (issue #571 sibling: Kinesis append race) ──────++ /** Runs {@code op} {@code opsPerThread} times on each of {@code threadCount} threads,+ * all released together, and returns any throwables observed. */+ private List<Throwable> runConcurrently(int threadCount, int opsPerThread, Runnable op)+ throws InterruptedException {+ ExecutorService pool = Executors.newFixedThreadPool(threadCount);+ CountDownLatch startGate = new CountDownLatch(1);+ CountDownLatch doneGate = new CountDownLatch(threadCount);+ List<Throwable> errors = Collections.synchronizedList(new ArrayList<>());+ try {+ for (int t = 0; t < threadCount; t++) {+ pool.submit(() -> {+ try {+ startGate.await();+ for (int i = 0; i < opsPerThread; i++) {+ op.run();+ }+ } catch (Throwable e) {+ errors.add(e);+ } finally {+ doneGate.countDown();+ }+ });+ }+ startGate.countDown();+ assertTrue(doneGate.await(60, TimeUnit.SECONDS),+ "concurrent producers did not complete within 60s");+ return errors;+ } finally {+ pool.shutdownNow();+ assertTrue(pool.awaitTermination(5, TimeUnit.SECONDS), "thread pool did not terminate");+ }+ }++ /** Drains every record of a single-shard stream, following NextShardIterator to the end. */+ private List<KinesisRecord> drainShard(String streamName, String type, String seq) {+ String shardId = kinesisService.describeStream(streamName, REGION)+ .getShards().getFirst().getShardId();+ String iterator = kinesisService.getShardIterator(streamName, shardId, type, seq, REGION);+ return drainFrom(iterator);+ }++ @SuppressWarnings("unchecked")+ private List<KinesisRecord> drainFrom(String iterator) {+ List<KinesisRecord> all = new ArrayList<>();+ while (true) {+ Map<String, Object> page = kinesisService.getRecords(iterator, 1000, REGION);+ List<KinesisRecord> records = (List<KinesisRecord>) page.get("Records");+ if (records.isEmpty()) {+ break;+ }+ all.addAll(records);+ iterator = (String) page.get("NextShardIterator");+ }+ return all;+ }++ private static void assertStrictlyIncreasing(List<KinesisRecord> records) {+ long previous = Long.MIN_VALUE;+ for (KinesisRecord r : records) {+ long current = Long.parseLong(r.getSequenceNumber());+ assertTrue(current > previous,+ () -> "sequence numbers must be strictly increasing in shard order");+ previous = current;+ }+ }++ @RepeatedTest(5)+ void concurrent_putRecord_preserves_every_record_and_ordering() throws InterruptedException {+ kinesisService.createStream("stress", 1, REGION);+ byte[] data = "payload".getBytes(StandardCharsets.UTF_8);+ int threads = 32;+ int opsPerThread = 50;+ int expected = threads * opsPerThread; // 1600++ List<Throwable> errors = runConcurrently(threads, opsPerThread,+ () -> kinesisService.putRecord("stress", data, "pk", REGION));+ assertTrue(errors.isEmpty(), () -> "unexpected errors: " + errors);++ List<KinesisRecord> all = drainShard("stress", "TRIM_HORIZON", null);+ assertEquals(expected, all.size(),+ "every concurrently appended record must survive");++ Set<String> distinct = new HashSet<>();+ all.forEach(r -> distinct.add(r.getSequenceNumber()));+ assertEquals(expected, distinct.size(), "every record must carry a distinct sequence number");++ assertStrictlyIncreasing(all);++ // AFTER_SEQUENCE_NUMBER continuation: everything strictly after position k, in order.+ int k = 799;+ String kSeq = all.get(k).getSequenceNumber();+ List<KinesisRecord> after = drainShard("stress", "AFTER_SEQUENCE_NUMBER", kSeq);+ List<String> expectedTail = new ArrayList<>();+ for (int i = k + 1; i < all.size(); i++) {+ expectedTail.add(all.get(i).getSequenceNumber());+ }+ List<String> actualTail = new ArrayList<>();+ after.forEach(r -> actualTail.add(r.getSequenceNumber()));+ assertEquals(expectedTail, actualTail,+ "AFTER_SEQUENCE_NUMBER must return exactly the records after position k, in order");+ }++ @Test+ void putRecord_appendCriticalSectionIsMutuallyExclusive() throws Exception {+ kinesisService.createStream("mx", 1, REGION);++ CountDownLatch entered = new CountDownLatch(1);+ CountDownLatch release = new CountDownLatch(1);+ AtomicBoolean firstIn = new AtomicBoolean(true);+ // Only the first producer to enter the critical section parks; it holds the stream lock+ // until released, so any second producer on the same stream must block behind it.+ kinesisService.putRecordAppendHook = () -> {+ if (firstIn.compareAndSet(true, false)) {+ entered.countDown();+ try {+ release.await();+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ }+ }+ };++ // T2 signals the instant before it calls putRecord and publishes its thread, so we can+ // prove it was actually scheduled and reached the critical section (BLOCKED on the stream+ // monitor), not merely never scheduled, which would let a timeout pass vacuously on a+ // loaded worker even if the lock had been removed.+ CountDownLatch t2Started = new CountDownLatch(1);+ AtomicReference<Thread> t2Thread = new AtomicReference<>();++ ExecutorService pool = Executors.newFixedThreadPool(2);+ try {+ Future<?> t1 = pool.submit(() ->+ kinesisService.putRecord("mx", "1".getBytes(StandardCharsets.UTF_8), "pk", REGION));+ assertTrue(entered.await(2, TimeUnit.SECONDS), "T1 must enter the append critical section");++ Future<?> t2 = pool.submit(() -> {+ t2Thread.set(Thread.currentThread());+ t2Started.countDown();+ return kinesisService.putRecord("mx", "2".getBytes(StandardCharsets.UTF_8), "pk", REGION);+ });++ assertTrue(t2Started.await(2, TimeUnit.SECONDS),+ "T2 must be scheduled and reach the putRecord call");++ // Deterministic barrier: T2 must actually reach and block on the stream monitor while+ // T1 holds it. Spin until its thread is BLOCKED (entering the synchronized section);+ // if the lock were gone T2 would run to completion instead and never park here.+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2);+ boolean blockedOnMonitor = false;+ while (System.nanoTime() < deadline) {+ Thread th = t2Thread.get();+ if (th != null && th.getState() == Thread.State.BLOCKED) {+ blockedOnMonitor = true;+ break;+ }+ if (t2.isDone()) {+ break;+ }+ Thread.sleep(5);+ }+ assertTrue(blockedOnMonitor,+ "T2 must block entering the append critical section while T1 holds the stream lock");++ // Mutual exclusion: T2 cannot complete while T1 holds the lock.+ assertThrows(TimeoutException.class, () -> t2.get(500, TimeUnit.MILLISECONDS),+ "a second producer must block while the first holds the stream lock");++ release.countDown();+ t1.get(2, TimeUnit.SECONDS);+ t2.get(2, TimeUnit.SECONDS);+ } finally {+ pool.shutdownNow();+ }++ List<KinesisRecord> all = drainShard("mx", "TRIM_HORIZON", null);+ assertEquals(2, all.size(), "both records must be persisted");+ long firstSeq = Long.parseLong(all.get(0).getSequenceNumber());+ long secondSeq = Long.parseLong(all.get(1).getSequenceNumber());+ assertTrue(firstSeq < secondSeq,+ "T1 completed its append first, so its lower sequence must sort first");+ }++ /**+ * A deleteStream that overlaps an in-flight append must be serialized behind it and must NOT+ * be able to resurrect the stream. P1 parks inside the append critical section holding the+ * stream monitor; the deleter can only run once P1 releases it, so the delete always wins and+ * the stream stays gone. Fails on the pre-fix code where deleteStream neither took the monitor+ * nor left it in place: the deleter would run immediately and P1's trailing store.put would+ * re-persist (resurrect) the deleted instance.+ */+ @Test+ void deleteStream_serializesBehindInFlightAppend_andDoesNotResurrect() throws Exception {+ kinesisService.createStream("res", 1, REGION);++ CountDownLatch entered = new CountDownLatch(1);+ CountDownLatch release = new CountDownLatch(1);+ AtomicBoolean firstIn = new AtomicBoolean(true);+ kinesisService.putRecordAppendHook = () -> {+ if (firstIn.compareAndSet(true, false)) {+ entered.countDown();+ try {+ release.await();+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ }+ }+ };++ ExecutorService pool = Executors.newFixedThreadPool(2);+ try {+ Future<?> producer = pool.submit(() ->+ kinesisService.putRecord("res", "p1".getBytes(StandardCharsets.UTF_8), "pk", REGION));+ assertTrue(entered.await(2, TimeUnit.SECONDS), "producer must enter the append critical section");++ // The deleter shares the stream monitor the parked producer holds, so it must block.+ Future<?> deleter = pool.submit(() -> {+ kinesisService.deleteStream("res", REGION);+ return null;+ });+ assertThrows(TimeoutException.class, () -> deleter.get(500, TimeUnit.MILLISECONDS),+ "deleteStream must block behind the in-flight append, not delete concurrently");++ release.countDown();+ producer.get(2, TimeUnit.SECONDS);+ deleter.get(2, TimeUnit.SECONDS);+ } finally {+ pool.shutdownNow();+ }++ // The delete ran after the producer's persist, so the stream must be gone, not resurrected+ // by a stale trailing store.put.+ assertTrue(kinesisService.listStreams(REGION).isEmpty(),+ "a deleted stream must not be resurrected by an overlapping append");+ assertThrows(AwsException.class, () -> kinesisService.describeStream("res", REGION),+ "the resurrected stream must not be describable");++ // And a freshly recreated stream starts empty: no stale record leaked across the lifecycle.+ kinesisService.createStream("res", 1, REGION);+ assertTrue(drainShard("res", "TRIM_HORIZON", null).isEmpty(),+ "a recreated stream must not carry records from the deleted instance");+ }++ /**+ * If a stream is deleted in the window after a producer resolved it but before it acquires the+ * append lock, the producer must fail cleanly (ResourceNotFoundException) rather than append to+ * and re-persist the stale instance. Deterministic via the pre-lock hook: the producer parks+ * WITHOUT holding the monitor, so the deleter wins the lock first. Fails on code lacking the+ * in-lock re-resolve: the producer would resurrect the stream and throw nothing.+ */+ @Test+ void putRecord_failsCleanly_whenStreamDeletedBeforeLockAcquired() throws Exception {+ kinesisService.createStream("gone", 1, REGION);++ CountDownLatch preEntered = new CountDownLatch(1);+ CountDownLatch preRelease = new CountDownLatch(1);+ AtomicBoolean firstIn = new AtomicBoolean(true);+ kinesisService.putRecordBeforeLockHook = () -> {+ if (firstIn.compareAndSet(true, false)) {+ preEntered.countDown();+ try {+ preRelease.await();+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ }+ }+ };++ AtomicReference<Throwable> thrown = new AtomicReference<>();+ ExecutorService pool = Executors.newFixedThreadPool(1);+ try {+ Future<?> producer = pool.submit(() -> {+ try {+ kinesisService.putRecord("gone", "p1".getBytes(StandardCharsets.UTF_8), "pk", REGION);+ } catch (Throwable t) {+ thrown.set(t);+ }+ return null;+ });++ assertTrue(preEntered.await(2, TimeUnit.SECONDS),+ "producer must reach the pre-lock window without holding the monitor");++ // Producer is parked without the monitor, so the deleter wins the lock and completes.+ kinesisService.deleteStream("gone", REGION);++ preRelease.countDown();+ producer.get(2, TimeUnit.SECONDS);+ } finally {+ pool.shutdownNow();+ }++ Throwable t = thrown.get();+ assertNotNull(t, "the append must fail once the stream is deleted, not silently resurrect it");+ assertInstanceOf(AwsException.class, t, () -> "unexpected failure: " + t);+ assertEquals("ResourceNotFoundException", ((AwsException) t).getErrorCode(),+ "a delete-before-append must surface the same missing-stream error as any absent stream");++ assertTrue(kinesisService.listStreams(REGION).isEmpty(),+ "the deleted stream must not be resurrected by the failed append");+ }++ private static final String META_STREAM = "meta";++ /**+ * A store that parks the first {@code get} after {@link #arm()} until released.+ *+ * <p>Every metadata write resolves its stream through exactly one {@code store.get}, and does+ * so while holding the stream monitor, so parking there wedges the writer inside its critical+ * section without needing a production test seam.+ */+ private static final class ParkOnResolveStore implements StorageBackend<String, KinesisStream> {+ private final StorageBackend<String, KinesisStream> delegate = new InMemoryStorage<>();+ private final AtomicBoolean armed = new AtomicBoolean();+ final CountDownLatch parked = new CountDownLatch(1);+ final CountDownLatch release = new CountDownLatch(1);++ void arm() {+ armed.set(true);+ }++ @Override+ public Optional<KinesisStream> get(String key) {+ if (armed.compareAndSet(true, false)) {+ parked.countDown();+ try {+ release.await();+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ }+ }+ return delegate.get(key);+ }++ @Override+ public void put(String key, KinesisStream value) {+ delegate.put(key, value);+ }++ @Override+ public void delete(String key) {+ delegate.delete(key);+ }++ @Override+ public List<KinesisStream> scan(Predicate<String> keyFilter) {+ return delegate.scan(keyFilter);+ }++ @Override+ public Set<String> keys() {+ return delegate.keys();+ }++ @Override+ public void flush() {+ delegate.flush();+ }++ @Override+ public void load() {+ delegate.load();+ }++ @Override+ public void clear() {+ delegate.clear();+ }+ }++ private static Arguments metadataWrite(String label, Consumer<KinesisService> setUp,+ Consumer<KinesisService> apply) {+ return Arguments.of(label, setUp, apply);+ }++ /**+ * Every KinesisService action that reads a stored stream, mutates it and persists it again.+ * {@code setUp} establishes any precondition the action needs to reach its persist rather than+ * short-circuit as a no-op, and runs before the store is armed.+ */+ static Stream<Arguments> metadataWrites() {+ return Stream.of(+ metadataWrite("updateStreamMode", s -> {},+ s -> s.updateStreamMode(META_STREAM, "ON_DEMAND", REGION)),+ metadataWrite("addTagsToStream", s -> {},+ s -> s.addTagsToStream(META_STREAM, Map.of("env", "test"), REGION)),+ metadataWrite("removeTagsFromStream",+ s -> s.addTagsToStream(META_STREAM, Map.of("env", "test"), REGION),+ s -> s.removeTagsFromStream(META_STREAM, List.of("env"), REGION)),+ metadataWrite("startStreamEncryption", s -> {},+ s -> s.startStreamEncryption(META_STREAM, "KMS", "alias/aws/kinesis", REGION)),+ metadataWrite("stopStreamEncryption",+ s -> s.startStreamEncryption(META_STREAM, "KMS", "alias/aws/kinesis", REGION),+ s -> s.stopStreamEncryption(META_STREAM, REGION)),+ metadataWrite("increaseStreamRetentionPeriod", s -> {},+ s -> s.increaseStreamRetentionPeriod(META_STREAM, 48, REGION)),+ metadataWrite("decreaseStreamRetentionPeriod",+ s -> s.increaseStreamRetentionPeriod(META_STREAM, 48, REGION),+ s -> s.decreaseStreamRetentionPeriod(META_STREAM, 24, REGION)),+ metadataWrite("enableEnhancedMonitoring", s -> {},+ s -> s.enableEnhancedMonitoring(META_STREAM, List.of("IncomingBytes"), REGION)),+ metadataWrite("disableEnhancedMonitoring",+ s -> s.enableEnhancedMonitoring(META_STREAM, List.of("IncomingBytes"), REGION),+ s -> s.disableEnhancedMonitoring(META_STREAM, List.of("IncomingBytes"), REGION)),+ metadataWrite("updateMaxRecordSize", s -> {},+ s -> s.updateMaxRecordSize(META_STREAM, 2048, REGION)));+ }++ /** Spins until {@code thread} is BLOCKED entering a monitor, or {@code task} completes. */+ private static boolean awaitBlockedOnMonitor(AtomicReference<Thread> thread, Future<?> task)+ throws InterruptedException {+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2);+ while (System.nanoTime() < deadline) {+ Thread t = thread.get();+ if (t != null && t.getState() == Thread.State.BLOCKED) {+ return true;+ }+ if (task.isDone()) {+ return false;+ }+ Thread.sleep(5);+ }+ return false;+ }++ /**+ * A metadata write that overlaps a deleteStream must not resurrect the stream.+ *+ * <p>The writer is wedged inside its own stream resolve, which it performs after taking the+ * per-stream monitor. A concurrent deleteStream shares that monitor, so it must block: the+ * delete therefore lands strictly after the writer's persist and the stream stays gone.+ *+ * <p>This fails on either half of the fix being absent. If a metadata path did not take the+ * monitor, or resolved the stream before taking it, the deleter would not block here, and the+ * writer's trailing {@code store.put} would re-insert the instance the delete had just removed.+ */+ @ParameterizedTest(name = "{0}")+ @MethodSource("metadataWrites")+ void metadataWrite_cannotResurrectAConcurrentlyDeletedStream(+ String label, Consumer<KinesisService> setUp, Consumer<KinesisService> apply) throws Exception {+ ParkOnResolveStore store = new ParkOnResolveStore();+ KinesisService service = new KinesisService(store, new InMemoryStorage<>(),+ new RegionResolver(REGION, "000000000000"));+ service.createStream(META_STREAM, 1, REGION);+ setUp.accept(service);+ store.arm();++ CountDownLatch deleterStarted = new CountDownLatch(1);+ AtomicReference<Thread> deleterThread = new AtomicReference<>();+ ExecutorService pool = Executors.newFixedThreadPool(2);+ try {+ Future<?> writer = pool.submit(() -> {+ apply.accept(service);+ return null;+ });+ assertTrue(store.parked.await(2, TimeUnit.SECONDS),+ () -> label + " must park inside the stream resolve it performs under the monitor");++ // The deleter publishes its thread and signals immediately before the call, so a+ // never-scheduled deleter cannot make the blocking assertions below pass vacuously.+ Future<?> deleter = pool.submit(() -> {+ deleterThread.set(Thread.currentThread());+ deleterStarted.countDown();+ service.deleteStream(META_STREAM, REGION);+ return null;+ });+ assertTrue(deleterStarted.await(2, TimeUnit.SECONDS),+ "the deleter must be scheduled and reach the deleteStream call");+ assertTrue(awaitBlockedOnMonitor(deleterThread, deleter),+ () -> "deleteStream must block on the stream monitor while " + label+ + " holds it, which is what proves the resolve happens inside the lock");+ assertThrows(TimeoutException.class, () -> deleter.get(500, TimeUnit.MILLISECONDS),+ () -> "deleteStream must not complete while " + label + " holds the stream monitor");++ store.release.countDown();+ // Neither may fail: the writer resolved a live stream, and the delete then removes it.+ writer.get(2, TimeUnit.SECONDS);+ deleter.get(2, TimeUnit.SECONDS);+ } finally {+ pool.shutdownNow();+ }++ assertTrue(service.listStreams(REGION).isEmpty(),+ () -> label + " must not resurrect a concurrently deleted stream");+ AwsException notFound = assertThrows(AwsException.class,+ () -> service.describeStream(META_STREAM, REGION),+ () -> "a stream resurrected by " + label + " would still be describable");+ assertEquals("ResourceNotFoundException", notFound.getErrorCode());++ // A stream recreated under the same name carries none of the deleted instance's metadata,+ // so nothing leaked across the lifecycle.+ KinesisStream recreated = service.createStream(META_STREAM, 1, REGION);+ assertTrue(recreated.getTags().isEmpty(), "a recreated stream must start untagged");+ assertEquals("PROVISIONED", recreated.getStreamMode(),+ "a recreated stream must start in the default stream mode");+ assertEquals(24, recreated.getRetentionPeriodHours(),+ "a recreated stream must start at the default retention period");+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisWalPersistenceTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisWalPersistenceTest.javanew file mode 100644index 0000000000..85ad56f93a--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisWalPersistenceTest.java@@ -0,0 +1,151 @@+package io.github.hectorvent.floci.services.kinesis;++import com.fasterxml.jackson.core.type.TypeReference;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.core.storage.WalStorage;+import io.github.hectorvent.floci.services.kinesis.model.KinesisRecord;+import io.github.hectorvent.floci.services.kinesis.model.KinesisStream;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.io.TempDir;++import java.nio.charset.StandardCharsets;+import java.nio.file.Files;+import java.nio.file.Path;+import java.nio.file.StandardCopyOption;+import java.util.ArrayList;+import java.util.Collections;+import java.util.HashSet;+import java.util.List;+import java.util.Map;+import java.util.Set;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.TimeUnit;++import static org.junit.jupiter.api.Assertions.*;++/**+ * Concurrent producers must not lose records across a WAL persist/reload cycle.+ *+ * <p>Serialize-order must equal append-order must equal WAL-write-order per stream key, so a+ * process restart replays every record in sequence order. See issue #571 (Kinesis append race).+ */+class KinesisWalPersistenceTest {++ private static final String REGION = "us-east-1";++ @TempDir+ Path tmp;++ @Test+ void concurrentPutRecords_survive_wal_reload() throws Exception {+ Path snapshot = tmp.resolve("kinesis-streams.json");+ Path wal = tmp.resolve("kinesis-streams.wal");+ RegionResolver rr = new RegionResolver("us-east-1", "000000000000");++ int threads = 16;+ int opsPerThread = 50;+ int expected = threads * opsPerThread; // 800+ byte[] data = "payload".getBytes(StandardCharsets.UTF_8);++ // Large compaction interval so replay (not snapshot) is what proves durability.+ WalStorage<String, KinesisStream> store1 = new WalStorage<>(+ snapshot, wal, new TypeReference<Map<String, KinesisStream>>() {}, 3_600_000L);+ WalStorage<String, KinesisStream> store2 = null;+ try {+ store1.load(); // opens the WAL writer: appends before load() are silent no-ops+ KinesisService writer = new KinesisService(store1, new InMemoryStorage<>(), rr);+ writer.createStream("wal-stream", 1, REGION);++ List<Throwable> errors = runConcurrently(threads, opsPerThread,+ () -> writer.putRecord("wal-stream", data, "pk", REGION));+ assertTrue(errors.isEmpty(), () -> "unexpected errors: " + errors);++ // Copy the still-uncompacted WAL (and snapshot, if one exists) aside for replay. Each+ // appendPut flushes to the file, so the copy holds every entry. Reloading from the copy+ // lets us shut store1 down cleanly in the finally: shutdown() compacts and truncates+ // store1's own WAL, but that no longer touches the files the reader replays.+ Path snapshot2 = tmp.resolve("reload-streams.json");+ Path wal2 = tmp.resolve("reload-streams.wal");+ Files.copy(wal, wal2, StandardCopyOption.REPLACE_EXISTING);+ if (Files.exists(snapshot)) {+ Files.copy(snapshot, snapshot2, StandardCopyOption.REPLACE_EXISTING);+ }++ // Fresh instance over the copied files: load() replays the WAL.+ store2 = new WalStorage<>(+ snapshot2, wal2, new TypeReference<Map<String, KinesisStream>>() {}, 3_600_000L);+ store2.load();+ KinesisService reader = new KinesisService(store2, new InMemoryStorage<>(), rr);++ List<KinesisRecord> all = drain(reader, "wal-stream");+ assertEquals(expected, all.size(), "every record must survive the WAL reload");++ Set<String> distinct = new HashSet<>();+ all.forEach(r -> distinct.add(r.getSequenceNumber()));+ assertEquals(expected, distinct.size(), "every reloaded record must carry a distinct sequence");++ long previous = Long.MIN_VALUE;+ for (KinesisRecord r : all) {+ long current = Long.parseLong(r.getSequenceNumber());+ assertTrue(current > previous, "reloaded records must be in strictly increasing sequence order");+ previous = current;+ }+ } finally {+ // Shut down both stores so neither leaves a compaction scheduler or open WAL writer.+ if (store2 != null) {+ store2.shutdown();+ }+ store1.shutdown();+ }+ }++ @SuppressWarnings("unchecked")+ private List<KinesisRecord> drain(KinesisService service, String streamName) {+ String shardId = service.describeStream(streamName, REGION).getShards().getFirst().getShardId();+ String iterator = service.getShardIterator(streamName, shardId, "TRIM_HORIZON", null, REGION);+ List<KinesisRecord> all = new ArrayList<>();+ while (true) {+ Map<String, Object> page = service.getRecords(iterator, 1000, REGION);+ List<KinesisRecord> records = (List<KinesisRecord>) page.get("Records");+ if (records.isEmpty()) {+ break;+ }+ all.addAll(records);+ iterator = (String) page.get("NextShardIterator");+ }+ return all;+ }++ private List<Throwable> runConcurrently(int threadCount, int opsPerThread, Runnable op)+ throws InterruptedException {+ ExecutorService pool = Executors.newFixedThreadPool(threadCount);+ CountDownLatch startGate = new CountDownLatch(1);+ CountDownLatch doneGate = new CountDownLatch(threadCount);+ List<Throwable> errors = Collections.synchronizedList(new ArrayList<>());+ try {+ for (int t = 0; t < threadCount; t++) {+ pool.submit(() -> {+ try {+ startGate.await();+ for (int i = 0; i < opsPerThread; i++) {+ op.run();+ }+ } catch (Throwable e) {+ errors.add(e);+ } finally {+ doneGate.countDown();+ }+ });+ }+ startGate.countDown();+ assertTrue(doneGate.await(60, TimeUnit.SECONDS), "producers did not complete within 60s");+ return errors;+ } finally {+ pool.shutdownNow();+ assertTrue(pool.awaitTermination(5, TimeUnit.SECONDS), "thread pool did not terminate");+ }+ }+}