Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(kinesis): stop dropping shard records under concurrent producers and resharding (#3000)

Guilherme Scaranse73deb13c01db
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarder.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarder.java
index b10b9c97de..fbd11390d6 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarder.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarder.java
@@ -14,6 +14,7 @@ import org.jboss.logging.Logger;
import java.time.Instant;
import java.util.List;
import java.util.UUID;
+import java.util.concurrent.atomic.AtomicLong;
@ApplicationScoped
public class KinesisStreamingForwarder {
@@ -22,6 +23,7 @@ public class KinesisStreamingForwarder {
private final KinesisService kinesisService;
private final ObjectMapper objectMapper;
+ private final AtomicLong forwardFailures = new AtomicLong();
@Inject
public KinesisStreamingForwarder(KinesisService kinesisService, ObjectMapper objectMapper) {
@@ -53,12 +55,22 @@ public class KinesisStreamingForwarder {
LOG.debugv("Forwarded DynamoDB event to Kinesis stream {0}: {1} on {2}",
streamName, eventName, table.getTableName());
} catch (Exception e) {
- LOG.warnv("Failed to forward DynamoDB event to Kinesis destination {0}: {1}",
- dest.getStreamArn(), e.getMessage());
+ // Do NOT rethrow: put/delete/update persist before forwarding, so a rethrow would
+ // turn a committed write into a spurious 5xx; the TTL sweep forwards mid-loop and
+ // persists only after the batch, so a rethrow would abort the sweep. Surface the
+ // drop loudly (ERROR + stack trace) and count it for process-local diagnostics.
+ forwardFailures.incrementAndGet();
+ LOG.errorv(e, "Dropped DynamoDB CDC event: failed to forward {0} on table {1} to Kinesis destination {2}",
+ eventName, table.getTableName(), dest.getStreamArn());
}
}
}
+ /** Process-local count of CDC events dropped because forwarding to Kinesis threw. */
+ public long getForwardFailureCount() {
+ return forwardFailures.get();
+ }
+
private ObjectNode buildPayload(String eventName, JsonNode keys,
JsonNode newImage, JsonNode oldImage,
String tableName, String region, Instant timestamp) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisInspectionController.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisInspectionController.java
index 2c6c1983e0..849e3eb2bc 100644
--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisInspectionController.java
+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisInspectionController.java
@@ -104,7 +104,7 @@ public class KinesisInspectionController {
node.put("StreamCreationTimestamp", stream.getStreamCreationTimestamp().toString());
}
node.put("OpenShardCount", stream.getShards().stream().filter(s -> !s.isClosed()).count());
- node.put("RecordCount", stream.getShards().stream().mapToInt(s -> s.getRecords().size()).sum());
+ node.put("RecordCount", stream.getShards().stream().mapToInt(KinesisShard::recordCount).sum());
ArrayNode enhancedMonitoring = node.putArray("EnhancedMonitoring");
Set<String> metrics = stream.getEnhancedMonitoringMetrics();
if (metrics != null) {
@@ -130,7 +130,7 @@ public class KinesisInspectionController {
ObjectNode node = objectMapper.createObjectNode();
node.put("ShardId", shard.getShardId());
node.put("Closed", shard.isClosed());
- node.put("RecordCount", shard.getRecords().size());
+ node.put("RecordCount", shard.recordCount());
if (shard.getParentShardId() != null) {
node.put("ParentShardId", shard.getParentShardId());
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java
index 23a54b175d..822f0debea 100644
--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandler.java
@@ -638,7 +638,7 @@ public class KinesisJsonHandler {
}
int maxResults = request.has("MaxResults") ? request.path("MaxResults").asInt(1000) : 1000;
- List<KinesisShard> page = shards.size() > maxResults ? shards.subList(0, maxResults) : shards;
+ List<KinesisShard> page = paginateShards(shards, maxResults);
ObjectNode response = objectMapper.createObjectNode();
ArrayNode shardsArray = response.putArray("Shards");
@@ -666,6 +666,18 @@ public class KinesisJsonHandler {
return Response.ok(response).build();
}
+ /**
+ * Take a ListShards page, snapshotting first. The shard list is a live
+ * {@link java.util.concurrent.CopyOnWriteArrayList}: its {@code subList} view is NOT an independent
+ * snapshot. It stays bound to the backing array and throws {@link java.util.ConcurrentModificationException}
+ * the moment a concurrent split/merge appends a shard. Copying to an immutable list first gives a page
+ * that reads consistently regardless of concurrent resharding.
+ */
+ static List<KinesisShard> paginateShards(List<KinesisShard> shards, int maxResults) {
+ List<KinesisShard> snapshot = List.copyOf(shards);
+ return snapshot.size() > maxResults ? snapshot.subList(0, maxResults) : snapshot;
+ }
+
private Response handleEnableEnhancedMonitoring(JsonNode request, String region) {
String streamName = resolveStreamName(request);
diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java
index d4807bceb8..5dd3e24f10 100644
--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java
@@ -17,6 +17,7 @@ import org.jboss.logging.Logger;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.util.*;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.atomic.AtomicLong;
import io.github.hectorvent.floci.core.resource.ExplorerResource;
import io.github.hectorvent.floci.core.resource.ResourceProvider;
@@ -47,6 +48,18 @@ public class KinesisService implements ResourceProvider {
private final StorageBackend<String, KinesisConsumer> consumerStore;
private final RegionResolver regionResolver;
private final AtomicLong sequenceGenerator = new AtomicLong(System.currentTimeMillis());
+ // One monitor per stream key, handed out by lockFor. Serializes sequence allocation + shard
+ // append + persist so the in-memory log, the assigned sequence order and the WAL write order
+ // all agree, and serializes every other read-modify-write on a stream against deleteStream.
+ private final ConcurrentHashMap<String, Object> streamAppendLocks = new ConcurrentHashMap<>();
+ // Package-private test seam, run inside the append critical section after the sequence is
+ // allocated and before the record is appended. Default no-op in production.
+ Runnable putRecordAppendHook = () -> {};
+ // Package-private test seam, run after the pre-lock resolve/validation and before the append
+ // lock is acquired. Lets a test deterministically wedge a producer in the window where a
+ // concurrent deleteStream can win the lock first, exercising the in-lock re-resolve guard.
+ // Default no-op in production.
+ Runnable putRecordBeforeLockHook = () -> {};
@Inject
public KinesisService(StorageFactory factory, RegionResolver regionResolver) {
@@ -118,20 +131,23 @@ public class KinesisService implements ResourceProvider {
throw new AwsException("InvalidArgumentException",
"StreamMode must be PROVISIONED or ON_DEMAND, got: " + streamMode, 400);
}
- KinesisStream stream = resolveStream(streamName, region);
- if (!"ACTIVE".equals(stream.getStreamStatus())) {
- throw new AwsException("ResourceInUseException",
- "Stream " + streamName + " is not ACTIVE (current state: " + stream.getStreamStatus() + ")", 400);
- }
- // Same-mode is a no-op. Mirrors the same-value behaviour in
- // increase/decreaseStreamRetentionPeriod (see #342). Avoids breaking
- // terraform-provider-aws which calls UpdateStreamMode on every refresh.
- if (streamMode.equals(stream.getStreamMode())) {
- return;
+ String key = regionKey(region, streamName);
+ synchronized (lockFor(key)) {
+ KinesisStream stream = resolveStream(streamName, region);
+ if (!"ACTIVE".equals(stream.getStreamStatus())) {
+ throw new AwsException("ResourceInUseException",
+ "Stream " + streamName + " is not ACTIVE (current state: " + stream.getStreamStatus() + ")", 400);
+ }
+ // Same-mode is a no-op. Mirrors the same-value behaviour in
+ // increase/decreaseStreamRetentionPeriod (see #342). Avoids breaking
+ // terraform-provider-aws which calls UpdateStreamMode on every refresh.
+ if (streamMode.equals(stream.getStreamMode())) {
+ return;
+ }
+ stream.setStreamMode(streamMode);
+ store.put(key, stream);
+ LOG.infov("Updated stream mode for {0} to {1}", streamName, streamMode);
}
- stream.setStreamMode(streamMode);
- store.put(regionKey(region, streamName), stream);
- LOG.infov("Updated stream mode for {0} to {1}", streamName, streamMode);
}
public List<String> listStreams(String region) {
@@ -193,20 +209,31 @@ public class KinesisService implements ResourceProvider {
public void deleteStream(String streamName, String region) {
String storageKey = regionKey(region, streamName);
- store.delete(storageKey);
+ // Delete under the per-stream monitor every writer takes, so the delete is serialized with
+ // any in-flight append, split, merge or metadata write on this stream. See lockFor for why
+ // the monitor is deliberately left in the map.
+ synchronized (lockFor(storageKey)) {
+ store.delete(storageKey);
+ }
LOG.infov("Deleted Kinesis stream: {0}", streamName);
}
public void addTagsToStream(String streamName, Map<String, String> tags, String region) {
- KinesisStream stream = resolveStream(streamName, region);
- stream.getTags().putAll(tags);
- store.put(regionKey(region, streamName), stream);
+ String key = regionKey(region, streamName);
+ synchronized (lockFor(key)) {
+ KinesisStream stream = resolveStream(streamName, region);
+ stream.getTags().putAll(tags);
+ store.put(key, stream);
+ }
}
public void removeTagsFromStream(String streamName, List<String> tagKeys, String region) {
- KinesisStream stream = resolveStream(streamName, region);
- tagKeys.forEach(stream.getTags()::remove);
- store.put(regionKey(region, streamName), stream);
+ String key = regionKey(region, streamName);
+ synchronized (lockFor(key)) {
+ KinesisStream stream = resolveStream(streamName, region);
+ tagKeys.forEach(stream.getTags()::remove);
+ store.put(key, stream);
+ }
}
public Map<String, String> listTagsForStream(String streamName, String region) {
@@ -214,75 +241,90 @@ public class KinesisService implements ResourceProvider {
}
public void startStreamEncryption(String streamName, String encryptionType, String keyId, String region) {
- KinesisStream stream = resolveStream(streamName, region);
- stream.setEncryptionType(encryptionType);
- stream.setKeyId(keyId);
- store.put(regionKey(region, streamName), stream);
+ String key = regionKey(region, streamName);
+ synchronized (lockFor(key)) {
+ KinesisStream stream = resolveStream(streamName, region);
+ stream.setEncryptionType(encryptionType);
+ stream.setKeyId(keyId);
+ store.put(key, stream);
+ }
}
public void increaseStreamRetentionPeriod(String streamName, int retentionPeriodHours, String region) {
- KinesisStream stream = resolveStream(streamName, region);
- if (retentionPeriodHours > 8760) {
- throw new AwsException("InvalidArgumentException",
- "Retention period must not exceed 8760 hours (365 days)", 400);
- }
- if (retentionPeriodHours < stream.getRetentionPeriodHours()) {
- throw new AwsException("InvalidArgumentException",
- "Requested retention period (" + retentionPeriodHours +
- " hours) must not be less than current retention period (" +
- stream.getRetentionPeriodHours() + " hours)", 400);
- }
- // Same value is a no-op on real AWS despite the API doc wording ("must be more than
- // current"). Proof: terraform-provider-aws calls IncreaseStreamRetentionPeriod on
- // stream creation unconditionally when retention_period is set (stream.go Create path),
- // so every default-retention TF stream would fail if AWS rejected same-value. See #342.
- if (retentionPeriodHours == stream.getRetentionPeriodHours()) {
- return;
+ String key = regionKey(region, streamName);
+ synchronized (lockFor(key)) {
+ KinesisStream stream = resolveStream(streamName, region);
+ if (retentionPeriodHours > 8760) {
+ throw new AwsException("InvalidArgumentException",
+ "Retention period must not exceed 8760 hours (365 days)", 400);
+ }
+ if (retentionPeriodHours < stream.getRetentionPeriodHours()) {
+ throw new AwsException("InvalidArgumentException",
+ "Requested retention period (" + retentionPeriodHours +
+ " hours) must not be less than current retention period (" +
+ stream.getRetentionPeriodHours() + " hours)", 400);
+ }
+ // Same value is a no-op on real AWS despite the API doc wording ("must be more than
+ // current"). Proof: terraform-provider-aws calls IncreaseStreamRetentionPeriod on
+ // stream creation unconditionally when retention_period is set (stream.go Create path),
+ // so every default-retention TF stream would fail if AWS rejected same-value. See #342.
+ if (retentionPeriodHours == stream.getRetentionPeriodHours()) {
+ return;
+ }
+ stream.setRetentionPeriodHours(retentionPeriodHours);
+ store.put(key, stream);
+ LOG.infov("Increased retention period for stream {0} to {1} hours", streamName, retentionPeriodHours);
}
- stream.setRetentionPeriodHours(retentionPeriodHours);
- store.put(regionKey(region, streamName), stream);
- LOG.infov("Increased retention period for stream {0} to {1} hours", streamName, retentionPeriodHours);
}
public void decreaseStreamRetentionPeriod(String streamName, int retentionPeriodHours, String region) {
- KinesisStream stream = resolveStream(streamName, region);
- if (retentionPeriodHours < 24) {
- throw new AwsException("InvalidArgumentException",
- "Retention period must not be less than 24 hours", 400);
- }
- if (retentionPeriodHours > stream.getRetentionPeriodHours()) {
- throw new AwsException("InvalidArgumentException",
- "Requested retention period (" + retentionPeriodHours +
- " hours) must not be greater than current retention period (" +
- stream.getRetentionPeriodHours() + " hours)", 400);
- }
- // Same value is a no-op on real AWS (mirrors IncreaseStreamRetentionPeriod). See #342.
- if (retentionPeriodHours == stream.getRetentionPeriodHours()) {
- return;
+ String key = regionKey(region, streamName);
+ synchronized (lockFor(key)) {
+ KinesisStream stream = resolveStream(streamName, region);
+ if (retentionPeriodHours < 24) {
+ throw new AwsException("InvalidArgumentException",
+ "Retention period must not be less than 24 hours", 400);
+ }
+ if (retentionPeriodHours > stream.getRetentionPeriodHours()) {
+ throw new AwsException("InvalidArgumentException",
+ "Requested retention period (" + retentionPeriodHours +
+ " hours) must not be greater than current retention period (" +
+ stream.getRetentionPeriodHours() + " hours)", 400);
+ }
+ // Same value is a no-op on real AWS (mirrors IncreaseStreamRetentionPeriod). See #342.
+ if (retentionPeriodHours == stream.getRetentionPeriodHours()) {
+ return;
+ }
+ stream.setRetentionPeriodHours(retentionPeriodHours);
+ store.put(key, stream);
+ LOG.infov("Decreased retention period for stream {0} to {1} hours", streamName, retentionPeriodHours);
}
- stream.setRetentionPeriodHours(retentionPeriodHours);
- store.put(regionKey(region, streamName), stream);
- LOG.infov("Decreased retention period for stream {0} to {1} hours", streamName, retentionPeriodHours);
}
public Set<String> enableEnhancedMonitoring(String streamName, List<String> metrics, String region) {
- KinesisStream stream = resolveStream(streamName, region);
- Set<String> current = new HashSet<>(stream.getEnhancedMonitoringMetrics());
- Set<String> desired = resolveMetrics(metrics);
- stream.getEnhancedMonitoringMetrics().addAll(desired);
- store.put(regionKey(region, streamName), stream);
- LOG.infov("Enabled enhanced monitoring for stream {0}: {1}", streamName, desired);
- return current;
+ String key = regionKey(region, streamName);
+ synchronized (lockFor(key)) {
+ KinesisStream stream = resolveStream(streamName, region);
+ Set<String> current = new HashSet<>(stream.getEnhancedMonitoringMetrics());
+ Set<String> desired = resolveMetrics(metrics);
+ stream.getEnhancedMonitoringMetrics().addAll(desired);
+ store.put(key, stream);
+ LOG.infov("Enabled enhanced monitoring for stream {0}: {1}", streamName, desired);
+ return current;
+ }
}
public Set<String> disableEnhancedMonitoring(String streamName, List<String> metrics, String region) {
- KinesisStream stream = resolveStream(streamName, region);
- Set<String> current = new HashSet<>(stream.getEnhancedMonitoringMetrics());
- Set<String> toRemove = resolveMetrics(metrics);
- stream.getEnhancedMonitoringMetrics().removeAll(toRemove);
- store.put(regionKey(region, streamName), stream);
- LOG.infov("Disabled enhanced monitoring for stream {0}: {1}", streamName, toRemove);
- return current;
+ String key = regionKey(region, streamName);
+ synchronized (lockFor(key)) {
+ KinesisStream stream = resolveStream(streamName, region);
+ Set<String> current = new HashSet<>(stream.getEnhancedMonitoringMetrics());
+ Set<String> toRemove = resolveMetrics(metrics);
+ stream.getEnhancedMonitoringMetrics().removeAll(toRemove);
+ store.put(key, stream);
+ LOG.infov("Disabled enhanced monitoring for stream {0}: {1}", streamName, toRemove);
+ return current;
+ }
}
private Set<String> resolveMetrics(List<String> metrics) {
@@ -306,80 +348,101 @@ public class KinesisService implements ResourceProvider {
}
public void stopStreamEncryption(String streamName, String region) {
- KinesisStream stream = resolveStream(streamName, region);
- stream.setEncryptionType("NONE");
- stream.setKeyId(null);
- store.put(regionKey(region, streamName), stream);
+ String key = regionKey(region, streamName);
+ synchronized (lockFor(key)) {
+ KinesisStream stream = resolveStream(streamName, region);
+ stream.setEncryptionType("NONE");
+ stream.setKeyId(null);
+ store.put(key, stream);
+ }
}
public void splitShard(String streamName, String shardId, String newStartingHashKey, String region) {
- KinesisStream stream = resolveStream(streamName, region);
- KinesisShard parent = stream.getShards().stream()
- .filter(s -> s.getShardId().equals(shardId))
- .findFirst()
- .orElseThrow(() -> new AwsException("ResourceNotFoundException", "Shard " + shardId + " not found", 400));
+ String key = regionKey(region, streamName);
+ // Share the producer critical section so a split can't race an append: selectShard runs
+ // under the same lock, so a producer never appends to a parent a completed split has closed.
+ synchronized (lockFor(key)) {
+ // Resolve under the lock so a concurrent deleteStream (same monitor) cannot leave us
+ // mutating and re-persisting a stale, deleted instance.
+ KinesisStream stream = resolveStream(streamName, region);
+ KinesisShard parent = stream.getShards().stream()
+ .filter(s -> s.getShardId().equals(shardId))
+ .findFirst()
+ .orElseThrow(() -> new AwsException("ResourceNotFoundException", "Shard " + shardId + " not found", 400));
- if (parent.isClosed()) {
- throw new AwsException("InvalidArgumentException", "Shard " + shardId + " is already closed", 400);
- }
+ if (parent.isClosed()) {
+ throw new AwsException("InvalidArgumentException", "Shard " + shardId + " is already closed", 400);
+ }
- parent.setClosed(true);
- parent.setSequenceNumberRange(new KinesisShard.SequenceNumberRange(
- parent.getSequenceNumberRange().startingSequenceNumber(),
- String.valueOf(sequenceGenerator.get())));
+ parent.setClosed(true);
+ parent.setSequenceNumberRange(new KinesisShard.SequenceNumberRange(
+ parent.getSequenceNumberRange().startingSequenceNumber(),
+ String.valueOf(sequenceGenerator.get())));
- String start = parent.getHashKeyRange().startingHashKey();
- String end = parent.getHashKeyRange().endingHashKey();
+ String start = parent.getHashKeyRange().startingHashKey();
+ String end = parent.getHashKeyRange().endingHashKey();
- KinesisShard child1 = new KinesisShard(nextShardId(stream), start, subtractOne(newStartingHashKey), String.valueOf(sequenceGenerator.get()));
- child1.setParentShardId(shardId);
+ // Derive both child ids from the base size WITHOUT relying on the first add: nextShardId
+ // is size-based, so two sequential adds would either collide (both ids computed pre-add)
+ // or expose a half-published topology (child1 visible before child2). Compute base+0 and
+ // base+1 up front so the ids stay distinct, then publish BOTH children in one atomic
+ // CopyOnWriteArrayList.addAll, so a lock-free reader can never observe {parent, child1}
+ // without child2.
+ int base = stream.getShards().size();
+ KinesisShard child1 = new KinesisShard(String.format("shardId-%012d", base), start, subtractOne(newStartingHashKey), String.valueOf(sequenceGenerator.get()));
+ child1.setParentShardId(shardId);
- KinesisShard child2 = new KinesisShard(nextShardId(stream), newStartingHashKey, end, String.valueOf(sequenceGenerator.get()));
- child2.setParentShardId(shardId);
+ KinesisShard child2 = new KinesisShard(String.format("shardId-%012d", base + 1), newStartingHashKey, end, String.valueOf(sequenceGenerator.get()));
+ child2.setParentShardId(shardId);
- stream.getShards().add(child1);
- stream.getShards().add(child2);
- store.put(regionKey(region, streamName), stream);
- LOG.infov("Split shard {0} in stream {1}", shardId, streamName);
+ stream.getShards().addAll(List.of(child1, child2));
+ store.put(key, stream);
+ LOG.infov("Split shard {0} in stream {1}", shardId, streamName);
+ }
}
public void mergeShards(String streamName, String shardId, String adjacentShardId, String region) {
- KinesisStream stream = resolveStream(streamName, region);
- KinesisShard shard1 = stream.getShards().stream()
- .filter(s -> s.getShardId().equals(shardId))
- .findFirst()
- .orElseThrow(() -> new AwsException("ResourceNotFoundException", "Shard " + shardId + " not found", 400));
- KinesisShard shard2 = stream.getShards().stream()
- .filter(s -> s.getShardId().equals(adjacentShardId))
- .findFirst()
- .orElseThrow(() -> new AwsException("ResourceNotFoundException", "Shard " + adjacentShardId + " not found", 400));
+ String key = regionKey(region, streamName);
+ synchronized (lockFor(key)) {
+ // Resolve under the lock so a concurrent deleteStream (same monitor) cannot leave us
+ // mutating and re-persisting a stale, deleted instance.
+ KinesisStream stream = resolveStream(streamName, region);
+ KinesisShard shard1 = stream.getShards().stream()
+ .filter(s -> s.getShardId().equals(shardId))
+ .findFirst()
+ .orElseThrow(() -> new AwsException("ResourceNotFoundException", "Shard " + shardId + " not found", 400));
+ KinesisShard shard2 = stream.getShards().stream()
+ .filter(s -> s.getShardId().equals(adjacentShardId))
+ .findFirst()
+ .orElseThrow(() -> new AwsException("ResourceNotFoundException", "Shard " + adjacentShardId + " not found", 400));
- if (shard1.isClosed() || shard2.isClosed()) {
- throw new AwsException("InvalidArgumentException", "One or both shards are already closed", 400);
- }
+ if (shard1.isClosed() || shard2.isClosed()) {
+ throw new AwsException("InvalidArgumentException", "One or both shards are already closed", 400);
+ }
- shard1.setClosed(true);
- shard2.setClosed(true);
- String seq = String.valueOf(sequenceGenerator.get());
- shard1.setSequenceNumberRange(new KinesisShard.SequenceNumberRange(shard1.getSequenceNumberRange().startingSequenceNumber(), seq));
- shard2.setSequenceNumberRange(new KinesisShard.SequenceNumberRange(shard2.getSequenceNumberRange().startingSequenceNumber(), seq));
+ shard1.setClosed(true);
+ shard2.setClosed(true);
+ String seq = String.valueOf(sequenceGenerator.get());
+ shard1.setSequenceNumberRange(new KinesisShard.SequenceNumberRange(shard1.getSequenceNumberRange().startingSequenceNumber(), seq));
+ shard2.setSequenceNumberRange(new KinesisShard.SequenceNumberRange(shard2.getSequenceNumberRange().startingSequenceNumber(), seq));
- // Combine hash ranges (assuming they are adjacent)
- java.math.BigInteger s1Start = new java.math.BigInteger(shard1.getHashKeyRange().startingHashKey());
- java.math.BigInteger s2Start = new java.math.BigInteger(shard2.getHashKeyRange().startingHashKey());
-
- String start = s1Start.min(s2Start).toString();
- java.math.BigInteger s1End = new java.math.BigInteger(shard1.getHashKeyRange().endingHashKey());
- java.math.BigInteger s2End = new java.math.BigInteger(shard2.getHashKeyRange().endingHashKey());
- String end = s1End.max(s2End).toString();
+ // Combine hash ranges (assuming they are adjacent)
+ java.math.BigInteger s1Start = new java.math.BigInteger(shard1.getHashKeyRange().startingHashKey());
+ java.math.BigInteger s2Start = new java.math.BigInteger(shard2.getHashKeyRange().startingHashKey());
- KinesisShard child = new KinesisShard(nextShardId(stream), start, end, seq);
- child.setParentShardId(shardId);
- child.setAdjacentParentShardId(adjacentShardId);
+ String start = s1Start.min(s2Start).toString();
+ java.math.BigInteger s1End = new java.math.BigInteger(shard1.getHashKeyRange().endingHashKey());
+ java.math.BigInteger s2End = new java.math.BigInteger(shard2.getHashKeyRange().endingHashKey());
+ String end = s1End.max(s2End).toString();
- stream.getShards().add(child);
- store.put(regionKey(region, streamName), stream);
- LOG.infov("Merged shards {0} and {1} in stream {2}", shardId, adjacentShardId, streamName);
+ KinesisShard child = new KinesisShard(nextShardId(stream), start, end, seq);
+ child.setParentShardId(shardId);
+ child.setAdjacentParentShardId(adjacentShardId);
+
+ stream.getShards().add(child);
+ store.put(key, stream);
+ LOG.infov("Merged shards {0} and {1} in stream {2}", shardId, adjacentShardId, streamName);
+ }
}
private String nextShardId(KinesisStream stream) {
@@ -452,35 +515,49 @@ public class KinesisService implements ResourceProvider {
}
public void updateMaxRecordSize(String streamName, int maxRecordSizeInKiB, String region) {
- KinesisStream stream = resolveStream(streamName, region);
- validateMaxRecordSize(maxRecordSizeInKiB, "ValidationException");
- if ("ON_DEMAND".equals(stream.getStreamMode())) {
- throw new AwsException("ValidationException",
- "UpdateMaxRecordSize is only supported for data streams with the provisioned capacity mode.",
- 400);
- }
- if (!"ACTIVE".equals(stream.getStreamStatus())) {
- throw new AwsException("ResourceInUseException",
- "Stream " + streamName + " is not ACTIVE (current state: "
- + stream.getStreamStatus() + ")", 400);
+ String key = regionKey(region, streamName);
+ synchronized (lockFor(key)) {
+ KinesisStream stream = resolveStream(streamName, region);
+ validateMaxRecordSize(maxRecordSizeInKiB, "ValidationException");
+ if ("ON_DEMAND".equals(stream.getStreamMode())) {
+ throw new AwsException("ValidationException",
+ "UpdateMaxRecordSize is only supported for data streams with the provisioned capacity mode.",
+ 400);
+ }
+ if (!"ACTIVE".equals(stream.getStreamStatus())) {
+ throw new AwsException("ResourceInUseException",
+ "Stream " + streamName + " is not ACTIVE (current state: "
+ + stream.getStreamStatus() + ")", 400);
+ }
+ stream.setMaxRecordSizeInKiB(maxRecordSizeInKiB);
+ store.put(key, stream);
+ LOG.infov("Updated max record size for {0} to {1} KiB", streamName, maxRecordSizeInKiB);
}
- stream.setMaxRecordSizeInKiB(maxRecordSizeInKiB);
- store.put(regionKey(region, streamName), stream);
- LOG.infov("Updated max record size for {0} to {1} KiB", streamName, maxRecordSizeInKiB);
}
public PutRecordResult putRecordWithShardId(String streamName, byte[] data, String partitionKey, String region) {
+ String key = regionKey(region, streamName);
KinesisStream stream = resolveStream(streamName, region);
validateRecordSize(stream, data, partitionKey);
- KinesisShard shard = selectShard(stream, partitionKey);
-
- String sequenceNumber = String.valueOf(sequenceGenerator.incrementAndGet());
- KinesisRecord record = new KinesisRecord(data, partitionKey, sequenceNumber, Instant.now());
-
- shard.getRecords().add(record);
- store.put(regionKey(region, streamName), stream);
-
- return new PutRecordResult(sequenceNumber, shard.getShardId());
+ putRecordBeforeLockHook.run();
+
+ // Serialize shard selection, sequence allocation, append and persistence per stream so
+ // concurrent producers can neither lose an append (plain ArrayList) nor store records
+ // out of sequence order (which would break AT_/AFTER_SEQUENCE_NUMBER scans and WAL replay).
+ synchronized (lockFor(key)) {
+ // Re-resolve under the lock. deleteStream holds this same monitor, so a stream deleted
+ // between the resolve above and here is now absent from the store. Appending to and
+ // persisting the stale pre-delete instance would resurrect a deleted stream, so bind to
+ // the instance currently in the store and fail like any missing stream if it is gone.
+ KinesisStream current = resolveStream(streamName, region);
+ KinesisShard shard = selectShard(current, partitionKey);
+ String sequenceNumber = String.valueOf(sequenceGenerator.incrementAndGet());
+ putRecordAppendHook.run();
+ KinesisRecord record = new KinesisRecord(data, partitionKey, sequenceNumber, Instant.now());
+ shard.addRecord(record);
+ store.put(key, current);
+ return new PutRecordResult(sequenceNumber, shard.getShardId());
+ }
}
public String getShardIterator(String streamName, String shardId, String type, String sequenceNumber, String region) {
@@ -527,7 +604,7 @@ public class KinesisService implements ResourceProvider {
.findFirst()
.orElseThrow(() -> new AwsException("ResourceNotFoundException", "Shard not found", 400));
// LATEST resumes from the tip snapshot taken now; other types resolve in getRecords.
- return "LATEST".equals(type) ? shard.getRecords().size() : 0;
+ return "LATEST".equals(type) ? shard.recordCount() : 0;
}
private int parseIteratorIndex(String value) {
@@ -786,6 +863,23 @@ public class KinesisService implements ResourceProvider {
return region + "::" + name;
}
+ /**
+ * The monitor guarding a stream key. Every path that mutates a stored stream takes it: record
+ * appends, split and merge, the metadata writes, and {@link #deleteStream}.
+ *
+ * <p>Holding it is only half the contract: a writer must also resolve the stream inside the
+ * monitor. A stream resolved before the monitor was acquired may already have been deleted, and
+ * persisting that stale instance would resurrect it or clobber a stream since recreated under
+ * the same name.
+ *
+ * <p>Monitors are never removed. One stable monitor per key is a bounded leak, whereas dropping
+ * one on delete would let a writer that already resolved the stream create a fresh monitor and
+ * persist the deleted instance under it, defeating the serialization.
+ */
+ private Object lockFor(String storageKey) {
+ return streamAppendLocks.computeIfAbsent(storageKey, k -> new Object());
+ }
+
// ─── Resource Explorer 2 ───────────────────────────────────────────────────
@Override
diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/model/KinesisShard.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/model/KinesisShard.java
index 1d088ceb51..58a8aa4963 100644
--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/model/KinesisShard.java
+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/model/KinesisShard.java
@@ -15,6 +15,7 @@ public class KinesisShard {
private String adjacentParentShardId;
private HashKeyRange hashKeyRange;
private SequenceNumberRange sequenceNumberRange;
+ private final Object recordsMonitor = new Object();
private List<KinesisRecord> records = new ArrayList<>();
private boolean closed = false;
private Instant creationTimestamp = Instant.now();
@@ -42,8 +43,38 @@ public class KinesisShard {
public SequenceNumberRange getSequenceNumberRange() { return sequenceNumberRange; }
public void setSequenceNumberRange(SequenceNumberRange range) { this.sequenceNumberRange = range; }
- public List<KinesisRecord> getRecords() { return records; }
- public void setRecords(List<KinesisRecord> records) { this.records = records; }
+ /**
+ * A shallow snapshot of this shard's records. Safe to index and iterate without a
+ * ConcurrentModificationException even while producers append concurrently. Structural
+ * mutation of the returned list does NOT affect the shard (append via {@link #addRecord});
+ * element references are shared, so per-element setters still write through.
+ */
+ public List<KinesisRecord> getRecords() {
+ synchronized (recordsMonitor) {
+ return new ArrayList<>(records);
+ }
+ }
+
+ /** Appends a record. The sole production mutation path for a shard's log. */
+ public void addRecord(KinesisRecord record) {
+ synchronized (recordsMonitor) {
+ records.add(record);
+ }
+ }
+
+ /** Number of records currently held, without copying the log. */
+ public int recordCount() {
+ synchronized (recordsMonitor) {
+ return records.size();
+ }
+ }
+
+ /** Jackson rehydration only; not safe for concurrent replacement during live traffic. */
+ public void setRecords(List<KinesisRecord> records) {
+ synchronized (recordsMonitor) {
+ this.records = records == null ? new ArrayList<>() : new ArrayList<>(records);
+ }
+ }
public boolean isClosed() { return closed; }
public void setClosed(boolean closed) { this.closed = closed; }
diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/model/KinesisStream.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/model/KinesisStream.java
index ca0130da39..d8af576ae1 100644
--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/model/KinesisStream.java
+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/model/KinesisStream.java
@@ -4,12 +4,12 @@ import com.fasterxml.jackson.annotation.JsonIgnoreProperties;
import io.quarkus.runtime.annotations.RegisterForReflection;
import java.time.Instant;
-import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.concurrent.CopyOnWriteArrayList;
@RegisterForReflection
@JsonIgnoreProperties(ignoreUnknown = true)
@@ -18,7 +18,7 @@ public class KinesisStream {
private String streamArn;
private String accountId;
private String streamStatus;
- private List<KinesisShard> shards = new ArrayList<>();
+ private volatile List<KinesisShard> shards = new CopyOnWriteArrayList<>();
private int retentionPeriodHours = 24;
private Instant streamCreationTimestamp;
private Map<String, String> tags = new HashMap<>();
@@ -51,7 +51,9 @@ public class KinesisStream {
public void setStreamStatus(String streamStatus) { this.streamStatus = streamStatus; }
public List<KinesisShard> getShards() { return shards; }
- public void setShards(List<KinesisShard> shards) { this.shards = shards; }
+ public void setShards(List<KinesisShard> shards) {
+ this.shards = shards == null ? new CopyOnWriteArrayList<>() : new CopyOnWriteArrayList<>(shards);
+ }
public int getRetentionPeriodHours() { return retentionPeriodHours; }
public void setRetentionPeriodHours(int retentionPeriodHours) { this.retentionPeriodHours = retentionPeriodHours; }
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbConcurrencyIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbConcurrencyIntegrationTest.java
index 3facae51a2..37d88e121c 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbConcurrencyIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbConcurrencyIntegrationTest.java
@@ -1,13 +1,19 @@
package io.github.hectorvent.floci.services.dynamodb;
import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
import io.github.hectorvent.floci.core.storage.InMemoryStorage;
import io.github.hectorvent.floci.core.storage.StorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;
import io.github.hectorvent.floci.services.dynamodb.model.ConditionalCheckFailedException;
import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;
+import io.github.hectorvent.floci.services.dynamodb.model.KinesisStreamingDestination;
import io.github.hectorvent.floci.services.dynamodb.model.StreamDescription;
import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;
+import io.github.hectorvent.floci.services.kinesis.KinesisService;
+import io.github.hectorvent.floci.services.kinesis.model.KinesisRecord;
+import io.github.hectorvent.floci.services.kinesis.model.KinesisStream;
import com.fasterxml.jackson.databind.JsonNode;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
@@ -17,6 +23,7 @@ import org.junit.jupiter.api.Test;
import java.util.ArrayList;
import java.util.Collections;
+import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -29,6 +36,10 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
/**
* Concurrency compatibility suite for {@link DynamoDbService}.
@@ -481,6 +492,89 @@ class DynamoDbConcurrencyIntegrationTest {
assertEquals(ops, previous, "final event must reflect final counter value");
}
+ /**
+ * End-to-end CDC wiring: concurrent putItem must forward EVERY record to the attached Kinesis
+ * destination, with no loss and per-shard sequence order preserved. This is the gap the suite's
+ * production wiring left uncovered (the shared {@code service} field wires {@code forwarder=null}).
+ * See issue #571 (Kinesis append race).
+ */
+ @RepeatedTest(REPEATS)
+ void concurrent_putItem_forwards_every_cdc_record_to_kinesis() throws Exception {
+ String region = "us-east-1";
+ StorageBackend<String, TableDefinition> tableStore = new InMemoryStorage<>();
+ // Public KinesisService ctor via a mocked StorageFactory that hands out in-memory
+ // backends keyed by file name (the FirehoseServiceTest pattern), because the
+ // package-private ctor is not visible from this package.
+ Map<String, AccountAwareStorageBackend<?>> backends = new HashMap<>();
+ StorageFactory storageFactory = mock(StorageFactory.class);
+ when(storageFactory.create(anyString(), anyString(), any()))
+ .thenAnswer(inv -> backends.computeIfAbsent(
+ inv.getArgument(0) + "/" + inv.getArgument(1),
+ k -> AccountAwareStorageBackend.inMemory("000000000000")));
+ KinesisService kinesis = new KinesisService(storageFactory,
+ new RegionResolver("us-east-1", "000000000000"));
+ KinesisStreamingForwarder forwarder = new KinesisStreamingForwarder(kinesis, mapper);
+ DynamoDbService svc = new DynamoDbService(
+ tableStore, new InMemoryStorage<>(),
+ new RegionResolver("us-east-1", "000000000000"), null, forwarder);
+
+ KinesisStream stream = kinesis.createStream("cdc-stream", 1, region);
+ TableDefinition table = svc.createTable("Events",
+ List.of(new KeySchemaElement("pk", "HASH"), new KeySchemaElement("sk", "RANGE")),
+ List.of(new AttributeDefinition("pk", "S"), new AttributeDefinition("sk", "S")),
+ 5L, 5L, region);
+ table.getKinesisStreamingDestinations().add(
+ new KinesisStreamingDestination(stream.getStreamArn()));
+
+ int threads = 16;
+ int opsPerThread = 25;
+ int expected = threads * opsPerThread; // 400
+ AtomicInteger idSource = new AtomicInteger();
+
+ List<Throwable> errors = runConcurrently(threads, () -> {
+ for (int i = 0; i < opsPerThread; i++) {
+ int id = idSource.getAndIncrement();
+ ObjectNode item = mapper.createObjectNode();
+ item.set("pk", stringAttr("fixed"));
+ item.set("sk", stringAttr("t-" + id));
+ svc.putItem("Events", item, region);
+ }
+ });
+ assertTrue(errors.isEmpty(), () -> "unexpected errors: " + errors);
+
+ // Paginated drain of the single shard.
+ String shardId = kinesis.describeStream("cdc-stream", region).getShards().getFirst().getShardId();
+ String iterator = kinesis.getShardIterator("cdc-stream", shardId, "TRIM_HORIZON", null, region);
+ List<KinesisRecord> drained = new ArrayList<>();
+ while (true) {
+ Map<String, Object> page = kinesis.getRecords(iterator, 1000, region);
+ @SuppressWarnings("unchecked")
+ List<KinesisRecord> records = (List<KinesisRecord>) page.get("Records");
+ if (records.isEmpty()) break;
+ drained.addAll(records);
+ iterator = (String) page.get("NextShardIterator");
+ }
+
+ assertEquals(expected, drained.size(), "every CDC event must reach Kinesis");
+
+ Set<String> sortKeys = new HashSet<>();
+ long previousSeq = Long.MIN_VALUE;
+ for (KinesisRecord record : drained) {
+ JsonNode payload = mapper.readTree(record.getData());
+ sortKeys.add(payload.get("dynamodb").get("Keys").get("sk").get("S").asText());
+ long seq = Long.parseLong(record.getSequenceNumber());
+ assertTrue(seq > previousSeq, "per-shard sequence numbers must be strictly increasing");
+ previousSeq = seq;
+ }
+
+ Set<String> expectedKeys = new HashSet<>();
+ for (int i = 0; i < expected; i++) {
+ expectedKeys.add("t-" + i);
+ }
+ assertEquals(expectedKeys, sortKeys, "the forwarded sk set must match exactly, with no gaps");
+ assertEquals(0L, forwarder.getForwardFailureCount(), "no CDC event may be dropped");
+ }
+
@Test
void baseline_single_threaded_still_works() {
String region = "eu-west-1";
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlForwardingTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlForwardingTest.java
new file mode 100644
index 0000000000..ea52c227b9
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbTtlForwardingTest.java
@@ -0,0 +1,178 @@
+package io.github.hectorvent.floci.services.dynamodb;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.core.storage.StorageBackend;
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;
+import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;
+import io.github.hectorvent.floci.services.dynamodb.model.KinesisStreamingDestination;
+import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;
+import io.github.hectorvent.floci.services.kinesis.KinesisService;
+import io.github.hectorvent.floci.services.kinesis.model.KinesisRecord;
+import io.github.hectorvent.floci.services.kinesis.model.KinesisStream;
+import org.junit.jupiter.api.Test;
+
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * TTL sweep must forward a REMOVE per expired item to Kinesis (happy path) and must isolate
+ * forwarding failures: a throwing forwarder cannot abort the sweep or block persistence.
+ * See issue #571.
+ */
+class DynamoDbTtlForwardingTest {
+
+ private static final String REGION = "us-east-1";
+
+ private final ObjectMapper mapper = new ObjectMapper();
+
+ private KinesisService realKinesis() {
+ Map<String, AccountAwareStorageBackend<?>> backends = new HashMap<>();
+ StorageFactory factory = mock(StorageFactory.class);
+ when(factory.create(anyString(), anyString(), any()))
+ .thenAnswer(inv -> backends.computeIfAbsent(
+ inv.getArgument(0) + "/" + inv.getArgument(1),
+ k -> AccountAwareStorageBackend.inMemory("000000000000")));
+ return new KinesisService(factory, new RegionResolver("us-east-1", "000000000000"));
+ }
+
+ private ObjectNode stringAttr(String v) {
+ ObjectNode n = mapper.createObjectNode();
+ n.put("S", v);
+ return n;
+ }
+
+ private ObjectNode numberAttr(long v) {
+ ObjectNode n = mapper.createObjectNode();
+ n.put("N", String.valueOf(v));
+ return n;
+ }
+
+ private void seedTtlTable(DynamoDbService svc, int expiredCount) {
+ svc.createTable("TtlTable",
+ List.of(new KeySchemaElement("pk", "HASH")),
+ List.of(new AttributeDefinition("pk", "S")),
+ 5L, 5L, REGION);
+ svc.updateTimeToLive("TtlTable", "expireAt", true, REGION);
+ long past = Instant.now().getEpochSecond() - 3600;
+ for (int i = 0; i < expiredCount; i++) {
+ ObjectNode item = mapper.createObjectNode();
+ item.set("pk", stringAttr("row-" + i));
+ item.set("expireAt", numberAttr(past));
+ svc.putItem("TtlTable", item, REGION);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private List<KinesisRecord> drain(KinesisService kinesis, String streamName) {
+ String shardId = kinesis.describeStream(streamName, REGION).getShards().getFirst().getShardId();
+ String iterator = kinesis.getShardIterator(streamName, shardId, "TRIM_HORIZON", null, REGION);
+ List<KinesisRecord> all = new ArrayList<>();
+ while (true) {
+ Map<String, Object> page = kinesis.getRecords(iterator, 1000, REGION);
+ List<KinesisRecord> records = (List<KinesisRecord>) page.get("Records");
+ if (records.isEmpty()) break;
+ all.addAll(records);
+ iterator = (String) page.get("NextShardIterator");
+ }
+ return all;
+ }
+
+ @Test
+ void ttlSweep_forwards_one_remove_per_expired_item() throws Exception {
+ StorageBackend<String, TableDefinition> tableStore = new InMemoryStorage<>();
+ StorageBackend<String, Map<String, JsonNode>> itemStore = new InMemoryStorage<>();
+ KinesisService kinesis = realKinesis();
+ KinesisStreamingForwarder forwarder = new KinesisStreamingForwarder(kinesis, mapper);
+ DynamoDbService svc = new DynamoDbService(
+ tableStore, itemStore, new RegionResolver("us-east-1", "000000000000"), null, forwarder);
+
+ KinesisStream stream = kinesis.createStream("ttl-stream", 1, REGION);
+ int expired = 5;
+ seedTtlTable(svc, expired);
+ // Attach the destination after createTable (live stored ref).
+ TableDefinition table = tableStore.get("us-east-1::TtlTable").orElseThrow();
+ table.getKinesisStreamingDestinations().add(new KinesisStreamingDestination(stream.getStreamArn()));
+
+ svc.deleteExpiredItems();
+
+ List<KinesisRecord> drained = drain(kinesis, "ttl-stream");
+ assertEquals(expired, drained.size(), "one Kinesis record per expired item");
+ for (KinesisRecord record : drained) {
+ JsonNode payload = mapper.readTree(record.getData());
+ assertEquals("REMOVE", payload.get("eventName").asText(), "TTL deletions forward as REMOVE");
+ }
+ assertEquals(0L, forwarder.getForwardFailureCount(), "happy path drops nothing");
+
+ for (int i = 0; i < expired; i++) {
+ ObjectNode key = mapper.createObjectNode();
+ key.set("pk", stringAttr("row-" + i));
+ assertNull(svc.getItem("TtlTable", key, REGION), "expired item must be gone");
+ }
+ }
+
+ @Test
+ void ttlSweep_isolates_forwarding_failures_and_still_persists() {
+ StorageBackend<String, TableDefinition> tableStore = new InMemoryStorage<>();
+ StorageBackend<String, Map<String, JsonNode>> itemStore = new InMemoryStorage<>();
+
+ KinesisService throwingKinesis = mock(KinesisService.class);
+ when(throwingKinesis.putRecord(anyString(), any(byte[].class), anyString(), anyString()))
+ .thenThrow(new RuntimeException("kinesis unavailable"));
+ KinesisStreamingForwarder forwarder = new KinesisStreamingForwarder(throwingKinesis, mapper);
+ DynamoDbService svc = new DynamoDbService(
+ tableStore, itemStore, new RegionResolver("us-east-1", "000000000000"), null, forwarder);
+
+ int expired = 5;
+ seedTtlTable(svc, expired);
+ TableDefinition table = tableStore.get("us-east-1::TtlTable").orElseThrow();
+ table.getKinesisStreamingDestinations().add(new KinesisStreamingDestination(
+ "arn:aws:kinesis:us-east-1:000000000000:stream/ttl-stream"));
+
+ // A forwarding failure must not propagate out of the sweep.
+ assertDoesNotThrow(svc::deleteExpiredItems);
+
+ assertEquals(expired, forwarder.getForwardFailureCount(),
+ "each failed forward must be counted, not swallowed silently");
+
+ // Removed in-memory AND persisted: a fresh service over the same stores must not see them.
+ for (int i = 0; i < expired; i++) {
+ ObjectNode key = mapper.createObjectNode();
+ key.set("pk", stringAttr("row-" + i));
+ assertNull(svc.getItem("TtlTable", key, REGION), "expired item removed from live state");
+ }
+
+ // Direct proof of persistence: the sweep must have written the trimmed map back to the
+ // underlying item store. Inspecting the stored map (rather than getItem) is what makes this
+ // fail if persistItemsForAccount is dropped: getItem would still return null for expired
+ // rows via isExpired even though the stale rows remained physically persisted. putItem
+ // during seeding persisted 'expired' rows here, so the store must now hold none of them.
+ Map<String, JsonNode> persisted = itemStore.get("us-east-1::TtlTable").orElseGet(Map::of);
+ assertTrue(persisted.isEmpty(),
+ () -> "TTL removals must be persisted to the item store; stale rows still present: "
+ + persisted.keySet());
+
+ // Corroborate via a fresh service reload over the same stores.
+ DynamoDbService reloaded = new DynamoDbService(
+ tableStore, itemStore, new RegionResolver("us-east-1", "000000000000"));
+ for (int i = 0; i < expired; i++) {
+ ObjectNode key = mapper.createObjectNode();
+ key.set("pk", stringAttr("row-" + i));
+ assertNull(reloaded.getItem("TtlTable", key, REGION), "expired removal must be persisted");
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarderTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarderTest.java
index 1b8a004626..04800d1533 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarderTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/KinesisStreamingForwarderTest.java
@@ -12,6 +12,7 @@ import org.junit.jupiter.api.Test;
import java.util.List;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.mockito.ArgumentMatchers.*;
import static org.mockito.Mockito.*;
@@ -98,5 +99,7 @@ class KinesisStreamingForwarderTest {
verify(kinesisService).putRecord(eq("stream-1"), any(byte[].class), anyString(), anyString());
verify(kinesisService).putRecord(eq("stream-2"), any(byte[].class), anyString(), anyString());
+ assertEquals(1L, forwarder.getForwardFailureCount(),
+ "the single failed forward must be counted");
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisListShardsConcurrencyTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisListShardsConcurrencyTest.java
new file mode 100644
index 0000000000..55b47d8c8f
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisListShardsConcurrencyTest.java
@@ -0,0 +1,77 @@
+package io.github.hectorvent.floci.services.kinesis;
+
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.services.kinesis.model.KinesisShard;
+import io.github.hectorvent.floci.services.kinesis.model.KinesisStream;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+
+/**
+ * Regression for the ListShards {@code CopyOnWriteArrayList} sub-list hazard.
+ *
+ * <p>Exercises {@link KinesisJsonHandler#paginateShards(List, int)} directly. That is the exact
+ * production step ListShards uses to take a {@code MaxResults} page, so the test needs no JAX-RS
+ * runtime (absent from the sandbox launcher) and no timing window.
+ */
+class KinesisListShardsConcurrencyTest {
+
+ private static final String REGION = "us-east-1";
+
+ private KinesisService service;
+
+ @BeforeEach
+ void setUp() {
+ service = new KinesisService(
+ new InMemoryStorage<>(),
+ new InMemoryStorage<>(),
+ new RegionResolver(REGION, "123456789012"));
+ }
+
+ /**
+ * A ListShards page must survive a concurrent split of the underlying stream.
+ *
+ * <p>ListShards takes {@code MaxResults < shard count}, so a sub-list page is taken; then a split
+ * appends two children to the live backing list. Pre-fix ({@code shards.subList(...)} over the live
+ * CopyOnWriteArrayList) reading the page throws {@link java.util.ConcurrentModificationException};
+ * post-fix (page taken from a {@code List.copyOf} snapshot) the page reflects the pre-split shards
+ * and reads cleanly.
+ */
+ @Test
+ void listShardsPageSurvivesConcurrentSplit() {
+ service.createStream("reshard-stream", 3, REGION);
+ KinesisStream stream = service.describeStream("reshard-stream", REGION);
+
+ // The page ListShards would build for MaxResults=2 against the live shard list.
+ List<KinesisShard> page = KinesisJsonHandler.paginateShards(stream.getShards(), 2);
+
+ // A concurrent split closes the parent and atomically appends two children, mutating the
+ // live backing list the pre-fix sub-list page still points at.
+ String firstShardId = stream.getShards().getFirst().getShardId();
+ service.splitShard("reshard-stream", firstShardId,
+ "170141183460469231731687303715884105728", REGION);
+
+ // Pre-fix: iterating/sizing the live sub-list here throws ConcurrentModificationException.
+ assertDoesNotThrow(() -> {
+ int seen = 0;
+ for (KinesisShard shard : page) {
+ shard.getShardId();
+ seen++;
+ }
+ assertEquals(2, seen);
+ });
+
+ // A consistent page: exactly the first two shards of the pre-split snapshot, in order.
+ assertEquals(2, page.size());
+ assertEquals("shardId-000000000000", page.get(0).getShardId());
+ assertEquals("shardId-000000000001", page.get(1).getShardId());
+
+ // The split itself completed on the underlying stream: 3 original + 2 children.
+ assertEquals(5, service.describeStream("reshard-stream", REGION).getShards().size());
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisServiceTest.java
index 42ee0afb1b..ec20fc862d 100644
--- a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisServiceTest.java
@@ -3,18 +3,39 @@ package io.github.hectorvent.floci.services.kinesis;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.services.kinesis.model.KinesisConsumer;
import io.github.hectorvent.floci.services.kinesis.model.KinesisRecord;
import io.github.hectorvent.floci.services.kinesis.model.KinesisShard;
import io.github.hectorvent.floci.services.kinesis.model.KinesisStream;
import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.RepeatedTest;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.Arguments;
+import org.junit.jupiter.params.provider.MethodSource;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Consumer;
+import java.util.function.Predicate;
+import java.util.stream.Stream;
import static org.junit.jupiter.api.Assertions.*;
@@ -288,7 +309,127 @@ class KinesisServiceTest {
KinesisStream updated = kinesisService.describeStream("my-stream", REGION);
assertEquals(3, updated.getShards().size());
- assertTrue(updated.getShards().getFirst().isClosed());
+ KinesisShard parent = updated.getShards().getFirst();
+ assertTrue(parent.isClosed());
+
+ // Both children must get distinct shard ids. nextShardId is size-based, so computing both
+ // ids before adding either child assigns the same id to both (regression guard).
+ List<String> shardIds = updated.getShards().stream().map(KinesisShard::getShardId).toList();
+ assertEquals(shardIds.size(), new HashSet<>(shardIds).size(), "shard ids must be unique after split");
+
+ // The single atomic addAll must publish a COMPLETE topology: both children present, each
+ // pointing at the parent, together tiling the parent's whole hash range with no gap/overlap.
+ List<KinesisShard> children = updated.getShards().stream()
+ .filter(s -> shardId.equals(s.getParentShardId()))
+ .toList();
+ assertEquals(2, children.size(), "a split must publish exactly two children");
+ KinesisShard child1 = children.get(0);
+ KinesisShard child2 = children.get(1);
+ assertEquals(parent.getHashKeyRange().startingHashKey(), child1.getHashKeyRange().startingHashKey());
+ assertEquals("170141183460469231731687303715884105727", child1.getHashKeyRange().endingHashKey());
+ assertEquals("170141183460469231731687303715884105728", child2.getHashKeyRange().startingHashKey());
+ assertEquals(parent.getHashKeyRange().endingHashKey(), child2.getHashKeyRange().endingHashKey());
+ }
+
+ /**
+ * A lock-free reader must never observe a half-published split: a closed parent with only one
+ * of its two children. splitShard publishes both children in one atomic CopyOnWriteArrayList.addAll,
+ * so every snapshot holds either zero or both split-children of any given parent.
+ *
+ * <p>Fail-without-fix: reverting the addAll to two sequential {@code add()} calls opens a window
+ * (between the two adds, widened by the child2 shard construction) in which the list holds child1
+ * but not child2; the hammering reader below catches it and records a violation.
+ */
+ @RepeatedTest(3)
+ void splitShard_concurrentReaderNeverSeesHalfPublishedTopology() throws Exception {
+ kinesisService.createStream("my-stream", 1, REGION);
+ KinesisStream stream = kinesisService.describeStream("my-stream", REGION);
+
+ AtomicReference<String> violation = new AtomicReference<>();
+ AtomicBoolean done = new AtomicBoolean(false);
+
+ Thread reader = new Thread(() -> {
+ while (!done.get()) {
+ // A fresh snapshot of the live list, exactly as a lock-free DescribeStream/ListShards sees it.
+ List<KinesisShard> snapshot = new ArrayList<>(stream.getShards());
+ Map<String, Integer> splitChildCount = new java.util.HashMap<>();
+ for (KinesisShard s : snapshot) {
+ // Split children carry a parent but no adjacent-parent (that marks a merge child).
+ if (s.getParentShardId() != null && s.getAdjacentParentShardId() == null) {
+ splitChildCount.merge(s.getParentShardId(), 1, Integer::sum);
+ }
+ }
+ for (Map.Entry<String, Integer> e : splitChildCount.entrySet()) {
+ if (e.getValue() != 2) {
+ violation.compareAndSet(null,
+ "parent " + e.getKey() + " observed with " + e.getValue()
+ + " split-children (expected 0 or 2)");
+ return;
+ }
+ }
+ }
+ });
+ reader.start();
+
+ // Split an open shard repeatedly; each split is one control-plane resharding with its own
+ // publish window. Always split an open shard so open shards remain available.
+ for (int i = 0; i < 3000 && violation.get() == null; i++) {
+ // Split the widest open shard so ranges never narrow to the point splitting is impossible;
+ // this keeps every one of the 3000 iterations a real split with its own publish window.
+ KinesisShard open = stream.getShards().stream()
+ .filter(s -> !s.isClosed())
+ .max(java.util.Comparator.comparing(s ->
+ new java.math.BigInteger(s.getHashKeyRange().endingHashKey())
+ .subtract(new java.math.BigInteger(s.getHashKeyRange().startingHashKey()))))
+ .orElseThrow();
+ java.math.BigInteger start = new java.math.BigInteger(open.getHashKeyRange().startingHashKey());
+ java.math.BigInteger end = new java.math.BigInteger(open.getHashKeyRange().endingHashKey());
+ java.math.BigInteger mid = start.add(end).divide(java.math.BigInteger.TWO);
+ if (mid.compareTo(start) <= 0 || mid.compareTo(end) >= 0) {
+ break; // range too narrow to split further
+ }
+ kinesisService.splitShard("my-stream", open.getShardId(), mid.toString(), REGION);
+ }
+ done.set(true);
+ reader.join(TimeUnit.SECONDS.toMillis(10));
+ assertFalse(reader.isAlive(), "reader thread did not terminate");
+
+ assertNull(violation.get(), () -> "half-published split observed: " + violation.get());
+ }
+
+ @Test
+ void getShards_readerMidIteration_survivesConcurrentSplit() {
+ // Two shards so the snapshot the reader opens still has an element left to read AFTER the
+ // concurrent split; a single-shard snapshot would be exhausted by the first next() and the
+ // second next() would throw NoSuchElementException even under COWAL.
+ kinesisService.createStream("my-stream", 2, REGION);
+ // describeStream returns the same KinesisStream instance the store holds, so these iterators
+ // are over the exact backing list splitShard mutates.
+ KinesisStream stream = kinesisService.describeStream("my-stream", REGION);
+ String shardId = stream.getShards().getFirst().getShardId();
+
+ // A second iterator opened before the split, used to assert the pre-split snapshot is fixed.
+ Iterator<KinesisShard> preSplitSnapshot = stream.getShards().iterator();
+ Iterator<KinesisShard> reader = stream.getShards().iterator();
+ reader.next();
+
+ kinesisService.splitShard("my-stream", shardId, "170141183460469231731687303715884105728", REGION);
+
+ // A plain ArrayList iterator throws ConcurrentModificationException here; COWAL keeps reading
+ // its fixed pre-split snapshot.
+ assertDoesNotThrow(reader::next);
+
+ // The snapshot opened before the split still reflects exactly the original two shards,
+ // regardless of the two children the split appended afterwards.
+ int snapshotSize = 0;
+ while (preSplitSnapshot.hasNext()) {
+ preSplitSnapshot.next();
+ snapshotSize++;
+ }
+ assertEquals(2, snapshotSize);
+
+ // A fresh read observes the completed reshard: 2 original shards + 2 children.
+ assertEquals(4, kinesisService.describeStream("my-stream", REGION).getShards().size());
}
@Test
@@ -444,4 +585,504 @@ class KinesisServiceTest {
.get("Records");
assertEquals(1, got.size(), "AT_TIMESTAMP boundary is >= (inclusive)");
}
+
+ // ─── Concurrency regression coverage (issue #571 sibling: Kinesis append race) ──────
+
+ /** Runs {@code op} {@code opsPerThread} times on each of {@code threadCount} threads,
+ * all released together, and returns any throwables observed. */
+ private List<Throwable> runConcurrently(int threadCount, int opsPerThread, Runnable op)
+ throws InterruptedException {
+ ExecutorService pool = Executors.newFixedThreadPool(threadCount);
+ CountDownLatch startGate = new CountDownLatch(1);
+ CountDownLatch doneGate = new CountDownLatch(threadCount);
+ List<Throwable> errors = Collections.synchronizedList(new ArrayList<>());
+ try {
+ for (int t = 0; t < threadCount; t++) {
+ pool.submit(() -> {
+ try {
+ startGate.await();
+ for (int i = 0; i < opsPerThread; i++) {
+ op.run();
+ }
+ } catch (Throwable e) {
+ errors.add(e);
+ } finally {
+ doneGate.countDown();
+ }
+ });
+ }
+ startGate.countDown();
+ assertTrue(doneGate.await(60, TimeUnit.SECONDS),
+ "concurrent producers did not complete within 60s");
+ return errors;
+ } finally {
+ pool.shutdownNow();
+ assertTrue(pool.awaitTermination(5, TimeUnit.SECONDS), "thread pool did not terminate");
+ }
+ }
+
+ /** Drains every record of a single-shard stream, following NextShardIterator to the end. */
+ private List<KinesisRecord> drainShard(String streamName, String type, String seq) {
+ String shardId = kinesisService.describeStream(streamName, REGION)
+ .getShards().getFirst().getShardId();
+ String iterator = kinesisService.getShardIterator(streamName, shardId, type, seq, REGION);
+ return drainFrom(iterator);
+ }
+
+ @SuppressWarnings("unchecked")
+ private List<KinesisRecord> drainFrom(String iterator) {
+ List<KinesisRecord> all = new ArrayList<>();
+ while (true) {
+ Map<String, Object> page = kinesisService.getRecords(iterator, 1000, REGION);
+ List<KinesisRecord> records = (List<KinesisRecord>) page.get("Records");
+ if (records.isEmpty()) {
+ break;
+ }
+ all.addAll(records);
+ iterator = (String) page.get("NextShardIterator");
+ }
+ return all;
+ }
+
+ private static void assertStrictlyIncreasing(List<KinesisRecord> records) {
+ long previous = Long.MIN_VALUE;
+ for (KinesisRecord r : records) {
+ long current = Long.parseLong(r.getSequenceNumber());
+ assertTrue(current > previous,
+ () -> "sequence numbers must be strictly increasing in shard order");
+ previous = current;
+ }
+ }
+
+ @RepeatedTest(5)
+ void concurrent_putRecord_preserves_every_record_and_ordering() throws InterruptedException {
+ kinesisService.createStream("stress", 1, REGION);
+ byte[] data = "payload".getBytes(StandardCharsets.UTF_8);
+ int threads = 32;
+ int opsPerThread = 50;
+ int expected = threads * opsPerThread; // 1600
+
+ List<Throwable> errors = runConcurrently(threads, opsPerThread,
+ () -> kinesisService.putRecord("stress", data, "pk", REGION));
+ assertTrue(errors.isEmpty(), () -> "unexpected errors: " + errors);
+
+ List<KinesisRecord> all = drainShard("stress", "TRIM_HORIZON", null);
+ assertEquals(expected, all.size(),
+ "every concurrently appended record must survive");
+
+ Set<String> distinct = new HashSet<>();
+ all.forEach(r -> distinct.add(r.getSequenceNumber()));
+ assertEquals(expected, distinct.size(), "every record must carry a distinct sequence number");
+
+ assertStrictlyIncreasing(all);
+
+ // AFTER_SEQUENCE_NUMBER continuation: everything strictly after position k, in order.
+ int k = 799;
+ String kSeq = all.get(k).getSequenceNumber();
+ List<KinesisRecord> after = drainShard("stress", "AFTER_SEQUENCE_NUMBER", kSeq);
+ List<String> expectedTail = new ArrayList<>();
+ for (int i = k + 1; i < all.size(); i++) {
+ expectedTail.add(all.get(i).getSequenceNumber());
+ }
+ List<String> actualTail = new ArrayList<>();
+ after.forEach(r -> actualTail.add(r.getSequenceNumber()));
+ assertEquals(expectedTail, actualTail,
+ "AFTER_SEQUENCE_NUMBER must return exactly the records after position k, in order");
+ }
+
+ @Test
+ void putRecord_appendCriticalSectionIsMutuallyExclusive() throws Exception {
+ kinesisService.createStream("mx", 1, REGION);
+
+ CountDownLatch entered = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ AtomicBoolean firstIn = new AtomicBoolean(true);
+ // Only the first producer to enter the critical section parks; it holds the stream lock
+ // until released, so any second producer on the same stream must block behind it.
+ kinesisService.putRecordAppendHook = () -> {
+ if (firstIn.compareAndSet(true, false)) {
+ entered.countDown();
+ try {
+ release.await();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ };
+
+ // T2 signals the instant before it calls putRecord and publishes its thread, so we can
+ // prove it was actually scheduled and reached the critical section (BLOCKED on the stream
+ // monitor), not merely never scheduled, which would let a timeout pass vacuously on a
+ // loaded worker even if the lock had been removed.
+ CountDownLatch t2Started = new CountDownLatch(1);
+ AtomicReference<Thread> t2Thread = new AtomicReference<>();
+
+ ExecutorService pool = Executors.newFixedThreadPool(2);
+ try {
+ Future<?> t1 = pool.submit(() ->
+ kinesisService.putRecord("mx", "1".getBytes(StandardCharsets.UTF_8), "pk", REGION));
+ assertTrue(entered.await(2, TimeUnit.SECONDS), "T1 must enter the append critical section");
+
+ Future<?> t2 = pool.submit(() -> {
+ t2Thread.set(Thread.currentThread());
+ t2Started.countDown();
+ return kinesisService.putRecord("mx", "2".getBytes(StandardCharsets.UTF_8), "pk", REGION);
+ });
+
+ assertTrue(t2Started.await(2, TimeUnit.SECONDS),
+ "T2 must be scheduled and reach the putRecord call");
+
+ // Deterministic barrier: T2 must actually reach and block on the stream monitor while
+ // T1 holds it. Spin until its thread is BLOCKED (entering the synchronized section);
+ // if the lock were gone T2 would run to completion instead and never park here.
+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2);
+ boolean blockedOnMonitor = false;
+ while (System.nanoTime() < deadline) {
+ Thread th = t2Thread.get();
+ if (th != null && th.getState() == Thread.State.BLOCKED) {
+ blockedOnMonitor = true;
+ break;
+ }
+ if (t2.isDone()) {
+ break;
+ }
+ Thread.sleep(5);
+ }
+ assertTrue(blockedOnMonitor,
+ "T2 must block entering the append critical section while T1 holds the stream lock");
+
+ // Mutual exclusion: T2 cannot complete while T1 holds the lock.
+ assertThrows(TimeoutException.class, () -> t2.get(500, TimeUnit.MILLISECONDS),
+ "a second producer must block while the first holds the stream lock");
+
+ release.countDown();
+ t1.get(2, TimeUnit.SECONDS);
+ t2.get(2, TimeUnit.SECONDS);
+ } finally {
+ pool.shutdownNow();
+ }
+
+ List<KinesisRecord> all = drainShard("mx", "TRIM_HORIZON", null);
+ assertEquals(2, all.size(), "both records must be persisted");
+ long firstSeq = Long.parseLong(all.get(0).getSequenceNumber());
+ long secondSeq = Long.parseLong(all.get(1).getSequenceNumber());
+ assertTrue(firstSeq < secondSeq,
+ "T1 completed its append first, so its lower sequence must sort first");
+ }
+
+ /**
+ * A deleteStream that overlaps an in-flight append must be serialized behind it and must NOT
+ * be able to resurrect the stream. P1 parks inside the append critical section holding the
+ * stream monitor; the deleter can only run once P1 releases it, so the delete always wins and
+ * the stream stays gone. Fails on the pre-fix code where deleteStream neither took the monitor
+ * nor left it in place: the deleter would run immediately and P1's trailing store.put would
+ * re-persist (resurrect) the deleted instance.
+ */
+ @Test
+ void deleteStream_serializesBehindInFlightAppend_andDoesNotResurrect() throws Exception {
+ kinesisService.createStream("res", 1, REGION);
+
+ CountDownLatch entered = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ AtomicBoolean firstIn = new AtomicBoolean(true);
+ kinesisService.putRecordAppendHook = () -> {
+ if (firstIn.compareAndSet(true, false)) {
+ entered.countDown();
+ try {
+ release.await();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ };
+
+ ExecutorService pool = Executors.newFixedThreadPool(2);
+ try {
+ Future<?> producer = pool.submit(() ->
+ kinesisService.putRecord("res", "p1".getBytes(StandardCharsets.UTF_8), "pk", REGION));
+ assertTrue(entered.await(2, TimeUnit.SECONDS), "producer must enter the append critical section");
+
+ // The deleter shares the stream monitor the parked producer holds, so it must block.
+ Future<?> deleter = pool.submit(() -> {
+ kinesisService.deleteStream("res", REGION);
+ return null;
+ });
+ assertThrows(TimeoutException.class, () -> deleter.get(500, TimeUnit.MILLISECONDS),
+ "deleteStream must block behind the in-flight append, not delete concurrently");
+
+ release.countDown();
+ producer.get(2, TimeUnit.SECONDS);
+ deleter.get(2, TimeUnit.SECONDS);
+ } finally {
+ pool.shutdownNow();
+ }
+
+ // The delete ran after the producer's persist, so the stream must be gone, not resurrected
+ // by a stale trailing store.put.
+ assertTrue(kinesisService.listStreams(REGION).isEmpty(),
+ "a deleted stream must not be resurrected by an overlapping append");
+ assertThrows(AwsException.class, () -> kinesisService.describeStream("res", REGION),
+ "the resurrected stream must not be describable");
+
+ // And a freshly recreated stream starts empty: no stale record leaked across the lifecycle.
+ kinesisService.createStream("res", 1, REGION);
+ assertTrue(drainShard("res", "TRIM_HORIZON", null).isEmpty(),
+ "a recreated stream must not carry records from the deleted instance");
+ }
+
+ /**
+ * If a stream is deleted in the window after a producer resolved it but before it acquires the
+ * append lock, the producer must fail cleanly (ResourceNotFoundException) rather than append to
+ * and re-persist the stale instance. Deterministic via the pre-lock hook: the producer parks
+ * WITHOUT holding the monitor, so the deleter wins the lock first. Fails on code lacking the
+ * in-lock re-resolve: the producer would resurrect the stream and throw nothing.
+ */
+ @Test
+ void putRecord_failsCleanly_whenStreamDeletedBeforeLockAcquired() throws Exception {
+ kinesisService.createStream("gone", 1, REGION);
+
+ CountDownLatch preEntered = new CountDownLatch(1);
+ CountDownLatch preRelease = new CountDownLatch(1);
+ AtomicBoolean firstIn = new AtomicBoolean(true);
+ kinesisService.putRecordBeforeLockHook = () -> {
+ if (firstIn.compareAndSet(true, false)) {
+ preEntered.countDown();
+ try {
+ preRelease.await();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ };
+
+ AtomicReference<Throwable> thrown = new AtomicReference<>();
+ ExecutorService pool = Executors.newFixedThreadPool(1);
+ try {
+ Future<?> producer = pool.submit(() -> {
+ try {
+ kinesisService.putRecord("gone", "p1".getBytes(StandardCharsets.UTF_8), "pk", REGION);
+ } catch (Throwable t) {
+ thrown.set(t);
+ }
+ return null;
+ });
+
+ assertTrue(preEntered.await(2, TimeUnit.SECONDS),
+ "producer must reach the pre-lock window without holding the monitor");
+
+ // Producer is parked without the monitor, so the deleter wins the lock and completes.
+ kinesisService.deleteStream("gone", REGION);
+
+ preRelease.countDown();
+ producer.get(2, TimeUnit.SECONDS);
+ } finally {
+ pool.shutdownNow();
+ }
+
+ Throwable t = thrown.get();
+ assertNotNull(t, "the append must fail once the stream is deleted, not silently resurrect it");
+ assertInstanceOf(AwsException.class, t, () -> "unexpected failure: " + t);
+ assertEquals("ResourceNotFoundException", ((AwsException) t).getErrorCode(),
+ "a delete-before-append must surface the same missing-stream error as any absent stream");
+
+ assertTrue(kinesisService.listStreams(REGION).isEmpty(),
+ "the deleted stream must not be resurrected by the failed append");
+ }
+
+ private static final String META_STREAM = "meta";
+
+ /**
+ * A store that parks the first {@code get} after {@link #arm()} until released.
+ *
+ * <p>Every metadata write resolves its stream through exactly one {@code store.get}, and does
+ * so while holding the stream monitor, so parking there wedges the writer inside its critical
+ * section without needing a production test seam.
+ */
+ private static final class ParkOnResolveStore implements StorageBackend<String, KinesisStream> {
+ private final StorageBackend<String, KinesisStream> delegate = new InMemoryStorage<>();
+ private final AtomicBoolean armed = new AtomicBoolean();
+ final CountDownLatch parked = new CountDownLatch(1);
+ final CountDownLatch release = new CountDownLatch(1);
+
+ void arm() {
+ armed.set(true);
+ }
+
+ @Override
+ public Optional<KinesisStream> get(String key) {
+ if (armed.compareAndSet(true, false)) {
+ parked.countDown();
+ try {
+ release.await();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ }
+ return delegate.get(key);
+ }
+
+ @Override
+ public void put(String key, KinesisStream value) {
+ delegate.put(key, value);
+ }
+
+ @Override
+ public void delete(String key) {
+ delegate.delete(key);
+ }
+
+ @Override
+ public List<KinesisStream> scan(Predicate<String> keyFilter) {
+ return delegate.scan(keyFilter);
+ }
+
+ @Override
+ public Set<String> keys() {
+ return delegate.keys();
+ }
+
+ @Override
+ public void flush() {
+ delegate.flush();
+ }
+
+ @Override
+ public void load() {
+ delegate.load();
+ }
+
+ @Override
+ public void clear() {
+ delegate.clear();
+ }
+ }
+
+ private static Arguments metadataWrite(String label, Consumer<KinesisService> setUp,
+ Consumer<KinesisService> apply) {
+ return Arguments.of(label, setUp, apply);
+ }
+
+ /**
+ * Every KinesisService action that reads a stored stream, mutates it and persists it again.
+ * {@code setUp} establishes any precondition the action needs to reach its persist rather than
+ * short-circuit as a no-op, and runs before the store is armed.
+ */
+ static Stream<Arguments> metadataWrites() {
+ return Stream.of(
+ metadataWrite("updateStreamMode", s -> {},
+ s -> s.updateStreamMode(META_STREAM, "ON_DEMAND", REGION)),
+ metadataWrite("addTagsToStream", s -> {},
+ s -> s.addTagsToStream(META_STREAM, Map.of("env", "test"), REGION)),
+ metadataWrite("removeTagsFromStream",
+ s -> s.addTagsToStream(META_STREAM, Map.of("env", "test"), REGION),
+ s -> s.removeTagsFromStream(META_STREAM, List.of("env"), REGION)),
+ metadataWrite("startStreamEncryption", s -> {},
+ s -> s.startStreamEncryption(META_STREAM, "KMS", "alias/aws/kinesis", REGION)),
+ metadataWrite("stopStreamEncryption",
+ s -> s.startStreamEncryption(META_STREAM, "KMS", "alias/aws/kinesis", REGION),
+ s -> s.stopStreamEncryption(META_STREAM, REGION)),
+ metadataWrite("increaseStreamRetentionPeriod", s -> {},
+ s -> s.increaseStreamRetentionPeriod(META_STREAM, 48, REGION)),
+ metadataWrite("decreaseStreamRetentionPeriod",
+ s -> s.increaseStreamRetentionPeriod(META_STREAM, 48, REGION),
+ s -> s.decreaseStreamRetentionPeriod(META_STREAM, 24, REGION)),
+ metadataWrite("enableEnhancedMonitoring", s -> {},
+ s -> s.enableEnhancedMonitoring(META_STREAM, List.of("IncomingBytes"), REGION)),
+ metadataWrite("disableEnhancedMonitoring",
+ s -> s.enableEnhancedMonitoring(META_STREAM, List.of("IncomingBytes"), REGION),
+ s -> s.disableEnhancedMonitoring(META_STREAM, List.of("IncomingBytes"), REGION)),
+ metadataWrite("updateMaxRecordSize", s -> {},
+ s -> s.updateMaxRecordSize(META_STREAM, 2048, REGION)));
+ }
+
+ /** Spins until {@code thread} is BLOCKED entering a monitor, or {@code task} completes. */
+ private static boolean awaitBlockedOnMonitor(AtomicReference<Thread> thread, Future<?> task)
+ throws InterruptedException {
+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(2);
+ while (System.nanoTime() < deadline) {
+ Thread t = thread.get();
+ if (t != null && t.getState() == Thread.State.BLOCKED) {
+ return true;
+ }
+ if (task.isDone()) {
+ return false;
+ }
+ Thread.sleep(5);
+ }
+ return false;
+ }
+
+ /**
+ * A metadata write that overlaps a deleteStream must not resurrect the stream.
+ *
+ * <p>The writer is wedged inside its own stream resolve, which it performs after taking the
+ * per-stream monitor. A concurrent deleteStream shares that monitor, so it must block: the
+ * delete therefore lands strictly after the writer's persist and the stream stays gone.
+ *
+ * <p>This fails on either half of the fix being absent. If a metadata path did not take the
+ * monitor, or resolved the stream before taking it, the deleter would not block here, and the
+ * writer's trailing {@code store.put} would re-insert the instance the delete had just removed.
+ */
+ @ParameterizedTest(name = "{0}")
+ @MethodSource("metadataWrites")
+ void metadataWrite_cannotResurrectAConcurrentlyDeletedStream(
+ String label, Consumer<KinesisService> setUp, Consumer<KinesisService> apply) throws Exception {
+ ParkOnResolveStore store = new ParkOnResolveStore();
+ KinesisService service = new KinesisService(store, new InMemoryStorage<>(),
+ new RegionResolver(REGION, "000000000000"));
+ service.createStream(META_STREAM, 1, REGION);
+ setUp.accept(service);
+ store.arm();
+
+ CountDownLatch deleterStarted = new CountDownLatch(1);
+ AtomicReference<Thread> deleterThread = new AtomicReference<>();
+ ExecutorService pool = Executors.newFixedThreadPool(2);
+ try {
+ Future<?> writer = pool.submit(() -> {
+ apply.accept(service);
+ return null;
+ });
+ assertTrue(store.parked.await(2, TimeUnit.SECONDS),
+ () -> label + " must park inside the stream resolve it performs under the monitor");
+
+ // The deleter publishes its thread and signals immediately before the call, so a
+ // never-scheduled deleter cannot make the blocking assertions below pass vacuously.
+ Future<?> deleter = pool.submit(() -> {
+ deleterThread.set(Thread.currentThread());
+ deleterStarted.countDown();
+ service.deleteStream(META_STREAM, REGION);
+ return null;
+ });
+ assertTrue(deleterStarted.await(2, TimeUnit.SECONDS),
+ "the deleter must be scheduled and reach the deleteStream call");
+ assertTrue(awaitBlockedOnMonitor(deleterThread, deleter),
+ () -> "deleteStream must block on the stream monitor while " + label
+ + " holds it, which is what proves the resolve happens inside the lock");
+ assertThrows(TimeoutException.class, () -> deleter.get(500, TimeUnit.MILLISECONDS),
+ () -> "deleteStream must not complete while " + label + " holds the stream monitor");
+
+ store.release.countDown();
+ // Neither may fail: the writer resolved a live stream, and the delete then removes it.
+ writer.get(2, TimeUnit.SECONDS);
+ deleter.get(2, TimeUnit.SECONDS);
+ } finally {
+ pool.shutdownNow();
+ }
+
+ assertTrue(service.listStreams(REGION).isEmpty(),
+ () -> label + " must not resurrect a concurrently deleted stream");
+ AwsException notFound = assertThrows(AwsException.class,
+ () -> service.describeStream(META_STREAM, REGION),
+ () -> "a stream resurrected by " + label + " would still be describable");
+ assertEquals("ResourceNotFoundException", notFound.getErrorCode());
+
+ // A stream recreated under the same name carries none of the deleted instance's metadata,
+ // so nothing leaked across the lifecycle.
+ KinesisStream recreated = service.createStream(META_STREAM, 1, REGION);
+ assertTrue(recreated.getTags().isEmpty(), "a recreated stream must start untagged");
+ assertEquals("PROVISIONED", recreated.getStreamMode(),
+ "a recreated stream must start in the default stream mode");
+ assertEquals(24, recreated.getRetentionPeriodHours(),
+ "a recreated stream must start at the default retention period");
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisWalPersistenceTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisWalPersistenceTest.java
new file mode 100644
index 0000000000..85ad56f93a
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisWalPersistenceTest.java
@@ -0,0 +1,151 @@
+package io.github.hectorvent.floci.services.kinesis;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.core.storage.WalStorage;
+import io.github.hectorvent.floci.services.kinesis.model.KinesisRecord;
+import io.github.hectorvent.floci.services.kinesis.model.KinesisStream;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+/**
+ * Concurrent producers must not lose records across a WAL persist/reload cycle.
+ *
+ * <p>Serialize-order must equal append-order must equal WAL-write-order per stream key, so a
+ * process restart replays every record in sequence order. See issue #571 (Kinesis append race).
+ */
+class KinesisWalPersistenceTest {
+
+ private static final String REGION = "us-east-1";
+
+ @TempDir
+ Path tmp;
+
+ @Test
+ void concurrentPutRecords_survive_wal_reload() throws Exception {
+ Path snapshot = tmp.resolve("kinesis-streams.json");
+ Path wal = tmp.resolve("kinesis-streams.wal");
+ RegionResolver rr = new RegionResolver("us-east-1", "000000000000");
+
+ int threads = 16;
+ int opsPerThread = 50;
+ int expected = threads * opsPerThread; // 800
+ byte[] data = "payload".getBytes(StandardCharsets.UTF_8);
+
+ // Large compaction interval so replay (not snapshot) is what proves durability.
+ WalStorage<String, KinesisStream> store1 = new WalStorage<>(
+ snapshot, wal, new TypeReference<Map<String, KinesisStream>>() {}, 3_600_000L);
+ WalStorage<String, KinesisStream> store2 = null;
+ try {
+ store1.load(); // opens the WAL writer: appends before load() are silent no-ops
+ KinesisService writer = new KinesisService(store1, new InMemoryStorage<>(), rr);
+ writer.createStream("wal-stream", 1, REGION);
+
+ List<Throwable> errors = runConcurrently(threads, opsPerThread,
+ () -> writer.putRecord("wal-stream", data, "pk", REGION));
+ assertTrue(errors.isEmpty(), () -> "unexpected errors: " + errors);
+
+ // Copy the still-uncompacted WAL (and snapshot, if one exists) aside for replay. Each
+ // appendPut flushes to the file, so the copy holds every entry. Reloading from the copy
+ // lets us shut store1 down cleanly in the finally: shutdown() compacts and truncates
+ // store1's own WAL, but that no longer touches the files the reader replays.
+ Path snapshot2 = tmp.resolve("reload-streams.json");
+ Path wal2 = tmp.resolve("reload-streams.wal");
+ Files.copy(wal, wal2, StandardCopyOption.REPLACE_EXISTING);
+ if (Files.exists(snapshot)) {
+ Files.copy(snapshot, snapshot2, StandardCopyOption.REPLACE_EXISTING);
+ }
+
+ // Fresh instance over the copied files: load() replays the WAL.
+ store2 = new WalStorage<>(
+ snapshot2, wal2, new TypeReference<Map<String, KinesisStream>>() {}, 3_600_000L);
+ store2.load();
+ KinesisService reader = new KinesisService(store2, new InMemoryStorage<>(), rr);
+
+ List<KinesisRecord> all = drain(reader, "wal-stream");
+ assertEquals(expected, all.size(), "every record must survive the WAL reload");
+
+ Set<String> distinct = new HashSet<>();
+ all.forEach(r -> distinct.add(r.getSequenceNumber()));
+ assertEquals(expected, distinct.size(), "every reloaded record must carry a distinct sequence");
+
+ long previous = Long.MIN_VALUE;
+ for (KinesisRecord r : all) {
+ long current = Long.parseLong(r.getSequenceNumber());
+ assertTrue(current > previous, "reloaded records must be in strictly increasing sequence order");
+ previous = current;
+ }
+ } finally {
+ // Shut down both stores so neither leaves a compaction scheduler or open WAL writer.
+ if (store2 != null) {
+ store2.shutdown();
+ }
+ store1.shutdown();
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private List<KinesisRecord> drain(KinesisService service, String streamName) {
+ String shardId = service.describeStream(streamName, REGION).getShards().getFirst().getShardId();
+ String iterator = service.getShardIterator(streamName, shardId, "TRIM_HORIZON", null, REGION);
+ List<KinesisRecord> all = new ArrayList<>();
+ while (true) {
+ Map<String, Object> page = service.getRecords(iterator, 1000, REGION);
+ List<KinesisRecord> records = (List<KinesisRecord>) page.get("Records");
+ if (records.isEmpty()) {
+ break;
+ }
+ all.addAll(records);
+ iterator = (String) page.get("NextShardIterator");
+ }
+ return all;
+ }
+
+ private List<Throwable> runConcurrently(int threadCount, int opsPerThread, Runnable op)
+ throws InterruptedException {
+ ExecutorService pool = Executors.newFixedThreadPool(threadCount);
+ CountDownLatch startGate = new CountDownLatch(1);
+ CountDownLatch doneGate = new CountDownLatch(threadCount);
+ List<Throwable> errors = Collections.synchronizedList(new ArrayList<>());
+ try {
+ for (int t = 0; t < threadCount; t++) {
+ pool.submit(() -> {
+ try {
+ startGate.await();
+ for (int i = 0; i < opsPerThread; i++) {
+ op.run();
+ }
+ } catch (Throwable e) {
+ errors.add(e);
+ } finally {
+ doneGate.countDown();
+ }
+ });
+ }
+ startGate.countDown();
+ assertTrue(doneGate.await(60, TimeUnit.SECONDS), "producers did not complete within 60s");
+ return errors;
+ } finally {
+ pool.shutdownNow();
+ assertTrue(pool.awaitTermination(5, TimeUnit.SECONDS), "thread pool did not terminate");
+ }
+ }
+}