Light, fluffy, and always free - The AWS Local Emulator alternative
fix(kinesis): expire shard records past the stream's retention period (#4224)
Niels Kootstraaddee350ba56
diff --git a/docs/services/kinesis.md b/docs/services/kinesis.mdindex 18c96fb3a2..3870e40bdb 100644--- a/docs/services/kinesis.md+++ b/docs/services/kinesis.md@@ -69,6 +69,10 @@ aws kinesis describe-stream --stream-arn arn:aws:kinesis:us-east-1:000000000000:A `LATEST` iterator is positioned at the shard tip at the moment the iterator is created, matching AWS: records written after the iterator was obtained are returned, records written before are not. This supports the standard tailing pattern: obtain a `LATEST` iterator, trigger the action that produces the record, then poll `GetRecords` following `NextShardIterator`.+## Record Retention++A stream's retention period (default 24 hours, `IncreaseStreamRetentionPeriod`/`DecreaseStreamRetentionPeriod` up to 8760 hours) is enforced: a record older than the current retention period is not returned by `GetRecords` or the `GET /_aws/kinesis/records` inspection endpoint, and is actually removed from memory rather than merely hidden. Expiry is checked lazily on each stream's next `PutRecord`/`PutRecords` or read after a record ages out; there is no scheduled sweep. After `DecreaseStreamRetentionPeriod`, records outside the new window are inaccessible from the next put or read, in line with AWS making them inaccessible almost immediately. Shard iterators (including `NextShardIterator` continuation tokens and `LATEST`) resolve by sequence number, so they keep working correctly across pruning even if the exact record they were positioned at has since expired.+## Record Routing`PutRecord` and `PutRecords` honor `ExplicitHashKey` when it is provided. The value must be a decimal integer in the Kinesis hash-key space, and records are written to the open shard whose `HashKeyRange` contains that value. Without `ExplicitHashKey`, Floci keeps using the partition key to choose a shard.diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.javaindex 25205598cf..137aae6424 100644--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java@@ -85,8 +85,10 @@ public class FirehoseService implements ResourceProvider {// DeliveryStartTimestamp and re-deliver the source stream's whole retained history to// S3. A Kinesis shard iterator is a self-describing token// (streamName|shardId|type|sequenceNumber|index|timestamp, base64) with no- // JVM-lifetime component, and KinesisService never trims a shard's record list, so a- // checkpoint written before a restart still resolves to the same position after one.+ // JVM-lifetime component, and KinesisService resolves it by sequence number, so a+ // checkpoint written before a restart still resolves to the same position after one,+ // even once retention has pruned older records. If the checkpointed record itself has+ // expired, reading resumes at the oldest retained record.//// This store holds only COMMITTED positions: a shard is advanced here once the// records read up to that point have actually landed in S3. Seediff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.javaindex 0ce6b79f3a..6c5e5c5a47 100644--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java@@ -16,6 +16,7 @@ import org.jboss.logging.Logger;import java.math.BigInteger;import java.nio.charset.StandardCharsets;+import java.time.Clock;import java.time.Duration;import java.time.Instant;import java.util.*;@@ -63,6 +64,7 @@ public class KinesisService implements ResourceProvider {private final StorageBackend<String, KinesisStream> store;private final StorageBackend<String, KinesisConsumer> consumerStore;private final RegionResolver regionResolver;+ private final Clock clock;private final AtomicLong sequenceGenerator = new AtomicLong(System.currentTimeMillis());// One monitor per stream key, handed out by lockFor. Serializes sequence allocation + shard// append + persist so the in-memory log, the assigned sequence order and the WAL write order@@ -89,9 +91,21 @@ public class KinesisService implements ResourceProvider {KinesisService(StorageBackend<String, KinesisStream> store,StorageBackend<String, KinesisConsumer> consumerStore,RegionResolver regionResolver) {+ this(store, consumerStore, regionResolver, Clock.systemUTC());+ }++ /**+ * Test seam: lets retention-period expiry be exercised deterministically instead+ * of depending on wall-clock time. Production code always goes through the public/{@code @Inject}+ * constructors, which pin the system clock.+ */+ KinesisService(StorageBackend<String, KinesisStream> store,+ StorageBackend<String, KinesisConsumer> consumerStore,+ RegionResolver regionResolver, Clock clock) {this.store = store;this.consumerStore = consumerStore;this.regionResolver = regionResolver;+ this.clock = clock;}public KinesisStream createStream(String streamName, int shardCount, String region) {@@ -772,9 +786,10 @@ public class KinesisService implements ResourceProvider {validateExplicitHashKey(explicitHashKey);validateRecordSize(current, data, partitionKey);KinesisShard shard = selectShard(current, partitionKey, explicitHashKey);+ pruneExpiredRecords(current, shard);String sequenceNumber = String.valueOf(sequenceGenerator.incrementAndGet());putRecordAppendHook.run();- KinesisRecord record = new KinesisRecord(data, partitionKey, sequenceNumber, Instant.now());+ KinesisRecord record = new KinesisRecord(data, partitionKey, sequenceNumber, clock.instant());shard.addRecord(record);persistStream(accountId, key, current);return new PutRecordResult(sequenceNumber, shard.getShardId());@@ -799,12 +814,14 @@ public class KinesisService implements ResourceProvider {// Format: streamName|shardId|type|sequenceNumber|index|timestampMillis// The 6th slot was added for AT_TIMESTAMP; empty for other iterator types.// Old 5-part iterators still decode via split(-1) compatibility in getRecords.- // For LATEST the index slot carries the shard tip at iterator creation time,- // so records written afterwards are visible to getRecords.+ // For LATEST the sequenceNumber slot carries the shard tip's sequence number at+ // iterator creation time (see resolveIteratorSequenceNumber), so records written+ // afterwards are visible to getRecords, and retention pruning of older records+ // can never shift what this iterator resolves to.String raw = String.format("%s|%s|%s|%s|%d|%s",streamName, shardId, type,- sequenceNumber != null ? sequenceNumber : "",- iteratorStartIndex(stream, shardId, type),+ resolveIteratorSequenceNumber(stream, shardId, type, sequenceNumber),+ 0,timestampMillis != null ? timestampMillis.toString() : "");return Base64.getEncoder().encodeToString(raw.getBytes(StandardCharsets.UTF_8));}@@ -826,14 +843,29 @@ public class KinesisService implements ResourceProvider {return parts;}- private int iteratorStartIndex(KinesisStream stream, String shardId, String type) {- // AWS validates the shard at GetShardIterator time for every iterator type.+ /**+ * The sequence-number reference encoded into a freshly-minted shard iterator. For LATEST this+ * is the shard's current tip record's sequence number (or {@code ""} if the shard has no+ * records yet); getRecords resolves it with the same sequence-number lookup used for+ * AFTER_SEQUENCE_NUMBER. A raw record-count/index snapshot would be invalidated by+ * retention pruning removing earlier records and shifting every later record's index; a+ * sequence number is stable under pruning since sequence numbers are never reused. Every other+ * iterator type passes its caller-supplied sequence number (or {@code ""}) straight through.+ *+ * <p>AWS validates the shard at GetShardIterator time for every iterator type, which the shard+ * lookup below still performs regardless of type.+ */+ private String resolveIteratorSequenceNumber(KinesisStream stream, String shardId, String type,+ String sequenceNumber) {KinesisShard shard = stream.getShards().stream().filter(s -> s.getShardId().equals(shardId)).findFirst().orElseThrow(() -> new AwsException("ResourceNotFoundException", "Shard not found", 400));- // LATEST resumes from the tip snapshot taken now; other types resolve in getRecords.- return "LATEST".equals(type) ? shard.recordCount() : 0;+ if (!"LATEST".equals(type)) {+ return sequenceNumber != null ? sequenceNumber : "";+ }+ List<KinesisRecord> records = shard.getRecords();+ return records.isEmpty() ? "" : records.get(records.size() - 1).getSequenceNumber();}private int parseIteratorIndex(String value) {@@ -866,15 +898,23 @@ public class KinesisService implements ResourceProvider {.filter(s -> s.getShardId().equals(shardId)).findFirst().orElseThrow(() -> new AwsException("ResourceNotFoundException", "Shard not found", 400));+ pruneExpiredRecords(stream, shard);- List<KinesisRecord> allRecords = shard.getRecords();+ KinesisShard.RecordsSnapshot snapshot = shard.snapshotRecords();+ List<KinesisRecord> allRecords = snapshot.records();int startIndex = 0;// Simple implementation of iterator types.- // LATEST resumes from the shard tip snapshot encoded at GetShardIterator time,- // so records appended after the iterator was obtained are returned.- if ("TRIM_HORIZON".equals(type) || "LATEST".equals(type)) {- startIndex = lastIndex;+ // LATEST resolves the same way as AFTER_SEQUENCE_NUMBER: it was encoded (at+ // GetShardIterator time) as the shard tip's sequence number, so records appended after the+ // iterator was obtained are returned. Resolving by sequence number rather than the shard's+ // raw index/count means retention pruning of earlier records (which shifts+ // every later record's array index) can never invalidate this reference: if the+ // referenced record has itself since been pruned, every record before it was necessarily+ // pruned too (pruning only ever removes a contiguous prefix), so falling through to the+ // startIndex=0 default below still correctly resumes at the oldest still-retained record.+ if ("TRIM_HORIZON".equals(type)) {+ startIndex = legacyResumeIndex(snapshot, lastIndex);} else if ("AT_SEQUENCE_NUMBER".equals(type)) {for (int i = 0; i < allRecords.size(); i++) {if (allRecords.get(i).getSequenceNumber().equals(startSeq)) {@@ -882,7 +922,7 @@ public class KinesisService implements ResourceProvider {break;}}- } else if ("AFTER_SEQUENCE_NUMBER".equals(type)) {+ } else if ("AFTER_SEQUENCE_NUMBER".equals(type) || "LATEST".equals(type)) {for (int i = 0; i < allRecords.size(); i++) {if (allRecords.get(i).getSequenceNumber().equals(startSeq)) {startIndex = i + 1;@@ -914,12 +954,7 @@ public class KinesisService implements ResourceProvider {nextIndex = i + 1;}- // Continuation iterator: type=TRIM_HORIZON + resume-at-nextIndex is the existing- // "resume by index" convention (the type label is misleading but preserved for compat).- // Timestamp slot empty on continuation.- String nextIterator = Base64.getEncoder().encodeToString(- String.format("%s|%s|%s|%s|%d|", streamName, shardId, "TRIM_HORIZON", "", nextIndex)- .getBytes(StandardCharsets.UTF_8));+ String nextIterator = buildContinuationIterator(streamName, shardId, allRecords, nextIndex);Map<String, Object> response = new HashMap<>();response.put("Records", result);@@ -958,12 +993,15 @@ public class KinesisService implements ResourceProvider {PriorityQueue<PeekedRecord> newest = new PriorityQueue<>(oldestFirst);stream.getShards().stream().filter(shard -> shardId == null || shardId.isBlank() || shard.getShardId().equals(shardId))- .forEach(shard -> shard.getRecords().forEach(record -> {- newest.add(new PeekedRecord(shard.getShardId(), record));- if (newest.size() > resolvedLimit) {- newest.poll();- }- }));+ .forEach(shard -> {+ pruneExpiredRecords(stream, shard);+ shard.getRecords().forEach(record -> {+ newest.add(new PeekedRecord(shard.getShardId(), record));+ if (newest.size() > resolvedLimit) {+ newest.poll();+ }+ });+ });return newest.stream().sorted(Comparator.comparing(peeked -> peeked.record().getApproximateArrivalTimestamp(),@@ -999,6 +1037,51 @@ public class KinesisService implements ResourceProvider {return Math.max(0L, tip.toEpochMilli() - lastReturned.toEpochMilli());}+ /**+ * Earlier versions encoded a continuation as TRIM_HORIZON plus a raw index into the shard's+ * log, and Firehose persists such iterators as checkpoints. Pruning shifts that index, so+ * subtract the records pruned since.+ */+ private static int legacyResumeIndex(KinesisShard.RecordsSnapshot snapshot, int index) {+ long resumeIndex = Math.max(0, index - snapshot.prunedRecordCount());+ return (int) Math.min(resumeIndex, snapshot.records().size());+ }++ /**+ * Builds the {@code NextShardIterator} a GetRecords call returns, positioned just after the+ * last record delivered (or at the log's start if none were). Encoded as+ * AFTER_SEQUENCE_NUMBER against the sequence number of the last-returned record (or, if+ * nothing has been returned yet, the original TRIM_HORIZON/index-0 convention) rather than a+ * raw resume index: retention pruning removes a prefix of the shard's record+ * list, which would silently shift a raw index, but never invalidates a sequence number.+ */+ private String buildContinuationIterator(String streamName, String shardId,+ List<KinesisRecord> allRecords, int nextIndex) {+ if (nextIndex <= 0) {+ return Base64.getEncoder().encodeToString(+ String.format("%s|%s|%s|%s|%d|", streamName, shardId, "TRIM_HORIZON", "", 0)+ .getBytes(StandardCharsets.UTF_8));+ }+ String resumeAfterSequenceNumber = allRecords.get(nextIndex - 1).getSequenceNumber();+ return Base64.getEncoder().encodeToString(+ String.format("%s|%s|%s|%s|%d|", streamName, shardId, "AFTER_SEQUENCE_NUMBER",+ resumeAfterSequenceNumber, 0)+ .getBytes(StandardCharsets.UTF_8));+ }++ /**+ * Lazily expires records older than {@code stream}'s current retention period from+ * {@code shard}, per the documented Kinesis retention-period contract: records past the+ * retention window become inaccessible, and decreasing the retention period makes+ * newly-out-of-window records inaccessible almost immediately. There is no background sweep+ * (YAGNI): every put and read touches only the shard(s) it actually visits, so pruning happens+ * on that shard's next put or read after the record ages out.+ */+ private void pruneExpiredRecords(KinesisStream stream, KinesisShard shard) {+ Instant cutoff = clock.instant().minus(Duration.ofHours(stream.getRetentionPeriodHours()));+ shard.pruneRecordsBefore(cutoff);+ }+private KinesisStream resolveStream(String streamName, String region) {return store.get(regionKey(region, streamName)).orElseThrow(() -> new AwsException("ResourceNotFoundException", "Stream " + streamName + " not found", 400));@@ -1043,8 +1126,8 @@ public class KinesisService implements ResourceProvider {KinesisStream stream = resolveStreamForAccount(accountId, streamName, region);String raw = String.format("%s|%s|%s|%s|%d|%s",streamName, shardId, type,- sequenceNumber != null ? sequenceNumber : "",- iteratorStartIndex(stream, shardId, type),+ resolveIteratorSequenceNumber(stream, shardId, type, sequenceNumber),+ 0,timestampMillis != null ? timestampMillis.toString() : "");return Base64.getEncoder().encodeToString(raw.getBytes(StandardCharsets.UTF_8));}@@ -1071,13 +1154,17 @@ public class KinesisService implements ResourceProvider {.filter(s -> s.getShardId().equals(shardId)).findFirst().orElseThrow(() -> new AwsException("ResourceNotFoundException", "Shard not found", 400));+ pruneExpiredRecords(stream, shard);- List<KinesisRecord> allRecords = shard.getRecords();+ KinesisShard.RecordsSnapshot snapshot = shard.snapshotRecords();+ List<KinesisRecord> allRecords = snapshot.records();int startIndex = 0;- // LATEST resumes from the shard tip snapshot encoded at GetShardIterator time.- if ("TRIM_HORIZON".equals(type) || "LATEST".equals(type)) {- startIndex = lastIndex;- } else if ("AFTER_SEQUENCE_NUMBER".equals(type)) {+ // LATEST resolves the same way as AFTER_SEQUENCE_NUMBER (see the longer explanation in+ // getRecords, above): it was encoded at GetShardIterator time as the shard tip's sequence+ // number, which retention pruning of earlier records can never invalidate.+ if ("TRIM_HORIZON".equals(type)) {+ startIndex = legacyResumeIndex(snapshot, lastIndex);+ } else if ("AFTER_SEQUENCE_NUMBER".equals(type) || "LATEST".equals(type)) {for (int i = 0; i < allRecords.size(); i++) {if (allRecords.get(i).getSequenceNumber().equals(startSeq)) {startIndex = i + 1;@@ -1107,9 +1194,7 @@ public class KinesisService implements ResourceProvider {nextIndex = i + 1;}- String nextIterator = Base64.getEncoder().encodeToString(- String.format("%s|%s|%s|%s|%d|", streamName, shardId, "TRIM_HORIZON", "", nextIndex)- .getBytes(StandardCharsets.UTF_8));+ String nextIterator = buildContinuationIterator(streamName, shardId, allRecords, nextIndex);Map<String, Object> response = new HashMap<>();response.put("Records", result);response.put("NextShardIterator", nextIterator);diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/model/KinesisShard.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/model/KinesisShard.javaindex 58a8aa4963..6395618278 100644--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/model/KinesisShard.java+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/model/KinesisShard.java@@ -17,6 +17,7 @@ public class KinesisShard {private SequenceNumberRange sequenceNumberRange;private final Object recordsMonitor = new Object();private List<KinesisRecord> records = new ArrayList<>();+ private long prunedRecordCount;private boolean closed = false;private Instant creationTimestamp = Instant.now();@@ -55,6 +56,20 @@ public class KinesisShard {}}+ /**+ * Records the shard held at one instant, with the number of records pruned before them.+ * Both values are read under one lock, so a concurrent prune cannot leave the count ahead of+ * the snapshot.+ */+ public RecordsSnapshot snapshotRecords() {+ synchronized (recordsMonitor) {+ return new RecordsSnapshot(new ArrayList<>(records), prunedRecordCount);+ }+ }++ /** A shard's records and the count of records pruned before the first of them. */+ public record RecordsSnapshot(List<KinesisRecord> records, long prunedRecordCount) {}+/** Appends a record. The sole production mutation path for a shard's log. */public void addRecord(KinesisRecord record) {synchronized (recordsMonitor) {@@ -69,6 +84,45 @@ public class KinesisShard {}}+ /**+ * Drops every record whose {@link KinesisRecord#getApproximateArrivalTimestamp()} is strictly+ * before {@code cutoff}, freeing the underlying storage rather than merely hiding them from+ * scans (bounded memory, per the Kinesis retention-period contract). Records are always+ * appended in arrival order, so expired records are a contiguous prefix of the log and this is+ * a single prefix trim, not a general filter. A record with a {@code null} timestamp is never+ * pruned (defensive: it should not occur in practice, but pruning it would be unverifiable).+ */+ public void pruneRecordsBefore(Instant cutoff) {+ synchronized (recordsMonitor) {+ int firstRetained = 0;+ while (firstRetained < records.size()) {+ Instant arrival = records.get(firstRetained).getApproximateArrivalTimestamp();+ if (arrival == null || !arrival.isBefore(cutoff)) {+ break;+ }+ firstRetained++;+ }+ if (firstRetained > 0) {+ records = new ArrayList<>(records.subList(firstRetained, records.size()));+ prunedRecordCount += firstRetained;+ }+ }+ }++ /** Records pruned from the front of the log since the shard was created. */+ public long getPrunedRecordCount() {+ synchronized (recordsMonitor) {+ return prunedRecordCount;+ }+ }++ /** Jackson rehydration only. */+ public void setPrunedRecordCount(long prunedRecordCount) {+ synchronized (recordsMonitor) {+ this.prunedRecordCount = prunedRecordCount;+ }+ }+/** Jackson rehydration only; not safe for concurrent replacement during live traffic. */public void setRecords(List<KinesisRecord> records) {synchronized (recordsMonitor) {diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.javaindex 5659acce54..31ce69432f 100644--- a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java@@ -15,7 +15,9 @@ import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import java.math.BigDecimal;+import java.time.Clock;import java.time.Instant;+import java.time.ZoneOffset;import java.util.Base64;import java.util.List;import java.util.Map;@@ -55,6 +57,18 @@ class KinesisJsonHandlerTest {handler = new KinesisJsonHandler(service, MAPPER, 300_000);}+ // Retention is measured against the service clock, so a test that plants a fixed arrival+ // timestamp pins the clock just after it instead of relying on the wall clock.+ private void useClockFixedAt(Instant now) {+ service = new KinesisService(+ new InMemoryStorage<>(),+ new InMemoryStorage<>(),+ new RegionResolver(REGION, ACCOUNT),+ Clock.fixed(now, ZoneOffset.UTC)+ );+ handler = new KinesisJsonHandler(service, MAPPER, 300_000);+ }+private void createStream(String name) {createStream(name, 1);}@@ -780,6 +794,7 @@ class KinesisJsonHandlerTest {@Testvoid getRecordsSerializesApproximateArrivalTimestampAsPlainDecimal() throws Exception {+ useClockFixedAt(Instant.ofEpochMilli(1_786_959_660_000L));createStream("test-stream");ObjectNode putReq = MAPPER.createObjectNode();@@ -821,6 +836,7 @@ class KinesisJsonHandlerTest {@Testvoid getRecordsWholeSecondArrivalTimestampRemainsNumeric() throws Exception {+ useClockFixedAt(Instant.ofEpochMilli(1_786_959_660_000L));createStream("test-stream");ObjectNode putReq = MAPPER.createObjectNode();diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisServiceTest.javaindex 5ae6df5389..6c43f810e0 100644--- a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisServiceTest.java@@ -8,6 +8,7 @@ import io.github.hectorvent.floci.services.kinesis.model.KinesisConsumer;import io.github.hectorvent.floci.services.kinesis.model.KinesisRecord;import io.github.hectorvent.floci.services.kinesis.model.KinesisShard;import io.github.hectorvent.floci.services.kinesis.model.KinesisStream;+import io.github.hectorvent.floci.testing.MutableClock;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.RepeatedTest;import org.junit.jupiter.api.Test;@@ -17,8 +18,12 @@ import org.junit.jupiter.params.provider.MethodSource;import java.math.BigInteger;import java.nio.charset.StandardCharsets;+import java.time.Clock;+import java.time.Duration;import java.time.Instant;+import java.time.ZoneOffset;import java.util.ArrayList;+import java.util.Base64;import java.util.Collections;import java.util.Comparator;import java.util.HashSet;@@ -205,22 +210,24 @@ class KinesisServiceTest {@Testvoid millisBehindLatestIsTimeDeltaWhenBatchLimitHit() {- kinesisService.createStream("my-stream", 1, REGION);- kinesisService.putRecord("my-stream", "a".getBytes(StandardCharsets.UTF_8), "pk", REGION);- kinesisService.putRecord("my-stream", "b".getBytes(StandardCharsets.UTF_8), "pk", REGION);- kinesisService.putRecord("my-stream", "c".getBytes(StandardCharsets.UTF_8), "pk", REGION);+ Instant base = Instant.parse("2026-01-01T00:00:00Z");+ KinesisService service = new KinesisService(new InMemoryStorage<>(), new InMemoryStorage<>(),+ new RegionResolver(REGION, "000000000000"), Clock.fixed(base.plusSeconds(5), ZoneOffset.UTC));+ service.createStream("my-stream", 1, REGION);+ service.putRecord("my-stream", "a".getBytes(StandardCharsets.UTF_8), "pk", REGION);+ service.putRecord("my-stream", "b".getBytes(StandardCharsets.UTF_8), "pk", REGION);+ service.putRecord("my-stream", "c".getBytes(StandardCharsets.UTF_8), "pk", REGION);// Overwrite timestamps so we can assert a deterministic delta.- KinesisShard shard = kinesisService.describeStream("my-stream", REGION).getShards().getFirst();+ KinesisShard shard = service.describeStream("my-stream", REGION).getShards().getFirst();List<KinesisRecord> records = shard.getRecords();- Instant base = Instant.parse("2026-01-01T00:00:00Z");records.get(0).setApproximateArrivalTimestamp(base);records.get(1).setApproximateArrivalTimestamp(base.plusMillis(1500));records.get(2).setApproximateArrivalTimestamp(base.plusMillis(4000));- String iterator = kinesisService.getShardIterator("my-stream", shard.getShardId(), "TRIM_HORIZON", null, REGION);+ String iterator = service.getShardIterator("my-stream", shard.getShardId(), "TRIM_HORIZON", null, REGION);- Map<String, Object> result = kinesisService.getRecords(iterator, 2, REGION);+ Map<String, Object> result = service.getRecords(iterator, 2, REGION);var returned = (List<?>) result.get("Records");assertEquals(2, returned.size());@@ -1279,4 +1286,187 @@ class KinesisServiceTest {assertEquals(24, recreated.getRetentionPeriodHours(),"a recreated stream must start at the default retention period");}++ // ─── Retention: expired records are pruned ──────────────────────────++ @Test+ void trimHorizonExcludesRecordsOlderThanRetentionPeriod() {+ MutableClock clock = new MutableClock();+ KinesisService service = new KinesisService(new InMemoryStorage<>(), new InMemoryStorage<>(),+ new RegionResolver("us-east-1", "000000000000"), clock);+ service.createStream("orders", 1, REGION); // default retention: 24 hours++ service.putRecord("orders", "expired".getBytes(StandardCharsets.UTF_8), "pk", REGION);+ clock.advance(Duration.ofHours(25)); // older than the 24-hour default retention+ service.putRecord("orders", "fresh".getBytes(StandardCharsets.UTF_8), "pk", REGION);++ String shardId = service.describeStream("orders", REGION).getShards().getFirst().getShardId();+ String iterator = service.getShardIterator("orders", shardId, "TRIM_HORIZON", null, REGION);+ @SuppressWarnings("unchecked")+ List<KinesisRecord> records = (List<KinesisRecord>) service.getRecords(iterator, 10, REGION).get("Records");++ List<String> data = records.stream()+ .map(r -> new String(r.getData(), StandardCharsets.UTF_8))+ .toList();+ assertEquals(List.of("fresh"), data,+ "a record older than the retention period must not be returned from TRIM_HORIZON");+ }++ @Test+ void decreaseStreamRetentionPeriodExcludesNowOutOfWindowRecords() {+ MutableClock clock = new MutableClock();+ KinesisService service = new KinesisService(new InMemoryStorage<>(), new InMemoryStorage<>(),+ new RegionResolver("us-east-1", "000000000000"), clock);+ service.createStream("orders", 1, REGION);+ service.increaseStreamRetentionPeriod("orders", 48, REGION);++ service.putRecord("orders", "old".getBytes(StandardCharsets.UTF_8), "pk", REGION);+ clock.advance(Duration.ofHours(30)); // within the 48h retention, but would exceed 24h+ service.putRecord("orders", "recent".getBytes(StandardCharsets.UTF_8), "pk", REGION);++ // Decreasing retention to 24h must make the 30-hour-old record inaccessible "almost+ // immediately" (per the AWS retention-period contract), well before it would otherwise+ // have aged out under the wider 48h window.+ service.decreaseStreamRetentionPeriod("orders", 24, REGION);++ String shardId = service.describeStream("orders", REGION).getShards().getFirst().getShardId();+ String iterator = service.getShardIterator("orders", shardId, "TRIM_HORIZON", null, REGION);+ @SuppressWarnings("unchecked")+ List<KinesisRecord> records = (List<KinesisRecord>) service.getRecords(iterator, 10, REGION).get("Records");++ List<String> data = records.stream()+ .map(r -> new String(r.getData(), StandardCharsets.UTF_8))+ .toList();+ assertEquals(List.of("recent"), data,+ "decreasing retention must exclude records older than the new period");+ }++ @Test+ void putRecordPrunesExpiredRecordsBoundingShardMemory() {+ MutableClock clock = new MutableClock();+ StorageBackend<String, KinesisStream> store = new InMemoryStorage<>();+ KinesisService service = new KinesisService(store, new InMemoryStorage<>(),+ new RegionResolver("us-east-1", "000000000000"), clock);+ service.createStream("orders", 1, REGION); // default retention: 24 hours++ service.putRecord("orders", "expired".getBytes(StandardCharsets.UTF_8), "pk", REGION);+ clock.advance(Duration.ofHours(25)); // older than the 24-hour default retention+ // A producer-only workload (no GetRecords calls at all) must still prune: memory must not+ // grow without bound just because nobody is reading.+ service.putRecord("orders", "fresh".getBytes(StandardCharsets.UTF_8), "pk", REGION);++ KinesisShard shard = store.get(REGION + "::orders").orElseThrow().getShards().getFirst();+ assertEquals(1, shard.getRecords().size(),+ "an expired record must be removed (not merely hidden) on put, so the shard's "+ + "memory stays bounded even without any reads");+ }++ @Test+ void continuationIteratorResumesCorrectlyAfterEarlierRecordsArePruned() {+ MutableClock clock = new MutableClock();+ KinesisService service = new KinesisService(new InMemoryStorage<>(), new InMemoryStorage<>(),+ new RegionResolver("us-east-1", "000000000000"), clock);+ service.createStream("orders", 1, REGION);+ service.increaseStreamRetentionPeriod("orders", 72, REGION);++ service.putRecord("orders", "r0".getBytes(StandardCharsets.UTF_8), "pk", REGION); // age will be 52h+ clock.advance(Duration.ofHours(50));+ service.putRecord("orders", "r1".getBytes(StandardCharsets.UTF_8), "pk", REGION); // age will be 2h+ clock.advance(Duration.ofHours(1));+ service.putRecord("orders", "r2".getBytes(StandardCharsets.UTF_8), "pk", REGION); // age will be 1h+ clock.advance(Duration.ofHours(1));+ service.putRecord("orders", "r3".getBytes(StandardCharsets.UTF_8), "pk", REGION); // age will be 0h++ String shardId = service.describeStream("orders", REGION).getShards().getFirst().getShardId();+ String iterator = service.getShardIterator("orders", shardId, "TRIM_HORIZON", null, REGION);++ // Consume only r0, capturing the continuation iterator while all four records are still+ // present (so it necessarily carries a resume position pointing just past r0).+ Map<String, Object> firstPage = service.getRecords(iterator, 1, REGION);+ @SuppressWarnings("unchecked")+ List<KinesisRecord> firstRecords = (List<KinesisRecord>) firstPage.get("Records");+ assertEquals(List.of("r0"), recordData(firstRecords));+ String continuation = (String) firstPage.get("NextShardIterator");++ // r0 (age 52h) ages out of a decreased 24h retention window; r1 (age 2h) stays retained.+ // This prunes r0 out of the shard's record list, shifting every later record's array index+ // down by one.+ service.decreaseStreamRetentionPeriod("orders", 24, REGION);++ Map<String, Object> secondPage = service.getRecords(continuation, 10, REGION);+ @SuppressWarnings("unchecked")+ List<KinesisRecord> secondRecords = (List<KinesisRecord>) secondPage.get("Records");+ assertEquals(List.of("r1", "r2", "r3"), recordData(secondRecords),+ "a continuation iterator must resume by sequence number, not a raw array index "+ + "that pruning has shifted, or it will skip or re-skip retained records");+ }++ @Test+ void indexContinuationIteratorFromAnEarlierVersionResumesAfterPruning() {+ MutableClock clock = new MutableClock();+ KinesisService service = new KinesisService(new InMemoryStorage<>(), new InMemoryStorage<>(),+ new RegionResolver("us-east-1", "000000000000"), clock);+ service.createStream("orders", 1, REGION);+ service.putRecord("orders", "r0".getBytes(StandardCharsets.UTF_8), "pk", REGION);+ service.putRecord("orders", "r1".getBytes(StandardCharsets.UTF_8), "pk", REGION);+ clock.advance(Duration.ofHours(23));+ service.putRecord("orders", "r2".getBytes(StandardCharsets.UTF_8), "pk", REGION);+ service.putRecord("orders", "r3".getBytes(StandardCharsets.UTF_8), "pk", REGION);+ String shardId = service.describeStream("orders", REGION).getShards().getFirst().getShardId();+ // Earlier versions encoded a continuation as TRIM_HORIZON plus the index of the next record;+ // Firehose persists such iterators as checkpoints. This one was issued after reading r0..r2.+ String legacyIterator = Base64.getEncoder().encodeToString(+ ("orders|" + shardId + "|TRIM_HORIZON||3|").getBytes(StandardCharsets.UTF_8));++ clock.advance(Duration.ofHours(2)); // r0 and r1 are now past the 24-hour retention++ @SuppressWarnings("unchecked")+ List<KinesisRecord> records = (List<KinesisRecord>) service.getRecords(legacyIterator, 10, REGION).get("Records");+ assertEquals(List.of("r3"), recordData(records));+ }++ @Test+ void peekRecordsExcludesRecordsOlderThanRetentionPeriod() {+ MutableClock clock = new MutableClock();+ KinesisService service = new KinesisService(new InMemoryStorage<>(), new InMemoryStorage<>(),+ new RegionResolver("us-east-1", "000000000000"), clock);+ service.createStream("orders", 1, REGION); // default retention: 24 hours++ service.putRecord("orders", "old".getBytes(StandardCharsets.UTF_8), "pk", REGION);+ // No further put after this: only the read path below can prune, isolating peekRecords'+ // own responsibility instead of piggybacking on the write-path pruning a later put would+ // trigger on the same shard.+ clock.advance(Duration.ofHours(25)); // older than the 24-hour default retention++ List<KinesisService.PeekedRecord> peeked = service.peekRecords("orders", null, 10, REGION);++ assertEquals(List.of(), peeked,+ "peekRecords must not surface a record older than the retention period");+ }++ @Test+ void getRecordsForAccountExcludesRecordsOlderThanRetentionPeriod() {+ MutableClock clock = new MutableClock();+ KinesisService service = new KinesisService(new InMemoryStorage<>(), new InMemoryStorage<>(),+ new RegionResolver("us-east-1", "000000000000"), clock);+ service.createStream("orders", 1, REGION); // default retention: 24 hours+ String shardId = service.describeStream("orders", REGION).getShards().getFirst().getShardId();+ String iterator = service.getShardIteratorForAccount(null, "orders", shardId, "TRIM_HORIZON", null, REGION);++ service.putRecord("orders", "old".getBytes(StandardCharsets.UTF_8), "pk", REGION);+ // No further put after this: only getRecordsForAccount's own read-path pruning is under+ // test here, not a later put's write-path pruning on the same shard.+ clock.advance(Duration.ofHours(25)); // older than the 24-hour default retention++ @SuppressWarnings("unchecked")+ List<KinesisRecord> records =+ (List<KinesisRecord>) service.getRecordsForAccount(null, iterator, 10, REGION).get("Records");++ assertEquals(List.of(), records,+ "getRecordsForAccount must not return a record older than the retention period");+ }++ private static List<String> recordData(List<KinesisRecord> records) {+ return records.stream().map(r -> new String(r.getData(), StandardCharsets.UTF_8)).toList();+ }}