Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(kinesis): expire shard records past the stream's retention period (#4224)

Niels Kootstraaddee350ba56
diff --git a/docs/services/kinesis.md b/docs/services/kinesis.md
index 18c96fb3a2..3870e40bdb 100644
--- a/docs/services/kinesis.md
+++ b/docs/services/kinesis.md
@@ -69,6 +69,10 @@ aws kinesis describe-stream --stream-arn arn:aws:kinesis:us-east-1:000000000000:
A `LATEST` iterator is positioned at the shard tip at the moment the iterator is created, matching AWS: records written after the iterator was obtained are returned, records written before are not. This supports the standard tailing pattern: obtain a `LATEST` iterator, trigger the action that produces the record, then poll `GetRecords` following `NextShardIterator`.
+## Record Retention
+
+A stream's retention period (default 24 hours, `IncreaseStreamRetentionPeriod`/`DecreaseStreamRetentionPeriod` up to 8760 hours) is enforced: a record older than the current retention period is not returned by `GetRecords` or the `GET /_aws/kinesis/records` inspection endpoint, and is actually removed from memory rather than merely hidden. Expiry is checked lazily on each stream's next `PutRecord`/`PutRecords` or read after a record ages out; there is no scheduled sweep. After `DecreaseStreamRetentionPeriod`, records outside the new window are inaccessible from the next put or read, in line with AWS making them inaccessible almost immediately. Shard iterators (including `NextShardIterator` continuation tokens and `LATEST`) resolve by sequence number, so they keep working correctly across pruning even if the exact record they were positioned at has since expired.
+
## Record Routing
`PutRecord` and `PutRecords` honor `ExplicitHashKey` when it is provided. The value must be a decimal integer in the Kinesis hash-key space, and records are written to the open shard whose `HashKeyRange` contains that value. Without `ExplicitHashKey`, Floci keeps using the partition key to choose a shard.
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
index 25205598cf..137aae6424 100644
--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
@@ -85,8 +85,10 @@ public class FirehoseService implements ResourceProvider {
// DeliveryStartTimestamp and re-deliver the source stream's whole retained history to
// S3. A Kinesis shard iterator is a self-describing token
// (streamName|shardId|type|sequenceNumber|index|timestamp, base64) with no
- // JVM-lifetime component, and KinesisService never trims a shard's record list, so a
- // checkpoint written before a restart still resolves to the same position after one.
+ // JVM-lifetime component, and KinesisService resolves it by sequence number, so a
+ // checkpoint written before a restart still resolves to the same position after one,
+ // even once retention has pruned older records. If the checkpointed record itself has
+ // expired, reading resumes at the oldest retained record.
//
// This store holds only COMMITTED positions: a shard is advanced here once the
// records read up to that point have actually landed in S3. See
diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java
index 0ce6b79f3a..6c5e5c5a47 100644
--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/KinesisService.java
@@ -16,6 +16,7 @@ import org.jboss.logging.Logger;
import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
+import java.time.Clock;
import java.time.Duration;
import java.time.Instant;
import java.util.*;
@@ -63,6 +64,7 @@ public class KinesisService implements ResourceProvider {
private final StorageBackend<String, KinesisStream> store;
private final StorageBackend<String, KinesisConsumer> consumerStore;
private final RegionResolver regionResolver;
+ private final Clock clock;
private final AtomicLong sequenceGenerator = new AtomicLong(System.currentTimeMillis());
// One monitor per stream key, handed out by lockFor. Serializes sequence allocation + shard
// append + persist so the in-memory log, the assigned sequence order and the WAL write order
@@ -89,9 +91,21 @@ public class KinesisService implements ResourceProvider {
KinesisService(StorageBackend<String, KinesisStream> store,
StorageBackend<String, KinesisConsumer> consumerStore,
RegionResolver regionResolver) {
+ this(store, consumerStore, regionResolver, Clock.systemUTC());
+ }
+
+ /**
+ * Test seam: lets retention-period expiry be exercised deterministically instead
+ * of depending on wall-clock time. Production code always goes through the public/{@code @Inject}
+ * constructors, which pin the system clock.
+ */
+ KinesisService(StorageBackend<String, KinesisStream> store,
+ StorageBackend<String, KinesisConsumer> consumerStore,
+ RegionResolver regionResolver, Clock clock) {
this.store = store;
this.consumerStore = consumerStore;
this.regionResolver = regionResolver;
+ this.clock = clock;
}
public KinesisStream createStream(String streamName, int shardCount, String region) {
@@ -772,9 +786,10 @@ public class KinesisService implements ResourceProvider {
validateExplicitHashKey(explicitHashKey);
validateRecordSize(current, data, partitionKey);
KinesisShard shard = selectShard(current, partitionKey, explicitHashKey);
+ pruneExpiredRecords(current, shard);
String sequenceNumber = String.valueOf(sequenceGenerator.incrementAndGet());
putRecordAppendHook.run();
- KinesisRecord record = new KinesisRecord(data, partitionKey, sequenceNumber, Instant.now());
+ KinesisRecord record = new KinesisRecord(data, partitionKey, sequenceNumber, clock.instant());
shard.addRecord(record);
persistStream(accountId, key, current);
return new PutRecordResult(sequenceNumber, shard.getShardId());
@@ -799,12 +814,14 @@ public class KinesisService implements ResourceProvider {
// Format: streamName|shardId|type|sequenceNumber|index|timestampMillis
// The 6th slot was added for AT_TIMESTAMP; empty for other iterator types.
// Old 5-part iterators still decode via split(-1) compatibility in getRecords.
- // For LATEST the index slot carries the shard tip at iterator creation time,
- // so records written afterwards are visible to getRecords.
+ // For LATEST the sequenceNumber slot carries the shard tip's sequence number at
+ // iterator creation time (see resolveIteratorSequenceNumber), so records written
+ // afterwards are visible to getRecords, and retention pruning of older records
+ // can never shift what this iterator resolves to.
String raw = String.format("%s|%s|%s|%s|%d|%s",
streamName, shardId, type,
- sequenceNumber != null ? sequenceNumber : "",
- iteratorStartIndex(stream, shardId, type),
+ resolveIteratorSequenceNumber(stream, shardId, type, sequenceNumber),
+ 0,
timestampMillis != null ? timestampMillis.toString() : "");
return Base64.getEncoder().encodeToString(raw.getBytes(StandardCharsets.UTF_8));
}
@@ -826,14 +843,29 @@ public class KinesisService implements ResourceProvider {
return parts;
}
- private int iteratorStartIndex(KinesisStream stream, String shardId, String type) {
- // AWS validates the shard at GetShardIterator time for every iterator type.
+ /**
+ * The sequence-number reference encoded into a freshly-minted shard iterator. For LATEST this
+ * is the shard's current tip record's sequence number (or {@code ""} if the shard has no
+ * records yet); getRecords resolves it with the same sequence-number lookup used for
+ * AFTER_SEQUENCE_NUMBER. A raw record-count/index snapshot would be invalidated by
+ * retention pruning removing earlier records and shifting every later record's index; a
+ * sequence number is stable under pruning since sequence numbers are never reused. Every other
+ * iterator type passes its caller-supplied sequence number (or {@code ""}) straight through.
+ *
+ * <p>AWS validates the shard at GetShardIterator time for every iterator type, which the shard
+ * lookup below still performs regardless of type.
+ */
+ private String resolveIteratorSequenceNumber(KinesisStream stream, String shardId, String type,
+ String sequenceNumber) {
KinesisShard shard = stream.getShards().stream()
.filter(s -> s.getShardId().equals(shardId))
.findFirst()
.orElseThrow(() -> new AwsException("ResourceNotFoundException", "Shard not found", 400));
- // LATEST resumes from the tip snapshot taken now; other types resolve in getRecords.
- return "LATEST".equals(type) ? shard.recordCount() : 0;
+ if (!"LATEST".equals(type)) {
+ return sequenceNumber != null ? sequenceNumber : "";
+ }
+ List<KinesisRecord> records = shard.getRecords();
+ return records.isEmpty() ? "" : records.get(records.size() - 1).getSequenceNumber();
}
private int parseIteratorIndex(String value) {
@@ -866,15 +898,23 @@ public class KinesisService implements ResourceProvider {
.filter(s -> s.getShardId().equals(shardId))
.findFirst()
.orElseThrow(() -> new AwsException("ResourceNotFoundException", "Shard not found", 400));
+ pruneExpiredRecords(stream, shard);
- List<KinesisRecord> allRecords = shard.getRecords();
+ KinesisShard.RecordsSnapshot snapshot = shard.snapshotRecords();
+ List<KinesisRecord> allRecords = snapshot.records();
int startIndex = 0;
// Simple implementation of iterator types.
- // LATEST resumes from the shard tip snapshot encoded at GetShardIterator time,
- // so records appended after the iterator was obtained are returned.
- if ("TRIM_HORIZON".equals(type) || "LATEST".equals(type)) {
- startIndex = lastIndex;
+ // LATEST resolves the same way as AFTER_SEQUENCE_NUMBER: it was encoded (at
+ // GetShardIterator time) as the shard tip's sequence number, so records appended after the
+ // iterator was obtained are returned. Resolving by sequence number rather than the shard's
+ // raw index/count means retention pruning of earlier records (which shifts
+ // every later record's array index) can never invalidate this reference: if the
+ // referenced record has itself since been pruned, every record before it was necessarily
+ // pruned too (pruning only ever removes a contiguous prefix), so falling through to the
+ // startIndex=0 default below still correctly resumes at the oldest still-retained record.
+ if ("TRIM_HORIZON".equals(type)) {
+ startIndex = legacyResumeIndex(snapshot, lastIndex);
} else if ("AT_SEQUENCE_NUMBER".equals(type)) {
for (int i = 0; i < allRecords.size(); i++) {
if (allRecords.get(i).getSequenceNumber().equals(startSeq)) {
@@ -882,7 +922,7 @@ public class KinesisService implements ResourceProvider {
break;
}
}
- } else if ("AFTER_SEQUENCE_NUMBER".equals(type)) {
+ } else if ("AFTER_SEQUENCE_NUMBER".equals(type) || "LATEST".equals(type)) {
for (int i = 0; i < allRecords.size(); i++) {
if (allRecords.get(i).getSequenceNumber().equals(startSeq)) {
startIndex = i + 1;
@@ -914,12 +954,7 @@ public class KinesisService implements ResourceProvider {
nextIndex = i + 1;
}
- // Continuation iterator: type=TRIM_HORIZON + resume-at-nextIndex is the existing
- // "resume by index" convention (the type label is misleading but preserved for compat).
- // Timestamp slot empty on continuation.
- String nextIterator = Base64.getEncoder().encodeToString(
- String.format("%s|%s|%s|%s|%d|", streamName, shardId, "TRIM_HORIZON", "", nextIndex)
- .getBytes(StandardCharsets.UTF_8));
+ String nextIterator = buildContinuationIterator(streamName, shardId, allRecords, nextIndex);
Map<String, Object> response = new HashMap<>();
response.put("Records", result);
@@ -958,12 +993,15 @@ public class KinesisService implements ResourceProvider {
PriorityQueue<PeekedRecord> newest = new PriorityQueue<>(oldestFirst);
stream.getShards().stream()
.filter(shard -> shardId == null || shardId.isBlank() || shard.getShardId().equals(shardId))
- .forEach(shard -> shard.getRecords().forEach(record -> {
- newest.add(new PeekedRecord(shard.getShardId(), record));
- if (newest.size() > resolvedLimit) {
- newest.poll();
- }
- }));
+ .forEach(shard -> {
+ pruneExpiredRecords(stream, shard);
+ shard.getRecords().forEach(record -> {
+ newest.add(new PeekedRecord(shard.getShardId(), record));
+ if (newest.size() > resolvedLimit) {
+ newest.poll();
+ }
+ });
+ });
return newest.stream()
.sorted(Comparator.comparing(peeked -> peeked.record().getApproximateArrivalTimestamp(),
@@ -999,6 +1037,51 @@ public class KinesisService implements ResourceProvider {
return Math.max(0L, tip.toEpochMilli() - lastReturned.toEpochMilli());
}
+ /**
+ * Earlier versions encoded a continuation as TRIM_HORIZON plus a raw index into the shard's
+ * log, and Firehose persists such iterators as checkpoints. Pruning shifts that index, so
+ * subtract the records pruned since.
+ */
+ private static int legacyResumeIndex(KinesisShard.RecordsSnapshot snapshot, int index) {
+ long resumeIndex = Math.max(0, index - snapshot.prunedRecordCount());
+ return (int) Math.min(resumeIndex, snapshot.records().size());
+ }
+
+ /**
+ * Builds the {@code NextShardIterator} a GetRecords call returns, positioned just after the
+ * last record delivered (or at the log's start if none were). Encoded as
+ * AFTER_SEQUENCE_NUMBER against the sequence number of the last-returned record (or, if
+ * nothing has been returned yet, the original TRIM_HORIZON/index-0 convention) rather than a
+ * raw resume index: retention pruning removes a prefix of the shard's record
+ * list, which would silently shift a raw index, but never invalidates a sequence number.
+ */
+ private String buildContinuationIterator(String streamName, String shardId,
+ List<KinesisRecord> allRecords, int nextIndex) {
+ if (nextIndex <= 0) {
+ return Base64.getEncoder().encodeToString(
+ String.format("%s|%s|%s|%s|%d|", streamName, shardId, "TRIM_HORIZON", "", 0)
+ .getBytes(StandardCharsets.UTF_8));
+ }
+ String resumeAfterSequenceNumber = allRecords.get(nextIndex - 1).getSequenceNumber();
+ return Base64.getEncoder().encodeToString(
+ String.format("%s|%s|%s|%s|%d|", streamName, shardId, "AFTER_SEQUENCE_NUMBER",
+ resumeAfterSequenceNumber, 0)
+ .getBytes(StandardCharsets.UTF_8));
+ }
+
+ /**
+ * Lazily expires records older than {@code stream}'s current retention period from
+ * {@code shard}, per the documented Kinesis retention-period contract: records past the
+ * retention window become inaccessible, and decreasing the retention period makes
+ * newly-out-of-window records inaccessible almost immediately. There is no background sweep
+ * (YAGNI): every put and read touches only the shard(s) it actually visits, so pruning happens
+ * on that shard's next put or read after the record ages out.
+ */
+ private void pruneExpiredRecords(KinesisStream stream, KinesisShard shard) {
+ Instant cutoff = clock.instant().minus(Duration.ofHours(stream.getRetentionPeriodHours()));
+ shard.pruneRecordsBefore(cutoff);
+ }
+
private KinesisStream resolveStream(String streamName, String region) {
return store.get(regionKey(region, streamName))
.orElseThrow(() -> new AwsException("ResourceNotFoundException", "Stream " + streamName + " not found", 400));
@@ -1043,8 +1126,8 @@ public class KinesisService implements ResourceProvider {
KinesisStream stream = resolveStreamForAccount(accountId, streamName, region);
String raw = String.format("%s|%s|%s|%s|%d|%s",
streamName, shardId, type,
- sequenceNumber != null ? sequenceNumber : "",
- iteratorStartIndex(stream, shardId, type),
+ resolveIteratorSequenceNumber(stream, shardId, type, sequenceNumber),
+ 0,
timestampMillis != null ? timestampMillis.toString() : "");
return Base64.getEncoder().encodeToString(raw.getBytes(StandardCharsets.UTF_8));
}
@@ -1071,13 +1154,17 @@ public class KinesisService implements ResourceProvider {
.filter(s -> s.getShardId().equals(shardId))
.findFirst()
.orElseThrow(() -> new AwsException("ResourceNotFoundException", "Shard not found", 400));
+ pruneExpiredRecords(stream, shard);
- List<KinesisRecord> allRecords = shard.getRecords();
+ KinesisShard.RecordsSnapshot snapshot = shard.snapshotRecords();
+ List<KinesisRecord> allRecords = snapshot.records();
int startIndex = 0;
- // LATEST resumes from the shard tip snapshot encoded at GetShardIterator time.
- if ("TRIM_HORIZON".equals(type) || "LATEST".equals(type)) {
- startIndex = lastIndex;
- } else if ("AFTER_SEQUENCE_NUMBER".equals(type)) {
+ // LATEST resolves the same way as AFTER_SEQUENCE_NUMBER (see the longer explanation in
+ // getRecords, above): it was encoded at GetShardIterator time as the shard tip's sequence
+ // number, which retention pruning of earlier records can never invalidate.
+ if ("TRIM_HORIZON".equals(type)) {
+ startIndex = legacyResumeIndex(snapshot, lastIndex);
+ } else if ("AFTER_SEQUENCE_NUMBER".equals(type) || "LATEST".equals(type)) {
for (int i = 0; i < allRecords.size(); i++) {
if (allRecords.get(i).getSequenceNumber().equals(startSeq)) {
startIndex = i + 1;
@@ -1107,9 +1194,7 @@ public class KinesisService implements ResourceProvider {
nextIndex = i + 1;
}
- String nextIterator = Base64.getEncoder().encodeToString(
- String.format("%s|%s|%s|%s|%d|", streamName, shardId, "TRIM_HORIZON", "", nextIndex)
- .getBytes(StandardCharsets.UTF_8));
+ String nextIterator = buildContinuationIterator(streamName, shardId, allRecords, nextIndex);
Map<String, Object> response = new HashMap<>();
response.put("Records", result);
response.put("NextShardIterator", nextIterator);
diff --git a/src/main/java/io/github/hectorvent/floci/services/kinesis/model/KinesisShard.java b/src/main/java/io/github/hectorvent/floci/services/kinesis/model/KinesisShard.java
index 58a8aa4963..6395618278 100644
--- a/src/main/java/io/github/hectorvent/floci/services/kinesis/model/KinesisShard.java
+++ b/src/main/java/io/github/hectorvent/floci/services/kinesis/model/KinesisShard.java
@@ -17,6 +17,7 @@ public class KinesisShard {
private SequenceNumberRange sequenceNumberRange;
private final Object recordsMonitor = new Object();
private List<KinesisRecord> records = new ArrayList<>();
+ private long prunedRecordCount;
private boolean closed = false;
private Instant creationTimestamp = Instant.now();
@@ -55,6 +56,20 @@ public class KinesisShard {
}
}
+ /**
+ * Records the shard held at one instant, with the number of records pruned before them.
+ * Both values are read under one lock, so a concurrent prune cannot leave the count ahead of
+ * the snapshot.
+ */
+ public RecordsSnapshot snapshotRecords() {
+ synchronized (recordsMonitor) {
+ return new RecordsSnapshot(new ArrayList<>(records), prunedRecordCount);
+ }
+ }
+
+ /** A shard's records and the count of records pruned before the first of them. */
+ public record RecordsSnapshot(List<KinesisRecord> records, long prunedRecordCount) {}
+
/** Appends a record. The sole production mutation path for a shard's log. */
public void addRecord(KinesisRecord record) {
synchronized (recordsMonitor) {
@@ -69,6 +84,45 @@ public class KinesisShard {
}
}
+ /**
+ * Drops every record whose {@link KinesisRecord#getApproximateArrivalTimestamp()} is strictly
+ * before {@code cutoff}, freeing the underlying storage rather than merely hiding them from
+ * scans (bounded memory, per the Kinesis retention-period contract). Records are always
+ * appended in arrival order, so expired records are a contiguous prefix of the log and this is
+ * a single prefix trim, not a general filter. A record with a {@code null} timestamp is never
+ * pruned (defensive: it should not occur in practice, but pruning it would be unverifiable).
+ */
+ public void pruneRecordsBefore(Instant cutoff) {
+ synchronized (recordsMonitor) {
+ int firstRetained = 0;
+ while (firstRetained < records.size()) {
+ Instant arrival = records.get(firstRetained).getApproximateArrivalTimestamp();
+ if (arrival == null || !arrival.isBefore(cutoff)) {
+ break;
+ }
+ firstRetained++;
+ }
+ if (firstRetained > 0) {
+ records = new ArrayList<>(records.subList(firstRetained, records.size()));
+ prunedRecordCount += firstRetained;
+ }
+ }
+ }
+
+ /** Records pruned from the front of the log since the shard was created. */
+ public long getPrunedRecordCount() {
+ synchronized (recordsMonitor) {
+ return prunedRecordCount;
+ }
+ }
+
+ /** Jackson rehydration only. */
+ public void setPrunedRecordCount(long prunedRecordCount) {
+ synchronized (recordsMonitor) {
+ this.prunedRecordCount = prunedRecordCount;
+ }
+ }
+
/** Jackson rehydration only; not safe for concurrent replacement during live traffic. */
public void setRecords(List<KinesisRecord> records) {
synchronized (recordsMonitor) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java
index 5659acce54..31ce69432f 100644
--- a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisJsonHandlerTest.java
@@ -15,7 +15,9 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import java.math.BigDecimal;
+import java.time.Clock;
import java.time.Instant;
+import java.time.ZoneOffset;
import java.util.Base64;
import java.util.List;
import java.util.Map;
@@ -55,6 +57,18 @@ class KinesisJsonHandlerTest {
handler = new KinesisJsonHandler(service, MAPPER, 300_000);
}
+ // Retention is measured against the service clock, so a test that plants a fixed arrival
+ // timestamp pins the clock just after it instead of relying on the wall clock.
+ private void useClockFixedAt(Instant now) {
+ service = new KinesisService(
+ new InMemoryStorage<>(),
+ new InMemoryStorage<>(),
+ new RegionResolver(REGION, ACCOUNT),
+ Clock.fixed(now, ZoneOffset.UTC)
+ );
+ handler = new KinesisJsonHandler(service, MAPPER, 300_000);
+ }
+
private void createStream(String name) {
createStream(name, 1);
}
@@ -780,6 +794,7 @@ class KinesisJsonHandlerTest {
@Test
void getRecordsSerializesApproximateArrivalTimestampAsPlainDecimal() throws Exception {
+ useClockFixedAt(Instant.ofEpochMilli(1_786_959_660_000L));
createStream("test-stream");
ObjectNode putReq = MAPPER.createObjectNode();
@@ -821,6 +836,7 @@ class KinesisJsonHandlerTest {
@Test
void getRecordsWholeSecondArrivalTimestampRemainsNumeric() throws Exception {
+ useClockFixedAt(Instant.ofEpochMilli(1_786_959_660_000L));
createStream("test-stream");
ObjectNode putReq = MAPPER.createObjectNode();
diff --git a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisServiceTest.java
index 5ae6df5389..6c43f810e0 100644
--- a/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/kinesis/KinesisServiceTest.java
@@ -8,6 +8,7 @@ import io.github.hectorvent.floci.services.kinesis.model.KinesisConsumer;
import io.github.hectorvent.floci.services.kinesis.model.KinesisRecord;
import io.github.hectorvent.floci.services.kinesis.model.KinesisShard;
import io.github.hectorvent.floci.services.kinesis.model.KinesisStream;
+import io.github.hectorvent.floci.testing.MutableClock;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.RepeatedTest;
import org.junit.jupiter.api.Test;
@@ -17,8 +18,12 @@ import org.junit.jupiter.params.provider.MethodSource;
import java.math.BigInteger;
import java.nio.charset.StandardCharsets;
+import java.time.Clock;
+import java.time.Duration;
import java.time.Instant;
+import java.time.ZoneOffset;
import java.util.ArrayList;
+import java.util.Base64;
import java.util.Collections;
import java.util.Comparator;
import java.util.HashSet;
@@ -205,22 +210,24 @@ class KinesisServiceTest {
@Test
void millisBehindLatestIsTimeDeltaWhenBatchLimitHit() {
- kinesisService.createStream("my-stream", 1, REGION);
- kinesisService.putRecord("my-stream", "a".getBytes(StandardCharsets.UTF_8), "pk", REGION);
- kinesisService.putRecord("my-stream", "b".getBytes(StandardCharsets.UTF_8), "pk", REGION);
- kinesisService.putRecord("my-stream", "c".getBytes(StandardCharsets.UTF_8), "pk", REGION);
+ Instant base = Instant.parse("2026-01-01T00:00:00Z");
+ KinesisService service = new KinesisService(new InMemoryStorage<>(), new InMemoryStorage<>(),
+ new RegionResolver(REGION, "000000000000"), Clock.fixed(base.plusSeconds(5), ZoneOffset.UTC));
+ service.createStream("my-stream", 1, REGION);
+ service.putRecord("my-stream", "a".getBytes(StandardCharsets.UTF_8), "pk", REGION);
+ service.putRecord("my-stream", "b".getBytes(StandardCharsets.UTF_8), "pk", REGION);
+ service.putRecord("my-stream", "c".getBytes(StandardCharsets.UTF_8), "pk", REGION);
// Overwrite timestamps so we can assert a deterministic delta.
- KinesisShard shard = kinesisService.describeStream("my-stream", REGION).getShards().getFirst();
+ KinesisShard shard = service.describeStream("my-stream", REGION).getShards().getFirst();
List<KinesisRecord> records = shard.getRecords();
- Instant base = Instant.parse("2026-01-01T00:00:00Z");
records.get(0).setApproximateArrivalTimestamp(base);
records.get(1).setApproximateArrivalTimestamp(base.plusMillis(1500));
records.get(2).setApproximateArrivalTimestamp(base.plusMillis(4000));
- String iterator = kinesisService.getShardIterator("my-stream", shard.getShardId(), "TRIM_HORIZON", null, REGION);
+ String iterator = service.getShardIterator("my-stream", shard.getShardId(), "TRIM_HORIZON", null, REGION);
- Map<String, Object> result = kinesisService.getRecords(iterator, 2, REGION);
+ Map<String, Object> result = service.getRecords(iterator, 2, REGION);
var returned = (List<?>) result.get("Records");
assertEquals(2, returned.size());
@@ -1279,4 +1286,187 @@ class KinesisServiceTest {
assertEquals(24, recreated.getRetentionPeriodHours(),
"a recreated stream must start at the default retention period");
}
+
+ // ─── Retention: expired records are pruned ──────────────────────────
+
+ @Test
+ void trimHorizonExcludesRecordsOlderThanRetentionPeriod() {
+ MutableClock clock = new MutableClock();
+ KinesisService service = new KinesisService(new InMemoryStorage<>(), new InMemoryStorage<>(),
+ new RegionResolver("us-east-1", "000000000000"), clock);
+ service.createStream("orders", 1, REGION); // default retention: 24 hours
+
+ service.putRecord("orders", "expired".getBytes(StandardCharsets.UTF_8), "pk", REGION);
+ clock.advance(Duration.ofHours(25)); // older than the 24-hour default retention
+ service.putRecord("orders", "fresh".getBytes(StandardCharsets.UTF_8), "pk", REGION);
+
+ String shardId = service.describeStream("orders", REGION).getShards().getFirst().getShardId();
+ String iterator = service.getShardIterator("orders", shardId, "TRIM_HORIZON", null, REGION);
+ @SuppressWarnings("unchecked")
+ List<KinesisRecord> records = (List<KinesisRecord>) service.getRecords(iterator, 10, REGION).get("Records");
+
+ List<String> data = records.stream()
+ .map(r -> new String(r.getData(), StandardCharsets.UTF_8))
+ .toList();
+ assertEquals(List.of("fresh"), data,
+ "a record older than the retention period must not be returned from TRIM_HORIZON");
+ }
+
+ @Test
+ void decreaseStreamRetentionPeriodExcludesNowOutOfWindowRecords() {
+ MutableClock clock = new MutableClock();
+ KinesisService service = new KinesisService(new InMemoryStorage<>(), new InMemoryStorage<>(),
+ new RegionResolver("us-east-1", "000000000000"), clock);
+ service.createStream("orders", 1, REGION);
+ service.increaseStreamRetentionPeriod("orders", 48, REGION);
+
+ service.putRecord("orders", "old".getBytes(StandardCharsets.UTF_8), "pk", REGION);
+ clock.advance(Duration.ofHours(30)); // within the 48h retention, but would exceed 24h
+ service.putRecord("orders", "recent".getBytes(StandardCharsets.UTF_8), "pk", REGION);
+
+ // Decreasing retention to 24h must make the 30-hour-old record inaccessible "almost
+ // immediately" (per the AWS retention-period contract), well before it would otherwise
+ // have aged out under the wider 48h window.
+ service.decreaseStreamRetentionPeriod("orders", 24, REGION);
+
+ String shardId = service.describeStream("orders", REGION).getShards().getFirst().getShardId();
+ String iterator = service.getShardIterator("orders", shardId, "TRIM_HORIZON", null, REGION);
+ @SuppressWarnings("unchecked")
+ List<KinesisRecord> records = (List<KinesisRecord>) service.getRecords(iterator, 10, REGION).get("Records");
+
+ List<String> data = records.stream()
+ .map(r -> new String(r.getData(), StandardCharsets.UTF_8))
+ .toList();
+ assertEquals(List.of("recent"), data,
+ "decreasing retention must exclude records older than the new period");
+ }
+
+ @Test
+ void putRecordPrunesExpiredRecordsBoundingShardMemory() {
+ MutableClock clock = new MutableClock();
+ StorageBackend<String, KinesisStream> store = new InMemoryStorage<>();
+ KinesisService service = new KinesisService(store, new InMemoryStorage<>(),
+ new RegionResolver("us-east-1", "000000000000"), clock);
+ service.createStream("orders", 1, REGION); // default retention: 24 hours
+
+ service.putRecord("orders", "expired".getBytes(StandardCharsets.UTF_8), "pk", REGION);
+ clock.advance(Duration.ofHours(25)); // older than the 24-hour default retention
+ // A producer-only workload (no GetRecords calls at all) must still prune: memory must not
+ // grow without bound just because nobody is reading.
+ service.putRecord("orders", "fresh".getBytes(StandardCharsets.UTF_8), "pk", REGION);
+
+ KinesisShard shard = store.get(REGION + "::orders").orElseThrow().getShards().getFirst();
+ assertEquals(1, shard.getRecords().size(),
+ "an expired record must be removed (not merely hidden) on put, so the shard's "
+ + "memory stays bounded even without any reads");
+ }
+
+ @Test
+ void continuationIteratorResumesCorrectlyAfterEarlierRecordsArePruned() {
+ MutableClock clock = new MutableClock();
+ KinesisService service = new KinesisService(new InMemoryStorage<>(), new InMemoryStorage<>(),
+ new RegionResolver("us-east-1", "000000000000"), clock);
+ service.createStream("orders", 1, REGION);
+ service.increaseStreamRetentionPeriod("orders", 72, REGION);
+
+ service.putRecord("orders", "r0".getBytes(StandardCharsets.UTF_8), "pk", REGION); // age will be 52h
+ clock.advance(Duration.ofHours(50));
+ service.putRecord("orders", "r1".getBytes(StandardCharsets.UTF_8), "pk", REGION); // age will be 2h
+ clock.advance(Duration.ofHours(1));
+ service.putRecord("orders", "r2".getBytes(StandardCharsets.UTF_8), "pk", REGION); // age will be 1h
+ clock.advance(Duration.ofHours(1));
+ service.putRecord("orders", "r3".getBytes(StandardCharsets.UTF_8), "pk", REGION); // age will be 0h
+
+ String shardId = service.describeStream("orders", REGION).getShards().getFirst().getShardId();
+ String iterator = service.getShardIterator("orders", shardId, "TRIM_HORIZON", null, REGION);
+
+ // Consume only r0, capturing the continuation iterator while all four records are still
+ // present (so it necessarily carries a resume position pointing just past r0).
+ Map<String, Object> firstPage = service.getRecords(iterator, 1, REGION);
+ @SuppressWarnings("unchecked")
+ List<KinesisRecord> firstRecords = (List<KinesisRecord>) firstPage.get("Records");
+ assertEquals(List.of("r0"), recordData(firstRecords));
+ String continuation = (String) firstPage.get("NextShardIterator");
+
+ // r0 (age 52h) ages out of a decreased 24h retention window; r1 (age 2h) stays retained.
+ // This prunes r0 out of the shard's record list, shifting every later record's array index
+ // down by one.
+ service.decreaseStreamRetentionPeriod("orders", 24, REGION);
+
+ Map<String, Object> secondPage = service.getRecords(continuation, 10, REGION);
+ @SuppressWarnings("unchecked")
+ List<KinesisRecord> secondRecords = (List<KinesisRecord>) secondPage.get("Records");
+ assertEquals(List.of("r1", "r2", "r3"), recordData(secondRecords),
+ "a continuation iterator must resume by sequence number, not a raw array index "
+ + "that pruning has shifted, or it will skip or re-skip retained records");
+ }
+
+ @Test
+ void indexContinuationIteratorFromAnEarlierVersionResumesAfterPruning() {
+ MutableClock clock = new MutableClock();
+ KinesisService service = new KinesisService(new InMemoryStorage<>(), new InMemoryStorage<>(),
+ new RegionResolver("us-east-1", "000000000000"), clock);
+ service.createStream("orders", 1, REGION);
+ service.putRecord("orders", "r0".getBytes(StandardCharsets.UTF_8), "pk", REGION);
+ service.putRecord("orders", "r1".getBytes(StandardCharsets.UTF_8), "pk", REGION);
+ clock.advance(Duration.ofHours(23));
+ service.putRecord("orders", "r2".getBytes(StandardCharsets.UTF_8), "pk", REGION);
+ service.putRecord("orders", "r3".getBytes(StandardCharsets.UTF_8), "pk", REGION);
+ String shardId = service.describeStream("orders", REGION).getShards().getFirst().getShardId();
+ // Earlier versions encoded a continuation as TRIM_HORIZON plus the index of the next record;
+ // Firehose persists such iterators as checkpoints. This one was issued after reading r0..r2.
+ String legacyIterator = Base64.getEncoder().encodeToString(
+ ("orders|" + shardId + "|TRIM_HORIZON||3|").getBytes(StandardCharsets.UTF_8));
+
+ clock.advance(Duration.ofHours(2)); // r0 and r1 are now past the 24-hour retention
+
+ @SuppressWarnings("unchecked")
+ List<KinesisRecord> records = (List<KinesisRecord>) service.getRecords(legacyIterator, 10, REGION).get("Records");
+ assertEquals(List.of("r3"), recordData(records));
+ }
+
+ @Test
+ void peekRecordsExcludesRecordsOlderThanRetentionPeriod() {
+ MutableClock clock = new MutableClock();
+ KinesisService service = new KinesisService(new InMemoryStorage<>(), new InMemoryStorage<>(),
+ new RegionResolver("us-east-1", "000000000000"), clock);
+ service.createStream("orders", 1, REGION); // default retention: 24 hours
+
+ service.putRecord("orders", "old".getBytes(StandardCharsets.UTF_8), "pk", REGION);
+ // No further put after this: only the read path below can prune, isolating peekRecords'
+ // own responsibility instead of piggybacking on the write-path pruning a later put would
+ // trigger on the same shard.
+ clock.advance(Duration.ofHours(25)); // older than the 24-hour default retention
+
+ List<KinesisService.PeekedRecord> peeked = service.peekRecords("orders", null, 10, REGION);
+
+ assertEquals(List.of(), peeked,
+ "peekRecords must not surface a record older than the retention period");
+ }
+
+ @Test
+ void getRecordsForAccountExcludesRecordsOlderThanRetentionPeriod() {
+ MutableClock clock = new MutableClock();
+ KinesisService service = new KinesisService(new InMemoryStorage<>(), new InMemoryStorage<>(),
+ new RegionResolver("us-east-1", "000000000000"), clock);
+ service.createStream("orders", 1, REGION); // default retention: 24 hours
+ String shardId = service.describeStream("orders", REGION).getShards().getFirst().getShardId();
+ String iterator = service.getShardIteratorForAccount(null, "orders", shardId, "TRIM_HORIZON", null, REGION);
+
+ service.putRecord("orders", "old".getBytes(StandardCharsets.UTF_8), "pk", REGION);
+ // No further put after this: only getRecordsForAccount's own read-path pruning is under
+ // test here, not a later put's write-path pruning on the same shard.
+ clock.advance(Duration.ofHours(25)); // older than the 24-hour default retention
+
+ @SuppressWarnings("unchecked")
+ List<KinesisRecord> records =
+ (List<KinesisRecord>) service.getRecordsForAccount(null, iterator, 10, REGION).get("Records");
+
+ assertEquals(List.of(), records,
+ "getRecordsForAccount must not return a record older than the retention period");
+ }
+
+ private static List<String> recordData(List<KinesisRecord> records) {
+ return records.stream().map(r -> new String(r.getData(), StandardCharsets.UTF_8)).toList();
+ }
}