Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(dynamodb): keep stream iterators stable after trimming (#3313)

Niels Kootstrab377134fcbe0
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java
index 6739f1e74e..da844fbd06 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java
@@ -34,6 +34,7 @@ public class DynamoDbStreamService {
public static final String SHARD_ID = "shardId-0000000001-00000000001";
static final int MAX_RECORDS = 1000;
+ private static final String ZERO_SEQUENCE_NUMBER = "000000000000000000000";
private static final DateTimeFormatter STREAM_LABEL_FORMAT =
DateTimeFormatter.ofPattern("yyyy-MM-dd'T'HH:mm:ss.SSS").withZone(ZoneOffset.UTC);
@@ -41,6 +42,7 @@ public class DynamoDbStreamService {
private final ConcurrentHashMap<String, StreamDescription> streams = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, ConcurrentLinkedDeque<DynamoDbStreamRecord>> records =
new ConcurrentHashMap<>();
+ private final ConcurrentHashMap<String, AtomicLong> streamRecordCounts = new ConcurrentHashMap<>();
private final AtomicLong sequenceCounter = new AtomicLong(0);
private final ObjectMapper objectMapper;
@@ -110,6 +112,7 @@ public class DynamoDbStreamService {
streams.put(key, sd);
records.put(streamArn, new ConcurrentLinkedDeque<>());
+ streamRecordCounts.put(streamArn, new AtomicLong());
LOG.infov("Enabled stream for table {0} in region {1}: {2}", tableName, region, streamArn);
return sd;
}
@@ -128,6 +131,7 @@ public class DynamoDbStreamService {
StreamDescription sd = streams.remove(key);
if (sd != null) {
records.remove(sd.getStreamArn());
+ streamRecordCounts.remove(sd.getStreamArn());
LOG.infov("Deleted stream for table {0} in region {1}", tableName, region);
}
}
@@ -166,9 +170,13 @@ public class DynamoDbStreamService {
ConcurrentLinkedDeque<DynamoDbStreamRecord> deque = records.get(sd.getStreamArn());
if (deque != null) {
- deque.addLast(record);
- while (deque.size() > MAX_RECORDS) {
- deque.pollFirst();
+ synchronized (deque) {
+ streamRecordCounts.computeIfAbsent(sd.getStreamArn(), ignored -> new AtomicLong())
+ .incrementAndGet();
+ deque.addLast(record);
+ while (deque.size() > MAX_RECORDS) {
+ deque.pollFirst();
+ }
}
}
}
@@ -233,25 +241,42 @@ public class DynamoDbStreamService {
}
ConcurrentLinkedDeque<DynamoDbStreamRecord> deque = records.get(streamArn);
- List<DynamoDbStreamRecord> snapshot = deque != null ? new ArrayList<>(deque) : List.of();
+ List<DynamoDbStreamRecord> snapshot;
+ long recordCount;
+ if (deque == null) {
+ snapshot = List.of();
+ recordCount = 0;
+ } else {
+ synchronized (deque) {
+ snapshot = new ArrayList<>(deque);
+ recordCount = streamRecordCounts.getOrDefault(streamArn, new AtomicLong()).get();
+ }
+ }
- int position = switch (iteratorType) {
- case "TRIM_HORIZON" -> 0;
- case "LATEST" -> snapshot.size();
- case "AT_SEQUENCE_NUMBER" -> findSequencePosition(snapshot, sequenceNumber, false);
- case "AFTER_SEQUENCE_NUMBER" -> findSequencePosition(snapshot, sequenceNumber, true);
+ String cursorSequence = switch (iteratorType) {
+ case "TRIM_HORIZON" -> snapshot.isEmpty() ? zeroSequence() : snapshot.get(0).getSequenceNumber();
+ case "LATEST" -> snapshot.isEmpty() ? zeroSequence() : snapshot.get(snapshot.size() - 1).getSequenceNumber();
+ case "AT_SEQUENCE_NUMBER", "AFTER_SEQUENCE_NUMBER" -> {
+ if (sequenceNumber == null || sequenceNumber.isBlank()) {
+ throw new AwsException("ValidationException",
+ "Sequence number is required for this iterator type", 400);
+ }
+ yield sequenceNumber;
+ }
default -> throw new AwsException("ValidationException",
"Unknown iterator type: " + iteratorType, 400);
};
- return encodeIterator(streamArn, position);
+ boolean inclusive = "TRIM_HORIZON".equals(iteratorType) || "AT_SEQUENCE_NUMBER".equals(iteratorType);
+ long cursorRecordCount = zeroSequence().equals(cursorSequence) ? recordCount : -1;
+ return encodeIterator(streamArn, cursorSequence, inclusive, cursorRecordCount);
}
- private int findSequencePosition(List<DynamoDbStreamRecord> records, String targetSeq, boolean after) {
+ private int findSequencePosition(List<DynamoDbStreamRecord> records, String targetSeq, boolean inclusive) {
for (int i = 0; i < records.size(); i++) {
String seq = records.get(i).getSequenceNumber();
int cmp = seq.compareTo(targetSeq);
- if (after ? cmp > 0 : cmp >= 0) {
+ if (inclusive ? cmp >= 0 : cmp > 0) {
return i;
}
}
@@ -263,26 +288,48 @@ public class DynamoDbStreamService {
public GetRecordsResult getRecords(String shardIterator, Integer limit) {
String[] parts = decodeIterator(shardIterator);
String streamArn = parts[0];
- int position;
- try {
- position = Integer.parseInt(parts[1]);
- } catch (NumberFormatException e) {
- throw new AwsException("ValidationException", "Invalid shard iterator", 400);
- }
+ String cursorSequence = parts[1];
+ boolean inclusive = Boolean.parseBoolean(parts[2]);
+ long cursorRecordCount = parseRecordCount(parts[3]);
ConcurrentLinkedDeque<DynamoDbStreamRecord> deque = records.get(streamArn);
- List<DynamoDbStreamRecord> snapshot = deque != null ? new ArrayList<>(deque) : List.of();
+ List<DynamoDbStreamRecord> snapshot;
+ long currentRecordCount;
+ if (deque == null) {
+ snapshot = List.of();
+ currentRecordCount = 0;
+ } else {
+ synchronized (deque) {
+ snapshot = new ArrayList<>(deque);
+ currentRecordCount = streamRecordCounts.getOrDefault(streamArn, new AtomicLong()).get();
+ }
+ }
+ int position = findSequencePosition(snapshot, cursorSequence, inclusive);
+ if (zeroSequence().equals(cursorSequence) && cursorRecordCount >= 0
+ && currentRecordCount - cursorRecordCount > MAX_RECORDS) {
+ throw new AwsException("TrimmedDataAccessException",
+ "The requested sequence number has been trimmed", 400);
+ }
+ if (!snapshot.isEmpty() && !zeroSequence().equals(cursorSequence)
+ && cursorSequence.compareTo(snapshot.get(0).getSequenceNumber()) < 0) {
+ throw new AwsException("TrimmedDataAccessException",
+ "The requested sequence number has been trimmed", 400);
+ }
int effectiveLimit = limit != null ? limit : 100;
int end = Math.min(position + effectiveLimit, snapshot.size());
List<DynamoDbStreamRecord> page = snapshot.subList(position, end);
- String nextIterator = encodeIterator(streamArn, end);
+ String nextSequence = page.isEmpty()
+ ? cursorSequence
+ : page.get(page.size() - 1).getSequenceNumber();
+ long nextRecordCount = zeroSequence().equals(nextSequence) ? cursorRecordCount : -1;
+ String nextIterator = encodeIterator(streamArn, nextSequence, page.isEmpty() && inclusive, nextRecordCount);
return new GetRecordsResult(new ArrayList<>(page), nextIterator);
}
- private String encodeIterator(String streamArn, int position) {
- String raw = streamArn + "|" + position;
+ private String encodeIterator(String streamArn, String sequenceNumber, boolean inclusive, long recordCount) {
+ String raw = streamArn + "|" + sequenceNumber + "|" + inclusive + "|" + recordCount;
return Base64.getEncoder().encodeToString(raw.getBytes());
}
@@ -290,15 +337,40 @@ public class DynamoDbStreamService {
try {
String raw = new String(Base64.getDecoder().decode(iterator));
int lastPipe = raw.lastIndexOf('|');
- if (lastPipe < 0) {
+ int sequencePipe = raw.lastIndexOf('|', lastPipe - 1);
+ int inclusivePipe = raw.lastIndexOf('|', sequencePipe - 1);
+ if (inclusivePipe < 0 || sequencePipe < 0 || lastPipe < 0 || lastPipe == raw.length() - 1) {
throw new AwsException("ValidationException", "Invalid shard iterator", 400);
}
- return new String[]{raw.substring(0, lastPipe), raw.substring(lastPipe + 1)};
+ String inclusive = raw.substring(sequencePipe + 1, lastPipe);
+ String recordCount = raw.substring(lastPipe + 1);
+ if (!"true".equals(inclusive) && !"false".equals(inclusive)) {
+ throw new AwsException("ValidationException", "Invalid shard iterator", 400);
+ }
+ try {
+ Long.parseLong(recordCount);
+ } catch (NumberFormatException e) {
+ throw new AwsException("ValidationException", "Invalid shard iterator", 400);
+ }
+ return new String[]{raw.substring(0, inclusivePipe), raw.substring(inclusivePipe + 1, sequencePipe),
+ inclusive, recordCount};
} catch (IllegalArgumentException e) {
throw new AwsException("ValidationException", "Invalid shard iterator", 400);
}
}
+ private long parseRecordCount(String recordCount) {
+ try {
+ return Long.parseLong(recordCount);
+ } catch (NumberFormatException e) {
+ throw new AwsException("ValidationException", "Invalid shard iterator", 400);
+ }
+ }
+
+ private String zeroSequence() {
+ return ZERO_SEQUENCE_NUMBER;
+ }
+
private String streamKey(String region, String tableName) {
return region + "::" + tableName;
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationVpcEndpointIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationVpcEndpointIntegrationTest.java
index 37b89345c5..5ce4776448 100644
--- a/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationVpcEndpointIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudformation/CloudFormationVpcEndpointIntegrationTest.java
@@ -1,11 +1,11 @@
package io.github.hectorvent.floci.services.cloudformation;
+import io.github.hectorvent.floci.core.common.XmlParser;
import io.quarkus.test.junit.QuarkusTest;
import org.junit.jupiter.api.Test;
import static io.restassured.RestAssured.given;
import static org.hamcrest.Matchers.containsString;
-import static org.hamcrest.Matchers.not;
/**
* End-to-end check that CloudFormation provisions AWS::EC2::VPCEndpoint for
@@ -57,7 +57,7 @@ class CloudFormationVpcEndpointIntegrationTest {
.then()
.statusCode(200);
- given()
+ String stackDescription = given()
.contentType("application/x-www-form-urlencoded")
.header("Authorization", CFN_AUTH)
.formParam("Action", "DescribeStacks")
@@ -66,17 +66,10 @@ class CloudFormationVpcEndpointIntegrationTest {
.post("/")
.then()
.statusCode(200)
- .body(containsString("<StackStatus>CREATE_COMPLETE</StackStatus>"))
- .body(containsString("<OutputValue>vpce-"));
-
- given()
- .formParam("Action", "DescribeVpcEndpoints")
- .header("Authorization", EC2_AUTH)
- .when()
- .post("/")
- .then()
- .statusCode(200)
- .body(containsString("com.amazonaws.us-east-1.s3"));
+ .extract().asString();
+ org.junit.jupiter.api.Assertions.assertTrue(stackDescription.contains("<StackStatus>CREATE_COMPLETE</StackStatus>"));
+ String endpointId = XmlParser.extractFirst(stackDescription, "OutputValue", null);
+ org.junit.jupiter.api.Assertions.assertNotNull(endpointId);
given()
.contentType("application/x-www-form-urlencoded")
@@ -88,13 +81,14 @@ class CloudFormationVpcEndpointIntegrationTest {
.then()
.statusCode(200);
- given()
+ String endpointsAfterDelete = given()
.formParam("Action", "DescribeVpcEndpoints")
.header("Authorization", EC2_AUTH)
.when()
.post("/")
.then()
.statusCode(200)
- .body(not(containsString("com.amazonaws.us-east-1.s3")));
+ .extract().asString();
+ org.junit.jupiter.api.Assertions.assertFalse(endpointsAfterDelete.contains(endpointId));
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamServiceTest.java
index d33413b17a..c92604bde7 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamServiceTest.java
@@ -1,5 +1,6 @@
package io.github.hectorvent.floci.services.dynamodb;
+import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.storage.InMemoryStorage;
import io.github.hectorvent.floci.core.storage.StorageBackend;
import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;
@@ -94,4 +95,173 @@ class DynamoDbStreamServiceTest {
assertEquals("ENABLED", stream.getStreamStatus());
}
+ @Test
+ void rejectsAnIteratorWhenItsSequenceNumberHasBeenTrimmed() throws Exception {
+ var table = createTestTableWithStream();
+ service.enableStream(table.getTableName(), TABLE_ARN, "NEW_IMAGE", "us-east-1");
+ JsonNode item = mapper.readTree("{\"userId\":{\"S\":\"u1\"}}");
+
+ service.captureEvent(table.getTableName(), "INSERT", null, item, table, "us-east-1");
+ String iterator = service.getShardIterator(
+ table.getStreamArn(), DynamoDbStreamService.SHARD_ID, "AT_SEQUENCE_NUMBER", "000000000000000000001");
+
+ for (int i = 0; i < DynamoDbStreamService.MAX_RECORDS; i++) {
+ service.captureEvent(table.getTableName(), "INSERT", null, item, table, "us-east-1");
+ }
+
+ AwsException exception = assertThrows(AwsException.class,
+ () -> service.getRecords(iterator, 100));
+
+ assertEquals("TrimmedDataAccessException", exception.getErrorCode());
+ }
+
+ @Test
+ void rejectsAnEmptyStreamIteratorWhenRecordsAreTrimmedBeforeFirstPoll() throws Exception {
+ var table = new TableDefinition("EmptyTable",
+ List.of(new KeySchemaElement("userId", "HASH")),
+ List.of(new AttributeDefinition("userId", "S")),
+ "us-east-1", "000000000000");
+ table.setStreamEnabled(true);
+ StreamDescription stream = service.enableStream(
+ table.getTableName(), TABLE_ARN, "NEW_IMAGE", "us-east-1");
+ JsonNode item = mapper.readTree("{\"userId\":{\"S\":\"u1\"}}");
+ String iterator = service.getShardIterator(
+ stream.getStreamArn(), DynamoDbStreamService.SHARD_ID, "TRIM_HORIZON", null);
+
+ for (int i = 0; i <= DynamoDbStreamService.MAX_RECORDS; i++) {
+ service.captureEvent(table.getTableName(), "INSERT", null, item, table, "us-east-1");
+ }
+
+ AwsException exception = assertThrows(AwsException.class,
+ () -> service.getRecords(iterator, 100));
+
+ assertEquals("TrimmedDataAccessException", exception.getErrorCode());
+ }
+
+ @Test
+ void rejectsAnEmptyLatestIteratorWhenRecordsAreTrimmedBeforeFirstPoll() throws Exception {
+ var table = new TableDefinition("EmptyLatestTable",
+ List.of(new KeySchemaElement("userId", "HASH")),
+ List.of(new AttributeDefinition("userId", "S")),
+ "us-east-1", "000000000000");
+ table.setStreamEnabled(true);
+ StreamDescription stream = service.enableStream(
+ table.getTableName(), TABLE_ARN, "NEW_IMAGE", "us-east-1");
+ JsonNode item = mapper.readTree("{\"userId\":{\"S\":\"u1\"}}");
+ String iterator = service.getShardIterator(
+ stream.getStreamArn(), DynamoDbStreamService.SHARD_ID, "LATEST", null);
+
+ for (int i = 0; i <= DynamoDbStreamService.MAX_RECORDS; i++) {
+ service.captureEvent(table.getTableName(), "INSERT", null, item, table, "us-east-1");
+ }
+
+ AwsException exception = assertThrows(AwsException.class,
+ () -> service.getRecords(iterator, 100));
+
+ assertEquals("TrimmedDataAccessException", exception.getErrorCode());
+ }
+
+ @Test
+ void allowsAnEmptyStreamIteratorWhenRetentionHasNotBeenExceeded() throws Exception {
+ var table = new TableDefinition("EmptyTable",
+ List.of(new KeySchemaElement("userId", "HASH")),
+ List.of(new AttributeDefinition("userId", "S")),
+ "us-east-1", "000000000000");
+ table.setStreamEnabled(true);
+ StreamDescription stream = service.enableStream(
+ table.getTableName(), TABLE_ARN, "NEW_IMAGE", "us-east-1");
+ JsonNode item = mapper.readTree("{\"userId\":{\"S\":\"u1\"}}");
+ String iterator = service.getShardIterator(
+ stream.getStreamArn(), DynamoDbStreamService.SHARD_ID, "TRIM_HORIZON", null);
+
+ for (int i = 0; i < DynamoDbStreamService.MAX_RECORDS; i++) {
+ service.captureEvent(table.getTableName(), "INSERT", null, item, table, "us-east-1");
+ }
+
+ var result = service.getRecords(iterator, 1);
+
+ assertEquals(1, result.records().size());
+ assertEquals("000000000000000000001", result.records().get(0).getSequenceNumber());
+ }
+
+ @Test
+ void appliesInclusiveAndExclusiveSequencePositions() throws Exception {
+ var table = createTestTableWithStream();
+ service.enableStream(table.getTableName(), TABLE_ARN, "NEW_IMAGE", "us-east-1");
+ JsonNode item = mapper.readTree("{\"userId\":{\"S\":\"u1\"}}");
+
+ service.captureEvent(table.getTableName(), "INSERT", null, item, table, "us-east-1");
+ service.captureEvent(table.getTableName(), "MODIFY", item, item, table, "us-east-1");
+
+ String at = service.getShardIterator(
+ table.getStreamArn(), DynamoDbStreamService.SHARD_ID,
+ "AT_SEQUENCE_NUMBER", "000000000000000000001");
+ String after = service.getShardIterator(
+ table.getStreamArn(), DynamoDbStreamService.SHARD_ID,
+ "AFTER_SEQUENCE_NUMBER", "000000000000000000001");
+
+ var atRecords = service.getRecords(at, 1).records();
+ var afterRecords = service.getRecords(after, 1).records();
+
+ assertEquals("000000000000000000001", atRecords.get(0).getSequenceNumber());
+ assertEquals("000000000000000000002", afterRecords.get(0).getSequenceNumber());
+ assertEquals("MODIFY", afterRecords.get(0).getEventName());
+ }
+
+ @Test
+ void resumesAtTheRetainedSequenceAfterTheHeadIsTrimmed() throws Exception {
+ var table = createTestTableWithStream();
+ service.enableStream(table.getTableName(), TABLE_ARN, "NEW_IMAGE", "us-east-1");
+ JsonNode item = mapper.readTree("{\"userId\":{\"S\":\"u1\"}}");
+
+ service.captureEvent(table.getTableName(), "EVENT_1", null, item, table, "us-east-1");
+ service.captureEvent(table.getTableName(), "EVENT_2", null, item, table, "us-east-1");
+ String iterator = service.getShardIterator(
+ table.getStreamArn(), DynamoDbStreamService.SHARD_ID,
+ "AT_SEQUENCE_NUMBER", "000000000000000000002");
+
+ for (int i = 3; i <= DynamoDbStreamService.MAX_RECORDS + 1; i++) {
+ service.captureEvent(table.getTableName(), "EVENT_" + i, null, item, table, "us-east-1");
+ }
+
+ var records = service.getRecords(iterator, 1).records();
+
+ assertEquals(1, records.size());
+ assertEquals("EVENT_2", records.get(0).getEventName());
+ }
+
+ @Test
+ void advancesToTheNextSequenceWithoutDuplicatingRecords() throws Exception {
+ var table = createTestTableWithStream();
+ service.enableStream(table.getTableName(), TABLE_ARN, "NEW_IMAGE", "us-east-1");
+ JsonNode item = mapper.readTree("{\"userId\":{\"S\":\"u1\"}}");
+
+ service.captureEvent(table.getTableName(), "EVENT_1", null, item, table, "us-east-1");
+ service.captureEvent(table.getTableName(), "EVENT_2", null, item, table, "us-east-1");
+ String iterator = service.getShardIterator(
+ table.getStreamArn(), DynamoDbStreamService.SHARD_ID,
+ "TRIM_HORIZON", null);
+
+ var firstPage = service.getRecords(iterator, 1);
+ var secondPage = service.getRecords(firstPage.nextShardIterator(), 1);
+
+ assertEquals("000000000000000000001", firstPage.records().get(0).getSequenceNumber());
+ assertEquals("000000000000000000002", secondPage.records().get(0).getSequenceNumber());
+ }
+
+ @Test
+ void requiresASequenceNumberForSequenceBasedIterators() {
+ var table = createTestTableWithStream();
+
+ AwsException nullSequence = assertThrows(AwsException.class,
+ () -> service.getShardIterator(table.getStreamArn(), DynamoDbStreamService.SHARD_ID,
+ "AT_SEQUENCE_NUMBER", null));
+ AwsException blankSequence = assertThrows(AwsException.class,
+ () -> service.getShardIterator(table.getStreamArn(), DynamoDbStreamService.SHARD_ID,
+ "AFTER_SEQUENCE_NUMBER", " "));
+
+ assertEquals("ValidationException", nullSequence.getErrorCode());
+ assertEquals("ValidationException", blankSequence.getErrorCode());
+ }
+
}