Light, fluffy, and always free - The AWS Local Emulator alternative
fix(dynamodb): scope streams and sequence counters by account (#4116)
Daniël van Zield53ea39be917
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.javaindex 95bbb67d58..f4bb01ee1b 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamService.java@@ -46,7 +46,8 @@ public class DynamoDbStreamService {private final ConcurrentHashMap<String, ConcurrentLinkedDeque<DynamoDbStreamRecord>> records =new ConcurrentHashMap<>();private final ConcurrentHashMap<String, AtomicLong> streamRecordCounts = new ConcurrentHashMap<>();- private final AtomicLong sequenceCounter = new AtomicLong(0);+ /** Per stream (the stream ARN embeds account + region + table) so accounts never share a counter. */+ private final ConcurrentHashMap<String, AtomicLong> sequenceCounters = new ConcurrentHashMap<>();private final ObjectMapper objectMapper;@@ -142,6 +143,7 @@ public class DynamoDbStreamService {if (sd != null) {records.remove(sd.getStreamArn());streamRecordCounts.remove(sd.getStreamArn());+ sequenceCounters.remove(sd.getStreamArn());LOG.infov("Deleted stream for table {0}", tableArn);}}@@ -160,7 +162,7 @@ public class DynamoDbStreamService {return;}- long seq = sequenceCounter.incrementAndGet();+ long seq = sequenceCounters.computeIfAbsent(sd.getStreamArn(), k -> new AtomicLong()).incrementAndGet();String sequenceNumber = String.format("%021d", seq);JsonNode sourceItem = newItem != null ? newItem : oldItem;diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbAccountIsolationIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbAccountIsolationIntegrationTest.javaindex 2bd2e74f58..dba85e3471 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbAccountIsolationIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbAccountIsolationIntegrationTest.java@@ -6,6 +6,7 @@ import org.junit.jupiter.api.BeforeAll;import org.junit.jupiter.api.Test;import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.containsString;import static org.hamcrest.Matchers.equalTo;/**@@ -79,6 +80,38 @@ class DynamoDbAccountIsolationIntegrationTest {.body("Count", equalTo(0));}+ @Test+ void streamsInSameNamedTableAreIsolatedBetweenAccounts() {+ String tableName = "account-isolation-stream-table";++ createTable(AUTH_ACCOUNT_1, tableName);+ createTable(AUTH_ACCOUNT_2, tableName);+ enableStream(AUTH_ACCOUNT_1, tableName);+ enableStream(AUTH_ACCOUNT_2, tableName);++ given()+ .header("X-Amz-Target", "DynamoDBStreams_20120810.ListStreams")+ .header("Authorization", AUTH_ACCOUNT_1)+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("{\"TableName\":\"%s\"}".formatted(tableName))+ .when().post("/")+ .then()+ .statusCode(200)+ .body("Streams.size()", equalTo(1))+ .body("Streams[0].StreamArn", containsString(":000000000001:"));++ given()+ .header("X-Amz-Target", "DynamoDBStreams_20120810.ListStreams")+ .header("Authorization", AUTH_ACCOUNT_2)+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("{\"TableName\":\"%s\"}".formatted(tableName))+ .when().post("/")+ .then()+ .statusCode(200)+ .body("Streams.size()", equalTo(1))+ .body("Streams[0].StreamArn", containsString(":000000000002:"));+ }+private static void createTable(String auth, String tableName) {given().header("X-Amz-Target", "DynamoDB_20120810.CreateTable")@@ -95,4 +128,19 @@ class DynamoDbAccountIsolationIntegrationTest {.when().post("/").then().statusCode(200);}++ private static void enableStream(String auth, String tableName) {+ given()+ .header("X-Amz-Target", "DynamoDB_20120810.UpdateTable")+ .header("Authorization", auth)+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("""+ {+ "TableName": "%s",+ "StreamSpecification": {"StreamEnabled": true, "StreamViewType": "NEW_IMAGE"}+ }+ """.formatted(tableName))+ .when().post("/")+ .then().statusCode(200);+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamServiceTest.javaindex fc9792bd86..a5412fc9a1 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbStreamServiceTest.java@@ -5,6 +5,7 @@ import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.InMemoryStorage;import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;+import io.github.hectorvent.floci.services.dynamodb.model.DynamoDbStreamRecord;import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;import io.github.hectorvent.floci.services.dynamodb.model.StreamDescription;import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;@@ -287,6 +288,11 @@ class DynamoDbStreamServiceTest {.toList();}+ private List<DynamoDbStreamRecord> recordsOf(String streamArn) {+ String iterator = service.getShardIterator(streamArn, DynamoDbStreamService.SHARD_ID, "TRIM_HORIZON", null);+ return service.getRecords(iterator, 10).records();+ }+private static List<String> streamArns(List<StreamDescription> streams) {return streams.stream().map(StreamDescription::getStreamArn).toList();}@@ -345,4 +351,51 @@ class DynamoDbStreamServiceTest {assertEquals("ValidationException", blankSequence.getErrorCode());}+ /**+ * Two accounts with a same-named table in one region must not share a stream. Before this,+ * the stream map was keyed by region and table name only, so the second account's+ * enableStream returned the first account's stream and every record landed on it.+ */+ @Test+ void sameNamedTablesInTwoAccountsStayIsolated() throws Exception {+ TableDefinition firstAccountTable = new TableDefinition("ViewTypeTable",+ List.of(new KeySchemaElement("userId", "HASH")),+ List.of(new AttributeDefinition("userId", "S")),+ "us-east-1", "000000000000");+ firstAccountTable.setStreamEnabled(true);+ TableDefinition otherAccountTable = new TableDefinition("ViewTypeTable",+ List.of(new KeySchemaElement("userId", "HASH")),+ List.of(new AttributeDefinition("userId", "S")),+ "us-east-1", "111111111111");+ otherAccountTable.setStreamEnabled(true);++ StreamDescription first = service.enableStream("ViewTypeTable", TABLE_ARN, "NEW_IMAGE", "us-east-1");+ StreamDescription second = service.enableStream(+ "ViewTypeTable", otherAccountTable.getTableArn(), "NEW_IMAGE", "us-east-1");++ assertNotEquals(first.getStreamArn(), second.getStreamArn());+ assertEquals(1, service.listStreams("ViewTypeTable", "000000000000", "us-east-1").size());+ assertEquals(1, service.listStreams("ViewTypeTable", "111111111111", "us-east-1").size());++ JsonNode item = mapper.readTree("{\"userId\":{\"S\":\"u1\"}}");+ service.captureEvent("INSERT", null, item, otherAccountTable, "us-east-1");++ List<DynamoDbStreamRecord> secondRecords = recordsOf(second.getStreamArn());+ assertEquals(1, secondRecords.size());+ assertEquals("000000000000000000001", secondRecords.get(0).getSequenceNumber());++ assertEquals(0, recordsOf(first.getStreamArn()).size());++ service.captureEvent("INSERT", null, item, firstAccountTable, "us-east-1");++ List<DynamoDbStreamRecord> firstRecords = recordsOf(first.getStreamArn());+ assertEquals(1, firstRecords.size());+ // A shared/global counter would number this record 000000000000000000002.+ assertEquals("000000000000000000001", firstRecords.get(0).getSequenceNumber());++ service.deleteStream(otherAccountTable.getTableArn());+ assertEquals(0, service.listStreams("ViewTypeTable", "111111111111", "us-east-1").size());+ assertEquals(1, service.listStreams("ViewTypeTable", "000000000000", "us-east-1").size());+ }+}