Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(dynamodb): enforce the aggregate batch write limit (#6633)

Andre Kurait6ff4700a228b
diff --git a/compatibility-tests/sdk-test-python/tests/test_dynamodb_batch_limit.py b/compatibility-tests/sdk-test-python/tests/test_dynamodb_batch_limit.py
new file mode 100644
index 0000000000..75c4633d69
--- /dev/null
+++ b/compatibility-tests/sdk-test-python/tests/test_dynamodb_batch_limit.py
@@ -0,0 +1,43 @@
+"""Whole-request BatchWriteItem limits through the AWS SDK."""
+
+from contextlib import ExitStack
+
+import pytest
+from botocore.exceptions import ClientError
+
+
[email protected]("count", [25, 26])
+def test_aggregate_batch_write_limit(dynamodb_client, unique_name, count):
+ names = [f"batch-first-{unique_name}", f"batch-second-{unique_name}"]
+ with ExitStack() as cleanup:
+ for name in names:
+ dynamodb_client.create_table(
+ TableName=name,
+ KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}],
+ AttributeDefinitions=[{"AttributeName": "pk", "AttributeType": "S"}],
+ BillingMode="PAY_PER_REQUEST",
+ )
+ cleanup.callback(dynamodb_client.delete_table, TableName=name)
+ dynamodb_client.get_waiter("table_exists").wait(TableName=name)
+ dynamodb_client.put_item(TableName=name, Item={"pk": {"S": "delete"}})
+ requests = {}
+ for name, size in zip(names, [13, count - 13]):
+ requests[name] = [{"DeleteRequest": {"Key": {"pk": {"S": "delete"}}}}]
+ requests[name].extend(
+ {"PutRequest": {"Item": {"pk": {"S": f"put-{index}"}}}}
+ for index in range(size - 1)
+ )
+ if count == 26:
+ with pytest.raises(ClientError) as error:
+ dynamodb_client.batch_write_item(RequestItems=requests)
+ assert error.value.response["Error"]["Code"] == "ValidationException"
+ for name in names:
+ assert dynamodb_client.scan(TableName=name, ConsistentRead=True)["Items"] == [
+ {"pk": {"S": "delete"}}
+ ]
+ else:
+ assert not dynamodb_client.batch_write_item(RequestItems=requests)["UnprocessedItems"]
+ for name, size in zip(names, [13, count - 13]):
+ items = dynamodb_client.scan(TableName=name, ConsistentRead=True)["Items"]
+ assert len(items) == size - 1
+ assert {"pk": {"S": "delete"}} not in items
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
index 9ddd61257b..509eaf0d2c 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
@@ -1878,6 +1878,11 @@ public class DynamoDbService {
}
}
+ long writeCount = requestItems.values().stream().mapToLong(List::size).sum();
+ if (writeCount > 25) {
+ throw new AwsException("ValidationException", "Too many items requested for the BatchWriteItem call", 400);
+ }
+
// A store that writes single items saves each one inside its item lock, as PutItem does: a
// pass after the locks are released could read an item, lose the race to a concurrent
// write of the same key, and then save the stale copy over it. A whole-table store is
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBatchWriteLimitIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBatchWriteLimitIntegrationTest.java
new file mode 100644
index 0000000000..698f3b02ec
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBatchWriteLimitIntegrationTest.java
@@ -0,0 +1,111 @@
+package io.github.hectorvent.floci.services.dynamodb;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.dataformat.cbor.CBORFactory;
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.util.List;
+import java.util.UUID;
+
+import static io.restassured.RestAssured.given;
+import static org.junit.jupiter.api.Assertions.*;
+
+@QuarkusTest
+class DynamoDbBatchWriteLimitIntegrationTest {
+
+ private static final ObjectMapper JSON = new ObjectMapper();
+ private static final ObjectMapper CBOR = new ObjectMapper(new CBORFactory());
+
+ @BeforeAll
+ static void configureJson() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"20120810", "20111205", "cbor"})
+ void aggregateLimitRejectsWholeBatchAndAcceptsTwentyFive(String protocol) throws Exception {
+ String first = "batch-first-" + UUID.randomUUID();
+ String second = "batch-second-" + UUID.randomUUID();
+ createTable(first);
+ try {
+ createTable(second);
+ try {
+ for (String table : List.of(first, second)) {
+ ObjectNode seed = JSON.createObjectNode().put("TableName", table);
+ seed.set("Item", key("delete"));
+ call("20120810", "PutItem", seed, 200);
+ }
+ ObjectNode request = JSON.createObjectNode();
+ ObjectNode tables = request.putObject("RequestItems");
+ tables.set(first, writes(protocol, 13));
+ tables.set(second, writes(protocol, 13));
+ JsonNode error = call(protocol, "BatchWriteItem", request, 400);
+ assertTrue(error.path("__type").asText().endsWith("ValidationException"));
+ for (String table : List.of(first, second)) {
+ JsonNode scan = call("20120810", "Scan", JSON.createObjectNode().put("TableName", table), 200);
+ assertEquals(1, scan.path("Count").asInt());
+ assertEquals(key("delete"), scan.path("Items").get(0));
+ }
+ ((ArrayNode) tables.get(second)).remove(12);
+ assertTrue(call(protocol, "BatchWriteItem", request, 200).path("UnprocessedItems").isEmpty());
+ assertEquals(12, call("20120810", "Scan", JSON.createObjectNode().put("TableName", first), 200)
+ .path("Count").asInt());
+ assertEquals(11, call("20120810", "Scan", JSON.createObjectNode().put("TableName", second), 200)
+ .path("Count").asInt());
+ } finally {
+ call("20120810", "DeleteTable", JSON.createObjectNode().put("TableName", second), 200);
+ }
+ } finally {
+ call("20120810", "DeleteTable", JSON.createObjectNode().put("TableName", first), 200);
+ }
+ }
+
+ private static void createTable(String name) throws Exception {
+ call("20120810", "CreateTable", JSON.readTree("""
+ {"TableName":"%s","KeySchema":[{"AttributeName":"pk","KeyType":"HASH"}],
+ "AttributeDefinitions":[{"AttributeName":"pk","AttributeType":"S"}],
+ "BillingMode":"PAY_PER_REQUEST"}
+ """.formatted(name)), 200);
+ }
+
+ private static ArrayNode writes(String protocol, int count) {
+ ArrayNode result = JSON.createArrayNode();
+ ObjectNode deleteKey = key("delete");
+ if ("20111205".equals(protocol)) {
+ deleteKey = JSON.createObjectNode();
+ deleteKey.putObject("HashKeyElement").put("S", "delete");
+ }
+ result.addObject().putObject("DeleteRequest").set("Key", deleteKey);
+ for (int index = 0; index < count - 1; index++) {
+ result.addObject().putObject("PutRequest").set("Item", key("put-" + index));
+ }
+ return result;
+ }
+
+ private static ObjectNode key(String value) {
+ ObjectNode key = JSON.createObjectNode();
+ key.putObject("pk").put("S", value);
+ return key;
+ }
+
+ private static JsonNode call(String protocol, String action, JsonNode request, int status) throws Exception {
+ if ("cbor".equals(protocol)) {
+ byte[] response = given().contentType("application/cbor").accept("application/cbor")
+ .body(CBOR.writeValueAsBytes(request))
+ .post("/service/DynamoDB_20120810/operation/" + action)
+ .then().statusCode(status).extract().asByteArray();
+ return CBOR.readTree(response);
+ }
+ String response = given().contentType("application/x-amz-json-1.0")
+ .header("X-Amz-Target", "DynamoDB_" + protocol + "." + action).body(request.toString())
+ .post("/").then().statusCode(status).extract().asString();
+ return JSON.readTree(response);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBatchWriteLimitServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBatchWriteLimitServiceTest.java
new file mode 100644
index 0000000000..34b0982fb1
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBatchWriteLimitServiceTest.java
@@ -0,0 +1,103 @@
+package io.github.hectorvent.floci.services.dynamodb;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;
+import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+class DynamoDbBatchWriteLimitServiceTest {
+
+ private static final String REGION = "us-east-1";
+ private final ObjectMapper mapper = new ObjectMapper();
+ private DynamoDbService service;
+
+ @BeforeEach
+ void setUp() {
+ service = new DynamoDbService(new InMemoryStorage<>());
+ for (String table : List.of("First", "Second")) {
+ service.createTable(table, List.of(new KeySchemaElement("pk", "HASH")),
+ List.of(new AttributeDefinition("pk", "S")), 5L, 5L, REGION);
+ service.putItem(table, key("delete"), REGION);
+ }
+ }
+
+ @Test
+ void acceptsTwentyFiveMixedWritesAcrossTables() {
+ assertTrue(service.batchWriteItem(batch(12, 13), REGION).unprocessedItems().isEmpty());
+ assertNull(service.getItem("First", key("delete"), REGION));
+ assertNull(service.getItem("Second", key("delete"), REGION));
+ assertNotNull(service.getItem("First", key("put-10"), REGION));
+ assertNotNull(service.getItem("Second", key("put-11"), REGION));
+ }
+
+ @Test
+ void rejectsTwentySixMixedWritesAcrossTablesBeforeAnyMutation() {
+ AwsException error = assertThrows(AwsException.class,
+ () -> service.batchWriteItem(batch(13, 13), REGION));
+ assertEquals("ValidationException", error.getErrorCode());
+ assertUnchanged();
+ }
+
+ @Test
+ void rejectsTwentySixWritesInOneTableBeforeAnyMutation() {
+ AwsException error = assertThrows(AwsException.class,
+ () -> service.batchWriteItem(Map.of("First", writes(26)), REGION));
+ assertEquals("ValidationException", error.getErrorCode());
+ assertUnchanged();
+ }
+
+ @Test
+ void preservesExistingValidationBeforeAggregateLimit() {
+ Map<String, List<JsonNode>> requests = batch(13, 13);
+ requests.get("Second").set(12, requests.get("Second").get(11));
+ AwsException duplicate = assertThrows(AwsException.class,
+ () -> service.batchWriteItem(requests, REGION));
+ assertEquals("Provided list of item keys contains duplicates", duplicate.getMessage());
+ assertUnchanged();
+ }
+
+ private void assertUnchanged() {
+ for (String table : List.of("First", "Second")) {
+ assertNotNull(service.getItem(table, key("delete"), REGION));
+ assertEquals(1, service.scan(table, null, null, null, null, null, null, REGION).items().size());
+ }
+ }
+
+ private Map<String, List<JsonNode>> batch(int first, int second) {
+ Map<String, List<JsonNode>> requests = new LinkedHashMap<>();
+ requests.put("First", writes(first));
+ requests.put("Second", writes(second));
+ return requests;
+ }
+
+ private List<JsonNode> writes(int count) {
+ List<JsonNode> result = new ArrayList<>();
+ ObjectNode delete = mapper.createObjectNode();
+ delete.putObject("DeleteRequest").set("Key", key("delete"));
+ result.add(delete);
+ for (int index = 0; index < count - 1; index++) {
+ ObjectNode put = mapper.createObjectNode();
+ put.putObject("PutRequest").set("Item", key("put-" + index));
+ result.add(put);
+ }
+ return result;
+ }
+
+ private ObjectNode key(String value) {
+ ObjectNode key = mapper.createObjectNode();
+ key.putObject("pk").put("S", value);
+ return key;
+ }
+}