Light, fluffy, and always free - The AWS Local Emulator alternative
fix(dynamodb): enforce the aggregate batch write limit (#6633)
Andre Kurait6ff4700a228b
diff --git a/compatibility-tests/sdk-test-python/tests/test_dynamodb_batch_limit.py b/compatibility-tests/sdk-test-python/tests/test_dynamodb_batch_limit.pynew file mode 100644index 0000000000..75c4633d69--- /dev/null+++ b/compatibility-tests/sdk-test-python/tests/test_dynamodb_batch_limit.py@@ -0,0 +1,43 @@+"""Whole-request BatchWriteItem limits through the AWS SDK."""++from contextlib import ExitStack++import pytest+from botocore.exceptions import ClientError++[email protected]("count", [25, 26])+def test_aggregate_batch_write_limit(dynamodb_client, unique_name, count):+ names = [f"batch-first-{unique_name}", f"batch-second-{unique_name}"]+ with ExitStack() as cleanup:+ for name in names:+ dynamodb_client.create_table(+ TableName=name,+ KeySchema=[{"AttributeName": "pk", "KeyType": "HASH"}],+ AttributeDefinitions=[{"AttributeName": "pk", "AttributeType": "S"}],+ BillingMode="PAY_PER_REQUEST",+ )+ cleanup.callback(dynamodb_client.delete_table, TableName=name)+ dynamodb_client.get_waiter("table_exists").wait(TableName=name)+ dynamodb_client.put_item(TableName=name, Item={"pk": {"S": "delete"}})+ requests = {}+ for name, size in zip(names, [13, count - 13]):+ requests[name] = [{"DeleteRequest": {"Key": {"pk": {"S": "delete"}}}}]+ requests[name].extend(+ {"PutRequest": {"Item": {"pk": {"S": f"put-{index}"}}}}+ for index in range(size - 1)+ )+ if count == 26:+ with pytest.raises(ClientError) as error:+ dynamodb_client.batch_write_item(RequestItems=requests)+ assert error.value.response["Error"]["Code"] == "ValidationException"+ for name in names:+ assert dynamodb_client.scan(TableName=name, ConsistentRead=True)["Items"] == [+ {"pk": {"S": "delete"}}+ ]+ else:+ assert not dynamodb_client.batch_write_item(RequestItems=requests)["UnprocessedItems"]+ for name, size in zip(names, [13, count - 13]):+ items = dynamodb_client.scan(TableName=name, ConsistentRead=True)["Items"]+ assert len(items) == size - 1+ assert {"pk": {"S": "delete"}} not in itemsdiff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.javaindex 9ddd61257b..509eaf0d2c 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java@@ -1878,6 +1878,11 @@ public class DynamoDbService {}}+ long writeCount = requestItems.values().stream().mapToLong(List::size).sum();+ if (writeCount > 25) {+ throw new AwsException("ValidationException", "Too many items requested for the BatchWriteItem call", 400);+ }+// A store that writes single items saves each one inside its item lock, as PutItem does: a// pass after the locks are released could read an item, lose the race to a concurrent// write of the same key, and then save the stale copy over it. A whole-table store isdiff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBatchWriteLimitIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBatchWriteLimitIntegrationTest.javanew file mode 100644index 0000000000..698f3b02ec--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBatchWriteLimitIntegrationTest.java@@ -0,0 +1,111 @@+package io.github.hectorvent.floci.services.dynamodb;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ArrayNode;+import com.fasterxml.jackson.databind.node.ObjectNode;+import com.fasterxml.jackson.dataformat.cbor.CBORFactory;+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;++import java.util.List;+import java.util.UUID;++import static io.restassured.RestAssured.given;+import static org.junit.jupiter.api.Assertions.*;++@QuarkusTest+class DynamoDbBatchWriteLimitIntegrationTest {++ private static final ObjectMapper JSON = new ObjectMapper();+ private static final ObjectMapper CBOR = new ObjectMapper(new CBORFactory());++ @BeforeAll+ static void configureJson() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @ParameterizedTest+ @ValueSource(strings = {"20120810", "20111205", "cbor"})+ void aggregateLimitRejectsWholeBatchAndAcceptsTwentyFive(String protocol) throws Exception {+ String first = "batch-first-" + UUID.randomUUID();+ String second = "batch-second-" + UUID.randomUUID();+ createTable(first);+ try {+ createTable(second);+ try {+ for (String table : List.of(first, second)) {+ ObjectNode seed = JSON.createObjectNode().put("TableName", table);+ seed.set("Item", key("delete"));+ call("20120810", "PutItem", seed, 200);+ }+ ObjectNode request = JSON.createObjectNode();+ ObjectNode tables = request.putObject("RequestItems");+ tables.set(first, writes(protocol, 13));+ tables.set(second, writes(protocol, 13));+ JsonNode error = call(protocol, "BatchWriteItem", request, 400);+ assertTrue(error.path("__type").asText().endsWith("ValidationException"));+ for (String table : List.of(first, second)) {+ JsonNode scan = call("20120810", "Scan", JSON.createObjectNode().put("TableName", table), 200);+ assertEquals(1, scan.path("Count").asInt());+ assertEquals(key("delete"), scan.path("Items").get(0));+ }+ ((ArrayNode) tables.get(second)).remove(12);+ assertTrue(call(protocol, "BatchWriteItem", request, 200).path("UnprocessedItems").isEmpty());+ assertEquals(12, call("20120810", "Scan", JSON.createObjectNode().put("TableName", first), 200)+ .path("Count").asInt());+ assertEquals(11, call("20120810", "Scan", JSON.createObjectNode().put("TableName", second), 200)+ .path("Count").asInt());+ } finally {+ call("20120810", "DeleteTable", JSON.createObjectNode().put("TableName", second), 200);+ }+ } finally {+ call("20120810", "DeleteTable", JSON.createObjectNode().put("TableName", first), 200);+ }+ }++ private static void createTable(String name) throws Exception {+ call("20120810", "CreateTable", JSON.readTree("""+ {"TableName":"%s","KeySchema":[{"AttributeName":"pk","KeyType":"HASH"}],+ "AttributeDefinitions":[{"AttributeName":"pk","AttributeType":"S"}],+ "BillingMode":"PAY_PER_REQUEST"}+ """.formatted(name)), 200);+ }++ private static ArrayNode writes(String protocol, int count) {+ ArrayNode result = JSON.createArrayNode();+ ObjectNode deleteKey = key("delete");+ if ("20111205".equals(protocol)) {+ deleteKey = JSON.createObjectNode();+ deleteKey.putObject("HashKeyElement").put("S", "delete");+ }+ result.addObject().putObject("DeleteRequest").set("Key", deleteKey);+ for (int index = 0; index < count - 1; index++) {+ result.addObject().putObject("PutRequest").set("Item", key("put-" + index));+ }+ return result;+ }++ private static ObjectNode key(String value) {+ ObjectNode key = JSON.createObjectNode();+ key.putObject("pk").put("S", value);+ return key;+ }++ private static JsonNode call(String protocol, String action, JsonNode request, int status) throws Exception {+ if ("cbor".equals(protocol)) {+ byte[] response = given().contentType("application/cbor").accept("application/cbor")+ .body(CBOR.writeValueAsBytes(request))+ .post("/service/DynamoDB_20120810/operation/" + action)+ .then().statusCode(status).extract().asByteArray();+ return CBOR.readTree(response);+ }+ String response = given().contentType("application/x-amz-json-1.0")+ .header("X-Amz-Target", "DynamoDB_" + protocol + "." + action).body(request.toString())+ .post("/").then().statusCode(status).extract().asString();+ return JSON.readTree(response);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBatchWriteLimitServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBatchWriteLimitServiceTest.javanew file mode 100644index 0000000000..34b0982fb1--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbBatchWriteLimitServiceTest.java@@ -0,0 +1,103 @@+package io.github.hectorvent.floci.services.dynamodb;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ObjectNode;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;+import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.util.ArrayList;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;++import static org.junit.jupiter.api.Assertions.*;++class DynamoDbBatchWriteLimitServiceTest {++ private static final String REGION = "us-east-1";+ private final ObjectMapper mapper = new ObjectMapper();+ private DynamoDbService service;++ @BeforeEach+ void setUp() {+ service = new DynamoDbService(new InMemoryStorage<>());+ for (String table : List.of("First", "Second")) {+ service.createTable(table, List.of(new KeySchemaElement("pk", "HASH")),+ List.of(new AttributeDefinition("pk", "S")), 5L, 5L, REGION);+ service.putItem(table, key("delete"), REGION);+ }+ }++ @Test+ void acceptsTwentyFiveMixedWritesAcrossTables() {+ assertTrue(service.batchWriteItem(batch(12, 13), REGION).unprocessedItems().isEmpty());+ assertNull(service.getItem("First", key("delete"), REGION));+ assertNull(service.getItem("Second", key("delete"), REGION));+ assertNotNull(service.getItem("First", key("put-10"), REGION));+ assertNotNull(service.getItem("Second", key("put-11"), REGION));+ }++ @Test+ void rejectsTwentySixMixedWritesAcrossTablesBeforeAnyMutation() {+ AwsException error = assertThrows(AwsException.class,+ () -> service.batchWriteItem(batch(13, 13), REGION));+ assertEquals("ValidationException", error.getErrorCode());+ assertUnchanged();+ }++ @Test+ void rejectsTwentySixWritesInOneTableBeforeAnyMutation() {+ AwsException error = assertThrows(AwsException.class,+ () -> service.batchWriteItem(Map.of("First", writes(26)), REGION));+ assertEquals("ValidationException", error.getErrorCode());+ assertUnchanged();+ }++ @Test+ void preservesExistingValidationBeforeAggregateLimit() {+ Map<String, List<JsonNode>> requests = batch(13, 13);+ requests.get("Second").set(12, requests.get("Second").get(11));+ AwsException duplicate = assertThrows(AwsException.class,+ () -> service.batchWriteItem(requests, REGION));+ assertEquals("Provided list of item keys contains duplicates", duplicate.getMessage());+ assertUnchanged();+ }++ private void assertUnchanged() {+ for (String table : List.of("First", "Second")) {+ assertNotNull(service.getItem(table, key("delete"), REGION));+ assertEquals(1, service.scan(table, null, null, null, null, null, null, REGION).items().size());+ }+ }++ private Map<String, List<JsonNode>> batch(int first, int second) {+ Map<String, List<JsonNode>> requests = new LinkedHashMap<>();+ requests.put("First", writes(first));+ requests.put("Second", writes(second));+ return requests;+ }++ private List<JsonNode> writes(int count) {+ List<JsonNode> result = new ArrayList<>();+ ObjectNode delete = mapper.createObjectNode();+ delete.putObject("DeleteRequest").set("Key", key("delete"));+ result.add(delete);+ for (int index = 0; index < count - 1; index++) {+ ObjectNode put = mapper.createObjectNode();+ put.putObject("PutRequest").set("Item", key("put-" + index));+ result.add(put);+ }+ return result;+ }++ private ObjectNode key(String value) {+ ObjectNode key = mapper.createObjectNode();+ key.putObject("pk").put("S", value);+ return key;+ }+}