Light, fluffy, and always free - The AWS Local Emulator alternative
fix(dynamodb): cancel TransactWriteItems on a key that does not match the schema (#3674)
TATSUNO “Taz” Yasuhiro487cacb3bc6c
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.javaindex 87579b6c04..4a64e37101 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java@@ -1711,6 +1711,14 @@ public class DynamoDbJsonHandler {"Transaction failed: The total size of all items in the transaction request cannot exceed 4 MB", 400);}+ List<JsonNode> transactItems = new ArrayList<>();+ transactItemsNode.forEach(transactItems::add);+ try {+ dynamoDbService.cancelOnKeySchemaMismatch(transactItems, region);+ } catch (TransactionCanceledException e) {+ return transactWriteCanceled(e);+ }+Map<String, TableDefinition> tableCache = new HashMap<>();Set<String> seen = new HashSet<>();for (JsonNode txItem : transactItemsNode) {@@ -1731,9 +1739,6 @@ public class DynamoDbJsonHandler {}}- List<JsonNode> transactItems = new ArrayList<>();- transactItemsNode.forEach(transactItems::add);-// ClientRequestToken makes TransactWriteItems idempotent within ~10 minutes.// Forward the token and the raw request body so the service can hash the body// and reject replays whose parameters changed.@@ -1745,23 +1750,27 @@ public class DynamoDbJsonHandler {dynamoDbService.transactWriteItems(transactItems, region, clientRequestToken, request);return Response.ok(objectMapper.createObjectNode()).build();} catch (TransactionCanceledException e) {- ObjectNode body = objectMapper.createObjectNode();- body.put("__type", "TransactionCanceledException");- body.put("message", e.getMessage());- ArrayNode reasons = body.putArray("CancellationReasons");- for (TransactionCanceledException.CancellationReason reason : e.getCancellationReasons()) {- ObjectNode r = objectMapper.createObjectNode();- r.put("Code", reason.code().isEmpty() ? "None" : reason.code());- if (!reason.code().isEmpty()) {- r.put("Message", reason.message() != null ? reason.message() : "The conditional request failed");- }- if (reason.item() != null) {- r.set("Item", reason.item());- }- reasons.add(r);+ return transactWriteCanceled(e);+ }+ }++ private Response transactWriteCanceled(TransactionCanceledException e) {+ ObjectNode body = objectMapper.createObjectNode();+ body.put("__type", "TransactionCanceledException");+ body.put("message", e.getMessage());+ ArrayNode reasons = body.putArray("CancellationReasons");+ for (TransactionCanceledException.CancellationReason reason : e.getCancellationReasons()) {+ ObjectNode r = objectMapper.createObjectNode();+ r.put("Code", reason.code().isEmpty() ? "None" : reason.code());+ if (!reason.code().isEmpty()) {+ r.put("Message", reason.message() != null ? reason.message() : "The conditional request failed");}- return Response.status(400).entity(body).build();+ if (reason.item() != null) {+ r.set("Item", reason.item());+ }+ reasons.add(r);}+ return Response.status(400).entity(body).build();}private Response handleTransactGetItems(JsonNode request, String region) {diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.javaindex f13d140dc2..a11f789a80 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java@@ -1437,8 +1437,12 @@ public class DynamoDbService implements ResourceProvider {throw new TransactionCanceledException(cancellationReasons);}- for (JsonNode transactItem : transactItems) {- validateTransactItem(transactItem, region, staged);+ for (int i = 0; i < transactItems.size(); i++) {+ try {+ validateTransactItem(transactItems.get(i), region, staged);+ } catch (KeySchemaMismatchException e) {+ throw cancelledByMember(transactItems.size(), i, e.getMessage());+ }}for (JsonNode transactItem : transactItems) {if (transactItem.has("Put")) {@@ -1539,6 +1543,58 @@ public class DynamoDbService implements ResourceProvider {"Nesting Levels have exceeded supported limits");}+ // AWS checks every member's key against the table schema, in order, before it looks for+ // duplicate items or evaluates a condition. The first mismatch cancels with that member's+ // reason alone. An empty key value still fails the whole request.+ void cancelOnKeySchemaMismatch(List<JsonNode> transactItems, String region) {+ for (int i = 0; i < transactItems.size(); i++) {+ try {+ validateTransactMemberKey(transactItems.get(i), region);+ } catch (KeySchemaMismatchException e) {+ throw cancelledByMember(transactItems.size(), i, e.getMessage());+ }+ }+ }++ private void validateTransactMemberKey(JsonNode transactItem, String region) {+ boolean isPut = transactItem.has("Put");+ JsonNode target = isPut ? transactItem.get("Put")+ : transactItem.has("Update") ? transactItem.get("Update")+ : transactItem.has("Delete") ? transactItem.get("Delete")+ : transactItem.get("ConditionCheck");+ JsonNode key = target == null ? null : target.get(isPut ? "Item" : "Key");+ if (key == null) {+ return;+ }+ String tableName = canonicalTableName(region, target.path("TableName").asText());+ TableDefinition table = requireActiveTable(regionKey(region, tableName), tableName);+ List<String> keyNames = table.getSortKeyName() == null+ ? List.of(table.getPartitionKeyName())+ : List.of(table.getPartitionKeyName(), table.getSortKeyName());+ for (String keyName : keyNames) {+ if (!key.has(keyName)) {+ throw new KeySchemaMismatchException(isPut+ ? "One or more parameter values were invalid: Missing the key " + keyName + " in the item"+ : "The provided key element does not match the schema");+ }+ }+ buildItemKey(table, key, isPut ? KeySurface.ITEM_BODY : KeySurface.KEY_ARGUMENT);+ if (isPut) {+ validateIndexKeyTypes(table, key, false);+ }+ }++ private static TransactionCanceledException cancelledByMember(int memberCount, int failedMember,+ String message) {+ List<TransactionCanceledException.CancellationReason> reasons = new ArrayList<>();+ for (int i = 0; i < memberCount; i++) {+ reasons.add(i == failedMember+ ? new TransactionCanceledException.CancellationReason("ValidationError", null, message)+ : new TransactionCanceledException.CancellationReason("", null));+ }+ return new TransactionCanceledException(reasons);+ }+private TransactionCanceledException.CancellationReason evaluateTransactCondition(JsonNode transactItem, String region) {return evaluateTransactCondition(transactItem, region, null);}@@ -3228,12 +3284,11 @@ public class DynamoDbService implements ResourceProvider {String expectedType = keyAttributeType(table, keyName);if (expectedType != null && !attr.has(expectedType)) {if (surface == KeySurface.ITEM_BODY) {- throw new AwsException("ValidationException",+ throw new KeySchemaMismatchException("One or more parameter values were invalid: Type mismatch for key " + keyName- + " expected: " + expectedType + " actual: " + attr.fieldNames().next(), 400);+ + " expected: " + expectedType + " actual: " + attr.fieldNames().next());}- throw new AwsException("ValidationException",- "The provided key element does not match the schema", 400);+ throw new KeySchemaMismatchException("The provided key element does not match the schema");}if (attr.has("S") && attr.get("S").asText().isEmpty()) {throw new AwsException("ValidationException",@@ -3289,10 +3344,10 @@ public class DynamoDbService implements ResourceProvider {validateAttributeValueShape(attr);String expectedType = keyAttributeType(table, attrName);if (expectedType != null && !attr.has(expectedType)) {- throw new AwsException("ValidationException",+ throw new KeySchemaMismatchException("One or more parameter values were invalid: Type mismatch for Index Key " + attrName+ " Expected: " + expectedType + " Actual: " + attr.fieldNames().next()- + " IndexName: " + indexName, 400);+ + " IndexName: " + indexName);}if (attr.has("S") && attr.get("S").asText().isEmpty()) {throw emptyIndexKeyValue("empty string value", indexName, attrName, isUpdate);diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/KeySchemaMismatchException.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/KeySchemaMismatchException.javanew file mode 100644index 0000000000..8062fc1b15--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/KeySchemaMismatchException.java@@ -0,0 +1,11 @@+package io.github.hectorvent.floci.services.dynamodb;++import io.github.hectorvent.floci.core.common.AwsException;++// A single-item call reports this as a plain ValidationException. A transaction cancels on it.+public class KeySchemaMismatchException extends AwsException {++ public KeySchemaMismatchException(String message) {+ super("ValidationException", message, 400);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandlerTest.javaindex fc529cc206..06212963be 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandlerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandlerTest.java@@ -1679,4 +1679,186 @@ class DynamoDbJsonHandlerTest {assertEquals("Nesting Levels have exceeded supported limits", reasons.get(1).get("Message").asText());assertNull(service.getItem("Users", item("userId", "u2"), "eu-west-1"));}++ private ObjectNode transactMember(String action, String tableName, String field, ObjectNode value) {+ ObjectNode op = mapper.createObjectNode();+ op.put("TableName", tableName);+ op.set(field, value);+ ObjectNode member = mapper.createObjectNode();+ member.set(action, op);+ return member;+ }++ private ObjectNode transactWriteOf(ObjectNode... members) {+ ArrayNode items = mapper.createArrayNode();+ for (ObjectNode member : members) {+ items.add(member);+ }+ ObjectNode request = mapper.createObjectNode();+ request.set("TransactItems", items);+ return request;+ }++ private String onlyValidationErrorMessage(Response response) {+ assertEquals(400, response.getStatus());+ JsonNode body = mapper.convertValue(response.getEntity(), JsonNode.class);+ assertEquals("TransactionCanceledException", body.get("__type").asText());+ assertEquals("Transaction cancelled, please refer cancellation reasons for specific reasons [ValidationError]",+ body.get("message").asText());+ JsonNode reason = body.get("CancellationReasons").get(0);+ assertEquals("ValidationError", reason.get("Code").asText());+ return reason.get("Message").asText();+ }++ @Test+ void transactWriteItemsCancelsOnAPutKeyOfTheWrongType() throws Exception {+ createUsersTable("eu-west-1");+ ObjectNode item = mapper.createObjectNode();+ item.set("userId", attributeValue("N", "5"));++ Response response = handler.handle("TransactWriteItems",+ transactWriteOf(transactMember("Put", "Users", "Item", item)), "eu-west-1");++ assertEquals("One or more parameter values were invalid: Type mismatch for key userId expected: S actual: N",+ onlyValidationErrorMessage(response));+ }++ @Test+ void transactWriteItemsCancelsOnAPutItemMissingItsKey() throws Exception {+ createUsersTable("eu-west-1");++ Response response = handler.handle("TransactWriteItems",+ transactWriteOf(transactMember("Put", "Users", "Item", item("name", "x"))), "eu-west-1");++ assertEquals("One or more parameter values were invalid: Missing the key userId in the item",+ onlyValidationErrorMessage(response));+ }++ @Test+ void transactWriteItemsCancelsOnADeleteKeyOfTheWrongTypeWithTheSchemaMessage() throws Exception {+ createUsersTable("eu-west-1");+ ObjectNode key = mapper.createObjectNode();+ key.set("userId", attributeValue("N", "5"));++ Response response = handler.handle("TransactWriteItems",+ transactWriteOf(transactMember("Delete", "Users", "Key", key)), "eu-west-1");++ assertEquals("The provided key element does not match the schema", onlyValidationErrorMessage(response));+ }++ @Test+ void transactWriteItemsCancelsOnAConditionCheckKeyMissingItsAttribute() throws Exception {+ createUsersTable("eu-west-1");++ Response response = handler.handle("TransactWriteItems",+ transactWriteOf(transactMember("ConditionCheck", "Users", "Key", mapper.createObjectNode())),+ "eu-west-1");++ assertEquals("The provided key element does not match the schema", onlyValidationErrorMessage(response));+ }++ // Checked against DynamoDB in eu-west-2: the failing condition and the duplicate pair report None.+ @Test+ void transactWriteItemsCancelsAtTheFirstKeyMismatchBeforeConditionsAndDuplicates() throws Exception {+ createUsersTable("eu-west-1");+ ObjectNode check = transactMember("ConditionCheck", "Users", "Key", item("userId", "u9"));+ ((ObjectNode) check.get("ConditionCheck")).put("ConditionExpression", "attribute_exists(userId)");+ ObjectNode firstWrong = mapper.createObjectNode();+ firstWrong.set("userId", attributeValue("N", "5"));+ ObjectNode secondWrong = mapper.createObjectNode();+ secondWrong.set("userId", attributeValue("N", "6"));++ Response response = handler.handle("TransactWriteItems", transactWriteOf(check,+ transactMember("Put", "Users", "Item", item("userId", "u1")),+ transactMember("Put", "Users", "Item", item("userId", "u1")),+ transactMember("Put", "Users", "Item", firstWrong),+ transactMember("Put", "Users", "Item", secondWrong)), "eu-west-1");++ assertEquals(400, response.getStatus());+ JsonNode body = mapper.convertValue(response.getEntity(), JsonNode.class);+ assertEquals("Transaction cancelled, please refer cancellation reasons for specific reasons "+ + "[None, None, None, ValidationError, None]", body.get("message").asText());+ assertEquals("One or more parameter values were invalid: Type mismatch for key userId expected: S actual: N",+ body.get("CancellationReasons").get(3).get("Message").asText());+ assertNull(service.getItem("Users", item("userId", "u1"), "eu-west-1"));+ }++ // Checked against DynamoDB in eu-west-2: a CREATING table is not found before any key is checked.+ @Test+ void transactWriteItemsAnswersResourceNotFoundForACreatingTableBeforeTheKeyCheck() {+ createUsersTable("eu-west-1").setTableStatus("CREATING");+ ObjectNode wrong = mapper.createObjectNode();+ wrong.set("userId", attributeValue("N", "5"));+ ObjectNode request = transactWriteOf(transactMember("Put", "Users", "Item", wrong));++ AwsException ex = assertThrows(AwsException.class,+ () -> handler.handle("TransactWriteItems", request, "eu-west-1"));+ assertEquals("ResourceNotFoundException", ex.getErrorCode());+ assertEquals("Requested resource not found", ex.getMessage());+ }++ @Test+ void transactWriteItemsFailsTheRequestOnAnEmptyKeyAheadOfAKeyMismatch() {+ createUsersTable("eu-west-1");+ ObjectNode wrong = mapper.createObjectNode();+ wrong.set("userId", attributeValue("N", "5"));+ ObjectNode request = transactWriteOf(+ transactMember("Put", "Users", "Item", item("userId", "")),+ transactMember("Put", "Users", "Item", wrong));++ AwsException ex = assertThrows(AwsException.class,+ () -> handler.handle("TransactWriteItems", request, "eu-west-1"));+ assertEquals("ValidationException", ex.getErrorCode());+ assertEquals("One or more parameter values are not valid. The AttributeValue for a key attribute "+ + "cannot contain an empty string value. Key: userId", ex.getMessage());+ }++ @Test+ void transactWriteItemsCancelsOnAPutIndexKeyOfTheWrongType() throws Exception {+ createProvisionedTableWithGsi("Movies", "eu-west-1");+ ObjectNode item = item("id", "m1");+ item.set("title", attributeValue("N", "1"));++ Response response = handler.handle("TransactWriteItems",+ transactWriteOf(transactMember("Put", "Movies", "Item", item)), "eu-west-1");++ assertEquals("One or more parameter values were invalid: Type mismatch for Index Key title "+ + "Expected: S Actual: N IndexName: TitleIndex", onlyValidationErrorMessage(response));+ }++ @Test+ void transactWriteItemsCancelsOnAnUpdateSettingAnIndexKeyOfTheWrongType() throws Exception {+ createProvisionedTableWithGsi("Movies", "eu-west-1");+ ObjectNode update = transactMember("Update", "Movies", "Key", item("id", "m1"));+ ((ObjectNode) update.get("Update")).put("UpdateExpression", "SET title = :t");+ ((ObjectNode) update.get("Update")).set("ExpressionAttributeValues",+ mapper.createObjectNode().set(":t", attributeValue("N", "1")));++ Response response = handler.handle("TransactWriteItems", transactWriteOf(update), "eu-west-1");++ assertEquals("One or more parameter values were invalid: Type mismatch for Index Key title "+ + "Expected: S Actual: N IndexName: TitleIndex", onlyValidationErrorMessage(response));+ assertNull(service.getItem("Movies", item("id", "m1"), "eu-west-1"));+ }++ @Test+ void executeTransactionCancelsOnAnUpdateSettingAnIndexKeyOfTheWrongType() throws Exception {+ createProvisionedTableWithGsi("Movies", "eu-west-1");+ ObjectNode putRequest = mapper.createObjectNode();+ putRequest.put("TableName", "Movies");+ putRequest.set("Item", item("id", "m1", "title", "Old"));+ handler.handle("PutItem", putRequest, "eu-west-1");+ ObjectNode statement = mapper.createObjectNode();+ statement.put("Statement", "UPDATE \"Movies\" SET title = ? WHERE id = ?");+ statement.set("Parameters", mapper.createArrayNode()+ .add(attributeValue("N", "1")).add(attributeValue("S", "m1")));+ ObjectNode request = mapper.createObjectNode();+ request.set("TransactStatements", mapper.createArrayNode().add(statement));++ Response response = handler.handle("ExecuteTransaction", request, "eu-west-1");++ assertEquals("One or more parameter values were invalid: Type mismatch for Index Key title "+ + "Expected: S Actual: N IndexName: TitleIndex", onlyValidationErrorMessage(response));+ assertEquals("Old", service.getItem("Movies", item("id", "m1"), "eu-west-1").get("title").get("S").asText());+ }}