Light, fluffy, and always free - The AWS Local Emulator alternative
fix(dynamodb): match AWS on UpdateItem results and request limits (#3447)
TATSUNO “Taz” Yasuhiro6d43e16d0efb
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbAttributeValueValidator.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbAttributeValueValidator.javaindex b937f1711d..b2cb835ed0 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbAttributeValueValidator.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbAttributeValueValidator.java@@ -15,6 +15,8 @@ final class DynamoDbAttributeValueValidator {private static final List<String> TYPES = List.of("S", "N", "B", "BOOL", "NULL", "SS", "NS", "BS", "L", "M");+ static final int MAX_NESTING_LEVELS = 32;+private DynamoDbAttributeValueValidator() {}static void validate(JsonNode value) {@@ -37,6 +39,61 @@ final class DynamoDbAttributeValueValidator {return types.getFirst();}+ // AWS counts a top-level attribute as level 1 and allows a leaf down to level 32.+ static void requireNestingWithinLimit(JsonNode attributes) {+ requireNestingWithinLimit(attributes, true);+ }++ // Batch and transact writes report the limit without the validation envelope.+ static void requireNestingWithinLimit(JsonNode attributes, boolean inValidationEnvelope) {+ if (!nestingWithinLimit(attributes)) {+ throw validationEx((inValidationEnvelope ? "1 validation error detected: " : "")+ + "Nesting Levels have exceeded supported limits: "+ + "Attributes in the item have nested levels beyond supported limit");+ }+ }++ // PartiQL reports the limit with this short wording, and a transaction turns it into a+ // cancellation reason.+ static final String NESTING_EXCEEDED = "Nesting Levels have exceeded supported limits";++ static void requireParameterNestingWithinLimit(JsonNode value) {+ if (!valueNestingWithinLimit(value)) {+ throw validationEx(NESTING_EXCEEDED);+ }+ }++ static boolean valueNestingWithinLimit(JsonNode value) {+ return depthOf(value) <= MAX_NESTING_LEVELS;+ }++ static boolean nestingWithinLimit(JsonNode attributes) {+ if (attributes == null || !attributes.isObject()) {+ return true;+ }+ for (var value : attributes) {+ if (!valueNestingWithinLimit(value)) {+ return false;+ }+ }+ return true;+ }++ private static int depthOf(JsonNode value) {+ if (!value.isObject()) {+ return 1;+ }+ var children = value.has("M") ? value.get("M") : value.get("L");+ if (children == null) {+ return 1;+ }+ var deepest = 0;+ for (var child : children) {+ deepest = Math.max(deepest, depthOf(child));+ }+ return 1 + deepest;+ }+private static void requireOneType(JsonNode value) {switch (typeOf(value)) {case "L" -> value.get("L").forEach(DynamoDbAttributeValueValidator::requireOneType);diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExpressionSize.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExpressionSize.javanew file mode 100644index 0000000000..7c5d389480--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExpressionSize.java@@ -0,0 +1,48 @@+package io.github.hectorvent.floci.services.dynamodb;++import io.github.hectorvent.floci.core.common.AwsException;++import java.nio.charset.StandardCharsets;++// AWS counts the raw expression text, before placeholders are resolved. Single-item write+// APIs wrap the error in the validation envelope, the others do not, and only the Scan+// FilterExpression and TransactWriteItems conditions report the size.+final class DynamoDbExpressionSize {++ static final int MAX_BYTES = 4096;++ private static final String EXCEEDED = "Expression size has exceeded the maximum allowed size;";++ private DynamoDbExpressionSize() {}++ static void checkWrite(String expression, String exprType) {+ if (exceeds(expression)) {+ throw validationEx("1 validation error detected: Invalid " + exprType + ": " + EXCEEDED);+ }+ }++ static void checkRead(String expression, String exprType) {+ if (exceeds(expression)) {+ throw validationEx("Invalid " + exprType + ": " + EXCEEDED);+ }+ }++ static void checkReadWithSize(String expression, String exprType) {+ if (exceeds(expression)) {+ throw validationEx("Invalid " + exprType + ": " + EXCEEDED+ + " expression size: " + byteLength(expression));+ }+ }++ private static boolean exceeds(String expression) {+ return expression != null && byteLength(expression) > MAX_BYTES;+ }++ private static int byteLength(String expression) {+ return expression.getBytes(StandardCharsets.UTF_8).length;+ }++ private static AwsException validationEx(String message) {+ return new AwsException("ValidationException", message, 400);+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.javaindex f4f2d50b7d..87579b6c04 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java@@ -411,8 +411,13 @@ public class DynamoDbJsonHandler {n + " validation error" + (n > 1 ? "s" : "") + " detected: "+ String.join("; ", validationErrors), 400);}+ DynamoDbExpressionSize.checkWrite(conditionExpression, "ConditionExpression");+ DynamoDbAttributeValueValidator.requireNestingWithinLimit(item);+ DynamoDbAttributeValueValidator.requireNestingWithinLimit(exprAttrValues);+ DynamoDbNumberUtils.requireStorable(exprAttrValues, true);JsonNode expected = request.has("Expected") ? request.get("Expected") : null;+ DynamoDbAttributeValueValidator.requireNestingWithinLimit(legacyValues(expected));String conditionalOperator = request.has("ConditionalOperator")? request.get("ConditionalOperator").asText() : "AND";@@ -482,6 +487,7 @@ public class DynamoDbJsonHandler {// Validate ProjectionExpression syntax and reserved words before item lookupif (projectionExpression != null) {+ DynamoDbExpressionSize.checkRead(projectionExpression, "ProjectionExpression");ProjectionEvaluator.validateSyntax(projectionExpression, "ProjectionExpression");DynamoDbReservedWords.check(projectionExpression, "ProjectionExpression");}@@ -539,6 +545,11 @@ public class DynamoDbJsonHandler {n + " validation error" + (n > 1 ? "s" : "") + " detected: "+ String.join("; ", delValidationErrors), 400);}+ DynamoDbExpressionSize.checkWrite(conditionExpression, "ConditionExpression");+ DynamoDbAttributeValueValidator.requireNestingWithinLimit(exprAttrValues);+ DynamoDbNumberUtils.requireStorable(exprAttrValues, true);+ DynamoDbAttributeValueValidator.requireNestingWithinLimit(+ legacyValues(request.has("Expected") ? request.get("Expected") : null));// EAN/EAV with no expression to reference them: AWS reports "can only be specified// when using expressions", not the "unused in expressions" wording (#2893).@@ -587,23 +598,30 @@ public class DynamoDbJsonHandler {String returnValues = request.path("ReturnValues").asText("NONE");String returnValuesOnConditionCheckFailure = request.path("ReturnValuesOnConditionCheckFailure").asText("NONE");- // Validate ReturnValues + ReturnConsumedCapacity together before table lookup- List<String> updValidationErrors = new ArrayList<>();+ // UpdateItem stops at the first invalid enum and reports one error, unlike PutItem+ // and DeleteItem which aggregate.String rccUpd = request.has("ReturnConsumedCapacity") ? request.get("ReturnConsumedCapacity").asText() : null;- if (rccUpd != null && !VALID_RETURN_CONSUMED_CAPACITY.contains(rccUpd)) {- updValidationErrors.add("Value '" + rccUpd + "' at 'returnConsumedCapacity' failed to satisfy constraint: "- + "Member must satisfy enum value set: [INDEXES, TOTAL, NONE]");- }if (!VALID_RETURN_VALUES_UPDATE.contains(returnValues)) {- updValidationErrors.add("Value '" + returnValues + "' at 'returnValues' failed to satisfy constraint: "- + "Member must satisfy enum value set: [NONE, ALL_OLD, ALL_NEW, UPDATED_OLD, UPDATED_NEW]");+ throw new AwsException("ValidationException",+ "1 validation error detected: Value '" + returnValues + "' at 'returnValues' "+ + "failed to satisfy constraint: "+ + "Member must satisfy enum value set: [NONE, ALL_OLD, ALL_NEW, UPDATED_OLD, UPDATED_NEW]", 400);}- if (!updValidationErrors.isEmpty()) {- int n = updValidationErrors.size();+ if (rccUpd != null && !VALID_RETURN_CONSUMED_CAPACITY.contains(rccUpd)) {throw new AwsException("ValidationException",- n + " validation error" + (n > 1 ? "s" : "") + " detected: "- + String.join("; ", updValidationErrors), 400);+ "1 validation error detected: Value '" + rccUpd + "' at 'returnConsumedCapacity' "+ + "failed to satisfy constraint: "+ + "Member must satisfy enum value set: [INDEXES, TOTAL, NONE]", 400);}+ DynamoDbExpressionSize.checkWrite(updateExpression, "UpdateExpression");+ DynamoDbExpressionSize.checkWrite(conditionExpression, "ConditionExpression");+ DynamoDbAttributeValueValidator.requireNestingWithinLimit(exprAttrValues);+ DynamoDbNumberUtils.requireStorable(exprAttrValues, true);+ var attributeUpdateValues = legacyValues(attributeUpdates);+ DynamoDbAttributeValueValidator.requireNestingWithinLimit(attributeUpdateValues);+ DynamoDbNumberUtils.requireStorable(attributeUpdateValues, true);+ DynamoDbAttributeValueValidator.requireNestingWithinLimit(+ legacyValues(request.has("Expected") ? request.get("Expected") : null));JsonNode updateData = attributeUpdates.isMissingNode() ? null : attributeUpdates;JsonNode expectedUpd = request.has("Expected") ? request.get("Expected") : null;@@ -674,14 +692,12 @@ public class DynamoDbJsonHandler {response.set("Attributes", result.newItem());} else if ("ALL_OLD" .equals(returnValues) && result.oldItem() != null) {response.set("Attributes", result.oldItem());- } else if ("UPDATED_NEW".equals(returnValues) && result.newItem() != null) {- // When oldItem is null (new item created), diff against the key so key- // attributes are excluded - matching AWS behavior where UPDATED_NEW- // returns only the attributes set by the expression.- JsonNode baseline = result.oldItem() != null ? result.oldItem() : key;- response.set("Attributes", getChangedAttributes(result.newItem(), baseline));- } else if ("UPDATED_OLD".equals(returnValues) && result.oldItem() != null) {- response.set("Attributes", getChangedAttributes(result.oldItem(), result.newItem()));+ } else if ("UPDATED_NEW".equals(returnValues) || "UPDATED_OLD".equals(returnValues)) {+ var updated = DynamoDbUpdatedAttributes.collect(result.touched(), "UPDATED_NEW".equals(returnValues));+ // AWS omits Attributes when no touched path has a value on that side.+ if (!updated.isEmpty()) {+ response.set("Attributes", updated);+ }}addItemCollectionMetrics(response, request, tableName, key, region);addWriteConsumedCapacity(response, request, tableName, region, result.oldItem(), result.newItem());@@ -812,29 +828,27 @@ public class DynamoDbJsonHandler {} catch (Exception ignored) {}}- private JsonNode getChangedAttributes(JsonNode preferredItem, JsonNode secondaryItem){- ObjectNode changedAttributes = objectMapper.createObjectNode();- Iterator<Map.Entry<String, JsonNode>> fields = preferredItem.fields();- while (fields.hasNext()) {- var entry = fields.next();- String attrName = entry.getKey();- JsonNode value = entry.getValue();+ private static final int MAX_TOTAL_SEGMENTS = 1_000_000;- if (secondaryItem.has(attrName)){- JsonNode secondaryValue = secondaryItem.get(attrName);- if (!value.equals(secondaryValue)){- changedAttributes.put(attrName, value);- }+ // The AttributeValues inside a legacy container. AttributeUpdates, Expected,+ // KeyConditions and QueryFilter carry them under Value or AttributeValueList.+ private ObjectNode legacyValues(JsonNode container) {+ var values = objectMapper.createObjectNode();+ if (container == null || !container.isObject()) {+ return values;+ }+ var i = 0;+ for (var spec : container) {+ if (spec.has("Value")) {+ values.set("v" + i++, spec.get("Value"));}- else {- changedAttributes.put(attrName, value);+ for (var value : spec.path("AttributeValueList")) {+ values.set("v" + i++, value);}}- return changedAttributes;+ return values;}- private static final int MAX_TOTAL_SEGMENTS = 1_000_000;-private static final Set<String> VALID_SELECT = Set.of("ALL_ATTRIBUTES", "ALL_PROJECTED_ATTRIBUTES", "SPECIFIC_ATTRIBUTES", "COUNT");@@ -912,6 +926,13 @@ public class DynamoDbJsonHandler {"Invalid KeyConditionExpression: The expression can not be empty;", 400);}+ DynamoDbExpressionSize.checkRead(keyConditionExpr, "KeyConditionExpression");+ DynamoDbExpressionSize.checkRead(filterExpr, "FilterExpression");+ DynamoDbExpressionSize.checkRead(projectionExpression, "ProjectionExpression");+ DynamoDbAttributeValueValidator.requireNestingWithinLimit(exprAttrValues);+ DynamoDbNumberUtils.requireStorable(exprAttrValues, false);+ DynamoDbAttributeValueValidator.requireNestingWithinLimit(legacyValues(keyConditions), false);+ DynamoDbAttributeValueValidator.requireNestingWithinLimit(legacyValues(queryFilter), false);ExpressionEvaluator.validateExpression(keyConditionExpr, "KeyConditionExpression", exprAttrNames, exprAttrValues);ExpressionEvaluator.validateExpression(filterExpr, "FilterExpression", exprAttrNames, exprAttrValues);ProjectionEvaluator.validateExpression(projectionExpression);@@ -1080,6 +1101,10 @@ public class DynamoDbJsonHandler {+ "Segment: " + segment + " is not less than TotalSegments: " + totalSegments, 400);}+ DynamoDbExpressionSize.checkReadWithSize(filterExpr, "FilterExpression");+ DynamoDbExpressionSize.checkRead(projectionExpressionScan, "ProjectionExpression");+ DynamoDbAttributeValueValidator.requireNestingWithinLimit(exprAttrValues);+ DynamoDbNumberUtils.requireStorable(exprAttrValues, false);ExpressionEvaluator.validateExpression(filterExpr, "FilterExpression", exprAttrNames, exprAttrValues);ProjectionEvaluator.validateExpression(projectionExpressionScan);@@ -1166,6 +1191,8 @@ public class DynamoDbJsonHandler {var entry = tables.next();List<JsonNode> writes = new ArrayList<>();for (JsonNode writeReq : entry.getValue()) {+ DynamoDbAttributeValueValidator.requireNestingWithinLimit(+ writeReq.path("PutRequest").get("Item"), false);writes.add(writeReq);}items.put(entry.getKey(), writes);@@ -1264,6 +1291,10 @@ public class DynamoDbJsonHandler {// Per-table check: max 100 keysfor (Map.Entry<String, JsonNode> entry : items.entrySet()) {+ var projection = entry.getValue().path("ProjectionExpression");+ if (projection.isTextual()) {+ DynamoDbExpressionSize.checkRead(projection.asText(), "ProjectionExpression");+ }JsonNode keysNode = entry.getValue().has("Keys") ? entry.getValue().get("Keys") : null;if (keysNode != null && keysNode.size() > 100) {throw new AwsException("ValidationException",@@ -1655,6 +1686,15 @@ public class DynamoDbJsonHandler {+ "Member must have length less than or equal to 100", 400);}+ for (JsonNode txItem : transactItemsNode) {+ for (JsonNode op : txItem) {+ DynamoDbExpressionSize.checkRead(op.path("UpdateExpression").textValue(), "UpdateExpression");+ DynamoDbExpressionSize.checkReadWithSize(op.path("ConditionExpression").textValue(),+ "ConditionExpression");+ DynamoDbAttributeValueValidator.requireNestingWithinLimit(op.get("Item"), false);+ }+ }+// Check 4MB total item size limitfinal int MAX_TRANSACTION_BYTES = 4 * 1024 * 1024;int totalSize = 0;@@ -1713,7 +1753,7 @@ public class DynamoDbJsonHandler {ObjectNode r = objectMapper.createObjectNode();r.put("Code", reason.code().isEmpty() ? "None" : reason.code());if (!reason.code().isEmpty()) {- r.put("Message", "The conditional request failed");+ r.put("Message", reason.message() != null ? reason.message() : "The conditional request failed");}if (reason.item() != null) {r.set("Item", reason.item());@@ -1742,6 +1782,11 @@ public class DynamoDbJsonHandler {+ "Member must have length less than or equal to 100", 400);}+ for (JsonNode txItem : transactItemsNode) {+ DynamoDbExpressionSize.checkRead(txItem.path("Get").path("ProjectionExpression").textValue(),+ "ProjectionExpression");+ }+Map<String, TableDefinition> tableCache = new HashMap<>();Set<String> seenGet = new HashSet<>();for (JsonNode txItem : transactItemsNode) {@@ -2617,12 +2662,13 @@ public class DynamoDbJsonHandler {throw new AwsException("ValidationException", "TransactStatements must not be empty", 400);}List<JsonNode> transactItems = new ArrayList<>();- for (JsonNode s : stmts) {- DynamoDbPartiQLParser.Stmt stmt = DynamoDbPartiQLParser.parse(- s.path("Statement").asText(), toPartiQLParams(s.path("Parameters")));- transactItems.add(partiQLHandler.toTransactItem(stmt, region));- }try {+ cancelOnTooDeepParameters(stmts);+ for (JsonNode s : stmts) {+ DynamoDbPartiQLParser.Stmt stmt = DynamoDbPartiQLParser.parse(+ s.path("Statement").asText(), toPartiQLParams(s.path("Parameters")));+ transactItems.add(partiQLHandler.toTransactItem(stmt, region));+ }dynamoDbService.transactWriteItems(transactItems, region);ObjectNode resp = objectMapper.createObjectNode();resp.set("Responses", objectMapper.createArrayNode());@@ -2635,7 +2681,8 @@ public class DynamoDbJsonHandler {for (TransactionCanceledException.CancellationReason reason : e.getCancellationReasons()) {ObjectNode r = objectMapper.createObjectNode();r.put("Code", reason.code().isEmpty() ? "None" : reason.code());- r.put("Message", reason.code().isEmpty() ? "" : "The conditional request failed");+ r.put("Message", reason.code().isEmpty() ? ""+ : reason.message() != null ? reason.message() : "The conditional request failed");if (reason.item() != null) {r.set("Item", reason.item());}@@ -2645,6 +2692,26 @@ public class DynamoDbJsonHandler {}}+ // AWS cancels the transaction, with that statement's reason, when a parameter is too deep.+ private void cancelOnTooDeepParameters(JsonNode stmts) {+ var reasons = new ArrayList<TransactionCanceledException.CancellationReason>();+ var cancelled = false;+ for (JsonNode s : stmts) {+ var tooDeep = false;+ for (var parameter : s.path("Parameters")) {+ tooDeep |= !DynamoDbAttributeValueValidator.valueNestingWithinLimit(parameter);+ }+ cancelled |= tooDeep;+ reasons.add(tooDeep+ ? new TransactionCanceledException.CancellationReason("ValidationError", null,+ DynamoDbAttributeValueValidator.NESTING_EXCEEDED)+ : new TransactionCanceledException.CancellationReason("", null));+ }+ if (cancelled) {+ throw new TransactionCanceledException(reasons);+ }+ }+private Response handleBatchExecuteStatement(JsonNode request, String region) {JsonNode stmts = request.path("Statements");if (stmts.isMissingNode() || !stmts.isArray() || stmts.isEmpty()) {diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbNumberUtils.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbNumberUtils.javaindex 9b69cb9b4f..5f6381f79e 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbNumberUtils.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbNumberUtils.java@@ -64,6 +64,71 @@ final class DynamoDbNumberUtils {return toNormalizedString(stripped);}+ // AWS checks every number in a request before the table lookup. Single-item write+ // APIs wrap the error in the validation envelope, Query and Scan do not.+ static void requireStorable(JsonNode attributes, boolean inValidationEnvelope) {+ if (attributes == null || !attributes.isObject()) {+ return;+ }+ for (var value : attributes) {+ requireStorableValue(value, inValidationEnvelope ? "1 validation error detected: " : "");+ }+ }++ private static void requireStorableValue(JsonNode value, String prefix) {+ if (!value.isObject()) {+ return;+ }+ if (value.has("N")) {+ requireStorableNumber(value.get("N").asText(), prefix);+ }+ for (var n : value.path("NS")) {+ requireStorableNumber(n.asText(), prefix);+ }+ for (var element : value.path("L")) {+ requireStorableValue(element, prefix);+ }+ for (var entry : value.path("M")) {+ requireStorableValue(entry, prefix);+ }+ }++ private static void requireStorableNumber(String numStr, String prefix) {+ BigDecimal bd;+ try {+ bd = new BigDecimal(numStr);+ } catch (NumberFormatException e) {+ throw new AwsException("ValidationException",+ "The parameter cannot be converted to a numeric value: " + numStr, 400);+ }+ var stripped = bd.stripTrailingZeros();+ if (stripped.precision() > 38) {+ throw new AwsException("ValidationException",+ prefix + "Attempting to store more than 38 significant digits in a Number", 400);+ }+ var abs = stripped.abs();+ if (abs.compareTo(MAX_ABS) >= 0) {+ throw new AwsException("ValidationException",+ prefix + "Number overflow. Attempting to store a number with magnitude larger than supported range", 400);+ }+ if (abs.signum() > 0 && abs.compareTo(MIN_ABS_NONZERO) < 0) {+ throw new AwsException("ValidationException",+ prefix + "Number underflow. Attempting to store a number with magnitude smaller than supported range", 400);+ }+ }++ // AWS answers with the overflow wording for any arithmetic result it cannot store,+ // a magnitude too small or too many significant digits included.+ static void checkArithmeticResult(BigDecimal result) {+ var stripped = result.stripTrailingZeros();+ var abs = stripped.abs();+ if (stripped.precision() > 38 || abs.compareTo(MAX_ABS) >= 0+ || (abs.signum() > 0 && abs.compareTo(MIN_ABS_NONZERO) < 0)) {+ throw new AwsException("ValidationException",+ "Number overflow. Attempting to store a number with magnitude larger than supported range", 400);+ }+ }+private static String toNormalizedString(BigDecimal bd) {// -0 -> 0if (bd.compareTo(BigDecimal.ZERO) == 0) {diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPartiQLParser.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPartiQLParser.javaindex f9a913636f..a8aa444e06 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPartiQLParser.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPartiQLParser.java@@ -170,6 +170,7 @@ public class DynamoDbPartiQLParser {static Stmt parse(String statement, List<JsonNode> parameters) {parameters.forEach(DynamoDbAttributeValueValidator::validate);+ parameters.forEach(DynamoDbAttributeValueValidator::requireParameterNestingWithinLimit);return new DynamoDbPartiQLParser(tokenize(statement.trim()), parameters).parseStmt();}diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.javaindex 3b59d19ca2..a5c5955259 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java@@ -814,9 +814,10 @@ public class DynamoDbService implements ResourceProvider {item = key.deepCopy();}+ var touchedPaths = new ArrayList<String>();// Apply UpdateExpression (modern format: "SET #n = :val, age = :age REMOVE attr")if (updateExpression != null) {- applyUpdateExpression(item, updateExpression, expressionAttrNames, expressionAttrValues);+ applyUpdateExpression(item, updateExpression, expressionAttrNames, expressionAttrValues, touchedPaths);}// Apply attribute updates (legacy format: AttributeUpdates)else if (attributeUpdates != null && attributeUpdates.isObject()) {@@ -824,6 +825,7 @@ public class DynamoDbService implements ResourceProvider {while (fields.hasNext()) {var entry = fields.next();String attrName = entry.getKey();+ touchedPaths.add(attrName);JsonNode update = entry.getValue();String action = update.has("Action") ? update.get("Action").asText() : "PUT";JsonNode value = update.get("Value");@@ -863,8 +865,10 @@ public class DynamoDbService implements ResourceProvider {BigDecimal delta = new BigDecimal(value.get("N").asText());BigDecimal current = curAttr != null && curAttr.has("N")? new BigDecimal(curAttr.get("N").asText()) : BigDecimal.ZERO;+ var sum = current.add(delta);+ DynamoDbNumberUtils.checkArithmeticResult(sum);ObjectNode numNode = objectMapper.createObjectNode();- numNode.put("N", current.add(delta).stripTrailingZeros().toPlainString());+ numNode.put("N", sum.stripTrailingZeros().toPlainString());item.set(attrName, numNode);} else {// Add elements to a set@@ -921,7 +925,13 @@ public class DynamoDbService implements ResourceProvider {streamEvent.run();}- return new UpdateResult(item, existing);+ var touched = new ArrayList<TouchedPath>();+ for (var path : touchedPaths) {+ var tokens = updateExpression != null ? parsePath(path, expressionAttrNames) : List.<Object>of(path);+ touched.add(new TouchedPath(tokens,+ existing == null ? null : valueAtTokens(existing, tokens), valueAtTokens(item, tokens)));+ }+ return new UpdateResult(item, existing, touched);});}@@ -1412,8 +1422,10 @@ public class DynamoDbService implements ResourceProvider {List<TransactionCanceledException.CancellationReason> cancellationReasons = new ArrayList<>();boolean hasFailed = false;for (JsonNode transactItem : transactItems) {- TransactionCanceledException.CancellationReason failReason =- evaluateTransactCondition(transactItem, region, staged);+ var failReason = transactUpdateValueTooDeep(transactItem);+ if (failReason == null) {+ failReason = evaluateTransactCondition(transactItem, region, staged);+ }if (failReason != null) {hasFailed = true;cancellationReasons.add(failReason);@@ -1516,6 +1528,17 @@ public class DynamoDbService implements ResourceProvider {return new TransactParticipant(storageKey, itemKey);}+ // AWS checks the depth of a transact Update's values as part of that member, so a too+ // deep value cancels the transaction instead of failing the request up front.+ private TransactionCanceledException.CancellationReason transactUpdateValueTooDeep(JsonNode transactItem) {+ var values = transactItem.path("Update").get("ExpressionAttributeValues");+ if (DynamoDbAttributeValueValidator.nestingWithinLimit(values)) {+ return null;+ }+ return new TransactionCanceledException.CancellationReason("ValidationError", null,+ "Nesting Levels have exceeded supported limits");+ }+private TransactionCanceledException.CancellationReason evaluateTransactCondition(JsonNode transactItem, String region) {return evaluateTransactCondition(transactItem, region, null);}@@ -2058,6 +2081,13 @@ public class DynamoDbService implements ResourceProvider {private void applyUpdateExpression(ObjectNode item, String expression,JsonNode exprAttrNames, JsonNode exprAttrValues) {+ applyUpdateExpression(item, expression, exprAttrNames, exprAttrValues, new ArrayList<>());+ }++ // Every action's target path is added to touched, which UPDATED_NEW and UPDATED_OLD read.+ private void applyUpdateExpression(ObjectNode item, String expression,+ JsonNode exprAttrNames, JsonNode exprAttrValues,+ List<String> touched) {// Parse SET and REMOVE clauses from expressions like:// "SET #n = :newName, age = :newAge REMOVE oldField"if (expression.isBlank()) {@@ -2077,16 +2107,16 @@ public class DynamoDbService implements ResourceProvider {String upper = remaining.toUpperCase();if (upper.startsWith("SET ")) {remaining = remaining.substring(4).trim();- remaining = applySetClause(item, remaining, exprAttrNames, exprAttrValues);+ remaining = applySetClause(item, remaining, exprAttrNames, exprAttrValues, touched);} else if (upper.startsWith("REMOVE ")) {remaining = remaining.substring(7).trim();- remaining = applyRemoveClause(item, remaining, exprAttrNames);+ remaining = applyRemoveClause(item, remaining, exprAttrNames, touched);} else if (upper.startsWith("ADD ")) {remaining = remaining.substring(4).trim();- remaining = applyAddClause(item, remaining, exprAttrNames, exprAttrValues);+ remaining = applyAddClause(item, remaining, exprAttrNames, exprAttrValues, touched);} else if (upper.startsWith("DELETE ")) {remaining = remaining.substring(7).trim();- remaining = applyDeleteClause(item, remaining, exprAttrNames, exprAttrValues);+ remaining = applyDeleteClause(item, remaining, exprAttrNames, exprAttrValues, touched);} else {// Unknown keyword — syntax errorString[] parts = remaining.split("\\s+", 3);@@ -2101,7 +2131,7 @@ public class DynamoDbService implements ResourceProvider {}private String applySetClause(ObjectNode item, String clause,- JsonNode exprAttrNames, JsonNode exprAttrValues) {+ JsonNode exprAttrNames, JsonNode exprAttrValues, List<String> touched) {// Parse comma-separated assignments: "attr = :val, #name = :val2"// Stop when we hit another clause keyword (REMOVE, ADD, DELETE) or endLOG.debugv("applySetClause: clause={0}, exprAttrNames={1}, exprAttrValues={2}",@@ -2124,6 +2154,7 @@ public class DynamoDbService implements ResourceProvider {if (eqIdx < 0) break;String attrPath = clause.substring(0, eqIdx).trim();+ touched.add(attrPath);String attrName = resolveAttributeName(attrPath, exprAttrNames);String rest = clause.substring(eqIdx + 1).trim();@@ -2176,6 +2207,7 @@ public class DynamoDbService implements ResourceProvider {BigDecimal left = new BigDecimal(leftVal.get("N").asText());BigDecimal right = new BigDecimal(rightVal.get("N").asText());BigDecimal result = (operator == '+') ? left.add(right) : left.subtract(right);+ DynamoDbNumberUtils.checkArithmeticResult(result);ObjectNode numNode = JsonNodeFactory.instance.objectNode();numNode.put("N", result.toPlainString());setValueAtPath(item, attrPath, numNode, exprAttrNames);@@ -2305,7 +2337,8 @@ public class DynamoDbService implements ResourceProvider {}}- private String applyRemoveClause(ObjectNode item, String clause, JsonNode exprAttrNames) {+ private String applyRemoveClause(ObjectNode item, String clause, JsonNode exprAttrNames,+ List<String> touched) {while (!clause.isEmpty()) {String upper = clause.toUpperCase();if (upper.startsWith("SET ") || upper.startsWith("ADD ") || upper.startsWith("DELETE ")) {@@ -2330,13 +2363,14 @@ public class DynamoDbService implements ResourceProvider {clause = "";}+ touched.add(attrPart);removeValueAtPath(item, attrPart, exprAttrNames);}return clause;}private String applyAddClause(ObjectNode item, String clause,- JsonNode exprAttrNames, JsonNode exprAttrValues) {+ JsonNode exprAttrNames, JsonNode exprAttrValues, List<String> touched) {while (!clause.isEmpty()) {String upper = clause.toUpperCase();if (upper.startsWith("SET ") || upper.startsWith("REMOVE ") || upper.startsWith("DELETE ")) {@@ -2348,6 +2382,7 @@ public class DynamoDbService implements ResourceProvider {if (parts.length < 2) break;String attrPath = parts[0];+ touched.add(attrPath);String valuePlaceholder = parts[1].replaceAll(",.*", "").trim();if (valuePlaceholder.startsWith(":") && exprAttrValues != null) {@@ -2395,7 +2430,9 @@ public class DynamoDbService implements ResourceProvider {try {BigDecimal existingNum = new BigDecimal(existingNumStr);BigDecimal addNum = new BigDecimal(addNumStr);- result.put("N", existingNum.add(addNum).toPlainString());+ var sum = existingNum.add(addNum);+ DynamoDbNumberUtils.checkArithmeticResult(sum);+ result.put("N", sum.toPlainString());return result;} catch (NumberFormatException e) {// Fall back to just setting the value@@ -2447,7 +2484,7 @@ public class DynamoDbService implements ResourceProvider {}private String applyDeleteClause(ObjectNode item, String clause,- JsonNode exprAttrNames, JsonNode exprAttrValues) {+ JsonNode exprAttrNames, JsonNode exprAttrValues, List<String> touched) {while (!clause.isEmpty()) {String upper = clause.toUpperCase();if (upper.startsWith("SET ") || upper.startsWith("REMOVE ") || upper.startsWith("ADD ") || upper.startsWith("DELETE ")) {@@ -2459,6 +2496,7 @@ public class DynamoDbService implements ResourceProvider {if (parts.length < 2) break;String attrPath = parts[0];+ touched.add(attrPath);String valuePlaceholder = parts[1].replaceAll(",.*", "").trim();if (valuePlaceholder.startsWith(":") && exprAttrValues != null) {@@ -2751,7 +2789,10 @@ public class DynamoDbService implements ResourceProvider {}private JsonNode getValueAtPath(JsonNode item, String path, JsonNode exprAttrNames) {- List<Object> tokens = parsePath(path, exprAttrNames);+ return valueAtTokens(item, parsePath(path, exprAttrNames));+ }++ private JsonNode valueAtTokens(JsonNode item, List<Object> tokens) {if (tokens.isEmpty()) return null;JsonNode current = item;for (int i = 0; i < tokens.size(); i++) {@@ -3205,21 +3246,29 @@ public class DynamoDbService implements ResourceProvider {+ " IndexName: " + indexName, 400);}if (attr.has("S") && attr.get("S").asText().isEmpty()) {- // AWS uses different wording for UpdateItem than for writes of a whole item.- if (isUpdate) {- throw new AwsException("ValidationException",- "One or more parameter values are not valid. The update expression attempted to "- + "update a secondary index key to a value that is not supported. "- + "The AttributeValue for a key attribute cannot contain an empty string value.", 400);- }- throw new AwsException("ValidationException",- "One or more parameter values are not valid. A value specified for a secondary "- + "index key is not supported. The AttributeValue for a key attribute cannot "- + "contain an empty string value. IndexName: " + indexName + ", IndexKey: " + attrName, 400);+ throw emptyIndexKeyValue("empty string value", indexName, attrName, isUpdate);+ }+ if (attr.has("B") && attr.get("B").asText().isEmpty()) {+ throw emptyIndexKeyValue("empty binary value", indexName, attrName, isUpdate);}}}+ // AWS uses different wording for UpdateItem than for writes of a whole item.+ private static AwsException emptyIndexKeyValue(String what, String indexName, String attrName,+ boolean isUpdate) {+ if (isUpdate) {+ return new AwsException("ValidationException",+ "One or more parameter values are not valid. The update expression attempted to "+ + "update a secondary index key to a value that is not supported. "+ + "The AttributeValue for a key attribute cannot contain an " + what + ".", 400);+ }+ return new AwsException("ValidationException",+ "One or more parameter values are not valid. A value specified for a secondary "+ + "index key is not supported. The AttributeValue for a key attribute cannot "+ + "contain an " + what + ". IndexName: " + indexName + ", IndexKey: " + attrName, 400);+ }+private String buildItemKeyFromNode(JsonNode item, String pkName, String skName) {return buildItemKeyFromNode(item, pkName, skName == null ? List.of() : List.of(skName));}@@ -3465,7 +3514,10 @@ public class DynamoDbService implements ResourceProvider {return table;}- public record UpdateResult(JsonNode newItem, JsonNode oldItem) {}+ public record UpdateResult(JsonNode newItem, JsonNode oldItem, List<TouchedPath> touched) {}++ // One path the update expression acted on, with the value there before and after.+ public record TouchedPath(List<Object> tokens, JsonNode oldValue, JsonNode newValue) {}// scannedBytes carries the pre-filter size of the read items: DynamoDB bills a// Query or Scan on what it read, not on what survived the filter or projection.diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbUpdatedAttributes.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbUpdatedAttributes.javanew file mode 100644index 0000000000..616c787b7a--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbUpdatedAttributes.java@@ -0,0 +1,68 @@+package io.github.hectorvent.floci.services.dynamodb;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.node.JsonNodeFactory;+import com.fasterxml.jackson.databind.node.ObjectNode;++import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;+import java.util.TreeMap;++// UPDATED_NEW and UPDATED_OLD return the value at every path the update touched, whether+// it changed or not. Nested paths come back as a fragment along the path, and touched list+// elements are packed in index order.+final class DynamoDbUpdatedAttributes {++ private DynamoDbUpdatedAttributes() {}++ static ObjectNode collect(List<DynamoDbService.TouchedPath> touched, boolean newValues) {+ var root = new Node();+ for (var path : touched) {+ var value = newValues ? path.newValue() : path.oldValue();+ if (value != null) {+ root.insert(path.tokens(), 0, value);+ }+ }+ var out = JsonNodeFactory.instance.objectNode();+ root.mapChildren.forEach((name, child) -> out.set(name, child.render()));+ return out;+ }++ private static final class Node {+ private JsonNode whole;+ private final Map<String, Node> mapChildren = new LinkedHashMap<>();+ private final TreeMap<Long, Node> listChildren = new TreeMap<>();++ void insert(List<Object> tokens, int i, JsonNode value) {+ if (whole != null) {+ return;+ }+ if (i == tokens.size()) {+ whole = value;+ mapChildren.clear();+ listChildren.clear();+ return;+ }+ var child = tokens.get(i) instanceof Long index+ ? listChildren.computeIfAbsent(index, ignored -> new Node())+ : mapChildren.computeIfAbsent((String) tokens.get(i), ignored -> new Node());+ child.insert(tokens, i + 1, value);+ }++ JsonNode render() {+ if (whole != null) {+ return whole;+ }+ var out = JsonNodeFactory.instance.objectNode();+ if (!listChildren.isEmpty()) {+ var list = out.putArray("L");+ listChildren.values().forEach(child -> list.add(child.render()));+ } else {+ var map = out.putObject("M");+ mapChildren.forEach((name, child) -> map.set(name, child.render()));+ }+ return out;+ }+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/TransactionCanceledException.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/TransactionCanceledException.javaindex 489d90539c..5e2bcf852f 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/TransactionCanceledException.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/TransactionCanceledException.java@@ -7,7 +7,11 @@ import java.util.List;public class TransactionCanceledException extends AwsException {- public record CancellationReason(String code, JsonNode item) {}+ public record CancellationReason(String code, JsonNode item, String message) {+ public CancellationReason(String code, JsonNode item) {+ this(code, item, null);+ }+ }private final List<CancellationReason> cancellationReasons;diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandlerTest.javaindex 6bbacf0ded..fc529cc206 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandlerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandlerTest.java@@ -1105,4 +1105,578 @@ class DynamoDbJsonHandlerTest {assertEquals(1, userSegments.get("alice").size(), "All alice items must belong to the exact same segment");assertEquals(1, userSegments.get("bob").size(), "All bob items must belong to the exact same segment");}++ private ObjectNode updateUserRequest() {+ var request = mapper.createObjectNode();+ request.put("TableName", "Users");+ request.set("Key", mapper.createObjectNode().set("userId", attributeValue("S", "u1")));+ return request;+ }++ @Test+ void updateItemUpdatedNewReturnsOnlyTheChangedMapFragment() throws Exception {+ createUsersTable("eu-west-1");+ var parent = mapper.createObjectNode();+ parent.set("keep", attributeValue("S", "k"));+ parent.set("child", attributeValue("S", "old"));+ var putRequest = mapper.createObjectNode();+ putRequest.put("TableName", "Users");+ var item = mapper.createObjectNode();+ item.set("userId", attributeValue("S", "u1"));+ item.set("parent", mapper.createObjectNode().set("M", parent));+ putRequest.set("Item", item);+ handler.handle("PutItem", putRequest, "eu-west-1");++ var request = updateUserRequest();+ request.put("UpdateExpression", "SET parent.child = :v");+ request.set("ExpressionAttributeValues",+ mapper.createObjectNode().set(":v", attributeValue("S", "new")));+ request.put("ReturnValues", "UPDATED_NEW");++ var response = handler.handle("UpdateItem", request, "eu-west-1");+ var body = mapper.convertValue(response.getEntity(), JsonNode.class);+ var attributes = body.get("Attributes");+ assertEquals("new", attributes.get("parent").get("M").get("child").get("S").asText());+ assertFalse(attributes.get("parent").get("M").has("keep"));+ }++ @Test+ void updateItemRemoveWithUpdatedNewOmitsAttributes() throws Exception {+ createUsersTable("eu-west-1");+ var putRequest = mapper.createObjectNode();+ putRequest.put("TableName", "Users");+ var item = mapper.createObjectNode();+ item.set("userId", attributeValue("S", "u1"));+ item.set("y", attributeValue("S", "drop"));+ putRequest.set("Item", item);+ handler.handle("PutItem", putRequest, "eu-west-1");++ var request = updateUserRequest();+ request.put("UpdateExpression", "REMOVE y");+ request.put("ReturnValues", "UPDATED_NEW");++ var response = handler.handle("UpdateItem", request, "eu-west-1");+ var body = mapper.convertValue(response.getEntity(), JsonNode.class);+ assertFalse(body.has("Attributes"));+ }++ @Test+ void updateItemReportsOnlyTheFirstInvalidEnum() {+ createUsersTable("eu-west-1");+ var request = updateUserRequest();+ request.put("ReturnValues", "INVALID");+ request.put("ReturnConsumedCapacity", "INVALID");++ var ex = assertThrows(AwsException.class,+ () -> handler.handle("UpdateItem", request, "eu-west-1"));+ assertEquals("ValidationException", ex.getErrorCode());+ assertTrue(ex.getMessage().startsWith("1 validation error detected:"), ex.getMessage());+ assertTrue(ex.getMessage().contains("'returnValues'"), ex.getMessage());+ }++ private static String nameOfBytes(int bytes) {+ return "a".repeat(bytes);+ }++ private ObjectNode singleValue(String placeholder) {+ return (ObjectNode) mapper.createObjectNode().set(placeholder, attributeValue("S", "x"));+ }++ @Test+ void updateItemRejectsAnUpdateExpressionOver4096BytesBeforeTheTableLookup() {+ var request = updateUserRequest();+ request.put("TableName", "Missing");+ request.put("UpdateExpression", "SET " + nameOfBytes(4088) + " = :v");+ request.set("ExpressionAttributeValues", singleValue(":v"));++ var ex = assertThrows(AwsException.class,+ () -> handler.handle("UpdateItem", request, "eu-west-1"));+ assertEquals("ValidationException", ex.getErrorCode());+ assertEquals("1 validation error detected: Invalid UpdateExpression: "+ + "Expression size has exceeded the maximum allowed size;", ex.getMessage());+ }++ @Test+ void updateItemAcceptsAnUpdateExpressionOfExactly4096Bytes() throws Exception {+ createUsersTable("eu-west-1");+ var request = updateUserRequest();+ request.put("UpdateExpression", "SET " + nameOfBytes(4087) + " = :v");+ request.set("ExpressionAttributeValues", singleValue(":v"));++ var response = handler.handle("UpdateItem", request, "eu-west-1");+ assertEquals(200, response.getStatus());+ }++ @Test+ void putItemRejectsAConditionExpressionOver4096Bytes() {+ createUsersTable("eu-west-1");+ var request = mapper.createObjectNode();+ request.put("TableName", "Users");+ request.set("Item", item("userId", "u1"));+ request.put("ConditionExpression", "attribute_not_exists(" + nameOfBytes(4075) + ")");++ var ex = assertThrows(AwsException.class,+ () -> handler.handle("PutItem", request, "eu-west-1"));+ assertEquals("1 validation error detected: Invalid ConditionExpression: "+ + "Expression size has exceeded the maximum allowed size;", ex.getMessage());+ }++ @Test+ void queryRejectsAFilterExpressionOver4096Bytes() {+ createUsersTable("eu-west-1");+ var request = mapper.createObjectNode();+ request.put("TableName", "Users");+ request.put("KeyConditionExpression", "userId = :pk");+ request.put("FilterExpression", nameOfBytes(4092) + " = :v");+ var values = singleValue(":pk");+ values.set(":v", attributeValue("S", "x"));+ request.set("ExpressionAttributeValues", values);++ var ex = assertThrows(AwsException.class,+ () -> handler.handle("Query", request, "eu-west-1"));+ assertEquals("Invalid FilterExpression: Expression size has exceeded the maximum allowed size;",+ ex.getMessage());+ }++ @Test+ void scanRejectsAFilterExpressionOver4096BytesAndReportsTheSize() {+ createUsersTable("eu-west-1");+ var request = mapper.createObjectNode();+ request.put("TableName", "Users");+ request.put("FilterExpression", nameOfBytes(4092) + " = :v");+ request.set("ExpressionAttributeValues", singleValue(":v"));++ var ex = assertThrows(AwsException.class,+ () -> handler.handle("Scan", request, "eu-west-1"));+ assertEquals("Invalid FilterExpression: Expression size has exceeded the maximum allowed size; "+ + "expression size: 4097", ex.getMessage());+ }++ @Test+ void getItemRejectsAProjectionExpressionOver4096Bytes() {+ createUsersTable("eu-west-1");+ var request = updateUserRequest();+ request.put("ProjectionExpression", nameOfBytes(4097));++ var ex = assertThrows(AwsException.class,+ () -> handler.handle("GetItem", request, "eu-west-1"));+ assertEquals("Invalid ProjectionExpression: Expression size has exceeded the maximum allowed size;",+ ex.getMessage());+ }++ private ObjectNode nestedMaps(int levels) {+ ObjectNode value = attributeValue("S", "leaf");+ for (var i = 0; i < levels; i++) {+ var map = mapper.createObjectNode();+ map.set("n", value);+ value = mapper.createObjectNode();+ value.set("M", map);+ }+ return value;+ }++ private ObjectNode putRequest(JsonNode value) {+ var request = mapper.createObjectNode();+ request.put("TableName", "Users");+ var item = item("userId", "u1");+ item.set("data", value);+ request.set("Item", item);+ return request;+ }++ private static final String NESTING_MESSAGE = "1 validation error detected: Nesting Levels have exceeded "+ + "supported limits: Attributes in the item have nested levels beyond supported limit";++ @Test+ void putItemAcceptsAnAttributeWithALeafAtLevel32() throws Exception {+ createUsersTable("eu-west-1");++ var response = handler.handle("PutItem", putRequest(nestedMaps(31)), "eu-west-1");+ assertEquals(200, response.getStatus());+ }++ @Test+ void putItemRejectsAnAttributeWithALeafAtLevel33BeforeTheTableLookup() {+ var request = putRequest(nestedMaps(32));+ request.put("TableName", "Missing");++ var ex = assertThrows(AwsException.class,+ () -> handler.handle("PutItem", request, "eu-west-1"));+ assertEquals("ValidationException", ex.getErrorCode());+ assertEquals(NESTING_MESSAGE, ex.getMessage());+ }++ @Test+ void updateItemRejectsAnExpressionAttributeValueWithALeafAtLevel33() {+ createUsersTable("eu-west-1");+ var request = updateUserRequest();+ request.put("UpdateExpression", "SET touched = :t");+ request.put("ConditionExpression", "#d = :deep");+ request.set("ExpressionAttributeNames", mapper.createObjectNode().put("#d", "data"));+ var values = singleValue(":t");+ values.set(":deep", nestedMaps(32));+ request.set("ExpressionAttributeValues", values);++ var ex = assertThrows(AwsException.class,+ () -> handler.handle("UpdateItem", request, "eu-west-1"));+ assertEquals(NESTING_MESSAGE, ex.getMessage());+ }++ private ObjectNode transactWrite(String action, ObjectNode op) {+ op.put("TableName", "Users");+ var member = mapper.createObjectNode();+ member.set(action, op);+ var request = mapper.createObjectNode();+ request.set("TransactItems", mapper.createArrayNode().add(member));+ return request;+ }++ @Test+ void transactWriteItemsRejectsAConditionExpressionOver4096BytesWithTheSizeBeforeTheTableLookup() {+ var put = mapper.createObjectNode();+ put.set("Item", item("userId", "u1"));+ put.put("ConditionExpression", "attribute_not_exists(" + nameOfBytes(4075) + ")");+ var request = transactWrite("Put", put);+ ((ObjectNode) request.get("TransactItems").get(0).get("Put")).put("TableName", "Missing");++ var ex = assertThrows(AwsException.class,+ () -> handler.handle("TransactWriteItems", request, "eu-west-1"));+ assertEquals("ValidationException", ex.getErrorCode());+ assertEquals("Invalid ConditionExpression: Expression size has exceeded the maximum allowed size; "+ + "expression size: 4097", ex.getMessage());+ }++ @Test+ void transactWriteItemsRejectsAnUpdateExpressionOver4096Bytes() {+ createUsersTable("eu-west-1");+ var update = mapper.createObjectNode();+ update.set("Key", item("userId", "u1"));+ update.put("UpdateExpression", "SET " + nameOfBytes(4088) + " = :v");+ update.set("ExpressionAttributeValues", singleValue(":v"));++ var ex = assertThrows(AwsException.class,+ () -> handler.handle("TransactWriteItems", transactWrite("Update", update), "eu-west-1"));+ assertEquals("Invalid UpdateExpression: Expression size has exceeded the maximum allowed size;",+ ex.getMessage());+ }++ @Test+ void transactWriteItemsRejectsAPutItemWithALeafAtLevel33() {+ createUsersTable("eu-west-1");+ var put = mapper.createObjectNode();+ var item = item("userId", "u1");+ item.set("data", nestedMaps(32));+ put.set("Item", item);++ var ex = assertThrows(AwsException.class,+ () -> handler.handle("TransactWriteItems", transactWrite("Put", put), "eu-west-1"));+ assertEquals("Nesting Levels have exceeded supported limits: "+ + "Attributes in the item have nested levels beyond supported limit", ex.getMessage());+ }++ @Test+ void batchWriteItemRejectsAPutItemWithALeafAtLevel33BeforeTheTableLookup() {+ var item = item("userId", "u1");+ item.set("data", nestedMaps(32));+ var putRequest = mapper.createObjectNode();+ putRequest.set("PutRequest", mapper.createObjectNode().set("Item", item));+ var request = mapper.createObjectNode();+ request.set("RequestItems", mapper.createObjectNode().set("Missing", mapper.createArrayNode().add(putRequest)));++ var ex = assertThrows(AwsException.class,+ () -> handler.handle("BatchWriteItem", request, "eu-west-1"));+ assertEquals("ValidationException", ex.getErrorCode());+ assertEquals("Nesting Levels have exceeded supported limits: "+ + "Attributes in the item have nested levels beyond supported limit", ex.getMessage());+ }++ @Test+ void transactWriteItemsCancelsOnAnUpdateValueWithALeafAtLevel33() throws Exception {+ createUsersTable("eu-west-1");+ var update = mapper.createObjectNode();+ update.set("Key", item("userId", "u1"));+ update.put("UpdateExpression", "SET deep = :deep");+ update.set("ExpressionAttributeValues", mapper.createObjectNode().set(":deep", nestedMaps(32)));++ var response = handler.handle("TransactWriteItems", transactWrite("Update", update), "eu-west-1");+ assertEquals(400, response.getStatus());+ var body = mapper.convertValue(response.getEntity(), JsonNode.class);+ assertEquals("TransactionCanceledException", body.get("__type").asText());+ var reason = body.get("CancellationReasons").get(0);+ assertEquals("ValidationError", reason.get("Code").asText());+ assertEquals("Nesting Levels have exceeded supported limits", reason.get("Message").asText());+ assertNull(service.getItem("Users", item("userId", "u1"), "eu-west-1"));+ }++ @Test+ void transactWriteItemsDoesNotCheckTheDepthOfAConditionCheckValue() throws Exception {+ createUsersTable("eu-west-1");+ var putRequest = mapper.createObjectNode();+ putRequest.put("TableName", "Users");+ putRequest.set("Item", item("userId", "u1", "marker", "x"));+ handler.handle("PutItem", putRequest, "eu-west-1");+ var check = mapper.createObjectNode();+ check.set("Key", item("userId", "u1"));+ check.put("ConditionExpression", "#d = :deep");+ check.set("ExpressionAttributeNames", mapper.createObjectNode().put("#d", "data"));+ check.set("ExpressionAttributeValues", mapper.createObjectNode().set(":deep", nestedMaps(32)));++ var response = handler.handle("TransactWriteItems", transactWrite("ConditionCheck", check), "eu-west-1");+ assertEquals(400, response.getStatus());+ var reason = mapper.convertValue(response.getEntity(), JsonNode.class).get("CancellationReasons").get(0);+ assertEquals("ConditionalCheckFailed", reason.get("Code").asText());+ assertEquals("The conditional request failed", reason.get("Message").asText());+ }++ @Test+ void updateItemRejectsAnOutOfRangeAddOperandBeforeTheTableLookup() {+ var request = updateUserRequest();+ request.put("TableName", "Missing");+ request.put("UpdateExpression", "ADD n :a");+ request.set("ExpressionAttributeValues", mapper.createObjectNode().set(":a", attributeValue("N", "1e126")));++ var ex = assertThrows(AwsException.class,+ () -> handler.handle("UpdateItem", request, "eu-west-1"));+ assertEquals("ValidationException", ex.getErrorCode());+ assertEquals("1 validation error detected: Number overflow. "+ + "Attempting to store a number with magnitude larger than supported range", ex.getMessage());+ }++ @Test+ void updateItemRejectsATooSmallExpressionAttributeValue() {+ createUsersTable("eu-west-1");+ var request = updateUserRequest();+ request.put("UpdateExpression", "SET x = :a");+ request.set("ExpressionAttributeValues", mapper.createObjectNode().set(":a", attributeValue("N", "1e-131")));++ var ex = assertThrows(AwsException.class,+ () -> handler.handle("UpdateItem", request, "eu-west-1"));+ assertEquals("1 validation error detected: Number underflow. "+ + "Attempting to store a number with magnitude smaller than supported range", ex.getMessage());+ }++ @Test+ void updateItemRejectsAnAttributeUpdatesValueWithTooManyDigits() {+ createUsersTable("eu-west-1");+ var request = updateUserRequest();+ var update = mapper.createObjectNode();+ update.put("Action", "ADD");+ update.set("Value", attributeValue("N", "123456789012345678901234567890123456789"));+ request.set("AttributeUpdates", mapper.createObjectNode().set("n", update));++ var ex = assertThrows(AwsException.class,+ () -> handler.handle("UpdateItem", request, "eu-west-1"));+ assertEquals("1 validation error detected: Attempting to store more than 38 significant digits in a Number",+ ex.getMessage());+ }++ @Test+ void queryRejectsAnOutOfRangeExpressionAttributeValueWithoutTheEnvelope() {+ createUsersTable("eu-west-1");+ var request = mapper.createObjectNode();+ request.put("TableName", "Users");+ request.put("KeyConditionExpression", "userId = :p");+ request.put("FilterExpression", "n = :a");+ var values = singleValue(":p");+ values.set(":a", attributeValue("N", "1e126"));+ request.set("ExpressionAttributeValues", values);++ var ex = assertThrows(AwsException.class,+ () -> handler.handle("Query", request, "eu-west-1"));+ assertEquals("Number overflow. Attempting to store a number with magnitude larger than supported range",+ ex.getMessage());+ }++ @Test+ void updateItemRejectsAnAttributeUpdatesValueWithALeafAtLevel33BeforeTheTableLookup() {+ var request = updateUserRequest();+ request.put("TableName", "Missing");+ var update = mapper.createObjectNode();+ update.put("Action", "PUT");+ update.set("Value", nestedMaps(32));+ request.set("AttributeUpdates", mapper.createObjectNode().set("data", update));++ var ex = assertThrows(AwsException.class,+ () -> handler.handle("UpdateItem", request, "eu-west-1"));+ assertEquals("ValidationException", ex.getErrorCode());+ assertEquals(NESTING_MESSAGE, ex.getMessage());+ }++ @Test+ void putItemRejectsAnExpectedValueWithALeafAtLevel33() {+ createUsersTable("eu-west-1");+ var request = putRequest(attributeValue("S", "x"));+ request.set("Expected", mapper.createObjectNode().set("data",+ mapper.createObjectNode().set("Value", nestedMaps(32))));++ var ex = assertThrows(AwsException.class,+ () -> handler.handle("PutItem", request, "eu-west-1"));+ assertEquals(NESTING_MESSAGE, ex.getMessage());+ }++ @Test+ void queryRejectsAQueryFilterValueWithALeafAtLevel33WithoutTheEnvelope() {+ createUsersTable("eu-west-1");+ var request = mapper.createObjectNode();+ request.put("TableName", "Users");+ var keyCondition = mapper.createObjectNode();+ keyCondition.put("ComparisonOperator", "EQ");+ keyCondition.set("AttributeValueList", mapper.createArrayNode().add(attributeValue("S", "u1")));+ request.set("KeyConditions", mapper.createObjectNode().set("userId", keyCondition));+ var filter = mapper.createObjectNode();+ filter.put("ComparisonOperator", "EQ");+ filter.set("AttributeValueList", mapper.createArrayNode().add(nestedMaps(32)));+ request.set("QueryFilter", mapper.createObjectNode().set("data", filter));++ var ex = assertThrows(AwsException.class,+ () -> handler.handle("Query", request, "eu-west-1"));+ assertEquals("Nesting Levels have exceeded supported limits: "+ + "Attributes in the item have nested levels beyond supported limit", ex.getMessage());+ }++ private void seedUpdateReturnValuesItem() throws Exception {+ createUsersTable("eu-west-1");+ var parent = mapper.createObjectNode();+ parent.set("keep", attributeValue("S", "k"));+ parent.set("child", attributeValue("S", "old"));+ var list = mapper.createArrayNode()+ .add(attributeValue("S", "l0")).add(attributeValue("S", "l1")).add(attributeValue("S", "l2"));+ var item = item("userId", "u1");+ item.set("parent", mapper.createObjectNode().set("M", parent));+ item.set("l", mapper.createObjectNode().set("L", list));+ var putRequest = mapper.createObjectNode();+ putRequest.put("TableName", "Users");+ putRequest.set("Item", item);+ handler.handle("PutItem", putRequest, "eu-west-1");+ }++ private JsonNode updateAttributes(String expression, ObjectNode values, String returnValues) throws Exception {+ var request = updateUserRequest();+ request.put("UpdateExpression", expression);+ if (values != null) {+ request.set("ExpressionAttributeValues", values);+ }+ request.put("ReturnValues", returnValues);+ var body = mapper.convertValue(handler.handle("UpdateItem", request, "eu-west-1").getEntity(), JsonNode.class);+ return body.get("Attributes");+ }++ @Test+ void updateItemUpdatedNewReturnsTheWholeMapWhenTheMapItselfIsSet() throws Exception {+ seedUpdateReturnValuesItem();+ var parent = mapper.createObjectNode();+ parent.set("keep", attributeValue("S", "k"));+ parent.set("child", attributeValue("S", "new"));+ var values = mapper.createObjectNode();+ values.set(":v", mapper.createObjectNode().set("M", parent));++ var attributes = updateAttributes("SET parent = :v", values, "UPDATED_NEW");+ assertEquals("k", attributes.get("parent").get("M").get("keep").get("S").asText());+ assertEquals("new", attributes.get("parent").get("M").get("child").get("S").asText());+ }++ @Test+ void updateItemUpdatedOldReturnsTheWholeOldMapWhenTheMapItselfIsSet() throws Exception {+ seedUpdateReturnValuesItem();+ var values = mapper.createObjectNode();+ values.set(":v", mapper.createObjectNode().set("M", mapper.createObjectNode().set("child", attributeValue("S", "new"))));++ var attributes = updateAttributes("SET parent = :v", values, "UPDATED_OLD");+ assertEquals("k", attributes.get("parent").get("M").get("keep").get("S").asText());+ assertEquals("old", attributes.get("parent").get("M").get("child").get("S").asText());+ }++ @Test+ void updateItemUpdatedNewReturnsANestedSetEvenWhenTheValueDidNotChange() throws Exception {+ seedUpdateReturnValuesItem();++ var attributes = updateAttributes("SET parent.child = :v", singleValueOf("old"), "UPDATED_NEW");+ assertEquals("old", attributes.get("parent").get("M").get("child").get("S").asText());+ assertFalse(attributes.get("parent").get("M").has("keep"));+ }++ @Test+ void updateItemUpdatedNewPacksTouchedListElementsInIndexOrder() throws Exception {+ seedUpdateReturnValuesItem();+ var values = mapper.createObjectNode();+ values.set(":v", attributeValue("S", "L2"));+ values.set(":w", attributeValue("S", "L0"));++ var attributes = updateAttributes("SET l[2] = :v, l[0] = :w", values, "UPDATED_NEW");+ var packed = attributes.get("l").get("L");+ assertEquals(2, packed.size());+ assertEquals("L0", packed.get(0).get("S").asText());+ assertEquals("L2", packed.get(1).get("S").asText());+ }++ @Test+ void updateItemUpdatedNewAfterRemovingAListElementReturnsTheElementNowAtThatIndex() throws Exception {+ seedUpdateReturnValuesItem();++ var attributes = updateAttributes("REMOVE l[1]", null, "UPDATED_NEW");+ assertEquals(1, attributes.get("l").get("L").size());+ assertEquals("l2", attributes.get("l").get("L").get(0).get("S").asText());+ }++ @Test+ void updateItemUpdatedNewOmitsAttributesWhenAppendingPastTheEndOfAList() throws Exception {+ seedUpdateReturnValuesItem();++ assertNull(updateAttributes("SET l[5] = :v", singleValueOf("L5"), "UPDATED_NEW"));+ }++ private ObjectNode singleValueOf(String value) {+ return (ObjectNode) mapper.createObjectNode().set(":v", attributeValue("S", value));+ }++ @Test+ void transactGetItemsRejectsAnOversizedProjectionBeforeTheTableLookup() {+ var get = mapper.createObjectNode();+ get.put("TableName", "Missing");+ get.set("Key", item("userId", "u1"));+ get.put("ProjectionExpression", nameOfBytes(4097));+ var request = mapper.createObjectNode();+ request.set("TransactItems", mapper.createArrayNode().add(mapper.createObjectNode().set("Get", get)));++ var ex = assertThrows(AwsException.class,+ () -> handler.handle("TransactGetItems", request, "eu-west-1"));+ assertEquals("Invalid ProjectionExpression: Expression size has exceeded the maximum allowed size;",+ ex.getMessage());+ }++ @Test+ void executeStatementRejectsATooDeepParameter() {+ createUsersTable("eu-west-1");+ var request = mapper.createObjectNode();+ request.put("Statement", "UPDATE \"Users\" SET deep=? WHERE userId=?");+ request.set("Parameters", mapper.createArrayNode().add(nestedMaps(32)).add(attributeValue("S", "u1")));++ var ex = assertThrows(AwsException.class,+ () -> handler.handle("ExecuteStatement", request, "eu-west-1"));+ assertEquals("ValidationException", ex.getErrorCode());+ assertEquals("Nesting Levels have exceeded supported limits", ex.getMessage());+ }++ @Test+ void executeTransactionCancelsOnATooDeepParameterWithThatStatementsReason() throws Exception {+ createUsersTable("eu-west-1");+ var fine = mapper.createObjectNode();+ fine.put("Statement", "UPDATE \"Users\" SET x=? WHERE userId=?");+ fine.set("Parameters", mapper.createArrayNode().add(attributeValue("S", "x")).add(attributeValue("S", "u1")));+ var deep = mapper.createObjectNode();+ deep.put("Statement", "INSERT INTO \"Users\" VALUE {'userId': ?, 'deep': ?}");+ deep.set("Parameters", mapper.createArrayNode().add(attributeValue("S", "u2")).add(nestedMaps(32)));+ var request = mapper.createObjectNode();+ request.set("TransactStatements", mapper.createArrayNode().add(fine).add(deep));++ var response = handler.handle("ExecuteTransaction", request, "eu-west-1");+ assertEquals(400, response.getStatus());+ var body = mapper.convertValue(response.getEntity(), JsonNode.class);+ assertEquals("TransactionCanceledException", body.get("__type").asText());+ var reasons = body.get("CancellationReasons");+ assertEquals("None", reasons.get(0).get("Code").asText());+ assertEquals("ValidationError", reasons.get(1).get("Code").asText());+ assertEquals("Nesting Levels have exceeded supported limits", reasons.get(1).get("Message").asText());+ assertNull(service.getItem("Users", item("userId", "u2"), "eu-west-1"));+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.javaindex 75e51c8429..06fbef0920 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java@@ -2625,6 +2625,50 @@ class DynamoDbServiceTest {+ "The AttributeValue for a key attribute cannot contain an empty string value.", ex.getMessage());}+ private void createBinaryIndexedTable(String region) {+ var gsi = new GlobalSecondaryIndex("gsib",+ List.of(new KeySchemaElement("bidx", "HASH")), null, "ALL", null);+ service.createTable("BinaryIndexed",+ List.of(new KeySchemaElement("pk", "HASH")),+ List.of(+ new AttributeDefinition("pk", "S"),+ new AttributeDefinition("bidx", "B")),+ 5L, 5L, List.of(gsi), region);+ }++ @Test+ void putItemEmptyBinaryGsiKeyThrowsValidationException() {+ var region = "eu-west-1";+ createBinaryIndexedTable(region);++ var item = item("pk", "p1");+ item.set("bidx", attributeValue("B", ""));++ var ex = assertThrows(AwsException.class, () ->+ service.putItem("BinaryIndexed", item, region));+ assertEquals("ValidationException", ex.getErrorCode());+ assertEquals("One or more parameter values are not valid. A value specified for a secondary "+ + "index key is not supported. The AttributeValue for a key attribute cannot "+ + "contain an empty binary value. IndexName: gsib, IndexKey: bidx", ex.getMessage());+ }++ @Test+ void updateItemSettingEmptyBinaryGsiKeyThrowsValidationException() {+ var region = "eu-west-1";+ createBinaryIndexedTable(region);++ var exprValues = mapper.createObjectNode();+ exprValues.set(":v", attributeValue("B", ""));++ var ex = assertThrows(AwsException.class, () ->+ service.updateItem("BinaryIndexed", item("pk", "p1"), null,+ "SET bidx = :v", null, exprValues, null, region));+ assertEquals("ValidationException", ex.getErrorCode());+ assertEquals("One or more parameter values are not valid. The update expression attempted to "+ + "update a secondary index key to a value that is not supported. "+ + "The AttributeValue for a key attribute cannot contain an empty binary value.", ex.getMessage());+ }+@Testvoid updateItemNullPartitionKeyThrowsValidationException() {String region = "eu-west-1";@@ -3451,6 +3495,41 @@ class DynamoDbServiceTest {"previousStatus should receive the pre-update value of status");}+ @Test+ void updateItemSetArithmeticOverflowThrowsValidationException() {+ var region = "eu-west-1";+ createUsersTable(region);+ var exprValues = mapper.createObjectNode();+ exprValues.set(":a", attributeValue("N", "9.9e125"));+ exprValues.set(":b", attributeValue("N", "9.9e125"));++ var ex = assertThrows(AwsException.class, () ->+ service.updateItem("Users", item("userId", "u1"), null,+ "SET n = :a + :b", null, exprValues, null, region));+ assertEquals("ValidationException", ex.getErrorCode());+ assertEquals("Number overflow. Attempting to store a number with magnitude larger than supported range",+ ex.getMessage());+ assertNull(service.getItem("Users", item("userId", "u1"), region));+ }++ @Test+ void updateItemAddOverflowThrowsValidationException() {+ var region = "eu-west-1";+ createUsersTable(region);+ var existing = item("userId", "u1");+ existing.set("n", attributeValue("N", "9.9e125"));+ service.putItem("Users", existing, region);+ var exprValues = mapper.createObjectNode();+ exprValues.set(":a", attributeValue("N", "9.9e125"));++ var ex = assertThrows(AwsException.class, () ->+ service.updateItem("Users", item("userId", "u1"), null,+ "ADD n :a", null, exprValues, null, region));+ assertEquals("ValidationException", ex.getErrorCode());+ assertEquals("Number overflow. Attempting to store a number with magnitude larger than supported range",+ ex.getMessage());+ }+@Testvoid updateItemSetParenthesizedArithmeticAppliesSubtraction() {String region = "eu-west-1";