Light, fluffy, and always free - The AWS Local Emulator alternative
fix(dynamodb): measure item size and UpdateItem statements like DynamoDB (#5934)
TATSUNO “Taz” Yasuhiro3b7782e5832c
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbItemSize.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbItemSize.javaindex f83f2afa31..6c17a57676 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbItemSize.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbItemSize.java@@ -7,9 +7,11 @@ import java.math.BigDecimal;import java.nio.charset.StandardCharsets;import java.util.Base64;import java.util.Iterator;+import java.util.LinkedHashMap;+import java.util.List;import java.util.Locale;import java.util.Map;-import java.util.Set;+import java.util.function.Function;/*** Calculates DynamoDB item size and enforces the 400KB limit.@@ -20,12 +22,20 @@ final class DynamoDbItemSize {static final String UPDATE_EXCEEDED = "Item size to update has exceeded the maximum allowed size";- // UpdateItem measures what the statement writes, name and value, not the item it stores,- // and charges these on top. Every other surface measures the finished item.+ // An UpdateExpression is also measured as a statement: these per action, the target paths+ // with a shared prefix counted once, and every operand a SET reads or an ADD writes.+ // Every other surface, AttributeUpdates included, measures only the finished item.private static final int UPDATE_COST = 3;private static final int SET_OR_ADD_COST = 19;- private static final int REMOVE_OR_DELETE_COST = 2;- private static final int LIST_INDEX_COST = 1;+ private static final int REMOVE_OR_DELETE_COST = 1;+ private static final int TARGET_CONTAINER_COST = 4;+ private static final int OPERAND_PATH_COST = 1;+ private static final int OPERAND_PATH_ELEMENT_COST = 14;+ private static final int PLUS_COST = 34;+ private static final int MINUS_COST = 37;+ private static final int LIST_APPEND_COST = 37;+ private static final int IF_NOT_EXISTS_COST = 39;+ private static final int CONTAINER_COST = 3;private static final int MAX_NUMBER_PAIRS = 20;private DynamoDbItemSize() {}@@ -45,23 +55,13 @@ final class DynamoDbItemSize {return calculateItemSize(item) <= MAX_ITEM_SIZE;}- static boolean updateItemWithinLimit(JsonNode item, Set<String> writtenAttributes, int actionCost) {- int total = 0;- int written = 0;- Iterator<Map.Entry<String, JsonNode>> fields = item.fields();- while (fields.hasNext()) {- Map.Entry<String, JsonNode> entry = fields.next();- int size = utf8Length(entry.getKey()) + attributeValueSize(entry.getValue());- total += size;- if (writtenAttributes.contains(entry.getKey())) {- written += size;- }- }- return total <= MAX_ITEM_SIZE && written + actionCost <= MAX_ITEM_SIZE;+ static boolean updateItemWithinLimit(JsonNode item, int statementSize) {+ return calculateItemSize(item) <= MAX_ITEM_SIZE && statementSize <= MAX_ITEM_SIZE;}- static int updateExpressionCost(String expression) {- int cost = UPDATE_COST;+ static int updateStatementSize(String expression, Function<String, List<Object>> paths, JsonNode values) {+ int size = UPDATE_COST;+ PathTree targets = new PathTree();String remaining = expression.trim().replaceAll("\\s+", " ");while (!remaining.isEmpty()) {String keyword = remaining.substring(0, Math.max(remaining.indexOf(' '), 0)).toUpperCase(Locale.ROOT);@@ -71,37 +71,95 @@ final class DynamoDbItemSize {remaining = nextClause < 0 ? "" : body.substring(nextClause);while (!actions.isBlank()) {int comma = DynamoDbService.findNextComma(actions);- cost += actionCost(keyword, comma < 0 ? actions : actions.substring(0, comma));+ String action = (comma < 0 ? actions : actions.substring(0, comma)).trim();+ size += actionSize(keyword, action, targets, paths, values);actions = comma < 0 ? "" : actions.substring(comma + 1);}}- return cost;+ return size + targets.size();+ }++ private static int actionSize(String clause, String action, PathTree targets,+ Function<String, List<Object>> paths, JsonNode values) {+ return switch (clause) {+ case "SET" -> {+ int assignment = action.indexOf('=');+ targets.add(paths.apply(action.substring(0, assignment)));+ yield SET_OR_ADD_COST + operandSize(action.substring(assignment + 1), paths, values);+ }+ case "ADD" -> {+ String[] parts = action.split(" ", 2);+ targets.add(paths.apply(parts[0]));+ yield SET_OR_ADD_COST + operandSize(parts[1], paths, values);+ }+ default -> {+ targets.add(paths.apply(action.split(" ", 2)[0]));+ yield REMOVE_OR_DELETE_COST;+ }+ };}- static int attributeUpdatesCost(JsonNode attributeUpdates) {- int cost = UPDATE_COST;- if (attributeUpdates == null || !attributeUpdates.isObject()) {- return cost;+ private static int operandSize(String operand, Function<String, List<Object>> paths, JsonNode values) {+ String expr = DynamoDbService.stripOuterParens(operand);+ int operator = DynamoDbService.findArithmeticOperator(expr);+ if (operator >= 0) {+ return (expr.charAt(operator) == '+' ? PLUS_COST : MINUS_COST)+ + operandSize(expr.substring(0, operator), paths, values)+ + operandSize(expr.substring(operator + 1), paths, values);+ }+ String lower = expr.toLowerCase(Locale.ROOT);+ if (DynamoDbService.startsWithFunctionCall(lower, "list_append")) {+ return LIST_APPEND_COST + argumentsSize(expr, paths, values);+ }+ if (DynamoDbService.startsWithFunctionCall(lower, "if_not_exists")) {+ return IF_NOT_EXISTS_COST + argumentsSize(expr, paths, values);+ }+ if (expr.startsWith(":")) {+ return values == null ? 0 : attributeValueSize(values.get(expr));}- for (JsonNode update : attributeUpdates) {- cost += "DELETE".equals(update.path("Action").asText("PUT"))- ? REMOVE_OR_DELETE_COST : SET_OR_ADD_COST;+ int size = OPERAND_PATH_COST;+ for (Object element : paths.apply(expr)) {+ size += OPERAND_PATH_ELEMENT_COST + pathElementSize(element);}- return cost;+ return size;}- private static int actionCost(String clause, String action) {- return switch (clause) {- case "SET" -> SET_OR_ADD_COST + (writesListElement(action) ? LIST_INDEX_COST : 0);- case "ADD" -> SET_OR_ADD_COST;- default -> REMOVE_OR_DELETE_COST;- };+ private static int argumentsSize(String call, Function<String, List<Object>> paths, JsonNode values) {+ String arguments = call.substring(call.indexOf('(') + 1, call.lastIndexOf(')'));+ int size = 0;+ while (!arguments.isBlank()) {+ int comma = DynamoDbService.findNextComma(arguments);+ size += operandSize(comma < 0 ? arguments : arguments.substring(0, comma), paths, values);+ arguments = comma < 0 ? "" : arguments.substring(comma + 1);+ }+ return size;+ }++ // A list index counts its decimal digits.+ private static int pathElementSize(Object element) {+ return element instanceof Long index ? Long.toString(index).length() : utf8Length((String) element);}- private static boolean writesListElement(String action) {- int bracket = action.indexOf('[');- int assignment = action.indexOf('=');- return bracket >= 0 && (assignment < 0 || bracket < assignment);+ private static final class PathTree {++ private final Map<Object, PathTree> children = new LinkedHashMap<>();++ void add(List<Object> path) {+ PathTree node = this;+ for (Object element : path) {+ node = node.children.computeIfAbsent(element, ignored -> new PathTree());+ }+ }++ int size() {+ int size = 0;+ for (Map.Entry<Object, PathTree> child : children.entrySet()) {+ PathTree nested = child.getValue();+ size += pathElementSize(child.getKey())+ + (nested.children.isEmpty() ? 0 : TARGET_CONTAINER_COST + nested.size());+ }+ return size;+ }}static int calculateItemSize(JsonNode item) {@@ -127,7 +185,7 @@ final class DynamoDbItemSize {if (attr.has("NULL")) return 1;if (attr.has("SS")) {int size = 0;- for (JsonNode e : attr.get("SS")) size += utf8Length(e.asText()) + 1;+ for (JsonNode e : attr.get("SS")) size += utf8Length(e.asText());return size;}if (attr.has("NS")) {@@ -139,20 +197,20 @@ final class DynamoDbItemSize {}if (attr.has("BS")) {int size = 0;- for (JsonNode e : attr.get("BS")) size += binarySize(e.asText()) + 1;+ for (JsonNode e : attr.get("BS")) size += binarySize(e.asText());return size;}if (attr.has("L")) {- int size = 0;- for (JsonNode e : attr.get("L")) size += attributeValueSize(e) + 3;+ int size = CONTAINER_COST;+ for (JsonNode e : attr.get("L")) size += 1 + attributeValueSize(e);return size;}if (attr.has("M")) {- int size = 0;+ int size = CONTAINER_COST;Iterator<Map.Entry<String, JsonNode>> fields = attr.get("M").fields();while (fields.hasNext()) {Map.Entry<String, JsonNode> entry = fields.next();- size += utf8Length(entry.getKey()) + 1 + attributeValueSize(entry.getValue()) + 3;+ size += utf8Length(entry.getKey()) + 1 + attributeValueSize(entry.getValue());}return size;}diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.javaindex 1c1249bbca..78c56ff7b2 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java@@ -1259,12 +1259,9 @@ public class DynamoDbService {// AWS validates index key values against the item the update produces.validateIndexKeyTypes(table, item, true);- if (sizeRule == UpdateSizeRule.UPDATE_ITEM) {- requireUpdateItemWithinSizeLimit(item,- writtenAttributes(touchedPaths, updateExpression, expressionAttrNames),- updateExpression != null- ? DynamoDbItemSize.updateExpressionCost(updateExpression)- : DynamoDbItemSize.attributeUpdatesCost(attributeUpdates));+ if (sizeRule == UpdateSizeRule.UPDATE_ITEM && updateExpression != null) {+ requireUpdateItemWithinSizeLimit(item, DynamoDbItemSize.updateStatementSize(updateExpression,+ path -> parsePath(path, expressionAttrNames), expressionAttrValues));} else {requireUpdatedItemWithinSizeLimit(item);}@@ -1308,18 +1305,6 @@ public class DynamoDbService {return updateExpression != null ? parsePath(path, expressionAttrNames) : List.<Object>of(path);}- private Set<String> writtenAttributes(List<String> touchedPaths, String updateExpression,- JsonNode expressionAttrNames) {- Set<String> names = new LinkedHashSet<>();- for (String path : touchedPaths) {- List<Object> tokens = pathTokens(path, updateExpression, expressionAttrNames);- if (!tokens.isEmpty() && tokens.get(0) instanceof String name) {- names.add(name);- }- }- return names;- }-public QueryResult query(String tableName, JsonNode keyConditions,JsonNode expressionAttrValues, String keyConditionExpression,String filterExpression, Integer limit, String region) {@@ -2169,9 +2154,8 @@ public class DynamoDbService {}}- private static void requireUpdateItemWithinSizeLimit(JsonNode item, Set<String> writtenAttributes,- int actionCost) {- if (!DynamoDbItemSize.updateItemWithinLimit(item, writtenAttributes, actionCost)) {+ private static void requireUpdateItemWithinSizeLimit(JsonNode item, int statementSize) {+ if (!DynamoDbItemSize.updateItemWithinLimit(item, statementSize)) {throw new UpdatedItemTooLargeException();}}@@ -2929,6 +2913,7 @@ public class DynamoDbService {throw new AwsException("ValidationException","Invalid UpdateExpression: The expression can not be empty;", 400);}+ requireClosedParentheses(expression, "");// AWS tokenizes UpdateExpression whitespace-insensitively, so collapse runs of// whitespace (newlines, tabs, multiple spaces) so clause-keyword dispatch and the// comma-separated action parsing below work regardless of formatting. Normalize@@ -3132,7 +3117,7 @@ public class DynamoDbService {* unwrapped form. Rejects forms like "(a) - (b)" where the outer parens do* not actually enclose the whole expression.*/- private static String stripOuterParens(String expr) {+ static String stripOuterParens(String expr) {String s = expr.trim();while (s.length() >= 2 && s.charAt(0) == '(' && s.charAt(s.length() - 1) == ')') {int depth = 0;@@ -3792,6 +3777,41 @@ public class DynamoDbService {}}+ // DynamoDB reads an unclosed call or group to the end of the expression and reports the+ // raw text from its last token on, trailing whitespace included. UpdateItem wraps the+ // error in the validation envelope, TransactWriteItems does not.+ static void requireClosedParentheses(String expression, String envelope) {+ int depth = 0;+ int lastToken = 0;+ for (int i = 0; i < expression.length(); i++) {+ char c = expression.charAt(i);+ if (Character.isWhitespace(c)) {+ continue;+ }+ if (c == '(') {+ depth++;+ } else if (c == ')') {+ depth--;+ if (depth < 0) {+ return;+ }+ }+ if (i == 0 || isUpdatePunctuation(c) || isUpdatePunctuation(expression.charAt(i - 1))+ || Character.isWhitespace(expression.charAt(i - 1))) {+ lastToken = i;+ }+ }+ if (depth > 0) {+ throw new AwsException("ValidationException",+ envelope + "Invalid UpdateExpression: Syntax error; token: \"<EOF>\", near: \""+ + expression.substring(lastToken) + "\"", 400);+ }+ }++ private static boolean isUpdatePunctuation(char c) {+ return "()[],.=+-".indexOf(c) >= 0;+ }+static int findNextComma(String s) {// Find next comma that is not inside a function callint depth = 0;@@ -3874,7 +3894,7 @@ public class DynamoDbService {* Finds the index of an arithmetic operator (+ or -) that is outside* function parentheses. Returns -1 if none found.*/- private int findArithmeticOperator(String expr) {+ static int findArithmeticOperator(String expr) {int depth = 0;for (int i = 0; i < expr.length(); i++) {char c = expr.charAt(i);@@ -3899,7 +3919,7 @@ public class DynamoDbService {* spaced form (e.g. PynamoDB always generates {@code "list_append (x, y)"}), which a plain* {@code startsWith(functionName + "(")} rejects.*/- private static boolean startsWithFunctionCall(String value, String functionName) {+ static boolean startsWithFunctionCall(String value, String functionName) {return value.startsWith(functionName)&& value.substring(functionName.length()).stripLeading().startsWith("(");}diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.javaindex 512c5b3133..035db086c8 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java@@ -819,6 +819,7 @@ public class NativeDynamoDbJsonHandler {throw new AwsException("ValidationException","Invalid UpdateExpression: Syntax error; token: \"" + token + "\", near: \"" + near + "\"", 400);}+ DynamoDbService.requireClosedParentheses(updateExpression, "1 validation error detected: ");}// Undefined tokens, then ConditionExpression's semantic errors, must both be caught// before the unused-EAN/EAV checks below.diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbItemSizeServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbItemSizeServiceTest.javaindex 9311f3012f..f3b0aada6b 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbItemSizeServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbItemSizeServiceTest.java@@ -15,7 +15,9 @@ import org.junit.jupiter.api.function.Executable;import org.junit.jupiter.params.ParameterizedTest;import org.junit.jupiter.params.provider.CsvSource;+import java.util.Base64;import java.util.List;+import java.util.function.IntFunction;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertNull;@@ -30,8 +32,16 @@ class DynamoDbItemSizeServiceTest {"Item size to update has exceeded the maximum allowed size";private static final int UPDATE_COST = 3;private static final int SET_OR_ADD_COST = 19;- private static final int REMOVE_OR_DELETE_COST = 2;+ private static final int REMOVE_OR_DELETE_COST = 1;private static final int LIST_INDEX_COST = 1;+ private static final int OPERAND_PATH_COST = 1;+ private static final int OPERAND_PATH_ELEMENT_COST = 14;+ private static final int PLUS_COST = 34;+ private static final int MINUS_COST = 37;+ private static final int LIST_APPEND_COST = 37;+ private static final int IF_NOT_EXISTS_COST = 39;+ private static final int CONTAINER_COST = 3;+ private static final int TARGET_CONTAINER_COST = 4;// "pk" plus a one-byte value, short enough that the ceiling falls below 400KB.private static final int SHORT_KEY_BYTES = 3;@@ -142,7 +152,7 @@ class DynamoDbItemSizeServiceTest {@Testvoid updateItemChargesTwoBytesForARemoveAlongsideASet() {int ceiling = MAX_ITEM_SIZE + SHORT_KEY_BYTES- - (UPDATE_COST + SET_OR_ADD_COST + REMOVE_OR_DELETE_COST);+ - (UPDATE_COST + SET_OR_ADD_COST + REMOVE_OR_DELETE_COST + "r".length());seed("a", "r", stringValue("z"));seed("b", "r", stringValue("z"));@@ -154,7 +164,7 @@ class DynamoDbItemSizeServiceTest {@Testvoid updateItemChargesTwoBytesForADeleteAlongsideASet() {int ceiling = MAX_ITEM_SIZE + SHORT_KEY_BYTES- - (UPDATE_COST + SET_OR_ADD_COST + REMOVE_OR_DELETE_COST);+ - (UPDATE_COST + SET_OR_ADD_COST + REMOVE_OR_DELETE_COST + "s".length());ObjectNode members = values(":members", stringSetValue("z"));seed("a", "s", stringSetValue("z"));seed("b", "s", stringSetValue("z"));@@ -180,7 +190,7 @@ class DynamoDbItemSizeServiceTest {void updateItemChargesOneMoreByteThroughAListIndex() {int ceiling = MAX_ITEM_SIZE + SHORT_KEY_BYTES- (UPDATE_COST + SET_OR_ADD_COST + LIST_INDEX_COST);- int elementBytes = 3;+ int elementBytes = CONTAINER_COST + 1;seed("a", "d", listValue(stringValue("x")));seed("b", "d", listValue(stringValue("x")));@@ -190,8 +200,9 @@ class DynamoDbItemSizeServiceTest {}@Test- void updateItemChargesAFunctionCallAsOneAction() {- int ceiling = MAX_ITEM_SIZE + SHORT_KEY_BYTES - (UPDATE_COST + SET_OR_ADD_COST);+ void updateItemChargesIfNotExistsAndTheOperandsItReads() {+ int ceiling = MAX_ITEM_SIZE + SHORT_KEY_BYTES+ - (UPDATE_COST + SET_OR_ADD_COST + IF_NOT_EXISTS_COST + operandPath("b"));update("a", "SET b = if_not_exists(b, :pad)", ceiling, 0);assertEquals(ceiling, storedBytes("a"));@@ -199,12 +210,10 @@ class DynamoDbItemSizeServiceTest {}@Test- void updateItemChargesALegacyAttributeUpdateLikeASet() {- int ceiling = MAX_ITEM_SIZE + SHORT_KEY_BYTES - (UPDATE_COST + SET_OR_ADD_COST);-- updateAttribute("a", ceiling);- assertEquals(ceiling, storedBytes("a"));- assertRefused(() -> updateAttribute("b", ceiling + 1));+ void updateItemMeasuresALegacyAttributeUpdateByTheFinishedItem() {+ updateAttribute("a", MAX_ITEM_SIZE);+ assertEquals(MAX_ITEM_SIZE, storedBytes("a"));+ assertRefused(() -> updateAttribute("b", MAX_ITEM_SIZE + 1));}@Test@@ -242,6 +251,168 @@ class DynamoDbItemSizeServiceTest {assertNull(service.getItem(TABLE, key("b"), REGION));}+ @Test+ void putItemMeasuresAListAtThreeBytesPlusOnePerElement() {+ ObjectNode attributes = JsonNodeFactory.instance.objectNode();+ attributes.set("l", listValue(stringValue("x"), stringValue("y")));+ attributes.set("e", listValue());++ assertPutCeiling(attributes, "l".length() + CONTAINER_COST + 2 * (1 + 1) + "e".length() + CONTAINER_COST);+ }++ @Test+ void putItemMeasuresAMapAtThreeBytesPlusOnePerEntryAndItsName() {+ ObjectNode map = JsonNodeFactory.instance.objectNode();+ map.putObject("M").set("k", stringValue("x"));+ ((ObjectNode) map.get("M")).set("kk", stringValue("y"));+ ObjectNode attributes = JsonNodeFactory.instance.objectNode();+ attributes.set("m", map);+ attributes.set("n", JsonNodeFactory.instance.objectNode().set("M", JsonNodeFactory.instance.objectNode()));++ assertPutCeiling(attributes, "m".length() + CONTAINER_COST + (1 + 1 + 1) + (2 + 1 + 1)+ + "n".length() + CONTAINER_COST);+ }++ @Test+ void putItemMeasuresStringAndBinarySetsByTheirMembersAlone() {+ ObjectNode binarySet = JsonNodeFactory.instance.objectNode();+ binarySet.putArray("BS")+ .add(Base64.getEncoder().encodeToString(new byte[] {1}))+ .add(Base64.getEncoder().encodeToString(new byte[] {2, 2}));+ ObjectNode attributes = JsonNodeFactory.instance.objectNode();+ attributes.set("ss", stringSetValue("x", "yy"));+ attributes.set("bs", binarySet);++ assertPutCeiling(attributes, "ss".length() + 1 + 2 + "bs".length() + 1 + 2);+ }++ @Test+ void refusedPutLeavesTheStoredItemAsItWas() {+ seed("a", "u", stringValue("y".repeat(10)));+ JsonNode before = service.getItem(TABLE, key("a"), REGION).deepCopy();+ ObjectNode over = key("a");+ over.set("b", stringValue(padding("a", MAX_ITEM_SIZE + 1, 0)));++ assertThrows(AwsException.class, () -> service.putItem(TABLE, over, REGION));+ assertEquals(before, service.getItem(TABLE, key("a"), REGION));+ }++ @Test+ void refusedUpdateLeavesTheStoredItemAsItWas() {+ seed("a", "u", stringValue("y".repeat(10)));+ JsonNode before = service.getItem(TABLE, key("a"), REGION).deepCopy();++ assertRefused(() -> update("a", "SET b = :pad", MAX_ITEM_SIZE + 1, "u".length() + 10));+ assertEquals(before, service.getItem(TABLE, key("a"), REGION));+ }++ @Test+ void updateItemChargesListAppendAndTheOperandsItReads() {+ int fixed = UPDATE_COST + SET_OR_ADD_COST + "l".length()+ + LIST_APPEND_COST + operandPath("l") + CONTAINER_COST + 1;++ assertUpdateCeiling(attributes("l", listValue(stringValue("x"))), "SET l = list_append(l, :list)",+ pad -> values(":list", listValue(stringValue("x".repeat(pad)))), MAX_ITEM_SIZE - fixed);+ }++ @Test+ void updateItemChargesIfNotExistsNestedInListAppend() {+ int fixed = UPDATE_COST + SET_OR_ADD_COST + "l".length() + LIST_APPEND_COST+ + IF_NOT_EXISTS_COST + operandPath("l") + CONTAINER_COST + CONTAINER_COST + 1;++ assertUpdateCeiling(null, "SET l = list_append(if_not_exists(l, :empty), :list)", pad -> {+ ObjectNode values = values(":list", listValue(stringValue("x".repeat(pad))));+ values.set(":empty", listValue());+ return values;+ }, MAX_ITEM_SIZE - fixed);+ }++ @Test+ void updateItemChargesTheSourcePathOfACopy() {+ int fixed = UPDATE_COST + 2 * SET_OR_ADD_COST + "a".length() + operandPath("b") + "c".length();++ assertUpdateCeiling(attributes("b", stringValue("y")), "SET a = b, c = :pad",+ pad -> values(":pad", stringValue("x".repeat(pad))), MAX_ITEM_SIZE - fixed);+ }++ @Test+ void updateItemChargesEveryElementOfANestedSourcePath() {+ ObjectNode list = listValue();+ for (int i = 0; i < 11; i++) {+ ((ArrayNode) list.get("L")).add(stringValue("x"));+ }+ int fixed = UPDATE_COST + 2 * SET_OR_ADD_COST + "a".length() + operandPath("d", "ee", "10") + "c".length();++ assertUpdateCeiling(attributes("d", mapValue("ee", list)), "SET a = d.ee[10], c = :pad",+ pad -> values(":pad", stringValue("x".repeat(pad))), MAX_ITEM_SIZE - fixed);+ }++ @Test+ void updateItemChargesPlusAndMinusDifferently() {+ int one = 2;+ int fixed = UPDATE_COST + 3 * SET_OR_ADD_COST+ + "n".length() + PLUS_COST + operandPath("n") + one+ + "m".length() + MINUS_COST + operandPath("m") + one+ + "c".length();+ ObjectNode seed = attributes("n", numberValue("1"));+ seed.set("m", numberValue("1"));++ assertUpdateCeiling(seed, "SET n = n + :one, m = m - :one, c = :pad", pad -> {+ ObjectNode values = values(":pad", stringValue("x".repeat(pad)));+ values.set(":one", numberValue("1"));+ return values;+ }, MAX_ITEM_SIZE - fixed);+ }++ @Test+ void updateItemCountsASharedTargetPrefixOnce() {+ int fixed = UPDATE_COST + 2 * SET_OR_ADD_COST+ + "d".length() + TARGET_CONTAINER_COST + "a".length() + "b".length() + "e".length();++ assertUpdateCeiling(attributes("d", JsonNodeFactory.instance.objectNode().set("M",+ JsonNodeFactory.instance.objectNode())), "SET d.a = :pad, d.b = :e", pad -> {+ ObjectNode values = values(":pad", stringValue("x".repeat(pad)));+ values.set(":e", stringValue("e"));+ return values;+ }, MAX_ITEM_SIZE - fixed);+ }++ @Test+ void updateItemCountsTheDigitsOfAListIndex() {+ ObjectNode list = listValue();+ for (int i = 0; i < 11; i++) {+ ((ArrayNode) list.get("L")).add(stringValue("x"));+ }+ int fixed = UPDATE_COST + SET_OR_ADD_COST + "l".length() + TARGET_CONTAINER_COST + "10".length();++ assertUpdateCeiling(attributes("l", list), "SET l[10] = :pad",+ pad -> values(":pad", stringValue("x".repeat(pad))), MAX_ITEM_SIZE - fixed);+ }++ @Test+ void updateItemChargesRemoveAndDeleteByTheirPathAlone() {+ ObjectNode seed = attributes("rrrrrrrrrr", stringValue("y"));+ seed.set("ssssssssss", stringSetValue("y", "z"));+ int fixed = UPDATE_COST + 2 * (REMOVE_OR_DELETE_COST + 10) + SET_OR_ADD_COST + "c".length();++ assertUpdateCeiling(seed, "REMOVE rrrrrrrrrr DELETE ssssssssss :z SET c = :pad", pad -> {+ ObjectNode values = values(":pad", stringValue("x".repeat(pad)));+ values.set(":z", stringSetValue("z".repeat(100)));+ return values;+ }, MAX_ITEM_SIZE - fixed);+ }++ @Test+ void updateItemChargesTheValueAnAddWritesNotTheSetItBuilds() {+ int fixed = UPDATE_COST + 2 * SET_OR_ADD_COST + "ss".length() + 1 + "c".length();++ assertUpdateCeiling(attributes("ss", stringSetValue("yy")), "ADD ss :v SET c = :pad", pad -> {+ ObjectNode values = values(":pad", stringValue("x".repeat(pad)));+ values.set(":v", stringSetValue("z"));+ return values;+ }, MAX_ITEM_SIZE - fixed);+ }+private void update(String keyValue, String expression, int totalBytes, int otherBytes) {update(keyValue, expression, totalBytes, otherBytes, JsonNodeFactory.instance.objectNode());}@@ -279,6 +450,49 @@ class DynamoDbItemSizeServiceTest {return member;}+ private void assertPutCeiling(ObjectNode attributes, int attributeBytes) {+ putPadded("a", MAX_ITEM_SIZE, attributes, attributeBytes);+ assertEquals(MAX_ITEM_SIZE, storedBytes("a"));+ AwsException refused = assertThrows(AwsException.class,+ () -> putPadded("b", MAX_ITEM_SIZE + 1, attributes, attributeBytes));+ assertEquals("Item size has exceeded the maximum allowed size", refused.getMessage());+ }++ private void putPadded(String keyValue, int totalBytes, ObjectNode attributes, int attributeBytes) {+ ObjectNode item = key(keyValue);+ item.setAll(attributes.deepCopy());+ item.set("b", stringValue(padding(keyValue, totalBytes, attributeBytes)));+ service.putItem(TABLE, item, REGION);+ }++ private void assertUpdateCeiling(ObjectNode seed, String expression, IntFunction<ObjectNode> values,+ int maxPadding) {+ for (String keyValue : List.of("a", "b")) {+ if (seed != null) {+ ObjectNode item = key(keyValue);+ item.setAll(seed.deepCopy());+ service.putItem(TABLE, item, REGION);+ }+ }+ service.updateItem(TABLE, key("a"), null, expression, null, values.apply(maxPadding), "NONE", REGION);+ assertRefused(() -> service.updateItem(TABLE, key("b"), null, expression, null,+ values.apply(maxPadding + 1), "NONE", REGION));+ }++ private static int operandPath(String... elements) {+ int size = OPERAND_PATH_COST;+ for (String element : elements) {+ size += OPERAND_PATH_ELEMENT_COST + element.length();+ }+ return size;+ }++ private ObjectNode attributes(String name, JsonNode value) {+ ObjectNode attributes = JsonNodeFactory.instance.objectNode();+ attributes.set(name, value);+ return attributes;+ }+private void seed(String keyValue, String attribute, JsonNode value) {ObjectNode item = key(keyValue);item.set(attribute, value);@@ -318,9 +532,12 @@ class DynamoDbItemSizeServiceTest {return value;}- private ObjectNode stringSetValue(String member) {+ private ObjectNode stringSetValue(String... members) {ObjectNode value = JsonNodeFactory.instance.objectNode();- value.putArray("SS").add(member);+ ArrayNode set = value.putArray("SS");+ for (String member : members) {+ set.add(member);+ }return value;}@@ -330,9 +547,12 @@ class DynamoDbItemSizeServiceTest {return map;}- private ObjectNode listValue(JsonNode element) {+ private ObjectNode listValue(JsonNode... elements) {ObjectNode list = JsonNodeFactory.instance.objectNode();- list.putArray("L").add(element);+ ArrayNode array = list.putArray("L");+ for (JsonNode element : elements) {+ array.add(element);+ }return list;}diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandlerTest.javaindex 652c61605a..73440622c8 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandlerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandlerTest.java@@ -1567,6 +1567,30 @@ class NativeDynamoDbJsonHandlerTest {+ "Expression size has exceeded the maximum allowed size;", ex.getMessage());}+ @Test+ void updateItemRejectsAnUnclosedCallAtTheEndBeforeTheTableLookup() {+ Map<String, String> nearByExpression = Map.of(+ "SET a = list_append(l, :v", ":v",+ "SET a = if_not_exists(a,:v", ":v",+ "SET a = list_append(l, :v ", ":v ",+ "SET a = list_append(if_not_exists(l, :v, :v)", ")",+ "SET a = if_not_exists(a.b[1], c[0]", "]",+ "SET a = list_append(", "(",+ "SET a = :v, b = if_not_exists(b, :v", ":v",+ "SET a = :v + (:v", ":v");+ nearByExpression.forEach((expression, near) -> {+ ObjectNode request = updateUserRequest();+ request.put("TableName", "Missing");+ request.put("UpdateExpression", expression);++ AwsException ex = assertThrows(AwsException.class,+ () -> handler.handle("UpdateItem", request, "eu-west-1"), expression);+ assertEquals("ValidationException", ex.getErrorCode());+ assertEquals("1 validation error detected: Invalid UpdateExpression: Syntax error; "+ + "token: \"<EOF>\", near: \"" + near + "\"", ex.getMessage());+ });+ }+@Testvoid updateItemAcceptsAnUpdateExpressionOfExactly4096Bytes() throws Exception {createUsersTable("eu-west-1");@@ -1779,6 +1803,20 @@ class NativeDynamoDbJsonHandlerTest {ex.getMessage());}+ @Test+ void transactWriteItemsRejectsAnUnclosedCallAtTheEnd() {+ createUsersTable("eu-west-1");+ ObjectNode update = mapper.createObjectNode();+ update.set("Key", item("userId", "u1"));+ update.put("UpdateExpression", "SET a = list_append(l, :v");+ update.set("ExpressionAttributeValues", singleValue(":v"));++ AwsException ex = assertThrows(AwsException.class,+ () -> handler.handle("TransactWriteItems", transactWrite("Update", update), "eu-west-1"));+ assertEquals("ValidationException", ex.getErrorCode());+ assertEquals("Invalid UpdateExpression: Syntax error; token: \"<EOF>\", near: \":v\"", ex.getMessage());+ }+@Testvoid transactWriteItemsRejectsAPutItemWithALeafAtLevel33() {createUsersTable("eu-west-1");