Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(dynamodb): measure item size and UpdateItem statements like DynamoDB (#5934)

TATSUNO “Taz” Yasuhiro3b7782e5832c
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbItemSize.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbItemSize.java
index f83f2afa31..6c17a57676 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbItemSize.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbItemSize.java
@@ -7,9 +7,11 @@ import java.math.BigDecimal;
import java.nio.charset.StandardCharsets;
import java.util.Base64;
import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.List;
import java.util.Locale;
import java.util.Map;
-import java.util.Set;
+import java.util.function.Function;
/**
* Calculates DynamoDB item size and enforces the 400KB limit.
@@ -20,12 +22,20 @@ final class DynamoDbItemSize {
static final String UPDATE_EXCEEDED = "Item size to update has exceeded the maximum allowed size";
- // UpdateItem measures what the statement writes, name and value, not the item it stores,
- // and charges these on top. Every other surface measures the finished item.
+ // An UpdateExpression is also measured as a statement: these per action, the target paths
+ // with a shared prefix counted once, and every operand a SET reads or an ADD writes.
+ // Every other surface, AttributeUpdates included, measures only the finished item.
private static final int UPDATE_COST = 3;
private static final int SET_OR_ADD_COST = 19;
- private static final int REMOVE_OR_DELETE_COST = 2;
- private static final int LIST_INDEX_COST = 1;
+ private static final int REMOVE_OR_DELETE_COST = 1;
+ private static final int TARGET_CONTAINER_COST = 4;
+ private static final int OPERAND_PATH_COST = 1;
+ private static final int OPERAND_PATH_ELEMENT_COST = 14;
+ private static final int PLUS_COST = 34;
+ private static final int MINUS_COST = 37;
+ private static final int LIST_APPEND_COST = 37;
+ private static final int IF_NOT_EXISTS_COST = 39;
+ private static final int CONTAINER_COST = 3;
private static final int MAX_NUMBER_PAIRS = 20;
private DynamoDbItemSize() {}
@@ -45,23 +55,13 @@ final class DynamoDbItemSize {
return calculateItemSize(item) <= MAX_ITEM_SIZE;
}
- static boolean updateItemWithinLimit(JsonNode item, Set<String> writtenAttributes, int actionCost) {
- int total = 0;
- int written = 0;
- Iterator<Map.Entry<String, JsonNode>> fields = item.fields();
- while (fields.hasNext()) {
- Map.Entry<String, JsonNode> entry = fields.next();
- int size = utf8Length(entry.getKey()) + attributeValueSize(entry.getValue());
- total += size;
- if (writtenAttributes.contains(entry.getKey())) {
- written += size;
- }
- }
- return total <= MAX_ITEM_SIZE && written + actionCost <= MAX_ITEM_SIZE;
+ static boolean updateItemWithinLimit(JsonNode item, int statementSize) {
+ return calculateItemSize(item) <= MAX_ITEM_SIZE && statementSize <= MAX_ITEM_SIZE;
}
- static int updateExpressionCost(String expression) {
- int cost = UPDATE_COST;
+ static int updateStatementSize(String expression, Function<String, List<Object>> paths, JsonNode values) {
+ int size = UPDATE_COST;
+ PathTree targets = new PathTree();
String remaining = expression.trim().replaceAll("\\s+", " ");
while (!remaining.isEmpty()) {
String keyword = remaining.substring(0, Math.max(remaining.indexOf(' '), 0)).toUpperCase(Locale.ROOT);
@@ -71,37 +71,95 @@ final class DynamoDbItemSize {
remaining = nextClause < 0 ? "" : body.substring(nextClause);
while (!actions.isBlank()) {
int comma = DynamoDbService.findNextComma(actions);
- cost += actionCost(keyword, comma < 0 ? actions : actions.substring(0, comma));
+ String action = (comma < 0 ? actions : actions.substring(0, comma)).trim();
+ size += actionSize(keyword, action, targets, paths, values);
actions = comma < 0 ? "" : actions.substring(comma + 1);
}
}
- return cost;
+ return size + targets.size();
+ }
+
+ private static int actionSize(String clause, String action, PathTree targets,
+ Function<String, List<Object>> paths, JsonNode values) {
+ return switch (clause) {
+ case "SET" -> {
+ int assignment = action.indexOf('=');
+ targets.add(paths.apply(action.substring(0, assignment)));
+ yield SET_OR_ADD_COST + operandSize(action.substring(assignment + 1), paths, values);
+ }
+ case "ADD" -> {
+ String[] parts = action.split(" ", 2);
+ targets.add(paths.apply(parts[0]));
+ yield SET_OR_ADD_COST + operandSize(parts[1], paths, values);
+ }
+ default -> {
+ targets.add(paths.apply(action.split(" ", 2)[0]));
+ yield REMOVE_OR_DELETE_COST;
+ }
+ };
}
- static int attributeUpdatesCost(JsonNode attributeUpdates) {
- int cost = UPDATE_COST;
- if (attributeUpdates == null || !attributeUpdates.isObject()) {
- return cost;
+ private static int operandSize(String operand, Function<String, List<Object>> paths, JsonNode values) {
+ String expr = DynamoDbService.stripOuterParens(operand);
+ int operator = DynamoDbService.findArithmeticOperator(expr);
+ if (operator >= 0) {
+ return (expr.charAt(operator) == '+' ? PLUS_COST : MINUS_COST)
+ + operandSize(expr.substring(0, operator), paths, values)
+ + operandSize(expr.substring(operator + 1), paths, values);
+ }
+ String lower = expr.toLowerCase(Locale.ROOT);
+ if (DynamoDbService.startsWithFunctionCall(lower, "list_append")) {
+ return LIST_APPEND_COST + argumentsSize(expr, paths, values);
+ }
+ if (DynamoDbService.startsWithFunctionCall(lower, "if_not_exists")) {
+ return IF_NOT_EXISTS_COST + argumentsSize(expr, paths, values);
+ }
+ if (expr.startsWith(":")) {
+ return values == null ? 0 : attributeValueSize(values.get(expr));
}
- for (JsonNode update : attributeUpdates) {
- cost += "DELETE".equals(update.path("Action").asText("PUT"))
- ? REMOVE_OR_DELETE_COST : SET_OR_ADD_COST;
+ int size = OPERAND_PATH_COST;
+ for (Object element : paths.apply(expr)) {
+ size += OPERAND_PATH_ELEMENT_COST + pathElementSize(element);
}
- return cost;
+ return size;
}
- private static int actionCost(String clause, String action) {
- return switch (clause) {
- case "SET" -> SET_OR_ADD_COST + (writesListElement(action) ? LIST_INDEX_COST : 0);
- case "ADD" -> SET_OR_ADD_COST;
- default -> REMOVE_OR_DELETE_COST;
- };
+ private static int argumentsSize(String call, Function<String, List<Object>> paths, JsonNode values) {
+ String arguments = call.substring(call.indexOf('(') + 1, call.lastIndexOf(')'));
+ int size = 0;
+ while (!arguments.isBlank()) {
+ int comma = DynamoDbService.findNextComma(arguments);
+ size += operandSize(comma < 0 ? arguments : arguments.substring(0, comma), paths, values);
+ arguments = comma < 0 ? "" : arguments.substring(comma + 1);
+ }
+ return size;
+ }
+
+ // A list index counts its decimal digits.
+ private static int pathElementSize(Object element) {
+ return element instanceof Long index ? Long.toString(index).length() : utf8Length((String) element);
}
- private static boolean writesListElement(String action) {
- int bracket = action.indexOf('[');
- int assignment = action.indexOf('=');
- return bracket >= 0 && (assignment < 0 || bracket < assignment);
+ private static final class PathTree {
+
+ private final Map<Object, PathTree> children = new LinkedHashMap<>();
+
+ void add(List<Object> path) {
+ PathTree node = this;
+ for (Object element : path) {
+ node = node.children.computeIfAbsent(element, ignored -> new PathTree());
+ }
+ }
+
+ int size() {
+ int size = 0;
+ for (Map.Entry<Object, PathTree> child : children.entrySet()) {
+ PathTree nested = child.getValue();
+ size += pathElementSize(child.getKey())
+ + (nested.children.isEmpty() ? 0 : TARGET_CONTAINER_COST + nested.size());
+ }
+ return size;
+ }
}
static int calculateItemSize(JsonNode item) {
@@ -127,7 +185,7 @@ final class DynamoDbItemSize {
if (attr.has("NULL")) return 1;
if (attr.has("SS")) {
int size = 0;
- for (JsonNode e : attr.get("SS")) size += utf8Length(e.asText()) + 1;
+ for (JsonNode e : attr.get("SS")) size += utf8Length(e.asText());
return size;
}
if (attr.has("NS")) {
@@ -139,20 +197,20 @@ final class DynamoDbItemSize {
}
if (attr.has("BS")) {
int size = 0;
- for (JsonNode e : attr.get("BS")) size += binarySize(e.asText()) + 1;
+ for (JsonNode e : attr.get("BS")) size += binarySize(e.asText());
return size;
}
if (attr.has("L")) {
- int size = 0;
- for (JsonNode e : attr.get("L")) size += attributeValueSize(e) + 3;
+ int size = CONTAINER_COST;
+ for (JsonNode e : attr.get("L")) size += 1 + attributeValueSize(e);
return size;
}
if (attr.has("M")) {
- int size = 0;
+ int size = CONTAINER_COST;
Iterator<Map.Entry<String, JsonNode>> fields = attr.get("M").fields();
while (fields.hasNext()) {
Map.Entry<String, JsonNode> entry = fields.next();
- size += utf8Length(entry.getKey()) + 1 + attributeValueSize(entry.getValue()) + 3;
+ size += utf8Length(entry.getKey()) + 1 + attributeValueSize(entry.getValue());
}
return size;
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
index 1c1249bbca..78c56ff7b2 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
@@ -1259,12 +1259,9 @@ public class DynamoDbService {
// AWS validates index key values against the item the update produces.
validateIndexKeyTypes(table, item, true);
- if (sizeRule == UpdateSizeRule.UPDATE_ITEM) {
- requireUpdateItemWithinSizeLimit(item,
- writtenAttributes(touchedPaths, updateExpression, expressionAttrNames),
- updateExpression != null
- ? DynamoDbItemSize.updateExpressionCost(updateExpression)
- : DynamoDbItemSize.attributeUpdatesCost(attributeUpdates));
+ if (sizeRule == UpdateSizeRule.UPDATE_ITEM && updateExpression != null) {
+ requireUpdateItemWithinSizeLimit(item, DynamoDbItemSize.updateStatementSize(updateExpression,
+ path -> parsePath(path, expressionAttrNames), expressionAttrValues));
} else {
requireUpdatedItemWithinSizeLimit(item);
}
@@ -1308,18 +1305,6 @@ public class DynamoDbService {
return updateExpression != null ? parsePath(path, expressionAttrNames) : List.<Object>of(path);
}
- private Set<String> writtenAttributes(List<String> touchedPaths, String updateExpression,
- JsonNode expressionAttrNames) {
- Set<String> names = new LinkedHashSet<>();
- for (String path : touchedPaths) {
- List<Object> tokens = pathTokens(path, updateExpression, expressionAttrNames);
- if (!tokens.isEmpty() && tokens.get(0) instanceof String name) {
- names.add(name);
- }
- }
- return names;
- }
-
public QueryResult query(String tableName, JsonNode keyConditions,
JsonNode expressionAttrValues, String keyConditionExpression,
String filterExpression, Integer limit, String region) {
@@ -2169,9 +2154,8 @@ public class DynamoDbService {
}
}
- private static void requireUpdateItemWithinSizeLimit(JsonNode item, Set<String> writtenAttributes,
- int actionCost) {
- if (!DynamoDbItemSize.updateItemWithinLimit(item, writtenAttributes, actionCost)) {
+ private static void requireUpdateItemWithinSizeLimit(JsonNode item, int statementSize) {
+ if (!DynamoDbItemSize.updateItemWithinLimit(item, statementSize)) {
throw new UpdatedItemTooLargeException();
}
}
@@ -2929,6 +2913,7 @@ public class DynamoDbService {
throw new AwsException("ValidationException",
"Invalid UpdateExpression: The expression can not be empty;", 400);
}
+ requireClosedParentheses(expression, "");
// AWS tokenizes UpdateExpression whitespace-insensitively, so collapse runs of
// whitespace (newlines, tabs, multiple spaces) so clause-keyword dispatch and the
// comma-separated action parsing below work regardless of formatting. Normalize
@@ -3132,7 +3117,7 @@ public class DynamoDbService {
* unwrapped form. Rejects forms like "(a) - (b)" where the outer parens do
* not actually enclose the whole expression.
*/
- private static String stripOuterParens(String expr) {
+ static String stripOuterParens(String expr) {
String s = expr.trim();
while (s.length() >= 2 && s.charAt(0) == '(' && s.charAt(s.length() - 1) == ')') {
int depth = 0;
@@ -3792,6 +3777,41 @@ public class DynamoDbService {
}
}
+ // DynamoDB reads an unclosed call or group to the end of the expression and reports the
+ // raw text from its last token on, trailing whitespace included. UpdateItem wraps the
+ // error in the validation envelope, TransactWriteItems does not.
+ static void requireClosedParentheses(String expression, String envelope) {
+ int depth = 0;
+ int lastToken = 0;
+ for (int i = 0; i < expression.length(); i++) {
+ char c = expression.charAt(i);
+ if (Character.isWhitespace(c)) {
+ continue;
+ }
+ if (c == '(') {
+ depth++;
+ } else if (c == ')') {
+ depth--;
+ if (depth < 0) {
+ return;
+ }
+ }
+ if (i == 0 || isUpdatePunctuation(c) || isUpdatePunctuation(expression.charAt(i - 1))
+ || Character.isWhitespace(expression.charAt(i - 1))) {
+ lastToken = i;
+ }
+ }
+ if (depth > 0) {
+ throw new AwsException("ValidationException",
+ envelope + "Invalid UpdateExpression: Syntax error; token: \"<EOF>\", near: \""
+ + expression.substring(lastToken) + "\"", 400);
+ }
+ }
+
+ private static boolean isUpdatePunctuation(char c) {
+ return "()[],.=+-".indexOf(c) >= 0;
+ }
+
static int findNextComma(String s) {
// Find next comma that is not inside a function call
int depth = 0;
@@ -3874,7 +3894,7 @@ public class DynamoDbService {
* Finds the index of an arithmetic operator (+ or -) that is outside
* function parentheses. Returns -1 if none found.
*/
- private int findArithmeticOperator(String expr) {
+ static int findArithmeticOperator(String expr) {
int depth = 0;
for (int i = 0; i < expr.length(); i++) {
char c = expr.charAt(i);
@@ -3899,7 +3919,7 @@ public class DynamoDbService {
* spaced form (e.g. PynamoDB always generates {@code "list_append (x, y)"}), which a plain
* {@code startsWith(functionName + "(")} rejects.
*/
- private static boolean startsWithFunctionCall(String value, String functionName) {
+ static boolean startsWithFunctionCall(String value, String functionName) {
return value.startsWith(functionName)
&& value.substring(functionName.length()).stripLeading().startsWith("(");
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java
index 512c5b3133..035db086c8 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandler.java
@@ -819,6 +819,7 @@ public class NativeDynamoDbJsonHandler {
throw new AwsException("ValidationException",
"Invalid UpdateExpression: Syntax error; token: \"" + token + "\", near: \"" + near + "\"", 400);
}
+ DynamoDbService.requireClosedParentheses(updateExpression, "1 validation error detected: ");
}
// Undefined tokens, then ConditionExpression's semantic errors, must both be caught
// before the unused-EAN/EAV checks below.
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbItemSizeServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbItemSizeServiceTest.java
index 9311f3012f..f3b0aada6b 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbItemSizeServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbItemSizeServiceTest.java
@@ -15,7 +15,9 @@ import org.junit.jupiter.api.function.Executable;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.CsvSource;
+import java.util.Base64;
import java.util.List;
+import java.util.function.IntFunction;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
@@ -30,8 +32,16 @@ class DynamoDbItemSizeServiceTest {
"Item size to update has exceeded the maximum allowed size";
private static final int UPDATE_COST = 3;
private static final int SET_OR_ADD_COST = 19;
- private static final int REMOVE_OR_DELETE_COST = 2;
+ private static final int REMOVE_OR_DELETE_COST = 1;
private static final int LIST_INDEX_COST = 1;
+ private static final int OPERAND_PATH_COST = 1;
+ private static final int OPERAND_PATH_ELEMENT_COST = 14;
+ private static final int PLUS_COST = 34;
+ private static final int MINUS_COST = 37;
+ private static final int LIST_APPEND_COST = 37;
+ private static final int IF_NOT_EXISTS_COST = 39;
+ private static final int CONTAINER_COST = 3;
+ private static final int TARGET_CONTAINER_COST = 4;
// "pk" plus a one-byte value, short enough that the ceiling falls below 400KB.
private static final int SHORT_KEY_BYTES = 3;
@@ -142,7 +152,7 @@ class DynamoDbItemSizeServiceTest {
@Test
void updateItemChargesTwoBytesForARemoveAlongsideASet() {
int ceiling = MAX_ITEM_SIZE + SHORT_KEY_BYTES
- - (UPDATE_COST + SET_OR_ADD_COST + REMOVE_OR_DELETE_COST);
+ - (UPDATE_COST + SET_OR_ADD_COST + REMOVE_OR_DELETE_COST + "r".length());
seed("a", "r", stringValue("z"));
seed("b", "r", stringValue("z"));
@@ -154,7 +164,7 @@ class DynamoDbItemSizeServiceTest {
@Test
void updateItemChargesTwoBytesForADeleteAlongsideASet() {
int ceiling = MAX_ITEM_SIZE + SHORT_KEY_BYTES
- - (UPDATE_COST + SET_OR_ADD_COST + REMOVE_OR_DELETE_COST);
+ - (UPDATE_COST + SET_OR_ADD_COST + REMOVE_OR_DELETE_COST + "s".length());
ObjectNode members = values(":members", stringSetValue("z"));
seed("a", "s", stringSetValue("z"));
seed("b", "s", stringSetValue("z"));
@@ -180,7 +190,7 @@ class DynamoDbItemSizeServiceTest {
void updateItemChargesOneMoreByteThroughAListIndex() {
int ceiling = MAX_ITEM_SIZE + SHORT_KEY_BYTES
- (UPDATE_COST + SET_OR_ADD_COST + LIST_INDEX_COST);
- int elementBytes = 3;
+ int elementBytes = CONTAINER_COST + 1;
seed("a", "d", listValue(stringValue("x")));
seed("b", "d", listValue(stringValue("x")));
@@ -190,8 +200,9 @@ class DynamoDbItemSizeServiceTest {
}
@Test
- void updateItemChargesAFunctionCallAsOneAction() {
- int ceiling = MAX_ITEM_SIZE + SHORT_KEY_BYTES - (UPDATE_COST + SET_OR_ADD_COST);
+ void updateItemChargesIfNotExistsAndTheOperandsItReads() {
+ int ceiling = MAX_ITEM_SIZE + SHORT_KEY_BYTES
+ - (UPDATE_COST + SET_OR_ADD_COST + IF_NOT_EXISTS_COST + operandPath("b"));
update("a", "SET b = if_not_exists(b, :pad)", ceiling, 0);
assertEquals(ceiling, storedBytes("a"));
@@ -199,12 +210,10 @@ class DynamoDbItemSizeServiceTest {
}
@Test
- void updateItemChargesALegacyAttributeUpdateLikeASet() {
- int ceiling = MAX_ITEM_SIZE + SHORT_KEY_BYTES - (UPDATE_COST + SET_OR_ADD_COST);
-
- updateAttribute("a", ceiling);
- assertEquals(ceiling, storedBytes("a"));
- assertRefused(() -> updateAttribute("b", ceiling + 1));
+ void updateItemMeasuresALegacyAttributeUpdateByTheFinishedItem() {
+ updateAttribute("a", MAX_ITEM_SIZE);
+ assertEquals(MAX_ITEM_SIZE, storedBytes("a"));
+ assertRefused(() -> updateAttribute("b", MAX_ITEM_SIZE + 1));
}
@Test
@@ -242,6 +251,168 @@ class DynamoDbItemSizeServiceTest {
assertNull(service.getItem(TABLE, key("b"), REGION));
}
+ @Test
+ void putItemMeasuresAListAtThreeBytesPlusOnePerElement() {
+ ObjectNode attributes = JsonNodeFactory.instance.objectNode();
+ attributes.set("l", listValue(stringValue("x"), stringValue("y")));
+ attributes.set("e", listValue());
+
+ assertPutCeiling(attributes, "l".length() + CONTAINER_COST + 2 * (1 + 1) + "e".length() + CONTAINER_COST);
+ }
+
+ @Test
+ void putItemMeasuresAMapAtThreeBytesPlusOnePerEntryAndItsName() {
+ ObjectNode map = JsonNodeFactory.instance.objectNode();
+ map.putObject("M").set("k", stringValue("x"));
+ ((ObjectNode) map.get("M")).set("kk", stringValue("y"));
+ ObjectNode attributes = JsonNodeFactory.instance.objectNode();
+ attributes.set("m", map);
+ attributes.set("n", JsonNodeFactory.instance.objectNode().set("M", JsonNodeFactory.instance.objectNode()));
+
+ assertPutCeiling(attributes, "m".length() + CONTAINER_COST + (1 + 1 + 1) + (2 + 1 + 1)
+ + "n".length() + CONTAINER_COST);
+ }
+
+ @Test
+ void putItemMeasuresStringAndBinarySetsByTheirMembersAlone() {
+ ObjectNode binarySet = JsonNodeFactory.instance.objectNode();
+ binarySet.putArray("BS")
+ .add(Base64.getEncoder().encodeToString(new byte[] {1}))
+ .add(Base64.getEncoder().encodeToString(new byte[] {2, 2}));
+ ObjectNode attributes = JsonNodeFactory.instance.objectNode();
+ attributes.set("ss", stringSetValue("x", "yy"));
+ attributes.set("bs", binarySet);
+
+ assertPutCeiling(attributes, "ss".length() + 1 + 2 + "bs".length() + 1 + 2);
+ }
+
+ @Test
+ void refusedPutLeavesTheStoredItemAsItWas() {
+ seed("a", "u", stringValue("y".repeat(10)));
+ JsonNode before = service.getItem(TABLE, key("a"), REGION).deepCopy();
+ ObjectNode over = key("a");
+ over.set("b", stringValue(padding("a", MAX_ITEM_SIZE + 1, 0)));
+
+ assertThrows(AwsException.class, () -> service.putItem(TABLE, over, REGION));
+ assertEquals(before, service.getItem(TABLE, key("a"), REGION));
+ }
+
+ @Test
+ void refusedUpdateLeavesTheStoredItemAsItWas() {
+ seed("a", "u", stringValue("y".repeat(10)));
+ JsonNode before = service.getItem(TABLE, key("a"), REGION).deepCopy();
+
+ assertRefused(() -> update("a", "SET b = :pad", MAX_ITEM_SIZE + 1, "u".length() + 10));
+ assertEquals(before, service.getItem(TABLE, key("a"), REGION));
+ }
+
+ @Test
+ void updateItemChargesListAppendAndTheOperandsItReads() {
+ int fixed = UPDATE_COST + SET_OR_ADD_COST + "l".length()
+ + LIST_APPEND_COST + operandPath("l") + CONTAINER_COST + 1;
+
+ assertUpdateCeiling(attributes("l", listValue(stringValue("x"))), "SET l = list_append(l, :list)",
+ pad -> values(":list", listValue(stringValue("x".repeat(pad)))), MAX_ITEM_SIZE - fixed);
+ }
+
+ @Test
+ void updateItemChargesIfNotExistsNestedInListAppend() {
+ int fixed = UPDATE_COST + SET_OR_ADD_COST + "l".length() + LIST_APPEND_COST
+ + IF_NOT_EXISTS_COST + operandPath("l") + CONTAINER_COST + CONTAINER_COST + 1;
+
+ assertUpdateCeiling(null, "SET l = list_append(if_not_exists(l, :empty), :list)", pad -> {
+ ObjectNode values = values(":list", listValue(stringValue("x".repeat(pad))));
+ values.set(":empty", listValue());
+ return values;
+ }, MAX_ITEM_SIZE - fixed);
+ }
+
+ @Test
+ void updateItemChargesTheSourcePathOfACopy() {
+ int fixed = UPDATE_COST + 2 * SET_OR_ADD_COST + "a".length() + operandPath("b") + "c".length();
+
+ assertUpdateCeiling(attributes("b", stringValue("y")), "SET a = b, c = :pad",
+ pad -> values(":pad", stringValue("x".repeat(pad))), MAX_ITEM_SIZE - fixed);
+ }
+
+ @Test
+ void updateItemChargesEveryElementOfANestedSourcePath() {
+ ObjectNode list = listValue();
+ for (int i = 0; i < 11; i++) {
+ ((ArrayNode) list.get("L")).add(stringValue("x"));
+ }
+ int fixed = UPDATE_COST + 2 * SET_OR_ADD_COST + "a".length() + operandPath("d", "ee", "10") + "c".length();
+
+ assertUpdateCeiling(attributes("d", mapValue("ee", list)), "SET a = d.ee[10], c = :pad",
+ pad -> values(":pad", stringValue("x".repeat(pad))), MAX_ITEM_SIZE - fixed);
+ }
+
+ @Test
+ void updateItemChargesPlusAndMinusDifferently() {
+ int one = 2;
+ int fixed = UPDATE_COST + 3 * SET_OR_ADD_COST
+ + "n".length() + PLUS_COST + operandPath("n") + one
+ + "m".length() + MINUS_COST + operandPath("m") + one
+ + "c".length();
+ ObjectNode seed = attributes("n", numberValue("1"));
+ seed.set("m", numberValue("1"));
+
+ assertUpdateCeiling(seed, "SET n = n + :one, m = m - :one, c = :pad", pad -> {
+ ObjectNode values = values(":pad", stringValue("x".repeat(pad)));
+ values.set(":one", numberValue("1"));
+ return values;
+ }, MAX_ITEM_SIZE - fixed);
+ }
+
+ @Test
+ void updateItemCountsASharedTargetPrefixOnce() {
+ int fixed = UPDATE_COST + 2 * SET_OR_ADD_COST
+ + "d".length() + TARGET_CONTAINER_COST + "a".length() + "b".length() + "e".length();
+
+ assertUpdateCeiling(attributes("d", JsonNodeFactory.instance.objectNode().set("M",
+ JsonNodeFactory.instance.objectNode())), "SET d.a = :pad, d.b = :e", pad -> {
+ ObjectNode values = values(":pad", stringValue("x".repeat(pad)));
+ values.set(":e", stringValue("e"));
+ return values;
+ }, MAX_ITEM_SIZE - fixed);
+ }
+
+ @Test
+ void updateItemCountsTheDigitsOfAListIndex() {
+ ObjectNode list = listValue();
+ for (int i = 0; i < 11; i++) {
+ ((ArrayNode) list.get("L")).add(stringValue("x"));
+ }
+ int fixed = UPDATE_COST + SET_OR_ADD_COST + "l".length() + TARGET_CONTAINER_COST + "10".length();
+
+ assertUpdateCeiling(attributes("l", list), "SET l[10] = :pad",
+ pad -> values(":pad", stringValue("x".repeat(pad))), MAX_ITEM_SIZE - fixed);
+ }
+
+ @Test
+ void updateItemChargesRemoveAndDeleteByTheirPathAlone() {
+ ObjectNode seed = attributes("rrrrrrrrrr", stringValue("y"));
+ seed.set("ssssssssss", stringSetValue("y", "z"));
+ int fixed = UPDATE_COST + 2 * (REMOVE_OR_DELETE_COST + 10) + SET_OR_ADD_COST + "c".length();
+
+ assertUpdateCeiling(seed, "REMOVE rrrrrrrrrr DELETE ssssssssss :z SET c = :pad", pad -> {
+ ObjectNode values = values(":pad", stringValue("x".repeat(pad)));
+ values.set(":z", stringSetValue("z".repeat(100)));
+ return values;
+ }, MAX_ITEM_SIZE - fixed);
+ }
+
+ @Test
+ void updateItemChargesTheValueAnAddWritesNotTheSetItBuilds() {
+ int fixed = UPDATE_COST + 2 * SET_OR_ADD_COST + "ss".length() + 1 + "c".length();
+
+ assertUpdateCeiling(attributes("ss", stringSetValue("yy")), "ADD ss :v SET c = :pad", pad -> {
+ ObjectNode values = values(":pad", stringValue("x".repeat(pad)));
+ values.set(":v", stringSetValue("z"));
+ return values;
+ }, MAX_ITEM_SIZE - fixed);
+ }
+
private void update(String keyValue, String expression, int totalBytes, int otherBytes) {
update(keyValue, expression, totalBytes, otherBytes, JsonNodeFactory.instance.objectNode());
}
@@ -279,6 +450,49 @@ class DynamoDbItemSizeServiceTest {
return member;
}
+ private void assertPutCeiling(ObjectNode attributes, int attributeBytes) {
+ putPadded("a", MAX_ITEM_SIZE, attributes, attributeBytes);
+ assertEquals(MAX_ITEM_SIZE, storedBytes("a"));
+ AwsException refused = assertThrows(AwsException.class,
+ () -> putPadded("b", MAX_ITEM_SIZE + 1, attributes, attributeBytes));
+ assertEquals("Item size has exceeded the maximum allowed size", refused.getMessage());
+ }
+
+ private void putPadded(String keyValue, int totalBytes, ObjectNode attributes, int attributeBytes) {
+ ObjectNode item = key(keyValue);
+ item.setAll(attributes.deepCopy());
+ item.set("b", stringValue(padding(keyValue, totalBytes, attributeBytes)));
+ service.putItem(TABLE, item, REGION);
+ }
+
+ private void assertUpdateCeiling(ObjectNode seed, String expression, IntFunction<ObjectNode> values,
+ int maxPadding) {
+ for (String keyValue : List.of("a", "b")) {
+ if (seed != null) {
+ ObjectNode item = key(keyValue);
+ item.setAll(seed.deepCopy());
+ service.putItem(TABLE, item, REGION);
+ }
+ }
+ service.updateItem(TABLE, key("a"), null, expression, null, values.apply(maxPadding), "NONE", REGION);
+ assertRefused(() -> service.updateItem(TABLE, key("b"), null, expression, null,
+ values.apply(maxPadding + 1), "NONE", REGION));
+ }
+
+ private static int operandPath(String... elements) {
+ int size = OPERAND_PATH_COST;
+ for (String element : elements) {
+ size += OPERAND_PATH_ELEMENT_COST + element.length();
+ }
+ return size;
+ }
+
+ private ObjectNode attributes(String name, JsonNode value) {
+ ObjectNode attributes = JsonNodeFactory.instance.objectNode();
+ attributes.set(name, value);
+ return attributes;
+ }
+
private void seed(String keyValue, String attribute, JsonNode value) {
ObjectNode item = key(keyValue);
item.set(attribute, value);
@@ -318,9 +532,12 @@ class DynamoDbItemSizeServiceTest {
return value;
}
- private ObjectNode stringSetValue(String member) {
+ private ObjectNode stringSetValue(String... members) {
ObjectNode value = JsonNodeFactory.instance.objectNode();
- value.putArray("SS").add(member);
+ ArrayNode set = value.putArray("SS");
+ for (String member : members) {
+ set.add(member);
+ }
return value;
}
@@ -330,9 +547,12 @@ class DynamoDbItemSizeServiceTest {
return map;
}
- private ObjectNode listValue(JsonNode element) {
+ private ObjectNode listValue(JsonNode... elements) {
ObjectNode list = JsonNodeFactory.instance.objectNode();
- list.putArray("L").add(element);
+ ArrayNode array = list.putArray("L");
+ for (JsonNode element : elements) {
+ array.add(element);
+ }
return list;
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandlerTest.java
index 652c61605a..73440622c8 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandlerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/NativeDynamoDbJsonHandlerTest.java
@@ -1567,6 +1567,30 @@ class NativeDynamoDbJsonHandlerTest {
+ "Expression size has exceeded the maximum allowed size;", ex.getMessage());
}
+ @Test
+ void updateItemRejectsAnUnclosedCallAtTheEndBeforeTheTableLookup() {
+ Map<String, String> nearByExpression = Map.of(
+ "SET a = list_append(l, :v", ":v",
+ "SET a = if_not_exists(a,:v", ":v",
+ "SET a = list_append(l, :v ", ":v ",
+ "SET a = list_append(if_not_exists(l, :v, :v)", ")",
+ "SET a = if_not_exists(a.b[1], c[0]", "]",
+ "SET a = list_append(", "(",
+ "SET a = :v, b = if_not_exists(b, :v", ":v",
+ "SET a = :v + (:v", ":v");
+ nearByExpression.forEach((expression, near) -> {
+ ObjectNode request = updateUserRequest();
+ request.put("TableName", "Missing");
+ request.put("UpdateExpression", expression);
+
+ AwsException ex = assertThrows(AwsException.class,
+ () -> handler.handle("UpdateItem", request, "eu-west-1"), expression);
+ assertEquals("ValidationException", ex.getErrorCode());
+ assertEquals("1 validation error detected: Invalid UpdateExpression: Syntax error; "
+ + "token: \"<EOF>\", near: \"" + near + "\"", ex.getMessage());
+ });
+ }
+
@Test
void updateItemAcceptsAnUpdateExpressionOfExactly4096Bytes() throws Exception {
createUsersTable("eu-west-1");
@@ -1779,6 +1803,20 @@ class NativeDynamoDbJsonHandlerTest {
ex.getMessage());
}
+ @Test
+ void transactWriteItemsRejectsAnUnclosedCallAtTheEnd() {
+ createUsersTable("eu-west-1");
+ ObjectNode update = mapper.createObjectNode();
+ update.set("Key", item("userId", "u1"));
+ update.put("UpdateExpression", "SET a = list_append(l, :v");
+ update.set("ExpressionAttributeValues", singleValue(":v"));
+
+ AwsException ex = assertThrows(AwsException.class,
+ () -> handler.handle("TransactWriteItems", transactWrite("Update", update), "eu-west-1"));
+ assertEquals("ValidationException", ex.getErrorCode());
+ assertEquals("Invalid UpdateExpression: Syntax error; token: \"<EOF>\", near: \":v\"", ex.getMessage());
+ }
+
@Test
void transactWriteItemsRejectsAPutItemWithALeafAtLevel33() {
createUsersTable("eu-west-1");