Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(dynamodb): return set members in DynamoDB's order (#6520)

TATSUNO “Taz” Yasuhiro2a5f52e0146b
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportRecords.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportRecords.java
index f355629216..adce675006 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportRecords.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportRecords.java
@@ -7,12 +7,10 @@ import com.fasterxml.jackson.databind.node.ObjectNode;
import io.github.hectorvent.floci.services.dynamodb.DynamoDbPointInTimeRecovery.ItemChange;
import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;
-import java.math.BigDecimal;
import java.time.Instant;
import java.time.temporal.ChronoUnit;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Base64;
import java.util.Comparator;
import java.util.List;
import java.util.Map;
@@ -81,18 +79,11 @@ final class DynamoDbExportRecords {
content.forEach(member -> members.add(sortedValue(member)));
yield NODES.objectNode().set("L", members);
}
- case "SS" -> sortedSet("SS", content, Comparator.comparing(JsonNode::asText, CODE_POINT_ORDER));
- case "NS" -> sortedSet("NS", content, Comparator.comparing(member -> new BigDecimal(member.asText())));
- case "BS" -> sortedSet("BS", content, (a, b) -> Arrays.compareUnsigned(
- Base64.getDecoder().decode(a.asText()), Base64.getDecoder().decode(b.asText())));
+ case "SS", "NS", "BS" -> {
+ String setType = typed.getKey();
+ yield NODES.objectNode().set(setType, DynamoDbSetOrder.sorted(setType, content));
+ }
default -> value;
};
}
-
- private static JsonNode sortedSet(String type, JsonNode members, Comparator<JsonNode> order) {
- List<JsonNode> sorted = new ArrayList<>();
- members.forEach(sorted::add);
- sorted.sort(order);
- return NODES.objectNode().set(type, NODES.arrayNode().addAll(sorted));
- }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbNumberUtils.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbNumberUtils.java
index e0a95f2a65..c5aa49744b 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbNumberUtils.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbNumberUtils.java
@@ -139,9 +139,9 @@ final class DynamoDbNumberUtils {
}
/**
- * Recursively normalizes all N-type number attribute values in the given item.
- * Returns a new ObjectNode with normalized values. Input must be a DynamoDB item
- * where attribute values are DynamoDB typed values (e.g. {S: "x"}, {N: "42"}).
+ * Recursively normalizes all N-type number attribute values in the given item and
+ * puts set members in DynamoDB's order. Returns a new ObjectNode. Input must be a
+ * DynamoDB item where attribute values are DynamoDB typed values (e.g. {S: "x"}, {N: "42"}).
*/
static ObjectNode normalizeNumbersInItem(JsonNode item) {
if (item == null || !item.isObject()) return (ObjectNode) item;
@@ -177,7 +177,14 @@ final class DynamoDbNumberUtils {
for (JsonNode elem : attr.get("NS")) {
ns.add(validateAndNormalize(elem.asText()));
}
- result.set("NS", ns);
+ result.set("NS", DynamoDbSetOrder.sorted("NS", ns));
+ return result;
+ }
+
+ if (attr.has("SS") || attr.has("BS")) {
+ String setType = attr.has("SS") ? "SS" : "BS";
+ ObjectNode result = JsonNodeFactory.instance.objectNode();
+ result.set(setType, DynamoDbSetOrder.sorted(setType, attr.get(setType)));
return result;
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
index 8fcbcbba7e..d9dca7efaa 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
@@ -1480,6 +1480,8 @@ public class DynamoDbService {
}
}
+ DynamoDbSetOrder.sortSetsIn(item);
+
// Reject any attempt to modify a key attribute
validateKeyNotModified(table, key, item);
requireItemNestingWithinLimit(item);
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbSetOrder.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbSetOrder.java
new file mode 100644
index 0000000000..41cd81894b
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbSetOrder.java
@@ -0,0 +1,104 @@
+package io.github.hectorvent.floci.services.dynamodb;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+import java.math.BigDecimal;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Comparator;
+import java.util.List;
+import java.util.function.Function;
+
+/**
+ * DynamoDB stores a set's members in its own order, not as written: strings by
+ * code point, numbers by value, binaries by unsigned byte.
+ */
+final class DynamoDbSetOrder {
+
+ private static final List<String> SET_TYPES = List.of("SS", "NS", "BS");
+
+ private record Keyed<K>(JsonNode member, K key) {}
+
+ private DynamoDbSetOrder() {}
+
+ /** Returns the members in DynamoDB's order, or the input itself when it is not an array. */
+ static JsonNode sorted(String setType, JsonNode members) {
+ if (!members.isArray()) {
+ return members;
+ }
+ return switch (setType) {
+ case "NS" -> sortedBy(members, DynamoDbSetOrder::numberOf, Comparator.naturalOrder());
+ case "BS" -> sortedBy(members, DynamoDbSetOrder::bytesOf, Arrays::compareUnsigned);
+ default -> sortedBy(members, DynamoDbSetOrder::utf8Of, Arrays::compareUnsigned);
+ };
+ }
+
+ /** Reorders every set in the attributes in place, through maps and lists. */
+ static void sortSetsIn(JsonNode attributes) {
+ if (attributes == null || !attributes.isObject()) {
+ return;
+ }
+ for (JsonNode value : attributes) {
+ sortSetsInValue(value);
+ }
+ }
+
+ private static void sortSetsInValue(JsonNode value) {
+ if (!(value instanceof ObjectNode object)) {
+ return;
+ }
+ for (String setType : SET_TYPES) {
+ if (object.has(setType)) {
+ object.set(setType, sorted(setType, object.get(setType)));
+ return;
+ }
+ }
+ if (object.has("M")) {
+ sortSetsIn(object.get("M"));
+ } else if (object.has("L")) {
+ object.get("L").forEach(DynamoDbSetOrder::sortSetsInValue);
+ }
+ }
+
+ private static <K> ArrayNode sortedBy(JsonNode members, Function<String, K> key, Comparator<K> order) {
+ List<Keyed<K>> keyed = new ArrayList<>(members.size());
+ for (JsonNode member : members) {
+ keyed.add(new Keyed<>(member, key.apply(member.asText())));
+ }
+ keyed.sort(Comparator.comparing(Keyed::key, order));
+ ArrayNode result = JsonNodeFactory.instance.arrayNode(keyed.size());
+ for (Keyed<K> entry : keyed) {
+ result.add(entry.member());
+ }
+ return result;
+ }
+
+ // UTF-8 byte order is code point order. String.compareTo is not: it orders by
+ // UTF-16 unit, which puts a supplementary character before U+E000..U+FFFF.
+ private static byte[] utf8Of(String text) {
+ return text.getBytes(StandardCharsets.UTF_8);
+ }
+
+ // Number syntax and base64 are checked when a request is read. A member that
+ // still does not parse keeps a stable place instead of failing the write.
+ private static BigDecimal numberOf(String text) {
+ try {
+ return new BigDecimal(text);
+ } catch (NumberFormatException ignored) {
+ return BigDecimal.ZERO;
+ }
+ }
+
+ private static byte[] bytesOf(String base64) {
+ try {
+ return Base64.getDecoder().decode(base64);
+ } catch (IllegalArgumentException ignored) {
+ return utf8Of(base64);
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbSetOrderIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbSetOrderIntegrationTest.java
new file mode 100644
index 0000000000..cda0509b4c
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbSetOrderIntegrationTest.java
@@ -0,0 +1,107 @@
+package io.github.hectorvent.floci.services.dynamodb;
+
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.response.ValidatableResponse;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.MethodOrderer;
+import org.junit.jupiter.api.Order;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.TestMethodOrder;
+
+import static io.github.hectorvent.floci.testing.DynamoDbRequests.ddb;
+import static org.hamcrest.Matchers.contains;
+
+/**
+ * A set comes back in DynamoDB's order however it was written: strings by code point,
+ * numbers by value, binaries by unsigned byte.
+ */
+@QuarkusTest
+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)
+class DynamoDbSetOrderIntegrationTest {
+
+ private static final String TABLE = "set-member-order";
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ @Order(1)
+ void putItemStoresEverySetInDynamoDbOrder() {
+ ddb("CreateTable", """
+ {
+ "TableName": "%s",
+ "AttributeDefinitions": [{"AttributeName":"pk","AttributeType":"S"}],
+ "KeySchema": [{"AttributeName":"pk","KeyType":"HASH"}],
+ "BillingMode": "PAY_PER_REQUEST"
+ }
+ """.formatted(TABLE))
+ .statusCode(200);
+ ddb("PutItem", """
+ {"TableName":"%s","Item":{"pk":{"S":"a"},
+ "ss":{"SS":["b","a","B","\\uff5e","\\ud83d\\ude00"]},
+ "ns":{"NS":["10","9","1e2","-1"]},
+ "bs":{"BS":["/w==","gA==","fw==","AQ==","AP8="]},
+ "m":{"M":{"deep":{"SS":["b","a",""]}}},
+ "l":{"L":[{"NS":["2","1"]}]}}}
+ """.formatted(TABLE))
+ .statusCode(200);
+ getItem("a")
+ .body("Item.ss.SS", contains("B", "a", "b", "~", "😀"))
+ .body("Item.ns.NS", contains("-1", "9", "10", "100"))
+ .body("Item.bs.BS", contains("AP8=", "AQ==", "fw==", "gA==", "/w=="))
+ .body("Item.m.M.deep.SS", contains("", "a", "b"))
+ .body("Item.l.L[0].NS", contains("1", "2"));
+ }
+
+ @Test
+ @Order(2)
+ void updateExpressionsKeepTheOrder() {
+ ddb("UpdateItem", """
+ {"TableName":"%s","Key":{"pk":{"S":"a"}},"ReturnValues":"ALL_NEW",
+ "UpdateExpression":"ADD ss :s, ns :n SET s2 = :v",
+ "ExpressionAttributeValues":{":s":{"SS":["c",""]},":n":{"NS":["5","-3"]},":v":{"SS":["z","y"]}}}
+ """.formatted(TABLE))
+ .statusCode(200)
+ .body("Attributes.ss.SS", contains("", "B", "a", "b", "c", "~", "😀"))
+ .body("Attributes.ns.NS", contains("-3", "-1", "5", "9", "10", "100"))
+ .body("Attributes.s2.SS", contains("y", "z"));
+ ddb("UpdateItem", """
+ {"TableName":"%s","Key":{"pk":{"S":"a"}},"ReturnValues":"UPDATED_NEW",
+ "AttributeUpdates":{"s2":{"Action":"ADD","Value":{"SS":["x"]}}}}
+ """.formatted(TABLE))
+ .statusCode(200)
+ .body("Attributes.s2.SS", contains("x", "y", "z"));
+ getItem("a")
+ .body("Item.ss.SS", contains("", "B", "a", "b", "c", "~", "😀"))
+ .body("Item.s2.SS", contains("x", "y", "z"));
+ }
+
+ @Test
+ @Order(3)
+ void batchAndTransactWritesKeepTheOrder() {
+ ddb("BatchWriteItem", """
+ {"RequestItems":{"%s":[{"PutRequest":{"Item":{"pk":{"S":"batch"},"ss":{"SS":["b","a"]}}}}]}}
+ """.formatted(TABLE))
+ .statusCode(200);
+ ddb("TransactWriteItems", """
+ {"TransactItems":[
+ {"Put":{"TableName":"%1$s","Item":{"pk":{"S":"tx"},"ns":{"NS":["2","1"]}}}},
+ {"Update":{"TableName":"%1$s","Key":{"pk":{"S":"tx2"}},"UpdateExpression":"ADD ss :v",
+ "ExpressionAttributeValues":{":v":{"SS":["d","c"]}}}}]}
+ """.formatted(TABLE))
+ .statusCode(200);
+ getItem("batch").body("Item.ss.SS", contains("a", "b"));
+ getItem("tx").body("Item.ns.NS", contains("1", "2"));
+ getItem("tx2").body("Item.ss.SS", contains("c", "d"));
+ }
+
+ private static ValidatableResponse getItem(String pk) {
+ return ddb("GetItem", """
+ {"TableName":"%s","Key":{"pk":{"S":"%s"}},"ConsistentRead":true}
+ """.formatted(TABLE, pk))
+ .statusCode(200);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbSetOrderTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbSetOrderTest.java
new file mode 100644
index 0000000000..7be76c1b98
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbSetOrderTest.java
@@ -0,0 +1,80 @@
+package io.github.hectorvent.floci.services.dynamodb;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import org.junit.jupiter.api.Test;
+
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertSame;
+
+class DynamoDbSetOrderTest {
+
+ private static final ObjectMapper MAPPER = new ObjectMapper();
+
+ @Test
+ void stringsOrderByCodePoint() {
+ List<String> sorted = sorted("SS",
+ List.of("b", "a", "", "B", "aa", "é", "Z", "~", "ab", "~", "😀"));
+ assertEquals(List.of("", "B", "Z", "a", "aa", "ab", "b", "~", "é", "~", "😀"), sorted);
+ }
+
+ @Test
+ void numbersOrderByValue() {
+ List<String> sorted = sorted("NS", List.of("10", "9", "-1", "1.5", "0", "1e2", "0.001", "-0.5"));
+ assertEquals(List.of("-1", "-0.5", "0", "0.001", "1.5", "9", "10", "1e2"), sorted);
+ }
+
+ @Test
+ void binariesOrderByUnsignedByte() {
+ List<String> sorted = sorted("BS", List.of(base64(0x80), base64(0x02), base64(0xff), base64(0x01, 0x00),
+ base64(0x00), base64(0xc0), base64(0x7f), base64(0x01), base64(0x00, 0xff), base64(0x01, 0x01)));
+ assertEquals(List.of(base64(0x00), base64(0x00, 0xff), base64(0x01), base64(0x01, 0x00), base64(0x01, 0x01),
+ base64(0x02), base64(0x7f), base64(0x80), base64(0xc0), base64(0xff)), sorted);
+ }
+
+ @Test
+ void sortSetsInReachesMapsAndLists() throws Exception {
+ ObjectNode item = (ObjectNode) MAPPER.readTree("""
+ {"pk":{"S":"a"},"ss":{"SS":["b","a"]},
+ "m":{"M":{"deep":{"SS":["b","a",""]},"nums":{"NS":["10","9"]}}},
+ "l":{"L":[{"NS":["2","1"]},{"M":{"inner":{"BS":["gA==","AQ=="]}}}]}}
+ """);
+ DynamoDbSetOrder.sortSetsIn(item);
+ assertEquals(MAPPER.readTree("""
+ {"pk":{"S":"a"},"ss":{"SS":["a","b"]},
+ "m":{"M":{"deep":{"SS":["","a","b"]},"nums":{"NS":["9","10"]}}},
+ "l":{"L":[{"NS":["1","2"]},{"M":{"inner":{"BS":["AQ==","gA=="]}}}]}}
+ """), item);
+ }
+
+ @Test
+ void membersThatDoNotParseKeepAPlace() {
+ assertEquals(List.of("abc", "0", "1"), sorted("NS", List.of("1", "abc", "0")));
+ assertEquals(List.of("AQ==", "not base64"), sorted("BS", List.of("not base64", "AQ==")));
+ }
+
+ @Test
+ void aMemberListThatIsNotAnArrayIsLeftAlone() throws Exception {
+ JsonNode members = MAPPER.readTree("\"x\"");
+ assertSame(members, DynamoDbSetOrder.sorted("SS", members));
+ }
+
+ private static List<String> sorted(String setType, List<String> members) {
+ List<String> result = new ArrayList<>();
+ DynamoDbSetOrder.sorted(setType, MAPPER.valueToTree(members)).forEach(node -> result.add(node.asText()));
+ return result;
+ }
+
+ private static String base64(int... bytes) {
+ byte[] raw = new byte[bytes.length];
+ for (int i = 0; i < bytes.length; i++) {
+ raw[i] = (byte) bytes[i];
+ }
+ return Base64.getEncoder().encodeToString(raw);
+ }
+}