Light, fluffy, and always free - The AWS Local Emulator alternative
fix(dynamodb): return set members in DynamoDB's order (#6520)
TATSUNO “Taz” Yasuhiro2a5f52e0146b
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportRecords.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportRecords.javaindex f355629216..adce675006 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportRecords.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbExportRecords.java@@ -7,12 +7,10 @@ import com.fasterxml.jackson.databind.node.ObjectNode;import io.github.hectorvent.floci.services.dynamodb.DynamoDbPointInTimeRecovery.ItemChange;import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;-import java.math.BigDecimal;import java.time.Instant;import java.time.temporal.ChronoUnit;import java.util.ArrayList;import java.util.Arrays;-import java.util.Base64;import java.util.Comparator;import java.util.List;import java.util.Map;@@ -81,18 +79,11 @@ final class DynamoDbExportRecords {content.forEach(member -> members.add(sortedValue(member)));yield NODES.objectNode().set("L", members);}- case "SS" -> sortedSet("SS", content, Comparator.comparing(JsonNode::asText, CODE_POINT_ORDER));- case "NS" -> sortedSet("NS", content, Comparator.comparing(member -> new BigDecimal(member.asText())));- case "BS" -> sortedSet("BS", content, (a, b) -> Arrays.compareUnsigned(- Base64.getDecoder().decode(a.asText()), Base64.getDecoder().decode(b.asText())));+ case "SS", "NS", "BS" -> {+ String setType = typed.getKey();+ yield NODES.objectNode().set(setType, DynamoDbSetOrder.sorted(setType, content));+ }default -> value;};}-- private static JsonNode sortedSet(String type, JsonNode members, Comparator<JsonNode> order) {- List<JsonNode> sorted = new ArrayList<>();- members.forEach(sorted::add);- sorted.sort(order);- return NODES.objectNode().set(type, NODES.arrayNode().addAll(sorted));- }}diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbNumberUtils.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbNumberUtils.javaindex e0a95f2a65..c5aa49744b 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbNumberUtils.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbNumberUtils.java@@ -139,9 +139,9 @@ final class DynamoDbNumberUtils {}/**- * Recursively normalizes all N-type number attribute values in the given item.- * Returns a new ObjectNode with normalized values. Input must be a DynamoDB item- * where attribute values are DynamoDB typed values (e.g. {S: "x"}, {N: "42"}).+ * Recursively normalizes all N-type number attribute values in the given item and+ * puts set members in DynamoDB's order. Returns a new ObjectNode. Input must be a+ * DynamoDB item where attribute values are DynamoDB typed values (e.g. {S: "x"}, {N: "42"}).*/static ObjectNode normalizeNumbersInItem(JsonNode item) {if (item == null || !item.isObject()) return (ObjectNode) item;@@ -177,7 +177,14 @@ final class DynamoDbNumberUtils {for (JsonNode elem : attr.get("NS")) {ns.add(validateAndNormalize(elem.asText()));}- result.set("NS", ns);+ result.set("NS", DynamoDbSetOrder.sorted("NS", ns));+ return result;+ }++ if (attr.has("SS") || attr.has("BS")) {+ String setType = attr.has("SS") ? "SS" : "BS";+ ObjectNode result = JsonNodeFactory.instance.objectNode();+ result.set(setType, DynamoDbSetOrder.sorted(setType, attr.get(setType)));return result;}diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.javaindex 8fcbcbba7e..d9dca7efaa 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java@@ -1480,6 +1480,8 @@ public class DynamoDbService {}}+ DynamoDbSetOrder.sortSetsIn(item);+// Reject any attempt to modify a key attributevalidateKeyNotModified(table, key, item);requireItemNestingWithinLimit(item);diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbSetOrder.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbSetOrder.javanew file mode 100644index 0000000000..41cd81894b--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbSetOrder.java@@ -0,0 +1,104 @@+package io.github.hectorvent.floci.services.dynamodb;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.node.ArrayNode;+import com.fasterxml.jackson.databind.node.JsonNodeFactory;+import com.fasterxml.jackson.databind.node.ObjectNode;++import java.math.BigDecimal;+import java.nio.charset.StandardCharsets;+import java.util.ArrayList;+import java.util.Arrays;+import java.util.Base64;+import java.util.Comparator;+import java.util.List;+import java.util.function.Function;++/**+ * DynamoDB stores a set's members in its own order, not as written: strings by+ * code point, numbers by value, binaries by unsigned byte.+ */+final class DynamoDbSetOrder {++ private static final List<String> SET_TYPES = List.of("SS", "NS", "BS");++ private record Keyed<K>(JsonNode member, K key) {}++ private DynamoDbSetOrder() {}++ /** Returns the members in DynamoDB's order, or the input itself when it is not an array. */+ static JsonNode sorted(String setType, JsonNode members) {+ if (!members.isArray()) {+ return members;+ }+ return switch (setType) {+ case "NS" -> sortedBy(members, DynamoDbSetOrder::numberOf, Comparator.naturalOrder());+ case "BS" -> sortedBy(members, DynamoDbSetOrder::bytesOf, Arrays::compareUnsigned);+ default -> sortedBy(members, DynamoDbSetOrder::utf8Of, Arrays::compareUnsigned);+ };+ }++ /** Reorders every set in the attributes in place, through maps and lists. */+ static void sortSetsIn(JsonNode attributes) {+ if (attributes == null || !attributes.isObject()) {+ return;+ }+ for (JsonNode value : attributes) {+ sortSetsInValue(value);+ }+ }++ private static void sortSetsInValue(JsonNode value) {+ if (!(value instanceof ObjectNode object)) {+ return;+ }+ for (String setType : SET_TYPES) {+ if (object.has(setType)) {+ object.set(setType, sorted(setType, object.get(setType)));+ return;+ }+ }+ if (object.has("M")) {+ sortSetsIn(object.get("M"));+ } else if (object.has("L")) {+ object.get("L").forEach(DynamoDbSetOrder::sortSetsInValue);+ }+ }++ private static <K> ArrayNode sortedBy(JsonNode members, Function<String, K> key, Comparator<K> order) {+ List<Keyed<K>> keyed = new ArrayList<>(members.size());+ for (JsonNode member : members) {+ keyed.add(new Keyed<>(member, key.apply(member.asText())));+ }+ keyed.sort(Comparator.comparing(Keyed::key, order));+ ArrayNode result = JsonNodeFactory.instance.arrayNode(keyed.size());+ for (Keyed<K> entry : keyed) {+ result.add(entry.member());+ }+ return result;+ }++ // UTF-8 byte order is code point order. String.compareTo is not: it orders by+ // UTF-16 unit, which puts a supplementary character before U+E000..U+FFFF.+ private static byte[] utf8Of(String text) {+ return text.getBytes(StandardCharsets.UTF_8);+ }++ // Number syntax and base64 are checked when a request is read. A member that+ // still does not parse keeps a stable place instead of failing the write.+ private static BigDecimal numberOf(String text) {+ try {+ return new BigDecimal(text);+ } catch (NumberFormatException ignored) {+ return BigDecimal.ZERO;+ }+ }++ private static byte[] bytesOf(String base64) {+ try {+ return Base64.getDecoder().decode(base64);+ } catch (IllegalArgumentException ignored) {+ return utf8Of(base64);+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbSetOrderIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbSetOrderIntegrationTest.javanew file mode 100644index 0000000000..cda0509b4c--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbSetOrderIntegrationTest.java@@ -0,0 +1,107 @@+package io.github.hectorvent.floci.services.dynamodb;++import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import io.restassured.response.ValidatableResponse;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.MethodOrderer;+import org.junit.jupiter.api.Order;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.TestMethodOrder;++import static io.github.hectorvent.floci.testing.DynamoDbRequests.ddb;+import static org.hamcrest.Matchers.contains;++/**+ * A set comes back in DynamoDB's order however it was written: strings by code point,+ * numbers by value, binaries by unsigned byte.+ */+@QuarkusTest+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class DynamoDbSetOrderIntegrationTest {++ private static final String TABLE = "set-member-order";++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @Test+ @Order(1)+ void putItemStoresEverySetInDynamoDbOrder() {+ ddb("CreateTable", """+ {+ "TableName": "%s",+ "AttributeDefinitions": [{"AttributeName":"pk","AttributeType":"S"}],+ "KeySchema": [{"AttributeName":"pk","KeyType":"HASH"}],+ "BillingMode": "PAY_PER_REQUEST"+ }+ """.formatted(TABLE))+ .statusCode(200);+ ddb("PutItem", """+ {"TableName":"%s","Item":{"pk":{"S":"a"},+ "ss":{"SS":["b","a","B","\\uff5e","\\ud83d\\ude00"]},+ "ns":{"NS":["10","9","1e2","-1"]},+ "bs":{"BS":["/w==","gA==","fw==","AQ==","AP8="]},+ "m":{"M":{"deep":{"SS":["b","a",""]}}},+ "l":{"L":[{"NS":["2","1"]}]}}}+ """.formatted(TABLE))+ .statusCode(200);+ getItem("a")+ .body("Item.ss.SS", contains("B", "a", "b", "~", "😀"))+ .body("Item.ns.NS", contains("-1", "9", "10", "100"))+ .body("Item.bs.BS", contains("AP8=", "AQ==", "fw==", "gA==", "/w=="))+ .body("Item.m.M.deep.SS", contains("", "a", "b"))+ .body("Item.l.L[0].NS", contains("1", "2"));+ }++ @Test+ @Order(2)+ void updateExpressionsKeepTheOrder() {+ ddb("UpdateItem", """+ {"TableName":"%s","Key":{"pk":{"S":"a"}},"ReturnValues":"ALL_NEW",+ "UpdateExpression":"ADD ss :s, ns :n SET s2 = :v",+ "ExpressionAttributeValues":{":s":{"SS":["c",""]},":n":{"NS":["5","-3"]},":v":{"SS":["z","y"]}}}+ """.formatted(TABLE))+ .statusCode(200)+ .body("Attributes.ss.SS", contains("", "B", "a", "b", "c", "~", "😀"))+ .body("Attributes.ns.NS", contains("-3", "-1", "5", "9", "10", "100"))+ .body("Attributes.s2.SS", contains("y", "z"));+ ddb("UpdateItem", """+ {"TableName":"%s","Key":{"pk":{"S":"a"}},"ReturnValues":"UPDATED_NEW",+ "AttributeUpdates":{"s2":{"Action":"ADD","Value":{"SS":["x"]}}}}+ """.formatted(TABLE))+ .statusCode(200)+ .body("Attributes.s2.SS", contains("x", "y", "z"));+ getItem("a")+ .body("Item.ss.SS", contains("", "B", "a", "b", "c", "~", "😀"))+ .body("Item.s2.SS", contains("x", "y", "z"));+ }++ @Test+ @Order(3)+ void batchAndTransactWritesKeepTheOrder() {+ ddb("BatchWriteItem", """+ {"RequestItems":{"%s":[{"PutRequest":{"Item":{"pk":{"S":"batch"},"ss":{"SS":["b","a"]}}}}]}}+ """.formatted(TABLE))+ .statusCode(200);+ ddb("TransactWriteItems", """+ {"TransactItems":[+ {"Put":{"TableName":"%1$s","Item":{"pk":{"S":"tx"},"ns":{"NS":["2","1"]}}}},+ {"Update":{"TableName":"%1$s","Key":{"pk":{"S":"tx2"}},"UpdateExpression":"ADD ss :v",+ "ExpressionAttributeValues":{":v":{"SS":["d","c"]}}}}]}+ """.formatted(TABLE))+ .statusCode(200);+ getItem("batch").body("Item.ss.SS", contains("a", "b"));+ getItem("tx").body("Item.ns.NS", contains("1", "2"));+ getItem("tx2").body("Item.ss.SS", contains("c", "d"));+ }++ private static ValidatableResponse getItem(String pk) {+ return ddb("GetItem", """+ {"TableName":"%s","Key":{"pk":{"S":"%s"}},"ConsistentRead":true}+ """.formatted(TABLE, pk))+ .statusCode(200);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbSetOrderTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbSetOrderTest.javanew file mode 100644index 0000000000..7be76c1b98--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbSetOrderTest.java@@ -0,0 +1,80 @@+package io.github.hectorvent.floci.services.dynamodb;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ObjectNode;+import org.junit.jupiter.api.Test;++import java.util.ArrayList;+import java.util.Base64;+import java.util.List;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertSame;++class DynamoDbSetOrderTest {++ private static final ObjectMapper MAPPER = new ObjectMapper();++ @Test+ void stringsOrderByCodePoint() {+ List<String> sorted = sorted("SS",+ List.of("b", "a", "", "B", "aa", "é", "Z", "~", "ab", "~", "😀"));+ assertEquals(List.of("", "B", "Z", "a", "aa", "ab", "b", "~", "é", "~", "😀"), sorted);+ }++ @Test+ void numbersOrderByValue() {+ List<String> sorted = sorted("NS", List.of("10", "9", "-1", "1.5", "0", "1e2", "0.001", "-0.5"));+ assertEquals(List.of("-1", "-0.5", "0", "0.001", "1.5", "9", "10", "1e2"), sorted);+ }++ @Test+ void binariesOrderByUnsignedByte() {+ List<String> sorted = sorted("BS", List.of(base64(0x80), base64(0x02), base64(0xff), base64(0x01, 0x00),+ base64(0x00), base64(0xc0), base64(0x7f), base64(0x01), base64(0x00, 0xff), base64(0x01, 0x01)));+ assertEquals(List.of(base64(0x00), base64(0x00, 0xff), base64(0x01), base64(0x01, 0x00), base64(0x01, 0x01),+ base64(0x02), base64(0x7f), base64(0x80), base64(0xc0), base64(0xff)), sorted);+ }++ @Test+ void sortSetsInReachesMapsAndLists() throws Exception {+ ObjectNode item = (ObjectNode) MAPPER.readTree("""+ {"pk":{"S":"a"},"ss":{"SS":["b","a"]},+ "m":{"M":{"deep":{"SS":["b","a",""]},"nums":{"NS":["10","9"]}}},+ "l":{"L":[{"NS":["2","1"]},{"M":{"inner":{"BS":["gA==","AQ=="]}}}]}}+ """);+ DynamoDbSetOrder.sortSetsIn(item);+ assertEquals(MAPPER.readTree("""+ {"pk":{"S":"a"},"ss":{"SS":["a","b"]},+ "m":{"M":{"deep":{"SS":["","a","b"]},"nums":{"NS":["9","10"]}}},+ "l":{"L":[{"NS":["1","2"]},{"M":{"inner":{"BS":["AQ==","gA=="]}}}]}}+ """), item);+ }++ @Test+ void membersThatDoNotParseKeepAPlace() {+ assertEquals(List.of("abc", "0", "1"), sorted("NS", List.of("1", "abc", "0")));+ assertEquals(List.of("AQ==", "not base64"), sorted("BS", List.of("not base64", "AQ==")));+ }++ @Test+ void aMemberListThatIsNotAnArrayIsLeftAlone() throws Exception {+ JsonNode members = MAPPER.readTree("\"x\"");+ assertSame(members, DynamoDbSetOrder.sorted("SS", members));+ }++ private static List<String> sorted(String setType, List<String> members) {+ List<String> result = new ArrayList<>();+ DynamoDbSetOrder.sorted(setType, MAPPER.valueToTree(members)).forEach(node -> result.add(node.asText()));+ return result;+ }++ private static String base64(int... bytes) {+ byte[] raw = new byte[bytes.length];+ for (int i = 0; i < bytes.length; i++) {+ raw[i] = (byte) bytes[i];+ }+ return Base64.getEncoder().encodeToString(raw);+ }+}