Light, fluffy, and always free - The AWS Local Emulator alternative
fix(dynamodb): escape composite key segments (#3804)
XuJian5340e88605f3
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.javaindex a11f789a80..539a017f15 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java@@ -183,20 +183,46 @@ public class DynamoDbService implements ResourceProvider {private void loadPersistedItems() {if (itemStore == null) return;+ Map<String, TableDefinition> persistedTables = persistedTablesByScopedKey();// No request scope at startup, so itemStore.keys() would only see the default account.// scanAllAccountsRaw() returns every account's items already in the "accountId/// region::tableName" key format itemsByTable expects.if (itemStore instanceof AccountAwareStorageBackend<Map<String, JsonNode>> aware) {aware.scanAllAccountsRaw().forEach((rawKey, items) ->- itemsByTable.put(rawKey, new ConcurrentSkipListMap<>(items)));+ itemsByTable.put(rawKey, rekeyPersistedItems(persistedTables.get(rawKey), items)));return;}for (String key : itemStore.keys()) {- itemStore.get(key).ifPresent(items ->- itemsByTable.put(scopedItemsKey(key), new ConcurrentSkipListMap<>(items)));+ String scopedKey = scopedItemsKey(key);+ itemStore.get(key).ifPresent(items -> itemsByTable.put(+ scopedKey, rekeyPersistedItems(persistedTables.get(scopedKey), items)));}}+ private Map<String, TableDefinition> persistedTablesByScopedKey() {+ if (tableStore instanceof AccountAwareStorageBackend<TableDefinition> aware) {+ return aware.scanAllAccountsRaw();+ }+ Map<String, TableDefinition> persistedTables = new HashMap<>();+ for (String key : tableStore.keys()) {+ tableStore.get(key).ifPresent(table -> persistedTables.put(scopedItemsKey(key), table));+ }+ return persistedTables;+ }++ private ConcurrentSkipListMap<String, JsonNode> rekeyPersistedItems(+ TableDefinition table, Map<String, JsonNode> persistedItems) {+ if (table == null) {+ return new ConcurrentSkipListMap<>(persistedItems);+ }+ ConcurrentSkipListMap<String, JsonNode> rekeyedItems = new ConcurrentSkipListMap<>();+ for (JsonNode item : persistedItems.values()) {+ rekeyedItems.put(buildItemKeyFromNode(+ item, table.getPartitionKeyName(), table.getSortKeyName()), item);+ }+ return rekeyedItems;+ }+private void persistItems(String storageKey) {if (itemStore == null) return;var items = currentItems(storageKey, false);@@ -1382,9 +1408,9 @@ public class DynamoDbService implements ResourceProvider {// prevents deadlock across concurrent transactions; ReentrantLock lets the inner// putItem/updateItem/deleteItem calls re-enter the same lock for free.//- // Ordering uses a tuple comparator — not a delimited string — so user-supplied- // bytes in an item's PK/SK value cannot collide two distinct participants- // into the same ordering key.+ // Ordering compares storageKey and itemKey as separate tuple fields. The item key's+ // PK/SK segments are escaped before they are joined, so delimiter bytes inside a+ // user-supplied key cannot collapse distinct transaction participants.TreeMap<TransactParticipant, ReentrantLock> toAcquire = new TreeMap<>(PARTICIPANT_ORDER);Set<TransactParticipant> seenParticipants = new HashSet<>();for (JsonNode transactItem : transactItems) {@@ -3223,7 +3249,7 @@ public class DynamoDbService implements ResourceProvider {validateKeyAttributeValue(table, pkAttr, pkName, surface);validateKeySize(pkAttr, true);- String pk = extractScalarValue(pkAttr);+ String pk = encodeKeySegment(extractScalarValue(pkAttr));String skName = table.getSortKeyName();if (skName != null) {JsonNode skAttr = item.get(skName);@@ -3232,11 +3258,23 @@ public class DynamoDbService implements ResourceProvider {}validateKeyAttributeValue(table, skAttr, skName, surface);validateKeySize(skAttr, false);- return pk + "#" + extractScalarValue(skAttr);+ return pk + "#" + encodeKeySegment(extractScalarValue(skAttr));}return pk;}+ // '#' separates composite key segments in the in-memory map. Escape it and the escape+ // character inside each segment so legal string key values cannot produce the same map key.+ static String encodeKeySegment(String value) {+ if (value == null) {+ return "";+ }+ if (value.indexOf('#') < 0 && value.indexOf('\\') < 0) {+ return value;+ }+ return value.replace("\\", "\\\\").replace("#", "\\#");+ }+private void validateKeySize(JsonNode attr, boolean partitionKey) {int limit = partitionKey ? 2048 : 1024;if ((attr.has("S") || attr.has("B")) && DynamoDbItemSize.attributeValueSize(attr) > limit) {@@ -3384,12 +3422,11 @@ public class DynamoDbService implements ResourceProvider {private String buildItemKeyFromNode(JsonNode item, String pkName, List<String> skNames) {JsonNode pkAttr = item.get(pkName);if (pkAttr == null) return "";- String pk = extractScalarValue(pkAttr);- StringBuilder key = new StringBuilder(pk != null ? pk : "");+ StringBuilder key = new StringBuilder(encodeKeySegment(extractScalarValue(pkAttr)));for (String skName : skNames) {JsonNode skAttr = item.get(skName);if (skAttr != null) {- key.append("#").append(extractScalarValue(skAttr));+ key.append("#").append(encodeKeySegment(extractScalarValue(skAttr)));}}return key.toString();diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbCompositeKeyEncodingTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbCompositeKeyEncodingTest.javanew file mode 100644index 0000000000..3f2f4218c0--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbCompositeKeyEncodingTest.java@@ -0,0 +1,171 @@+package io.github.hectorvent.floci.services.dynamodb;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ObjectNode;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.core.storage.StorageBackend;+import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;+import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;+import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.util.HashSet;+import java.util.List;+import java.util.Map;+import java.util.Set;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNotNull;++class DynamoDbCompositeKeyEncodingTest {++ private static final String REGION = "us-east-1";++ private final ObjectMapper mapper = new ObjectMapper();+ private DynamoDbService service;++ @BeforeEach+ void setUp() {+ service = new DynamoDbService(new InMemoryStorage<>());+ createOrdersTable(service);+ }++ @Test+ void putAndGetKeepDelimiterBearingCompositeKeysDistinct() {+ ObjectNode first = order("A#", "B", "FIRST");+ ObjectNode second = order("A", "#B", "SECOND");++ service.putItem("Orders", first, REGION);+ service.putItem("Orders", second, REGION);++ assertEquals("FIRST", marker(service.getItem("Orders", key("A#", "B"), REGION)));+ assertEquals("SECOND", marker(service.getItem("Orders", key("A", "#B"), REGION)));+ assertEquals(2, service.scan("Orders", null, null, null, null, null, null, REGION).items().size());++ service.deleteItem("Orders", key("A#", "B"), REGION);+ assertEquals("SECOND", marker(service.getItem("Orders", key("A", "#B"), REGION)));+ }++ @Test+ void transactWriteAcceptsDelimiterBearingKeysAsDifferentItems() {+ service.transactWriteItems(List.of(+ transactPut(order("X#", "Y", "FIRST")),+ transactPut(order("X", "#Y", "SECOND"))), REGION);++ assertEquals("FIRST", marker(service.getItem("Orders", key("X#", "Y"), REGION)));+ assertEquals("SECOND", marker(service.getItem("Orders", key("X", "#Y"), REGION)));+ }++ @Test+ void batchWriteAcceptsDelimiterBearingKeysAsDifferentItems() {+ service.batchWriteItem(Map.of("Orders", List.of(+ batchPut(order("P#", "Q", "FIRST")),+ batchPut(order("P", "#Q", "SECOND")))), REGION);++ assertEquals("FIRST", marker(service.getItem("Orders", key("P#", "Q"), REGION)));+ assertEquals("SECOND", marker(service.getItem("Orders", key("P", "#Q"), REGION)));+ }++ @Test+ void backslashesRemainUnambiguousWhenKeySegmentsAreEscaped() {+ service.putItem("Orders", order("A\\", "#B", "FIRST"), REGION);+ service.putItem("Orders", order("A\\#", "B", "SECOND"), REGION);++ assertEquals("FIRST", marker(service.getItem("Orders", key("A\\", "#B"), REGION)));+ assertEquals("SECOND", marker(service.getItem("Orders", key("A\\#", "B"), REGION)));+ }++ @Test+ void scanPaginationContinuesAfterDelimiterBearingKey() {+ service.putItem("Orders", order("A#", "B", "FIRST"), REGION);+ service.putItem("Orders", order("A", "#B", "SECOND"), REGION);++ DynamoDbService.ScanResult firstPage =+ service.scan("Orders", null, null, null, null, 1, null, REGION);+ assertEquals(1, firstPage.items().size());+ assertNotNull(firstPage.lastEvaluatedKey());++ DynamoDbService.ScanResult secondPage = service.scan(+ "Orders", null, null, null, null, 1, firstPage.lastEvaluatedKey(), REGION);+ assertEquals(1, secondPage.items().size());++ Set<String> markers = new HashSet<>();+ markers.add(marker(firstPage.items().getFirst()));+ markers.add(marker(secondPage.items().getFirst()));+ assertEquals(Set.of("FIRST", "SECOND"), markers);+ }++ @Test+ void startupRekeysItemsPersistedWithLegacyDelimitedAddress() {+ String accountId = "222222222222";+ StorageBackend<String, TableDefinition> tableStore =+ AccountAwareStorageBackend.inMemory(accountId);+ StorageBackend<String, Map<String, JsonNode>> itemStore =+ AccountAwareStorageBackend.inMemory(accountId);+ RegionResolver resolver = new RegionResolver(REGION, accountId);+ DynamoDbService beforeRestart = new DynamoDbService(tableStore, itemStore, resolver);+ createOrdersTable(beforeRestart);++ ObjectNode persisted = order("A#", "B", "LEGACY");+ itemStore.put(REGION + "::Orders", Map.of("A##B", persisted));++ DynamoDbService restarted = new DynamoDbService(tableStore, itemStore, resolver);++ assertEquals("LEGACY", marker(restarted.getItem("Orders", key("A#", "B"), REGION)));+ }++ private void createOrdersTable(DynamoDbService target) {+ target.createTable("Orders",+ List.of(+ new KeySchemaElement("customerId", "HASH"),+ new KeySchemaElement("orderId", "RANGE")),+ List.of(+ new AttributeDefinition("customerId", "S"),+ new AttributeDefinition("orderId", "S")),+ 5L, 5L, REGION);+ }++ private ObjectNode order(String customerId, String orderId, String marker) {+ ObjectNode item = key(customerId, orderId);+ item.set("marker", attribute("S", marker));+ return item;+ }++ private ObjectNode key(String customerId, String orderId) {+ ObjectNode key = mapper.createObjectNode();+ key.set("customerId", attribute("S", customerId));+ key.set("orderId", attribute("S", orderId));+ return key;+ }++ private ObjectNode attribute(String type, String value) {+ ObjectNode attribute = mapper.createObjectNode();+ attribute.put(type, value);+ return attribute;+ }++ private ObjectNode transactPut(JsonNode item) {+ ObjectNode put = mapper.createObjectNode();+ put.put("TableName", "Orders");+ put.set("Item", item);+ ObjectNode request = mapper.createObjectNode();+ request.set("Put", put);+ return request;+ }++ private ObjectNode batchPut(JsonNode item) {+ ObjectNode put = mapper.createObjectNode();+ put.set("Item", item);+ ObjectNode request = mapper.createObjectNode();+ request.set("PutRequest", put);+ return request;+ }++ private String marker(JsonNode item) {+ return item.path("marker").path("S").asText();+ }+}