Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(dynamodb): escape composite key segments (#3804)

XuJian5340e88605f3
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
index a11f789a80..539a017f15 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
@@ -183,20 +183,46 @@ public class DynamoDbService implements ResourceProvider {
private void loadPersistedItems() {
if (itemStore == null) return;
+ Map<String, TableDefinition> persistedTables = persistedTablesByScopedKey();
// No request scope at startup, so itemStore.keys() would only see the default account.
// scanAllAccountsRaw() returns every account's items already in the "accountId/
// region::tableName" key format itemsByTable expects.
if (itemStore instanceof AccountAwareStorageBackend<Map<String, JsonNode>> aware) {
aware.scanAllAccountsRaw().forEach((rawKey, items) ->
- itemsByTable.put(rawKey, new ConcurrentSkipListMap<>(items)));
+ itemsByTable.put(rawKey, rekeyPersistedItems(persistedTables.get(rawKey), items)));
return;
}
for (String key : itemStore.keys()) {
- itemStore.get(key).ifPresent(items ->
- itemsByTable.put(scopedItemsKey(key), new ConcurrentSkipListMap<>(items)));
+ String scopedKey = scopedItemsKey(key);
+ itemStore.get(key).ifPresent(items -> itemsByTable.put(
+ scopedKey, rekeyPersistedItems(persistedTables.get(scopedKey), items)));
}
}
+ private Map<String, TableDefinition> persistedTablesByScopedKey() {
+ if (tableStore instanceof AccountAwareStorageBackend<TableDefinition> aware) {
+ return aware.scanAllAccountsRaw();
+ }
+ Map<String, TableDefinition> persistedTables = new HashMap<>();
+ for (String key : tableStore.keys()) {
+ tableStore.get(key).ifPresent(table -> persistedTables.put(scopedItemsKey(key), table));
+ }
+ return persistedTables;
+ }
+
+ private ConcurrentSkipListMap<String, JsonNode> rekeyPersistedItems(
+ TableDefinition table, Map<String, JsonNode> persistedItems) {
+ if (table == null) {
+ return new ConcurrentSkipListMap<>(persistedItems);
+ }
+ ConcurrentSkipListMap<String, JsonNode> rekeyedItems = new ConcurrentSkipListMap<>();
+ for (JsonNode item : persistedItems.values()) {
+ rekeyedItems.put(buildItemKeyFromNode(
+ item, table.getPartitionKeyName(), table.getSortKeyName()), item);
+ }
+ return rekeyedItems;
+ }
+
private void persistItems(String storageKey) {
if (itemStore == null) return;
var items = currentItems(storageKey, false);
@@ -1382,9 +1408,9 @@ public class DynamoDbService implements ResourceProvider {
// prevents deadlock across concurrent transactions; ReentrantLock lets the inner
// putItem/updateItem/deleteItem calls re-enter the same lock for free.
//
- // Ordering uses a tuple comparator — not a delimited string — so user-supplied
- // bytes in an item's PK/SK value cannot collide two distinct participants
- // into the same ordering key.
+ // Ordering compares storageKey and itemKey as separate tuple fields. The item key's
+ // PK/SK segments are escaped before they are joined, so delimiter bytes inside a
+ // user-supplied key cannot collapse distinct transaction participants.
TreeMap<TransactParticipant, ReentrantLock> toAcquire = new TreeMap<>(PARTICIPANT_ORDER);
Set<TransactParticipant> seenParticipants = new HashSet<>();
for (JsonNode transactItem : transactItems) {
@@ -3223,7 +3249,7 @@ public class DynamoDbService implements ResourceProvider {
validateKeyAttributeValue(table, pkAttr, pkName, surface);
validateKeySize(pkAttr, true);
- String pk = extractScalarValue(pkAttr);
+ String pk = encodeKeySegment(extractScalarValue(pkAttr));
String skName = table.getSortKeyName();
if (skName != null) {
JsonNode skAttr = item.get(skName);
@@ -3232,11 +3258,23 @@ public class DynamoDbService implements ResourceProvider {
}
validateKeyAttributeValue(table, skAttr, skName, surface);
validateKeySize(skAttr, false);
- return pk + "#" + extractScalarValue(skAttr);
+ return pk + "#" + encodeKeySegment(extractScalarValue(skAttr));
}
return pk;
}
+ // '#' separates composite key segments in the in-memory map. Escape it and the escape
+ // character inside each segment so legal string key values cannot produce the same map key.
+ static String encodeKeySegment(String value) {
+ if (value == null) {
+ return "";
+ }
+ if (value.indexOf('#') < 0 && value.indexOf('\\') < 0) {
+ return value;
+ }
+ return value.replace("\\", "\\\\").replace("#", "\\#");
+ }
+
private void validateKeySize(JsonNode attr, boolean partitionKey) {
int limit = partitionKey ? 2048 : 1024;
if ((attr.has("S") || attr.has("B")) && DynamoDbItemSize.attributeValueSize(attr) > limit) {
@@ -3384,12 +3422,11 @@ public class DynamoDbService implements ResourceProvider {
private String buildItemKeyFromNode(JsonNode item, String pkName, List<String> skNames) {
JsonNode pkAttr = item.get(pkName);
if (pkAttr == null) return "";
- String pk = extractScalarValue(pkAttr);
- StringBuilder key = new StringBuilder(pk != null ? pk : "");
+ StringBuilder key = new StringBuilder(encodeKeySegment(extractScalarValue(pkAttr)));
for (String skName : skNames) {
JsonNode skAttr = item.get(skName);
if (skAttr != null) {
- key.append("#").append(extractScalarValue(skAttr));
+ key.append("#").append(encodeKeySegment(extractScalarValue(skAttr)));
}
}
return key.toString();
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbCompositeKeyEncodingTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbCompositeKeyEncodingTest.java
new file mode 100644
index 0000000000..3f2f4218c0
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbCompositeKeyEncodingTest.java
@@ -0,0 +1,171 @@
+package io.github.hectorvent.floci.services.dynamodb;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.core.storage.StorageBackend;
+import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;
+import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;
+import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+
+class DynamoDbCompositeKeyEncodingTest {
+
+ private static final String REGION = "us-east-1";
+
+ private final ObjectMapper mapper = new ObjectMapper();
+ private DynamoDbService service;
+
+ @BeforeEach
+ void setUp() {
+ service = new DynamoDbService(new InMemoryStorage<>());
+ createOrdersTable(service);
+ }
+
+ @Test
+ void putAndGetKeepDelimiterBearingCompositeKeysDistinct() {
+ ObjectNode first = order("A#", "B", "FIRST");
+ ObjectNode second = order("A", "#B", "SECOND");
+
+ service.putItem("Orders", first, REGION);
+ service.putItem("Orders", second, REGION);
+
+ assertEquals("FIRST", marker(service.getItem("Orders", key("A#", "B"), REGION)));
+ assertEquals("SECOND", marker(service.getItem("Orders", key("A", "#B"), REGION)));
+ assertEquals(2, service.scan("Orders", null, null, null, null, null, null, REGION).items().size());
+
+ service.deleteItem("Orders", key("A#", "B"), REGION);
+ assertEquals("SECOND", marker(service.getItem("Orders", key("A", "#B"), REGION)));
+ }
+
+ @Test
+ void transactWriteAcceptsDelimiterBearingKeysAsDifferentItems() {
+ service.transactWriteItems(List.of(
+ transactPut(order("X#", "Y", "FIRST")),
+ transactPut(order("X", "#Y", "SECOND"))), REGION);
+
+ assertEquals("FIRST", marker(service.getItem("Orders", key("X#", "Y"), REGION)));
+ assertEquals("SECOND", marker(service.getItem("Orders", key("X", "#Y"), REGION)));
+ }
+
+ @Test
+ void batchWriteAcceptsDelimiterBearingKeysAsDifferentItems() {
+ service.batchWriteItem(Map.of("Orders", List.of(
+ batchPut(order("P#", "Q", "FIRST")),
+ batchPut(order("P", "#Q", "SECOND")))), REGION);
+
+ assertEquals("FIRST", marker(service.getItem("Orders", key("P#", "Q"), REGION)));
+ assertEquals("SECOND", marker(service.getItem("Orders", key("P", "#Q"), REGION)));
+ }
+
+ @Test
+ void backslashesRemainUnambiguousWhenKeySegmentsAreEscaped() {
+ service.putItem("Orders", order("A\\", "#B", "FIRST"), REGION);
+ service.putItem("Orders", order("A\\#", "B", "SECOND"), REGION);
+
+ assertEquals("FIRST", marker(service.getItem("Orders", key("A\\", "#B"), REGION)));
+ assertEquals("SECOND", marker(service.getItem("Orders", key("A\\#", "B"), REGION)));
+ }
+
+ @Test
+ void scanPaginationContinuesAfterDelimiterBearingKey() {
+ service.putItem("Orders", order("A#", "B", "FIRST"), REGION);
+ service.putItem("Orders", order("A", "#B", "SECOND"), REGION);
+
+ DynamoDbService.ScanResult firstPage =
+ service.scan("Orders", null, null, null, null, 1, null, REGION);
+ assertEquals(1, firstPage.items().size());
+ assertNotNull(firstPage.lastEvaluatedKey());
+
+ DynamoDbService.ScanResult secondPage = service.scan(
+ "Orders", null, null, null, null, 1, firstPage.lastEvaluatedKey(), REGION);
+ assertEquals(1, secondPage.items().size());
+
+ Set<String> markers = new HashSet<>();
+ markers.add(marker(firstPage.items().getFirst()));
+ markers.add(marker(secondPage.items().getFirst()));
+ assertEquals(Set.of("FIRST", "SECOND"), markers);
+ }
+
+ @Test
+ void startupRekeysItemsPersistedWithLegacyDelimitedAddress() {
+ String accountId = "222222222222";
+ StorageBackend<String, TableDefinition> tableStore =
+ AccountAwareStorageBackend.inMemory(accountId);
+ StorageBackend<String, Map<String, JsonNode>> itemStore =
+ AccountAwareStorageBackend.inMemory(accountId);
+ RegionResolver resolver = new RegionResolver(REGION, accountId);
+ DynamoDbService beforeRestart = new DynamoDbService(tableStore, itemStore, resolver);
+ createOrdersTable(beforeRestart);
+
+ ObjectNode persisted = order("A#", "B", "LEGACY");
+ itemStore.put(REGION + "::Orders", Map.of("A##B", persisted));
+
+ DynamoDbService restarted = new DynamoDbService(tableStore, itemStore, resolver);
+
+ assertEquals("LEGACY", marker(restarted.getItem("Orders", key("A#", "B"), REGION)));
+ }
+
+ private void createOrdersTable(DynamoDbService target) {
+ target.createTable("Orders",
+ List.of(
+ new KeySchemaElement("customerId", "HASH"),
+ new KeySchemaElement("orderId", "RANGE")),
+ List.of(
+ new AttributeDefinition("customerId", "S"),
+ new AttributeDefinition("orderId", "S")),
+ 5L, 5L, REGION);
+ }
+
+ private ObjectNode order(String customerId, String orderId, String marker) {
+ ObjectNode item = key(customerId, orderId);
+ item.set("marker", attribute("S", marker));
+ return item;
+ }
+
+ private ObjectNode key(String customerId, String orderId) {
+ ObjectNode key = mapper.createObjectNode();
+ key.set("customerId", attribute("S", customerId));
+ key.set("orderId", attribute("S", orderId));
+ return key;
+ }
+
+ private ObjectNode attribute(String type, String value) {
+ ObjectNode attribute = mapper.createObjectNode();
+ attribute.put(type, value);
+ return attribute;
+ }
+
+ private ObjectNode transactPut(JsonNode item) {
+ ObjectNode put = mapper.createObjectNode();
+ put.put("TableName", "Orders");
+ put.set("Item", item);
+ ObjectNode request = mapper.createObjectNode();
+ request.set("Put", put);
+ return request;
+ }
+
+ private ObjectNode batchPut(JsonNode item) {
+ ObjectNode put = mapper.createObjectNode();
+ put.set("Item", item);
+ ObjectNode request = mapper.createObjectNode();
+ request.set("PutRequest", put);
+ return request;
+ }
+
+ private String marker(JsonNode item) {
+ return item.path("marker").path("S").asText();
+ }
+}