Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(dynamodb): partition parallel scan by partition key hash (#3225) (#3429)

Vũ Thế Huy9c9aa670e436
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java
index 9908704605..f4f2d50b7d 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java
@@ -1115,18 +1115,9 @@ public class DynamoDbJsonHandler {
DynamoDbService.ScanResult result = dynamoDbService.scan(
tableName, filterExpr, exprAttrNames, exprAttrValues, scanFilter, limit,
- exclusiveStartKey, indexNameScan, region);
+ exclusiveStartKey, indexNameScan, segment, totalSegments, region);
List<JsonNode> scanItems = result.items();
- // Apply parallel scan segment partitioning
- if (segment != null && totalSegments != null && totalSegments > 1) {
- final int seg = segment, total = totalSegments;
- final List<JsonNode> allItems = scanItems;
- scanItems = new ArrayList<>();
- for (int si = 0; si < allItems.size(); si++) {
- if (si % total == seg) scanItems.add(allItems.get(si));
- }
- }
// Apply index projection (KEYS_ONLY / INCLUDE) when scanning a secondary index
if (indexNameScan != null && projectionExpressionScan == null && attributesToGetScan == null) {
scanItems = applyIndexProjection(scanItems, scanTable, scanAccessPath, select);
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
index 7b74c00306..3b59d19ca2 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
@@ -1095,6 +1095,14 @@ public class DynamoDbService implements ResourceProvider {
JsonNode expressionAttrNames, JsonNode expressionAttrValues,
JsonNode scanFilter, Integer limit, JsonNode exclusiveStartKey,
String indexName, String region) {
+ return scan(tableName, filterExpression, expressionAttrNames, expressionAttrValues,
+ scanFilter, limit, exclusiveStartKey, indexName, null, null, region);
+ }
+
+ public ScanResult scan(String tableName, String filterExpression,
+ JsonNode expressionAttrNames, JsonNode expressionAttrValues,
+ JsonNode scanFilter, Integer limit, JsonNode exclusiveStartKey,
+ String indexName, Integer segment, Integer totalSegments, String region) {
DynamoDbReservedWords.check(filterExpression, "FilterExpression");
String canonicalTableName = canonicalTableName(region, tableName);
String storageKey = regionKey(region, canonicalTableName);
@@ -1140,6 +1148,12 @@ public class DynamoDbService implements ResourceProvider {
&& (lekSkName == null || hasNonNullAttribute(item, lekSkName)))) {
continue;
}
+ if (segment != null && totalSegments != null && totalSegments > 1) {
+ JsonNode pkAttr = item.get(lekPkName);
+ if (computeSegment(pkAttr, totalSegments) != segment) {
+ continue;
+ }
+ }
// Stop at whichever boundary the read reaches first: the 1 MB cap or
// Limit. The size check comes first — per the API reference, "if the
// processed dataset size exceeds 1 MB before DynamoDB reaches this
@@ -3269,6 +3283,27 @@ public class DynamoDbService implements ResourceProvider {
return keyNode;
}
+ int computeSegment(JsonNode pkAttr, int totalSegments) {
+ if (totalSegments <= 1 || pkAttr == null) {
+ return 0;
+ }
+ String scalar = extractScalarValue(pkAttr);
+ if (scalar == null) {
+ return 0;
+ }
+ try {
+ MessageDigest md = MessageDigest.getInstance("MD5");
+ byte[] digest = md.digest(scalar.getBytes(StandardCharsets.UTF_8));
+ long val = ((long) (digest[0] & 0xFF) << 24)
+ | ((long) (digest[1] & 0xFF) << 16)
+ | ((long) (digest[2] & 0xFF) << 8)
+ | ((long) (digest[3] & 0xFF));
+ return (int) (val % totalSegments);
+ } catch (NoSuchAlgorithmException e) {
+ return Math.floorMod(scalar.hashCode(), totalSegments);
+ }
+ }
+
private String extractScalarValue(JsonNode attrValue) {
if (attrValue == null) return null;
if (attrValue.has("S")) return attrValue.get("S").asText();
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandlerTest.java
index f578d9a5d7..6bbacf0ded 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandlerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandlerTest.java
@@ -18,8 +18,11 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.nio.file.Path;
+import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import static org.junit.jupiter.api.Assertions.*;
@@ -954,4 +957,152 @@ class DynamoDbJsonHandlerTest {
"""), "eu-west-1");
assertEquals(200, response.getStatus());
}
+
+ @Test
+ void parallelScanSegmentsAreStableWhenItemsDeleted() throws Exception {
+ String region = "eu-west-1";
+ createUsersTable(region);
+
+ for (int i = 0; i < 10; i++) {
+ service.putItem("Users", item("userId", "item-" + i), region);
+ }
+
+ int totalSegments = 10;
+ int totalDeleted = 0;
+ for (int segment = 0; segment < totalSegments; segment++) {
+ var scanReq = mapper.createObjectNode();
+ scanReq.put("TableName", "Users");
+ scanReq.put("Segment", segment);
+ scanReq.put("TotalSegments", totalSegments);
+
+ var response = handler.handle("Scan", scanReq, region);
+ assertEquals(200, response.getStatus());
+ var entity = mapper.readTree(response.getEntity().toString());
+ var items = entity.get("Items");
+ for (var item : items) {
+ totalDeleted++;
+ String key = item.get("userId").get("S").asText();
+ service.deleteItem("Users", item("userId", key), region);
+ }
+ }
+
+ assertEquals(10, totalDeleted, "All 10 items should have been returned and deleted across segments");
+
+ var checkResponse = handler.handle("Scan", json("""
+ {"TableName": "Users", "Select": "COUNT"}
+ """), region);
+ var checkEntity = mapper.readTree(checkResponse.getEntity().toString());
+ assertEquals(0, checkEntity.get("Count").asInt(), "No remaining items should be in the table");
+ }
+
+ @Test
+ void parallelScanPaginationWithLimit() throws Exception {
+ String region = "eu-west-1";
+ createUsersTable(region);
+
+ for (int i = 0; i < 20; i++) {
+ service.putItem("Users", item("userId", "user-" + i), region);
+ }
+
+ int totalSegments = 4;
+ Set<String> collectedKeys = new HashSet<>();
+ for (int segment = 0; segment < totalSegments; segment++) {
+ JsonNode exclusiveStartKey = null;
+ do {
+ var scanReq = mapper.createObjectNode();
+ scanReq.put("TableName", "Users");
+ scanReq.put("Segment", segment);
+ scanReq.put("TotalSegments", totalSegments);
+ scanReq.put("Limit", 2);
+ if (exclusiveStartKey != null) {
+ scanReq.set("ExclusiveStartKey", exclusiveStartKey);
+ }
+
+ var response = handler.handle("Scan", scanReq, region);
+ assertEquals(200, response.getStatus());
+ var entity = mapper.readTree(response.getEntity().toString());
+ var items = entity.get("Items");
+ assertTrue(items.size() <= 2, "Page size must not exceed limit");
+ for (var item : items) {
+ String key = item.get("userId").get("S").asText();
+ assertTrue(collectedKeys.add(key), "Item must not be returned multiple times: " + key);
+ }
+ exclusiveStartKey = entity.get("LastEvaluatedKey");
+ } while (exclusiveStartKey != null && !exclusiveStartKey.isNull());
+ }
+
+ assertEquals(20, collectedKeys.size(), "All 20 items must be collected across paginated segments");
+ }
+
+ @Test
+ void parallelScanSelectCount() throws Exception {
+ String region = "eu-west-1";
+ createUsersTable(region);
+
+ for (int i = 0; i < 15; i++) {
+ service.putItem("Users", item("userId", "id-" + i), region);
+ }
+
+ int totalSegments = 5;
+ int sumCount = 0;
+ int sumScannedCount = 0;
+ for (int segment = 0; segment < totalSegments; segment++) {
+ var scanReq = mapper.createObjectNode();
+ scanReq.put("TableName", "Users");
+ scanReq.put("Segment", segment);
+ scanReq.put("TotalSegments", totalSegments);
+ scanReq.put("Select", "COUNT");
+
+ var response = handler.handle("Scan", scanReq, region);
+ assertEquals(200, response.getStatus());
+ var entity = mapper.readTree(response.getEntity().toString());
+ sumCount += entity.get("Count").asInt();
+ sumScannedCount += entity.get("ScannedCount").asInt();
+ }
+
+ assertEquals(15, sumCount, "Sum of Count across segments must equal total items");
+ assertEquals(15, sumScannedCount, "Sum of ScannedCount across segments must equal total items");
+ }
+
+ @Test
+ void parallelScanSamePartitionKeySameSegment() throws Exception {
+ String region = "eu-west-1";
+ service.createTable("Orders",
+ List.of(
+ new KeySchemaElement("userId", "HASH"),
+ new KeySchemaElement("orderId", "RANGE")),
+ List.of(
+ new AttributeDefinition("userId", "S"),
+ new AttributeDefinition("orderId", "S")),
+ 5L, 5L, region);
+
+ for (int i = 0; i < 5; i++) {
+ service.putItem("Orders", item("userId", "alice", "orderId", "ord-" + i), region);
+ service.putItem("Orders", item("userId", "bob", "orderId", "ord-" + i), region);
+ }
+
+ int totalSegments = 10;
+ Map<String, Set<Integer>> userSegments = new HashMap<>();
+ userSegments.put("alice", new HashSet<>());
+ userSegments.put("bob", new HashSet<>());
+
+ for (int segment = 0; segment < totalSegments; segment++) {
+ var scanReq = mapper.createObjectNode();
+ scanReq.put("TableName", "Orders");
+ scanReq.put("Segment", segment);
+ scanReq.put("TotalSegments", totalSegments);
+
+ var response = handler.handle("Scan", scanReq, region);
+ assertEquals(200, response.getStatus());
+ var entity = mapper.readTree(response.getEntity().toString());
+ var items = entity.get("Items");
+ for (var item : items) {
+ String userId = item.get("userId").get("S").asText();
+ userSegments.get(userId).add(segment);
+ }
+ }
+
+ assertEquals(1, userSegments.get("alice").size(), "All alice items must belong to the exact same segment");
+ assertEquals(1, userSegments.get("bob").size(), "All bob items must belong to the exact same segment");
+ }
}