Light, fluffy, and always free - The AWS Local Emulator alternative
fix(dynamodb): partition parallel scan by partition key hash (#3225) (#3429)
Vũ Thế Huy9c9aa670e436
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.javaindex 9908704605..f4f2d50b7d 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandler.java@@ -1115,18 +1115,9 @@ public class DynamoDbJsonHandler {DynamoDbService.ScanResult result = dynamoDbService.scan(tableName, filterExpr, exprAttrNames, exprAttrValues, scanFilter, limit,- exclusiveStartKey, indexNameScan, region);+ exclusiveStartKey, indexNameScan, segment, totalSegments, region);List<JsonNode> scanItems = result.items();- // Apply parallel scan segment partitioning- if (segment != null && totalSegments != null && totalSegments > 1) {- final int seg = segment, total = totalSegments;- final List<JsonNode> allItems = scanItems;- scanItems = new ArrayList<>();- for (int si = 0; si < allItems.size(); si++) {- if (si % total == seg) scanItems.add(allItems.get(si));- }- }// Apply index projection (KEYS_ONLY / INCLUDE) when scanning a secondary indexif (indexNameScan != null && projectionExpressionScan == null && attributesToGetScan == null) {scanItems = applyIndexProjection(scanItems, scanTable, scanAccessPath, select);diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.javaindex 7b74c00306..3b59d19ca2 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java@@ -1095,6 +1095,14 @@ public class DynamoDbService implements ResourceProvider {JsonNode expressionAttrNames, JsonNode expressionAttrValues,JsonNode scanFilter, Integer limit, JsonNode exclusiveStartKey,String indexName, String region) {+ return scan(tableName, filterExpression, expressionAttrNames, expressionAttrValues,+ scanFilter, limit, exclusiveStartKey, indexName, null, null, region);+ }++ public ScanResult scan(String tableName, String filterExpression,+ JsonNode expressionAttrNames, JsonNode expressionAttrValues,+ JsonNode scanFilter, Integer limit, JsonNode exclusiveStartKey,+ String indexName, Integer segment, Integer totalSegments, String region) {DynamoDbReservedWords.check(filterExpression, "FilterExpression");String canonicalTableName = canonicalTableName(region, tableName);String storageKey = regionKey(region, canonicalTableName);@@ -1140,6 +1148,12 @@ public class DynamoDbService implements ResourceProvider {&& (lekSkName == null || hasNonNullAttribute(item, lekSkName)))) {continue;}+ if (segment != null && totalSegments != null && totalSegments > 1) {+ JsonNode pkAttr = item.get(lekPkName);+ if (computeSegment(pkAttr, totalSegments) != segment) {+ continue;+ }+ }// Stop at whichever boundary the read reaches first: the 1 MB cap or// Limit. The size check comes first — per the API reference, "if the// processed dataset size exceeds 1 MB before DynamoDB reaches this@@ -3269,6 +3283,27 @@ public class DynamoDbService implements ResourceProvider {return keyNode;}+ int computeSegment(JsonNode pkAttr, int totalSegments) {+ if (totalSegments <= 1 || pkAttr == null) {+ return 0;+ }+ String scalar = extractScalarValue(pkAttr);+ if (scalar == null) {+ return 0;+ }+ try {+ MessageDigest md = MessageDigest.getInstance("MD5");+ byte[] digest = md.digest(scalar.getBytes(StandardCharsets.UTF_8));+ long val = ((long) (digest[0] & 0xFF) << 24)+ | ((long) (digest[1] & 0xFF) << 16)+ | ((long) (digest[2] & 0xFF) << 8)+ | ((long) (digest[3] & 0xFF));+ return (int) (val % totalSegments);+ } catch (NoSuchAlgorithmException e) {+ return Math.floorMod(scalar.hashCode(), totalSegments);+ }+ }+private String extractScalarValue(JsonNode attrValue) {if (attrValue == null) return null;if (attrValue.has("S")) return attrValue.get("S").asText();diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandlerTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandlerTest.javaindex f578d9a5d7..6bbacf0ded 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandlerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbJsonHandlerTest.java@@ -18,8 +18,11 @@ import org.junit.jupiter.api.Test;import org.junit.jupiter.api.io.TempDir;import java.nio.file.Path;+import java.util.HashMap;+import java.util.HashSet;import java.util.List;import java.util.Map;+import java.util.Set;import static org.junit.jupiter.api.Assertions.*;@@ -954,4 +957,152 @@ class DynamoDbJsonHandlerTest {"""), "eu-west-1");assertEquals(200, response.getStatus());}++ @Test+ void parallelScanSegmentsAreStableWhenItemsDeleted() throws Exception {+ String region = "eu-west-1";+ createUsersTable(region);++ for (int i = 0; i < 10; i++) {+ service.putItem("Users", item("userId", "item-" + i), region);+ }++ int totalSegments = 10;+ int totalDeleted = 0;+ for (int segment = 0; segment < totalSegments; segment++) {+ var scanReq = mapper.createObjectNode();+ scanReq.put("TableName", "Users");+ scanReq.put("Segment", segment);+ scanReq.put("TotalSegments", totalSegments);++ var response = handler.handle("Scan", scanReq, region);+ assertEquals(200, response.getStatus());+ var entity = mapper.readTree(response.getEntity().toString());+ var items = entity.get("Items");+ for (var item : items) {+ totalDeleted++;+ String key = item.get("userId").get("S").asText();+ service.deleteItem("Users", item("userId", key), region);+ }+ }++ assertEquals(10, totalDeleted, "All 10 items should have been returned and deleted across segments");++ var checkResponse = handler.handle("Scan", json("""+ {"TableName": "Users", "Select": "COUNT"}+ """), region);+ var checkEntity = mapper.readTree(checkResponse.getEntity().toString());+ assertEquals(0, checkEntity.get("Count").asInt(), "No remaining items should be in the table");+ }++ @Test+ void parallelScanPaginationWithLimit() throws Exception {+ String region = "eu-west-1";+ createUsersTable(region);++ for (int i = 0; i < 20; i++) {+ service.putItem("Users", item("userId", "user-" + i), region);+ }++ int totalSegments = 4;+ Set<String> collectedKeys = new HashSet<>();+ for (int segment = 0; segment < totalSegments; segment++) {+ JsonNode exclusiveStartKey = null;+ do {+ var scanReq = mapper.createObjectNode();+ scanReq.put("TableName", "Users");+ scanReq.put("Segment", segment);+ scanReq.put("TotalSegments", totalSegments);+ scanReq.put("Limit", 2);+ if (exclusiveStartKey != null) {+ scanReq.set("ExclusiveStartKey", exclusiveStartKey);+ }++ var response = handler.handle("Scan", scanReq, region);+ assertEquals(200, response.getStatus());+ var entity = mapper.readTree(response.getEntity().toString());+ var items = entity.get("Items");+ assertTrue(items.size() <= 2, "Page size must not exceed limit");+ for (var item : items) {+ String key = item.get("userId").get("S").asText();+ assertTrue(collectedKeys.add(key), "Item must not be returned multiple times: " + key);+ }+ exclusiveStartKey = entity.get("LastEvaluatedKey");+ } while (exclusiveStartKey != null && !exclusiveStartKey.isNull());+ }++ assertEquals(20, collectedKeys.size(), "All 20 items must be collected across paginated segments");+ }++ @Test+ void parallelScanSelectCount() throws Exception {+ String region = "eu-west-1";+ createUsersTable(region);++ for (int i = 0; i < 15; i++) {+ service.putItem("Users", item("userId", "id-" + i), region);+ }++ int totalSegments = 5;+ int sumCount = 0;+ int sumScannedCount = 0;+ for (int segment = 0; segment < totalSegments; segment++) {+ var scanReq = mapper.createObjectNode();+ scanReq.put("TableName", "Users");+ scanReq.put("Segment", segment);+ scanReq.put("TotalSegments", totalSegments);+ scanReq.put("Select", "COUNT");++ var response = handler.handle("Scan", scanReq, region);+ assertEquals(200, response.getStatus());+ var entity = mapper.readTree(response.getEntity().toString());+ sumCount += entity.get("Count").asInt();+ sumScannedCount += entity.get("ScannedCount").asInt();+ }++ assertEquals(15, sumCount, "Sum of Count across segments must equal total items");+ assertEquals(15, sumScannedCount, "Sum of ScannedCount across segments must equal total items");+ }++ @Test+ void parallelScanSamePartitionKeySameSegment() throws Exception {+ String region = "eu-west-1";+ service.createTable("Orders",+ List.of(+ new KeySchemaElement("userId", "HASH"),+ new KeySchemaElement("orderId", "RANGE")),+ List.of(+ new AttributeDefinition("userId", "S"),+ new AttributeDefinition("orderId", "S")),+ 5L, 5L, region);++ for (int i = 0; i < 5; i++) {+ service.putItem("Orders", item("userId", "alice", "orderId", "ord-" + i), region);+ service.putItem("Orders", item("userId", "bob", "orderId", "ord-" + i), region);+ }++ int totalSegments = 10;+ Map<String, Set<Integer>> userSegments = new HashMap<>();+ userSegments.put("alice", new HashSet<>());+ userSegments.put("bob", new HashSet<>());++ for (int segment = 0; segment < totalSegments; segment++) {+ var scanReq = mapper.createObjectNode();+ scanReq.put("TableName", "Orders");+ scanReq.put("Segment", segment);+ scanReq.put("TotalSegments", totalSegments);++ var response = handler.handle("Scan", scanReq, region);+ assertEquals(200, response.getStatus());+ var entity = mapper.readTree(response.getEntity().toString());+ var items = entity.get("Items");+ for (var item : items) {+ String userId = item.get("userId").get("S").asText();+ userSegments.get(userId).add(segment);+ }+ }++ assertEquals(1, userSegments.get("alice").size(), "All alice items must belong to the exact same segment");+ assertEquals(1, userSegments.get("bob").size(), "All bob items must belong to the exact same segment");+ }}