Light, fluffy, and always free - The AWS Local Emulator alternative
fix(dynamodb): fix GSI query pagination infinite loop when items share sort key (#637)
Hector Ventura77db7d24efa0
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.javaindex 9a3c570897..c5ca5292b6 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java@@ -533,11 +533,19 @@ public class DynamoDbService {// Apply ExclusiveStartKey offsetif (exclusiveStartKey != null) {- TableDefinition finalTable = table;- String startItemKey = buildItemKeyFromNode(exclusiveStartKey, pkName, skName);+ String tablePkName = table.getPartitionKeyName();+ String tableSkName = table.getSortKeyName();+ boolean hasTableKeys = exclusiveStartKey.has(tablePkName);++ String startItemKey = hasTableKeys+ ? buildItemKeyFromNode(exclusiveStartKey, tablePkName, tableSkName)+ : buildItemKeyFromNode(exclusiveStartKey, pkName, skName);+int startIdx = -1;for (int i = 0; i < results.size(); i++) {- String thisKey = buildItemKeyFromNode(results.get(i), pkName, skName);+ String thisKey = hasTableKeys+ ? buildItemKeyFromNode(results.get(i), tablePkName, tableSkName)+ : buildItemKeyFromNode(results.get(i), pkName, skName);if (thisKey.equals(startItemKey)) {startIdx = i;break;@@ -553,7 +561,7 @@ public class DynamoDbService {if (limit != null && limit > 0 && evaluatedItems.size() > limit) {JsonNode lastItem = evaluatedItems.get(limit - 1);- lastEvaluatedKey = buildKeyNode(table, lastItem, pkName, skName);+ lastEvaluatedKey = buildKeyNode(table, lastItem, pkName, skName, indexName != null);evaluatedItems = new ArrayList<>(evaluatedItems.subList(0, limit));}@@ -1775,6 +1783,11 @@ public class DynamoDbService {}JsonNode buildKeyNode(TableDefinition table, JsonNode item, String pkName, String skName) {+ return buildKeyNode(table, item, pkName, skName, false);+ }++ JsonNode buildKeyNode(TableDefinition table, JsonNode item,+ String pkName, String skName, boolean isIndexQuery) {com.fasterxml.jackson.databind.node.ObjectNode keyNode =com.fasterxml.jackson.databind.node.JsonNodeFactory.instance.objectNode();JsonNode pkAttr = item.get(pkName);@@ -1787,6 +1800,16 @@ public class DynamoDbService {keyNode.set(skName, skAttr);}}+ if (isIndexQuery) {+ String tablePk = table.getPartitionKeyName();+ String tableSk = table.getSortKeyName();+ if (!tablePk.equals(pkName) && item.get(tablePk) != null) {+ keyNode.set(tablePk, item.get(tablePk));+ }+ if (tableSk != null && !tableSk.equals(skName) && item.get(tableSk) != null) {+ keyNode.set(tableSk, item.get(tableSk));+ }+ }return keyNode;}diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIntegrationTest.javaindex 0441a837be..4b274a6e59 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIntegrationTest.java@@ -1,5 +1,7 @@package io.github.hectorvent.floci.services.dynamodb;+import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;import io.quarkus.test.junit.QuarkusTest;import io.restassured.response.Response;@@ -9,6 +11,10 @@ import org.junit.jupiter.api.Order;import org.junit.jupiter.api.Test;import org.junit.jupiter.api.TestMethodOrder;+import java.util.ArrayList;+import java.util.HashSet;+import java.util.List;+import java.util.Set;import java.util.zip.CRC32;import static io.restassured.RestAssured.given;@@ -1768,4 +1774,129 @@ given()crc.update(bytes);return crc.getValue();}++ @Test+ @Order(33)+ void gsiQueryPaginationWithSharedSortKey() throws Exception {+ ObjectMapper mapper = new ObjectMapper();+ String tableName = "gsi-pagination-test";++ given()+ .header("X-Amz-Target", "DynamoDB_20120810.CreateTable")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("""+ {+ "TableName": "%s",+ "KeySchema": [+ {"AttributeName": "PK", "KeyType": "HASH"},+ {"AttributeName": "SK", "KeyType": "RANGE"}+ ],+ "AttributeDefinitions": [+ {"AttributeName": "PK", "AttributeType": "S"},+ {"AttributeName": "SK", "AttributeType": "S"},+ {"AttributeName": "GSI1PK", "AttributeType": "S"},+ {"AttributeName": "GSI1SK", "AttributeType": "S"}+ ],+ "BillingMode": "PAY_PER_REQUEST",+ "GlobalSecondaryIndexes": [{+ "IndexName": "GSI1",+ "KeySchema": [+ {"AttributeName": "GSI1PK", "KeyType": "HASH"},+ {"AttributeName": "GSI1SK", "KeyType": "RANGE"}+ ],+ "Projection": {"ProjectionType": "ALL"}+ }]+ }+ """.formatted(tableName))+ .when().post("/")+ .then().statusCode(200);++ // 5 items all sharing (GSI1PK="ITEM", GSI1SK="SAME"), unique base-table PK/SK+ for (String id : new String[]{"ITEM_a", "ITEM_b", "ITEM_c", "ITEM_d", "ITEM_e"}) {+ given()+ .header("X-Amz-Target", "DynamoDB_20120810.PutItem")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body("""+ {+ "TableName": "%s",+ "Item": {+ "PK": {"S": "%s"},+ "SK": {"S": "DETAIL"},+ "GSI1PK": {"S": "ITEM"},+ "GSI1SK": {"S": "SAME"}+ }+ }+ """.formatted(tableName, id))+ .when().post("/")+ .then().statusCode(200);+ }++ List<String> allCollected = new ArrayList<>();+ Set<String> seenLeks = new HashSet<>();+ JsonNode exclusiveStartKey = null;+ int pages = 0;++ do {+ String body;+ if (exclusiveStartKey == null) {+ body = """+ {+ "TableName": "%s",+ "IndexName": "GSI1",+ "KeyConditionExpression": "GSI1PK = :pk",+ "ExpressionAttributeValues": {":pk": {"S": "ITEM"}},+ "Limit": 2+ }+ """.formatted(tableName);+ } else {+ body = """+ {+ "TableName": "%s",+ "IndexName": "GSI1",+ "KeyConditionExpression": "GSI1PK = :pk",+ "ExpressionAttributeValues": {":pk": {"S": "ITEM"}},+ "Limit": 2,+ "ExclusiveStartKey": %s+ }+ """.formatted(tableName, mapper.writeValueAsString(exclusiveStartKey));+ }++ String responseBody = given()+ .header("X-Amz-Target", "DynamoDB_20120810.Query")+ .contentType(DYNAMODB_CONTENT_TYPE)+ .body(body)+ .when().post("/")+ .then().statusCode(200).extract().body().asString();++ JsonNode root = mapper.readTree(responseBody);+ pages++;++ for (JsonNode item : root.path("Items")) {+ allCollected.add(item.path("PK").path("S").asText());+ }++ JsonNode lek = root.path("LastEvaluatedKey");+ if (lek.isMissingNode() || lek.isNull()) {+ exclusiveStartKey = null;+ } else {+ // LEK must contain all four keys+ assertNotNull(lek.get("GSI1PK"), "LastEvaluatedKey missing GSI1PK");+ assertNotNull(lek.get("GSI1SK"), "LastEvaluatedKey missing GSI1SK");+ assertNotNull(lek.get("PK"), "LastEvaluatedKey missing PK");+ assertNotNull(lek.get("SK"), "LastEvaluatedKey missing SK");++ // LEK must be unique across pages — cursor must advance+ String lekStr = lek.toString();+ assertEquals(false, seenLeks.contains(lekStr),+ "LastEvaluatedKey repeated — infinite pagination loop: " + lekStr);+ seenLeks.add(lekStr);+ exclusiveStartKey = lek;+ }+ } while (exclusiveStartKey != null && pages < 10);++ // All 5 distinct items returned exactly once+ assertEquals(5, allCollected.size(), "Expected 5 items total, got: " + allCollected);+ assertEquals(Set.of("ITEM_a", "ITEM_b", "ITEM_c", "ITEM_d", "ITEM_e"), new HashSet<>(allCollected));+ assertEquals(3, pages, "Expected ceil(5/2)=3 pages");+ }}