Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(dynamodb): fix GSI query pagination infinite loop when items share sort key (#637)

Hector Ventura77db7d24efa0
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
index 9a3c570897..c5ca5292b6 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
@@ -533,11 +533,19 @@ public class DynamoDbService {
// Apply ExclusiveStartKey offset
if (exclusiveStartKey != null) {
- TableDefinition finalTable = table;
- String startItemKey = buildItemKeyFromNode(exclusiveStartKey, pkName, skName);
+ String tablePkName = table.getPartitionKeyName();
+ String tableSkName = table.getSortKeyName();
+ boolean hasTableKeys = exclusiveStartKey.has(tablePkName);
+
+ String startItemKey = hasTableKeys
+ ? buildItemKeyFromNode(exclusiveStartKey, tablePkName, tableSkName)
+ : buildItemKeyFromNode(exclusiveStartKey, pkName, skName);
+
int startIdx = -1;
for (int i = 0; i < results.size(); i++) {
- String thisKey = buildItemKeyFromNode(results.get(i), pkName, skName);
+ String thisKey = hasTableKeys
+ ? buildItemKeyFromNode(results.get(i), tablePkName, tableSkName)
+ : buildItemKeyFromNode(results.get(i), pkName, skName);
if (thisKey.equals(startItemKey)) {
startIdx = i;
break;
@@ -553,7 +561,7 @@ public class DynamoDbService {
if (limit != null && limit > 0 && evaluatedItems.size() > limit) {
JsonNode lastItem = evaluatedItems.get(limit - 1);
- lastEvaluatedKey = buildKeyNode(table, lastItem, pkName, skName);
+ lastEvaluatedKey = buildKeyNode(table, lastItem, pkName, skName, indexName != null);
evaluatedItems = new ArrayList<>(evaluatedItems.subList(0, limit));
}
@@ -1775,6 +1783,11 @@ public class DynamoDbService {
}
JsonNode buildKeyNode(TableDefinition table, JsonNode item, String pkName, String skName) {
+ return buildKeyNode(table, item, pkName, skName, false);
+ }
+
+ JsonNode buildKeyNode(TableDefinition table, JsonNode item,
+ String pkName, String skName, boolean isIndexQuery) {
com.fasterxml.jackson.databind.node.ObjectNode keyNode =
com.fasterxml.jackson.databind.node.JsonNodeFactory.instance.objectNode();
JsonNode pkAttr = item.get(pkName);
@@ -1787,6 +1800,16 @@ public class DynamoDbService {
keyNode.set(skName, skAttr);
}
}
+ if (isIndexQuery) {
+ String tablePk = table.getPartitionKeyName();
+ String tableSk = table.getSortKeyName();
+ if (!tablePk.equals(pkName) && item.get(tablePk) != null) {
+ keyNode.set(tablePk, item.get(tablePk));
+ }
+ if (tableSk != null && !tableSk.equals(skName) && item.get(tableSk) != null) {
+ keyNode.set(tableSk, item.get(tableSk));
+ }
+ }
return keyNode;
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIntegrationTest.java
index 0441a837be..4b274a6e59 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbIntegrationTest.java
@@ -1,5 +1,7 @@
package io.github.hectorvent.floci.services.dynamodb;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
import io.quarkus.test.junit.QuarkusTest;
import io.restassured.response.Response;
@@ -9,6 +11,10 @@ import org.junit.jupiter.api.Order;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestMethodOrder;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
import java.util.zip.CRC32;
import static io.restassured.RestAssured.given;
@@ -1768,4 +1774,129 @@ given()
crc.update(bytes);
return crc.getValue();
}
+
+ @Test
+ @Order(33)
+ void gsiQueryPaginationWithSharedSortKey() throws Exception {
+ ObjectMapper mapper = new ObjectMapper();
+ String tableName = "gsi-pagination-test";
+
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.CreateTable")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("""
+ {
+ "TableName": "%s",
+ "KeySchema": [
+ {"AttributeName": "PK", "KeyType": "HASH"},
+ {"AttributeName": "SK", "KeyType": "RANGE"}
+ ],
+ "AttributeDefinitions": [
+ {"AttributeName": "PK", "AttributeType": "S"},
+ {"AttributeName": "SK", "AttributeType": "S"},
+ {"AttributeName": "GSI1PK", "AttributeType": "S"},
+ {"AttributeName": "GSI1SK", "AttributeType": "S"}
+ ],
+ "BillingMode": "PAY_PER_REQUEST",
+ "GlobalSecondaryIndexes": [{
+ "IndexName": "GSI1",
+ "KeySchema": [
+ {"AttributeName": "GSI1PK", "KeyType": "HASH"},
+ {"AttributeName": "GSI1SK", "KeyType": "RANGE"}
+ ],
+ "Projection": {"ProjectionType": "ALL"}
+ }]
+ }
+ """.formatted(tableName))
+ .when().post("/")
+ .then().statusCode(200);
+
+ // 5 items all sharing (GSI1PK="ITEM", GSI1SK="SAME"), unique base-table PK/SK
+ for (String id : new String[]{"ITEM_a", "ITEM_b", "ITEM_c", "ITEM_d", "ITEM_e"}) {
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.PutItem")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("""
+ {
+ "TableName": "%s",
+ "Item": {
+ "PK": {"S": "%s"},
+ "SK": {"S": "DETAIL"},
+ "GSI1PK": {"S": "ITEM"},
+ "GSI1SK": {"S": "SAME"}
+ }
+ }
+ """.formatted(tableName, id))
+ .when().post("/")
+ .then().statusCode(200);
+ }
+
+ List<String> allCollected = new ArrayList<>();
+ Set<String> seenLeks = new HashSet<>();
+ JsonNode exclusiveStartKey = null;
+ int pages = 0;
+
+ do {
+ String body;
+ if (exclusiveStartKey == null) {
+ body = """
+ {
+ "TableName": "%s",
+ "IndexName": "GSI1",
+ "KeyConditionExpression": "GSI1PK = :pk",
+ "ExpressionAttributeValues": {":pk": {"S": "ITEM"}},
+ "Limit": 2
+ }
+ """.formatted(tableName);
+ } else {
+ body = """
+ {
+ "TableName": "%s",
+ "IndexName": "GSI1",
+ "KeyConditionExpression": "GSI1PK = :pk",
+ "ExpressionAttributeValues": {":pk": {"S": "ITEM"}},
+ "Limit": 2,
+ "ExclusiveStartKey": %s
+ }
+ """.formatted(tableName, mapper.writeValueAsString(exclusiveStartKey));
+ }
+
+ String responseBody = given()
+ .header("X-Amz-Target", "DynamoDB_20120810.Query")
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body(body)
+ .when().post("/")
+ .then().statusCode(200).extract().body().asString();
+
+ JsonNode root = mapper.readTree(responseBody);
+ pages++;
+
+ for (JsonNode item : root.path("Items")) {
+ allCollected.add(item.path("PK").path("S").asText());
+ }
+
+ JsonNode lek = root.path("LastEvaluatedKey");
+ if (lek.isMissingNode() || lek.isNull()) {
+ exclusiveStartKey = null;
+ } else {
+ // LEK must contain all four keys
+ assertNotNull(lek.get("GSI1PK"), "LastEvaluatedKey missing GSI1PK");
+ assertNotNull(lek.get("GSI1SK"), "LastEvaluatedKey missing GSI1SK");
+ assertNotNull(lek.get("PK"), "LastEvaluatedKey missing PK");
+ assertNotNull(lek.get("SK"), "LastEvaluatedKey missing SK");
+
+ // LEK must be unique across pages — cursor must advance
+ String lekStr = lek.toString();
+ assertEquals(false, seenLeks.contains(lekStr),
+ "LastEvaluatedKey repeated — infinite pagination loop: " + lekStr);
+ seenLeks.add(lekStr);
+ exclusiveStartKey = lek;
+ }
+ } while (exclusiveStartKey != null && pages < 10);
+
+ // All 5 distinct items returned exactly once
+ assertEquals(5, allCollected.size(), "Expected 5 items total, got: " + allCollected);
+ assertEquals(Set.of("ITEM_a", "ITEM_b", "ITEM_c", "ITEM_d", "ITEM_e"), new HashSet<>(allCollected));
+ assertEquals(3, pages, "Expected ceil(5/2)=3 pages");
+ }
}