Light, fluffy, and always free - The AWS Local Emulator alternative
fix(kms): page ListKeys and ListAliases with Limit and Marker (#6269)
Allen Sanbornaf3dc4866e48
diff --git a/src/main/java/io/github/hectorvent/floci/services/kms/KmsJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/kms/KmsJsonHandler.javaindex dec10a13da..8c2bd5a0a9 100644--- a/src/main/java/io/github/hectorvent/floci/services/kms/KmsJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/kms/KmsJsonHandler.java@@ -174,18 +174,63 @@ public class KmsJsonHandler {}private Response handleListKeys(JsonNode request, String region) {- List<KmsKey> keys = service.listKeys(region);+ KmsService.Page<KmsKey> page = service.listKeys(region, marker(request), limit(request));ObjectNode response = objectMapper.createObjectNode();ArrayNode array = response.putArray("Keys");- for (KmsKey k : keys) {+ for (KmsKey k : page.items()) {ObjectNode entry = array.addObject();entry.put("KeyId", k.getKeyId());entry.put("KeyArn", k.getArn());}- response.put("Truncated", false);+ putTruncation(response, page);return Response.ok(response).build();}+ /** MarkerType has length >= 1 in the model, so a present but empty Marker is refused, not ignored. */+ private static String marker(JsonNode request) {+ JsonNode node = request.path("Marker");+ if (node.isMissingNode() || node.isNull()) {+ return null;+ }+ if (!node.isTextual()) {+ // MarkerType is a string; any other JSON value is refused rather than coerced.+ String token = node.isNumber() ? "NUMBER_VALUE"+ : node.isBoolean() ? (node.booleanValue() ? "TRUE_VALUE" : "FALSE_VALUE")+ : node.isArray() ? "START_ARRAY" : "START_OBJECT";+ throw new AwsException("SerializationException", token + " can not be converted to a String", 400);+ }+ String marker = node.textValue();+ if (marker.isEmpty()) {+ throw SmithyValidationErrors.validationException("Value '' at 'marker' failed to satisfy constraint: "+ + "Member must have length greater than or equal to 1");+ }+ return marker;+ }++ /**+ * The model types Limit as an integer in LimitType [1..1000], so anything else is refused+ * rather than coerced. A Limit above an action's documented page size is clamped by the service.+ */+ private static Integer limit(JsonNode request) {+ JsonNode limit = request.path("Limit");+ if (limit.isMissingNode() || limit.isNull()) {+ return null;+ }+ if (!limit.isIntegralNumber() || !limit.canConvertToInt()) {+ throw new AwsException("SerializationException",+ (limit.isNumber() ? "NUMBER_VALUE" : "STRING_VALUE") + " can not be converted to an Integer", 400);+ }+ validateRange("limit", limit.intValue(), 1, 1000);+ return limit.intValue();+ }++ private static void putTruncation(ObjectNode response, KmsService.Page<?> page) {+ response.put("Truncated", page.truncated());+ if (page.truncated()) {+ response.put("NextMarker", page.nextMarker());+ }+ }+private Response handleListGrants(JsonNode request, String region) {String keyId = request.path("KeyId").asText();String marker = request.path("Marker").isMissingNode() ? null : request.path("Marker").asText(null);@@ -467,17 +512,17 @@ public class KmsJsonHandler {private Response handleListAliases(JsonNode request, String region) {JsonNode keyIdNode = request.path("KeyId");String keyId = (keyIdNode.isMissingNode() || keyIdNode.isNull()) ? null : keyIdNode.asText();- List<KmsAlias> aliases = service.listAliases(keyId, region);+ KmsService.Page<KmsAlias> page = service.listAliases(keyId, region, marker(request), limit(request));ObjectNode response = objectMapper.createObjectNode();ArrayNode array = response.putArray("Aliases");- for (KmsAlias a : aliases) {+ for (KmsAlias a : page.items()) {ObjectNode entry = array.addObject();entry.put("AliasName", a.getAliasName());entry.put("AliasArn", a.getAliasArn());entry.put("TargetKeyId", a.getTargetKeyId());entry.put("CreationDate", a.getCreationDate());}- response.put("Truncated", false);+ putTruncation(response, page);return Response.ok(response).build();}diff --git a/src/main/java/io/github/hectorvent/floci/services/kms/KmsService.java b/src/main/java/io/github/hectorvent/floci/services/kms/KmsService.javaindex 904b86d969..a35b616446 100644--- a/src/main/java/io/github/hectorvent/floci/services/kms/KmsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/kms/KmsService.java@@ -7,6 +7,7 @@ import io.github.hectorvent.floci.core.common.AwsRegions;import io.github.hectorvent.floci.core.common.Base64Url;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.common.ReservedTags;+import io.github.hectorvent.floci.core.common.SignedMarkers;import io.github.hectorvent.floci.core.common.SmithyValidationErrors;import io.github.hectorvent.floci.core.resource.ExplorerResource;import io.github.hectorvent.floci.core.resource.ResourceProvider;@@ -35,6 +36,7 @@ import java.security.*;import java.time.Duration;import java.time.Instant;import java.util.*;+import java.util.function.Function;import java.util.regex.Pattern;import static io.github.hectorvent.floci.services.kms.model.KmsMessageType.RAW;@@ -677,6 +679,66 @@ public class KmsService implements ResourceProvider {return result;}+ public record Page<T>(List<T> items, String nextMarker) {+ public boolean truncated() {+ return nextMarker != null;+ }+ }++ // LimitType in the model is [1..1000]; the ListKeys docs allow up to 1000, ListAliases up to 100.+ static final int MAX_LIST_KEYS_LIMIT = 1000;+ static final int MAX_LIST_ALIASES_LIMIT = 100;++ public Page<KmsKey> listKeys(String region, String marker, Integer limit) {+ return page(listKeys(region), KmsKey::getKeyId,+ "ListKeys\n" + regionResolver.getAccountId() + "\n" + region, marker, limit, MAX_LIST_KEYS_LIMIT);+ }++ public Page<KmsAlias> listAliases(String keyId, String region, String marker, Integer limit) {+ // Resolve the filter once: an alias given as KeyId could be repointed between two lookups,+ // leaving a page of one key's aliases with a marker signed for another.+ String filter = keyId == null || keyId.isBlank() ? null : resolveKey(keyId, region).getKeyId();+ return page(listAliases(filter, region), KmsAlias::getAliasName,+ "ListAliases\n" + regionResolver.getAccountId() + "\n" + region + "\n" + (filter == null ? "" : filter),+ marker, limit, MAX_LIST_ALIASES_LIMIT);+ }++ /** Signs each ListKeys/ListAliases Marker, so only a marker this service issued is accepted. */+ private final SignedMarkers listMarkers = new SignedMarkers();++ /**+ * Pages {@code items} sorted by id. Marker is a signed cursor naming the id of the last item+ * returned; the next page starts at the first id sorting after it, so deleting that item+ * between pages does not break the listing (as upstream IAM pagination does). The cursor also+ * carries {@code context} (action, account, region and filter), so a marker is only accepted by the+ * listing that issued it. Any other marker is InvalidMarkerException. Limit is clamped to+ * [1..max]; without a Limit the whole remainder is returned, so callers that never page see+ * every entry.+ */+ private <T> Page<T> page(List<T> items, Function<T, String> id, String context,+ String marker, Integer limit, int max) {+ String scope = context + "\n";+ List<T> sorted = items.stream().sorted(Comparator.comparing(id)).toList();+ int startIndex = 0;+ if (marker != null) {+ // A blank marker was not issued here, so it is refused rather than restarting the listing.+ String cursor = listMarkers.cursorOf(marker.trim());+ if (cursor == null || !cursor.startsWith(scope)) {+ throw new AwsException("InvalidMarkerException",+ "The request was rejected because the marker that specifies where pagination should next begin is not valid.", 400);+ }+ String anchor = cursor.substring(scope.length());+ while (startIndex < sorted.size() && id.apply(sorted.get(startIndex)).compareTo(anchor) <= 0) {+ startIndex++;+ }+ }+ int endIndex = limit == null ? sorted.size()+ : Math.min(startIndex + Math.clamp(limit, 1, max), sorted.size());+ List<T> pageItems = sorted.subList(startIndex, endIndex);+ String nextMarker = endIndex < sorted.size() ? listMarkers.issue(scope + id.apply(pageItems.getLast())) : null;+ return new Page<>(pageItems, nextMarker);+ }+private Map<String, Object> grantToMap(KmsGrant grant) {Map<String, Object> result = new HashMap<>();result.put("GrantId", grant.getGrantId());diff --git a/src/test/java/io/github/hectorvent/floci/services/kms/KmsIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/kms/KmsIntegrationTest.javaindex acd4c82004..0c81e7748b 100644--- a/src/test/java/io/github/hectorvent/floci/services/kms/KmsIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/kms/KmsIntegrationTest.java@@ -22,8 +22,10 @@ import java.security.PublicKey;import java.security.spec.ECGenParameterSpec;import java.security.spec.MGF1ParameterSpec;import java.security.spec.X509EncodedKeySpec;+import java.util.ArrayList;import java.util.Arrays;import java.util.Base64;+import java.util.HashSet;import java.util.List;import static io.restassured.RestAssured.given;@@ -859,6 +861,368 @@ class KmsIntegrationTest {.body("__type", equalTo("InvalidMarkerException"));}+ @Test+ void listKeysPagesWithLimitAndMarker() {+ for (int i = 0; i < 2; i++) {+ given()+ .header("X-Amz-Target", "TrentService.CreateKey")+ .contentType(KMS_CONTENT_TYPE)+ .body("{}")+ .when().post("/")+ .then().statusCode(200);+ }++ JsonPath unpaged = given()+ .header("X-Amz-Target", "TrentService.ListKeys")+ .contentType(KMS_CONTENT_TYPE)+ .body("{}")+ .when().post("/")+ .then()+ .statusCode(200)+ .body("Truncated", equalTo(false))+ .body("$", not(hasKey("NextMarker")))+ .extract().jsonPath();+ List<String> allKeyIds = unpaged.getList("Keys.KeyId");+ assertTrue(allKeyIds.size() >= 2);++ List<String> pagedKeyIds = new ArrayList<>();+ String marker = null;+ do {+ String body = marker == null ? "{\"Limit\":1}" : "{\"Limit\":1,\"Marker\":\"" + marker + "\"}";+ JsonPath page = given()+ .header("X-Amz-Target", "TrentService.ListKeys")+ .contentType(KMS_CONTENT_TYPE)+ .body(body)+ .when().post("/")+ .then()+ .statusCode(200)+ .body("Keys.size()", equalTo(1))+ .extract().jsonPath();+ pagedKeyIds.addAll(page.getList("Keys.KeyId"));+ marker = page.getBoolean("Truncated") ? page.getString("NextMarker") : null;+ assertTrue(pagedKeyIds.size() <= allKeyIds.size(), "paging did not terminate");+ } while (marker != null);++ assertEquals(new HashSet<>(allKeyIds), new HashSet<>(pagedKeyIds));+ assertEquals(allKeyIds.size(), pagedKeyIds.size());+ }++ @Test+ void listAliasesPagesAfterKeyIdFilter() {+ String keyId = given()+ .header("X-Amz-Target", "TrentService.CreateKey")+ .contentType(KMS_CONTENT_TYPE)+ .body("{}")+ .when().post("/")+ .then().statusCode(200)+ .extract().path("KeyMetadata.KeyId");++ String prefix = "alias/paging-" + System.nanoTime() + "-";+ for (int i = 0; i < 3; i++) {+ given()+ .header("X-Amz-Target", "TrentService.CreateAlias")+ .contentType(KMS_CONTENT_TYPE)+ .body("{\"AliasName\":\"" + prefix + i + "\",\"TargetKeyId\":\"" + keyId + "\"}")+ .when().post("/")+ .then().statusCode(200);+ }++ JsonPath first = given()+ .header("X-Amz-Target", "TrentService.ListAliases")+ .contentType(KMS_CONTENT_TYPE)+ .body("{\"KeyId\":\"" + keyId + "\",\"Limit\":2}")+ .when().post("/")+ .then()+ .statusCode(200)+ .body("Aliases.size()", equalTo(2))+ .body("Truncated", equalTo(true))+ .body("NextMarker", notNullValue())+ .extract().jsonPath();++ JsonPath second = given()+ .header("X-Amz-Target", "TrentService.ListAliases")+ .contentType(KMS_CONTENT_TYPE)+ .body("{\"KeyId\":\"" + keyId + "\",\"Limit\":2,\"Marker\":\"" + first.getString("NextMarker") + "\"}")+ .when().post("/")+ .then()+ .statusCode(200)+ .body("Aliases.size()", equalTo(1))+ .body("Truncated", equalTo(false))+ .body("$", not(hasKey("NextMarker")))+ .extract().jsonPath();++ List<String> names = new ArrayList<>(first.getList("Aliases.AliasName"));+ names.addAll(second.getList("Aliases.AliasName"));+ assertEquals(List.of(prefix + 0, prefix + 1, prefix + 2), names);+ assertTrue(second.getList("Aliases.TargetKeyId").stream().allMatch(keyId::equals));+ }++ @Test+ void listAliasesResumesAfterMarkedAliasIsDeleted() {+ String keyId = given()+ .header("X-Amz-Target", "TrentService.CreateKey")+ .contentType(KMS_CONTENT_TYPE)+ .body("{}")+ .when().post("/")+ .then().statusCode(200)+ .extract().path("KeyMetadata.KeyId");++ String prefix = "alias/resume-" + System.nanoTime() + "-";+ for (int i = 0; i < 3; i++) {+ given()+ .header("X-Amz-Target", "TrentService.CreateAlias")+ .contentType(KMS_CONTENT_TYPE)+ .body("{\"AliasName\":\"" + prefix + i + "\",\"TargetKeyId\":\"" + keyId + "\"}")+ .when().post("/")+ .then().statusCode(200);+ }++ String marker = given()+ .header("X-Amz-Target", "TrentService.ListAliases")+ .contentType(KMS_CONTENT_TYPE)+ .body("{\"KeyId\":\"" + keyId + "\",\"Limit\":1}")+ .when().post("/")+ .then()+ .statusCode(200)+ .body("Aliases.AliasName", equalTo(List.of(prefix + 0)))+ .body("Truncated", equalTo(true))+ .extract().path("NextMarker");++ // Delete-while-listing, as Terraform and cloud-nuke do: the alias the marker names is gone.+ given()+ .header("X-Amz-Target", "TrentService.DeleteAlias")+ .contentType(KMS_CONTENT_TYPE)+ .body("{\"AliasName\":\"" + prefix + 0 + "\"}")+ .when().post("/")+ .then().statusCode(200);++ given()+ .header("X-Amz-Target", "TrentService.ListAliases")+ .contentType(KMS_CONTENT_TYPE)+ .body("{\"KeyId\":\"" + keyId + "\",\"Limit\":1,\"Marker\":\"" + marker + "\"}")+ .when().post("/")+ .then()+ .statusCode(200)+ .body("Aliases.AliasName", equalTo(List.of(prefix + 1)))+ .body("Truncated", equalTo(true));+ }++ @Test+ void listAliasesClampsLimitToDocumentedMaximum() {+ String keyId = given()+ .header("X-Amz-Target", "TrentService.CreateKey")+ .contentType(KMS_CONTENT_TYPE)+ .body("{}")+ .when().post("/")+ .then().statusCode(200)+ .extract().path("KeyMetadata.KeyId");++ String prefix = "alias/clamp-" + System.nanoTime() + "-";+ for (int i = 0; i < 101; i++) {+ given()+ .header("X-Amz-Target", "TrentService.CreateAlias")+ .contentType(KMS_CONTENT_TYPE)+ .body("{\"AliasName\":\"" + prefix + i + "\",\"TargetKeyId\":\"" + keyId + "\"}")+ .when().post("/")+ .then().statusCode(200);+ }++ // The ListAliases reference allows Limit 1-100, so a larger Limit returns at most 100.+ given()+ .header("X-Amz-Target", "TrentService.ListAliases")+ .contentType(KMS_CONTENT_TYPE)+ .body("{\"KeyId\":\"" + keyId + "\",\"Limit\":1000}")+ .when().post("/")+ .then()+ .statusCode(200)+ .body("Aliases.size()", equalTo(100))+ .body("Truncated", equalTo(true))+ .body("NextMarker", notNullValue());+ }++ @Test+ void listMarkerIsRejectedOutsideTheListingThatIssuedIt() {+ String keyId = given()+ .header("X-Amz-Target", "TrentService.CreateKey")+ .contentType(KMS_CONTENT_TYPE)+ .body("{}")+ .when().post("/")+ .then().statusCode(200)+ .extract().path("KeyMetadata.KeyId");+ String prefix = "alias/context-" + System.nanoTime() + "-";+ for (int i = 0; i < 2; i++) {+ given()+ .header("X-Amz-Target", "TrentService.CreateAlias")+ .contentType(KMS_CONTENT_TYPE)+ .body("{\"AliasName\":\"" + prefix + i + "\",\"TargetKeyId\":\"" + keyId + "\"}")+ .when().post("/")+ .then().statusCode(200);+ }+ String marker = given()+ .header("X-Amz-Target", "TrentService.ListAliases")+ .contentType(KMS_CONTENT_TYPE)+ .body("{\"KeyId\":\"" + keyId + "\",\"Limit\":1}")+ .when().post("/")+ .then().statusCode(200)+ .body("Truncated", equalTo(true))+ .extract().path("NextMarker");++ // Same action without the KeyId filter, and another action, must not accept it.+ for (String[] reuse : new String[][] {+ {"ListAliases", "{\"Marker\":\"" + marker + "\"}"},+ {"ListKeys", "{\"Marker\":\"" + marker + "\"}"}}) {+ given()+ .header("X-Amz-Target", "TrentService." + reuse[0])+ .contentType(KMS_CONTENT_TYPE)+ .body(reuse[1])+ .when().post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidMarkerException"));+ }+ }++ @Test+ void listAliasesMarkerIsBoundToTheResolvedKeyId() {+ String keyId = given()+ .header("X-Amz-Target", "TrentService.CreateKey")+ .contentType(KMS_CONTENT_TYPE)+ .body("{}")+ .when().post("/")+ .then().statusCode(200)+ .extract().path("KeyMetadata.KeyId");+ String prefix = "alias/bound-" + System.nanoTime() + "-";+ for (int i = 0; i < 3; i++) {+ given()+ .header("X-Amz-Target", "TrentService.CreateAlias")+ .contentType(KMS_CONTENT_TYPE)+ .body("{\"AliasName\":\"" + prefix + i + "\",\"TargetKeyId\":\"" + keyId + "\"}")+ .when().post("/")+ .then().statusCode(200);+ }++ // A marker minted with the filter as an alias name pages on with the filter as the key id,+ // and the other way round: the marker names the key the page was filtered by.+ String[][] pairs = {{prefix + 0, keyId}, {keyId, prefix + 0}};+ for (String[] pair : pairs) {+ String marker = given()+ .header("X-Amz-Target", "TrentService.ListAliases")+ .contentType(KMS_CONTENT_TYPE)+ .body("{\"KeyId\":\"" + pair[0] + "\",\"Limit\":1}")+ .when().post("/")+ .then().statusCode(200)+ .body("Aliases.AliasName", equalTo(List.of(prefix + 0)))+ .extract().path("NextMarker");+ given()+ .header("X-Amz-Target", "TrentService.ListAliases")+ .contentType(KMS_CONTENT_TYPE)+ .body("{\"KeyId\":\"" + pair[1] + "\",\"Limit\":1,\"Marker\":\"" + marker + "\"}")+ .when().post("/")+ .then().statusCode(200)+ .body("Aliases.AliasName", equalTo(List.of(prefix + 1)));+ }+ }++ @ParameterizedTest+ @CsvSource(delimiter = '|', value = {+ "ListKeys|0|ValidationException",+ "ListAliases|-1|ValidationException",+ "ListAliases|1001|ValidationException",+ "ListKeys|\"abc\"|SerializationException",+ "ListAliases|1.5|SerializationException"})+ void listKeysAndAliasesRejectLimitOutsideTheModel(String action, String limit, String errorType) {+ given()+ .header("X-Amz-Target", "TrentService." + action)+ .contentType(KMS_CONTENT_TYPE)+ .body("{\"Limit\":" + limit + "}")+ .when().post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo(errorType));+ }++ @ParameterizedTest+ @CsvSource(delimiter = '|', value = {+ "ListKeys|''|ValidationException",+ "ListAliases|''|ValidationException",+ "ListKeys|' '|InvalidMarkerException",+ "ListAliases|' '|InvalidMarkerException"})+ void listKeysAndAliasesRejectBlankMarker(String action, String marker, String errorType) {+ given()+ .header("X-Amz-Target", "TrentService." + action)+ .contentType(KMS_CONTENT_TYPE)+ .body("{\"Marker\":\"" + marker + "\"}")+ .when().post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo(errorType));+ }++ @ParameterizedTest+ @CsvSource(delimiter = '|', value = {+ "ListKeys|{}", "ListKeys|[]", "ListKeys|1", "ListKeys|true",+ "ListAliases|{}", "ListAliases|[]", "ListAliases|1", "ListAliases|true"})+ void listKeysAndAliasesRejectNonStringMarker(String action, String marker) {+ given()+ .header("X-Amz-Target", "TrentService." + action)+ .contentType(KMS_CONTENT_TYPE)+ .body("{\"Marker\":" + marker + "}")+ .when().post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("SerializationException"));+ }++ @ParameterizedTest+ @CsvSource({"ListKeys", "ListAliases"})+ void listMarkerIsRejectedInAnotherAccount(String action) {+ String issuer = "AWS4-HMAC-SHA256 Credential=000000000101/20260215/us-east-1/kms/aws4_request, "+ + "SignedHeaders=host, Signature=abc";+ String other = "AWS4-HMAC-SHA256 Credential=000000000102/20260215/us-east-1/kms/aws4_request, "+ + "SignedHeaders=host, Signature=abc";+ // Both accounts list the AWS managed keys and aliases, so the first page is truncated.+ String marker = given()+ .header("Authorization", issuer)+ .header("X-Amz-Target", "TrentService." + action)+ .contentType(KMS_CONTENT_TYPE)+ .body("{\"Limit\":1}")+ .when().post("/")+ .then().statusCode(200)+ .body("Truncated", equalTo(true))+ .extract().path("NextMarker");++ given()+ .header("Authorization", other)+ .header("X-Amz-Target", "TrentService." + action)+ .contentType(KMS_CONTENT_TYPE)+ .body("{\"Limit\":1,\"Marker\":\"" + marker + "\"}")+ .when().post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidMarkerException"));++ given()+ .header("Authorization", issuer)+ .header("X-Amz-Target", "TrentService." + action)+ .contentType(KMS_CONTENT_TYPE)+ .body("{\"Limit\":1,\"Marker\":\"" + marker + "\"}")+ .when().post("/")+ .then().statusCode(200);+ }++ @ParameterizedTest+ @CsvSource({"ListKeys", "ListAliases"})+ void listKeysAndAliasesRejectBadMarker(String action) {+ given()+ .header("X-Amz-Target", "TrentService." + action)+ .contentType(KMS_CONTENT_TYPE)+ .body("{\"Marker\":\"bad-marker\"}")+ .when().post("/")+ .then()+ .statusCode(400)+ .body("__type", equalTo("InvalidMarkerException"));+ }+@Testvoid listRetirableGrantsReturnsMatchingGrantsThroughJsonHandler() {String keyId = given()