Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3): honor max-keys=0 and reject invalid max-keys values (#3510)
Niels Kootstrab994d58e142a
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.javaindex a5c5955259..f13446e755 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java@@ -3910,7 +3910,7 @@ public class DynamoDbService implements ResourceProvider {throw new AwsException("AccessDenied","Access Denied (Service: Amazon S3; Status Code: 403; Error Code: AccessDenied)", 403);}- var objects = s3Service.listObjects(bucket, prefix, null, 0);+ var objects = s3Service.listObjects(bucket, prefix, null, Integer.MAX_VALUE);if (objects.isEmpty()) {failImport(desc, "S3NoSuchKey", "No objects found under s3://" + bucket + "/" + prefix);} else {diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.javaindex 0048e9de8e..056ab0d8d1 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java@@ -486,7 +486,7 @@ public class S3Controller {public Response listObjects(@PathParam("bucket") String bucket,@QueryParam("prefix") String prefix,@QueryParam("delimiter") String delimiter,- @QueryParam("max-keys") Integer maxKeys,+ @QueryParam("max-keys") String maxKeys,@QueryParam("list-type") String listType,@QueryParam("continuation-token") String continuationToken,@QueryParam("start-after") String startAfter,@@ -639,7 +639,7 @@ public class S3Controller {s3Service.authorizeListBucket(bucket, authorization);- int max = (maxKeys != null && maxKeys > 0) ? maxKeys : 1000;+ int max = resolveMaxKeys(maxKeys);boolean v1 = !"2".equals(listType);String effectiveStartAfter = v1 && marker != null ? marker : startAfter;String effectiveContinuationToken = v1 ? null : continuationToken;@@ -1655,13 +1655,13 @@ public class S3Controller {return Response.ok(xml.build()).type(MediaType.APPLICATION_XML).build();}- private Response handleListObjectVersions(String bucket, String prefix, String delimiter, Integer maxKeys,+ private Response handleListObjectVersions(String bucket, String prefix, String delimiter, String maxKeys,String keyMarker, String versionIdMarker, String encodingType) {if (hasText(versionIdMarker) && !hasText(keyMarker)) {throw new AwsException("InvalidArgument","A version-id marker cannot be specified without a key marker.", 400);}- int max = (maxKeys != null && maxKeys > 0) ? maxKeys : 1000;+ int max = resolveMaxKeys(maxKeys);S3Service.ListVersionsResult result =s3Service.listObjectVersions(bucket, prefix, delimiter, max, keyMarker, versionIdMarker);XmlBuilder xml = new XmlBuilder()@@ -3710,6 +3710,25 @@ public class S3Controller {return Response.ok(xml.build()).build();}+ /**+ * AWS accepts any integer from 0 to {@link Integer#MAX_VALUE} for {@code max-keys} and treats+ * a missing parameter as 1000. Zero is a valid request for an empty page. Anything else,+ * negative, non-numeric or overflowing, is rejected with the same InvalidArgument response.+ */+ private static int resolveMaxKeys(String maxKeys) {+ if (maxKeys == null) {+ return 1000;+ }+ if (maxKeys.matches("\\d{1,10}") && Long.parseLong(maxKeys) <= Integer.MAX_VALUE) {+ return Integer.parseInt(maxKeys);+ }+ Map<String, Object> detail = new LinkedHashMap<>();+ detail.put("ArgumentName", "maxKeys");+ detail.put("ArgumentValue", maxKeys);+ throw new AwsException("InvalidArgument",+ "Argument maxKeys must be an integer between 0 and 2147483647", 400, detail);+ }+private Response handlePutBucketWebsite(String bucket, byte[] body) {String xml = new String(body, StandardCharsets.UTF_8);String indexDoc = XmlParser.extractFirst(xml, "Suffix", null);diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.javaindex 78c0ea265c..51a070dc8b 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java@@ -1381,7 +1381,7 @@ public class S3Service implements Resettable, ResourceProvider {// see https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.htmlboolean isTruncated = false;String nextContinuationToken = null;- if (maxKeys > 0) {+ if (maxKeys >= 0) {List<S3Object> limitedObjects = new ArrayList<>();List<String> limitedPrefixes = new ArrayList<>();int count = 0;@@ -1400,7 +1400,10 @@ public class S3Service implements Resettable, ResourceProvider {}count++;}- isTruncated = directObjectCount < allObjects.size() || commonPrefixCount < commonPrefixes.size();+ // max-keys=0 is a valid request for an empty page: AWS answers it with IsTruncated=false+ // and no continuation token even when the bucket has more objects.+ isTruncated = maxKeys > 0+ && (directObjectCount < allObjects.size() || commonPrefixCount < commonPrefixes.size());if (isTruncated) {nextContinuationToken = lastEmittedKey;}@@ -1615,7 +1618,7 @@ public class S3Service implements Resettable, ResourceProvider {boolean isTruncated = false;String nextKeyMarker = null;String nextVersionIdMarker = null;- if (maxKeys > 0) {+ if (maxKeys >= 0) {List<S3Object> pageVersions = new ArrayList<>();List<String> pagePrefixes = new ArrayList<>();int vIdx = 0;@@ -1641,7 +1644,7 @@ public class S3Service implements Resettable, ResourceProvider {}}- isTruncated = vIdx < versions.size() || cpIdx < commonPrefixes.size();+ isTruncated = maxKeys > 0 && (vIdx < versions.size() || cpIdx < commonPrefixes.size());if (!isTruncated) {nextKeyMarker = null;nextVersionIdMarker = null;diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.javaindex 06fbef0920..b9f9612678 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java@@ -4017,7 +4017,7 @@ class DynamoDbServiceTest {private static S3Service s3With(S3Object... objects) {var s3 = mock(S3Service.class);- when(s3.listObjects("bucket", "imp/", null, 0)).thenReturn(List.of(objects));+ when(s3.listObjects("bucket", "imp/", null, Integer.MAX_VALUE)).thenReturn(List.of(objects));for (var object : objects) {when(s3.getObjectMetadata("bucket", object.getKey(), null)).thenReturn(object);when(s3.openObjectStream("bucket", object.getKey(), null))@@ -4165,7 +4165,7 @@ class DynamoDbServiceTest {@Testvoid runImport_missingBucket_failsWithS3NoSuchBucket() {var s3 = mock(S3Service.class);- when(s3.listObjects("missing", "imp/", null, 0))+ when(s3.listObjects("missing", "imp/", null, Integer.MAX_VALUE)).thenThrow(new AwsException("NoSuchBucket", "The specified bucket does not exist.", 404));var svc = serviceWithS3(s3, new InMemoryStorage<>());createUsersTableInCreating(svc);@@ -4182,7 +4182,7 @@ class DynamoDbServiceTest {@Testvoid runImport_otherS3Error_reportsAnS3FailureCode() {var s3 = mock(S3Service.class);- when(s3.listObjects("bucket", "imp/", null, 0))+ when(s3.listObjects("bucket", "imp/", null, Integer.MAX_VALUE)).thenThrow(new AwsException("AccessDenied", "Access Denied", 403));var svc = serviceWithS3(s3, new InMemoryStorage<>());createUsersTableInCreating(svc);diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.javaindex 349afd7722..5ddbf33412 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java@@ -13,6 +13,7 @@ import java.security.NoSuchAlgorithmException;import java.nio.charset.StandardCharsets;import java.util.Arrays;import java.util.Base64;+import java.util.List;import static io.restassured.RestAssured.given;import static org.hamcrest.Matchers.*;@@ -2245,6 +2246,51 @@ class S3IntegrationTest {// --- ListObjectsV2 pagination ---+ @Test+ @Order(100)+ void listObjectsValidatesMaxKeys() {+ String bucket = "max-keys-test-bucket";+ given().when().put("/" + bucket).then().statusCode(200);+ given().body("a").when().put("/" + bucket + "/a.txt").then().statusCode(200);+ try {+ // max-keys=0 is a valid request for an empty page: no keys, and IsTruncated stays false+ // even though the bucket has objects.+ for (String query : List.of("?max-keys=0", "?list-type=2&max-keys=0")) {+ given()+ .when()+ .get("/" + bucket + query)+ .then()+ .statusCode(200)+ .body(containsString("<MaxKeys>0</MaxKeys>"))+ .body(containsString("<IsTruncated>false</IsTruncated>"))+ .body(not(containsString("<Key>")))+ .body(not(containsString("<NextContinuationToken>")));+ }+ given()+ .when()+ .get("/" + bucket + "?versions&max-keys=0")+ .then()+ .statusCode(200)+ .body(containsString("<MaxKeys>0</MaxKeys>"))+ .body(containsString("<IsTruncated>false</IsTruncated>"))+ .body(not(containsString("<Version>")));++ for (String invalid : List.of("-1", "abc", "2147483648")) {+ given()+ .when()+ .get("/" + bucket + "?list-type=2&max-keys=" + invalid)+ .then()+ .statusCode(400)+ .body(containsString("<Code>InvalidArgument</Code>"))+ .body(containsString("<ArgumentName>maxKeys</ArgumentName>"))+ .body(containsString("<ArgumentValue>" + invalid + "</ArgumentValue>"));+ }+ } finally {+ given().when().delete("/" + bucket + "/a.txt");+ given().when().delete("/" + bucket);+ }+ }+@Test@Order(101)void listObjectsV2StartAfterFiltersResults() {