Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(s3): honor max-keys=0 and reject invalid max-keys values (#3510)

Niels Kootstrab994d58e142a
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
index a5c5955259..f13446e755 100644
--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbService.java
@@ -3910,7 +3910,7 @@ public class DynamoDbService implements ResourceProvider {
throw new AwsException("AccessDenied",
"Access Denied (Service: Amazon S3; Status Code: 403; Error Code: AccessDenied)", 403);
}
- var objects = s3Service.listObjects(bucket, prefix, null, 0);
+ var objects = s3Service.listObjects(bucket, prefix, null, Integer.MAX_VALUE);
if (objects.isEmpty()) {
failImport(desc, "S3NoSuchKey", "No objects found under s3://" + bucket + "/" + prefix);
} else {
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
index 0048e9de8e..056ab0d8d1 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
@@ -486,7 +486,7 @@ public class S3Controller {
public Response listObjects(@PathParam("bucket") String bucket,
@QueryParam("prefix") String prefix,
@QueryParam("delimiter") String delimiter,
- @QueryParam("max-keys") Integer maxKeys,
+ @QueryParam("max-keys") String maxKeys,
@QueryParam("list-type") String listType,
@QueryParam("continuation-token") String continuationToken,
@QueryParam("start-after") String startAfter,
@@ -639,7 +639,7 @@ public class S3Controller {
s3Service.authorizeListBucket(bucket, authorization);
- int max = (maxKeys != null && maxKeys > 0) ? maxKeys : 1000;
+ int max = resolveMaxKeys(maxKeys);
boolean v1 = !"2".equals(listType);
String effectiveStartAfter = v1 && marker != null ? marker : startAfter;
String effectiveContinuationToken = v1 ? null : continuationToken;
@@ -1655,13 +1655,13 @@ public class S3Controller {
return Response.ok(xml.build()).type(MediaType.APPLICATION_XML).build();
}
- private Response handleListObjectVersions(String bucket, String prefix, String delimiter, Integer maxKeys,
+ private Response handleListObjectVersions(String bucket, String prefix, String delimiter, String maxKeys,
String keyMarker, String versionIdMarker, String encodingType) {
if (hasText(versionIdMarker) && !hasText(keyMarker)) {
throw new AwsException("InvalidArgument",
"A version-id marker cannot be specified without a key marker.", 400);
}
- int max = (maxKeys != null && maxKeys > 0) ? maxKeys : 1000;
+ int max = resolveMaxKeys(maxKeys);
S3Service.ListVersionsResult result =
s3Service.listObjectVersions(bucket, prefix, delimiter, max, keyMarker, versionIdMarker);
XmlBuilder xml = new XmlBuilder()
@@ -3710,6 +3710,25 @@ public class S3Controller {
return Response.ok(xml.build()).build();
}
+ /**
+ * AWS accepts any integer from 0 to {@link Integer#MAX_VALUE} for {@code max-keys} and treats
+ * a missing parameter as 1000. Zero is a valid request for an empty page. Anything else,
+ * negative, non-numeric or overflowing, is rejected with the same InvalidArgument response.
+ */
+ private static int resolveMaxKeys(String maxKeys) {
+ if (maxKeys == null) {
+ return 1000;
+ }
+ if (maxKeys.matches("\\d{1,10}") && Long.parseLong(maxKeys) <= Integer.MAX_VALUE) {
+ return Integer.parseInt(maxKeys);
+ }
+ Map<String, Object> detail = new LinkedHashMap<>();
+ detail.put("ArgumentName", "maxKeys");
+ detail.put("ArgumentValue", maxKeys);
+ throw new AwsException("InvalidArgument",
+ "Argument maxKeys must be an integer between 0 and 2147483647", 400, detail);
+ }
+
private Response handlePutBucketWebsite(String bucket, byte[] body) {
String xml = new String(body, StandardCharsets.UTF_8);
String indexDoc = XmlParser.extractFirst(xml, "Suffix", null);
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
index 78c0ea265c..51a070dc8b 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
@@ -1381,7 +1381,7 @@ public class S3Service implements Resettable, ResourceProvider {
// see https://docs.aws.amazon.com/AmazonS3/latest/API/API_ListObjectsV2.html
boolean isTruncated = false;
String nextContinuationToken = null;
- if (maxKeys > 0) {
+ if (maxKeys >= 0) {
List<S3Object> limitedObjects = new ArrayList<>();
List<String> limitedPrefixes = new ArrayList<>();
int count = 0;
@@ -1400,7 +1400,10 @@ public class S3Service implements Resettable, ResourceProvider {
}
count++;
}
- isTruncated = directObjectCount < allObjects.size() || commonPrefixCount < commonPrefixes.size();
+ // max-keys=0 is a valid request for an empty page: AWS answers it with IsTruncated=false
+ // and no continuation token even when the bucket has more objects.
+ isTruncated = maxKeys > 0
+ && (directObjectCount < allObjects.size() || commonPrefixCount < commonPrefixes.size());
if (isTruncated) {
nextContinuationToken = lastEmittedKey;
}
@@ -1615,7 +1618,7 @@ public class S3Service implements Resettable, ResourceProvider {
boolean isTruncated = false;
String nextKeyMarker = null;
String nextVersionIdMarker = null;
- if (maxKeys > 0) {
+ if (maxKeys >= 0) {
List<S3Object> pageVersions = new ArrayList<>();
List<String> pagePrefixes = new ArrayList<>();
int vIdx = 0;
@@ -1641,7 +1644,7 @@ public class S3Service implements Resettable, ResourceProvider {
}
}
- isTruncated = vIdx < versions.size() || cpIdx < commonPrefixes.size();
+ isTruncated = maxKeys > 0 && (vIdx < versions.size() || cpIdx < commonPrefixes.size());
if (!isTruncated) {
nextKeyMarker = null;
nextVersionIdMarker = null;
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java
index 06fbef0920..b9f9612678 100644
--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbServiceTest.java
@@ -4017,7 +4017,7 @@ class DynamoDbServiceTest {
private static S3Service s3With(S3Object... objects) {
var s3 = mock(S3Service.class);
- when(s3.listObjects("bucket", "imp/", null, 0)).thenReturn(List.of(objects));
+ when(s3.listObjects("bucket", "imp/", null, Integer.MAX_VALUE)).thenReturn(List.of(objects));
for (var object : objects) {
when(s3.getObjectMetadata("bucket", object.getKey(), null)).thenReturn(object);
when(s3.openObjectStream("bucket", object.getKey(), null))
@@ -4165,7 +4165,7 @@ class DynamoDbServiceTest {
@Test
void runImport_missingBucket_failsWithS3NoSuchBucket() {
var s3 = mock(S3Service.class);
- when(s3.listObjects("missing", "imp/", null, 0))
+ when(s3.listObjects("missing", "imp/", null, Integer.MAX_VALUE))
.thenThrow(new AwsException("NoSuchBucket", "The specified bucket does not exist.", 404));
var svc = serviceWithS3(s3, new InMemoryStorage<>());
createUsersTableInCreating(svc);
@@ -4182,7 +4182,7 @@ class DynamoDbServiceTest {
@Test
void runImport_otherS3Error_reportsAnS3FailureCode() {
var s3 = mock(S3Service.class);
- when(s3.listObjects("bucket", "imp/", null, 0))
+ when(s3.listObjects("bucket", "imp/", null, Integer.MAX_VALUE))
.thenThrow(new AwsException("AccessDenied", "Access Denied", 403));
var svc = serviceWithS3(s3, new InMemoryStorage<>());
createUsersTableInCreating(svc);
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java
index 349afd7722..5ddbf33412 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3IntegrationTest.java
@@ -13,6 +13,7 @@ import java.security.NoSuchAlgorithmException;
import java.nio.charset.StandardCharsets;
import java.util.Arrays;
import java.util.Base64;
+import java.util.List;
import static io.restassured.RestAssured.given;
import static org.hamcrest.Matchers.*;
@@ -2245,6 +2246,51 @@ class S3IntegrationTest {
// --- ListObjectsV2 pagination ---
+ @Test
+ @Order(100)
+ void listObjectsValidatesMaxKeys() {
+ String bucket = "max-keys-test-bucket";
+ given().when().put("/" + bucket).then().statusCode(200);
+ given().body("a").when().put("/" + bucket + "/a.txt").then().statusCode(200);
+ try {
+ // max-keys=0 is a valid request for an empty page: no keys, and IsTruncated stays false
+ // even though the bucket has objects.
+ for (String query : List.of("?max-keys=0", "?list-type=2&max-keys=0")) {
+ given()
+ .when()
+ .get("/" + bucket + query)
+ .then()
+ .statusCode(200)
+ .body(containsString("<MaxKeys>0</MaxKeys>"))
+ .body(containsString("<IsTruncated>false</IsTruncated>"))
+ .body(not(containsString("<Key>")))
+ .body(not(containsString("<NextContinuationToken>")));
+ }
+ given()
+ .when()
+ .get("/" + bucket + "?versions&max-keys=0")
+ .then()
+ .statusCode(200)
+ .body(containsString("<MaxKeys>0</MaxKeys>"))
+ .body(containsString("<IsTruncated>false</IsTruncated>"))
+ .body(not(containsString("<Version>")));
+
+ for (String invalid : List.of("-1", "abc", "2147483648")) {
+ given()
+ .when()
+ .get("/" + bucket + "?list-type=2&max-keys=" + invalid)
+ .then()
+ .statusCode(400)
+ .body(containsString("<Code>InvalidArgument</Code>"))
+ .body(containsString("<ArgumentName>maxKeys</ArgumentName>"))
+ .body(containsString("<ArgumentValue>" + invalid + "</ArgumentValue>"));
+ }
+ } finally {
+ given().when().delete("/" + bucket + "/a.txt");
+ given().when().delete("/" + bucket);
+ }
+ }
+
@Test
@Order(101)
void listObjectsV2StartAfterFiltersResults() {