Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3): store x-amz-meta-* passed in a presigned URL query string (#4753)
Niels Kootstra419e84026039
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilter.java b/src/main/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilter.javaindex 8e5032f547..4d7522b4f1 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilter.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilter.java@@ -150,10 +150,10 @@ public class PreSignedUrlFilter implements ContainerRequestFilter {return;}- List<String> unsignedChecksumHeaders = unsignedChecksumHeaders(+ List<String> unsignedHeaders = unsignedHeadersRequiringSignature(requestContext.getHeaders().keySet(), queryParams.getFirst("X-Amz-SignedHeaders"));- if (!unsignedChecksumHeaders.isEmpty()) {- requestContext.abortWith(headersNotSignedResponse(unsignedChecksumHeaders));+ if (!unsignedHeaders.isEmpty()) {+ requestContext.abortWith(headersNotSignedResponse(unsignedHeaders));}return;}@@ -308,7 +308,7 @@ public class PreSignedUrlFilter implements ContainerRequestFilter {return value == null ? "" : value.trim().replaceAll(" +", " ");}- static List<String> unsignedChecksumHeaders(Set<String> requestHeaderNames, String signedHeaders) {+ static List<String> unsignedHeadersRequiringSignature(Set<String> requestHeaderNames, String signedHeaders) {Set<String> normalizedSignedHeaders = List.of(signedHeaders.split(";")).stream().map(name -> name.toLowerCase(Locale.ROOT))@@ -316,7 +316,7 @@ public class PreSignedUrlFilter implements ContainerRequestFilter {return requestHeaderNames.stream().map(name -> name.toLowerCase(Locale.ROOT))- .filter(CHECKSUM_HEADERS_REQUIRING_SIGNATURE::contains)+ .filter(name -> CHECKSUM_HEADERS_REQUIRING_SIGNATURE.contains(name) || name.startsWith("x-amz-meta-")).filter(name -> !normalizedSignedHeaders.contains(name)).distinct().sorted()diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.javaindex 56405a4c55..00717d838f 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java@@ -884,7 +884,7 @@ public class S3Controller {String sseCustomerKeyMd5 = httpHeaders.getHeaderString("x-amz-server-side-encryption-customer-key-MD5");String cannedAcl = httpHeaders.getHeaderString("x-amz-acl");s3Service.authorizePutObject(bucket, key, authorization);- S3Object obj = s3Service.putObject(bucket, key, data, contentType, extractUserMetadata(httpHeaders),+ S3Object obj = s3Service.putObject(bucket, key, data, contentType, extractUserMetadata(httpHeaders, uriInfo),new PutObjectOptions().withStorageClass(httpHeaders.getHeaderString("x-amz-storage-class")).withContentEncoding(persistedEncoding)@@ -1484,7 +1484,7 @@ public class S3Controller {if (hasQueryParam(uriInfo, "uploads")) {s3Service.authorizeObjectWrite(bucket, key, "s3:PutObject", authorization);MultipartUpload upload = s3Service.initiateMultipartUpload(bucket, key, contentType,- extractUserMetadata(httpHeaders),+ extractUserMetadata(httpHeaders, uriInfo),httpHeaders.getHeaderString("x-amz-storage-class"),httpHeaders.getHeaderString("Content-Disposition"),httpHeaders.getHeaderString("x-amz-server-side-encryption"),@@ -2768,7 +2768,7 @@ public class S3Controller {sourceObject.versionId(),new CopyObjectOptions().withMetadataDirective(httpHeaders.getHeaderString("x-amz-metadata-directive"))- .withReplacementMetadata(extractUserMetadata(httpHeaders))+ .withReplacementMetadata(extractUserMetadata(httpHeaders, uriInfo)).withTaggingDirective(taggingDirective).withReplacementTagging(replacementTagging).withStorageClass(httpHeaders.getHeaderString("x-amz-storage-class"))@@ -2972,19 +2972,31 @@ public class S3Controller {}}- private Map<String, String> extractUserMetadata(HttpHeaders httpHeaders) {+ /**+ * Reads {@code x-amz-meta-*} user metadata from the request headers and, for presigned URLs+ * whose SDK hoisted them there, from the query string. A query parameter wins over a header+ * with the same key, because the presigned URL signature covers the query value.+ */+ private Map<String, String> extractUserMetadata(HttpHeaders httpHeaders, UriInfo uriInfo) {Map<String, String> metadata = new LinkedHashMap<>();- for (Map.Entry<String, List<String>> entry : httpHeaders.getRequestHeaders().entrySet()) {- String headerName = entry.getKey().toLowerCase(Locale.ROOT);- if (!headerName.startsWith("x-amz-meta-")) {+ if (uriInfo != null) {+ addUserMetadata(metadata, uriInfo.getQueryParameters());+ }+ addUserMetadata(metadata, httpHeaders.getRequestHeaders());+ return metadata;+ }++ private static void addUserMetadata(Map<String, String> metadata, Map<String, List<String>> source) {+ for (Map.Entry<String, List<String>> entry : source.entrySet()) {+ String name = entry.getKey().toLowerCase(Locale.ROOT);+ if (!name.startsWith("x-amz-meta-")) {continue;}- String key = headerName.substring("x-amz-meta-".length());+ String key = name.substring("x-amz-meta-".length());if (!key.isBlank() && !entry.getValue().isEmpty()) {- metadata.put(key, entry.getValue().get(0));+ metadata.putIfAbsent(key, entry.getValue().get(0));}}- return metadata;}static String resolveHeaderOrQueryParam(HttpHeaders httpHeaders, UriInfo uriInfo, String name) {diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilterTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilterTest.javaindex c6ef00ee9e..01169b8b5f 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilterTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilterTest.java@@ -156,7 +156,7 @@ class PreSignedUrlFilterTest {"x-amz-checksum-sha1","x-amz-checksum-sha256","x-amz-sdk-checksum-algorithm"),- PreSignedUrlFilter.unsignedChecksumHeaders(+ PreSignedUrlFilter.unsignedHeadersRequiringSignature(Set.of("Host","X-Amz-Checksum-Algorithm",@@ -173,18 +173,27 @@ class PreSignedUrlFilterTest {void acceptsChecksumHeadersIncludedInSignedHeaders() {assertEquals(List.of(),- PreSignedUrlFilter.unsignedChecksumHeaders(+ PreSignedUrlFilter.unsignedHeadersRequiringSignature(Set.of("host", "x-amz-checksum-algorithm", "x-amz-checksum-crc32","x-amz-checksum-sha256", "x-amz-sdk-checksum-algorithm"),"host;x-amz-checksum-algorithm;x-amz-checksum-crc32;"+ "x-amz-checksum-sha256;x-amz-sdk-checksum-algorithm"));}+ @Test+ void reportsUnsignedUserMetadataHeaders() {+ assertEquals(+ List.of("x-amz-meta-source"),+ PreSignedUrlFilter.unsignedHeadersRequiringSignature(+ Set.of("host", "X-Amz-Meta-Source", "x-amz-meta-signed"),+ "host;x-amz-meta-signed"));+ }+@Testvoid ignoresHeadersOutsideChecksumFamily() {assertEquals(List.of(),- PreSignedUrlFilter.unsignedChecksumHeaders(+ PreSignedUrlFilter.unsignedHeadersRequiringSignature(Set.of("content-type", "user-agent", "x-amz-content-sha256","x-amz-user-agent", "x-amz-checksum-type"),"host"));diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3PresignedChecksumIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3PresignedChecksumIntegrationTest.javaindex 95eecc6f67..6492b41a12 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3PresignedChecksumIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3PresignedChecksumIntegrationTest.java@@ -212,6 +212,135 @@ class S3PresignedChecksumIntegrationTest {.header("x-amz-checksum-sha256", equalTo(expectedSha256));}+ @Test+ void presignedPutWithUserMetadataInQueryStoresTheMetadata() throws Exception {+ createBucket();+ String path = "/" + BUCKET + "/query-metadata.txt";+ String url = presign("PUT", path,+ Map.of("x-amz-meta-source", "presigned-put", "x-amz-meta-Uploaded-By", "tester"), Map.of());++ given()+ .urlEncodingEnabled(false)+ .body("x".getBytes(StandardCharsets.UTF_8))+ .when()+ .put(url)+ .then()+ .statusCode(200);++ given()+ .filter(LOCAL_SIGNER)+ .when()+ .head(path)+ .then()+ .statusCode(200)+ .header("x-amz-meta-source", equalTo("presigned-put"))+ .header("x-amz-meta-uploaded-by", equalTo("tester"));+ }++ @Test+ void presignedPutWithUnsignedMetadataHeaderIsRejectedAsHeadersNotSigned() throws Exception {+ createBucket();+ String path = "/" + BUCKET + "/unsigned-metadata-header.txt";+ String url = presign("PUT", path, Map.of("x-amz-meta-source", "signed"), Map.of());++ given()+ .urlEncodingEnabled(false)+ .header("x-amz-meta-source", "altered")+ .body("x".getBytes(StandardCharsets.UTF_8))+ .when()+ .put(url)+ .then()+ .statusCode(403)+ .body(containsString("<Code>AccessDenied</Code>"))+ .body(containsString("<HeadersNotSigned>x-amz-meta-source</HeadersNotSigned>"));++ given()+ .filter(LOCAL_SIGNER)+ .when()+ .head(path)+ .then()+ .statusCode(404);+ }++ @Test+ void presignedCreateMultipartUploadWithUserMetadataInQueryStoresTheMetadata() throws Exception {+ createBucket();+ String path = "/" + BUCKET + "/multipart-query-metadata.txt";+ String initiateUrl = presign("POST", path,+ Map.of("uploads", "", "x-amz-meta-source", "presigned-multipart"), Map.of());++ String uploadId = given()+ .urlEncodingEnabled(false)+ .when()+ .post(initiateUrl)+ .then()+ .statusCode(200)+ .extract().xmlPath().getString("InitiateMultipartUploadResult.UploadId");++ String etag = given()+ .filter(LOCAL_SIGNER)+ .body("part".getBytes(StandardCharsets.UTF_8))+ .when()+ .put(path + "?partNumber=1&uploadId=" + uploadId)+ .then()+ .statusCode(200)+ .extract().header("ETag");++ given()+ .filter(LOCAL_SIGNER)+ .body("<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>" + etag+ + "</ETag></Part></CompleteMultipartUpload>")+ .when()+ .post(path + "?uploadId=" + uploadId)+ .then()+ .statusCode(200);++ given()+ .filter(LOCAL_SIGNER)+ .when()+ .head(path)+ .then()+ .statusCode(200)+ .header("x-amz-meta-source", equalTo("presigned-multipart"));+ }++ @Test+ void presignedCopyObjectWithReplacedUserMetadataInQueryStoresTheMetadata() throws Exception {+ createBucket();+ String sourcePath = "/" + BUCKET + "/metadata-copy-source.txt";+ given()+ .filter(LOCAL_SIGNER)+ .header("x-amz-meta-source", "original")+ .body("copy me".getBytes(StandardCharsets.UTF_8))+ .when()+ .put(sourcePath)+ .then()+ .statusCode(200);++ String destPath = "/" + BUCKET + "/metadata-copy-dest.txt";+ String copyUrl = presign("PUT", destPath,+ Map.of("x-amz-meta-source", "copied"),+ Map.of("x-amz-copy-source", sourcePath, "x-amz-metadata-directive", "REPLACE"));++ given()+ .urlEncodingEnabled(false)+ .header("x-amz-copy-source", sourcePath)+ .header("x-amz-metadata-directive", "REPLACE")+ .when()+ .put(copyUrl)+ .then()+ .statusCode(200)+ .body(containsString("CopyObjectResult"));++ given()+ .filter(LOCAL_SIGNER)+ .when()+ .head(destPath)+ .then()+ .statusCode(200)+ .header("x-amz-meta-source", equalTo("copied"));+ }+private static void createBucket() {given().filter(LOCAL_SIGNER)