Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(s3): store x-amz-meta-* passed in a presigned URL query string (#4753)

Niels Kootstra419e84026039
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilter.java b/src/main/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilter.java
index 8e5032f547..4d7522b4f1 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilter.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilter.java
@@ -150,10 +150,10 @@ public class PreSignedUrlFilter implements ContainerRequestFilter {
return;
}
- List<String> unsignedChecksumHeaders = unsignedChecksumHeaders(
+ List<String> unsignedHeaders = unsignedHeadersRequiringSignature(
requestContext.getHeaders().keySet(), queryParams.getFirst("X-Amz-SignedHeaders"));
- if (!unsignedChecksumHeaders.isEmpty()) {
- requestContext.abortWith(headersNotSignedResponse(unsignedChecksumHeaders));
+ if (!unsignedHeaders.isEmpty()) {
+ requestContext.abortWith(headersNotSignedResponse(unsignedHeaders));
}
return;
}
@@ -308,7 +308,7 @@ public class PreSignedUrlFilter implements ContainerRequestFilter {
return value == null ? "" : value.trim().replaceAll(" +", " ");
}
- static List<String> unsignedChecksumHeaders(Set<String> requestHeaderNames, String signedHeaders) {
+ static List<String> unsignedHeadersRequiringSignature(Set<String> requestHeaderNames, String signedHeaders) {
Set<String> normalizedSignedHeaders = List.of(signedHeaders.split(";"))
.stream()
.map(name -> name.toLowerCase(Locale.ROOT))
@@ -316,7 +316,7 @@ public class PreSignedUrlFilter implements ContainerRequestFilter {
return requestHeaderNames.stream()
.map(name -> name.toLowerCase(Locale.ROOT))
- .filter(CHECKSUM_HEADERS_REQUIRING_SIGNATURE::contains)
+ .filter(name -> CHECKSUM_HEADERS_REQUIRING_SIGNATURE.contains(name) || name.startsWith("x-amz-meta-"))
.filter(name -> !normalizedSignedHeaders.contains(name))
.distinct()
.sorted()
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
index 56405a4c55..00717d838f 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
@@ -884,7 +884,7 @@ public class S3Controller {
String sseCustomerKeyMd5 = httpHeaders.getHeaderString("x-amz-server-side-encryption-customer-key-MD5");
String cannedAcl = httpHeaders.getHeaderString("x-amz-acl");
s3Service.authorizePutObject(bucket, key, authorization);
- S3Object obj = s3Service.putObject(bucket, key, data, contentType, extractUserMetadata(httpHeaders),
+ S3Object obj = s3Service.putObject(bucket, key, data, contentType, extractUserMetadata(httpHeaders, uriInfo),
new PutObjectOptions()
.withStorageClass(httpHeaders.getHeaderString("x-amz-storage-class"))
.withContentEncoding(persistedEncoding)
@@ -1484,7 +1484,7 @@ public class S3Controller {
if (hasQueryParam(uriInfo, "uploads")) {
s3Service.authorizeObjectWrite(bucket, key, "s3:PutObject", authorization);
MultipartUpload upload = s3Service.initiateMultipartUpload(bucket, key, contentType,
- extractUserMetadata(httpHeaders),
+ extractUserMetadata(httpHeaders, uriInfo),
httpHeaders.getHeaderString("x-amz-storage-class"),
httpHeaders.getHeaderString("Content-Disposition"),
httpHeaders.getHeaderString("x-amz-server-side-encryption"),
@@ -2768,7 +2768,7 @@ public class S3Controller {
sourceObject.versionId(),
new CopyObjectOptions()
.withMetadataDirective(httpHeaders.getHeaderString("x-amz-metadata-directive"))
- .withReplacementMetadata(extractUserMetadata(httpHeaders))
+ .withReplacementMetadata(extractUserMetadata(httpHeaders, uriInfo))
.withTaggingDirective(taggingDirective)
.withReplacementTagging(replacementTagging)
.withStorageClass(httpHeaders.getHeaderString("x-amz-storage-class"))
@@ -2972,19 +2972,31 @@ public class S3Controller {
}
}
- private Map<String, String> extractUserMetadata(HttpHeaders httpHeaders) {
+ /**
+ * Reads {@code x-amz-meta-*} user metadata from the request headers and, for presigned URLs
+ * whose SDK hoisted them there, from the query string. A query parameter wins over a header
+ * with the same key, because the presigned URL signature covers the query value.
+ */
+ private Map<String, String> extractUserMetadata(HttpHeaders httpHeaders, UriInfo uriInfo) {
Map<String, String> metadata = new LinkedHashMap<>();
- for (Map.Entry<String, List<String>> entry : httpHeaders.getRequestHeaders().entrySet()) {
- String headerName = entry.getKey().toLowerCase(Locale.ROOT);
- if (!headerName.startsWith("x-amz-meta-")) {
+ if (uriInfo != null) {
+ addUserMetadata(metadata, uriInfo.getQueryParameters());
+ }
+ addUserMetadata(metadata, httpHeaders.getRequestHeaders());
+ return metadata;
+ }
+
+ private static void addUserMetadata(Map<String, String> metadata, Map<String, List<String>> source) {
+ for (Map.Entry<String, List<String>> entry : source.entrySet()) {
+ String name = entry.getKey().toLowerCase(Locale.ROOT);
+ if (!name.startsWith("x-amz-meta-")) {
continue;
}
- String key = headerName.substring("x-amz-meta-".length());
+ String key = name.substring("x-amz-meta-".length());
if (!key.isBlank() && !entry.getValue().isEmpty()) {
- metadata.put(key, entry.getValue().get(0));
+ metadata.putIfAbsent(key, entry.getValue().get(0));
}
}
- return metadata;
}
static String resolveHeaderOrQueryParam(HttpHeaders httpHeaders, UriInfo uriInfo, String name) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilterTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilterTest.java
index c6ef00ee9e..01169b8b5f 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilterTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilterTest.java
@@ -156,7 +156,7 @@ class PreSignedUrlFilterTest {
"x-amz-checksum-sha1",
"x-amz-checksum-sha256",
"x-amz-sdk-checksum-algorithm"),
- PreSignedUrlFilter.unsignedChecksumHeaders(
+ PreSignedUrlFilter.unsignedHeadersRequiringSignature(
Set.of(
"Host",
"X-Amz-Checksum-Algorithm",
@@ -173,18 +173,27 @@ class PreSignedUrlFilterTest {
void acceptsChecksumHeadersIncludedInSignedHeaders() {
assertEquals(
List.of(),
- PreSignedUrlFilter.unsignedChecksumHeaders(
+ PreSignedUrlFilter.unsignedHeadersRequiringSignature(
Set.of("host", "x-amz-checksum-algorithm", "x-amz-checksum-crc32",
"x-amz-checksum-sha256", "x-amz-sdk-checksum-algorithm"),
"host;x-amz-checksum-algorithm;x-amz-checksum-crc32;"
+ "x-amz-checksum-sha256;x-amz-sdk-checksum-algorithm"));
}
+ @Test
+ void reportsUnsignedUserMetadataHeaders() {
+ assertEquals(
+ List.of("x-amz-meta-source"),
+ PreSignedUrlFilter.unsignedHeadersRequiringSignature(
+ Set.of("host", "X-Amz-Meta-Source", "x-amz-meta-signed"),
+ "host;x-amz-meta-signed"));
+ }
+
@Test
void ignoresHeadersOutsideChecksumFamily() {
assertEquals(
List.of(),
- PreSignedUrlFilter.unsignedChecksumHeaders(
+ PreSignedUrlFilter.unsignedHeadersRequiringSignature(
Set.of("content-type", "user-agent", "x-amz-content-sha256",
"x-amz-user-agent", "x-amz-checksum-type"),
"host"));
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3PresignedChecksumIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3PresignedChecksumIntegrationTest.java
index 95eecc6f67..6492b41a12 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3PresignedChecksumIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3PresignedChecksumIntegrationTest.java
@@ -212,6 +212,135 @@ class S3PresignedChecksumIntegrationTest {
.header("x-amz-checksum-sha256", equalTo(expectedSha256));
}
+ @Test
+ void presignedPutWithUserMetadataInQueryStoresTheMetadata() throws Exception {
+ createBucket();
+ String path = "/" + BUCKET + "/query-metadata.txt";
+ String url = presign("PUT", path,
+ Map.of("x-amz-meta-source", "presigned-put", "x-amz-meta-Uploaded-By", "tester"), Map.of());
+
+ given()
+ .urlEncodingEnabled(false)
+ .body("x".getBytes(StandardCharsets.UTF_8))
+ .when()
+ .put(url)
+ .then()
+ .statusCode(200);
+
+ given()
+ .filter(LOCAL_SIGNER)
+ .when()
+ .head(path)
+ .then()
+ .statusCode(200)
+ .header("x-amz-meta-source", equalTo("presigned-put"))
+ .header("x-amz-meta-uploaded-by", equalTo("tester"));
+ }
+
+ @Test
+ void presignedPutWithUnsignedMetadataHeaderIsRejectedAsHeadersNotSigned() throws Exception {
+ createBucket();
+ String path = "/" + BUCKET + "/unsigned-metadata-header.txt";
+ String url = presign("PUT", path, Map.of("x-amz-meta-source", "signed"), Map.of());
+
+ given()
+ .urlEncodingEnabled(false)
+ .header("x-amz-meta-source", "altered")
+ .body("x".getBytes(StandardCharsets.UTF_8))
+ .when()
+ .put(url)
+ .then()
+ .statusCode(403)
+ .body(containsString("<Code>AccessDenied</Code>"))
+ .body(containsString("<HeadersNotSigned>x-amz-meta-source</HeadersNotSigned>"));
+
+ given()
+ .filter(LOCAL_SIGNER)
+ .when()
+ .head(path)
+ .then()
+ .statusCode(404);
+ }
+
+ @Test
+ void presignedCreateMultipartUploadWithUserMetadataInQueryStoresTheMetadata() throws Exception {
+ createBucket();
+ String path = "/" + BUCKET + "/multipart-query-metadata.txt";
+ String initiateUrl = presign("POST", path,
+ Map.of("uploads", "", "x-amz-meta-source", "presigned-multipart"), Map.of());
+
+ String uploadId = given()
+ .urlEncodingEnabled(false)
+ .when()
+ .post(initiateUrl)
+ .then()
+ .statusCode(200)
+ .extract().xmlPath().getString("InitiateMultipartUploadResult.UploadId");
+
+ String etag = given()
+ .filter(LOCAL_SIGNER)
+ .body("part".getBytes(StandardCharsets.UTF_8))
+ .when()
+ .put(path + "?partNumber=1&uploadId=" + uploadId)
+ .then()
+ .statusCode(200)
+ .extract().header("ETag");
+
+ given()
+ .filter(LOCAL_SIGNER)
+ .body("<CompleteMultipartUpload><Part><PartNumber>1</PartNumber><ETag>" + etag
+ + "</ETag></Part></CompleteMultipartUpload>")
+ .when()
+ .post(path + "?uploadId=" + uploadId)
+ .then()
+ .statusCode(200);
+
+ given()
+ .filter(LOCAL_SIGNER)
+ .when()
+ .head(path)
+ .then()
+ .statusCode(200)
+ .header("x-amz-meta-source", equalTo("presigned-multipart"));
+ }
+
+ @Test
+ void presignedCopyObjectWithReplacedUserMetadataInQueryStoresTheMetadata() throws Exception {
+ createBucket();
+ String sourcePath = "/" + BUCKET + "/metadata-copy-source.txt";
+ given()
+ .filter(LOCAL_SIGNER)
+ .header("x-amz-meta-source", "original")
+ .body("copy me".getBytes(StandardCharsets.UTF_8))
+ .when()
+ .put(sourcePath)
+ .then()
+ .statusCode(200);
+
+ String destPath = "/" + BUCKET + "/metadata-copy-dest.txt";
+ String copyUrl = presign("PUT", destPath,
+ Map.of("x-amz-meta-source", "copied"),
+ Map.of("x-amz-copy-source", sourcePath, "x-amz-metadata-directive", "REPLACE"));
+
+ given()
+ .urlEncodingEnabled(false)
+ .header("x-amz-copy-source", sourcePath)
+ .header("x-amz-metadata-directive", "REPLACE")
+ .when()
+ .put(copyUrl)
+ .then()
+ .statusCode(200)
+ .body(containsString("CopyObjectResult"));
+
+ given()
+ .filter(LOCAL_SIGNER)
+ .when()
+ .head(destPath)
+ .then()
+ .statusCode(200)
+ .header("x-amz-meta-source", equalTo("copied"));
+ }
+
private static void createBucket() {
given()
.filter(LOCAL_SIGNER)