Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3): honor x-amz-tagging query parameter on presigned PutObject and CreateMultipartUpload (#3615)
Mohamed Amine ALOUINI52bafb7e5c12
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.javaindex 1c5ecf631b..f69d41fe7d 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java@@ -812,7 +812,7 @@ public class S3Controller {return handleCopyObject(copySource, bucket, key, contentType, httpHeaders, authorization);}- Map<String, String> inlineTags = parseInlineTaggingHeader(tagging);+ Map<String, String> inlineTags = parseInlineTaggingHeader(resolveInlineTaggingSource(tagging, uriInfo));String lockMode = httpHeaders.getHeaderString("x-amz-object-lock-mode");String retainUntilStr = httpHeaders.getHeaderString("x-amz-object-lock-retain-until-date");@@ -1388,7 +1388,8 @@ public class S3Controller {httpHeaders.getHeaderString("x-amz-server-side-encryption-customer-key-MD5"),getChecksumAlgorithm(httpHeaders),httpHeaders.getHeaderString("x-amz-checksum-type"),- parseInlineTaggingHeader(httpHeaders.getHeaderString("x-amz-tagging")));+ parseInlineTaggingHeader(+ resolveInlineTaggingSource(httpHeaders.getHeaderString("x-amz-tagging"), uriInfo)));String xml = new XmlBuilder().raw("<?xml version=\"1.0\" encoding=\"UTF-8\"?>").start("InitiateMultipartUploadResult", AwsNamespaces.S3)@@ -3606,8 +3607,24 @@ public class S3Controller {private static final int MAX_INLINE_TAGGING_HEADER_BYTES = 8 * 1024;/**- * Parses an {@code x-amz-tagging} request-header value (URL-encoded- * {@code k=v&k=v}) into a tag map. Returns an empty map for null or blank input.+ * Resolves the inline {@code x-amz-tagging} value for PutObject and CreateMultipartUpload.+ * The signed request header wins; otherwise the value comes from the {@code x-amz-tagging}+ * query parameter, where SDK presigners hoist the header while signing only {@code host}.+ * JAX-RS decodes the query value once, which yields the same URL-encoded {@code k=v&k=v}+ * string the header form carries, so both sources feed+ * {@link #parseInlineTaggingHeader(String)} unchanged. Real S3 honors both forms.+ */+ private static String resolveInlineTaggingSource(String taggingHeader, UriInfo uriInfo) {+ if (taggingHeader != null) {+ return taggingHeader;+ }+ return uriInfo.getQueryParameters().getFirst("x-amz-tagging");+ }++ /**+ * Parses an {@code x-amz-tagging} value (URL-encoded {@code k=v&k=v}), taken from the+ * request header or the presigned-URL query parameter, into a tag map. Returns an empty+ * map for null or blank input.** <p>Note: the error codes thrown here ({@code InvalidArgument} for malformed input,* {@code BadRequest} for exceeding the 10-tag limit) match real-AWS S3 behaviordiff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3PresignedPutTaggingIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3PresignedPutTaggingIntegrationTest.javaindex 21125fb1a2..0defb8abdf 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3PresignedPutTaggingIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3PresignedPutTaggingIntegrationTest.java@@ -1,6 +1,7 @@package io.github.hectorvent.floci.services.s3;import io.quarkus.test.junit.QuarkusTest;+import io.restassured.RestAssured;import jakarta.inject.Inject;import org.junit.jupiter.api.Test;@@ -9,7 +10,9 @@ import java.util.Arrays;import java.util.stream.Collectors;import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.containsString;import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.not;import static org.hamcrest.Matchers.notNullValue;import static org.junit.jupiter.api.Assertions.assertTrue;@@ -78,5 +81,135 @@ class S3PresignedPutTaggingIntegrationTest {.then().statusCode(200).body(equalTo("uploaded via presigned PUT with tagging"));++ given()+ .when()+ .get("/" + BUCKET + "/tagged-via-presign.txt?tagging")+ .then()+ .statusCode(200)+ .body(containsString("<Key>tag</Key>"))+ .body(containsString("<Value>test</Value>"));+ }++ /**+ * Regression test for #3608: SDK presigners hoist {@code x-amz-tagging} into the query+ * string and sign only {@code host}, so the uploader sends no tagging header at all.+ * Real S3 applies the tags from the query parameter; Floci must do the same.+ *+ * <p>The header form is itself URL-encoded ({@code c=d%20e}), and the presigner+ * URL-encodes that whole value once more when it becomes a query parameter, which is+ * why the tag string below is double-encoded.+ */+ @Test+ void presignedPutWithTaggingQueryParameterAppliesTags() {+ createBucket();+ String key = "tagged-via-query.txt";+ URI uri = presign(key, "PUT");++ given()+ .urlEncodingEnabled(false)+ .contentType("text/plain")+ .body("uploaded via presigned PUT with tagging in the query string")+ .when()+ .put(uri.getRawPath() + "?" + uri.getRawQuery() + "&x-amz-tagging=a%3Db%26c%3Dd%2520e")+ .then()+ .statusCode(200)+ .header("ETag", notNullValue());++ given()+ .when()+ .get("/" + BUCKET + "/" + key + "?tagging")+ .then()+ .statusCode(200)+ .body(containsString("<Key>a</Key>"))+ .body(containsString("<Value>b</Value>"))+ .body(containsString("<Key>c</Key>"))+ .body(containsString("<Value>d e</Value>"));+ }++ @Test+ void presignedPutWithTaggingHeaderAndQueryParameterPrefersHeader() {+ createBucket();+ String key = "tagged-via-header-and-query.txt";+ URI uri = presign(key, "PUT");++ given()+ .urlEncodingEnabled(false)+ .contentType("text/plain")+ .header("x-amz-tagging", "header=1")+ .body("the signed header wins over the query parameter")+ .when()+ .put(uri.getRawPath() + "?" + uri.getRawQuery() + "&x-amz-tagging=query%3D1")+ .then()+ .statusCode(200);++ given()+ .when()+ .get("/" + BUCKET + "/" + key + "?tagging")+ .then()+ .statusCode(200)+ .body(containsString("<Key>header</Key>"))+ .body(containsString("<Value>1</Value>"))+ .body(not(containsString("<Key>query</Key>")));+ }++ @Test+ void presignedCreateMultipartUploadWithTaggingQueryParameterAppliesTags() {+ createBucket();+ String key = "tagged-multipart-via-query.txt";+ String objectPath = "/" + BUCKET + "/" + key;+ URI uri = presign(key, "POST");++ String uploadId = given()+ .urlEncodingEnabled(false)+ .when()+ .post(uri.getRawPath() + "?uploads&x-amz-tagging=a%3Db%26c%3Dd&" + uri.getRawQuery())+ .then()+ .statusCode(200)+ .extract().xmlPath().getString("InitiateMultipartUploadResult.UploadId");++ String partETag = given()+ .body("part-one")+ .when()+ .put(objectPath + "?uploadId=" + uploadId + "&partNumber=1")+ .then()+ .statusCode(200)+ .extract().header("ETag");++ String completeXml = """+ <CompleteMultipartUpload>+ <Part><PartNumber>1</PartNumber><ETag>%s</ETag></Part>+ </CompleteMultipartUpload>""".formatted(partETag);++ given()+ .contentType("application/xml")+ .body(completeXml)+ .when()+ .post(objectPath + "?uploadId=" + uploadId)+ .then()+ .statusCode(200);++ given()+ .when()+ .get(objectPath + "?tagging")+ .then()+ .statusCode(200)+ .body(containsString("<Key>a</Key>"))+ .body(containsString("<Value>b</Value>"))+ .body(containsString("<Key>c</Key>"))+ .body(containsString("<Value>d</Value>"));+ }++ private static void createBucket() {+ given()+ .when()+ .put("/" + BUCKET)+ .then()+ .statusCode(200);+ }++ private URI presign(String key, String method) {+ String fullBaseUrl = "http://localhost:" + RestAssured.port;+ return URI.create(presignGenerator.generatePresignedUrl(fullBaseUrl, BUCKET, key, method, 3600));}}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3PresignedTaggingSigV4EnforcementIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3PresignedTaggingSigV4EnforcementIntegrationTest.javanew file mode 100644index 0000000000..551898f311--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3PresignedTaggingSigV4EnforcementIntegrationTest.java@@ -0,0 +1,261 @@+package io.github.hectorvent.floci.services.s3;++import io.github.hectorvent.floci.testutil.S3RequestSigner;+import io.quarkus.test.junit.QuarkusTest;+import io.quarkus.test.junit.TestProfile;+import io.restassured.RestAssured;+import org.junit.jupiter.api.Test;++import javax.crypto.Mac;+import javax.crypto.spec.SecretKeySpec;+import java.nio.charset.StandardCharsets;+import java.security.MessageDigest;+import java.time.Instant;+import java.time.ZoneOffset;+import java.time.format.DateTimeFormatter;+import java.util.Map;+import java.util.TreeMap;+import java.util.stream.Collectors;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.containsString;+import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.not;++/**+ * Companion to {@link S3PresignedPutTaggingIntegrationTest} for #3608, run with+ * {@code floci.services.s3.enforce-auth} on so {@link PreSignedUrlFilter} verifies the SigV4+ * signature. The URLs here are built the way SDK presigners build them: every query+ * parameter, {@code x-amz-tagging} included, is part of the canonical query string, and only+ * the headers listed in {@code X-Amz-SignedHeaders} are covered by the signature. This proves+ * a real presigned request carrying tags in the query passes verification and reaches the+ * query fallback, and that a tag parameter added after signing is rejected.+ */+@QuarkusTest+@TestProfile(S3AuthEnforcementIntegrationTest.S3AuthProfile.class)+class S3PresignedTaggingSigV4EnforcementIntegrationTest {++ private static final String BUCKET = "presigned-tagging-sigv4-bucket";+ private static final String ACCESS_KEY = "test";+ private static final String SECRET_KEY = "test";+ private static final S3RequestSigner LOCAL_SIGNER = S3RequestSigner.signedAs(ACCESS_KEY, SECRET_KEY);+ private static final DateTimeFormatter AMZ_DATE =+ DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss'Z'").withZone(ZoneOffset.UTC);++ @Test+ void presignedPutWithTaggingInSignedQueryAppliesTagsUnderEnforcedAuth() throws Exception {+ createBucket();+ String path = "/" + BUCKET + "/tagged-via-signed-query.txt";+ // Header form is URL-encoded (c=d%20e); the presigner encodes that whole value again as+ // a query parameter, so the wire carries x-amz-tagging=a%3Db%26c%3Dd%2520e.+ String url = presign("PUT", path, Map.of("x-amz-tagging", "a=b&c=d%20e"), Map.of());++ given()+ .urlEncodingEnabled(false)+ .contentType("text/plain")+ .body("uploaded via SigV4-presigned PUT with tagging in the query string")+ .when()+ .put(url)+ .then()+ .statusCode(200);++ given()+ .filter(LOCAL_SIGNER)+ .when()+ .get(path + "?tagging")+ .then()+ .statusCode(200)+ .body(containsString("<Key>a</Key>"))+ .body(containsString("<Value>b</Value>"))+ .body(containsString("<Key>c</Key>"))+ .body(containsString("<Value>d e</Value>"));+ }++ @Test+ void presignedPutWithTaggingAppendedAfterSigningIsRejectedUnderEnforcedAuth() throws Exception {+ createBucket();+ String path = "/" + BUCKET + "/tampered-tagging.txt";+ String url = presign("PUT", path, Map.of(), Map.of());++ given()+ .urlEncodingEnabled(false)+ .contentType("text/plain")+ .body("the tag parameter is not covered by the signature")+ .when()+ .put(url + "&x-amz-tagging=a%3Db")+ .then()+ .statusCode(403)+ .body("Error.Code", equalTo("SignatureDoesNotMatch"));+ }++ @Test+ void presignedPutWithSignedTaggingHeaderAndTaggingQueryParameterPrefersHeader() throws Exception {+ createBucket();+ String path = "/" + BUCKET + "/tagged-via-signed-header-and-query.txt";+ String url = presign("PUT", path,+ Map.of("x-amz-tagging", "query=1"),+ Map.of("x-amz-tagging", "header=1"));++ given()+ .urlEncodingEnabled(false)+ .contentType("text/plain")+ .header("x-amz-tagging", "header=1")+ .body("the signed header wins over the query parameter")+ .when()+ .put(url)+ .then()+ .statusCode(200);++ given()+ .filter(LOCAL_SIGNER)+ .when()+ .get(path + "?tagging")+ .then()+ .statusCode(200)+ .body(containsString("<Key>header</Key>"))+ .body(containsString("<Value>1</Value>"))+ .body(not(containsString("<Key>query</Key>")));+ }++ @Test+ void presignedCreateMultipartUploadWithTaggingInSignedQueryAppliesTagsUnderEnforcedAuth()+ throws Exception {+ createBucket();+ String path = "/" + BUCKET + "/tagged-multipart-via-signed-query.txt";+ String url = presign("POST", path, Map.of("uploads", "", "x-amz-tagging", "a=b&c=d"), Map.of());++ String uploadId = given()+ .urlEncodingEnabled(false)+ .when()+ .post(url)+ .then()+ .statusCode(200)+ .extract().xmlPath().getString("InitiateMultipartUploadResult.UploadId");++ String partETag = given()+ .filter(LOCAL_SIGNER)+ .body("part-one")+ .when()+ .put(path + "?uploadId=" + uploadId + "&partNumber=1")+ .then()+ .statusCode(200)+ .extract().header("ETag");++ String completeXml = """+ <CompleteMultipartUpload>+ <Part><PartNumber>1</PartNumber><ETag>%s</ETag></Part>+ </CompleteMultipartUpload>""".formatted(partETag);++ given()+ .filter(LOCAL_SIGNER)+ .contentType("application/xml")+ .body(completeXml)+ .when()+ .post(path + "?uploadId=" + uploadId)+ .then()+ .statusCode(200);++ given()+ .filter(LOCAL_SIGNER)+ .when()+ .get(path + "?tagging")+ .then()+ .statusCode(200)+ .body(containsString("<Key>a</Key>"))+ .body(containsString("<Value>b</Value>"))+ .body(containsString("<Key>c</Key>"))+ .body(containsString("<Value>d</Value>"));+ }++ private static void createBucket() {+ given()+ .filter(LOCAL_SIGNER)+ .when()+ .put("/" + BUCKET)+ .then()+ .statusCode(200);+ }++ /**+ * Builds a SigV4 query-string-authenticated URL like an SDK presigner: {@code queryParams}+ * holds decoded values that all become part of the canonical query string, and+ * {@code signedHeaders} lists the request headers, besides {@code host}, that the signature+ * covers. The payload is unsigned, as with every SDK-presigned S3 upload.+ */+ private static String presign(String method, String path, Map<String, String> queryParams,+ Map<String, String> signedHeaders) throws Exception {+ String amzDate = AMZ_DATE.format(Instant.now());+ String date = amzDate.substring(0, 8);+ String credentialScope = date + "/us-east-1/s3/aws4_request";++ TreeMap<String, String> headers = new TreeMap<>(signedHeaders);+ headers.put("host", "localhost:" + RestAssured.port);+ String signedHeaderNames = String.join(";", headers.keySet());++ TreeMap<String, String> params = new TreeMap<>(queryParams);+ params.put("X-Amz-Algorithm", "AWS4-HMAC-SHA256");+ params.put("X-Amz-Credential", ACCESS_KEY + "/" + credentialScope);+ params.put("X-Amz-Date", amzDate);+ params.put("X-Amz-Expires", "3600");+ params.put("X-Amz-SignedHeaders", signedHeaderNames);+ String canonicalQuery = params.entrySet().stream()+ .map(entry -> uriEncode(entry.getKey()) + "=" + uriEncode(entry.getValue()))+ .collect(Collectors.joining("&"));+ String canonicalHeaders = headers.entrySet().stream()+ .map(entry -> entry.getKey() + ":" + entry.getValue().trim() + "\n")+ .collect(Collectors.joining());++ String canonicalRequest = method + "\n"+ + path + "\n"+ + canonicalQuery + "\n"+ + canonicalHeaders + "\n"+ + signedHeaderNames + "\n"+ + "UNSIGNED-PAYLOAD";+ String stringToSign = "AWS4-HMAC-SHA256\n"+ + amzDate + "\n"+ + credentialScope + "\n"+ + sha256Hex(canonicalRequest);++ byte[] kDate = hmacSha256(("AWS4" + SECRET_KEY).getBytes(StandardCharsets.UTF_8), date);+ byte[] kRegion = hmacSha256(kDate, "us-east-1");+ byte[] kService = hmacSha256(kRegion, "s3");+ byte[] signingKey = hmacSha256(kService, "aws4_request");+ String signature = hexEncode(hmacSha256(signingKey, stringToSign));++ return path + "?" + canonicalQuery + "&X-Amz-Signature=" + signature;+ }++ /** RFC 3986 encoding as SigV4 requires it: only unreserved characters stay literal. */+ private static String uriEncode(String value) {+ StringBuilder encoded = new StringBuilder();+ for (byte b : value.getBytes(StandardCharsets.UTF_8)) {+ int c = b & 0xFF;+ if ((c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9')+ || c == '-' || c == '_' || c == '.' || c == '~') {+ encoded.append((char) c);+ } else {+ encoded.append('%').append(String.format("%02X", c));+ }+ }+ return encoded.toString();+ }++ private static byte[] hmacSha256(byte[] key, String data) throws Exception {+ Mac mac = Mac.getInstance("HmacSHA256");+ mac.init(new SecretKeySpec(key, "HmacSHA256"));+ return mac.doFinal(data.getBytes(StandardCharsets.UTF_8));+ }++ private static String sha256Hex(String input) throws Exception {+ MessageDigest digest = MessageDigest.getInstance("SHA-256");+ return hexEncode(digest.digest(input.getBytes(StandardCharsets.UTF_8)));+ }++ private static String hexEncode(byte[] bytes) {+ StringBuilder hex = new StringBuilder(bytes.length * 2);+ for (byte b : bytes) {+ hex.append(String.format("%02x", b));+ }+ return hex.toString();+ }+}