Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3): reject unsigned checksum headers on presigned requests (#4153)
Roberto Perez Alcolea343e1574fad2
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3PresignedUrlSigV4VerificationTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3PresignedUrlSigV4VerificationTest.javaindex a326be67aa..57d027c00b 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3PresignedUrlSigV4VerificationTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3PresignedUrlSigV4VerificationTest.java@@ -5,6 +5,8 @@ import org.junit.jupiter.api.Assumptions;import org.junit.jupiter.api.BeforeAll;import org.junit.jupiter.api.DisplayName;import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.MethodSource;import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;import software.amazon.awssdk.core.sync.RequestBody;@@ -13,18 +15,32 @@ import software.amazon.awssdk.services.iam.IamClient;import software.amazon.awssdk.services.iam.model.CreateAccessKeyResponse;import software.amazon.awssdk.services.s3.S3Client;import software.amazon.awssdk.services.s3.S3Configuration;+import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm;import software.amazon.awssdk.services.s3.model.CreateBucketRequest;+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse;import software.amazon.awssdk.services.s3.model.GetObjectRequest;import software.amazon.awssdk.services.s3.model.PutObjectRequest;import software.amazon.awssdk.services.s3.model.S3Exception;+import software.amazon.awssdk.services.s3.model.UploadPartRequest;import software.amazon.awssdk.services.s3.presigner.S3Presigner;import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest;+import software.amazon.awssdk.services.s3.presigner.model.PresignedUploadPartRequest;+import software.amazon.awssdk.services.s3.presigner.model.UploadPartPresignRequest;+import java.io.InputStream;+import java.io.OutputStream;+import java.net.HttpURLConnection;import java.net.URI;import java.net.http.HttpClient;import java.net.http.HttpRequest;import java.net.http.HttpResponse;+import java.nio.charset.StandardCharsets;import java.time.Duration;+import java.util.List;+import java.util.Locale;+import java.util.Map;+import java.util.stream.Stream;import static org.assertj.core.api.Assertions.assertThat;@@ -190,6 +206,86 @@ class S3PresignedUrlSigV4VerificationTest {assertThat(r.body()).contains("InvalidAccessKeyId");}+ @ParameterizedTest(name = "{0}")+ @MethodSource("checksumCases")+ @DisplayName("unsigned checksum header on presigned multipart PUT is rejected")+ void unsignedChecksumHeaderOnPresignedUploadPartIsRejected(ChecksumCase checksum) throws Exception {+ assumeEnforcementEnabled();++ String key = "unsigned-checksum-" + checksum.algorithm().toString().toLowerCase(Locale.ROOT) + ".bin";+ CreateMultipartUploadResponse upload = s3.createMultipartUpload(CreateMultipartUploadRequest.builder()+ .bucket(BUCKET)+ .key(key)+ .checksumAlgorithm(checksum.algorithm())+ .build());+ try {+ PresignedUploadPartRequest unsignedChecksum;+ PresignedUploadPartRequest signedChecksum;+ try (S3Presigner presigner = presigner()) {+ unsignedChecksum = presigner.presignUploadPart(UploadPartPresignRequest.builder()+ .signatureDuration(Duration.ofMinutes(5))+ .uploadPartRequest(UploadPartRequest.builder()+ .bucket(BUCKET)+ .key(key)+ .uploadId(upload.uploadId())+ .partNumber(1)+ .build())+ .build());+ UploadPartRequest.Builder signedRequest = UploadPartRequest.builder()+ .bucket(BUCKET)+ .key(key)+ .uploadId(upload.uploadId())+ .partNumber(1);+ signedChecksum = presigner.presignUploadPart(UploadPartPresignRequest.builder()+ .signatureDuration(Duration.ofMinutes(5))+ .uploadPartRequest(withChecksum(signedRequest, checksum))+ .build());+ }++ HttpResult rejected = httpPut(unsignedChecksum,+ Map.of(checksum.headerName(), checksum.value()), "123456789");+ assertThat(rejected.status()).isEqualTo(403);+ assertThat(rejected.body()).contains("<Code>AccessDenied</Code>");+ assertThat(rejected.body()).contains(+ "<Message>There were headers present in the request which were not signed</Message>");+ assertThat(rejected.body()).contains(+ "<HeadersNotSigned>" + checksum.headerName() + "</HeadersNotSigned>");++ assertThat(signedChecksum.httpRequest().headers()).containsKey(checksum.headerName());+ HttpResult accepted = httpPut(signedChecksum, Map.of(), "123456789");+ assertThat(accepted.status()).isEqualTo(200);+ } finally {+ s3.abortMultipartUpload(r -> r.bucket(BUCKET).key(key).uploadId(upload.uploadId()));+ }+ }++ private static Stream<ChecksumCase> checksumCases() {+ return Stream.of(+ new ChecksumCase(ChecksumAlgorithm.CRC32,+ "x-amz-checksum-crc32", "y/Q5Jg=="),+ new ChecksumCase(ChecksumAlgorithm.CRC32_C,+ "x-amz-checksum-crc32c", "4waSgw=="),+ new ChecksumCase(ChecksumAlgorithm.CRC64_NVME,+ "x-amz-checksum-crc64nvme", "rosUhgp5mIg="),+ new ChecksumCase(ChecksumAlgorithm.SHA1,+ "x-amz-checksum-sha1", "98O8HYCOBHMq32eZZczDTKeuNEE="),+ new ChecksumCase(ChecksumAlgorithm.SHA256,+ "x-amz-checksum-sha256", "FeKw08M4keuw8e9gnsQZQgwg4yDOlMZfvIwzEkSOsiU="));+ }++ private static UploadPartRequest withChecksum(+ UploadPartRequest.Builder request, ChecksumCase checksum) {+ return switch (checksum.algorithm()) {+ case CRC32 -> request.checksumCRC32(checksum.value()).build();+ case CRC32_C -> request.checksumCRC32C(checksum.value()).build();+ case CRC64_NVME -> request.checksumCRC64NVME(checksum.value()).build();+ case SHA1 -> request.checksumSHA1(checksum.value()).build();+ case SHA256 -> request.checksumSHA256(checksum.value()).build();+ default -> throw new IllegalArgumentException(+ "Unsupported checksum algorithm: " + checksum.algorithm());+ };+ }+private static boolean probeEnforcementEnabled() {var unknownS3 = S3Client.builder().endpointOverride(TestFixtures.endpoint())@@ -245,4 +341,43 @@ class S3PresignedUrlSigV4VerificationTest {HttpRequest.newBuilder(URI.create(url)).GET().build(),HttpResponse.BodyHandlers.ofString());}++ private static HttpResult httpPut(PresignedUploadPartRequest presigned,+ Map<String, String> additionalHeaders,+ String body) throws Exception {+ HttpURLConnection connection = (HttpURLConnection) presigned.url().openConnection();+ connection.setRequestMethod("PUT");+ connection.setDoOutput(true);+ for (Map.Entry<String, List<String>> entry : presigned.httpRequest().headers().entrySet()) {+ for (String value : entry.getValue()) {+ connection.addRequestProperty(entry.getKey(), value);+ }+ }+ additionalHeaders.forEach(connection::addRequestProperty);++ byte[] bytes = body.getBytes(StandardCharsets.UTF_8);+ connection.setFixedLengthStreamingMode(bytes.length);+ try (OutputStream output = connection.getOutputStream()) {+ output.write(bytes);+ }++ int status = connection.getResponseCode();+ InputStream responseStream = status >= 400 ? connection.getErrorStream() : connection.getInputStream();+ if (responseStream == null) {+ return new HttpResult(status, "");+ }+ try (InputStream input = responseStream) {+ return new HttpResult(status, new String(input.readAllBytes(), StandardCharsets.UTF_8));+ }+ }++ private record HttpResult(int status, String body) {+ }++ private record ChecksumCase(ChecksumAlgorithm algorithm, String headerName, String value) {+ @Override+ public String toString() {+ return algorithm.toString();+ }+ }}diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilter.java b/src/main/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilter.javaindex 40677ba4c0..826f98076a 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilter.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilter.java@@ -20,7 +20,9 @@ import java.security.MessageDigest;import java.util.ArrayList;import java.util.Comparator;import java.util.List;+import java.util.Locale;import java.util.Optional;+import java.util.Set;import java.util.stream.Collectors;@Provider@@ -29,6 +31,15 @@ public class PreSignedUrlFilter implements ContainerRequestFilter {private static final Logger LOG = Logger.getLogger(PreSignedUrlFilter.class);private static final String LEGACY_ACCESS_KEY_ID = "test";private static final String LEGACY_SECRET_KEY = "test";+ private static final Set<String> CHECKSUM_HEADERS_REQUIRING_SIGNATURE =+ Set.of(+ "x-amz-checksum-algorithm",+ "x-amz-checksum-crc32",+ "x-amz-checksum-crc32c",+ "x-amz-checksum-crc64nvme",+ "x-amz-checksum-sha1",+ "x-amz-checksum-sha256",+ "x-amz-sdk-checksum-algorithm");private final PreSignedUrlGenerator presignGenerator;private final S3Service s3Service;@@ -135,6 +146,13 @@ public class PreSignedUrlFilter implements ContainerRequestFilter {if (!verifySigV4Signature(requestContext, signature, secretKey)) {requestContext.abortWith(errorResponse(403, "SignatureDoesNotMatch","The request signature we calculated does not match the signature you provided."));+ return;+ }++ List<String> unsignedChecksumHeaders = unsignedChecksumHeaders(+ requestContext.getHeaders().keySet(), queryParams.getFirst("X-Amz-SignedHeaders"));+ if (!unsignedChecksumHeaders.isEmpty()) {+ requestContext.abortWith(headersNotSignedResponse(unsignedChecksumHeaders));}} else if (presignGenerator.shouldValidateSignatures()) {String path = requestContext.getUriInfo().getPath();@@ -294,6 +312,21 @@ public class PreSignedUrlFilter implements ContainerRequestFilter {return value == null ? "" : value.trim().replaceAll(" +", " ");}+ static List<String> unsignedChecksumHeaders(Set<String> requestHeaderNames, String signedHeaders) {+ Set<String> normalizedSignedHeaders = List.of(signedHeaders.split(";"))+ .stream()+ .map(name -> name.toLowerCase(Locale.ROOT))+ .collect(Collectors.toSet());++ return requestHeaderNames.stream()+ .map(name -> name.toLowerCase(Locale.ROOT))+ .filter(CHECKSUM_HEADERS_REQUIRING_SIGNATURE::contains)+ .filter(name -> !normalizedSignedHeaders.contains(name))+ .distinct()+ .sorted()+ .toList();+ }+static String awsUriEncode(String value) {StringBuilder encoded = new StringBuilder();for (byte b : value.getBytes(StandardCharsets.UTF_8)) {@@ -319,4 +352,16 @@ public class PreSignedUrlFilter implements ContainerRequestFilter {.build();return Response.status(status).entity(xml).type(MediaType.APPLICATION_XML).build();}++ private static Response headersNotSignedResponse(List<String> headerNames) {+ String xml = new XmlBuilder()+ .raw("<?xml version=\"1.0\" encoding=\"UTF-8\"?>")+ .start("Error")+ .elem("Code", "AccessDenied")+ .elem("Message", "There were headers present in the request which were not signed")+ .elem("HeadersNotSigned", String.join(",", headerNames))+ .end("Error")+ .build();+ return Response.status(403).entity(xml).type(MediaType.APPLICATION_XML).build();+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilterTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilterTest.javaindex c449941eed..c6ef00ee9e 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilterTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilterTest.java@@ -7,6 +7,8 @@ import jakarta.ws.rs.core.MultivaluedMap;import org.junit.jupiter.api.Test;import java.lang.reflect.Method;+import java.util.List;+import java.util.Set;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertNull;@@ -142,4 +144,49 @@ class PreSignedUrlFilterTest {"X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20260101T000000Z",PreSignedUrlFilter.buildCanonicalQueryString(params));}++ @Test+ void identifiesUnsignedChecksumHeadersCaseInsensitively() {+ assertEquals(+ List.of(+ "x-amz-checksum-algorithm",+ "x-amz-checksum-crc32",+ "x-amz-checksum-crc32c",+ "x-amz-checksum-crc64nvme",+ "x-amz-checksum-sha1",+ "x-amz-checksum-sha256",+ "x-amz-sdk-checksum-algorithm"),+ PreSignedUrlFilter.unsignedChecksumHeaders(+ Set.of(+ "Host",+ "X-Amz-Checksum-Algorithm",+ "X-Amz-Checksum-CRC32",+ "X-Amz-Checksum-CRC32C",+ "X-Amz-Checksum-CRC64NVME",+ "X-Amz-Checksum-SHA1",+ "X-Amz-Checksum-SHA256",+ "X-Amz-SDK-Checksum-Algorithm"),+ "host"));+ }++ @Test+ void acceptsChecksumHeadersIncludedInSignedHeaders() {+ assertEquals(+ List.of(),+ PreSignedUrlFilter.unsignedChecksumHeaders(+ Set.of("host", "x-amz-checksum-algorithm", "x-amz-checksum-crc32",+ "x-amz-checksum-sha256", "x-amz-sdk-checksum-algorithm"),+ "host;x-amz-checksum-algorithm;x-amz-checksum-crc32;"+ + "x-amz-checksum-sha256;x-amz-sdk-checksum-algorithm"));+ }++ @Test+ void ignoresHeadersOutsideChecksumFamily() {+ assertEquals(+ List.of(),+ PreSignedUrlFilter.unsignedChecksumHeaders(+ Set.of("content-type", "user-agent", "x-amz-content-sha256",+ "x-amz-user-agent", "x-amz-checksum-type"),+ "host"));+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3AuthEnforcementIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3AuthEnforcementIntegrationTest.javaindex d19d9f42d3..460bb655dc 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3AuthEnforcementIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3AuthEnforcementIntegrationTest.java@@ -668,6 +668,49 @@ class S3AuthEnforcementIntegrationTest {.body(containsString("SignatureDoesNotMatch"));}+ @Test+ @Order(25)+ void presignedPutWithUnsignedChecksumHeaderIsRejected() {+ String bucket = "auth-presigned-checksum-bucket";+ String key = "part.bin";+ String path = "/" + bucket + "/" + key;+ given().filter(LOCAL_SIGNER).when().put("/" + bucket).then().statusCode(200);++ try {+ String signature = presignedSignature("PUT", path, "test", "test", "3600");+ Map<String, String> checksumHeaders = Map.of(+ "x-amz-checksum-algorithm", "CRC32",+ "x-amz-checksum-crc32", "y/Q5Jg==",+ "x-amz-checksum-crc32c", "4waSgw==",+ "x-amz-checksum-crc64nvme", "rosUhgp5mIg=",+ "x-amz-checksum-sha1", "98O8HYCOBHMq32eZZczDTKeuNEE=",+ "x-amz-checksum-sha256", "FeKw08M4keuw8e9gnsQZQgwg4yDOlMZfvIwzEkSOsiU=",+ "x-amz-sdk-checksum-algorithm", "CRC32");+ for (Map.Entry<String, String> checksum : checksumHeaders.entrySet()) {+ presignedRequest(signature)+ .header(checksum.getKey(), checksum.getValue())+ .body("123456789")+ .when()+ .put(path)+ .then()+ .statusCode(403)+ .body("Error.Code", equalTo("AccessDenied"))+ .body("Error.Message", equalTo(+ "There were headers present in the request which were not signed"))+ .body("Error.HeadersNotSigned", equalTo(checksum.getKey()));+ }++ given()+ .filter(LOCAL_SIGNER)+ .when()+ .get(path)+ .then()+ .statusCode(404);+ } finally {+ given().filter(LOCAL_SIGNER).when().delete("/" + bucket);+ }+ }+@Test@Order(26)void presignedRequestWithExpiresExceedingMaxIsRejected() {