Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(s3): reject unsigned checksum headers on presigned requests (#4153)

Roberto Perez Alcolea343e1574fad2
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3PresignedUrlSigV4VerificationTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3PresignedUrlSigV4VerificationTest.java
index a326be67aa..57d027c00b 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3PresignedUrlSigV4VerificationTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/S3PresignedUrlSigV4VerificationTest.java
@@ -5,6 +5,8 @@ import org.junit.jupiter.api.Assumptions;
import org.junit.jupiter.api.BeforeAll;
import org.junit.jupiter.api.DisplayName;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.MethodSource;
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
import software.amazon.awssdk.core.sync.RequestBody;
@@ -13,18 +15,32 @@ import software.amazon.awssdk.services.iam.IamClient;
import software.amazon.awssdk.services.iam.model.CreateAccessKeyResponse;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.S3Configuration;
+import software.amazon.awssdk.services.s3.model.ChecksumAlgorithm;
import software.amazon.awssdk.services.s3.model.CreateBucketRequest;
+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadRequest;
+import software.amazon.awssdk.services.s3.model.CreateMultipartUploadResponse;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.model.S3Exception;
+import software.amazon.awssdk.services.s3.model.UploadPartRequest;
import software.amazon.awssdk.services.s3.presigner.S3Presigner;
import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest;
+import software.amazon.awssdk.services.s3.presigner.model.PresignedUploadPartRequest;
+import software.amazon.awssdk.services.s3.presigner.model.UploadPartPresignRequest;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.HttpURLConnection;
import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
+import java.nio.charset.StandardCharsets;
import java.time.Duration;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.stream.Stream;
import static org.assertj.core.api.Assertions.assertThat;
@@ -190,6 +206,86 @@ class S3PresignedUrlSigV4VerificationTest {
assertThat(r.body()).contains("InvalidAccessKeyId");
}
+ @ParameterizedTest(name = "{0}")
+ @MethodSource("checksumCases")
+ @DisplayName("unsigned checksum header on presigned multipart PUT is rejected")
+ void unsignedChecksumHeaderOnPresignedUploadPartIsRejected(ChecksumCase checksum) throws Exception {
+ assumeEnforcementEnabled();
+
+ String key = "unsigned-checksum-" + checksum.algorithm().toString().toLowerCase(Locale.ROOT) + ".bin";
+ CreateMultipartUploadResponse upload = s3.createMultipartUpload(CreateMultipartUploadRequest.builder()
+ .bucket(BUCKET)
+ .key(key)
+ .checksumAlgorithm(checksum.algorithm())
+ .build());
+ try {
+ PresignedUploadPartRequest unsignedChecksum;
+ PresignedUploadPartRequest signedChecksum;
+ try (S3Presigner presigner = presigner()) {
+ unsignedChecksum = presigner.presignUploadPart(UploadPartPresignRequest.builder()
+ .signatureDuration(Duration.ofMinutes(5))
+ .uploadPartRequest(UploadPartRequest.builder()
+ .bucket(BUCKET)
+ .key(key)
+ .uploadId(upload.uploadId())
+ .partNumber(1)
+ .build())
+ .build());
+ UploadPartRequest.Builder signedRequest = UploadPartRequest.builder()
+ .bucket(BUCKET)
+ .key(key)
+ .uploadId(upload.uploadId())
+ .partNumber(1);
+ signedChecksum = presigner.presignUploadPart(UploadPartPresignRequest.builder()
+ .signatureDuration(Duration.ofMinutes(5))
+ .uploadPartRequest(withChecksum(signedRequest, checksum))
+ .build());
+ }
+
+ HttpResult rejected = httpPut(unsignedChecksum,
+ Map.of(checksum.headerName(), checksum.value()), "123456789");
+ assertThat(rejected.status()).isEqualTo(403);
+ assertThat(rejected.body()).contains("<Code>AccessDenied</Code>");
+ assertThat(rejected.body()).contains(
+ "<Message>There were headers present in the request which were not signed</Message>");
+ assertThat(rejected.body()).contains(
+ "<HeadersNotSigned>" + checksum.headerName() + "</HeadersNotSigned>");
+
+ assertThat(signedChecksum.httpRequest().headers()).containsKey(checksum.headerName());
+ HttpResult accepted = httpPut(signedChecksum, Map.of(), "123456789");
+ assertThat(accepted.status()).isEqualTo(200);
+ } finally {
+ s3.abortMultipartUpload(r -> r.bucket(BUCKET).key(key).uploadId(upload.uploadId()));
+ }
+ }
+
+ private static Stream<ChecksumCase> checksumCases() {
+ return Stream.of(
+ new ChecksumCase(ChecksumAlgorithm.CRC32,
+ "x-amz-checksum-crc32", "y/Q5Jg=="),
+ new ChecksumCase(ChecksumAlgorithm.CRC32_C,
+ "x-amz-checksum-crc32c", "4waSgw=="),
+ new ChecksumCase(ChecksumAlgorithm.CRC64_NVME,
+ "x-amz-checksum-crc64nvme", "rosUhgp5mIg="),
+ new ChecksumCase(ChecksumAlgorithm.SHA1,
+ "x-amz-checksum-sha1", "98O8HYCOBHMq32eZZczDTKeuNEE="),
+ new ChecksumCase(ChecksumAlgorithm.SHA256,
+ "x-amz-checksum-sha256", "FeKw08M4keuw8e9gnsQZQgwg4yDOlMZfvIwzEkSOsiU="));
+ }
+
+ private static UploadPartRequest withChecksum(
+ UploadPartRequest.Builder request, ChecksumCase checksum) {
+ return switch (checksum.algorithm()) {
+ case CRC32 -> request.checksumCRC32(checksum.value()).build();
+ case CRC32_C -> request.checksumCRC32C(checksum.value()).build();
+ case CRC64_NVME -> request.checksumCRC64NVME(checksum.value()).build();
+ case SHA1 -> request.checksumSHA1(checksum.value()).build();
+ case SHA256 -> request.checksumSHA256(checksum.value()).build();
+ default -> throw new IllegalArgumentException(
+ "Unsupported checksum algorithm: " + checksum.algorithm());
+ };
+ }
+
private static boolean probeEnforcementEnabled() {
var unknownS3 = S3Client.builder()
.endpointOverride(TestFixtures.endpoint())
@@ -245,4 +341,43 @@ class S3PresignedUrlSigV4VerificationTest {
HttpRequest.newBuilder(URI.create(url)).GET().build(),
HttpResponse.BodyHandlers.ofString());
}
+
+ private static HttpResult httpPut(PresignedUploadPartRequest presigned,
+ Map<String, String> additionalHeaders,
+ String body) throws Exception {
+ HttpURLConnection connection = (HttpURLConnection) presigned.url().openConnection();
+ connection.setRequestMethod("PUT");
+ connection.setDoOutput(true);
+ for (Map.Entry<String, List<String>> entry : presigned.httpRequest().headers().entrySet()) {
+ for (String value : entry.getValue()) {
+ connection.addRequestProperty(entry.getKey(), value);
+ }
+ }
+ additionalHeaders.forEach(connection::addRequestProperty);
+
+ byte[] bytes = body.getBytes(StandardCharsets.UTF_8);
+ connection.setFixedLengthStreamingMode(bytes.length);
+ try (OutputStream output = connection.getOutputStream()) {
+ output.write(bytes);
+ }
+
+ int status = connection.getResponseCode();
+ InputStream responseStream = status >= 400 ? connection.getErrorStream() : connection.getInputStream();
+ if (responseStream == null) {
+ return new HttpResult(status, "");
+ }
+ try (InputStream input = responseStream) {
+ return new HttpResult(status, new String(input.readAllBytes(), StandardCharsets.UTF_8));
+ }
+ }
+
+ private record HttpResult(int status, String body) {
+ }
+
+ private record ChecksumCase(ChecksumAlgorithm algorithm, String headerName, String value) {
+ @Override
+ public String toString() {
+ return algorithm.toString();
+ }
+ }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilter.java b/src/main/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilter.java
index 40677ba4c0..826f98076a 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilter.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilter.java
@@ -20,7 +20,9 @@ import java.security.MessageDigest;
import java.util.ArrayList;
import java.util.Comparator;
import java.util.List;
+import java.util.Locale;
import java.util.Optional;
+import java.util.Set;
import java.util.stream.Collectors;
@Provider
@@ -29,6 +31,15 @@ public class PreSignedUrlFilter implements ContainerRequestFilter {
private static final Logger LOG = Logger.getLogger(PreSignedUrlFilter.class);
private static final String LEGACY_ACCESS_KEY_ID = "test";
private static final String LEGACY_SECRET_KEY = "test";
+ private static final Set<String> CHECKSUM_HEADERS_REQUIRING_SIGNATURE =
+ Set.of(
+ "x-amz-checksum-algorithm",
+ "x-amz-checksum-crc32",
+ "x-amz-checksum-crc32c",
+ "x-amz-checksum-crc64nvme",
+ "x-amz-checksum-sha1",
+ "x-amz-checksum-sha256",
+ "x-amz-sdk-checksum-algorithm");
private final PreSignedUrlGenerator presignGenerator;
private final S3Service s3Service;
@@ -135,6 +146,13 @@ public class PreSignedUrlFilter implements ContainerRequestFilter {
if (!verifySigV4Signature(requestContext, signature, secretKey)) {
requestContext.abortWith(errorResponse(403, "SignatureDoesNotMatch",
"The request signature we calculated does not match the signature you provided."));
+ return;
+ }
+
+ List<String> unsignedChecksumHeaders = unsignedChecksumHeaders(
+ requestContext.getHeaders().keySet(), queryParams.getFirst("X-Amz-SignedHeaders"));
+ if (!unsignedChecksumHeaders.isEmpty()) {
+ requestContext.abortWith(headersNotSignedResponse(unsignedChecksumHeaders));
}
} else if (presignGenerator.shouldValidateSignatures()) {
String path = requestContext.getUriInfo().getPath();
@@ -294,6 +312,21 @@ public class PreSignedUrlFilter implements ContainerRequestFilter {
return value == null ? "" : value.trim().replaceAll(" +", " ");
}
+ static List<String> unsignedChecksumHeaders(Set<String> requestHeaderNames, String signedHeaders) {
+ Set<String> normalizedSignedHeaders = List.of(signedHeaders.split(";"))
+ .stream()
+ .map(name -> name.toLowerCase(Locale.ROOT))
+ .collect(Collectors.toSet());
+
+ return requestHeaderNames.stream()
+ .map(name -> name.toLowerCase(Locale.ROOT))
+ .filter(CHECKSUM_HEADERS_REQUIRING_SIGNATURE::contains)
+ .filter(name -> !normalizedSignedHeaders.contains(name))
+ .distinct()
+ .sorted()
+ .toList();
+ }
+
static String awsUriEncode(String value) {
StringBuilder encoded = new StringBuilder();
for (byte b : value.getBytes(StandardCharsets.UTF_8)) {
@@ -319,4 +352,16 @@ public class PreSignedUrlFilter implements ContainerRequestFilter {
.build();
return Response.status(status).entity(xml).type(MediaType.APPLICATION_XML).build();
}
+
+ private static Response headersNotSignedResponse(List<String> headerNames) {
+ String xml = new XmlBuilder()
+ .raw("<?xml version=\"1.0\" encoding=\"UTF-8\"?>")
+ .start("Error")
+ .elem("Code", "AccessDenied")
+ .elem("Message", "There were headers present in the request which were not signed")
+ .elem("HeadersNotSigned", String.join(",", headerNames))
+ .end("Error")
+ .build();
+ return Response.status(403).entity(xml).type(MediaType.APPLICATION_XML).build();
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilterTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilterTest.java
index c449941eed..c6ef00ee9e 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilterTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilterTest.java
@@ -7,6 +7,8 @@ import jakarta.ws.rs.core.MultivaluedMap;
import org.junit.jupiter.api.Test;
import java.lang.reflect.Method;
+import java.util.List;
+import java.util.Set;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNull;
@@ -142,4 +144,49 @@ class PreSignedUrlFilterTest {
"X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Date=20260101T000000Z",
PreSignedUrlFilter.buildCanonicalQueryString(params));
}
+
+ @Test
+ void identifiesUnsignedChecksumHeadersCaseInsensitively() {
+ assertEquals(
+ List.of(
+ "x-amz-checksum-algorithm",
+ "x-amz-checksum-crc32",
+ "x-amz-checksum-crc32c",
+ "x-amz-checksum-crc64nvme",
+ "x-amz-checksum-sha1",
+ "x-amz-checksum-sha256",
+ "x-amz-sdk-checksum-algorithm"),
+ PreSignedUrlFilter.unsignedChecksumHeaders(
+ Set.of(
+ "Host",
+ "X-Amz-Checksum-Algorithm",
+ "X-Amz-Checksum-CRC32",
+ "X-Amz-Checksum-CRC32C",
+ "X-Amz-Checksum-CRC64NVME",
+ "X-Amz-Checksum-SHA1",
+ "X-Amz-Checksum-SHA256",
+ "X-Amz-SDK-Checksum-Algorithm"),
+ "host"));
+ }
+
+ @Test
+ void acceptsChecksumHeadersIncludedInSignedHeaders() {
+ assertEquals(
+ List.of(),
+ PreSignedUrlFilter.unsignedChecksumHeaders(
+ Set.of("host", "x-amz-checksum-algorithm", "x-amz-checksum-crc32",
+ "x-amz-checksum-sha256", "x-amz-sdk-checksum-algorithm"),
+ "host;x-amz-checksum-algorithm;x-amz-checksum-crc32;"
+ + "x-amz-checksum-sha256;x-amz-sdk-checksum-algorithm"));
+ }
+
+ @Test
+ void ignoresHeadersOutsideChecksumFamily() {
+ assertEquals(
+ List.of(),
+ PreSignedUrlFilter.unsignedChecksumHeaders(
+ Set.of("content-type", "user-agent", "x-amz-content-sha256",
+ "x-amz-user-agent", "x-amz-checksum-type"),
+ "host"));
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3AuthEnforcementIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3AuthEnforcementIntegrationTest.java
index d19d9f42d3..460bb655dc 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3AuthEnforcementIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3AuthEnforcementIntegrationTest.java
@@ -668,6 +668,49 @@ class S3AuthEnforcementIntegrationTest {
.body(containsString("SignatureDoesNotMatch"));
}
+ @Test
+ @Order(25)
+ void presignedPutWithUnsignedChecksumHeaderIsRejected() {
+ String bucket = "auth-presigned-checksum-bucket";
+ String key = "part.bin";
+ String path = "/" + bucket + "/" + key;
+ given().filter(LOCAL_SIGNER).when().put("/" + bucket).then().statusCode(200);
+
+ try {
+ String signature = presignedSignature("PUT", path, "test", "test", "3600");
+ Map<String, String> checksumHeaders = Map.of(
+ "x-amz-checksum-algorithm", "CRC32",
+ "x-amz-checksum-crc32", "y/Q5Jg==",
+ "x-amz-checksum-crc32c", "4waSgw==",
+ "x-amz-checksum-crc64nvme", "rosUhgp5mIg=",
+ "x-amz-checksum-sha1", "98O8HYCOBHMq32eZZczDTKeuNEE=",
+ "x-amz-checksum-sha256", "FeKw08M4keuw8e9gnsQZQgwg4yDOlMZfvIwzEkSOsiU=",
+ "x-amz-sdk-checksum-algorithm", "CRC32");
+ for (Map.Entry<String, String> checksum : checksumHeaders.entrySet()) {
+ presignedRequest(signature)
+ .header(checksum.getKey(), checksum.getValue())
+ .body("123456789")
+ .when()
+ .put(path)
+ .then()
+ .statusCode(403)
+ .body("Error.Code", equalTo("AccessDenied"))
+ .body("Error.Message", equalTo(
+ "There were headers present in the request which were not signed"))
+ .body("Error.HeadersNotSigned", equalTo(checksum.getKey()));
+ }
+
+ given()
+ .filter(LOCAL_SIGNER)
+ .when()
+ .get(path)
+ .then()
+ .statusCode(404);
+ } finally {
+ given().filter(LOCAL_SIGNER).when().delete("/" + bucket);
+ }
+ }
+
@Test
@Order(26)
void presignedRequestWithExpiresExceedingMaxIsRejected() {