Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3): verify SigV4 signatures against the wire path for leading-slash keys (#4125)
Matheus André2bf92a9beb2c
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilter.java b/src/main/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilter.javaindex 3db376e687..40677ba4c0 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilter.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilter.java@@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.s3;import io.github.hectorvent.floci.core.common.XmlBuilder;import io.github.hectorvent.floci.core.common.auth.SigV4RequestValidator;import io.github.hectorvent.floci.services.iam.IamService;+import io.quarkus.vertx.http.runtime.CurrentVertxRequest;import jakarta.inject.Inject;import jakarta.ws.rs.container.ContainerRequestContext;import jakarta.ws.rs.container.ContainerRequestFilter;@@ -32,14 +33,17 @@ public class PreSignedUrlFilter implements ContainerRequestFilter {private final PreSignedUrlGenerator presignGenerator;private final S3Service s3Service;private final IamService iamService;+ private final CurrentVertxRequest currentVertxRequest;@Injectpublic PreSignedUrlFilter(PreSignedUrlGenerator presignGenerator,S3Service s3Service,- IamService iamService) {+ IamService iamService,+ CurrentVertxRequest currentVertxRequest) {this.presignGenerator = presignGenerator;this.s3Service = s3Service;this.iamService = iamService;+ this.currentVertxRequest = currentVertxRequest;}@Override@@ -196,8 +200,9 @@ public class PreSignedUrlFilter implements ContainerRequestFilter {}}- // Canonical request- String path = requestUri.getRawPath();+ // Canonical request: the wire path, since a leading-slash key travels (and is+ // signed) as a double slash that JAX-RS would otherwise collapse.+ String path = S3HeaderSignatureFilter.signedPath(currentVertxRequest, requestUri);String canonicalQueryString = buildCanonicalQueryString(queryParams);String payloadHash = requestContext.getHeaderString("x-amz-content-sha256");if (payloadHash == null) {diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.javaindex 83508e5f75..77d2acbb5e 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java@@ -3760,16 +3760,21 @@ public class S3Controller {/*** True when the raw request path carries no bucket prefix because the bucket came from the* Host header: the rewritten request URI is then exactly "/" + bucket + rawPath, which is- * what {@link S3VirtualHostFilter} produces and what a path-style request never is.+ * what {@link S3VirtualHostFilter} produces and what a path-style request never is. JAX-RS+ * collapses consecutive slashes before the filter sees the path, so a leading-slash key+ * ({@code //file.txt} on the wire) is rewritten to {@code /<bucket>/file.txt}; the collapsed+ * form is compared as well so that key keeps its slash.*/private static boolean isVirtualHostedRawPath(UriInfo uriInfo, String bucket, String rawPath) {if (rawPath.isEmpty() || rawPath.charAt(0) != '/') {return false;}String rewritten = uriInfo.getRequestUri().getRawPath();- return rewritten != null- && !rewritten.equals(rawPath)- && rewritten.equals("/" + bucket + rawPath);+ if (rewritten == null || rewritten.equals(rawPath)) {+ return false;+ }+ return rewritten.equals("/" + bucket + rawPath)+ || rewritten.equals("/" + bucket + rawPath.replaceAll("/{2,}", "/"));}private void validateKeyNoTraversal(String key) {diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3HeaderSignatureFilter.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3HeaderSignatureFilter.javaindex ae8e7768e9..e18009cec5 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3HeaderSignatureFilter.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3HeaderSignatureFilter.java@@ -2,6 +2,8 @@ package io.github.hectorvent.floci.services.s3;import io.github.hectorvent.floci.core.common.auth.SigV4RequestValidator;import io.github.hectorvent.floci.services.iam.IamService;+import io.quarkus.vertx.http.runtime.CurrentVertxRequest;+import io.vertx.ext.web.RoutingContext;import jakarta.annotation.Priority;import jakarta.inject.Inject;import jakarta.ws.rs.Priorities;@@ -31,7 +33,7 @@ import java.util.Optional;* <p>The presigned placements are already verified elsewhere ({@link PreSignedUrlFilter} for the* query string, {@link S3PostPolicySigner} for a browser POST). This filter closes the remaining* placement, the one every AWS SDK uses for ordinary calls: the canonical request is rebuilt from- * the request as it arrived (raw path, canonical query string, the headers named in+ * the request as it arrived (the wire path, canonical query string, the headers named in* {@code SignedHeaders}, and the {@code x-amz-content-sha256} value the client signed), the* signature is derived with the key's secret, and the two are compared in constant time. When the* declared content hash is a real digest rather than an {@code UNSIGNED-PAYLOAD} or@@ -75,14 +77,17 @@ public class S3HeaderSignatureFilter implements ContainerRequestFilter {private final S3Service s3Service;private final IamService iamService;+ private final CurrentVertxRequest currentVertxRequest;@ContextResourceInfo resourceInfo;@Inject- public S3HeaderSignatureFilter(S3Service s3Service, IamService iamService) {+ public S3HeaderSignatureFilter(S3Service s3Service, IamService iamService,+ CurrentVertxRequest currentVertxRequest) {this.s3Service = s3Service;this.iamService = iamService;+ this.currentVertxRequest = currentVertxRequest;}@Override@@ -196,7 +201,7 @@ public class S3HeaderSignatureFilter implements ContainerRequestFilter {}String canonicalRequest = ctx.getMethod() + "\n"- + requestUri.getRawPath() + "\n"+ + signedPath(currentVertxRequest, requestUri) + "\n"+ PreSignedUrlFilter.buildCanonicalQueryString(ctx.getUriInfo().getQueryParameters()) + "\n"+ canonicalHeaders + "\n"+ signedHeaders + "\n"@@ -212,6 +217,25 @@ public class S3HeaderSignatureFilter implements ContainerRequestFilter {signature.getBytes(StandardCharsets.UTF_8));}+ /**+ * The path the client signed: the request line exactly as it arrived, before JAX-RS collapsed+ * consecutive slashes in {@code UriInfo}. S3 never normalizes the canonical URI, so a key with+ * a leading slash ({@code /email.txt}) is sent, and signed, as {@code /bucket//email.txt}+ * (or {@code //email.txt} virtual-hosted). Verifying against the normalized path would reject+ * every such request and, conversely, let a signature over {@code /bucket/email.txt} pass for+ * the other object. Falls back to {@code requestUri} when no Vert.x request is current.+ */+ static String signedPath(CurrentVertxRequest currentVertxRequest, URI requestUri) {+ RoutingContext routingContext = currentVertxRequest != null ? currentVertxRequest.getCurrent() : null;+ if (routingContext != null && routingContext.request() != null) {+ String path = routingContext.request().path();+ if (path != null && !path.isEmpty()) {+ return path;+ }+ }+ return requestUri.getRawPath();+ }+/*** Hashes the entity and hands it back to the request untouched. The S3 controllers already* receive the body as a {@code byte[]}, so buffering it here costs nothing extra. The streamdiff --git a/src/test/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilterTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilterTest.javaindex 2c6619e0a1..c449941eed 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilterTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/PreSignedUrlFilterTest.java@@ -38,7 +38,7 @@ class PreSignedUrlFilterTest {@Testvoid resolveSecretKeyRejectsUnregisteredNumericAccessKeyId() throws Exception {IamService iamService = IamServiceTestHelper.iamServiceWithAccessKey("AKIDUNRELATED", "unrelated-secret");- PreSignedUrlFilter filter = new PreSignedUrlFilter(null, null, iamService);+ PreSignedUrlFilter filter = new PreSignedUrlFilter(null, null, iamService, null);assertNull(resolveSecretKey(filter, "123456789012"));}@@ -47,7 +47,7 @@ class PreSignedUrlFilterTest {void temporaryCredentialRequiresMatchingSessionToken() throws Exception {IamService iamService = IamServiceTestHelper.iamServiceWithSessionCredential("ASIAS3EXAMPLE", "temporary-secret", "issued-token", java.time.Instant.now().plusSeconds(3600));- PreSignedUrlFilter filter = new PreSignedUrlFilter(null, null, iamService);+ PreSignedUrlFilter filter = new PreSignedUrlFilter(null, null, iamService, null);assertEquals("temporary-secret", resolveSecretKey(filter, "ASIAS3EXAMPLE", "issued-token"));assertNull(resolveSecretKey(filter, "ASIAS3EXAMPLE", null));@@ -58,7 +58,7 @@ class PreSignedUrlFilterTest {void temporaryCredentialIsRejectedAfterExpiration() throws Exception {IamService iamService = IamServiceTestHelper.iamServiceWithSessionCredential("ASIAS3EXPIRED", "temporary-secret", "issued-token", java.time.Instant.now().minusSeconds(1));- PreSignedUrlFilter filter = new PreSignedUrlFilter(null, null, iamService);+ PreSignedUrlFilter filter = new PreSignedUrlFilter(null, null, iamService, null);assertNull(resolveSecretKey(filter, "ASIAS3EXPIRED", "issued-token"));}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3LeadingSlashKeyAuthEnforcementIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3LeadingSlashKeyAuthEnforcementIntegrationTest.javanew file mode 100644index 0000000000..c8dda1c00c--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3LeadingSlashKeyAuthEnforcementIntegrationTest.java@@ -0,0 +1,264 @@+package io.github.hectorvent.floci.services.s3;++import io.github.hectorvent.floci.testutil.S3RequestSigner;+import io.quarkus.test.junit.QuarkusTest;+import io.quarkus.test.junit.TestProfile;+import io.restassured.RestAssured;+import org.junit.jupiter.api.MethodOrderer;+import org.junit.jupiter.api.Order;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.TestMethodOrder;++import java.io.ByteArrayOutputStream;+import java.io.InputStream;+import java.io.OutputStream;+import java.net.Socket;+import java.net.URI;+import java.net.URLEncoder;+import java.net.http.HttpClient;+import java.net.http.HttpRequest;+import java.net.http.HttpResponse;+import java.nio.charset.StandardCharsets;+import java.security.MessageDigest;+import java.time.Instant;+import java.time.ZoneOffset;+import java.time.format.DateTimeFormatter;+import java.util.LinkedHashMap;+import java.util.Map;+import javax.crypto.Mac;+import javax.crypto.spec.SecretKeySpec;++import static io.restassured.RestAssured.given;+import static org.hamcrest.MatcherAssert.assertThat;+import static org.hamcrest.Matchers.containsString;+import static org.hamcrest.Matchers.equalTo;+import static org.junit.jupiter.api.Assertions.assertEquals;++/**+ * A key that starts with a slash ({@code /email.txt}) travels as a double slash on the wire+ * ({@code PUT /bucket//email.txt}, or {@code PUT //email.txt} virtual-hosted), and the client+ * signs that wire path verbatim: S3 never normalizes the canonical URI. JAX-RS collapses the+ * consecutive slashes in {@code UriInfo}, so the signature filters have to verify against the+ * path as it arrived, not the normalized one, or every such request is rejected with+ * {@code SignatureDoesNotMatch} under {@code enforce-auth}.+ *+ * <p>RestAssured normalizes double slashes too, so the requests here go through+ * {@link HttpClient} (path-style) or a raw socket (virtual-hosted, where the {@code Host}+ * header must be set by hand).+ */+@QuarkusTest+@TestProfile(S3AuthEnforcementIntegrationTest.S3AuthProfile.class)+@TestMethodOrder(MethodOrderer.OrderAnnotation.class)+class S3LeadingSlashKeyAuthEnforcementIntegrationTest {++ private static final String BUCKET = "auth-leading-slash-bucket";+ private static final String KEY = "/email.txt";+ private static final String VHOST_KEY = "/vhost.txt";+ private static final S3RequestSigner LOCAL_SIGNER = S3RequestSigner.signedAs("test", "test");+ private static final DateTimeFormatter AMZ_DATE =+ DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmmss'Z'").withZone(ZoneOffset.UTC);++ @Test+ @Order(1)+ void createBucket() {+ given().filter(LOCAL_SIGNER).when().put("/" + BUCKET).then().statusCode(200);+ }++ @Test+ @Order(2)+ void headerSignedPathStyleRequestAcceptsLeadingSlashKey() throws Exception {+ URI uri = URI.create(base() + "/" + BUCKET + "/" + KEY);+ byte[] body = "slash body".getBytes(StandardCharsets.UTF_8);++ HttpResponse<String> put = send("PUT", uri, body, LOCAL_SIGNER);+ assertEquals(200, put.statusCode(), put.body());++ HttpResponse<String> get = send("GET", uri, new byte[0], LOCAL_SIGNER);+ assertEquals(200, get.statusCode(), get.body());+ assertEquals("slash body", get.body());++ given()+ .filter(LOCAL_SIGNER)+ .when()+ .get("/" + BUCKET + "?list-type=2")+ .then()+ .statusCode(200)+ .body(containsString("<Key>" + KEY + "</Key>"));+ }++ @Test+ @Order(3)+ void headerSignedRequestSignedForNormalizedPathIsStillRejected() throws Exception {+ // A signature over "/bucket/email.txt" must not verify a request for "/bucket//email.txt":+ // the two are different objects, and S3 signs the wire path verbatim.+ URI signedUri = URI.create(base() + "/" + BUCKET + "/email.txt");+ URI sentUri = URI.create(base() + "/" + BUCKET + "/" + KEY);+ byte[] body = "forged".getBytes(StandardCharsets.UTF_8);++ HttpRequest.Builder request = HttpRequest.newBuilder(sentUri)+ .PUT(HttpRequest.BodyPublishers.ofByteArray(body));+ LOCAL_SIGNER.headersFor("PUT", signedUri, body).forEach(request::header);+ HttpResponse<String> response = HttpClient.newHttpClient()+ .send(request.build(), HttpResponse.BodyHandlers.ofString());++ assertEquals(403, response.statusCode(), response.body());+ assertThat(response.body(), containsString("SignatureDoesNotMatch"));+ }++ @Test+ @Order(4)+ void headerSignedVirtualHostedRequestAcceptsLeadingSlashKey() throws Exception {+ String authority = BUCKET + ".localhost:" + RestAssured.port;+ URI uri = URI.create("http://" + authority + "/" + VHOST_KEY);+ byte[] body = "vhost body".getBytes(StandardCharsets.UTF_8);++ RawResponse put = sendRaw("PUT", uri, body, LOCAL_SIGNER.headersFor("PUT", uri, body));+ assertEquals(200, put.status(), put.body());++ RawResponse get = sendRaw("GET", uri, new byte[0], LOCAL_SIGNER.headersFor("GET", uri, new byte[0]));+ assertEquals(200, get.status(), get.body());+ assertEquals("vhost body", get.body());++ // The key keeps its leading slash: "//vhost.txt" on the wire is "/vhost.txt", not "vhost.txt".+ given()+ .filter(LOCAL_SIGNER)+ .when()+ .get("/" + BUCKET + "?list-type=2")+ .then()+ .statusCode(200)+ .body(containsString("<Key>" + VHOST_KEY + "</Key>"));+ }++ @Test+ @Order(5)+ void presignedPathStyleRequestAcceptsLeadingSlashKey() throws Exception {+ String path = "/" + BUCKET + "/" + KEY;+ String timestamp = AMZ_DATE.format(Instant.now());+ String query = presignedQuery("GET", path, "localhost:" + RestAssured.port, timestamp);+ URI uri = URI.create(base() + path + "?" + query);++ HttpResponse<String> response = HttpClient.newHttpClient()+ .send(HttpRequest.newBuilder(uri).GET().build(), HttpResponse.BodyHandlers.ofString());++ assertEquals(200, response.statusCode(), response.body());+ assertEquals("slash body", response.body());+ }++ @Test+ @Order(6)+ void presignedVirtualHostedRequestAcceptsLeadingSlashKey() throws Exception {+ String authority = BUCKET + ".localhost:" + RestAssured.port;+ String path = "/" + VHOST_KEY;+ String timestamp = AMZ_DATE.format(Instant.now());+ String query = presignedQuery("GET", path, authority, timestamp);+ URI uri = URI.create("http://" + authority + path + "?" + query);++ RawResponse response = sendRaw("GET", uri, new byte[0], Map.of());++ assertEquals(200, response.status(), response.body());+ assertEquals("vhost body", response.body());+ }++ @Test+ @Order(7)+ void cleanup() throws Exception {+ HttpResponse<String> delete = send("DELETE", URI.create(base() + "/" + BUCKET + "/" + KEY),+ new byte[0], LOCAL_SIGNER);+ assertEquals(204, delete.statusCode(), delete.body());+ HttpResponse<String> deleteVhost = send("DELETE", URI.create(base() + "/" + BUCKET + "/" + VHOST_KEY),+ new byte[0], LOCAL_SIGNER);+ assertEquals(204, deleteVhost.statusCode(), deleteVhost.body());+ given().filter(LOCAL_SIGNER).when().delete("/" + BUCKET).then().statusCode(204);+ }++ private static String base() {+ return "http://localhost:" + RestAssured.port;+ }++ private static HttpResponse<String> send(String method, URI uri, byte[] body, S3RequestSigner signer)+ throws Exception {+ HttpRequest.Builder request = HttpRequest.newBuilder(uri)+ .method(method, HttpRequest.BodyPublishers.ofByteArray(body));+ signer.headersFor(method, uri, body).forEach(request::header);+ return HttpClient.newHttpClient().send(request.build(), HttpResponse.BodyHandlers.ofString());+ }++ /**+ * Sends the request over a plain socket so the {@code Host} header can name the bucket+ * (HttpClient refuses to set it) and the double slash reaches the server untouched.+ */+ private static RawResponse sendRaw(String method, URI uri, byte[] body, Map<String, String> headers)+ throws Exception {+ String target = uri.getRawPath() + (uri.getRawQuery() != null ? "?" + uri.getRawQuery() : "");+ StringBuilder head = new StringBuilder()+ .append(method).append(' ').append(target).append(" HTTP/1.1\r\n")+ .append("Host: ").append(uri.getHost()).append(':').append(uri.getPort()).append("\r\n")+ .append("Connection: close\r\n")+ .append("Content-Length: ").append(body.length).append("\r\n");+ headers.forEach((name, value) -> head.append(name).append(": ").append(value).append("\r\n"));+ head.append("\r\n");++ try (Socket socket = new Socket("localhost", RestAssured.port)) {+ OutputStream out = socket.getOutputStream();+ out.write(head.toString().getBytes(StandardCharsets.ISO_8859_1));+ out.write(body);+ out.flush();+ InputStream in = socket.getInputStream();+ ByteArrayOutputStream buffer = new ByteArrayOutputStream();+ in.transferTo(buffer);+ String raw = buffer.toString(StandardCharsets.UTF_8);+ int status = Integer.parseInt(raw.substring(9, 12));+ int bodyStart = raw.indexOf("\r\n\r\n");+ return new RawResponse(status, bodyStart >= 0 ? raw.substring(bodyStart + 4) : "");+ }+ }++ private record RawResponse(int status, String body) {+ }++ private static String presignedQuery(String method, String path, String authority, String timestamp)+ throws Exception {+ String date = timestamp.substring(0, 8);+ String credentialScope = date + "/us-east-1/s3/aws4_request";+ Map<String, String> params = new LinkedHashMap<>();+ params.put("X-Amz-Algorithm", "AWS4-HMAC-SHA256");+ params.put("X-Amz-Credential", URLEncoder.encode("test/" + credentialScope, StandardCharsets.UTF_8));+ params.put("X-Amz-Date", timestamp);+ params.put("X-Amz-Expires", "3600");+ params.put("X-Amz-SignedHeaders", "host");+ String canonicalQuery = params.entrySet().stream()+ .map(e -> e.getKey() + "=" + e.getValue())+ .reduce((a, b) -> a + "&" + b).orElseThrow();++ String canonicalRequest = method + "\n"+ + path + "\n"+ + canonicalQuery + "\n"+ + "host:" + authority + "\n\n"+ + "host\n"+ + "UNSIGNED-PAYLOAD";+ String stringToSign = "AWS4-HMAC-SHA256\n"+ + timestamp + "\n"+ + credentialScope + "\n"+ + hex(MessageDigest.getInstance("SHA-256").digest(canonicalRequest.getBytes(StandardCharsets.UTF_8)));++ byte[] kDate = hmac(("AWS4test").getBytes(StandardCharsets.UTF_8), date);+ byte[] kRegion = hmac(kDate, "us-east-1");+ byte[] kService = hmac(kRegion, "s3");+ byte[] signingKey = hmac(kService, "aws4_request");+ return canonicalQuery + "&X-Amz-Signature=" + hex(hmac(signingKey, stringToSign));+ }++ private static byte[] hmac(byte[] key, String data) throws Exception {+ Mac mac = Mac.getInstance("HmacSHA256");+ mac.init(new SecretKeySpec(key, "HmacSHA256"));+ return mac.doFinal(data.getBytes(StandardCharsets.UTF_8));+ }++ private static String hex(byte[] bytes) {+ StringBuilder hex = new StringBuilder(bytes.length * 2);+ for (byte b : bytes) {+ hex.append(String.format("%02x", b));+ }+ return hex.toString();+ }+}