Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3): reject a PutObject or UploadPart over 5 GiB (#5469)
Priamo Germosen6d2aaba6a6e7
diff --git a/docs/services/s3.md b/docs/services/s3.mdindex 9ee0ccc1b2..160771352f 100644--- a/docs/services/s3.md+++ b/docs/services/s3.md@@ -412,10 +412,20 @@ way, and assemble a multipart upload file to file, so no object passes through t`FLOCI_STORAGE_SERVICES_S3_MODE=persistent` to keep only S3 on disk. Every request body is stillcapped by `FLOCI_PROTOCOLS_MAX_REQUEST_SIZE`, 2048 MB by default: to accept the 5 GiB S3 allows inone `PutObject` or `UploadPart`, raise it a little past 5120, since the aws-chunked framing SDKs-send makes a request slightly larger than its data. With-[signature verification](#signature-verification) on, a request signed over its whole payload, with-a hex `x-amz-content-sha256` rather than `UNSIGNED-PAYLOAD` or a `STREAMING-` value, is still read-into memory to check that hash, so it is limited to about 2 GiB. While it runs,+send makes a request slightly larger than its data. A request whose `Content-Length` is over the+limit gets an empty `413 Request Entity Too Large` before S3 sees it. As on S3, more than 5 GiB of+data in one `PutObject` or `UploadPart` is rejected with `EntityTooLarge`, whatever the limit is+raised to: before any of it is read when the request declares its size, in `Content-Length` or, for+an aws-chunked body, `x-amz-decoded-content-length`, and otherwise as soon as it passes 5 GiB. As+S3's does, the error names the most allowed in `MaxSizeAllowed` and, when the request declared it,+the size proposed in `ProposedSize`. A request rejected before its body is read that sent+`Expect: 100-continue` is answered with `Connection: close` and its connection closed, since the+client never sends that body. With [signature verification](#signature-verification) on, a request+signed over its whole payload, with a hex `x-amz-content-sha256` rather than `UNSIGNED-PAYLOAD` or a+`STREAMING-` value, is still read into memory to check that hash, so it is limited to about 2 GiB.+So is a body sent with `Content-Encoding: aws-chunked` but neither a `STREAMING-`+`x-amz-content-sha256` nor an `x-amz-trailer`, which is read whole to be decoded and fails with+`EntityTooLarge` past that. While it runs,`CompleteMultipartUpload` needs free disk space for twice the object, its parts and the assembledcopy, until the parts are removed. Multipart uploads in progress do not survive a restart or areset, and their files are removed when either happens. `UploadPartCopy` reads only the copieddiff --git a/src/main/java/io/github/hectorvent/floci/core/common/ExpectContinueConnectionFilter.java b/src/main/java/io/github/hectorvent/floci/core/common/ExpectContinueConnectionFilter.javanew file mode 100644index 0000000000..bddc6c3df8--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/core/common/ExpectContinueConnectionFilter.java@@ -0,0 +1,49 @@+package io.github.hectorvent.floci.core.common;++import io.quarkus.vertx.http.runtime.CurrentVertxRequest;+import io.vertx.core.http.HttpHeaders;+import io.vertx.core.http.HttpServerRequest;+import io.vertx.core.http.HttpVersion;+import io.vertx.ext.web.RoutingContext;+import jakarta.ws.rs.container.ContainerRequestContext;+import jakarta.ws.rs.container.ContainerResponseContext;+import jakarta.ws.rs.container.ContainerResponseFilter;+import jakarta.ws.rs.ext.Provider;++/**+ * Closes the connection after answering a request that sent {@code Expect: 100-continue} before any+ * of its body arrived, as when an upload is rejected on its headers alone. The client never sends a+ * body it was not told to continue with, so a connection kept open would read the client's next+ * request as that body. The response says {@code Connection: close}, so the client does not reuse it+ * either.+ */+@Provider+public class ExpectContinueConnectionFilter implements ContainerResponseFilter {++ private final CurrentVertxRequest currentVertxRequest;++ public ExpectContinueConnectionFilter(CurrentVertxRequest currentVertxRequest) {+ this.currentVertxRequest = currentVertxRequest;+ }++ @Override+ public void filter(ContainerRequestContext requestContext, ContainerResponseContext responseContext) {+ RoutingContext context = currentVertxRequest.getCurrent();+ if (context == null || !awaitsUnsentBody(context.request())) {+ return;+ }+ responseContext.getHeaders().putSingle(HttpHeaders.CONNECTION.toString(), HttpHeaders.CLOSE.toString());+ context.addEndHandler(ended -> context.request().connection().close());+ }++ /**+ * Whether the request is waiting to be told to continue before it sends its body. An HTTP/2+ * connection carries other requests, so it is never closed for one of them.+ */+ private static boolean awaitsUnsentBody(HttpServerRequest request) {+ return request.version() != HttpVersion.HTTP_2+ && request.headers().contains(HttpHeaders.EXPECT, HttpHeaders.CONTINUE, true)+ && !request.isEnded()+ && request.bytesRead() == 0;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/DigestingInputStream.java b/src/main/java/io/github/hectorvent/floci/services/s3/DigestingInputStream.javaindex 3cedc5f72f..928e8de7ea 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/DigestingInputStream.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/DigestingInputStream.java@@ -15,18 +15,21 @@ import java.util.Map;/*** Passes an upload body through while computing its MD5, the checksums it is checked against or- * stored with, and its size, so a body can be validated and stored without being held whole.+ * stored with, and its size, so a body can be validated and stored without being held whole. A body+ * that runs past the most its request may upload fails with EntityTooLarge, whatever size it declared.*/final class DigestingInputStream extends FilterInputStream {private final MessageDigest md5;private final Map<ChecksumAlgorithm, S3Checksum.Calculator> calculators = new EnumMap<>(ChecksumAlgorithm.class);private final Map<ChecksumAlgorithm, String> checksums = new EnumMap<>(ChecksumAlgorithm.class);+ private final long maxSize;private byte[] md5Digest;private long size;- DigestingInputStream(InputStream in, Collection<ChecksumAlgorithm> algorithms) {+ DigestingInputStream(InputStream in, Collection<ChecksumAlgorithm> algorithms, long maxSize) {super(in);+ this.maxSize = maxSize;try {this.md5 = MessageDigest.getInstance("MD5");} catch (NoSuchAlgorithmException e) {@@ -106,6 +109,9 @@ final class DigestingInputStream extends FilterInputStream {}private void update(byte[] buffer, int offset, int length) {+ if (size + length > maxSize) {+ throw S3Service.entityTooLarge(maxSize);+ }md5.update(buffer, offset, length);for (S3Checksum.Calculator calculator : calculators.values()) {calculator.update(buffer, offset, length);diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.javaindex dad09d0b6a..328b603984 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java@@ -847,6 +847,7 @@ public class S3Controller {return handleUploadPartCopy(copySource, bucket, key, uploadId, partNumber, httpHeaders, authorization);}+ requireDeclaredSizeWithin(httpHeaders, contentEncoding, contentSha256, S3Service.MAX_PART_SIZE);Part part = s3Service.storePart(bucket, key, uploadId, partNumber,decodedBody(body, contentEncoding, contentSha256, trailerHeader(httpHeaders, uriInfo)),uploadChecksums(httpHeaders, uriInfo),@@ -890,6 +891,7 @@ public class S3Controller {String sseCustomerKeyMd5 = httpHeaders.getHeaderString("x-amz-server-side-encryption-customer-key-MD5");String cannedAcl = httpHeaders.getHeaderString("x-amz-acl");s3Service.authorizePutObject(bucket, key, authorization);+ requireDeclaredSizeWithin(httpHeaders, contentEncoding, contentSha256, S3Service.MAX_PUT_OBJECT_SIZE);S3Object obj = s3Service.putObject(bucket, key,decodedBody(body, contentEncoding, contentSha256, trailerHeader(httpHeaders, uriInfo)),uploadChecksums(httpHeaders, uriInfo), contentType, extractUserMetadata(httpHeaders, uriInfo),@@ -2163,27 +2165,55 @@ public class S3Controller {* The object data of an upload body, read as it arrives. A streaming payload, and any aws-chunked* body that declares {@code x-amz-trailer}, is decoded from its framing on the way through so the* trailing checksum can be checked. A Content-Encoding naming aws-chunked without either of those- * does not guarantee framing, so that body is read whole and kept as sent when it does not decode.+ * does not guarantee framing, so that body is read whole, failing with EntityTooLarge past the most+ * one array holds, and kept as sent when it does not decode.*/private InputStream decodedBody(InputStream body, String contentEncoding, String contentSha256,String trailer) {InputStream raw = body != null ? body : InputStream.nullInputStream();- boolean awsChunked = contentEncoding != null- && contentEncoding.toLowerCase(Locale.ROOT).contains("aws-chunked");+ boolean awsChunked = namesAwsChunked(contentEncoding);if (declaresStreamingPayload(contentSha256) || (awsChunked && trailer != null && !trailer.isBlank())) {return new AwsChunkedInputStream(raw, trailerChecksumAlgorithm(trailer));}if (awsChunked) {- return new ByteArrayInputStream(decodeAwsChunked(readWholeBody(raw), contentEncoding, contentSha256));+ byte[] whole = S3Service.readWithin(raw, S3Service.MAX_IN_MEMORY_OBJECT_SIZE, readSoFar -> LOG.warnv(+ "Rejecting an aws-chunked body of more than {0} bytes: sent without a streaming x-amz-content-sha256 or an x-amz-trailer, it is read whole to be decoded",+ S3Service.MAX_IN_MEMORY_OBJECT_SIZE));+ return new ByteArrayInputStream(decodeAwsChunked(whole, contentEncoding, contentSha256));}return raw;}+ /**+ * Rejects an upload that declares more data than {@code maxSize} before any of it is read. An+ * aws-chunked body declares its data in {@code x-amz-decoded-content-length}, since its+ * Content-Length also counts the framing. A body that declares no size, or a size that is not a+ * number, is held to the limit as it is read.+ */+ static void requireDeclaredSizeWithin(HttpHeaders httpHeaders, String contentEncoding, String contentSha256,+ long maxSize) {+ boolean awsChunked = declaresStreamingPayload(contentSha256) || namesAwsChunked(contentEncoding);+ String declared = httpHeaders.getHeaderString(awsChunked ? "x-amz-decoded-content-length" : "Content-Length");+ if (declared == null) {+ return;+ }+ long size;+ try {+ size = Long.parseLong(declared.trim());+ } catch (NumberFormatException ignored) {+ // Not a size, so nothing is known before the body is read, which is held to the limit as it arrives.+ return;+ }+ if (size > maxSize) {+ throw S3Service.entityTooLarge(size, maxSize);+ }+ }+private static String trailerHeader(HttpHeaders httpHeaders, UriInfo uriInfo) {return resolveHeaderOrQueryParam(httpHeaders, uriInfo, "x-amz-trailer");}- /** The whole of a body that is small by nature, such as a subresource's XML, or must be read whole to be decoded. */+ /** The whole of a body that is small by nature, such as a subresource's XML. */private static byte[] readWholeBody(InputStream body) {if (body == null) {return new byte[0];@@ -2201,8 +2231,7 @@ public class S3Controller {*/private byte[] decodeAwsChunked(byte[] body, String contentEncoding, String contentSha256) {boolean declaresStreamingPayload = declaresStreamingPayload(contentSha256);- boolean isAwsChunked = declaresStreamingPayload || (contentEncoding != null- && contentEncoding.toLowerCase(Locale.ROOT).contains("aws-chunked"));+ boolean isAwsChunked = declaresStreamingPayload || namesAwsChunked(contentEncoding);if (!isAwsChunked) {return body;}@@ -2257,6 +2286,11 @@ public class S3Controller {}}+ /** Whether a Content-Encoding names aws-chunked, alone or with other codings. */+ private static boolean namesAwsChunked(String contentEncoding) {+ return contentEncoding != null && contentEncoding.toLowerCase(Locale.ROOT).contains("aws-chunked");+ }+/** Whether x-amz-content-sha256 declares a streaming payload, which is always aws-chunked. */static boolean declaresStreamingPayload(String contentSha256) {return "STREAMING-AWS4-HMAC-SHA256-PAYLOAD".equals(contentSha256)@@ -3401,8 +3435,15 @@ public class S3Controller {xmlBuilder.elem("Condition", condition);}// S3 InvalidArgument responses carry ArgumentName and ArgumentValue so the SDK can- // surface which input was rejected. They travel through AwsException.extendedData.+ // surface which input was rejected, and EntityTooLarge the proposed and allowed sizes.+ // They travel through AwsException.extendedData.if (e.getExtendedData() != null) {+ for (String sizeField : List.of("ProposedSize", "MaxSizeAllowed")) {+ Object size = e.getExtendedData().get(sizeField);+ if (size != null) {+ xmlBuilder.elem(sizeField, size.toString());+ }+ }Object resource = e.getExtendedData().get("Resource");if (resource != null) {xmlBuilder.elem("Resource", resource.toString());@@ -4290,8 +4331,7 @@ public class S3Controller {if (contentSha256 != null && contentSha256.startsWith("STREAMING-")) {return true;}- String contentEncoding = httpHeaders.getHeaderString("Content-Encoding");- if (contentEncoding != null && contentEncoding.toLowerCase(Locale.ROOT).contains("aws-chunked")) {+ if (namesAwsChunked(httpHeaders.getHeaderString("Content-Encoding"))) {return true;}if (httpHeaders.getHeaderString("x-amz-decoded-content-length") != nulldiff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.javaindex 2aac0d6537..5d823348e3 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java@@ -85,6 +85,7 @@ import java.util.concurrent.locks.ReentrantLock;import java.util.concurrent.locks.ReentrantReadWriteLock;import java.util.function.Consumer;import java.util.function.Function;+import java.util.function.LongConsumer;import java.util.function.Supplier;import java.util.regex.Matcher;import java.util.regex.Pattern;@@ -220,7 +221,7 @@ public class S3Service implements Resettable, ResourceProvider {return locks;}// Memory mode keeps an object in one byte array; this is the largest the JDK reliably allocates.- private static final long MAX_IN_MEMORY_OBJECT_SIZE = Integer.MAX_VALUE - 8;+ static final long MAX_IN_MEMORY_OBJECT_SIZE = Integer.MAX_VALUE - 8;// Directory under .multipart where a streamed part or object body is written before it is// published. Upload IDs are UUIDs, so it never collides with an upload's own directory, and the// sweep at startup and reset clears it with them.@@ -633,7 +634,8 @@ public class S3Service implements Resettable, ResourceProvider {* it in memory in the disk-backed modes. Everything the request alone decides is checked before* any of the body is read, so a write that cannot succeed fails without transferring it. The body* is staged on disk with its ETag and checksum worked out as it arrives, then moved into place- * like an assembled multipart object.+ * like an assembled multipart object. A body of more than 5 GiB, the most S3 takes in one+ * PutObject, fails with EntityTooLarge as soon as it passes that.*/S3Object putObject(String bucketName, String key, InputStream body, UploadChecksums checksums,String contentType, Map<String, String> metadata, PutObjectOptions options) {@@ -661,7 +663,7 @@ public class S3Service implements Resettable, ResourceProvider {ChecksumAlgorithm computed = effectiveOptions.getClientChecksum() != null ? null: declared != null ? declared : ChecksumAlgorithm.CRC64NVME;DigestingInputStream digests = new DigestingInputStream(body,- digestAlgorithms(checksums, computed));+ digestAlgorithms(checksums, computed), MAX_PUT_OBJECT_SIZE);Path staged = stageBody(digests);try {checksums.verify(digests.md5(), digests::checksum);@@ -2303,6 +2305,28 @@ public class S3Service implements Resettable, ResourceProvider {// The most one part of a multipart upload can hold, per S3's multipart upload limits.static final long MAX_PART_SIZE = 5L * 1024 * 1024 * 1024;+ // The User Guide's "up to 5 GB" in one PutObject, read as 5 GiB like CopyObject's 5 GB limit;+ // a larger object is uploaded in parts.+ static final long MAX_PUT_OBJECT_SIZE = 5L * 1024 * 1024 * 1024;++ /**+ * The message of S3's {@code EntityTooLarge} error as S3 sends it, which is not the wording of the+ * error code table's description.+ */+ static final String ENTITY_TOO_LARGE_MESSAGE = "Your proposed upload exceeds the maximum allowed size";++ /** S3's error for an upload over the most it takes, naming the size proposed and the most allowed. */+ static AwsException entityTooLarge(long proposedSize, long maxSizeAllowed) {+ return new AwsException("EntityTooLarge", ENTITY_TOO_LARGE_MESSAGE, 400,+ Map.of("ProposedSize", proposedSize, "MaxSizeAllowed", maxSizeAllowed));+ }++ /** The same error for a body that ran past the most allowed before its whole size was known. */+ static AwsException entityTooLarge(long maxSizeAllowed) {+ return new AwsException("EntityTooLarge", ENTITY_TOO_LARGE_MESSAGE, 400,+ Map.of("MaxSizeAllowed", maxSizeAllowed));+ }+static void requireCopyableSize(S3Object source) {if (source.getSize() > MAX_COPY_OBJECT_SOURCE_SIZE) {throw new AwsException("InvalidRequest",@@ -3898,7 +3922,9 @@ public class S3Service implements Resettable, ResourceProvider {* body is staged outside the upload's directory and outside any lock, so parts of one upload still* arrive in parallel and a slow client never holds up the bucket's other multipart operations. Only* once it has passed its checks is it moved into the upload and its record published, under the- * upload's lock, so an upload aborted or completed meanwhile fails the part with NoSuchUpload.+ * upload's lock, so an upload aborted or completed meanwhile fails the part with NoSuchUpload. A+ * body of more than 5 GiB, the most one part holds, fails with EntityTooLarge as soon as it+ * passes that.*/Part storePart(String bucket, String key, String uploadId, int partNumber, InputStream body,UploadChecksums checksums,@@ -3920,7 +3946,7 @@ public class S3Service implements Resettable, ResourceProvider {}ChecksumAlgorithm algorithm = declared != null ? declared : ChecksumAlgorithm.CRC64NVME;DigestingInputStream digests = new DigestingInputStream(body,- digestAlgorithms(checksums, algorithm));+ digestAlgorithms(checksums, algorithm), MAX_PART_SIZE);Path staged = stageBody(digests);try {checksums.verify(digests.md5(), digests::checksum);@@ -3956,19 +3982,36 @@ public class S3Service implements Resettable, ResourceProvider {* with EntityTooLarge once it outgrows the largest array the JDK allocates.*/private static byte[] readVerified(InputStream body, UploadChecksums checksums, String upload) {- DigestingInputStream digests = new DigestingInputStream(body, digestAlgorithms(checksums, null));+ // The memory cap is lower than the most S3 takes in one body, so it is the limit here.+ DigestingInputStream digests = new DigestingInputStream(body, digestAlgorithms(checksums, null),+ Long.MAX_VALUE);+ byte[] data = readWithin(digests, MAX_IN_MEMORY_OBJECT_SIZE,+ readSoFar -> warnTooLargeForMemory(upload, readSoFar));+ checksums.verify(digests.md5(), digests::checksum);+ verifyChunkedTrailers(body, digests, checksums);+ return data;+ }++ /**+ * Reads all of {@code body} into one array, failing with EntityTooLarge once it passes+ * {@code maxSize}, after telling {@code tooLarge} how many bytes it had read by then.+ */+ static byte[] readWithin(InputStream body, long maxSize, LongConsumer tooLarge) {ByteArrayOutputStream out = new ByteArrayOutputStream();byte[] buffer = new byte[64 * 1024];try {- for (int read = digests.read(buffer); read >= 0; read = digests.read(buffer)) {- requireFitsInMemory(upload, (long) out.size() + read);+ for (int read = body.read(buffer); read >= 0; read = body.read(buffer)) {+ long readSoFar = (long) out.size() + read;+ if (readSoFar > maxSize) {+ tooLarge.accept(readSoFar);+ // The rest of the body is not read, so its whole size is not known to name.+ throw entityTooLarge(maxSize);+ }out.write(buffer, 0, read);}} catch (IOException e) {throw new UncheckedIOException("Failed to read the upload body", e);}- checksums.verify(digests.md5(), digests::checksum);- verifyChunkedTrailers(body, digests, checksums);return out.toByteArray();}@@ -4065,8 +4108,7 @@ public class S3Service implements Resettable, ResourceProvider {copySourceSseCustomerHeaders.keyMd5());CopySourceRange range = CopySourceRange.parse(copySourceRange, source.getSize());if (range.length() > MAX_PART_SIZE) {- throw new AwsException("EntityTooLarge",- "Your proposed upload exceeds the maximum allowed object size.", 400);+ throw entityTooLarge(range.length(), MAX_PART_SIZE);}return storePart(destBucket, destKey, uploadId, partNumber,new ByteRangeInputStream(read.body(), range.first(), range.length()), UploadChecksums.NONE,@@ -4261,17 +4303,22 @@ public class S3Service implements Resettable, ResourceProvider {/*** Rejects an upload that memory mode could not hold in one byte array, with the error S3 gives an- * object over its size limit, so it fails at the part that crosses the limit instead of running- * the heap out of memory or overflowing the array size at completion.+ * object over its size limit, naming the upload's size and the most memory mode holds, so it fails+ * at the part that crosses the limit instead of running the heap out of memory or overflowing the+ * array size at completion.*/private static void requireFitsInMemory(String upload, long uploadSize) {if (uploadSize > MAX_IN_MEMORY_OBJECT_SIZE) {- LOG.warnv("{0} would hold {1} bytes, over the {2} an object can have in memory storage mode; set FLOCI_STORAGE_SERVICES_S3_MODE=persistent for larger objects",- upload, uploadSize, MAX_IN_MEMORY_OBJECT_SIZE);- throw new AwsException("EntityTooLarge", "Your proposed upload exceeds the maximum allowed object size.", 400);+ warnTooLargeForMemory(upload, uploadSize);+ throw entityTooLarge(uploadSize, MAX_IN_MEMORY_OBJECT_SIZE);}}+ private static void warnTooLargeForMemory(String upload, long uploadSize) {+ LOG.warnv("{0} would hold {1} bytes, over the {2} an object can have in memory storage mode; set FLOCI_STORAGE_SERVICES_S3_MODE=persistent for larger objects",+ upload, uploadSize, MAX_IN_MEMORY_OBJECT_SIZE);+ }+/*** Copies the in-memory bytes of {@code parts}, in order, into one array sized to the total* upload. The total is measured on the arrays being copied, so it holds whatever the records say.diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/DigestingInputStreamTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/DigestingInputStreamTest.javanew file mode 100644index 0000000000..a4ea5cef1e--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3/DigestingInputStreamTest.java@@ -0,0 +1,52 @@+package io.github.hectorvent.floci.services.s3;++import io.github.hectorvent.floci.core.common.AwsException;+import org.junit.jupiter.api.Test;++import java.io.ByteArrayInputStream;+import java.io.IOException;+import java.nio.charset.StandardCharsets;+import java.util.List;+import java.util.Map;++import static org.junit.jupiter.api.Assertions.assertArrayEquals;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertThrows;++class DigestingInputStreamTest {++ @Test+ void aBodyOfExactlyItsLimitIsReadWhole() throws IOException {+ DigestingInputStream body = new DigestingInputStream(+ new ByteArrayInputStream("0123456789".getBytes(StandardCharsets.UTF_8)), List.of(), 10);++ assertArrayEquals("0123456789".getBytes(StandardCharsets.UTF_8), body.readAllBytes());+ assertEquals(10, body.size());+ }++ @Test+ void aBodyThatRunsPastItsLimitIsEntityTooLarge() {+ DigestingInputStream body = new DigestingInputStream(+ new ByteArrayInputStream("0123456789A".getBytes(StandardCharsets.UTF_8)), List.of(), 10);++ AwsException error = assertThrows(AwsException.class, body::readAllBytes);++ assertEquals("EntityTooLarge", error.getErrorCode());+ assertEquals(400, error.getHttpStatus());+ assertEquals("Your proposed upload exceeds the maximum allowed size", error.getMessage());+ // Only the bytes read so far are known, so only the limit is named.+ assertEquals(Map.of("MaxSizeAllowed", 10L), error.getExtendedData());+ }++ @Test+ void aSingleByteReadPastTheLimitIsEntityTooLarge() throws IOException {+ DigestingInputStream body = new DigestingInputStream(+ new ByteArrayInputStream("ab".getBytes(StandardCharsets.UTF_8)), List.of(), 1);++ assertEquals('a', body.read());+ AwsException error = assertThrows(AwsException.class, body::read);++ assertEquals("EntityTooLarge", error.getErrorCode());+ assertEquals(1, body.size());+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3AwsChunkedIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3AwsChunkedIntegrationTest.javaindex 64b157cff7..d885e9a68b 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3AwsChunkedIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3AwsChunkedIntegrationTest.java@@ -16,6 +16,10 @@ class S3AwsChunkedIntegrationTest {private static final String STREAMING_UNSIGNED = "STREAMING-UNSIGNED-PAYLOAD-TRAILER";private static final String INCOMPLETE_BODY_MESSAGE ="You did not provide the number of bytes specified by the Content-Length HTTP header.";+ private static final String ENTITY_TOO_LARGE_MESSAGE = "Your proposed upload exceeds the maximum allowed size";+ // S3 names the size proposed and the most allowed, in this order, between the message and the request id.+ private static final String ENTITY_TOO_LARGE_SIZES =+ "</Message><ProposedSize>5368709121</ProposedSize><MaxSizeAllowed>5368709120</MaxSizeAllowed><RequestId>";@Testvoid putObject_chunkShorterThanItsDeclaredSize_isIncompleteBodyAndNotStored() {@@ -301,6 +305,60 @@ class S3AwsChunkedIntegrationTest {.body(not(containsString("<Part>")));}+ @Test+ void putObject_declaringMoreThanFiveGibibytes_isEntityTooLargeAndNotStored() {+ String bucket = createBucket("chunked-too-large");++ given()+ .header("x-amz-content-sha256", STREAMING_UNSIGNED)+ .header("Content-Encoding", "aws-chunked")+ .header("x-amz-decoded-content-length", "5368709121")+ .body("5\r\nhello\r\n0\r\n\r\n")+ .when()+ .put("/" + bucket + "/object.txt")+ .then()+ .statusCode(400)+ .body(containsString("<Code>EntityTooLarge</Code>"))+ .body(containsString("<Message>" + ENTITY_TOO_LARGE_MESSAGE + ENTITY_TOO_LARGE_SIZES));+ assertObjectAbsent(bucket, "object.txt");++ given()+ .header("x-amz-content-sha256", STREAMING_UNSIGNED)+ .header("Content-Encoding", "aws-chunked")+ .header("x-amz-decoded-content-length", "5")+ .body("5\r\nhello\r\n0\r\n\r\n")+ .when()+ .put("/" + bucket + "/object.txt")+ .then()+ .statusCode(200);+ }++ @Test+ void uploadPart_declaringMoreThanFiveGibibytes_isEntityTooLargeAndNoPartIsStored() {+ String bucket = createBucket("chunked-part-too-large");+ String uploadId = given()+ .when().post("/" + bucket + "/object.txt?uploads")+ .then().statusCode(200)+ .extract().xmlPath().getString("InitiateMultipartUploadResult.UploadId");++ given()+ .header("x-amz-content-sha256", STREAMING_UNSIGNED)+ .header("Content-Encoding", "aws-chunked")+ .header("x-amz-decoded-content-length", "5368709121")+ .body("5\r\nhello\r\n0\r\n\r\n")+ .when()+ .put("/" + bucket + "/object.txt?partNumber=1&uploadId=" + uploadId)+ .then()+ .statusCode(400)+ .body(containsString("<Code>EntityTooLarge</Code>"))+ .body(containsString("<Message>" + ENTITY_TOO_LARGE_MESSAGE + ENTITY_TOO_LARGE_SIZES));++ given()+ .when().get("/" + bucket + "/object.txt?uploadId=" + uploadId)+ .then().statusCode(200)+ .body(not(containsString("<Part>")));+ }+@Testvoid putObject_chunkedBodyWithoutATrailer_storesTheDecodedPayload() {String bucket = createBucket("chunked-no-trailer");diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ControllerDeclaredSizeTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ControllerDeclaredSizeTest.javanew file mode 100644index 0000000000..97579c59ea--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ControllerDeclaredSizeTest.java@@ -0,0 +1,67 @@+package io.github.hectorvent.floci.services.s3;++import io.github.hectorvent.floci.core.common.AwsException;+import jakarta.ws.rs.core.HttpHeaders;+import org.junit.jupiter.api.Test;++import java.util.Map;++import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.when;++class S3ControllerDeclaredSizeTest {++ private static final long LIMIT = S3Service.MAX_PUT_OBJECT_SIZE;+ private static final String STREAMING = "STREAMING-UNSIGNED-PAYLOAD-TRAILER";++ private static HttpHeaders headers(String contentLength, String decodedContentLength) {+ HttpHeaders headers = mock(HttpHeaders.class);+ when(headers.getHeaderString("Content-Length")).thenReturn(contentLength);+ when(headers.getHeaderString("x-amz-decoded-content-length")).thenReturn(decodedContentLength);+ return headers;+ }++ @Test+ void aBodyOverTheLimitIsEntityTooLargeBeforeItIsRead() {+ AwsException error = assertThrows(AwsException.class, () -> S3Controller.requireDeclaredSizeWithin(+ headers(String.valueOf(LIMIT + 1), null), null, "UNSIGNED-PAYLOAD", LIMIT));++ assertEquals("EntityTooLarge", error.getErrorCode());+ assertEquals(400, error.getHttpStatus());+ assertEquals("Your proposed upload exceeds the maximum allowed size", error.getMessage());+ assertEquals(Map.of("ProposedSize", LIMIT + 1, "MaxSizeAllowed", LIMIT), error.getExtendedData());+ }++ @Test+ void aBodyOfExactlyTheLimitIsAccepted() {+ assertDoesNotThrow(() -> S3Controller.requireDeclaredSizeWithin(+ headers(String.valueOf(LIMIT), null), null, "UNSIGNED-PAYLOAD", LIMIT));+ }++ @Test+ void anAwsChunkedBodyIsMeasuredByItsDecodedLengthNotItsFraming() {+ // The framing takes the Content-Length past the limit while the data is exactly the limit.+ assertDoesNotThrow(() -> S3Controller.requireDeclaredSizeWithin(+ headers(String.valueOf(LIMIT + 4096), String.valueOf(LIMIT)), "aws-chunked", STREAMING, LIMIT));+ assertDoesNotThrow(() -> S3Controller.requireDeclaredSizeWithin(+ headers(String.valueOf(LIMIT + 4096), String.valueOf(LIMIT)), null, STREAMING, LIMIT));++ assertThrows(AwsException.class, () -> S3Controller.requireDeclaredSizeWithin(+ headers("100", String.valueOf(LIMIT + 1)), "aws-chunked", STREAMING, LIMIT));+ assertThrows(AwsException.class, () -> S3Controller.requireDeclaredSizeWithin(+ headers("100", String.valueOf(LIMIT + 1)), "gzip, aws-chunked", "UNSIGNED-PAYLOAD", LIMIT));+ }++ @Test+ void aBodyThatDeclaresNoUsableSizeIsLeftToTheLimitAsItIsRead() {+ assertDoesNotThrow(() -> S3Controller.requireDeclaredSizeWithin(+ headers(null, null), null, "UNSIGNED-PAYLOAD", LIMIT));+ assertDoesNotThrow(() -> S3Controller.requireDeclaredSizeWithin(+ headers(String.valueOf(LIMIT + 1), null), "aws-chunked", STREAMING, LIMIT));+ assertDoesNotThrow(() -> S3Controller.requireDeclaredSizeWithin(+ headers("100", "not-a-size"), "aws-chunked", STREAMING, LIMIT));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ExpectContinueIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ExpectContinueIntegrationTest.javanew file mode 100644index 0000000000..1f0bb6e966--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ExpectContinueIntegrationTest.java@@ -0,0 +1,150 @@+package io.github.hectorvent.floci.services.s3;++import io.quarkus.test.junit.QuarkusTest;+import io.restassured.RestAssured;+import org.junit.jupiter.api.Test;++import java.io.ByteArrayOutputStream;+import java.io.IOException;+import java.io.InputStream;+import java.io.OutputStream;+import java.net.Socket;+import java.nio.charset.StandardCharsets;+import java.util.HashMap;+import java.util.Locale;+import java.util.Map;+import java.util.UUID;++import static io.restassured.RestAssured.given;+import static org.hamcrest.MatcherAssert.assertThat;+import static org.hamcrest.Matchers.containsString;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNull;++/**+ * Uploads sent with {@code Expect: 100-continue} over a raw connection, since an HTTP client library+ * hides whether the connection was kept or closed.+ */+@QuarkusTest+class S3ExpectContinueIntegrationTest {++ @Test+ void putObject_rejectedBeforeItsBodyIsSent_closesTheConnection() throws IOException {+ String bucket = createBucket("expect-too-large");++ try (Socket socket = connect()) {+ send(socket, "PUT /" + bucket + "/big.bin HTTP/1.1\r\n"+ + "Host: localhost\r\n"+ + "Content-Encoding: aws-chunked\r\n"+ + "x-amz-content-sha256: STREAMING-UNSIGNED-PAYLOAD-TRAILER\r\n"+ + "x-amz-decoded-content-length: 5368709121\r\n"+ + "Content-Length: 100\r\n"+ + "Expect: 100-continue\r\n"+ + "\r\n");++ ResponseHead head = readHead(socket.getInputStream());+ assertEquals(400, head.status());+ assertEquals("close", head.header("connection"));+ // Reading to the end returns only once Floci closes the connection.+ assertThat(readToEnd(socket), containsString("<Code>EntityTooLarge</Code>"));+ }+ }++ @Test+ void putObject_toAMissingBucketBeforeItsBodyIsSent_closesTheConnection() throws IOException {+ try (Socket socket = connect()) {+ send(socket, "PUT /missing-" + UUID.randomUUID() + "/object.txt HTTP/1.1\r\n"+ + "Host: localhost\r\n"+ + "Content-Length: 5\r\n"+ + "Expect: 100-continue\r\n"+ + "\r\n");++ ResponseHead head = readHead(socket.getInputStream());+ assertEquals(404, head.status());+ assertEquals("close", head.header("connection"));+ assertThat(readToEnd(socket), containsString("<Code>NoSuchBucket</Code>"));+ }+ }++ @Test+ void putObject_whoseBodyIsReadAfterContinue_keepsTheConnection() throws IOException {+ String bucket = createBucket("expect-kept");++ try (Socket socket = connect()) {+ InputStream in = socket.getInputStream();+ send(socket, "PUT /" + bucket + "/object.txt HTTP/1.1\r\n"+ + "Host: localhost\r\n"+ + "Content-Length: 5\r\n"+ + "Expect: 100-continue\r\n"+ + "\r\n");+ assertEquals(100, readHead(in).status());+ send(socket, "hello");+ ResponseHead stored = readHead(in);+ assertEquals(200, stored.status());+ assertNull(stored.header("connection"));+ readBody(in, stored);++ send(socket, "GET /" + bucket + "/object.txt HTTP/1.1\r\n"+ + "Host: localhost\r\n"+ + "\r\n");+ ResponseHead read = readHead(in);+ assertEquals(200, read.status());+ assertEquals("hello", readBody(in, read));+ }+ }++ private static Socket connect() throws IOException {+ Socket socket = new Socket("localhost", RestAssured.port);+ // A connection Floci leaves open fails the read instead of leaving it waiting.+ socket.setSoTimeout(10_000);+ return socket;+ }++ private static void send(Socket socket, String text) throws IOException {+ OutputStream out = socket.getOutputStream();+ out.write(text.getBytes(StandardCharsets.ISO_8859_1));+ out.flush();+ }++ private static ResponseHead readHead(InputStream in) throws IOException {+ ByteArrayOutputStream bytes = new ByteArrayOutputStream();+ int matched = 0;+ while (matched < 4) {+ int next = in.read();+ if (next < 0) {+ throw new IOException("connection closed before the response head ended");+ }+ bytes.write(next);+ matched = next == "\r\n\r\n".charAt(matched) ? matched + 1 : (next == '\r' ? 1 : 0);+ }+ String[] lines = bytes.toString(StandardCharsets.ISO_8859_1).split("\r\n");+ Map<String, String> headers = new HashMap<>();+ for (int i = 1; i < lines.length; i++) {+ int colon = lines[i].indexOf(':');+ headers.put(lines[i].substring(0, colon).trim().toLowerCase(Locale.ROOT), lines[i].substring(colon + 1).trim());+ }+ return new ResponseHead(Integer.parseInt(lines[0].split(" ")[1]), headers);+ }++ private static String readBody(InputStream in, ResponseHead head) throws IOException {+ int length = Integer.parseInt(head.header("content-length"));+ return new String(in.readNBytes(length), StandardCharsets.UTF_8);+ }++ private static String readToEnd(Socket socket) throws IOException {+ return new String(socket.getInputStream().readAllBytes(), StandardCharsets.UTF_8);+ }++ private static String createBucket(String prefix) {+ String bucket = prefix + "-" + UUID.randomUUID().toString().substring(0, 8);+ given().when().put("/" + bucket).then().statusCode(200);+ return bucket;+ }++ private record ResponseHead(int status, Map<String, String> headers) {++ String header(String name) {+ return headers.get(name);+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.javaindex d325bb8030..c6cb3d159c 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java@@ -569,7 +569,9 @@ class S3MultipartServiceTest {assertEquals("EntityTooLarge", error.getErrorCode());assertEquals(400, error.getHttpStatus());- assertEquals("Your proposed upload exceeds the maximum allowed object size.", error.getMessage());+ assertEquals("Your proposed upload exceeds the maximum allowed size", error.getMessage());+ assertEquals(Map.of("ProposedSize", (long) Integer.MAX_VALUE, "MaxSizeAllowed", Integer.MAX_VALUE - 8L),+ error.getExtendedData());assertNull(upload.getParts().get(2), "a rejected part is not recorded");s3Service.uploadPart("test-bucket", "huge.bin", upload.getUploadId(), 1, new byte[16]);assertEquals(16, upload.getParts().get(1).getSize(), "re-uploading a part replaces its size in the total");diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceReadWithinTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceReadWithinTest.javanew file mode 100644index 0000000000..871326dcfb--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceReadWithinTest.java@@ -0,0 +1,46 @@+package io.github.hectorvent.floci.services.s3;++import io.github.hectorvent.floci.core.common.AwsException;+import org.junit.jupiter.api.Test;++import java.io.ByteArrayInputStream;+import java.nio.charset.StandardCharsets;+import java.util.ArrayList;+import java.util.List;+import java.util.Map;++import static org.junit.jupiter.api.Assertions.assertArrayEquals;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;++class S3ServiceReadWithinTest {++ @Test+ void aBodyOfExactlyItsLimitIsReadWhole() {+ List<Long> tooLarge = new ArrayList<>();++ byte[] read = S3Service.readWithin(body("0123456789"), 10, tooLarge::add);++ assertArrayEquals("0123456789".getBytes(StandardCharsets.UTF_8), read);+ assertTrue(tooLarge.isEmpty());+ }++ @Test+ void aBodyThatRunsPastItsLimitIsEntityTooLargeNamingOnlyTheLimit() {+ List<Long> tooLarge = new ArrayList<>();++ AwsException error = assertThrows(AwsException.class,+ () -> S3Service.readWithin(body("0123456789A"), 10, tooLarge::add));++ assertEquals("EntityTooLarge", error.getErrorCode());+ assertEquals(400, error.getHttpStatus());+ assertEquals("Your proposed upload exceeds the maximum allowed size", error.getMessage());+ assertEquals(Map.of("MaxSizeAllowed", 10L), error.getExtendedData());+ assertEquals(List.of(11L), tooLarge);+ }++ private static ByteArrayInputStream body(String text) {+ return new ByteArrayInputStream(text.getBytes(StandardCharsets.UTF_8));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3StreamedUploadTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3StreamedUploadTest.javaindex 613dd02305..f5c37a800e 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3StreamedUploadTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3StreamedUploadTest.java@@ -284,7 +284,9 @@ class S3StreamedUploadTest {assertEquals("EntityTooLarge", error.getErrorCode());assertEquals(400, error.getHttpStatus());- assertEquals("Your proposed upload exceeds the maximum allowed object size.", error.getMessage());+ assertEquals("Your proposed upload exceeds the maximum allowed size", error.getMessage());+ assertEquals(Map.of("ProposedSize", overLimit, "MaxSizeAllowed", S3Service.MAX_PART_SIZE),+ error.getExtendedData());}assertTrue(s3Service.getMultipartUpload("bucket", "copy.bin", upload.getUploadId()).getParts().isEmpty());assertNothingStaged();