Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(s3): reject a PutObject or UploadPart over 5 GiB (#5469)

Priamo Germosen6d2aaba6a6e7
diff --git a/docs/services/s3.md b/docs/services/s3.md
index 9ee0ccc1b2..160771352f 100644
--- a/docs/services/s3.md
+++ b/docs/services/s3.md
@@ -412,10 +412,20 @@ way, and assemble a multipart upload file to file, so no object passes through t
`FLOCI_STORAGE_SERVICES_S3_MODE=persistent` to keep only S3 on disk. Every request body is still
capped by `FLOCI_PROTOCOLS_MAX_REQUEST_SIZE`, 2048 MB by default: to accept the 5 GiB S3 allows in
one `PutObject` or `UploadPart`, raise it a little past 5120, since the aws-chunked framing SDKs
-send makes a request slightly larger than its data. With
-[signature verification](#signature-verification) on, a request signed over its whole payload, with
-a hex `x-amz-content-sha256` rather than `UNSIGNED-PAYLOAD` or a `STREAMING-` value, is still read
-into memory to check that hash, so it is limited to about 2 GiB. While it runs,
+send makes a request slightly larger than its data. A request whose `Content-Length` is over the
+limit gets an empty `413 Request Entity Too Large` before S3 sees it. As on S3, more than 5 GiB of
+data in one `PutObject` or `UploadPart` is rejected with `EntityTooLarge`, whatever the limit is
+raised to: before any of it is read when the request declares its size, in `Content-Length` or, for
+an aws-chunked body, `x-amz-decoded-content-length`, and otherwise as soon as it passes 5 GiB. As
+S3's does, the error names the most allowed in `MaxSizeAllowed` and, when the request declared it,
+the size proposed in `ProposedSize`. A request rejected before its body is read that sent
+`Expect: 100-continue` is answered with `Connection: close` and its connection closed, since the
+client never sends that body. With [signature verification](#signature-verification) on, a request
+signed over its whole payload, with a hex `x-amz-content-sha256` rather than `UNSIGNED-PAYLOAD` or a
+`STREAMING-` value, is still read into memory to check that hash, so it is limited to about 2 GiB.
+So is a body sent with `Content-Encoding: aws-chunked` but neither a `STREAMING-`
+`x-amz-content-sha256` nor an `x-amz-trailer`, which is read whole to be decoded and fails with
+`EntityTooLarge` past that. While it runs,
`CompleteMultipartUpload` needs free disk space for twice the object, its parts and the assembled
copy, until the parts are removed. Multipart uploads in progress do not survive a restart or a
reset, and their files are removed when either happens. `UploadPartCopy` reads only the copied
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/ExpectContinueConnectionFilter.java b/src/main/java/io/github/hectorvent/floci/core/common/ExpectContinueConnectionFilter.java
new file mode 100644
index 0000000000..bddc6c3df8
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/core/common/ExpectContinueConnectionFilter.java
@@ -0,0 +1,49 @@
+package io.github.hectorvent.floci.core.common;
+
+import io.quarkus.vertx.http.runtime.CurrentVertxRequest;
+import io.vertx.core.http.HttpHeaders;
+import io.vertx.core.http.HttpServerRequest;
+import io.vertx.core.http.HttpVersion;
+import io.vertx.ext.web.RoutingContext;
+import jakarta.ws.rs.container.ContainerRequestContext;
+import jakarta.ws.rs.container.ContainerResponseContext;
+import jakarta.ws.rs.container.ContainerResponseFilter;
+import jakarta.ws.rs.ext.Provider;
+
+/**
+ * Closes the connection after answering a request that sent {@code Expect: 100-continue} before any
+ * of its body arrived, as when an upload is rejected on its headers alone. The client never sends a
+ * body it was not told to continue with, so a connection kept open would read the client's next
+ * request as that body. The response says {@code Connection: close}, so the client does not reuse it
+ * either.
+ */
+@Provider
+public class ExpectContinueConnectionFilter implements ContainerResponseFilter {
+
+ private final CurrentVertxRequest currentVertxRequest;
+
+ public ExpectContinueConnectionFilter(CurrentVertxRequest currentVertxRequest) {
+ this.currentVertxRequest = currentVertxRequest;
+ }
+
+ @Override
+ public void filter(ContainerRequestContext requestContext, ContainerResponseContext responseContext) {
+ RoutingContext context = currentVertxRequest.getCurrent();
+ if (context == null || !awaitsUnsentBody(context.request())) {
+ return;
+ }
+ responseContext.getHeaders().putSingle(HttpHeaders.CONNECTION.toString(), HttpHeaders.CLOSE.toString());
+ context.addEndHandler(ended -> context.request().connection().close());
+ }
+
+ /**
+ * Whether the request is waiting to be told to continue before it sends its body. An HTTP/2
+ * connection carries other requests, so it is never closed for one of them.
+ */
+ private static boolean awaitsUnsentBody(HttpServerRequest request) {
+ return request.version() != HttpVersion.HTTP_2
+ && request.headers().contains(HttpHeaders.EXPECT, HttpHeaders.CONTINUE, true)
+ && !request.isEnded()
+ && request.bytesRead() == 0;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/DigestingInputStream.java b/src/main/java/io/github/hectorvent/floci/services/s3/DigestingInputStream.java
index 3cedc5f72f..928e8de7ea 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/DigestingInputStream.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/DigestingInputStream.java
@@ -15,18 +15,21 @@ import java.util.Map;
/**
* Passes an upload body through while computing its MD5, the checksums it is checked against or
- * stored with, and its size, so a body can be validated and stored without being held whole.
+ * stored with, and its size, so a body can be validated and stored without being held whole. A body
+ * that runs past the most its request may upload fails with EntityTooLarge, whatever size it declared.
*/
final class DigestingInputStream extends FilterInputStream {
private final MessageDigest md5;
private final Map<ChecksumAlgorithm, S3Checksum.Calculator> calculators = new EnumMap<>(ChecksumAlgorithm.class);
private final Map<ChecksumAlgorithm, String> checksums = new EnumMap<>(ChecksumAlgorithm.class);
+ private final long maxSize;
private byte[] md5Digest;
private long size;
- DigestingInputStream(InputStream in, Collection<ChecksumAlgorithm> algorithms) {
+ DigestingInputStream(InputStream in, Collection<ChecksumAlgorithm> algorithms, long maxSize) {
super(in);
+ this.maxSize = maxSize;
try {
this.md5 = MessageDigest.getInstance("MD5");
} catch (NoSuchAlgorithmException e) {
@@ -106,6 +109,9 @@ final class DigestingInputStream extends FilterInputStream {
}
private void update(byte[] buffer, int offset, int length) {
+ if (size + length > maxSize) {
+ throw S3Service.entityTooLarge(maxSize);
+ }
md5.update(buffer, offset, length);
for (S3Checksum.Calculator calculator : calculators.values()) {
calculator.update(buffer, offset, length);
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
index dad09d0b6a..328b603984 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
@@ -847,6 +847,7 @@ public class S3Controller {
return handleUploadPartCopy(
copySource, bucket, key, uploadId, partNumber, httpHeaders, authorization);
}
+ requireDeclaredSizeWithin(httpHeaders, contentEncoding, contentSha256, S3Service.MAX_PART_SIZE);
Part part = s3Service.storePart(bucket, key, uploadId, partNumber,
decodedBody(body, contentEncoding, contentSha256, trailerHeader(httpHeaders, uriInfo)),
uploadChecksums(httpHeaders, uriInfo),
@@ -890,6 +891,7 @@ public class S3Controller {
String sseCustomerKeyMd5 = httpHeaders.getHeaderString("x-amz-server-side-encryption-customer-key-MD5");
String cannedAcl = httpHeaders.getHeaderString("x-amz-acl");
s3Service.authorizePutObject(bucket, key, authorization);
+ requireDeclaredSizeWithin(httpHeaders, contentEncoding, contentSha256, S3Service.MAX_PUT_OBJECT_SIZE);
S3Object obj = s3Service.putObject(bucket, key,
decodedBody(body, contentEncoding, contentSha256, trailerHeader(httpHeaders, uriInfo)),
uploadChecksums(httpHeaders, uriInfo), contentType, extractUserMetadata(httpHeaders, uriInfo),
@@ -2163,27 +2165,55 @@ public class S3Controller {
* The object data of an upload body, read as it arrives. A streaming payload, and any aws-chunked
* body that declares {@code x-amz-trailer}, is decoded from its framing on the way through so the
* trailing checksum can be checked. A Content-Encoding naming aws-chunked without either of those
- * does not guarantee framing, so that body is read whole and kept as sent when it does not decode.
+ * does not guarantee framing, so that body is read whole, failing with EntityTooLarge past the most
+ * one array holds, and kept as sent when it does not decode.
*/
private InputStream decodedBody(InputStream body, String contentEncoding, String contentSha256,
String trailer) {
InputStream raw = body != null ? body : InputStream.nullInputStream();
- boolean awsChunked = contentEncoding != null
- && contentEncoding.toLowerCase(Locale.ROOT).contains("aws-chunked");
+ boolean awsChunked = namesAwsChunked(contentEncoding);
if (declaresStreamingPayload(contentSha256) || (awsChunked && trailer != null && !trailer.isBlank())) {
return new AwsChunkedInputStream(raw, trailerChecksumAlgorithm(trailer));
}
if (awsChunked) {
- return new ByteArrayInputStream(decodeAwsChunked(readWholeBody(raw), contentEncoding, contentSha256));
+ byte[] whole = S3Service.readWithin(raw, S3Service.MAX_IN_MEMORY_OBJECT_SIZE, readSoFar -> LOG.warnv(
+ "Rejecting an aws-chunked body of more than {0} bytes: sent without a streaming x-amz-content-sha256 or an x-amz-trailer, it is read whole to be decoded",
+ S3Service.MAX_IN_MEMORY_OBJECT_SIZE));
+ return new ByteArrayInputStream(decodeAwsChunked(whole, contentEncoding, contentSha256));
}
return raw;
}
+ /**
+ * Rejects an upload that declares more data than {@code maxSize} before any of it is read. An
+ * aws-chunked body declares its data in {@code x-amz-decoded-content-length}, since its
+ * Content-Length also counts the framing. A body that declares no size, or a size that is not a
+ * number, is held to the limit as it is read.
+ */
+ static void requireDeclaredSizeWithin(HttpHeaders httpHeaders, String contentEncoding, String contentSha256,
+ long maxSize) {
+ boolean awsChunked = declaresStreamingPayload(contentSha256) || namesAwsChunked(contentEncoding);
+ String declared = httpHeaders.getHeaderString(awsChunked ? "x-amz-decoded-content-length" : "Content-Length");
+ if (declared == null) {
+ return;
+ }
+ long size;
+ try {
+ size = Long.parseLong(declared.trim());
+ } catch (NumberFormatException ignored) {
+ // Not a size, so nothing is known before the body is read, which is held to the limit as it arrives.
+ return;
+ }
+ if (size > maxSize) {
+ throw S3Service.entityTooLarge(size, maxSize);
+ }
+ }
+
private static String trailerHeader(HttpHeaders httpHeaders, UriInfo uriInfo) {
return resolveHeaderOrQueryParam(httpHeaders, uriInfo, "x-amz-trailer");
}
- /** The whole of a body that is small by nature, such as a subresource's XML, or must be read whole to be decoded. */
+ /** The whole of a body that is small by nature, such as a subresource's XML. */
private static byte[] readWholeBody(InputStream body) {
if (body == null) {
return new byte[0];
@@ -2201,8 +2231,7 @@ public class S3Controller {
*/
private byte[] decodeAwsChunked(byte[] body, String contentEncoding, String contentSha256) {
boolean declaresStreamingPayload = declaresStreamingPayload(contentSha256);
- boolean isAwsChunked = declaresStreamingPayload || (contentEncoding != null
- && contentEncoding.toLowerCase(Locale.ROOT).contains("aws-chunked"));
+ boolean isAwsChunked = declaresStreamingPayload || namesAwsChunked(contentEncoding);
if (!isAwsChunked) {
return body;
}
@@ -2257,6 +2286,11 @@ public class S3Controller {
}
}
+ /** Whether a Content-Encoding names aws-chunked, alone or with other codings. */
+ private static boolean namesAwsChunked(String contentEncoding) {
+ return contentEncoding != null && contentEncoding.toLowerCase(Locale.ROOT).contains("aws-chunked");
+ }
+
/** Whether x-amz-content-sha256 declares a streaming payload, which is always aws-chunked. */
static boolean declaresStreamingPayload(String contentSha256) {
return "STREAMING-AWS4-HMAC-SHA256-PAYLOAD".equals(contentSha256)
@@ -3401,8 +3435,15 @@ public class S3Controller {
xmlBuilder.elem("Condition", condition);
}
// S3 InvalidArgument responses carry ArgumentName and ArgumentValue so the SDK can
- // surface which input was rejected. They travel through AwsException.extendedData.
+ // surface which input was rejected, and EntityTooLarge the proposed and allowed sizes.
+ // They travel through AwsException.extendedData.
if (e.getExtendedData() != null) {
+ for (String sizeField : List.of("ProposedSize", "MaxSizeAllowed")) {
+ Object size = e.getExtendedData().get(sizeField);
+ if (size != null) {
+ xmlBuilder.elem(sizeField, size.toString());
+ }
+ }
Object resource = e.getExtendedData().get("Resource");
if (resource != null) {
xmlBuilder.elem("Resource", resource.toString());
@@ -4290,8 +4331,7 @@ public class S3Controller {
if (contentSha256 != null && contentSha256.startsWith("STREAMING-")) {
return true;
}
- String contentEncoding = httpHeaders.getHeaderString("Content-Encoding");
- if (contentEncoding != null && contentEncoding.toLowerCase(Locale.ROOT).contains("aws-chunked")) {
+ if (namesAwsChunked(httpHeaders.getHeaderString("Content-Encoding"))) {
return true;
}
if (httpHeaders.getHeaderString("x-amz-decoded-content-length") != null
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
index 2aac0d6537..5d823348e3 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
@@ -85,6 +85,7 @@ import java.util.concurrent.locks.ReentrantLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Consumer;
import java.util.function.Function;
+import java.util.function.LongConsumer;
import java.util.function.Supplier;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -220,7 +221,7 @@ public class S3Service implements Resettable, ResourceProvider {
return locks;
}
// Memory mode keeps an object in one byte array; this is the largest the JDK reliably allocates.
- private static final long MAX_IN_MEMORY_OBJECT_SIZE = Integer.MAX_VALUE - 8;
+ static final long MAX_IN_MEMORY_OBJECT_SIZE = Integer.MAX_VALUE - 8;
// Directory under .multipart where a streamed part or object body is written before it is
// published. Upload IDs are UUIDs, so it never collides with an upload's own directory, and the
// sweep at startup and reset clears it with them.
@@ -633,7 +634,8 @@ public class S3Service implements Resettable, ResourceProvider {
* it in memory in the disk-backed modes. Everything the request alone decides is checked before
* any of the body is read, so a write that cannot succeed fails without transferring it. The body
* is staged on disk with its ETag and checksum worked out as it arrives, then moved into place
- * like an assembled multipart object.
+ * like an assembled multipart object. A body of more than 5 GiB, the most S3 takes in one
+ * PutObject, fails with EntityTooLarge as soon as it passes that.
*/
S3Object putObject(String bucketName, String key, InputStream body, UploadChecksums checksums,
String contentType, Map<String, String> metadata, PutObjectOptions options) {
@@ -661,7 +663,7 @@ public class S3Service implements Resettable, ResourceProvider {
ChecksumAlgorithm computed = effectiveOptions.getClientChecksum() != null ? null
: declared != null ? declared : ChecksumAlgorithm.CRC64NVME;
DigestingInputStream digests = new DigestingInputStream(body,
- digestAlgorithms(checksums, computed));
+ digestAlgorithms(checksums, computed), MAX_PUT_OBJECT_SIZE);
Path staged = stageBody(digests);
try {
checksums.verify(digests.md5(), digests::checksum);
@@ -2303,6 +2305,28 @@ public class S3Service implements Resettable, ResourceProvider {
// The most one part of a multipart upload can hold, per S3's multipart upload limits.
static final long MAX_PART_SIZE = 5L * 1024 * 1024 * 1024;
+ // The User Guide's "up to 5 GB" in one PutObject, read as 5 GiB like CopyObject's 5 GB limit;
+ // a larger object is uploaded in parts.
+ static final long MAX_PUT_OBJECT_SIZE = 5L * 1024 * 1024 * 1024;
+
+ /**
+ * The message of S3's {@code EntityTooLarge} error as S3 sends it, which is not the wording of the
+ * error code table's description.
+ */
+ static final String ENTITY_TOO_LARGE_MESSAGE = "Your proposed upload exceeds the maximum allowed size";
+
+ /** S3's error for an upload over the most it takes, naming the size proposed and the most allowed. */
+ static AwsException entityTooLarge(long proposedSize, long maxSizeAllowed) {
+ return new AwsException("EntityTooLarge", ENTITY_TOO_LARGE_MESSAGE, 400,
+ Map.of("ProposedSize", proposedSize, "MaxSizeAllowed", maxSizeAllowed));
+ }
+
+ /** The same error for a body that ran past the most allowed before its whole size was known. */
+ static AwsException entityTooLarge(long maxSizeAllowed) {
+ return new AwsException("EntityTooLarge", ENTITY_TOO_LARGE_MESSAGE, 400,
+ Map.of("MaxSizeAllowed", maxSizeAllowed));
+ }
+
static void requireCopyableSize(S3Object source) {
if (source.getSize() > MAX_COPY_OBJECT_SOURCE_SIZE) {
throw new AwsException("InvalidRequest",
@@ -3898,7 +3922,9 @@ public class S3Service implements Resettable, ResourceProvider {
* body is staged outside the upload's directory and outside any lock, so parts of one upload still
* arrive in parallel and a slow client never holds up the bucket's other multipart operations. Only
* once it has passed its checks is it moved into the upload and its record published, under the
- * upload's lock, so an upload aborted or completed meanwhile fails the part with NoSuchUpload.
+ * upload's lock, so an upload aborted or completed meanwhile fails the part with NoSuchUpload. A
+ * body of more than 5 GiB, the most one part holds, fails with EntityTooLarge as soon as it
+ * passes that.
*/
Part storePart(String bucket, String key, String uploadId, int partNumber, InputStream body,
UploadChecksums checksums,
@@ -3920,7 +3946,7 @@ public class S3Service implements Resettable, ResourceProvider {
}
ChecksumAlgorithm algorithm = declared != null ? declared : ChecksumAlgorithm.CRC64NVME;
DigestingInputStream digests = new DigestingInputStream(body,
- digestAlgorithms(checksums, algorithm));
+ digestAlgorithms(checksums, algorithm), MAX_PART_SIZE);
Path staged = stageBody(digests);
try {
checksums.verify(digests.md5(), digests::checksum);
@@ -3956,19 +3982,36 @@ public class S3Service implements Resettable, ResourceProvider {
* with EntityTooLarge once it outgrows the largest array the JDK allocates.
*/
private static byte[] readVerified(InputStream body, UploadChecksums checksums, String upload) {
- DigestingInputStream digests = new DigestingInputStream(body, digestAlgorithms(checksums, null));
+ // The memory cap is lower than the most S3 takes in one body, so it is the limit here.
+ DigestingInputStream digests = new DigestingInputStream(body, digestAlgorithms(checksums, null),
+ Long.MAX_VALUE);
+ byte[] data = readWithin(digests, MAX_IN_MEMORY_OBJECT_SIZE,
+ readSoFar -> warnTooLargeForMemory(upload, readSoFar));
+ checksums.verify(digests.md5(), digests::checksum);
+ verifyChunkedTrailers(body, digests, checksums);
+ return data;
+ }
+
+ /**
+ * Reads all of {@code body} into one array, failing with EntityTooLarge once it passes
+ * {@code maxSize}, after telling {@code tooLarge} how many bytes it had read by then.
+ */
+ static byte[] readWithin(InputStream body, long maxSize, LongConsumer tooLarge) {
ByteArrayOutputStream out = new ByteArrayOutputStream();
byte[] buffer = new byte[64 * 1024];
try {
- for (int read = digests.read(buffer); read >= 0; read = digests.read(buffer)) {
- requireFitsInMemory(upload, (long) out.size() + read);
+ for (int read = body.read(buffer); read >= 0; read = body.read(buffer)) {
+ long readSoFar = (long) out.size() + read;
+ if (readSoFar > maxSize) {
+ tooLarge.accept(readSoFar);
+ // The rest of the body is not read, so its whole size is not known to name.
+ throw entityTooLarge(maxSize);
+ }
out.write(buffer, 0, read);
}
} catch (IOException e) {
throw new UncheckedIOException("Failed to read the upload body", e);
}
- checksums.verify(digests.md5(), digests::checksum);
- verifyChunkedTrailers(body, digests, checksums);
return out.toByteArray();
}
@@ -4065,8 +4108,7 @@ public class S3Service implements Resettable, ResourceProvider {
copySourceSseCustomerHeaders.keyMd5());
CopySourceRange range = CopySourceRange.parse(copySourceRange, source.getSize());
if (range.length() > MAX_PART_SIZE) {
- throw new AwsException("EntityTooLarge",
- "Your proposed upload exceeds the maximum allowed object size.", 400);
+ throw entityTooLarge(range.length(), MAX_PART_SIZE);
}
return storePart(destBucket, destKey, uploadId, partNumber,
new ByteRangeInputStream(read.body(), range.first(), range.length()), UploadChecksums.NONE,
@@ -4261,17 +4303,22 @@ public class S3Service implements Resettable, ResourceProvider {
/**
* Rejects an upload that memory mode could not hold in one byte array, with the error S3 gives an
- * object over its size limit, so it fails at the part that crosses the limit instead of running
- * the heap out of memory or overflowing the array size at completion.
+ * object over its size limit, naming the upload's size and the most memory mode holds, so it fails
+ * at the part that crosses the limit instead of running the heap out of memory or overflowing the
+ * array size at completion.
*/
private static void requireFitsInMemory(String upload, long uploadSize) {
if (uploadSize > MAX_IN_MEMORY_OBJECT_SIZE) {
- LOG.warnv("{0} would hold {1} bytes, over the {2} an object can have in memory storage mode; set FLOCI_STORAGE_SERVICES_S3_MODE=persistent for larger objects",
- upload, uploadSize, MAX_IN_MEMORY_OBJECT_SIZE);
- throw new AwsException("EntityTooLarge", "Your proposed upload exceeds the maximum allowed object size.", 400);
+ warnTooLargeForMemory(upload, uploadSize);
+ throw entityTooLarge(uploadSize, MAX_IN_MEMORY_OBJECT_SIZE);
}
}
+ private static void warnTooLargeForMemory(String upload, long uploadSize) {
+ LOG.warnv("{0} would hold {1} bytes, over the {2} an object can have in memory storage mode; set FLOCI_STORAGE_SERVICES_S3_MODE=persistent for larger objects",
+ upload, uploadSize, MAX_IN_MEMORY_OBJECT_SIZE);
+ }
+
/**
* Copies the in-memory bytes of {@code parts}, in order, into one array sized to the total
* upload. The total is measured on the arrays being copied, so it holds whatever the records say.
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/DigestingInputStreamTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/DigestingInputStreamTest.java
new file mode 100644
index 0000000000..a4ea5cef1e
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/DigestingInputStreamTest.java
@@ -0,0 +1,52 @@
+package io.github.hectorvent.floci.services.s3;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+class DigestingInputStreamTest {
+
+ @Test
+ void aBodyOfExactlyItsLimitIsReadWhole() throws IOException {
+ DigestingInputStream body = new DigestingInputStream(
+ new ByteArrayInputStream("0123456789".getBytes(StandardCharsets.UTF_8)), List.of(), 10);
+
+ assertArrayEquals("0123456789".getBytes(StandardCharsets.UTF_8), body.readAllBytes());
+ assertEquals(10, body.size());
+ }
+
+ @Test
+ void aBodyThatRunsPastItsLimitIsEntityTooLarge() {
+ DigestingInputStream body = new DigestingInputStream(
+ new ByteArrayInputStream("0123456789A".getBytes(StandardCharsets.UTF_8)), List.of(), 10);
+
+ AwsException error = assertThrows(AwsException.class, body::readAllBytes);
+
+ assertEquals("EntityTooLarge", error.getErrorCode());
+ assertEquals(400, error.getHttpStatus());
+ assertEquals("Your proposed upload exceeds the maximum allowed size", error.getMessage());
+ // Only the bytes read so far are known, so only the limit is named.
+ assertEquals(Map.of("MaxSizeAllowed", 10L), error.getExtendedData());
+ }
+
+ @Test
+ void aSingleByteReadPastTheLimitIsEntityTooLarge() throws IOException {
+ DigestingInputStream body = new DigestingInputStream(
+ new ByteArrayInputStream("ab".getBytes(StandardCharsets.UTF_8)), List.of(), 1);
+
+ assertEquals('a', body.read());
+ AwsException error = assertThrows(AwsException.class, body::read);
+
+ assertEquals("EntityTooLarge", error.getErrorCode());
+ assertEquals(1, body.size());
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3AwsChunkedIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3AwsChunkedIntegrationTest.java
index 64b157cff7..d885e9a68b 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3AwsChunkedIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3AwsChunkedIntegrationTest.java
@@ -16,6 +16,10 @@ class S3AwsChunkedIntegrationTest {
private static final String STREAMING_UNSIGNED = "STREAMING-UNSIGNED-PAYLOAD-TRAILER";
private static final String INCOMPLETE_BODY_MESSAGE =
"You did not provide the number of bytes specified by the Content-Length HTTP header.";
+ private static final String ENTITY_TOO_LARGE_MESSAGE = "Your proposed upload exceeds the maximum allowed size";
+ // S3 names the size proposed and the most allowed, in this order, between the message and the request id.
+ private static final String ENTITY_TOO_LARGE_SIZES =
+ "</Message><ProposedSize>5368709121</ProposedSize><MaxSizeAllowed>5368709120</MaxSizeAllowed><RequestId>";
@Test
void putObject_chunkShorterThanItsDeclaredSize_isIncompleteBodyAndNotStored() {
@@ -301,6 +305,60 @@ class S3AwsChunkedIntegrationTest {
.body(not(containsString("<Part>")));
}
+ @Test
+ void putObject_declaringMoreThanFiveGibibytes_isEntityTooLargeAndNotStored() {
+ String bucket = createBucket("chunked-too-large");
+
+ given()
+ .header("x-amz-content-sha256", STREAMING_UNSIGNED)
+ .header("Content-Encoding", "aws-chunked")
+ .header("x-amz-decoded-content-length", "5368709121")
+ .body("5\r\nhello\r\n0\r\n\r\n")
+ .when()
+ .put("/" + bucket + "/object.txt")
+ .then()
+ .statusCode(400)
+ .body(containsString("<Code>EntityTooLarge</Code>"))
+ .body(containsString("<Message>" + ENTITY_TOO_LARGE_MESSAGE + ENTITY_TOO_LARGE_SIZES));
+ assertObjectAbsent(bucket, "object.txt");
+
+ given()
+ .header("x-amz-content-sha256", STREAMING_UNSIGNED)
+ .header("Content-Encoding", "aws-chunked")
+ .header("x-amz-decoded-content-length", "5")
+ .body("5\r\nhello\r\n0\r\n\r\n")
+ .when()
+ .put("/" + bucket + "/object.txt")
+ .then()
+ .statusCode(200);
+ }
+
+ @Test
+ void uploadPart_declaringMoreThanFiveGibibytes_isEntityTooLargeAndNoPartIsStored() {
+ String bucket = createBucket("chunked-part-too-large");
+ String uploadId = given()
+ .when().post("/" + bucket + "/object.txt?uploads")
+ .then().statusCode(200)
+ .extract().xmlPath().getString("InitiateMultipartUploadResult.UploadId");
+
+ given()
+ .header("x-amz-content-sha256", STREAMING_UNSIGNED)
+ .header("Content-Encoding", "aws-chunked")
+ .header("x-amz-decoded-content-length", "5368709121")
+ .body("5\r\nhello\r\n0\r\n\r\n")
+ .when()
+ .put("/" + bucket + "/object.txt?partNumber=1&uploadId=" + uploadId)
+ .then()
+ .statusCode(400)
+ .body(containsString("<Code>EntityTooLarge</Code>"))
+ .body(containsString("<Message>" + ENTITY_TOO_LARGE_MESSAGE + ENTITY_TOO_LARGE_SIZES));
+
+ given()
+ .when().get("/" + bucket + "/object.txt?uploadId=" + uploadId)
+ .then().statusCode(200)
+ .body(not(containsString("<Part>")));
+ }
+
@Test
void putObject_chunkedBodyWithoutATrailer_storesTheDecodedPayload() {
String bucket = createBucket("chunked-no-trailer");
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ControllerDeclaredSizeTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ControllerDeclaredSizeTest.java
new file mode 100644
index 0000000000..97579c59ea
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ControllerDeclaredSizeTest.java
@@ -0,0 +1,67 @@
+package io.github.hectorvent.floci.services.s3;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import jakarta.ws.rs.core.HttpHeaders;
+import org.junit.jupiter.api.Test;
+
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class S3ControllerDeclaredSizeTest {
+
+ private static final long LIMIT = S3Service.MAX_PUT_OBJECT_SIZE;
+ private static final String STREAMING = "STREAMING-UNSIGNED-PAYLOAD-TRAILER";
+
+ private static HttpHeaders headers(String contentLength, String decodedContentLength) {
+ HttpHeaders headers = mock(HttpHeaders.class);
+ when(headers.getHeaderString("Content-Length")).thenReturn(contentLength);
+ when(headers.getHeaderString("x-amz-decoded-content-length")).thenReturn(decodedContentLength);
+ return headers;
+ }
+
+ @Test
+ void aBodyOverTheLimitIsEntityTooLargeBeforeItIsRead() {
+ AwsException error = assertThrows(AwsException.class, () -> S3Controller.requireDeclaredSizeWithin(
+ headers(String.valueOf(LIMIT + 1), null), null, "UNSIGNED-PAYLOAD", LIMIT));
+
+ assertEquals("EntityTooLarge", error.getErrorCode());
+ assertEquals(400, error.getHttpStatus());
+ assertEquals("Your proposed upload exceeds the maximum allowed size", error.getMessage());
+ assertEquals(Map.of("ProposedSize", LIMIT + 1, "MaxSizeAllowed", LIMIT), error.getExtendedData());
+ }
+
+ @Test
+ void aBodyOfExactlyTheLimitIsAccepted() {
+ assertDoesNotThrow(() -> S3Controller.requireDeclaredSizeWithin(
+ headers(String.valueOf(LIMIT), null), null, "UNSIGNED-PAYLOAD", LIMIT));
+ }
+
+ @Test
+ void anAwsChunkedBodyIsMeasuredByItsDecodedLengthNotItsFraming() {
+ // The framing takes the Content-Length past the limit while the data is exactly the limit.
+ assertDoesNotThrow(() -> S3Controller.requireDeclaredSizeWithin(
+ headers(String.valueOf(LIMIT + 4096), String.valueOf(LIMIT)), "aws-chunked", STREAMING, LIMIT));
+ assertDoesNotThrow(() -> S3Controller.requireDeclaredSizeWithin(
+ headers(String.valueOf(LIMIT + 4096), String.valueOf(LIMIT)), null, STREAMING, LIMIT));
+
+ assertThrows(AwsException.class, () -> S3Controller.requireDeclaredSizeWithin(
+ headers("100", String.valueOf(LIMIT + 1)), "aws-chunked", STREAMING, LIMIT));
+ assertThrows(AwsException.class, () -> S3Controller.requireDeclaredSizeWithin(
+ headers("100", String.valueOf(LIMIT + 1)), "gzip, aws-chunked", "UNSIGNED-PAYLOAD", LIMIT));
+ }
+
+ @Test
+ void aBodyThatDeclaresNoUsableSizeIsLeftToTheLimitAsItIsRead() {
+ assertDoesNotThrow(() -> S3Controller.requireDeclaredSizeWithin(
+ headers(null, null), null, "UNSIGNED-PAYLOAD", LIMIT));
+ assertDoesNotThrow(() -> S3Controller.requireDeclaredSizeWithin(
+ headers(String.valueOf(LIMIT + 1), null), "aws-chunked", STREAMING, LIMIT));
+ assertDoesNotThrow(() -> S3Controller.requireDeclaredSizeWithin(
+ headers("100", "not-a-size"), "aws-chunked", STREAMING, LIMIT));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ExpectContinueIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ExpectContinueIntegrationTest.java
new file mode 100644
index 0000000000..1f0bb6e966
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ExpectContinueIntegrationTest.java
@@ -0,0 +1,150 @@
+package io.github.hectorvent.floci.services.s3;
+
+import io.quarkus.test.junit.QuarkusTest;
+import io.restassured.RestAssured;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.Socket;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Locale;
+import java.util.Map;
+import java.util.UUID;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.MatcherAssert.assertThat;
+import static org.hamcrest.Matchers.containsString;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+
+/**
+ * Uploads sent with {@code Expect: 100-continue} over a raw connection, since an HTTP client library
+ * hides whether the connection was kept or closed.
+ */
+@QuarkusTest
+class S3ExpectContinueIntegrationTest {
+
+ @Test
+ void putObject_rejectedBeforeItsBodyIsSent_closesTheConnection() throws IOException {
+ String bucket = createBucket("expect-too-large");
+
+ try (Socket socket = connect()) {
+ send(socket, "PUT /" + bucket + "/big.bin HTTP/1.1\r\n"
+ + "Host: localhost\r\n"
+ + "Content-Encoding: aws-chunked\r\n"
+ + "x-amz-content-sha256: STREAMING-UNSIGNED-PAYLOAD-TRAILER\r\n"
+ + "x-amz-decoded-content-length: 5368709121\r\n"
+ + "Content-Length: 100\r\n"
+ + "Expect: 100-continue\r\n"
+ + "\r\n");
+
+ ResponseHead head = readHead(socket.getInputStream());
+ assertEquals(400, head.status());
+ assertEquals("close", head.header("connection"));
+ // Reading to the end returns only once Floci closes the connection.
+ assertThat(readToEnd(socket), containsString("<Code>EntityTooLarge</Code>"));
+ }
+ }
+
+ @Test
+ void putObject_toAMissingBucketBeforeItsBodyIsSent_closesTheConnection() throws IOException {
+ try (Socket socket = connect()) {
+ send(socket, "PUT /missing-" + UUID.randomUUID() + "/object.txt HTTP/1.1\r\n"
+ + "Host: localhost\r\n"
+ + "Content-Length: 5\r\n"
+ + "Expect: 100-continue\r\n"
+ + "\r\n");
+
+ ResponseHead head = readHead(socket.getInputStream());
+ assertEquals(404, head.status());
+ assertEquals("close", head.header("connection"));
+ assertThat(readToEnd(socket), containsString("<Code>NoSuchBucket</Code>"));
+ }
+ }
+
+ @Test
+ void putObject_whoseBodyIsReadAfterContinue_keepsTheConnection() throws IOException {
+ String bucket = createBucket("expect-kept");
+
+ try (Socket socket = connect()) {
+ InputStream in = socket.getInputStream();
+ send(socket, "PUT /" + bucket + "/object.txt HTTP/1.1\r\n"
+ + "Host: localhost\r\n"
+ + "Content-Length: 5\r\n"
+ + "Expect: 100-continue\r\n"
+ + "\r\n");
+ assertEquals(100, readHead(in).status());
+ send(socket, "hello");
+ ResponseHead stored = readHead(in);
+ assertEquals(200, stored.status());
+ assertNull(stored.header("connection"));
+ readBody(in, stored);
+
+ send(socket, "GET /" + bucket + "/object.txt HTTP/1.1\r\n"
+ + "Host: localhost\r\n"
+ + "\r\n");
+ ResponseHead read = readHead(in);
+ assertEquals(200, read.status());
+ assertEquals("hello", readBody(in, read));
+ }
+ }
+
+ private static Socket connect() throws IOException {
+ Socket socket = new Socket("localhost", RestAssured.port);
+ // A connection Floci leaves open fails the read instead of leaving it waiting.
+ socket.setSoTimeout(10_000);
+ return socket;
+ }
+
+ private static void send(Socket socket, String text) throws IOException {
+ OutputStream out = socket.getOutputStream();
+ out.write(text.getBytes(StandardCharsets.ISO_8859_1));
+ out.flush();
+ }
+
+ private static ResponseHead readHead(InputStream in) throws IOException {
+ ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ int matched = 0;
+ while (matched < 4) {
+ int next = in.read();
+ if (next < 0) {
+ throw new IOException("connection closed before the response head ended");
+ }
+ bytes.write(next);
+ matched = next == "\r\n\r\n".charAt(matched) ? matched + 1 : (next == '\r' ? 1 : 0);
+ }
+ String[] lines = bytes.toString(StandardCharsets.ISO_8859_1).split("\r\n");
+ Map<String, String> headers = new HashMap<>();
+ for (int i = 1; i < lines.length; i++) {
+ int colon = lines[i].indexOf(':');
+ headers.put(lines[i].substring(0, colon).trim().toLowerCase(Locale.ROOT), lines[i].substring(colon + 1).trim());
+ }
+ return new ResponseHead(Integer.parseInt(lines[0].split(" ")[1]), headers);
+ }
+
+ private static String readBody(InputStream in, ResponseHead head) throws IOException {
+ int length = Integer.parseInt(head.header("content-length"));
+ return new String(in.readNBytes(length), StandardCharsets.UTF_8);
+ }
+
+ private static String readToEnd(Socket socket) throws IOException {
+ return new String(socket.getInputStream().readAllBytes(), StandardCharsets.UTF_8);
+ }
+
+ private static String createBucket(String prefix) {
+ String bucket = prefix + "-" + UUID.randomUUID().toString().substring(0, 8);
+ given().when().put("/" + bucket).then().statusCode(200);
+ return bucket;
+ }
+
+ private record ResponseHead(int status, Map<String, String> headers) {
+
+ String header(String name) {
+ return headers.get(name);
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java
index d325bb8030..c6cb3d159c 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3MultipartServiceTest.java
@@ -569,7 +569,9 @@ class S3MultipartServiceTest {
assertEquals("EntityTooLarge", error.getErrorCode());
assertEquals(400, error.getHttpStatus());
- assertEquals("Your proposed upload exceeds the maximum allowed object size.", error.getMessage());
+ assertEquals("Your proposed upload exceeds the maximum allowed size", error.getMessage());
+ assertEquals(Map.of("ProposedSize", (long) Integer.MAX_VALUE, "MaxSizeAllowed", Integer.MAX_VALUE - 8L),
+ error.getExtendedData());
assertNull(upload.getParts().get(2), "a rejected part is not recorded");
s3Service.uploadPart("test-bucket", "huge.bin", upload.getUploadId(), 1, new byte[16]);
assertEquals(16, upload.getParts().get(1).getSize(), "re-uploading a part replaces its size in the total");
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceReadWithinTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceReadWithinTest.java
new file mode 100644
index 0000000000..871326dcfb
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceReadWithinTest.java
@@ -0,0 +1,46 @@
+package io.github.hectorvent.floci.services.s3;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayInputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class S3ServiceReadWithinTest {
+
+ @Test
+ void aBodyOfExactlyItsLimitIsReadWhole() {
+ List<Long> tooLarge = new ArrayList<>();
+
+ byte[] read = S3Service.readWithin(body("0123456789"), 10, tooLarge::add);
+
+ assertArrayEquals("0123456789".getBytes(StandardCharsets.UTF_8), read);
+ assertTrue(tooLarge.isEmpty());
+ }
+
+ @Test
+ void aBodyThatRunsPastItsLimitIsEntityTooLargeNamingOnlyTheLimit() {
+ List<Long> tooLarge = new ArrayList<>();
+
+ AwsException error = assertThrows(AwsException.class,
+ () -> S3Service.readWithin(body("0123456789A"), 10, tooLarge::add));
+
+ assertEquals("EntityTooLarge", error.getErrorCode());
+ assertEquals(400, error.getHttpStatus());
+ assertEquals("Your proposed upload exceeds the maximum allowed size", error.getMessage());
+ assertEquals(Map.of("MaxSizeAllowed", 10L), error.getExtendedData());
+ assertEquals(List.of(11L), tooLarge);
+ }
+
+ private static ByteArrayInputStream body(String text) {
+ return new ByteArrayInputStream(text.getBytes(StandardCharsets.UTF_8));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3StreamedUploadTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3StreamedUploadTest.java
index 613dd02305..f5c37a800e 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3StreamedUploadTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3StreamedUploadTest.java
@@ -284,7 +284,9 @@ class S3StreamedUploadTest {
assertEquals("EntityTooLarge", error.getErrorCode());
assertEquals(400, error.getHttpStatus());
- assertEquals("Your proposed upload exceeds the maximum allowed object size.", error.getMessage());
+ assertEquals("Your proposed upload exceeds the maximum allowed size", error.getMessage());
+ assertEquals(Map.of("ProposedSize", overLimit, "MaxSizeAllowed", S3Service.MAX_PART_SIZE),
+ error.getExtendedData());
}
assertTrue(s3Service.getMultipartUpload("bucket", "copy.bin", upload.getUploadId()).getParts().isEmpty());
assertNothingStaged();