Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(s3): stream PutObject and UploadPart bodies to disk (#5106)

Priamo Germosen583de3010c7f
diff --git a/docs/services/s3.md b/docs/services/s3.md
index cd7626b306..e3b86d29b0 100644
--- a/docs/services/s3.md
+++ b/docs/services/s3.md
@@ -389,10 +389,17 @@ still rewritten on every change. See
[Storage Modes](../configuration/storage.md#journaled-stores-under-persistent-mode).
In `memory` mode each object body is a single in-memory byte array, so an object can hold at most
-about 2 GiB: an `UploadPart` that takes a multipart upload past that, or a
-`CompleteMultipartUpload` over it, fails with `EntityTooLarge`. The disk-backed modes assemble a
-multipart upload file to file, so the object never passes through the heap; set
-`FLOCI_STORAGE_SERVICES_S3_MODE=persistent` to keep only S3 on disk. While it runs,
+about 2 GiB: a `PutObject` over that, an `UploadPart` that takes a multipart upload past it, or a
+`CompleteMultipartUpload` over it fails with `EntityTooLarge`. The disk-backed modes write a
+`PutObject` or `UploadPart` body to disk as it arrives, working out its ETag and checksums on the
+way, and assemble a multipart upload file to file, so no object passes through the heap; set
+`FLOCI_STORAGE_SERVICES_S3_MODE=persistent` to keep only S3 on disk. Every request body is still
+capped by `FLOCI_PROTOCOLS_MAX_REQUEST_SIZE`, 2048 MB by default: to accept the 5 GiB S3 allows in
+one `PutObject` or `UploadPart`, raise it a little past 5120, since the aws-chunked framing SDKs
+send makes a request slightly larger than its data. With
+[signature verification](#signature-verification) on, a request signed over its whole payload, with
+a hex `x-amz-content-sha256` rather than `UNSIGNED-PAYLOAD` or a `STREAMING-` value, is still read
+into memory to check that hash, so it is limited to about 2 GiB. While it runs,
`CompleteMultipartUpload` needs free disk space for twice the object, its parts and the assembled
copy, until the parts are removed. Multipart uploads in progress do not survive a restart or a
reset, and their files are removed when either happens. `UploadPartCopy` reads only the copied
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/AwsChunkedInputStream.java b/src/main/java/io/github/hectorvent/floci/services/s3/AwsChunkedInputStream.java
new file mode 100644
index 0000000000..107f30465e
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/AwsChunkedInputStream.java
@@ -0,0 +1,120 @@
+package io.github.hectorvent.floci.services.s3;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+
+import java.io.BufferedInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+
+/**
+ * Decodes an aws-chunked body as it is read, so a streaming upload is never held whole. Each chunk
+ * header ({@code hex-size[;chunk-signature=...]}) is consumed and only the chunk data comes out, up
+ * to the final empty chunk and the trailer lines and empty line after it. Chunk signatures and
+ * trailing checksums are not checked, as they were not when the body was decoded in one piece. A
+ * body that breaks the framing or ends early fails with S3's {@code IncompleteBody}.
+ */
+final class AwsChunkedInputStream extends InputStream {
+
+ // A chunk header or trailer line is a size, a signature and a few separators; anything longer
+ // is not framing.
+ private static final int MAX_LINE_LENGTH = 8192;
+
+ private final InputStream in;
+ private long remainingInChunk;
+ private boolean finished;
+
+ AwsChunkedInputStream(InputStream in) {
+ this.in = new BufferedInputStream(in);
+ }
+
+ @Override
+ public int read() throws IOException {
+ byte[] single = new byte[1];
+ int read = read(single, 0, 1);
+ return read < 0 ? -1 : single[0] & 0xFF;
+ }
+
+ @Override
+ public int read(byte[] buffer, int offset, int length) throws IOException {
+ if (length == 0) {
+ return 0;
+ }
+ if (remainingInChunk == 0 && !nextChunk()) {
+ return -1;
+ }
+ int read = in.read(buffer, offset, (int) Math.min(length, remainingInChunk));
+ if (read < 0) {
+ throw incompleteBody();
+ }
+ remainingInChunk -= read;
+ if (remainingInChunk == 0) {
+ readLineBreak();
+ }
+ return read;
+ }
+
+ @Override
+ public void close() throws IOException {
+ in.close();
+ }
+
+ /** Reads the next chunk header; false once the final chunk and its trailer are consumed. */
+ private boolean nextChunk() throws IOException {
+ if (finished) {
+ return false;
+ }
+ String header = readLine();
+ int semicolon = header.indexOf(';');
+ String hexSize = (semicolon >= 0 ? header.substring(0, semicolon) : header).trim();
+ long size;
+ try {
+ size = Long.parseLong(hexSize, 16);
+ } catch (NumberFormatException e) {
+ throw incompleteBody();
+ }
+ if (size < 0) {
+ throw incompleteBody();
+ }
+ if (size > 0) {
+ remainingInChunk = size;
+ return true;
+ }
+ // The final chunk: trailer lines such as x-amz-checksum-crc32, if any, then the empty line
+ // that ends the body.
+ String trailer = readLine();
+ while (!trailer.isEmpty()) {
+ trailer = readLine();
+ }
+ finished = true;
+ return false;
+ }
+
+ private void readLineBreak() throws IOException {
+ int next = in.read();
+ if (next == '\r') {
+ next = in.read();
+ }
+ if (next != '\n') {
+ throw incompleteBody();
+ }
+ }
+
+ /** The next line, trimmed and without its line break; the body may not end inside one. */
+ private String readLine() throws IOException {
+ ByteArrayOutputStream line = new ByteArrayOutputStream();
+ for (int next = in.read(); next != '\n'; next = in.read()) {
+ if (next < 0 || line.size() >= MAX_LINE_LENGTH) {
+ throw incompleteBody();
+ }
+ line.write(next);
+ }
+ return line.toString(StandardCharsets.ISO_8859_1).trim();
+ }
+
+ static AwsException incompleteBody() {
+ return new AwsException("IncompleteBody",
+ "You did not provide the number of bytes specified by the Content-Length HTTP header.", 400);
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/DigestingInputStream.java b/src/main/java/io/github/hectorvent/floci/services/s3/DigestingInputStream.java
new file mode 100644
index 0000000000..3cedc5f72f
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/DigestingInputStream.java
@@ -0,0 +1,115 @@
+package io.github.hectorvent.floci.services.s3;
+
+import io.github.hectorvent.floci.services.s3.model.ChecksumAlgorithm;
+import io.github.hectorvent.floci.services.s3.model.S3Checksum;
+
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+import java.util.Collection;
+import java.util.EnumMap;
+import java.util.HexFormat;
+import java.util.Map;
+
+/**
+ * Passes an upload body through while computing its MD5, the checksums it is checked against or
+ * stored with, and its size, so a body can be validated and stored without being held whole.
+ */
+final class DigestingInputStream extends FilterInputStream {
+
+ private final MessageDigest md5;
+ private final Map<ChecksumAlgorithm, S3Checksum.Calculator> calculators = new EnumMap<>(ChecksumAlgorithm.class);
+ private final Map<ChecksumAlgorithm, String> checksums = new EnumMap<>(ChecksumAlgorithm.class);
+ private byte[] md5Digest;
+ private long size;
+
+ DigestingInputStream(InputStream in, Collection<ChecksumAlgorithm> algorithms) {
+ super(in);
+ try {
+ this.md5 = MessageDigest.getInstance("MD5");
+ } catch (NoSuchAlgorithmException e) {
+ throw new IllegalStateException("MD5 algorithm is not available", e);
+ }
+ for (ChecksumAlgorithm algorithm : algorithms) {
+ calculators.put(algorithm, S3Checksum.calculator(algorithm));
+ }
+ }
+
+ @Override
+ public int read() throws IOException {
+ int next = super.read();
+ if (next >= 0) {
+ byte[] single = {(byte) next};
+ update(single, 0, 1);
+ }
+ return next;
+ }
+
+ @Override
+ public int read(byte[] buffer, int offset, int length) throws IOException {
+ int read = super.read(buffer, offset, length);
+ if (read > 0) {
+ update(buffer, offset, read);
+ }
+ return read;
+ }
+
+ @Override
+ public long skip(long n) throws IOException {
+ // Skipped bytes would be missing from the digests, so they are read instead.
+ byte[] buffer = new byte[8192];
+ long skipped = 0;
+ while (skipped < n) {
+ int read = read(buffer, 0, (int) Math.min(buffer.length, n - skipped));
+ if (read < 0) {
+ break;
+ }
+ skipped += read;
+ }
+ return skipped;
+ }
+
+ @Override
+ public boolean markSupported() {
+ return false;
+ }
+
+ long size() {
+ return size;
+ }
+
+ /** The MD5 of everything read; reading on after asking for it is not supported. */
+ byte[] md5() {
+ if (md5Digest == null) {
+ md5Digest = md5.digest();
+ }
+ return md5Digest.clone();
+ }
+
+ /** The MD5 as an S3 ETag, quoted lowercase hex. */
+ String eTag() {
+ return "\"" + HexFormat.of().formatHex(md5()) + "\"";
+ }
+
+ /**
+ * The Base64 checksum of everything read, for an algorithm the stream was created with. Like
+ * {@link #md5}, it is final once asked for.
+ */
+ String checksum(ChecksumAlgorithm algorithm) {
+ S3Checksum.Calculator calculator = calculators.get(algorithm);
+ if (calculator == null) {
+ throw new IllegalArgumentException(algorithm + " was not computed for this body");
+ }
+ return checksums.computeIfAbsent(algorithm, ignored -> calculator.value());
+ }
+
+ private void update(byte[] buffer, int offset, int length) {
+ md5.update(buffer, offset, length);
+ for (S3Checksum.Calculator calculator : calculators.values()) {
+ calculator.update(buffer, offset, length);
+ }
+ size += length;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
index 08e0187b6b..0057083e03 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
@@ -47,12 +47,14 @@ import jakarta.ws.rs.core.MultivaluedMap;
import jakarta.ws.rs.core.Response;
import jakarta.ws.rs.core.StreamingOutput;
import jakarta.ws.rs.core.UriInfo;
+import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.EOFException;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.io.UncheckedIOException;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.security.MessageDigest;
@@ -64,6 +66,7 @@ import java.time.format.DateTimeParseException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Base64;
+import java.util.EnumMap;
import java.util.HashMap;
import java.util.Iterator;
import java.util.LinkedHashMap;
@@ -416,7 +419,7 @@ public class S3Controller {
? Integer.parseInt(uriInfo.getQueryParameters().getFirst("partNumber")) : null,
uriInfo,
httpHeaders,
- body);
+ body != null ? new ByteArrayInputStream(body) : null);
}
if (bucket.length() > 63) {
throw new AwsException("InvalidBucketName", "The specified bucket is not valid.", 400);
@@ -798,7 +801,7 @@ public class S3Controller {
@QueryParam("partNumber") Integer partNumber,
@Context UriInfo uriInfo,
@Context HttpHeaders httpHeaders,
- byte[] body) {
+ InputStream body) {
try {
key = extractObjectKey(uriInfo, bucket, key);
S3Service.RequestAuthorization authorization = S3RequestAuthorizationParser.parseIfRequired(
@@ -806,7 +809,7 @@ public class S3Controller {
if (hasQueryParam(uriInfo, "tagging")) {
s3Service.authorizeObjectWrite(bucket, key, "s3:PutObjectTagging", authorization);
- return handlePutObjectTagging(bucket, key, body);
+ return handlePutObjectTagging(bucket, key, readWholeBody(body));
}
if (hasQueryParam(uriInfo, "retention")) {
s3Service.authorizeObjectWrite(bucket, key, "s3:PutObjectRetention", authorization);
@@ -814,17 +817,17 @@ public class S3Controller {
s3Service.authorizeObjectWrite(bucket, key, "s3:BypassGovernanceRetention", authorization);
}
return handlePutObjectRetention(bucket, key,
- uriInfo.getQueryParameters().getFirst("versionId"), httpHeaders, body);
+ uriInfo.getQueryParameters().getFirst("versionId"), httpHeaders, readWholeBody(body));
}
if (hasQueryParam(uriInfo, "legal-hold")) {
s3Service.authorizeObjectWrite(bucket, key, "s3:PutObjectLegalHold", authorization);
return handlePutObjectLegalHold(bucket, key,
- uriInfo.getQueryParameters().getFirst("versionId"), body);
+ uriInfo.getQueryParameters().getFirst("versionId"), readWholeBody(body));
}
if (hasQueryParam(uriInfo, "acl")) {
s3Service.authorizeObjectWrite(bucket, key, "s3:PutObjectAcl", authorization);
s3Service.putObjectAcl(bucket, key, uriInfo.getQueryParameters().getFirst("versionId"),
- new String(body, StandardCharsets.UTF_8),
+ new String(readWholeBody(body), StandardCharsets.UTF_8),
httpHeaders.getHeaderString("x-amz-acl"),
httpHeaders.getHeaderString("x-amz-grant-read"),
httpHeaders.getHeaderString("x-amz-grant-write"),
@@ -836,7 +839,7 @@ public class S3Controller {
if (hasQueryParam(uriInfo, "annotation")) {
s3Service.authorizeObjectWrite(bucket, key, "s3:PutObjectAnnotation", authorization);
- return handlePutObjectAnnotation(bucket, key, body, uriInfo, httpHeaders);
+ return handlePutObjectAnnotation(bucket, key, readWholeBody(body), uriInfo, httpHeaders);
}
if (uploadId != null && partNumber != null) {
@@ -845,9 +848,8 @@ public class S3Controller {
return handleUploadPartCopy(
copySource, bucket, key, uploadId, partNumber, httpHeaders, authorization);
}
- byte[] partData = decodeAwsChunked(body, contentEncoding, contentSha256);
- validateChecksumHeaders(httpHeaders, uriInfo, partData, getChecksumAlgorithm(httpHeaders, uriInfo));
- Part part = s3Service.storePart(bucket, key, uploadId, partNumber, partData,
+ Part part = s3Service.storePart(bucket, key, uploadId, partNumber,
+ decodedBody(body, contentEncoding, contentSha256), uploadChecksums(httpHeaders, uriInfo),
httpHeaders.getHeaderString("x-amz-server-side-encryption-customer-algorithm"),
httpHeaders.getHeaderString("x-amz-server-side-encryption-customer-key"),
httpHeaders.getHeaderString("x-amz-server-side-encryption-customer-key-MD5"));
@@ -875,9 +877,7 @@ public class S3Controller {
String legalHold = httpHeaders.getHeaderString("x-amz-object-lock-legal-hold");
Instant retainUntil = retainUntilStr != null ? Instant.parse(retainUntilStr) : null;
- byte[] data = decodeAwsChunked(body, contentEncoding, contentSha256);
String checksumAlgorithm = getChecksumAlgorithm(httpHeaders, uriInfo);
- validateChecksumHeaders(httpHeaders, uriInfo, data, checksumAlgorithm);
String persistedEncoding = toPersistedContentEncoding(contentEncoding);
String contentDisposition = httpHeaders.getHeaderString("Content-Disposition");
String cacheControl = httpHeaders.getHeaderString("Cache-Control");
@@ -888,7 +888,8 @@ public class S3Controller {
String sseCustomerKeyMd5 = httpHeaders.getHeaderString("x-amz-server-side-encryption-customer-key-MD5");
String cannedAcl = httpHeaders.getHeaderString("x-amz-acl");
s3Service.authorizePutObject(bucket, key, authorization);
- S3Object obj = s3Service.putObject(bucket, key, data, contentType, extractUserMetadata(httpHeaders, uriInfo),
+ S3Object obj = s3Service.putObject(bucket, key, decodedBody(body, contentEncoding, contentSha256),
+ uploadChecksums(httpHeaders, uriInfo), contentType, extractUserMetadata(httpHeaders, uriInfo),
new PutObjectOptions()
.withStorageClass(httpHeaders.getHeaderString("x-amz-storage-class"))
.withContentEncoding(persistedEncoding)
@@ -918,7 +919,7 @@ public class S3Controller {
resp.header("x-amz-version-id", obj.getVersionId());
}
appendPutObjectResponseHeaders(resp, obj);
- emitCloudTrailEvent("PutObject", bucket, key, data == null ? 0 : data.length, 0L, null, null);
+ emitCloudTrailEvent("PutObject", bucket, key, obj.getSize(), 0L, null, null);
return resp.build();
} catch (AwsException e) {
emitCloudTrailEvent("PutObject", bucket, key, 0L, 0L, e.getErrorCode(), e.getMessage());
@@ -2097,22 +2098,61 @@ public class S3Controller {
// --- AWS Chunked Decoding ---
+ /**
+ * The object data of an upload body, read as it arrives. A streaming payload is decoded from its
+ * aws-chunked framing on the way through. A Content-Encoding naming aws-chunked without that
+ * declaration does not guarantee framing, so that body is read whole and kept as sent when it
+ * does not decode.
+ */
+ private InputStream decodedBody(InputStream body, String contentEncoding, String contentSha256) {
+ InputStream raw = body != null ? body : InputStream.nullInputStream();
+ if (declaresStreamingPayload(contentSha256)) {
+ return new AwsChunkedInputStream(raw);
+ }
+ if (contentEncoding != null && contentEncoding.toLowerCase(Locale.ROOT).contains("aws-chunked")) {
+ return new ByteArrayInputStream(decodeAwsChunked(readWholeBody(raw), contentEncoding, contentSha256));
+ }
+ return raw;
+ }
+
+ /** The whole of a body that is small by nature, such as a subresource's XML, or must be read whole to be decoded. */
+ private static byte[] readWholeBody(InputStream body) {
+ if (body == null) {
+ return new byte[0];
+ }
+ try {
+ return body.readAllBytes();
+ } catch (IOException e) {
+ throw new UncheckedIOException("Failed to read the request body", e);
+ }
+ }
+
/**
* Decodes aws-chunked transfer encoding used by AWS SDK v2 with SigV4 chunk signing.
* Format: hex-size;chunk-signature=sig\r\n data \r\n ... 0;chunk-signature=sig\r\n
*/
private byte[] decodeAwsChunked(byte[] body, String contentEncoding, String contentSha256) {
- boolean declaresStreamingPayload = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD".equals(contentSha256)
- || "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER".equals(contentSha256)
- || "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD".equals(contentSha256)
- || "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER".equals(contentSha256)
- || "STREAMING-UNSIGNED-PAYLOAD-TRAILER".equals(contentSha256);
+ boolean declaresStreamingPayload = declaresStreamingPayload(contentSha256);
boolean isAwsChunked = declaresStreamingPayload || (contentEncoding != null
&& contentEncoding.toLowerCase(Locale.ROOT).contains("aws-chunked"));
if (!isAwsChunked) {
return body;
}
+ if (declaresStreamingPayload) {
+ // A streaming payload is always framed, so a body that breaks the framing is rejected
+ // rather than stored with its chunk headers as the object's content.
+ try (AwsChunkedInputStream decoded = new AwsChunkedInputStream(new ByteArrayInputStream(body))) {
+ return decoded.readAllBytes();
+ } catch (AwsException e) {
+ LOG.debugv("Rejecting malformed aws-chunked body: {0}", e.getMessage());
+ throw e;
+ } catch (IOException e) {
+ throw new UncheckedIOException("Failed to decode an aws-chunked body", e);
+ }
+ }
+ // Without that declaration, a Content-Encoding naming aws-chunked does not guarantee
+ // framing, so a body that does not decode is kept as sent.
try {
ByteArrayOutputStream out = new ByteArrayOutputStream();
String raw = new String(body, StandardCharsets.ISO_8859_1);
@@ -2126,8 +2166,7 @@ public class S3Controller {
String hexSize = semiColon >= 0 ? line.substring(0, semiColon) : line;
int chunkSize = Integer.parseInt(hexSize.trim(), 16);
if (chunkSize == 0) {
- // A streaming body ends with an empty line, after any trailer headers.
- complete = !declaresStreamingPayload || reachesEmptyLine(raw, lineEnd + 1);
+ complete = true;
break;
}
@@ -2138,43 +2177,25 @@ public class S3Controller {
pos = dataStart + chunkSize;
if (pos < raw.length() && raw.charAt(pos) == '\r') pos++;
- boolean lineBreak = pos < raw.length() && raw.charAt(pos) == '\n';
- if (lineBreak) pos++;
- if (declaresStreamingPayload && !lineBreak) {
- throw new IllegalArgumentException("aws-chunked chunk is not followed by a line break");
- }
+ if (pos < raw.length() && raw.charAt(pos) == '\n') pos++;
}
if (!complete) {
throw new IllegalArgumentException("aws-chunked body ends before its final chunk");
}
return out.toByteArray();
} catch (Exception e) {
- // A streaming payload is always framed, so storing it raw would save the chunk
- // headers as the object's content. Without that declaration, a Content-Encoding
- // naming aws-chunked does not guarantee framing, and the body is kept as sent.
- if (declaresStreamingPayload) {
- LOG.debugv("Rejecting malformed aws-chunked body: {0}", e.getMessage());
- throw new AwsException("IncompleteBody",
- "You did not provide the number of bytes specified by the Content-Length HTTP header.",
- 400);
- }
LOG.debugv("Failed to decode aws-chunked body, using raw: {0}", e.getMessage());
return body;
}
}
- private static boolean reachesEmptyLine(String raw, int pos) {
- while (pos < raw.length()) {
- int lineEnd = raw.indexOf('\n', pos);
- if (lineEnd < 0) {
- return false;
- }
- if (raw.substring(pos, lineEnd).trim().isEmpty()) {
- return true;
- }
- pos = lineEnd + 1;
- }
- return false;
+ /** Whether x-amz-content-sha256 declares a streaming payload, which is always aws-chunked. */
+ static boolean declaresStreamingPayload(String contentSha256) {
+ return "STREAMING-AWS4-HMAC-SHA256-PAYLOAD".equals(contentSha256)
+ || "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER".equals(contentSha256)
+ || "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD".equals(contentSha256)
+ || "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER".equals(contentSha256)
+ || "STREAMING-UNSIGNED-PAYLOAD-TRAILER".equals(contentSha256);
}
/**
@@ -2406,7 +2427,7 @@ public class S3Controller {
String versionId = uriInfo.getQueryParameters().getFirst("versionId");
String algorithmHeader = getChecksumAlgorithm(httpHeaders, uriInfo);
ChecksumAlgorithm algorithm = ChecksumAlgorithm.fromWireValue(algorithmHeader);
- validateChecksumHeaders(httpHeaders, uriInfo, payload, algorithmHeader);
+ validateChecksumHeaders(httpHeaders, uriInfo, payload);
ObjectAnnotation annotation = s3Service.putObjectAnnotation(bucket, key, annotationName,
versionId, payload, httpHeaders.getHeaderString("x-amz-object-if-match"), algorithm);
Response.ResponseBuilder response = Response.ok(putObjectAnnotationXml(annotation))
@@ -3114,59 +3135,27 @@ public class S3Controller {
return algorithm;
}
- private void validateChecksumHeaders(HttpHeaders httpHeaders, UriInfo uriInfo, byte[] data, String algorithm) {
- validateContentMd5(httpHeaders.getHeaderString("Content-MD5"), data);
-
- String sha1 = resolveHeaderOrQueryParam(httpHeaders, uriInfo, "x-amz-checksum-sha1");
- if (sha1 != null && !sha1.equals(S3Checksum.sha1Base64(data))) {
- throw new AwsException("BadDigest", "The SHA1 checksum you specified did not match the payload.", 400);
- }
-
- String sha256 = resolveHeaderOrQueryParam(httpHeaders, uriInfo, "x-amz-checksum-sha256");
- if (sha256 != null && !sha256.equals(S3Checksum.sha256Base64(data))) {
- throw new AwsException("BadDigest", "The SHA256 checksum you specified did not match the payload.", 400);
- }
-
- String crc32 = resolveHeaderOrQueryParam(httpHeaders, uriInfo, "x-amz-checksum-crc32");
- if (crc32 != null && !crc32.equals(S3Checksum.crc32Base64(data))) {
- throw new AwsException("BadDigest", "The CRC32 checksum you specified did not match the payload.", 400);
- }
-
- String crc32c = resolveHeaderOrQueryParam(httpHeaders, uriInfo, "x-amz-checksum-crc32c");
- if (crc32c != null && !crc32c.equals(S3Checksum.crc32cBase64(data))) {
- throw new AwsException("BadDigest", "The CRC32C checksum you specified did not match the payload.", 400);
- }
-
- String crc64nvme = resolveHeaderOrQueryParam(httpHeaders, uriInfo, "x-amz-checksum-crc64nvme");
- if (crc64nvme != null && !crc64nvme.equals(S3Checksum.crc64NvmeBase64(data))) {
- throw new AwsException("BadDigest", "The CRC64NVME checksum you specified did not match the payload.", 400);
- }
- }
-
- // S3 answers InvalidDigest for a Content-MD5 that is not the base64 of a 16-byte digest, and
- // BadDigest for a well-formed one that does not match the payload.
- private static void validateContentMd5(String contentMd5, byte[] data) {
- if (contentMd5 == null) {
- return;
- }
- byte[] expected;
+ private void validateChecksumHeaders(HttpHeaders httpHeaders, UriInfo uriInfo, byte[] data) {
+ UploadChecksums checksums = uploadChecksums(httpHeaders, uriInfo);
+ checksums.requireWellFormedContentMd5();
+ byte[] md5;
try {
- expected = Base64.getDecoder().decode(contentMd5.trim());
- } catch (IllegalArgumentException e) {
- expected = null;
- }
- if (expected == null || expected.length != 16) {
- throw new AwsException("InvalidDigest", "The Content-MD5 you specified is not valid.", 400);
- }
- byte[] actual;
- try {
- actual = MessageDigest.getInstance("MD5").digest(data);
+ md5 = MessageDigest.getInstance("MD5").digest(data);
} catch (NoSuchAlgorithmException e) {
throw new IllegalStateException("MD5 algorithm is not available", e);
}
- if (!MessageDigest.isEqual(expected, actual)) {
- throw new AwsException("BadDigest", "The Content-MD5 you specified did not match what we received.", 400);
+ checksums.verify(md5, claimedAlgorithm -> claimedAlgorithm.compute(data));
+ }
+
+ private UploadChecksums uploadChecksums(HttpHeaders httpHeaders, UriInfo uriInfo) {
+ Map<ChecksumAlgorithm, String> claimed = new EnumMap<>(ChecksumAlgorithm.class);
+ for (ChecksumAlgorithm algorithm : ChecksumAlgorithm.values()) {
+ String value = resolveHeaderOrQueryParam(httpHeaders, uriInfo, "x-amz-checksum-" + algorithm.wireValue());
+ if (value != null) {
+ claimed.put(algorithm, value);
+ }
}
+ return new UploadChecksums(httpHeaders.getHeaderString("Content-MD5"), claimed);
}
static boolean isWebsiteRequest(HttpHeaders httpHeaders, UriInfo uriInfo) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
index d51bf6156b..d4e596c4fd 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
@@ -40,6 +40,7 @@ import jakarta.ws.rs.core.MultivaluedHashMap;
import org.jboss.logging.Logger;
import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
@@ -65,6 +66,7 @@ import java.util.Arrays;
import java.util.Base64;
import java.util.Collections;
import java.util.Comparator;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.HexFormat;
import java.util.LinkedHashMap;
@@ -215,6 +217,10 @@ public class S3Service implements Resettable, ResourceProvider {
}
// Memory mode keeps an object in one byte array; this is the largest the JDK reliably allocates.
private static final long MAX_IN_MEMORY_OBJECT_SIZE = Integer.MAX_VALUE - 8;
+ // Directory under .multipart where a streamed part or object body is written before it is
+ // published. Upload IDs are UUIDs, so it never collides with an upload's own directory, and the
+ // sweep at startup and reset clears it with them.
+ private static final String INCOMING_BODIES = ".incoming";
// Upload ID to part bytes, keyed by each part's storage ID.
private final ConcurrentHashMap<String, Map<String, byte[]>> memoryMultipartStore = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, MultipartUpload> multipartUploads = new ConcurrentHashMap<>();
@@ -618,6 +624,62 @@ public class S3Service implements Resettable, ResourceProvider {
return createObject(bucketName, key, data, contentType, metadata, options, "ObjectCreated:Put");
}
+ /**
+ * Stores an object streamed from {@code body}, checked against {@code checksums}, without holding
+ * it in memory in the disk-backed modes. Everything the request alone decides is checked before
+ * any of the body is read, so a write that cannot succeed fails without transferring it. The body
+ * is staged on disk with its ETag and checksum worked out as it arrives, then moved into place
+ * like an assembled multipart object.
+ */
+ S3Object putObject(String bucketName, String key, InputStream body, UploadChecksums checksums,
+ String contentType, Map<String, String> metadata, PutObjectOptions options) {
+ PutObjectOptions effectiveOptions = options != null ? options : new PutObjectOptions();
+ checksums.requireWellFormedContentMd5();
+ AccountAwareStorageBackend.OwnedEntry<Bucket> checkedBucket = resolveBucketEntry(bucketName)
+ .orElseThrow(() -> new AwsException("NoSuchBucket",
+ "The specified bucket does not exist.", 404));
+ rejectConflictingServerSideEncryption(
+ normalizeServerSideEncryption(effectiveOptions.getServerSideEncryption()),
+ validateSseCustomerKey(effectiveOptions.getSseCustomerAlgorithm(),
+ effectiveOptions.getSseCustomerKey(), effectiveOptions.getSseCustomerKeyMd5()));
+ ChecksumAlgorithm declared = ChecksumAlgorithm.fromWireValue(effectiveOptions.getChecksumAlgorithm());
+ // Checked again under the bucket lock when the object is stored; this only spares a transfer.
+ checkWritePreconditions(bucketName, key, effectiveOptions.getIfMatch(), effectiveOptions.getIfNoneMatch());
+ if (inMemory) {
+ byte[] data = readVerified(body, checksums, "PutObject of " + bucketName + "/" + key);
+ S3Object object = storeObject(checkedBucket, bucketName, key, new BytesBody(data),
+ contentType, metadata, null, null, effectiveOptions, null);
+ fireNotifications(bucketName, key, "ObjectCreated:Put", object);
+ return object;
+ }
+
+ // A checksum the client sent is stored as sent; otherwise the declared algorithm's, or CRC64NVME.
+ ChecksumAlgorithm computed = effectiveOptions.getClientChecksum() != null ? null
+ : declared != null ? declared : ChecksumAlgorithm.CRC64NVME;
+ Set<ChecksumAlgorithm> algorithms = EnumSet.noneOf(ChecksumAlgorithm.class);
+ algorithms.addAll(checksums.algorithms());
+ if (computed != null) {
+ algorithms.add(computed);
+ }
+ DigestingInputStream digests = new DigestingInputStream(body, algorithms);
+ Path staged = stageBody(digests);
+ try {
+ checksums.verify(digests.md5(), digests::checksum);
+ S3Checksum checksum = null;
+ if (computed != null) {
+ checksum = new S3Checksum();
+ checksum.setValueFor(computed, digests.checksum(computed));
+ checksum.setChecksumType(ChecksumType.FULL_OBJECT);
+ }
+ S3Object object = storeObject(checkedBucket, bucketName, key, new AssembledBody(staged, digests.size()),
+ contentType, metadata, checksum, null, effectiveOptions, digests.eTag());
+ fireNotifications(bucketName, key, "ObjectCreated:Put", object);
+ return object;
+ } finally {
+ deleteQuietly(staged, "staged body of an object that was not stored");
+ }
+ }
+
public S3Object postObject(String bucketName, String key, byte[] data,
String contentType, Map<String, String> metadata) {
return postObject(bucketName, key, data, contentType, metadata, new PutObjectOptions());
@@ -671,19 +733,39 @@ public class S3Service implements Resettable, ResourceProvider {
private S3Object storeObject(String bucketName, String key, ObjectBody body,
String contentType, Map<String, String> metadata,
S3Checksum checksum, List<Part> parts, PutObjectOptions options, String eTag) {
+ return storeObject(null, bucketName, key, body, contentType, metadata, checksum, parts, options, eTag);
+ }
+
+ /**
+ * Stores the object, and with {@code checkedBucket} given, only in that bucket: an upload checked
+ * against a bucket that was deleted while its body arrived fails with NoSuchBucket rather than
+ * landing in a bucket created under the same name since, whose policy it was never checked
+ * against. A bucket is created once and keeps its instance until it is deleted, so the instance
+ * tells one bucket of a name from another.
+ */
+ private S3Object storeObject(AccountAwareStorageBackend.OwnedEntry<Bucket> checkedBucket,
+ String bucketName, String key, ObjectBody body,
+ String contentType, Map<String, String> metadata,
+ S3Checksum checksum, List<Part> parts, PutObjectOptions options, String eTag) {
AccountAwareStorageBackend.OwnedEntry<Bucket> ownedBucket = resolveBucketEntry(bucketName)
.orElseThrow(() -> new AwsException("NoSuchBucket",
"The specified bucket does not exist.", 404));
Bucket bucket = ownedBucket.value();
synchronized (bucket) {
+ // Rechecked under the bucket's monitor, which deleting the bucket also holds.
+ if (checkedBucket != null && resolveBucketEntry(bucketName)
+ .map(current -> current.value() != checkedBucket.value())
+ .orElse(true)) {
+ throw new AwsException("NoSuchBucket", "The specified bucket does not exist.", 404);
+ }
return storeObjectInternal(ownedBucket.account(), bucket, bucketName, key, body,
contentType, metadata, checksum, parts, options, eTag);
}
}
/**
- * The body of an object being stored: the bytes of a single-request write, or the file a
- * multipart upload was assembled into, which is moved into place instead of being read back.
+ * The body of an object being stored: bytes already in memory, or a file on disk, which is moved
+ * into place instead of being read back.
*/
private sealed interface ObjectBody permits BytesBody, AssembledBody {
long size();
@@ -697,9 +779,10 @@ public class S3Service implements Resettable, ResourceProvider {
}
/**
- * A body already on disk that is moved into place: an assembled multipart object, or the pinned
- * file of a copy source. It is stored with an ETag and checksum worked out beforehand, since
- * computing either here would mean reading the whole file under the bucket lock.
+ * A body already on disk that is moved into place: an assembled multipart object, the pinned file
+ * of a copy source, or a streamed PutObject body. It is stored with an ETag and checksum worked
+ * out beforehand, since computing either here would mean reading the whole file under the bucket
+ * lock.
*/
private record AssembledBody(Path file, long size) implements ObjectBody { }
@@ -3734,7 +3817,7 @@ public class S3Service implements Resettable, ResourceProvider {
.filter(part -> part.getPartNumber() != partNumber)
.mapToLong(Part::getSize)
.sum();
- requireFitsInMemory(uploadId, otherParts + data.length);
+ requireFitsInMemory("Multipart upload " + uploadId, otherParts + data.length);
memoryMultipartStore.get(uploadId).put(storageId, data);
} else {
Path partPath = dataRoot.resolve(".multipart").resolve(uploadId).resolve(storageId);
@@ -3758,6 +3841,104 @@ public class S3Service implements Resettable, ResourceProvider {
});
}
+ /**
+ * Stores one part streamed from {@code body}, checked against {@code checksums}, without holding it
+ * in memory in the disk-backed modes. The upload is checked before any of the body is read. The
+ * body is staged outside the upload's directory and outside any lock, so parts of one upload still
+ * arrive in parallel and a slow client never holds up the bucket's other multipart operations. Only
+ * once it has passed its checks is it moved into the upload and its record published, under the
+ * upload's lock, so an upload aborted or completed meanwhile fails the part with NoSuchUpload.
+ */
+ Part storePart(String bucket, String key, String uploadId, int partNumber, InputStream body,
+ UploadChecksums checksums,
+ String sseCustomerAlgorithm, String sseCustomerKey, String sseCustomerKeyMd5) {
+ checksums.requireWellFormedContentMd5();
+ ChecksumAlgorithm declared = withMultipartOperationLock(bucket, uploadId, () -> {
+ MultipartUpload upload = getMultipartUpload(bucket, key, uploadId);
+ if (partNumber < 1 || partNumber > 10000) {
+ throw new AwsException("InvalidArgument",
+ "Part number must be between 1 and 10000.", 400);
+ }
+ validateSseCustomerAccess(upload, sseCustomerAlgorithm, sseCustomerKey, sseCustomerKeyMd5);
+ return upload.getChecksumAlgorithm();
+ });
+ if (inMemory) {
+ byte[] data = readVerified(body, checksums, "Multipart upload " + uploadId);
+ return storePart(bucket, key, uploadId, partNumber, data,
+ sseCustomerAlgorithm, sseCustomerKey, sseCustomerKeyMd5);
+ }
+ ChecksumAlgorithm algorithm = declared != null ? declared : ChecksumAlgorithm.CRC64NVME;
+ Set<ChecksumAlgorithm> algorithms = EnumSet.of(algorithm);
+ algorithms.addAll(checksums.algorithms());
+ DigestingInputStream digests = new DigestingInputStream(body, algorithms);
+ Path staged = stageBody(digests);
+ try {
+ checksums.verify(digests.md5(), digests::checksum);
+ S3Checksum partChecksum = new S3Checksum();
+ partChecksum.setValueFor(algorithm, digests.checksum(algorithm));
+ return withMultipartOperationLock(bucket, uploadId, () -> {
+ MultipartUpload upload = getMultipartUpload(bucket, key, uploadId);
+ String storageId = partNumber + "-" + UUID.randomUUID();
+ try {
+ Files.move(staged, dataRoot.resolve(".multipart").resolve(uploadId).resolve(storageId),
+ StandardCopyOption.ATOMIC_MOVE);
+ } catch (IOException e) {
+ throw new UncheckedIOException("Failed to write multipart part", e);
+ }
+ Part part = new Part(partNumber, digests.eTag(), digests.size());
+ part.setChecksum(partChecksum);
+ part.setStorageId(storageId);
+ Part replaced = upload.getParts().put(partNumber, part);
+ if (replaced != null) {
+ discardPartBytes(uploadId, replaced);
+ }
+ LOG.debugv("Uploaded part {0} for upload {1} ({2} bytes)", partNumber, uploadId, digests.size());
+ return part;
+ });
+ } finally {
+ deleteQuietly(staged, "staged part of an upload that was not stored");
+ }
+ }
+
+ /**
+ * Reads a memory-mode upload body into one array, checked against {@code checksums}, failing
+ * with EntityTooLarge once it outgrows the largest array the JDK allocates.
+ */
+ private static byte[] readVerified(InputStream body, UploadChecksums checksums, String upload) {
+ DigestingInputStream digests = new DigestingInputStream(body, checksums.algorithms());
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ byte[] buffer = new byte[64 * 1024];
+ try {
+ for (int read = digests.read(buffer); read >= 0; read = digests.read(buffer)) {
+ requireFitsInMemory(upload, (long) out.size() + read);
+ out.write(buffer, 0, read);
+ }
+ } catch (IOException e) {
+ throw new UncheckedIOException("Failed to read the upload body", e);
+ }
+ checksums.verify(digests.md5(), digests::checksum);
+ return out.toByteArray();
+ }
+
+ /**
+ * Writes a streamed body to a new file under the staging directory and returns it. The file is
+ * deleted if the body fails partway, so on return the caller owns it.
+ */
+ private Path stageBody(InputStream body) {
+ Path staged = dataRoot.resolve(".multipart").resolve(INCOMING_BODIES).resolve(UUID.randomUUID().toString());
+ try {
+ Files.createDirectories(staged.getParent());
+ Files.copy(body, staged);
+ return staged;
+ } catch (IOException e) {
+ deleteQuietly(staged, "partly staged upload body");
+ throw new UncheckedIOException("Failed to write the upload body", e);
+ } catch (RuntimeException e) {
+ deleteQuietly(staged, "partly staged upload body");
+ throw e;
+ }
+ }
+
/**
* Drops the bytes of a part a re-upload replaced, once the new record is published, so no record
* ever names bytes that changed after it was written.
@@ -3940,7 +4121,7 @@ public class S3Service implements Resettable, ResourceProvider {
validateCompleteChecksumType(algorithm, storedChecksumType, ChecksumType.fromWireValue(checksumType));
if (inMemory) {
- requireFitsInMemory(uploadId, parts.stream().mapToLong(Part::getSize).sum());
+ requireFitsInMemory("Multipart upload " + uploadId, parts.stream().mapToLong(Part::getSize).sum());
}
// Concatenate parts in order
@@ -4020,10 +4201,10 @@ public class S3Service implements Resettable, ResourceProvider {
* object over its size limit, so it fails at the part that crosses the limit instead of running
* the heap out of memory or overflowing the array size at completion.
*/
- private static void requireFitsInMemory(String uploadId, long uploadSize) {
+ private static void requireFitsInMemory(String upload, long uploadSize) {
if (uploadSize > MAX_IN_MEMORY_OBJECT_SIZE) {
- LOG.warnv("Multipart upload {0} would hold {1} bytes, over the {2} an object can have in memory storage mode; set FLOCI_STORAGE_SERVICES_S3_MODE=persistent for larger objects",
- uploadId, uploadSize, MAX_IN_MEMORY_OBJECT_SIZE);
+ LOG.warnv("{0} would hold {1} bytes, over the {2} an object can have in memory storage mode; set FLOCI_STORAGE_SERVICES_S3_MODE=persistent for larger objects",
+ upload, uploadSize, MAX_IN_MEMORY_OBJECT_SIZE);
throw new AwsException("EntityTooLarge", "Your proposed upload exceeds the maximum allowed object size.", 400);
}
}
@@ -4043,7 +4224,7 @@ public class S3Service implements Resettable, ResourceProvider {
}
totalSize += partData[i].length;
}
- requireFitsInMemory(uploadId, totalSize);
+ requireFitsInMemory("Multipart upload " + uploadId, totalSize);
byte[] allData = new byte[(int) totalSize];
int offset = 0;
for (byte[] part : partData) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/UploadChecksums.java b/src/main/java/io/github/hectorvent/floci/services/s3/UploadChecksums.java
new file mode 100644
index 0000000000..e0d33ba820
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/UploadChecksums.java
@@ -0,0 +1,66 @@
+package io.github.hectorvent.floci.services.s3;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.s3.model.ChecksumAlgorithm;
+
+import java.security.MessageDigest;
+import java.util.Base64;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.function.Function;
+
+/**
+ * The integrity headers an upload body is checked against: Content-MD5 and the x-amz-checksum-*
+ * values. A malformed Content-MD5 is rejected before any of the body is read; the digests are
+ * compared once it has been.
+ */
+record UploadChecksums(String contentMd5, Map<ChecksumAlgorithm, String> claimed) {
+
+ // The order the checksums have always been checked in, so a body failing several reports the same one.
+ private static final List<ChecksumAlgorithm> CHECK_ORDER = List.of(ChecksumAlgorithm.SHA1,
+ ChecksumAlgorithm.SHA256, ChecksumAlgorithm.CRC32, ChecksumAlgorithm.CRC32C, ChecksumAlgorithm.CRC64NVME);
+
+ UploadChecksums {
+ claimed = Map.copyOf(claimed);
+ }
+
+ /** The algorithms the request claims a checksum for, which the body has to be hashed with. */
+ Set<ChecksumAlgorithm> algorithms() {
+ return claimed.keySet();
+ }
+
+ // S3 answers InvalidDigest for a Content-MD5 that is not the base64 of a 16-byte digest, and
+ // BadDigest for a well-formed one that does not match the payload.
+ void requireWellFormedContentMd5() {
+ if (contentMd5 != null) {
+ expectedMd5();
+ }
+ }
+
+ void verify(byte[] actualMd5, Function<ChecksumAlgorithm, String> actualChecksum) {
+ if (contentMd5 != null && !MessageDigest.isEqual(expectedMd5(), actualMd5)) {
+ throw new AwsException("BadDigest", "The Content-MD5 you specified did not match what we received.", 400);
+ }
+ for (ChecksumAlgorithm algorithm : CHECK_ORDER) {
+ String claimedValue = claimed.get(algorithm);
+ if (claimedValue != null && !claimedValue.equals(actualChecksum.apply(algorithm))) {
+ throw new AwsException("BadDigest", "The " + algorithm.name()
+ + " checksum you specified did not match the payload.", 400);
+ }
+ }
+ }
+
+ private byte[] expectedMd5() {
+ byte[] expected;
+ try {
+ expected = Base64.getDecoder().decode(contentMd5.trim());
+ } catch (IllegalArgumentException e) {
+ expected = null;
+ }
+ if (expected == null || expected.length != 16) {
+ throw new AwsException("InvalidDigest", "The Content-MD5 you specified is not valid.", 400);
+ }
+ return expected;
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/AwsChunkedInputStreamTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/AwsChunkedInputStreamTest.java
new file mode 100644
index 0000000000..a18282a093
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/AwsChunkedInputStreamTest.java
@@ -0,0 +1,96 @@
+package io.github.hectorvent.floci.services.s3;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Random;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+class AwsChunkedInputStreamTest {
+
+ @Test
+ void decodesSignedChunksUpToTheFinalChunk() throws IOException {
+ assertEquals("hello world", decode("5;chunk-signature=abc\r\nhello\r\n6;chunk-signature=def\r\n world\r\n"
+ + "0;chunk-signature=ghi\r\n\r\n"));
+ }
+
+ @Test
+ void skipsTrailerLinesAfterTheFinalChunk() throws IOException {
+ assertEquals("hello", decode("5\r\nhello\r\n0\r\nx-amz-checksum-crc32:NhCmhg==\r\n"
+ + "x-amz-trailer-signature:ghi\r\n\r\n"));
+ }
+
+ @Test
+ void acceptsBareLineFeeds() throws IOException {
+ assertEquals("hello", decode("5\nhello\n0\n\n"));
+ }
+
+ @Test
+ void decodesManyChunksWhateverTheReadSize() throws IOException {
+ Random random = new Random(5080);
+ byte[] payload = new byte[200_003];
+ random.nextBytes(payload);
+ ByteArrayOutputStream framed = new ByteArrayOutputStream();
+ for (int offset = 0; offset < payload.length; offset += 8192) {
+ int size = Math.min(8192, payload.length - offset);
+ framed.writeBytes((Integer.toHexString(size) + ";chunk-signature=sig\r\n").getBytes(StandardCharsets.US_ASCII));
+ framed.write(payload, offset, size);
+ framed.writeBytes("\r\n".getBytes(StandardCharsets.US_ASCII));
+ }
+ framed.writeBytes("0;chunk-signature=sig\r\n\r\n".getBytes(StandardCharsets.US_ASCII));
+
+ for (int readSize : List.of(1, 7, 4096, 65_536)) {
+ try (InputStream in = new AwsChunkedInputStream(new ByteArrayInputStream(framed.toByteArray()))) {
+ ByteArrayOutputStream decoded = new ByteArrayOutputStream();
+ byte[] buffer = new byte[readSize];
+ for (int read = in.read(buffer); read >= 0; read = in.read(buffer)) {
+ decoded.write(buffer, 0, read);
+ }
+ assertArrayEquals(payload, decoded.toByteArray(), "read size " + readSize);
+ }
+ }
+ }
+
+ @Test
+ void readsOneByteAtATime() throws IOException {
+ try (InputStream in = new AwsChunkedInputStream(new ByteArrayInputStream(
+ "3\r\nabc\r\n0\r\n\r\n".getBytes(StandardCharsets.US_ASCII)))) {
+ assertEquals('a', in.read());
+ assertEquals('b', in.read());
+ assertEquals('c', in.read());
+ assertEquals(-1, in.read());
+ assertEquals(-1, in.read(), "the stream stays at its end");
+ }
+ }
+
+ @Test
+ void brokenFramingIsAnIncompleteBody() {
+ for (String body : List.of(
+ "10\r\nonly-five\r\n",
+ "5\r\nhello\r\n",
+ "five\r\nhello\r\n0\r\n\r\n",
+ "5\r\nhello0\r\n\r\n",
+ "5\r\nhello\r\n0\r\nx-amz-checksum-crc32:NhCmhg==\r\n",
+ "5;chunk-signature=abc\r\nhello\r\n0;chunk-signature=def\r\n",
+ "-5\r\nhello\r\n0\r\n\r\n",
+ "")) {
+ AwsException error = assertThrows(AwsException.class, () -> decode(body), body);
+ assertEquals("IncompleteBody", error.getErrorCode(), body);
+ assertEquals(400, error.getHttpStatus(), body);
+ }
+ }
+
+ private static String decode(String framed) throws IOException {
+ try (InputStream in = new AwsChunkedInputStream(
+ new ByteArrayInputStream(framed.getBytes(StandardCharsets.ISO_8859_1)))) {
+ return new String(in.readAllBytes(), StandardCharsets.ISO_8859_1);
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3StreamedUploadTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3StreamedUploadTest.java
new file mode 100644
index 0000000000..02639f2979
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3StreamedUploadTest.java
@@ -0,0 +1,337 @@
+package io.github.hectorvent.floci.services.s3;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.services.s3.model.ChecksumAlgorithm;
+import io.github.hectorvent.floci.services.s3.model.ChecksumType;
+import io.github.hectorvent.floci.services.s3.model.MultipartUpload;
+import io.github.hectorvent.floci.services.s3.model.Part;
+import io.github.hectorvent.floci.services.s3.model.PutObjectOptions;
+import io.github.hectorvent.floci.services.s3.model.S3Object;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.ByteArrayInputStream;
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.security.MessageDigest;
+import java.time.Duration;
+import java.util.Base64;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Stream;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+/**
+ * PutObject and UploadPart bodies read as a stream, as the controller hands them over, so a
+ * disk-backed upload is never held in memory whole.
+ */
+class S3StreamedUploadTest {
+
+ private static final byte[] BODY = "a body that arrives as a stream".getBytes(StandardCharsets.UTF_8);
+ private static final UploadChecksums NO_CHECKSUMS = new UploadChecksums(null, Map.of());
+
+ @TempDir
+ Path tempDir;
+
+ private S3Service s3Service;
+ private Path dataRoot;
+
+ @BeforeEach
+ void setUp() {
+ dataRoot = tempDir.resolve("s3");
+ s3Service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), dataRoot, false);
+ s3Service.createBucket("bucket", "us-east-1");
+ }
+
+ @Test
+ void aStreamedPutStoresWhatTheBytePathStores() {
+ S3Object fromBytes = s3Service.putObject("bucket", "bytes.txt", BODY, "text/plain", Map.of());
+
+ S3Object streamed = s3Service.putObject("bucket", "streamed.txt", new ByteArrayInputStream(BODY),
+ NO_CHECKSUMS, "text/plain", Map.of(), new PutObjectOptions());
+
+ assertEquals(fromBytes.getETag(), streamed.getETag());
+ assertEquals(fromBytes.getChecksum().getChecksumCRC64NVME(), streamed.getChecksum().getChecksumCRC64NVME());
+ assertEquals(ChecksumType.FULL_OBJECT, streamed.getChecksum().getChecksumType());
+ assertEquals(BODY.length, streamed.getSize());
+ assertArrayEquals(BODY, s3Service.getObject("bucket", "streamed.txt").getData());
+ assertNothingStaged();
+ }
+
+ @Test
+ void aStreamedPutStoresTheDeclaredAlgorithmsChecksum() {
+ S3Object streamed = s3Service.putObject("bucket", "sha.txt", new ByteArrayInputStream(BODY),
+ NO_CHECKSUMS, "text/plain", Map.of(), new PutObjectOptions().withChecksumAlgorithm("SHA256"));
+
+ assertEquals(ChecksumAlgorithm.SHA256.compute(BODY), streamed.getChecksum().getChecksumSHA256());
+ assertNull(streamed.getChecksum().getChecksumCRC64NVME());
+ }
+
+ @Test
+ void aStreamedPutIntoAVersionedBucketKeepsEachVersion() {
+ s3Service.putBucketVersioning("bucket", "Enabled");
+ byte[] second = "the second version".getBytes(StandardCharsets.UTF_8);
+ S3Object first = s3Service.putObject("bucket", "versioned.txt", new ByteArrayInputStream(BODY),
+ NO_CHECKSUMS, "text/plain", Map.of(), new PutObjectOptions());
+ s3Service.putObject("bucket", "versioned.txt", new ByteArrayInputStream(second),
+ NO_CHECKSUMS, "text/plain", Map.of(), new PutObjectOptions());
+
+ assertArrayEquals(second, s3Service.getObject("bucket", "versioned.txt").getData());
+ assertArrayEquals(BODY, s3Service.getObject("bucket", "versioned.txt", first.getVersionId()).getData());
+ assertNothingStaged();
+ }
+
+ @Test
+ void aBodyThatFailsItsChecksumIsNotStored() {
+ UploadChecksums wrong = new UploadChecksums(null,
+ Map.of(ChecksumAlgorithm.CRC32, ChecksumAlgorithm.CRC32.compute("something else".getBytes(StandardCharsets.UTF_8))));
+
+ AwsException error = assertThrows(AwsException.class, () -> s3Service.putObject("bucket", "bad.txt",
+ new ByteArrayInputStream(BODY), wrong, "text/plain", Map.of(), new PutObjectOptions()));
+
+ assertEquals("BadDigest", error.getErrorCode());
+ assertEquals("NoSuchKey", assertThrows(AwsException.class,
+ () -> s3Service.headObject("bucket", "bad.txt")).getErrorCode());
+ assertNothingStaged();
+ }
+
+ @Test
+ void aBodyThatMatchesItsContentMd5AndChecksumIsStored() throws Exception {
+ String md5 = Base64.getEncoder().encodeToString(MessageDigest.getInstance("MD5").digest(BODY));
+ UploadChecksums right = new UploadChecksums(md5, Map.of(ChecksumAlgorithm.SHA1, ChecksumAlgorithm.SHA1.compute(BODY)));
+
+ s3Service.putObject("bucket", "good.txt", new ByteArrayInputStream(BODY), right,
+ "text/plain", Map.of(), new PutObjectOptions());
+
+ assertArrayEquals(BODY, s3Service.getObject("bucket", "good.txt").getData());
+ }
+
+ @Test
+ void aWriteThatCannotSucceedFailsBeforeTheBodyIsRead() {
+ s3Service.putObject("bucket", "existing.txt", BODY, "text/plain", Map.of());
+
+ assertEquals("PreconditionFailed", assertThrows(AwsException.class, () -> s3Service.putObject("bucket",
+ "existing.txt", unreadable(), NO_CHECKSUMS, "text/plain", Map.of(),
+ new PutObjectOptions().withIfNoneMatch("*"))).getErrorCode());
+ assertEquals("NoSuchBucket", assertThrows(AwsException.class, () -> s3Service.putObject("missing-bucket",
+ "key.txt", unreadable(), NO_CHECKSUMS, "text/plain", Map.of(),
+ new PutObjectOptions())).getErrorCode());
+ assertEquals("InvalidDigest", assertThrows(AwsException.class, () -> s3Service.putObject("bucket",
+ "key.txt", unreadable(), new UploadChecksums("not-an-md5", Map.of()), "text/plain", Map.of(),
+ new PutObjectOptions())).getErrorCode());
+ assertEquals("InvalidArgument", assertThrows(AwsException.class, () -> s3Service.putObject("bucket",
+ "key.txt", unreadable(), NO_CHECKSUMS, "text/plain", Map.of(),
+ new PutObjectOptions().withChecksumAlgorithm("NOT-AN-ALGORITHM"))).getErrorCode());
+ }
+
+ @Test
+ void aBodyThatBreaksPartwayLeavesNothingBehind() {
+ assertThrows(UncheckedIOException.class, () -> s3Service.putObject("bucket", "broken.txt",
+ breaksAfter(BODY), NO_CHECKSUMS, "text/plain", Map.of(), new PutObjectOptions()));
+
+ AwsException framing = assertThrows(AwsException.class, () -> s3Service.putObject("bucket", "framing.txt",
+ new AwsChunkedInputStream(new ByteArrayInputStream("10\r\nonly-five\r\n".getBytes(StandardCharsets.US_ASCII))),
+ NO_CHECKSUMS, "text/plain", Map.of(), new PutObjectOptions()));
+
+ assertEquals("IncompleteBody", framing.getErrorCode());
+ assertEquals("NoSuchKey", assertThrows(AwsException.class,
+ () -> s3Service.headObject("bucket", "broken.txt")).getErrorCode());
+ assertNothingStaged();
+ }
+
+ @Test
+ void aStreamedPartStoresWhatTheBytePathStores() {
+ byte[] part2 = "and a second part".getBytes(StandardCharsets.UTF_8);
+ MultipartUpload upload = s3Service.initiateMultipartUpload("bucket", "multipart.bin", null);
+ Part fromBytes = s3Service.storePart("bucket", "multipart.bin", upload.getUploadId(), 1, BODY, null, null, null);
+
+ Part streamed = s3Service.storePart("bucket", "multipart.bin", upload.getUploadId(), 2,
+ new ByteArrayInputStream(part2), NO_CHECKSUMS, null, null, null);
+ Part streamedAgain = s3Service.storePart("bucket", "multipart.bin", upload.getUploadId(), 1,
+ new ByteArrayInputStream(BODY), NO_CHECKSUMS, null, null, null);
+
+ assertEquals(fromBytes.getETag(), streamedAgain.getETag());
+ assertEquals(fromBytes.getChecksum().getChecksumCRC64NVME(), streamedAgain.getChecksum().getChecksumCRC64NVME());
+ assertEquals(part2.length, streamed.getSize());
+ s3Service.completeMultipartUpload("bucket", "multipart.bin", upload.getUploadId(), List.of(1, 2), null, null);
+ byte[] whole = new byte[BODY.length + part2.length];
+ System.arraycopy(BODY, 0, whole, 0, BODY.length);
+ System.arraycopy(part2, 0, whole, BODY.length, part2.length);
+ assertArrayEquals(whole, s3Service.getObject("bucket", "multipart.bin").getData());
+ assertNothingStaged();
+ }
+
+ @Test
+ void aPartThatFailsItsChecksumIsNotRecorded() {
+ MultipartUpload upload = s3Service.initiateMultipartUpload("bucket", "multipart.bin", null);
+ UploadChecksums wrong = new UploadChecksums(null,
+ Map.of(ChecksumAlgorithm.SHA256, ChecksumAlgorithm.SHA256.compute("other".getBytes(StandardCharsets.UTF_8))));
+
+ assertEquals("BadDigest", assertThrows(AwsException.class, () -> s3Service.storePart("bucket",
+ "multipart.bin", upload.getUploadId(), 1, new ByteArrayInputStream(BODY), wrong,
+ null, null, null)).getErrorCode());
+
+ assertTrue(s3Service.getMultipartUpload("bucket", "multipart.bin", upload.getUploadId()).getParts().isEmpty());
+ assertNothingStaged();
+ }
+
+ @Test
+ void aPartForAnUploadThatCannotTakeItFailsBeforeItsBodyIsRead() {
+ MultipartUpload upload = s3Service.initiateMultipartUpload("bucket", "multipart.bin", null);
+
+ assertEquals("NoSuchUpload", assertThrows(AwsException.class, () -> s3Service.storePart("bucket",
+ "multipart.bin", "no-such-upload", 1, unreadable(), NO_CHECKSUMS, null, null, null)).getErrorCode());
+ assertEquals("InvalidArgument", assertThrows(AwsException.class, () -> s3Service.storePart("bucket",
+ "multipart.bin", upload.getUploadId(), 10001, unreadable(), NO_CHECKSUMS, null, null, null)).getErrorCode());
+ }
+
+ @Test
+ void anUploadAbortedWhileAPartArrivesFailsThePartAndLeavesNothingBehind() {
+ MultipartUpload upload = s3Service.initiateMultipartUpload("bucket", "multipart.bin", null);
+ InputStream body = onFirstRead(BODY,
+ () -> s3Service.abortMultipartUpload("bucket", "multipart.bin", upload.getUploadId()));
+
+ assertEquals("NoSuchUpload", assertThrows(AwsException.class, () -> s3Service.storePart("bucket",
+ "multipart.bin", upload.getUploadId(), 1, body, NO_CHECKSUMS, null, null, null)).getErrorCode());
+
+ assertFalse(Files.exists(dataRoot.resolve(".multipart").resolve(upload.getUploadId())));
+ assertNothingStaged();
+ }
+
+ @Test
+ void aPartStillArrivingDoesNotHoldUpDeletingItsBucket() {
+ MultipartUpload upload = s3Service.initiateMultipartUpload("bucket", "multipart.bin", null);
+ // Deleting the bucket takes its multipart lock exclusively, so this would never return if the
+ // part held that lock while its body arrived.
+ InputStream body = onFirstRead(BODY, () -> s3Service.deleteBucket("bucket"));
+
+ AwsException error = assertTimeoutPreemptively(Duration.ofSeconds(30), () -> assertThrows(AwsException.class,
+ () -> s3Service.storePart("bucket", "multipart.bin", upload.getUploadId(), 1, body,
+ NO_CHECKSUMS, null, null, null)));
+
+ assertTrue(List.of("NoSuchUpload", "NoSuchBucket").contains(error.getErrorCode()), error.getErrorCode());
+ assertNothingStaged();
+ }
+
+ @Test
+ void memoryModeReadsTheStreamIntoTheBytePaths() {
+ S3Service memory = memoryService();
+ S3Object fromBytes = memory.putObject("bucket", "bytes.txt", BODY, "text/plain", Map.of());
+
+ S3Object streamed = memory.putObject("bucket", "streamed.txt", new ByteArrayInputStream(BODY),
+ NO_CHECKSUMS, "text/plain", Map.of(), new PutObjectOptions());
+ MultipartUpload upload = memory.initiateMultipartUpload("bucket", "multipart.bin", null);
+ memory.storePart("bucket", "multipart.bin", upload.getUploadId(), 1, new ByteArrayInputStream(BODY),
+ NO_CHECKSUMS, null, null, null);
+ memory.completeMultipartUpload("bucket", "multipart.bin", upload.getUploadId(), List.of(1), null, null);
+
+ assertEquals(fromBytes.getETag(), streamed.getETag());
+ assertArrayEquals(BODY, memory.getObject("bucket", "streamed.txt").getData());
+ assertArrayEquals(BODY, memory.getObject("bucket", "multipart.bin").getData());
+ UploadChecksums wrong = new UploadChecksums(null,
+ Map.of(ChecksumAlgorithm.CRC32C, ChecksumAlgorithm.CRC32C.compute("other".getBytes(StandardCharsets.UTF_8))));
+ assertEquals("BadDigest", assertThrows(AwsException.class, () -> memory.putObject("bucket", "bad.txt",
+ new ByteArrayInputStream(BODY), wrong, "text/plain", Map.of(), new PutObjectOptions())).getErrorCode());
+ }
+
+ @Test
+ void aBucketRecreatedWhileABodyArrivesDoesNotReceiveTheObject() {
+ for (S3Service service : List.of(s3Service, memoryService())) {
+ service.createBucket("recreated", "us-east-1");
+ // The request was checked against the first bucket; the second has its own policy.
+ InputStream body = onFirstRead(BODY, () -> {
+ service.deleteBucket("recreated");
+ service.createBucket("recreated", "us-east-1");
+ });
+
+ assertEquals("NoSuchBucket", assertThrows(AwsException.class, () -> service.putObject("recreated",
+ "object.txt", body, NO_CHECKSUMS, "text/plain", Map.of(), new PutObjectOptions())).getErrorCode());
+
+ assertEquals("NoSuchKey", assertThrows(AwsException.class,
+ () -> service.headObject("recreated", "object.txt")).getErrorCode());
+ }
+ assertNothingStaged();
+ }
+
+ @Test
+ void memoryModeChecksAPartsUploadBeforeReadingItsBody() {
+ S3Service memory = memoryService();
+ MultipartUpload upload = memory.initiateMultipartUpload("bucket", "multipart.bin", null);
+
+ assertEquals("NoSuchUpload", assertThrows(AwsException.class, () -> memory.storePart("bucket",
+ "multipart.bin", "no-such-upload", 1, unreadable(), NO_CHECKSUMS, null, null, null)).getErrorCode());
+ assertEquals("InvalidArgument", assertThrows(AwsException.class, () -> memory.storePart("bucket",
+ "multipart.bin", upload.getUploadId(), 0, unreadable(), NO_CHECKSUMS, null, null, null)).getErrorCode());
+ }
+
+ private S3Service memoryService() {
+ S3Service memory = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), dataRoot, true);
+ memory.createBucket("bucket", "us-east-1");
+ return memory;
+ }
+
+ private void assertNothingStaged() {
+ Path incoming = dataRoot.resolve(".multipart").resolve(".incoming");
+ if (!Files.isDirectory(incoming)) {
+ return;
+ }
+ try (Stream<Path> files = Files.list(incoming)) {
+ assertEquals(List.of(), files.toList(), "staged bodies left behind");
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ }
+
+ /** A body the write should never get to reading. */
+ private static InputStream unreadable() {
+ return new InputStream() {
+ @Override
+ public int read() {
+ throw new AssertionError("the body was read");
+ }
+
+ @Override
+ public int read(byte[] buffer, int offset, int length) {
+ throw new AssertionError("the body was read");
+ }
+ };
+ }
+
+ /** {@code data}, then a failure, as when a client disconnects partway. */
+ private static InputStream breaksAfter(byte[] data) {
+ return new FilterInputStream(new ByteArrayInputStream(data)) {
+ @Override
+ public int read(byte[] buffer, int offset, int length) throws IOException {
+ int read = super.read(buffer, offset, length);
+ if (read < 0) {
+ throw new IOException("connection reset");
+ }
+ return read;
+ }
+ };
+ }
+
+ /** {@code data}, running {@code action} the first time any of it is read. */
+ private static InputStream onFirstRead(byte[] data, Runnable action) {
+ return new ByteArrayInputStream(data) {
+ private boolean started;
+
+ @Override
+ public synchronized int read(byte[] buffer, int offset, int length) {
+ if (!started) {
+ started = true;
+ action.run();
+ }
+ return super.read(buffer, offset, length);
+ }
+ };
+ }
+}