Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3): stream PutObject and UploadPart bodies to disk (#5106)
Priamo Germosen583de3010c7f
diff --git a/docs/services/s3.md b/docs/services/s3.mdindex cd7626b306..e3b86d29b0 100644--- a/docs/services/s3.md+++ b/docs/services/s3.md@@ -389,10 +389,17 @@ still rewritten on every change. See[Storage Modes](../configuration/storage.md#journaled-stores-under-persistent-mode).In `memory` mode each object body is a single in-memory byte array, so an object can hold at most-about 2 GiB: an `UploadPart` that takes a multipart upload past that, or a-`CompleteMultipartUpload` over it, fails with `EntityTooLarge`. The disk-backed modes assemble a-multipart upload file to file, so the object never passes through the heap; set-`FLOCI_STORAGE_SERVICES_S3_MODE=persistent` to keep only S3 on disk. While it runs,+about 2 GiB: a `PutObject` over that, an `UploadPart` that takes a multipart upload past it, or a+`CompleteMultipartUpload` over it fails with `EntityTooLarge`. The disk-backed modes write a+`PutObject` or `UploadPart` body to disk as it arrives, working out its ETag and checksums on the+way, and assemble a multipart upload file to file, so no object passes through the heap; set+`FLOCI_STORAGE_SERVICES_S3_MODE=persistent` to keep only S3 on disk. Every request body is still+capped by `FLOCI_PROTOCOLS_MAX_REQUEST_SIZE`, 2048 MB by default: to accept the 5 GiB S3 allows in+one `PutObject` or `UploadPart`, raise it a little past 5120, since the aws-chunked framing SDKs+send makes a request slightly larger than its data. With+[signature verification](#signature-verification) on, a request signed over its whole payload, with+a hex `x-amz-content-sha256` rather than `UNSIGNED-PAYLOAD` or a `STREAMING-` value, is still read+into memory to check that hash, so it is limited to about 2 GiB. While it runs,`CompleteMultipartUpload` needs free disk space for twice the object, its parts and the assembledcopy, until the parts are removed. Multipart uploads in progress do not survive a restart or areset, and their files are removed when either happens. `UploadPartCopy` reads only the copieddiff --git a/src/main/java/io/github/hectorvent/floci/services/s3/AwsChunkedInputStream.java b/src/main/java/io/github/hectorvent/floci/services/s3/AwsChunkedInputStream.javanew file mode 100644index 0000000000..107f30465e--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/s3/AwsChunkedInputStream.java@@ -0,0 +1,120 @@+package io.github.hectorvent.floci.services.s3;++import io.github.hectorvent.floci.core.common.AwsException;++import java.io.BufferedInputStream;+import java.io.ByteArrayOutputStream;+import java.io.IOException;+import java.io.InputStream;+import java.nio.charset.StandardCharsets;++/**+ * Decodes an aws-chunked body as it is read, so a streaming upload is never held whole. Each chunk+ * header ({@code hex-size[;chunk-signature=...]}) is consumed and only the chunk data comes out, up+ * to the final empty chunk and the trailer lines and empty line after it. Chunk signatures and+ * trailing checksums are not checked, as they were not when the body was decoded in one piece. A+ * body that breaks the framing or ends early fails with S3's {@code IncompleteBody}.+ */+final class AwsChunkedInputStream extends InputStream {++ // A chunk header or trailer line is a size, a signature and a few separators; anything longer+ // is not framing.+ private static final int MAX_LINE_LENGTH = 8192;++ private final InputStream in;+ private long remainingInChunk;+ private boolean finished;++ AwsChunkedInputStream(InputStream in) {+ this.in = new BufferedInputStream(in);+ }++ @Override+ public int read() throws IOException {+ byte[] single = new byte[1];+ int read = read(single, 0, 1);+ return read < 0 ? -1 : single[0] & 0xFF;+ }++ @Override+ public int read(byte[] buffer, int offset, int length) throws IOException {+ if (length == 0) {+ return 0;+ }+ if (remainingInChunk == 0 && !nextChunk()) {+ return -1;+ }+ int read = in.read(buffer, offset, (int) Math.min(length, remainingInChunk));+ if (read < 0) {+ throw incompleteBody();+ }+ remainingInChunk -= read;+ if (remainingInChunk == 0) {+ readLineBreak();+ }+ return read;+ }++ @Override+ public void close() throws IOException {+ in.close();+ }++ /** Reads the next chunk header; false once the final chunk and its trailer are consumed. */+ private boolean nextChunk() throws IOException {+ if (finished) {+ return false;+ }+ String header = readLine();+ int semicolon = header.indexOf(';');+ String hexSize = (semicolon >= 0 ? header.substring(0, semicolon) : header).trim();+ long size;+ try {+ size = Long.parseLong(hexSize, 16);+ } catch (NumberFormatException e) {+ throw incompleteBody();+ }+ if (size < 0) {+ throw incompleteBody();+ }+ if (size > 0) {+ remainingInChunk = size;+ return true;+ }+ // The final chunk: trailer lines such as x-amz-checksum-crc32, if any, then the empty line+ // that ends the body.+ String trailer = readLine();+ while (!trailer.isEmpty()) {+ trailer = readLine();+ }+ finished = true;+ return false;+ }++ private void readLineBreak() throws IOException {+ int next = in.read();+ if (next == '\r') {+ next = in.read();+ }+ if (next != '\n') {+ throw incompleteBody();+ }+ }++ /** The next line, trimmed and without its line break; the body may not end inside one. */+ private String readLine() throws IOException {+ ByteArrayOutputStream line = new ByteArrayOutputStream();+ for (int next = in.read(); next != '\n'; next = in.read()) {+ if (next < 0 || line.size() >= MAX_LINE_LENGTH) {+ throw incompleteBody();+ }+ line.write(next);+ }+ return line.toString(StandardCharsets.ISO_8859_1).trim();+ }++ static AwsException incompleteBody() {+ return new AwsException("IncompleteBody",+ "You did not provide the number of bytes specified by the Content-Length HTTP header.", 400);+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/DigestingInputStream.java b/src/main/java/io/github/hectorvent/floci/services/s3/DigestingInputStream.javanew file mode 100644index 0000000000..3cedc5f72f--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/s3/DigestingInputStream.java@@ -0,0 +1,115 @@+package io.github.hectorvent.floci.services.s3;++import io.github.hectorvent.floci.services.s3.model.ChecksumAlgorithm;+import io.github.hectorvent.floci.services.s3.model.S3Checksum;++import java.io.FilterInputStream;+import java.io.IOException;+import java.io.InputStream;+import java.security.MessageDigest;+import java.security.NoSuchAlgorithmException;+import java.util.Collection;+import java.util.EnumMap;+import java.util.HexFormat;+import java.util.Map;++/**+ * Passes an upload body through while computing its MD5, the checksums it is checked against or+ * stored with, and its size, so a body can be validated and stored without being held whole.+ */+final class DigestingInputStream extends FilterInputStream {++ private final MessageDigest md5;+ private final Map<ChecksumAlgorithm, S3Checksum.Calculator> calculators = new EnumMap<>(ChecksumAlgorithm.class);+ private final Map<ChecksumAlgorithm, String> checksums = new EnumMap<>(ChecksumAlgorithm.class);+ private byte[] md5Digest;+ private long size;++ DigestingInputStream(InputStream in, Collection<ChecksumAlgorithm> algorithms) {+ super(in);+ try {+ this.md5 = MessageDigest.getInstance("MD5");+ } catch (NoSuchAlgorithmException e) {+ throw new IllegalStateException("MD5 algorithm is not available", e);+ }+ for (ChecksumAlgorithm algorithm : algorithms) {+ calculators.put(algorithm, S3Checksum.calculator(algorithm));+ }+ }++ @Override+ public int read() throws IOException {+ int next = super.read();+ if (next >= 0) {+ byte[] single = {(byte) next};+ update(single, 0, 1);+ }+ return next;+ }++ @Override+ public int read(byte[] buffer, int offset, int length) throws IOException {+ int read = super.read(buffer, offset, length);+ if (read > 0) {+ update(buffer, offset, read);+ }+ return read;+ }++ @Override+ public long skip(long n) throws IOException {+ // Skipped bytes would be missing from the digests, so they are read instead.+ byte[] buffer = new byte[8192];+ long skipped = 0;+ while (skipped < n) {+ int read = read(buffer, 0, (int) Math.min(buffer.length, n - skipped));+ if (read < 0) {+ break;+ }+ skipped += read;+ }+ return skipped;+ }++ @Override+ public boolean markSupported() {+ return false;+ }++ long size() {+ return size;+ }++ /** The MD5 of everything read; reading on after asking for it is not supported. */+ byte[] md5() {+ if (md5Digest == null) {+ md5Digest = md5.digest();+ }+ return md5Digest.clone();+ }++ /** The MD5 as an S3 ETag, quoted lowercase hex. */+ String eTag() {+ return "\"" + HexFormat.of().formatHex(md5()) + "\"";+ }++ /**+ * The Base64 checksum of everything read, for an algorithm the stream was created with. Like+ * {@link #md5}, it is final once asked for.+ */+ String checksum(ChecksumAlgorithm algorithm) {+ S3Checksum.Calculator calculator = calculators.get(algorithm);+ if (calculator == null) {+ throw new IllegalArgumentException(algorithm + " was not computed for this body");+ }+ return checksums.computeIfAbsent(algorithm, ignored -> calculator.value());+ }++ private void update(byte[] buffer, int offset, int length) {+ md5.update(buffer, offset, length);+ for (S3Checksum.Calculator calculator : calculators.values()) {+ calculator.update(buffer, offset, length);+ }+ size += length;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.javaindex 08e0187b6b..0057083e03 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java@@ -47,12 +47,14 @@ import jakarta.ws.rs.core.MultivaluedMap;import jakarta.ws.rs.core.Response;import jakarta.ws.rs.core.StreamingOutput;import jakarta.ws.rs.core.UriInfo;+import java.io.ByteArrayInputStream;import java.io.ByteArrayOutputStream;import java.io.Closeable;import java.io.EOFException;import java.io.IOException;import java.io.InputStream;import java.io.OutputStream;+import java.io.UncheckedIOException;import java.net.URLDecoder;import java.nio.charset.StandardCharsets;import java.security.MessageDigest;@@ -64,6 +66,7 @@ import java.time.format.DateTimeParseException;import java.util.ArrayList;import java.util.Arrays;import java.util.Base64;+import java.util.EnumMap;import java.util.HashMap;import java.util.Iterator;import java.util.LinkedHashMap;@@ -416,7 +419,7 @@ public class S3Controller {? Integer.parseInt(uriInfo.getQueryParameters().getFirst("partNumber")) : null,uriInfo,httpHeaders,- body);+ body != null ? new ByteArrayInputStream(body) : null);}if (bucket.length() > 63) {throw new AwsException("InvalidBucketName", "The specified bucket is not valid.", 400);@@ -798,7 +801,7 @@ public class S3Controller {@QueryParam("partNumber") Integer partNumber,@Context UriInfo uriInfo,@Context HttpHeaders httpHeaders,- byte[] body) {+ InputStream body) {try {key = extractObjectKey(uriInfo, bucket, key);S3Service.RequestAuthorization authorization = S3RequestAuthorizationParser.parseIfRequired(@@ -806,7 +809,7 @@ public class S3Controller {if (hasQueryParam(uriInfo, "tagging")) {s3Service.authorizeObjectWrite(bucket, key, "s3:PutObjectTagging", authorization);- return handlePutObjectTagging(bucket, key, body);+ return handlePutObjectTagging(bucket, key, readWholeBody(body));}if (hasQueryParam(uriInfo, "retention")) {s3Service.authorizeObjectWrite(bucket, key, "s3:PutObjectRetention", authorization);@@ -814,17 +817,17 @@ public class S3Controller {s3Service.authorizeObjectWrite(bucket, key, "s3:BypassGovernanceRetention", authorization);}return handlePutObjectRetention(bucket, key,- uriInfo.getQueryParameters().getFirst("versionId"), httpHeaders, body);+ uriInfo.getQueryParameters().getFirst("versionId"), httpHeaders, readWholeBody(body));}if (hasQueryParam(uriInfo, "legal-hold")) {s3Service.authorizeObjectWrite(bucket, key, "s3:PutObjectLegalHold", authorization);return handlePutObjectLegalHold(bucket, key,- uriInfo.getQueryParameters().getFirst("versionId"), body);+ uriInfo.getQueryParameters().getFirst("versionId"), readWholeBody(body));}if (hasQueryParam(uriInfo, "acl")) {s3Service.authorizeObjectWrite(bucket, key, "s3:PutObjectAcl", authorization);s3Service.putObjectAcl(bucket, key, uriInfo.getQueryParameters().getFirst("versionId"),- new String(body, StandardCharsets.UTF_8),+ new String(readWholeBody(body), StandardCharsets.UTF_8),httpHeaders.getHeaderString("x-amz-acl"),httpHeaders.getHeaderString("x-amz-grant-read"),httpHeaders.getHeaderString("x-amz-grant-write"),@@ -836,7 +839,7 @@ public class S3Controller {if (hasQueryParam(uriInfo, "annotation")) {s3Service.authorizeObjectWrite(bucket, key, "s3:PutObjectAnnotation", authorization);- return handlePutObjectAnnotation(bucket, key, body, uriInfo, httpHeaders);+ return handlePutObjectAnnotation(bucket, key, readWholeBody(body), uriInfo, httpHeaders);}if (uploadId != null && partNumber != null) {@@ -845,9 +848,8 @@ public class S3Controller {return handleUploadPartCopy(copySource, bucket, key, uploadId, partNumber, httpHeaders, authorization);}- byte[] partData = decodeAwsChunked(body, contentEncoding, contentSha256);- validateChecksumHeaders(httpHeaders, uriInfo, partData, getChecksumAlgorithm(httpHeaders, uriInfo));- Part part = s3Service.storePart(bucket, key, uploadId, partNumber, partData,+ Part part = s3Service.storePart(bucket, key, uploadId, partNumber,+ decodedBody(body, contentEncoding, contentSha256), uploadChecksums(httpHeaders, uriInfo),httpHeaders.getHeaderString("x-amz-server-side-encryption-customer-algorithm"),httpHeaders.getHeaderString("x-amz-server-side-encryption-customer-key"),httpHeaders.getHeaderString("x-amz-server-side-encryption-customer-key-MD5"));@@ -875,9 +877,7 @@ public class S3Controller {String legalHold = httpHeaders.getHeaderString("x-amz-object-lock-legal-hold");Instant retainUntil = retainUntilStr != null ? Instant.parse(retainUntilStr) : null;- byte[] data = decodeAwsChunked(body, contentEncoding, contentSha256);String checksumAlgorithm = getChecksumAlgorithm(httpHeaders, uriInfo);- validateChecksumHeaders(httpHeaders, uriInfo, data, checksumAlgorithm);String persistedEncoding = toPersistedContentEncoding(contentEncoding);String contentDisposition = httpHeaders.getHeaderString("Content-Disposition");String cacheControl = httpHeaders.getHeaderString("Cache-Control");@@ -888,7 +888,8 @@ public class S3Controller {String sseCustomerKeyMd5 = httpHeaders.getHeaderString("x-amz-server-side-encryption-customer-key-MD5");String cannedAcl = httpHeaders.getHeaderString("x-amz-acl");s3Service.authorizePutObject(bucket, key, authorization);- S3Object obj = s3Service.putObject(bucket, key, data, contentType, extractUserMetadata(httpHeaders, uriInfo),+ S3Object obj = s3Service.putObject(bucket, key, decodedBody(body, contentEncoding, contentSha256),+ uploadChecksums(httpHeaders, uriInfo), contentType, extractUserMetadata(httpHeaders, uriInfo),new PutObjectOptions().withStorageClass(httpHeaders.getHeaderString("x-amz-storage-class")).withContentEncoding(persistedEncoding)@@ -918,7 +919,7 @@ public class S3Controller {resp.header("x-amz-version-id", obj.getVersionId());}appendPutObjectResponseHeaders(resp, obj);- emitCloudTrailEvent("PutObject", bucket, key, data == null ? 0 : data.length, 0L, null, null);+ emitCloudTrailEvent("PutObject", bucket, key, obj.getSize(), 0L, null, null);return resp.build();} catch (AwsException e) {emitCloudTrailEvent("PutObject", bucket, key, 0L, 0L, e.getErrorCode(), e.getMessage());@@ -2097,22 +2098,61 @@ public class S3Controller {// --- AWS Chunked Decoding ---+ /**+ * The object data of an upload body, read as it arrives. A streaming payload is decoded from its+ * aws-chunked framing on the way through. A Content-Encoding naming aws-chunked without that+ * declaration does not guarantee framing, so that body is read whole and kept as sent when it+ * does not decode.+ */+ private InputStream decodedBody(InputStream body, String contentEncoding, String contentSha256) {+ InputStream raw = body != null ? body : InputStream.nullInputStream();+ if (declaresStreamingPayload(contentSha256)) {+ return new AwsChunkedInputStream(raw);+ }+ if (contentEncoding != null && contentEncoding.toLowerCase(Locale.ROOT).contains("aws-chunked")) {+ return new ByteArrayInputStream(decodeAwsChunked(readWholeBody(raw), contentEncoding, contentSha256));+ }+ return raw;+ }++ /** The whole of a body that is small by nature, such as a subresource's XML, or must be read whole to be decoded. */+ private static byte[] readWholeBody(InputStream body) {+ if (body == null) {+ return new byte[0];+ }+ try {+ return body.readAllBytes();+ } catch (IOException e) {+ throw new UncheckedIOException("Failed to read the request body", e);+ }+ }+/*** Decodes aws-chunked transfer encoding used by AWS SDK v2 with SigV4 chunk signing.* Format: hex-size;chunk-signature=sig\r\n data \r\n ... 0;chunk-signature=sig\r\n*/private byte[] decodeAwsChunked(byte[] body, String contentEncoding, String contentSha256) {- boolean declaresStreamingPayload = "STREAMING-AWS4-HMAC-SHA256-PAYLOAD".equals(contentSha256)- || "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER".equals(contentSha256)- || "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD".equals(contentSha256)- || "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER".equals(contentSha256)- || "STREAMING-UNSIGNED-PAYLOAD-TRAILER".equals(contentSha256);+ boolean declaresStreamingPayload = declaresStreamingPayload(contentSha256);boolean isAwsChunked = declaresStreamingPayload || (contentEncoding != null&& contentEncoding.toLowerCase(Locale.ROOT).contains("aws-chunked"));if (!isAwsChunked) {return body;}+ if (declaresStreamingPayload) {+ // A streaming payload is always framed, so a body that breaks the framing is rejected+ // rather than stored with its chunk headers as the object's content.+ try (AwsChunkedInputStream decoded = new AwsChunkedInputStream(new ByteArrayInputStream(body))) {+ return decoded.readAllBytes();+ } catch (AwsException e) {+ LOG.debugv("Rejecting malformed aws-chunked body: {0}", e.getMessage());+ throw e;+ } catch (IOException e) {+ throw new UncheckedIOException("Failed to decode an aws-chunked body", e);+ }+ }+ // Without that declaration, a Content-Encoding naming aws-chunked does not guarantee+ // framing, so a body that does not decode is kept as sent.try {ByteArrayOutputStream out = new ByteArrayOutputStream();String raw = new String(body, StandardCharsets.ISO_8859_1);@@ -2126,8 +2166,7 @@ public class S3Controller {String hexSize = semiColon >= 0 ? line.substring(0, semiColon) : line;int chunkSize = Integer.parseInt(hexSize.trim(), 16);if (chunkSize == 0) {- // A streaming body ends with an empty line, after any trailer headers.- complete = !declaresStreamingPayload || reachesEmptyLine(raw, lineEnd + 1);+ complete = true;break;}@@ -2138,43 +2177,25 @@ public class S3Controller {pos = dataStart + chunkSize;if (pos < raw.length() && raw.charAt(pos) == '\r') pos++;- boolean lineBreak = pos < raw.length() && raw.charAt(pos) == '\n';- if (lineBreak) pos++;- if (declaresStreamingPayload && !lineBreak) {- throw new IllegalArgumentException("aws-chunked chunk is not followed by a line break");- }+ if (pos < raw.length() && raw.charAt(pos) == '\n') pos++;}if (!complete) {throw new IllegalArgumentException("aws-chunked body ends before its final chunk");}return out.toByteArray();} catch (Exception e) {- // A streaming payload is always framed, so storing it raw would save the chunk- // headers as the object's content. Without that declaration, a Content-Encoding- // naming aws-chunked does not guarantee framing, and the body is kept as sent.- if (declaresStreamingPayload) {- LOG.debugv("Rejecting malformed aws-chunked body: {0}", e.getMessage());- throw new AwsException("IncompleteBody",- "You did not provide the number of bytes specified by the Content-Length HTTP header.",- 400);- }LOG.debugv("Failed to decode aws-chunked body, using raw: {0}", e.getMessage());return body;}}- private static boolean reachesEmptyLine(String raw, int pos) {- while (pos < raw.length()) {- int lineEnd = raw.indexOf('\n', pos);- if (lineEnd < 0) {- return false;- }- if (raw.substring(pos, lineEnd).trim().isEmpty()) {- return true;- }- pos = lineEnd + 1;- }- return false;+ /** Whether x-amz-content-sha256 declares a streaming payload, which is always aws-chunked. */+ static boolean declaresStreamingPayload(String contentSha256) {+ return "STREAMING-AWS4-HMAC-SHA256-PAYLOAD".equals(contentSha256)+ || "STREAMING-AWS4-HMAC-SHA256-PAYLOAD-TRAILER".equals(contentSha256)+ || "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD".equals(contentSha256)+ || "STREAMING-AWS4-ECDSA-P256-SHA256-PAYLOAD-TRAILER".equals(contentSha256)+ || "STREAMING-UNSIGNED-PAYLOAD-TRAILER".equals(contentSha256);}/**@@ -2406,7 +2427,7 @@ public class S3Controller {String versionId = uriInfo.getQueryParameters().getFirst("versionId");String algorithmHeader = getChecksumAlgorithm(httpHeaders, uriInfo);ChecksumAlgorithm algorithm = ChecksumAlgorithm.fromWireValue(algorithmHeader);- validateChecksumHeaders(httpHeaders, uriInfo, payload, algorithmHeader);+ validateChecksumHeaders(httpHeaders, uriInfo, payload);ObjectAnnotation annotation = s3Service.putObjectAnnotation(bucket, key, annotationName,versionId, payload, httpHeaders.getHeaderString("x-amz-object-if-match"), algorithm);Response.ResponseBuilder response = Response.ok(putObjectAnnotationXml(annotation))@@ -3114,59 +3135,27 @@ public class S3Controller {return algorithm;}- private void validateChecksumHeaders(HttpHeaders httpHeaders, UriInfo uriInfo, byte[] data, String algorithm) {- validateContentMd5(httpHeaders.getHeaderString("Content-MD5"), data);-- String sha1 = resolveHeaderOrQueryParam(httpHeaders, uriInfo, "x-amz-checksum-sha1");- if (sha1 != null && !sha1.equals(S3Checksum.sha1Base64(data))) {- throw new AwsException("BadDigest", "The SHA1 checksum you specified did not match the payload.", 400);- }-- String sha256 = resolveHeaderOrQueryParam(httpHeaders, uriInfo, "x-amz-checksum-sha256");- if (sha256 != null && !sha256.equals(S3Checksum.sha256Base64(data))) {- throw new AwsException("BadDigest", "The SHA256 checksum you specified did not match the payload.", 400);- }-- String crc32 = resolveHeaderOrQueryParam(httpHeaders, uriInfo, "x-amz-checksum-crc32");- if (crc32 != null && !crc32.equals(S3Checksum.crc32Base64(data))) {- throw new AwsException("BadDigest", "The CRC32 checksum you specified did not match the payload.", 400);- }-- String crc32c = resolveHeaderOrQueryParam(httpHeaders, uriInfo, "x-amz-checksum-crc32c");- if (crc32c != null && !crc32c.equals(S3Checksum.crc32cBase64(data))) {- throw new AwsException("BadDigest", "The CRC32C checksum you specified did not match the payload.", 400);- }-- String crc64nvme = resolveHeaderOrQueryParam(httpHeaders, uriInfo, "x-amz-checksum-crc64nvme");- if (crc64nvme != null && !crc64nvme.equals(S3Checksum.crc64NvmeBase64(data))) {- throw new AwsException("BadDigest", "The CRC64NVME checksum you specified did not match the payload.", 400);- }- }-- // S3 answers InvalidDigest for a Content-MD5 that is not the base64 of a 16-byte digest, and- // BadDigest for a well-formed one that does not match the payload.- private static void validateContentMd5(String contentMd5, byte[] data) {- if (contentMd5 == null) {- return;- }- byte[] expected;+ private void validateChecksumHeaders(HttpHeaders httpHeaders, UriInfo uriInfo, byte[] data) {+ UploadChecksums checksums = uploadChecksums(httpHeaders, uriInfo);+ checksums.requireWellFormedContentMd5();+ byte[] md5;try {- expected = Base64.getDecoder().decode(contentMd5.trim());- } catch (IllegalArgumentException e) {- expected = null;- }- if (expected == null || expected.length != 16) {- throw new AwsException("InvalidDigest", "The Content-MD5 you specified is not valid.", 400);- }- byte[] actual;- try {- actual = MessageDigest.getInstance("MD5").digest(data);+ md5 = MessageDigest.getInstance("MD5").digest(data);} catch (NoSuchAlgorithmException e) {throw new IllegalStateException("MD5 algorithm is not available", e);}- if (!MessageDigest.isEqual(expected, actual)) {- throw new AwsException("BadDigest", "The Content-MD5 you specified did not match what we received.", 400);+ checksums.verify(md5, claimedAlgorithm -> claimedAlgorithm.compute(data));+ }++ private UploadChecksums uploadChecksums(HttpHeaders httpHeaders, UriInfo uriInfo) {+ Map<ChecksumAlgorithm, String> claimed = new EnumMap<>(ChecksumAlgorithm.class);+ for (ChecksumAlgorithm algorithm : ChecksumAlgorithm.values()) {+ String value = resolveHeaderOrQueryParam(httpHeaders, uriInfo, "x-amz-checksum-" + algorithm.wireValue());+ if (value != null) {+ claimed.put(algorithm, value);+ }}+ return new UploadChecksums(httpHeaders.getHeaderString("Content-MD5"), claimed);}static boolean isWebsiteRequest(HttpHeaders httpHeaders, UriInfo uriInfo) {diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.javaindex d51bf6156b..d4e596c4fd 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java@@ -40,6 +40,7 @@ import jakarta.ws.rs.core.MultivaluedHashMap;import org.jboss.logging.Logger;import java.io.ByteArrayInputStream;+import java.io.ByteArrayOutputStream;import java.io.Closeable;import java.io.IOException;import java.io.InputStream;@@ -65,6 +66,7 @@ import java.util.Arrays;import java.util.Base64;import java.util.Collections;import java.util.Comparator;+import java.util.EnumSet;import java.util.HashMap;import java.util.HexFormat;import java.util.LinkedHashMap;@@ -215,6 +217,10 @@ public class S3Service implements Resettable, ResourceProvider {}// Memory mode keeps an object in one byte array; this is the largest the JDK reliably allocates.private static final long MAX_IN_MEMORY_OBJECT_SIZE = Integer.MAX_VALUE - 8;+ // Directory under .multipart where a streamed part or object body is written before it is+ // published. Upload IDs are UUIDs, so it never collides with an upload's own directory, and the+ // sweep at startup and reset clears it with them.+ private static final String INCOMING_BODIES = ".incoming";// Upload ID to part bytes, keyed by each part's storage ID.private final ConcurrentHashMap<String, Map<String, byte[]>> memoryMultipartStore = new ConcurrentHashMap<>();private final ConcurrentHashMap<String, MultipartUpload> multipartUploads = new ConcurrentHashMap<>();@@ -618,6 +624,62 @@ public class S3Service implements Resettable, ResourceProvider {return createObject(bucketName, key, data, contentType, metadata, options, "ObjectCreated:Put");}+ /**+ * Stores an object streamed from {@code body}, checked against {@code checksums}, without holding+ * it in memory in the disk-backed modes. Everything the request alone decides is checked before+ * any of the body is read, so a write that cannot succeed fails without transferring it. The body+ * is staged on disk with its ETag and checksum worked out as it arrives, then moved into place+ * like an assembled multipart object.+ */+ S3Object putObject(String bucketName, String key, InputStream body, UploadChecksums checksums,+ String contentType, Map<String, String> metadata, PutObjectOptions options) {+ PutObjectOptions effectiveOptions = options != null ? options : new PutObjectOptions();+ checksums.requireWellFormedContentMd5();+ AccountAwareStorageBackend.OwnedEntry<Bucket> checkedBucket = resolveBucketEntry(bucketName)+ .orElseThrow(() -> new AwsException("NoSuchBucket",+ "The specified bucket does not exist.", 404));+ rejectConflictingServerSideEncryption(+ normalizeServerSideEncryption(effectiveOptions.getServerSideEncryption()),+ validateSseCustomerKey(effectiveOptions.getSseCustomerAlgorithm(),+ effectiveOptions.getSseCustomerKey(), effectiveOptions.getSseCustomerKeyMd5()));+ ChecksumAlgorithm declared = ChecksumAlgorithm.fromWireValue(effectiveOptions.getChecksumAlgorithm());+ // Checked again under the bucket lock when the object is stored; this only spares a transfer.+ checkWritePreconditions(bucketName, key, effectiveOptions.getIfMatch(), effectiveOptions.getIfNoneMatch());+ if (inMemory) {+ byte[] data = readVerified(body, checksums, "PutObject of " + bucketName + "/" + key);+ S3Object object = storeObject(checkedBucket, bucketName, key, new BytesBody(data),+ contentType, metadata, null, null, effectiveOptions, null);+ fireNotifications(bucketName, key, "ObjectCreated:Put", object);+ return object;+ }++ // A checksum the client sent is stored as sent; otherwise the declared algorithm's, or CRC64NVME.+ ChecksumAlgorithm computed = effectiveOptions.getClientChecksum() != null ? null+ : declared != null ? declared : ChecksumAlgorithm.CRC64NVME;+ Set<ChecksumAlgorithm> algorithms = EnumSet.noneOf(ChecksumAlgorithm.class);+ algorithms.addAll(checksums.algorithms());+ if (computed != null) {+ algorithms.add(computed);+ }+ DigestingInputStream digests = new DigestingInputStream(body, algorithms);+ Path staged = stageBody(digests);+ try {+ checksums.verify(digests.md5(), digests::checksum);+ S3Checksum checksum = null;+ if (computed != null) {+ checksum = new S3Checksum();+ checksum.setValueFor(computed, digests.checksum(computed));+ checksum.setChecksumType(ChecksumType.FULL_OBJECT);+ }+ S3Object object = storeObject(checkedBucket, bucketName, key, new AssembledBody(staged, digests.size()),+ contentType, metadata, checksum, null, effectiveOptions, digests.eTag());+ fireNotifications(bucketName, key, "ObjectCreated:Put", object);+ return object;+ } finally {+ deleteQuietly(staged, "staged body of an object that was not stored");+ }+ }+public S3Object postObject(String bucketName, String key, byte[] data,String contentType, Map<String, String> metadata) {return postObject(bucketName, key, data, contentType, metadata, new PutObjectOptions());@@ -671,19 +733,39 @@ public class S3Service implements Resettable, ResourceProvider {private S3Object storeObject(String bucketName, String key, ObjectBody body,String contentType, Map<String, String> metadata,S3Checksum checksum, List<Part> parts, PutObjectOptions options, String eTag) {+ return storeObject(null, bucketName, key, body, contentType, metadata, checksum, parts, options, eTag);+ }++ /**+ * Stores the object, and with {@code checkedBucket} given, only in that bucket: an upload checked+ * against a bucket that was deleted while its body arrived fails with NoSuchBucket rather than+ * landing in a bucket created under the same name since, whose policy it was never checked+ * against. A bucket is created once and keeps its instance until it is deleted, so the instance+ * tells one bucket of a name from another.+ */+ private S3Object storeObject(AccountAwareStorageBackend.OwnedEntry<Bucket> checkedBucket,+ String bucketName, String key, ObjectBody body,+ String contentType, Map<String, String> metadata,+ S3Checksum checksum, List<Part> parts, PutObjectOptions options, String eTag) {AccountAwareStorageBackend.OwnedEntry<Bucket> ownedBucket = resolveBucketEntry(bucketName).orElseThrow(() -> new AwsException("NoSuchBucket","The specified bucket does not exist.", 404));Bucket bucket = ownedBucket.value();synchronized (bucket) {+ // Rechecked under the bucket's monitor, which deleting the bucket also holds.+ if (checkedBucket != null && resolveBucketEntry(bucketName)+ .map(current -> current.value() != checkedBucket.value())+ .orElse(true)) {+ throw new AwsException("NoSuchBucket", "The specified bucket does not exist.", 404);+ }return storeObjectInternal(ownedBucket.account(), bucket, bucketName, key, body,contentType, metadata, checksum, parts, options, eTag);}}/**- * The body of an object being stored: the bytes of a single-request write, or the file a- * multipart upload was assembled into, which is moved into place instead of being read back.+ * The body of an object being stored: bytes already in memory, or a file on disk, which is moved+ * into place instead of being read back.*/private sealed interface ObjectBody permits BytesBody, AssembledBody {long size();@@ -697,9 +779,10 @@ public class S3Service implements Resettable, ResourceProvider {}/**- * A body already on disk that is moved into place: an assembled multipart object, or the pinned- * file of a copy source. It is stored with an ETag and checksum worked out beforehand, since- * computing either here would mean reading the whole file under the bucket lock.+ * A body already on disk that is moved into place: an assembled multipart object, the pinned file+ * of a copy source, or a streamed PutObject body. It is stored with an ETag and checksum worked+ * out beforehand, since computing either here would mean reading the whole file under the bucket+ * lock.*/private record AssembledBody(Path file, long size) implements ObjectBody { }@@ -3734,7 +3817,7 @@ public class S3Service implements Resettable, ResourceProvider {.filter(part -> part.getPartNumber() != partNumber).mapToLong(Part::getSize).sum();- requireFitsInMemory(uploadId, otherParts + data.length);+ requireFitsInMemory("Multipart upload " + uploadId, otherParts + data.length);memoryMultipartStore.get(uploadId).put(storageId, data);} else {Path partPath = dataRoot.resolve(".multipart").resolve(uploadId).resolve(storageId);@@ -3758,6 +3841,104 @@ public class S3Service implements Resettable, ResourceProvider {});}+ /**+ * Stores one part streamed from {@code body}, checked against {@code checksums}, without holding it+ * in memory in the disk-backed modes. The upload is checked before any of the body is read. The+ * body is staged outside the upload's directory and outside any lock, so parts of one upload still+ * arrive in parallel and a slow client never holds up the bucket's other multipart operations. Only+ * once it has passed its checks is it moved into the upload and its record published, under the+ * upload's lock, so an upload aborted or completed meanwhile fails the part with NoSuchUpload.+ */+ Part storePart(String bucket, String key, String uploadId, int partNumber, InputStream body,+ UploadChecksums checksums,+ String sseCustomerAlgorithm, String sseCustomerKey, String sseCustomerKeyMd5) {+ checksums.requireWellFormedContentMd5();+ ChecksumAlgorithm declared = withMultipartOperationLock(bucket, uploadId, () -> {+ MultipartUpload upload = getMultipartUpload(bucket, key, uploadId);+ if (partNumber < 1 || partNumber > 10000) {+ throw new AwsException("InvalidArgument",+ "Part number must be between 1 and 10000.", 400);+ }+ validateSseCustomerAccess(upload, sseCustomerAlgorithm, sseCustomerKey, sseCustomerKeyMd5);+ return upload.getChecksumAlgorithm();+ });+ if (inMemory) {+ byte[] data = readVerified(body, checksums, "Multipart upload " + uploadId);+ return storePart(bucket, key, uploadId, partNumber, data,+ sseCustomerAlgorithm, sseCustomerKey, sseCustomerKeyMd5);+ }+ ChecksumAlgorithm algorithm = declared != null ? declared : ChecksumAlgorithm.CRC64NVME;+ Set<ChecksumAlgorithm> algorithms = EnumSet.of(algorithm);+ algorithms.addAll(checksums.algorithms());+ DigestingInputStream digests = new DigestingInputStream(body, algorithms);+ Path staged = stageBody(digests);+ try {+ checksums.verify(digests.md5(), digests::checksum);+ S3Checksum partChecksum = new S3Checksum();+ partChecksum.setValueFor(algorithm, digests.checksum(algorithm));+ return withMultipartOperationLock(bucket, uploadId, () -> {+ MultipartUpload upload = getMultipartUpload(bucket, key, uploadId);+ String storageId = partNumber + "-" + UUID.randomUUID();+ try {+ Files.move(staged, dataRoot.resolve(".multipart").resolve(uploadId).resolve(storageId),+ StandardCopyOption.ATOMIC_MOVE);+ } catch (IOException e) {+ throw new UncheckedIOException("Failed to write multipart part", e);+ }+ Part part = new Part(partNumber, digests.eTag(), digests.size());+ part.setChecksum(partChecksum);+ part.setStorageId(storageId);+ Part replaced = upload.getParts().put(partNumber, part);+ if (replaced != null) {+ discardPartBytes(uploadId, replaced);+ }+ LOG.debugv("Uploaded part {0} for upload {1} ({2} bytes)", partNumber, uploadId, digests.size());+ return part;+ });+ } finally {+ deleteQuietly(staged, "staged part of an upload that was not stored");+ }+ }++ /**+ * Reads a memory-mode upload body into one array, checked against {@code checksums}, failing+ * with EntityTooLarge once it outgrows the largest array the JDK allocates.+ */+ private static byte[] readVerified(InputStream body, UploadChecksums checksums, String upload) {+ DigestingInputStream digests = new DigestingInputStream(body, checksums.algorithms());+ ByteArrayOutputStream out = new ByteArrayOutputStream();+ byte[] buffer = new byte[64 * 1024];+ try {+ for (int read = digests.read(buffer); read >= 0; read = digests.read(buffer)) {+ requireFitsInMemory(upload, (long) out.size() + read);+ out.write(buffer, 0, read);+ }+ } catch (IOException e) {+ throw new UncheckedIOException("Failed to read the upload body", e);+ }+ checksums.verify(digests.md5(), digests::checksum);+ return out.toByteArray();+ }++ /**+ * Writes a streamed body to a new file under the staging directory and returns it. The file is+ * deleted if the body fails partway, so on return the caller owns it.+ */+ private Path stageBody(InputStream body) {+ Path staged = dataRoot.resolve(".multipart").resolve(INCOMING_BODIES).resolve(UUID.randomUUID().toString());+ try {+ Files.createDirectories(staged.getParent());+ Files.copy(body, staged);+ return staged;+ } catch (IOException e) {+ deleteQuietly(staged, "partly staged upload body");+ throw new UncheckedIOException("Failed to write the upload body", e);+ } catch (RuntimeException e) {+ deleteQuietly(staged, "partly staged upload body");+ throw e;+ }+ }+/*** Drops the bytes of a part a re-upload replaced, once the new record is published, so no record* ever names bytes that changed after it was written.@@ -3940,7 +4121,7 @@ public class S3Service implements Resettable, ResourceProvider {validateCompleteChecksumType(algorithm, storedChecksumType, ChecksumType.fromWireValue(checksumType));if (inMemory) {- requireFitsInMemory(uploadId, parts.stream().mapToLong(Part::getSize).sum());+ requireFitsInMemory("Multipart upload " + uploadId, parts.stream().mapToLong(Part::getSize).sum());}// Concatenate parts in order@@ -4020,10 +4201,10 @@ public class S3Service implements Resettable, ResourceProvider {* object over its size limit, so it fails at the part that crosses the limit instead of running* the heap out of memory or overflowing the array size at completion.*/- private static void requireFitsInMemory(String uploadId, long uploadSize) {+ private static void requireFitsInMemory(String upload, long uploadSize) {if (uploadSize > MAX_IN_MEMORY_OBJECT_SIZE) {- LOG.warnv("Multipart upload {0} would hold {1} bytes, over the {2} an object can have in memory storage mode; set FLOCI_STORAGE_SERVICES_S3_MODE=persistent for larger objects",- uploadId, uploadSize, MAX_IN_MEMORY_OBJECT_SIZE);+ LOG.warnv("{0} would hold {1} bytes, over the {2} an object can have in memory storage mode; set FLOCI_STORAGE_SERVICES_S3_MODE=persistent for larger objects",+ upload, uploadSize, MAX_IN_MEMORY_OBJECT_SIZE);throw new AwsException("EntityTooLarge", "Your proposed upload exceeds the maximum allowed object size.", 400);}}@@ -4043,7 +4224,7 @@ public class S3Service implements Resettable, ResourceProvider {}totalSize += partData[i].length;}- requireFitsInMemory(uploadId, totalSize);+ requireFitsInMemory("Multipart upload " + uploadId, totalSize);byte[] allData = new byte[(int) totalSize];int offset = 0;for (byte[] part : partData) {diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/UploadChecksums.java b/src/main/java/io/github/hectorvent/floci/services/s3/UploadChecksums.javanew file mode 100644index 0000000000..e0d33ba820--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/s3/UploadChecksums.java@@ -0,0 +1,66 @@+package io.github.hectorvent.floci.services.s3;++import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.s3.model.ChecksumAlgorithm;++import java.security.MessageDigest;+import java.util.Base64;+import java.util.List;+import java.util.Map;+import java.util.Set;+import java.util.function.Function;++/**+ * The integrity headers an upload body is checked against: Content-MD5 and the x-amz-checksum-*+ * values. A malformed Content-MD5 is rejected before any of the body is read; the digests are+ * compared once it has been.+ */+record UploadChecksums(String contentMd5, Map<ChecksumAlgorithm, String> claimed) {++ // The order the checksums have always been checked in, so a body failing several reports the same one.+ private static final List<ChecksumAlgorithm> CHECK_ORDER = List.of(ChecksumAlgorithm.SHA1,+ ChecksumAlgorithm.SHA256, ChecksumAlgorithm.CRC32, ChecksumAlgorithm.CRC32C, ChecksumAlgorithm.CRC64NVME);++ UploadChecksums {+ claimed = Map.copyOf(claimed);+ }++ /** The algorithms the request claims a checksum for, which the body has to be hashed with. */+ Set<ChecksumAlgorithm> algorithms() {+ return claimed.keySet();+ }++ // S3 answers InvalidDigest for a Content-MD5 that is not the base64 of a 16-byte digest, and+ // BadDigest for a well-formed one that does not match the payload.+ void requireWellFormedContentMd5() {+ if (contentMd5 != null) {+ expectedMd5();+ }+ }++ void verify(byte[] actualMd5, Function<ChecksumAlgorithm, String> actualChecksum) {+ if (contentMd5 != null && !MessageDigest.isEqual(expectedMd5(), actualMd5)) {+ throw new AwsException("BadDigest", "The Content-MD5 you specified did not match what we received.", 400);+ }+ for (ChecksumAlgorithm algorithm : CHECK_ORDER) {+ String claimedValue = claimed.get(algorithm);+ if (claimedValue != null && !claimedValue.equals(actualChecksum.apply(algorithm))) {+ throw new AwsException("BadDigest", "The " + algorithm.name()+ + " checksum you specified did not match the payload.", 400);+ }+ }+ }++ private byte[] expectedMd5() {+ byte[] expected;+ try {+ expected = Base64.getDecoder().decode(contentMd5.trim());+ } catch (IllegalArgumentException e) {+ expected = null;+ }+ if (expected == null || expected.length != 16) {+ throw new AwsException("InvalidDigest", "The Content-MD5 you specified is not valid.", 400);+ }+ return expected;+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/AwsChunkedInputStreamTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/AwsChunkedInputStreamTest.javanew file mode 100644index 0000000000..a18282a093--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3/AwsChunkedInputStreamTest.java@@ -0,0 +1,96 @@+package io.github.hectorvent.floci.services.s3;++import io.github.hectorvent.floci.core.common.AwsException;+import org.junit.jupiter.api.Test;++import java.io.ByteArrayInputStream;+import java.io.ByteArrayOutputStream;+import java.io.IOException;+import java.io.InputStream;+import java.nio.charset.StandardCharsets;+import java.util.List;+import java.util.Random;++import static org.junit.jupiter.api.Assertions.*;++class AwsChunkedInputStreamTest {++ @Test+ void decodesSignedChunksUpToTheFinalChunk() throws IOException {+ assertEquals("hello world", decode("5;chunk-signature=abc\r\nhello\r\n6;chunk-signature=def\r\n world\r\n"+ + "0;chunk-signature=ghi\r\n\r\n"));+ }++ @Test+ void skipsTrailerLinesAfterTheFinalChunk() throws IOException {+ assertEquals("hello", decode("5\r\nhello\r\n0\r\nx-amz-checksum-crc32:NhCmhg==\r\n"+ + "x-amz-trailer-signature:ghi\r\n\r\n"));+ }++ @Test+ void acceptsBareLineFeeds() throws IOException {+ assertEquals("hello", decode("5\nhello\n0\n\n"));+ }++ @Test+ void decodesManyChunksWhateverTheReadSize() throws IOException {+ Random random = new Random(5080);+ byte[] payload = new byte[200_003];+ random.nextBytes(payload);+ ByteArrayOutputStream framed = new ByteArrayOutputStream();+ for (int offset = 0; offset < payload.length; offset += 8192) {+ int size = Math.min(8192, payload.length - offset);+ framed.writeBytes((Integer.toHexString(size) + ";chunk-signature=sig\r\n").getBytes(StandardCharsets.US_ASCII));+ framed.write(payload, offset, size);+ framed.writeBytes("\r\n".getBytes(StandardCharsets.US_ASCII));+ }+ framed.writeBytes("0;chunk-signature=sig\r\n\r\n".getBytes(StandardCharsets.US_ASCII));++ for (int readSize : List.of(1, 7, 4096, 65_536)) {+ try (InputStream in = new AwsChunkedInputStream(new ByteArrayInputStream(framed.toByteArray()))) {+ ByteArrayOutputStream decoded = new ByteArrayOutputStream();+ byte[] buffer = new byte[readSize];+ for (int read = in.read(buffer); read >= 0; read = in.read(buffer)) {+ decoded.write(buffer, 0, read);+ }+ assertArrayEquals(payload, decoded.toByteArray(), "read size " + readSize);+ }+ }+ }++ @Test+ void readsOneByteAtATime() throws IOException {+ try (InputStream in = new AwsChunkedInputStream(new ByteArrayInputStream(+ "3\r\nabc\r\n0\r\n\r\n".getBytes(StandardCharsets.US_ASCII)))) {+ assertEquals('a', in.read());+ assertEquals('b', in.read());+ assertEquals('c', in.read());+ assertEquals(-1, in.read());+ assertEquals(-1, in.read(), "the stream stays at its end");+ }+ }++ @Test+ void brokenFramingIsAnIncompleteBody() {+ for (String body : List.of(+ "10\r\nonly-five\r\n",+ "5\r\nhello\r\n",+ "five\r\nhello\r\n0\r\n\r\n",+ "5\r\nhello0\r\n\r\n",+ "5\r\nhello\r\n0\r\nx-amz-checksum-crc32:NhCmhg==\r\n",+ "5;chunk-signature=abc\r\nhello\r\n0;chunk-signature=def\r\n",+ "-5\r\nhello\r\n0\r\n\r\n",+ "")) {+ AwsException error = assertThrows(AwsException.class, () -> decode(body), body);+ assertEquals("IncompleteBody", error.getErrorCode(), body);+ assertEquals(400, error.getHttpStatus(), body);+ }+ }++ private static String decode(String framed) throws IOException {+ try (InputStream in = new AwsChunkedInputStream(+ new ByteArrayInputStream(framed.getBytes(StandardCharsets.ISO_8859_1)))) {+ return new String(in.readAllBytes(), StandardCharsets.ISO_8859_1);+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3StreamedUploadTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3StreamedUploadTest.javanew file mode 100644index 0000000000..02639f2979--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3StreamedUploadTest.java@@ -0,0 +1,337 @@+package io.github.hectorvent.floci.services.s3;++import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.services.s3.model.ChecksumAlgorithm;+import io.github.hectorvent.floci.services.s3.model.ChecksumType;+import io.github.hectorvent.floci.services.s3.model.MultipartUpload;+import io.github.hectorvent.floci.services.s3.model.Part;+import io.github.hectorvent.floci.services.s3.model.PutObjectOptions;+import io.github.hectorvent.floci.services.s3.model.S3Object;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.io.TempDir;++import java.io.ByteArrayInputStream;+import java.io.FilterInputStream;+import java.io.IOException;+import java.io.InputStream;+import java.io.UncheckedIOException;+import java.nio.charset.StandardCharsets;+import java.nio.file.Files;+import java.nio.file.Path;+import java.security.MessageDigest;+import java.time.Duration;+import java.util.Base64;+import java.util.List;+import java.util.Map;+import java.util.stream.Stream;++import static org.junit.jupiter.api.Assertions.*;++/**+ * PutObject and UploadPart bodies read as a stream, as the controller hands them over, so a+ * disk-backed upload is never held in memory whole.+ */+class S3StreamedUploadTest {++ private static final byte[] BODY = "a body that arrives as a stream".getBytes(StandardCharsets.UTF_8);+ private static final UploadChecksums NO_CHECKSUMS = new UploadChecksums(null, Map.of());++ @TempDir+ Path tempDir;++ private S3Service s3Service;+ private Path dataRoot;++ @BeforeEach+ void setUp() {+ dataRoot = tempDir.resolve("s3");+ s3Service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), dataRoot, false);+ s3Service.createBucket("bucket", "us-east-1");+ }++ @Test+ void aStreamedPutStoresWhatTheBytePathStores() {+ S3Object fromBytes = s3Service.putObject("bucket", "bytes.txt", BODY, "text/plain", Map.of());++ S3Object streamed = s3Service.putObject("bucket", "streamed.txt", new ByteArrayInputStream(BODY),+ NO_CHECKSUMS, "text/plain", Map.of(), new PutObjectOptions());++ assertEquals(fromBytes.getETag(), streamed.getETag());+ assertEquals(fromBytes.getChecksum().getChecksumCRC64NVME(), streamed.getChecksum().getChecksumCRC64NVME());+ assertEquals(ChecksumType.FULL_OBJECT, streamed.getChecksum().getChecksumType());+ assertEquals(BODY.length, streamed.getSize());+ assertArrayEquals(BODY, s3Service.getObject("bucket", "streamed.txt").getData());+ assertNothingStaged();+ }++ @Test+ void aStreamedPutStoresTheDeclaredAlgorithmsChecksum() {+ S3Object streamed = s3Service.putObject("bucket", "sha.txt", new ByteArrayInputStream(BODY),+ NO_CHECKSUMS, "text/plain", Map.of(), new PutObjectOptions().withChecksumAlgorithm("SHA256"));++ assertEquals(ChecksumAlgorithm.SHA256.compute(BODY), streamed.getChecksum().getChecksumSHA256());+ assertNull(streamed.getChecksum().getChecksumCRC64NVME());+ }++ @Test+ void aStreamedPutIntoAVersionedBucketKeepsEachVersion() {+ s3Service.putBucketVersioning("bucket", "Enabled");+ byte[] second = "the second version".getBytes(StandardCharsets.UTF_8);+ S3Object first = s3Service.putObject("bucket", "versioned.txt", new ByteArrayInputStream(BODY),+ NO_CHECKSUMS, "text/plain", Map.of(), new PutObjectOptions());+ s3Service.putObject("bucket", "versioned.txt", new ByteArrayInputStream(second),+ NO_CHECKSUMS, "text/plain", Map.of(), new PutObjectOptions());++ assertArrayEquals(second, s3Service.getObject("bucket", "versioned.txt").getData());+ assertArrayEquals(BODY, s3Service.getObject("bucket", "versioned.txt", first.getVersionId()).getData());+ assertNothingStaged();+ }++ @Test+ void aBodyThatFailsItsChecksumIsNotStored() {+ UploadChecksums wrong = new UploadChecksums(null,+ Map.of(ChecksumAlgorithm.CRC32, ChecksumAlgorithm.CRC32.compute("something else".getBytes(StandardCharsets.UTF_8))));++ AwsException error = assertThrows(AwsException.class, () -> s3Service.putObject("bucket", "bad.txt",+ new ByteArrayInputStream(BODY), wrong, "text/plain", Map.of(), new PutObjectOptions()));++ assertEquals("BadDigest", error.getErrorCode());+ assertEquals("NoSuchKey", assertThrows(AwsException.class,+ () -> s3Service.headObject("bucket", "bad.txt")).getErrorCode());+ assertNothingStaged();+ }++ @Test+ void aBodyThatMatchesItsContentMd5AndChecksumIsStored() throws Exception {+ String md5 = Base64.getEncoder().encodeToString(MessageDigest.getInstance("MD5").digest(BODY));+ UploadChecksums right = new UploadChecksums(md5, Map.of(ChecksumAlgorithm.SHA1, ChecksumAlgorithm.SHA1.compute(BODY)));++ s3Service.putObject("bucket", "good.txt", new ByteArrayInputStream(BODY), right,+ "text/plain", Map.of(), new PutObjectOptions());++ assertArrayEquals(BODY, s3Service.getObject("bucket", "good.txt").getData());+ }++ @Test+ void aWriteThatCannotSucceedFailsBeforeTheBodyIsRead() {+ s3Service.putObject("bucket", "existing.txt", BODY, "text/plain", Map.of());++ assertEquals("PreconditionFailed", assertThrows(AwsException.class, () -> s3Service.putObject("bucket",+ "existing.txt", unreadable(), NO_CHECKSUMS, "text/plain", Map.of(),+ new PutObjectOptions().withIfNoneMatch("*"))).getErrorCode());+ assertEquals("NoSuchBucket", assertThrows(AwsException.class, () -> s3Service.putObject("missing-bucket",+ "key.txt", unreadable(), NO_CHECKSUMS, "text/plain", Map.of(),+ new PutObjectOptions())).getErrorCode());+ assertEquals("InvalidDigest", assertThrows(AwsException.class, () -> s3Service.putObject("bucket",+ "key.txt", unreadable(), new UploadChecksums("not-an-md5", Map.of()), "text/plain", Map.of(),+ new PutObjectOptions())).getErrorCode());+ assertEquals("InvalidArgument", assertThrows(AwsException.class, () -> s3Service.putObject("bucket",+ "key.txt", unreadable(), NO_CHECKSUMS, "text/plain", Map.of(),+ new PutObjectOptions().withChecksumAlgorithm("NOT-AN-ALGORITHM"))).getErrorCode());+ }++ @Test+ void aBodyThatBreaksPartwayLeavesNothingBehind() {+ assertThrows(UncheckedIOException.class, () -> s3Service.putObject("bucket", "broken.txt",+ breaksAfter(BODY), NO_CHECKSUMS, "text/plain", Map.of(), new PutObjectOptions()));++ AwsException framing = assertThrows(AwsException.class, () -> s3Service.putObject("bucket", "framing.txt",+ new AwsChunkedInputStream(new ByteArrayInputStream("10\r\nonly-five\r\n".getBytes(StandardCharsets.US_ASCII))),+ NO_CHECKSUMS, "text/plain", Map.of(), new PutObjectOptions()));++ assertEquals("IncompleteBody", framing.getErrorCode());+ assertEquals("NoSuchKey", assertThrows(AwsException.class,+ () -> s3Service.headObject("bucket", "broken.txt")).getErrorCode());+ assertNothingStaged();+ }++ @Test+ void aStreamedPartStoresWhatTheBytePathStores() {+ byte[] part2 = "and a second part".getBytes(StandardCharsets.UTF_8);+ MultipartUpload upload = s3Service.initiateMultipartUpload("bucket", "multipart.bin", null);+ Part fromBytes = s3Service.storePart("bucket", "multipart.bin", upload.getUploadId(), 1, BODY, null, null, null);++ Part streamed = s3Service.storePart("bucket", "multipart.bin", upload.getUploadId(), 2,+ new ByteArrayInputStream(part2), NO_CHECKSUMS, null, null, null);+ Part streamedAgain = s3Service.storePart("bucket", "multipart.bin", upload.getUploadId(), 1,+ new ByteArrayInputStream(BODY), NO_CHECKSUMS, null, null, null);++ assertEquals(fromBytes.getETag(), streamedAgain.getETag());+ assertEquals(fromBytes.getChecksum().getChecksumCRC64NVME(), streamedAgain.getChecksum().getChecksumCRC64NVME());+ assertEquals(part2.length, streamed.getSize());+ s3Service.completeMultipartUpload("bucket", "multipart.bin", upload.getUploadId(), List.of(1, 2), null, null);+ byte[] whole = new byte[BODY.length + part2.length];+ System.arraycopy(BODY, 0, whole, 0, BODY.length);+ System.arraycopy(part2, 0, whole, BODY.length, part2.length);+ assertArrayEquals(whole, s3Service.getObject("bucket", "multipart.bin").getData());+ assertNothingStaged();+ }++ @Test+ void aPartThatFailsItsChecksumIsNotRecorded() {+ MultipartUpload upload = s3Service.initiateMultipartUpload("bucket", "multipart.bin", null);+ UploadChecksums wrong = new UploadChecksums(null,+ Map.of(ChecksumAlgorithm.SHA256, ChecksumAlgorithm.SHA256.compute("other".getBytes(StandardCharsets.UTF_8))));++ assertEquals("BadDigest", assertThrows(AwsException.class, () -> s3Service.storePart("bucket",+ "multipart.bin", upload.getUploadId(), 1, new ByteArrayInputStream(BODY), wrong,+ null, null, null)).getErrorCode());++ assertTrue(s3Service.getMultipartUpload("bucket", "multipart.bin", upload.getUploadId()).getParts().isEmpty());+ assertNothingStaged();+ }++ @Test+ void aPartForAnUploadThatCannotTakeItFailsBeforeItsBodyIsRead() {+ MultipartUpload upload = s3Service.initiateMultipartUpload("bucket", "multipart.bin", null);++ assertEquals("NoSuchUpload", assertThrows(AwsException.class, () -> s3Service.storePart("bucket",+ "multipart.bin", "no-such-upload", 1, unreadable(), NO_CHECKSUMS, null, null, null)).getErrorCode());+ assertEquals("InvalidArgument", assertThrows(AwsException.class, () -> s3Service.storePart("bucket",+ "multipart.bin", upload.getUploadId(), 10001, unreadable(), NO_CHECKSUMS, null, null, null)).getErrorCode());+ }++ @Test+ void anUploadAbortedWhileAPartArrivesFailsThePartAndLeavesNothingBehind() {+ MultipartUpload upload = s3Service.initiateMultipartUpload("bucket", "multipart.bin", null);+ InputStream body = onFirstRead(BODY,+ () -> s3Service.abortMultipartUpload("bucket", "multipart.bin", upload.getUploadId()));++ assertEquals("NoSuchUpload", assertThrows(AwsException.class, () -> s3Service.storePart("bucket",+ "multipart.bin", upload.getUploadId(), 1, body, NO_CHECKSUMS, null, null, null)).getErrorCode());++ assertFalse(Files.exists(dataRoot.resolve(".multipart").resolve(upload.getUploadId())));+ assertNothingStaged();+ }++ @Test+ void aPartStillArrivingDoesNotHoldUpDeletingItsBucket() {+ MultipartUpload upload = s3Service.initiateMultipartUpload("bucket", "multipart.bin", null);+ // Deleting the bucket takes its multipart lock exclusively, so this would never return if the+ // part held that lock while its body arrived.+ InputStream body = onFirstRead(BODY, () -> s3Service.deleteBucket("bucket"));++ AwsException error = assertTimeoutPreemptively(Duration.ofSeconds(30), () -> assertThrows(AwsException.class,+ () -> s3Service.storePart("bucket", "multipart.bin", upload.getUploadId(), 1, body,+ NO_CHECKSUMS, null, null, null)));++ assertTrue(List.of("NoSuchUpload", "NoSuchBucket").contains(error.getErrorCode()), error.getErrorCode());+ assertNothingStaged();+ }++ @Test+ void memoryModeReadsTheStreamIntoTheBytePaths() {+ S3Service memory = memoryService();+ S3Object fromBytes = memory.putObject("bucket", "bytes.txt", BODY, "text/plain", Map.of());++ S3Object streamed = memory.putObject("bucket", "streamed.txt", new ByteArrayInputStream(BODY),+ NO_CHECKSUMS, "text/plain", Map.of(), new PutObjectOptions());+ MultipartUpload upload = memory.initiateMultipartUpload("bucket", "multipart.bin", null);+ memory.storePart("bucket", "multipart.bin", upload.getUploadId(), 1, new ByteArrayInputStream(BODY),+ NO_CHECKSUMS, null, null, null);+ memory.completeMultipartUpload("bucket", "multipart.bin", upload.getUploadId(), List.of(1), null, null);++ assertEquals(fromBytes.getETag(), streamed.getETag());+ assertArrayEquals(BODY, memory.getObject("bucket", "streamed.txt").getData());+ assertArrayEquals(BODY, memory.getObject("bucket", "multipart.bin").getData());+ UploadChecksums wrong = new UploadChecksums(null,+ Map.of(ChecksumAlgorithm.CRC32C, ChecksumAlgorithm.CRC32C.compute("other".getBytes(StandardCharsets.UTF_8))));+ assertEquals("BadDigest", assertThrows(AwsException.class, () -> memory.putObject("bucket", "bad.txt",+ new ByteArrayInputStream(BODY), wrong, "text/plain", Map.of(), new PutObjectOptions())).getErrorCode());+ }++ @Test+ void aBucketRecreatedWhileABodyArrivesDoesNotReceiveTheObject() {+ for (S3Service service : List.of(s3Service, memoryService())) {+ service.createBucket("recreated", "us-east-1");+ // The request was checked against the first bucket; the second has its own policy.+ InputStream body = onFirstRead(BODY, () -> {+ service.deleteBucket("recreated");+ service.createBucket("recreated", "us-east-1");+ });++ assertEquals("NoSuchBucket", assertThrows(AwsException.class, () -> service.putObject("recreated",+ "object.txt", body, NO_CHECKSUMS, "text/plain", Map.of(), new PutObjectOptions())).getErrorCode());++ assertEquals("NoSuchKey", assertThrows(AwsException.class,+ () -> service.headObject("recreated", "object.txt")).getErrorCode());+ }+ assertNothingStaged();+ }++ @Test+ void memoryModeChecksAPartsUploadBeforeReadingItsBody() {+ S3Service memory = memoryService();+ MultipartUpload upload = memory.initiateMultipartUpload("bucket", "multipart.bin", null);++ assertEquals("NoSuchUpload", assertThrows(AwsException.class, () -> memory.storePart("bucket",+ "multipart.bin", "no-such-upload", 1, unreadable(), NO_CHECKSUMS, null, null, null)).getErrorCode());+ assertEquals("InvalidArgument", assertThrows(AwsException.class, () -> memory.storePart("bucket",+ "multipart.bin", upload.getUploadId(), 0, unreadable(), NO_CHECKSUMS, null, null, null)).getErrorCode());+ }++ private S3Service memoryService() {+ S3Service memory = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), dataRoot, true);+ memory.createBucket("bucket", "us-east-1");+ return memory;+ }++ private void assertNothingStaged() {+ Path incoming = dataRoot.resolve(".multipart").resolve(".incoming");+ if (!Files.isDirectory(incoming)) {+ return;+ }+ try (Stream<Path> files = Files.list(incoming)) {+ assertEquals(List.of(), files.toList(), "staged bodies left behind");+ } catch (IOException e) {+ throw new UncheckedIOException(e);+ }+ }++ /** A body the write should never get to reading. */+ private static InputStream unreadable() {+ return new InputStream() {+ @Override+ public int read() {+ throw new AssertionError("the body was read");+ }++ @Override+ public int read(byte[] buffer, int offset, int length) {+ throw new AssertionError("the body was read");+ }+ };+ }++ /** {@code data}, then a failure, as when a client disconnects partway. */+ private static InputStream breaksAfter(byte[] data) {+ return new FilterInputStream(new ByteArrayInputStream(data)) {+ @Override+ public int read(byte[] buffer, int offset, int length) throws IOException {+ int read = super.read(buffer, offset, length);+ if (read < 0) {+ throw new IOException("connection reset");+ }+ return read;+ }+ };+ }++ /** {@code data}, running {@code action} the first time any of it is read. */+ private static InputStream onFirstRead(byte[] data, Runnable action) {+ return new ByteArrayInputStream(data) {+ private boolean started;++ @Override+ public synchronized int read(byte[] buffer, int offset, int length) {+ if (!started) {+ started = true;+ action.run();+ }+ return super.read(buffer, offset, length);+ }+ };+ }+}