Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3): stream GetObject bodies from disk instead of the heap (#4691)
Niels Kootstra888a71892469
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.javaindex ae1bbe54f1..4cf17286dd 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java@@ -46,6 +46,11 @@ import jakarta.ws.rs.core.Response;import jakarta.ws.rs.core.StreamingOutput;import jakarta.ws.rs.core.UriInfo;import java.io.ByteArrayOutputStream;+import java.io.Closeable;+import java.io.EOFException;+import java.io.IOException;+import java.io.InputStream;+import java.io.OutputStream;import java.net.URLDecoder;import java.nio.charset.StandardCharsets;import java.time.Instant;@@ -62,6 +67,7 @@ import java.util.Locale;import java.util.Map;import java.util.Optional;import java.util.Set;+import java.util.function.Function;import javax.xml.stream.XMLStreamConstants;import javax.xml.stream.XMLStreamException;import javax.xml.stream.XMLStreamReader;@@ -1000,39 +1006,51 @@ public class S3Controller {mergedAttributes, maxParts, partNumberMarker);}s3Service.authorizeGetObject(bucket, key, versionId, authorization);- // Fetch metadata and body as one atomic snapshot: resolving the body lazily at- // entity-write time (openObjectStream) races concurrent overwrites and can pair one- // version's Content-Length/checksum headers with another version's bytes.- S3Object obj = s3Service.getObject(bucket, key, versionId);- if (hasPreconditions(ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince)) {- // Evaluate preconditions against the same snapshot that is served: a separate- // metadata fetch could approve one version (e.g. If-Match for a CAS read) while a- // concurrent overwrite swaps in another before the body is resolved.- Response preconditionResponse = checkPreconditions(obj, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince);- if (preconditionResponse != null) {- return preconditionResponse;+ // Open metadata and body as one atomic snapshot: resolving the body lazily at+ // entity-write time races concurrent overwrites and can pair one version's+ // Content-Length/checksum headers with another version's bytes. The open stream keeps+ // serving the snapshot's bytes, and is closed here unless a response took it over.+ S3Service.ObjectRead read = s3Service.openObject(bucket, key, versionId);+ S3Object obj = read.object();+ boolean bodyHandedOff = false;+ try {+ if (hasPreconditions(ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince)) {+ // Evaluate preconditions against the same snapshot that is served: a separate+ // metadata fetch could approve one version (e.g. If-Match for a CAS read) while a+ // concurrent overwrite swaps in another before the body is resolved.+ Response preconditionResponse = checkPreconditions(obj, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince);+ if (preconditionResponse != null) {+ return preconditionResponse;+ }+ }+ S3Service.validateSseCustomerAccess(+ obj,+ httpHeaders.getHeaderString("x-amz-server-side-encryption-customer-algorithm"),+ httpHeaders.getHeaderString("x-amz-server-side-encryption-customer-key"),+ httpHeaders.getHeaderString("x-amz-server-side-encryption-customer-key-MD5"));+ ResponseHeaderOverrides overrides = new ResponseHeaderOverrides(+ responseContentType, responseContentLanguage, responseExpires,+ responseCacheControl, responseContentDisposition, responseContentEncoding);+ if (overrides.hasAny() && !S3RequestAuthorizationParser.isSigned(httpHeaders, uriInfo)) {+ return xmlErrorResponse(new AwsException("InvalidRequest",+ "Request specific response headers cannot be used for anonymous GET requests.", 400));}- }- S3Service.validateSseCustomerAccess(- obj,- httpHeaders.getHeaderString("x-amz-server-side-encryption-customer-algorithm"),- httpHeaders.getHeaderString("x-amz-server-side-encryption-customer-key"),- httpHeaders.getHeaderString("x-amz-server-side-encryption-customer-key-MD5"));- ResponseHeaderOverrides overrides = new ResponseHeaderOverrides(- responseContentType, responseContentLanguage, responseExpires,- responseCacheControl, responseContentDisposition, responseContentEncoding);- if (overrides.hasAny() && !S3RequestAuthorizationParser.isSigned(httpHeaders, uriInfo)) {- return xmlErrorResponse(new AwsException("InvalidRequest",- "Request specific response headers cannot be used for anonymous GET requests.", 400));- }- boolean includeChecksum = "ENABLED".equalsIgnoreCase(resolveHeaderOrQueryParam(checksumMode, uriInfo, "x-amz-checksum-mode"));- if (rangeHeader != null && rangeHeader.startsWith("bytes=")) {- return handleRangeRequest(obj, rangeHeader, overrides, includeChecksum);- }+ boolean includeChecksum = "ENABLED".equalsIgnoreCase(resolveHeaderOrQueryParam(checksumMode, uriInfo, "x-amz-checksum-mode"));+ if (rangeHeader != null && rangeHeader.startsWith("bytes=")) {+ Response rangeResponse = handleRangeRequest(obj, read.body(), rangeHeader, overrides, includeChecksum);+ bodyHandedOff = rangeResponse.getEntity() instanceof StreamingOutput;+ return rangeResponse;+ }- emitCloudTrailEvent("GetObject", bucket, key, 0L, obj.getSize(), null, null);- return fullObjectResponse(obj, overrides, includeChecksum);+ emitCloudTrailEvent("GetObject", bucket, key, 0L, obj.getSize(), null, null);+ bodyHandedOff = true;+ return fullObjectResponse(obj, read.body(), overrides, includeChecksum);+ } finally {+ if (!bodyHandedOff) {+ closeQuietly(read);+ }+ }} catch (AwsException e) {emitCloudTrailEvent("GetObject", bucket, key, 0L, 0L, e.getErrorCode(), e.getMessage());if (S3Service.isWebsiteErrorDocumentTrigger(e) && isWebsiteRequest(httpHeaders, uriInfo)) {@@ -1045,11 +1063,29 @@ public class S3Controller {}}- private Response fullObjectResponse(S3Object obj, ResponseHeaderOverrides overrides,+ /** Streams {@code body}, opened together with {@code obj} by {@link S3Service#openObject}, and closes it. */+ private Response fullObjectResponse(S3Object obj, InputStream body, ResponseHeaderOverrides overrides,boolean includeChecksum) {- byte[] data = objectDataSnapshot(obj);- StreamingOutput stream = output -> output.write(data);- return objectResponseHeaders(Response.ok(stream), obj, overrides, includeChecksum).build();+ return streamingResponse(body,+ stream -> objectResponseHeaders(Response.ok(stream), obj, overrides, includeChecksum).build());+ }++ /**+ * Builds a response whose entity streams {@code body} and then closes it. If building the+ * response fails, the entity is never written, so {@code body} is closed here instead.+ */+ static Response streamingResponse(InputStream body, Function<StreamingOutput, Response> build) {+ StreamingOutput stream = output -> {+ try (InputStream in = body) {+ in.transferTo(output);+ }+ };+ try {+ return build.apply(stream);+ } catch (RuntimeException e) {+ closeQuietly(body);+ throw e;+ }}/** Applies the standard GetObject response headers derived from {@code obj} to {@code resp}. */@@ -1068,7 +1104,7 @@ public class S3Controller {return resp;}- private Response handleRangeRequest(S3Object obj, String rangeHeader,+ private Response handleRangeRequest(S3Object obj, InputStream body, String rangeHeader,ResponseHeaderOverrides overrides,boolean includeChecksum) {long totalSize = obj.getSize();@@ -1102,15 +1138,18 @@ public class S3Controller {if (start < 0 || start >= totalSize || start > end) {if (totalSize == 0 && rangeSpec.startsWith("-")) {- return fullObjectResponse(obj, overrides, includeChecksum);+ return fullObjectResponse(obj, body, overrides, includeChecksum);}return invalidRangeResponse(totalSize);}long length = end - start + 1;- byte[] data = objectDataSnapshot(obj);- // start/length fit in int: they are bounded by the snapshot's length (a byte[]).- StreamingOutput stream = output -> output.write(data, (int) start, (int) length);+ StreamingOutput stream = output -> {+ try (InputStream in = body) {+ in.skipNBytes(start);+ copyExactly(in, output, length);+ }+ };var resp = Response.status(206).entity(stream).header("Content-Type", overrides.contentType() != null ? overrides.contentType() : obj.getContentType())@@ -1127,24 +1166,29 @@ public class S3Controller {return resp.build();}- /**- * Returns the body bytes captured together with {@code obj}'s metadata by- * {@link S3Service#getObject}. Serving the response from this snapshot (instead of re-reading- * the store at entity-write time) guarantees the body always matches the already-committed- * Content-Length/ETag/checksum headers, even when a concurrent PutObject overwrites the key.- */- private static byte[] objectDataSnapshot(S3Object obj) {- byte[] data = obj.getData();- if (data == null) {- throw new IllegalStateException("S3 object data snapshot is missing for "- + obj.getBucketName() + "/" + obj.getKey());+ private Response websiteObjectResponse(S3Service.ObjectRead read, ResponseHeaderOverrides overrides) {+ return fullObjectResponse(read.object(), read.body(), overrides, false);+ }++ private static void copyExactly(InputStream in, OutputStream out, long length) throws IOException {+ byte[] buffer = new byte[64 * 1024];+ long remaining = length;+ while (remaining > 0) {+ int n = in.read(buffer, 0, (int) Math.min(buffer.length, remaining));+ if (n < 0) {+ throw new EOFException("S3 object ended " + remaining + " bytes before the requested range");+ }+ out.write(buffer, 0, n);+ remaining -= n;}- if (data.length != obj.getSize()) {- throw new IllegalStateException("S3 object data snapshot for " + obj.getBucketName()- + "/" + obj.getKey() + " has " + data.length + " bytes but metadata declares "- + obj.getSize() + "; serving it would corrupt the response framing");+ }++ private static void closeQuietly(Closeable closeable) {+ try {+ closeable.close();+ } catch (IOException e) {+ LOG.debugv("Failed to close S3 object stream: {0}", e.getMessage());}- return data;}private Response invalidRangeResponse(long totalSize) {@@ -3109,7 +3153,7 @@ public class S3Controller {// overwrite of the index document cannot tear the response.case S3Service.WebsiteResolution.ServeObject(String key, S3Object object) ->includeBody- ? fullObjectResponse(s3Service.getObject(bucket, key), noOverrides, false)+ ? websiteObjectResponse(s3Service.openObject(bucket, key, null), noOverrides): objectResponseHeaders(Response.ok(), object, noOverrides, false).build();// The query string is deliberately dropped: real S3 answers// GET /photos?code=abc&state=xyz with a bare "Location: /photos/" (verified against adiff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.javaindex ee5c6c16a4..bd646478ab 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java@@ -34,16 +34,20 @@ import jakarta.ws.rs.core.MultivaluedHashMap;import org.jboss.logging.Logger;import java.io.ByteArrayInputStream;+import java.io.Closeable;import java.io.IOException;import java.io.InputStream;import java.io.UncheckedIOException;import java.net.URLEncoder;+import java.nio.channels.Channels;+import java.nio.channels.FileChannel;import java.nio.charset.StandardCharsets;import java.nio.file.NoSuchFileException;import java.nio.file.AtomicMoveNotSupportedException;import java.nio.file.Files;import java.nio.file.Path;import java.nio.file.StandardCopyOption;+import java.nio.file.StandardOpenOption;import java.security.MessageDigest;import java.security.NoSuchAlgorithmException;import java.time.Instant;@@ -66,6 +70,8 @@ import java.util.Set;import java.util.UUID;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.locks.ReentrantLock;+import java.util.function.Consumer;+import java.util.function.Function;import java.util.regex.Matcher;import java.util.regex.Pattern;import io.github.hectorvent.floci.core.resource.ExplorerResource;@@ -1336,6 +1342,20 @@ public class S3Service implements Resettable, ResourceProvider {* token is only ever observed when untouched, and untouched means nothing to race against.*/private S3Object getLatestObject(String bucketName, String key) {+ Snapshot<byte[]> snapshot = readLatestSnapshot(bucketName, key,+ account -> readFile(account, bucketName, key), data -> { });+ snapshot.object().setData(snapshot.body());+ return snapshot.object();+ }++ private record Snapshot<B>(S3Object object, B body) { }++ /**+ * The optimistic read described on {@link #getLatestObject}, for any body type: {@code readBody}+ * gets the bucket owner's account and {@code discard} releases a body read during a race.+ */+ private <B> Snapshot<B> readLatestSnapshot(String bucketName, String key,+ Function<String, B> readBody, Consumer<B> discard) {AccountAwareStorageBackend.OwnedEntry<Bucket> ownedBucket = resolveBucketEntry(bucketName).orElseThrow(() -> new AwsException("NoSuchBucket","The specified bucket does not exist.", 404));@@ -1347,15 +1367,15 @@ public class S3Service implements Resettable, ResourceProvider {// with a clear error instead of spinning forever re-reading the file and exhausting the heap.for (int attempt = 0; attempt < 10_000; attempt++) {S3Object obj = getObjectMetadata(bucketName, key, null);- byte[] data = readFile(bucketOwnerAccount, bucketName, key);+ B body = readBody.apply(bucketOwnerAccount);synchronized (bucket) {S3Object current = resolveObjectForAccount(bucketOwnerAccount, storeKey).orElse(null);if (current != null && !current.isDeleteMarker()&& Objects.equals(current.getDataGeneration(), obj.getDataGeneration())) {- obj.setData(data);- return obj;+ return new Snapshot<>(obj, body);}}+ discard.accept(body);// A concurrent overwrite (or delete) landed mid-read; retry against the new state.}throw new IllegalStateException(@@ -1364,6 +1384,76 @@ public class S3Service implements Resettable, ResourceProvider {+ "metadata/data resolution paths disagree about where this key lives");}+ /** An object's metadata and an open stream over the body that matches it. */+ public record ObjectRead(S3Object object, InputStream body) implements Closeable {+ @Override+ public void close() throws IOException {+ body.close();+ }+ }++ /**+ * Like {@link #getObject}, but the body is an open stream instead of a byte array, so serving+ * a large object does not load it into the heap. Every write replaces the object file with an+ * atomic move, so a file opened for the matching generation keeps returning that generation's+ * bytes even if the key is overwritten while the stream is read. The caller closes the result.+ */+ public ObjectRead openObject(String bucketName, String key, String versionId) {+ if (inMemory) {+ S3Object obj = getObject(bucketName, key, versionId);+ return new ObjectRead(obj, new ByteArrayInputStream(obj.getData()));+ }+ if (versionId == null) {+ Snapshot<FileChannel> snapshot = readLatestSnapshot(bucketName, key,+ account -> openForRead(resolveObjectPathForRead(account, bucketName, key)),+ S3Service::closeQuietly);+ return objectRead(snapshot.object(), snapshot.body());+ }+ String bucketOwnerAccount = resolveBucketEntry(bucketName)+ .orElseThrow(() -> new AwsException("NoSuchBucket",+ "The specified bucket does not exist.", 404))+ .account();+ S3Object obj = getObjectMetadata(bucketName, key, versionId);+ Path path = "null".equals(versionId)+ ? resolveObjectPathForRead(bucketOwnerAccount, bucketName, key)+ : resolveVersionedPathForRead(bucketOwnerAccount, bucketName, key, versionId);+ return objectRead(obj, openForRead(path));+ }++ private static ObjectRead objectRead(S3Object obj, FileChannel channel) {+ try {+ long fileSize = channel.size();+ if (fileSize != obj.getSize()) {+ throw new IllegalStateException("S3 object file for " + obj.getBucketName() + "/"+ + obj.getKey() + " has " + fileSize + " bytes but metadata declares "+ + obj.getSize() + "; serving it would corrupt the response framing");+ }+ } catch (IOException e) {+ closeQuietly(channel);+ throw new UncheckedIOException("Failed to read S3 object file size", e);+ } catch (RuntimeException e) {+ closeQuietly(channel);+ throw e;+ }+ return new ObjectRead(obj, Channels.newInputStream(channel));+ }++ private static FileChannel openForRead(Path path) {+ try {+ return FileChannel.open(path, StandardOpenOption.READ);+ } catch (IOException e) {+ throw new UncheckedIOException("Failed to open S3 object file", e);+ }+ }++ private static void closeQuietly(Closeable closeable) {+ try {+ closeable.close();+ } catch (IOException ignored) {+ // Nothing was read from it; closing is best effort.+ }+ }+public S3Object headObject(String bucketName, String key) {return headObject(bucketName, key, null);}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ObjectStreamTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ObjectStreamTest.javanew file mode 100644index 0000000000..4f37ac11fa--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ObjectStreamTest.java@@ -0,0 +1,87 @@+package io.github.hectorvent.floci.services.s3;++import com.sun.management.ThreadMXBean;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.io.TempDir;++import java.io.ByteArrayInputStream;+import java.io.InputStream;+import java.lang.management.ManagementFactory;+import java.nio.charset.StandardCharsets;+import java.nio.file.Path;+import java.util.Map;+import java.util.concurrent.atomic.AtomicBoolean;++import static org.junit.jupiter.api.Assertions.assertArrayEquals;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.junit.jupiter.api.Assumptions.assumeTrue;++class S3ObjectStreamTest {++ @TempDir+ Path tempDir;++ @Test+ void readingAPersistedObjectDoesNotLoadItIntoTheHeap() throws Exception {+ S3Service s3 = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), tempDir.resolve("s3"), false);+ s3.createBucket("large-objects", "us-east-1");+ int objectSize = 64 * 1024 * 1024;+ s3.putObject("large-objects", "blob.bin", new byte[objectSize], "application/octet-stream", Map.of());++ ThreadMXBean threads = (ThreadMXBean) ManagementFactory.getThreadMXBean();+ long threadId = Thread.currentThread().threadId();+ byte[] buffer = new byte[64 * 1024];+ long total = 0;+ assumeTrue(threads.isThreadAllocatedMemorySupported() && threads.isThreadAllocatedMemoryEnabled(),+ "thread allocation tracking is unavailable");+ long allocatedBefore = threads.getThreadAllocatedBytes(threadId);+ try (S3Service.ObjectRead read = s3.openObject("large-objects", "blob.bin", null)) {+ int n;+ while ((n = read.body().read(buffer)) > 0) {+ total += n;+ }+ }+ long allocated = threads.getThreadAllocatedBytes(threadId) - allocatedBefore;++ assertEquals(objectSize, total);+ assertTrue(allocated < objectSize / 4,+ "reading a " + objectSize + " byte object allocated " + allocated + " bytes");+ }++ @Test+ void overwriteAfterOpenDoesNotChangeTheOpenedBody() throws Exception {+ S3Service s3 = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), tempDir.resolve("s3"), false);+ s3.createBucket("objects", "us-east-1");+ byte[] first = "first version".getBytes(StandardCharsets.UTF_8);+ s3.putObject("objects", "key.txt", first, "text/plain", Map.of());++ try (S3Service.ObjectRead read = s3.openObject("objects", "key.txt", null)) {+ s3.putObject("objects", "key.txt", "second, longer version".getBytes(StandardCharsets.UTF_8),+ "text/plain", Map.of());+ InputStream body = read.body();++ assertEquals(first.length, read.object().getSize());+ assertArrayEquals(first, body.readAllBytes());+ }+ }++ @Test+ void responseThatFailsToBuildClosesTheObjectStream() {+ AtomicBoolean closed = new AtomicBoolean();+ InputStream body = new ByteArrayInputStream(new byte[16]) {+ @Override+ public void close() {+ closed.set(true);+ }+ };++ assertThrows(NullPointerException.class, () -> S3Controller.streamingResponse(body, stream -> {+ throw new NullPointerException("Last-Modified is missing");+ }));++ assertTrue(closed.get(), "the object stream was left open");+ }+}