Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(s3): stream GetObject bodies from disk instead of the heap (#4691)

Niels Kootstra888a71892469
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
index ae1bbe54f1..4cf17286dd 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
@@ -46,6 +46,11 @@ import jakarta.ws.rs.core.Response;
import jakarta.ws.rs.core.StreamingOutput;
import jakarta.ws.rs.core.UriInfo;
import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
import java.time.Instant;
@@ -62,6 +67,7 @@ import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
+import java.util.function.Function;
import javax.xml.stream.XMLStreamConstants;
import javax.xml.stream.XMLStreamException;
import javax.xml.stream.XMLStreamReader;
@@ -1000,39 +1006,51 @@ public class S3Controller {
mergedAttributes, maxParts, partNumberMarker);
}
s3Service.authorizeGetObject(bucket, key, versionId, authorization);
- // Fetch metadata and body as one atomic snapshot: resolving the body lazily at
- // entity-write time (openObjectStream) races concurrent overwrites and can pair one
- // version's Content-Length/checksum headers with another version's bytes.
- S3Object obj = s3Service.getObject(bucket, key, versionId);
- if (hasPreconditions(ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince)) {
- // Evaluate preconditions against the same snapshot that is served: a separate
- // metadata fetch could approve one version (e.g. If-Match for a CAS read) while a
- // concurrent overwrite swaps in another before the body is resolved.
- Response preconditionResponse = checkPreconditions(obj, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince);
- if (preconditionResponse != null) {
- return preconditionResponse;
+ // Open metadata and body as one atomic snapshot: resolving the body lazily at
+ // entity-write time races concurrent overwrites and can pair one version's
+ // Content-Length/checksum headers with another version's bytes. The open stream keeps
+ // serving the snapshot's bytes, and is closed here unless a response took it over.
+ S3Service.ObjectRead read = s3Service.openObject(bucket, key, versionId);
+ S3Object obj = read.object();
+ boolean bodyHandedOff = false;
+ try {
+ if (hasPreconditions(ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince)) {
+ // Evaluate preconditions against the same snapshot that is served: a separate
+ // metadata fetch could approve one version (e.g. If-Match for a CAS read) while a
+ // concurrent overwrite swaps in another before the body is resolved.
+ Response preconditionResponse = checkPreconditions(obj, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince);
+ if (preconditionResponse != null) {
+ return preconditionResponse;
+ }
+ }
+ S3Service.validateSseCustomerAccess(
+ obj,
+ httpHeaders.getHeaderString("x-amz-server-side-encryption-customer-algorithm"),
+ httpHeaders.getHeaderString("x-amz-server-side-encryption-customer-key"),
+ httpHeaders.getHeaderString("x-amz-server-side-encryption-customer-key-MD5"));
+ ResponseHeaderOverrides overrides = new ResponseHeaderOverrides(
+ responseContentType, responseContentLanguage, responseExpires,
+ responseCacheControl, responseContentDisposition, responseContentEncoding);
+ if (overrides.hasAny() && !S3RequestAuthorizationParser.isSigned(httpHeaders, uriInfo)) {
+ return xmlErrorResponse(new AwsException("InvalidRequest",
+ "Request specific response headers cannot be used for anonymous GET requests.", 400));
}
- }
- S3Service.validateSseCustomerAccess(
- obj,
- httpHeaders.getHeaderString("x-amz-server-side-encryption-customer-algorithm"),
- httpHeaders.getHeaderString("x-amz-server-side-encryption-customer-key"),
- httpHeaders.getHeaderString("x-amz-server-side-encryption-customer-key-MD5"));
- ResponseHeaderOverrides overrides = new ResponseHeaderOverrides(
- responseContentType, responseContentLanguage, responseExpires,
- responseCacheControl, responseContentDisposition, responseContentEncoding);
- if (overrides.hasAny() && !S3RequestAuthorizationParser.isSigned(httpHeaders, uriInfo)) {
- return xmlErrorResponse(new AwsException("InvalidRequest",
- "Request specific response headers cannot be used for anonymous GET requests.", 400));
- }
- boolean includeChecksum = "ENABLED".equalsIgnoreCase(resolveHeaderOrQueryParam(checksumMode, uriInfo, "x-amz-checksum-mode"));
- if (rangeHeader != null && rangeHeader.startsWith("bytes=")) {
- return handleRangeRequest(obj, rangeHeader, overrides, includeChecksum);
- }
+ boolean includeChecksum = "ENABLED".equalsIgnoreCase(resolveHeaderOrQueryParam(checksumMode, uriInfo, "x-amz-checksum-mode"));
+ if (rangeHeader != null && rangeHeader.startsWith("bytes=")) {
+ Response rangeResponse = handleRangeRequest(obj, read.body(), rangeHeader, overrides, includeChecksum);
+ bodyHandedOff = rangeResponse.getEntity() instanceof StreamingOutput;
+ return rangeResponse;
+ }
- emitCloudTrailEvent("GetObject", bucket, key, 0L, obj.getSize(), null, null);
- return fullObjectResponse(obj, overrides, includeChecksum);
+ emitCloudTrailEvent("GetObject", bucket, key, 0L, obj.getSize(), null, null);
+ bodyHandedOff = true;
+ return fullObjectResponse(obj, read.body(), overrides, includeChecksum);
+ } finally {
+ if (!bodyHandedOff) {
+ closeQuietly(read);
+ }
+ }
} catch (AwsException e) {
emitCloudTrailEvent("GetObject", bucket, key, 0L, 0L, e.getErrorCode(), e.getMessage());
if (S3Service.isWebsiteErrorDocumentTrigger(e) && isWebsiteRequest(httpHeaders, uriInfo)) {
@@ -1045,11 +1063,29 @@ public class S3Controller {
}
}
- private Response fullObjectResponse(S3Object obj, ResponseHeaderOverrides overrides,
+ /** Streams {@code body}, opened together with {@code obj} by {@link S3Service#openObject}, and closes it. */
+ private Response fullObjectResponse(S3Object obj, InputStream body, ResponseHeaderOverrides overrides,
boolean includeChecksum) {
- byte[] data = objectDataSnapshot(obj);
- StreamingOutput stream = output -> output.write(data);
- return objectResponseHeaders(Response.ok(stream), obj, overrides, includeChecksum).build();
+ return streamingResponse(body,
+ stream -> objectResponseHeaders(Response.ok(stream), obj, overrides, includeChecksum).build());
+ }
+
+ /**
+ * Builds a response whose entity streams {@code body} and then closes it. If building the
+ * response fails, the entity is never written, so {@code body} is closed here instead.
+ */
+ static Response streamingResponse(InputStream body, Function<StreamingOutput, Response> build) {
+ StreamingOutput stream = output -> {
+ try (InputStream in = body) {
+ in.transferTo(output);
+ }
+ };
+ try {
+ return build.apply(stream);
+ } catch (RuntimeException e) {
+ closeQuietly(body);
+ throw e;
+ }
}
/** Applies the standard GetObject response headers derived from {@code obj} to {@code resp}. */
@@ -1068,7 +1104,7 @@ public class S3Controller {
return resp;
}
- private Response handleRangeRequest(S3Object obj, String rangeHeader,
+ private Response handleRangeRequest(S3Object obj, InputStream body, String rangeHeader,
ResponseHeaderOverrides overrides,
boolean includeChecksum) {
long totalSize = obj.getSize();
@@ -1102,15 +1138,18 @@ public class S3Controller {
if (start < 0 || start >= totalSize || start > end) {
if (totalSize == 0 && rangeSpec.startsWith("-")) {
- return fullObjectResponse(obj, overrides, includeChecksum);
+ return fullObjectResponse(obj, body, overrides, includeChecksum);
}
return invalidRangeResponse(totalSize);
}
long length = end - start + 1;
- byte[] data = objectDataSnapshot(obj);
- // start/length fit in int: they are bounded by the snapshot's length (a byte[]).
- StreamingOutput stream = output -> output.write(data, (int) start, (int) length);
+ StreamingOutput stream = output -> {
+ try (InputStream in = body) {
+ in.skipNBytes(start);
+ copyExactly(in, output, length);
+ }
+ };
var resp = Response.status(206)
.entity(stream)
.header("Content-Type", overrides.contentType() != null ? overrides.contentType() : obj.getContentType())
@@ -1127,24 +1166,29 @@ public class S3Controller {
return resp.build();
}
- /**
- * Returns the body bytes captured together with {@code obj}'s metadata by
- * {@link S3Service#getObject}. Serving the response from this snapshot (instead of re-reading
- * the store at entity-write time) guarantees the body always matches the already-committed
- * Content-Length/ETag/checksum headers, even when a concurrent PutObject overwrites the key.
- */
- private static byte[] objectDataSnapshot(S3Object obj) {
- byte[] data = obj.getData();
- if (data == null) {
- throw new IllegalStateException("S3 object data snapshot is missing for "
- + obj.getBucketName() + "/" + obj.getKey());
+ private Response websiteObjectResponse(S3Service.ObjectRead read, ResponseHeaderOverrides overrides) {
+ return fullObjectResponse(read.object(), read.body(), overrides, false);
+ }
+
+ private static void copyExactly(InputStream in, OutputStream out, long length) throws IOException {
+ byte[] buffer = new byte[64 * 1024];
+ long remaining = length;
+ while (remaining > 0) {
+ int n = in.read(buffer, 0, (int) Math.min(buffer.length, remaining));
+ if (n < 0) {
+ throw new EOFException("S3 object ended " + remaining + " bytes before the requested range");
+ }
+ out.write(buffer, 0, n);
+ remaining -= n;
}
- if (data.length != obj.getSize()) {
- throw new IllegalStateException("S3 object data snapshot for " + obj.getBucketName()
- + "/" + obj.getKey() + " has " + data.length + " bytes but metadata declares "
- + obj.getSize() + "; serving it would corrupt the response framing");
+ }
+
+ private static void closeQuietly(Closeable closeable) {
+ try {
+ closeable.close();
+ } catch (IOException e) {
+ LOG.debugv("Failed to close S3 object stream: {0}", e.getMessage());
}
- return data;
}
private Response invalidRangeResponse(long totalSize) {
@@ -3109,7 +3153,7 @@ public class S3Controller {
// overwrite of the index document cannot tear the response.
case S3Service.WebsiteResolution.ServeObject(String key, S3Object object) ->
includeBody
- ? fullObjectResponse(s3Service.getObject(bucket, key), noOverrides, false)
+ ? websiteObjectResponse(s3Service.openObject(bucket, key, null), noOverrides)
: objectResponseHeaders(Response.ok(), object, noOverrides, false).build();
// The query string is deliberately dropped: real S3 answers
// GET /photos?code=abc&state=xyz with a bare "Location: /photos/" (verified against a
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
index ee5c6c16a4..bd646478ab 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
@@ -34,16 +34,20 @@ import jakarta.ws.rs.core.MultivaluedHashMap;
import org.jboss.logging.Logger;
import java.io.ByteArrayInputStream;
+import java.io.Closeable;
import java.io.IOException;
import java.io.InputStream;
import java.io.UncheckedIOException;
import java.net.URLEncoder;
+import java.nio.channels.Channels;
+import java.nio.channels.FileChannel;
import java.nio.charset.StandardCharsets;
import java.nio.file.NoSuchFileException;
import java.nio.file.AtomicMoveNotSupportedException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardCopyOption;
+import java.nio.file.StandardOpenOption;
import java.security.MessageDigest;
import java.security.NoSuchAlgorithmException;
import java.time.Instant;
@@ -66,6 +70,8 @@ import java.util.Set;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.ReentrantLock;
+import java.util.function.Consumer;
+import java.util.function.Function;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import io.github.hectorvent.floci.core.resource.ExplorerResource;
@@ -1336,6 +1342,20 @@ public class S3Service implements Resettable, ResourceProvider {
* token is only ever observed when untouched, and untouched means nothing to race against.
*/
private S3Object getLatestObject(String bucketName, String key) {
+ Snapshot<byte[]> snapshot = readLatestSnapshot(bucketName, key,
+ account -> readFile(account, bucketName, key), data -> { });
+ snapshot.object().setData(snapshot.body());
+ return snapshot.object();
+ }
+
+ private record Snapshot<B>(S3Object object, B body) { }
+
+ /**
+ * The optimistic read described on {@link #getLatestObject}, for any body type: {@code readBody}
+ * gets the bucket owner's account and {@code discard} releases a body read during a race.
+ */
+ private <B> Snapshot<B> readLatestSnapshot(String bucketName, String key,
+ Function<String, B> readBody, Consumer<B> discard) {
AccountAwareStorageBackend.OwnedEntry<Bucket> ownedBucket = resolveBucketEntry(bucketName)
.orElseThrow(() -> new AwsException("NoSuchBucket",
"The specified bucket does not exist.", 404));
@@ -1347,15 +1367,15 @@ public class S3Service implements Resettable, ResourceProvider {
// with a clear error instead of spinning forever re-reading the file and exhausting the heap.
for (int attempt = 0; attempt < 10_000; attempt++) {
S3Object obj = getObjectMetadata(bucketName, key, null);
- byte[] data = readFile(bucketOwnerAccount, bucketName, key);
+ B body = readBody.apply(bucketOwnerAccount);
synchronized (bucket) {
S3Object current = resolveObjectForAccount(bucketOwnerAccount, storeKey).orElse(null);
if (current != null && !current.isDeleteMarker()
&& Objects.equals(current.getDataGeneration(), obj.getDataGeneration())) {
- obj.setData(data);
- return obj;
+ return new Snapshot<>(obj, body);
}
}
+ discard.accept(body);
// A concurrent overwrite (or delete) landed mid-read; retry against the new state.
}
throw new IllegalStateException(
@@ -1364,6 +1384,76 @@ public class S3Service implements Resettable, ResourceProvider {
+ "metadata/data resolution paths disagree about where this key lives");
}
+ /** An object's metadata and an open stream over the body that matches it. */
+ public record ObjectRead(S3Object object, InputStream body) implements Closeable {
+ @Override
+ public void close() throws IOException {
+ body.close();
+ }
+ }
+
+ /**
+ * Like {@link #getObject}, but the body is an open stream instead of a byte array, so serving
+ * a large object does not load it into the heap. Every write replaces the object file with an
+ * atomic move, so a file opened for the matching generation keeps returning that generation's
+ * bytes even if the key is overwritten while the stream is read. The caller closes the result.
+ */
+ public ObjectRead openObject(String bucketName, String key, String versionId) {
+ if (inMemory) {
+ S3Object obj = getObject(bucketName, key, versionId);
+ return new ObjectRead(obj, new ByteArrayInputStream(obj.getData()));
+ }
+ if (versionId == null) {
+ Snapshot<FileChannel> snapshot = readLatestSnapshot(bucketName, key,
+ account -> openForRead(resolveObjectPathForRead(account, bucketName, key)),
+ S3Service::closeQuietly);
+ return objectRead(snapshot.object(), snapshot.body());
+ }
+ String bucketOwnerAccount = resolveBucketEntry(bucketName)
+ .orElseThrow(() -> new AwsException("NoSuchBucket",
+ "The specified bucket does not exist.", 404))
+ .account();
+ S3Object obj = getObjectMetadata(bucketName, key, versionId);
+ Path path = "null".equals(versionId)
+ ? resolveObjectPathForRead(bucketOwnerAccount, bucketName, key)
+ : resolveVersionedPathForRead(bucketOwnerAccount, bucketName, key, versionId);
+ return objectRead(obj, openForRead(path));
+ }
+
+ private static ObjectRead objectRead(S3Object obj, FileChannel channel) {
+ try {
+ long fileSize = channel.size();
+ if (fileSize != obj.getSize()) {
+ throw new IllegalStateException("S3 object file for " + obj.getBucketName() + "/"
+ + obj.getKey() + " has " + fileSize + " bytes but metadata declares "
+ + obj.getSize() + "; serving it would corrupt the response framing");
+ }
+ } catch (IOException e) {
+ closeQuietly(channel);
+ throw new UncheckedIOException("Failed to read S3 object file size", e);
+ } catch (RuntimeException e) {
+ closeQuietly(channel);
+ throw e;
+ }
+ return new ObjectRead(obj, Channels.newInputStream(channel));
+ }
+
+ private static FileChannel openForRead(Path path) {
+ try {
+ return FileChannel.open(path, StandardOpenOption.READ);
+ } catch (IOException e) {
+ throw new UncheckedIOException("Failed to open S3 object file", e);
+ }
+ }
+
+ private static void closeQuietly(Closeable closeable) {
+ try {
+ closeable.close();
+ } catch (IOException ignored) {
+ // Nothing was read from it; closing is best effort.
+ }
+ }
+
public S3Object headObject(String bucketName, String key) {
return headObject(bucketName, key, null);
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ObjectStreamTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ObjectStreamTest.java
new file mode 100644
index 0000000000..4f37ac11fa
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ObjectStreamTest.java
@@ -0,0 +1,87 @@
+package io.github.hectorvent.floci.services.s3;
+
+import com.sun.management.ThreadMXBean;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.lang.management.ManagementFactory;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Path;
+import java.util.Map;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assumptions.assumeTrue;
+
+class S3ObjectStreamTest {
+
+ @TempDir
+ Path tempDir;
+
+ @Test
+ void readingAPersistedObjectDoesNotLoadItIntoTheHeap() throws Exception {
+ S3Service s3 = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), tempDir.resolve("s3"), false);
+ s3.createBucket("large-objects", "us-east-1");
+ int objectSize = 64 * 1024 * 1024;
+ s3.putObject("large-objects", "blob.bin", new byte[objectSize], "application/octet-stream", Map.of());
+
+ ThreadMXBean threads = (ThreadMXBean) ManagementFactory.getThreadMXBean();
+ long threadId = Thread.currentThread().threadId();
+ byte[] buffer = new byte[64 * 1024];
+ long total = 0;
+ assumeTrue(threads.isThreadAllocatedMemorySupported() && threads.isThreadAllocatedMemoryEnabled(),
+ "thread allocation tracking is unavailable");
+ long allocatedBefore = threads.getThreadAllocatedBytes(threadId);
+ try (S3Service.ObjectRead read = s3.openObject("large-objects", "blob.bin", null)) {
+ int n;
+ while ((n = read.body().read(buffer)) > 0) {
+ total += n;
+ }
+ }
+ long allocated = threads.getThreadAllocatedBytes(threadId) - allocatedBefore;
+
+ assertEquals(objectSize, total);
+ assertTrue(allocated < objectSize / 4,
+ "reading a " + objectSize + " byte object allocated " + allocated + " bytes");
+ }
+
+ @Test
+ void overwriteAfterOpenDoesNotChangeTheOpenedBody() throws Exception {
+ S3Service s3 = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), tempDir.resolve("s3"), false);
+ s3.createBucket("objects", "us-east-1");
+ byte[] first = "first version".getBytes(StandardCharsets.UTF_8);
+ s3.putObject("objects", "key.txt", first, "text/plain", Map.of());
+
+ try (S3Service.ObjectRead read = s3.openObject("objects", "key.txt", null)) {
+ s3.putObject("objects", "key.txt", "second, longer version".getBytes(StandardCharsets.UTF_8),
+ "text/plain", Map.of());
+ InputStream body = read.body();
+
+ assertEquals(first.length, read.object().getSize());
+ assertArrayEquals(first, body.readAllBytes());
+ }
+ }
+
+ @Test
+ void responseThatFailsToBuildClosesTheObjectStream() {
+ AtomicBoolean closed = new AtomicBoolean();
+ InputStream body = new ByteArrayInputStream(new byte[16]) {
+ @Override
+ public void close() {
+ closed.set(true);
+ }
+ };
+
+ assertThrows(NullPointerException.class, () -> S3Controller.streamingResponse(body, stream -> {
+ throw new NullPointerException("Last-Modified is missing");
+ }));
+
+ assertTrue(closed.get(), "the object stream was left open");
+ }
+}