Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(s3): make GetObject atomic against concurrent same-key overwrites (#2874)

Thomas Kosiewskib4aab999f75b
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
index 2393efd5be..d58475d666 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Controller.java
@@ -39,7 +39,6 @@ import jakarta.ws.rs.core.Response;
import jakarta.ws.rs.core.StreamingOutput;
import jakarta.ws.rs.core.UriInfo;
import java.io.ByteArrayOutputStream;
-import java.io.InputStream;
import java.io.StringReader;
import java.net.URLDecoder;
import java.nio.charset.StandardCharsets;
@@ -232,7 +231,7 @@ public class S3Controller {
S3Service.RequestAuthorization authorization = S3RequestAuthorizationParser.parseIfRequired(
s3Service.isAuthEnforced(), httpHeaders, uriInfo);
if (isWebsiteRequest(httpHeaders, uriInfo)) {
- Response websiteResponse = serveWebsiteObject(bucket, "", authorization);
+ Response websiteResponse = serveWebsiteObject(bucket, "", authorization, false);
if (websiteResponse != null) {
return headOnlyResponse(websiteResponse);
}
@@ -551,7 +550,7 @@ public class S3Controller {
// /?code=...&state=... must return index.html, not a ListObjects response. (?list-type and
// other sub-resource queries only reach the REST endpoint, never a website host.)
if (isWebsiteRequest(httpHeaders, uriInfo)) {
- Response website = serveWebsiteObject(bucket, "", authorization);
+ Response website = serveWebsiteObject(bucket, "", authorization, true);
if (website != null) {
return website;
}
@@ -787,7 +786,7 @@ public class S3Controller {
s3Service.isAuthEnforced(), httpHeaders, uriInfo);
if (isWebsiteRequest(httpHeaders, uriInfo)) {
- Response website = serveWebsiteObject(bucket, key, authorization);
+ Response website = serveWebsiteObject(bucket, key, authorization, true);
if (website != null) {
return website;
}
@@ -831,15 +830,19 @@ public class S3Controller {
mergedAttributes, maxParts, partNumberMarker);
}
s3Service.authorizeGetObject(bucket, key, versionId, authorization);
+ // Fetch metadata and body as one atomic snapshot: resolving the body lazily at
+ // entity-write time (openObjectStream) races concurrent overwrites and can pair one
+ // version's Content-Length/checksum headers with another version's bytes.
+ S3Object obj = s3Service.getObject(bucket, key, versionId);
if (hasPreconditions(ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince)) {
- // Fetch metadata only to evaluate preconditions, avoiding loading the full object unnecessarily.
- S3Object metadata = s3Service.headObject(bucket, key, versionId);
- Response preconditionResponse = checkPreconditions(metadata, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince);
+ // Evaluate preconditions against the same snapshot that is served: a separate
+ // metadata fetch could approve one version (e.g. If-Match for a CAS read) while a
+ // concurrent overwrite swaps in another before the body is resolved.
+ Response preconditionResponse = checkPreconditions(obj, ifMatch, ifNoneMatch, ifModifiedSince, ifUnmodifiedSince);
if (preconditionResponse != null) {
return preconditionResponse;
}
}
- S3Object obj = s3Service.headObject(bucket, key, versionId);
S3Service.validateSseCustomerAccess(
obj,
httpHeaders.getHeaderString("x-amz-server-side-encryption-customer-algorithm"),
@@ -855,11 +858,11 @@ public class S3Controller {
boolean includeChecksum = "ENABLED".equalsIgnoreCase(checksumMode);
if (rangeHeader != null && rangeHeader.startsWith("bytes=")) {
- return handleRangeRequest(bucket, key, versionId, obj, rangeHeader, overrides, includeChecksum);
+ return handleRangeRequest(obj, rangeHeader, overrides, includeChecksum);
}
emitCloudTrailEvent("GetObject", bucket, key, 0L, obj.getSize(), null, null);
- return fullObjectResponse(bucket, key, versionId, obj, overrides, includeChecksum);
+ return fullObjectResponse(obj, overrides, includeChecksum);
} catch (AwsException e) {
emitCloudTrailEvent("GetObject", bucket, key, 0L, 0L, e.getErrorCode(), e.getMessage());
if (S3Service.isWebsiteErrorDocumentTrigger(e) && isWebsiteRequest(httpHeaders, uriInfo)) {
@@ -872,16 +875,18 @@ public class S3Controller {
}
}
- private Response fullObjectResponse(String bucket, String key, String versionId,
- S3Object obj, ResponseHeaderOverrides overrides,
+ private Response fullObjectResponse(S3Object obj, ResponseHeaderOverrides overrides,
boolean includeChecksum) {
- StreamingOutput stream = output -> {
- try (InputStream input = s3Service.openObjectStream(bucket, key, versionId)) {
- input.transferTo(output);
- }
- };
- var resp = Response.ok(stream)
- .header("Content-Type", overrides.contentType() != null ? overrides.contentType() : obj.getContentType())
+ byte[] data = objectDataSnapshot(obj);
+ StreamingOutput stream = output -> output.write(data);
+ return objectResponseHeaders(Response.ok(stream), obj, overrides, includeChecksum).build();
+ }
+
+ /** Applies the standard GetObject response headers derived from {@code obj} to {@code resp}. */
+ private Response.ResponseBuilder objectResponseHeaders(Response.ResponseBuilder resp, S3Object obj,
+ ResponseHeaderOverrides overrides,
+ boolean includeChecksum) {
+ resp.header("Content-Type", overrides.contentType() != null ? overrides.contentType() : obj.getContentType())
.header("Content-Length", obj.getSize())
.header("ETag", obj.getETag())
.header("Last-Modified", RFC_822.format(obj.getLastModified()))
@@ -890,11 +895,10 @@ public class S3Controller {
resp.header("x-amz-version-id", obj.getVersionId());
}
appendObjectHeaders(resp, obj, overrides, includeChecksum);
- return resp.build();
+ return resp;
}
- private Response handleRangeRequest(String bucket, String key, String versionId,
- S3Object obj, String rangeHeader,
+ private Response handleRangeRequest(S3Object obj, String rangeHeader,
ResponseHeaderOverrides overrides,
boolean includeChecksum) {
long totalSize = obj.getSize();
@@ -928,18 +932,15 @@ public class S3Controller {
if (start < 0 || start >= totalSize || start > end) {
if (totalSize == 0 && rangeSpec.startsWith("-")) {
- return fullObjectResponse(bucket, key, versionId, obj, overrides, includeChecksum);
+ return fullObjectResponse(obj, overrides, includeChecksum);
}
return invalidRangeResponse(totalSize);
}
long length = end - start + 1;
- StreamingOutput stream = output -> {
- try (InputStream input = s3Service.openObjectStream(bucket, key, versionId)) {
- input.skipNBytes(start);
- transferLimited(input, output, length);
- }
- };
+ byte[] data = objectDataSnapshot(obj);
+ // start/length fit in int: they are bounded by the snapshot's length (a byte[]).
+ StreamingOutput stream = output -> output.write(data, (int) start, (int) length);
var resp = Response.status(206)
.entity(stream)
.header("Content-Type", overrides.contentType() != null ? overrides.contentType() : obj.getContentType())
@@ -956,18 +957,24 @@ public class S3Controller {
return resp.build();
}
- private static void transferLimited(InputStream input, java.io.OutputStream output, long bytes)
- throws java.io.IOException {
- byte[] buffer = new byte[8192];
- long remaining = bytes;
- while (remaining > 0) {
- int count = input.read(buffer, 0, (int) Math.min(buffer.length, remaining));
- if (count < 0) {
- throw new java.io.EOFException("Object stream ended before the requested range was fully written.");
- }
- output.write(buffer, 0, count);
- remaining -= count;
+ /**
+ * Returns the body bytes captured together with {@code obj}'s metadata by
+ * {@link S3Service#getObject}. Serving the response from this snapshot (instead of re-reading
+ * the store at entity-write time) guarantees the body always matches the already-committed
+ * Content-Length/ETag/checksum headers, even when a concurrent PutObject overwrites the key.
+ */
+ private static byte[] objectDataSnapshot(S3Object obj) {
+ byte[] data = obj.getData();
+ if (data == null) {
+ throw new IllegalStateException("S3 object data snapshot is missing for "
+ + obj.getBucketName() + "/" + obj.getKey());
+ }
+ if (data.length != obj.getSize()) {
+ throw new IllegalStateException("S3 object data snapshot for " + obj.getBucketName()
+ + "/" + obj.getKey() + " has " + data.length + " bytes but metadata declares "
+ + obj.getSize() + "; serving it would corrupt the response framing");
}
+ return data;
}
private Response invalidRangeResponse(long totalSize) {
@@ -1010,7 +1017,7 @@ public class S3Controller {
authorization = S3RequestAuthorizationParser.parseIfRequired(
s3Service.isAuthEnforced(), httpHeaders, uriInfo);
if (isWebsiteRequest(httpHeaders, uriInfo)) {
- Response websiteResponse = serveWebsiteObject(bucket, key, authorization);
+ Response websiteResponse = serveWebsiteObject(bucket, key, authorization, false);
if (websiteResponse != null) {
return headOnlyResponse(websiteResponse);
}
@@ -2355,13 +2362,14 @@ public class S3Controller {
* (a no-op unless S3 auth enforcement is enabled), matching the object-serving path.
*/
private Response serveWebsiteObject(String bucket, String key,
- S3Service.RequestAuthorization authorization) {
+ S3Service.RequestAuthorization authorization,
+ boolean includeBody) {
// The routing layer strips a trailing slash from the object key, so the "directory" intent
// has to be recovered from the raw request path before handing off to the service.
String rawPath = currentVertxRequest.getCurrent().request().path();
return renderWebsiteResolution(bucket,
s3Service.resolveWebsiteRequest(bucket, key, rawPath.endsWith("/"), authorization),
- rawPath);
+ rawPath, includeBody);
}
private Response serveWebsiteErrorResponse(String bucket,
@@ -2369,7 +2377,7 @@ public class S3Controller {
AwsException cause) {
try {
return renderWebsiteResolution(bucket,
- s3Service.resolveWebsiteError(bucket, authorization, cause.getHttpStatus()), null);
+ s3Service.resolveWebsiteError(bucket, authorization, cause.getHttpStatus()), null, true);
} catch (AwsException websiteException) {
return xmlErrorResponse(websiteException);
}
@@ -2378,16 +2386,22 @@ public class S3Controller {
/**
* Render a {@link S3Service.WebsiteResolution} as HTTP. {@code rawPath} is only needed for the
* directory redirect; pass {@code null} where that outcome cannot occur. Returns {@code null}
- * for {@code NotAWebsite}, meaning "fall through to the normal object path".
+ * for {@code NotAWebsite}, meaning "fall through to the normal object path". With
+ * {@code includeBody} false (HEAD requests) the served object's bytes are never loaded; the
+ * headers come from the resolution's metadata snapshot.
*/
private Response renderWebsiteResolution(String bucket, S3Service.WebsiteResolution resolution,
- String rawPath) {
+ String rawPath, boolean includeBody) {
+ var noOverrides = new ResponseHeaderOverrides(null, null, null, null, null, null);
return switch (resolution) {
// A website endpoint serves the index document with no response-header overrides and no
// checksum headers (no viewer sends response-* or x-amz-checksum-mode to a website endpoint).
+ // For GET, the body is re-fetched as one atomic metadata+data snapshot so a concurrent
+ // overwrite of the index document cannot tear the response.
case S3Service.WebsiteResolution.ServeObject(String key, S3Object object) ->
- fullObjectResponse(bucket, key, null, object,
- new ResponseHeaderOverrides(null, null, null, null, null, null), false);
+ includeBody
+ ? fullObjectResponse(s3Service.getObject(bucket, key), noOverrides, false)
+ : objectResponseHeaders(Response.ok(), object, noOverrides, false).build();
// The query string is deliberately dropped: real S3 answers
// GET /photos?code=abc&state=xyz with a bare "Location: /photos/" (verified against a
// live website endpoint in us-east-1, same for HEAD and for nested prefixes).
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
index e2a01a0471..018851c8f3 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
@@ -885,15 +885,27 @@ public class S3Service implements Resettable, ResourceProvider {
}
public S3Object getObject(String bucketName, String key, String versionId) {
- S3Object obj = getObjectMetadata(bucketName, key, versionId);
+ Bucket bucket = resolveBucket(bucketName)
+ .orElseThrow(() -> new AwsException("NoSuchBucket",
+ "The specified bucket does not exist.", 404));
+ // Writers install metadata (objectStore) and data (writeFile) as two separate stores
+ // while holding the bucket monitor (storeObject). Pairing the two reads under the same
+ // monitor keeps a GET atomic against a concurrent overwrite: size/ETag/checksums and the
+ // body always describe the same version. Reading them unlocked can pair version N's
+ // metadata with version N+1's bytes, which on the wire becomes a body that disagrees
+ // with the declared Content-Length and corrupts the connection's HTTP framing. AWS S3
+ // gives a concurrent read exactly one complete version, never a mix.
+ synchronized (bucket) {
+ S3Object obj = getObjectMetadata(bucketName, key, versionId);
- // Read from versioned file if available, otherwise from latest
- if (versionId != null) {
- obj.setData(readVersionedFile(bucketName, key, versionId));
- } else {
- obj.setData(readFile(bucketName, key));
+ // Read from versioned file if available, otherwise from latest
+ if (versionId != null) {
+ obj.setData(readVersionedFile(bucketName, key, versionId));
+ } else {
+ obj.setData(readFile(bucketName, key));
+ }
+ return obj;
}
- return obj;
}
public S3Object headObject(String bucketName, String key) {
@@ -1641,6 +1653,8 @@ public class S3Service implements Resettable, ResourceProvider {
String indexKey = prefix.isEmpty() ? index : prefix + "/" + index;
try {
authorizeGetObject(bucket, indexKey, null, authorization);
+ // Metadata only: the controller fetches the body atomically itself when the
+ // request actually needs one (GET), so HEAD website requests never load it.
return new WebsiteResolution.ServeObject(indexKey, headObject(bucket, indexKey, null));
} catch (AwsException e) {
if (!isWebsiteErrorDocumentTrigger(e)) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ConcurrentWriteReadTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ConcurrentWriteReadTest.java
index ebb803a5b0..ebc9d1d7e8 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3ConcurrentWriteReadTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ConcurrentWriteReadTest.java
@@ -6,6 +6,7 @@ import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
import java.nio.file.Path;
+import java.util.Arrays;
import java.util.Map;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
@@ -87,6 +88,83 @@ class S3ConcurrentWriteReadTest {
assertEquals(true, reads.get() > 0, "reader threads never observed the object");
}
+ /**
+ * Regression test for torn GET responses under same-key overwrites: metadata (size, ETag,
+ * checksums — the source of {@code Content-Length} and {@code x-amz-checksum-*} response
+ * headers) and body bytes must always come from the same version. Writers install metadata
+ * and data as two separate stores under the bucket monitor, so a getObject that reads the
+ * pair without that monitor can return version N's metadata with version N+1's bytes; on the
+ * wire that surfaces as a body longer than the declared Content-Length, whose surplus bytes
+ * corrupt the keep-alive connection's framing. AWS S3 reads are atomic: one complete version,
+ * never a mix.
+ */
+ @Test
+ void concurrentOverwriteNeverPairsMetadataWithAnotherVersionsBody() throws Exception {
+ S3Service s3 = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(),
+ tempDir.resolve("s3"), false);
+ String bucket = "torn-read-repro";
+ String key = "torn-probe";
+ s3.createBucket(bucket, "us-east-1");
+
+ byte[] bodyA = new byte[256];
+ Arrays.fill(bodyA, (byte) 'a');
+ byte[] bodyB = new byte[300];
+ Arrays.fill(bodyB, (byte) 'b');
+ String eTagA = new S3Object(bucket, key, bodyA, null).getETag();
+ String eTagB = new S3Object(bucket, key, bodyB, null).getETag();
+ s3.putObject(bucket, key, bodyA, "application/octet-stream", Map.of());
+
+ int writers = 2;
+ int readers = 4;
+ int iterations = 4000;
+ AtomicReference<String> firstTear = new AtomicReference<>();
+ AtomicInteger reads = new AtomicInteger();
+ volatileDone done = new volatileDone();
+
+ Thread[] threads = new Thread[writers + readers];
+ for (int w = 0; w < writers; w++) {
+ threads[w] = new Thread(() -> {
+ for (int i = 0; i < iterations && firstTear.get() == null; i++) {
+ s3.putObject(bucket, key, i % 2 == 0 ? bodyA : bodyB,
+ "application/octet-stream", Map.of());
+ }
+ done.count.incrementAndGet();
+ });
+ }
+ for (int r = 0; r < readers; r++) {
+ threads[writers + r] = new Thread(() -> {
+ while (done.count.get() < writers && firstTear.get() == null) {
+ S3Object obj = s3.getObject(bucket, key);
+ byte[] data = obj.getData();
+ reads.incrementAndGet();
+ String tear = null;
+ if (data == null || (!Arrays.equals(data, bodyA) && !Arrays.equals(data, bodyB))) {
+ tear = "body is neither version: length="
+ + (data == null ? "null" : data.length);
+ } else if (obj.getSize() != data.length) {
+ tear = "declared size=" + obj.getSize() + " but body has " + data.length
+ + " bytes (Content-Length/body mismatch)";
+ } else if (!obj.getETag().equals(data.length == bodyA.length ? eTagA : eTagB)) {
+ tear = "ETag " + obj.getETag() + " does not match the returned "
+ + data.length + "-byte body";
+ }
+ if (tear != null) {
+ firstTear.compareAndSet(null, tear);
+ return;
+ }
+ }
+ });
+ }
+ for (Thread t : threads) t.start();
+ for (Thread t : threads) {
+ t.join(60_000);
+ assertFalse(t.isAlive(), "thread did not finish within the join timeout");
+ }
+
+ assertNull(firstTear.get(), firstTear.get());
+ assertEquals(true, reads.get() > 0, "reader threads never observed the object");
+ }
+
/** Tiny holder so the reader loop can see writers finishing without a shared executor. */
private static final class volatileDone {
final AtomicInteger count = new AtomicInteger();