Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(s3): shrink getObject's lock to a cheap seqlock check, not the I/O (#2901)

Priamo Germosen3a212d83605a
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
index 018851c8f3..a03ee91bc2 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
@@ -482,6 +482,9 @@ public class S3Service implements Resettable, ResourceProvider {
String versionId = UUID.randomUUID().toString();
object.setVersionId(versionId);
object.setLatest(true);
+ // Doubles as this object's dataGeneration token (see the field's javadoc on
+ // S3Object) - reusing versionId needs no extra random value.
+ object.setDataGeneration(versionId);
// Check lock protection on the current latest before overwriting
String latestKey = objectKey(bucketName, key);
@@ -501,20 +504,29 @@ public class S3Service implements Resettable, ResourceProvider {
effectiveOptions.getRetainUntilDate(),
effectiveOptions.getLegalHoldStatus());
+ // Write the body before publishing metadata: getObject's optimistic read (see
+ // getLatestObject) relies on a generation only ever becoming visible in objectStore
+ // once its file is fully on disk, or a reader could see the new dataGeneration and
+ // still read the previous write's bytes underneath it. Write the fresh, not-yet-
+ // referenced versioned file first and the shared canonical file last: if the
+ // versioned write fails, the canonical file - which unlocked GETs already associate
+ // with the still-unpublished previous generation - is never touched, so a concurrent
+ // GET can't observe corrupted "latest" bytes paired with the old metadata.
+ writeVersionedFile(bucketName, key, versionId, data);
+ writeFile(bucketName, key, data);
+ // Release the cached payload before publishing: once objectStore.put makes this
+ // instance visible to other threads, a concurrent getObject can hold a reference to
+ // it (copyObject reads getData() without any lock) and race this null-out otherwise.
+ object.setData(null);
// Store versioned copy and update latest pointer
objectStore.put(versionedKey(bucketName, key, versionId), object);
objectStore.put(latestKey, object);
- writeFile(bucketName, key, data);
- writeVersionedFile(bucketName, key, versionId, data);
LOG.debugv("Put versioned object: {0}/{1} v={2} ({3} bytes)", bucketName, key, versionId, data.length);
} else {
+ S3Object prev = objectStore.get(objectKey(bucketName, key)).orElse(null);
// Check lock protection on the existing object before overwriting
- if (bucket.isObjectLockEnabled()) {
- objectStore.get(objectKey(bucketName, key)).ifPresent(prev -> {
- if (!prev.isDeleteMarker()) {
- checkLockProtection(prev, false);
- }
- });
+ if (bucket.isObjectLockEnabled() && prev != null && !prev.isDeleteMarker()) {
+ checkLockProtection(prev, false);
}
// Apply lock fields from request or bucket default
@@ -523,12 +535,20 @@ public class S3Service implements Resettable, ResourceProvider {
effectiveOptions.getRetainUntilDate(),
effectiveOptions.getLegalHoldStatus());
- objectStore.put(objectKey(bucketName, key), object);
+ // A fresh per-write token, compared by getObject's optimistic read against a
+ // later re-read of this same field to detect a concurrent overwrite - see the
+ // dataGeneration javadoc on S3Object.
+ object.setDataGeneration(UUID.randomUUID().toString());
+
+ // Write the body before publishing metadata - see the comment in the versioned
+ // branch above; the same ordering requirement applies here.
writeFile(bucketName, key, data);
+ // Release the cached payload before publishing - see the comment in the versioned
+ // branch above; the same race applies here.
+ object.setData(null);
+ objectStore.put(objectKey(bucketName, key), object);
LOG.debugv("Put object: {0}/{1} ({2} bytes)", bucketName, key, data.length);
}
- // Release cached payload reference - data is now persisted to disk (or to memoryDataStore in inMemory mode)
- object.setData(null);
return object;
}
@@ -885,27 +905,60 @@ public class S3Service implements Resettable, ResourceProvider {
}
public S3Object getObject(String bucketName, String key, String versionId) {
+ if (versionId != null) {
+ // An explicit version's file is immutable once written (see storeObjectInternal) and
+ // never reused by a later PUT, so this pairing can never race a concurrent overwrite.
+ S3Object obj = getObjectMetadata(bucketName, key, versionId);
+ obj.setData(readVersionedFile(bucketName, key, versionId));
+ return obj;
+ }
+ return getLatestObject(bucketName, key);
+ }
+
+ /**
+ * Reads the "latest" object without locking against a concurrent overwrite for the
+ * (potentially slow) metadata read or file read, while still never pairing one write's
+ * metadata with another write's bytes. storeObjectInternal stamps every write with a fresh,
+ * random dataGeneration and, critically, always finishes writing the file for that generation
+ * before publishing metadata that names it (see the write-ordering comment in
+ * storeObjectInternal) - so a generation can only become visible in objectStore once its
+ * bytes are already on disk. This is a seqlock-style optimistic read: read the metadata, read
+ * the file, then re-read the metadata's dataGeneration under the bucket monitor and compare it
+ * to the first read. That re-read can only happen either fully before or fully after any
+ * single write's monitor-held publish, so an unchanged token proves no write completed while
+ * this read was in flight; combined with the write-before-publish ordering, that also proves
+ * the file read - which happened after the first metadata read, and therefore after that
+ * generation's file was already written - cannot have observed an earlier, stale generation's
+ * bytes. A change (or a concurrent delete) means an overwrite landed mid-read, so the whole
+ * read is retried. Objects written before this scheme existed have no dataGeneration recorded;
+ * since nothing can concurrently overwrite a key without immediately stamping one, a null
+ * token is only ever observed when untouched, and untouched means nothing to race against.
+ */
+ private S3Object getLatestObject(String bucketName, String key) {
Bucket bucket = resolveBucket(bucketName)
.orElseThrow(() -> new AwsException("NoSuchBucket",
"The specified bucket does not exist.", 404));
- // Writers install metadata (objectStore) and data (writeFile) as two separate stores
- // while holding the bucket monitor (storeObject). Pairing the two reads under the same
- // monitor keeps a GET atomic against a concurrent overwrite: size/ETag/checksums and the
- // body always describe the same version. Reading them unlocked can pair version N's
- // metadata with version N+1's bytes, which on the wire becomes a body that disagrees
- // with the declared Content-Length and corrupts the connection's HTTP framing. AWS S3
- // gives a concurrent read exactly one complete version, never a mix.
- synchronized (bucket) {
- S3Object obj = getObjectMetadata(bucketName, key, versionId);
-
- // Read from versioned file if available, otherwise from latest
- if (versionId != null) {
- obj.setData(readVersionedFile(bucketName, key, versionId));
- } else {
- obj.setData(readFile(bucketName, key));
+ String storeKey = objectKey(bucketName, key);
+ // A genuine race only ever needs a retry or two; this bound exists so a resolution bug
+ // (the recheck disagreeing with getObjectMetadata about where this key lives) fails loudly
+ // with a clear error instead of spinning forever re-reading the file and exhausting the heap.
+ for (int attempt = 0; attempt < 10_000; attempt++) {
+ S3Object obj = getObjectMetadata(bucketName, key, null);
+ byte[] data = readFile(bucketName, key);
+ synchronized (bucket) {
+ S3Object current = resolveObject(storeKey).orElse(null);
+ if (current != null && !current.isDeleteMarker()
+ && Objects.equals(current.getDataGeneration(), obj.getDataGeneration())) {
+ obj.setData(data);
+ return obj;
+ }
}
- return obj;
+ // A concurrent overwrite (or delete) landed mid-read; retry against the new state.
}
+ throw new IllegalStateException(
+ "getObject retry limit exceeded for " + bucketName + "/" + key
+ + " - the object is either under sustained concurrent overwrite or the "
+ + "metadata/data resolution paths disagree about where this key lives");
}
public S3Object headObject(String bucketName, String key) {
@@ -1106,12 +1159,11 @@ public class S3Service implements Resettable, ResourceProvider {
});
return toDelete;
} else {
+ S3Object existing = objectStore.get(objectKey(bucketName, key)).orElse(null);
// Check lock on the non-versioned object before delete
- objectStore.get(objectKey(bucketName, key)).ifPresent(obj -> {
- if (!obj.isDeleteMarker()) {
- checkLockProtection(obj, bypassGovernance);
- }
- });
+ if (existing != null && !existing.isDeleteMarker()) {
+ checkLockProtection(existing, bypassGovernance);
+ }
// Non-versioned delete
objectStore.delete(objectKey(bucketName, key));
deleteFile(bucketName, key);
@@ -3267,6 +3319,7 @@ public class S3Service implements Resettable, ResourceProvider {
copy.setRetainUntilDate(source.getRetainUntilDate());
copy.setLegalHoldStatus(source.getLegalHoldStatus());
copy.setAcl(source.getAcl());
+ copy.setDataGeneration(source.getDataGeneration());
return copy;
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java
index 98b6bffa05..ddac70f939 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java
@@ -43,6 +43,14 @@ public class S3Object {
private String legalHoldStatus; // "ON" | "OFF" | null
private String acl;
+ // Internal-only per-write stamp (not an S3 versionId, and set even when bucket versioning is
+ // off - see S3Service#getLatestObject). A fresh value is assigned every time this key is
+ // overwritten; comparing two reads of it is how a GET detects whether a concurrent overwrite
+ // landed in the middle of reading this object's metadata and body, without holding a lock
+ // for that read's full duration.
+ @JsonIgnore
+ private String dataGeneration;
+
public S3Object() {
this.metadata = new HashMap<>();
this.storageClass = "STANDARD";
@@ -143,6 +151,9 @@ public class S3Object {
public String getAcl() { return acl; }
public void setAcl(String acl) { this.acl = acl; }
+ public String getDataGeneration() { return dataGeneration; }
+ public void setDataGeneration(String dataGeneration) { this.dataGeneration = dataGeneration; }
+
private static String computeETag(byte[] data) {
try {
var md = java.security.MessageDigest.getInstance("MD5");