Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3): shrink getObject's lock to a cheap seqlock check, not the I/O (#2901)
Priamo Germosen3a212d83605a
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.javaindex 018851c8f3..a03ee91bc2 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java@@ -482,6 +482,9 @@ public class S3Service implements Resettable, ResourceProvider {String versionId = UUID.randomUUID().toString();object.setVersionId(versionId);object.setLatest(true);+ // Doubles as this object's dataGeneration token (see the field's javadoc on+ // S3Object) - reusing versionId needs no extra random value.+ object.setDataGeneration(versionId);// Check lock protection on the current latest before overwritingString latestKey = objectKey(bucketName, key);@@ -501,20 +504,29 @@ public class S3Service implements Resettable, ResourceProvider {effectiveOptions.getRetainUntilDate(),effectiveOptions.getLegalHoldStatus());+ // Write the body before publishing metadata: getObject's optimistic read (see+ // getLatestObject) relies on a generation only ever becoming visible in objectStore+ // once its file is fully on disk, or a reader could see the new dataGeneration and+ // still read the previous write's bytes underneath it. Write the fresh, not-yet-+ // referenced versioned file first and the shared canonical file last: if the+ // versioned write fails, the canonical file - which unlocked GETs already associate+ // with the still-unpublished previous generation - is never touched, so a concurrent+ // GET can't observe corrupted "latest" bytes paired with the old metadata.+ writeVersionedFile(bucketName, key, versionId, data);+ writeFile(bucketName, key, data);+ // Release the cached payload before publishing: once objectStore.put makes this+ // instance visible to other threads, a concurrent getObject can hold a reference to+ // it (copyObject reads getData() without any lock) and race this null-out otherwise.+ object.setData(null);// Store versioned copy and update latest pointerobjectStore.put(versionedKey(bucketName, key, versionId), object);objectStore.put(latestKey, object);- writeFile(bucketName, key, data);- writeVersionedFile(bucketName, key, versionId, data);LOG.debugv("Put versioned object: {0}/{1} v={2} ({3} bytes)", bucketName, key, versionId, data.length);} else {+ S3Object prev = objectStore.get(objectKey(bucketName, key)).orElse(null);// Check lock protection on the existing object before overwriting- if (bucket.isObjectLockEnabled()) {- objectStore.get(objectKey(bucketName, key)).ifPresent(prev -> {- if (!prev.isDeleteMarker()) {- checkLockProtection(prev, false);- }- });+ if (bucket.isObjectLockEnabled() && prev != null && !prev.isDeleteMarker()) {+ checkLockProtection(prev, false);}// Apply lock fields from request or bucket default@@ -523,12 +535,20 @@ public class S3Service implements Resettable, ResourceProvider {effectiveOptions.getRetainUntilDate(),effectiveOptions.getLegalHoldStatus());- objectStore.put(objectKey(bucketName, key), object);+ // A fresh per-write token, compared by getObject's optimistic read against a+ // later re-read of this same field to detect a concurrent overwrite - see the+ // dataGeneration javadoc on S3Object.+ object.setDataGeneration(UUID.randomUUID().toString());++ // Write the body before publishing metadata - see the comment in the versioned+ // branch above; the same ordering requirement applies here.writeFile(bucketName, key, data);+ // Release the cached payload before publishing - see the comment in the versioned+ // branch above; the same race applies here.+ object.setData(null);+ objectStore.put(objectKey(bucketName, key), object);LOG.debugv("Put object: {0}/{1} ({2} bytes)", bucketName, key, data.length);}- // Release cached payload reference - data is now persisted to disk (or to memoryDataStore in inMemory mode)- object.setData(null);return object;}@@ -885,27 +905,60 @@ public class S3Service implements Resettable, ResourceProvider {}public S3Object getObject(String bucketName, String key, String versionId) {+ if (versionId != null) {+ // An explicit version's file is immutable once written (see storeObjectInternal) and+ // never reused by a later PUT, so this pairing can never race a concurrent overwrite.+ S3Object obj = getObjectMetadata(bucketName, key, versionId);+ obj.setData(readVersionedFile(bucketName, key, versionId));+ return obj;+ }+ return getLatestObject(bucketName, key);+ }++ /**+ * Reads the "latest" object without locking against a concurrent overwrite for the+ * (potentially slow) metadata read or file read, while still never pairing one write's+ * metadata with another write's bytes. storeObjectInternal stamps every write with a fresh,+ * random dataGeneration and, critically, always finishes writing the file for that generation+ * before publishing metadata that names it (see the write-ordering comment in+ * storeObjectInternal) - so a generation can only become visible in objectStore once its+ * bytes are already on disk. This is a seqlock-style optimistic read: read the metadata, read+ * the file, then re-read the metadata's dataGeneration under the bucket monitor and compare it+ * to the first read. That re-read can only happen either fully before or fully after any+ * single write's monitor-held publish, so an unchanged token proves no write completed while+ * this read was in flight; combined with the write-before-publish ordering, that also proves+ * the file read - which happened after the first metadata read, and therefore after that+ * generation's file was already written - cannot have observed an earlier, stale generation's+ * bytes. A change (or a concurrent delete) means an overwrite landed mid-read, so the whole+ * read is retried. Objects written before this scheme existed have no dataGeneration recorded;+ * since nothing can concurrently overwrite a key without immediately stamping one, a null+ * token is only ever observed when untouched, and untouched means nothing to race against.+ */+ private S3Object getLatestObject(String bucketName, String key) {Bucket bucket = resolveBucket(bucketName).orElseThrow(() -> new AwsException("NoSuchBucket","The specified bucket does not exist.", 404));- // Writers install metadata (objectStore) and data (writeFile) as two separate stores- // while holding the bucket monitor (storeObject). Pairing the two reads under the same- // monitor keeps a GET atomic against a concurrent overwrite: size/ETag/checksums and the- // body always describe the same version. Reading them unlocked can pair version N's- // metadata with version N+1's bytes, which on the wire becomes a body that disagrees- // with the declared Content-Length and corrupts the connection's HTTP framing. AWS S3- // gives a concurrent read exactly one complete version, never a mix.- synchronized (bucket) {- S3Object obj = getObjectMetadata(bucketName, key, versionId);-- // Read from versioned file if available, otherwise from latest- if (versionId != null) {- obj.setData(readVersionedFile(bucketName, key, versionId));- } else {- obj.setData(readFile(bucketName, key));+ String storeKey = objectKey(bucketName, key);+ // A genuine race only ever needs a retry or two; this bound exists so a resolution bug+ // (the recheck disagreeing with getObjectMetadata about where this key lives) fails loudly+ // with a clear error instead of spinning forever re-reading the file and exhausting the heap.+ for (int attempt = 0; attempt < 10_000; attempt++) {+ S3Object obj = getObjectMetadata(bucketName, key, null);+ byte[] data = readFile(bucketName, key);+ synchronized (bucket) {+ S3Object current = resolveObject(storeKey).orElse(null);+ if (current != null && !current.isDeleteMarker()+ && Objects.equals(current.getDataGeneration(), obj.getDataGeneration())) {+ obj.setData(data);+ return obj;+ }}- return obj;+ // A concurrent overwrite (or delete) landed mid-read; retry against the new state.}+ throw new IllegalStateException(+ "getObject retry limit exceeded for " + bucketName + "/" + key+ + " - the object is either under sustained concurrent overwrite or the "+ + "metadata/data resolution paths disagree about where this key lives");}public S3Object headObject(String bucketName, String key) {@@ -1106,12 +1159,11 @@ public class S3Service implements Resettable, ResourceProvider {});return toDelete;} else {+ S3Object existing = objectStore.get(objectKey(bucketName, key)).orElse(null);// Check lock on the non-versioned object before delete- objectStore.get(objectKey(bucketName, key)).ifPresent(obj -> {- if (!obj.isDeleteMarker()) {- checkLockProtection(obj, bypassGovernance);- }- });+ if (existing != null && !existing.isDeleteMarker()) {+ checkLockProtection(existing, bypassGovernance);+ }// Non-versioned deleteobjectStore.delete(objectKey(bucketName, key));deleteFile(bucketName, key);@@ -3267,6 +3319,7 @@ public class S3Service implements Resettable, ResourceProvider {copy.setRetainUntilDate(source.getRetainUntilDate());copy.setLegalHoldStatus(source.getLegalHoldStatus());copy.setAcl(source.getAcl());+ copy.setDataGeneration(source.getDataGeneration());return copy;}diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.javaindex 98b6bffa05..ddac70f939 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/model/S3Object.java@@ -43,6 +43,14 @@ public class S3Object {private String legalHoldStatus; // "ON" | "OFF" | nullprivate String acl;+ // Internal-only per-write stamp (not an S3 versionId, and set even when bucket versioning is+ // off - see S3Service#getLatestObject). A fresh value is assigned every time this key is+ // overwritten; comparing two reads of it is how a GET detects whether a concurrent overwrite+ // landed in the middle of reading this object's metadata and body, without holding a lock+ // for that read's full duration.+ @JsonIgnore+ private String dataGeneration;+public S3Object() {this.metadata = new HashMap<>();this.storageClass = "STANDARD";@@ -143,6 +151,9 @@ public class S3Object {public String getAcl() { return acl; }public void setAcl(String acl) { this.acl = acl; }+ public String getDataGeneration() { return dataGeneration; }+ public void setDataGeneration(String dataGeneration) { this.dataGeneration = dataGeneration; }+private static String computeETag(byte[] data) {try {var md = java.security.MessageDigest.getInstance("MD5");