Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3): isolate object byte storage between accounts (#2241)
Dmitriy Polomoshnovcdb2c36dfe62
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.javaindex d2d82782cc..ec463503a5 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java@@ -40,6 +40,7 @@ import java.time.Instant;import java.time.format.DateTimeFormatter;import java.util.*;import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.locks.ReentrantLock;import java.util.regex.Matcher;import java.util.regex.Pattern;@@ -71,6 +72,19 @@ public class S3Service implements Resettable {private final Path dataRoot;private final boolean inMemory;private final ConcurrentHashMap<String, byte[]> memoryDataStore = new ConcurrentHashMap<>();+ // Guards disk writes/deletes against a racing legacy migration for the same path (see+ // copyLegacyFileIfPresent()). Fixed-size stripes keep memory bounded, unlike a per-path+ // map that would need reference counting to ever shrink safely.+ private static final int DISK_FILE_LOCK_STRIPES = 256;+ private final ReentrantLock[] diskFileLocks = newLockStripes(DISK_FILE_LOCK_STRIPES);++ private static ReentrantLock[] newLockStripes(int count) {+ ReentrantLock[] locks = new ReentrantLock[count];+ for (int i = 0; i < count; i++) {+ locks[i] = new ReentrantLock();+ }+ return locks;+ }private final ConcurrentHashMap<String, Map<Integer, byte[]>> memoryMultipartStore = new ConcurrentHashMap<>();private final ConcurrentHashMap<String, MultipartUpload> multipartUploads = new ConcurrentHashMap<>();@@ -183,6 +197,12 @@ public class S3Service implements Resettable {}public Bucket createBucket(String bucketName, String region) {+ if (ACCOUNT_STORAGE_ROOT.equals(bucketName)) {+ // Floci doesn't otherwise validate bucket name format, but this name must stay+ // unavailable — it's the root every account's disk storage lives under.+ throw new AwsException("InvalidBucketName",+ "The specified bucket is not valid.", 400);+ }var existing = bucketStore.get(bucketName);if (existing.isPresent()) {Bucket bucket = existing.get();@@ -217,10 +237,10 @@ public class S3Service implements Resettable {bucketStore.delete(bucketName);if (inMemory) {- String prefix = bucketName + "/";+ String prefix = ownerId() + "/" + bucketName + "/";memoryDataStore.keySet().removeIf(k -> k.startsWith(prefix));} else {- deleteDirectory(dataRoot.resolve(bucketName));+ deleteDirectory(dataRoot.resolve(ACCOUNT_STORAGE_ROOT).resolve(ownerId()).resolve(bucketName));}LOG.infov("Deleted bucket: {0}", bucketName);}@@ -659,8 +679,8 @@ public class S3Service implements Resettable {getObjectMetadata(bucketName, key, versionId);if (inMemory) {byte[] data = versionId != null- ? memoryDataStore.get(versionedKey(bucketName, key, versionId))- : memoryDataStore.get(objectKey(bucketName, key));+ ? memoryDataStore.get(physicalVersionedKey(bucketName, key, versionId))+ : memoryDataStore.get(physicalKey(bucketName, key));if (data == null) {throw new IllegalStateException("S3 object data is missing for " + bucketName + "/" + key);}@@ -668,8 +688,8 @@ public class S3Service implements Resettable {}try {Path path = versionId != null- ? resolveVersionedPath(bucketName, key, versionId)- : resolveObjectPath(bucketName, key);+ ? resolveVersionedPathForRead(bucketName, key, versionId)+ : resolveObjectPathForRead(bucketName, key);return Files.newInputStream(path);} catch (IOException e) {throw new UncheckedIOException("Failed to open S3 object stream", e);@@ -2564,14 +2584,29 @@ public class S3Service implements Resettable {private static final String DATA_SUFFIX = ".s3data";+ // A 12-digit bucket name is valid on S3 and would otherwise collide with an account ID,+ // making dataRoot/<accountId>/... indistinguishable from the legacy dataRoot/<bucket>/...+ // layout. A leading "." keeps this namespace unreachable by any real bucket name.+ private static final String ACCOUNT_STORAGE_ROOT = ".accounts";++ // Unlike bucketStore/objectStore, object bytes get no automatic account prefixing — two+ // accounts can own a bucket named "orders" and would collide here without this scoping.+ private String physicalKey(String bucketName, String key) {+ return ownerId() + "/" + objectKey(bucketName, key);+ }++ private String physicalVersionedKey(String bucketName, String key, String versionId) {+ return ownerId() + "/" + versionedKey(bucketName, key, versionId);+ }+private Path resolveObjectPath(String bucketName, String key) {- Path bucketDir = dataRoot.resolve(bucketName).normalize();-+ Path bucketDir = dataRoot.resolve(ACCOUNT_STORAGE_ROOT).resolve(ownerId()).resolve(bucketName).normalize();+String safeKey = key;while (safeKey.startsWith("/")) {safeKey = safeKey.substring(1);}-+Path resolved = bucketDir.resolve(safeKey + DATA_SUFFIX).normalize();if (!resolved.startsWith(bucketDir)) {throw new AwsException("InvalidKey", "The specified key is invalid.", 400);@@ -2579,14 +2614,35 @@ public class S3Service implements Resettable {return resolved;}+ private Path legacyObjectPath(String bucketName, String key) {+ String safeKey = key;+ while (safeKey.startsWith("/")) {+ safeKey = safeKey.substring(1);+ }+ return dataRoot.resolve(bucketName).normalize().resolve(safeKey + DATA_SUFFIX);+ }++ /**+ * Resolves the path for a read, copying in a legacy-layout file if present. Reads only —+ * a write/delete ({@link #resolveObjectPath}) must never touch legacy data, since it isn't+ * known to belong to any one account and two accounts can share a bucket name. Copying+ * (not moving) leaves the ambiguous source in place so every account that reads it gets+ * its own copy.+ */+ private Path resolveObjectPathForRead(String bucketName, String key) {+ Path resolved = resolveObjectPath(bucketName, key);+ copyLegacyFileIfPresent(legacyObjectPath(bucketName, key), resolved);+ return resolved;+ }+private Path resolveVersionedPath(String bucketName, String key, String versionId) {- Path baseDir = dataRoot.resolve(".versions").resolve(bucketName).normalize();-+ Path baseDir = dataRoot.resolve(ACCOUNT_STORAGE_ROOT).resolve(ownerId()).resolve(".versions").resolve(bucketName).normalize();+String safeKey = key;while (safeKey.startsWith("/")) {safeKey = safeKey.substring(1);}-+Path resolved = baseDir.resolve(safeKey).resolve(versionId + DATA_SUFFIX).normalize();if (!resolved.startsWith(baseDir)) {throw new AwsException("InvalidKey", "The specified key is invalid.", 400);@@ -2594,26 +2650,86 @@ public class S3Service implements Resettable {return resolved;}+ private Path legacyVersionedPath(String bucketName, String key, String versionId) {+ String safeKey = key;+ while (safeKey.startsWith("/")) {+ safeKey = safeKey.substring(1);+ }+ return dataRoot.resolve(".versions").resolve(bucketName).normalize()+ .resolve(safeKey).resolve(versionId + DATA_SUFFIX);+ }++ /** Read-only counterpart of {@link #resolveObjectPathForRead} for versioned objects. */+ private Path resolveVersionedPathForRead(String bucketName, String key, String versionId) {+ Path resolved = resolveVersionedPath(bucketName, key, versionId);+ copyLegacyFileIfPresent(legacyVersionedPath(bucketName, key, versionId), resolved);+ return resolved;+ }++ private ReentrantLock diskFileLock(Path path) {+ // A stripe collision just serializes unrelated paths — safe, unlike a map entry.+ return diskFileLocks[Math.floorMod(path.hashCode(), diskFileLocks.length)];+ }++ /**+ * Copies in a legacy file for {@code newPath}, under the same lock {@link #writeFile}/+ * {@link #deleteFile} hold for that path — otherwise a concurrent write could land its+ * real content and then be silently clobbered by a racing legacy copy.+ */+ private void copyLegacyFileIfPresent(Path legacyPath, Path newPath) {+ if (!Files.exists(legacyPath)) {+ return;+ }+ ReentrantLock lock = diskFileLock(newPath);+ lock.lock();+ try {+ if (Files.exists(newPath)) {+ return;+ }+ Files.createDirectories(newPath.getParent());+ Files.copy(legacyPath, newPath);+ } catch (IOException e) {+ // A failed copy can leave newPath truncated, which would wrongly look+ // "already migrated" on a later retry — clean it up before rethrowing.+ deleteQuietly(newPath, "partially-copied legacy S3 object file, so a later read can retry migration");+ throw new UncheckedIOException("Failed to copy legacy S3 object file to account-scoped layout", e);+ } finally {+ lock.unlock();+ }+ }++ private void deleteQuietly(Path path, String reason) {+ try {+ Files.deleteIfExists(path);+ } catch (IOException e) {+ LOG.warnv(e, "Failed to delete {0} ({1})", path, reason);+ }+ }+private void writeVersionedFile(String bucketName, String key, String versionId, byte[] data) {if (inMemory) {- memoryDataStore.put(versionedKey(bucketName, key, versionId), data);+ memoryDataStore.put(physicalVersionedKey(bucketName, key, versionId), data);return;}+ Path filePath = resolveVersionedPath(bucketName, key, versionId);+ ReentrantLock lock = diskFileLock(filePath);+ lock.lock();try {- Path filePath = resolveVersionedPath(bucketName, key, versionId);Files.createDirectories(filePath.getParent());Files.write(filePath, data);} catch (IOException e) {throw new UncheckedIOException("Failed to write versioned S3 object file", e);+ } finally {+ lock.unlock();}}private byte[] readVersionedFile(String bucketName, String key, String versionId) {if (inMemory) {- return memoryDataStore.get(versionedKey(bucketName, key, versionId));+ return memoryDataStore.get(physicalVersionedKey(bucketName, key, versionId));}try {- return Files.readAllBytes(resolveVersionedPath(bucketName, key, versionId));+ return Files.readAllBytes(resolveVersionedPathForRead(bucketName, key, versionId));} catch (IOException e) {throw new UncheckedIOException("Failed to read versioned S3 object file", e);}@@ -2621,24 +2737,28 @@ public class S3Service implements Resettable {private void writeFile(String bucketName, String key, byte[] data) {if (inMemory) {- memoryDataStore.put(objectKey(bucketName, key), data);+ memoryDataStore.put(physicalKey(bucketName, key), data);return;}+ Path filePath = resolveObjectPath(bucketName, key);+ ReentrantLock lock = diskFileLock(filePath);+ lock.lock();try {- Path filePath = resolveObjectPath(bucketName, key);Files.createDirectories(filePath.getParent());Files.write(filePath, data);} catch (IOException e) {throw new UncheckedIOException("Failed to write S3 object file", e);+ } finally {+ lock.unlock();}}private byte[] readFile(String bucketName, String key) {if (inMemory) {- return memoryDataStore.get(objectKey(bucketName, key));+ return memoryDataStore.get(physicalKey(bucketName, key));}try {- return Files.readAllBytes(resolveObjectPath(bucketName, key));+ return Files.readAllBytes(resolveObjectPathForRead(bucketName, key));} catch (IOException e) {throw new UncheckedIOException("Failed to read S3 object file", e);}@@ -2646,25 +2766,35 @@ public class S3Service implements Resettable {private void deleteFile(String bucketName, String key) {if (inMemory) {- memoryDataStore.remove(objectKey(bucketName, key));+ memoryDataStore.remove(physicalKey(bucketName, key));return;}+ Path filePath = resolveObjectPath(bucketName, key);+ ReentrantLock lock = diskFileLock(filePath);+ lock.lock();try {- Files.deleteIfExists(resolveObjectPath(bucketName, key));+ Files.deleteIfExists(filePath);} catch (IOException e) {LOG.errorv(e, "Failed to delete S3 object file: {0}/{1}", bucketName, key);+ } finally {+ lock.unlock();}}private void deleteVersionedFile(String bucketName, String key, String versionId) {if (inMemory) {- memoryDataStore.remove(versionedKey(bucketName, key, versionId));+ memoryDataStore.remove(physicalVersionedKey(bucketName, key, versionId));return;}+ Path filePath = resolveVersionedPath(bucketName, key, versionId);+ ReentrantLock lock = diskFileLock(filePath);+ lock.lock();try {- Files.deleteIfExists(resolveVersionedPath(bucketName, key, versionId));+ Files.deleteIfExists(filePath);} catch (IOException e) {LOG.errorv(e, "Failed to delete versioned S3 object file: {0}/{1} v={2}", bucketName, key, versionId);+ } finally {+ lock.unlock();}}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3AccountIsolationIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3AccountIsolationIntegrationTest.javanew file mode 100644index 0000000000..85c69c5f88--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3AccountIsolationIntegrationTest.java@@ -0,0 +1,107 @@+package io.github.hectorvent.floci.services.s3;++import io.quarkus.test.junit.QuarkusTest;+import org.junit.jupiter.api.Test;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.containsString;+import static org.hamcrest.Matchers.not;++/**+ * Verifies S3 object bytes are isolated between accounts, mirroring+ * {@link io.github.hectorvent.floci.core.common.AccountIsolationIntegrationTest}'s SQS coverage.+ * Two accounts may own a same-named bucket in Floci (bucket names are only unique per account+ * here, unlike real S3); object bytes must not collide when they do.+ */+@QuarkusTest+class S3AccountIsolationIntegrationTest {++ private static final String AUTH_ACCOUNT_1 =+ "AWS4-HMAC-SHA256 Credential=000000000001/20260215/us-east-1/s3/aws4_request, SignedHeaders=host, Signature=abc";+ private static final String AUTH_ACCOUNT_2 =+ "AWS4-HMAC-SHA256 Credential=000000000002/20260215/us-east-1/s3/aws4_request, SignedHeaders=host, Signature=abc";++ @Test+ void objectsInSameNamedBucketAreIsolatedBetweenAccounts() {+ String bucket = "account-isolation-shared-bucket";++ createBucket(AUTH_ACCOUNT_1, bucket);+ createBucket(AUTH_ACCOUNT_2, bucket);++ // Both write the same key in their own bucket — exercises the byte-storage+ // collision directly, not just metadata isolation.+ given()+ .header("Authorization", AUTH_ACCOUNT_1)+ .body("account-1-data")+ .when()+ .put("/" + bucket + "/shared-key.txt")+ .then()+ .statusCode(200);++ given()+ .header("Authorization", AUTH_ACCOUNT_2)+ .body("account-2-data")+ .when()+ .put("/" + bucket + "/shared-key.txt")+ .then()+ .statusCode(200);++ given()+ .header("Authorization", AUTH_ACCOUNT_1)+ .when()+ .get("/" + bucket + "/shared-key.txt")+ .then()+ .statusCode(200)+ .body(containsString("account-1-data"));++ given()+ .header("Authorization", AUTH_ACCOUNT_2)+ .when()+ .get("/" + bucket + "/shared-key.txt")+ .then()+ .statusCode(200)+ .body(containsString("account-2-data"));+ }++ @Test+ void deletingBucketDoesNotRemoveAnotherAccountsSameNamedBucketData() {+ String bucket = "account-isolation-delete-bucket";++ createBucket(AUTH_ACCOUNT_1, bucket);+ createBucket(AUTH_ACCOUNT_2, bucket);++ given()+ .header("Authorization", AUTH_ACCOUNT_2)+ .body("account-2-data")+ .when()+ .put("/" + bucket + "/kept-key.txt")+ .then()+ .statusCode(200);++ // Account 1 deletes its own (empty) same-named bucket.+ given()+ .header("Authorization", AUTH_ACCOUNT_1)+ .when()+ .delete("/" + bucket)+ .then()+ .statusCode(204);++ // Account 2's object must be unaffected.+ given()+ .header("Authorization", AUTH_ACCOUNT_2)+ .when()+ .get("/" + bucket + "/kept-key.txt")+ .then()+ .statusCode(200)+ .body(not(containsString("<Error>")));+ }++ private static void createBucket(String auth, String bucket) {+ given()+ .header("Authorization", auth)+ .when()+ .put("/" + bucket)+ .then()+ .statusCode(200);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3LegacyDiskLayoutIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3LegacyDiskLayoutIntegrationTest.javanew file mode 100644index 0000000000..d18119b8c4--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3LegacyDiskLayoutIntegrationTest.java@@ -0,0 +1,268 @@+package io.github.hectorvent.floci.services.s3;++import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.services.lambda.LambdaService;+import io.github.hectorvent.floci.services.s3.model.Bucket;+import io.github.hectorvent.floci.services.s3.model.S3Object;+import org.junit.jupiter.api.Assumptions;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.condition.DisabledOnOs;+import org.junit.jupiter.api.condition.OS;+import org.junit.jupiter.api.io.TempDir;++import java.io.UncheckedIOException;+import java.nio.charset.StandardCharsets;+import java.nio.file.Files;+import java.nio.file.Path;+import java.nio.file.StandardCopyOption;+import java.util.ArrayList;+import java.util.List;+import java.util.concurrent.CyclicBarrier;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;+import java.util.concurrent.TimeUnit;++import static org.junit.jupiter.api.Assertions.assertArrayEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.junit.jupiter.api.Assertions.fail;++/**+ * Before account-scoped object storage, every disk-mode S3 object lived at+ * {@code dataRoot/bucketName/key.s3data}, with no account segment. Upgrading past that change+ * must not lose access to objects already on disk, and — since a legacy file isn't known to+ * belong to any one account, and two accounts can share a bucket name — migrating it must never+ * let one account's write/delete steal or destroy another account's still-unmigrated object.+ */+class S3LegacyDiskLayoutIntegrationTest {++ @TempDir+ Path tempDir;++ @Test+ void existingObjectWrittenUnderTheLegacyUnscopedPathIsStillReadable() throws Exception {+ Path dataRoot = tempDir.resolve("s3");+ S3Service s3Service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), dataRoot, false);++ s3Service.createBucket("legacy-bucket", "us-east-1");+ byte[] data = "pre-upgrade-content".getBytes(StandardCharsets.UTF_8);+ // Write normally, then relocate to the legacy path to simulate pre-upgrade data.+ s3Service.putObject("legacy-bucket", "legacy-key.txt", data, "text/plain", null);++ Path newLayoutFile = dataRoot.resolve(".accounts").resolve("000000000000").resolve("legacy-bucket").resolve("legacy-key.txt.s3data");+ Path legacyLayoutFile = dataRoot.resolve("legacy-bucket").resolve("legacy-key.txt.s3data");+ assertTrue(Files.exists(newLayoutFile), "test setup: object should initially be written at the new path");++ Files.createDirectories(legacyLayoutFile.getParent());+ Files.move(newLayoutFile, legacyLayoutFile, StandardCopyOption.REPLACE_EXISTING);++ S3Object retrieved = s3Service.getObject("legacy-bucket", "legacy-key.txt");+ assertArrayEquals(data, retrieved.getData(),+ "an object physically stored under the pre-upgrade unscoped layout must still be readable");+ }++ @Test+ void bucketNamedLikeTheDefaultAccountIdDoesNotCollideWithAccountScopedStorage() throws Exception {+ // A 12-digit bucket name is valid on S3 and shares its shape with an account ID: a+ // legacy bucket "000000000000" with key "orders/report.csv" must not collide with+ // account "000000000000"'s own "orders" bucket, key "report.csv".+ Path dataRoot = tempDir.resolve("s3");+ S3Service s3Service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), dataRoot, false);++ String accountIdShapedBucket = "000000000000";+ s3Service.createBucket(accountIdShapedBucket, "us-east-1");+ byte[] legacyData = "legacy-bucket-named-like-an-account-id".getBytes(StandardCharsets.UTF_8);+ s3Service.putObject(accountIdShapedBucket, "orders/report.csv", legacyData, "text/plain", null);++ Path newLayoutFile = dataRoot.resolve(".accounts").resolve("000000000000")+ .resolve(accountIdShapedBucket).resolve("orders/report.csv.s3data");+ Path legacyLayoutFile = dataRoot.resolve(accountIdShapedBucket).resolve("orders/report.csv.s3data");+ Files.createDirectories(legacyLayoutFile.getParent());+ Files.move(newLayoutFile, legacyLayoutFile, StandardCopyOption.REPLACE_EXISTING);++ // The default account's own, unrelated "orders" bucket and "report.csv" key.+ s3Service.createBucket("orders", "us-east-1");+ byte[] newData = "unrelated-object-in-the-orders-bucket".getBytes(StandardCharsets.UTF_8);+ s3Service.putObject("orders", "report.csv", newData, "text/plain", null);++ assertArrayEquals(newData, s3Service.getObject("orders", "report.csv").getData(),+ "the unrelated new-layout object must not be shadowed by the account-ID-shaped legacy bucket");+ assertArrayEquals(legacyData, s3Service.getObject(accountIdShapedBucket, "orders/report.csv").getData(),+ "the legacy bucket named like an account ID must still resolve to its own, distinct content");+ }++ @Test+ void bucketNamedLikeTheReservedAccountStorageRootIsRejected() {+ Path dataRoot = tempDir.resolve("s3");+ S3Service s3Service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), dataRoot, false);++ assertThrows(io.github.hectorvent.floci.core.common.AwsException.class,+ () -> s3Service.createBucket(".accounts", "us-east-1"),+ "a bucket named exactly like the reserved account-storage root must be rejected, "+ + "since Floci doesn't otherwise validate bucket name format at all");+ }++ @Test+ @DisabledOnOs(OS.WINDOWS)+ void migrationFailureIsNotPermanentAndDoesNotLeaveAFileBehind() throws Exception {+ // Note: permission-denied fails before Files.copy creates a destination on this JVM,+ // so this doesn't reproduce a genuinely partial write (disk-full mid-copy isn't+ // reliably forceable for a file this small) — it verifies the property that matters+ // to a caller: a transient failure isn't permanent, and a retry recovers.+ Assumptions.assumeFalse("root".equals(System.getProperty("user.name")),+ "root ignores file read permissions");++ Path dataRoot = tempDir.resolve("s3");+ S3Service s3Service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), dataRoot, false);+ s3Service.createBucket("legacy-bucket", "us-east-1");+ byte[] data = "pre-upgrade-content".getBytes(StandardCharsets.UTF_8);+ s3Service.putObject("legacy-bucket", "legacy-key.txt", data, "text/plain", null);++ Path newLayoutFile = dataRoot.resolve(".accounts").resolve("000000000000").resolve("legacy-bucket").resolve("legacy-key.txt.s3data");+ Path legacyLayoutFile = dataRoot.resolve("legacy-bucket").resolve("legacy-key.txt.s3data");+ Files.createDirectories(legacyLayoutFile.getParent());+ Files.move(newLayoutFile, legacyLayoutFile, StandardCopyOption.REPLACE_EXISTING);++ assertTrue(legacyLayoutFile.toFile().setReadable(false));+ try {+ assertThrows(UncheckedIOException.class, () -> s3Service.getObject("legacy-bucket", "legacy-key.txt"));+ assertFalse(Files.exists(newLayoutFile),+ "a failed migration must not leave a file behind that would satisfy the "+ + "exists-check on a later attempt and permanently shadow the intact legacy source");+ } finally {+ assertTrue(legacyLayoutFile.toFile().setReadable(true));+ }++ S3Object retrieved = s3Service.getObject("legacy-bucket", "legacy-key.txt");+ assertArrayEquals(data, retrieved.getData(),+ "a retry after a failed migration must recover the still-intact legacy content");+ }++ @Test+ void anotherAccountsWriteMustNotDestroyThisAccountsUnmigratedLegacyObject() throws Exception {+ Path dataRoot = tempDir.resolve("s3");+ // Two accounts sharing the underlying stores, like production — isolation comes only+ // from AccountAwareStorageBackend's key prefixing.+ InMemoryStorage<String, Bucket> sharedBucketStore = new InMemoryStorage<>();+ InMemoryStorage<String, S3Object> sharedObjectStore = new InMemoryStorage<>();++ S3Service s3ForA = new S3Service(+ new AccountAwareStorageBackend<>(sharedBucketStore, null, "111111111111"),+ new AccountAwareStorageBackend<>(sharedObjectStore, null, "111111111111"),+ dataRoot, false, (LambdaService) null, new RegionResolver("us-east-1", "111111111111"));+ S3Service s3ForB = new S3Service(+ new AccountAwareStorageBackend<>(sharedBucketStore, null, "222222222222"),+ new AccountAwareStorageBackend<>(sharedObjectStore, null, "222222222222"),+ dataRoot, false, (LambdaService) null, new RegionResolver("us-east-1", "222222222222"));++ s3ForA.createBucket("shared-bucket", "us-east-1");+ byte[] legacyData = "account-A-legacy-content".getBytes(StandardCharsets.UTF_8);+ s3ForA.putObject("shared-bucket", "shared-key.txt", legacyData, "text/plain", null);++ // A hasn't read it since upgrading, so it's still sitting at the legacy path.+ Path newLayoutFile = dataRoot.resolve(".accounts").resolve("111111111111").resolve("shared-bucket").resolve("shared-key.txt.s3data");+ Path legacyLayoutFile = dataRoot.resolve("shared-bucket").resolve("shared-key.txt.s3data");+ Files.createDirectories(legacyLayoutFile.getParent());+ Files.move(newLayoutFile, legacyLayoutFile, StandardCopyOption.REPLACE_EXISTING);++ // B is unrelated to A and just happens to reuse the same bucket/key names (bucket+ // names are only unique per account in Floci) — pure coincidence, not shared history.+ s3ForB.createBucket("shared-bucket", "us-east-1");+ byte[] bData = "account-B-content".getBytes(StandardCharsets.UTF_8);+ s3ForB.putObject("shared-bucket", "shared-key.txt", bData, "text/plain", null);++ assertArrayEquals(bData, s3ForB.getObject("shared-bucket", "shared-key.txt").getData());+ assertArrayEquals(legacyData, s3ForA.getObject("shared-bucket", "shared-key.txt").getData(),+ "account B's unrelated write must not destroy account A's still-unmigrated legacy object");+ }++ @Test+ void concurrentOverwriteNeverLosesToARacingLegacyMigration() throws Exception {+ // A racing legacy-migration read must never reinstall stale bytes on top of a write+ // that already landed. Repeated across many keys, since a lock-based fix can only be+ // shown to hold under real concurrent scheduling, not forced into one interleaving.+ int rounds = 50;+ ExecutorService pool = Executors.newFixedThreadPool(2);+ try {+ for (int i = 0; i < rounds; i++) {+ Path dataRoot = tempDir.resolve("race-" + i).resolve("s3");+ S3Service s3Service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), dataRoot, false);+ String bucket = "legacy-bucket";+ String key = "key.txt";+ s3Service.createBucket(bucket, "us-east-1");++ byte[] legacyData = "legacy-content".getBytes(StandardCharsets.UTF_8);+ s3Service.putObject(bucket, key, legacyData, "text/plain", null);+ Path newLayoutFile = dataRoot.resolve(".accounts").resolve("000000000000").resolve(bucket).resolve(key + ".s3data");+ Path legacyLayoutFile = dataRoot.resolve(bucket).resolve(key + ".s3data");+ Files.createDirectories(legacyLayoutFile.getParent());+ Files.move(newLayoutFile, legacyLayoutFile, StandardCopyOption.REPLACE_EXISTING);++ byte[] writeData = ("write-content-" + i).getBytes(StandardCharsets.UTF_8);+ CyclicBarrier barrier = new CyclicBarrier(2);+ Future<byte[]> readFuture = pool.submit(() -> {+ barrier.await(5, TimeUnit.SECONDS);+ return s3Service.getObject(bucket, key).getData();+ });+ Future<?> writeFuture = pool.submit(() -> {+ barrier.await(5, TimeUnit.SECONDS);+ s3Service.putObject(bucket, key, writeData, "text/plain", null);+ return null;+ });+ readFuture.get(5, TimeUnit.SECONDS);+ writeFuture.get(5, TimeUnit.SECONDS);++ assertArrayEquals(writeData, s3Service.getObject(bucket, key).getData(),+ "a completed write must never be clobbered by a racing legacy migration (round " + i + ")");+ }+ } finally {+ pool.shutdownNow();+ }+ }++ @Test+ void concurrentFirstReadsOfTheSameLegacyObjectBothSucceed() throws Exception {+ Path dataRoot = tempDir.resolve("s3");+ S3Service s3Service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), dataRoot, false);+ s3Service.createBucket("legacy-bucket", "us-east-1");++ // Migration copies rather than moves, so there's no "source vanished" case to race —+ // this just confirms many concurrent first reads never fail, regardless of scheduling.+ int rounds = 50;+ ExecutorService pool = Executors.newFixedThreadPool(2);+ try {+ for (int i = 0; i < rounds; i++) {+ String key = "legacy-key-" + i + ".txt";+ byte[] data = ("pre-upgrade-content-" + i).getBytes(StandardCharsets.UTF_8);+ s3Service.putObject("legacy-bucket", key, data, "text/plain", null);++ Path newLayoutFile = dataRoot.resolve(".accounts").resolve("000000000000").resolve("legacy-bucket").resolve(key + ".s3data");+ Path legacyLayoutFile = dataRoot.resolve("legacy-bucket").resolve(key + ".s3data");+ Files.createDirectories(legacyLayoutFile.getParent());+ Files.move(newLayoutFile, legacyLayoutFile, StandardCopyOption.REPLACE_EXISTING);++ CyclicBarrier barrier = new CyclicBarrier(2);+ List<Future<byte[]>> futures = new ArrayList<>();+ for (int t = 0; t < 2; t++) {+ futures.add(pool.submit(() -> {+ barrier.await(5, TimeUnit.SECONDS);+ return s3Service.getObject("legacy-bucket", key).getData();+ }));+ }+ for (Future<byte[]> future : futures) {+ try {+ assertArrayEquals(data, future.get(5, TimeUnit.SECONDS));+ } catch (Exception e) {+ fail("concurrent first read of a legacy-layout object must not fail (round " + i + ")", e);+ }+ }+ }+ } finally {+ pool.shutdownNow();+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.javaindex 3fcc70b4cf..d84e2ef238 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java@@ -192,7 +192,7 @@ class S3ServiceTest {byte[] data = "file content".getBytes(StandardCharsets.UTF_8);s3Service.putObject("test-bucket", "docs/readme.txt", data, "text/plain", null);- Path filePath = tempDir.resolve("s3/test-bucket/docs/readme.txt.s3data");+ Path filePath = tempDir.resolve("s3/.accounts/000000000000/test-bucket/docs/readme.txt.s3data");assertTrue(Files.exists(filePath));assertArrayEquals(data, assertDoesNotThrow(() -> Files.readAllBytes(filePath)));}@@ -240,7 +240,7 @@ class S3ServiceTest {s3Service.createBucket("test-bucket", "us-east-1");s3Service.putObject("test-bucket", "file.txt", "data".getBytes(), null, null);- Path filePath = tempDir.resolve("s3/test-bucket/file.txt.s3data");+ Path filePath = tempDir.resolve("s3/.accounts/000000000000/test-bucket/file.txt.s3data");assertTrue(Files.exists(filePath));s3Service.deleteObject("test-bucket", "file.txt");@@ -350,7 +350,7 @@ class S3ServiceTest {S3Object retrieved = s3Service.getObject("dest-bucket", "copy.txt");assertArrayEquals("content".getBytes(), retrieved.getData());- assertTrue(Files.exists(tempDir.resolve("s3/dest-bucket/copy.txt.s3data")));+ assertTrue(Files.exists(tempDir.resolve("s3/.accounts/000000000000/dest-bucket/copy.txt.s3data")));}@Test@@ -435,7 +435,7 @@ class S3ServiceTest {S3Object marker = s3Service.getObject("test-bucket", "output.parquet");assertArrayEquals(markerData, marker.getData());- Path bucketDir = tempDir.resolve("s3/test-bucket");+ Path bucketDir = tempDir.resolve("s3/.accounts/000000000000/test-bucket");assertTrue(Files.isDirectory(bucketDir.resolve("output.parquet")));assertTrue(Files.isRegularFile(bucketDir.resolve("output.parquet.s3data")));assertTrue(Files.isRegularFile(bucketDir.resolve("output.parquet/part-0001.parquet.s3data")));@@ -458,7 +458,7 @@ class S3ServiceTest {S3Object child = s3Service.getObject("test-bucket", "output.parquet/part-0001.parquet");assertArrayEquals(childData, child.getData());- Path bucketDir = tempDir.resolve("s3/test-bucket");+ Path bucketDir = tempDir.resolve("s3/.accounts/000000000000/test-bucket");assertTrue(Files.isRegularFile(bucketDir.resolve("output.parquet.s3data")));assertTrue(Files.isDirectory(bucketDir.resolve("output.parquet")));assertTrue(Files.isRegularFile(bucketDir.resolve("output.parquet/part-0001.parquet.s3data")));diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3VersioningServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3VersioningServiceTest.javaindex d70b92d13e..e98e3168b1 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3VersioningServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3VersioningServiceTest.java@@ -195,7 +195,9 @@ class S3VersioningServiceTest {S3Object v1 = diskService.putObject("versioned-bucket", "test.txt","v1".getBytes(StandardCharsets.UTF_8), "text/plain", null);- Path versionedPath = tempDir.resolve(".versions")+ Path versionedPath = tempDir.resolve(".accounts")+ .resolve("000000000000")+ .resolve(".versions").resolve("versioned-bucket").resolve("test.txt").resolve(v1.getVersionId() + ".s3data");