Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(s3): isolate object byte storage between accounts (#2241)

Dmitriy Polomoshnovcdb2c36dfe62
diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
index d2d82782cc..ec463503a5 100644
--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java
@@ -40,6 +40,7 @@ import java.time.Instant;
import java.time.format.DateTimeFormatter;
import java.util.*;
import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.locks.ReentrantLock;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -71,6 +72,19 @@ public class S3Service implements Resettable {
private final Path dataRoot;
private final boolean inMemory;
private final ConcurrentHashMap<String, byte[]> memoryDataStore = new ConcurrentHashMap<>();
+ // Guards disk writes/deletes against a racing legacy migration for the same path (see
+ // copyLegacyFileIfPresent()). Fixed-size stripes keep memory bounded, unlike a per-path
+ // map that would need reference counting to ever shrink safely.
+ private static final int DISK_FILE_LOCK_STRIPES = 256;
+ private final ReentrantLock[] diskFileLocks = newLockStripes(DISK_FILE_LOCK_STRIPES);
+
+ private static ReentrantLock[] newLockStripes(int count) {
+ ReentrantLock[] locks = new ReentrantLock[count];
+ for (int i = 0; i < count; i++) {
+ locks[i] = new ReentrantLock();
+ }
+ return locks;
+ }
private final ConcurrentHashMap<String, Map<Integer, byte[]>> memoryMultipartStore = new ConcurrentHashMap<>();
private final ConcurrentHashMap<String, MultipartUpload> multipartUploads = new ConcurrentHashMap<>();
@@ -183,6 +197,12 @@ public class S3Service implements Resettable {
}
public Bucket createBucket(String bucketName, String region) {
+ if (ACCOUNT_STORAGE_ROOT.equals(bucketName)) {
+ // Floci doesn't otherwise validate bucket name format, but this name must stay
+ // unavailable — it's the root every account's disk storage lives under.
+ throw new AwsException("InvalidBucketName",
+ "The specified bucket is not valid.", 400);
+ }
var existing = bucketStore.get(bucketName);
if (existing.isPresent()) {
Bucket bucket = existing.get();
@@ -217,10 +237,10 @@ public class S3Service implements Resettable {
bucketStore.delete(bucketName);
if (inMemory) {
- String prefix = bucketName + "/";
+ String prefix = ownerId() + "/" + bucketName + "/";
memoryDataStore.keySet().removeIf(k -> k.startsWith(prefix));
} else {
- deleteDirectory(dataRoot.resolve(bucketName));
+ deleteDirectory(dataRoot.resolve(ACCOUNT_STORAGE_ROOT).resolve(ownerId()).resolve(bucketName));
}
LOG.infov("Deleted bucket: {0}", bucketName);
}
@@ -659,8 +679,8 @@ public class S3Service implements Resettable {
getObjectMetadata(bucketName, key, versionId);
if (inMemory) {
byte[] data = versionId != null
- ? memoryDataStore.get(versionedKey(bucketName, key, versionId))
- : memoryDataStore.get(objectKey(bucketName, key));
+ ? memoryDataStore.get(physicalVersionedKey(bucketName, key, versionId))
+ : memoryDataStore.get(physicalKey(bucketName, key));
if (data == null) {
throw new IllegalStateException("S3 object data is missing for " + bucketName + "/" + key);
}
@@ -668,8 +688,8 @@ public class S3Service implements Resettable {
}
try {
Path path = versionId != null
- ? resolveVersionedPath(bucketName, key, versionId)
- : resolveObjectPath(bucketName, key);
+ ? resolveVersionedPathForRead(bucketName, key, versionId)
+ : resolveObjectPathForRead(bucketName, key);
return Files.newInputStream(path);
} catch (IOException e) {
throw new UncheckedIOException("Failed to open S3 object stream", e);
@@ -2564,14 +2584,29 @@ public class S3Service implements Resettable {
private static final String DATA_SUFFIX = ".s3data";
+ // A 12-digit bucket name is valid on S3 and would otherwise collide with an account ID,
+ // making dataRoot/<accountId>/... indistinguishable from the legacy dataRoot/<bucket>/...
+ // layout. A leading "." keeps this namespace unreachable by any real bucket name.
+ private static final String ACCOUNT_STORAGE_ROOT = ".accounts";
+
+ // Unlike bucketStore/objectStore, object bytes get no automatic account prefixing — two
+ // accounts can own a bucket named "orders" and would collide here without this scoping.
+ private String physicalKey(String bucketName, String key) {
+ return ownerId() + "/" + objectKey(bucketName, key);
+ }
+
+ private String physicalVersionedKey(String bucketName, String key, String versionId) {
+ return ownerId() + "/" + versionedKey(bucketName, key, versionId);
+ }
+
private Path resolveObjectPath(String bucketName, String key) {
- Path bucketDir = dataRoot.resolve(bucketName).normalize();
-
+ Path bucketDir = dataRoot.resolve(ACCOUNT_STORAGE_ROOT).resolve(ownerId()).resolve(bucketName).normalize();
+
String safeKey = key;
while (safeKey.startsWith("/")) {
safeKey = safeKey.substring(1);
}
-
+
Path resolved = bucketDir.resolve(safeKey + DATA_SUFFIX).normalize();
if (!resolved.startsWith(bucketDir)) {
throw new AwsException("InvalidKey", "The specified key is invalid.", 400);
@@ -2579,14 +2614,35 @@ public class S3Service implements Resettable {
return resolved;
}
+ private Path legacyObjectPath(String bucketName, String key) {
+ String safeKey = key;
+ while (safeKey.startsWith("/")) {
+ safeKey = safeKey.substring(1);
+ }
+ return dataRoot.resolve(bucketName).normalize().resolve(safeKey + DATA_SUFFIX);
+ }
+
+ /**
+ * Resolves the path for a read, copying in a legacy-layout file if present. Reads only —
+ * a write/delete ({@link #resolveObjectPath}) must never touch legacy data, since it isn't
+ * known to belong to any one account and two accounts can share a bucket name. Copying
+ * (not moving) leaves the ambiguous source in place so every account that reads it gets
+ * its own copy.
+ */
+ private Path resolveObjectPathForRead(String bucketName, String key) {
+ Path resolved = resolveObjectPath(bucketName, key);
+ copyLegacyFileIfPresent(legacyObjectPath(bucketName, key), resolved);
+ return resolved;
+ }
+
private Path resolveVersionedPath(String bucketName, String key, String versionId) {
- Path baseDir = dataRoot.resolve(".versions").resolve(bucketName).normalize();
-
+ Path baseDir = dataRoot.resolve(ACCOUNT_STORAGE_ROOT).resolve(ownerId()).resolve(".versions").resolve(bucketName).normalize();
+
String safeKey = key;
while (safeKey.startsWith("/")) {
safeKey = safeKey.substring(1);
}
-
+
Path resolved = baseDir.resolve(safeKey).resolve(versionId + DATA_SUFFIX).normalize();
if (!resolved.startsWith(baseDir)) {
throw new AwsException("InvalidKey", "The specified key is invalid.", 400);
@@ -2594,26 +2650,86 @@ public class S3Service implements Resettable {
return resolved;
}
+ private Path legacyVersionedPath(String bucketName, String key, String versionId) {
+ String safeKey = key;
+ while (safeKey.startsWith("/")) {
+ safeKey = safeKey.substring(1);
+ }
+ return dataRoot.resolve(".versions").resolve(bucketName).normalize()
+ .resolve(safeKey).resolve(versionId + DATA_SUFFIX);
+ }
+
+ /** Read-only counterpart of {@link #resolveObjectPathForRead} for versioned objects. */
+ private Path resolveVersionedPathForRead(String bucketName, String key, String versionId) {
+ Path resolved = resolveVersionedPath(bucketName, key, versionId);
+ copyLegacyFileIfPresent(legacyVersionedPath(bucketName, key, versionId), resolved);
+ return resolved;
+ }
+
+ private ReentrantLock diskFileLock(Path path) {
+ // A stripe collision just serializes unrelated paths — safe, unlike a map entry.
+ return diskFileLocks[Math.floorMod(path.hashCode(), diskFileLocks.length)];
+ }
+
+ /**
+ * Copies in a legacy file for {@code newPath}, under the same lock {@link #writeFile}/
+ * {@link #deleteFile} hold for that path — otherwise a concurrent write could land its
+ * real content and then be silently clobbered by a racing legacy copy.
+ */
+ private void copyLegacyFileIfPresent(Path legacyPath, Path newPath) {
+ if (!Files.exists(legacyPath)) {
+ return;
+ }
+ ReentrantLock lock = diskFileLock(newPath);
+ lock.lock();
+ try {
+ if (Files.exists(newPath)) {
+ return;
+ }
+ Files.createDirectories(newPath.getParent());
+ Files.copy(legacyPath, newPath);
+ } catch (IOException e) {
+ // A failed copy can leave newPath truncated, which would wrongly look
+ // "already migrated" on a later retry — clean it up before rethrowing.
+ deleteQuietly(newPath, "partially-copied legacy S3 object file, so a later read can retry migration");
+ throw new UncheckedIOException("Failed to copy legacy S3 object file to account-scoped layout", e);
+ } finally {
+ lock.unlock();
+ }
+ }
+
+ private void deleteQuietly(Path path, String reason) {
+ try {
+ Files.deleteIfExists(path);
+ } catch (IOException e) {
+ LOG.warnv(e, "Failed to delete {0} ({1})", path, reason);
+ }
+ }
+
private void writeVersionedFile(String bucketName, String key, String versionId, byte[] data) {
if (inMemory) {
- memoryDataStore.put(versionedKey(bucketName, key, versionId), data);
+ memoryDataStore.put(physicalVersionedKey(bucketName, key, versionId), data);
return;
}
+ Path filePath = resolveVersionedPath(bucketName, key, versionId);
+ ReentrantLock lock = diskFileLock(filePath);
+ lock.lock();
try {
- Path filePath = resolveVersionedPath(bucketName, key, versionId);
Files.createDirectories(filePath.getParent());
Files.write(filePath, data);
} catch (IOException e) {
throw new UncheckedIOException("Failed to write versioned S3 object file", e);
+ } finally {
+ lock.unlock();
}
}
private byte[] readVersionedFile(String bucketName, String key, String versionId) {
if (inMemory) {
- return memoryDataStore.get(versionedKey(bucketName, key, versionId));
+ return memoryDataStore.get(physicalVersionedKey(bucketName, key, versionId));
}
try {
- return Files.readAllBytes(resolveVersionedPath(bucketName, key, versionId));
+ return Files.readAllBytes(resolveVersionedPathForRead(bucketName, key, versionId));
} catch (IOException e) {
throw new UncheckedIOException("Failed to read versioned S3 object file", e);
}
@@ -2621,24 +2737,28 @@ public class S3Service implements Resettable {
private void writeFile(String bucketName, String key, byte[] data) {
if (inMemory) {
- memoryDataStore.put(objectKey(bucketName, key), data);
+ memoryDataStore.put(physicalKey(bucketName, key), data);
return;
}
+ Path filePath = resolveObjectPath(bucketName, key);
+ ReentrantLock lock = diskFileLock(filePath);
+ lock.lock();
try {
- Path filePath = resolveObjectPath(bucketName, key);
Files.createDirectories(filePath.getParent());
Files.write(filePath, data);
} catch (IOException e) {
throw new UncheckedIOException("Failed to write S3 object file", e);
+ } finally {
+ lock.unlock();
}
}
private byte[] readFile(String bucketName, String key) {
if (inMemory) {
- return memoryDataStore.get(objectKey(bucketName, key));
+ return memoryDataStore.get(physicalKey(bucketName, key));
}
try {
- return Files.readAllBytes(resolveObjectPath(bucketName, key));
+ return Files.readAllBytes(resolveObjectPathForRead(bucketName, key));
} catch (IOException e) {
throw new UncheckedIOException("Failed to read S3 object file", e);
}
@@ -2646,25 +2766,35 @@ public class S3Service implements Resettable {
private void deleteFile(String bucketName, String key) {
if (inMemory) {
- memoryDataStore.remove(objectKey(bucketName, key));
+ memoryDataStore.remove(physicalKey(bucketName, key));
return;
}
+ Path filePath = resolveObjectPath(bucketName, key);
+ ReentrantLock lock = diskFileLock(filePath);
+ lock.lock();
try {
- Files.deleteIfExists(resolveObjectPath(bucketName, key));
+ Files.deleteIfExists(filePath);
} catch (IOException e) {
LOG.errorv(e, "Failed to delete S3 object file: {0}/{1}", bucketName, key);
+ } finally {
+ lock.unlock();
}
}
private void deleteVersionedFile(String bucketName, String key, String versionId) {
if (inMemory) {
- memoryDataStore.remove(versionedKey(bucketName, key, versionId));
+ memoryDataStore.remove(physicalVersionedKey(bucketName, key, versionId));
return;
}
+ Path filePath = resolveVersionedPath(bucketName, key, versionId);
+ ReentrantLock lock = diskFileLock(filePath);
+ lock.lock();
try {
- Files.deleteIfExists(resolveVersionedPath(bucketName, key, versionId));
+ Files.deleteIfExists(filePath);
} catch (IOException e) {
LOG.errorv(e, "Failed to delete versioned S3 object file: {0}/{1} v={2}", bucketName, key, versionId);
+ } finally {
+ lock.unlock();
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3AccountIsolationIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3AccountIsolationIntegrationTest.java
new file mode 100644
index 0000000000..85c69c5f88
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3AccountIsolationIntegrationTest.java
@@ -0,0 +1,107 @@
+package io.github.hectorvent.floci.services.s3;
+
+import io.quarkus.test.junit.QuarkusTest;
+import org.junit.jupiter.api.Test;
+
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.containsString;
+import static org.hamcrest.Matchers.not;
+
+/**
+ * Verifies S3 object bytes are isolated between accounts, mirroring
+ * {@link io.github.hectorvent.floci.core.common.AccountIsolationIntegrationTest}'s SQS coverage.
+ * Two accounts may own a same-named bucket in Floci (bucket names are only unique per account
+ * here, unlike real S3); object bytes must not collide when they do.
+ */
+@QuarkusTest
+class S3AccountIsolationIntegrationTest {
+
+ private static final String AUTH_ACCOUNT_1 =
+ "AWS4-HMAC-SHA256 Credential=000000000001/20260215/us-east-1/s3/aws4_request, SignedHeaders=host, Signature=abc";
+ private static final String AUTH_ACCOUNT_2 =
+ "AWS4-HMAC-SHA256 Credential=000000000002/20260215/us-east-1/s3/aws4_request, SignedHeaders=host, Signature=abc";
+
+ @Test
+ void objectsInSameNamedBucketAreIsolatedBetweenAccounts() {
+ String bucket = "account-isolation-shared-bucket";
+
+ createBucket(AUTH_ACCOUNT_1, bucket);
+ createBucket(AUTH_ACCOUNT_2, bucket);
+
+ // Both write the same key in their own bucket — exercises the byte-storage
+ // collision directly, not just metadata isolation.
+ given()
+ .header("Authorization", AUTH_ACCOUNT_1)
+ .body("account-1-data")
+ .when()
+ .put("/" + bucket + "/shared-key.txt")
+ .then()
+ .statusCode(200);
+
+ given()
+ .header("Authorization", AUTH_ACCOUNT_2)
+ .body("account-2-data")
+ .when()
+ .put("/" + bucket + "/shared-key.txt")
+ .then()
+ .statusCode(200);
+
+ given()
+ .header("Authorization", AUTH_ACCOUNT_1)
+ .when()
+ .get("/" + bucket + "/shared-key.txt")
+ .then()
+ .statusCode(200)
+ .body(containsString("account-1-data"));
+
+ given()
+ .header("Authorization", AUTH_ACCOUNT_2)
+ .when()
+ .get("/" + bucket + "/shared-key.txt")
+ .then()
+ .statusCode(200)
+ .body(containsString("account-2-data"));
+ }
+
+ @Test
+ void deletingBucketDoesNotRemoveAnotherAccountsSameNamedBucketData() {
+ String bucket = "account-isolation-delete-bucket";
+
+ createBucket(AUTH_ACCOUNT_1, bucket);
+ createBucket(AUTH_ACCOUNT_2, bucket);
+
+ given()
+ .header("Authorization", AUTH_ACCOUNT_2)
+ .body("account-2-data")
+ .when()
+ .put("/" + bucket + "/kept-key.txt")
+ .then()
+ .statusCode(200);
+
+ // Account 1 deletes its own (empty) same-named bucket.
+ given()
+ .header("Authorization", AUTH_ACCOUNT_1)
+ .when()
+ .delete("/" + bucket)
+ .then()
+ .statusCode(204);
+
+ // Account 2's object must be unaffected.
+ given()
+ .header("Authorization", AUTH_ACCOUNT_2)
+ .when()
+ .get("/" + bucket + "/kept-key.txt")
+ .then()
+ .statusCode(200)
+ .body(not(containsString("<Error>")));
+ }
+
+ private static void createBucket(String auth, String bucket) {
+ given()
+ .header("Authorization", auth)
+ .when()
+ .put("/" + bucket)
+ .then()
+ .statusCode(200);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3LegacyDiskLayoutIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3LegacyDiskLayoutIntegrationTest.java
new file mode 100644
index 0000000000..d18119b8c4
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3LegacyDiskLayoutIntegrationTest.java
@@ -0,0 +1,268 @@
+package io.github.hectorvent.floci.services.s3;
+
+import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
+import io.github.hectorvent.floci.core.storage.InMemoryStorage;
+import io.github.hectorvent.floci.services.lambda.LambdaService;
+import io.github.hectorvent.floci.services.s3.model.Bucket;
+import io.github.hectorvent.floci.services.s3.model.S3Object;
+import org.junit.jupiter.api.Assumptions;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.condition.DisabledOnOs;
+import org.junit.jupiter.api.condition.OS;
+import org.junit.jupiter.api.io.TempDir;
+
+import java.io.UncheckedIOException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.CyclicBarrier;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.assertArrayEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
+
+/**
+ * Before account-scoped object storage, every disk-mode S3 object lived at
+ * {@code dataRoot/bucketName/key.s3data}, with no account segment. Upgrading past that change
+ * must not lose access to objects already on disk, and — since a legacy file isn't known to
+ * belong to any one account, and two accounts can share a bucket name — migrating it must never
+ * let one account's write/delete steal or destroy another account's still-unmigrated object.
+ */
+class S3LegacyDiskLayoutIntegrationTest {
+
+ @TempDir
+ Path tempDir;
+
+ @Test
+ void existingObjectWrittenUnderTheLegacyUnscopedPathIsStillReadable() throws Exception {
+ Path dataRoot = tempDir.resolve("s3");
+ S3Service s3Service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), dataRoot, false);
+
+ s3Service.createBucket("legacy-bucket", "us-east-1");
+ byte[] data = "pre-upgrade-content".getBytes(StandardCharsets.UTF_8);
+ // Write normally, then relocate to the legacy path to simulate pre-upgrade data.
+ s3Service.putObject("legacy-bucket", "legacy-key.txt", data, "text/plain", null);
+
+ Path newLayoutFile = dataRoot.resolve(".accounts").resolve("000000000000").resolve("legacy-bucket").resolve("legacy-key.txt.s3data");
+ Path legacyLayoutFile = dataRoot.resolve("legacy-bucket").resolve("legacy-key.txt.s3data");
+ assertTrue(Files.exists(newLayoutFile), "test setup: object should initially be written at the new path");
+
+ Files.createDirectories(legacyLayoutFile.getParent());
+ Files.move(newLayoutFile, legacyLayoutFile, StandardCopyOption.REPLACE_EXISTING);
+
+ S3Object retrieved = s3Service.getObject("legacy-bucket", "legacy-key.txt");
+ assertArrayEquals(data, retrieved.getData(),
+ "an object physically stored under the pre-upgrade unscoped layout must still be readable");
+ }
+
+ @Test
+ void bucketNamedLikeTheDefaultAccountIdDoesNotCollideWithAccountScopedStorage() throws Exception {
+ // A 12-digit bucket name is valid on S3 and shares its shape with an account ID: a
+ // legacy bucket "000000000000" with key "orders/report.csv" must not collide with
+ // account "000000000000"'s own "orders" bucket, key "report.csv".
+ Path dataRoot = tempDir.resolve("s3");
+ S3Service s3Service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), dataRoot, false);
+
+ String accountIdShapedBucket = "000000000000";
+ s3Service.createBucket(accountIdShapedBucket, "us-east-1");
+ byte[] legacyData = "legacy-bucket-named-like-an-account-id".getBytes(StandardCharsets.UTF_8);
+ s3Service.putObject(accountIdShapedBucket, "orders/report.csv", legacyData, "text/plain", null);
+
+ Path newLayoutFile = dataRoot.resolve(".accounts").resolve("000000000000")
+ .resolve(accountIdShapedBucket).resolve("orders/report.csv.s3data");
+ Path legacyLayoutFile = dataRoot.resolve(accountIdShapedBucket).resolve("orders/report.csv.s3data");
+ Files.createDirectories(legacyLayoutFile.getParent());
+ Files.move(newLayoutFile, legacyLayoutFile, StandardCopyOption.REPLACE_EXISTING);
+
+ // The default account's own, unrelated "orders" bucket and "report.csv" key.
+ s3Service.createBucket("orders", "us-east-1");
+ byte[] newData = "unrelated-object-in-the-orders-bucket".getBytes(StandardCharsets.UTF_8);
+ s3Service.putObject("orders", "report.csv", newData, "text/plain", null);
+
+ assertArrayEquals(newData, s3Service.getObject("orders", "report.csv").getData(),
+ "the unrelated new-layout object must not be shadowed by the account-ID-shaped legacy bucket");
+ assertArrayEquals(legacyData, s3Service.getObject(accountIdShapedBucket, "orders/report.csv").getData(),
+ "the legacy bucket named like an account ID must still resolve to its own, distinct content");
+ }
+
+ @Test
+ void bucketNamedLikeTheReservedAccountStorageRootIsRejected() {
+ Path dataRoot = tempDir.resolve("s3");
+ S3Service s3Service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), dataRoot, false);
+
+ assertThrows(io.github.hectorvent.floci.core.common.AwsException.class,
+ () -> s3Service.createBucket(".accounts", "us-east-1"),
+ "a bucket named exactly like the reserved account-storage root must be rejected, "
+ + "since Floci doesn't otherwise validate bucket name format at all");
+ }
+
+ @Test
+ @DisabledOnOs(OS.WINDOWS)
+ void migrationFailureIsNotPermanentAndDoesNotLeaveAFileBehind() throws Exception {
+ // Note: permission-denied fails before Files.copy creates a destination on this JVM,
+ // so this doesn't reproduce a genuinely partial write (disk-full mid-copy isn't
+ // reliably forceable for a file this small) — it verifies the property that matters
+ // to a caller: a transient failure isn't permanent, and a retry recovers.
+ Assumptions.assumeFalse("root".equals(System.getProperty("user.name")),
+ "root ignores file read permissions");
+
+ Path dataRoot = tempDir.resolve("s3");
+ S3Service s3Service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), dataRoot, false);
+ s3Service.createBucket("legacy-bucket", "us-east-1");
+ byte[] data = "pre-upgrade-content".getBytes(StandardCharsets.UTF_8);
+ s3Service.putObject("legacy-bucket", "legacy-key.txt", data, "text/plain", null);
+
+ Path newLayoutFile = dataRoot.resolve(".accounts").resolve("000000000000").resolve("legacy-bucket").resolve("legacy-key.txt.s3data");
+ Path legacyLayoutFile = dataRoot.resolve("legacy-bucket").resolve("legacy-key.txt.s3data");
+ Files.createDirectories(legacyLayoutFile.getParent());
+ Files.move(newLayoutFile, legacyLayoutFile, StandardCopyOption.REPLACE_EXISTING);
+
+ assertTrue(legacyLayoutFile.toFile().setReadable(false));
+ try {
+ assertThrows(UncheckedIOException.class, () -> s3Service.getObject("legacy-bucket", "legacy-key.txt"));
+ assertFalse(Files.exists(newLayoutFile),
+ "a failed migration must not leave a file behind that would satisfy the "
+ + "exists-check on a later attempt and permanently shadow the intact legacy source");
+ } finally {
+ assertTrue(legacyLayoutFile.toFile().setReadable(true));
+ }
+
+ S3Object retrieved = s3Service.getObject("legacy-bucket", "legacy-key.txt");
+ assertArrayEquals(data, retrieved.getData(),
+ "a retry after a failed migration must recover the still-intact legacy content");
+ }
+
+ @Test
+ void anotherAccountsWriteMustNotDestroyThisAccountsUnmigratedLegacyObject() throws Exception {
+ Path dataRoot = tempDir.resolve("s3");
+ // Two accounts sharing the underlying stores, like production — isolation comes only
+ // from AccountAwareStorageBackend's key prefixing.
+ InMemoryStorage<String, Bucket> sharedBucketStore = new InMemoryStorage<>();
+ InMemoryStorage<String, S3Object> sharedObjectStore = new InMemoryStorage<>();
+
+ S3Service s3ForA = new S3Service(
+ new AccountAwareStorageBackend<>(sharedBucketStore, null, "111111111111"),
+ new AccountAwareStorageBackend<>(sharedObjectStore, null, "111111111111"),
+ dataRoot, false, (LambdaService) null, new RegionResolver("us-east-1", "111111111111"));
+ S3Service s3ForB = new S3Service(
+ new AccountAwareStorageBackend<>(sharedBucketStore, null, "222222222222"),
+ new AccountAwareStorageBackend<>(sharedObjectStore, null, "222222222222"),
+ dataRoot, false, (LambdaService) null, new RegionResolver("us-east-1", "222222222222"));
+
+ s3ForA.createBucket("shared-bucket", "us-east-1");
+ byte[] legacyData = "account-A-legacy-content".getBytes(StandardCharsets.UTF_8);
+ s3ForA.putObject("shared-bucket", "shared-key.txt", legacyData, "text/plain", null);
+
+ // A hasn't read it since upgrading, so it's still sitting at the legacy path.
+ Path newLayoutFile = dataRoot.resolve(".accounts").resolve("111111111111").resolve("shared-bucket").resolve("shared-key.txt.s3data");
+ Path legacyLayoutFile = dataRoot.resolve("shared-bucket").resolve("shared-key.txt.s3data");
+ Files.createDirectories(legacyLayoutFile.getParent());
+ Files.move(newLayoutFile, legacyLayoutFile, StandardCopyOption.REPLACE_EXISTING);
+
+ // B is unrelated to A and just happens to reuse the same bucket/key names (bucket
+ // names are only unique per account in Floci) — pure coincidence, not shared history.
+ s3ForB.createBucket("shared-bucket", "us-east-1");
+ byte[] bData = "account-B-content".getBytes(StandardCharsets.UTF_8);
+ s3ForB.putObject("shared-bucket", "shared-key.txt", bData, "text/plain", null);
+
+ assertArrayEquals(bData, s3ForB.getObject("shared-bucket", "shared-key.txt").getData());
+ assertArrayEquals(legacyData, s3ForA.getObject("shared-bucket", "shared-key.txt").getData(),
+ "account B's unrelated write must not destroy account A's still-unmigrated legacy object");
+ }
+
+ @Test
+ void concurrentOverwriteNeverLosesToARacingLegacyMigration() throws Exception {
+ // A racing legacy-migration read must never reinstall stale bytes on top of a write
+ // that already landed. Repeated across many keys, since a lock-based fix can only be
+ // shown to hold under real concurrent scheduling, not forced into one interleaving.
+ int rounds = 50;
+ ExecutorService pool = Executors.newFixedThreadPool(2);
+ try {
+ for (int i = 0; i < rounds; i++) {
+ Path dataRoot = tempDir.resolve("race-" + i).resolve("s3");
+ S3Service s3Service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), dataRoot, false);
+ String bucket = "legacy-bucket";
+ String key = "key.txt";
+ s3Service.createBucket(bucket, "us-east-1");
+
+ byte[] legacyData = "legacy-content".getBytes(StandardCharsets.UTF_8);
+ s3Service.putObject(bucket, key, legacyData, "text/plain", null);
+ Path newLayoutFile = dataRoot.resolve(".accounts").resolve("000000000000").resolve(bucket).resolve(key + ".s3data");
+ Path legacyLayoutFile = dataRoot.resolve(bucket).resolve(key + ".s3data");
+ Files.createDirectories(legacyLayoutFile.getParent());
+ Files.move(newLayoutFile, legacyLayoutFile, StandardCopyOption.REPLACE_EXISTING);
+
+ byte[] writeData = ("write-content-" + i).getBytes(StandardCharsets.UTF_8);
+ CyclicBarrier barrier = new CyclicBarrier(2);
+ Future<byte[]> readFuture = pool.submit(() -> {
+ barrier.await(5, TimeUnit.SECONDS);
+ return s3Service.getObject(bucket, key).getData();
+ });
+ Future<?> writeFuture = pool.submit(() -> {
+ barrier.await(5, TimeUnit.SECONDS);
+ s3Service.putObject(bucket, key, writeData, "text/plain", null);
+ return null;
+ });
+ readFuture.get(5, TimeUnit.SECONDS);
+ writeFuture.get(5, TimeUnit.SECONDS);
+
+ assertArrayEquals(writeData, s3Service.getObject(bucket, key).getData(),
+ "a completed write must never be clobbered by a racing legacy migration (round " + i + ")");
+ }
+ } finally {
+ pool.shutdownNow();
+ }
+ }
+
+ @Test
+ void concurrentFirstReadsOfTheSameLegacyObjectBothSucceed() throws Exception {
+ Path dataRoot = tempDir.resolve("s3");
+ S3Service s3Service = new S3Service(new InMemoryStorage<>(), new InMemoryStorage<>(), dataRoot, false);
+ s3Service.createBucket("legacy-bucket", "us-east-1");
+
+ // Migration copies rather than moves, so there's no "source vanished" case to race —
+ // this just confirms many concurrent first reads never fail, regardless of scheduling.
+ int rounds = 50;
+ ExecutorService pool = Executors.newFixedThreadPool(2);
+ try {
+ for (int i = 0; i < rounds; i++) {
+ String key = "legacy-key-" + i + ".txt";
+ byte[] data = ("pre-upgrade-content-" + i).getBytes(StandardCharsets.UTF_8);
+ s3Service.putObject("legacy-bucket", key, data, "text/plain", null);
+
+ Path newLayoutFile = dataRoot.resolve(".accounts").resolve("000000000000").resolve("legacy-bucket").resolve(key + ".s3data");
+ Path legacyLayoutFile = dataRoot.resolve("legacy-bucket").resolve(key + ".s3data");
+ Files.createDirectories(legacyLayoutFile.getParent());
+ Files.move(newLayoutFile, legacyLayoutFile, StandardCopyOption.REPLACE_EXISTING);
+
+ CyclicBarrier barrier = new CyclicBarrier(2);
+ List<Future<byte[]>> futures = new ArrayList<>();
+ for (int t = 0; t < 2; t++) {
+ futures.add(pool.submit(() -> {
+ barrier.await(5, TimeUnit.SECONDS);
+ return s3Service.getObject("legacy-bucket", key).getData();
+ }));
+ }
+ for (Future<byte[]> future : futures) {
+ try {
+ assertArrayEquals(data, future.get(5, TimeUnit.SECONDS));
+ } catch (Exception e) {
+ fail("concurrent first read of a legacy-layout object must not fail (round " + i + ")", e);
+ }
+ }
+ }
+ } finally {
+ pool.shutdownNow();
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java
index 3fcc70b4cf..d84e2ef238 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ServiceTest.java
@@ -192,7 +192,7 @@ class S3ServiceTest {
byte[] data = "file content".getBytes(StandardCharsets.UTF_8);
s3Service.putObject("test-bucket", "docs/readme.txt", data, "text/plain", null);
- Path filePath = tempDir.resolve("s3/test-bucket/docs/readme.txt.s3data");
+ Path filePath = tempDir.resolve("s3/.accounts/000000000000/test-bucket/docs/readme.txt.s3data");
assertTrue(Files.exists(filePath));
assertArrayEquals(data, assertDoesNotThrow(() -> Files.readAllBytes(filePath)));
}
@@ -240,7 +240,7 @@ class S3ServiceTest {
s3Service.createBucket("test-bucket", "us-east-1");
s3Service.putObject("test-bucket", "file.txt", "data".getBytes(), null, null);
- Path filePath = tempDir.resolve("s3/test-bucket/file.txt.s3data");
+ Path filePath = tempDir.resolve("s3/.accounts/000000000000/test-bucket/file.txt.s3data");
assertTrue(Files.exists(filePath));
s3Service.deleteObject("test-bucket", "file.txt");
@@ -350,7 +350,7 @@ class S3ServiceTest {
S3Object retrieved = s3Service.getObject("dest-bucket", "copy.txt");
assertArrayEquals("content".getBytes(), retrieved.getData());
- assertTrue(Files.exists(tempDir.resolve("s3/dest-bucket/copy.txt.s3data")));
+ assertTrue(Files.exists(tempDir.resolve("s3/.accounts/000000000000/dest-bucket/copy.txt.s3data")));
}
@Test
@@ -435,7 +435,7 @@ class S3ServiceTest {
S3Object marker = s3Service.getObject("test-bucket", "output.parquet");
assertArrayEquals(markerData, marker.getData());
- Path bucketDir = tempDir.resolve("s3/test-bucket");
+ Path bucketDir = tempDir.resolve("s3/.accounts/000000000000/test-bucket");
assertTrue(Files.isDirectory(bucketDir.resolve("output.parquet")));
assertTrue(Files.isRegularFile(bucketDir.resolve("output.parquet.s3data")));
assertTrue(Files.isRegularFile(bucketDir.resolve("output.parquet/part-0001.parquet.s3data")));
@@ -458,7 +458,7 @@ class S3ServiceTest {
S3Object child = s3Service.getObject("test-bucket", "output.parquet/part-0001.parquet");
assertArrayEquals(childData, child.getData());
- Path bucketDir = tempDir.resolve("s3/test-bucket");
+ Path bucketDir = tempDir.resolve("s3/.accounts/000000000000/test-bucket");
assertTrue(Files.isRegularFile(bucketDir.resolve("output.parquet.s3data")));
assertTrue(Files.isDirectory(bucketDir.resolve("output.parquet")));
assertTrue(Files.isRegularFile(bucketDir.resolve("output.parquet/part-0001.parquet.s3data")));
diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3VersioningServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3VersioningServiceTest.java
index d70b92d13e..e98e3168b1 100644
--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3VersioningServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3VersioningServiceTest.java
@@ -195,7 +195,9 @@ class S3VersioningServiceTest {
S3Object v1 = diskService.putObject("versioned-bucket", "test.txt",
"v1".getBytes(StandardCharsets.UTF_8), "text/plain", null);
- Path versionedPath = tempDir.resolve(".versions")
+ Path versionedPath = tempDir.resolve(".accounts")
+ .resolve("000000000000")
+ .resolve(".versions")
.resolve("versioned-bucket")
.resolve("test.txt")
.resolve(v1.getVersionId() + ".s3data");