Light, fluffy, and always free - The AWS Local Emulator alternative
fix(s3): journal the object index under persistent storage (#4642)
Abhigyacea6d3e99f26
diff --git a/docs/configuration/advanced/application-yml.md b/docs/configuration/advanced/application-yml.mdindex 662c2deadc..bd2f0eae32 100644--- a/docs/configuration/advanced/application-yml.md+++ b/docs/configuration/advanced/application-yml.md@@ -58,8 +58,8 @@ floci:# mount-user: "1001:1001" # PosixUser: run mounting containers as uid[:gid]# mount-group-add: 2000 # supplementary gid added to mounting containerswal:- # Also the cadence at which journaled stores under persistent mode (CloudWatch Logs events)- # fold their .wal file into the store's JSON file.+ # Also the cadence at which journaled stores under persistent mode (CloudWatch Logs events,+ # the S3 object index) fold their .wal file into the store's JSON file.compaction-interval-ms: 30000services:ssm:diff --git a/docs/configuration/environment-variables.md b/docs/configuration/environment-variables.mdindex c1625bd590..fbcb7c215d 100644--- a/docs/configuration/environment-variables.md+++ b/docs/configuration/environment-variables.md@@ -83,7 +83,7 @@ See [TLS / HTTPS](./tls.md) for SDK configuration examples and WebSocket (`wss:/| `FLOCI_STORAGE_PERSISTENT_PATH` | `./data` | Container-side directory for persistent and hybrid storage || `FLOCI_STORAGE_HOST_PERSISTENT_PATH` | `./data` | Host-side path for Docker volume bind-mounts (RDS, OpenSearch, MSK, ECR data). When unset, Floci uses named Docker volumes || `FLOCI_STORAGE_PRUNE_VOLUMES_ON_DELETE` | `false` | Remove named Docker volumes immediately when the resource is deleted |-| `FLOCI_STORAGE_WAL_COMPACTION_INTERVAL_MS` | `30000` | How often (ms) the WAL compaction runs. Applies to `wal` mode and to the stores that are journaled under `persistent` mode (CloudWatch Logs events) |+| `FLOCI_STORAGE_WAL_COMPACTION_INTERVAL_MS` | `30000` | How often (ms) the WAL compaction runs. Applies to `wal` mode and to the stores that are journaled under `persistent` mode (CloudWatch Logs events, the S3 object index) |### Per-service storage overridesdiff --git a/docs/configuration/storage.md b/docs/configuration/storage.mdindex c97cf08367..cfadb20d2b 100644--- a/docs/configuration/storage.md+++ b/docs/configuration/storage.md@@ -29,10 +29,12 @@ current after every call but makes the cost of one write grow with the size of tAppend-heavy stores are journaled instead: a change is appended to a `.wal` file next to thestore, and the store's JSON file is rewritten from memory on the `FLOCI_STORAGE_WAL_COMPACTION_INTERVAL_MS`cadence and at shutdown, and only when something changed. Today this applies to CloudWatch Logs-events (`cwlogs-events.json` with `cwlogs-events.wal`). After a clean shutdown the JSON file holds-every event; while Floci runs it can be up to one compaction interval behind, and the journal is-replayed on the next start. An existing `cwlogs-events.json` from an older version is picked up as-the first snapshot without any migration.+events (`cwlogs-events.json` with `cwlogs-events.wal`) and the S3 object index+(`s3-objects.json` with `s3-objects.wal`), which holds the metadata of every object in every+bucket. After a clean shutdown the JSON file holds every entry; while Floci runs it can be up to+one compaction interval behind, and the journal is replayed on the next start. An existing+`cwlogs-events.json` or `s3-objects.json` from an older version is picked up as the first snapshot+without any migration.## Per-Service Overridediff --git a/docs/services/s3.md b/docs/services/s3.mdindex 2cd95946ed..032c386912 100644--- a/docs/services/s3.md+++ b/docs/services/s3.md@@ -335,6 +335,17 @@ These AWS S3 features have no handler in Floci. Calls will return an error (typi| `FLOCI_SERVICES_S3_ENFORCE_AUTH` | `false` | Enable bucket policy evaluation for authenticated callers || `FLOCI_SERVICES_S3_GLOBAL_BUCKET_NAMESPACE` | `false` | Enable cross-account bucket and object resolution |+### Storage++Object bodies are files under the S3 data directory; the object index, `s3-objects.json`, holds+the metadata of every object in every bucket. Under `persistent` mode the index is journaled+instead of rewritten on every `PutObject`, `CopyObject` or `DeleteObject`: the change is appended+to `s3-objects.wal`, and `s3-objects.json` is rewritten from memory on the+`FLOCI_STORAGE_WAL_COMPACTION_INTERVAL_MS` cadence and at shutdown, so the cost of a write no+longer grows with the number of stored objects. Bucket definitions and the other S3 stores are+still rewritten on every change. See+[Storage Modes](../configuration/storage.md#journaled-stores-under-persistent-mode).+## Examples```bashdiff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.javaindex 701dc0211f..acaa475cfe 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3Service.java@@ -12,6 +12,7 @@ import io.github.hectorvent.floci.core.common.Resettable;import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.core.storage.WriteProfile;import io.github.hectorvent.floci.services.iam.IamPolicyEvaluator;import io.github.hectorvent.floci.services.iam.IamPolicyEvaluator.ResourcePolicyDecision;import io.github.hectorvent.floci.services.iam.IamService;@@ -174,7 +175,7 @@ public class S3Service implements Resettable, ResourceProvider {}),storageFactory.create("s3", "s3-objects.json",new TypeReference<Map<String, S3Object>>() {- }),+ }, WriteProfile.APPEND_HEAVY),storageFactory.create("s3", "s3-annotations.json",new TypeReference<Map<String, ObjectAnnotation>>() {}),@@ -2083,7 +2084,7 @@ public class S3Service implements Resettable, ResourceProvider {getStoredObjectEntry(bucketName, key, null);S3Object obj = ownedObject.value();obj.setTags(tags != null ? tags : new java.util.HashMap<>());- putObjectForAccount(ownedObject.account(), objectKey(bucketName, key), obj);+ putObjectMetadataForAccount(ownedObject.account(), bucketName, key, obj);LOG.debugv("Put tags on object: {0}/{1}", bucketName, key);}@@ -2102,7 +2103,7 @@ public class S3Service implements Resettable, ResourceProvider {getStoredObjectEntry(bucketName, key, null);S3Object obj = ownedObject.value();obj.setTags(new java.util.HashMap<>());- putObjectForAccount(ownedObject.account(), objectKey(bucketName, key), obj);+ putObjectMetadataForAccount(ownedObject.account(), bucketName, key, obj);LOG.debugv("Deleted tags from object: {0}/{1}", bucketName, key);}@@ -3098,9 +3099,6 @@ public class S3Service implements Resettable, ResourceProvider {String mode, Instant retainUntil, boolean bypassGovernance) {AccountAwareStorageBackend.OwnedEntry<S3Object> ownedObject =getStoredObjectEntry(bucketName, key, versionId);- String storeKey = versionId != null- ? versionedKey(bucketName, key, versionId)- : objectKey(bucketName, key);S3Object obj = ownedObject.value();boolean activeComplianceRetention = "COMPLIANCE".equals(obj.getObjectLockMode())@@ -3135,7 +3133,7 @@ public class S3Service implements Resettable, ResourceProvider {obj.setObjectLockMode(mode);obj.setRetainUntilDate(retainUntil);- putObjectForAccount(ownedObject.account(), storeKey, obj);+ putObjectMetadataForAccount(ownedObject.account(), bucketName, key, obj);LOG.debugv("Set retention on {0}/{1}: mode={2}, until={3}", bucketName, key, mode, retainUntil);}@@ -3146,12 +3144,9 @@ public class S3Service implements Resettable, ResourceProvider {public void putObjectLegalHold(String bucketName, String key, String versionId, String status) {AccountAwareStorageBackend.OwnedEntry<S3Object> ownedObject =getStoredObjectEntry(bucketName, key, versionId);- String storeKey = versionId != null- ? versionedKey(bucketName, key, versionId)- : objectKey(bucketName, key);S3Object obj = ownedObject.value();obj.setLegalHoldStatus(status);- putObjectForAccount(ownedObject.account(), storeKey, obj);+ putObjectMetadataForAccount(ownedObject.account(), bucketName, key, obj);LOG.debugv("Set legal hold on {0}/{1}: {2}", bucketName, key, status);}@@ -3425,7 +3420,11 @@ public class S3Service implements Resettable, ResourceProvider {object.setSseCustomerAlgorithm(upload.getSseCustomerAlgorithm());object.setSseCustomerKeyMd5(upload.getSseCustomerKeyMd5());}- objectStore.put(objectKey(bucket, key), object);+ String bucketOwnerAccount = resolveBucketEntry(bucket)+ .map(AccountAwareStorageBackend.OwnedEntry::account)+ .orElseThrow(() -> new AwsException("NoSuchBucket",+ "The specified bucket does not exist.", 404));+ putObjectMetadataForAccount(bucketOwnerAccount, bucket, key, object);// CleanupcleanupMultipart(uploadId);@@ -3782,8 +3781,7 @@ public class S3Service implements Resettable, ResourceProvider {String newAcl = resolvedAcl != null ? resolvedAcl : (bodyAcl.isBlank() ? null : bodyAcl);rejectPublicAclWhenBlocked(bucketName, newAcl);obj.setAcl(newAcl);- String storeKey = (versionId != null) ? versionedKey(bucketName, key, versionId) : objectKey(bucketName, key);- putObjectForAccount(ownedObject.account(), storeKey, obj);+ putObjectMetadataForAccount(ownedObject.account(), bucketName, key, obj);}/**@@ -4865,6 +4863,34 @@ public class S3Service implements Resettable, ResourceProvider {objectStore.put(storeKey, object);}+ /**+ * Persists a metadata change to one stored object. The current version of a versioned object+ * is indexed under both its version key and the latest key, so both entries are written: the+ * journaled object store replays each entry on its own, and writing one would bring the+ * other back with the old metadata after a restart. The latest key is written only while it+ * still refers to this object, checked under the bucket lock that puts and deletes hold, so a+ * concurrent overwrite or delete marker is never replaced by the object it superseded.+ */+ private void putObjectMetadataForAccount(String accountId, String bucketName, String key, S3Object object) {+ Bucket bucket = resolveBucket(bucketName)+ .orElseThrow(() -> new AwsException("NoSuchBucket",+ "The specified bucket does not exist.", 404));+ synchronized (bucket) {+ String latestKey = objectKey(bucketName, key);+ String versionId = object.getVersionId();+ S3Object latest = resolveObjectForAccount(accountId, latestKey).orElse(null);+ boolean isCurrent = latest == object+ || (versionId != null && latest != null && !latest.isDeleteMarker()+ && versionId.equals(latest.getVersionId()));+ if (versionId != null) {+ putObjectForAccount(accountId, versionedKey(bucketName, key, versionId), object);+ }+ if (isCurrent) {+ putObjectForAccount(accountId, latestKey, object);+ }+ }+ }+private String objectKey(String bucketName, String key) {return bucketName + "/" + key;}diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex 1ccadd0247..5962d97796 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -163,8 +163,8 @@ floci:# mount-user: # run mounting containers as "uid[:gid]" (access point PosixUser)# mount-group-add: # supplementary gid added to mounting containerswal:- # Also the cadence at which journaled stores under persistent mode (CloudWatch Logs events)- # fold their .wal file into the store's JSON file.+ # Also the cadence at which journaled stores under persistent mode (CloudWatch Logs events,+ # the S3 object index) fold their .wal file into the store's JSON file.compaction-interval-ms: 30000services:ssm:diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.javaindex dd5aa083a0..9fcf29d884 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/DynamoDbStreamsEventSourcePollerTest.java@@ -11,6 +11,7 @@ import io.github.hectorvent.floci.core.common.RequestScopes;import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.InMemoryStorage;import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.core.storage.WriteProfile;import io.github.hectorvent.floci.services.dynamodb.backend.DynamoDbStreamReader;import io.github.hectorvent.floci.services.dynamodb.backend.DynamoDbStreamReader.CheckpointLifetime;import io.github.hectorvent.floci.services.dynamodb.backend.DynamoDbStreamReader.Position;@@ -38,6 +39,7 @@ import org.junit.jupiter.params.ParameterizedTest;import org.junit.jupiter.params.provider.NullSource;import org.junit.jupiter.params.provider.ValueSource;import org.mockito.ArgumentCaptor;+import org.mockito.stubbing.Answer;import java.nio.charset.StandardCharsets;import java.nio.file.Path;@@ -869,7 +871,7 @@ class DynamoDbStreamsEventSourcePollerTest {buckets.putForAccount(bucketOwnerAccountId, "my-failed-events", new Bucket("my-failed-events"));StorageFactory storageFactory = mock(StorageFactory.class);- when(storageFactory.create(eq("s3"), anyString(), any())).thenAnswer(invocation -> {+ Answer<Object> s3Stores = invocation -> {String fileName = invocation.getArgument(1);return switch (fileName) {case "s3-buckets.json" -> buckets;@@ -878,7 +880,9 @@ class DynamoDbStreamsEventSourcePollerTest {case "s3-account-public-access-block.json" -> accountPublicAccessBlocks;default -> throw new AssertionError("Unexpected S3 store: " + fileName);};- });+ };+ when(storageFactory.create(eq("s3"), anyString(), any())).thenAnswer(s3Stores);+ when(storageFactory.create(eq("s3"), anyString(), any(), any(WriteProfile.class))).thenAnswer(s3Stores);EmulatorConfig.StorageConfig storageConfig = mock(EmulatorConfig.StorageConfig.class);when(storageConfig.persistentPath()).thenReturn(Path.of("target", "s3-onfailure-test").toString());diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3GlobalBucketNamespaceTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3GlobalBucketNamespaceTest.javaindex 1e7acf9859..a23643da72 100644--- a/src/test/java/io/github/hectorvent/floci/services/s3/S3GlobalBucketNamespaceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3GlobalBucketNamespaceTest.java@@ -5,12 +5,14 @@ import io.github.hectorvent.floci.core.common.RequestContext;import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.InMemoryStorage;import io.github.hectorvent.floci.services.s3.model.Bucket;+import io.github.hectorvent.floci.services.s3.model.MultipartUpload;import io.github.hectorvent.floci.services.s3.model.S3Object;import jakarta.enterprise.inject.Instance;import org.junit.jupiter.api.Test;import java.nio.file.Path;import java.time.Instant;+import java.util.List;import java.util.Map;import java.util.concurrent.atomic.AtomicReference;@@ -275,4 +277,36 @@ class S3GlobalBucketNamespaceTest {assertTrue(objects.getForAccount(ACCOUNT_B, versionKey).isEmpty(),"cross-account version metadata writes must stay in the owner partition");}++ @Test+ void completeMultipartUploadCrossAccountStoresTheVersionInTheOwnerPartitionOnly() {+ Instance<RequestContext> ctx = mutableContext();+ AccountAwareStorageBackend<Bucket> buckets =+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), ctx, DEFAULT_ACCT);+ AccountAwareStorageBackend<S3Object> objects =+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), ctx, DEFAULT_ACCT);+ S3Service globalNs = new S3Service(+ buckets, objects, Path.of("s3-gns-multipart-test"), true, true);++ caller.set(ACCOUNT_A);+ globalNs.createBucket("shared-multipart-bucket", "us-east-1");+ globalNs.putBucketVersioning("shared-multipart-bucket", "Enabled");++ caller.set(ACCOUNT_B);+ MultipartUpload upload = globalNs.initiateMultipartUpload(+ "shared-multipart-bucket", "object.bin", "application/octet-stream");+ globalNs.uploadPart("shared-multipart-bucket", "object.bin", upload.getUploadId(), 1,+ "payload".getBytes(UTF_8));+ String versionId = globalNs.completeMultipartUpload("shared-multipart-bucket", "object.bin",+ upload.getUploadId(), List.of(1), null, null).getVersionId();++ String currentKey = "shared-multipart-bucket/object.bin";+ String versionKey = currentKey + "#v#" + versionId;+ assertTrue(objects.getForAccount(ACCOUNT_A, currentKey).isPresent());+ assertTrue(objects.getForAccount(ACCOUNT_A, versionKey).isPresent());+ assertTrue(objects.getForAccount(ACCOUNT_B, currentKey).isEmpty(),+ "a cross-account multipart completion must not create a caller shadow object");+ assertTrue(objects.getForAccount(ACCOUNT_B, versionKey).isEmpty(),+ "a cross-account multipart completion must not create a caller shadow version");+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/s3/S3ObjectIndexJournalTest.java b/src/test/java/io/github/hectorvent/floci/services/s3/S3ObjectIndexJournalTest.javanew file mode 100644index 0000000000..94421f8bc3--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/s3/S3ObjectIndexJournalTest.java@@ -0,0 +1,214 @@+package io.github.hectorvent.floci.services.s3;++import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.ServiceConfigAccess;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.io.TempDir;++import java.io.IOException;+import java.nio.charset.StandardCharsets;+import java.nio.file.Files;+import java.nio.file.Path;+import java.nio.file.StandardCopyOption;+import java.util.Map;+import java.util.Optional;+import java.util.stream.Stream;++import static org.junit.jupiter.api.Assertions.assertArrayEquals;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.when;++/**+ * Under {@code persistent} storage a PutObject is appended to the object-index journal instead of+ * rewriting {@code s3-objects.json}, which holds the metadata of every object in every bucket, so+ * the cost of one write no longer grows with the total object count. The bucket store keeps its+ * rewritten-on-every-call file.+ */+class S3ObjectIndexJournalTest {++ private static final String ACCOUNT = "000000000000";+ private static final String REGION = "us-east-1";+ private static final String BUCKET = "journal-bucket";+ private static final String VERSIONED_BUCKET = "journal-versioned";++ @TempDir+ Path root;++ @Test+ void putObjectUnderPersistentModeAppendsToTheJournalInsteadOfRewritingTheIndex() throws IOException {+ Path directory = root.resolve("live");+ Path journal = directory.resolve("s3-objects.wal");+ Path index = directory.resolve("s3-objects.json");+ Path bucketStore = directory.resolve("s3-buckets.json");++ StorageFactory first = newFactory(directory);+ S3Service s3 = newService(first, directory);+ s3.createBucket(BUCKET, REGION);+ assertTrue(Files.readString(bucketStore).contains(BUCKET), "the bucket store is still written on every call");++ long journalSize = 0;+ for (int i = 0; i < 3; i++) {+ s3.putObject(BUCKET, key(i), body(i), "text/plain", Map.of());+ assertTrue(journalSize(journal) > journalSize, "every PutObject must append to the journal");+ journalSize = journalSize(journal);+ assertFalse(Files.exists(index), "a PutObject must not rewrite the object index");+ }+ s3.deleteObject(BUCKET, key(2));+ assertTrue(journalSize(journal) > journalSize, "a DeleteObject must append to the journal");+ first.shutdownAll();++ assertTrue(Files.exists(index), "a clean shutdown folds the journal into the object index");+ StorageFactory second = newFactory(directory);+ try {+ S3Service reopened = newService(second, directory);+ for (int i = 0; i < 2; i++) {+ assertArrayEquals(body(i), reopened.getObject(BUCKET, key(i)).getData());+ }+ assertEquals(2, reopened.listObjects(BUCKET, null, null, 100).size());+ } finally {+ second.shutdownAll();+ }+ }++ @Test+ void journaledEntriesAreReplayedWhenTheProcessStopsWithoutCompacting() throws IOException {+ Path directory = root.resolve("live");+ Path afterCrash = root.resolve("after-crash");+ StorageFactory first = newFactory(directory);+ String v1;+ try {+ S3Service s3 = newService(first, directory);+ s3.createBucket(BUCKET, REGION);+ s3.createBucket(VERSIONED_BUCKET, REGION);+ s3.putBucketVersioning(VERSIONED_BUCKET, "Enabled");+ s3.putObject(BUCKET, key(0), body(0), "text/plain", Map.of());+ v1 = s3.putObject(VERSIONED_BUCKET, key(0), body(1), "text/plain", Map.of()).getVersionId();+ s3.putObject(VERSIONED_BUCKET, key(0), body(2), "text/plain", Map.of());+ assertFalse(Files.exists(directory.resolve("s3-objects.json")), "nothing has compacted the journal yet");+ copyAsLeftByAKill(directory, afterCrash);+ } finally {+ first.shutdownAll();+ }++ StorageFactory second = newFactory(afterCrash);+ try {+ S3Service reopened = newService(second, afterCrash);+ assertArrayEquals(body(0), reopened.getObject(BUCKET, key(0)).getData());+ assertArrayEquals(body(2), reopened.getObject(VERSIONED_BUCKET, key(0)).getData());+ assertArrayEquals(body(1), reopened.getObject(VERSIONED_BUCKET, key(0), v1).getData());+ } finally {+ second.shutdownAll();+ }+ }++ @Test+ void aTagUpdateOnTheLatestKeyIsReplayedUnderTheCurrentVersionToo() throws IOException {+ Path directory = root.resolve("live");+ Path afterCrash = root.resolve("after-crash");+ StorageFactory first = newFactory(directory);+ String current;+ try {+ S3Service s3 = newService(first, directory);+ s3.createBucket(VERSIONED_BUCKET, REGION);+ s3.putBucketVersioning(VERSIONED_BUCKET, "Enabled");+ current = s3.putObject(VERSIONED_BUCKET, key(0), body(0), "text/plain", Map.of()).getVersionId();+ s3.putObjectTagging(VERSIONED_BUCKET, key(0), Map.of("stage", "reviewed"));+ copyAsLeftByAKill(directory, afterCrash);+ } finally {+ first.shutdownAll();+ }++ StorageFactory second = newFactory(afterCrash);+ try {+ S3Service reopened = newService(second, afterCrash);+ assertEquals(Map.of("stage", "reviewed"), reopened.getObjectTagging(VERSIONED_BUCKET, key(0), current));+ } finally {+ second.shutdownAll();+ }+ }++ @Test+ void aLegalHoldSetByVersionIdOnTheCurrentVersionIsReplayedUnderTheLatestKeyToo() throws IOException {+ Path directory = root.resolve("live");+ Path afterCrash = root.resolve("after-crash");+ StorageFactory first = newFactory(directory);+ try {+ S3Service s3 = newService(first, directory);+ s3.createBucket(VERSIONED_BUCKET, REGION);+ s3.putBucketVersioning(VERSIONED_BUCKET, "Enabled");+ String current = s3.putObject(VERSIONED_BUCKET, key(0), body(0), "text/plain", Map.of()).getVersionId();+ s3.putObjectLegalHold(VERSIONED_BUCKET, key(0), current, "ON");+ copyAsLeftByAKill(directory, afterCrash);+ } finally {+ first.shutdownAll();+ }++ StorageFactory second = newFactory(afterCrash);+ try {+ S3Service reopened = newService(second, afterCrash);+ assertEquals("ON", reopened.getObjectLegalHold(VERSIONED_BUCKET, key(0), null).getLegalHoldStatus());+ } finally {+ second.shutdownAll();+ }+ }++ /**+ * The storage directory as a process killed now would leave it, with no shutdown compaction.+ * Replaying a copy keeps the restarted factory from sharing the journal with the live writer.+ */+ private static void copyAsLeftByAKill(Path source, Path target) throws IOException {+ try (Stream<Path> paths = Files.walk(source)) {+ for (Path path : (Iterable<Path>) paths::iterator) {+ Path copy = target.resolve(source.relativize(path).toString());+ if (Files.isDirectory(path)) {+ Files.createDirectories(copy);+ } else {+ Files.copy(path, copy, StandardCopyOption.REPLACE_EXISTING);+ }+ }+ }+ }++ private static String key(int i) {+ return "logos/object-" + i + ".txt";+ }++ private static byte[] body(int i) {+ return ("body " + i).getBytes(StandardCharsets.UTF_8);+ }++ private static long journalSize(Path journal) throws IOException {+ return Files.exists(journal) ? Files.size(journal) : 0L;+ }++ private static S3Service newService(StorageFactory factory, Path storage) {+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);+ when(config.storage().persistentPath()).thenReturn(storage.toString());+ when(config.storage().mode()).thenReturn("persistent");+ when(config.storage().services().s3().mode()).thenReturn(Optional.of("persistent"));+ when(config.effectiveBaseUrl()).thenReturn("http://localhost:4566");+ when(config.services().s3().enforceAuth()).thenReturn(false);+ when(config.services().s3().globalBucketNamespace()).thenReturn(false);+ return new S3Service(factory, config, null, null, null, null, null,+ new RegionResolver(REGION, ACCOUNT), new ObjectMapper(), null);+ }++ private static StorageFactory newFactory(Path storage) {+ EmulatorConfig config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);+ when(config.defaultAccountId()).thenReturn(ACCOUNT);+ when(config.storage().persistentPath()).thenReturn(storage.toString());+ when(config.storage().wal().compactionIntervalMs()).thenReturn(3_600_000L);+ ServiceConfigAccess access = mock(ServiceConfigAccess.class);+ when(access.storageMode(anyString())).thenReturn("persistent");+ when(access.storageFlushInterval(anyString())).thenReturn(3_600_000L);+ return new StorageFactory(config, access);+ }+}