Light, fluffy, and always free - The AWS Local Emulator alternative
fix(storage): publish replacement batches atomically (#6324)
Andre Kurait686d0b5f022b
diff --git a/docs/configuration/storage.md b/docs/configuration/storage.mdindex df1fa66f58..4e279055de 100644--- a/docs/configuration/storage.md+++ b/docs/configuration/storage.md@@ -11,6 +11,28 @@ Floci supports four storage backends. You can set a global default and override| `hybrid` | Yes | In-memory reads, async flush to disk | General local development || `wal` | Yes | Append-only write-ahead log with compaction | High-write workloads |+### Atomic replacement batches++Services that need to replace related records together can use the opt-in+`StorageBackend.putAllAtomically` operation. Values must be copied before modification;+the operation does not make existing mutable objects immutable. Memory publication and+individual scans see the whole batch. Separate reads are not a transaction. Atomic batches+copy the store's key/value map before publishing it, so their memory and CPU cost grows+with the number of stored entries. Reads remain lock-free during snapshot writes and WAL+compaction; ordinary writes retain their existing storage policy.++Persistent storage writes the complete candidate snapshot before publishing it in memory.+WAL storage writes a single framed batch, so replay applies either the whole batch or none+of a truncated frame. Both report write errors to the caller. Hybrid storage swaps in the prepared+batch map and retains its configured asynchronous flush interval; an unflushed batch can still be lost on restart. Ordinary `putAll` keeps its+existing batching behavior and does not provide this atomicity guarantee. Partitioned+stores do not implement the opt-in operation.++WAL replay still accepts existing PUT and DELETE records. Older Floci versions cannot+read the new batch record format; do not downgrade a store containing batch records+without first making a backup and completing a clean shutdown that compacts its WAL.+This is a batch/replay guarantee, not a claim of hardware power-loss durability.+## Global Configuration| Variable | Default | Description |diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackend.java b/src/main/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackend.javaindex 230f06c012..e9c0cd54f7 100644--- a/src/main/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackend.java+++ b/src/main/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackend.java@@ -82,6 +82,18 @@ public class AccountAwareStorageBackend<V> implements StorageBackend<String, V>delegate.putAll(prefixedEntries);}+ @Override+ public void putAllAtomically(Map<String, V> entries) {+ putAllAtomicallyForAccount(prefix(), entries);+ }++ public void putAllAtomicallyForAccount(String accountId, Map<String, V> entries) {+ Map<String, V> prefixedEntries = new LinkedHashMap<>();+ entries.forEach((key, value) -> prefixedEntries.put(accountId + "/" + key, value));+ delegate.putAllAtomically(prefixedEntries);+ trackOwner(accountId);+ }+@Overridepublic Optional<V> get(String key) {String prefixedKey = prefixed(key);diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/HybridStorage.java b/src/main/java/io/github/hectorvent/floci/core/storage/HybridStorage.javaindex 54b1c114b9..ff1675c686 100644--- a/src/main/java/io/github/hectorvent/floci/core/storage/HybridStorage.java+++ b/src/main/java/io/github/hectorvent/floci/core/storage/HybridStorage.java@@ -10,7 +10,6 @@ import java.io.IOException;import java.nio.file.Files;import java.nio.file.Path;import java.util.ArrayList;-import java.util.Collections;import java.util.List;import java.util.Map;import java.util.Optional;@@ -29,11 +28,12 @@ public class HybridStorage<K, V> implements StorageBackend<K, V> {private static final Logger LOG = Logger.getLogger(HybridStorage.class);- private final ConcurrentHashMap<K, V> store = new ConcurrentHashMap<>();+ private volatile ConcurrentHashMap<K, V> store = new ConcurrentHashMap<>();private final Path filePath;private final ObjectMapper objectMapper;private final TypeReference<Map<K, V>> typeReference;private final ScheduledExecutorService scheduler;+ private final Object flushLock = new Object();private final AtomicBoolean dirty = new AtomicBoolean(false);public HybridStorage(Path filePath, TypeReference<Map<K, V>> typeReference, long flushIntervalMs) {@@ -48,18 +48,26 @@ public class HybridStorage<K, V> implements StorageBackend<K, V> {}@Override- public void put(K key, V value) {+ public synchronized void put(K key, V value) {store.put(key, value);dirty.set(true);}+ @Override+ public synchronized void putAllAtomically(Map<K, V> entries) {+ ConcurrentHashMap<K, V> candidate = new ConcurrentHashMap<>(store);+ candidate.putAll(Map.copyOf(entries));+ store = candidate;+ dirty.set(true);+ }+@Overridepublic Optional<V> get(K key) {return Optional.ofNullable(store.get(key));}@Override- public void delete(K key) {+ public synchronized void delete(K key) {store.remove(key);dirty.set(true);}@@ -74,7 +82,7 @@ public class HybridStorage<K, V> implements StorageBackend<K, V> {@Overridepublic Set<K> keys() {- return Collections.unmodifiableSet(store.keySet());+ return Set.copyOf(store.keySet());}@Override@@ -99,7 +107,7 @@ public class HybridStorage<K, V> implements StorageBackend<K, V> {}@Override- public void clear() {+ public synchronized void clear() {store.clear();dirty.set(true);}@@ -115,13 +123,16 @@ public class HybridStorage<K, V> implements StorageBackend<K, V> {}}- private synchronized void persistToDisk() {- try {- StorageFiles.writeAtomically(filePath, tempFile -> objectMapper.writeValue(tempFile.toFile(), store));- LOG.debugv("Flushed {0} entries to {1}", store.size(), filePath);- } catch (IOException e) {- LOG.errorv(e, "Failed to persist data to {0}", filePath);- dirty.set(true);+ private void persistToDisk() {+ synchronized (flushLock) {+ Map<K, V> snapshot = store;+ try {+ StorageFiles.writeAtomically(filePath, tempFile -> objectMapper.writeValue(tempFile.toFile(), snapshot));+ LOG.debugv("Flushed {0} entries to {1}", snapshot.size(), filePath);+ } catch (IOException e) {+ LOG.errorv(e, "Failed to persist data to {0}", filePath);+ dirty.set(true);+ }}}}diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/InMemoryStorage.java b/src/main/java/io/github/hectorvent/floci/core/storage/InMemoryStorage.javaindex 49d24fdc5b..630e9ee6d4 100644--- a/src/main/java/io/github/hectorvent/floci/core/storage/InMemoryStorage.java+++ b/src/main/java/io/github/hectorvent/floci/core/storage/InMemoryStorage.java@@ -1,8 +1,8 @@package io.github.hectorvent.floci.core.storage;import java.util.ArrayList;-import java.util.Collections;import java.util.List;+import java.util.Map;import java.util.Optional;import java.util.Set;import java.util.concurrent.ConcurrentHashMap;@@ -14,20 +14,27 @@ import java.util.function.Predicate;*/public class InMemoryStorage<K, V> implements StorageBackend<K, V> {- private final ConcurrentHashMap<K, V> store = new ConcurrentHashMap<>();+ private volatile ConcurrentHashMap<K, V> store = new ConcurrentHashMap<>();@Override- public void put(K key, V value) {+ public synchronized void put(K key, V value) {store.put(key, value);}+ @Override+ public synchronized void putAllAtomically(Map<K, V> entries) {+ ConcurrentHashMap<K, V> candidate = new ConcurrentHashMap<>(store);+ candidate.putAll(Map.copyOf(entries));+ store = candidate;+ }+@Overridepublic Optional<V> get(K key) {return Optional.ofNullable(store.get(key));}@Override- public void delete(K key) {+ public synchronized void delete(K key) {store.remove(key);}@@ -44,7 +51,7 @@ public class InMemoryStorage<K, V> implements StorageBackend<K, V> {@Overridepublic Set<K> keys() {- return Collections.unmodifiableSet(store.keySet());+ return Set.copyOf(store.keySet());}@Override@@ -53,12 +60,12 @@ public class InMemoryStorage<K, V> implements StorageBackend<K, V> {}@Override- public void load() {+ public synchronized void load() {// No-op for in-memory storage}@Override- public void clear() {+ public synchronized void clear() {store.clear();}}diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/PersistentStorage.java b/src/main/java/io/github/hectorvent/floci/core/storage/PersistentStorage.javaindex 6b0e26ccbb..8df755766e 100644--- a/src/main/java/io/github/hectorvent/floci/core/storage/PersistentStorage.java+++ b/src/main/java/io/github/hectorvent/floci/core/storage/PersistentStorage.java@@ -7,10 +7,10 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;import org.jboss.logging.Logger;import java.io.IOException;+import java.io.UncheckedIOException;import java.nio.file.Files;import java.nio.file.Path;import java.util.ArrayList;-import java.util.Collections;import java.util.List;import java.util.Map;import java.util.Optional;@@ -29,7 +29,7 @@ public class PersistentStorage<K, V> implements StorageBackend<K, V> {private static final Logger LOG = Logger.getLogger(PersistentStorage.class);- private final ConcurrentHashMap<K, V> store = new ConcurrentHashMap<>();+ private volatile ConcurrentHashMap<K, V> store = new ConcurrentHashMap<>();private final Path filePath;private final ObjectMapper objectMapper;private final TypeReference<Map<K, V>> typeReference;@@ -44,13 +44,13 @@ public class PersistentStorage<K, V> implements StorageBackend<K, V> {}@Override- public void put(K key, V value) {+ public synchronized void put(K key, V value) {store.put(key, value);persistToDisk();}@Override- public void putAll(Map<K, V> entries) {+ public synchronized void putAll(Map<K, V> entries) {if (entries.isEmpty()) {return;}@@ -58,13 +58,29 @@ public class PersistentStorage<K, V> implements StorageBackend<K, V> {persistToDisk();}+ @Override+ public synchronized void putAllAtomically(Map<K, V> entries) {+ Map<K, V> replacements = Map.copyOf(entries);+ if (replacements.isEmpty()) {+ return;+ }+ ConcurrentHashMap<K, V> candidate = new ConcurrentHashMap<>(store);+ candidate.putAll(replacements);+ try {+ StorageFiles.writeAtomically(filePath, tempFile -> objectMapper.writeValue(tempFile.toFile(), candidate));+ } catch (IOException exception) {+ throw new UncheckedIOException("Failed to commit atomic batch to " + filePath, exception);+ }+ store = candidate;+ }+@Overridepublic Optional<V> get(K key) {return Optional.ofNullable(store.get(key));}@Override- public void delete(K key) {+ public synchronized void delete(K key) {store.remove(key);persistToDisk();}@@ -79,7 +95,7 @@ public class PersistentStorage<K, V> implements StorageBackend<K, V> {@Overridepublic Set<K> keys() {- return Collections.unmodifiableSet(store.keySet());+ return Set.copyOf(store.keySet());}@Override@@ -88,7 +104,7 @@ public class PersistentStorage<K, V> implements StorageBackend<K, V> {}@Override- public void load() {+ public synchronized void load() {if (!Files.exists(filePath)) {LOG.debugv("No persistent file found at {0}, starting with empty store", filePath);return;@@ -108,7 +124,7 @@ public class PersistentStorage<K, V> implements StorageBackend<K, V> {}@Override- public void clear() {+ public synchronized void clear() {store.clear();persistToDisk();}diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/StorageBackend.java b/src/main/java/io/github/hectorvent/floci/core/storage/StorageBackend.javaindex 1cb745af8f..c7ba2ab5d8 100644--- a/src/main/java/io/github/hectorvent/floci/core/storage/StorageBackend.java+++ b/src/main/java/io/github/hectorvent/floci/core/storage/StorageBackend.java@@ -25,6 +25,16 @@ public interface StorageBackend<K, V> {entries.forEach(this::put);}+ /**+ * Publish replacement values as one batch, including in snapshots and WAL replay.+ * Callers must not mutate values already stored here. Persistent and WAL backends+ * report write failures before publishing; hybrid retains its asynchronous durability.+ * Unsupported backends reject the operation without changing any entries.+ */+ default void putAllAtomically(Map<K, V> entries) {+ throw new UnsupportedOperationException("Atomic batches are not supported by this backend");+ }+Optional<V> get(K key);void delete(K key);diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/WalStorage.java b/src/main/java/io/github/hectorvent/floci/core/storage/WalStorage.javaindex 3c35521369..54b295452b 100644--- a/src/main/java/io/github/hectorvent/floci/core/storage/WalStorage.java+++ b/src/main/java/io/github/hectorvent/floci/core/storage/WalStorage.java@@ -8,19 +8,27 @@ import com.fasterxml.jackson.dataformat.cbor.CBORFactory;import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;import org.jboss.logging.Logger;-import java.io.*;+import java.io.BufferedInputStream;+import java.io.BufferedOutputStream;+import java.io.ByteArrayOutputStream;+import java.io.DataInputStream;+import java.io.DataOutputStream;+import java.io.EOFException;+import java.io.IOException;+import java.io.UncheckedIOException;+import java.nio.ByteBuffer;import java.nio.channels.FileChannel;import java.nio.file.Files;import java.nio.file.Path;import java.nio.file.StandardOpenOption;import java.util.ArrayList;-import java.util.Collections;import java.util.HashMap;import java.util.List;import java.util.Map;import java.util.Optional;import java.util.Set;-import java.util.concurrent.*;+import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.ScheduledExecutorService;import java.util.concurrent.locks.ReentrantReadWriteLock;import java.util.function.Predicate;import java.util.stream.Collectors;@@ -34,6 +42,7 @@ import java.util.stream.Collectors;* Binary WAL entry format:* PUT: [0x01] [4-byte key length] [key bytes] [4-byte value length] [value bytes]* DELETE: [0x02] [4-byte key length] [key bytes]+ * BATCH: [0x03] [4-byte map length] [CBOR map bytes]** Key and value bytes are serialized via Jackson CBOR (compact binary format).* Snapshot files use indented JSON for debuggability.@@ -44,8 +53,9 @@ public class WalStorage<K, V> implements StorageBackend<K, V> {static final byte OP_PUT = 0x01;static final byte OP_DELETE = 0x02;+ static final byte OP_BATCH = 0x03;- private final ConcurrentHashMap<K, V> store = new ConcurrentHashMap<>();+ private volatile ConcurrentHashMap<K, V> store = new ConcurrentHashMap<>();private final Path snapshotPath;private final Path walPath;private final ObjectMapper snapshotMapper;@@ -107,6 +117,63 @@ public class WalStorage<K, V> implements StorageBackend<K, V> {}}+ /** A batch is one length-delimited CBOR map, applied only after the entire frame decodes. */+ @Override+ public void putAllAtomically(Map<K, V> entries) {+ Map<K, V> replacements = Map.copyOf(entries);+ if (replacements.isEmpty()) {+ return;+ }+ byte[] payload;+ try {+ payload = walMapper.writeValueAsBytes(replacements);+ } catch (IOException exception) {+ throw new UncheckedIOException("Failed to encode atomic WAL batch", exception);+ }+ compactionLock.writeLock().lock();+ try {+ if (walWriter == null) {+ throw new IllegalStateException("WAL writer is not open");+ }+ ConcurrentHashMap<K, V> candidate = new ConcurrentHashMap<>(store);+ candidate.putAll(replacements);+ walWriter.flush();+ FileChannel channel = FileChannel.open(walPath, StandardOpenOption.WRITE);+ try {+ long offset = channel.size();+ channel.position(offset);+ ByteBuffer record = ByteBuffer.allocate(1 + Integer.BYTES + payload.length);+ record.put(OP_BATCH).putInt(payload.length).put(payload).flip();+ try {+ while (record.hasRemaining()) {+ channel.write(record);+ }+ channel.force(true);+ } catch (IOException exception) {+ try {+ channel.truncate(offset);+ channel.force(true);+ } catch (IOException rollbackFailure) {+ exception.addSuppressed(rollbackFailure);+ }+ throw exception;+ }+ } finally {+ try {+ channel.close();+ } catch (IOException exception) {+ LOG.warnv(exception, "Failed to close atomic batch channel for {0}", walPath);+ }+ }+ store = candidate;+ dirty = true;+ } catch (IOException exception) {+ throw new UncheckedIOException("Failed to commit atomic WAL batch to " + walPath, exception);+ } finally {+ compactionLock.writeLock().unlock();+ }+ }+@Overridepublic Optional<V> get(K key) {return Optional.ofNullable(store.get(key));@@ -134,7 +201,7 @@ public class WalStorage<K, V> implements StorageBackend<K, V> {@Overridepublic Set<K> keys() {- return Collections.unmodifiableSet(store.keySet());+ return Set.copyOf(store.keySet());}@Override@@ -190,8 +257,13 @@ public class WalStorage<K, V> implements StorageBackend<K, V> {public void shutdown() {StorageFiles.stop(scheduler);- compact();- closeWalWriter();+ compactionLock.writeLock().lock();+ try {+ compact();+ closeWalWriter();+ } finally {+ compactionLock.writeLock().unlock();+ }}/**@@ -337,7 +409,15 @@ public class WalStorage<K, V> implements StorageBackend<K, V> {break;}long recordLength = 1 + 4 + keyBytes.length;- if (op == OP_PUT) {+ if (op == OP_BATCH) {+ Map<K, V> batch = walMapper.readValue(keyBytes, typeReference);+ if (batch == null || batch.entrySet().stream()+ .anyMatch(entry -> entry.getKey() == null || entry.getValue() == null)) {+ LOG.warnv("Invalid WAL batch at offset {0} in {1}, stopping replay", consumed, walPath);+ break;+ }+ store.putAll(batch);+ } else if (op == OP_PUT) {byte[] valueBytes = readFrame(in, fileSize - consumed - recordLength);if (valueBytes == null) {break;diff --git a/src/test/java/io/github/hectorvent/floci/core/storage/AtomicStorageBatchTest.java b/src/test/java/io/github/hectorvent/floci/core/storage/AtomicStorageBatchTest.javanew file mode 100644index 0000000000..0318a92096--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/core/storage/AtomicStorageBatchTest.java@@ -0,0 +1,321 @@+package io.github.hectorvent.floci.core.storage;++import com.fasterxml.jackson.core.type.TypeReference;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.io.TempDir;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;++import java.io.UncheckedIOException;+import java.nio.file.Files;+import java.nio.file.Path;+import java.util.Arrays;+import java.util.Map;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;+import java.util.concurrent.TimeUnit;++import static org.junit.jupiter.api.Assertions.*;++class AtomicStorageBatchTest {+ private static final TypeReference<Map<String, Integer>> TYPE = new TypeReference<>() {};++ @TempDir+ Path directory;++ @ParameterizedTest+ @ValueSource(strings = {"memory", "persistent", "hybrid", "wal"})+ void publishesWholeBatchWithAccountIsolation(String mode) {+ StorageBackend<String, Integer> backend = backend(mode);+ backend.load();+ try {+ AccountAwareStorageBackend<Integer> store =+ new AccountAwareStorageBackend<>(backend, null, "111111111111");+ store.putAllAtomically(Map.of("source", 3, "destination", 1));+ store.putAllAtomicallyForAccount("222222222222", Map.of("source", 8, "destination", 2));+ store.putAllAtomically(Map.of("source", 1, "destination", 3));+ assertEquals(4, store.scan(key -> true).stream().mapToInt(Integer::intValue).sum());+ assertEquals(10, store.scanForAccount("222222222222", key -> true)+ .stream().mapToInt(Integer::intValue).sum());+ assertEquals(1, store.get("source").orElseThrow());+ assertEquals(8, store.getForAccount("222222222222", "source").orElseThrow());+ // Enable the owner index, then publish a new account through the batch path.+ assertTrue(store.findAnyAccount("missing").isEmpty());+ store.putAllAtomicallyForAccount("333333333333", Map.of("unique", 9));+ assertEquals("333333333333", store.findAnyAccountEntry("unique").orElseThrow().account());+ } finally {+ close(backend);+ }+ }++ @Test+ void persistentFailureLeavesMemoryAndPreviousSnapshotUnchanged() throws Exception {+ PersistentStorage<String, Integer> store = new PersistentStorage<>(directory.resolve("data.json"), TYPE);+ store.putAllAtomically(Map.of("source", 3, "destination", 1));+ Path snapshot = directory.resolve("data.json");+ Path backup = directory.resolve("backup.json");+ Files.move(snapshot, backup);+ Files.createDirectory(snapshot);+ Files.writeString(snapshot.resolve("blocker"), "owned test file");+ assertThrows(UncheckedIOException.class,+ () -> store.putAllAtomically(Map.of("source", 1, "destination", 3)));+ assertEquals(3, store.get("source").orElseThrow());+ assertEquals(1, store.get("destination").orElseThrow());+ PersistentStorage<String, Integer> recovered = new PersistentStorage<>(backup, TYPE);+ recovered.load();+ assertEquals(3, recovered.get("source").orElseThrow());+ Files.delete(snapshot.resolve("blocker"));+ Files.delete(snapshot);+ Files.move(backup, snapshot);+ store.putAllAtomically(Map.of("source", 1, "destination", 3));+ recovered = new PersistentStorage<>(snapshot, TYPE);+ recovered.load();+ assertEquals(1, recovered.get("source").orElseThrow());+ assertEquals(3, recovered.get("destination").orElseThrow());+ }++ @Test+ void hybridBatchDoesNotBecomeWriteThrough() throws Exception {+ HybridStorage<String, Integer> store = new HybridStorage<>(directory.resolve("data.json"), TYPE, Long.MAX_VALUE);+ try {+ store.putAllAtomically(Map.of("source", 3, "destination", 1));+ store.flush();+ byte[] before = Files.readAllBytes(directory.resolve("data.json"));+ store.putAllAtomically(Map.of("source", 1, "destination", 3));+ assertArrayEquals(before, Files.readAllBytes(directory.resolve("data.json")));+ store.flush();+ PersistentStorage<String, Integer> recovered = new PersistentStorage<>(directory.resolve("data.json"), TYPE);+ recovered.load();+ assertEquals(1, recovered.get("source").orElseThrow());+ assertEquals(3, recovered.get("destination").orElseThrow());+ } finally {+ store.shutdown();+ }+ }++ @Test+ void hybridSnapshotAndBatchSharePublicationBoundary() throws Exception {+ TypeReference<Map<String, SnapshotValue>> type = new TypeReference<>() {};+ HybridStorage<String, SnapshotValue> store =+ new HybridStorage<>(directory.resolve("data.json"), type, Long.MAX_VALUE);+ CountDownLatch serializing = new CountDownLatch(1);+ CountDownLatch resume = new CountDownLatch(1);+ SnapshotValue source = new SnapshotValue(3, serializing, resume);+ store.putAllAtomically(Map.of("source", source, "destination", new SnapshotValue(1, null, null)));+ try (ExecutorService workers = Executors.newFixedThreadPool(2)) {+ Future<?> flush = workers.submit(store::flush);+ try {+ assertTrue(serializing.await(10, TimeUnit.SECONDS));+ Future<?> batch = workers.submit(() -> store.putAllAtomically(+ Map.of("source", new SnapshotValue(1, null, null),+ "destination", new SnapshotValue(3, null, null))));+ batch.get(10, TimeUnit.SECONDS);+ Future<Integer> read = workers.submit(() -> store.get("source").orElseThrow().getCount());+ assertEquals(1, read.get(10, TimeUnit.SECONDS));+ resume.countDown();+ flush.get(10, TimeUnit.SECONDS);+ PersistentStorage<String, SnapshotValue> recovered =+ new PersistentStorage<>(directory.resolve("data.json"), type);+ recovered.load();+ assertEquals(3, recovered.get("source").orElseThrow().getCount());+ assertEquals(1, recovered.get("destination").orElseThrow().getCount());+ assertEquals(1, store.get("source").orElseThrow().getCount());+ assertEquals(3, store.get("destination").orElseThrow().getCount());+ } finally {+ resume.countDown();+ }+ } finally {+ store.shutdown();+ }+ }++ @ParameterizedTest+ @ValueSource(strings = {"persistent", "wal"})+ void readsContinueWhileSnapshotSerializationIsPaused(String mode) throws Exception {+ TypeReference<Map<String, SnapshotValue>> type = new TypeReference<>() {};+ StorageBackend<String, SnapshotValue> store = "wal".equals(mode)+ ? new WalStorage<>(directory.resolve("data.json"), directory.resolve("data.wal"), type, Long.MAX_VALUE)+ : new PersistentStorage<>(directory.resolve("data.json"), type);+ store.load();+ store.putAllAtomically(Map.of("value", new SnapshotValue(3, null, null)));+ CountDownLatch serializing = new CountDownLatch(1);+ CountDownLatch resume = new CountDownLatch(1);+ SnapshotValue blocking = new SnapshotValue(4, serializing, resume);+ try (ExecutorService workers = Executors.newFixedThreadPool(2)) {+ Future<?> write;+ if ("wal".equals(mode)) {+ // Activate the pause only after the ordinary WAL append has encoded the value.+ SnapshotValue value = new SnapshotValue(4, null, null);+ store.put("value", value);+ value.pause(serializing, resume);+ write = workers.submit(store::flush);+ } else {+ write = workers.submit(() -> store.putAllAtomically(Map.of("value", blocking)));+ }+ try {+ assertTrue(serializing.await(10, TimeUnit.SECONDS));+ Future<SnapshotValue> read = workers.submit(() -> store.get("value").orElseThrow());+ assertEquals("wal".equals(mode) ? 4 : 3, read.get(10, TimeUnit.SECONDS).count);+ Future<Integer> scan = workers.submit(() -> store.scan(key -> true).size());+ assertEquals(1, scan.get(10, TimeUnit.SECONDS));+ } finally {+ resume.countDown();+ }+ write.get(10, TimeUnit.SECONDS);+ } finally {+ if (store instanceof WalStorage<String, SnapshotValue> wal) {+ wal.shutdown();+ }+ }+ }++ @Test+ void inMemoryReadersDoNotQueueBehindAnotherScan() throws Exception {+ InMemoryStorage<String, Integer> store = new InMemoryStorage<>();+ store.put("value", 1);+ CountDownLatch scanning = new CountDownLatch(1);+ CountDownLatch resume = new CountDownLatch(1);+ try (ExecutorService workers = Executors.newFixedThreadPool(2)) {+ Future<?> scan = workers.submit(() -> store.scan(key -> {+ scanning.countDown();+ try {+ assertTrue(resume.await(10, TimeUnit.SECONDS));+ } catch (InterruptedException exception) {+ Thread.currentThread().interrupt();+ throw new AssertionError(exception);+ }+ return true;+ }));+ try {+ assertTrue(scanning.await(10, TimeUnit.SECONDS));+ Future<Integer> read = workers.submit(() -> store.get("value").orElseThrow());+ assertEquals(1, read.get(10, TimeUnit.SECONDS));+ } finally {+ resume.countDown();+ }+ scan.get(10, TimeUnit.SECONDS);+ }+ }++ public static class SnapshotValue {+ private int count;+ private CountDownLatch serializing;+ private CountDownLatch resume;++ public SnapshotValue() {+ this(0, null, null);+ }++ SnapshotValue(int count, CountDownLatch serializing, CountDownLatch resume) {+ this.count = count;+ this.serializing = serializing;+ this.resume = resume;+ }++ void pause(CountDownLatch started, CountDownLatch release) {+ serializing = started;+ resume = release;+ }++ public int getCount() {+ if (serializing != null) {+ serializing.countDown();+ try {+ assertTrue(resume.await(10, TimeUnit.SECONDS));+ } catch (InterruptedException exception) {+ Thread.currentThread().interrupt();+ throw new AssertionError(exception);+ }+ }+ return count;+ }++ public void setCount(int count) {+ this.count = count;+ }+ }++ @Test+ void everyTornBatchPrefixRetainsLegacyRecordsAndAllowsLaterWrites() throws Exception {+ Path snapshot = directory.resolve("original.json");+ Path log = directory.resolve("original.wal");+ WalStorage<String, Integer> store = new WalStorage<>(snapshot, log, TYPE, Long.MAX_VALUE);+ store.load();+ try {+ // Ordinary writes retain the old PUT/DELETE encoding.+ store.put("source", 3);+ store.put("destination", 1);+ store.put("deleted", 1);+ store.delete("deleted");+ int oldLength = Files.readAllBytes(log).length;+ store.putAllAtomically(Map.of("source", 1, "destination", 3));+ byte[] complete = Files.readAllBytes(log);+ for (int length = oldLength; length <= complete.length; length++) {+ Path replayLog = directory.resolve("replay-" + length + ".wal");+ Path replaySnapshot = directory.resolve("replay-" + length + ".json");+ Files.write(replayLog, Arrays.copyOf(complete, length));+ WalStorage<String, Integer> recovered = new WalStorage<>(replaySnapshot, replayLog, TYPE, Long.MAX_VALUE);+ recovered.load();+ try {+ assertEquals(length == complete.length ? 1 : 3, recovered.get("source").orElseThrow(),+ "prefix " + length);+ assertEquals(length == complete.length ? 3 : 1, recovered.get("destination").orElseThrow(),+ "prefix " + length);+ assertTrue(recovered.get("deleted").isEmpty());+ recovered.putAllAtomically(Map.of("later", 7));+ Map<String, Integer> replayed = WalStorage.readContents(replaySnapshot, replayLog, TYPE);+ assertEquals(7, replayed.get("later"));+ assertEquals(4, replayed.get("source") + replayed.get("destination"));+ } finally {+ recovered.shutdown();+ }+ }+ } finally {+ store.shutdown();+ }+ }++ @Test+ void walAppendFailureDoesNotPublishAndCanRetry() throws Exception {+ Path log = directory.resolve("data.wal");+ WalStorage<String, Integer> store = new WalStorage<>(directory.resolve("data.json"), log, TYPE, Long.MAX_VALUE);+ store.load();+ try {+ store.putAllAtomically(Map.of("source", 3, "destination", 1));+ Path saved = directory.resolve("saved.wal");+ Files.move(log, saved);+ Files.createDirectory(log);+ assertThrows(UncheckedIOException.class,+ () -> store.putAllAtomically(Map.of("source", 1, "destination", 3)));+ assertEquals(3, store.get("source").orElseThrow());+ assertEquals(1, store.get("destination").orElseThrow());+ Files.delete(log);+ Files.move(saved, log);+ store.putAllAtomically(Map.of("source", 1, "destination", 3));+ assertEquals(Map.of("source", 1, "destination", 3),+ WalStorage.readContents(directory.resolve("data.json"), log, TYPE));+ } finally {+ store.shutdown();+ }+ }++ private StorageBackend<String, Integer> backend(String mode) {+ return switch (mode) {+ case "persistent" -> new PersistentStorage<>(directory.resolve("data.json"), TYPE);+ case "hybrid" -> new HybridStorage<>(directory.resolve("data.json"), TYPE, Long.MAX_VALUE);+ case "wal" -> new WalStorage<>(directory.resolve("data.json"), directory.resolve("data.wal"),+ TYPE, Long.MAX_VALUE);+ default -> new InMemoryStorage<>();+ };+ }++ private void close(StorageBackend<String, Integer> backend) {+ if (backend instanceof WalStorage<String, Integer> wal) {+ wal.shutdown();+ } else if (backend instanceof HybridStorage<String, Integer> hybrid) {+ hybrid.shutdown();+ }+ }+}