Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(storage): publish replacement batches atomically (#6324)

Andre Kurait686d0b5f022b
diff --git a/docs/configuration/storage.md b/docs/configuration/storage.md
index df1fa66f58..4e279055de 100644
--- a/docs/configuration/storage.md
+++ b/docs/configuration/storage.md
@@ -11,6 +11,28 @@ Floci supports four storage backends. You can set a global default and override
| `hybrid` | Yes | In-memory reads, async flush to disk | General local development |
| `wal` | Yes | Append-only write-ahead log with compaction | High-write workloads |
+### Atomic replacement batches
+
+Services that need to replace related records together can use the opt-in
+`StorageBackend.putAllAtomically` operation. Values must be copied before modification;
+the operation does not make existing mutable objects immutable. Memory publication and
+individual scans see the whole batch. Separate reads are not a transaction. Atomic batches
+copy the store's key/value map before publishing it, so their memory and CPU cost grows
+with the number of stored entries. Reads remain lock-free during snapshot writes and WAL
+compaction; ordinary writes retain their existing storage policy.
+
+Persistent storage writes the complete candidate snapshot before publishing it in memory.
+WAL storage writes a single framed batch, so replay applies either the whole batch or none
+of a truncated frame. Both report write errors to the caller. Hybrid storage swaps in the prepared
+batch map and retains its configured asynchronous flush interval; an unflushed batch can still be lost on restart. Ordinary `putAll` keeps its
+existing batching behavior and does not provide this atomicity guarantee. Partitioned
+stores do not implement the opt-in operation.
+
+WAL replay still accepts existing PUT and DELETE records. Older Floci versions cannot
+read the new batch record format; do not downgrade a store containing batch records
+without first making a backup and completing a clean shutdown that compacts its WAL.
+This is a batch/replay guarantee, not a claim of hardware power-loss durability.
+
## Global Configuration
| Variable | Default | Description |
diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackend.java b/src/main/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackend.java
index 230f06c012..e9c0cd54f7 100644
--- a/src/main/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackend.java
+++ b/src/main/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackend.java
@@ -82,6 +82,18 @@ public class AccountAwareStorageBackend<V> implements StorageBackend<String, V>
delegate.putAll(prefixedEntries);
}
+ @Override
+ public void putAllAtomically(Map<String, V> entries) {
+ putAllAtomicallyForAccount(prefix(), entries);
+ }
+
+ public void putAllAtomicallyForAccount(String accountId, Map<String, V> entries) {
+ Map<String, V> prefixedEntries = new LinkedHashMap<>();
+ entries.forEach((key, value) -> prefixedEntries.put(accountId + "/" + key, value));
+ delegate.putAllAtomically(prefixedEntries);
+ trackOwner(accountId);
+ }
+
@Override
public Optional<V> get(String key) {
String prefixedKey = prefixed(key);
diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/HybridStorage.java b/src/main/java/io/github/hectorvent/floci/core/storage/HybridStorage.java
index 54b1c114b9..ff1675c686 100644
--- a/src/main/java/io/github/hectorvent/floci/core/storage/HybridStorage.java
+++ b/src/main/java/io/github/hectorvent/floci/core/storage/HybridStorage.java
@@ -10,7 +10,6 @@ import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -29,11 +28,12 @@ public class HybridStorage<K, V> implements StorageBackend<K, V> {
private static final Logger LOG = Logger.getLogger(HybridStorage.class);
- private final ConcurrentHashMap<K, V> store = new ConcurrentHashMap<>();
+ private volatile ConcurrentHashMap<K, V> store = new ConcurrentHashMap<>();
private final Path filePath;
private final ObjectMapper objectMapper;
private final TypeReference<Map<K, V>> typeReference;
private final ScheduledExecutorService scheduler;
+ private final Object flushLock = new Object();
private final AtomicBoolean dirty = new AtomicBoolean(false);
public HybridStorage(Path filePath, TypeReference<Map<K, V>> typeReference, long flushIntervalMs) {
@@ -48,18 +48,26 @@ public class HybridStorage<K, V> implements StorageBackend<K, V> {
}
@Override
- public void put(K key, V value) {
+ public synchronized void put(K key, V value) {
store.put(key, value);
dirty.set(true);
}
+ @Override
+ public synchronized void putAllAtomically(Map<K, V> entries) {
+ ConcurrentHashMap<K, V> candidate = new ConcurrentHashMap<>(store);
+ candidate.putAll(Map.copyOf(entries));
+ store = candidate;
+ dirty.set(true);
+ }
+
@Override
public Optional<V> get(K key) {
return Optional.ofNullable(store.get(key));
}
@Override
- public void delete(K key) {
+ public synchronized void delete(K key) {
store.remove(key);
dirty.set(true);
}
@@ -74,7 +82,7 @@ public class HybridStorage<K, V> implements StorageBackend<K, V> {
@Override
public Set<K> keys() {
- return Collections.unmodifiableSet(store.keySet());
+ return Set.copyOf(store.keySet());
}
@Override
@@ -99,7 +107,7 @@ public class HybridStorage<K, V> implements StorageBackend<K, V> {
}
@Override
- public void clear() {
+ public synchronized void clear() {
store.clear();
dirty.set(true);
}
@@ -115,13 +123,16 @@ public class HybridStorage<K, V> implements StorageBackend<K, V> {
}
}
- private synchronized void persistToDisk() {
- try {
- StorageFiles.writeAtomically(filePath, tempFile -> objectMapper.writeValue(tempFile.toFile(), store));
- LOG.debugv("Flushed {0} entries to {1}", store.size(), filePath);
- } catch (IOException e) {
- LOG.errorv(e, "Failed to persist data to {0}", filePath);
- dirty.set(true);
+ private void persistToDisk() {
+ synchronized (flushLock) {
+ Map<K, V> snapshot = store;
+ try {
+ StorageFiles.writeAtomically(filePath, tempFile -> objectMapper.writeValue(tempFile.toFile(), snapshot));
+ LOG.debugv("Flushed {0} entries to {1}", snapshot.size(), filePath);
+ } catch (IOException e) {
+ LOG.errorv(e, "Failed to persist data to {0}", filePath);
+ dirty.set(true);
+ }
}
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/InMemoryStorage.java b/src/main/java/io/github/hectorvent/floci/core/storage/InMemoryStorage.java
index 49d24fdc5b..630e9ee6d4 100644
--- a/src/main/java/io/github/hectorvent/floci/core/storage/InMemoryStorage.java
+++ b/src/main/java/io/github/hectorvent/floci/core/storage/InMemoryStorage.java
@@ -1,8 +1,8 @@
package io.github.hectorvent.floci.core.storage;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
@@ -14,20 +14,27 @@ import java.util.function.Predicate;
*/
public class InMemoryStorage<K, V> implements StorageBackend<K, V> {
- private final ConcurrentHashMap<K, V> store = new ConcurrentHashMap<>();
+ private volatile ConcurrentHashMap<K, V> store = new ConcurrentHashMap<>();
@Override
- public void put(K key, V value) {
+ public synchronized void put(K key, V value) {
store.put(key, value);
}
+ @Override
+ public synchronized void putAllAtomically(Map<K, V> entries) {
+ ConcurrentHashMap<K, V> candidate = new ConcurrentHashMap<>(store);
+ candidate.putAll(Map.copyOf(entries));
+ store = candidate;
+ }
+
@Override
public Optional<V> get(K key) {
return Optional.ofNullable(store.get(key));
}
@Override
- public void delete(K key) {
+ public synchronized void delete(K key) {
store.remove(key);
}
@@ -44,7 +51,7 @@ public class InMemoryStorage<K, V> implements StorageBackend<K, V> {
@Override
public Set<K> keys() {
- return Collections.unmodifiableSet(store.keySet());
+ return Set.copyOf(store.keySet());
}
@Override
@@ -53,12 +60,12 @@ public class InMemoryStorage<K, V> implements StorageBackend<K, V> {
}
@Override
- public void load() {
+ public synchronized void load() {
// No-op for in-memory storage
}
@Override
- public void clear() {
+ public synchronized void clear() {
store.clear();
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/PersistentStorage.java b/src/main/java/io/github/hectorvent/floci/core/storage/PersistentStorage.java
index 6b0e26ccbb..8df755766e 100644
--- a/src/main/java/io/github/hectorvent/floci/core/storage/PersistentStorage.java
+++ b/src/main/java/io/github/hectorvent/floci/core/storage/PersistentStorage.java
@@ -7,10 +7,10 @@ import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import org.jboss.logging.Logger;
import java.io.IOException;
+import java.io.UncheckedIOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.Optional;
@@ -29,7 +29,7 @@ public class PersistentStorage<K, V> implements StorageBackend<K, V> {
private static final Logger LOG = Logger.getLogger(PersistentStorage.class);
- private final ConcurrentHashMap<K, V> store = new ConcurrentHashMap<>();
+ private volatile ConcurrentHashMap<K, V> store = new ConcurrentHashMap<>();
private final Path filePath;
private final ObjectMapper objectMapper;
private final TypeReference<Map<K, V>> typeReference;
@@ -44,13 +44,13 @@ public class PersistentStorage<K, V> implements StorageBackend<K, V> {
}
@Override
- public void put(K key, V value) {
+ public synchronized void put(K key, V value) {
store.put(key, value);
persistToDisk();
}
@Override
- public void putAll(Map<K, V> entries) {
+ public synchronized void putAll(Map<K, V> entries) {
if (entries.isEmpty()) {
return;
}
@@ -58,13 +58,29 @@ public class PersistentStorage<K, V> implements StorageBackend<K, V> {
persistToDisk();
}
+ @Override
+ public synchronized void putAllAtomically(Map<K, V> entries) {
+ Map<K, V> replacements = Map.copyOf(entries);
+ if (replacements.isEmpty()) {
+ return;
+ }
+ ConcurrentHashMap<K, V> candidate = new ConcurrentHashMap<>(store);
+ candidate.putAll(replacements);
+ try {
+ StorageFiles.writeAtomically(filePath, tempFile -> objectMapper.writeValue(tempFile.toFile(), candidate));
+ } catch (IOException exception) {
+ throw new UncheckedIOException("Failed to commit atomic batch to " + filePath, exception);
+ }
+ store = candidate;
+ }
+
@Override
public Optional<V> get(K key) {
return Optional.ofNullable(store.get(key));
}
@Override
- public void delete(K key) {
+ public synchronized void delete(K key) {
store.remove(key);
persistToDisk();
}
@@ -79,7 +95,7 @@ public class PersistentStorage<K, V> implements StorageBackend<K, V> {
@Override
public Set<K> keys() {
- return Collections.unmodifiableSet(store.keySet());
+ return Set.copyOf(store.keySet());
}
@Override
@@ -88,7 +104,7 @@ public class PersistentStorage<K, V> implements StorageBackend<K, V> {
}
@Override
- public void load() {
+ public synchronized void load() {
if (!Files.exists(filePath)) {
LOG.debugv("No persistent file found at {0}, starting with empty store", filePath);
return;
@@ -108,7 +124,7 @@ public class PersistentStorage<K, V> implements StorageBackend<K, V> {
}
@Override
- public void clear() {
+ public synchronized void clear() {
store.clear();
persistToDisk();
}
diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/StorageBackend.java b/src/main/java/io/github/hectorvent/floci/core/storage/StorageBackend.java
index 1cb745af8f..c7ba2ab5d8 100644
--- a/src/main/java/io/github/hectorvent/floci/core/storage/StorageBackend.java
+++ b/src/main/java/io/github/hectorvent/floci/core/storage/StorageBackend.java
@@ -25,6 +25,16 @@ public interface StorageBackend<K, V> {
entries.forEach(this::put);
}
+ /**
+ * Publish replacement values as one batch, including in snapshots and WAL replay.
+ * Callers must not mutate values already stored here. Persistent and WAL backends
+ * report write failures before publishing; hybrid retains its asynchronous durability.
+ * Unsupported backends reject the operation without changing any entries.
+ */
+ default void putAllAtomically(Map<K, V> entries) {
+ throw new UnsupportedOperationException("Atomic batches are not supported by this backend");
+ }
+
Optional<V> get(K key);
void delete(K key);
diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/WalStorage.java b/src/main/java/io/github/hectorvent/floci/core/storage/WalStorage.java
index 3c35521369..54b295452b 100644
--- a/src/main/java/io/github/hectorvent/floci/core/storage/WalStorage.java
+++ b/src/main/java/io/github/hectorvent/floci/core/storage/WalStorage.java
@@ -8,19 +8,27 @@ import com.fasterxml.jackson.dataformat.cbor.CBORFactory;
import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
import org.jboss.logging.Logger;
-import java.io.*;
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.ByteBuffer;
import java.nio.channels.FileChannel;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.StandardOpenOption;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.Set;
-import java.util.concurrent.*;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.function.Predicate;
import java.util.stream.Collectors;
@@ -34,6 +42,7 @@ import java.util.stream.Collectors;
* Binary WAL entry format:
* PUT: [0x01] [4-byte key length] [key bytes] [4-byte value length] [value bytes]
* DELETE: [0x02] [4-byte key length] [key bytes]
+ * BATCH: [0x03] [4-byte map length] [CBOR map bytes]
*
* Key and value bytes are serialized via Jackson CBOR (compact binary format).
* Snapshot files use indented JSON for debuggability.
@@ -44,8 +53,9 @@ public class WalStorage<K, V> implements StorageBackend<K, V> {
static final byte OP_PUT = 0x01;
static final byte OP_DELETE = 0x02;
+ static final byte OP_BATCH = 0x03;
- private final ConcurrentHashMap<K, V> store = new ConcurrentHashMap<>();
+ private volatile ConcurrentHashMap<K, V> store = new ConcurrentHashMap<>();
private final Path snapshotPath;
private final Path walPath;
private final ObjectMapper snapshotMapper;
@@ -107,6 +117,63 @@ public class WalStorage<K, V> implements StorageBackend<K, V> {
}
}
+ /** A batch is one length-delimited CBOR map, applied only after the entire frame decodes. */
+ @Override
+ public void putAllAtomically(Map<K, V> entries) {
+ Map<K, V> replacements = Map.copyOf(entries);
+ if (replacements.isEmpty()) {
+ return;
+ }
+ byte[] payload;
+ try {
+ payload = walMapper.writeValueAsBytes(replacements);
+ } catch (IOException exception) {
+ throw new UncheckedIOException("Failed to encode atomic WAL batch", exception);
+ }
+ compactionLock.writeLock().lock();
+ try {
+ if (walWriter == null) {
+ throw new IllegalStateException("WAL writer is not open");
+ }
+ ConcurrentHashMap<K, V> candidate = new ConcurrentHashMap<>(store);
+ candidate.putAll(replacements);
+ walWriter.flush();
+ FileChannel channel = FileChannel.open(walPath, StandardOpenOption.WRITE);
+ try {
+ long offset = channel.size();
+ channel.position(offset);
+ ByteBuffer record = ByteBuffer.allocate(1 + Integer.BYTES + payload.length);
+ record.put(OP_BATCH).putInt(payload.length).put(payload).flip();
+ try {
+ while (record.hasRemaining()) {
+ channel.write(record);
+ }
+ channel.force(true);
+ } catch (IOException exception) {
+ try {
+ channel.truncate(offset);
+ channel.force(true);
+ } catch (IOException rollbackFailure) {
+ exception.addSuppressed(rollbackFailure);
+ }
+ throw exception;
+ }
+ } finally {
+ try {
+ channel.close();
+ } catch (IOException exception) {
+ LOG.warnv(exception, "Failed to close atomic batch channel for {0}", walPath);
+ }
+ }
+ store = candidate;
+ dirty = true;
+ } catch (IOException exception) {
+ throw new UncheckedIOException("Failed to commit atomic WAL batch to " + walPath, exception);
+ } finally {
+ compactionLock.writeLock().unlock();
+ }
+ }
+
@Override
public Optional<V> get(K key) {
return Optional.ofNullable(store.get(key));
@@ -134,7 +201,7 @@ public class WalStorage<K, V> implements StorageBackend<K, V> {
@Override
public Set<K> keys() {
- return Collections.unmodifiableSet(store.keySet());
+ return Set.copyOf(store.keySet());
}
@Override
@@ -190,8 +257,13 @@ public class WalStorage<K, V> implements StorageBackend<K, V> {
public void shutdown() {
StorageFiles.stop(scheduler);
- compact();
- closeWalWriter();
+ compactionLock.writeLock().lock();
+ try {
+ compact();
+ closeWalWriter();
+ } finally {
+ compactionLock.writeLock().unlock();
+ }
}
/**
@@ -337,7 +409,15 @@ public class WalStorage<K, V> implements StorageBackend<K, V> {
break;
}
long recordLength = 1 + 4 + keyBytes.length;
- if (op == OP_PUT) {
+ if (op == OP_BATCH) {
+ Map<K, V> batch = walMapper.readValue(keyBytes, typeReference);
+ if (batch == null || batch.entrySet().stream()
+ .anyMatch(entry -> entry.getKey() == null || entry.getValue() == null)) {
+ LOG.warnv("Invalid WAL batch at offset {0} in {1}, stopping replay", consumed, walPath);
+ break;
+ }
+ store.putAll(batch);
+ } else if (op == OP_PUT) {
byte[] valueBytes = readFrame(in, fileSize - consumed - recordLength);
if (valueBytes == null) {
break;
diff --git a/src/test/java/io/github/hectorvent/floci/core/storage/AtomicStorageBatchTest.java b/src/test/java/io/github/hectorvent/floci/core/storage/AtomicStorageBatchTest.java
new file mode 100644
index 0000000000..0318a92096
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/core/storage/AtomicStorageBatchTest.java
@@ -0,0 +1,321 @@
+package io.github.hectorvent.floci.core.storage;
+
+import com.fasterxml.jackson.core.type.TypeReference;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.Arrays;
+import java.util.Map;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.*;
+
+class AtomicStorageBatchTest {
+ private static final TypeReference<Map<String, Integer>> TYPE = new TypeReference<>() {};
+
+ @TempDir
+ Path directory;
+
+ @ParameterizedTest
+ @ValueSource(strings = {"memory", "persistent", "hybrid", "wal"})
+ void publishesWholeBatchWithAccountIsolation(String mode) {
+ StorageBackend<String, Integer> backend = backend(mode);
+ backend.load();
+ try {
+ AccountAwareStorageBackend<Integer> store =
+ new AccountAwareStorageBackend<>(backend, null, "111111111111");
+ store.putAllAtomically(Map.of("source", 3, "destination", 1));
+ store.putAllAtomicallyForAccount("222222222222", Map.of("source", 8, "destination", 2));
+ store.putAllAtomically(Map.of("source", 1, "destination", 3));
+ assertEquals(4, store.scan(key -> true).stream().mapToInt(Integer::intValue).sum());
+ assertEquals(10, store.scanForAccount("222222222222", key -> true)
+ .stream().mapToInt(Integer::intValue).sum());
+ assertEquals(1, store.get("source").orElseThrow());
+ assertEquals(8, store.getForAccount("222222222222", "source").orElseThrow());
+ // Enable the owner index, then publish a new account through the batch path.
+ assertTrue(store.findAnyAccount("missing").isEmpty());
+ store.putAllAtomicallyForAccount("333333333333", Map.of("unique", 9));
+ assertEquals("333333333333", store.findAnyAccountEntry("unique").orElseThrow().account());
+ } finally {
+ close(backend);
+ }
+ }
+
+ @Test
+ void persistentFailureLeavesMemoryAndPreviousSnapshotUnchanged() throws Exception {
+ PersistentStorage<String, Integer> store = new PersistentStorage<>(directory.resolve("data.json"), TYPE);
+ store.putAllAtomically(Map.of("source", 3, "destination", 1));
+ Path snapshot = directory.resolve("data.json");
+ Path backup = directory.resolve("backup.json");
+ Files.move(snapshot, backup);
+ Files.createDirectory(snapshot);
+ Files.writeString(snapshot.resolve("blocker"), "owned test file");
+ assertThrows(UncheckedIOException.class,
+ () -> store.putAllAtomically(Map.of("source", 1, "destination", 3)));
+ assertEquals(3, store.get("source").orElseThrow());
+ assertEquals(1, store.get("destination").orElseThrow());
+ PersistentStorage<String, Integer> recovered = new PersistentStorage<>(backup, TYPE);
+ recovered.load();
+ assertEquals(3, recovered.get("source").orElseThrow());
+ Files.delete(snapshot.resolve("blocker"));
+ Files.delete(snapshot);
+ Files.move(backup, snapshot);
+ store.putAllAtomically(Map.of("source", 1, "destination", 3));
+ recovered = new PersistentStorage<>(snapshot, TYPE);
+ recovered.load();
+ assertEquals(1, recovered.get("source").orElseThrow());
+ assertEquals(3, recovered.get("destination").orElseThrow());
+ }
+
+ @Test
+ void hybridBatchDoesNotBecomeWriteThrough() throws Exception {
+ HybridStorage<String, Integer> store = new HybridStorage<>(directory.resolve("data.json"), TYPE, Long.MAX_VALUE);
+ try {
+ store.putAllAtomically(Map.of("source", 3, "destination", 1));
+ store.flush();
+ byte[] before = Files.readAllBytes(directory.resolve("data.json"));
+ store.putAllAtomically(Map.of("source", 1, "destination", 3));
+ assertArrayEquals(before, Files.readAllBytes(directory.resolve("data.json")));
+ store.flush();
+ PersistentStorage<String, Integer> recovered = new PersistentStorage<>(directory.resolve("data.json"), TYPE);
+ recovered.load();
+ assertEquals(1, recovered.get("source").orElseThrow());
+ assertEquals(3, recovered.get("destination").orElseThrow());
+ } finally {
+ store.shutdown();
+ }
+ }
+
+ @Test
+ void hybridSnapshotAndBatchSharePublicationBoundary() throws Exception {
+ TypeReference<Map<String, SnapshotValue>> type = new TypeReference<>() {};
+ HybridStorage<String, SnapshotValue> store =
+ new HybridStorage<>(directory.resolve("data.json"), type, Long.MAX_VALUE);
+ CountDownLatch serializing = new CountDownLatch(1);
+ CountDownLatch resume = new CountDownLatch(1);
+ SnapshotValue source = new SnapshotValue(3, serializing, resume);
+ store.putAllAtomically(Map.of("source", source, "destination", new SnapshotValue(1, null, null)));
+ try (ExecutorService workers = Executors.newFixedThreadPool(2)) {
+ Future<?> flush = workers.submit(store::flush);
+ try {
+ assertTrue(serializing.await(10, TimeUnit.SECONDS));
+ Future<?> batch = workers.submit(() -> store.putAllAtomically(
+ Map.of("source", new SnapshotValue(1, null, null),
+ "destination", new SnapshotValue(3, null, null))));
+ batch.get(10, TimeUnit.SECONDS);
+ Future<Integer> read = workers.submit(() -> store.get("source").orElseThrow().getCount());
+ assertEquals(1, read.get(10, TimeUnit.SECONDS));
+ resume.countDown();
+ flush.get(10, TimeUnit.SECONDS);
+ PersistentStorage<String, SnapshotValue> recovered =
+ new PersistentStorage<>(directory.resolve("data.json"), type);
+ recovered.load();
+ assertEquals(3, recovered.get("source").orElseThrow().getCount());
+ assertEquals(1, recovered.get("destination").orElseThrow().getCount());
+ assertEquals(1, store.get("source").orElseThrow().getCount());
+ assertEquals(3, store.get("destination").orElseThrow().getCount());
+ } finally {
+ resume.countDown();
+ }
+ } finally {
+ store.shutdown();
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(strings = {"persistent", "wal"})
+ void readsContinueWhileSnapshotSerializationIsPaused(String mode) throws Exception {
+ TypeReference<Map<String, SnapshotValue>> type = new TypeReference<>() {};
+ StorageBackend<String, SnapshotValue> store = "wal".equals(mode)
+ ? new WalStorage<>(directory.resolve("data.json"), directory.resolve("data.wal"), type, Long.MAX_VALUE)
+ : new PersistentStorage<>(directory.resolve("data.json"), type);
+ store.load();
+ store.putAllAtomically(Map.of("value", new SnapshotValue(3, null, null)));
+ CountDownLatch serializing = new CountDownLatch(1);
+ CountDownLatch resume = new CountDownLatch(1);
+ SnapshotValue blocking = new SnapshotValue(4, serializing, resume);
+ try (ExecutorService workers = Executors.newFixedThreadPool(2)) {
+ Future<?> write;
+ if ("wal".equals(mode)) {
+ // Activate the pause only after the ordinary WAL append has encoded the value.
+ SnapshotValue value = new SnapshotValue(4, null, null);
+ store.put("value", value);
+ value.pause(serializing, resume);
+ write = workers.submit(store::flush);
+ } else {
+ write = workers.submit(() -> store.putAllAtomically(Map.of("value", blocking)));
+ }
+ try {
+ assertTrue(serializing.await(10, TimeUnit.SECONDS));
+ Future<SnapshotValue> read = workers.submit(() -> store.get("value").orElseThrow());
+ assertEquals("wal".equals(mode) ? 4 : 3, read.get(10, TimeUnit.SECONDS).count);
+ Future<Integer> scan = workers.submit(() -> store.scan(key -> true).size());
+ assertEquals(1, scan.get(10, TimeUnit.SECONDS));
+ } finally {
+ resume.countDown();
+ }
+ write.get(10, TimeUnit.SECONDS);
+ } finally {
+ if (store instanceof WalStorage<String, SnapshotValue> wal) {
+ wal.shutdown();
+ }
+ }
+ }
+
+ @Test
+ void inMemoryReadersDoNotQueueBehindAnotherScan() throws Exception {
+ InMemoryStorage<String, Integer> store = new InMemoryStorage<>();
+ store.put("value", 1);
+ CountDownLatch scanning = new CountDownLatch(1);
+ CountDownLatch resume = new CountDownLatch(1);
+ try (ExecutorService workers = Executors.newFixedThreadPool(2)) {
+ Future<?> scan = workers.submit(() -> store.scan(key -> {
+ scanning.countDown();
+ try {
+ assertTrue(resume.await(10, TimeUnit.SECONDS));
+ } catch (InterruptedException exception) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError(exception);
+ }
+ return true;
+ }));
+ try {
+ assertTrue(scanning.await(10, TimeUnit.SECONDS));
+ Future<Integer> read = workers.submit(() -> store.get("value").orElseThrow());
+ assertEquals(1, read.get(10, TimeUnit.SECONDS));
+ } finally {
+ resume.countDown();
+ }
+ scan.get(10, TimeUnit.SECONDS);
+ }
+ }
+
+ public static class SnapshotValue {
+ private int count;
+ private CountDownLatch serializing;
+ private CountDownLatch resume;
+
+ public SnapshotValue() {
+ this(0, null, null);
+ }
+
+ SnapshotValue(int count, CountDownLatch serializing, CountDownLatch resume) {
+ this.count = count;
+ this.serializing = serializing;
+ this.resume = resume;
+ }
+
+ void pause(CountDownLatch started, CountDownLatch release) {
+ serializing = started;
+ resume = release;
+ }
+
+ public int getCount() {
+ if (serializing != null) {
+ serializing.countDown();
+ try {
+ assertTrue(resume.await(10, TimeUnit.SECONDS));
+ } catch (InterruptedException exception) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError(exception);
+ }
+ }
+ return count;
+ }
+
+ public void setCount(int count) {
+ this.count = count;
+ }
+ }
+
+ @Test
+ void everyTornBatchPrefixRetainsLegacyRecordsAndAllowsLaterWrites() throws Exception {
+ Path snapshot = directory.resolve("original.json");
+ Path log = directory.resolve("original.wal");
+ WalStorage<String, Integer> store = new WalStorage<>(snapshot, log, TYPE, Long.MAX_VALUE);
+ store.load();
+ try {
+ // Ordinary writes retain the old PUT/DELETE encoding.
+ store.put("source", 3);
+ store.put("destination", 1);
+ store.put("deleted", 1);
+ store.delete("deleted");
+ int oldLength = Files.readAllBytes(log).length;
+ store.putAllAtomically(Map.of("source", 1, "destination", 3));
+ byte[] complete = Files.readAllBytes(log);
+ for (int length = oldLength; length <= complete.length; length++) {
+ Path replayLog = directory.resolve("replay-" + length + ".wal");
+ Path replaySnapshot = directory.resolve("replay-" + length + ".json");
+ Files.write(replayLog, Arrays.copyOf(complete, length));
+ WalStorage<String, Integer> recovered = new WalStorage<>(replaySnapshot, replayLog, TYPE, Long.MAX_VALUE);
+ recovered.load();
+ try {
+ assertEquals(length == complete.length ? 1 : 3, recovered.get("source").orElseThrow(),
+ "prefix " + length);
+ assertEquals(length == complete.length ? 3 : 1, recovered.get("destination").orElseThrow(),
+ "prefix " + length);
+ assertTrue(recovered.get("deleted").isEmpty());
+ recovered.putAllAtomically(Map.of("later", 7));
+ Map<String, Integer> replayed = WalStorage.readContents(replaySnapshot, replayLog, TYPE);
+ assertEquals(7, replayed.get("later"));
+ assertEquals(4, replayed.get("source") + replayed.get("destination"));
+ } finally {
+ recovered.shutdown();
+ }
+ }
+ } finally {
+ store.shutdown();
+ }
+ }
+
+ @Test
+ void walAppendFailureDoesNotPublishAndCanRetry() throws Exception {
+ Path log = directory.resolve("data.wal");
+ WalStorage<String, Integer> store = new WalStorage<>(directory.resolve("data.json"), log, TYPE, Long.MAX_VALUE);
+ store.load();
+ try {
+ store.putAllAtomically(Map.of("source", 3, "destination", 1));
+ Path saved = directory.resolve("saved.wal");
+ Files.move(log, saved);
+ Files.createDirectory(log);
+ assertThrows(UncheckedIOException.class,
+ () -> store.putAllAtomically(Map.of("source", 1, "destination", 3)));
+ assertEquals(3, store.get("source").orElseThrow());
+ assertEquals(1, store.get("destination").orElseThrow());
+ Files.delete(log);
+ Files.move(saved, log);
+ store.putAllAtomically(Map.of("source", 1, "destination", 3));
+ assertEquals(Map.of("source", 1, "destination", 3),
+ WalStorage.readContents(directory.resolve("data.json"), log, TYPE));
+ } finally {
+ store.shutdown();
+ }
+ }
+
+ private StorageBackend<String, Integer> backend(String mode) {
+ return switch (mode) {
+ case "persistent" -> new PersistentStorage<>(directory.resolve("data.json"), TYPE);
+ case "hybrid" -> new HybridStorage<>(directory.resolve("data.json"), TYPE, Long.MAX_VALUE);
+ case "wal" -> new WalStorage<>(directory.resolve("data.json"), directory.resolve("data.wal"),
+ TYPE, Long.MAX_VALUE);
+ default -> new InMemoryStorage<>();
+ };
+ }
+
+ private void close(StorageBackend<String, Integer> backend) {
+ if (backend instanceof WalStorage<String, Integer> wal) {
+ wal.shutdown();
+ } else if (backend instanceof HybridStorage<String, Integer> hybrid) {
+ hybrid.shutdown();
+ }
+ }
+}