Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(dynamodb): journal per-table items one record per item in wal mode (#6310)

Ben Wart4c49f23c805a
diff --git a/docs/services/dynamodb.md b/docs/services/dynamodb.md
index ef2a88516c..7e31d8cf26 100644
--- a/docs/services/dynamodb.md
+++ b/docs/services/dynamodb.md
@@ -258,9 +258,9 @@ With the default `single-file` layout, every table's items live in one `dynamodb
|---|---|
| `persistent` | Rewrites the one file that holds the item |
| `hybrid` | Marks that file dirty; dirty files are rewritten on the flush interval |
-| `wal` | Not supported yet: the items stay in the `single-file` layout, and a warning is logged at startup |
+| `wal` | Appends one record for the item to `dynamodb-items/journal.wal`; dirty files are rewritten and the journal emptied on `FLOCI_STORAGE_WAL_COMPACTION_INTERVAL_MS` |
-On first start with `per-table`, an existing `dynamodb-items.json` or a `wal`-mode `dynamodb-items-snapshot.json` and `dynamodb-items.wal` is migrated into the directory, and those files are renamed with a `.migrated` suffix. When both kinds exist, the one written last is migrated. The migration is written to a staging directory and moved in place in one step, so a crash leaves either no directory or a complete one. If it cannot be written, Floci fails to start and the old files stay untouched. Once the directory holds data, old files are ignored. Switching back to `single-file` does not read the directory. A write that cannot be saved where its mode promises fails with an error, instead of only being logged: in `persistent` mode the shard file must be written before the write returns. The change stays in memory and is written by the next flush. Changing the shard count rewrites the files on the next start. The other DynamoDB stores (tables, backups, point-in-time recovery history, exports, imports) keep their single files.
+On first start with `per-table`, in any storage mode, an existing `dynamodb-items.json` or a `wal`-mode `dynamodb-items-snapshot.json` and `dynamodb-items.wal` is migrated into the directory, and those files are renamed with a `.migrated` suffix. When both kinds exist, the one written last is migrated. The migration is written to a staging directory and moved in place in one step, so a crash leaves either no directory or a complete one. If it cannot be written, Floci fails to start and the old files stay untouched. Once the directory holds data, old files are ignored. Switching back to `single-file` does not read the directory. A write that cannot be saved where its mode promises fails with an error, instead of only being logged: in `persistent` mode the shard file must be written, and in `wal` mode a change the journal cannot take is written to its shard file at once. The change stays in memory and is written by the next flush. Changing the shard count rewrites the files on the next start. Starting `per-table` in `persistent` or `hybrid` mode after a `wal` run that ended without compacting writes the changes left in `journal.wal` into the shard files and removes it. If `journal.wal` cannot be read, Floci fails to start and leaves it untouched. The other DynamoDB stores (tables, backups, point-in-time recovery history, exports, imports) keep their single files.
## Examples
diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/EntryJournal.java b/src/main/java/io/github/hectorvent/floci/core/storage/EntryJournal.java
new file mode 100644
index 0000000000..989729d7cd
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/core/storage/EntryJournal.java
@@ -0,0 +1,336 @@
+package io.github.hectorvent.floci.core.storage;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JavaType;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.dataformat.cbor.CBORFactory;
+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;
+import org.jboss.logging.Logger;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.channels.FileChannel;
+import java.nio.charset.CharacterCodingException;
+import java.nio.charset.CodingErrorAction;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.util.Map;
+import java.util.function.Consumer;
+
+/**
+ * The append-only journal of a {@link PartitionedFileStorage} in journal mode: one record per
+ * changed map entry, so a change costs the size of that entry, not of the value it belongs to.
+ *
+ * <p>Record format, every string UTF-8 and every value Jackson CBOR, each length-prefixed with a
+ * 4-byte length:
+ * <pre>
+ * PUT_ENTRY: [0x01] [key] [entry key] [entry value]
+ * REMOVE_ENTRY: [0x02] [key] [entry key]
+ * PUT: [0x03] [key] [whole map]
+ * DELETE: [0x04] [key]
+ * </pre>
+ *
+ * <p>Replay stops at the first record that cannot be framed or decoded (a tail torn by a crash)
+ * and cuts the file there, so records appended afterwards are not hidden behind the garbage.
+ *
+ * @param <E> the type of one map entry's value
+ */
+final class EntryJournal<E> {
+
+ private static final Logger LOG = Logger.getLogger(EntryJournal.class);
+
+ static final byte OP_PUT_ENTRY = 0x01;
+ static final byte OP_REMOVE_ENTRY = 0x02;
+ static final byte OP_PUT = 0x03;
+ static final byte OP_DELETE = 0x04;
+
+ /** One replayed change. */
+ sealed interface Change<E> {
+ String key();
+ }
+
+ record PutEntry<E>(String key, String entryKey, E value) implements Change<E> {}
+
+ record RemoveEntry<E>(String key, String entryKey) implements Change<E> {}
+
+ record Put<E>(String key, Map<String, E> value) implements Change<E> {}
+
+ record Delete<E>(String key) implements Change<E> {}
+
+ private final Path path;
+ private final ObjectMapper mapper;
+ private final JavaType entryType;
+ private final JavaType entriesType;
+ private volatile DataOutputStream writer;
+ // Runs just before a record is written to the file. Tests use it to make a write fail midway.
+ private volatile WriteHook beforeWrite = file -> { };
+
+ /** A step run against the journal file that may fail like a write to it. */
+ interface WriteHook {
+ void run(Path file) throws IOException;
+ }
+
+ EntryJournal(Path path, JavaType entryType, JavaType entriesType) {
+ this.path = path;
+ this.entryType = entryType;
+ this.entriesType = entriesType;
+ this.mapper = new ObjectMapper(new CBORFactory());
+ this.mapper.registerModule(new JavaTimeModule());
+ }
+
+ // Each append returns whether its record reached the journal file.
+
+ boolean putEntry(String key, String entryKey, E value) {
+ return append(OP_PUT_ENTRY, key, entryKey, value);
+ }
+
+ boolean removeEntry(String key, String entryKey) {
+ return append(OP_REMOVE_ENTRY, key, entryKey, null);
+ }
+
+ boolean put(String key, Map<String, E> value) {
+ return append(OP_PUT, key, null, value);
+ }
+
+ boolean delete(String key) {
+ return append(OP_DELETE, key, null, null);
+ }
+
+ /** Whether the journal holds any record. */
+ boolean isEmpty() {
+ try {
+ return !Files.exists(path) || Files.size(path) == 0;
+ } catch (IOException e) {
+ LOG.warnv(e, "Failed to read the size of {0}", path);
+ return false;
+ }
+ }
+
+ /**
+ * Applies every readable record in order and returns how many were applied. A torn or
+ * undecodable tail ends replay and is cut off.
+ *
+ * @throws IOException when the file could not be read to its end: records past the failure
+ * were not applied and are kept, so the caller must not empty the journal
+ */
+ int replay(Consumer<Change<E>> apply) throws IOException {
+ if (!Files.exists(path)) {
+ return 0;
+ }
+ if (Files.isDirectory(path)) {
+ // Holds no records; the first append fails and compaction replaces it with a file.
+ LOG.warnv("{0} is a directory, not a journal", path);
+ return 0;
+ }
+ long fileSize = Files.size(path);
+ int replayed = 0;
+ long consumed = 0;
+ try (DataInputStream in = new DataInputStream(new BufferedInputStream(Files.newInputStream(path)))) {
+ while (consumed < fileSize) {
+ long[] position = {consumed + 1};
+ byte op = in.readByte();
+ String key = readString(in, fileSize, position);
+ Change<E> change = switch (op) {
+ case OP_PUT_ENTRY -> {
+ String entryKey = readString(in, fileSize, position);
+ byte[] value = readFrame(in, fileSize, position);
+ E decoded = value == null ? null : mapper.readValue(value, entryType);
+ yield decoded == null || entryKey == null || key == null
+ ? null : new PutEntry<>(key, entryKey, decoded);
+ }
+ case OP_REMOVE_ENTRY -> {
+ String entryKey = readString(in, fileSize, position);
+ yield entryKey == null || key == null ? null : new RemoveEntry<>(key, entryKey);
+ }
+ case OP_PUT -> {
+ byte[] value = readFrame(in, fileSize, position);
+ Map<String, E> decoded = value == null ? null : mapper.readValue(value, entriesType);
+ // A null map, or one holding a null key or value, cannot be stored: the
+ // record is as unreadable as a torn one.
+ yield decoded == null || key == null || hasNull(decoded) ? null : new Put<>(key, decoded);
+ }
+ case OP_DELETE -> key == null ? null : new Delete<>(key);
+ default -> null;
+ };
+ if (change == null) {
+ LOG.warnv("Unreadable record at offset {0} of {1}, stopping replay", consumed, path);
+ break;
+ }
+ apply.accept(change);
+ consumed = position[0];
+ replayed++;
+ }
+ } catch (EOFException | JsonProcessingException e) {
+ LOG.warnv("{0} is unreadable from offset {1}: {2}", path, consumed, e.getMessage());
+ }
+ if (consumed < fileSize) {
+ cutTail(consumed, fileSize);
+ }
+ return replayed;
+ }
+
+ void open() {
+ close();
+ try {
+ Files.createDirectories(path.getParent());
+ writer = new DataOutputStream(new BufferedOutputStream(Files.newOutputStream(path,
+ StandardOpenOption.CREATE, StandardOpenOption.APPEND)));
+ } catch (IOException e) {
+ LOG.errorv(e, "Failed to open journal {0}", path);
+ }
+ }
+
+ void close() {
+ DataOutputStream out = writer;
+ writer = null;
+ if (out != null) {
+ try {
+ out.close();
+ } catch (IOException e) {
+ LOG.errorv(e, "Failed to close journal {0}", path);
+ }
+ }
+ }
+
+ /** Drops every record; the caller holds off appends while it does. */
+ void truncate() throws IOException {
+ close();
+ try {
+ Files.deleteIfExists(path);
+ } finally {
+ // Reopen even when the delete failed, so later records are still appended.
+ open();
+ }
+ }
+
+ void deleteFile() {
+ close();
+ try {
+ Files.deleteIfExists(path);
+ } catch (IOException e) {
+ LOG.errorv(e, "Failed to delete journal {0}", path);
+ }
+ }
+
+ void setBeforeWrite(WriteHook hook) {
+ this.beforeWrite = hook;
+ }
+
+ private boolean append(byte op, String key, String entryKey, Object value) {
+ DataOutputStream out = writer;
+ if (out == null) {
+ LOG.errorv("Journal {0} is not open; a change to {1} was not journaled", path, key);
+ return false;
+ }
+ try {
+ // Encode off the shared writer so the lock is held for one write.
+ ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ DataOutputStream record = new DataOutputStream(bytes);
+ record.writeByte(op);
+ writeFrame(record, key.getBytes(StandardCharsets.UTF_8));
+ if (entryKey != null) {
+ writeFrame(record, entryKey.getBytes(StandardCharsets.UTF_8));
+ }
+ if (op == OP_PUT_ENTRY || op == OP_PUT) {
+ writeFrame(record, mapper.writeValueAsBytes(value));
+ }
+ record.flush();
+ synchronized (out) {
+ if (writer != out) {
+ LOG.errorv("Journal {0} was closed; a change to {1} was not journaled", path, key);
+ return false;
+ }
+ try {
+ beforeWrite.run(path);
+ bytes.writeTo(out);
+ out.flush();
+ } catch (IOException e) {
+ // Part of the record may be on disk. A later record appended behind it would be
+ // cut off with it on replay, so stop appending until compaction replaces the file.
+ writer = null;
+ closeQuietly(out);
+ throw e;
+ }
+ }
+ return true;
+ } catch (IOException e) {
+ LOG.errorv(e, "Failed to append to journal {0}; it takes no more records until it is compacted", path);
+ return false;
+ }
+ }
+
+ private void closeQuietly(DataOutputStream out) {
+ try {
+ out.close();
+ } catch (IOException e) {
+ LOG.warnv(e, "Failed to close journal {0} after a failed append", path);
+ }
+ }
+
+ private static void writeFrame(DataOutputStream out, byte[] bytes) throws IOException {
+ out.writeInt(bytes.length);
+ out.write(bytes);
+ }
+
+ private static boolean hasNull(Map<?, ?> map) {
+ for (Map.Entry<?, ?> entry : map.entrySet()) {
+ if (entry.getKey() == null || entry.getValue() == null) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ /** Reads one frame as strict UTF-8, or {@code null} when it is torn or not valid UTF-8. */
+ private static String readString(DataInputStream in, long fileSize, long[] position) throws IOException {
+ byte[] bytes = readFrame(in, fileSize, position);
+ if (bytes == null) {
+ return null;
+ }
+ try {
+ // A lenient decode would swap bad bytes for U+FFFD and apply the change to another key.
+ return StandardCharsets.UTF_8.newDecoder()
+ .onMalformedInput(CodingErrorAction.REPORT)
+ .onUnmappableCharacter(CodingErrorAction.REPORT)
+ .decode(ByteBuffer.wrap(bytes))
+ .toString();
+ } catch (CharacterCodingException e) {
+ return null;
+ }
+ }
+
+ /** Reads one length-prefixed frame, or {@code null} when its length runs past the file. */
+ private static byte[] readFrame(DataInputStream in, long fileSize, long[] position) throws IOException {
+ if (fileSize - position[0] < 4) {
+ return null;
+ }
+ int length = in.readInt();
+ if (length < 0 || length > fileSize - position[0] - 4) {
+ return null;
+ }
+ byte[] bytes = in.readNBytes(length);
+ if (bytes.length != length) {
+ return null;
+ }
+ position[0] += 4 + length;
+ return bytes;
+ }
+
+ private void cutTail(long lastGoodOffset, long fileSize) {
+ try (FileChannel channel = FileChannel.open(path, StandardOpenOption.WRITE)) {
+ channel.truncate(lastGoodOffset);
+ LOG.warnv("Cut {0} unreadable bytes from the end of {1} at offset {2}; the records in them are lost",
+ fileSize - lastGoodOffset, path, lastGoodOffset);
+ } catch (IOException e) {
+ LOG.errorv(e, "Failed to cut the unreadable tail of {0}", path);
+ }
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/PartitionedFileStorage.java b/src/main/java/io/github/hectorvent/floci/core/storage/PartitionedFileStorage.java
index 921cd8d533..e88db18854 100644
--- a/src/main/java/io/github/hectorvent/floci/core/storage/PartitionedFileStorage.java
+++ b/src/main/java/io/github/hectorvent/floci/core/storage/PartitionedFileStorage.java
@@ -31,7 +31,9 @@ import java.util.Optional;
import java.util.Set;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.locks.ReentrantLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import java.util.function.BooleanSupplier;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@@ -52,7 +54,12 @@ import java.util.stream.Stream;
*
* <p>A {@link Mode#SYNC} store writes a shard's file before the mutating call returns, like
* {@link PersistentStorage}. A {@link Mode#PERIODIC} store marks the shard dirty and writes every
- * dirty shard on the flush interval, like {@link HybridStorage}.
+ * dirty shard on the flush interval, like {@link HybridStorage}. A {@link Mode#JOURNAL} store
+ * appends one record per changed entry to {@value #JOURNAL_FILE} in the directory, like
+ * {@link WalStorage}, but its compaction rewrites only the shards changed since the last one
+ * instead of a snapshot of the whole store. Load replays the journal over the shard files. A
+ * store in another mode that finds a journal left by a journal-mode run writes its changes into
+ * the shard files on load and removes it.
*
* <p>While the directory holds no data, a legacy single-file store (the {@code Map<String,
* Map<String, E>>} file the other backends write, or a {@code wal} snapshot and log) is migrated:
@@ -66,6 +73,7 @@ public class PartitionedFileStorage<E> implements StorageBackend<String, Map<Str
private static final Logger LOG = Logger.getLogger(PartitionedFileStorage.class);
static final String FILE_SUFFIX = ".json";
+ static final String JOURNAL_FILE = "journal.wal";
static final String MIGRATING_SUFFIX = ".migrating";
private static final String KEY_FIELD = "key";
private static final String ENTRIES_FIELD = "entries";
@@ -76,7 +84,12 @@ public class PartitionedFileStorage<E> implements StorageBackend<String, Map<Str
/** Its shard file is rewritten before the mutating call returns. */
SYNC,
/** Dirty shard files are rewritten on the flush interval. */
- PERIODIC
+ PERIODIC,
+ /**
+ * One record per changed entry is appended to a journal before the mutating call returns;
+ * dirty shard files are rewritten and the journal emptied on the compaction interval.
+ */
+ JOURNAL
}
/** One shard of one key: the unit that is written to a file. */
@@ -116,16 +129,32 @@ public class PartitionedFileStorage<E> implements StorageBackend<String, Map<Str
private final ObjectReader entriesReader;
private final ObjectWriter entriesWriter;
private final ScheduledExecutorService scheduler;
- // Shard writes hold the read lock, and load and clear hold the write lock, so a write that
- // read a shard before a clear cannot put its file back afterwards, and the flush schedule,
- // which starts with the constructor, cannot write while load is filling the store.
- private final ReentrantReadWriteLock fileLock = new ReentrantReadWriteLock();
+ private final EntryJournal<E> journal;
+ // Outside journal mode: the journal a crashed journal-mode run may have left, read once on load.
+ private final EntryJournal<E> leftoverJournal;
+ // Journal mode: changes hold the read lock while they update memory and append their record,
+ // and compaction holds the write lock while it writes dirty shards and truncates the journal,
+ // so the journal never drops a record whose change is not yet in a shard file. In every mode,
+ // shard writes hold the read lock and load and clear hold the write lock.
+ private final ReentrantReadWriteLock compactionLock = new ReentrantReadWriteLock();
+ // Journal mode: a change holds its key's lock while it updates memory and appends its record,
+ // so two changes to one key are journaled in the order they were applied. Replay follows the
+ // journal's order; changes to different keys commute, so they need no common order.
+ private final ConcurrentHashMap<String, ReentrantLock> changeLocks = new ConcurrentHashMap<>();
+ // Runs inside a journaled change after memory was updated and before its record is appended:
+ // the moment a second change to the same key could overtake it. Tests use it to stage that.
+ private volatile Runnable beforeJournalAppend = () -> { };
+ // Runs inside compaction between the last shard write and emptying the journal: the moment a
+ // change that slipped past the lock would be lost. Tests use it to stage that change.
+ private volatile Runnable beforeJournalTruncate = () -> { };
// Runs inside a shard write after the shard was read and before its file is replaced: the
// moment a clear() that slipped past the lock would be undone. Tests use it to stage that.
private volatile Runnable beforeShardWrite = () -> { };
// Runs on load after superseded files were rewritten under their current names and before
// the old files are removed. Tests use it to make only the removal fail.
private volatile Runnable beforeSupersededDelete = () -> { };
+ // Set once load() has replayed the journal; compaction waits for it.
+ private volatile boolean loaded;
/**
* @param directory where the shard files live
@@ -133,7 +162,8 @@ public class PartitionedFileStorage<E> implements StorageBackend<String, Map<Str
* @param entryType the type of one map entry's value
* @param shardCount how many files each key is split into, at least one
* @param mode when a change reaches disk
- * @param flushIntervalMs the flush interval of a {@link Mode#PERIODIC} store
+ * @param flushIntervalMs the flush interval of a {@link Mode#PERIODIC} store, or the
+ * compaction interval of a {@link Mode#JOURNAL} store
*/
public PartitionedFileStorage(Path directory, Path legacyFile, Class<E> entryType, int shardCount,
Mode mode, long flushIntervalMs) {
@@ -159,6 +189,10 @@ public class PartitionedFileStorage<E> implements StorageBackend<String, Map<Str
JavaType entriesType = mapper.getTypeFactory().constructMapType(Map.class, String.class, entryType);
this.entriesReader = mapper.readerFor(entriesType);
this.entriesWriter = mapper.writerFor(entriesType);
+ EntryJournal<E> journalFile = new EntryJournal<>(directory.resolve(JOURNAL_FILE),
+ mapper.constructType(entryType), entriesType);
+ this.journal = mode == Mode.JOURNAL ? journalFile : null;
+ this.leftoverJournal = mode == Mode.JOURNAL ? null : journalFile;
this.scheduler = mode == Mode.SYNC
? null
: StorageFiles.scheduleEvery("partitioned-storage-flush", flushIntervalMs, this::flushQuietly);
@@ -168,8 +202,15 @@ public class PartitionedFileStorage<E> implements StorageBackend<String, Map<Str
@Override
public void put(String key, Map<String, E> value) {
- applyPut(key, value);
- persistChange(allShardsOf(key));
+ boolean journaled;
+ ReentrantLock lock = beginChange(key);
+ try {
+ applyPut(key, value);
+ journaled = journal == null || journalAppend(() -> journal.put(key, value));
+ } finally {
+ endChange(lock);
+ }
+ persistChange(journaled, allShardsOf(key));
}
@Override
@@ -186,8 +227,15 @@ public class PartitionedFileStorage<E> implements StorageBackend<String, Map<Str
@Override
public void delete(String key) {
- applyDelete(key);
- persistChange(allShardsOf(key));
+ boolean journaled;
+ ReentrantLock lock = beginChange(key);
+ try {
+ applyDelete(key);
+ journaled = journal == null || journalAppend(() -> journal.delete(key));
+ } finally {
+ endChange(lock);
+ }
+ persistChange(journaled, allShardsOf(key));
}
@Override
@@ -204,14 +252,14 @@ public class PartitionedFileStorage<E> implements StorageBackend<String, Map<Str
}
/**
- * Writes every dirty shard.
+ * Writes every dirty shard; in journal mode this is a compaction.
*
* @throws UncheckedIOException when a shard could not be written; it stays dirty and is
* written again by the next flush
*/
@Override
public void flush() {
- int failed = writeDirtyShards();
+ int failed = journal != null ? compact() : writeDirtyShards();
if (failed > 0) {
throw new UncheckedIOException(new IOException(
"Could not write " + failed + " shards of " + directory + "; see the errors logged above"));
@@ -220,26 +268,79 @@ public class PartitionedFileStorage<E> implements StorageBackend<String, Map<Str
@Override
public synchronized void load() {
- // The flush schedule starts with the constructor: hold it off while the store is filled.
- fileLock.writeLock().lock();
+ // The flush schedule starts with the constructor: hold compaction off until the journal
+ // has been replayed, or it could empty a journal whose records are not yet in memory.
+ compactionLock.writeLock().lock();
try {
- store.clear();
- dirty.clear();
- migrateLegacyFileIfNeeded();
- if (Files.isDirectory(directory)) {
- loadDirectory();
- }
+ loadLocked();
} finally {
- fileLock.writeLock().unlock();
+ compactionLock.writeLock().unlock();
+ }
+ }
+
+ private void loadLocked() {
+ loaded = false;
+ store.clear();
+ dirty.clear();
+ migrateLegacyFileIfNeeded();
+ if (Files.isDirectory(directory)) {
+ loadDirectory();
+ }
+ if (journal != null) {
+ replayJournal(journal);
+ journal.open();
+ } else if (!leftoverJournal.isEmpty()) {
+ foldLeftoverJournal();
+ }
+ loaded = true;
+ }
+
+ private void replayJournal(EntryJournal<E> source) {
+ int replayed;
+ try {
+ replayed = source.replay(this::applyReplayed);
+ } catch (IOException e) {
+ // Compaction would write the partial state and empty the journal, losing the
+ // records not yet read: refuse to start instead, as a failed migration does.
+ throw new UncheckedIOException("Failed to replay the journal of " + directory
+ + "; it is untouched, fix the cause and restart", e);
+ }
+ if (replayed > 0) {
+ LOG.infov("Replayed {0} journal records into {1}", replayed, directory);
+ }
+ }
+
+ /**
+ * Folds the journal of a journal-mode run that ended without compacting into the shard files,
+ * then removes it. Ignored, its changes would be lost now and replayed as stale ones over newer
+ * shard files once the store is back in journal mode.
+ */
+ private void foldLeftoverJournal() {
+ Path file = directory.resolve(JOURNAL_FILE);
+ LOG.warnv("Found {0}, left by a wal-mode run; writing its changes into the shard files", file);
+ replayJournal(leftoverJournal);
+ writeDirtyShards();
+ if (!dirty.isEmpty()) {
+ throw new IllegalStateException("Could not write " + dirty.size() + " shards of " + directory
+ + " from " + file + "; it is kept, fix the cause and restart");
+ }
+ try {
+ Files.deleteIfExists(file);
+ } catch (IOException e) {
+ throw new UncheckedIOException("Failed to remove " + file
+ + "; its changes are in the shard files, remove it and restart", e);
}
}
@Override
public synchronized void clear() {
- fileLock.writeLock().lock();
+ compactionLock.writeLock().lock();
try {
store.clear();
dirty.clear();
+ if (journal != null) {
+ journal.deleteFile();
+ }
if (Files.isDirectory(directory)) {
try (DirectoryStream<Path> files = Files.newDirectoryStream(directory, "*" + FILE_SUFFIX)) {
for (Path file : files) {
@@ -249,8 +350,11 @@ public class PartitionedFileStorage<E> implements StorageBackend<String, Map<Str
LOG.errorv(e, "Failed to clear {0}", directory);
}
}
+ if (journal != null) {
+ journal.open();
+ }
} finally {
- fileLock.writeLock().unlock();
+ compactionLock.writeLock().unlock();
}
}
@@ -258,43 +362,70 @@ public class PartitionedFileStorage<E> implements StorageBackend<String, Map<Str
@Override
public void putEntry(String key, String entryKey, E entryValue) {
- persistChange(List.of(applyPutEntry(key, entryKey, entryValue)));
+ ShardId id;
+ boolean journaled;
+ ReentrantLock lock = beginChange(key);
+ try {
+ id = applyPutEntry(key, entryKey, entryValue);
+ journaled = journal == null || journalAppend(() -> journal.putEntry(key, entryKey, entryValue));
+ } finally {
+ endChange(lock);
+ }
+ persistChange(journaled, List.of(id));
}
@Override
public void removeEntry(String key, String entryKey) {
- ShardId id = applyRemoveEntry(key, entryKey);
+ ShardId id;
+ boolean journaled;
+ ReentrantLock lock = beginChange(key);
+ try {
+ id = applyRemoveEntry(key, entryKey);
+ journaled = id == null || journal == null || journalAppend(() -> journal.removeEntry(key, entryKey));
+ } finally {
+ endChange(lock);
+ }
if (id != null) {
- persistChange(List.of(id));
+ persistChange(journaled, List.of(id));
}
}
// ---
- /** Stops the flush schedule and writes what is still dirty. */
+ /** Stops the flush schedule and writes what is still dirty; a journal is compacted and closed. */
public void shutdown() {
if (scheduler != null) {
StorageFiles.stop(scheduler);
}
- flush();
+ try {
+ flush();
+ } finally {
+ if (journal != null) {
+ journal.close();
+ }
+ }
}
/**
- * Writes the changed shards of a sync store before the call returns; a periodic store leaves
- * them to the next flush.
+ * Makes a change that is already in memory durable where its mode promises it: a sync store
+ * writes the changed shards now, and a journal store whose record could not be appended
+ * compacts now instead, so the change still reaches a shard file before the call returns.
+ * (Writing only the changed shard would put it ahead of older records still in the journal,
+ * which a replay would then apply over it.) A periodic store leaves it to the next flush.
*
- * @throws UncheckedIOException when a shard could not be written; the change stays in memory
+ * @throws UncheckedIOException when the change could not be made durable; it stays in memory
* and dirty, so a later flush writes it
*/
- private void persistChange(List<ShardId> changed) {
- if (mode != Mode.SYNC) {
- return;
- }
+ private void persistChange(boolean journaled, List<ShardId> changed) {
int failed = 0;
- for (ShardId id : changed) {
- if (!writeShard(id)) {
- failed++;
+ if (mode == Mode.SYNC) {
+ for (ShardId id : changed) {
+ if (!writeShard(id)) {
+ failed++;
+ }
}
+ } else if (!journaled) {
+ failed = compact();
}
if (failed > 0) {
throw new UncheckedIOException(new IOException(
@@ -303,7 +434,46 @@ public class PartitionedFileStorage<E> implements StorageBackend<String, Map<Str
}
}
- // The apply methods change memory and mark what they change dirty.
+ /**
+ * Writes every dirty shard and then empties the journal. A shard that fails to write keeps
+ * the journal intact, so the next load replays what that shard file is missing.
+ *
+ * @return how many shards could not be written, plus one when the journal could not be emptied
+ */
+ private int compact() {
+ compactionLock.writeLock().lock();
+ try {
+ // Before the first load the journal still holds the last run's records, unreplayed.
+ if (!loaded || (dirty.isEmpty() && journal.isEmpty())) {
+ return 0;
+ }
+ int failed = writeDirtyShards();
+ if (failed > 0) {
+ LOG.errorv("Could not write {0} shards of {1}; keeping the journal", failed, directory);
+ return failed;
+ }
+ beforeJournalTruncate.run();
+ journal.truncate();
+ return 0;
+ } catch (IOException e) {
+ LOG.errorv(e, "Failed to truncate the journal of {0}", directory);
+ return 1;
+ } finally {
+ compactionLock.writeLock().unlock();
+ }
+ }
+
+ private void applyReplayed(EntryJournal.Change<E> change) {
+ switch (change) {
+ case EntryJournal.PutEntry<E> put -> applyPutEntry(put.key(), put.entryKey(), put.value());
+ case EntryJournal.RemoveEntry<E> remove -> applyRemoveEntry(remove.key(), remove.entryKey());
+ case EntryJournal.Put<E> put -> applyPut(put.key(), put.value());
+ case EntryJournal.Delete<E> delete -> applyDelete(delete.key());
+ }
+ }
+
+ // The apply methods change memory and mark what they change dirty. Live changes call them
+ // under the change lock and then journal the change; replay calls them for journal records.
private ShardId applyPutEntry(String key, String entryKey, E entryValue) {
ShardId id = new ShardId(key, shardOf(entryKey));
@@ -333,6 +503,29 @@ public class PartitionedFileStorage<E> implements StorageBackend<String, Map<Str
markAll(key);
}
+ /** Journal mode: takes the change side of the compaction lock, then the key's lock, and returns the latter. */
+ private ReentrantLock beginChange(String key) {
+ if (journal == null) {
+ return null;
+ }
+ compactionLock.readLock().lock();
+ ReentrantLock lock = changeLocks.computeIfAbsent(key, ignored -> new ReentrantLock());
+ lock.lock();
+ return lock;
+ }
+
+ private void endChange(ReentrantLock lock) {
+ if (lock != null) {
+ lock.unlock();
+ compactionLock.readLock().unlock();
+ }
+ }
+
+ private boolean journalAppend(BooleanSupplier append) {
+ beforeJournalAppend.run();
+ return append.getAsBoolean();
+ }
+
/** Writes every dirty shard and returns how many could not be written. */
private int writeDirtyShards() {
int failed = 0;
@@ -344,6 +537,18 @@ public class PartitionedFileStorage<E> implements StorageBackend<String, Map<Str
return failed;
}
+ void setBeforeJournalAppend(Runnable hook) {
+ this.beforeJournalAppend = hook;
+ }
+
+ void setBeforeJournalWrite(EntryJournal.WriteHook hook) {
+ journal.setBeforeWrite(hook);
+ }
+
+ void setBeforeJournalTruncate(Runnable hook) {
+ this.beforeJournalTruncate = hook;
+ }
+
void setBeforeShardWrite(Runnable hook) {
this.beforeShardWrite = hook;
}
@@ -438,13 +643,13 @@ public class PartitionedFileStorage<E> implements StorageBackend<String, Map<Str
*/
private boolean writeShard(ShardId id) {
// clear() holds the write side, so a write that read a shard before the clear cannot put
- // its file back after the clear removed it. Load already holds the write side, which lets
- // it take the read side here.
- fileLock.readLock().lock();
+ // its file back after the clear removed it. Compaction and load already hold the write
+ // side, which lets them take the read side here.
+ compactionLock.readLock().lock();
try {
return writeShardLocked(id);
} finally {
- fileLock.readLock().unlock();
+ compactionLock.readLock().unlock();
}
}
@@ -606,7 +811,8 @@ public class PartitionedFileStorage<E> implements StorageBackend<String, Map<Str
}
}
if (Files.isDirectory(directory)) {
- // Empty (holdsData said so): make way for the move.
+ // Empty apart from an empty journal (holdsData said so): make way for the move.
+ Files.deleteIfExists(directory.resolve(JOURNAL_FILE));
Files.delete(directory);
}
Files.move(staging, directory, StandardCopyOption.ATOMIC_MOVE);
@@ -629,13 +835,19 @@ public class PartitionedFileStorage<E> implements StorageBackend<String, Map<Str
entries.size(), sources, directory);
}
- /** Whether {@code dir} holds anything at all. */
+ /** Whether {@code dir} holds a shard file or a journal with records in it. */
private static boolean holdsData(Path dir) {
if (!Files.isDirectory(dir)) {
return false;
}
try (DirectoryStream<Path> files = Files.newDirectoryStream(dir)) {
- return files.iterator().hasNext();
+ for (Path file : files) {
+ boolean emptyJournal = file.getFileName().toString().equals(JOURNAL_FILE) && Files.size(file) == 0;
+ if (!emptyJournal) {
+ return true;
+ }
+ }
+ return false;
} catch (IOException e) {
throw new UncheckedIOException("Failed to list " + dir, e);
}
diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/StorageFactory.java b/src/main/java/io/github/hectorvent/floci/core/storage/StorageFactory.java
index f78d17b48c..342fa690b6 100644
--- a/src/main/java/io/github/hectorvent/floci/core/storage/StorageFactory.java
+++ b/src/main/java/io/github/hectorvent/floci/core/storage/StorageFactory.java
@@ -156,9 +156,10 @@ public class StorageFactory {
* when {@link AccountAwareStorageBackend#writesEntries()} is true.
*
* <p>{@code persistent} writes a shard's file on every change and {@code hybrid} writes dirty
- * shards on the flush interval. {@code memory} keeps the store in memory, and {@code wal}
- * keeps its single-file {@link WalStorage} for now. An existing single-file store (a
- * {@code wal}-mode snapshot and log included) is migrated into the directory on first load.
+ * shards on the flush interval. {@code wal} journals one record per changed entry and writes
+ * dirty shards on the WAL compaction interval. {@code memory} keeps the store in memory. An
+ * existing single-file store (a {@code wal}-mode snapshot and log included) is migrated into
+ * the directory on first load.
*/
public synchronized <E> AccountAwareStorageBackend<Map<String, E>> createPartitioned(
String serviceName, String fileName, TypeReference<Map<String, Map<String, E>>> typeReference,
@@ -167,11 +168,6 @@ public class StorageFactory {
if ("memory".equals(mode)) {
return create(serviceName, fileName, typeReference);
}
- if ("wal".equals(mode)) {
- LOG.warnv("The per-table layout does not support wal mode yet; storing {0} for service {1} "
- + "in the single-file layout", fileName, serviceName);
- return create(serviceName, fileName, typeReference);
- }
Path basePath = Path.of(config.storage().persistentPath());
Path directory = basePath.resolve(fileName.replace(".json", ""));
StorageBackend<?, ?> existing = backendsByPath.get(directory);
@@ -191,6 +187,10 @@ public class StorageFactory {
writeMode = PartitionedFileStorage.Mode.PERIODIC;
interval = resolveFlushInterval(serviceName);
}
+ case "wal" -> {
+ writeMode = PartitionedFileStorage.Mode.JOURNAL;
+ interval = config.storage().wal().compactionIntervalMs();
+ }
default -> throw new IllegalArgumentException("Unknown storage mode: " + mode);
}
PartitionedFileStorage<E> inner = new PartitionedFileStorage<>(directory,
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index dae05ffb4d..b61269f03a 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -211,8 +211,9 @@ floci:
flush-interval-ms: 5000
# single-file: every table in dynamodb-items.json, rewritten on any item change.
# per-table: files per table under dynamodb-items/, rewritten only where an item changed
- # (persistent: on the write, hybrid: on the flush interval; wal keeps single-file for
- # now). An existing single-file store is migrated on first start.
+ # (persistent: on the write, hybrid: on the flush interval; wal: each write appends one
+ # record to dynamodb-items/journal.wal and changed files are rewritten on the WAL
+ # compaction interval). An existing single-file store is migrated on first start.
item-layout: single-file # FLOCI_STORAGE_SERVICES_DYNAMODB_ITEM_LAYOUT
# per-table only: files per table. A write rewrites only its item's file; changing the
# count rewrites the files on the next start.
diff --git a/src/test/java/io/github/hectorvent/floci/core/storage/PartitionedFileStorageJournalTest.java b/src/test/java/io/github/hectorvent/floci/core/storage/PartitionedFileStorageJournalTest.java
new file mode 100644
index 0000000000..50a735ff7e
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/core/storage/PartitionedFileStorageJournalTest.java
@@ -0,0 +1,466 @@
+package io.github.hectorvent.floci.core.storage;
+
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.io.UncheckedIOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardOpenOption;
+import java.nio.file.attribute.FileTime;
+import java.nio.file.attribute.PosixFilePermission;
+import java.nio.file.attribute.PosixFilePermissions;
+import java.time.Instant;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Stream;
+
+import static org.junit.jupiter.api.Assertions.*;
+import static org.junit.jupiter.api.Assumptions.assumeFalse;
+
+/** {@link PartitionedFileStorage} in {@link PartitionedFileStorage.Mode#JOURNAL} mode. */
+class PartitionedFileStorageJournalTest {
+
+ private static final FileTime LONG_AGO = FileTime.from(Instant.parse("2020-01-01T00:00:00Z"));
+
+ @TempDir
+ Path tempDir;
+
+ private final List<PartitionedFileStorage<String>> opened = new ArrayList<>();
+
+ @AfterEach
+ void tearDown() {
+ opened.forEach(PartitionedFileStorage::shutdown);
+ }
+
+ /** Opens the store the way a restart after a crash would: whatever an earlier instance left. */
+ private PartitionedFileStorage<String> open(int shards) {
+ PartitionedFileStorage<String> storage = new PartitionedFileStorage<>(tempDir.resolve("items"),
+ tempDir.resolve("items.json"), String.class, shards, PartitionedFileStorage.Mode.JOURNAL,
+ Long.MAX_VALUE);
+ storage.load();
+ opened.add(storage);
+ return storage;
+ }
+
+ private Path journal() {
+ return tempDir.resolve("items").resolve(PartitionedFileStorage.JOURNAL_FILE);
+ }
+
+ private long shardFileCount() throws Exception {
+ try (Stream<Path> files = Files.list(tempDir.resolve("items"))) {
+ return files.filter(f -> f.getFileName().toString().endsWith(".json")).count();
+ }
+ }
+
+ @Test
+ void aWriteCostsOneRecordWhateverTheSizeOfItsKey() throws Exception {
+ PartitionedFileStorage<String> storage = open(4);
+ for (int i = 0; i < 100; i++) {
+ storage.putEntry("big", "entry-" + i, "value");
+ }
+ long before = Files.size(journal());
+ storage.putEntry("big", "entry-x", "value");
+ long costAtOneHundred = Files.size(journal()) - before;
+
+ for (int i = 100; i < 10_000; i++) {
+ storage.putEntry("big", "entry-" + i, "value");
+ }
+ before = Files.size(journal());
+ storage.putEntry("big", "entry-y", "value");
+ long costAtTenThousand = Files.size(journal()) - before;
+
+ assertTrue(costAtOneHundred > 0);
+ assertEquals(costAtOneHundred, costAtTenThousand);
+ assertEquals(0, shardFileCount(), "shard files are only written by compaction");
+ }
+
+ @Test
+ void aRestartWithoutCompactionReplaysEveryChange() {
+ PartitionedFileStorage<String> crashed = open(4);
+ crashed.put("table", Map.of("a", "1", "b", "2", "c", "3"));
+ crashed.putEntry("table", "d", "4");
+ crashed.removeEntry("table", "a");
+ crashed.putEntry("gone", "x", "1");
+ crashed.delete("gone");
+
+ PartitionedFileStorage<String> restarted = open(4);
+
+ assertEquals(Map.of("b", "2", "c", "3", "d", "4"), restarted.get("table").orElseThrow());
+ assertTrue(restarted.get("gone").isEmpty());
+ }
+
+ @Test
+ void aCompactionBeforeTheFirstLoadKeepsTheJournal() {
+ PartitionedFileStorage<String> crashed = open(1);
+ crashed.putEntry("table", "a", "1");
+
+ // The compaction schedule starts with the constructor, so it can run before load().
+ PartitionedFileStorage<String> restarting = new PartitionedFileStorage<>(tempDir.resolve("items"),
+ tempDir.resolve("items.json"), String.class, 1, PartitionedFileStorage.Mode.JOURNAL, Long.MAX_VALUE);
+ opened.add(restarting);
+ restarting.flush();
+ restarting.load();
+
+ assertEquals(Map.of("a", "1"), restarting.get("table").orElseThrow());
+ }
+
+ @Test
+ void aRecordThatDecodesToNullEndsReplayInsteadOfFailingLoad() throws Exception {
+ byte[] cborNull = {(byte) 0xF6};
+ byte[] mapWithNullValue = {(byte) 0xA1, 0x61, 'k', (byte) 0xF6};
+ List<byte[]> badRecords = List.of(
+ record(EntryJournal.OP_PUT_ENTRY, "table".getBytes(), "k".getBytes(), cborNull),
+ record(EntryJournal.OP_PUT, "table".getBytes(), cborNull),
+ record(EntryJournal.OP_PUT, "table".getBytes(), mapWithNullValue));
+ open(1).putEntry("table", "a", "1");
+
+ for (byte[] bad : badRecords) {
+ long goodLength = Files.size(journal());
+ Files.write(journal(), bad, StandardOpenOption.APPEND);
+
+ PartitionedFileStorage<String> restarted = open(1);
+
+ assertEquals(Map.of("a", "1"), restarted.get("table").orElseThrow());
+ assertEquals(goodLength, Files.size(journal()), "the unreadable record is cut off");
+ }
+ }
+
+ @Test
+ void aRecordWithAKeyThatIsNotUtf8EndsReplay() throws Exception {
+ byte[] cborX = {0x61, 'x'};
+ byte[] notUtf8 = {'t', 'a', 'b', 'l', (byte) 0xFF};
+ List<byte[]> badRecords = List.of(
+ record(EntryJournal.OP_PUT_ENTRY, notUtf8, "k".getBytes(), cborX),
+ record(EntryJournal.OP_PUT_ENTRY, "table".getBytes(), new byte[]{(byte) 0xC3, 0x28}, cborX));
+ open(1).putEntry("table", "a", "1");
+
+ for (byte[] bad : badRecords) {
+ long goodLength = Files.size(journal());
+ Files.write(journal(), bad, StandardOpenOption.APPEND);
+
+ PartitionedFileStorage<String> restarted = open(1);
+
+ assertEquals(Set.of("table"), restarted.keys());
+ assertEquals(Map.of("a", "1"), restarted.get("table").orElseThrow());
+ assertEquals(goodLength, Files.size(journal()), "the unreadable record is cut off");
+ }
+ }
+
+ /** One journal record: the op byte, then each part as a length-prefixed frame. */
+ private static byte[] record(byte op, byte[]... parts) throws IOException {
+ ByteArrayOutputStream bytes = new ByteArrayOutputStream();
+ DataOutputStream out = new DataOutputStream(bytes);
+ out.writeByte(op);
+ for (byte[] part : parts) {
+ out.writeInt(part.length);
+ out.write(part);
+ }
+ out.flush();
+ return bytes.toByteArray();
+ }
+
+ @Test
+ void compactionRewritesOnlyChangedShardsAndEmptiesTheJournal() throws Exception {
+ PartitionedFileStorage<String> storage = open(8);
+ Map<String, String> entries = new HashMap<>();
+ for (int i = 0; i < 200; i++) {
+ entries.put("entry-" + i, "value-" + i);
+ }
+ storage.put("big", entries);
+ storage.flush();
+ assertEquals(8, shardFileCount());
+ assertEquals(0, Files.size(journal()));
+ try (Stream<Path> files = Files.list(tempDir.resolve("items"))) {
+ for (Path file : files.toList()) {
+ Files.setLastModifiedTime(file, LONG_AGO);
+ }
+ }
+ Path touched = storage.fileFor("big", "entry-7");
+
+ storage.putEntry("big", "entry-7", "changed");
+ storage.flush();
+
+ try (Stream<Path> files = Files.list(tempDir.resolve("items"))) {
+ for (Path file : files.filter(f -> f.getFileName().toString().endsWith(".json")).toList()) {
+ assertEquals(file.equals(touched), !LONG_AGO.equals(Files.getLastModifiedTime(file)),
+ file.getFileName().toString());
+ }
+ }
+ assertEquals(0, Files.size(journal()));
+ entries.put("entry-7", "changed");
+ assertEquals(entries, open(8).get("big").orElseThrow());
+ }
+
+ @Test
+ void anIdleCompactionWritesNothing() throws Exception {
+ PartitionedFileStorage<String> storage = open(1);
+ storage.putEntry("table", "a", "1");
+ storage.flush();
+ Path file = storage.fileFor("table");
+ Files.setLastModifiedTime(file, LONG_AGO);
+
+ storage.flush();
+
+ assertEquals(LONG_AGO, Files.getLastModifiedTime(file));
+ }
+
+ @Test
+ void aTornTailIsCutAndLaterRecordsSurvive() throws Exception {
+ PartitionedFileStorage<String> crashed = open(1);
+ crashed.putEntry("table", "a", "1");
+ crashed.putEntry("table", "b", "2");
+ Files.write(journal(), new byte[]{0x01, 0x00, 0x00, 0x7f}, StandardOpenOption.APPEND);
+
+ PartitionedFileStorage<String> restarted = open(1);
+ assertEquals(Map.of("a", "1", "b", "2"), restarted.get("table").orElseThrow());
+ restarted.putEntry("table", "c", "3");
+
+ assertEquals(Map.of("a", "1", "b", "2", "c", "3"), open(1).get("table").orElseThrow());
+ }
+
+ @Test
+ void replayOverShardFilesAppliesOnlyTheNewerChanges() {
+ PartitionedFileStorage<String> storage = open(2);
+ storage.put("table", Map.of("a", "1", "b", "2"));
+ storage.flush();
+ storage.putEntry("table", "a", "changed");
+ storage.removeEntry("table", "b");
+
+ assertEquals(Map.of("a", "changed"), open(2).get("table").orElseThrow());
+ }
+
+ @Test
+ void clearRemovesTheJournalAndEveryFile() throws Exception {
+ PartitionedFileStorage<String> storage = open(2);
+ storage.put("table", Map.of("a", "1"));
+ storage.flush();
+ storage.putEntry("table", "b", "2");
+
+ storage.clear();
+
+ assertEquals(0, shardFileCount());
+ assertTrue(open(2).keys().isEmpty());
+ }
+
+ @Test
+ void shutdownCompacts() throws Exception {
+ PartitionedFileStorage<String> storage = open(1);
+ storage.putEntry("table", "a", "1");
+
+ storage.shutdown();
+ opened.remove(storage);
+
+ assertEquals(0, Files.size(journal()));
+ assertEquals(1, shardFileCount());
+ }
+
+ @Test
+ void aWriteRacingCompactionIsNeverLost() throws Exception {
+ PartitionedFileStorage<String> live = open(1);
+ live.putEntry("table", "early", "1");
+ Thread[] writer = new Thread[1];
+ live.setBeforeJournalTruncate(() -> {
+ // The shards are written and the journal is about to be emptied. A write now must
+ // wait for the compaction to finish, or its journal record is dropped unwritten.
+ writer[0] = Thread.ofPlatform().start(() -> live.putEntry("table", "late", "2"));
+ StorageTestThreads.awaitParkedOrDone(writer[0]);
+ });
+
+ live.flush();
+ live.setBeforeJournalTruncate(() -> { });
+ writer[0].join(TimeUnit.SECONDS.toMillis(10));
+
+ assertFalse(writer[0].isAlive());
+ assertEquals(Map.of("early", "1", "late", "2"), recoverCopy().get("table").orElseThrow());
+ }
+
+ @Test
+ void twoWritesToOneEntryReplayInTheOrderTheyWereApplied() throws Exception {
+ PartitionedFileStorage<String> live = open(1);
+ Thread[] second = new Thread[1];
+ live.setBeforeJournalAppend(() -> {
+ // The first write is in memory and about to be journaled. A second write to the same
+ // entry must wait, or its record lands first and a replay ends on the older value.
+ live.setBeforeJournalAppend(() -> { });
+ second[0] = Thread.ofPlatform().start(() -> live.putEntry("table", "a", "second"));
+ StorageTestThreads.awaitParkedOrDone(second[0]);
+ });
+
+ live.putEntry("table", "a", "first");
+ second[0].join(TimeUnit.SECONDS.toMillis(10));
+
+ assertFalse(second[0].isAlive());
+ assertEquals(Map.of("a", "second"), live.get("table").orElseThrow());
+ assertEquals(Map.of("a", "second"), recoverCopy().get("table").orElseThrow());
+ }
+
+ /**
+ * What a restart would recover right now, read from a copy of the directory so that opening
+ * it, which replays and later compacts, cannot change the files the live store is using.
+ */
+ private PartitionedFileStorage<String> recoverCopy() throws Exception {
+ Path copy = tempDir.resolve("recovered");
+ Files.createDirectories(copy);
+ try (Stream<Path> files = Files.list(tempDir.resolve("items"))) {
+ for (Path file : files.toList()) {
+ Files.copy(file, copy.resolve(file.getFileName()));
+ }
+ }
+ PartitionedFileStorage<String> recovered = new PartitionedFileStorage<>(copy, (Path) null, String.class, 1,
+ PartitionedFileStorage.Mode.JOURNAL, Long.MAX_VALUE);
+ recovered.load();
+ opened.add(recovered);
+ return recovered;
+ }
+
+ @Test
+ void aChangeTheJournalCannotTakeIsCompactedIntoItsShardAtOnce() throws Exception {
+ // An empty directory where the journal goes: it cannot be opened, so the first append
+ // fails, while the compaction that follows can remove it and start a real journal.
+ Files.createDirectories(journal());
+ PartitionedFileStorage<String> storage = open(1);
+
+ storage.putEntry("table", "a", "1");
+
+ assertTrue(Files.exists(storage.fileFor("table")), "the change reached its shard file");
+ assertTrue(Files.isRegularFile(journal()));
+ storage.putEntry("table", "b", "2");
+ assertTrue(Files.size(journal()) > 0, "later changes are journaled again");
+ assertEquals(Map.of("a", "1", "b", "2"), recoverCopy().get("table").orElseThrow());
+ }
+
+ @Test
+ void aWriteAfterATornAppendIsNeverAcknowledgedBehindIt() throws Exception {
+ PartitionedFileStorage<String> crashed = open(1);
+ crashed.putEntry("table", "a", "1");
+ // A non-empty directory where the shard file goes makes every fallback compaction fail.
+ Path blocker = crashed.fileFor("table");
+ Files.createDirectories(blocker);
+ Files.writeString(blocker.resolve("child"), "x");
+ crashed.setBeforeJournalWrite(file -> {
+ // The write fails after part of the record reached the file.
+ crashed.setBeforeJournalWrite(ignored -> { });
+ Files.write(file, new byte[]{EntryJournal.OP_PUT_ENTRY, 0x00, 0x00, 0x7f}, StandardOpenOption.APPEND);
+ throw new IOException("disk full");
+ });
+
+ assertThrows(UncheckedIOException.class, () -> crashed.putEntry("table", "b", "2"));
+ assertThrows(UncheckedIOException.class, () -> crashed.putEntry("table", "c", "3"),
+ "a record appended behind the torn one would be lost on replay, so it is not acknowledged");
+
+ Files.delete(blocker.resolve("child"));
+ Files.delete(blocker);
+ crashed.putEntry("table", "d", "4");
+
+ assertEquals(Map.of("a", "1", "b", "2", "c", "3", "d", "4"), open(1).get("table").orElseThrow());
+ }
+
+ @Test
+ void aJournalThatCannotBeReadFailsLoadAndIsNeverCompactedAway() throws Exception {
+ PartitionedFileStorage<String> crashed = open(1);
+ crashed.putEntry("table", "a", "1");
+ StorageTestFiles.assumePosixPermissions(journal());
+ Set<PosixFilePermission> original = Files.getPosixFilePermissions(journal());
+ PartitionedFileStorage<String> restarting = new PartitionedFileStorage<>(tempDir.resolve("items"),
+ tempDir.resolve("items.json"), String.class, 1, PartitionedFileStorage.Mode.JOURNAL, Long.MAX_VALUE);
+ opened.add(restarting);
+ Files.setPosixFilePermissions(journal(), PosixFilePermissions.fromString("---------"));
+ try {
+ // A root user reads through the mode, so there is nothing to test there.
+ assumeFalse(Files.isReadable(journal()), "the journal stays readable for this user");
+
+ assertThrows(UncheckedIOException.class, restarting::load);
+ restarting.flush();
+ } finally {
+ Files.setPosixFilePermissions(journal(), original);
+ }
+
+ assertTrue(Files.size(journal()) > 0, "the unreplayed records are kept");
+ assertEquals(Map.of("a", "1"), open(1).get("table").orElseThrow());
+ }
+
+ @ParameterizedTest
+ @EnumSource(value = PartitionedFileStorage.Mode.class, names = {"SYNC", "PERIODIC"})
+ void anotherModeWritesALeftoverJournalIntoItsShardsAndRemovesIt(PartitionedFileStorage.Mode mode) {
+ PartitionedFileStorage<String> crashed = open(1);
+ crashed.put("table", Map.of("a", "1", "b", "2"));
+ crashed.flush();
+ crashed.putEntry("table", "a", "journaled");
+
+ PartitionedFileStorage<String> switched = new PartitionedFileStorage<>(tempDir.resolve("items"),
+ tempDir.resolve("items.json"), String.class, 1, mode, Long.MAX_VALUE);
+ opened.add(switched);
+ switched.load();
+
+ assertEquals(Map.of("a", "journaled", "b", "2"), switched.get("table").orElseThrow());
+ assertFalse(Files.exists(journal()), "the journal is folded into the shard files and removed");
+ switched.putEntry("table", "a", "newer");
+ switched.flush();
+
+ // Back in journal mode, nothing stale is replayed over the newer shard file.
+ assertEquals(Map.of("a", "newer", "b", "2"), open(1).get("table").orElseThrow());
+ }
+
+ @Test
+ void aShardThatCannotBeWrittenKeepsTheJournal() throws Exception {
+ PartitionedFileStorage<String> crashed = open(1);
+ crashed.putEntry("table", "a", "1");
+ // A non-empty directory where the shard file goes makes only that write fail.
+ Path blocker = crashed.fileFor("table");
+ Files.createDirectories(blocker);
+ Files.writeString(blocker.resolve("child"), "x");
+
+ assertThrows(UncheckedIOException.class, crashed::flush, "a shard left unwritten is reported");
+
+ assertTrue(Files.size(journal()) > 0, "the journal still holds the unwritten change");
+ Files.delete(blocker.resolve("child"));
+ Files.delete(blocker);
+ assertEquals(Map.of("a", "1"), open(1).get("table").orElseThrow());
+ }
+
+ @Test
+ void aJournalThatCannotBeEmptiedStaysOpenForLaterChanges() throws Exception {
+ PartitionedFileStorage<String> crashed = open(1);
+ crashed.putEntry("table", "a", "1");
+ Path directory = tempDir.resolve("items");
+ StorageTestFiles.assumePosixPermissions(directory);
+ Set<PosixFilePermission> original = Files.getPosixFilePermissions(directory);
+ crashed.setBeforeJournalTruncate(() -> {
+ // The shards are written; now make emptying the journal fail.
+ try {
+ Files.setPosixFilePermissions(directory, PosixFilePermissions.fromString("r-xr-xr-x"));
+ } catch (IOException e) {
+ throw new UncheckedIOException(e);
+ }
+ });
+ try {
+ UncheckedIOException failure = null;
+ try {
+ crashed.flush();
+ } catch (UncheckedIOException e) {
+ failure = e;
+ }
+ // A root user writes through the read-only mode, so there is nothing to test there.
+ assumeFalse(Files.isWritable(directory), "the directory stays writable for this user");
+ assertNotNull(failure, "a journal left unemptied is reported");
+ } finally {
+ crashed.setBeforeJournalTruncate(() -> { });
+ Files.setPosixFilePermissions(directory, original);
+ }
+
+ crashed.putEntry("table", "b", "2");
+
+ assertEquals(Map.of("a", "1", "b", "2"), open(1).get("table").orElseThrow());
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/core/storage/PartitionedFileStorageTest.java b/src/test/java/io/github/hectorvent/floci/core/storage/PartitionedFileStorageTest.java
index 2e3d0bf1c5..3f0918e26d 100644
--- a/src/test/java/io/github/hectorvent/floci/core/storage/PartitionedFileStorageTest.java
+++ b/src/test/java/io/github/hectorvent/floci/core/storage/PartitionedFileStorageTest.java
@@ -290,6 +290,7 @@ class PartitionedFileStorageTest {
Path legacy = tempDir.resolve("items.json");
new ObjectMapper().writeValue(legacy.toFile(), Map.of("a", Map.of("k", "1")));
Files.createDirectories(tempDir.resolve("items"));
+ Files.createFile(tempDir.resolve("items").resolve(PartitionedFileStorage.JOURNAL_FILE));
assertEquals(Map.of("k", "1"), open(PartitionedFileStorage.Mode.SYNC).get("a").orElseThrow());
assertTrue(Files.exists(tempDir.resolve("items.json.migrated")));
diff --git a/src/test/java/io/github/hectorvent/floci/core/storage/StorageFactoryPartitionedTest.java b/src/test/java/io/github/hectorvent/floci/core/storage/StorageFactoryPartitionedTest.java
index d397d171ee..59d5132d1d 100644
--- a/src/test/java/io/github/hectorvent/floci/core/storage/StorageFactoryPartitionedTest.java
+++ b/src/test/java/io/github/hectorvent/floci/core/storage/StorageFactoryPartitionedTest.java
@@ -76,13 +76,37 @@ class StorageFactoryPartitionedTest {
}
@Test
- void walModeKeepsTheSingleFileStoreForNow() {
+ void walModeJournalsChangesAndCompactsOnShutdown() {
+ StorageFactory factory = newFactory("wal");
+ AccountAwareStorageBackend<Map<String, String>> store =
+ factory.createPartitioned("svc", "svc-items.json", TYPE, String.class, 1);
+
+ store.putEntry("table-a", "k", "v");
+ assertEquals(0, fileCount(tempDir.resolve("svc-items")));
+ assertTrue(Files.exists(tempDir.resolve("svc-items").resolve(PartitionedFileStorage.JOURNAL_FILE)));
+
+ factory.shutdownAll();
+ assertEquals(1, fileCount(tempDir.resolve("svc-items")));
+ assertEquals(Map.of("k", "v"), newFactory("wal")
+ .createPartitioned("svc", "svc-items.json", TYPE, String.class, 1).get("table-a").orElseThrow());
+ }
+
+ @Test
+ void walModeMigratesALegacyWalStoreIncludingItsUncompactedLog() {
+ WalStorage<String, Map<String, String>> legacy = new WalStorage<>(tempDir.resolve("svc-items-snapshot.json"),
+ tempDir.resolve("svc-items.wal"), TYPE, Long.MAX_VALUE);
+ legacy.load();
+ legacy.put("000000000000/table-a", Map.of("k", "v"));
+ legacy.flush();
+ legacy.put("000000000000/table-b", Map.of("k", "only-in-the-log"));
+
AccountAwareStorageBackend<Map<String, String>> store = newFactory("wal")
- .createPartitioned("svc", "svc-items.json", TYPE, String.class, 4);
+ .createPartitioned("svc", "svc-items.json", TYPE, String.class, 1);
- assertFalse(store.writesEntries());
- store.put("table-a", Map.of("k", "v"));
- assertFalse(Files.exists(tempDir.resolve("svc-items")));
+ assertEquals(Map.of("k", "v"), store.get("table-a").orElseThrow());
+ assertEquals(Map.of("k", "only-in-the-log"), store.get("table-b").orElseThrow());
+ assertFalse(Files.exists(tempDir.resolve("svc-items.wal")));
+ assertTrue(Files.exists(tempDir.resolve("svc-items-snapshot.json.migrated")));
}
@Test
@@ -112,7 +136,7 @@ class StorageFactoryPartitionedTest {
// What a crash right after the directory was created leaves behind.
Files.createDirectories(tempDir.resolve("svc-items"));
- AccountAwareStorageBackend<Map<String, String>> store = newFactory("hybrid")
+ AccountAwareStorageBackend<Map<String, String>> store = newFactory("wal")
.createPartitioned("svc", "svc-items.json", TYPE, String.class, 1);
assertEquals(Map.of("k", "v"), store.get("table-a").orElseThrow());
@@ -131,7 +155,7 @@ class StorageFactoryPartitionedTest {
Files.setLastModifiedTime(tempDir.resolve("svc-items-snapshot.json"), older);
Files.setLastModifiedTime(tempDir.resolve("svc-items.wal"), older);
- AccountAwareStorageBackend<Map<String, String>> store = newFactory("hybrid")
+ AccountAwareStorageBackend<Map<String, String>> store = newFactory("wal")
.createPartitioned("svc", "svc-items.json", TYPE, String.class, 1);
assertEquals(Map.of("k", "from-plain"), store.get("table-a").orElseThrow());
@@ -153,14 +177,14 @@ class StorageFactoryPartitionedTest {
Files.setPosixFilePermissions(staging, PosixFilePermissions.fromString("r-xr-xr-x"));
try {
assumeFalse(Files.isWritable(staging), "the directory stays writable for this user");
- assertThrows(UncheckedIOException.class, () -> newFactory("hybrid")
+ assertThrows(UncheckedIOException.class, () -> newFactory("wal")
.createPartitioned("svc", "svc-items.json", TYPE, String.class, 1));
} finally {
Files.setPosixFilePermissions(staging, PosixFilePermissions.fromString("rwxr-xr-x"));
}
assertArrayEquals(logged, Files.readAllBytes(log));
- assertEquals(Map.of("k", "only-in-the-log"), newFactory("hybrid")
+ assertEquals(Map.of("k", "only-in-the-log"), newFactory("wal")
.createPartitioned("svc", "svc-items.json", TYPE, String.class, 1).get("table-a").orElseThrow());
legacy.shutdown();
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPartitionedWalStorageIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPartitionedWalStorageIntegrationTest.java
new file mode 100644
index 0000000000..7ccc7c8107
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPartitionedWalStorageIntegrationTest.java
@@ -0,0 +1,124 @@
+package io.github.hectorvent.floci.services.dynamodb;
+
+import io.github.hectorvent.floci.core.storage.StorageFactory;
+import io.github.hectorvent.floci.testing.DynamoDbPartitionedWalStorageProfile;
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.TestProfile;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.List;
+import java.util.UUID;
+import java.util.stream.Stream;
+
+import static io.github.hectorvent.floci.testing.DynamoDbPartitionedWalStorageProfile.STORAGE_DIR;
+import static io.restassured.RestAssured.given;
+import static org.hamcrest.Matchers.equalTo;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * The {@code per-table} item layout in {@code wal} mode, end to end through the application:
+ * the configuration selects the partitioned item store, a write lands in its journal rather than
+ * in a single items file, and a compaction moves it into shard files, one set per account.
+ */
+@QuarkusTest
+@TestProfile(DynamoDbPartitionedWalStorageProfile.class)
+class DynamoDbPartitionedWalStorageIntegrationTest {
+
+ private static final String DYNAMODB_CONTENT_TYPE = "application/x-amz-json-1.0";
+ private static final String AUTH_ACCOUNT_1 =
+ "AWS4-HMAC-SHA256 Credential=000000000001/20260215/us-east-1/dynamodb/aws4_request, SignedHeaders=host, Signature=abc";
+ private static final String AUTH_ACCOUNT_2 =
+ "AWS4-HMAC-SHA256 Credential=000000000002/20260215/us-east-1/dynamodb/aws4_request, SignedHeaders=host, Signature=abc";
+
+ @Inject
+ StorageFactory storageFactory;
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ void itemsGoThroughTheJournalIntoPerAccountShardFiles() throws IOException {
+ // The directory outlives a run, so each run uses its own table name.
+ String table = "partitioned-wal-" + UUID.randomUUID();
+ Path itemsDirectory = Path.of(STORAGE_DIR, "dynamodb-items");
+ Path journal = itemsDirectory.resolve("journal.wal");
+ try {
+ for (String auth : List.of(AUTH_ACCOUNT_1, AUTH_ACCOUNT_2)) {
+ call(auth, "CreateTable", """
+ {"TableName": "%s", "BillingMode": "PAY_PER_REQUEST",
+ "KeySchema": [{"AttributeName": "pk", "KeyType": "HASH"}],
+ "AttributeDefinitions": [{"AttributeName": "pk", "AttributeType": "S"}]}
+ """.formatted(table));
+ }
+ call(AUTH_ACCOUNT_1, "PutItem", item(table, "one"));
+ call(AUTH_ACCOUNT_2, "PutItem", item(table, "two"));
+
+ assertTrue(Files.size(journal) > 0, "the writes are in the journal");
+ assertFalse(Files.exists(Path.of(STORAGE_DIR, "dynamodb-items.json")));
+
+ storageFactory.flushAll();
+
+ assertEquals(0, Files.size(journal), "compaction empties the journal");
+ assertEquals(2, shardFilesOf(itemsDirectory, table), "one shard file per account's table");
+ getItem(AUTH_ACCOUNT_1, table, "one");
+ getItem(AUTH_ACCOUNT_2, table, "two");
+ } finally {
+ for (String auth : List.of(AUTH_ACCOUNT_1, AUTH_ACCOUNT_2)) {
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.DeleteTable")
+ .header("Authorization", auth)
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("{\"TableName\": \"%s\"}".formatted(table))
+ .when().post("/");
+ }
+ }
+ }
+
+ private static String item(String table, String value) {
+ return """
+ {"TableName": "%s", "Item": {"pk": {"S": "item"}, "v": {"S": "%s"}}}
+ """.formatted(table, value);
+ }
+
+ private static void call(String auth, String action, String body) {
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810." + action)
+ .header("Authorization", auth)
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body(body)
+ .when().post("/")
+ .then().statusCode(200);
+ }
+
+ private static void getItem(String auth, String table, String expected) {
+ given()
+ .header("X-Amz-Target", "DynamoDB_20120810.GetItem")
+ .header("Authorization", auth)
+ .contentType(DYNAMODB_CONTENT_TYPE)
+ .body("{\"TableName\": \"%s\", \"Key\": {\"pk\": {\"S\": \"item\"}}}".formatted(table))
+ .when().post("/")
+ .then()
+ .statusCode(200)
+ .body("Item.v.S", equalTo(expected));
+ }
+
+ /** Shard files whose readable name prefix carries the table name, under any account. */
+ private static long shardFilesOf(Path directory, String table) throws IOException {
+ String safeName = table.replaceAll("[^A-Za-z0-9._-]", "_");
+ try (Stream<Path> files = Files.list(directory)) {
+ return files.map(f -> f.getFileName().toString())
+ .filter(name -> name.endsWith(".json") && name.contains(safeName))
+ .count();
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/testing/DynamoDbPartitionedWalStorageProfile.java b/src/test/java/io/github/hectorvent/floci/testing/DynamoDbPartitionedWalStorageProfile.java
new file mode 100644
index 0000000000..6159394b20
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/testing/DynamoDbPartitionedWalStorageProfile.java
@@ -0,0 +1,25 @@
+package io.github.hectorvent.floci.testing;
+
+import io.quarkus.test.junit.QuarkusTestProfile;
+
+import java.util.Map;
+
+/**
+ * DynamoDB in {@code wal} mode with its items in the {@code per-table} layout, split into four
+ * shards, on a directory of its own. Every other service stays on the test default, memory.
+ * {@code DynamoDbPartitionedWalStorageIntegrationTest} relies on all of it: it checks that the
+ * configuration reaches the item store and that the item store writes the journal and shards.
+ */
+public class DynamoDbPartitionedWalStorageProfile implements QuarkusTestProfile {
+
+ public static final String STORAGE_DIR = "target/dynamodb-partitioned-wal-it";
+
+ @Override
+ public Map<String, String> getConfigOverrides() {
+ return Map.of(
+ "floci.storage.services.dynamodb.mode", "wal",
+ "floci.storage.services.dynamodb.item-layout", "per-table",
+ "floci.storage.services.dynamodb.item-shards", "4",
+ "floci.storage.persistent-path", STORAGE_DIR);
+ }
+}