Light, fluffy, and always free - The AWS Local Emulator alternative
perf(dynamodb): stop copying the change history on each write to a table with point-in-time recovery (#6824)
TATSUNO “Taz” Yasuhiro8ad890148aa7
diff --git a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPointInTimeRecovery.java b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPointInTimeRecovery.javaindex 43a4e880fb..44a66d9aa3 100644--- a/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPointInTimeRecovery.java+++ b/src/main/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPointInTimeRecovery.java@@ -10,10 +10,12 @@ import java.time.Clock;import java.time.Duration;import java.time.Instant;import java.time.temporal.ChronoUnit;+import java.util.ArrayDeque;import java.util.ArrayList;import java.util.Comparator;import java.util.HashMap;import java.util.HashSet;+import java.util.Iterator;import java.util.LinkedHashMap;import java.util.List;import java.util.Map;@@ -36,7 +38,7 @@ final class DynamoDbPointInTimeRecovery {/** The changes of one table, oldest first. */private static final class Changes {- private final List<PointInTimeChange> list = new ArrayList<>();+ private final ArrayDeque<PointInTimeChange> list = new ArrayDeque<>();private long nextSequence;private boolean dropped;// itemsAt leaves out a change recorded after it read the list, so that change must fall after its time.@@ -62,11 +64,13 @@ final class DynamoDbPointInTimeRecovery {store.get(key).ifPresent(change -> stored.put(defaultAccountId + "/" + key, change));}}- stored.forEach((key, change) -> changesByTable- .computeIfAbsent(key.substring(0, key.lastIndexOf('#')), ignored -> new Changes())- .list.add(change));+ stored.entrySet().stream()+ .sorted(Comparator.comparingLong(entry -> entry.getValue().getSequence()))+ .forEach(entry -> changesByTable+ .computeIfAbsent(entry.getKey().substring(0, entry.getKey().lastIndexOf('#')),+ ignored -> new Changes())+ .list.add(entry.getValue()));for (Changes changes : changesByTable.values()) {- changes.list.sort(Comparator.comparingLong(PointInTimeChange::getSequence));changes.nextSequence = changes.list.getLast().getSequence() + 1;}}@@ -105,12 +109,9 @@ final class DynamoDbPointInTimeRecovery {changes.list.add(change);put(accountId, storageKey, change);Instant cutoff = now.minus(recoveryPeriod(table));- int expired = 0;- while (expired < changes.list.size() && changes.list.get(expired).getTime().isBefore(cutoff)) {- delete(accountId, storageKey, changes.list.get(expired));- expired++;+ while (changes.list.getFirst().getTime().isBefore(cutoff)) {+ delete(accountId, storageKey, changes.list.removeFirst());}- changes.list.subList(0, expired).clear();}}@@ -224,11 +225,16 @@ final class DynamoDbPointInTimeRecovery {JsonNode of(String itemKey) {synchronized (changes) {- int i = changes.list.size();- while (0 < i && read <= changes.list.get(i - 1).getSequence()) {- i--;+ List<PointInTimeChange> unread = new ArrayList<>();+ Iterator<PointInTimeChange> newestFirst = changes.list.descendingIterator();+ while (newestFirst.hasNext()) {+ PointInTimeChange change = newestFirst.next();+ if (change.getSequence() < read) {+ break;+ }+ unread.add(change);}- for (PointInTimeChange change : changes.list.subList(i, changes.list.size())) {+ for (PointInTimeChange change : unread.reversed()) {if (!replaced.containsKey(change.getItemKey())) {replaced.put(change.getItemKey(), change.getOldItem());}diff --git a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPointInTimeRecoveryTest.java b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPointInTimeRecoveryTest.javaindex 5880901e4b..1e659f37ae 100644--- a/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPointInTimeRecoveryTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/dynamodb/DynamoDbPointInTimeRecoveryTest.java@@ -4,9 +4,11 @@ import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;import com.fasterxml.jackson.databind.node.ObjectNode;import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.services.dynamodb.DynamoDbPointInTimeRecovery.ItemChange;import io.github.hectorvent.floci.services.dynamodb.model.AttributeDefinition;import io.github.hectorvent.floci.services.dynamodb.model.KeySchemaElement;+import io.github.hectorvent.floci.services.dynamodb.model.PointInTimeChange;import io.github.hectorvent.floci.services.dynamodb.model.TableDefinition;import io.github.hectorvent.floci.testing.MutableClock;import org.junit.jupiter.api.Test;@@ -28,12 +30,25 @@ class DynamoDbPointInTimeRecoveryTest {private static final String TABLE_KEY = "us-east-1::Items";private final MutableClock clock = new MutableClock();+ private final StorageBackend<String, PointInTimeChange> store = new InMemoryStorage<>();private final DynamoDbPointInTimeRecovery recovery =- new DynamoDbPointInTimeRecovery(new InMemoryStorage<>(), ACCOUNT, Duration.ofMinutes(5), clock);+ new DynamoDbPointInTimeRecovery(store, ACCOUNT, Duration.ofMinutes(5), clock);private final TableDefinition table = new TableDefinition("Items",List.of(new KeySchemaElement("pk", "HASH")), List.of(new AttributeDefinition("pk", "S")));private final Map<String, JsonNode> live = new ConcurrentHashMap<>();+ @Test+ void recordItemDropsTheChangesThatLeftTheRecoveryPeriod() {+ table.setPointInTimeRecoveryEnabled(true);+ table.setPointInTimeRecoveryRecoveryPeriodInDays(1);+ for (String value : List.of("A", "B", "C")) {+ recovery.recordItem(ACCOUNT, TABLE_KEY, table, "k", item(value));+ clock.advance(Duration.ofHours(12));+ }++ assertEquals(Set.of(TABLE_KEY + "#1", TABLE_KEY + "#2"), store.keys());+ }+@Testvoid changesBetweenWaitsForAWriteThatIsRecordedButNotYetWritten() {table.setPointInTimeRecoveryEnabled(true);