Light, fluffy, and always free - The AWS Local Emulator alternative
fix(cloudwatchlogs): persist event batches once (#3767)
Aniket Shuklabb67a443ce8f
diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackend.java b/src/main/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackend.javaindex f66bdc95be..5f77fdc3f9 100644--- a/src/main/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackend.java+++ b/src/main/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackend.java@@ -74,6 +74,14 @@ public class AccountAwareStorageBackend<V> implements StorageBackend<String, V>trackOwner(account);}+ @Override+ public void putAll(Map<String, V> entries) {+ String prefix = prefix() + "/";+ Map<String, V> prefixedEntries = new LinkedHashMap<>();+ entries.forEach((key, value) -> prefixedEntries.put(prefix + key, value));+ delegate.putAll(prefixedEntries);+ }+@Overridepublic Optional<V> get(String key) {String prefixedKey = prefixed(key);@@ -384,6 +392,13 @@ public class AccountAwareStorageBackend<V> implements StorageBackend<String, V>trackOwner(accountId);}+ public void putAllForAccount(String accountId, Map<String, V> entries) {+ String prefix = accountId + "/";+ Map<String, V> prefixedEntries = new LinkedHashMap<>();+ entries.forEach((key, value) -> prefixedEntries.put(prefix + key, value));+ delegate.putAll(prefixedEntries);+ }+public void deleteForAccount(String accountId, String key) {delegate.delete(accountId + "/" + key);}diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/PersistentStorage.java b/src/main/java/io/github/hectorvent/floci/core/storage/PersistentStorage.javaindex e908265e00..dc4a7da20d 100644--- a/src/main/java/io/github/hectorvent/floci/core/storage/PersistentStorage.java+++ b/src/main/java/io/github/hectorvent/floci/core/storage/PersistentStorage.java@@ -50,6 +50,15 @@ public class PersistentStorage<K, V> implements StorageBackend<K, V> {persistToDisk();}+ @Override+ public void putAll(Map<K, V> entries) {+ if (entries.isEmpty()) {+ return;+ }+ store.putAll(entries);+ persistToDisk();+ }+@Overridepublic Optional<V> get(K key) {return Optional.ofNullable(store.get(key));diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/StorageBackend.java b/src/main/java/io/github/hectorvent/floci/core/storage/StorageBackend.javaindex 7866ff13e5..1cb745af8f 100644--- a/src/main/java/io/github/hectorvent/floci/core/storage/StorageBackend.java+++ b/src/main/java/io/github/hectorvent/floci/core/storage/StorageBackend.java@@ -1,6 +1,7 @@package io.github.hectorvent.floci.core.storage;import java.util.List;+import java.util.Map;import java.util.Optional;import java.util.Set;import java.util.function.Predicate;@@ -15,6 +16,15 @@ public interface StorageBackend<K, V> {void put(K key, V value);+ /**+ * Store several entries as one logical write. Backends that can persist a batch more+ * efficiently should override this method; the default preserves compatibility for+ * lightweight and service-specific implementations.+ */+ default void putAll(Map<K, V> entries) {+ entries.forEach(this::put);+ }+Optional<V> get(K key);void delete(K key);diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.javaindex aca5ac6be9..a475196b08 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java@@ -614,6 +614,7 @@ public class CloudWatchLogsService implements ResourceProvider {Long minTs = null;Long maxTs = null;List<LogEvent> stored = new ArrayList<>(events.size());+ Map<String, LogEvent> logEvents = new LinkedHashMap<>();for (Map<String, Object> evt : events) {long ts = toLong(evt.get("timestamp"), now);@@ -627,13 +628,14 @@ public class CloudWatchLogsService implements ResourceProvider {logEvent.setSequence(ingestionSequence.incrementAndGet());String eventKey = eventKey(region, groupName, streamName, ts, logEvent.getEventId());- putForAccount(eventStore, accountId, eventKey, logEvent);+ logEvents.put(eventKey, logEvent);stored.add(logEvent);totalBytes += msg.getBytes().length + 26; // approx overheadif (minTs == null || ts < minTs) { minTs = ts; }if (maxTs == null || ts > maxTs) { maxTs = ts; }}+ putAllForAccount(eventStore, accountId, logEvents);evictEventsPastRetention(accountId, region, groupName, now);evictEventsBeyondCapacity(accountId);@@ -734,6 +736,17 @@ public class CloudWatchLogsService implements ResourceProvider {store.put(key, value);}+ private <V> void putAllForAccount(+ StorageBackend<String, V> store, String accountId, Map<String, V> entries) {+ if (accountId != null && store instanceof AccountAwareStorageBackend<?> rawAware) {+ @SuppressWarnings("unchecked")+ AccountAwareStorageBackend<V> aware = (AccountAwareStorageBackend<V>) rawAware;+ aware.putAllForAccount(accountId, entries);+ return;+ }+ store.putAll(entries);+ }+private <V> Set<String> keysForAccount(StorageBackend<String, V> store, String accountId) {if (accountId != null && store instanceof AccountAwareStorageBackend<?> rawAware) {@SuppressWarnings("unchecked")diff --git a/src/test/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackendTest.java b/src/test/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackendTest.javaindex afaffa62ce..92452d191e 100644--- a/src/test/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackendTest.java+++ b/src/test/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackendTest.java@@ -35,6 +35,20 @@ class AccountAwareStorageBackendTest {assertEquals("111111111111", storage.accountId());}+ @Test+ void putAllPrefixesEveryEntryWithTheRequestScope() {+ InMemoryStorage<String, String> raw = new InMemoryStorage<>();+ AccountAwareStorageBackend<String> storage =+ new AccountAwareStorageBackend<>(raw, null, "111111111111");++ storage.putAll(Map.of("key1", "value1", "key2", "value2"));++ assertEquals("value1", raw.get("111111111111/key1").orElseThrow());+ assertEquals("value2", raw.get("111111111111/key2").orElseThrow());+ assertTrue(raw.get("key1").isEmpty());+ assertTrue(raw.get("key2").isEmpty());+ }+@Testvoid guardedLegacyLookupMigratesMatchingRawValue() {InMemoryStorage<String, String> raw = new InMemoryStorage<>();diff --git a/src/test/java/io/github/hectorvent/floci/core/storage/PersistentStorageTest.java b/src/test/java/io/github/hectorvent/floci/core/storage/PersistentStorageTest.javaindex 63114931b1..860a4b181f 100644--- a/src/test/java/io/github/hectorvent/floci/core/storage/PersistentStorageTest.java+++ b/src/test/java/io/github/hectorvent/floci/core/storage/PersistentStorageTest.java@@ -43,6 +43,22 @@ class PersistentStorageTest {assertEquals("value2", store2.get("key2").orElseThrow());}+ @Test+ void putAllPersistsEveryEntry() {+ Path filePath = tempDir.resolve("batch-persist-test.json");+ PersistentStorage<String, String> writer = new PersistentStorage<>(+ filePath, new TypeReference<Map<String, String>>() {});+ writer.putAll(Map.of("key1", "value1", "key2", "value2", "key3", "value3"));++ PersistentStorage<String, String> reader = new PersistentStorage<>(+ filePath, new TypeReference<Map<String, String>>() {});+ reader.load();++ assertEquals("value1", reader.get("key1").orElseThrow());+ assertEquals("value2", reader.get("key2").orElseThrow());+ assertEquals("value3", reader.get("key3").orElseThrow());+ }+@Testvoid delete() {storage.put("key1", "value1");diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.javaindex 0791a64778..62a21e5cd9 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java@@ -818,6 +818,49 @@ class CloudWatchLogsServiceTest {assertNotNull(stream.getLastIngestionTime());}+ @Test+ void putLogEventsStoresTheBatchWithOneBackendWrite() {+ String accountId = "111111111111";+ CountingStorageBackend<String, LogEvent> rawEvents = new CountingStorageBackend<>();+ CloudWatchLogsService batchService = new CloudWatchLogsService(+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, "000000000000"),+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, "000000000000"),+ new AccountAwareStorageBackend<>(rawEvents, null, "000000000000"),+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, "000000000000"),+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, "000000000000"),+ 10_000,+ new RegionResolver(REGION, "000000000000"));+ batchService.createLogGroupForAccount(accountId, "/app/logs", null, null, REGION);+ batchService.createLogStreamForAccount(accountId, "/app/logs", "stream-1", REGION);++ batchService.putLogEventsForAccount(accountId, "/app/logs", "stream-1", List.of(+ Map.of("timestamp", 1_000L, "message", "first"),+ Map.of("timestamp", 2_000L, "message", "second"),+ Map.of("timestamp", 3_000L, "message", "third")), REGION);++ assertEquals(1, rawEvents.putAllCalls);+ assertEquals(0, rawEvents.putCalls);+ assertEquals(3, rawEvents.keys().size());+ assertTrue(rawEvents.keys().stream().allMatch(key -> key.startsWith(accountId + "/")));+ }++ private static final class CountingStorageBackend<K, V> extends InMemoryStorage<K, V> {+ private int putCalls;+ private int putAllCalls;++ @Override+ public void put(K key, V value) {+ putCalls++;+ super.put(key, value);+ }++ @Override+ public void putAll(Map<K, V> entries) {+ putAllCalls++;+ entries.forEach((key, value) -> super.put(key, value));+ }+ }+@Testvoid maxEventsPerQueryIsRespected() {CloudWatchLogsService limitedService = new CloudWatchLogsService(