Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(cloudwatchlogs): persist event batches once (#3767)

Aniket Shuklabb67a443ce8f
diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackend.java b/src/main/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackend.java
index f66bdc95be..5f77fdc3f9 100644
--- a/src/main/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackend.java
+++ b/src/main/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackend.java
@@ -74,6 +74,14 @@ public class AccountAwareStorageBackend<V> implements StorageBackend<String, V>
trackOwner(account);
}
+ @Override
+ public void putAll(Map<String, V> entries) {
+ String prefix = prefix() + "/";
+ Map<String, V> prefixedEntries = new LinkedHashMap<>();
+ entries.forEach((key, value) -> prefixedEntries.put(prefix + key, value));
+ delegate.putAll(prefixedEntries);
+ }
+
@Override
public Optional<V> get(String key) {
String prefixedKey = prefixed(key);
@@ -384,6 +392,13 @@ public class AccountAwareStorageBackend<V> implements StorageBackend<String, V>
trackOwner(accountId);
}
+ public void putAllForAccount(String accountId, Map<String, V> entries) {
+ String prefix = accountId + "/";
+ Map<String, V> prefixedEntries = new LinkedHashMap<>();
+ entries.forEach((key, value) -> prefixedEntries.put(prefix + key, value));
+ delegate.putAll(prefixedEntries);
+ }
+
public void deleteForAccount(String accountId, String key) {
delegate.delete(accountId + "/" + key);
}
diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/PersistentStorage.java b/src/main/java/io/github/hectorvent/floci/core/storage/PersistentStorage.java
index e908265e00..dc4a7da20d 100644
--- a/src/main/java/io/github/hectorvent/floci/core/storage/PersistentStorage.java
+++ b/src/main/java/io/github/hectorvent/floci/core/storage/PersistentStorage.java
@@ -50,6 +50,15 @@ public class PersistentStorage<K, V> implements StorageBackend<K, V> {
persistToDisk();
}
+ @Override
+ public void putAll(Map<K, V> entries) {
+ if (entries.isEmpty()) {
+ return;
+ }
+ store.putAll(entries);
+ persistToDisk();
+ }
+
@Override
public Optional<V> get(K key) {
return Optional.ofNullable(store.get(key));
diff --git a/src/main/java/io/github/hectorvent/floci/core/storage/StorageBackend.java b/src/main/java/io/github/hectorvent/floci/core/storage/StorageBackend.java
index 7866ff13e5..1cb745af8f 100644
--- a/src/main/java/io/github/hectorvent/floci/core/storage/StorageBackend.java
+++ b/src/main/java/io/github/hectorvent/floci/core/storage/StorageBackend.java
@@ -1,6 +1,7 @@
package io.github.hectorvent.floci.core.storage;
import java.util.List;
+import java.util.Map;
import java.util.Optional;
import java.util.Set;
import java.util.function.Predicate;
@@ -15,6 +16,15 @@ public interface StorageBackend<K, V> {
void put(K key, V value);
+ /**
+ * Store several entries as one logical write. Backends that can persist a batch more
+ * efficiently should override this method; the default preserves compatibility for
+ * lightweight and service-specific implementations.
+ */
+ default void putAll(Map<K, V> entries) {
+ entries.forEach(this::put);
+ }
+
Optional<V> get(K key);
void delete(K key);
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java
index aca5ac6be9..a475196b08 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java
@@ -614,6 +614,7 @@ public class CloudWatchLogsService implements ResourceProvider {
Long minTs = null;
Long maxTs = null;
List<LogEvent> stored = new ArrayList<>(events.size());
+ Map<String, LogEvent> logEvents = new LinkedHashMap<>();
for (Map<String, Object> evt : events) {
long ts = toLong(evt.get("timestamp"), now);
@@ -627,13 +628,14 @@ public class CloudWatchLogsService implements ResourceProvider {
logEvent.setSequence(ingestionSequence.incrementAndGet());
String eventKey = eventKey(region, groupName, streamName, ts, logEvent.getEventId());
- putForAccount(eventStore, accountId, eventKey, logEvent);
+ logEvents.put(eventKey, logEvent);
stored.add(logEvent);
totalBytes += msg.getBytes().length + 26; // approx overhead
if (minTs == null || ts < minTs) { minTs = ts; }
if (maxTs == null || ts > maxTs) { maxTs = ts; }
}
+ putAllForAccount(eventStore, accountId, logEvents);
evictEventsPastRetention(accountId, region, groupName, now);
evictEventsBeyondCapacity(accountId);
@@ -734,6 +736,17 @@ public class CloudWatchLogsService implements ResourceProvider {
store.put(key, value);
}
+ private <V> void putAllForAccount(
+ StorageBackend<String, V> store, String accountId, Map<String, V> entries) {
+ if (accountId != null && store instanceof AccountAwareStorageBackend<?> rawAware) {
+ @SuppressWarnings("unchecked")
+ AccountAwareStorageBackend<V> aware = (AccountAwareStorageBackend<V>) rawAware;
+ aware.putAllForAccount(accountId, entries);
+ return;
+ }
+ store.putAll(entries);
+ }
+
private <V> Set<String> keysForAccount(StorageBackend<String, V> store, String accountId) {
if (accountId != null && store instanceof AccountAwareStorageBackend<?> rawAware) {
@SuppressWarnings("unchecked")
diff --git a/src/test/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackendTest.java b/src/test/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackendTest.java
index afaffa62ce..92452d191e 100644
--- a/src/test/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackendTest.java
+++ b/src/test/java/io/github/hectorvent/floci/core/storage/AccountAwareStorageBackendTest.java
@@ -35,6 +35,20 @@ class AccountAwareStorageBackendTest {
assertEquals("111111111111", storage.accountId());
}
+ @Test
+ void putAllPrefixesEveryEntryWithTheRequestScope() {
+ InMemoryStorage<String, String> raw = new InMemoryStorage<>();
+ AccountAwareStorageBackend<String> storage =
+ new AccountAwareStorageBackend<>(raw, null, "111111111111");
+
+ storage.putAll(Map.of("key1", "value1", "key2", "value2"));
+
+ assertEquals("value1", raw.get("111111111111/key1").orElseThrow());
+ assertEquals("value2", raw.get("111111111111/key2").orElseThrow());
+ assertTrue(raw.get("key1").isEmpty());
+ assertTrue(raw.get("key2").isEmpty());
+ }
+
@Test
void guardedLegacyLookupMigratesMatchingRawValue() {
InMemoryStorage<String, String> raw = new InMemoryStorage<>();
diff --git a/src/test/java/io/github/hectorvent/floci/core/storage/PersistentStorageTest.java b/src/test/java/io/github/hectorvent/floci/core/storage/PersistentStorageTest.java
index 63114931b1..860a4b181f 100644
--- a/src/test/java/io/github/hectorvent/floci/core/storage/PersistentStorageTest.java
+++ b/src/test/java/io/github/hectorvent/floci/core/storage/PersistentStorageTest.java
@@ -43,6 +43,22 @@ class PersistentStorageTest {
assertEquals("value2", store2.get("key2").orElseThrow());
}
+ @Test
+ void putAllPersistsEveryEntry() {
+ Path filePath = tempDir.resolve("batch-persist-test.json");
+ PersistentStorage<String, String> writer = new PersistentStorage<>(
+ filePath, new TypeReference<Map<String, String>>() {});
+ writer.putAll(Map.of("key1", "value1", "key2", "value2", "key3", "value3"));
+
+ PersistentStorage<String, String> reader = new PersistentStorage<>(
+ filePath, new TypeReference<Map<String, String>>() {});
+ reader.load();
+
+ assertEquals("value1", reader.get("key1").orElseThrow());
+ assertEquals("value2", reader.get("key2").orElseThrow());
+ assertEquals("value3", reader.get("key3").orElseThrow());
+ }
+
@Test
void delete() {
storage.put("key1", "value1");
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java
index 0791a64778..62a21e5cd9 100644
--- a/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java
@@ -818,6 +818,49 @@ class CloudWatchLogsServiceTest {
assertNotNull(stream.getLastIngestionTime());
}
+ @Test
+ void putLogEventsStoresTheBatchWithOneBackendWrite() {
+ String accountId = "111111111111";
+ CountingStorageBackend<String, LogEvent> rawEvents = new CountingStorageBackend<>();
+ CloudWatchLogsService batchService = new CloudWatchLogsService(
+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, "000000000000"),
+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, "000000000000"),
+ new AccountAwareStorageBackend<>(rawEvents, null, "000000000000"),
+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, "000000000000"),
+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, "000000000000"),
+ 10_000,
+ new RegionResolver(REGION, "000000000000"));
+ batchService.createLogGroupForAccount(accountId, "/app/logs", null, null, REGION);
+ batchService.createLogStreamForAccount(accountId, "/app/logs", "stream-1", REGION);
+
+ batchService.putLogEventsForAccount(accountId, "/app/logs", "stream-1", List.of(
+ Map.of("timestamp", 1_000L, "message", "first"),
+ Map.of("timestamp", 2_000L, "message", "second"),
+ Map.of("timestamp", 3_000L, "message", "third")), REGION);
+
+ assertEquals(1, rawEvents.putAllCalls);
+ assertEquals(0, rawEvents.putCalls);
+ assertEquals(3, rawEvents.keys().size());
+ assertTrue(rawEvents.keys().stream().allMatch(key -> key.startsWith(accountId + "/")));
+ }
+
+ private static final class CountingStorageBackend<K, V> extends InMemoryStorage<K, V> {
+ private int putCalls;
+ private int putAllCalls;
+
+ @Override
+ public void put(K key, V value) {
+ putCalls++;
+ super.put(key, value);
+ }
+
+ @Override
+ public void putAll(Map<K, V> entries) {
+ putAllCalls++;
+ entries.forEach((key, value) -> super.put(key, value));
+ }
+ }
+
@Test
void maxEventsPerQueryIsRespected() {
CloudWatchLogsService limitedService = new CloudWatchLogsService(