Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(cloudwatchlogs): bound stored events and enforce retention (#3611)

Chris Jovanovski6ce3fbbd2e85
diff --git a/docs/configuration/advanced/application-yml.md b/docs/configuration/advanced/application-yml.md
index 3dd8b01877..f4494d2518 100644
--- a/docs/configuration/advanced/application-yml.md
+++ b/docs/configuration/advanced/application-yml.md
@@ -66,7 +66,7 @@ floci:
lambda:
flush-interval-ms: 5000
cloudwatchlogs:
- flush-interval-ms: 5000
+ flush-interval-ms: 15000
cloudwatchmetrics:
flush-interval-ms: 5000
secretsmanager:
@@ -200,6 +200,7 @@ floci:
cloudwatchlogs:
enabled: true
max-events-per-query: 10000
+ max-stored-events: 20000 # per account; oldest events are evicted once the store exceeds this
cloudwatchmetrics:
enabled: true
diff --git a/docs/configuration/environment-variables.md b/docs/configuration/environment-variables.md
index d31b7f6514..e7678ac154 100644
--- a/docs/configuration/environment-variables.md
+++ b/docs/configuration/environment-variables.md
@@ -266,6 +266,7 @@ See [Initialization Hooks](./initialization-hooks.md) for lifecycle phases and s
|---|---|---|
| `FLOCI_SERVICES_CLOUDWATCHLOGS_ENABLED` | `true` | Enable the CloudWatch Logs service |
| `FLOCI_SERVICES_CLOUDWATCHLOGS_MAX_EVENTS_PER_QUERY` | `10000` | Maximum log events returned by a single `FilterLogEvents` or `GetLogEvents` call, and the upper bound for a Logs Insights `limit` |
+| `FLOCI_SERVICES_CLOUDWATCHLOGS_MAX_STORED_EVENTS` | `20000` | Maximum log events retained per account across all groups; oldest events are evicted first |
| `FLOCI_SERVICES_CLOUDWATCHLOGS_QUERY_COMPLETION_DELAY_MS` | `0` | Artificial Logs Insights query delay. With `0` a query completes immediately; a positive value emulates the asynchronous `Running` → `Complete` lifecycle |
### CloudWatch Metrics
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
index e706677b48..cd1dcfc7c1 100644
--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
@@ -370,7 +370,8 @@ public interface EmulatorConfig {
interface CloudWatchLogsStorageConfig {
Optional<String> mode();
- @WithDefault("5000")
+ // Log events are the largest and most write-heavy store, so they flush less often than the rest.
+ @WithDefault("15000")
long flushIntervalMs();
}
@@ -1515,6 +1516,14 @@ public interface EmulatorConfig {
@WithDefault("10000")
int maxEventsPerQuery();
+ /**
+ * Upper bound on log events kept across all groups. The store is one JSON document rewritten
+ * in full on every flush, so an unbounded store turns a chatty or retrying Lambda into a
+ * sustained multi-hundred-MB/s disk writer. Oldest events are evicted first once exceeded.
+ */
+ @WithDefault("20000")
+ int maxStoredEvents();
+
/**
* Artificial Logs Insights query completion delay, in milliseconds. With the default 0,
* queries complete immediately (fast local dev). A positive value emulates the real
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java
index ccac6debbc..5030f0b123 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java
@@ -60,6 +60,8 @@ public class CloudWatchLogsService implements ResourceProvider {
private final StorageBackend<String, ResourcePolicy> resourcePolicyStore;
private final RegionResolver regionResolver;
private final int maxEventsPerQuery;
+ /** Ceiling on stored events across every group in an account; the oldest are evicted first once exceeded. */
+ private final int maxStoredEvents;
/**
* Monotonic counter assigning an ingestion sequence to each stored event. Seeded from
* the highest sequence already in the store so ordering survives persistence reloads.
@@ -105,6 +107,7 @@ public class CloudWatchLogsService implements ResourceProvider {
storageFactory.create("cloudwatchlogs", "cwlogs-resource-policies.json",
new TypeReference<>() {}),
config.services().cloudwatchlogs().maxEventsPerQuery(),
+ config.services().cloudwatchlogs().maxStoredEvents(),
regionResolver,
config.services().cloudwatchlogs().queryCompletionDelayMs(),
System::currentTimeMillis
@@ -118,7 +121,18 @@ public class CloudWatchLogsService implements ResourceProvider {
int maxEventsPerQuery,
RegionResolver regionResolver) {
this(groupStore, streamStore, eventStore, subscriptionFilterStore, new InMemoryStorage<>(),
- maxEventsPerQuery, regionResolver, 0L, System::currentTimeMillis);
+ maxEventsPerQuery, Integer.MAX_VALUE, regionResolver, 0L, System::currentTimeMillis);
+ }
+
+ CloudWatchLogsService(StorageBackend<String, LogGroup> groupStore,
+ StorageBackend<String, LogStream> streamStore,
+ StorageBackend<String, LogEvent> eventStore,
+ StorageBackend<String, SubscriptionFilter> subscriptionFilterStore,
+ int maxEventsPerQuery,
+ int maxStoredEvents,
+ RegionResolver regionResolver) {
+ this(groupStore, streamStore, eventStore, subscriptionFilterStore, new InMemoryStorage<>(),
+ maxEventsPerQuery, maxStoredEvents, regionResolver, 0L, System::currentTimeMillis);
}
CloudWatchLogsService(StorageBackend<String, LogGroup> groupStore,
@@ -130,7 +144,7 @@ public class CloudWatchLogsService implements ResourceProvider {
long queryCompletionDelayMs,
LongSupplier clock) {
this(groupStore, streamStore, eventStore, subscriptionFilterStore, new InMemoryStorage<>(),
- maxEventsPerQuery, regionResolver, queryCompletionDelayMs, clock);
+ maxEventsPerQuery, Integer.MAX_VALUE, regionResolver, queryCompletionDelayMs, clock);
}
CloudWatchLogsService(StorageBackend<String, LogGroup> groupStore,
@@ -139,6 +153,7 @@ public class CloudWatchLogsService implements ResourceProvider {
StorageBackend<String, SubscriptionFilter> subscriptionFilterStore,
StorageBackend<String, ResourcePolicy> resourcePolicyStore,
int maxEventsPerQuery,
+ int maxStoredEvents,
RegionResolver regionResolver,
long queryCompletionDelayMs,
LongSupplier clock) {
@@ -148,6 +163,7 @@ public class CloudWatchLogsService implements ResourceProvider {
this.subscriptionFilterStore = subscriptionFilterStore;
this.resourcePolicyStore = resourcePolicyStore;
this.maxEventsPerQuery = maxEventsPerQuery;
+ this.maxStoredEvents = maxStoredEvents;
this.regionResolver = regionResolver;
long maxSequence = eventStore.scan(k -> true).stream()
.mapToLong(LogEvent::getSequence)
@@ -560,6 +576,9 @@ public class CloudWatchLogsService implements ResourceProvider {
if (maxTs == null || ts > maxTs) { maxTs = ts; }
}
+ evictEventsPastRetention(accountId, region, groupName, now);
+ evictEventsBeyondCapacity(accountId);
+
// Update stream metadata
if (minTs != null) {
if (stream.getFirstEventTimestamp() == null || minTs < stream.getFirstEventTimestamp()) {
@@ -578,6 +597,60 @@ public class CloudWatchLogsService implements ResourceProvider {
return nextToken;
}
+ /**
+ * Drops this group's events older than its retention policy. AWS expires them lazily in the
+ * background; doing it on ingest keeps the on-disk store from growing past what a
+ * {@code PutRetentionPolicy} promised.
+ */
+ private void evictEventsPastRetention(String accountId, String region, String groupName, long now) {
+ Integer retentionInDays = getForAccount(groupStore, accountId, groupKey(region, groupName))
+ .map(LogGroup::getRetentionInDays)
+ .orElse(null);
+ if (retentionInDays == null) {
+ return;
+ }
+ long cutoff = now - retentionInDays * 86_400_000L;
+ String groupPrefix = region + "::" + groupName + "::";
+ for (String key : List.copyOf(keysForAccount(eventStore, accountId))) {
+ if (!key.startsWith(groupPrefix)) {
+ continue;
+ }
+ boolean expired = getForAccount(eventStore, accountId, key)
+ .map(event -> event.getTimestamp() < cutoff).orElse(false);
+ if (expired) {
+ deleteForAccount(eventStore, accountId, key);
+ }
+ }
+ }
+
+ /**
+ * Keeps an account's event store under {@link #maxStoredEvents} by dropping the oldest events.
+ * The store is persisted as a single document rewritten in full on each flush, so its size
+ * is the cost of every flush; without a ceiling a chatty function turns log ingestion into a
+ * sustained disk writer.
+ * <p>
+ * The ceiling is best-effort rather than atomic: this method is not synchronized, so two
+ * concurrent PutLogEvents calls for the same account can each scan and evict independently and
+ * briefly overshoot the cap. The next ingest corrects the drift, which mirrors how CloudWatch
+ * Logs itself deletes expired events lazily rather than at an exact boundary.
+ */
+ private void evictEventsBeyondCapacity(String accountId) {
+ List<String> keys = List.copyOf(keysForAccount(eventStore, accountId));
+ int excess = keys.size() - maxStoredEvents;
+ if (excess <= 0) {
+ return;
+ }
+ List<Map.Entry<String, LogEvent>> oldestFirst = new ArrayList<>(keys.size());
+ for (String key : keys) {
+ getForAccount(eventStore, accountId, key).ifPresent(event -> oldestFirst.add(Map.entry(key, event)));
+ }
+ oldestFirst.sort(Map.Entry.comparingByValue(EVENT_ORDER));
+ for (Map.Entry<String, LogEvent> entry : oldestFirst.subList(0, Math.min(excess, oldestFirst.size()))) {
+ deleteForAccount(eventStore, accountId, entry.getKey());
+ }
+ LOG.debugv("Evicted {0} oldest log event(s) to stay within the {1}-event store ceiling", excess, maxStoredEvents);
+ }
+
private <V> Optional<V> getForAccount(
StorageBackend<String, V> store, String accountId, String key) {
if (accountId != null && store instanceof AccountAwareStorageBackend<?> rawAware) {
@@ -599,6 +672,25 @@ public class CloudWatchLogsService implements ResourceProvider {
store.put(key, value);
}
+ private <V> Set<String> keysForAccount(StorageBackend<String, V> store, String accountId) {
+ if (accountId != null && store instanceof AccountAwareStorageBackend<?> rawAware) {
+ @SuppressWarnings("unchecked")
+ AccountAwareStorageBackend<V> aware = (AccountAwareStorageBackend<V>) rawAware;
+ return aware.keysForAccount(accountId);
+ }
+ return store.keys();
+ }
+
+ private <V> void deleteForAccount(StorageBackend<String, V> store, String accountId, String key) {
+ if (accountId != null && store instanceof AccountAwareStorageBackend<?> rawAware) {
+ @SuppressWarnings("unchecked")
+ AccountAwareStorageBackend<V> aware = (AccountAwareStorageBackend<V>) rawAware;
+ aware.deleteForAccount(accountId, key);
+ return;
+ }
+ store.delete(key);
+ }
+
public record LogEventsResult(List<LogEvent> events, String nextForwardToken, String nextBackwardToken) {}
public LogEventsResult getLogEvents(String groupName, String streamName,
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index f6d371a5bc..5b3c1b3389 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -152,7 +152,7 @@ floci:
lambda:
flush-interval-ms: 5000
cloudwatchlogs:
- flush-interval-ms: 5000
+ flush-interval-ms: 15000
cloudwatchmetrics:
flush-interval-ms: 5000
secretsmanager:
@@ -412,6 +412,7 @@ floci:
cloudwatchlogs:
enabled: true
max-events-per-query: 10000
+ max-stored-events: 20000
cloudwatchmetrics:
enabled: true
secretsmanager:
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java
index 6997432b96..f569ce3b57 100644
--- a/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java
@@ -108,6 +108,74 @@ class CloudWatchLogsServiceTest {
.count());
}
+ @Test
+ void explicitAccountRetentionEvictionLeavesOtherAccountEventsUntouched() {
+ InMemoryStorage<String, LogGroup> rawGroups = new InMemoryStorage<>();
+ InMemoryStorage<String, LogStream> rawStreams = new InMemoryStorage<>();
+ InMemoryStorage<String, LogEvent> rawEvents = new InMemoryStorage<>();
+ String accountA = "111111111111";
+ String accountB = "222222222222";
+ CloudWatchLogsService accountService = new CloudWatchLogsService(
+ new AccountAwareStorageBackend<>(rawGroups, null, accountB),
+ new AccountAwareStorageBackend<>(rawStreams, null, accountB),
+ new AccountAwareStorageBackend<>(rawEvents, null, accountB),
+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, accountB),
+ 10_000, new RegionResolver(REGION, accountB));
+ long now = System.currentTimeMillis();
+ long twoDaysAgo = now - 2 * 86_400_000L;
+
+ accountService.createLogGroupForAccount(accountA, "/app/logs", 1, null, REGION);
+ accountService.createLogStreamForAccount(accountA, "/app/logs", "stream", REGION);
+ accountService.createLogGroupForAccount(accountB, "/app/logs", null, null, REGION);
+ accountService.createLogStreamForAccount(accountB, "/app/logs", "stream", REGION);
+ accountService.putLogEventsForAccount(
+ accountB, "/app/logs", "stream", eventsAt(twoDaysAgo), REGION);
+ accountService.putLogEventsForAccount(
+ accountA, "/app/logs", "stream", eventsAt(twoDaysAgo, now), REGION);
+
+ String eventPrefix = REGION + "::/app/logs::stream::";
+ assertEquals(1, rawEvents.keys().stream()
+ .filter(key -> key.startsWith(accountA + "/" + eventPrefix))
+ .count());
+ assertEquals(1, rawEvents.keys().stream()
+ .filter(key -> key.startsWith(accountB + "/" + eventPrefix))
+ .count());
+ }
+
+ @Test
+ void explicitAccountCapacityEvictionDoesNotCountOtherAccountEvents() {
+ InMemoryStorage<String, LogGroup> rawGroups = new InMemoryStorage<>();
+ InMemoryStorage<String, LogStream> rawStreams = new InMemoryStorage<>();
+ InMemoryStorage<String, LogEvent> rawEvents = new InMemoryStorage<>();
+ String accountA = "111111111111";
+ String accountB = "222222222222";
+ CloudWatchLogsService accountService = new CloudWatchLogsService(
+ new AccountAwareStorageBackend<>(rawGroups, null, accountB),
+ new AccountAwareStorageBackend<>(rawStreams, null, accountB),
+ new AccountAwareStorageBackend<>(rawEvents, null, accountB),
+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, accountB),
+ 10_000, 2, new RegionResolver(REGION, accountB));
+
+ accountService.createLogGroupForAccount(accountA, "/app/logs", null, null, REGION);
+ accountService.createLogStreamForAccount(accountA, "/app/logs", "stream", REGION);
+ for (long timestamp : List.of(1L, 2L, 3L)) {
+ LogEvent event = new LogEvent();
+ event.setTimestamp(timestamp);
+ event.setEventId("event-" + timestamp);
+ rawEvents.put(accountB + "/event-" + timestamp, event);
+ }
+
+ accountService.putLogEventsForAccount(
+ accountA, "/app/logs", "stream", eventsAt(4L), REGION);
+
+ assertEquals(3, rawEvents.keys().stream()
+ .filter(key -> key.startsWith(accountB + "/"))
+ .count());
+ assertEquals(1, rawEvents.keys().stream()
+ .filter(key -> key.startsWith(accountA + "/"))
+ .count());
+ }
+
@Test
void deleteLogGroup() {
service.createLogGroup("/app/logs", null, null, REGION);
@@ -168,6 +236,89 @@ class CloudWatchLogsServiceTest {
assertNull(group.getRetentionInDays());
}
+ // ──────────────────────────── Stored event ceiling ────────────────────────────
+
+ private static CloudWatchLogsService serviceWithStoredEventCeiling(int maxStoredEvents) {
+ return new CloudWatchLogsService(
+ new InMemoryStorage<>(),
+ new InMemoryStorage<>(),
+ new InMemoryStorage<>(),
+ new InMemoryStorage<>(),
+ 10000,
+ maxStoredEvents,
+ new RegionResolver("us-east-1", "000000000000")
+ );
+ }
+
+ private static List<Map<String, Object>> eventsAt(long... timestamps) {
+ List<Map<String, Object>> events = new ArrayList<>();
+ for (long timestamp : timestamps) {
+ events.add(Map.of("timestamp", timestamp, "message", "event@" + timestamp));
+ }
+ return events;
+ }
+
+ private static List<Long> storedTimestamps(CloudWatchLogsService target, String group, String stream) {
+ return target.getLogEvents(group, stream, null, null, 100, true, null, REGION).events().stream()
+ .map(LogEvent::getTimestamp)
+ .toList();
+ }
+
+ @Test
+ void putLogEventsEvictsTheOldestEventsBeyondTheStoredEventCeiling() {
+ CloudWatchLogsService capped = serviceWithStoredEventCeiling(3);
+ capped.createLogGroup("/app/logs", null, null, REGION);
+ capped.createLogStream("/app/logs", "stream-1", REGION);
+ capped.createLogStream("/app/logs", "stream-2", REGION);
+
+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(1000, 2000, 3000), REGION);
+ capped.putLogEvents("/app/logs", "stream-2", eventsAt(4000, 5000), REGION);
+
+ assertEquals(List.of(3000L), storedTimestamps(capped, "/app/logs", "stream-1"),
+ "the oldest events across every stream go first, the store stays within the ceiling");
+ assertEquals(List.of(4000L, 5000L), storedTimestamps(capped, "/app/logs", "stream-2"));
+ }
+
+ @Test
+ void putLogEventsKeepsEverythingWhileUnderTheStoredEventCeiling() {
+ CloudWatchLogsService capped = serviceWithStoredEventCeiling(3);
+ capped.createLogGroup("/app/logs", null, null, REGION);
+ capped.createLogStream("/app/logs", "stream-1", REGION);
+
+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(1000, 2000, 3000), REGION);
+
+ assertEquals(List.of(1000L, 2000L, 3000L), storedTimestamps(capped, "/app/logs", "stream-1"));
+ }
+
+ @Test
+ void putLogEventsDropsEventsOlderThanTheGroupRetentionPolicy() {
+ service.createLogGroup("/app/logs", 1, null, REGION);
+ service.createLogStream("/app/logs", "stream-1", REGION);
+ long now = System.currentTimeMillis();
+ long twoDaysAgo = now - 2 * 86_400_000L;
+
+ service.putLogEvents("/app/logs", "stream-1", eventsAt(twoDaysAgo, now), REGION);
+
+ assertEquals(List.of(now), storedTimestamps(service, "/app/logs", "stream-1"),
+ "events past the retention window are not kept on disk waiting for a background sweep");
+ }
+
+ @Test
+ void putLogEventsLeavesOtherGroupsAloneWhenApplyingRetention() {
+ service.createLogGroup("/short", 1, null, REGION);
+ service.createLogStream("/short", "stream-1", REGION);
+ service.createLogGroup("/forever", null, null, REGION);
+ service.createLogStream("/forever", "stream-1", REGION);
+ long now = System.currentTimeMillis();
+ long twoDaysAgo = now - 2 * 86_400_000L;
+
+ service.putLogEvents("/forever", "stream-1", eventsAt(twoDaysAgo), REGION);
+ service.putLogEvents("/short", "stream-1", eventsAt(twoDaysAgo), REGION);
+
+ assertEquals(List.of(twoDaysAgo), storedTimestamps(service, "/forever", "stream-1"));
+ assertEquals(List.of(), storedTimestamps(service, "/short", "stream-1"));
+ }
+
// ──────────────────────────── KMS key association ────────────────────────────
@Test