Light, fluffy, and always free - The AWS Local Emulator alternative
fix(cloudwatchlogs): bound stored events and enforce retention (#3611)
Chris Jovanovski6ce3fbbd2e85
diff --git a/docs/configuration/advanced/application-yml.md b/docs/configuration/advanced/application-yml.mdindex 3dd8b01877..f4494d2518 100644--- a/docs/configuration/advanced/application-yml.md+++ b/docs/configuration/advanced/application-yml.md@@ -66,7 +66,7 @@ floci:lambda:flush-interval-ms: 5000cloudwatchlogs:- flush-interval-ms: 5000+ flush-interval-ms: 15000cloudwatchmetrics:flush-interval-ms: 5000secretsmanager:@@ -200,6 +200,7 @@ floci:cloudwatchlogs:enabled: truemax-events-per-query: 10000+ max-stored-events: 20000 # per account; oldest events are evicted once the store exceeds thiscloudwatchmetrics:enabled: truediff --git a/docs/configuration/environment-variables.md b/docs/configuration/environment-variables.mdindex d31b7f6514..e7678ac154 100644--- a/docs/configuration/environment-variables.md+++ b/docs/configuration/environment-variables.md@@ -266,6 +266,7 @@ See [Initialization Hooks](./initialization-hooks.md) for lifecycle phases and s|---|---|---|| `FLOCI_SERVICES_CLOUDWATCHLOGS_ENABLED` | `true` | Enable the CloudWatch Logs service || `FLOCI_SERVICES_CLOUDWATCHLOGS_MAX_EVENTS_PER_QUERY` | `10000` | Maximum log events returned by a single `FilterLogEvents` or `GetLogEvents` call, and the upper bound for a Logs Insights `limit` |+| `FLOCI_SERVICES_CLOUDWATCHLOGS_MAX_STORED_EVENTS` | `20000` | Maximum log events retained per account across all groups; oldest events are evicted first || `FLOCI_SERVICES_CLOUDWATCHLOGS_QUERY_COMPLETION_DELAY_MS` | `0` | Artificial Logs Insights query delay. With `0` a query completes immediately; a positive value emulates the asynchronous `Running` → `Complete` lifecycle |### CloudWatch Metricsdiff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex e706677b48..cd1dcfc7c1 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -370,7 +370,8 @@ public interface EmulatorConfig {interface CloudWatchLogsStorageConfig {Optional<String> mode();- @WithDefault("5000")+ // Log events are the largest and most write-heavy store, so they flush less often than the rest.+ @WithDefault("15000")long flushIntervalMs();}@@ -1515,6 +1516,14 @@ public interface EmulatorConfig {@WithDefault("10000")int maxEventsPerQuery();+ /**+ * Upper bound on log events kept across all groups. The store is one JSON document rewritten+ * in full on every flush, so an unbounded store turns a chatty or retrying Lambda into a+ * sustained multi-hundred-MB/s disk writer. Oldest events are evicted first once exceeded.+ */+ @WithDefault("20000")+ int maxStoredEvents();+/*** Artificial Logs Insights query completion delay, in milliseconds. With the default 0,* queries complete immediately (fast local dev). A positive value emulates the realdiff --git a/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.javaindex ccac6debbc..5030f0b123 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java@@ -60,6 +60,8 @@ public class CloudWatchLogsService implements ResourceProvider {private final StorageBackend<String, ResourcePolicy> resourcePolicyStore;private final RegionResolver regionResolver;private final int maxEventsPerQuery;+ /** Ceiling on stored events across every group in an account; the oldest are evicted first once exceeded. */+ private final int maxStoredEvents;/*** Monotonic counter assigning an ingestion sequence to each stored event. Seeded from* the highest sequence already in the store so ordering survives persistence reloads.@@ -105,6 +107,7 @@ public class CloudWatchLogsService implements ResourceProvider {storageFactory.create("cloudwatchlogs", "cwlogs-resource-policies.json",new TypeReference<>() {}),config.services().cloudwatchlogs().maxEventsPerQuery(),+ config.services().cloudwatchlogs().maxStoredEvents(),regionResolver,config.services().cloudwatchlogs().queryCompletionDelayMs(),System::currentTimeMillis@@ -118,7 +121,18 @@ public class CloudWatchLogsService implements ResourceProvider {int maxEventsPerQuery,RegionResolver regionResolver) {this(groupStore, streamStore, eventStore, subscriptionFilterStore, new InMemoryStorage<>(),- maxEventsPerQuery, regionResolver, 0L, System::currentTimeMillis);+ maxEventsPerQuery, Integer.MAX_VALUE, regionResolver, 0L, System::currentTimeMillis);+ }++ CloudWatchLogsService(StorageBackend<String, LogGroup> groupStore,+ StorageBackend<String, LogStream> streamStore,+ StorageBackend<String, LogEvent> eventStore,+ StorageBackend<String, SubscriptionFilter> subscriptionFilterStore,+ int maxEventsPerQuery,+ int maxStoredEvents,+ RegionResolver regionResolver) {+ this(groupStore, streamStore, eventStore, subscriptionFilterStore, new InMemoryStorage<>(),+ maxEventsPerQuery, maxStoredEvents, regionResolver, 0L, System::currentTimeMillis);}CloudWatchLogsService(StorageBackend<String, LogGroup> groupStore,@@ -130,7 +144,7 @@ public class CloudWatchLogsService implements ResourceProvider {long queryCompletionDelayMs,LongSupplier clock) {this(groupStore, streamStore, eventStore, subscriptionFilterStore, new InMemoryStorage<>(),- maxEventsPerQuery, regionResolver, queryCompletionDelayMs, clock);+ maxEventsPerQuery, Integer.MAX_VALUE, regionResolver, queryCompletionDelayMs, clock);}CloudWatchLogsService(StorageBackend<String, LogGroup> groupStore,@@ -139,6 +153,7 @@ public class CloudWatchLogsService implements ResourceProvider {StorageBackend<String, SubscriptionFilter> subscriptionFilterStore,StorageBackend<String, ResourcePolicy> resourcePolicyStore,int maxEventsPerQuery,+ int maxStoredEvents,RegionResolver regionResolver,long queryCompletionDelayMs,LongSupplier clock) {@@ -148,6 +163,7 @@ public class CloudWatchLogsService implements ResourceProvider {this.subscriptionFilterStore = subscriptionFilterStore;this.resourcePolicyStore = resourcePolicyStore;this.maxEventsPerQuery = maxEventsPerQuery;+ this.maxStoredEvents = maxStoredEvents;this.regionResolver = regionResolver;long maxSequence = eventStore.scan(k -> true).stream().mapToLong(LogEvent::getSequence)@@ -560,6 +576,9 @@ public class CloudWatchLogsService implements ResourceProvider {if (maxTs == null || ts > maxTs) { maxTs = ts; }}+ evictEventsPastRetention(accountId, region, groupName, now);+ evictEventsBeyondCapacity(accountId);+// Update stream metadataif (minTs != null) {if (stream.getFirstEventTimestamp() == null || minTs < stream.getFirstEventTimestamp()) {@@ -578,6 +597,60 @@ public class CloudWatchLogsService implements ResourceProvider {return nextToken;}+ /**+ * Drops this group's events older than its retention policy. AWS expires them lazily in the+ * background; doing it on ingest keeps the on-disk store from growing past what a+ * {@code PutRetentionPolicy} promised.+ */+ private void evictEventsPastRetention(String accountId, String region, String groupName, long now) {+ Integer retentionInDays = getForAccount(groupStore, accountId, groupKey(region, groupName))+ .map(LogGroup::getRetentionInDays)+ .orElse(null);+ if (retentionInDays == null) {+ return;+ }+ long cutoff = now - retentionInDays * 86_400_000L;+ String groupPrefix = region + "::" + groupName + "::";+ for (String key : List.copyOf(keysForAccount(eventStore, accountId))) {+ if (!key.startsWith(groupPrefix)) {+ continue;+ }+ boolean expired = getForAccount(eventStore, accountId, key)+ .map(event -> event.getTimestamp() < cutoff).orElse(false);+ if (expired) {+ deleteForAccount(eventStore, accountId, key);+ }+ }+ }++ /**+ * Keeps an account's event store under {@link #maxStoredEvents} by dropping the oldest events.+ * The store is persisted as a single document rewritten in full on each flush, so its size+ * is the cost of every flush; without a ceiling a chatty function turns log ingestion into a+ * sustained disk writer.+ * <p>+ * The ceiling is best-effort rather than atomic: this method is not synchronized, so two+ * concurrent PutLogEvents calls for the same account can each scan and evict independently and+ * briefly overshoot the cap. The next ingest corrects the drift, which mirrors how CloudWatch+ * Logs itself deletes expired events lazily rather than at an exact boundary.+ */+ private void evictEventsBeyondCapacity(String accountId) {+ List<String> keys = List.copyOf(keysForAccount(eventStore, accountId));+ int excess = keys.size() - maxStoredEvents;+ if (excess <= 0) {+ return;+ }+ List<Map.Entry<String, LogEvent>> oldestFirst = new ArrayList<>(keys.size());+ for (String key : keys) {+ getForAccount(eventStore, accountId, key).ifPresent(event -> oldestFirst.add(Map.entry(key, event)));+ }+ oldestFirst.sort(Map.Entry.comparingByValue(EVENT_ORDER));+ for (Map.Entry<String, LogEvent> entry : oldestFirst.subList(0, Math.min(excess, oldestFirst.size()))) {+ deleteForAccount(eventStore, accountId, entry.getKey());+ }+ LOG.debugv("Evicted {0} oldest log event(s) to stay within the {1}-event store ceiling", excess, maxStoredEvents);+ }+private <V> Optional<V> getForAccount(StorageBackend<String, V> store, String accountId, String key) {if (accountId != null && store instanceof AccountAwareStorageBackend<?> rawAware) {@@ -599,6 +672,25 @@ public class CloudWatchLogsService implements ResourceProvider {store.put(key, value);}+ private <V> Set<String> keysForAccount(StorageBackend<String, V> store, String accountId) {+ if (accountId != null && store instanceof AccountAwareStorageBackend<?> rawAware) {+ @SuppressWarnings("unchecked")+ AccountAwareStorageBackend<V> aware = (AccountAwareStorageBackend<V>) rawAware;+ return aware.keysForAccount(accountId);+ }+ return store.keys();+ }++ private <V> void deleteForAccount(StorageBackend<String, V> store, String accountId, String key) {+ if (accountId != null && store instanceof AccountAwareStorageBackend<?> rawAware) {+ @SuppressWarnings("unchecked")+ AccountAwareStorageBackend<V> aware = (AccountAwareStorageBackend<V>) rawAware;+ aware.deleteForAccount(accountId, key);+ return;+ }+ store.delete(key);+ }+public record LogEventsResult(List<LogEvent> events, String nextForwardToken, String nextBackwardToken) {}public LogEventsResult getLogEvents(String groupName, String streamName,diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex f6d371a5bc..5b3c1b3389 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -152,7 +152,7 @@ floci:lambda:flush-interval-ms: 5000cloudwatchlogs:- flush-interval-ms: 5000+ flush-interval-ms: 15000cloudwatchmetrics:flush-interval-ms: 5000secretsmanager:@@ -412,6 +412,7 @@ floci:cloudwatchlogs:enabled: truemax-events-per-query: 10000+ max-stored-events: 20000cloudwatchmetrics:enabled: truesecretsmanager:diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.javaindex 6997432b96..f569ce3b57 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java@@ -108,6 +108,74 @@ class CloudWatchLogsServiceTest {.count());}+ @Test+ void explicitAccountRetentionEvictionLeavesOtherAccountEventsUntouched() {+ InMemoryStorage<String, LogGroup> rawGroups = new InMemoryStorage<>();+ InMemoryStorage<String, LogStream> rawStreams = new InMemoryStorage<>();+ InMemoryStorage<String, LogEvent> rawEvents = new InMemoryStorage<>();+ String accountA = "111111111111";+ String accountB = "222222222222";+ CloudWatchLogsService accountService = new CloudWatchLogsService(+ new AccountAwareStorageBackend<>(rawGroups, null, accountB),+ new AccountAwareStorageBackend<>(rawStreams, null, accountB),+ new AccountAwareStorageBackend<>(rawEvents, null, accountB),+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, accountB),+ 10_000, new RegionResolver(REGION, accountB));+ long now = System.currentTimeMillis();+ long twoDaysAgo = now - 2 * 86_400_000L;++ accountService.createLogGroupForAccount(accountA, "/app/logs", 1, null, REGION);+ accountService.createLogStreamForAccount(accountA, "/app/logs", "stream", REGION);+ accountService.createLogGroupForAccount(accountB, "/app/logs", null, null, REGION);+ accountService.createLogStreamForAccount(accountB, "/app/logs", "stream", REGION);+ accountService.putLogEventsForAccount(+ accountB, "/app/logs", "stream", eventsAt(twoDaysAgo), REGION);+ accountService.putLogEventsForAccount(+ accountA, "/app/logs", "stream", eventsAt(twoDaysAgo, now), REGION);++ String eventPrefix = REGION + "::/app/logs::stream::";+ assertEquals(1, rawEvents.keys().stream()+ .filter(key -> key.startsWith(accountA + "/" + eventPrefix))+ .count());+ assertEquals(1, rawEvents.keys().stream()+ .filter(key -> key.startsWith(accountB + "/" + eventPrefix))+ .count());+ }++ @Test+ void explicitAccountCapacityEvictionDoesNotCountOtherAccountEvents() {+ InMemoryStorage<String, LogGroup> rawGroups = new InMemoryStorage<>();+ InMemoryStorage<String, LogStream> rawStreams = new InMemoryStorage<>();+ InMemoryStorage<String, LogEvent> rawEvents = new InMemoryStorage<>();+ String accountA = "111111111111";+ String accountB = "222222222222";+ CloudWatchLogsService accountService = new CloudWatchLogsService(+ new AccountAwareStorageBackend<>(rawGroups, null, accountB),+ new AccountAwareStorageBackend<>(rawStreams, null, accountB),+ new AccountAwareStorageBackend<>(rawEvents, null, accountB),+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, accountB),+ 10_000, 2, new RegionResolver(REGION, accountB));++ accountService.createLogGroupForAccount(accountA, "/app/logs", null, null, REGION);+ accountService.createLogStreamForAccount(accountA, "/app/logs", "stream", REGION);+ for (long timestamp : List.of(1L, 2L, 3L)) {+ LogEvent event = new LogEvent();+ event.setTimestamp(timestamp);+ event.setEventId("event-" + timestamp);+ rawEvents.put(accountB + "/event-" + timestamp, event);+ }++ accountService.putLogEventsForAccount(+ accountA, "/app/logs", "stream", eventsAt(4L), REGION);++ assertEquals(3, rawEvents.keys().stream()+ .filter(key -> key.startsWith(accountB + "/"))+ .count());+ assertEquals(1, rawEvents.keys().stream()+ .filter(key -> key.startsWith(accountA + "/"))+ .count());+ }+@Testvoid deleteLogGroup() {service.createLogGroup("/app/logs", null, null, REGION);@@ -168,6 +236,89 @@ class CloudWatchLogsServiceTest {assertNull(group.getRetentionInDays());}+ // ──────────────────────────── Stored event ceiling ────────────────────────────++ private static CloudWatchLogsService serviceWithStoredEventCeiling(int maxStoredEvents) {+ return new CloudWatchLogsService(+ new InMemoryStorage<>(),+ new InMemoryStorage<>(),+ new InMemoryStorage<>(),+ new InMemoryStorage<>(),+ 10000,+ maxStoredEvents,+ new RegionResolver("us-east-1", "000000000000")+ );+ }++ private static List<Map<String, Object>> eventsAt(long... timestamps) {+ List<Map<String, Object>> events = new ArrayList<>();+ for (long timestamp : timestamps) {+ events.add(Map.of("timestamp", timestamp, "message", "event@" + timestamp));+ }+ return events;+ }++ private static List<Long> storedTimestamps(CloudWatchLogsService target, String group, String stream) {+ return target.getLogEvents(group, stream, null, null, 100, true, null, REGION).events().stream()+ .map(LogEvent::getTimestamp)+ .toList();+ }++ @Test+ void putLogEventsEvictsTheOldestEventsBeyondTheStoredEventCeiling() {+ CloudWatchLogsService capped = serviceWithStoredEventCeiling(3);+ capped.createLogGroup("/app/logs", null, null, REGION);+ capped.createLogStream("/app/logs", "stream-1", REGION);+ capped.createLogStream("/app/logs", "stream-2", REGION);++ capped.putLogEvents("/app/logs", "stream-1", eventsAt(1000, 2000, 3000), REGION);+ capped.putLogEvents("/app/logs", "stream-2", eventsAt(4000, 5000), REGION);++ assertEquals(List.of(3000L), storedTimestamps(capped, "/app/logs", "stream-1"),+ "the oldest events across every stream go first, the store stays within the ceiling");+ assertEquals(List.of(4000L, 5000L), storedTimestamps(capped, "/app/logs", "stream-2"));+ }++ @Test+ void putLogEventsKeepsEverythingWhileUnderTheStoredEventCeiling() {+ CloudWatchLogsService capped = serviceWithStoredEventCeiling(3);+ capped.createLogGroup("/app/logs", null, null, REGION);+ capped.createLogStream("/app/logs", "stream-1", REGION);++ capped.putLogEvents("/app/logs", "stream-1", eventsAt(1000, 2000, 3000), REGION);++ assertEquals(List.of(1000L, 2000L, 3000L), storedTimestamps(capped, "/app/logs", "stream-1"));+ }++ @Test+ void putLogEventsDropsEventsOlderThanTheGroupRetentionPolicy() {+ service.createLogGroup("/app/logs", 1, null, REGION);+ service.createLogStream("/app/logs", "stream-1", REGION);+ long now = System.currentTimeMillis();+ long twoDaysAgo = now - 2 * 86_400_000L;++ service.putLogEvents("/app/logs", "stream-1", eventsAt(twoDaysAgo, now), REGION);++ assertEquals(List.of(now), storedTimestamps(service, "/app/logs", "stream-1"),+ "events past the retention window are not kept on disk waiting for a background sweep");+ }++ @Test+ void putLogEventsLeavesOtherGroupsAloneWhenApplyingRetention() {+ service.createLogGroup("/short", 1, null, REGION);+ service.createLogStream("/short", "stream-1", REGION);+ service.createLogGroup("/forever", null, null, REGION);+ service.createLogStream("/forever", "stream-1", REGION);+ long now = System.currentTimeMillis();+ long twoDaysAgo = now - 2 * 86_400_000L;++ service.putLogEvents("/forever", "stream-1", eventsAt(twoDaysAgo), REGION);+ service.putLogEvents("/short", "stream-1", eventsAt(twoDaysAgo), REGION);++ assertEquals(List.of(twoDaysAgo), storedTimestamps(service, "/forever", "stream-1"));+ assertEquals(List.of(), storedTimestamps(service, "/short", "stream-1"));+ }+// ──────────────────────────── KMS key association ────────────────────────────@Test