Light, fluffy, and always free - The AWS Local Emulator alternative
perf(logs): evict past the ceiling and retention without scanning the store (#5167)
Robin Baeckmanc229311f8fc0
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.javaindex 36c5a69790..5fb6adac67 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java@@ -6,6 +6,7 @@ import io.github.hectorvent.floci.core.common.AwsArnUtils;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.Base64Url;import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.Resettable;import io.github.hectorvent.floci.core.resource.ExplorerResource;import io.github.hectorvent.floci.core.resource.ResourceProvider;import io.github.hectorvent.floci.core.resource.SupportedResourceType;@@ -47,7 +48,7 @@ import java.util.function.LongSupplier;import static io.github.hectorvent.floci.services.cloudwatch.logs.MetricFilterRules.invalid;@ApplicationScoped-public class CloudWatchLogsService implements ResourceProvider {+public class CloudWatchLogsService implements ResourceProvider, Resettable {private static final Logger LOG = Logger.getLogger(CloudWatchLogsService.class);private static final int MAX_REGEX_FILTERS_PER_LOG_GROUP = 5;@@ -67,6 +68,8 @@ public class CloudWatchLogsService implements ResourceProvider {private final StorageBackend<String, LogGroup> groupStore;private final StorageBackend<String, LogStream> streamStore;private final StorageBackend<String, LogEvent> eventStore;+ private final Map<String, LogEventIndex> eventIndexes = new ConcurrentHashMap<>();+ private volatile boolean resetting;private final StorageBackend<String, SubscriptionFilter> subscriptionFilterStore;private final StorageBackend<String, MetricFilter> metricFilterStore;private final StorageBackend<String, ResourcePolicy> resourcePolicyStore;@@ -225,6 +228,32 @@ public class CloudWatchLogsService implements ResourceProvider {this.logGroupDeleted = logGroupDeleted;}+ /** Each reset hook empties every index and marks it stale; the next write refills from the store. */+ @Override+ public void beforeReset() {+ resetting = true;+ clearEveryIndex();+ }++ @Override+ public void clear() {+ clearEveryIndex();+ }++ @Override+ public void afterReset() {+ clearEveryIndex();+ resetting = false;+ }++ private void clearEveryIndex() {+ for (LogEventIndex index : eventIndexes.values()) {+ synchronized (index) {+ index.clear();+ }+ }+ }+// ──────────────────────────── Log Groups ────────────────────────────public void createLogGroup(String name, Integer retentionInDays, Map<String, String> tags, String region) {@@ -304,8 +333,7 @@ public class CloudWatchLogsService implements ResourceProvider {LogStream stream = streamStore.get(sk).orElse(null);if (stream != null && name.equals(stream.getLogGroupName())&& sk.equals(streamKey(region, name, stream.getLogStreamName()))) {- deleteEventsForStream(region, name, stream.getLogStreamName());- streamStore.delete(sk);+ deleteStreamAndEvents(region, name, stream.getLogStreamName(), sk);}}metricFilterStore.keys().stream().filter(k -> k.startsWith(streamPrefix)@@ -483,8 +511,7 @@ public class CloudWatchLogsService implements ResourceProvider {.orElseThrow(() -> new AwsException("ResourceNotFoundException","The specified log stream does not exist: " + streamName, 400));- deleteEventsForStream(region, groupName, streamName);- streamStore.delete(streamKey);+ deleteStreamAndEvents(region, groupName, streamName, streamKey);LOG.infov("Deleted log stream: {0}/{1}", groupName, streamName);}@@ -637,7 +664,7 @@ public class CloudWatchLogsService implements ResourceProvider {String accountId, String groupName, String streamName,List<Map<String, Object>> events, String region) {String streamKey = streamKey(region, groupName, streamName);- LogStream stream = getForAccount(streamStore, accountId, streamKey)+ getForAccount(streamStore, accountId, streamKey).orElseThrow(() -> new AwsException("ResourceNotFoundException","The specified log stream does not exist: " + streamName, 400));@@ -667,25 +694,42 @@ public class CloudWatchLogsService implements ResourceProvider {if (minTs == null || ts < minTs) { minTs = ts; }if (maxTs == null || ts > maxTs) { maxTs = ts; }}- putAllForAccount(eventStore, accountId, logEvents);-- evictEventsPastRetention(accountId, region, groupName, now);- evictEventsBeyondCapacity(accountId);+ String nextToken = UUID.randomUUID().toString();+ LogEventIndex index = eventIndex(indexAccount(accountId));+ synchronized (index) {+ // Again under the lock: DeleteLogStream removes the stream and its events while holding it,+ // and the stream's metadata is written here too, so a put can't bring a deleted stream back.+ LogStream stream = getForAccount(streamStore, accountId, streamKey)+ .orElseThrow(() -> new AwsException("ResourceNotFoundException",+ "The specified log stream does not exist: " + streamName, 400));+ boolean done = false;+ try {+ refillIfStale(index, accountId);+ putAllForAccount(eventStore, accountId, logEvents);+ String group = streamKeyPrefix(region, groupName);+ logEvents.forEach((key, event) -> index.add(key, group, event));+ evictEventsPastRetention(accountId, index, region, groupName, now);+ evictEventsBeyondCapacity(accountId, index);+ done = true;+ } finally {+ if (!done || resetting) {+ index.markStale();+ }+ }- // Update stream metadata- if (minTs != null) {- if (stream.getFirstEventTimestamp() == null || minTs < stream.getFirstEventTimestamp()) {- stream.setFirstEventTimestamp(minTs);+ if (minTs != null) {+ if (stream.getFirstEventTimestamp() == null || minTs < stream.getFirstEventTimestamp()) {+ stream.setFirstEventTimestamp(minTs);+ }}+ if (maxTs != null) {+ stream.setLastEventTimestamp(maxTs);+ }+ stream.setLastIngestionTime(now);+ stream.setStoredBytes(stream.getStoredBytes() + totalBytes);+ stream.setUploadSequenceToken(nextToken);+ putForAccount(streamStore, accountId, streamKey, stream);}- if (maxTs != null) {- stream.setLastEventTimestamp(maxTs);- }- stream.setLastIngestionTime(now);- stream.setStoredBytes(stream.getStoredBytes() + totalBytes);- String nextToken = UUID.randomUUID().toString();- stream.setUploadSequenceToken(nextToken);- putForAccount(streamStore, accountId, streamKey, stream);if (logEventsIngested != null && !stored.isEmpty()) {logEventsIngested.fire(new LogEventsIngested(accountId, region, groupName, streamName, List.copyOf(stored)));@@ -698,7 +742,8 @@ public class CloudWatchLogsService implements ResourceProvider {* background; doing it on ingest keeps the on-disk store from growing past what a* {@code PutRetentionPolicy} promised.*/- private void evictEventsPastRetention(String accountId, String region, String groupName, long now) {+ private void evictEventsPastRetention(String accountId, LogEventIndex index, String region, String groupName,+ long now) {Integer retentionInDays = getForAccount(groupStore, accountId, groupKey(region, groupName)).map(LogGroup::getRetentionInDays).orElse(null);@@ -706,16 +751,9 @@ public class CloudWatchLogsService implements ResourceProvider {return;}long cutoff = now - retentionInDays * 86_400_000L;- String groupPrefix = region + "::" + groupName + "::";- for (String key : List.copyOf(keysForAccount(eventStore, accountId))) {- if (!key.startsWith(groupPrefix)) {- continue;- }- boolean expired = getForAccount(eventStore, accountId, key)- .map(event -> event.getTimestamp() < cutoff).orElse(false);- if (expired) {- deleteForAccount(eventStore, accountId, key);- }+ for (String key : index.olderThan(streamKeyPrefix(region, groupName), cutoff)) {+ deleteForAccount(eventStore, accountId, key);+ index.remove(key);}}@@ -726,28 +764,47 @@ public class CloudWatchLogsService implements ResourceProvider {* only on the compaction interval, but without a ceiling a chatty function would still grow* every snapshot without bound.* <p>- * The ceiling is best-effort rather than atomic: this method is not synchronized, so two- * concurrent PutLogEvents calls for the same account can each scan and evict independently and- * briefly overshoot the cap. The next ingest corrects the drift, which mirrors how CloudWatch- * Logs itself deletes expired events lazily rather than at an exact boundary.+ * The oldest events come from the account's {@link LogEventIndex}, so an ingest at the+ * ceiling costs a few index operations rather than a sort of the whole store.*/- private void evictEventsBeyondCapacity(String accountId) {- List<String> keys = List.copyOf(keysForAccount(eventStore, accountId));- int excess = keys.size() - maxStoredEvents;+ private void evictEventsBeyondCapacity(String accountId, LogEventIndex index) {+ int excess = index.size() - maxStoredEvents;if (excess <= 0) {return;}- List<Map.Entry<String, LogEvent>> oldestFirst = new ArrayList<>(keys.size());- for (String key : keys) {- getForAccount(eventStore, accountId, key).ifPresent(event -> oldestFirst.add(Map.entry(key, event)));- }- oldestFirst.sort(Map.Entry.comparingByValue(EVENT_ORDER));- for (Map.Entry<String, LogEvent> entry : oldestFirst.subList(0, Math.min(excess, oldestFirst.size()))) {- deleteForAccount(eventStore, accountId, entry.getKey());+ for (String key : index.oldest(excess)) {+ deleteForAccount(eventStore, accountId, key);+ index.remove(key);}LOG.debugv("Evicted {0} oldest log event(s) to stay within the {1}-event store ceiling", excess, maxStoredEvents);}+ /** The account's event index; every write to the account's events holds its monitor. */+ LogEventIndex eventIndex(String account) {+ return eventIndexes.computeIfAbsent(account, a -> new LogEventIndex());+ }++ /** Refills a stale index from the store. The caller holds the index's monitor. */+ private void refillIfStale(LogEventIndex index, String accountId) {+ if (!index.isStale()) {+ return;+ }+ index.startRefill();+ for (String key : keysForAccount(eventStore, accountId)) {+ getForAccount(eventStore, accountId, key)+ .ifPresent(event -> index.add(key, groupPrefixFromEventKey(key), event));+ }+ index.markFilled();+ }++ /** The partition {@code accountId} writes to, or the request's when null; "" for a plain store. */+ private String indexAccount(String accountId) {+ if (eventStore instanceof AccountAwareStorageBackend<?> aware) {+ return accountId != null ? accountId : aware.accountId();+ }+ return "";+ }+private <V> Optional<V> getForAccount(StorageBackend<String, V> store, String accountId, String key) {if (accountId != null && store instanceof AccountAwareStorageBackend<?> rawAware) {@@ -1282,14 +1339,32 @@ public class CloudWatchLogsService implements ResourceProvider {// ──────────────────────────── Helpers ────────────────────────────- private void deleteEventsForStream(String region, String groupName, String streamName) {+ /**+ * The record and its events go in one hold of the index lock, record first, so a put sees either+ * the stream with its events or neither, and a stream recreated under the same name cannot write+ * until the old one's events are gone.+ */+ private void deleteStreamAndEvents(String region, String groupName, String streamName, String streamKey) {String eventPrefix = eventKeyPrefix(region, groupName, streamName);String groupPrefix = streamKeyPrefix(region, groupName);- List<String> keys = eventStore.keys().stream()- .filter(k -> k.startsWith(eventPrefix)- && streamName.equals(streamNameFromEventKey(k, groupPrefix)))- .toList();- keys.forEach(eventStore::delete);+ LogEventIndex index = eventIndex(indexAccount(null));+ synchronized (index) {+ boolean done = false;+ try {+ streamStore.delete(streamKey);+ List<String> keys = eventStore.keys().stream()+ .filter(k -> k.startsWith(eventPrefix)+ && streamName.equals(streamNameFromEventKey(k, groupPrefix)))+ .toList();+ keys.forEach(eventStore::delete);+ keys.forEach(index::remove);+ done = true;+ } finally {+ if (!done || resetting) {+ index.markStale();+ }+ }+ }}public String buildArn(String groupName, String region) {@@ -1337,6 +1412,17 @@ public class CloudWatchLogsService implements ResourceProvider {: eventKey.substring(groupPrefix.length(), timestampSeparator);}+ /**+ * The {@code <region>::<group>::} an event key starts with. AWS allows no ':' in a stream name,+ * and the timestamp and uuid are generated here, so the prefix ends at the third "::" from the right.+ */+ private static String groupPrefixFromEventKey(String eventKey) {+ int uuidSeparator = eventKey.lastIndexOf("::");+ int timestampSeparator = eventKey.lastIndexOf("::", uuidSeparator - 1);+ int streamSeparator = eventKey.lastIndexOf("::", timestampSeparator - 1);+ return eventKey.substring(0, streamSeparator + 2);+ }+static String subscriptionFilterKeyPrefix(String region, String logGroupName) {return region + "::" + logGroupName + "::filter::";}diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/LogEventIndex.java b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/LogEventIndex.javanew file mode 100644index 0000000000..859499b06a--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/LogEventIndex.java@@ -0,0 +1,121 @@+package io.github.hectorvent.floci.services.cloudwatch.logs;++import io.github.hectorvent.floci.services.cloudwatch.logs.model.LogEvent;++import java.util.ArrayList;+import java.util.Comparator;+import java.util.HashMap;+import java.util.List;+import java.util.Map;+import java.util.NavigableSet;+import java.util.TreeSet;++/**+ * One account's stored event keys, oldest first, so the stored-event ceiling and the retention+ * policy find the events to drop without listing the whole store on every PutLogEvents.+ * Ordered like the store's events are read: timestamp, then ingestion sequence, then event id.+ * Not thread-safe; callers synchronize on the instance. An account keeps one instance, so its+ * monitor is the one lock for that account's writes; a stale index is refilled in place.+ */+final class LogEventIndex {++ private boolean stale = true;++ private record Entry(long timestamp, long sequence, String eventId, String key, String group) {}++ private static final Comparator<Entry> ORDER = Comparator.comparingLong(Entry::timestamp)+ .thenComparingLong(Entry::sequence)+ .thenComparing(Entry::eventId, Comparator.nullsFirst(Comparator.<String>naturalOrder()))+ .thenComparing(Entry::key);++ private final NavigableSet<Entry> oldestFirst = new TreeSet<>(ORDER);+ private final Map<String, NavigableSet<Entry>> byGroup = new HashMap<>();+ private final Map<String, Entry> byKey = new HashMap<>();++ /** Indexes the event under {@code group}, its {@code region::group::}. */+ void add(String key, String group, LogEvent event) {+ if (byKey.containsKey(key)) {+ return;+ }+ Entry entry = new Entry(event.getTimestamp(), event.getSequence(), event.getEventId(), key, group);+ byKey.put(key, entry);+ oldestFirst.add(entry);+ byGroup.computeIfAbsent(group, g -> new TreeSet<>(ORDER)).add(entry);+ }++ /** True until filled, and again after a write that may not have reached the store. */+ boolean isStale() {+ return stale;+ }++ void markStale() {+ stale = true;+ }++ /** Drops every entry and marks the index stale, for a store that was wiped. */+ void clear() {+ startRefill();+ markStale();+ }++ /** Empties the index for a refill from the store; {@link #markFilled()} ends the refill. */+ void startRefill() {+ oldestFirst.clear();+ byGroup.clear();+ byKey.clear();+ }++ void markFilled() {+ stale = false;+ }++ int size() {+ return oldestFirst.size();+ }++ /** The keys of the {@code count} oldest events, oldest first. Removes nothing. */+ List<String> oldest(int count) {+ List<String> keys = new ArrayList<>(Math.max(0, count));+ for (Entry entry : oldestFirst) {+ if (keys.size() >= count) {+ break;+ }+ keys.add(entry.key());+ }+ return keys;+ }++ /** The keys of the group's events older than {@code cutoff}, oldest first. Removes nothing. */+ List<String> olderThan(String group, long cutoff) {+ List<String> keys = new ArrayList<>();+ NavigableSet<Entry> events = byGroup.get(group);+ if (events != null) {+ for (Entry entry : events) {+ if (entry.timestamp() >= cutoff) {+ break;+ }+ keys.add(entry.key());+ }+ }+ return keys;+ }++ /** Removes the event stored under {@code key}, if the index holds it. */+ void remove(String key) {+ Entry entry = byKey.remove(key);+ if (entry != null) {+ oldestFirst.remove(entry);+ removeFromGroup(entry);+ }+ }++ private void removeFromGroup(Entry entry) {+ NavigableSet<Entry> events = byGroup.get(entry.group());+ if (events != null) {+ events.remove(entry);+ if (events.isEmpty()) {+ byGroup.remove(entry.group());+ }+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.javaindex 21d1ae5ded..bb928f887d 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java@@ -10,12 +10,19 @@ import io.github.hectorvent.floci.services.cloudwatch.logs.model.LogStream;import io.github.hectorvent.floci.services.cloudwatch.logs.model.SubscriptionFilter;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.Timeout;import org.junit.jupiter.params.ParameterizedTest;import org.junit.jupiter.params.provider.ValueSource;import java.util.ArrayList;+import java.util.Collections;import java.util.List;import java.util.Map;+import java.util.Optional;+import java.util.Set;+import java.util.concurrent.atomic.AtomicInteger;+import java.util.concurrent.atomic.AtomicReference;+import java.util.function.Predicate;import static org.junit.jupiter.api.Assertions.*;@@ -338,6 +345,115 @@ class CloudWatchLogsServiceTest {"events past the retention window are not kept on disk waiting for a background sweep");}+ @Test+ void aStreamDeletedAfterThePutChecksItLeavesNoEventsBehind() {+ // The stream is deleted between the put's first look at it and the put taking the lock.+ AtomicReference<Runnable> deleteLater = new AtomicReference<>();+ InMemoryStorage<String, LogStream> streams = new InMemoryStorage<>() {+ private boolean seen;++ @Override+ public Optional<LogStream> get(String key) {+ Optional<LogStream> stream = super.get(key);+ if (!seen && stream.isPresent() && stream.get().getLogStreamName().equals("doomed")) {+ seen = true;+ deleteLater.get().run();+ }+ return stream;+ }+ };+ InMemoryStorage<String, LogEvent> events = new InMemoryStorage<>();+ CloudWatchLogsService logs = new CloudWatchLogsService(new InMemoryStorage<>(), streams, events,+ new InMemoryStorage<>(), new InMemoryStorage<>(), 10000, 10000,+ new RegionResolver(REGION, "000000000000"));+ logs.createLogGroup("/app", null, null, REGION);+ logs.createLogStream("/app", "doomed", REGION);+ deleteLater.set(() -> logs.deleteLogStream("/app", "doomed", REGION));++ assertThrows(AwsException.class,+ () -> logs.putLogEvents("/app", "doomed", eventsAt(System.currentTimeMillis()), REGION));+ assertEquals(List.of(), List.copyOf(events.keys()));+ }++ @Test+ @Timeout(value = 10, threadMode = Timeout.ThreadMode.SEPARATE_THREAD)+ void aStreamRecreatedWhileTheOldOneIsDeletedKeepsItsNewEvents() throws InterruptedException {+ // As the old stream's record goes, another caller recreates it and writes to it.+ AtomicReference<Thread> recreate = new AtomicReference<>();+ InMemoryStorage<String, LogStream> streams = new InMemoryStorage<>() {+ @Override+ public void delete(String key) {+ super.delete(key);+ Thread writer = recreate.getAndSet(null);+ if (writer == null) {+ return;+ }+ startAndWaitUntilFinishedOrStuck(writer);+ }+ };+ InMemoryStorage<String, LogEvent> events = new InMemoryStorage<>();+ CloudWatchLogsService logs = new CloudWatchLogsService(new InMemoryStorage<>(), streams, events,+ new InMemoryStorage<>(), new InMemoryStorage<>(), 10000, 10000,+ new RegionResolver(REGION, "000000000000"));+ logs.createLogGroup("/app", null, null, REGION);+ logs.createLogStream("/app", "reused", REGION);+ long now = System.currentTimeMillis();+ logs.putLogEvents("/app", "reused", eventsAt(now - 1000), REGION);+ Thread writer = new Thread(() -> {+ logs.createLogStream("/app", "reused", REGION);+ logs.putLogEvents("/app", "reused", eventsAt(now), REGION);+ });+ recreate.set(writer);++ logs.deleteLogStream("/app", "reused", REGION);+ writer.join();++ assertEquals(List.of(now), storedTimestamps(logs, "/app", "reused"));+ }++ @Test+ @Timeout(value = 10, threadMode = Timeout.ThreadMode.SEPARATE_THREAD)+ void aStreamDeletedAsThePutRecordsItsMetadataStaysDeleted() throws InterruptedException {+ // The delete arrives as the put writes the stream's metadata back.+ AtomicReference<Thread> deleter = new AtomicReference<>();+ InMemoryStorage<String, LogStream> streams = new InMemoryStorage<>() {+ @Override+ public void put(String key, LogStream value) {+ Thread delete = deleter.getAndSet(null);+ if (delete != null) {+ startAndWaitUntilFinishedOrStuck(delete);+ }+ super.put(key, value);+ }+ };+ CloudWatchLogsService logs = new CloudWatchLogsService(new InMemoryStorage<>(), streams,+ new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(), 10000, 10000,+ new RegionResolver(REGION, "000000000000"));+ logs.createLogGroup("/app", null, null, REGION);+ logs.createLogStream("/app", "doomed", REGION);+ Thread delete = new Thread(() -> logs.deleteLogStream("/app", "doomed", REGION));+ deleter.set(delete);++ logs.putLogEvents("/app", "doomed", eventsAt(System.currentTimeMillis()), REGION);+ delete.join();++ assertEquals(List.of(), logs.describeLogStreams("/app", null, REGION));+ }++ /** Runs the thread until it ends or waits on a lock the caller may hold. */+ private static void startAndWaitUntilFinishedOrStuck(Thread thread) {+ thread.start();+ long deadline = System.nanoTime() + 5_000_000_000L;+ while (thread.isAlive() && System.nanoTime() < deadline) {+ Thread.State state = thread.getState();+ if (state == Thread.State.BLOCKED || state == Thread.State.WAITING+ || state == Thread.State.TIMED_WAITING) {+ return;+ }+ Thread.onSpinWait();+ }+ }+@Testvoid putLogEventsLeavesOtherGroupsAloneWhenApplyingRetention() {service.createLogGroup("/short", 1, null, REGION);@@ -354,6 +470,399 @@ class CloudWatchLogsServiceTest {assertEquals(List.of(), storedTimestamps(service, "/short", "stream-1"));}+ @Test+ void putLogEventsAtTheCeilingDoesNotEnumerateTheStorePerCall() {+ CountingStorage<LogEvent> events = new CountingStorage<>();+ CloudWatchLogsService capped = new CloudWatchLogsService(+ new InMemoryStorage<>(), new InMemoryStorage<>(), events,+ new InMemoryStorage<>(), new InMemoryStorage<>(),+ 10000, 100, new RegionResolver(REGION, "000000000000"));+ capped.createLogGroup("/app/logs", 30, null, REGION);+ capped.createLogStream("/app/logs", "stream-1", REGION);+ long now = System.currentTimeMillis();+ events.enumerations = 0;++ for (int i = 0; i < 1_000; i++) {+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(now + i), REGION);+ }++ // Exactly once: the first call fills the index.+ int listed = events.enumerations;+ assertEquals(100, events.keys().size(), "the store stays at the ceiling");+ assertEquals(1, listed,+ "1,000 single-event calls listed the whole store " + listed+ + " times; the index fills once and eviction should not grow with the store");+ }++ @Test+ void putLogEventsEvictsExactlyAfterAStreamIsDeleted() {+ CloudWatchLogsService capped = serviceWithStoredEventCeiling(3);+ capped.createLogGroup("/app/logs", null, null, REGION);+ capped.createLogStream("/app/logs", "kept", REGION);+ capped.createLogStream("/app/logs", "deleted", REGION);+ capped.putLogEvents("/app/logs", "kept", eventsAt(1000), REGION);+ capped.putLogEvents("/app/logs", "deleted", eventsAt(2000, 3000), REGION);++ capped.deleteLogStream("/app/logs", "deleted", REGION);+ capped.putLogEvents("/app/logs", "kept", eventsAt(4000), REGION);++ assertEquals(List.of(1000L, 4000L), storedTimestamps(capped, "/app/logs", "kept"),+ "the deleted stream's events no longer count toward the ceiling");+ }++ @Test+ void putLogEventsEvictsExactlyAfterAGroupIsDeleted() {+ CloudWatchLogsService capped = serviceWithStoredEventCeiling(3);+ capped.createLogGroup("/kept", null, null, REGION);+ capped.createLogStream("/kept", "stream-1", REGION);+ capped.createLogGroup("/deleted", null, null, REGION);+ capped.createLogStream("/deleted", "stream-1", REGION);+ capped.putLogEvents("/kept", "stream-1", eventsAt(1000), REGION);+ capped.putLogEvents("/deleted", "stream-1", eventsAt(2000, 3000), REGION);++ capped.deleteLogGroup("/deleted", REGION);+ capped.putLogEvents("/kept", "stream-1", eventsAt(4000), REGION);++ assertEquals(List.of(1000L, 4000L), storedTimestamps(capped, "/kept", "stream-1"));+ }++ @Test+ void putLogEventsCountsEventsStoredBeforeARestart() {+ InMemoryStorage<String, LogGroup> groups = new InMemoryStorage<>();+ InMemoryStorage<String, LogStream> streams = new InMemoryStorage<>();+ InMemoryStorage<String, LogEvent> events = new InMemoryStorage<>();+ CloudWatchLogsService before = new CloudWatchLogsService(groups, streams, events,+ new InMemoryStorage<>(), new InMemoryStorage<>(), 10000, 3,+ new RegionResolver(REGION, "000000000000"));+ before.createLogGroup("/app/logs", null, null, REGION);+ before.createLogStream("/app/logs", "stream-1", REGION);+ before.putLogEvents("/app/logs", "stream-1", eventsAt(1000, 2000, 3000), REGION);++ CloudWatchLogsService after = new CloudWatchLogsService(groups, streams, events,+ new InMemoryStorage<>(), new InMemoryStorage<>(), 10000, 3,+ new RegionResolver(REGION, "000000000000"));+ after.putLogEvents("/app/logs", "stream-1", eventsAt(4000), REGION);++ assertEquals(List.of(2000L, 3000L, 4000L), storedTimestamps(after, "/app/logs", "stream-1"));+ }++ @Test+ void putLogEventsDropsAnExpiredEventThatArrivesAfterNewerOnes() {+ service.createLogGroup("/app/logs", 1, null, REGION);+ service.createLogStream("/app/logs", "stream-1", REGION);+ long now = System.currentTimeMillis();++ service.putLogEvents("/app/logs", "stream-1", eventsAt(now), REGION);+ service.putLogEvents("/app/logs", "stream-1", eventsAt(now - 2 * 86_400_000L), REGION);++ assertEquals(List.of(now), storedTimestamps(service, "/app/logs", "stream-1"));+ }++ @Test+ void putLogEventsAfterAResetCountsOnlyWhatIsStoredSinceTheReset() {+ InMemoryStorage<String, LogGroup> groups = new InMemoryStorage<>();+ InMemoryStorage<String, LogStream> streams = new InMemoryStorage<>();+ InMemoryStorage<String, LogEvent> events = new InMemoryStorage<>();+ CloudWatchLogsService capped = new CloudWatchLogsService(groups, streams, events,+ new InMemoryStorage<>(), new InMemoryStorage<>(), 10000, 3,+ new RegionResolver(REGION, "000000000000"));+ capped.createLogGroup("/app/logs", null, null, REGION);+ capped.createLogStream("/app/logs", "stream-1", REGION);+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(1000, 2000, 3000), REGION);++ groups.clear();+ streams.clear();+ events.clear();+ capped.clear();+ capped.createLogGroup("/app/logs", null, null, REGION);+ capped.createLogStream("/app/logs", "stream-1", REGION);+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(500), REGION);++ assertEquals(List.of(500L), storedTimestamps(capped, "/app/logs", "stream-1"),+ "events wiped by the reset no longer count toward the ceiling");+ }++ @Test+ void eachAccountKeepsItsOwnCeiling() {+ InMemoryStorage<String, LogEvent> rawEvents = new InMemoryStorage<>();+ String accountA = "111111111111";+ String accountB = "222222222222";+ CloudWatchLogsService accountService = new CloudWatchLogsService(+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, accountB),+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, accountB),+ new AccountAwareStorageBackend<>(rawEvents, null, accountB),+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, accountB),+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, accountB),+ 10_000, 2, new RegionResolver(REGION, accountB));+ for (String account : List.of(accountA, accountB)) {+ accountService.createLogGroupForAccount(account, "/app/logs", null, null, REGION);+ accountService.createLogStreamForAccount(account, "/app/logs", "stream", REGION);+ }++ accountService.putLogEventsForAccount(accountA, "/app/logs", "stream", eventsAt(1, 2, 3), REGION);+ accountService.putLogEventsForAccount(accountB, "/app/logs", "stream", eventsAt(4), REGION);+ accountService.putLogEventsForAccount(accountA, "/app/logs", "stream", eventsAt(5), REGION);++ assertEquals(List.of(3L, 5L), rawTimestamps(rawEvents, accountA),+ "account A drops its own oldest, whatever account B stores");+ assertEquals(List.of(4L), rawTimestamps(rawEvents, accountB));+ }++ private static List<Long> rawTimestamps(InMemoryStorage<String, LogEvent> rawEvents, String account) {+ return rawEvents.keys().stream()+ .filter(key -> key.startsWith(account + "/"))+ .map(key -> rawEvents.get(key).orElseThrow().getTimestamp())+ .sorted()+ .toList();+ }++ @Test+ void aStreamDeletedInTheRequestAccountLeavesThatAccountsCeilingExact() {+ String account = "222222222222";+ CloudWatchLogsService accountService = new CloudWatchLogsService(+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, account),+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, account),+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, account),+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, account),+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, account),+ 10_000, 3, new RegionResolver(REGION, account));+ accountService.createLogGroupForAccount(account, "/app/logs", null, null, REGION);+ accountService.createLogStreamForAccount(account, "/app/logs", "kept", REGION);+ accountService.createLogStreamForAccount(account, "/app/logs", "deleted", REGION);+ accountService.putLogEventsForAccount(account, "/app/logs", "kept", eventsAt(1000), REGION);+ accountService.putLogEventsForAccount(account, "/app/logs", "deleted", eventsAt(2000, 3000), REGION);++ accountService.deleteLogStream("/app/logs", "deleted", REGION);+ accountService.putLogEventsForAccount(account, "/app/logs", "kept", eventsAt(4000), REGION);++ assertEquals(List.of(1000L, 4000L), storedTimestamps(accountService, "/app/logs", "kept"));+ }++ @Test+ // Unsynchronized writers can corrupt a TreeSet into a loop that never ends: fail, don't hang.+ @Timeout(60)+ void concurrentPutsAtTheCeilingKeepExactlyTheNewestEvents() throws InterruptedException {+ CeilingWatchStorage events = new CeilingWatchStorage();+ CloudWatchLogsService capped = new CloudWatchLogsService(+ new InMemoryStorage<>(), new InMemoryStorage<>(), events,+ new InMemoryStorage<>(), new InMemoryStorage<>(),+ 10000, 100, new RegionResolver(REGION, "000000000000"));+ capped.createLogGroup("/app/logs", null, null, REGION);+ capped.createLogStream("/app/logs", "stream-1", REGION);+ int threads = 8;+ int perThread = 250;+ List<Thread> workers = new ArrayList<>();+ List<Throwable> failures = Collections.synchronizedList(new ArrayList<>());+ for (int t = 0; t < threads; t++) {+ int base = t * perThread;+ Thread worker = new Thread(() -> {+ try {+ for (int i = 0; i < perThread; i++) {+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(1000L + base + i), REGION);+ }+ } catch (Throwable e) {+ failures.add(e);+ }+ });+ workers.add(worker);+ worker.start();+ }+ for (Thread worker : workers) {+ worker.join();+ }++ assertEquals(List.of(), failures);+ assertTrue(events.largestBeforeAWrite.get() <= 100,+ "a write began with " + events.largestBeforeAWrite.get() + " events stored, over the ceiling");+ List<Long> newest = new ArrayList<>();+ for (long ts = 1000L + threads * perThread - 100; ts < 1000L + threads * perThread; ts++) {+ newest.add(ts);+ }+ assertEquals(newest, storedTimestamps(capped, "/app/logs", "stream-1"));+ }++ @Test+ void aStoreWriteThatFailsPartWayIsCountedOnTheNextCall() {+ FailingOnceStorage events = new FailingOnceStorage();+ CloudWatchLogsService capped = new CloudWatchLogsService(+ new InMemoryStorage<>(), new InMemoryStorage<>(), events,+ new InMemoryStorage<>(), new InMemoryStorage<>(),+ 10000, 3, new RegionResolver(REGION, "000000000000"));+ capped.createLogGroup("/app/logs", null, null, REGION);+ capped.createLogStream("/app/logs", "stream-1", REGION);+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(1000, 2000), REGION);++ events.failNextPutAll = true;+ assertThrows(IllegalStateException.class,+ () -> capped.putLogEvents("/app/logs", "stream-1", eventsAt(3000, 4000), REGION));+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(5000), REGION);++ assertEquals(List.of(3000L, 4000L, 5000L), storedTimestamps(capped, "/app/logs", "stream-1"),+ "events the failed call did store still count toward the ceiling");+ }++ @Test+ void anErrorAfterTheStoreWriteIsCountedOnTheNextCall() {+ FailingOnceStorage events = new FailingOnceStorage();+ CloudWatchLogsService capped = new CloudWatchLogsService(+ new InMemoryStorage<>(), new InMemoryStorage<>(), events,+ new InMemoryStorage<>(), new InMemoryStorage<>(),+ 10000, 3, new RegionResolver(REGION, "000000000000"));+ capped.createLogGroup("/app/logs", null, null, REGION);+ capped.createLogStream("/app/logs", "stream-1", REGION);+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(1000, 2000), REGION);++ events.errorNextPutAll = true;+ assertThrows(OutOfMemoryError.class,+ () -> capped.putLogEvents("/app/logs", "stream-1", eventsAt(3000, 4000), REGION));+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(5000), REGION);++ assertEquals(List.of(3000L, 4000L, 5000L), storedTimestamps(capped, "/app/logs", "stream-1"),+ "an Error, not only a RuntimeException, leaves the index to be refilled");+ }++ @Test+ void anAccountKeepsOneIndexThroughAFailedWriteAndAReset() {+ // Writers queued on an index's monitor must never find a different index in the map,+ // or two writers would change one account's events under two locks.+ FailingOnceStorage events = new FailingOnceStorage();+ CloudWatchLogsService capped = new CloudWatchLogsService(+ new InMemoryStorage<>(), new InMemoryStorage<>(), events,+ new InMemoryStorage<>(), new InMemoryStorage<>(),+ 10000, 3, new RegionResolver(REGION, "000000000000"));+ capped.createLogGroup("/app/logs", null, null, REGION);+ capped.createLogStream("/app/logs", "stream-1", REGION);+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(1000), REGION);+ LogEventIndex index = capped.eventIndex("");++ events.failNextPutAll = true;+ assertThrows(IllegalStateException.class,+ () -> capped.putLogEvents("/app/logs", "stream-1", eventsAt(2000), REGION));+ assertSame(index, capped.eventIndex(""), "a failed write keeps the account's index");++ capped.beforeReset();+ capped.clear();+ capped.afterReset();+ assertSame(index, capped.eventIndex(""), "a reset keeps the account's index");+ }++ @Test+ void aResetReleasesTheIndexedEventsWithoutWaitingForTheNextWrite() {+ CloudWatchLogsService capped = serviceWithStoredEventCeiling(10);+ capped.createLogGroup("/app/logs", null, null, REGION);+ capped.createLogStream("/app/logs", "stream-1", REGION);+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(1000, 2000, 3000), REGION);+ LogEventIndex index = capped.eventIndex("");+ assertEquals(3, index.size());++ capped.beforeReset();+ assertEquals(0, index.size(), "beforeReset() alone releases the entries");+ assertTrue(index.isStale(), "the next write still refills from the store");++ capped.putLogEvents("/app/logs", "stream-1", eventsAt(4000), REGION);+ capped.clear();+ assertEquals(0, index.size(), "clear() alone releases what was written during the reset");+ assertTrue(index.isStale());++ capped.putLogEvents("/app/logs", "stream-1", eventsAt(5000), REGION);+ capped.afterReset();+ assertEquals(0, index.size(), "afterReset() alone releases what was written before it");+ assertTrue(index.isStale());+ }++ @Test+ void putLogEventsDuringAResetDoNotSkewTheCeilingAfterIt() {+ InMemoryStorage<String, LogGroup> groups = new InMemoryStorage<>();+ InMemoryStorage<String, LogStream> streams = new InMemoryStorage<>();+ InMemoryStorage<String, LogEvent> events = new InMemoryStorage<>();+ CloudWatchLogsService capped = new CloudWatchLogsService(groups, streams, events,+ new InMemoryStorage<>(), new InMemoryStorage<>(), 10000, 3,+ new RegionResolver(REGION, "000000000000"));+ capped.createLogGroup("/app/logs", null, null, REGION);+ capped.createLogStream("/app/logs", "stream-1", REGION);+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(1000, 2000, 3000), REGION);++ // The order EmulatorInfoController resets in: beforeReset, the storage wipe, clear,+ // afterReset. Calls land on both sides of the wipe, before the service is told.+ capped.beforeReset();+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(4000), REGION);+ groups.clear();+ streams.clear();+ events.clear();+ capped.createLogGroup("/app/logs", null, null, REGION);+ capped.createLogStream("/app/logs", "stream-1", REGION);+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(500), REGION);+ capped.clear();+ capped.afterReset();+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(600, 700), REGION);++ assertEquals(List.of(500L, 600L, 700L), storedTimestamps(capped, "/app/logs", "stream-1"),+ "events wiped by the reset do not count toward the ceiling, before clear() or after");+ }++ @Test+ void aStoredEventWithoutAnIdDoesNotBreakIngest() {+ InMemoryStorage<String, LogEvent> events = new InMemoryStorage<>();+ CloudWatchLogsService capped = new CloudWatchLogsService(+ new InMemoryStorage<>(), new InMemoryStorage<>(), events,+ new InMemoryStorage<>(), new InMemoryStorage<>(),+ 10000, 2, new RegionResolver(REGION, "000000000000"));+ capped.createLogGroup("/app/logs", null, null, REGION);+ capped.createLogStream("/app/logs", "stream-1", REGION);+ LogEvent legacy = new LogEvent();+ legacy.setTimestamp(1000);+ legacy.setMessage("legacy");+ events.put(REGION + "::/app/logs::stream-1::000000000001000::legacy", legacy);++ capped.putLogEvents("/app/logs", "stream-1", eventsAt(2000, 3000), REGION);++ assertEquals(List.of(2000L, 3000L), storedTimestamps(capped, "/app/logs", "stream-1"));+ }++ private static final class FailingOnceStorage extends InMemoryStorage<String, LogEvent> {+ boolean failNextPutAll;+ boolean errorNextPutAll;++ @Override+ public void putAll(Map<String, LogEvent> entries) {+ super.putAll(entries);+ if (failNextPutAll) {+ failNextPutAll = false;+ throw new IllegalStateException("journal write failed after the entries were stored");+ }+ if (errorNextPutAll) {+ errorNextPutAll = false;+ throw new OutOfMemoryError("ran out after the entries were stored");+ }+ }+ }++ private static final class CeilingWatchStorage extends InMemoryStorage<String, LogEvent> {+ final AtomicInteger largestBeforeAWrite = new AtomicInteger();++ @Override+ public void putAll(Map<String, LogEvent> entries) {+ largestBeforeAWrite.accumulateAndGet(keys().size(), Math::max);+ super.putAll(entries);+ }+ }++ private static final class CountingStorage<V> extends InMemoryStorage<String, V> {+ int enumerations;++ @Override+ public Set<String> keys() {+ enumerations++;+ return super.keys();+ }++ @Override+ public List<V> scan(Predicate<String> keyFilter) {+ enumerations++;+ return super.scan(keyFilter);+ }+ }+// ──────────────────────────── KMS key association ────────────────────────────@Test