Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

perf(logs): evict past the ceiling and retention without scanning the store (#5167)

Robin Baeckmanc229311f8fc0
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java
index 36c5a69790..5fb6adac67 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java
@@ -6,6 +6,7 @@ import io.github.hectorvent.floci.core.common.AwsArnUtils;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.Base64Url;
import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.common.Resettable;
import io.github.hectorvent.floci.core.resource.ExplorerResource;
import io.github.hectorvent.floci.core.resource.ResourceProvider;
import io.github.hectorvent.floci.core.resource.SupportedResourceType;
@@ -47,7 +48,7 @@ import java.util.function.LongSupplier;
import static io.github.hectorvent.floci.services.cloudwatch.logs.MetricFilterRules.invalid;
@ApplicationScoped
-public class CloudWatchLogsService implements ResourceProvider {
+public class CloudWatchLogsService implements ResourceProvider, Resettable {
private static final Logger LOG = Logger.getLogger(CloudWatchLogsService.class);
private static final int MAX_REGEX_FILTERS_PER_LOG_GROUP = 5;
@@ -67,6 +68,8 @@ public class CloudWatchLogsService implements ResourceProvider {
private final StorageBackend<String, LogGroup> groupStore;
private final StorageBackend<String, LogStream> streamStore;
private final StorageBackend<String, LogEvent> eventStore;
+ private final Map<String, LogEventIndex> eventIndexes = new ConcurrentHashMap<>();
+ private volatile boolean resetting;
private final StorageBackend<String, SubscriptionFilter> subscriptionFilterStore;
private final StorageBackend<String, MetricFilter> metricFilterStore;
private final StorageBackend<String, ResourcePolicy> resourcePolicyStore;
@@ -225,6 +228,32 @@ public class CloudWatchLogsService implements ResourceProvider {
this.logGroupDeleted = logGroupDeleted;
}
+ /** Each reset hook empties every index and marks it stale; the next write refills from the store. */
+ @Override
+ public void beforeReset() {
+ resetting = true;
+ clearEveryIndex();
+ }
+
+ @Override
+ public void clear() {
+ clearEveryIndex();
+ }
+
+ @Override
+ public void afterReset() {
+ clearEveryIndex();
+ resetting = false;
+ }
+
+ private void clearEveryIndex() {
+ for (LogEventIndex index : eventIndexes.values()) {
+ synchronized (index) {
+ index.clear();
+ }
+ }
+ }
+
// ──────────────────────────── Log Groups ────────────────────────────
public void createLogGroup(String name, Integer retentionInDays, Map<String, String> tags, String region) {
@@ -304,8 +333,7 @@ public class CloudWatchLogsService implements ResourceProvider {
LogStream stream = streamStore.get(sk).orElse(null);
if (stream != null && name.equals(stream.getLogGroupName())
&& sk.equals(streamKey(region, name, stream.getLogStreamName()))) {
- deleteEventsForStream(region, name, stream.getLogStreamName());
- streamStore.delete(sk);
+ deleteStreamAndEvents(region, name, stream.getLogStreamName(), sk);
}
}
metricFilterStore.keys().stream().filter(k -> k.startsWith(streamPrefix)
@@ -483,8 +511,7 @@ public class CloudWatchLogsService implements ResourceProvider {
.orElseThrow(() -> new AwsException("ResourceNotFoundException",
"The specified log stream does not exist: " + streamName, 400));
- deleteEventsForStream(region, groupName, streamName);
- streamStore.delete(streamKey);
+ deleteStreamAndEvents(region, groupName, streamName, streamKey);
LOG.infov("Deleted log stream: {0}/{1}", groupName, streamName);
}
@@ -637,7 +664,7 @@ public class CloudWatchLogsService implements ResourceProvider {
String accountId, String groupName, String streamName,
List<Map<String, Object>> events, String region) {
String streamKey = streamKey(region, groupName, streamName);
- LogStream stream = getForAccount(streamStore, accountId, streamKey)
+ getForAccount(streamStore, accountId, streamKey)
.orElseThrow(() -> new AwsException("ResourceNotFoundException",
"The specified log stream does not exist: " + streamName, 400));
@@ -667,25 +694,42 @@ public class CloudWatchLogsService implements ResourceProvider {
if (minTs == null || ts < minTs) { minTs = ts; }
if (maxTs == null || ts > maxTs) { maxTs = ts; }
}
- putAllForAccount(eventStore, accountId, logEvents);
-
- evictEventsPastRetention(accountId, region, groupName, now);
- evictEventsBeyondCapacity(accountId);
+ String nextToken = UUID.randomUUID().toString();
+ LogEventIndex index = eventIndex(indexAccount(accountId));
+ synchronized (index) {
+ // Again under the lock: DeleteLogStream removes the stream and its events while holding it,
+ // and the stream's metadata is written here too, so a put can't bring a deleted stream back.
+ LogStream stream = getForAccount(streamStore, accountId, streamKey)
+ .orElseThrow(() -> new AwsException("ResourceNotFoundException",
+ "The specified log stream does not exist: " + streamName, 400));
+ boolean done = false;
+ try {
+ refillIfStale(index, accountId);
+ putAllForAccount(eventStore, accountId, logEvents);
+ String group = streamKeyPrefix(region, groupName);
+ logEvents.forEach((key, event) -> index.add(key, group, event));
+ evictEventsPastRetention(accountId, index, region, groupName, now);
+ evictEventsBeyondCapacity(accountId, index);
+ done = true;
+ } finally {
+ if (!done || resetting) {
+ index.markStale();
+ }
+ }
- // Update stream metadata
- if (minTs != null) {
- if (stream.getFirstEventTimestamp() == null || minTs < stream.getFirstEventTimestamp()) {
- stream.setFirstEventTimestamp(minTs);
+ if (minTs != null) {
+ if (stream.getFirstEventTimestamp() == null || minTs < stream.getFirstEventTimestamp()) {
+ stream.setFirstEventTimestamp(minTs);
+ }
}
+ if (maxTs != null) {
+ stream.setLastEventTimestamp(maxTs);
+ }
+ stream.setLastIngestionTime(now);
+ stream.setStoredBytes(stream.getStoredBytes() + totalBytes);
+ stream.setUploadSequenceToken(nextToken);
+ putForAccount(streamStore, accountId, streamKey, stream);
}
- if (maxTs != null) {
- stream.setLastEventTimestamp(maxTs);
- }
- stream.setLastIngestionTime(now);
- stream.setStoredBytes(stream.getStoredBytes() + totalBytes);
- String nextToken = UUID.randomUUID().toString();
- stream.setUploadSequenceToken(nextToken);
- putForAccount(streamStore, accountId, streamKey, stream);
if (logEventsIngested != null && !stored.isEmpty()) {
logEventsIngested.fire(new LogEventsIngested(accountId, region, groupName, streamName, List.copyOf(stored)));
@@ -698,7 +742,8 @@ public class CloudWatchLogsService implements ResourceProvider {
* background; doing it on ingest keeps the on-disk store from growing past what a
* {@code PutRetentionPolicy} promised.
*/
- private void evictEventsPastRetention(String accountId, String region, String groupName, long now) {
+ private void evictEventsPastRetention(String accountId, LogEventIndex index, String region, String groupName,
+ long now) {
Integer retentionInDays = getForAccount(groupStore, accountId, groupKey(region, groupName))
.map(LogGroup::getRetentionInDays)
.orElse(null);
@@ -706,16 +751,9 @@ public class CloudWatchLogsService implements ResourceProvider {
return;
}
long cutoff = now - retentionInDays * 86_400_000L;
- String groupPrefix = region + "::" + groupName + "::";
- for (String key : List.copyOf(keysForAccount(eventStore, accountId))) {
- if (!key.startsWith(groupPrefix)) {
- continue;
- }
- boolean expired = getForAccount(eventStore, accountId, key)
- .map(event -> event.getTimestamp() < cutoff).orElse(false);
- if (expired) {
- deleteForAccount(eventStore, accountId, key);
- }
+ for (String key : index.olderThan(streamKeyPrefix(region, groupName), cutoff)) {
+ deleteForAccount(eventStore, accountId, key);
+ index.remove(key);
}
}
@@ -726,28 +764,47 @@ public class CloudWatchLogsService implements ResourceProvider {
* only on the compaction interval, but without a ceiling a chatty function would still grow
* every snapshot without bound.
* <p>
- * The ceiling is best-effort rather than atomic: this method is not synchronized, so two
- * concurrent PutLogEvents calls for the same account can each scan and evict independently and
- * briefly overshoot the cap. The next ingest corrects the drift, which mirrors how CloudWatch
- * Logs itself deletes expired events lazily rather than at an exact boundary.
+ * The oldest events come from the account's {@link LogEventIndex}, so an ingest at the
+ * ceiling costs a few index operations rather than a sort of the whole store.
*/
- private void evictEventsBeyondCapacity(String accountId) {
- List<String> keys = List.copyOf(keysForAccount(eventStore, accountId));
- int excess = keys.size() - maxStoredEvents;
+ private void evictEventsBeyondCapacity(String accountId, LogEventIndex index) {
+ int excess = index.size() - maxStoredEvents;
if (excess <= 0) {
return;
}
- List<Map.Entry<String, LogEvent>> oldestFirst = new ArrayList<>(keys.size());
- for (String key : keys) {
- getForAccount(eventStore, accountId, key).ifPresent(event -> oldestFirst.add(Map.entry(key, event)));
- }
- oldestFirst.sort(Map.Entry.comparingByValue(EVENT_ORDER));
- for (Map.Entry<String, LogEvent> entry : oldestFirst.subList(0, Math.min(excess, oldestFirst.size()))) {
- deleteForAccount(eventStore, accountId, entry.getKey());
+ for (String key : index.oldest(excess)) {
+ deleteForAccount(eventStore, accountId, key);
+ index.remove(key);
}
LOG.debugv("Evicted {0} oldest log event(s) to stay within the {1}-event store ceiling", excess, maxStoredEvents);
}
+ /** The account's event index; every write to the account's events holds its monitor. */
+ LogEventIndex eventIndex(String account) {
+ return eventIndexes.computeIfAbsent(account, a -> new LogEventIndex());
+ }
+
+ /** Refills a stale index from the store. The caller holds the index's monitor. */
+ private void refillIfStale(LogEventIndex index, String accountId) {
+ if (!index.isStale()) {
+ return;
+ }
+ index.startRefill();
+ for (String key : keysForAccount(eventStore, accountId)) {
+ getForAccount(eventStore, accountId, key)
+ .ifPresent(event -> index.add(key, groupPrefixFromEventKey(key), event));
+ }
+ index.markFilled();
+ }
+
+ /** The partition {@code accountId} writes to, or the request's when null; "" for a plain store. */
+ private String indexAccount(String accountId) {
+ if (eventStore instanceof AccountAwareStorageBackend<?> aware) {
+ return accountId != null ? accountId : aware.accountId();
+ }
+ return "";
+ }
+
private <V> Optional<V> getForAccount(
StorageBackend<String, V> store, String accountId, String key) {
if (accountId != null && store instanceof AccountAwareStorageBackend<?> rawAware) {
@@ -1282,14 +1339,32 @@ public class CloudWatchLogsService implements ResourceProvider {
// ──────────────────────────── Helpers ────────────────────────────
- private void deleteEventsForStream(String region, String groupName, String streamName) {
+ /**
+ * The record and its events go in one hold of the index lock, record first, so a put sees either
+ * the stream with its events or neither, and a stream recreated under the same name cannot write
+ * until the old one's events are gone.
+ */
+ private void deleteStreamAndEvents(String region, String groupName, String streamName, String streamKey) {
String eventPrefix = eventKeyPrefix(region, groupName, streamName);
String groupPrefix = streamKeyPrefix(region, groupName);
- List<String> keys = eventStore.keys().stream()
- .filter(k -> k.startsWith(eventPrefix)
- && streamName.equals(streamNameFromEventKey(k, groupPrefix)))
- .toList();
- keys.forEach(eventStore::delete);
+ LogEventIndex index = eventIndex(indexAccount(null));
+ synchronized (index) {
+ boolean done = false;
+ try {
+ streamStore.delete(streamKey);
+ List<String> keys = eventStore.keys().stream()
+ .filter(k -> k.startsWith(eventPrefix)
+ && streamName.equals(streamNameFromEventKey(k, groupPrefix)))
+ .toList();
+ keys.forEach(eventStore::delete);
+ keys.forEach(index::remove);
+ done = true;
+ } finally {
+ if (!done || resetting) {
+ index.markStale();
+ }
+ }
+ }
}
public String buildArn(String groupName, String region) {
@@ -1337,6 +1412,17 @@ public class CloudWatchLogsService implements ResourceProvider {
: eventKey.substring(groupPrefix.length(), timestampSeparator);
}
+ /**
+ * The {@code <region>::<group>::} an event key starts with. AWS allows no ':' in a stream name,
+ * and the timestamp and uuid are generated here, so the prefix ends at the third "::" from the right.
+ */
+ private static String groupPrefixFromEventKey(String eventKey) {
+ int uuidSeparator = eventKey.lastIndexOf("::");
+ int timestampSeparator = eventKey.lastIndexOf("::", uuidSeparator - 1);
+ int streamSeparator = eventKey.lastIndexOf("::", timestampSeparator - 1);
+ return eventKey.substring(0, streamSeparator + 2);
+ }
+
static String subscriptionFilterKeyPrefix(String region, String logGroupName) {
return region + "::" + logGroupName + "::filter::";
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/LogEventIndex.java b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/LogEventIndex.java
new file mode 100644
index 0000000000..859499b06a
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/LogEventIndex.java
@@ -0,0 +1,121 @@
+package io.github.hectorvent.floci.services.cloudwatch.logs;
+
+import io.github.hectorvent.floci.services.cloudwatch.logs.model.LogEvent;
+
+import java.util.ArrayList;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.NavigableSet;
+import java.util.TreeSet;
+
+/**
+ * One account's stored event keys, oldest first, so the stored-event ceiling and the retention
+ * policy find the events to drop without listing the whole store on every PutLogEvents.
+ * Ordered like the store's events are read: timestamp, then ingestion sequence, then event id.
+ * Not thread-safe; callers synchronize on the instance. An account keeps one instance, so its
+ * monitor is the one lock for that account's writes; a stale index is refilled in place.
+ */
+final class LogEventIndex {
+
+ private boolean stale = true;
+
+ private record Entry(long timestamp, long sequence, String eventId, String key, String group) {}
+
+ private static final Comparator<Entry> ORDER = Comparator.comparingLong(Entry::timestamp)
+ .thenComparingLong(Entry::sequence)
+ .thenComparing(Entry::eventId, Comparator.nullsFirst(Comparator.<String>naturalOrder()))
+ .thenComparing(Entry::key);
+
+ private final NavigableSet<Entry> oldestFirst = new TreeSet<>(ORDER);
+ private final Map<String, NavigableSet<Entry>> byGroup = new HashMap<>();
+ private final Map<String, Entry> byKey = new HashMap<>();
+
+ /** Indexes the event under {@code group}, its {@code region::group::}. */
+ void add(String key, String group, LogEvent event) {
+ if (byKey.containsKey(key)) {
+ return;
+ }
+ Entry entry = new Entry(event.getTimestamp(), event.getSequence(), event.getEventId(), key, group);
+ byKey.put(key, entry);
+ oldestFirst.add(entry);
+ byGroup.computeIfAbsent(group, g -> new TreeSet<>(ORDER)).add(entry);
+ }
+
+ /** True until filled, and again after a write that may not have reached the store. */
+ boolean isStale() {
+ return stale;
+ }
+
+ void markStale() {
+ stale = true;
+ }
+
+ /** Drops every entry and marks the index stale, for a store that was wiped. */
+ void clear() {
+ startRefill();
+ markStale();
+ }
+
+ /** Empties the index for a refill from the store; {@link #markFilled()} ends the refill. */
+ void startRefill() {
+ oldestFirst.clear();
+ byGroup.clear();
+ byKey.clear();
+ }
+
+ void markFilled() {
+ stale = false;
+ }
+
+ int size() {
+ return oldestFirst.size();
+ }
+
+ /** The keys of the {@code count} oldest events, oldest first. Removes nothing. */
+ List<String> oldest(int count) {
+ List<String> keys = new ArrayList<>(Math.max(0, count));
+ for (Entry entry : oldestFirst) {
+ if (keys.size() >= count) {
+ break;
+ }
+ keys.add(entry.key());
+ }
+ return keys;
+ }
+
+ /** The keys of the group's events older than {@code cutoff}, oldest first. Removes nothing. */
+ List<String> olderThan(String group, long cutoff) {
+ List<String> keys = new ArrayList<>();
+ NavigableSet<Entry> events = byGroup.get(group);
+ if (events != null) {
+ for (Entry entry : events) {
+ if (entry.timestamp() >= cutoff) {
+ break;
+ }
+ keys.add(entry.key());
+ }
+ }
+ return keys;
+ }
+
+ /** Removes the event stored under {@code key}, if the index holds it. */
+ void remove(String key) {
+ Entry entry = byKey.remove(key);
+ if (entry != null) {
+ oldestFirst.remove(entry);
+ removeFromGroup(entry);
+ }
+ }
+
+ private void removeFromGroup(Entry entry) {
+ NavigableSet<Entry> events = byGroup.get(entry.group());
+ if (events != null) {
+ events.remove(entry);
+ if (events.isEmpty()) {
+ byGroup.remove(entry.group());
+ }
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java
index 21d1ae5ded..bb928f887d 100644
--- a/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java
@@ -10,12 +10,19 @@ import io.github.hectorvent.floci.services.cloudwatch.logs.model.LogStream;
import io.github.hectorvent.floci.services.cloudwatch.logs.model.SubscriptionFilter;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.Timeout;
import org.junit.jupiter.params.ParameterizedTest;
import org.junit.jupiter.params.provider.ValueSource;
import java.util.ArrayList;
+import java.util.Collections;
import java.util.List;
import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.function.Predicate;
import static org.junit.jupiter.api.Assertions.*;
@@ -338,6 +345,115 @@ class CloudWatchLogsServiceTest {
"events past the retention window are not kept on disk waiting for a background sweep");
}
+ @Test
+ void aStreamDeletedAfterThePutChecksItLeavesNoEventsBehind() {
+ // The stream is deleted between the put's first look at it and the put taking the lock.
+ AtomicReference<Runnable> deleteLater = new AtomicReference<>();
+ InMemoryStorage<String, LogStream> streams = new InMemoryStorage<>() {
+ private boolean seen;
+
+ @Override
+ public Optional<LogStream> get(String key) {
+ Optional<LogStream> stream = super.get(key);
+ if (!seen && stream.isPresent() && stream.get().getLogStreamName().equals("doomed")) {
+ seen = true;
+ deleteLater.get().run();
+ }
+ return stream;
+ }
+ };
+ InMemoryStorage<String, LogEvent> events = new InMemoryStorage<>();
+ CloudWatchLogsService logs = new CloudWatchLogsService(new InMemoryStorage<>(), streams, events,
+ new InMemoryStorage<>(), new InMemoryStorage<>(), 10000, 10000,
+ new RegionResolver(REGION, "000000000000"));
+ logs.createLogGroup("/app", null, null, REGION);
+ logs.createLogStream("/app", "doomed", REGION);
+ deleteLater.set(() -> logs.deleteLogStream("/app", "doomed", REGION));
+
+ assertThrows(AwsException.class,
+ () -> logs.putLogEvents("/app", "doomed", eventsAt(System.currentTimeMillis()), REGION));
+ assertEquals(List.of(), List.copyOf(events.keys()));
+ }
+
+ @Test
+ @Timeout(value = 10, threadMode = Timeout.ThreadMode.SEPARATE_THREAD)
+ void aStreamRecreatedWhileTheOldOneIsDeletedKeepsItsNewEvents() throws InterruptedException {
+ // As the old stream's record goes, another caller recreates it and writes to it.
+ AtomicReference<Thread> recreate = new AtomicReference<>();
+ InMemoryStorage<String, LogStream> streams = new InMemoryStorage<>() {
+ @Override
+ public void delete(String key) {
+ super.delete(key);
+ Thread writer = recreate.getAndSet(null);
+ if (writer == null) {
+ return;
+ }
+ startAndWaitUntilFinishedOrStuck(writer);
+ }
+ };
+ InMemoryStorage<String, LogEvent> events = new InMemoryStorage<>();
+ CloudWatchLogsService logs = new CloudWatchLogsService(new InMemoryStorage<>(), streams, events,
+ new InMemoryStorage<>(), new InMemoryStorage<>(), 10000, 10000,
+ new RegionResolver(REGION, "000000000000"));
+ logs.createLogGroup("/app", null, null, REGION);
+ logs.createLogStream("/app", "reused", REGION);
+ long now = System.currentTimeMillis();
+ logs.putLogEvents("/app", "reused", eventsAt(now - 1000), REGION);
+ Thread writer = new Thread(() -> {
+ logs.createLogStream("/app", "reused", REGION);
+ logs.putLogEvents("/app", "reused", eventsAt(now), REGION);
+ });
+ recreate.set(writer);
+
+ logs.deleteLogStream("/app", "reused", REGION);
+ writer.join();
+
+ assertEquals(List.of(now), storedTimestamps(logs, "/app", "reused"));
+ }
+
+ @Test
+ @Timeout(value = 10, threadMode = Timeout.ThreadMode.SEPARATE_THREAD)
+ void aStreamDeletedAsThePutRecordsItsMetadataStaysDeleted() throws InterruptedException {
+ // The delete arrives as the put writes the stream's metadata back.
+ AtomicReference<Thread> deleter = new AtomicReference<>();
+ InMemoryStorage<String, LogStream> streams = new InMemoryStorage<>() {
+ @Override
+ public void put(String key, LogStream value) {
+ Thread delete = deleter.getAndSet(null);
+ if (delete != null) {
+ startAndWaitUntilFinishedOrStuck(delete);
+ }
+ super.put(key, value);
+ }
+ };
+ CloudWatchLogsService logs = new CloudWatchLogsService(new InMemoryStorage<>(), streams,
+ new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>(), 10000, 10000,
+ new RegionResolver(REGION, "000000000000"));
+ logs.createLogGroup("/app", null, null, REGION);
+ logs.createLogStream("/app", "doomed", REGION);
+ Thread delete = new Thread(() -> logs.deleteLogStream("/app", "doomed", REGION));
+ deleter.set(delete);
+
+ logs.putLogEvents("/app", "doomed", eventsAt(System.currentTimeMillis()), REGION);
+ delete.join();
+
+ assertEquals(List.of(), logs.describeLogStreams("/app", null, REGION));
+ }
+
+ /** Runs the thread until it ends or waits on a lock the caller may hold. */
+ private static void startAndWaitUntilFinishedOrStuck(Thread thread) {
+ thread.start();
+ long deadline = System.nanoTime() + 5_000_000_000L;
+ while (thread.isAlive() && System.nanoTime() < deadline) {
+ Thread.State state = thread.getState();
+ if (state == Thread.State.BLOCKED || state == Thread.State.WAITING
+ || state == Thread.State.TIMED_WAITING) {
+ return;
+ }
+ Thread.onSpinWait();
+ }
+ }
+
@Test
void putLogEventsLeavesOtherGroupsAloneWhenApplyingRetention() {
service.createLogGroup("/short", 1, null, REGION);
@@ -354,6 +470,399 @@ class CloudWatchLogsServiceTest {
assertEquals(List.of(), storedTimestamps(service, "/short", "stream-1"));
}
+ @Test
+ void putLogEventsAtTheCeilingDoesNotEnumerateTheStorePerCall() {
+ CountingStorage<LogEvent> events = new CountingStorage<>();
+ CloudWatchLogsService capped = new CloudWatchLogsService(
+ new InMemoryStorage<>(), new InMemoryStorage<>(), events,
+ new InMemoryStorage<>(), new InMemoryStorage<>(),
+ 10000, 100, new RegionResolver(REGION, "000000000000"));
+ capped.createLogGroup("/app/logs", 30, null, REGION);
+ capped.createLogStream("/app/logs", "stream-1", REGION);
+ long now = System.currentTimeMillis();
+ events.enumerations = 0;
+
+ for (int i = 0; i < 1_000; i++) {
+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(now + i), REGION);
+ }
+
+ // Exactly once: the first call fills the index.
+ int listed = events.enumerations;
+ assertEquals(100, events.keys().size(), "the store stays at the ceiling");
+ assertEquals(1, listed,
+ "1,000 single-event calls listed the whole store " + listed
+ + " times; the index fills once and eviction should not grow with the store");
+ }
+
+ @Test
+ void putLogEventsEvictsExactlyAfterAStreamIsDeleted() {
+ CloudWatchLogsService capped = serviceWithStoredEventCeiling(3);
+ capped.createLogGroup("/app/logs", null, null, REGION);
+ capped.createLogStream("/app/logs", "kept", REGION);
+ capped.createLogStream("/app/logs", "deleted", REGION);
+ capped.putLogEvents("/app/logs", "kept", eventsAt(1000), REGION);
+ capped.putLogEvents("/app/logs", "deleted", eventsAt(2000, 3000), REGION);
+
+ capped.deleteLogStream("/app/logs", "deleted", REGION);
+ capped.putLogEvents("/app/logs", "kept", eventsAt(4000), REGION);
+
+ assertEquals(List.of(1000L, 4000L), storedTimestamps(capped, "/app/logs", "kept"),
+ "the deleted stream's events no longer count toward the ceiling");
+ }
+
+ @Test
+ void putLogEventsEvictsExactlyAfterAGroupIsDeleted() {
+ CloudWatchLogsService capped = serviceWithStoredEventCeiling(3);
+ capped.createLogGroup("/kept", null, null, REGION);
+ capped.createLogStream("/kept", "stream-1", REGION);
+ capped.createLogGroup("/deleted", null, null, REGION);
+ capped.createLogStream("/deleted", "stream-1", REGION);
+ capped.putLogEvents("/kept", "stream-1", eventsAt(1000), REGION);
+ capped.putLogEvents("/deleted", "stream-1", eventsAt(2000, 3000), REGION);
+
+ capped.deleteLogGroup("/deleted", REGION);
+ capped.putLogEvents("/kept", "stream-1", eventsAt(4000), REGION);
+
+ assertEquals(List.of(1000L, 4000L), storedTimestamps(capped, "/kept", "stream-1"));
+ }
+
+ @Test
+ void putLogEventsCountsEventsStoredBeforeARestart() {
+ InMemoryStorage<String, LogGroup> groups = new InMemoryStorage<>();
+ InMemoryStorage<String, LogStream> streams = new InMemoryStorage<>();
+ InMemoryStorage<String, LogEvent> events = new InMemoryStorage<>();
+ CloudWatchLogsService before = new CloudWatchLogsService(groups, streams, events,
+ new InMemoryStorage<>(), new InMemoryStorage<>(), 10000, 3,
+ new RegionResolver(REGION, "000000000000"));
+ before.createLogGroup("/app/logs", null, null, REGION);
+ before.createLogStream("/app/logs", "stream-1", REGION);
+ before.putLogEvents("/app/logs", "stream-1", eventsAt(1000, 2000, 3000), REGION);
+
+ CloudWatchLogsService after = new CloudWatchLogsService(groups, streams, events,
+ new InMemoryStorage<>(), new InMemoryStorage<>(), 10000, 3,
+ new RegionResolver(REGION, "000000000000"));
+ after.putLogEvents("/app/logs", "stream-1", eventsAt(4000), REGION);
+
+ assertEquals(List.of(2000L, 3000L, 4000L), storedTimestamps(after, "/app/logs", "stream-1"));
+ }
+
+ @Test
+ void putLogEventsDropsAnExpiredEventThatArrivesAfterNewerOnes() {
+ service.createLogGroup("/app/logs", 1, null, REGION);
+ service.createLogStream("/app/logs", "stream-1", REGION);
+ long now = System.currentTimeMillis();
+
+ service.putLogEvents("/app/logs", "stream-1", eventsAt(now), REGION);
+ service.putLogEvents("/app/logs", "stream-1", eventsAt(now - 2 * 86_400_000L), REGION);
+
+ assertEquals(List.of(now), storedTimestamps(service, "/app/logs", "stream-1"));
+ }
+
+ @Test
+ void putLogEventsAfterAResetCountsOnlyWhatIsStoredSinceTheReset() {
+ InMemoryStorage<String, LogGroup> groups = new InMemoryStorage<>();
+ InMemoryStorage<String, LogStream> streams = new InMemoryStorage<>();
+ InMemoryStorage<String, LogEvent> events = new InMemoryStorage<>();
+ CloudWatchLogsService capped = new CloudWatchLogsService(groups, streams, events,
+ new InMemoryStorage<>(), new InMemoryStorage<>(), 10000, 3,
+ new RegionResolver(REGION, "000000000000"));
+ capped.createLogGroup("/app/logs", null, null, REGION);
+ capped.createLogStream("/app/logs", "stream-1", REGION);
+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(1000, 2000, 3000), REGION);
+
+ groups.clear();
+ streams.clear();
+ events.clear();
+ capped.clear();
+ capped.createLogGroup("/app/logs", null, null, REGION);
+ capped.createLogStream("/app/logs", "stream-1", REGION);
+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(500), REGION);
+
+ assertEquals(List.of(500L), storedTimestamps(capped, "/app/logs", "stream-1"),
+ "events wiped by the reset no longer count toward the ceiling");
+ }
+
+ @Test
+ void eachAccountKeepsItsOwnCeiling() {
+ InMemoryStorage<String, LogEvent> rawEvents = new InMemoryStorage<>();
+ String accountA = "111111111111";
+ String accountB = "222222222222";
+ CloudWatchLogsService accountService = new CloudWatchLogsService(
+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, accountB),
+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, accountB),
+ new AccountAwareStorageBackend<>(rawEvents, null, accountB),
+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, accountB),
+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, accountB),
+ 10_000, 2, new RegionResolver(REGION, accountB));
+ for (String account : List.of(accountA, accountB)) {
+ accountService.createLogGroupForAccount(account, "/app/logs", null, null, REGION);
+ accountService.createLogStreamForAccount(account, "/app/logs", "stream", REGION);
+ }
+
+ accountService.putLogEventsForAccount(accountA, "/app/logs", "stream", eventsAt(1, 2, 3), REGION);
+ accountService.putLogEventsForAccount(accountB, "/app/logs", "stream", eventsAt(4), REGION);
+ accountService.putLogEventsForAccount(accountA, "/app/logs", "stream", eventsAt(5), REGION);
+
+ assertEquals(List.of(3L, 5L), rawTimestamps(rawEvents, accountA),
+ "account A drops its own oldest, whatever account B stores");
+ assertEquals(List.of(4L), rawTimestamps(rawEvents, accountB));
+ }
+
+ private static List<Long> rawTimestamps(InMemoryStorage<String, LogEvent> rawEvents, String account) {
+ return rawEvents.keys().stream()
+ .filter(key -> key.startsWith(account + "/"))
+ .map(key -> rawEvents.get(key).orElseThrow().getTimestamp())
+ .sorted()
+ .toList();
+ }
+
+ @Test
+ void aStreamDeletedInTheRequestAccountLeavesThatAccountsCeilingExact() {
+ String account = "222222222222";
+ CloudWatchLogsService accountService = new CloudWatchLogsService(
+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, account),
+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, account),
+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, account),
+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, account),
+ new AccountAwareStorageBackend<>(new InMemoryStorage<>(), null, account),
+ 10_000, 3, new RegionResolver(REGION, account));
+ accountService.createLogGroupForAccount(account, "/app/logs", null, null, REGION);
+ accountService.createLogStreamForAccount(account, "/app/logs", "kept", REGION);
+ accountService.createLogStreamForAccount(account, "/app/logs", "deleted", REGION);
+ accountService.putLogEventsForAccount(account, "/app/logs", "kept", eventsAt(1000), REGION);
+ accountService.putLogEventsForAccount(account, "/app/logs", "deleted", eventsAt(2000, 3000), REGION);
+
+ accountService.deleteLogStream("/app/logs", "deleted", REGION);
+ accountService.putLogEventsForAccount(account, "/app/logs", "kept", eventsAt(4000), REGION);
+
+ assertEquals(List.of(1000L, 4000L), storedTimestamps(accountService, "/app/logs", "kept"));
+ }
+
+ @Test
+ // Unsynchronized writers can corrupt a TreeSet into a loop that never ends: fail, don't hang.
+ @Timeout(60)
+ void concurrentPutsAtTheCeilingKeepExactlyTheNewestEvents() throws InterruptedException {
+ CeilingWatchStorage events = new CeilingWatchStorage();
+ CloudWatchLogsService capped = new CloudWatchLogsService(
+ new InMemoryStorage<>(), new InMemoryStorage<>(), events,
+ new InMemoryStorage<>(), new InMemoryStorage<>(),
+ 10000, 100, new RegionResolver(REGION, "000000000000"));
+ capped.createLogGroup("/app/logs", null, null, REGION);
+ capped.createLogStream("/app/logs", "stream-1", REGION);
+ int threads = 8;
+ int perThread = 250;
+ List<Thread> workers = new ArrayList<>();
+ List<Throwable> failures = Collections.synchronizedList(new ArrayList<>());
+ for (int t = 0; t < threads; t++) {
+ int base = t * perThread;
+ Thread worker = new Thread(() -> {
+ try {
+ for (int i = 0; i < perThread; i++) {
+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(1000L + base + i), REGION);
+ }
+ } catch (Throwable e) {
+ failures.add(e);
+ }
+ });
+ workers.add(worker);
+ worker.start();
+ }
+ for (Thread worker : workers) {
+ worker.join();
+ }
+
+ assertEquals(List.of(), failures);
+ assertTrue(events.largestBeforeAWrite.get() <= 100,
+ "a write began with " + events.largestBeforeAWrite.get() + " events stored, over the ceiling");
+ List<Long> newest = new ArrayList<>();
+ for (long ts = 1000L + threads * perThread - 100; ts < 1000L + threads * perThread; ts++) {
+ newest.add(ts);
+ }
+ assertEquals(newest, storedTimestamps(capped, "/app/logs", "stream-1"));
+ }
+
+ @Test
+ void aStoreWriteThatFailsPartWayIsCountedOnTheNextCall() {
+ FailingOnceStorage events = new FailingOnceStorage();
+ CloudWatchLogsService capped = new CloudWatchLogsService(
+ new InMemoryStorage<>(), new InMemoryStorage<>(), events,
+ new InMemoryStorage<>(), new InMemoryStorage<>(),
+ 10000, 3, new RegionResolver(REGION, "000000000000"));
+ capped.createLogGroup("/app/logs", null, null, REGION);
+ capped.createLogStream("/app/logs", "stream-1", REGION);
+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(1000, 2000), REGION);
+
+ events.failNextPutAll = true;
+ assertThrows(IllegalStateException.class,
+ () -> capped.putLogEvents("/app/logs", "stream-1", eventsAt(3000, 4000), REGION));
+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(5000), REGION);
+
+ assertEquals(List.of(3000L, 4000L, 5000L), storedTimestamps(capped, "/app/logs", "stream-1"),
+ "events the failed call did store still count toward the ceiling");
+ }
+
+ @Test
+ void anErrorAfterTheStoreWriteIsCountedOnTheNextCall() {
+ FailingOnceStorage events = new FailingOnceStorage();
+ CloudWatchLogsService capped = new CloudWatchLogsService(
+ new InMemoryStorage<>(), new InMemoryStorage<>(), events,
+ new InMemoryStorage<>(), new InMemoryStorage<>(),
+ 10000, 3, new RegionResolver(REGION, "000000000000"));
+ capped.createLogGroup("/app/logs", null, null, REGION);
+ capped.createLogStream("/app/logs", "stream-1", REGION);
+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(1000, 2000), REGION);
+
+ events.errorNextPutAll = true;
+ assertThrows(OutOfMemoryError.class,
+ () -> capped.putLogEvents("/app/logs", "stream-1", eventsAt(3000, 4000), REGION));
+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(5000), REGION);
+
+ assertEquals(List.of(3000L, 4000L, 5000L), storedTimestamps(capped, "/app/logs", "stream-1"),
+ "an Error, not only a RuntimeException, leaves the index to be refilled");
+ }
+
+ @Test
+ void anAccountKeepsOneIndexThroughAFailedWriteAndAReset() {
+ // Writers queued on an index's monitor must never find a different index in the map,
+ // or two writers would change one account's events under two locks.
+ FailingOnceStorage events = new FailingOnceStorage();
+ CloudWatchLogsService capped = new CloudWatchLogsService(
+ new InMemoryStorage<>(), new InMemoryStorage<>(), events,
+ new InMemoryStorage<>(), new InMemoryStorage<>(),
+ 10000, 3, new RegionResolver(REGION, "000000000000"));
+ capped.createLogGroup("/app/logs", null, null, REGION);
+ capped.createLogStream("/app/logs", "stream-1", REGION);
+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(1000), REGION);
+ LogEventIndex index = capped.eventIndex("");
+
+ events.failNextPutAll = true;
+ assertThrows(IllegalStateException.class,
+ () -> capped.putLogEvents("/app/logs", "stream-1", eventsAt(2000), REGION));
+ assertSame(index, capped.eventIndex(""), "a failed write keeps the account's index");
+
+ capped.beforeReset();
+ capped.clear();
+ capped.afterReset();
+ assertSame(index, capped.eventIndex(""), "a reset keeps the account's index");
+ }
+
+ @Test
+ void aResetReleasesTheIndexedEventsWithoutWaitingForTheNextWrite() {
+ CloudWatchLogsService capped = serviceWithStoredEventCeiling(10);
+ capped.createLogGroup("/app/logs", null, null, REGION);
+ capped.createLogStream("/app/logs", "stream-1", REGION);
+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(1000, 2000, 3000), REGION);
+ LogEventIndex index = capped.eventIndex("");
+ assertEquals(3, index.size());
+
+ capped.beforeReset();
+ assertEquals(0, index.size(), "beforeReset() alone releases the entries");
+ assertTrue(index.isStale(), "the next write still refills from the store");
+
+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(4000), REGION);
+ capped.clear();
+ assertEquals(0, index.size(), "clear() alone releases what was written during the reset");
+ assertTrue(index.isStale());
+
+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(5000), REGION);
+ capped.afterReset();
+ assertEquals(0, index.size(), "afterReset() alone releases what was written before it");
+ assertTrue(index.isStale());
+ }
+
+ @Test
+ void putLogEventsDuringAResetDoNotSkewTheCeilingAfterIt() {
+ InMemoryStorage<String, LogGroup> groups = new InMemoryStorage<>();
+ InMemoryStorage<String, LogStream> streams = new InMemoryStorage<>();
+ InMemoryStorage<String, LogEvent> events = new InMemoryStorage<>();
+ CloudWatchLogsService capped = new CloudWatchLogsService(groups, streams, events,
+ new InMemoryStorage<>(), new InMemoryStorage<>(), 10000, 3,
+ new RegionResolver(REGION, "000000000000"));
+ capped.createLogGroup("/app/logs", null, null, REGION);
+ capped.createLogStream("/app/logs", "stream-1", REGION);
+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(1000, 2000, 3000), REGION);
+
+ // The order EmulatorInfoController resets in: beforeReset, the storage wipe, clear,
+ // afterReset. Calls land on both sides of the wipe, before the service is told.
+ capped.beforeReset();
+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(4000), REGION);
+ groups.clear();
+ streams.clear();
+ events.clear();
+ capped.createLogGroup("/app/logs", null, null, REGION);
+ capped.createLogStream("/app/logs", "stream-1", REGION);
+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(500), REGION);
+ capped.clear();
+ capped.afterReset();
+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(600, 700), REGION);
+
+ assertEquals(List.of(500L, 600L, 700L), storedTimestamps(capped, "/app/logs", "stream-1"),
+ "events wiped by the reset do not count toward the ceiling, before clear() or after");
+ }
+
+ @Test
+ void aStoredEventWithoutAnIdDoesNotBreakIngest() {
+ InMemoryStorage<String, LogEvent> events = new InMemoryStorage<>();
+ CloudWatchLogsService capped = new CloudWatchLogsService(
+ new InMemoryStorage<>(), new InMemoryStorage<>(), events,
+ new InMemoryStorage<>(), new InMemoryStorage<>(),
+ 10000, 2, new RegionResolver(REGION, "000000000000"));
+ capped.createLogGroup("/app/logs", null, null, REGION);
+ capped.createLogStream("/app/logs", "stream-1", REGION);
+ LogEvent legacy = new LogEvent();
+ legacy.setTimestamp(1000);
+ legacy.setMessage("legacy");
+ events.put(REGION + "::/app/logs::stream-1::000000000001000::legacy", legacy);
+
+ capped.putLogEvents("/app/logs", "stream-1", eventsAt(2000, 3000), REGION);
+
+ assertEquals(List.of(2000L, 3000L), storedTimestamps(capped, "/app/logs", "stream-1"));
+ }
+
+ private static final class FailingOnceStorage extends InMemoryStorage<String, LogEvent> {
+ boolean failNextPutAll;
+ boolean errorNextPutAll;
+
+ @Override
+ public void putAll(Map<String, LogEvent> entries) {
+ super.putAll(entries);
+ if (failNextPutAll) {
+ failNextPutAll = false;
+ throw new IllegalStateException("journal write failed after the entries were stored");
+ }
+ if (errorNextPutAll) {
+ errorNextPutAll = false;
+ throw new OutOfMemoryError("ran out after the entries were stored");
+ }
+ }
+ }
+
+ private static final class CeilingWatchStorage extends InMemoryStorage<String, LogEvent> {
+ final AtomicInteger largestBeforeAWrite = new AtomicInteger();
+
+ @Override
+ public void putAll(Map<String, LogEvent> entries) {
+ largestBeforeAWrite.accumulateAndGet(keys().size(), Math::max);
+ super.putAll(entries);
+ }
+ }
+
+ private static final class CountingStorage<V> extends InMemoryStorage<String, V> {
+ int enumerations;
+
+ @Override
+ public Set<String> keys() {
+ enumerations++;
+ return super.keys();
+ }
+
+ @Override
+ public List<V> scan(Predicate<String> keyFilter) {
+ enumerations++;
+ return super.scan(keyFilter);
+ }
+ }
+
// ──────────────────────────── KMS key association ────────────────────────────
@Test