Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

perf(logs): filter log events by time before sorting (#4474)

Niels Kootstrabf47dd719345
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java
index c6fb91b07e..71896205b3 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java
@@ -788,11 +788,13 @@ public class CloudWatchLogsService implements ResourceProvider {
String eventPrefix = eventKeyPrefix(region, groupName, streamName);
List<LogEvent> all = eventStore.scan(k -> k.startsWith(eventPrefix));
- all.sort(EVENT_ORDER);
+ // Filter before sorting so a narrow time window only pays the comparison cost for
+ // the events it keeps, instead of sorting the whole stream and discarding most of it.
List<LogEvent> filtered = all.stream()
.filter(e -> (startTime == null || e.getTimestamp() >= startTime)
&& (endTime == null || e.getTimestamp() <= endTime))
+ .sorted(EVENT_ORDER)
.toList();
int total = filtered.size();
@@ -872,13 +874,14 @@ public class CloudWatchLogsService implements ResourceProvider {
eventStore.get(key).ifPresent(e -> all.add(new FilteredEvent(streamName, e)));
}
- all.sort(Comparator.comparing(FilteredEvent::event, EVENT_ORDER));
-
+ // Filter before sorting so a narrow time window or pattern only pays the comparison
+ // cost for the events it keeps, instead of sorting every stored event up front.
List<FilteredEvent> matches = all.stream()
.filter(f -> (startTime == null || f.event().getTimestamp() >= startTime)
&& (endTime == null || f.event().getTimestamp() <= endTime))
.filter(f -> filterPattern == null || filterPattern.isBlank()
|| f.event().getMessage().contains(filterPattern))
+ .sorted(Comparator.comparing(FilteredEvent::event, EVENT_ORDER))
.toList();
// The cursor indexes matches rather than stored events, which is why the window is applied
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java
index bb865e826d..ca6401fef5 100644
--- a/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java
@@ -1078,6 +1078,53 @@ class CloudWatchLogsServiceTest {
assertEquals("f/5", result.nextForwardToken());
}
+ @Test
+ void getLogEventsNarrowWindowPaginatesOnlyMatchingEventsAcrossManyOutOfWindowEvents() {
+ // The window must be applied before pagination, not after: 6 in-window matches
+ // surrounded by 200 out-of-window events must page exactly like a 6-event stream.
+ // A refactor that paginates the full scan before filtering would return the wrong
+ // messages here and drift the tokens off the match count.
+ service.createLogGroup("/app/logs", null, null, REGION);
+ service.createLogStream("/app/logs", "stream-1", REGION);
+ long base = System.currentTimeMillis();
+ putEvents("/app/logs", "stream-1", base, 200);
+
+ long startTime = base + 97;
+ long endTime = base + 102;
+
+ CloudWatchLogsService.LogEventsResult page1 = service.getLogEvents(
+ "/app/logs", "stream-1", startTime, endTime, 2, true, null, REGION);
+ assertEquals(List.of("msg-97", "msg-98"),
+ page1.events().stream().map(LogEvent::getMessage).toList());
+ assertEquals("f/2", page1.nextForwardToken());
+
+ CloudWatchLogsService.LogEventsResult page2 = service.getLogEvents(
+ "/app/logs", "stream-1", startTime, endTime, 2, true, page1.nextForwardToken(), REGION);
+ assertEquals(List.of("msg-99", "msg-100"),
+ page2.events().stream().map(LogEvent::getMessage).toList());
+ assertEquals("f/4", page2.nextForwardToken());
+
+ CloudWatchLogsService.LogEventsResult page3 = service.getLogEvents(
+ "/app/logs", "stream-1", startTime, endTime, 2, true, page2.nextForwardToken(), REGION);
+ assertEquals(List.of("msg-101", "msg-102"),
+ page3.events().stream().map(LogEvent::getMessage).toList());
+ assertEquals("f/6", page3.nextForwardToken());
+
+ CloudWatchLogsService.LogEventsResult atEnd = service.getLogEvents(
+ "/app/logs", "stream-1", startTime, endTime, 2, true, page3.nextForwardToken(), REGION);
+ assertEquals(0, atEnd.events().size());
+ assertEquals("f/6", atEnd.nextForwardToken(), "token must echo back to signal end of the window");
+
+ // startFromHead=false with no token must start from the tail of the *filtered*
+ // window, not the tail of the full 200-event stream.
+ CloudWatchLogsService.LogEventsResult tail = service.getLogEvents(
+ "/app/logs", "stream-1", startTime, endTime, 2, false, null, REGION);
+ assertEquals(List.of("msg-101", "msg-102"),
+ tail.events().stream().map(LogEvent::getMessage).toList());
+ assertEquals("b/4", tail.nextBackwardToken());
+ assertEquals("f/6", tail.nextForwardToken());
+ }
+
@Test
void filterLogEventsPagesForwardToTheNewestMatches() {
service.createLogGroup("/app/logs", null, null, REGION);
@@ -1203,6 +1250,38 @@ class CloudWatchLogsServiceTest {
assertNull(page2.nextToken());
}
+ @Test
+ void filterLogEventsNarrowWindowPaginatesOnlyMatchingEventsAcrossManyOutOfWindowEvents() {
+ // Same guard as GetLogEvents: the window must be applied before pagination, so 6
+ // in-window matches surrounded by 200 out-of-window events page like a 6-event scan,
+ // not like the full one.
+ service.createLogGroup("/app/logs", null, null, REGION);
+ service.createLogStream("/app/logs", "stream-1", REGION);
+ long base = System.currentTimeMillis();
+ putEvents("/app/logs", "stream-1", base, 200);
+
+ long startTime = base + 97;
+ long endTime = base + 102;
+
+ CloudWatchLogsService.FilteredLogEventsResult page1 = service.filterLogEvents(
+ "/app/logs", null, startTime, endTime, null, 2, null, REGION);
+ assertEquals(List.of("msg-97", "msg-98"),
+ page1.events().stream().map(f -> f.event().getMessage()).toList());
+ assertEquals("f/2", page1.nextToken());
+
+ CloudWatchLogsService.FilteredLogEventsResult page2 = service.filterLogEvents(
+ "/app/logs", null, startTime, endTime, null, 2, page1.nextToken(), REGION);
+ assertEquals(List.of("msg-99", "msg-100"),
+ page2.events().stream().map(f -> f.event().getMessage()).toList());
+ assertEquals("f/4", page2.nextToken());
+
+ CloudWatchLogsService.FilteredLogEventsResult page3 = service.filterLogEvents(
+ "/app/logs", null, startTime, endTime, null, 2, page2.nextToken(), REGION);
+ assertEquals(List.of("msg-101", "msg-102"),
+ page3.events().stream().map(f -> f.event().getMessage()).toList());
+ assertNull(page3.nextToken(), "a page that exactly exhausts the matches is the last one");
+ }
+
@Test
void filterLogEventsPaginatesAcrossStreamsKeepingAttribution() {
service.createLogGroup("/app/logs", null, null, REGION);