Light, fluffy, and always free - The AWS Local Emulator alternative
perf(logs): filter log events by time before sorting (#4474)
Niels Kootstrabf47dd719345
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.javaindex c6fb91b07e..71896205b3 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java@@ -788,11 +788,13 @@ public class CloudWatchLogsService implements ResourceProvider {String eventPrefix = eventKeyPrefix(region, groupName, streamName);List<LogEvent> all = eventStore.scan(k -> k.startsWith(eventPrefix));- all.sort(EVENT_ORDER);+ // Filter before sorting so a narrow time window only pays the comparison cost for+ // the events it keeps, instead of sorting the whole stream and discarding most of it.List<LogEvent> filtered = all.stream().filter(e -> (startTime == null || e.getTimestamp() >= startTime)&& (endTime == null || e.getTimestamp() <= endTime))+ .sorted(EVENT_ORDER).toList();int total = filtered.size();@@ -872,13 +874,14 @@ public class CloudWatchLogsService implements ResourceProvider {eventStore.get(key).ifPresent(e -> all.add(new FilteredEvent(streamName, e)));}- all.sort(Comparator.comparing(FilteredEvent::event, EVENT_ORDER));-+ // Filter before sorting so a narrow time window or pattern only pays the comparison+ // cost for the events it keeps, instead of sorting every stored event up front.List<FilteredEvent> matches = all.stream().filter(f -> (startTime == null || f.event().getTimestamp() >= startTime)&& (endTime == null || f.event().getTimestamp() <= endTime)).filter(f -> filterPattern == null || filterPattern.isBlank()|| f.event().getMessage().contains(filterPattern))+ .sorted(Comparator.comparing(FilteredEvent::event, EVENT_ORDER)).toList();// The cursor indexes matches rather than stored events, which is why the window is applieddiff --git a/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.javaindex bb865e826d..ca6401fef5 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java@@ -1078,6 +1078,53 @@ class CloudWatchLogsServiceTest {assertEquals("f/5", result.nextForwardToken());}+ @Test+ void getLogEventsNarrowWindowPaginatesOnlyMatchingEventsAcrossManyOutOfWindowEvents() {+ // The window must be applied before pagination, not after: 6 in-window matches+ // surrounded by 200 out-of-window events must page exactly like a 6-event stream.+ // A refactor that paginates the full scan before filtering would return the wrong+ // messages here and drift the tokens off the match count.+ service.createLogGroup("/app/logs", null, null, REGION);+ service.createLogStream("/app/logs", "stream-1", REGION);+ long base = System.currentTimeMillis();+ putEvents("/app/logs", "stream-1", base, 200);++ long startTime = base + 97;+ long endTime = base + 102;++ CloudWatchLogsService.LogEventsResult page1 = service.getLogEvents(+ "/app/logs", "stream-1", startTime, endTime, 2, true, null, REGION);+ assertEquals(List.of("msg-97", "msg-98"),+ page1.events().stream().map(LogEvent::getMessage).toList());+ assertEquals("f/2", page1.nextForwardToken());++ CloudWatchLogsService.LogEventsResult page2 = service.getLogEvents(+ "/app/logs", "stream-1", startTime, endTime, 2, true, page1.nextForwardToken(), REGION);+ assertEquals(List.of("msg-99", "msg-100"),+ page2.events().stream().map(LogEvent::getMessage).toList());+ assertEquals("f/4", page2.nextForwardToken());++ CloudWatchLogsService.LogEventsResult page3 = service.getLogEvents(+ "/app/logs", "stream-1", startTime, endTime, 2, true, page2.nextForwardToken(), REGION);+ assertEquals(List.of("msg-101", "msg-102"),+ page3.events().stream().map(LogEvent::getMessage).toList());+ assertEquals("f/6", page3.nextForwardToken());++ CloudWatchLogsService.LogEventsResult atEnd = service.getLogEvents(+ "/app/logs", "stream-1", startTime, endTime, 2, true, page3.nextForwardToken(), REGION);+ assertEquals(0, atEnd.events().size());+ assertEquals("f/6", atEnd.nextForwardToken(), "token must echo back to signal end of the window");++ // startFromHead=false with no token must start from the tail of the *filtered*+ // window, not the tail of the full 200-event stream.+ CloudWatchLogsService.LogEventsResult tail = service.getLogEvents(+ "/app/logs", "stream-1", startTime, endTime, 2, false, null, REGION);+ assertEquals(List.of("msg-101", "msg-102"),+ tail.events().stream().map(LogEvent::getMessage).toList());+ assertEquals("b/4", tail.nextBackwardToken());+ assertEquals("f/6", tail.nextForwardToken());+ }+@Testvoid filterLogEventsPagesForwardToTheNewestMatches() {service.createLogGroup("/app/logs", null, null, REGION);@@ -1203,6 +1250,38 @@ class CloudWatchLogsServiceTest {assertNull(page2.nextToken());}+ @Test+ void filterLogEventsNarrowWindowPaginatesOnlyMatchingEventsAcrossManyOutOfWindowEvents() {+ // Same guard as GetLogEvents: the window must be applied before pagination, so 6+ // in-window matches surrounded by 200 out-of-window events page like a 6-event scan,+ // not like the full one.+ service.createLogGroup("/app/logs", null, null, REGION);+ service.createLogStream("/app/logs", "stream-1", REGION);+ long base = System.currentTimeMillis();+ putEvents("/app/logs", "stream-1", base, 200);++ long startTime = base + 97;+ long endTime = base + 102;++ CloudWatchLogsService.FilteredLogEventsResult page1 = service.filterLogEvents(+ "/app/logs", null, startTime, endTime, null, 2, null, REGION);+ assertEquals(List.of("msg-97", "msg-98"),+ page1.events().stream().map(f -> f.event().getMessage()).toList());+ assertEquals("f/2", page1.nextToken());++ CloudWatchLogsService.FilteredLogEventsResult page2 = service.filterLogEvents(+ "/app/logs", null, startTime, endTime, null, 2, page1.nextToken(), REGION);+ assertEquals(List.of("msg-99", "msg-100"),+ page2.events().stream().map(f -> f.event().getMessage()).toList());+ assertEquals("f/4", page2.nextToken());++ CloudWatchLogsService.FilteredLogEventsResult page3 = service.filterLogEvents(+ "/app/logs", null, startTime, endTime, null, 2, page2.nextToken(), REGION);+ assertEquals(List.of("msg-101", "msg-102"),+ page3.events().stream().map(f -> f.event().getMessage()).toList());+ assertNull(page3.nextToken(), "a page that exactly exhausts the matches is the last one");+ }+@Testvoid filterLogEventsPaginatesAcrossStreamsKeepingAttribution() {service.createLogGroup("/app/logs", null, null, REGION);