Light, fluffy, and always free - The AWS Local Emulator alternative
fix(cloudwatch-logs): preserve ingestion order for same-timestamp events (#1592)
Stuart Lang7f76009cdb64
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.javaindex f56d866236..fd2c528070 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java@@ -20,18 +20,35 @@ import java.util.HashMap;import java.util.List;import java.util.Map;import java.util.UUID;+import java.util.concurrent.atomic.AtomicLong;@ApplicationScopedpublic class CloudWatchLogsService {private static final Logger LOG = Logger.getLogger(CloudWatchLogsService.class);+ /**+ * Orders events by timestamp, then by ingestion sequence so that events sharing the same+ * millisecond timestamp are returned in the order they were ingested (matching CloudWatch Logs).+ * Falls back to {@code eventId} so ordering stays deterministic even for legacy events that+ * predate {@code sequence} and therefore share the default value of {@code 0}.+ */+ private static final Comparator<LogEvent> EVENT_ORDER =+ Comparator.comparingLong(LogEvent::getTimestamp)+ .thenComparingLong(LogEvent::getSequence)+ .thenComparing(LogEvent::getEventId);+private final StorageBackend<String, LogGroup> groupStore;private final StorageBackend<String, LogStream> streamStore;private final StorageBackend<String, LogEvent> eventStore;private final StorageBackend<String, SubscriptionFilter> subscriptionFilterStore;private final RegionResolver regionResolver;private final int maxEventsPerQuery;+ /**+ * Monotonic counter assigning an ingestion sequence to each stored event. Seeded from+ * the highest sequence already in the store so ordering survives persistence reloads.+ */+ private final AtomicLong ingestionSequence;@Injectpublic CloudWatchLogsService(StorageFactory storageFactory,@@ -63,6 +80,11 @@ public class CloudWatchLogsService {this.subscriptionFilterStore = subscriptionFilterStore;this.maxEventsPerQuery = maxEventsPerQuery;this.regionResolver = regionResolver;+ long maxSequence = eventStore.scan(k -> true).stream()+ .mapToLong(LogEvent::getSequence)+ .max()+ .orElse(0L);+ this.ingestionSequence = new AtomicLong(maxSequence);}// ──────────────────────────── Log Groups ────────────────────────────@@ -247,6 +269,7 @@ public class CloudWatchLogsService {logEvent.setTimestamp(ts);logEvent.setMessage(msg);logEvent.setIngestionTime(now);+ logEvent.setSequence(ingestionSequence.incrementAndGet());String eventKey = eventKey(region, groupName, streamName, ts, logEvent.getEventId());eventStore.put(eventKey, logEvent);@@ -284,7 +307,7 @@ public class CloudWatchLogsService {String eventPrefix = eventKeyPrefix(region, groupName, streamName);List<LogEvent> all = eventStore.scan(k -> k.startsWith(eventPrefix));- all.sort(Comparator.comparingLong(LogEvent::getTimestamp));+ all.sort(EVENT_ORDER);List<LogEvent> filtered = all.stream().filter(e -> (startTime == null || e.getTimestamp() >= startTime)@@ -345,7 +368,7 @@ public class CloudWatchLogsService {all.addAll(eventStore.scan(k -> k.startsWith(groupPrefix)));}- all.sort(Comparator.comparingLong(LogEvent::getTimestamp));+ all.sort(EVENT_ORDER);List<LogEvent> result = all.stream().filter(e -> (startTime == null || e.getTimestamp() >= startTime)diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/model/LogEvent.java b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/model/LogEvent.javaindex 59e36c8ee4..43338d5122 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/model/LogEvent.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/model/LogEvent.java@@ -11,6 +11,12 @@ public class LogEvent {private long timestamp;private String message;private long ingestionTime;+ /**+ * Monotonically increasing ingestion sequence number, used to preserve ingestion+ * order as a tie-breaker when multiple events share the same millisecond timestamp.+ * Matches CloudWatch Logs, which returns same-timestamp events in ingestion order.+ */+ private long sequence;public LogEvent() {}@@ -25,4 +31,7 @@ public class LogEvent {public long getIngestionTime() { return ingestionTime; }public void setIngestionTime(long ingestionTime) { this.ingestionTime = ingestionTime; }++ public long getSequence() { return sequence; }+ public void setSequence(long sequence) { this.sequence = sequence; }}diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.javaindex f2a61b7abb..e5e9733bc6 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java@@ -10,6 +10,7 @@ import io.github.hectorvent.floci.services.cloudwatch.logs.model.SubscriptionFilimport org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;+import java.util.ArrayList;import java.util.List;import java.util.Map;@@ -174,6 +175,53 @@ class CloudWatchLogsServiceTest {assertEquals("second", result.events().get(1).getMessage());}+ @Test+ void getLogEventsPreservesIngestionOrderForSameTimestamp() {+ // Regression for issue #1584: events written in order within the same millisecond+ // must come back from GetLogEvents in ingestion order, not shuffled.+ service.createLogGroup("/app/logs", null, null, REGION);+ service.createLogStream("/app/logs", "stream-1", REGION);++ long ts = System.currentTimeMillis();+ List<Map<String, Object>> events = new ArrayList<>();+ for (int i = 0; i < 10; i++) {+ events.add(Map.of("timestamp", ts, "message", "SEQLINE-" + i));+ }+ service.putLogEvents("/app/logs", "stream-1", events, REGION);++ CloudWatchLogsService.LogEventsResult result = service.getLogEvents(+ "/app/logs", "stream-1", null, null, 100, true, null, REGION);++ assertEquals(10, result.events().size());+ for (int i = 0; i < 10; i++) {+ assertEquals("SEQLINE-" + i, result.events().get(i).getMessage(),+ "event at index " + i + " out of ingestion order");+ }+ }++ @Test+ void filterLogEventsPreservesIngestionOrderForSameTimestamp() {+ // Same-timestamp ordering must also hold for FilterLogEvents.+ service.createLogGroup("/app/logs", null, null, REGION);+ service.createLogStream("/app/logs", "stream-1", REGION);++ long ts = System.currentTimeMillis();+ List<Map<String, Object>> events = new ArrayList<>();+ for (int i = 0; i < 10; i++) {+ events.add(Map.of("timestamp", ts, "message", "SEQLINE-" + i));+ }+ service.putLogEvents("/app/logs", "stream-1", events, REGION);++ CloudWatchLogsService.FilteredLogEventsResult result = service.filterLogEvents(+ "/app/logs", null, null, null, "SEQLINE", 100, REGION);++ assertEquals(10, result.events().size());+ for (int i = 0; i < 10; i++) {+ assertEquals("SEQLINE-" + i, result.events().get(i).getMessage(),+ "event at index " + i + " out of ingestion order");+ }+ }+@Testvoid getLogEventsWithTimeRange() {service.createLogGroup("/app/logs", null, null, REGION);@@ -270,7 +318,7 @@ class CloudWatchLogsServiceTest {// ──────────────────────────── GetLogEvents pagination (issue #90) ────────────────────────────private void putEvents(String group, String stream, long baseTs, int count) {- List<Map<String, Object>> events = new java.util.ArrayList<>();+ List<Map<String, Object>> events = new ArrayList<>();for (int i = 0; i < count; i++) {events.add(Map.of("timestamp", baseTs + i, "message", "msg-" + i));}