Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(cloudwatch-logs): preserve ingestion order for same-timestamp events (#1592)

Stuart Lang7f76009cdb64
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java
index f56d866236..fd2c528070 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsService.java
@@ -20,18 +20,35 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.UUID;
+import java.util.concurrent.atomic.AtomicLong;
@ApplicationScoped
public class CloudWatchLogsService {
private static final Logger LOG = Logger.getLogger(CloudWatchLogsService.class);
+ /**
+ * Orders events by timestamp, then by ingestion sequence so that events sharing the same
+ * millisecond timestamp are returned in the order they were ingested (matching CloudWatch Logs).
+ * Falls back to {@code eventId} so ordering stays deterministic even for legacy events that
+ * predate {@code sequence} and therefore share the default value of {@code 0}.
+ */
+ private static final Comparator<LogEvent> EVENT_ORDER =
+ Comparator.comparingLong(LogEvent::getTimestamp)
+ .thenComparingLong(LogEvent::getSequence)
+ .thenComparing(LogEvent::getEventId);
+
private final StorageBackend<String, LogGroup> groupStore;
private final StorageBackend<String, LogStream> streamStore;
private final StorageBackend<String, LogEvent> eventStore;
private final StorageBackend<String, SubscriptionFilter> subscriptionFilterStore;
private final RegionResolver regionResolver;
private final int maxEventsPerQuery;
+ /**
+ * Monotonic counter assigning an ingestion sequence to each stored event. Seeded from
+ * the highest sequence already in the store so ordering survives persistence reloads.
+ */
+ private final AtomicLong ingestionSequence;
@Inject
public CloudWatchLogsService(StorageFactory storageFactory,
@@ -63,6 +80,11 @@ public class CloudWatchLogsService {
this.subscriptionFilterStore = subscriptionFilterStore;
this.maxEventsPerQuery = maxEventsPerQuery;
this.regionResolver = regionResolver;
+ long maxSequence = eventStore.scan(k -> true).stream()
+ .mapToLong(LogEvent::getSequence)
+ .max()
+ .orElse(0L);
+ this.ingestionSequence = new AtomicLong(maxSequence);
}
// ──────────────────────────── Log Groups ────────────────────────────
@@ -247,6 +269,7 @@ public class CloudWatchLogsService {
logEvent.setTimestamp(ts);
logEvent.setMessage(msg);
logEvent.setIngestionTime(now);
+ logEvent.setSequence(ingestionSequence.incrementAndGet());
String eventKey = eventKey(region, groupName, streamName, ts, logEvent.getEventId());
eventStore.put(eventKey, logEvent);
@@ -284,7 +307,7 @@ public class CloudWatchLogsService {
String eventPrefix = eventKeyPrefix(region, groupName, streamName);
List<LogEvent> all = eventStore.scan(k -> k.startsWith(eventPrefix));
- all.sort(Comparator.comparingLong(LogEvent::getTimestamp));
+ all.sort(EVENT_ORDER);
List<LogEvent> filtered = all.stream()
.filter(e -> (startTime == null || e.getTimestamp() >= startTime)
@@ -345,7 +368,7 @@ public class CloudWatchLogsService {
all.addAll(eventStore.scan(k -> k.startsWith(groupPrefix)));
}
- all.sort(Comparator.comparingLong(LogEvent::getTimestamp));
+ all.sort(EVENT_ORDER);
List<LogEvent> result = all.stream()
.filter(e -> (startTime == null || e.getTimestamp() >= startTime)
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/model/LogEvent.java b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/model/LogEvent.java
index 59e36c8ee4..43338d5122 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/model/LogEvent.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudwatch/logs/model/LogEvent.java
@@ -11,6 +11,12 @@ public class LogEvent {
private long timestamp;
private String message;
private long ingestionTime;
+ /**
+ * Monotonically increasing ingestion sequence number, used to preserve ingestion
+ * order as a tie-breaker when multiple events share the same millisecond timestamp.
+ * Matches CloudWatch Logs, which returns same-timestamp events in ingestion order.
+ */
+ private long sequence;
public LogEvent() {}
@@ -25,4 +31,7 @@ public class LogEvent {
public long getIngestionTime() { return ingestionTime; }
public void setIngestionTime(long ingestionTime) { this.ingestionTime = ingestionTime; }
+
+ public long getSequence() { return sequence; }
+ public void setSequence(long sequence) { this.sequence = sequence; }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java
index f2a61b7abb..e5e9733bc6 100644
--- a/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudwatch/logs/CloudWatchLogsServiceTest.java
@@ -10,6 +10,7 @@ import io.github.hectorvent.floci.services.cloudwatch.logs.model.SubscriptionFil
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import java.util.ArrayList;
import java.util.List;
import java.util.Map;
@@ -174,6 +175,53 @@ class CloudWatchLogsServiceTest {
assertEquals("second", result.events().get(1).getMessage());
}
+ @Test
+ void getLogEventsPreservesIngestionOrderForSameTimestamp() {
+ // Regression for issue #1584: events written in order within the same millisecond
+ // must come back from GetLogEvents in ingestion order, not shuffled.
+ service.createLogGroup("/app/logs", null, null, REGION);
+ service.createLogStream("/app/logs", "stream-1", REGION);
+
+ long ts = System.currentTimeMillis();
+ List<Map<String, Object>> events = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ events.add(Map.of("timestamp", ts, "message", "SEQLINE-" + i));
+ }
+ service.putLogEvents("/app/logs", "stream-1", events, REGION);
+
+ CloudWatchLogsService.LogEventsResult result = service.getLogEvents(
+ "/app/logs", "stream-1", null, null, 100, true, null, REGION);
+
+ assertEquals(10, result.events().size());
+ for (int i = 0; i < 10; i++) {
+ assertEquals("SEQLINE-" + i, result.events().get(i).getMessage(),
+ "event at index " + i + " out of ingestion order");
+ }
+ }
+
+ @Test
+ void filterLogEventsPreservesIngestionOrderForSameTimestamp() {
+ // Same-timestamp ordering must also hold for FilterLogEvents.
+ service.createLogGroup("/app/logs", null, null, REGION);
+ service.createLogStream("/app/logs", "stream-1", REGION);
+
+ long ts = System.currentTimeMillis();
+ List<Map<String, Object>> events = new ArrayList<>();
+ for (int i = 0; i < 10; i++) {
+ events.add(Map.of("timestamp", ts, "message", "SEQLINE-" + i));
+ }
+ service.putLogEvents("/app/logs", "stream-1", events, REGION);
+
+ CloudWatchLogsService.FilteredLogEventsResult result = service.filterLogEvents(
+ "/app/logs", null, null, null, "SEQLINE", 100, REGION);
+
+ assertEquals(10, result.events().size());
+ for (int i = 0; i < 10; i++) {
+ assertEquals("SEQLINE-" + i, result.events().get(i).getMessage(),
+ "event at index " + i + " out of ingestion order");
+ }
+ }
+
@Test
void getLogEventsWithTimeRange() {
service.createLogGroup("/app/logs", null, null, REGION);
@@ -270,7 +318,7 @@ class CloudWatchLogsServiceTest {
// ──────────────────────────── GetLogEvents pagination (issue #90) ────────────────────────────
private void putEvents(String group, String stream, long baseTs, int count) {
- List<Map<String, Object>> events = new java.util.ArrayList<>();
+ List<Map<String, Object>> events = new ArrayList<>();
for (int i = 0; i < count; i++) {
events.add(Map.of("timestamp", baseTs + i, "message", "msg-" + i));
}