Skip to content

Ministack: Free, open-source local AWS emulator - 60+ services, Terraform compatible, real databases. Free forever. MIT licensed.

fix(logs): preserve event ordering with bounded stream cursors (#2229)

Kinonef325c255754
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 3f8ad0a9..d3eac414 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -8,6 +8,7 @@ Versioning follows [Semantic Versioning](https://semver.org/).
## [Unreleased]
### Fixed
+- **CloudWatch Logs — ordered, bounded event reads** — keep stream histories chronological for API, Lambda, CodeBuild and DynamoDB import logs, including late batches and restored snapshots. `FilterLogEvents` seeks to time bounds and merges stream cursors until its limit is reached instead of collecting, filtering and sorting every event; newer events in an earlier-scanned stream cannot displace older matches. Equal timestamps retain stream-selection and arrival order, `GetLogEvents` reads late arrivals chronologically with tokens anchored to returned events, so pages neither repeat nor skip events on the unread side, and Live Tail still receives the exact newly ingested batch.
- **Lambda — conditional function and alias updates** — `UpdateFunctionCode`, `UpdateFunctionConfiguration`, and `UpdateAlias` reject a stale `RevisionId` with `PreconditionFailedException` (412) before changing state or invalidating workers. Competing updates to the same revision are serialized per function or alias, while unrelated resources remain independent.
- **SQS — empty policy updates** — `SetQueueAttributes` removes `Policy`, `RedrivePolicy`, and `RedriveAllowPolicy` when explicitly set to an empty string, so Terraform policy deletion succeeds and subsequent attribute reads omit the removed policy. Invalid nonempty redrive policies and numeric attributes are still rejected before any attributes change.
diff --git a/ministack/services/cloudwatch_logs.py b/ministack/services/cloudwatch_logs.py
index c1023d9a..9b22097b 100644
--- a/ministack/services/cloudwatch_logs.py
+++ b/ministack/services/cloudwatch_logs.py
@@ -26,12 +26,15 @@ import base64
import contextlib
import copy
import fnmatch
+import heapq
import json
import logging
import os
import re
+import threading
import time
import zlib
+from bisect import bisect_left, bisect_right
from datetime import datetime, timezone
from ministack.core.arn import ArnParseError, parse_arn
@@ -52,6 +55,8 @@ REGION = os.environ.get("MINISTACK_REGION", "us-east-1")
_log_groups = AccountRegionScopedDict()
+# Protect appends and cursor bounds; never hold this during delivery or filtering.
+_log_events_lock = threading.Lock()
# group_name -> {
# arn, creationTime, retentionInDays (int|None), tags: {str: str},
# subscriptionFilters: {filterName: {filterName, logGroupName, filterPattern,
@@ -206,7 +211,12 @@ def load_persisted_state(data):
def _restore_state(data):
if data:
- _log_groups.update(data.get("log_groups", {}))
+ with _log_events_lock:
+ _log_groups.update(data.get("log_groups", {}))
+ # Older snapshots stored arrival order, including late events.
+ for _, group in _log_groups.all_items():
+ for stream in group.get("streams", {}).values():
+ stream["events"] = sorted(stream["events"], key=_event_timestamp)
_destinations.update(data.get("destinations", {}))
_resource_policies.update(data.get("resource_policies", {}))
_log_group_policies.update(data.get("log_group_policies", {}))
@@ -571,6 +581,36 @@ def _describe_log_streams(data):
# Log events
# ---------------------------------------------------------------------------
+def _event_timestamp(event):
+ return event["timestamp"]
+
+
+def _append_log_events(stream, events):
+ """Keep chronological history and stable ties for every log producer.
+
+ Chronological batches extend the existing array without copying history.
+ A late batch replaces it with a stable merge, leaving captured readers'
+ arrays intact. Readers capture their array and end bound under the lock.
+ """
+ if not events:
+ return
+ ordered = sorted(events, key=_event_timestamp)
+ first, last = ordered[0]["timestamp"], ordered[-1]["timestamp"]
+ with _log_events_lock:
+ history = stream["events"]
+ late = bool(history and history[-1]["timestamp"] > first)
+ if late:
+ stream["events"] = list(heapq.merge(history, ordered, key=_event_timestamp))
+ else:
+ history.extend(ordered)
+ if stream["firstEventTimestamp"] is None or first < stream["firstEventTimestamp"]:
+ stream["firstEventTimestamp"] = first
+ if stream["lastEventTimestamp"] is None or last > stream["lastEventTimestamp"]:
+ stream["lastEventTimestamp"] = last
+ stream["lastIngestionTime"] = events[-1]["ingestionTime"]
+ logger.debug("Stored log events count=%d late_batch=%s", len(events), late)
+
+
def _put_log_events(data):
group = data.get("logGroupName")
stream = data.get("logStreamName")
@@ -589,12 +629,13 @@ def _put_log_events(data):
s = _log_groups[group]["streams"][stream]
now_ms = int(time.time() * 1000)
+ stored = []
for e in events:
ts = e.get("timestamp", now_ms)
msg = e.get("message", "")
ptr = new_uuid()
- s["events"].append({
+ stored.append({
"timestamp": ts,
"message": msg,
"ingestionTime": now_ms,
@@ -608,18 +649,13 @@ def _put_log_events(data):
log_stream=stream,
)
- if s["firstEventTimestamp"] is None or ts < s["firstEventTimestamp"]:
- s["firstEventTimestamp"] = ts
- if s["lastEventTimestamp"] is None or ts > s["lastEventTimestamp"]:
- s["lastEventTimestamp"] = ts
- s["lastIngestionTime"] = now_ms
+ _append_log_events(s, stored)
token = str(int(s["uploadSequenceToken"]) + 1)
s["uploadSequenceToken"] = token
_fanout_to_subscription_filters(group, stream, events)
# Fan out the stored shape (ingestionTime filled) to any Live Tail sessions.
- stored = s["events"][-len(events) :] if events else []
_fanout_to_live_tail_sessions(group, stream, stored)
return json_response({"nextSequenceToken": token})
@@ -1185,6 +1221,33 @@ def _resolve_log_group_name(data):
return ident
+def _event_cursor(history, size, position, forward):
+ """Name a GetLogEvents boundary by the returned event beside it.
+
+ The cursor is "<timestamp>:<k>", the k-th position within that timestamp's
+ run. Late events stay on their own side of the anchor, and equal-time
+ arrivals sort after their ties, so the boundary never shifts. A backward
+ boundary at the tail anchors on the last event, so later events stay
+ after it. "-" marks the head of the stream, or a boundary in an empty one.
+ """
+ anchor = position - 1 if forward or position == size else position
+ if not 0 <= anchor < size:
+ return "-"
+ timestamp = history[anchor]["timestamp"]
+ return f"{timestamp}:{position - bisect_left(history, timestamp, hi=size, key=_event_timestamp)}"
+
+
+def _event_token_position(history, size, token):
+ direction, _, cursor = token.partition("/")
+ if direction not in ("f", "b"):
+ raise ValueError(token)
+ forward = direction == "f"
+ if cursor == "-":
+ return forward, 0 if forward else size
+ timestamp, offset = cursor.split(":")
+ return forward, bisect_left(history, int(timestamp), hi=size, key=_event_timestamp) + int(offset)
+
+
def _get_log_events(data):
group = _resolve_log_group_name(data)
stream = data.get("logStreamName")
@@ -1205,37 +1268,35 @@ def _get_log_events(data):
f"The specified log stream does not exist: {stream}", 400,
)
- all_events = _log_groups[group]["streams"][stream]["events"]
-
- filtered = all_events
- if start_time is not None:
- filtered = [e for e in filtered if e["timestamp"] >= start_time]
- if end_time is not None:
- filtered = [e for e in filtered if e["timestamp"] <= end_time]
+ with _log_events_lock:
+ history = _log_groups[group]["streams"][stream]["events"]
+ size = len(history)
+ lower = bisect_left(history, start_time, hi=size, key=_event_timestamp) if start_time is not None else 0
+ upper = bisect_right(history, end_time, lo=lower, hi=size, key=_event_timestamp) if end_time is not None else size
- # Parse offset from token: f/<offset> for forward, b/<offset> for backward
- offset = 0
+ forward = start_from_head
+ position = lower if forward else upper
if next_token:
try:
- offset = int(next_token.split("/", 1)[1])
- except (IndexError, ValueError):
- offset = 0
-
- if start_from_head or (next_token and next_token.startswith("f/")):
- page = filtered[offset:offset + limit]
- new_forward = f"f/{offset + len(page)}"
- new_backward = f"b/{offset}"
+ forward, position = _event_token_position(history, size, next_token)
+ except ValueError:
+ return error_response_json("InvalidParameterException", "The specified nextToken is invalid.", 400)
+ position = min(max(position, lower), upper)
+ if forward:
+ start, end = position, min(position + limit, upper)
else:
- end = len(filtered) - offset if next_token and next_token.startswith("b/") else len(filtered)
- start = max(0, end - limit)
- page = filtered[start:end]
- new_forward = f"f/{end}"
- new_backward = f"b/{len(filtered) - start}"
+ start, end = max(lower, position - limit), position
+ page = history[start:end]
+ forward_token = f"f/{_event_cursor(history, size, end, True)}"
+ backward_token = f"b/{_event_cursor(history, size, start, False)}"
# AWS behaviour: when at end of stream, return the caller's token
# so SDK clients stop paginating
- forward_token = next_token if (next_token and len(page) < limit) else new_forward
- backward_token = next_token if (next_token and offset == 0 and next_token.startswith("b/")) else new_backward
+ if next_token and not page:
+ if forward:
+ forward_token = next_token
+ else:
+ backward_token = next_token
# OutputLogEvent is only {timestamp, message, ingestionTime}; strip internal
# fields (e.g. the eventId used by FilterLogEvents / GetLogRecord).
@@ -1281,6 +1342,11 @@ def _compile_filter_pattern(raw: str):
return _matches
+def _stream_event_range(events, stream_name, start, end):
+ for index in range(start, end):
+ yield events[index], stream_name
+
+
def _filter_log_events(data):
group = _resolve_log_group_name(data)
raw_pattern = data.get("filterPattern", "")
@@ -1298,27 +1364,34 @@ def _filter_log_events(data):
events = []
searched = []
+ ranges = []
streams = _log_groups[group]["streams"]
target_streams = stream_names if stream_names else list(streams.keys())
- for sn in target_streams:
- if sn not in streams:
- continue
- searched.append({"logStreamName": sn, "searchedCompletely": True})
- for e in streams[sn]["events"]:
- ts = e["timestamp"]
- if start_time is not None and ts < start_time:
- continue
- if end_time is not None and ts > end_time:
- continue
- if not pattern_fn(e.get("message", "")):
+ with _log_events_lock:
+ for sn in target_streams:
+ stream = streams.get(sn)
+ if stream is None:
continue
- events.append({**e, "logStreamName": sn})
+ searched.append({"logStreamName": sn, "searchedCompletely": True})
+ history = stream["events"]
+ start = bisect_left(history, start_time, key=_event_timestamp) if start_time is not None else 0
+ end = bisect_right(history, end_time, key=_event_timestamp) if end_time is not None else len(history)
+ ranges.append(_stream_event_range(history, sn, start, end))
+
+ # merge is stable across input streams and only pulls the next candidate.
+ scanned = 0
+ for event, sn in heapq.merge(*ranges, key=lambda entry: entry[0]["timestamp"]):
+ if len(events) >= limit:
+ break
+ scanned += 1
+ if pattern_fn(event.get("message", "")):
+ events.append({**event, "logStreamName": sn})
if len(events) >= limit:
break
-
- events.sort(key=lambda ev: ev["timestamp"])
- return json_response({"events": events[:limit], "searchedLogStreams": searched})
+ logger.debug("FilterLogEvents streams=%d scanned=%d returned=%d limit=%s",
+ len(ranges), scanned, len(events), limit)
+ return json_response({"events": events, "searchedLogStreams": searched})
# ---------------------------------------------------------------------------
diff --git a/ministack/services/codebuild.py b/ministack/services/codebuild.py
index 0a6b3a78..c28663b3 100644
--- a/ministack/services/codebuild.py
+++ b/ministack/services/codebuild.py
@@ -277,12 +277,7 @@ def _log_sink(build):
def emit(line):
ts = int(time.time() * 1000)
- events = stream["events"]
- events.append({"timestamp": ts, "message": line, "ingestionTime": ts})
- if stream["firstEventTimestamp"] is None:
- stream["firstEventTimestamp"] = ts
- stream["lastEventTimestamp"] = ts
- stream["lastIngestionTime"] = ts
+ _cwl._append_log_events(stream, [{"timestamp": ts, "message": line, "ingestionTime": ts}])
return emit
diff --git a/ministack/services/dynamodb.py b/ministack/services/dynamodb.py
index 0fe703e0..1d91f0b6 100644
--- a/ministack/services/dynamodb.py
+++ b/ministack/services/dynamodb.py
@@ -5628,13 +5628,15 @@ def _import_log_stream(import_arn):
def _log_import_error(import_arn, bucket, key, item_index, message):
"""One item-level error, in the shape the documented log event carries:
``{"itemS3Pointer": {...}, "importArn": ..., "errorMessages": [...]}``."""
+ from ministack.services import cloudwatch_logs as _cwl
+
try:
stream = _import_log_stream(import_arn)
except Exception:
logger.debug("DynamoDB import %s: error log unavailable", import_arn, exc_info=True)
return
ts = int(time.time() * 1000)
- stream["events"].append({
+ _cwl._append_log_events(stream, [{
"timestamp": ts,
"message": json.dumps({
"itemS3Pointer": {"bucket": bucket, "key": key, "itemIndex": item_index},
@@ -5642,11 +5644,7 @@ def _log_import_error(import_arn, bucket, key, item_index, message):
"errorMessages": [message],
}),
"ingestionTime": ts,
- })
- if stream["firstEventTimestamp"] is None:
- stream["firstEventTimestamp"] = ts
- stream["lastEventTimestamp"] = ts
- stream["lastIngestionTime"] = ts
+ }])
def _import_source_objects(s3_source, compression):
diff --git a/ministack/services/lambda_svc.py b/ministack/services/lambda_svc.py
index 5bb71f26..9e53e958 100644
--- a/ministack/services/lambda_svc.py
+++ b/ministack/services/lambda_svc.py
@@ -5032,12 +5032,10 @@ def _emit_lambda_logs(func: dict, request_id: str, log_text: str,
lines.append(_report())
else:
lines = [start, *(log_text.splitlines() if log_text else []), end, _report()]
- for line in lines:
- stream["events"].append({"timestamp": now_ms, "message": line, "ingestionTime": now_ms})
- if stream["firstEventTimestamp"] is None:
- stream["firstEventTimestamp"] = now_ms
- stream["lastEventTimestamp"] = now_ms
- stream["lastIngestionTime"] = now_ms
+ _cwl._append_log_events(stream, [
+ {"timestamp": now_ms, "message": line, "ingestionTime": now_ms}
+ for line in lines
+ ])
# Forward to any subscription filters on this group (e.g. Lambda log
# processor pattern), matching AWS (#896).
_cwl._fanout_to_subscription_filters(
diff --git a/tests/test_cloudwatch_logs_order.py b/tests/test_cloudwatch_logs_order.py
new file mode 100644
index 00000000..8cc18d76
--- /dev/null
+++ b/tests/test_cloudwatch_logs_order.py
@@ -0,0 +1,363 @@
+"""In-process regressions for FilterLogEvents ordering across streams."""
+
+import json
+import time
+import uuid
+
+import pytest
+
+from ministack.services import cloudwatch_logs as logs
+
+
+def log_group():
+ group = f"/test/filter-pattern/{uuid.uuid4().hex}"
+ assert logs._create_log_group({"logGroupName": group})[0] == 200
+ yield group
+ assert logs._delete_log_group({"logGroupName": group})[0] == 200
+
+
+def put_messages(group, stream, messages):
+ assert logs._create_log_stream({
+ "logGroupName": group, "logStreamName": stream,
+ })[0] == 200
+ response = logs._put_log_events({
+ "logGroupName": group,
+ "logStreamName": stream,
+ "logEvents": [{"timestamp": timestamp, "message": message}
+ for timestamp, message in messages],
+ })
+ assert response[0] == 200
+
+
+def test_filter_applies_limit_after_global_stream_order(log_group):
+ now = int(time.time() * 1000)
+ # The stream scanned first has newer events than the second stream.
+ put_messages(log_group, "newer", [(now + 30, "match newer")])
+ put_messages(log_group, "older", [
+ (now, "unrelated"), (now + 10, "match first"), (now + 20, "match second"),
+ ])
+ response = logs._filter_log_events({
+ "logGroupName": log_group, "filterPattern": "match", "limit": 1,
+ })
+ assert response[0] == 200
+ assert [e["message"] for e in json.loads(response[2])["events"]] == ["match first"]
+
+
+def test_filter_does_not_stop_later_stream_after_partial_global_limit(log_group):
+ now = int(time.time() * 1000)
+ put_messages(log_group, "newer", [(now + 30, "match newer")])
+ put_messages(log_group, "older", [(now + 10, "match first"), (now + 20, "match second")])
+ response = logs._filter_log_events({
+ "logGroupName": log_group, "filterPattern": "match", "limit": 2,
+ })
+ assert response[0] == 200
+ assert [e["message"] for e in json.loads(response[2])["events"]] == [
+ "match first", "match second",
+ ]
+
+
+def filter_events(group, **parameters):
+ response = logs._filter_log_events({"logGroupName": group, **parameters})
+ assert response[0] == 200
+ return json.loads(response[2])["events"]
+
+
+def count_filter_calls(monkeypatch):
+ compile_pattern = logs._compile_filter_pattern
+ checked = []
+
+ def compile_counted(pattern):
+ matcher = compile_pattern(pattern)
+
+ def matches(message):
+ checked.append(message)
+ return matcher(message)
+
+ return matches
+
+ monkeypatch.setattr(logs, "_compile_filter_pattern", compile_counted)
+ return checked
+
+
+def test_limit_stops_filtering_after_oldest_matches(log_group, monkeypatch):
+ now = int(time.time() * 1000)
+ put_messages(log_group, "a", [(now + i, f"match {i}") for i in range(2, 202, 2)])
+ put_messages(log_group, "b", [(now + i, f"match {i}") for i in range(1, 202, 2)])
+ # A later request can carry events older than those already stored.
+ assert logs._put_log_events({
+ "logGroupName": log_group, "logStreamName": "a",
+ "logEvents": [{"timestamp": now, "message": "skip"}],
+ })[0] == 200
+ checked = count_filter_calls(monkeypatch)
+ events = filter_events(log_group, filterPattern="match", limit=2)
+ assert [event["message"] for event in events] == ["match 1", "match 2"]
+ assert checked == ["skip", "match 1", "match 2"]
+
+
+def test_time_window_seeks_without_scanning_history(log_group):
+ now = int(time.time() * 1000)
+ put_messages(log_group, "history", [(now + i, f"event {i}") for i in range(8192)])
+ timestamp_reads = []
+
+ class CountedEvent(dict):
+ def __getitem__(self, key):
+ if key == "timestamp":
+ timestamp_reads.append(None)
+ return super().__getitem__(key)
+
+ stream = logs._log_groups[log_group]["streams"]["history"]
+ stream["events"] = [CountedEvent(event) for event in stream["events"]]
+ events = filter_events(log_group, startTime=now + 8000, endTime=now + 8001, limit=10)
+ assert [event["message"] for event in events] == ["event 8000", "event 8001"]
+ # Allow headroom for binary searches and merge setup, but no linear scan.
+ assert len(timestamp_reads) < 100
+
+
+def test_late_batch_keeps_reads_ordered_and_live_delivery_exact(log_group, monkeypatch):
+ now = int(time.time() * 1000)
+ put_messages(log_group, "s", [(now + 30, "newer")])
+ deliveries = []
+ monkeypatch.setattr(logs, "_fanout_to_live_tail_sessions",
+ lambda group, stream, events: deliveries.extend(events))
+ assert logs._put_log_events({
+ "logGroupName": log_group, "logStreamName": "s",
+ "logEvents": [
+ {"timestamp": now + 10, "message": "older"},
+ {"timestamp": now + 20, "message": "middle"},
+ ],
+ })[0] == 200
+ response = logs._get_log_events({
+ "logGroupName": log_group, "logStreamName": "s", "startFromHead": True,
+ })
+ assert response[0] == 200
+ assert [event["message"] for event in json.loads(response[2])["events"]] == [
+ "older", "middle", "newer",
+ ]
+ assert [event["message"] for event in deliveries] == ["older", "middle"]
+ stored = filter_events(log_group)
+ assert [event["eventId"] for event in stored[:2]] == [event["eventId"] for event in deliveries]
+
+
[email protected]("stream_order", [["a", "b"], ["b", "a"]])
+def test_equal_timestamps_keep_selected_stream_and_arrival_order(log_group, stream_order):
+ now = int(time.time() * 1000)
+ for stream in ("a", "b"):
+ put_messages(log_group, stream, [(now, f"{stream} first")])
+ assert logs._put_log_events({
+ "logGroupName": log_group, "logStreamName": stream,
+ "logEvents": [{"timestamp": now, "message": f"{stream} second"}],
+ })[0] == 200
+ events = filter_events(log_group, logStreamNames=stream_order, limit=3)
+ first, second = stream_order
+ assert [event["message"] for event in events] == [
+ f"{first} first", f"{first} second", f"{second} first",
+ ]
+
+
+def test_legacy_persistence_restores_order_before_limited_reads(log_group, monkeypatch, tmp_path):
+ from ministack.core import persistence
+
+ now = int(time.time() * 1000)
+ put_messages(log_group, "s", [(now + i, f"event {i}") for i in range(20)])
+ state = logs.get_state()
+ state["log_groups"][log_group]["streams"]["s"]["events"].reverse()
+ monkeypatch.setattr(persistence, "PERSIST_STATE", True)
+ monkeypatch.setattr(persistence, "STATE_DIR", str(tmp_path))
+ persistence.save_state("cloudwatch_logs", state)
+ assert logs._delete_log_group({"logGroupName": log_group})[0] == 200
+ logs.load_persisted_state(persistence.load_state("cloudwatch_logs"))
+ checked = count_filter_calls(monkeypatch)
+ events = filter_events(log_group, limit=1)
+ assert [event["message"] for event in events] == ["event 0"]
+ assert checked == ["event 0"]
+
+
[email protected]("producer", ["codebuild", "dynamodb"])
+def test_native_writers_keep_order_after_future_dated_api_event(log_group, monkeypatch, producer):
+ now = int(time.time() * 1000)
+ if producer == "codebuild":
+ from ministack.services import codebuild
+
+ stream_name = "native"
+ emit = codebuild._log_sink({"logs": {"groupName": log_group, "streamName": stream_name}})
+ monkeypatch.setattr(codebuild.time, "time", lambda: now / 1000)
+ else:
+ from ministack.services import dynamodb
+
+ monkeypatch.setattr(dynamodb, "_IMPORT_LOG_GROUP", log_group)
+ monkeypatch.setattr(dynamodb.time, "time", lambda: now / 1000)
+ import_id = uuid.uuid4().hex
+ stream_name = f"{import_id}/error"
+
+ def emit(message):
+ dynamodb._log_import_error(f"arn:aws:dynamodb:us-east-1:000000000000:table/t/import/{import_id}",
+ "bucket", "key", 0, message)
+
+ emit("initial")
+ assert logs._put_log_events({
+ "logGroupName": log_group, "logStreamName": stream_name,
+ "logEvents": [{"timestamp": now + 3000, "message": "future API event"}],
+ })[0] == 200
+ emit("later native event")
+ events = filter_events(log_group, limit=2)
+ assert len(events) == 2
+ assert [event["timestamp"] for event in events] == [now, now]
+ assert "initial" in events[0]["message"]
+ assert "later native event" in events[1]["message"]
+
+
+def test_reader_remains_consistent_during_late_arrival(log_group, monkeypatch):
+ now = int(time.time() * 1000)
+ put_messages(log_group, "s", [(now + 10, "ten"), (now + 20, "twenty")])
+ inserted = False
+
+ def matcher(message):
+ nonlocal inserted
+ if not inserted:
+ inserted = True
+ assert logs._put_log_events({
+ "logGroupName": log_group, "logStreamName": "s",
+ "logEvents": [{"timestamp": now, "message": "zero"}],
+ })[0] == 200
+ return True
+
+ monkeypatch.setattr(logs, "_compile_filter_pattern", lambda pattern: matcher)
+ assert [event["message"] for event in filter_events(log_group, limit=2)] == ["ten", "twenty"]
+ assert [event["message"] for event in filter_events(log_group, limit=2)] == ["zero", "ten"]
+
+
+def test_mixed_batches_and_ranges_match_stable_reference(log_group):
+ import random
+
+ randomizer = random.Random(731)
+ now = int(time.time() * 1000)
+ arrivals = {name: [] for name in ("a", "b", "c")}
+ for name in arrivals:
+ assert logs._create_log_stream({"logGroupName": log_group, "logStreamName": name})[0] == 200
+ for batch in range(30):
+ name = randomizer.choice(list(arrivals))
+ records = [{"timestamp": now + randomizer.randrange(12),
+ "message": f"{'keep' if index % 2 else 'drop'} {batch}-{index}"}
+ for index in range(4)]
+ arrivals[name].extend(records)
+ assert logs._put_log_events({
+ "logGroupName": log_group, "logStreamName": name, "logEvents": records,
+ })[0] == 200
+ for selection in (["a", "b", "c"], ["c", "a"], ["missing", "b"]):
+ reference = sorted((dict(event, logStreamName=name)
+ for name in selection for event in arrivals.get(name, [])),
+ key=lambda event: event["timestamp"])
+ for lower, upper in ((0, 11), (3, 7), (5, 5), (9, 2)):
+ expected = [event for event in reference if now + lower <= event["timestamp"] <= now + upper
+ and "keep" in event["message"]]
+ for limit in (1, 7, 10000):
+ actual = filter_events(log_group, logStreamNames=selection, filterPattern="keep",
+ startTime=now + lower, endTime=now + upper, limit=limit)
+ assert [(event["timestamp"], event["message"], event["logStreamName"]) for event in actual] == [
+ (event["timestamp"], event["message"], event["logStreamName"])
+ for event in expected[:limit]
+ ]
+
+
+def test_concurrent_late_batches_preserve_every_event(log_group):
+ from concurrent.futures import ThreadPoolExecutor
+ from threading import Barrier
+
+ from ministack.core.responses import get_account_id, get_region, request_scope
+
+ now = int(time.time() * 1000)
+ put_messages(log_group, "shared", [])
+ account, region = get_account_id(), get_region()
+ barrier = Barrier(4)
+
+ def write(worker):
+ with request_scope(account, region):
+ barrier.wait(timeout=5)
+ for batch in reversed(range(8)):
+ timestamp = now + batch * 4 + worker
+ assert logs._put_log_events({
+ "logGroupName": log_group, "logStreamName": "shared",
+ "logEvents": [{"timestamp": timestamp, "message": f"event {timestamp}"}],
+ })[0] == 200
+
+ with ThreadPoolExecutor(max_workers=4) as executor:
+ futures = [executor.submit(write, worker) for worker in range(4)]
+ for future in futures:
+ future.result(timeout=10)
+ events = filter_events(log_group, limit=100)
+ assert [event["timestamp"] for event in events] == list(range(now, now + 32))
+ stream = logs._log_groups[log_group]["streams"]["shared"]
+ assert stream["firstEventTimestamp"] == now
+ assert stream["lastEventTimestamp"] == now + 31
+
+
+def get_page(group, stream, **parameters):
+ response = logs._get_log_events({"logGroupName": group, "logStreamName": stream, **parameters})
+ assert response[0] == 200
+ body = json.loads(response[2])
+ return [event["message"] for event in body["events"]], body
+
+
+def test_forward_token_survives_late_arrivals(log_group):
+ now = int(time.time() * 1000)
+ put_messages(log_group, "s", [(now + 10, "e10"), (now + 20, "e20"), (now + 30, "e30")])
+ messages, body = get_page(log_group, "s", startFromHead=True, limit=2)
+ assert messages == ["e10", "e20"]
+ # One event lands behind the cursor, one between the cursor and stored history.
+ assert logs._put_log_events({
+ "logGroupName": log_group, "logStreamName": "s",
+ "logEvents": [{"timestamp": now + 15, "message": "e15"},
+ {"timestamp": now + 25, "message": "e25"}],
+ })[0] == 200
+ messages, _ = get_page(log_group, "s", nextToken=body["nextForwardToken"], limit=2)
+ assert messages == ["e25", "e30"]
+
+
+def test_backward_token_survives_late_arrivals(log_group):
+ now = int(time.time() * 1000)
+ put_messages(log_group, "s", [(now + 10, "e10"), (now + 20, "e20"), (now + 30, "e30")])
+ messages, body = get_page(log_group, "s", limit=2)
+ assert messages == ["e20", "e30"]
+ # e15 is older than everything read so far; e25 is behind the backward cursor.
+ assert logs._put_log_events({
+ "logGroupName": log_group, "logStreamName": "s",
+ "logEvents": [{"timestamp": now + 15, "message": "e15"},
+ {"timestamp": now + 25, "message": "e25"}],
+ })[0] == 200
+ messages, _ = get_page(log_group, "s", nextToken=body["nextBackwardToken"], limit=2)
+ assert messages == ["e10", "e15"]
+
+
+def test_backward_token_from_tail_excludes_later_events(log_group):
+ now = int(time.time() * 1000)
+ put_messages(log_group, "s", [(now + 10, "e10"), (now + 20, "e20")])
+ _, body = get_page(log_group, "s", startFromHead=True)
+ messages, body = get_page(log_group, "s", nextToken=body["nextForwardToken"])
+ assert messages == []
+ # e30 arrives after the reader reached the tail.
+ assert logs._put_log_events({
+ "logGroupName": log_group, "logStreamName": "s",
+ "logEvents": [{"timestamp": now + 30, "message": "e30"}],
+ })[0] == 200
+ messages, _ = get_page(log_group, "s", nextToken=body["nextBackwardToken"])
+ assert messages == ["e10", "e20"]
+
+
+def test_forward_token_splits_equal_timestamps_exactly_once(log_group):
+ now = int(time.time() * 1000)
+ put_messages(log_group, "s", [(now, "a"), (now, "b"), (now, "c")])
+ seen = []
+ messages, body = get_page(log_group, "s", startFromHead=True, limit=1)
+ seen += messages
+ # An equal-time arrival sorts after existing ties, so it is still ahead of the cursor.
+ assert logs._put_log_events({
+ "logGroupName": log_group, "logStreamName": "s",
+ "logEvents": [{"timestamp": now, "message": "d"}],
+ })[0] == 200
+ for _ in range(5):
+ messages, body = get_page(log_group, "s", nextToken=body["nextForwardToken"], limit=1)
+ if not messages:
+ break
+ seen += messages
+ assert seen == ["a", "b", "c", "d"]