Light, fluffy, and always free - The AWS Local Emulator alternative
fix: bound CloudTrail delivery retries (#3298)
Niels Kootstrad44a3fef62c3
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/CloudTrailTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/CloudTrailTest.javaindex 38fb0156b7..bf78cc19e5 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/CloudTrailTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/CloudTrailTest.java@@ -87,7 +87,7 @@ class CloudTrailTest {GetTrailStatusResponse started = cloudTrail.getTrailStatus(r -> r.name(trailName));assertThat(started.isLogging()).isTrue();- assertThat(started.latestDeliveryTime()).isNotNull();+ assertThat(started.latestDeliveryTime()).isNull();cloudTrail.stopLogging(r -> r.name(trailName));diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailEntry.java b/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailEntry.javaindex 13387abb1d..e4c9a119f3 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailEntry.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailEntry.java@@ -22,7 +22,17 @@ public record CloudTrailEntry(boolean logging,Long startLoggingTime,Long stopLoggingTime,- Map<String, String> tags) {+ Map<String, String> tags,+ Long latestDeliveryTime,+ String latestDeliveryError) {++ public CloudTrailEntry(Trail trail, List<EventSelector> selectors,+ List<AdvancedEventSelector> advancedSelectors, boolean logging,+ Long startLoggingTime, Long stopLoggingTime,+ Map<String, String> tags) {+ this(trail, selectors, advancedSelectors, logging, startLoggingTime, stopLoggingTime,+ tags, null, null);+ }// Not Map.copyOf: AWS allows a tag with a null value (AddTags omitting "Value"),// and Map.copyOf/Map.of reject null values.@@ -32,36 +42,52 @@ public record CloudTrailEntry(public CloudTrailEntry withTrail(Trail updated) {return new CloudTrailEntry(updated, selectors, advancedSelectors, logging,- startLoggingTime, stopLoggingTime, tags);+ startLoggingTime, stopLoggingTime, tags,+ latestDeliveryTime, latestDeliveryError);}/** Sets basic selectors, clearing any advanced selectors — a trail holds one kind or the other. */public CloudTrailEntry withSelectors(List<EventSelector> updated, boolean hasCustomSelectors) {Trail updatedTrail = withHasCustomSelectors(hasCustomSelectors);return new CloudTrailEntry(updatedTrail, updated, List.of(), logging,- startLoggingTime, stopLoggingTime, tags);+ startLoggingTime, stopLoggingTime, tags,+ latestDeliveryTime, latestDeliveryError);}/** Sets advanced selectors, clearing any basic selectors — a trail holds one kind or the other. */public CloudTrailEntry withAdvancedSelectors(List<AdvancedEventSelector> updated, boolean hasCustomSelectors) {Trail updatedTrail = withHasCustomSelectors(hasCustomSelectors);return new CloudTrailEntry(updatedTrail, List.of(), updated, logging,- startLoggingTime, stopLoggingTime, tags);+ startLoggingTime, stopLoggingTime, tags,+ latestDeliveryTime, latestDeliveryError);}public CloudTrailEntry withTags(Map<String, String> updated) {return new CloudTrailEntry(trail, selectors, advancedSelectors, logging,- startLoggingTime, stopLoggingTime, updated);+ startLoggingTime, stopLoggingTime, updated,+ latestDeliveryTime, latestDeliveryError);}public CloudTrailEntry startLogging(long time) {return new CloudTrailEntry(trail, selectors, advancedSelectors, true, time,- stopLoggingTime, tags);+ stopLoggingTime, tags,+ latestDeliveryTime, latestDeliveryError);}public CloudTrailEntry stopLogging(long time) {return new CloudTrailEntry(trail, selectors, advancedSelectors, false, startLoggingTime,- time, tags);+ time, tags,+ latestDeliveryTime, latestDeliveryError);+ }++ public CloudTrailEntry withDeliveryFailure(String error) {+ return new CloudTrailEntry(trail, selectors, advancedSelectors, logging,+ startLoggingTime, stopLoggingTime, tags, latestDeliveryTime, error);+ }++ public CloudTrailEntry withDeliverySuccess(long time) {+ return new CloudTrailEntry(trail, selectors, advancedSelectors, logging,+ startLoggingTime, stopLoggingTime, tags, time, null);}private Trail withHasCustomSelectors(boolean hasCustomSelectors) {diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailJsonHandler.javaindex 74565c315a..0154be3096 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailJsonHandler.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailJsonHandler.java@@ -206,7 +206,12 @@ public class CloudTrailJsonHandler {resp.put("IsLogging", status.logging());if (status.startLoggingTime() != null) {resp.put("StartLoggingTime", status.startLoggingTime() / 1000.0);- resp.put("LatestDeliveryTime", status.startLoggingTime() / 1000.0);+ }+ if (status.latestDeliveryTime() != null) {+ resp.put("LatestDeliveryTime", status.latestDeliveryTime() / 1000.0);+ }+ if (status.latestDeliveryError() != null) {+ resp.put("LatestDeliveryError", status.latestDeliveryError());}if (status.stopLoggingTime() != null) {resp.put("StopLoggingTime", status.stopLoggingTime() / 1000.0);diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailLogWriter.java b/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailLogWriter.javaindex 51bbeba875..fc81b98c3b 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailLogWriter.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailLogWriter.java@@ -1,9 +1,10 @@package io.github.hectorvent.floci.services.cloudtrail;-import com.fasterxml.jackson.core.JsonGenerator;import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ArrayNode;import com.fasterxml.jackson.databind.node.ObjectNode;import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.services.cloudtrail.model.Trail;import io.github.hectorvent.floci.services.s3.S3Service;@@ -48,10 +49,10 @@ import java.util.zip.GZIPOutputStream;public class CloudTrailLogWriter {private static final Logger LOG = Logger.getLogger(CloudTrailLogWriter.class);+ static final int MAX_RECORDS_PER_LOG_FILE = 1_000;private static final DateTimeFormatter PATH_DATE = DateTimeFormatter.ofPattern("yyyy/MM/dd");private static final DateTimeFormatter FILE_TS = DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmm'Z'");- static final int MAX_RECORDS_PER_LOG_FILE = 1_000;private static final String FILENAME_RAND_ALPHABET ="abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";@@ -96,7 +97,7 @@ public class CloudTrailLogWriter {void stop(@Observes ShutdownEvent event) {if (executor != null) {// Best-effort final flush so shutdown doesn't lose buffered events.- try { flushAll(); } catch (Exception e) { LOG.warnv(e, "CloudTrail final flush on shutdown failed — some records may be lost"); }+ try { flushAll(); } catch (Exception e) { LOG.warnv(e, "CloudTrail final flush on shutdown failed: some records may be lost"); }executor.shutdownNow();executor = null;}@@ -161,9 +162,9 @@ public class CloudTrailLogWriter {private int flushTrail(CloudTrailService.TrailKey key) {Trail trail = cloudTrailService.getTrail(key.region(), key.trailName());if (trail == null) {- // Trail was deleted while records were pending. Drop only the- // records that existed when this flush cycle started.- return cloudTrailService.drainPendingRecords(key, MAX_RECORDS_PER_LOG_FILE).size();+ // Trail was deleted while records were pending: drop them.+ cloudTrailService.discardPendingRecords(key);+ return 0;}List<ObjectNode> records = cloudTrailService.drainPendingRecords(key, MAX_RECORDS_PER_LOG_FILE);@@ -186,19 +187,30 @@ public class CloudTrailLogWriter {} catch (RuntimeException e) {// Re-queue so records survive the failed flush and are retried next cycle.cloudTrailService.requeueRecords(key, records);+ try {+ cloudTrailService.recordDeliveryFailure(key, deliveryError(e));+ } catch (RuntimeException statusError) {+ LOG.warnv(statusError, "CloudTrail delivery failure status update failed for trail {0}", key.trailName());+ }LOG.warnv(e, "CloudTrail flush failed for trail {0} ({1} records re-queued)",key.trailName(), records.size());throw e;}+ try {+ cloudTrailService.recordDeliverySuccess(key, System.currentTimeMillis());+ } catch (RuntimeException e) {+ LOG.warnv(e, "CloudTrail delivery success status update failed for trail {0}", key.trailName());+ }+ cloudTrailService.completeDelivery(key);- // The write above already succeeded and durably delivered the records —+ // The write above already succeeded and durably delivered the records:// from here on, records must never be re-queued. Doing so on a failure// in this block would deliver the same batch to S3 again next flush.try {// This write goes straight to S3Service, bypassing the HTTP-facing// S3Controller that normally emits data events for API-driven puts.// Any trail whose selector matches its own destination bucket must- // still see its own deliveries — that's the real circular-logging+ // still see its own deliveries: that is the real circular-logging// behavior (issue #1192 / PR #1194) this emulator exists to prove.cloudTrailService.emitS3DataEvent(CloudTrailService.S3EventInput.builder().region(key.eventRegion())@@ -221,19 +233,25 @@ public class CloudTrailLogWriter {return records.size();}+ private String deliveryError(RuntimeException e) {+ String message = e.getMessage() == null ? "" : ": " + e.getMessage();+ if (e instanceof AwsException awsException) {+ return awsException.getErrorCode() + message;+ }+ return e.getClass().getSimpleName() + message;+ }+private byte[] serializeAndGzip(List<ObjectNode> records) {+ ObjectNode envelope = mapper.createObjectNode();+ ArrayNode arr = envelope.putArray("Records");+ for (ObjectNode r : records) {+ arr.add(r);+ }try {- ByteArrayOutputStream baos = new ByteArrayOutputStream();+ byte[] json = mapper.writeValueAsBytes(envelope);+ ByteArrayOutputStream baos = new ByteArrayOutputStream(Math.max(64, json.length / 4));try (GZIPOutputStream gz = new GZIPOutputStream(baos)) {- try (JsonGenerator generator = mapper.getFactory().createGenerator(gz)) {- generator.writeStartObject();- generator.writeArrayFieldStart("Records");- for (ObjectNode record : records) {- generator.writeTree(record);- }- generator.writeEndArray();- generator.writeEndObject();- }+ gz.write(json);}return baos.toByteArray();} catch (Exception e) {diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailService.java b/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailService.javaindex 74f96611ee..fb8f6c7d0a 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailService.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailService.java@@ -23,15 +23,16 @@ import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.jboss.logging.Logger;+import java.nio.charset.StandardCharsets;import java.time.Instant;import java.time.format.DateTimeFormatter;+import java.util.ArrayDeque;import java.util.ArrayList;import java.util.List;-import java.util.ListIterator;import java.util.Map;import java.util.UUID;import java.util.concurrent.ConcurrentHashMap;-import java.util.concurrent.ConcurrentLinkedDeque;+import java.util.function.Function;import java.util.function.Supplier;@ApplicationScoped@@ -41,14 +42,16 @@ public class CloudTrailService {private static final String EVENT_VERSION = "1.11";private static final String S3_EVENT_SOURCE = "s3.amazonaws.com";+ static final int MAX_PENDING_RECORDS_PER_TRAIL = 1024;+ static final long MAX_PENDING_BYTES_PER_TRAIL = 4L * 1024L * 1024L;private final StorageBackend<String, CloudTrailEntry> store;private final RegionResolver regionResolver;private final IamService iamService;private final ObjectMapper mapper;- /** Per-trail pending record buffers — ephemeral, never persisted. */- private final ConcurrentHashMap<TrailKey, ConcurrentLinkedDeque<ObjectNode>> pendingRecordsByTrail =+ /** Per-trail pending record buffers: ephemeral, never persisted. */+ private final ConcurrentHashMap<PendingTrailKey, PendingRecordBuffer> pendingRecordsByTrail =new ConcurrentHashMap<>();/**@@ -112,7 +115,8 @@ public class CloudTrailService {name, arn, s3BucketName, s3KeyPrefix, snsTopicArn,includeGlobalServiceEvents, isMultiRegionTrail, region,enableLogFileValidation, false, false, isOrganizationTrail);- store.put(key, new CloudTrailEntry(trail, List.of(), List.of(), false, null, null, initialTags));+ store.put(key, new CloudTrailEntry(trail, List.of(), List.of(), false, null, null,+ initialTags, null, null));return trail;}@@ -241,15 +245,16 @@ public class CloudTrailService {public TrailStatus getTrailStatus(String region, String trailNameOrArn) {Trail trail = findTrailOrThrow(region, trailNameOrArn);return store.get(regionKey(trail.homeRegion(), trail.name()))- .map(e -> new TrailStatus(e.logging(), e.startLoggingTime(), e.stopLoggingTime()))- .orElse(new TrailStatus(false, null, null));+ .map(e -> new TrailStatus(e.logging(), e.startLoggingTime(), e.stopLoggingTime(),+ e.latestDeliveryTime(), e.latestDeliveryError()))+ .orElse(new TrailStatus(false, null, null, null, null));}// --- Tagging ---//// AddTags/RemoveTags/ListTags identify the trail solely by ARN (ResourceId /// ResourceIdList), unlike every other CloudTrail action here which also accepts a- // bare trail name — so these don't take a `region` parameter.+ // bare trail name, so these do not take a `region` parameter.private static final int MAX_TAGS_PER_RESOURCE = 50;@@ -354,8 +359,13 @@ public class CloudTrailService {for (MatchedTrail mt : matched) {ObjectNode copy = record.deepCopy();copy.put("recipientAccountId", regionResolver.getAccountId());- queueFor(new TrailKey(mt.region(), mt.trail().name(), region)).add(copy);- LOG.tracev("Emitted CloudTrail event {0} for trail {1}", in.eventName(), mt.trail().name());+ boolean accepted = append(new TrailKey(mt.region(), mt.trail().name(), region), copy);+ if (accepted) {+ LOG.tracev("Emitted CloudTrail event {0} for trail {1}", in.eventName(), mt.trail().name());+ } else {+ LOG.tracev("Dropped CloudTrail event {0} for trail {1}: retry buffer is full",+ in.eventName(), mt.trail().name());+ }}} catch (Exception e) {// Never let emission take down an S3 op.@@ -365,17 +375,32 @@ public class CloudTrailService {}public void requeueRecords(TrailKey key, List<ObjectNode> records) {- if (records.isEmpty()) {- return;- }- ConcurrentLinkedDeque<ObjectNode> q = queueFor(key);- ListIterator<ObjectNode> it = records.listIterator(records.size());- // Add in reverse so the first failed record remains first.- while (it.hasPrevious()) {- q.addFirst(it.previous());+ if (!records.isEmpty()) {+ List<PendingRecord> pending = records.stream()+ .map(record -> new PendingRecord(record, estimatedRecordBytes(record)))+ .toList();+ pendingRecordsByTrail.compute(pendingTrailKey(key), (ignored, buffer) -> {+ PendingRecordBuffer updated = buffer == null ? new PendingRecordBuffer() : buffer;+ updated.requeueFront(key.eventRegion(), pending);+ return updated.isEmpty() ? null : updated;+ });}}+ public void completeDelivery(TrailKey key) {+ pendingRecordsByTrail.computeIfPresent(pendingTrailKey(key), (ignored, buffer) -> {+ buffer.completeDelivery(key.eventRegion());+ return buffer.isEmpty() ? null : buffer;+ });+ }++ public void discardPendingRecords(TrailKey key) {+ pendingRecordsByTrail.computeIfPresent(pendingTrailKey(key), (ignored, buffer) -> {+ buffer.discard(key.eventRegion());+ return buffer.isEmpty() ? null : buffer;+ });+ }+public List<ObjectNode> drainPendingRecords(TrailKey key) {return drainPendingRecords(key, Integer.MAX_VALUE);}@@ -384,41 +409,68 @@ public class CloudTrailService {if (maxRecords <= 0) {return List.of();}- ConcurrentLinkedDeque<ObjectNode> q = pendingRecordsByTrail.get(key);- if (q == null) {- return List.of();- }List<ObjectNode> drained = new ArrayList<>();- ObjectNode r;- while (drained.size() < maxRecords && (r = q.pollFirst()) != null) {- drained.add(r);- }- return drained;- }-- public int pendingRecordCount(TrailKey key) {- ConcurrentLinkedDeque<ObjectNode> q = pendingRecordsByTrail.get(key);- return q == null ? 0 : q.size();+ pendingRecordsByTrail.compute(pendingTrailKey(key), (ignored, buffer) -> {+ if (buffer == null) {+ return null;+ }+ drained.addAll(buffer.drain(key.eventRegion(), maxRecords));+ return buffer.isEmpty() ? null : buffer;+ });+ return drained.isEmpty() ? List.of() : drained;}public List<TrailKey> trailsWithPendingRecords() {List<TrailKey> result = new ArrayList<>();- for (Map.Entry<TrailKey, ConcurrentLinkedDeque<ObjectNode>> e : pendingRecordsByTrail.entrySet()) {- if (!e.getValue().isEmpty()) {- result.add(e.getKey());+ for (Map.Entry<PendingTrailKey, PendingRecordBuffer> e : pendingRecordsByTrail.entrySet()) {+ for (String eventRegion : e.getValue().eventRegions()) {+ result.add(new TrailKey(e.getKey().region(), e.getKey().trailName(), eventRegion));}}return result;}+ public int pendingRecordCount(TrailKey key) {+ PendingRecordBuffer buffer = pendingRecordsByTrail.get(pendingTrailKey(key));+ return buffer == null ? 0 : buffer.pendingCount(key.eventRegion());+ }+public Trail getTrail(String region, String trailName) {return store.get(regionKey(region, trailName)).map(CloudTrailEntry::trail).orElse(null);}- private ConcurrentLinkedDeque<ObjectNode> queueFor(TrailKey key) {- return pendingRecordsByTrail.computeIfAbsent(key, k -> new ConcurrentLinkedDeque<>());+ public void recordDeliveryFailure(TrailKey key, String error) {+ updateDeliveryStatus(key, entry -> entry.withDeliveryFailure(error));+ }++ public void recordDeliverySuccess(TrailKey key, long time) {+ updateDeliveryStatus(key, entry -> entry.withDeliverySuccess(time));+ }++ private void updateDeliveryStatus(TrailKey key, Function<CloudTrailEntry, CloudTrailEntry> update) {+ String storeKey = regionKey(key.region(), key.trailName());+ withTrailLock(storeKey, () -> store.get(storeKey).ifPresent(entry -> store.put(storeKey, update.apply(entry))));+ }++ private boolean append(TrailKey key, ObjectNode record) {+ long recordBytes = estimatedRecordBytes(record);+ boolean[] accepted = {false};+ pendingRecordsByTrail.compute(pendingTrailKey(key), (ignored, buffer) -> {+ PendingRecordBuffer updated = buffer == null ? new PendingRecordBuffer() : buffer;+ accepted[0] = updated.append(key.eventRegion(), record, recordBytes);+ return updated.isEmpty() ? null : updated;+ });+ return accepted[0];+ }++ private long estimatedRecordBytes(ObjectNode record) {+ try {+ return mapper.writeValueAsBytes(record).length;+ } catch (Exception e) {+ return record.toString().getBytes(StandardCharsets.UTF_8).length;+ }}/**@@ -429,6 +481,137 @@ public class CloudTrailService {*/public record TrailKey(String region, String trailName, String eventRegion) {}+ private record PendingTrailKey(String region, String trailName) {}++ private record PendingRecord(ObjectNode record, long byteCount) {}++ private final class PendingRecordBuffer {+ private final Map<String, ArrayDeque<PendingRecord>> recordsByRegion = new ConcurrentHashMap<>();+ private final Map<String, List<PendingRecord>> inFlightByRegion = new ConcurrentHashMap<>();+ private int recordCount;+ private long byteCount;++ synchronized boolean append(String eventRegion, ObjectNode record, long recordBytes) {+ if (!inFlightByRegion.isEmpty()+ && (recordCount >= MAX_PENDING_RECORDS_PER_TRAIL+ || byteCount + recordBytes > MAX_PENDING_BYTES_PER_TRAIL)) {+ return false;+ }+ recordsByRegion.computeIfAbsent(eventRegion, ignored -> new ArrayDeque<>())+ .addLast(new PendingRecord(record, recordBytes));+ recordCount++;+ byteCount += recordBytes;+ return true;+ }++ synchronized void requeueFront(String eventRegion, List<PendingRecord> drained) {+ ArrayDeque<PendingRecord> records = recordsByRegion.computeIfAbsent(+ eventRegion, ignored -> new ArrayDeque<>());+ boolean wasInFlight = inFlightByRegion.remove(eventRegion) != null;+ for (int i = drained.size() - 1; i >= 0; i--) {+ PendingRecord pending = drained.get(i);+ records.addFirst(pending);+ if (!wasInFlight) {+ recordCount++;+ byteCount += pending.byteCount();+ }+ }+ trimTailToLimit(eventRegion);+ }++ synchronized List<ObjectNode> drain(String eventRegion) {+ return drain(eventRegion, Integer.MAX_VALUE);+ }++ synchronized List<ObjectNode> drain(String eventRegion, int maxRecords) {+ ArrayDeque<PendingRecord> records = recordsByRegion.remove(eventRegion);+ if (records == null || records.isEmpty()) {+ return List.of();+ }+ List<PendingRecord> selected = new ArrayList<>(Math.min(records.size(), maxRecords));+ while (selected.size() < maxRecords && !records.isEmpty()) {+ selected.add(records.removeFirst());+ }+ if (!records.isEmpty()) {+ recordsByRegion.put(eventRegion, records);+ }+ inFlightByRegion.put(eventRegion, selected);+ List<ObjectNode> drained = new ArrayList<>(selected.size());+ for (PendingRecord pending : selected) {+ drained.add(pending.record());+ }+ return drained;+ }++ synchronized void completeDelivery(String eventRegion) {+ List<PendingRecord> inFlight = inFlightByRegion.remove(eventRegion);+ if (inFlight == null) {+ return;+ }+ for (PendingRecord pending : inFlight) {+ recordCount--;+ byteCount -= pending.byteCount();+ }+ }++ synchronized void discard(String eventRegion) {+ ArrayDeque<PendingRecord> queued = recordsByRegion.remove(eventRegion);+ if (queued != null) {+ for (PendingRecord pending : queued) {+ recordCount--;+ byteCount -= pending.byteCount();+ }+ }+ List<PendingRecord> inFlight = inFlightByRegion.remove(eventRegion);+ if (inFlight != null) {+ for (PendingRecord pending : inFlight) {+ recordCount--;+ byteCount -= pending.byteCount();+ }+ }+ }++ synchronized boolean isEmpty() {+ return recordCount == 0 && inFlightByRegion.isEmpty();+ }++ synchronized int pendingCount(String eventRegion) {+ ArrayDeque<PendingRecord> records = recordsByRegion.get(eventRegion);+ List<PendingRecord> inFlight = inFlightByRegion.get(eventRegion);+ return (records == null ? 0 : records.size()) + (inFlight == null ? 0 : inFlight.size());+ }++ synchronized List<String> eventRegions() {+ return java.util.stream.Stream.concat(recordsByRegion.keySet().stream(), inFlightByRegion.keySet().stream())+ .distinct()+ .filter(eventRegion -> {+ ArrayDeque<PendingRecord> records = recordsByRegion.get(eventRegion);+ return (records != null && !records.isEmpty()) || inFlightByRegion.containsKey(eventRegion);+ })+ .toList();+ }++ private void trimTailToLimit(String preferredRegion) {+ while (recordCount > MAX_PENDING_RECORDS_PER_TRAIL+ || byteCount > MAX_PENDING_BYTES_PER_TRAIL) {+ ArrayDeque<PendingRecord> records = recordsByRegion.get(preferredRegion);+ if (records == null || records.isEmpty()) {+ records = recordsByRegion.values().stream()+ .filter(queue -> !queue.isEmpty())+ .findFirst()+ .orElseThrow();+ }+ PendingRecord dropped = records.removeLast();+ recordCount--;+ byteCount -= dropped.byteCount();+ }+ }+ }++ private static PendingTrailKey pendingTrailKey(TrailKey key) {+ return new PendingTrailKey(key.region(), key.trailName());+ }+// --- Helpers ---private List<MatchedTrail> trailsMatching(String region, S3EventInput in) {@@ -493,7 +676,7 @@ public class CloudTrailService {private boolean matchesAnyAdvancedSelector(List<AdvancedEventSelector> selectors, S3EventInput in) {String arn = "arn:aws:s3:::" + in.bucketName() + (in.key() != null ? "/" + in.key() : "");// Bucket-level operations (e.g. ListObjects) have no object key and are reported- // by CloudTrail as AWS::S3::Bucket resources, not AWS::S3::Object — matching real+ // by CloudTrail as AWS::S3::Bucket resources, not AWS::S3::Object: matching real// AWS behavior, an AWS::S3::Object DataResource selector must never match them.String resourceType = in.key() != null ? "AWS::S3::Object" : "AWS::S3::Bucket";for (AdvancedEventSelector sel : selectors) {@@ -778,7 +961,12 @@ public class CloudTrailService {return colon < 0 ? key : key.substring(0, colon);}- public record TrailStatus(boolean logging, Long startLoggingTime, Long stopLoggingTime) {}+ public record TrailStatus(boolean logging, Long startLoggingTime, Long stopLoggingTime,+ Long latestDeliveryTime, String latestDeliveryError) {+ public TrailStatus(boolean logging, Long startLoggingTime, Long stopLoggingTime) {+ this(logging, startLoggingTime, stopLoggingTime, null, null);+ }+ }private record MatchedTrail(Trail trail, String region) {}diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailLogWriterBoundedRetryIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailLogWriterBoundedRetryIntegrationTest.javanew file mode 100644index 0000000000..0d3a0ed83f--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailLogWriterBoundedRetryIntegrationTest.java@@ -0,0 +1,330 @@+package io.github.hectorvent.floci.services.cloudtrail;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.databind.node.ObjectNode;+import io.github.hectorvent.floci.core.common.XmlParser;+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;+import io.quarkus.test.junit.QuarkusTest;+import jakarta.inject.Inject;+import org.junit.jupiter.api.BeforeAll;+import org.junit.jupiter.api.Test;++import java.io.ByteArrayInputStream;+import java.util.ArrayList;+import java.util.HashSet;+import java.util.List;+import java.util.Set;+import java.util.UUID;+import java.util.zip.GZIPInputStream;++import static io.restassured.RestAssured.given;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertTrue;++@QuarkusTest+class CloudTrailLogWriterBoundedRetryIntegrationTest {++ private static final String CT_TARGET = "CloudTrail_20131101.";+ private static final String JSON11 = "application/x-amz-json-1.1";+ private static final int RETRY_RECORD_LIMIT = 1024;+ private static final int PRODUCED_EVENTS = RETRY_RECORD_LIMIT + 80;++ @Inject+ CloudTrailLogWriter writer;++ @Inject+ CloudTrailService cloudTrailService;++ @BeforeAll+ static void configureRestAssured() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ }++ @Test+ void retryStateStaysBoundedAndDeliversRetainedPrefixAfterDestinationRecovery() throws Exception {+ String suffix = UUID.randomUUID().toString().substring(0, 8);+ String sourceBucket = "bounded-source-" + suffix;+ String destinationBucket = "bounded-logs-" + suffix;+ String trailName = "bounded-trail-" + suffix;+ String region = "us-east-1";++ createBucket(sourceBucket);+ createTrail(trailName, destinationBucket);+ selectSourceBucket(trailName, sourceBucket);+ startLogging(trailName);++ JsonNode startedStatus = getTrailStatus(trailName);+ assertFalse(startedStatus.has("LatestDeliveryTime"),+ "starting a trail must not claim that a delivery already happened: " + startedStatus);+ assertFalse(startedStatus.has("LatestDeliveryAttemptTime"),+ "GetTrailStatus must not expose the retired delivery-attempt field: " + startedStatus);++ for (int i = 0; i < PRODUCED_EVENTS; i++) {+ putObject(sourceBucket, eventKey(i), "event-" + i);+ if (i % 100 == 99) {+ writer.flushNow();+ }+ }+ writer.flushNow();++ JsonNode failedStatus = getTrailStatus(trailName);+ assertTrue(failedStatus.hasNonNull("LatestDeliveryError"),+ "failed S3 delivery must be visible through GetTrailStatus: " + failedStatus);+ assertTrue(failedStatus.path("LatestDeliveryError").asText().contains("NoSuchBucket"),+ "GetTrailStatus should expose the S3 delivery error: " + failedStatus);++ createBucket(destinationBucket);+ writer.flushNow();++ List<JsonNode> delivered = deliveredRecords(destinationBucket);+ assertEquals(RETRY_RECORD_LIMIT, delivered.size(),+ "only the retained retry prefix should be delivered after recovery");+ for (int i = 0; i < RETRY_RECORD_LIMIT; i++) {+ assertEquals(eventKey(i), delivered.get(i).path("requestParameters").path("key").asText());+ }+ assertFalse(delivered.stream().anyMatch(r ->+ eventKey(RETRY_RECORD_LIMIT).equals(r.path("requestParameters").path("key").asText())),+ "events produced after the retry buffer filled should remain dropped");++ JsonNode recoveredStatus = getTrailStatus(trailName);+ assertFalse(recoveredStatus.has("LatestDeliveryError"),+ "successful delivery should clear LatestDeliveryError: " + recoveredStatus);+ assertTrue(recoveredStatus.hasNonNull("LatestDeliveryTime"),+ "successful delivery should update LatestDeliveryTime: " + recoveredStatus);+ }++ @Test+ void healthyBurstAboveRetryLimitIsDeliveredWithoutLoss() throws Exception {+ String suffix = UUID.randomUUID().toString().substring(0, 8);+ String sourceBucket = "healthy-burst-source-" + suffix;+ String destinationBucket = "healthy-burst-logs-" + suffix;+ String trailName = "healthy-burst-trail-" + suffix;++ createBucket(sourceBucket);+ createBucket(destinationBucket);+ createTrail(trailName, destinationBucket);+ selectSourceBucket(trailName, sourceBucket);+ startLogging(trailName);++ for (int i = 0; i < PRODUCED_EVENTS; i++) {+ putObject(sourceBucket, eventKey(i), "event-" + i);+ }+ writer.flushNow();++ List<JsonNode> delivered = deliveredRecords(destinationBucket);+ assertEquals(PRODUCED_EVENTS, delivered.size(),+ "a healthy burst must not be limited by the retry budget");+ Set<String> deliveredKeys = new HashSet<>();+ for (JsonNode record : delivered) {+ deliveredKeys.add(record.path("requestParameters").path("key").asText());+ }+ for (int i = 0; i < PRODUCED_EVENTS; i++) {+ assertTrue(deliveredKeys.contains(eventKey(i)));+ }+ }++ @Test+ void retryStateStaysBoundedByBytes() throws Exception {+ String suffix = UUID.randomUUID().toString().substring(0, 8);+ String sourceBucket = "bounded-bytes-source-" + suffix;+ String destinationBucket = "bounded-bytes-logs-" + suffix;+ String trailName = "bounded-bytes-trail-" + suffix;+ int producedEvents = 750;++ createBucket(sourceBucket);+ createTrail(trailName, destinationBucket);+ selectSourceBucket(trailName, sourceBucket);+ startLogging(trailName);++ String largeUserAgent = "x".repeat(6000);+ for (int i = 0; i < producedEvents; i++) {+ cloudTrailService.emitS3DataEvent(CloudTrailService.S3EventInput.builder()+ .region("us-east-1")+ .eventName("PutObject")+ .bucketName(sourceBucket)+ .key(eventKey(i))+ .userAgent(largeUserAgent)+ .eventTimeMillis(System.currentTimeMillis())+ .build());+ if (i % 100 == 99) {+ writer.flushNow();+ }+ }+ writer.flushNow();++ createBucket(destinationBucket);+ writer.flushNow();++ List<JsonNode> delivered = deliveredRecords(destinationBucket);+ assertTrue(delivered.size() < producedEvents,+ "the byte limit must drop records before the record limit: " + delivered.size());+ assertTrue(delivered.size() > 0, "the byte-limited retry buffer should retain records");+ assertEquals(eventKey(0),+ delivered.get(0).path("requestParameters").path("key").asText());+ }++ @Test+ void retryRecordLimitIsSharedAcrossEventRegions() throws Exception {+ String suffix = UUID.randomUUID().toString().substring(0, 8);+ String sourceBucket = "bounded-regions-source-" + suffix;+ String destinationBucket = "bounded-regions-logs-" + suffix;+ String trailName = "bounded-regions-trail-" + suffix;++ createBucket(sourceBucket);+ createTrail(trailName, destinationBucket, true);+ selectSourceBucket(trailName, sourceBucket);+ startLogging(trailName);++ emitEvents(sourceBucket, "us-east-1", 600, "east");+ writer.flushNow();+ emitEvents(sourceBucket, "us-west-2", 600, "west");+ writer.flushNow();++ createBucket(destinationBucket);+ writer.flushNow();++ List<JsonNode> delivered = deliveredRecords(destinationBucket);+ assertEquals(RETRY_RECORD_LIMIT, delivered.size(),+ "the record cap must apply to the trail, not each event region");+ assertTrue(delivered.stream().anyMatch(record -> "us-east-1".equals(record.path("awsRegion").asText())));+ assertTrue(delivered.stream().anyMatch(record -> "us-west-2".equals(record.path("awsRegion").asText())));+ }++ @Test+ void retryBudgetIncludesRecordsInFlightAcrossEventRegions() {+ String suffix = UUID.randomUUID().toString().substring(0, 8);+ String sourceBucket = "in-flight-source-" + suffix;+ String destinationBucket = "in-flight-logs-" + suffix;+ String trailName = "in-flight-trail-" + suffix;++ createBucket(sourceBucket);+ createTrail(trailName, destinationBucket, true);+ selectSourceBucket(trailName, sourceBucket);+ startLogging(trailName);++ emitEvents(sourceBucket, "us-east-1", RETRY_RECORD_LIMIT, "east");+ CloudTrailService.TrailKey eastKey =+ new CloudTrailService.TrailKey("us-east-1", trailName, "us-east-1");+ List<ObjectNode> inFlight = cloudTrailService.drainPendingRecords(eastKey);++ emitEvents(sourceBucket, "us-west-2", 80, "west");+ cloudTrailService.requeueRecords(eastKey, inFlight);++ List<ObjectNode> retainedEast = cloudTrailService.drainPendingRecords(eastKey);+ List<ObjectNode> retainedWest = cloudTrailService.drainPendingRecords(+ new CloudTrailService.TrailKey("us-east-1", trailName, "us-west-2"));++ assertEquals(RETRY_RECORD_LIMIT, retainedEast.size(),+ "in-flight records must reserve the trail-wide retry budget");+ assertTrue(retainedWest.isEmpty(),+ "events from another region must not displace older in-flight records");+ for (int i = 0; i < RETRY_RECORD_LIMIT; i++) {+ assertEquals("east/" + eventKey(i),+ retainedEast.get(i).path("requestParameters").path("key").asText());+ }+ }++ private static void createTrail(String trailName, String destinationBucket) {+ createTrail(trailName, destinationBucket, false);+ }++ private static void createTrail(String trailName, String destinationBucket, boolean multiRegion) {+ invokeCloudTrail("CreateTrail", String.format("""+ {"Name":"%s","S3BucketName":"%s","IsMultiRegionTrail":%s}+ """, trailName, destinationBucket, multiRegion))+ .then().statusCode(200);+ }++ private static void selectSourceBucket(String trailName, String sourceBucket) {+ invokeCloudTrail("PutEventSelectors", String.format("""+ {+ "TrailName": "%s",+ "EventSelectors": [+ {+ "ReadWriteType": "All",+ "IncludeManagementEvents": false,+ "DataResources": [+ {"Type": "AWS::S3::Object", "Values": ["arn:aws:s3:::%s/"]}+ ]+ }+ ]+ }+ """, trailName, sourceBucket))+ .then().statusCode(200);+ }++ private static void startLogging(String trailName) {+ invokeCloudTrail("StartLogging", String.format("{\"Name\":\"%s\"}", trailName))+ .then().statusCode(200);+ }++ private static JsonNode getTrailStatus(String trailName) throws Exception {+ String body = invokeCloudTrail("GetTrailStatus", String.format("{\"Name\":\"%s\"}", trailName))+ .then().statusCode(200)+ .extract().asString();+ return new ObjectMapper().readTree(body);+ }++ private static io.restassured.response.Response invokeCloudTrail(String action, String body) {+ return given()+ .header("X-Amz-Target", CT_TARGET + action)+ .contentType(JSON11)+ .body(body)+ .when().post("/");+ }++ private static void createBucket(String name) {+ given().when().put("/" + name).then().statusCode(200);+ }++ private static void putObject(String bucket, String key, String body) {+ given().body(body)+ .when().put("/" + bucket + "/" + key)+ .then().statusCode(200);+ }++ private void emitEvents(String sourceBucket, String region, int count, String prefix) {+ for (int i = 0; i < count; i++) {+ cloudTrailService.emitS3DataEvent(CloudTrailService.S3EventInput.builder()+ .region(region)+ .eventName("PutObject")+ .bucketName(sourceBucket)+ .key(prefix + "/" + eventKey(i))+ .eventTimeMillis(System.currentTimeMillis())+ .build());+ }+ }++ private static String eventKey(int index) {+ return String.format("events/%04d.txt", index);+ }++ private static List<JsonNode> deliveredRecords(String bucket) throws Exception {+ ObjectMapper mapper = new ObjectMapper();+ List<JsonNode> records = new ArrayList<>();+ for (String key : listLogKeys(bucket)) {+ byte[] gz = given().when().get("/" + bucket + "/" + key)+ .then().statusCode(200).extract().asByteArray();+ byte[] json;+ try (GZIPInputStream gzin = new GZIPInputStream(new ByteArrayInputStream(gz))) {+ json = gzin.readAllBytes();+ }+ mapper.readTree(json).path("Records").forEach(records::add);+ }+ return records;+ }++ private static List<String> listLogKeys(String bucket) {+ String xml = given().when().get("/" + bucket + "?list-type=2")+ .then().statusCode(200).extract().asString();+ List<String> keys = new ArrayList<>();+ for (String key : XmlParser.extractAll(xml, "Key")) {+ if (key.contains("/CloudTrail/") && key.endsWith(".json.gz")) {+ keys.add(key);+ }+ }+ return keys;+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailLogWriterRequeueTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailLogWriterRequeueTest.javaindex d8b645fd83..438661a0b3 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailLogWriterRequeueTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailLogWriterRequeueTest.java@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.cloudtrail;import com.fasterxml.jackson.databind.ObjectMapper;import com.fasterxml.jackson.databind.node.ObjectNode;+import io.github.hectorvent.floci.core.common.XmlParser;import io.github.hectorvent.floci.services.cloudtrail.model.Trail;import io.quarkus.test.junit.QuarkusMock;import io.quarkus.test.junit.QuarkusTest;@@ -117,15 +118,6 @@ class CloudTrailLogWriterRequeueTest {private static List<String> listKeys(String bucket) {String xml = given().when().get("/" + bucket + "?list-type=2").then().statusCode(200).extract().asString();- List<String> keys = new ArrayList<>();- int from = 0;- while (true) {- int open = xml.indexOf("<Key>", from);- if (open < 0) break;- int close = xml.indexOf("</Key>", open);- keys.add(xml.substring(open + 5, close));- from = close + 6;- }- return keys;+ return XmlParser.extractAll(xml, "Key");}}diff --git a/src/test/resources/application.yml b/src/test/resources/application.ymlindex 90263e4774..566d489102 100644--- a/src/test/resources/application.yml+++ b/src/test/resources/application.yml@@ -340,6 +340,7 @@ floci:emit-mode: offcloudtrail:enabled: true+ flush-interval-seconds: 3600lightsail:enabled: truecloudcontrol: