Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix: bound CloudTrail delivery retries (#3298)

Niels Kootstrad44a3fef62c3
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/CloudTrailTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/CloudTrailTest.java
index 38fb0156b7..bf78cc19e5 100644
--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/CloudTrailTest.java
+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/CloudTrailTest.java
@@ -87,7 +87,7 @@ class CloudTrailTest {
GetTrailStatusResponse started = cloudTrail.getTrailStatus(r -> r.name(trailName));
assertThat(started.isLogging()).isTrue();
- assertThat(started.latestDeliveryTime()).isNotNull();
+ assertThat(started.latestDeliveryTime()).isNull();
cloudTrail.stopLogging(r -> r.name(trailName));
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailEntry.java b/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailEntry.java
index 13387abb1d..e4c9a119f3 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailEntry.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailEntry.java
@@ -22,7 +22,17 @@ public record CloudTrailEntry(
boolean logging,
Long startLoggingTime,
Long stopLoggingTime,
- Map<String, String> tags) {
+ Map<String, String> tags,
+ Long latestDeliveryTime,
+ String latestDeliveryError) {
+
+ public CloudTrailEntry(Trail trail, List<EventSelector> selectors,
+ List<AdvancedEventSelector> advancedSelectors, boolean logging,
+ Long startLoggingTime, Long stopLoggingTime,
+ Map<String, String> tags) {
+ this(trail, selectors, advancedSelectors, logging, startLoggingTime, stopLoggingTime,
+ tags, null, null);
+ }
// Not Map.copyOf: AWS allows a tag with a null value (AddTags omitting "Value"),
// and Map.copyOf/Map.of reject null values.
@@ -32,36 +42,52 @@ public record CloudTrailEntry(
public CloudTrailEntry withTrail(Trail updated) {
return new CloudTrailEntry(updated, selectors, advancedSelectors, logging,
- startLoggingTime, stopLoggingTime, tags);
+ startLoggingTime, stopLoggingTime, tags,
+ latestDeliveryTime, latestDeliveryError);
}
/** Sets basic selectors, clearing any advanced selectors — a trail holds one kind or the other. */
public CloudTrailEntry withSelectors(List<EventSelector> updated, boolean hasCustomSelectors) {
Trail updatedTrail = withHasCustomSelectors(hasCustomSelectors);
return new CloudTrailEntry(updatedTrail, updated, List.of(), logging,
- startLoggingTime, stopLoggingTime, tags);
+ startLoggingTime, stopLoggingTime, tags,
+ latestDeliveryTime, latestDeliveryError);
}
/** Sets advanced selectors, clearing any basic selectors — a trail holds one kind or the other. */
public CloudTrailEntry withAdvancedSelectors(List<AdvancedEventSelector> updated, boolean hasCustomSelectors) {
Trail updatedTrail = withHasCustomSelectors(hasCustomSelectors);
return new CloudTrailEntry(updatedTrail, List.of(), updated, logging,
- startLoggingTime, stopLoggingTime, tags);
+ startLoggingTime, stopLoggingTime, tags,
+ latestDeliveryTime, latestDeliveryError);
}
public CloudTrailEntry withTags(Map<String, String> updated) {
return new CloudTrailEntry(trail, selectors, advancedSelectors, logging,
- startLoggingTime, stopLoggingTime, updated);
+ startLoggingTime, stopLoggingTime, updated,
+ latestDeliveryTime, latestDeliveryError);
}
public CloudTrailEntry startLogging(long time) {
return new CloudTrailEntry(trail, selectors, advancedSelectors, true, time,
- stopLoggingTime, tags);
+ stopLoggingTime, tags,
+ latestDeliveryTime, latestDeliveryError);
}
public CloudTrailEntry stopLogging(long time) {
return new CloudTrailEntry(trail, selectors, advancedSelectors, false, startLoggingTime,
- time, tags);
+ time, tags,
+ latestDeliveryTime, latestDeliveryError);
+ }
+
+ public CloudTrailEntry withDeliveryFailure(String error) {
+ return new CloudTrailEntry(trail, selectors, advancedSelectors, logging,
+ startLoggingTime, stopLoggingTime, tags, latestDeliveryTime, error);
+ }
+
+ public CloudTrailEntry withDeliverySuccess(long time) {
+ return new CloudTrailEntry(trail, selectors, advancedSelectors, logging,
+ startLoggingTime, stopLoggingTime, tags, time, null);
}
private Trail withHasCustomSelectors(boolean hasCustomSelectors) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailJsonHandler.java b/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailJsonHandler.java
index 74565c315a..0154be3096 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailJsonHandler.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailJsonHandler.java
@@ -206,7 +206,12 @@ public class CloudTrailJsonHandler {
resp.put("IsLogging", status.logging());
if (status.startLoggingTime() != null) {
resp.put("StartLoggingTime", status.startLoggingTime() / 1000.0);
- resp.put("LatestDeliveryTime", status.startLoggingTime() / 1000.0);
+ }
+ if (status.latestDeliveryTime() != null) {
+ resp.put("LatestDeliveryTime", status.latestDeliveryTime() / 1000.0);
+ }
+ if (status.latestDeliveryError() != null) {
+ resp.put("LatestDeliveryError", status.latestDeliveryError());
}
if (status.stopLoggingTime() != null) {
resp.put("StopLoggingTime", status.stopLoggingTime() / 1000.0);
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailLogWriter.java b/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailLogWriter.java
index 51bbeba875..fc81b98c3b 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailLogWriter.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailLogWriter.java
@@ -1,9 +1,10 @@
package io.github.hectorvent.floci.services.cloudtrail;
-import com.fasterxml.jackson.core.JsonGenerator;
import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
import com.fasterxml.jackson.databind.node.ObjectNode;
import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
import io.github.hectorvent.floci.services.cloudtrail.model.Trail;
import io.github.hectorvent.floci.services.s3.S3Service;
@@ -48,10 +49,10 @@ import java.util.zip.GZIPOutputStream;
public class CloudTrailLogWriter {
private static final Logger LOG = Logger.getLogger(CloudTrailLogWriter.class);
+ static final int MAX_RECORDS_PER_LOG_FILE = 1_000;
private static final DateTimeFormatter PATH_DATE = DateTimeFormatter.ofPattern("yyyy/MM/dd");
private static final DateTimeFormatter FILE_TS = DateTimeFormatter.ofPattern("yyyyMMdd'T'HHmm'Z'");
- static final int MAX_RECORDS_PER_LOG_FILE = 1_000;
private static final String FILENAME_RAND_ALPHABET =
"abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
@@ -96,7 +97,7 @@ public class CloudTrailLogWriter {
void stop(@Observes ShutdownEvent event) {
if (executor != null) {
// Best-effort final flush so shutdown doesn't lose buffered events.
- try { flushAll(); } catch (Exception e) { LOG.warnv(e, "CloudTrail final flush on shutdown failed — some records may be lost"); }
+ try { flushAll(); } catch (Exception e) { LOG.warnv(e, "CloudTrail final flush on shutdown failed: some records may be lost"); }
executor.shutdownNow();
executor = null;
}
@@ -161,9 +162,9 @@ public class CloudTrailLogWriter {
private int flushTrail(CloudTrailService.TrailKey key) {
Trail trail = cloudTrailService.getTrail(key.region(), key.trailName());
if (trail == null) {
- // Trail was deleted while records were pending. Drop only the
- // records that existed when this flush cycle started.
- return cloudTrailService.drainPendingRecords(key, MAX_RECORDS_PER_LOG_FILE).size();
+ // Trail was deleted while records were pending: drop them.
+ cloudTrailService.discardPendingRecords(key);
+ return 0;
}
List<ObjectNode> records = cloudTrailService.drainPendingRecords(key, MAX_RECORDS_PER_LOG_FILE);
@@ -186,19 +187,30 @@ public class CloudTrailLogWriter {
} catch (RuntimeException e) {
// Re-queue so records survive the failed flush and are retried next cycle.
cloudTrailService.requeueRecords(key, records);
+ try {
+ cloudTrailService.recordDeliveryFailure(key, deliveryError(e));
+ } catch (RuntimeException statusError) {
+ LOG.warnv(statusError, "CloudTrail delivery failure status update failed for trail {0}", key.trailName());
+ }
LOG.warnv(e, "CloudTrail flush failed for trail {0} ({1} records re-queued)",
key.trailName(), records.size());
throw e;
}
+ try {
+ cloudTrailService.recordDeliverySuccess(key, System.currentTimeMillis());
+ } catch (RuntimeException e) {
+ LOG.warnv(e, "CloudTrail delivery success status update failed for trail {0}", key.trailName());
+ }
+ cloudTrailService.completeDelivery(key);
- // The write above already succeeded and durably delivered the records —
+ // The write above already succeeded and durably delivered the records:
// from here on, records must never be re-queued. Doing so on a failure
// in this block would deliver the same batch to S3 again next flush.
try {
// This write goes straight to S3Service, bypassing the HTTP-facing
// S3Controller that normally emits data events for API-driven puts.
// Any trail whose selector matches its own destination bucket must
- // still see its own deliveries — that's the real circular-logging
+ // still see its own deliveries: that is the real circular-logging
// behavior (issue #1192 / PR #1194) this emulator exists to prove.
cloudTrailService.emitS3DataEvent(CloudTrailService.S3EventInput.builder()
.region(key.eventRegion())
@@ -221,19 +233,25 @@ public class CloudTrailLogWriter {
return records.size();
}
+ private String deliveryError(RuntimeException e) {
+ String message = e.getMessage() == null ? "" : ": " + e.getMessage();
+ if (e instanceof AwsException awsException) {
+ return awsException.getErrorCode() + message;
+ }
+ return e.getClass().getSimpleName() + message;
+ }
+
private byte[] serializeAndGzip(List<ObjectNode> records) {
+ ObjectNode envelope = mapper.createObjectNode();
+ ArrayNode arr = envelope.putArray("Records");
+ for (ObjectNode r : records) {
+ arr.add(r);
+ }
try {
- ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ byte[] json = mapper.writeValueAsBytes(envelope);
+ ByteArrayOutputStream baos = new ByteArrayOutputStream(Math.max(64, json.length / 4));
try (GZIPOutputStream gz = new GZIPOutputStream(baos)) {
- try (JsonGenerator generator = mapper.getFactory().createGenerator(gz)) {
- generator.writeStartObject();
- generator.writeArrayFieldStart("Records");
- for (ObjectNode record : records) {
- generator.writeTree(record);
- }
- generator.writeEndArray();
- generator.writeEndObject();
- }
+ gz.write(json);
}
return baos.toByteArray();
} catch (Exception e) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailService.java b/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailService.java
index 74f96611ee..fb8f6c7d0a 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailService.java
@@ -23,15 +23,16 @@ import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
+import java.nio.charset.StandardCharsets;
import java.time.Instant;
import java.time.format.DateTimeFormatter;
+import java.util.ArrayDeque;
import java.util.ArrayList;
import java.util.List;
-import java.util.ListIterator;
import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentLinkedDeque;
+import java.util.function.Function;
import java.util.function.Supplier;
@ApplicationScoped
@@ -41,14 +42,16 @@ public class CloudTrailService {
private static final String EVENT_VERSION = "1.11";
private static final String S3_EVENT_SOURCE = "s3.amazonaws.com";
+ static final int MAX_PENDING_RECORDS_PER_TRAIL = 1024;
+ static final long MAX_PENDING_BYTES_PER_TRAIL = 4L * 1024L * 1024L;
private final StorageBackend<String, CloudTrailEntry> store;
private final RegionResolver regionResolver;
private final IamService iamService;
private final ObjectMapper mapper;
- /** Per-trail pending record buffers — ephemeral, never persisted. */
- private final ConcurrentHashMap<TrailKey, ConcurrentLinkedDeque<ObjectNode>> pendingRecordsByTrail =
+ /** Per-trail pending record buffers: ephemeral, never persisted. */
+ private final ConcurrentHashMap<PendingTrailKey, PendingRecordBuffer> pendingRecordsByTrail =
new ConcurrentHashMap<>();
/**
@@ -112,7 +115,8 @@ public class CloudTrailService {
name, arn, s3BucketName, s3KeyPrefix, snsTopicArn,
includeGlobalServiceEvents, isMultiRegionTrail, region,
enableLogFileValidation, false, false, isOrganizationTrail);
- store.put(key, new CloudTrailEntry(trail, List.of(), List.of(), false, null, null, initialTags));
+ store.put(key, new CloudTrailEntry(trail, List.of(), List.of(), false, null, null,
+ initialTags, null, null));
return trail;
}
@@ -241,15 +245,16 @@ public class CloudTrailService {
public TrailStatus getTrailStatus(String region, String trailNameOrArn) {
Trail trail = findTrailOrThrow(region, trailNameOrArn);
return store.get(regionKey(trail.homeRegion(), trail.name()))
- .map(e -> new TrailStatus(e.logging(), e.startLoggingTime(), e.stopLoggingTime()))
- .orElse(new TrailStatus(false, null, null));
+ .map(e -> new TrailStatus(e.logging(), e.startLoggingTime(), e.stopLoggingTime(),
+ e.latestDeliveryTime(), e.latestDeliveryError()))
+ .orElse(new TrailStatus(false, null, null, null, null));
}
// --- Tagging ---
//
// AddTags/RemoveTags/ListTags identify the trail solely by ARN (ResourceId /
// ResourceIdList), unlike every other CloudTrail action here which also accepts a
- // bare trail name — so these don't take a `region` parameter.
+ // bare trail name, so these do not take a `region` parameter.
private static final int MAX_TAGS_PER_RESOURCE = 50;
@@ -354,8 +359,13 @@ public class CloudTrailService {
for (MatchedTrail mt : matched) {
ObjectNode copy = record.deepCopy();
copy.put("recipientAccountId", regionResolver.getAccountId());
- queueFor(new TrailKey(mt.region(), mt.trail().name(), region)).add(copy);
- LOG.tracev("Emitted CloudTrail event {0} for trail {1}", in.eventName(), mt.trail().name());
+ boolean accepted = append(new TrailKey(mt.region(), mt.trail().name(), region), copy);
+ if (accepted) {
+ LOG.tracev("Emitted CloudTrail event {0} for trail {1}", in.eventName(), mt.trail().name());
+ } else {
+ LOG.tracev("Dropped CloudTrail event {0} for trail {1}: retry buffer is full",
+ in.eventName(), mt.trail().name());
+ }
}
} catch (Exception e) {
// Never let emission take down an S3 op.
@@ -365,17 +375,32 @@ public class CloudTrailService {
}
public void requeueRecords(TrailKey key, List<ObjectNode> records) {
- if (records.isEmpty()) {
- return;
- }
- ConcurrentLinkedDeque<ObjectNode> q = queueFor(key);
- ListIterator<ObjectNode> it = records.listIterator(records.size());
- // Add in reverse so the first failed record remains first.
- while (it.hasPrevious()) {
- q.addFirst(it.previous());
+ if (!records.isEmpty()) {
+ List<PendingRecord> pending = records.stream()
+ .map(record -> new PendingRecord(record, estimatedRecordBytes(record)))
+ .toList();
+ pendingRecordsByTrail.compute(pendingTrailKey(key), (ignored, buffer) -> {
+ PendingRecordBuffer updated = buffer == null ? new PendingRecordBuffer() : buffer;
+ updated.requeueFront(key.eventRegion(), pending);
+ return updated.isEmpty() ? null : updated;
+ });
}
}
+ public void completeDelivery(TrailKey key) {
+ pendingRecordsByTrail.computeIfPresent(pendingTrailKey(key), (ignored, buffer) -> {
+ buffer.completeDelivery(key.eventRegion());
+ return buffer.isEmpty() ? null : buffer;
+ });
+ }
+
+ public void discardPendingRecords(TrailKey key) {
+ pendingRecordsByTrail.computeIfPresent(pendingTrailKey(key), (ignored, buffer) -> {
+ buffer.discard(key.eventRegion());
+ return buffer.isEmpty() ? null : buffer;
+ });
+ }
+
public List<ObjectNode> drainPendingRecords(TrailKey key) {
return drainPendingRecords(key, Integer.MAX_VALUE);
}
@@ -384,41 +409,68 @@ public class CloudTrailService {
if (maxRecords <= 0) {
return List.of();
}
- ConcurrentLinkedDeque<ObjectNode> q = pendingRecordsByTrail.get(key);
- if (q == null) {
- return List.of();
- }
List<ObjectNode> drained = new ArrayList<>();
- ObjectNode r;
- while (drained.size() < maxRecords && (r = q.pollFirst()) != null) {
- drained.add(r);
- }
- return drained;
- }
-
- public int pendingRecordCount(TrailKey key) {
- ConcurrentLinkedDeque<ObjectNode> q = pendingRecordsByTrail.get(key);
- return q == null ? 0 : q.size();
+ pendingRecordsByTrail.compute(pendingTrailKey(key), (ignored, buffer) -> {
+ if (buffer == null) {
+ return null;
+ }
+ drained.addAll(buffer.drain(key.eventRegion(), maxRecords));
+ return buffer.isEmpty() ? null : buffer;
+ });
+ return drained.isEmpty() ? List.of() : drained;
}
public List<TrailKey> trailsWithPendingRecords() {
List<TrailKey> result = new ArrayList<>();
- for (Map.Entry<TrailKey, ConcurrentLinkedDeque<ObjectNode>> e : pendingRecordsByTrail.entrySet()) {
- if (!e.getValue().isEmpty()) {
- result.add(e.getKey());
+ for (Map.Entry<PendingTrailKey, PendingRecordBuffer> e : pendingRecordsByTrail.entrySet()) {
+ for (String eventRegion : e.getValue().eventRegions()) {
+ result.add(new TrailKey(e.getKey().region(), e.getKey().trailName(), eventRegion));
}
}
return result;
}
+ public int pendingRecordCount(TrailKey key) {
+ PendingRecordBuffer buffer = pendingRecordsByTrail.get(pendingTrailKey(key));
+ return buffer == null ? 0 : buffer.pendingCount(key.eventRegion());
+ }
+
public Trail getTrail(String region, String trailName) {
return store.get(regionKey(region, trailName))
.map(CloudTrailEntry::trail)
.orElse(null);
}
- private ConcurrentLinkedDeque<ObjectNode> queueFor(TrailKey key) {
- return pendingRecordsByTrail.computeIfAbsent(key, k -> new ConcurrentLinkedDeque<>());
+ public void recordDeliveryFailure(TrailKey key, String error) {
+ updateDeliveryStatus(key, entry -> entry.withDeliveryFailure(error));
+ }
+
+ public void recordDeliverySuccess(TrailKey key, long time) {
+ updateDeliveryStatus(key, entry -> entry.withDeliverySuccess(time));
+ }
+
+ private void updateDeliveryStatus(TrailKey key, Function<CloudTrailEntry, CloudTrailEntry> update) {
+ String storeKey = regionKey(key.region(), key.trailName());
+ withTrailLock(storeKey, () -> store.get(storeKey).ifPresent(entry -> store.put(storeKey, update.apply(entry))));
+ }
+
+ private boolean append(TrailKey key, ObjectNode record) {
+ long recordBytes = estimatedRecordBytes(record);
+ boolean[] accepted = {false};
+ pendingRecordsByTrail.compute(pendingTrailKey(key), (ignored, buffer) -> {
+ PendingRecordBuffer updated = buffer == null ? new PendingRecordBuffer() : buffer;
+ accepted[0] = updated.append(key.eventRegion(), record, recordBytes);
+ return updated.isEmpty() ? null : updated;
+ });
+ return accepted[0];
+ }
+
+ private long estimatedRecordBytes(ObjectNode record) {
+ try {
+ return mapper.writeValueAsBytes(record).length;
+ } catch (Exception e) {
+ return record.toString().getBytes(StandardCharsets.UTF_8).length;
+ }
}
/**
@@ -429,6 +481,137 @@ public class CloudTrailService {
*/
public record TrailKey(String region, String trailName, String eventRegion) {}
+ private record PendingTrailKey(String region, String trailName) {}
+
+ private record PendingRecord(ObjectNode record, long byteCount) {}
+
+ private final class PendingRecordBuffer {
+ private final Map<String, ArrayDeque<PendingRecord>> recordsByRegion = new ConcurrentHashMap<>();
+ private final Map<String, List<PendingRecord>> inFlightByRegion = new ConcurrentHashMap<>();
+ private int recordCount;
+ private long byteCount;
+
+ synchronized boolean append(String eventRegion, ObjectNode record, long recordBytes) {
+ if (!inFlightByRegion.isEmpty()
+ && (recordCount >= MAX_PENDING_RECORDS_PER_TRAIL
+ || byteCount + recordBytes > MAX_PENDING_BYTES_PER_TRAIL)) {
+ return false;
+ }
+ recordsByRegion.computeIfAbsent(eventRegion, ignored -> new ArrayDeque<>())
+ .addLast(new PendingRecord(record, recordBytes));
+ recordCount++;
+ byteCount += recordBytes;
+ return true;
+ }
+
+ synchronized void requeueFront(String eventRegion, List<PendingRecord> drained) {
+ ArrayDeque<PendingRecord> records = recordsByRegion.computeIfAbsent(
+ eventRegion, ignored -> new ArrayDeque<>());
+ boolean wasInFlight = inFlightByRegion.remove(eventRegion) != null;
+ for (int i = drained.size() - 1; i >= 0; i--) {
+ PendingRecord pending = drained.get(i);
+ records.addFirst(pending);
+ if (!wasInFlight) {
+ recordCount++;
+ byteCount += pending.byteCount();
+ }
+ }
+ trimTailToLimit(eventRegion);
+ }
+
+ synchronized List<ObjectNode> drain(String eventRegion) {
+ return drain(eventRegion, Integer.MAX_VALUE);
+ }
+
+ synchronized List<ObjectNode> drain(String eventRegion, int maxRecords) {
+ ArrayDeque<PendingRecord> records = recordsByRegion.remove(eventRegion);
+ if (records == null || records.isEmpty()) {
+ return List.of();
+ }
+ List<PendingRecord> selected = new ArrayList<>(Math.min(records.size(), maxRecords));
+ while (selected.size() < maxRecords && !records.isEmpty()) {
+ selected.add(records.removeFirst());
+ }
+ if (!records.isEmpty()) {
+ recordsByRegion.put(eventRegion, records);
+ }
+ inFlightByRegion.put(eventRegion, selected);
+ List<ObjectNode> drained = new ArrayList<>(selected.size());
+ for (PendingRecord pending : selected) {
+ drained.add(pending.record());
+ }
+ return drained;
+ }
+
+ synchronized void completeDelivery(String eventRegion) {
+ List<PendingRecord> inFlight = inFlightByRegion.remove(eventRegion);
+ if (inFlight == null) {
+ return;
+ }
+ for (PendingRecord pending : inFlight) {
+ recordCount--;
+ byteCount -= pending.byteCount();
+ }
+ }
+
+ synchronized void discard(String eventRegion) {
+ ArrayDeque<PendingRecord> queued = recordsByRegion.remove(eventRegion);
+ if (queued != null) {
+ for (PendingRecord pending : queued) {
+ recordCount--;
+ byteCount -= pending.byteCount();
+ }
+ }
+ List<PendingRecord> inFlight = inFlightByRegion.remove(eventRegion);
+ if (inFlight != null) {
+ for (PendingRecord pending : inFlight) {
+ recordCount--;
+ byteCount -= pending.byteCount();
+ }
+ }
+ }
+
+ synchronized boolean isEmpty() {
+ return recordCount == 0 && inFlightByRegion.isEmpty();
+ }
+
+ synchronized int pendingCount(String eventRegion) {
+ ArrayDeque<PendingRecord> records = recordsByRegion.get(eventRegion);
+ List<PendingRecord> inFlight = inFlightByRegion.get(eventRegion);
+ return (records == null ? 0 : records.size()) + (inFlight == null ? 0 : inFlight.size());
+ }
+
+ synchronized List<String> eventRegions() {
+ return java.util.stream.Stream.concat(recordsByRegion.keySet().stream(), inFlightByRegion.keySet().stream())
+ .distinct()
+ .filter(eventRegion -> {
+ ArrayDeque<PendingRecord> records = recordsByRegion.get(eventRegion);
+ return (records != null && !records.isEmpty()) || inFlightByRegion.containsKey(eventRegion);
+ })
+ .toList();
+ }
+
+ private void trimTailToLimit(String preferredRegion) {
+ while (recordCount > MAX_PENDING_RECORDS_PER_TRAIL
+ || byteCount > MAX_PENDING_BYTES_PER_TRAIL) {
+ ArrayDeque<PendingRecord> records = recordsByRegion.get(preferredRegion);
+ if (records == null || records.isEmpty()) {
+ records = recordsByRegion.values().stream()
+ .filter(queue -> !queue.isEmpty())
+ .findFirst()
+ .orElseThrow();
+ }
+ PendingRecord dropped = records.removeLast();
+ recordCount--;
+ byteCount -= dropped.byteCount();
+ }
+ }
+ }
+
+ private static PendingTrailKey pendingTrailKey(TrailKey key) {
+ return new PendingTrailKey(key.region(), key.trailName());
+ }
+
// --- Helpers ---
private List<MatchedTrail> trailsMatching(String region, S3EventInput in) {
@@ -493,7 +676,7 @@ public class CloudTrailService {
private boolean matchesAnyAdvancedSelector(List<AdvancedEventSelector> selectors, S3EventInput in) {
String arn = "arn:aws:s3:::" + in.bucketName() + (in.key() != null ? "/" + in.key() : "");
// Bucket-level operations (e.g. ListObjects) have no object key and are reported
- // by CloudTrail as AWS::S3::Bucket resources, not AWS::S3::Object — matching real
+ // by CloudTrail as AWS::S3::Bucket resources, not AWS::S3::Object: matching real
// AWS behavior, an AWS::S3::Object DataResource selector must never match them.
String resourceType = in.key() != null ? "AWS::S3::Object" : "AWS::S3::Bucket";
for (AdvancedEventSelector sel : selectors) {
@@ -778,7 +961,12 @@ public class CloudTrailService {
return colon < 0 ? key : key.substring(0, colon);
}
- public record TrailStatus(boolean logging, Long startLoggingTime, Long stopLoggingTime) {}
+ public record TrailStatus(boolean logging, Long startLoggingTime, Long stopLoggingTime,
+ Long latestDeliveryTime, String latestDeliveryError) {
+ public TrailStatus(boolean logging, Long startLoggingTime, Long stopLoggingTime) {
+ this(logging, startLoggingTime, stopLoggingTime, null, null);
+ }
+ }
private record MatchedTrail(Trail trail, String region) {}
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailLogWriterBoundedRetryIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailLogWriterBoundedRetryIntegrationTest.java
new file mode 100644
index 0000000000..0d3a0ed83f
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailLogWriterBoundedRetryIntegrationTest.java
@@ -0,0 +1,330 @@
+package io.github.hectorvent.floci.services.cloudtrail;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.github.hectorvent.floci.core.common.XmlParser;
+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;
+import io.quarkus.test.junit.QuarkusTest;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.BeforeAll;
+import org.junit.jupiter.api.Test;
+
+import java.io.ByteArrayInputStream;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+import java.util.UUID;
+import java.util.zip.GZIPInputStream;
+
+import static io.restassured.RestAssured.given;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+@QuarkusTest
+class CloudTrailLogWriterBoundedRetryIntegrationTest {
+
+ private static final String CT_TARGET = "CloudTrail_20131101.";
+ private static final String JSON11 = "application/x-amz-json-1.1";
+ private static final int RETRY_RECORD_LIMIT = 1024;
+ private static final int PRODUCED_EVENTS = RETRY_RECORD_LIMIT + 80;
+
+ @Inject
+ CloudTrailLogWriter writer;
+
+ @Inject
+ CloudTrailService cloudTrailService;
+
+ @BeforeAll
+ static void configureRestAssured() {
+ RestAssuredJsonUtils.configureAwsContentTypes();
+ }
+
+ @Test
+ void retryStateStaysBoundedAndDeliversRetainedPrefixAfterDestinationRecovery() throws Exception {
+ String suffix = UUID.randomUUID().toString().substring(0, 8);
+ String sourceBucket = "bounded-source-" + suffix;
+ String destinationBucket = "bounded-logs-" + suffix;
+ String trailName = "bounded-trail-" + suffix;
+ String region = "us-east-1";
+
+ createBucket(sourceBucket);
+ createTrail(trailName, destinationBucket);
+ selectSourceBucket(trailName, sourceBucket);
+ startLogging(trailName);
+
+ JsonNode startedStatus = getTrailStatus(trailName);
+ assertFalse(startedStatus.has("LatestDeliveryTime"),
+ "starting a trail must not claim that a delivery already happened: " + startedStatus);
+ assertFalse(startedStatus.has("LatestDeliveryAttemptTime"),
+ "GetTrailStatus must not expose the retired delivery-attempt field: " + startedStatus);
+
+ for (int i = 0; i < PRODUCED_EVENTS; i++) {
+ putObject(sourceBucket, eventKey(i), "event-" + i);
+ if (i % 100 == 99) {
+ writer.flushNow();
+ }
+ }
+ writer.flushNow();
+
+ JsonNode failedStatus = getTrailStatus(trailName);
+ assertTrue(failedStatus.hasNonNull("LatestDeliveryError"),
+ "failed S3 delivery must be visible through GetTrailStatus: " + failedStatus);
+ assertTrue(failedStatus.path("LatestDeliveryError").asText().contains("NoSuchBucket"),
+ "GetTrailStatus should expose the S3 delivery error: " + failedStatus);
+
+ createBucket(destinationBucket);
+ writer.flushNow();
+
+ List<JsonNode> delivered = deliveredRecords(destinationBucket);
+ assertEquals(RETRY_RECORD_LIMIT, delivered.size(),
+ "only the retained retry prefix should be delivered after recovery");
+ for (int i = 0; i < RETRY_RECORD_LIMIT; i++) {
+ assertEquals(eventKey(i), delivered.get(i).path("requestParameters").path("key").asText());
+ }
+ assertFalse(delivered.stream().anyMatch(r ->
+ eventKey(RETRY_RECORD_LIMIT).equals(r.path("requestParameters").path("key").asText())),
+ "events produced after the retry buffer filled should remain dropped");
+
+ JsonNode recoveredStatus = getTrailStatus(trailName);
+ assertFalse(recoveredStatus.has("LatestDeliveryError"),
+ "successful delivery should clear LatestDeliveryError: " + recoveredStatus);
+ assertTrue(recoveredStatus.hasNonNull("LatestDeliveryTime"),
+ "successful delivery should update LatestDeliveryTime: " + recoveredStatus);
+ }
+
+ @Test
+ void healthyBurstAboveRetryLimitIsDeliveredWithoutLoss() throws Exception {
+ String suffix = UUID.randomUUID().toString().substring(0, 8);
+ String sourceBucket = "healthy-burst-source-" + suffix;
+ String destinationBucket = "healthy-burst-logs-" + suffix;
+ String trailName = "healthy-burst-trail-" + suffix;
+
+ createBucket(sourceBucket);
+ createBucket(destinationBucket);
+ createTrail(trailName, destinationBucket);
+ selectSourceBucket(trailName, sourceBucket);
+ startLogging(trailName);
+
+ for (int i = 0; i < PRODUCED_EVENTS; i++) {
+ putObject(sourceBucket, eventKey(i), "event-" + i);
+ }
+ writer.flushNow();
+
+ List<JsonNode> delivered = deliveredRecords(destinationBucket);
+ assertEquals(PRODUCED_EVENTS, delivered.size(),
+ "a healthy burst must not be limited by the retry budget");
+ Set<String> deliveredKeys = new HashSet<>();
+ for (JsonNode record : delivered) {
+ deliveredKeys.add(record.path("requestParameters").path("key").asText());
+ }
+ for (int i = 0; i < PRODUCED_EVENTS; i++) {
+ assertTrue(deliveredKeys.contains(eventKey(i)));
+ }
+ }
+
+ @Test
+ void retryStateStaysBoundedByBytes() throws Exception {
+ String suffix = UUID.randomUUID().toString().substring(0, 8);
+ String sourceBucket = "bounded-bytes-source-" + suffix;
+ String destinationBucket = "bounded-bytes-logs-" + suffix;
+ String trailName = "bounded-bytes-trail-" + suffix;
+ int producedEvents = 750;
+
+ createBucket(sourceBucket);
+ createTrail(trailName, destinationBucket);
+ selectSourceBucket(trailName, sourceBucket);
+ startLogging(trailName);
+
+ String largeUserAgent = "x".repeat(6000);
+ for (int i = 0; i < producedEvents; i++) {
+ cloudTrailService.emitS3DataEvent(CloudTrailService.S3EventInput.builder()
+ .region("us-east-1")
+ .eventName("PutObject")
+ .bucketName(sourceBucket)
+ .key(eventKey(i))
+ .userAgent(largeUserAgent)
+ .eventTimeMillis(System.currentTimeMillis())
+ .build());
+ if (i % 100 == 99) {
+ writer.flushNow();
+ }
+ }
+ writer.flushNow();
+
+ createBucket(destinationBucket);
+ writer.flushNow();
+
+ List<JsonNode> delivered = deliveredRecords(destinationBucket);
+ assertTrue(delivered.size() < producedEvents,
+ "the byte limit must drop records before the record limit: " + delivered.size());
+ assertTrue(delivered.size() > 0, "the byte-limited retry buffer should retain records");
+ assertEquals(eventKey(0),
+ delivered.get(0).path("requestParameters").path("key").asText());
+ }
+
+ @Test
+ void retryRecordLimitIsSharedAcrossEventRegions() throws Exception {
+ String suffix = UUID.randomUUID().toString().substring(0, 8);
+ String sourceBucket = "bounded-regions-source-" + suffix;
+ String destinationBucket = "bounded-regions-logs-" + suffix;
+ String trailName = "bounded-regions-trail-" + suffix;
+
+ createBucket(sourceBucket);
+ createTrail(trailName, destinationBucket, true);
+ selectSourceBucket(trailName, sourceBucket);
+ startLogging(trailName);
+
+ emitEvents(sourceBucket, "us-east-1", 600, "east");
+ writer.flushNow();
+ emitEvents(sourceBucket, "us-west-2", 600, "west");
+ writer.flushNow();
+
+ createBucket(destinationBucket);
+ writer.flushNow();
+
+ List<JsonNode> delivered = deliveredRecords(destinationBucket);
+ assertEquals(RETRY_RECORD_LIMIT, delivered.size(),
+ "the record cap must apply to the trail, not each event region");
+ assertTrue(delivered.stream().anyMatch(record -> "us-east-1".equals(record.path("awsRegion").asText())));
+ assertTrue(delivered.stream().anyMatch(record -> "us-west-2".equals(record.path("awsRegion").asText())));
+ }
+
+ @Test
+ void retryBudgetIncludesRecordsInFlightAcrossEventRegions() {
+ String suffix = UUID.randomUUID().toString().substring(0, 8);
+ String sourceBucket = "in-flight-source-" + suffix;
+ String destinationBucket = "in-flight-logs-" + suffix;
+ String trailName = "in-flight-trail-" + suffix;
+
+ createBucket(sourceBucket);
+ createTrail(trailName, destinationBucket, true);
+ selectSourceBucket(trailName, sourceBucket);
+ startLogging(trailName);
+
+ emitEvents(sourceBucket, "us-east-1", RETRY_RECORD_LIMIT, "east");
+ CloudTrailService.TrailKey eastKey =
+ new CloudTrailService.TrailKey("us-east-1", trailName, "us-east-1");
+ List<ObjectNode> inFlight = cloudTrailService.drainPendingRecords(eastKey);
+
+ emitEvents(sourceBucket, "us-west-2", 80, "west");
+ cloudTrailService.requeueRecords(eastKey, inFlight);
+
+ List<ObjectNode> retainedEast = cloudTrailService.drainPendingRecords(eastKey);
+ List<ObjectNode> retainedWest = cloudTrailService.drainPendingRecords(
+ new CloudTrailService.TrailKey("us-east-1", trailName, "us-west-2"));
+
+ assertEquals(RETRY_RECORD_LIMIT, retainedEast.size(),
+ "in-flight records must reserve the trail-wide retry budget");
+ assertTrue(retainedWest.isEmpty(),
+ "events from another region must not displace older in-flight records");
+ for (int i = 0; i < RETRY_RECORD_LIMIT; i++) {
+ assertEquals("east/" + eventKey(i),
+ retainedEast.get(i).path("requestParameters").path("key").asText());
+ }
+ }
+
+ private static void createTrail(String trailName, String destinationBucket) {
+ createTrail(trailName, destinationBucket, false);
+ }
+
+ private static void createTrail(String trailName, String destinationBucket, boolean multiRegion) {
+ invokeCloudTrail("CreateTrail", String.format("""
+ {"Name":"%s","S3BucketName":"%s","IsMultiRegionTrail":%s}
+ """, trailName, destinationBucket, multiRegion))
+ .then().statusCode(200);
+ }
+
+ private static void selectSourceBucket(String trailName, String sourceBucket) {
+ invokeCloudTrail("PutEventSelectors", String.format("""
+ {
+ "TrailName": "%s",
+ "EventSelectors": [
+ {
+ "ReadWriteType": "All",
+ "IncludeManagementEvents": false,
+ "DataResources": [
+ {"Type": "AWS::S3::Object", "Values": ["arn:aws:s3:::%s/"]}
+ ]
+ }
+ ]
+ }
+ """, trailName, sourceBucket))
+ .then().statusCode(200);
+ }
+
+ private static void startLogging(String trailName) {
+ invokeCloudTrail("StartLogging", String.format("{\"Name\":\"%s\"}", trailName))
+ .then().statusCode(200);
+ }
+
+ private static JsonNode getTrailStatus(String trailName) throws Exception {
+ String body = invokeCloudTrail("GetTrailStatus", String.format("{\"Name\":\"%s\"}", trailName))
+ .then().statusCode(200)
+ .extract().asString();
+ return new ObjectMapper().readTree(body);
+ }
+
+ private static io.restassured.response.Response invokeCloudTrail(String action, String body) {
+ return given()
+ .header("X-Amz-Target", CT_TARGET + action)
+ .contentType(JSON11)
+ .body(body)
+ .when().post("/");
+ }
+
+ private static void createBucket(String name) {
+ given().when().put("/" + name).then().statusCode(200);
+ }
+
+ private static void putObject(String bucket, String key, String body) {
+ given().body(body)
+ .when().put("/" + bucket + "/" + key)
+ .then().statusCode(200);
+ }
+
+ private void emitEvents(String sourceBucket, String region, int count, String prefix) {
+ for (int i = 0; i < count; i++) {
+ cloudTrailService.emitS3DataEvent(CloudTrailService.S3EventInput.builder()
+ .region(region)
+ .eventName("PutObject")
+ .bucketName(sourceBucket)
+ .key(prefix + "/" + eventKey(i))
+ .eventTimeMillis(System.currentTimeMillis())
+ .build());
+ }
+ }
+
+ private static String eventKey(int index) {
+ return String.format("events/%04d.txt", index);
+ }
+
+ private static List<JsonNode> deliveredRecords(String bucket) throws Exception {
+ ObjectMapper mapper = new ObjectMapper();
+ List<JsonNode> records = new ArrayList<>();
+ for (String key : listLogKeys(bucket)) {
+ byte[] gz = given().when().get("/" + bucket + "/" + key)
+ .then().statusCode(200).extract().asByteArray();
+ byte[] json;
+ try (GZIPInputStream gzin = new GZIPInputStream(new ByteArrayInputStream(gz))) {
+ json = gzin.readAllBytes();
+ }
+ mapper.readTree(json).path("Records").forEach(records::add);
+ }
+ return records;
+ }
+
+ private static List<String> listLogKeys(String bucket) {
+ String xml = given().when().get("/" + bucket + "?list-type=2")
+ .then().statusCode(200).extract().asString();
+ List<String> keys = new ArrayList<>();
+ for (String key : XmlParser.extractAll(xml, "Key")) {
+ if (key.contains("/CloudTrail/") && key.endsWith(".json.gz")) {
+ keys.add(key);
+ }
+ }
+ return keys;
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailLogWriterRequeueTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailLogWriterRequeueTest.java
index d8b645fd83..438661a0b3 100644
--- a/src/test/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailLogWriterRequeueTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailLogWriterRequeueTest.java
@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.cloudtrail;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
+import io.github.hectorvent.floci.core.common.XmlParser;
import io.github.hectorvent.floci.services.cloudtrail.model.Trail;
import io.quarkus.test.junit.QuarkusMock;
import io.quarkus.test.junit.QuarkusTest;
@@ -117,15 +118,6 @@ class CloudTrailLogWriterRequeueTest {
private static List<String> listKeys(String bucket) {
String xml = given().when().get("/" + bucket + "?list-type=2")
.then().statusCode(200).extract().asString();
- List<String> keys = new ArrayList<>();
- int from = 0;
- while (true) {
- int open = xml.indexOf("<Key>", from);
- if (open < 0) break;
- int close = xml.indexOf("</Key>", open);
- keys.add(xml.substring(open + 5, close));
- from = close + 6;
- }
- return keys;
+ return XmlParser.extractAll(xml, "Key");
}
}
diff --git a/src/test/resources/application.yml b/src/test/resources/application.yml
index 90263e4774..566d489102 100644
--- a/src/test/resources/application.yml
+++ b/src/test/resources/application.yml
@@ -340,6 +340,7 @@ floci:
emit-mode: off
cloudtrail:
enabled: true
+ flush-interval-seconds: 3600
lightsail:
enabled: true
cloudcontrol: