Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(cloudtrail): serialize trail mutations to prevent lost updates (#2911)

Priamo Germosen8c2619f96549
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailService.java b/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailService.java
index 01fb9af806..32c8614115 100644
--- a/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailService.java
@@ -27,6 +27,7 @@ import java.util.Map;
import java.util.UUID;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.function.Supplier;
@ApplicationScoped
public class CloudTrailService {
@@ -45,6 +46,15 @@ public class CloudTrailService {
private final ConcurrentHashMap<TrailKey, ConcurrentLinkedQueue<ObjectNode>> pendingRecordsByTrail =
new ConcurrentHashMap<>();
+ /**
+ * Backs {@link #withTrailLock}: serializes read-modify-write mutations to the same trail. A
+ * plain {@code get} then {@code put} on the store is not atomic across the pair, so two
+ * overlapping mutations of the same trail, even from different actions, e.g. AddTags racing
+ * StartLogging, can interleave such that the second {@code put} replaces the whole entry the
+ * first read, silently discarding whatever the first call changed.
+ */
+ private final ConcurrentHashMap<String, Object> trailLocks = new ConcurrentHashMap<>();
+
@Inject
public CloudTrailService(StorageFactory storageFactory, RegionResolver regionResolver,
IamService iamService, ObjectMapper mapper) {
@@ -82,7 +92,8 @@ public class CloudTrailService {
public void deleteTrail(String region, String trailNameOrArn) {
Trail trail = findTrailOrThrow(region, trailNameOrArn);
- store.delete(regionKey(trail.homeRegion(), trail.name()));
+ String key = regionKey(trail.homeRegion(), trail.name());
+ withTrailLock(key, () -> store.delete(key));
pendingRecordsByTrail.keySet().removeIf(k -> k.trailName().equals(trail.name()));
}
@@ -90,23 +101,28 @@ public class CloudTrailService {
String s3BucketName, String s3KeyPrefix, String snsTopicArn,
Boolean includeGlobalServiceEvents, Boolean isMultiRegionTrail,
Boolean enableLogFileValidation, Boolean isOrganizationTrail) {
- Trail existing = findTrailOrThrow(region, trailNameOrArn);
- Trail updated = new Trail(
- existing.name(),
- existing.trailArn(),
- s3BucketName != null ? s3BucketName : existing.s3BucketName(),
- s3KeyPrefix != null ? s3KeyPrefix : existing.s3KeyPrefix(),
- snsTopicArn != null ? snsTopicArn : existing.snsTopicArn(),
- includeGlobalServiceEvents != null ? includeGlobalServiceEvents : existing.includeGlobalServiceEvents(),
- isMultiRegionTrail != null ? isMultiRegionTrail : existing.isMultiRegionTrail(),
- existing.homeRegion(),
- enableLogFileValidation != null ? enableLogFileValidation : existing.logFileValidationEnabled(),
- existing.hasCustomEventSelectors(),
- existing.hasInsightSelectors(),
- isOrganizationTrail != null ? isOrganizationTrail : existing.isOrganizationTrail());
- String key = regionKey(existing.homeRegion(), existing.name());
- store.get(key).ifPresent(entry -> store.put(key, entry.withTrail(updated)));
- return updated;
+ Trail resolved = findTrailOrThrow(region, trailNameOrArn);
+ String key = regionKey(resolved.homeRegion(), resolved.name());
+ return withTrailLock(key, () -> {
+ CloudTrailEntry entry = store.get(key).orElseThrow(() -> new AwsException(
+ "TrailNotFoundException", "Unknown trail: " + trailNameOrArn, 400));
+ Trail existing = entry.trail();
+ Trail updated = new Trail(
+ existing.name(),
+ existing.trailArn(),
+ s3BucketName != null ? s3BucketName : existing.s3BucketName(),
+ s3KeyPrefix != null ? s3KeyPrefix : existing.s3KeyPrefix(),
+ snsTopicArn != null ? snsTopicArn : existing.snsTopicArn(),
+ includeGlobalServiceEvents != null ? includeGlobalServiceEvents : existing.includeGlobalServiceEvents(),
+ isMultiRegionTrail != null ? isMultiRegionTrail : existing.isMultiRegionTrail(),
+ existing.homeRegion(),
+ enableLogFileValidation != null ? enableLogFileValidation : existing.logFileValidationEnabled(),
+ existing.hasCustomEventSelectors(),
+ existing.hasInsightSelectors(),
+ isOrganizationTrail != null ? isOrganizationTrail : existing.isOrganizationTrail());
+ store.put(key, entry.withTrail(updated));
+ return updated;
+ });
}
public List<Trail> describeTrails(String region, List<String> trailNameOrArnList) {
@@ -135,7 +151,7 @@ public class CloudTrailService {
Trail trail = findTrailOrThrow(region, trailNameOrArn);
List<EventSelector> normalized = selectors == null ? List.of() : List.copyOf(selectors);
String key = regionKey(trail.homeRegion(), trail.name());
- store.get(key).ifPresent(entry -> store.put(key, entry.withSelectors(normalized, true)));
+ withTrailLock(key, () -> store.get(key).ifPresent(entry -> store.put(key, entry.withSelectors(normalized, true))));
return normalized;
}
@@ -149,13 +165,13 @@ public class CloudTrailService {
public void startLogging(String region, String trailNameOrArn) {
Trail trail = findTrailOrThrow(region, trailNameOrArn);
String key = regionKey(trail.homeRegion(), trail.name());
- store.get(key).ifPresent(entry -> store.put(key, entry.startLogging(System.currentTimeMillis())));
+ withTrailLock(key, () -> store.get(key).ifPresent(entry -> store.put(key, entry.startLogging(System.currentTimeMillis()))));
}
public void stopLogging(String region, String trailNameOrArn) {
Trail trail = findTrailOrThrow(region, trailNameOrArn);
String key = regionKey(trail.homeRegion(), trail.name());
- store.get(key).ifPresent(entry -> store.put(key, entry.stopLogging(System.currentTimeMillis())));
+ withTrailLock(key, () -> store.get(key).ifPresent(entry -> store.put(key, entry.stopLogging(System.currentTimeMillis()))));
}
public TrailStatus getTrailStatus(String region, String trailNameOrArn) {
@@ -174,22 +190,30 @@ public class CloudTrailService {
private static final int MAX_TAGS_PER_RESOURCE = 50;
public void addTags(String resourceId, Map<String, String> tagsToAdd) {
- CloudTrailEntry entry = findEntryByArnOrThrow(resourceId);
- Map<String, String> merged = entry.mutableTags();
- merged.putAll(tagsToAdd);
- if (merged.size() > MAX_TAGS_PER_RESOURCE) {
- throw new AwsException("TagsLimitExceededException",
- "Tag limit exceeded for resource " + resourceId
- + ". Maximum allowed: " + MAX_TAGS_PER_RESOURCE + ".", 400);
- }
- store.put(regionKey(entry.trail().homeRegion(), entry.trail().name()), entry.withTags(merged));
+ String key = findKeyByArnOrThrow(resourceId);
+ withTrailLock(key, () -> {
+ CloudTrailEntry entry = store.get(key).orElseThrow(() -> new AwsException(
+ "ResourceNotFoundException", "Resource not found: " + resourceId, 400));
+ Map<String, String> merged = entry.mutableTags();
+ merged.putAll(tagsToAdd);
+ if (merged.size() > MAX_TAGS_PER_RESOURCE) {
+ throw new AwsException("TagsLimitExceededException",
+ "Tag limit exceeded for resource " + resourceId
+ + ". Maximum allowed: " + MAX_TAGS_PER_RESOURCE + ".", 400);
+ }
+ store.put(key, entry.withTags(merged));
+ });
}
public void removeTags(String resourceId, List<String> tagKeys) {
- CloudTrailEntry entry = findEntryByArnOrThrow(resourceId);
- Map<String, String> remaining = entry.mutableTags();
- tagKeys.forEach(remaining::remove);
- store.put(regionKey(entry.trail().homeRegion(), entry.trail().name()), entry.withTags(remaining));
+ String key = findKeyByArnOrThrow(resourceId);
+ withTrailLock(key, () -> {
+ CloudTrailEntry entry = store.get(key).orElseThrow(() -> new AwsException(
+ "ResourceNotFoundException", "Resource not found: " + resourceId, 400));
+ Map<String, String> remaining = entry.mutableTags();
+ tagKeys.forEach(remaining::remove);
+ store.put(key, entry.withTags(remaining));
+ });
}
public Map<String, String> listTags(String resourceId) {
@@ -197,6 +221,15 @@ public class CloudTrailService {
}
private CloudTrailEntry findEntryByArnOrThrow(String resourceId) {
+ String key = findKeyByArnOrThrow(resourceId);
+ return store.get(key).orElseThrow(() -> new AwsException(
+ "ResourceNotFoundException", "Resource not found: " + resourceId, 400));
+ }
+
+ /** Validates {@code resourceId} as a trail ARN and resolves it to a storage key, without
+ * reading the entry itself. Callers that go on to mutate the entry must re-read it inside
+ * {@link #withTrailLock} rather than reuse a value read here. */
+ private String findKeyByArnOrThrow(String resourceId) {
AwsArnUtils.Arn arn;
try {
arn = AwsArnUtils.parse(resourceId);
@@ -211,13 +244,39 @@ public class CloudTrailService {
for (String k : store.keys()) {
CloudTrailEntry entry = store.get(k).orElse(null);
if (entry != null && resourceId.equals(entry.trail().trailArn())) {
- return entry;
+ return k;
}
}
throw new AwsException("ResourceNotFoundException",
"Resource not found: " + resourceId, 400);
}
+ /**
+ * Runs {@code action} with exclusive access to trail {@code key}, so overlapping mutations of
+ * the same trail (even from different actions) can't interleave and clobber one another.
+ * Piggybacks on {@link ConcurrentHashMap#compute}'s documented per-key atomicity as the
+ * mutex, rather than a plain lock-object map: returning null from the remapping function
+ * drops the bookkeeping entry the instant the call finishes, so {@code trailLocks} never
+ * accumulates one entry per trail ever mutated, unlike a map of retained lock objects would.
+ */
+ private <T> T withTrailLock(String key, Supplier<T> action) {
+ Object[] box = new Object[1];
+ trailLocks.compute(key, (k, v) -> {
+ box[0] = action.get();
+ return null;
+ });
+ @SuppressWarnings("unchecked")
+ T result = (T) box[0];
+ return result;
+ }
+
+ private void withTrailLock(String key, Runnable action) {
+ withTrailLock(key, () -> {
+ action.run();
+ return null;
+ });
+ }
+
// --- Data plane: called by S3 (and other services) when an op happens ---
public void emitS3DataEvent(S3EventInput in) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailTagConcurrencyTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailTagConcurrencyTest.java
new file mode 100644
index 0000000000..c120995b5b
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailTagConcurrencyTest.java
@@ -0,0 +1,133 @@
+package io.github.hectorvent.floci.services.cloudtrail;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.services.cloudtrail.model.Trail;
+import io.quarkus.test.junit.QuarkusTest;
+import jakarta.inject.Inject;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * AddTags/RemoveTags and every other trail mutation are a read-modify-write against the same
+ * stored {@link CloudTrailEntry}. Without a monitor shared across all of them, two overlapping
+ * mutations of the same trail can each read before either writes, so whichever write lands last
+ * silently discards the other's change (floci-io/floci#2904).
+ */
+@QuarkusTest
+class CloudTrailTagConcurrencyTest {
+
+ private static final String REGION = "us-east-1";
+
+ @Inject
+ CloudTrailService service;
+
+ @Test
+ void addTagsRacingStartLoggingLosesNeitherChange() throws Exception {
+ for (int attempt = 0; attempt < 25; attempt++) {
+ String name = "race-trail-" + UUID.randomUUID().toString().substring(0, 8);
+ Trail trail = service.createTrail(REGION, name, "some-bucket-" + attempt, null, null,
+ true, false, false, false);
+
+ CountDownLatch start = new CountDownLatch(1);
+ AtomicReference<Throwable> unexpected = new AtomicReference<>();
+
+ Thread tagger = new Thread(() -> {
+ await(start);
+ try {
+ service.addTags(trail.trailArn(), Map.of("env", "race"));
+ } catch (Throwable t) {
+ unexpected.set(t);
+ }
+ });
+ Thread starter = new Thread(() -> {
+ await(start);
+ try {
+ service.startLogging(REGION, name);
+ } catch (Throwable t) {
+ unexpected.set(t);
+ }
+ });
+
+ tagger.start();
+ starter.start();
+ start.countDown();
+ tagger.join(TimeUnit.SECONDS.toMillis(10));
+ starter.join(TimeUnit.SECONDS.toMillis(10));
+ assertFalse(tagger.isAlive() || starter.isAlive(), "a tag/logging call never finished");
+ assertNull(unexpected.get(), () -> "unexpected failure: " + unexpected.get());
+
+ assertEquals("race", service.listTags(trail.trailArn()).get("env"),
+ "attempt " + attempt + ": AddTags was lost to a concurrent StartLogging");
+ assertTrue(service.getTrailStatus(REGION, name).logging(),
+ "attempt " + attempt + ": StartLogging was lost to a concurrent AddTags");
+
+ service.deleteTrail(REGION, name);
+ }
+ }
+
+ @Test
+ void taggingDuringADeleteDoesNotReviveTheTrail() throws Exception {
+ for (int attempt = 0; attempt < 25; attempt++) {
+ String name = "del-race-trail-" + UUID.randomUUID().toString().substring(0, 8);
+ Trail trail = service.createTrail(REGION, name, "some-bucket-" + attempt, null, null,
+ true, false, false, false);
+
+ CountDownLatch start = new CountDownLatch(1);
+ AtomicReference<Throwable> unexpected = new AtomicReference<>();
+
+ Thread tagger = new Thread(() -> {
+ await(start);
+ try {
+ service.addTags(trail.trailArn(), Map.of("env", "race"));
+ } catch (AwsException expected) {
+ // Losing to the delete is a legitimate outcome; being answered wrong is not.
+ if (!"ResourceNotFoundException".equals(expected.getErrorCode())) {
+ unexpected.set(expected);
+ }
+ } catch (Throwable t) {
+ unexpected.set(t);
+ }
+ });
+ Thread deleter = new Thread(() -> {
+ await(start);
+ try {
+ service.deleteTrail(REGION, name);
+ } catch (Throwable t) {
+ unexpected.set(t);
+ }
+ });
+
+ tagger.start();
+ deleter.start();
+ start.countDown();
+ tagger.join(TimeUnit.SECONDS.toMillis(10));
+ deleter.join(TimeUnit.SECONDS.toMillis(10));
+ assertFalse(tagger.isAlive() || deleter.isAlive(),
+ "tagger/deleter did not finish, deadlock between the monitors");
+ assertNull(unexpected.get(), () -> "unexpected failure: " + unexpected.get());
+
+ assertTrue(service.describeTrails(REGION, List.of(name)).isEmpty(),
+ "attempt " + attempt + ": trail " + name + " came back after being deleted");
+ }
+ }
+
+ private static void await(CountDownLatch latch) {
+ try {
+ latch.await();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(e);
+ }
+ }
+}