Light, fluffy, and always free - The AWS Local Emulator alternative
fix(cloudtrail): serialize trail mutations to prevent lost updates (#2911)
Priamo Germosen8c2619f96549
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailService.java b/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailService.javaindex 01fb9af806..32c8614115 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailService.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailService.java@@ -27,6 +27,7 @@ import java.util.Map;import java.util.UUID;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.ConcurrentLinkedQueue;+import java.util.function.Supplier;@ApplicationScopedpublic class CloudTrailService {@@ -45,6 +46,15 @@ public class CloudTrailService {private final ConcurrentHashMap<TrailKey, ConcurrentLinkedQueue<ObjectNode>> pendingRecordsByTrail =new ConcurrentHashMap<>();+ /**+ * Backs {@link #withTrailLock}: serializes read-modify-write mutations to the same trail. A+ * plain {@code get} then {@code put} on the store is not atomic across the pair, so two+ * overlapping mutations of the same trail, even from different actions, e.g. AddTags racing+ * StartLogging, can interleave such that the second {@code put} replaces the whole entry the+ * first read, silently discarding whatever the first call changed.+ */+ private final ConcurrentHashMap<String, Object> trailLocks = new ConcurrentHashMap<>();+@Injectpublic CloudTrailService(StorageFactory storageFactory, RegionResolver regionResolver,IamService iamService, ObjectMapper mapper) {@@ -82,7 +92,8 @@ public class CloudTrailService {public void deleteTrail(String region, String trailNameOrArn) {Trail trail = findTrailOrThrow(region, trailNameOrArn);- store.delete(regionKey(trail.homeRegion(), trail.name()));+ String key = regionKey(trail.homeRegion(), trail.name());+ withTrailLock(key, () -> store.delete(key));pendingRecordsByTrail.keySet().removeIf(k -> k.trailName().equals(trail.name()));}@@ -90,23 +101,28 @@ public class CloudTrailService {String s3BucketName, String s3KeyPrefix, String snsTopicArn,Boolean includeGlobalServiceEvents, Boolean isMultiRegionTrail,Boolean enableLogFileValidation, Boolean isOrganizationTrail) {- Trail existing = findTrailOrThrow(region, trailNameOrArn);- Trail updated = new Trail(- existing.name(),- existing.trailArn(),- s3BucketName != null ? s3BucketName : existing.s3BucketName(),- s3KeyPrefix != null ? s3KeyPrefix : existing.s3KeyPrefix(),- snsTopicArn != null ? snsTopicArn : existing.snsTopicArn(),- includeGlobalServiceEvents != null ? includeGlobalServiceEvents : existing.includeGlobalServiceEvents(),- isMultiRegionTrail != null ? isMultiRegionTrail : existing.isMultiRegionTrail(),- existing.homeRegion(),- enableLogFileValidation != null ? enableLogFileValidation : existing.logFileValidationEnabled(),- existing.hasCustomEventSelectors(),- existing.hasInsightSelectors(),- isOrganizationTrail != null ? isOrganizationTrail : existing.isOrganizationTrail());- String key = regionKey(existing.homeRegion(), existing.name());- store.get(key).ifPresent(entry -> store.put(key, entry.withTrail(updated)));- return updated;+ Trail resolved = findTrailOrThrow(region, trailNameOrArn);+ String key = regionKey(resolved.homeRegion(), resolved.name());+ return withTrailLock(key, () -> {+ CloudTrailEntry entry = store.get(key).orElseThrow(() -> new AwsException(+ "TrailNotFoundException", "Unknown trail: " + trailNameOrArn, 400));+ Trail existing = entry.trail();+ Trail updated = new Trail(+ existing.name(),+ existing.trailArn(),+ s3BucketName != null ? s3BucketName : existing.s3BucketName(),+ s3KeyPrefix != null ? s3KeyPrefix : existing.s3KeyPrefix(),+ snsTopicArn != null ? snsTopicArn : existing.snsTopicArn(),+ includeGlobalServiceEvents != null ? includeGlobalServiceEvents : existing.includeGlobalServiceEvents(),+ isMultiRegionTrail != null ? isMultiRegionTrail : existing.isMultiRegionTrail(),+ existing.homeRegion(),+ enableLogFileValidation != null ? enableLogFileValidation : existing.logFileValidationEnabled(),+ existing.hasCustomEventSelectors(),+ existing.hasInsightSelectors(),+ isOrganizationTrail != null ? isOrganizationTrail : existing.isOrganizationTrail());+ store.put(key, entry.withTrail(updated));+ return updated;+ });}public List<Trail> describeTrails(String region, List<String> trailNameOrArnList) {@@ -135,7 +151,7 @@ public class CloudTrailService {Trail trail = findTrailOrThrow(region, trailNameOrArn);List<EventSelector> normalized = selectors == null ? List.of() : List.copyOf(selectors);String key = regionKey(trail.homeRegion(), trail.name());- store.get(key).ifPresent(entry -> store.put(key, entry.withSelectors(normalized, true)));+ withTrailLock(key, () -> store.get(key).ifPresent(entry -> store.put(key, entry.withSelectors(normalized, true))));return normalized;}@@ -149,13 +165,13 @@ public class CloudTrailService {public void startLogging(String region, String trailNameOrArn) {Trail trail = findTrailOrThrow(region, trailNameOrArn);String key = regionKey(trail.homeRegion(), trail.name());- store.get(key).ifPresent(entry -> store.put(key, entry.startLogging(System.currentTimeMillis())));+ withTrailLock(key, () -> store.get(key).ifPresent(entry -> store.put(key, entry.startLogging(System.currentTimeMillis()))));}public void stopLogging(String region, String trailNameOrArn) {Trail trail = findTrailOrThrow(region, trailNameOrArn);String key = regionKey(trail.homeRegion(), trail.name());- store.get(key).ifPresent(entry -> store.put(key, entry.stopLogging(System.currentTimeMillis())));+ withTrailLock(key, () -> store.get(key).ifPresent(entry -> store.put(key, entry.stopLogging(System.currentTimeMillis()))));}public TrailStatus getTrailStatus(String region, String trailNameOrArn) {@@ -174,22 +190,30 @@ public class CloudTrailService {private static final int MAX_TAGS_PER_RESOURCE = 50;public void addTags(String resourceId, Map<String, String> tagsToAdd) {- CloudTrailEntry entry = findEntryByArnOrThrow(resourceId);- Map<String, String> merged = entry.mutableTags();- merged.putAll(tagsToAdd);- if (merged.size() > MAX_TAGS_PER_RESOURCE) {- throw new AwsException("TagsLimitExceededException",- "Tag limit exceeded for resource " + resourceId- + ". Maximum allowed: " + MAX_TAGS_PER_RESOURCE + ".", 400);- }- store.put(regionKey(entry.trail().homeRegion(), entry.trail().name()), entry.withTags(merged));+ String key = findKeyByArnOrThrow(resourceId);+ withTrailLock(key, () -> {+ CloudTrailEntry entry = store.get(key).orElseThrow(() -> new AwsException(+ "ResourceNotFoundException", "Resource not found: " + resourceId, 400));+ Map<String, String> merged = entry.mutableTags();+ merged.putAll(tagsToAdd);+ if (merged.size() > MAX_TAGS_PER_RESOURCE) {+ throw new AwsException("TagsLimitExceededException",+ "Tag limit exceeded for resource " + resourceId+ + ". Maximum allowed: " + MAX_TAGS_PER_RESOURCE + ".", 400);+ }+ store.put(key, entry.withTags(merged));+ });}public void removeTags(String resourceId, List<String> tagKeys) {- CloudTrailEntry entry = findEntryByArnOrThrow(resourceId);- Map<String, String> remaining = entry.mutableTags();- tagKeys.forEach(remaining::remove);- store.put(regionKey(entry.trail().homeRegion(), entry.trail().name()), entry.withTags(remaining));+ String key = findKeyByArnOrThrow(resourceId);+ withTrailLock(key, () -> {+ CloudTrailEntry entry = store.get(key).orElseThrow(() -> new AwsException(+ "ResourceNotFoundException", "Resource not found: " + resourceId, 400));+ Map<String, String> remaining = entry.mutableTags();+ tagKeys.forEach(remaining::remove);+ store.put(key, entry.withTags(remaining));+ });}public Map<String, String> listTags(String resourceId) {@@ -197,6 +221,15 @@ public class CloudTrailService {}private CloudTrailEntry findEntryByArnOrThrow(String resourceId) {+ String key = findKeyByArnOrThrow(resourceId);+ return store.get(key).orElseThrow(() -> new AwsException(+ "ResourceNotFoundException", "Resource not found: " + resourceId, 400));+ }++ /** Validates {@code resourceId} as a trail ARN and resolves it to a storage key, without+ * reading the entry itself. Callers that go on to mutate the entry must re-read it inside+ * {@link #withTrailLock} rather than reuse a value read here. */+ private String findKeyByArnOrThrow(String resourceId) {AwsArnUtils.Arn arn;try {arn = AwsArnUtils.parse(resourceId);@@ -211,13 +244,39 @@ public class CloudTrailService {for (String k : store.keys()) {CloudTrailEntry entry = store.get(k).orElse(null);if (entry != null && resourceId.equals(entry.trail().trailArn())) {- return entry;+ return k;}}throw new AwsException("ResourceNotFoundException","Resource not found: " + resourceId, 400);}+ /**+ * Runs {@code action} with exclusive access to trail {@code key}, so overlapping mutations of+ * the same trail (even from different actions) can't interleave and clobber one another.+ * Piggybacks on {@link ConcurrentHashMap#compute}'s documented per-key atomicity as the+ * mutex, rather than a plain lock-object map: returning null from the remapping function+ * drops the bookkeeping entry the instant the call finishes, so {@code trailLocks} never+ * accumulates one entry per trail ever mutated, unlike a map of retained lock objects would.+ */+ private <T> T withTrailLock(String key, Supplier<T> action) {+ Object[] box = new Object[1];+ trailLocks.compute(key, (k, v) -> {+ box[0] = action.get();+ return null;+ });+ @SuppressWarnings("unchecked")+ T result = (T) box[0];+ return result;+ }++ private void withTrailLock(String key, Runnable action) {+ withTrailLock(key, () -> {+ action.run();+ return null;+ });+ }+// --- Data plane: called by S3 (and other services) when an op happens ---public void emitS3DataEvent(S3EventInput in) {diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailTagConcurrencyTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailTagConcurrencyTest.javanew file mode 100644index 0000000000..c120995b5b--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailTagConcurrencyTest.java@@ -0,0 +1,133 @@+package io.github.hectorvent.floci.services.cloudtrail;++import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.services.cloudtrail.model.Trail;+import io.quarkus.test.junit.QuarkusTest;+import jakarta.inject.Inject;+import org.junit.jupiter.api.Test;++import java.util.List;+import java.util.Map;+import java.util.UUID;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicReference;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertTrue;++/**+ * AddTags/RemoveTags and every other trail mutation are a read-modify-write against the same+ * stored {@link CloudTrailEntry}. Without a monitor shared across all of them, two overlapping+ * mutations of the same trail can each read before either writes, so whichever write lands last+ * silently discards the other's change (floci-io/floci#2904).+ */+@QuarkusTest+class CloudTrailTagConcurrencyTest {++ private static final String REGION = "us-east-1";++ @Inject+ CloudTrailService service;++ @Test+ void addTagsRacingStartLoggingLosesNeitherChange() throws Exception {+ for (int attempt = 0; attempt < 25; attempt++) {+ String name = "race-trail-" + UUID.randomUUID().toString().substring(0, 8);+ Trail trail = service.createTrail(REGION, name, "some-bucket-" + attempt, null, null,+ true, false, false, false);++ CountDownLatch start = new CountDownLatch(1);+ AtomicReference<Throwable> unexpected = new AtomicReference<>();++ Thread tagger = new Thread(() -> {+ await(start);+ try {+ service.addTags(trail.trailArn(), Map.of("env", "race"));+ } catch (Throwable t) {+ unexpected.set(t);+ }+ });+ Thread starter = new Thread(() -> {+ await(start);+ try {+ service.startLogging(REGION, name);+ } catch (Throwable t) {+ unexpected.set(t);+ }+ });++ tagger.start();+ starter.start();+ start.countDown();+ tagger.join(TimeUnit.SECONDS.toMillis(10));+ starter.join(TimeUnit.SECONDS.toMillis(10));+ assertFalse(tagger.isAlive() || starter.isAlive(), "a tag/logging call never finished");+ assertNull(unexpected.get(), () -> "unexpected failure: " + unexpected.get());++ assertEquals("race", service.listTags(trail.trailArn()).get("env"),+ "attempt " + attempt + ": AddTags was lost to a concurrent StartLogging");+ assertTrue(service.getTrailStatus(REGION, name).logging(),+ "attempt " + attempt + ": StartLogging was lost to a concurrent AddTags");++ service.deleteTrail(REGION, name);+ }+ }++ @Test+ void taggingDuringADeleteDoesNotReviveTheTrail() throws Exception {+ for (int attempt = 0; attempt < 25; attempt++) {+ String name = "del-race-trail-" + UUID.randomUUID().toString().substring(0, 8);+ Trail trail = service.createTrail(REGION, name, "some-bucket-" + attempt, null, null,+ true, false, false, false);++ CountDownLatch start = new CountDownLatch(1);+ AtomicReference<Throwable> unexpected = new AtomicReference<>();++ Thread tagger = new Thread(() -> {+ await(start);+ try {+ service.addTags(trail.trailArn(), Map.of("env", "race"));+ } catch (AwsException expected) {+ // Losing to the delete is a legitimate outcome; being answered wrong is not.+ if (!"ResourceNotFoundException".equals(expected.getErrorCode())) {+ unexpected.set(expected);+ }+ } catch (Throwable t) {+ unexpected.set(t);+ }+ });+ Thread deleter = new Thread(() -> {+ await(start);+ try {+ service.deleteTrail(REGION, name);+ } catch (Throwable t) {+ unexpected.set(t);+ }+ });++ tagger.start();+ deleter.start();+ start.countDown();+ tagger.join(TimeUnit.SECONDS.toMillis(10));+ deleter.join(TimeUnit.SECONDS.toMillis(10));+ assertFalse(tagger.isAlive() || deleter.isAlive(),+ "tagger/deleter did not finish, deadlock between the monitors");+ assertNull(unexpected.get(), () -> "unexpected failure: " + unexpected.get());++ assertTrue(service.describeTrails(REGION, List.of(name)).isEmpty(),+ "attempt " + attempt + ": trail " + name + " came back after being deleted");+ }+ }++ private static void await(CountDownLatch latch) {+ try {+ latch.await();+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new IllegalStateException(e);+ }+ }+}