Light, fluffy, and always free - The AWS Local Emulator alternative
fix(cloudtrail): isolate account buffers and partition matching (#6689)
jwoa959303ec287235
diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailLogWriter.java b/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailLogWriter.javaindex 254be2eab1..e0c5f425cb 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailLogWriter.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailLogWriter.java@@ -6,6 +6,7 @@ import com.fasterxml.jackson.databind.node.ObjectNode;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.RequestScopes;import io.github.hectorvent.floci.core.common.ServicePrincipals;import io.github.hectorvent.floci.services.cloudtrail.model.Trail;import io.github.hectorvent.floci.services.s3.S3Service;@@ -116,7 +117,7 @@ public class CloudTrailLogWriter {try {for (CloudTrailService.TrailKey key : cloudTrailService.trailsWithPendingRecords()) {try {- flushTrailBatches(key);+ RequestScopes.runAs(key.accountId(), key.region(), () -> flushTrailBatches(key));} catch (RuntimeException e) {LOG.warnv(e, "CloudTrail log flush failed for trail {0} in {1}",key.trailName(), key.region());diff --git a/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailService.java b/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailService.javaindex d1d697f798..7e2e57dd41 100644--- a/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailService.java+++ b/src/main/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailService.java@@ -128,7 +128,8 @@ public class CloudTrailService {Trail trail = findTrailOrThrow(region, trailNameOrArn);String key = regionKey(trail.homeRegion(), trail.name());withTrailLock(key, () -> store.delete(key));- pendingRecordsByTrail.keySet().removeIf(k -> k.trailName().equals(trail.name()));+ pendingRecordsByTrail.keySet().removeIf(k -> k.trailName().equals(trail.name())+ && k.region().equals(trail.homeRegion()) && k.accountId().equals(regionResolver.getAccountId()));}public Trail updateTrail(String region, String trailNameOrArn,@@ -428,7 +429,7 @@ public class CloudTrailService {List<TrailKey> result = new ArrayList<>();for (Map.Entry<PendingTrailKey, PendingRecordBuffer> e : pendingRecordsByTrail.entrySet()) {for (String eventRegion : e.getValue().eventRegions()) {- result.add(new TrailKey(e.getKey().region(), e.getKey().trailName(), eventRegion));+ result.add(new TrailKey(e.getKey().region(), e.getKey().trailName(), eventRegion, e.getKey().accountId()));}}return result;@@ -482,10 +483,15 @@ public class CloudTrailService {* {@code region} is the trail's home region (used for trail store lookups).* {@code eventRegion} is the region where the event occurred (used for the S3 delivery path).* For single-region trails these are the same; for multi-region trails they differ.+ * {@code accountId} preserves the owning account for background delivery.*/- public record TrailKey(String region, String trailName, String eventRegion) {}+ public record TrailKey(String region, String trailName, String eventRegion, String accountId) {+ public TrailKey(String region, String trailName, String eventRegion) {+ this(region, trailName, eventRegion, null);+ }+ }- private record PendingTrailKey(String region, String trailName) {}+ private record PendingTrailKey(String region, String trailName, String accountId) {}private record PendingRecord(ObjectNode record, long byteCount) {}@@ -612,8 +618,9 @@ public class CloudTrailService {}}- private static PendingTrailKey pendingTrailKey(TrailKey key) {- return new PendingTrailKey(key.region(), key.trailName());+ private PendingTrailKey pendingTrailKey(TrailKey key) {+ return new PendingTrailKey(key.region(), key.trailName(),+ key.accountId() != null ? key.accountId() : regionResolver.getAccountId());}// --- Helpers ---@@ -631,6 +638,9 @@ public class CloudTrailService {if (!sameRegion && !trail.isMultiRegionTrail()) {continue;}+ if (!regionResolver.partitionForRegion(region).equals(regionResolver.partitionForRegion(trail.homeRegion()))) {+ continue;+ }if (!entry.logging()) {continue;}diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailAccountDeliveryIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailAccountDeliveryIntegrationTest.javanew file mode 100644index 0000000000..24c7f7e990--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailAccountDeliveryIntegrationTest.java@@ -0,0 +1,179 @@+package io.github.hectorvent.floci.services.cloudtrail;++import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.RequestScopes;+import io.github.hectorvent.floci.services.cloudtrail.model.DataResource;+import io.github.hectorvent.floci.services.cloudtrail.model.EventSelector;+import io.github.hectorvent.floci.services.s3.S3Service;+import io.github.hectorvent.floci.services.s3.model.S3Object;+import io.quarkus.test.junit.QuarkusTest;+import jakarta.inject.Inject;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;++import java.io.ByteArrayInputStream;+import java.util.List;+import java.util.Map;+import java.util.UUID;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.TimeUnit;+import java.util.zip.GZIPInputStream;++import static org.junit.jupiter.api.Assertions.*;++@QuarkusTest+class CloudTrailAccountDeliveryIntegrationTest {+ private static final String REGION = "eu-west-1";+ private static final String FIRST_ACCOUNT = "661100000004";+ private static final String SECOND_ACCOUNT = "661100000005";++ @Inject+ CloudTrailService trails;++ @Inject+ CloudTrailLogWriter writer;++ @Inject+ S3Service s3;++ @Inject+ ObjectMapper mapper;++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void equalTrailNamesKeepAccountBuffersAndBackgroundDeliverySeparate(boolean deleteFirstTrail) throws Exception {+ String name = "account-events-" + UUID.randomUUID();+ Map<String, String> buckets = Map.of(FIRST_ACCOUNT, name + "-first", SECOND_ACCOUNT, name + "-second");+ try {+ buckets.forEach((account, bucket) -> createS3DataTrail(account, REGION, name, bucket));+ RequestScopes.runAs(FIRST_ACCOUNT, REGION, () -> trails.startLogging(REGION, name));+ RequestScopes.runAs(SECOND_ACCOUNT, REGION, () -> {+ trails.emitS3DataEvent(event());+ assertEquals(0, trails.pendingRecordCount(new CloudTrailService.TrailKey(REGION, name, REGION)),+ "another account's trail cannot enable events");+ trails.startLogging(REGION, name);+ });+ for (String account : buckets.keySet()) {+ RequestScopes.runAs(account, REGION, () -> {+ trails.emitS3DataEvent(event());+ assertEquals(1, trails.pendingRecordCount(new CloudTrailService.TrailKey(REGION, name, REGION)));+ });+ }+ if (deleteFirstTrail) {+ RequestScopes.runAs(FIRST_ACCOUNT, REGION, () -> trails.deleteTrail(REGION, name));+ }+ RequestScopes.runAs(SECOND_ACCOUNT, REGION, () -> assertEquals(1,+ trails.pendingRecordCount(new CloudTrailService.TrailKey(REGION, name, REGION)),+ "deleting one account's trail must preserve the other account's pending records"));++ flushLogsOnWorker();++ for (String account : buckets.keySet()) {+ if (deleteFirstTrail && FIRST_ACCOUNT.equals(account)) {+ RequestScopes.runAs(account, REGION, () -> assertTrue(+ s3.listObjects(buckets.get(account), null, null, 100).isEmpty()));+ continue;+ }+ JsonNode records = readDeliveredRecords(account, REGION, buckets.get(account));+ assertEquals(1, records.size());+ assertEquals("Data", records.get(0).path("eventCategory").asText());+ assertEquals(account, records.get(0).path("recipientAccountId").asText());+ assertEquals(account, records.get(0).path("userIdentity").path("accountId").asText());+ RequestScopes.runAs(account, REGION, () -> assertEquals(0,+ trails.pendingRecordCount(new CloudTrailService.TrailKey(REGION, name, REGION))));+ }+ } finally {+ buckets.forEach((account, bucket) -> deleteTrailAndBucket(account, REGION, name, bucket));+ }+ }++ @Test+ void deletingTrailByArnPreservesSameNamedTrailInAnotherHomeRegion() throws Exception {+ String retainedRegion = "us-west-2";+ String name = "region-events-" + UUID.randomUUID();+ Map<String, String> buckets = Map.of(REGION, name + "-deleted", retainedRegion, name + "-retained");+ try {+ buckets.forEach((region, bucket) -> {+ createS3DataTrail(FIRST_ACCOUNT, region, name, bucket);+ RequestScopes.runAs(FIRST_ACCOUNT, region, () -> {+ trails.startLogging(region, name);+ trails.emitS3DataEvent(event(region));+ assertEquals(1, trails.pendingRecordCount(new CloudTrailService.TrailKey(region, name, region)));+ });+ });+ String deletedArn = RequestScopes.callAs(FIRST_ACCOUNT, REGION,+ () -> trails.getTrail(REGION, name).trailArn());+ RequestScopes.runAs(FIRST_ACCOUNT, retainedRegion, () -> {+ trails.deleteTrail(retainedRegion, deletedArn);+ assertEquals(0, trails.pendingRecordCount(new CloudTrailService.TrailKey(REGION, name, REGION)));+ assertEquals(1, trails.pendingRecordCount(new CloudTrailService.TrailKey(retainedRegion, name, retainedRegion)),+ "deleting by ARN must preserve the same account's other home-region buffer");+ });++ flushLogsOnWorker();++ RequestScopes.runAs(FIRST_ACCOUNT, REGION, () -> assertTrue(+ s3.listObjects(buckets.get(REGION), null, null, 100).isEmpty()));+ JsonNode records = readDeliveredRecords(FIRST_ACCOUNT, retainedRegion, buckets.get(retainedRegion));+ assertEquals(1, records.size());+ assertEquals(FIRST_ACCOUNT, records.get(0).path("recipientAccountId").asText());+ assertEquals(retainedRegion, records.get(0).path("awsRegion").asText());+ assertEquals("PutObject", records.get(0).path("eventName").asText());+ RequestScopes.runAs(FIRST_ACCOUNT, retainedRegion, () -> assertEquals(0,+ trails.pendingRecordCount(new CloudTrailService.TrailKey(retainedRegion, name, retainedRegion))));+ } finally {+ buckets.forEach((region, bucket) -> deleteTrailAndBucket(FIRST_ACCOUNT, region, name, bucket));+ }+ }++ private void createS3DataTrail(String account, String region, String name, String bucket) {+ RequestScopes.runAs(account, region, () -> {+ s3.createBucket(bucket, region);+ trails.createTrail(region, name, bucket, null, null, false, false, false, false);+ trails.putEventSelectors(region, name, List.of(new EventSelector("All", false,+ List.of(new DataResource("AWS::S3::Object", List.of("arn:aws:s3:::source/"))), List.of())));+ });+ }++ private void flushLogsOnWorker() throws Exception {+ try (ExecutorService worker = Executors.newSingleThreadExecutor()) {+ worker.submit(writer::flushNow).get(10, TimeUnit.SECONDS);+ }+ }++ private JsonNode readDeliveredRecords(String account, String region, String bucket) throws Exception {+ List<S3Object> objects = RequestScopes.callAs(account, region,+ () -> s3.listObjects(bucket, null, null, 100));+ assertEquals(1, objects.size(), "each trail must receive its own log file");+ assertTrue(objects.getFirst().getKey().startsWith("AWSLogs/" + account + "/CloudTrail/" + region + "/"));+ byte[] content = RequestScopes.callAs(account, region,+ () -> s3.getObject(bucket, objects.getFirst().getKey()).getData());+ try (GZIPInputStream gzip = new GZIPInputStream(new ByteArrayInputStream(content))) {+ return mapper.readTree(gzip).path("Records");+ }+ }++ private void deleteTrailAndBucket(String account, String region, String name, String bucket) {+ RequestScopes.runAs(account, region, () -> {+ if (trails.getTrail(region, name) != null) {+ trails.deleteTrail(region, name);+ }+ for (S3Object object : s3.listObjects(bucket, null, null, 100)) {+ s3.deleteObject(bucket, object.getKey());+ }+ s3.deleteBucket(bucket);+ });+ }++ private CloudTrailService.S3EventInput event() {+ return event(REGION);+ }++ private CloudTrailService.S3EventInput event(String region) {+ return CloudTrailService.S3EventInput.builder()+ .region(region).eventName("PutObject").bucketName("source").key("example.txt").build();+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailSelectorMatchingTest.java b/src/test/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailSelectorMatchingTest.javaindex 0358cedae4..03de0e8203 100644--- a/src/test/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailSelectorMatchingTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/cloudtrail/CloudTrailSelectorMatchingTest.java@@ -9,6 +9,8 @@ import io.github.hectorvent.floci.services.cloudtrail.model.DataResource;import io.github.hectorvent.floci.services.cloudtrail.model.EventSelector;import io.github.hectorvent.floci.services.iam.IamService;import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.CsvSource;import java.util.List;@@ -177,6 +179,42 @@ class CloudTrailSelectorMatchingTest {assertFalse(CloudTrailService.isReadOnlyEvent("RestoreObject"));}+ // --- trail matching ---++ @ParameterizedTest+ @CsvSource({+ "us-east-1,us-east-1,us-east-1,false,true",+ "us-east-1,us-east-1,eu-west-1,false,false",+ "us-east-1,us-east-1,eu-west-1,true,true",+ "us-east-1,us-east-1,cn-north-1,true,false",+ "us-east-1,us-east-1,us-gov-west-1,true,false",+ "cn-north-1,cn-north-1,cn-northwest-1,true,true",+ "cn-north-1,cn-north-1,us-east-1,true,false",+ "us-gov-west-1,us-gov-west-1,us-gov-east-1,true,true",+ "us-gov-west-1,us-gov-west-1,us-east-1,true,false",+ "cn-north-1,cn-north-1,xx-nowhere-9,true,true",+ "cn-north-1,xx-nowhere-9,cn-northwest-1,true,true",+ "cn-north-1,xx-nowhere-9,us-east-1,true,false",+ "us-gov-west-1,us-gov-west-1,xx-nowhere-9,true,true",+ "us-gov-west-1,xx-nowhere-9,us-gov-east-1,true,true",+ "us-gov-west-1,xx-nowhere-9,cn-north-1,true,false"+ })+ void multiRegionTrailsMatchOnlyWithinTheirPartition(String defaultRegion, String homeRegion, String eventRegion,+ boolean multiRegion, boolean expected) {+ CloudTrailService service = cloudTrailService(defaultRegion);+ service.createTrail(homeRegion, "partition-trail", "trail-logs", null, null,+ false, multiRegion, false, false);+ service.putEventSelectors(homeRegion, "partition-trail", List.of(new EventSelector("All", false,+ List.of(new DataResource("AWS::S3::Object", List.of("arn:aws:s3:::source/"))), List.of())));+ service.startLogging(homeRegion, "partition-trail");++ service.emitS3DataEvent(CloudTrailService.S3EventInput.builder()+ .region(eventRegion).eventName("PutObject").bucketName("source").key("example.txt").build());++ assertEquals(expected ? 1 : 0, service.pendingRecordCount(+ new CloudTrailService.TrailKey(homeRegion, "partition-trail", eventRegion)));+ }+// --- a ReadOnly trail ---/**@@ -185,11 +223,7 @@ class CloudTrailSelectorMatchingTest {*/@Testvoid readOnlyTrail_keepsDeniedReadsAsReadOnly_andLeavesOutDeniedWrites() {- StorageFactory storageFactory = mock(StorageFactory.class);- when(storageFactory.create(anyString(), anyString(), any()))- .thenAnswer(invocation -> AccountAwareStorageBackend.inMemory(ACCOUNT));- CloudTrailService service = new CloudTrailService(storageFactory, new RegionResolver(REGION, ACCOUNT),- mock(IamService.class), new ObjectMapper());+ CloudTrailService service = cloudTrailService(REGION);service.createTrail(REGION, "reads", "trail-logs", null, null, true, false, false, false);service.putEventSelectors(REGION, "reads", List.of(new EventSelector("ReadOnly", false,List.of(new DataResource("AWS::S3::Object", List.of("arn:aws:s3:::source/"))), null)));@@ -228,4 +262,12 @@ class CloudTrailSelectorMatchingTest {// should default to safe-no-write semantics if the eventName is missing.assertTrue(CloudTrailService.isReadOnlyEvent(null));}++ private CloudTrailService cloudTrailService(String defaultRegion) {+ StorageFactory storageFactory = mock(StorageFactory.class);+ when(storageFactory.create(anyString(), anyString(), any()))+ .thenReturn(AccountAwareStorageBackend.inMemory(ACCOUNT));+ return new CloudTrailService(storageFactory, new RegionResolver(defaultRegion, ACCOUNT),+ mock(IamService.class), new ObjectMapper());+ }}