Light, fluffy, and always free - The AWS Local Emulator alternative
fix: create Firehose buckets and backfill EKS OIDC issuers in the resource's region (#5099)
Hector Ventura4d9585583ddf
diff --git a/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java b/src/main/java/io/github/hectorvent/floci/services/eks/EksService.javaindex 1485c5f16a..91cb497c01 100644--- a/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java+++ b/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java@@ -293,7 +293,9 @@ public class EksService implements TagHandler, ResourceProvider {cluster.getIdentity().getOidc().getIssuer());continue;}- String issuer = oidcService.newIssuerUrl(config.defaultRegion());+ // The cluster's own region, from its ARN: the issuer must match the one create mints there.+ String issuer = oidcService.newIssuerUrl(+ AwsArnUtils.regionOrDefault(cluster.getArn(), config.defaultRegion()));cluster.setIdentity(new ClusterIdentity(new OidcIdentity(issuer)));oidcService.ensureKeyForAccount(accountId, cluster.getName(), issuer);putClusterForAccount(accountId, cluster);diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.javaindex 137aae6424..447652bf61 100644--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java@@ -901,7 +901,7 @@ public class FirehoseService implements ResourceProvider {S3Destination s3 = stream.s3Destination();List<byte[]> records = toFlush;if (s3 != null && s3.isProcessingEnabled()) {- ensureBucket(bucket);+ ensureBucket(bucket, stream);FirehoseLambdaTransformer.Outcome transformed =lambdaTransformer.transform(stream, bucket, records, clock.instant());LOG.infov("Transformed {0} records from stream {1} ({2} dropped, {3} failed)",@@ -916,7 +916,7 @@ public class FirehoseService implements ResourceProvider {}}if (s3 != null && s3.isDataFormatConversionEnabled()) {- ensureBucket(bucket);+ ensureBucket(bucket, stream);FirehoseParquetConverter.Outcome outcome =parquetConverter.deliver(stream, bucket, records, clock.instant());LOG.infov("Converted {0} records ({1} failed) from stream {2} to s3://{3}/{4}",@@ -930,7 +930,7 @@ public class FirehoseService implements ResourceProvider {String key = S3ObjectKeyResolver.resolveKey(s3, stream.getDeliveryStreamName(),stream.getVersionId(), clock.instant(), compression);- ensureBucket(bucket);+ ensureBucket(bucket, stream);// Records are arbitrary bytes, so they are concatenated as bytes: routing// them through a String would corrupt any payload that is not valid UTF-8.@@ -993,10 +993,22 @@ public class FirehoseService implements ResourceProvider {return DEFAULT_BUCKET;}- private void ensureBucket(String bucket) {+ /**+ * Creates a missing destination bucket. A stream's own destination goes in the stream's region,+ * from its ARN; the shared default bucket, used by every stream without one, stays in the default+ * region so the first stream to deliver does not decide it for all the others.+ */+ private void ensureBucket(String bucket, DeliveryStreamDescription stream) {+ S3Destination s3 = stream.s3Destination();+ boolean ownDestination = s3 != null && s3.bucketName() != null;+ String region = ownDestination+ ? AwsArnUtils.regionOrDefault(stream.getDeliveryStreamARN(), regionResolver.getDefaultRegion())+ : regionResolver.getDefaultRegion();try {- s3Service.createBucket(bucket, regionResolver.getDefaultRegion());- } catch (Exception ignored) {}+ s3Service.createBucket(bucket, region);+ } catch (Exception ignored) {+ // The bucket usually exists already; a real problem with it surfaces on the write that follows.+ }}// ─── Resource Explorer 2 ───────────────────────────────────────────────────diff --git a/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.javaindex 8367668774..7c472c50f9 100644--- a/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java@@ -276,6 +276,28 @@ class EksServiceTest {assertTrue(oidcService.findVerificationKey(issuer).isPresent());}+ @Test+ void initBackfillsTheOidcIssuerInTheClustersOwnRegion() {+ StorageBackend<String, Cluster> clusterStore = new InMemoryStorage<>();+ StorageBackend<String, ClusterOidcKey> keyStore = new InMemoryStorage<>();++ Cluster legacy = new Cluster();+ legacy.setName("eu-legacy-cluster");+ legacy.setArn("arn:aws:eks:eu-west-1:000000000000:cluster/eu-legacy-cluster");+ legacy.setStatus(ClusterStatus.ACTIVE);+ clusterStore.put("eu-legacy-cluster", legacy);++ EksOidcService oidcService = new EksOidcService(+ fixedStorageFactory(keyStore), new ObjectMapper());+ EksService restarted = new EksService(fixedStorageFactory(clusterStore), testConfig(),+ new RegionResolver("us-east-1", "000000000000"), null, null, oidcService,+ mock(EksAccessEntryService.class), mock(EksPodIdentityAssociationService.class));+ restarted.init();++ String issuer = clusterStore.get("eu-legacy-cluster").orElseThrow().getIdentity().getOidc().getIssuer();+ assertTrue(issuer.startsWith("https://oidc.eks.eu-west-1.amazonaws.com/id/"), issuer);+ }+@Testvoid initLeavesAnExistingOidcIssuerUnchanged() {StorageBackend<String, Cluster> clusterStore = new InMemoryStorage<>();diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.javaindex 52eec55d0f..5545a5ca9b 100644--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java@@ -248,6 +248,43 @@ class FirehoseServiceTest {.getDeliveryStreamName());}+ @Test+ void aMissingDestinationBucketIsCreatedInTheStreamsRegion() {+ S3Destination s3 = new S3Destination();+ s3.setBucketArn("arn:aws:s3:::eu-stream-bucket");+ firehoseService.createDeliveryStream("eu-west-1", "000000000000", "eu-stream", s3, List.of(), null, null);+ firehoseService.putRecord("000000000000", "eu-west-1", "eu-stream",+ new Record("one".getBytes(StandardCharsets.UTF_8)));++ firehoseService.flush("000000000000", "eu-west-1", "eu-stream");++ verify(s3Service).createBucket("eu-stream-bucket", "eu-west-1");+ }++ @Test+ void aDestinationNamedLikeTheDefaultBucketFollowsItsStream() {+ S3Destination s3 = new S3Destination();+ s3.setBucketArn("arn:aws:s3:::floci-firehose-results");+ firehoseService.createDeliveryStream("eu-west-1", "000000000000", "eu-named", s3, List.of(), null, null);+ firehoseService.putRecord("000000000000", "eu-west-1", "eu-named",+ new Record("one".getBytes(StandardCharsets.UTF_8)));++ firehoseService.flush("000000000000", "eu-west-1", "eu-named");++ verify(s3Service).createBucket("floci-firehose-results", "eu-west-1");+ }++ @Test+ void theSharedDefaultBucketStaysInTheDefaultRegion() {+ firehoseService.createDeliveryStream("eu-west-1", "000000000000", "eu-default", null, List.of(), null, null);+ firehoseService.putRecord("000000000000", "eu-west-1", "eu-default",+ new Record("one".getBytes(StandardCharsets.UTF_8)));++ firehoseService.flush("000000000000", "eu-west-1", "eu-default");++ verify(s3Service).createBucket("floci-firehose-results", "us-east-1");+ }+@Testvoid sameNameIsIndependentForEachAccountRegionDimension() {List<String[]> owners = List.of(