Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix: create Firehose buckets and backfill EKS OIDC issuers in the resource's region (#5099)

Hector Ventura4d9585583ddf
diff --git a/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java b/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java
index 1485c5f16a..91cb497c01 100644
--- a/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/eks/EksService.java
@@ -293,7 +293,9 @@ public class EksService implements TagHandler, ResourceProvider {
cluster.getIdentity().getOidc().getIssuer());
continue;
}
- String issuer = oidcService.newIssuerUrl(config.defaultRegion());
+ // The cluster's own region, from its ARN: the issuer must match the one create mints there.
+ String issuer = oidcService.newIssuerUrl(
+ AwsArnUtils.regionOrDefault(cluster.getArn(), config.defaultRegion()));
cluster.setIdentity(new ClusterIdentity(new OidcIdentity(issuer)));
oidcService.ensureKeyForAccount(accountId, cluster.getName(), issuer);
putClusterForAccount(accountId, cluster);
diff --git a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
index 137aae6424..447652bf61 100644
--- a/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/firehose/FirehoseService.java
@@ -901,7 +901,7 @@ public class FirehoseService implements ResourceProvider {
S3Destination s3 = stream.s3Destination();
List<byte[]> records = toFlush;
if (s3 != null && s3.isProcessingEnabled()) {
- ensureBucket(bucket);
+ ensureBucket(bucket, stream);
FirehoseLambdaTransformer.Outcome transformed =
lambdaTransformer.transform(stream, bucket, records, clock.instant());
LOG.infov("Transformed {0} records from stream {1} ({2} dropped, {3} failed)",
@@ -916,7 +916,7 @@ public class FirehoseService implements ResourceProvider {
}
}
if (s3 != null && s3.isDataFormatConversionEnabled()) {
- ensureBucket(bucket);
+ ensureBucket(bucket, stream);
FirehoseParquetConverter.Outcome outcome =
parquetConverter.deliver(stream, bucket, records, clock.instant());
LOG.infov("Converted {0} records ({1} failed) from stream {2} to s3://{3}/{4}",
@@ -930,7 +930,7 @@ public class FirehoseService implements ResourceProvider {
String key = S3ObjectKeyResolver.resolveKey(s3, stream.getDeliveryStreamName(),
stream.getVersionId(), clock.instant(), compression);
- ensureBucket(bucket);
+ ensureBucket(bucket, stream);
// Records are arbitrary bytes, so they are concatenated as bytes: routing
// them through a String would corrupt any payload that is not valid UTF-8.
@@ -993,10 +993,22 @@ public class FirehoseService implements ResourceProvider {
return DEFAULT_BUCKET;
}
- private void ensureBucket(String bucket) {
+ /**
+ * Creates a missing destination bucket. A stream's own destination goes in the stream's region,
+ * from its ARN; the shared default bucket, used by every stream without one, stays in the default
+ * region so the first stream to deliver does not decide it for all the others.
+ */
+ private void ensureBucket(String bucket, DeliveryStreamDescription stream) {
+ S3Destination s3 = stream.s3Destination();
+ boolean ownDestination = s3 != null && s3.bucketName() != null;
+ String region = ownDestination
+ ? AwsArnUtils.regionOrDefault(stream.getDeliveryStreamARN(), regionResolver.getDefaultRegion())
+ : regionResolver.getDefaultRegion();
try {
- s3Service.createBucket(bucket, regionResolver.getDefaultRegion());
- } catch (Exception ignored) {}
+ s3Service.createBucket(bucket, region);
+ } catch (Exception ignored) {
+ // The bucket usually exists already; a real problem with it surfaces on the write that follows.
+ }
}
// ─── Resource Explorer 2 ───────────────────────────────────────────────────
diff --git a/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java
index 8367668774..7c472c50f9 100644
--- a/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/eks/EksServiceTest.java
@@ -276,6 +276,28 @@ class EksServiceTest {
assertTrue(oidcService.findVerificationKey(issuer).isPresent());
}
+ @Test
+ void initBackfillsTheOidcIssuerInTheClustersOwnRegion() {
+ StorageBackend<String, Cluster> clusterStore = new InMemoryStorage<>();
+ StorageBackend<String, ClusterOidcKey> keyStore = new InMemoryStorage<>();
+
+ Cluster legacy = new Cluster();
+ legacy.setName("eu-legacy-cluster");
+ legacy.setArn("arn:aws:eks:eu-west-1:000000000000:cluster/eu-legacy-cluster");
+ legacy.setStatus(ClusterStatus.ACTIVE);
+ clusterStore.put("eu-legacy-cluster", legacy);
+
+ EksOidcService oidcService = new EksOidcService(
+ fixedStorageFactory(keyStore), new ObjectMapper());
+ EksService restarted = new EksService(fixedStorageFactory(clusterStore), testConfig(),
+ new RegionResolver("us-east-1", "000000000000"), null, null, oidcService,
+ mock(EksAccessEntryService.class), mock(EksPodIdentityAssociationService.class));
+ restarted.init();
+
+ String issuer = clusterStore.get("eu-legacy-cluster").orElseThrow().getIdentity().getOidc().getIssuer();
+ assertTrue(issuer.startsWith("https://oidc.eks.eu-west-1.amazonaws.com/id/"), issuer);
+ }
+
@Test
void initLeavesAnExistingOidcIssuerUnchanged() {
StorageBackend<String, Cluster> clusterStore = new InMemoryStorage<>();
diff --git a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java
index 52eec55d0f..5545a5ca9b 100644
--- a/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/firehose/FirehoseServiceTest.java
@@ -248,6 +248,43 @@ class FirehoseServiceTest {
.getDeliveryStreamName());
}
+ @Test
+ void aMissingDestinationBucketIsCreatedInTheStreamsRegion() {
+ S3Destination s3 = new S3Destination();
+ s3.setBucketArn("arn:aws:s3:::eu-stream-bucket");
+ firehoseService.createDeliveryStream("eu-west-1", "000000000000", "eu-stream", s3, List.of(), null, null);
+ firehoseService.putRecord("000000000000", "eu-west-1", "eu-stream",
+ new Record("one".getBytes(StandardCharsets.UTF_8)));
+
+ firehoseService.flush("000000000000", "eu-west-1", "eu-stream");
+
+ verify(s3Service).createBucket("eu-stream-bucket", "eu-west-1");
+ }
+
+ @Test
+ void aDestinationNamedLikeTheDefaultBucketFollowsItsStream() {
+ S3Destination s3 = new S3Destination();
+ s3.setBucketArn("arn:aws:s3:::floci-firehose-results");
+ firehoseService.createDeliveryStream("eu-west-1", "000000000000", "eu-named", s3, List.of(), null, null);
+ firehoseService.putRecord("000000000000", "eu-west-1", "eu-named",
+ new Record("one".getBytes(StandardCharsets.UTF_8)));
+
+ firehoseService.flush("000000000000", "eu-west-1", "eu-named");
+
+ verify(s3Service).createBucket("floci-firehose-results", "eu-west-1");
+ }
+
+ @Test
+ void theSharedDefaultBucketStaysInTheDefaultRegion() {
+ firehoseService.createDeliveryStream("eu-west-1", "000000000000", "eu-default", null, List.of(), null, null);
+ firehoseService.putRecord("000000000000", "eu-west-1", "eu-default",
+ new Record("one".getBytes(StandardCharsets.UTF_8)));
+
+ firehoseService.flush("000000000000", "eu-west-1", "eu-default");
+
+ verify(s3Service).createBucket("floci-firehose-results", "us-east-1");
+ }
+
@Test
void sameNameIsIndependentForEachAccountRegionDimension() {
List<String[]> owners = List.of(