Light, fluffy, and always free - The AWS Local Emulator alternative
feat(msk): support SCRAM secret associations (#6221)
Isak Lindbeck756e72273ce0
diff --git a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/MskTest.java b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/MskTest.javaindex b9736ce86d..79a81653de 100644--- a/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/MskTest.java+++ b/compatibility-tests/sdk-test-java/src/test/java/com/floci/test/MskTest.java@@ -1,10 +1,16 @@package com.floci.test;+import org.jboss.logging.Logger;import org.junit.jupiter.api.*;import software.amazon.awssdk.services.kafka.KafkaClient;import software.amazon.awssdk.services.kafka.model.*;+import software.amazon.awssdk.services.kms.KmsClient;+import software.amazon.awssdk.services.kms.model.CreateKeyRequest;+import software.amazon.awssdk.services.kms.model.NotFoundException;+import software.amazon.awssdk.services.secretsmanager.SecretsManagerClient;+import software.amazon.awssdk.services.secretsmanager.model.CreateSecretResponse;+import software.amazon.awssdk.services.secretsmanager.model.ResourceNotFoundException;-import java.util.List;import java.util.Map;import static org.assertj.core.api.Assertions.assertThat;@@ -13,8 +19,11 @@ import static org.assertj.core.api.Assertions.assertThat;@TestMethodOrder(MethodOrderer.OrderAnnotation.class)class MskTest {+ private static final Logger LOG = Logger.getLogger(MskTest.class);private static KafkaClient kafka;private static String clusterArn;+ private static String scramSecretArn;+ private static String scramKeyId;private static final String CLUSTER_NAME = TestFixtures.uniqueName("msk-cluster");@BeforeAll@@ -32,6 +41,24 @@ class MskTest {}kafka.close();}+ if (scramSecretArn != null) {+ try (SecretsManagerClient secrets = TestFixtures.secretsManagerClient()) {+ secrets.deleteSecret(request -> request.secretId(scramSecretArn).forceDeleteWithoutRecovery(true));+ } catch (ResourceNotFoundException expected) {+ // An already-absent secret needs no further cleanup.+ } catch (Exception e) {+ LOG.warnv(e, "Failed to delete SCRAM secret {0} during test cleanup", scramSecretArn);+ }+ }+ if (scramKeyId != null) {+ try (KmsClient kms = TestFixtures.kmsClient()) {+ kms.scheduleKeyDeletion(request -> request.keyId(scramKeyId).pendingWindowInDays(7));+ } catch (NotFoundException expected) {+ // An already-absent key needs no further cleanup.+ } catch (Exception e) {+ LOG.warnv(e, "Failed to schedule deletion of SCRAM KMS key {0} during test cleanup", scramKeyId);+ }+ }}@Test@@ -143,6 +170,47 @@ class MskTest {@Test@Order(6)+ void batchAssociateScramSecret() {+ try (SecretsManagerClient secrets = TestFixtures.secretsManagerClient();+ KmsClient kms = TestFixtures.kmsClient()) {+ scramKeyId = kms.createKey(CreateKeyRequest.builder().build()).keyMetadata().keyId();+ CreateSecretResponse secret = secrets.createSecret(request -> request+ .name("AmazonMSK_" + TestFixtures.uniqueName("scram"))+ .kmsKeyId(scramKeyId)+ .secretString("{\"username\":\"alice\",\"password\":\"password\"}"));+ scramSecretArn = secret.arn();++ BatchAssociateScramSecretResponse response = kafka.batchAssociateScramSecret(request -> request+ .clusterArn(clusterArn).secretArnList(scramSecretArn));++ assertThat(response.clusterArn()).isEqualTo(clusterArn);+ assertThat(response.sdkHttpResponse().statusCode()).isEqualTo(200);+ assertThat(response.unprocessedScramSecrets()).isEmpty();+ }+ }++ @Test+ @Order(7)+ void listScramSecrets() {+ ListScramSecretsResponse response = kafka.listScramSecrets(request -> request.clusterArn(clusterArn));++ assertThat(response.sdkHttpResponse().statusCode()).isEqualTo(200);+ assertThat(response.secretArnList()).containsExactly(scramSecretArn);+ }++ @Test+ @Order(8)+ void batchDisassociateScramSecret() {+ BatchDisassociateScramSecretResponse response = kafka.batchDisassociateScramSecret(request -> request+ .clusterArn(clusterArn).secretArnList(scramSecretArn));++ assertThat(response.clusterArn()).isEqualTo(clusterArn);+ assertThat(response.sdkHttpResponse().statusCode()).isEqualTo(200);+ assertThat(response.unprocessedScramSecrets()).isEmpty();+ }++ @Test+ @Order(9)void deleteCluster() {DeleteClusterResponse response = kafka.deleteCluster(DeleteClusterRequest.builder().clusterArn(clusterArn)diff --git a/docs/services/msk.md b/docs/services/msk.mdindex 1210486a81..f92b9188cc 100644--- a/docs/services/msk.md+++ b/docs/services/msk.md@@ -17,6 +17,9 @@ Floci emulates Amazon MSK by orchestrating **Redpanda** containers. This provide| `DescribeClusterV2` | Get cluster metadata and state using V2 API || `DeleteCluster` | Stops and removes the Redpanda container || `GetBootstrapBrokers` | Get the connection strings for the cluster |+| `BatchAssociateScramSecret` | Validate Secrets Manager credentials and store secret associations |+| `ListScramSecrets` | List the secret ARNs associated with a cluster |+| `BatchDisassociateScramSecret` | Remove secret associations || `CreateConfiguration` | Create a broker configuration (`server.properties`) || `ListConfigurations` | List all configurations || `DescribeConfiguration` | Get configuration metadata and latest revision |@@ -112,6 +115,18 @@ Cluster and configuration tags are managed through `/v1/tags/{resourceArn}`Configuration tags are reachable only through the tag endpoints, because AWS's`DescribeConfiguration` response has no tags member.+### SCRAM Secrets Association++`BatchAssociateScramSecret` accepts up to 10 Secrets Manager ARNs and stores their associations.+Secrets must belong to the cluster's account, region and partition, use the `AmazonMSK_`+name prefix and a customer-managed symmetric KMS key ID, not an `alias/...` reference.+The secrets must contain JSON such as `{"username":"alice","password":"pw"}`.++`ListScramSecrets` supports pagination. `BatchDisassociateScramSecret` removes up to+10 associations. Item failures appear in+`unprocessedScramSecrets`; reassociation and removal of an unassociated ARN return+`InvalidSecretArn`.+## Examples```bash@@ -132,6 +147,12 @@ aws kafka list-clusters --endpoint-url $AWS_ENDPOINT_URLCLUSTER_ARN=$(aws kafka list-clusters --query 'ClusterInfoList[0].ClusterArn' --output text --endpoint-url $AWS_ENDPOINT_URL)aws kafka get-bootstrap-brokers --cluster-arn $CLUSTER_ARN --endpoint-url $AWS_ENDPOINT_URL+# Associate an existing secret+aws kafka batch-associate-scram-secret \+ --cluster-arn "$CLUSTER_ARN" \+ --secret-arn-list "arn:aws:secretsmanager:us-east-1:000000000000:secret:AmazonMSK_my-user" \+ --endpoint-url $AWS_ENDPOINT_URL+# Delete a clusteraws kafka delete-cluster --cluster-arn $CLUSTER_ARN --endpoint-url $AWS_ENDPOINT_URL```diff --git a/src/main/java/io/github/hectorvent/floci/services/msk/MskController.java b/src/main/java/io/github/hectorvent/floci/services/msk/MskController.javaindex af2ce3f7a8..87897af919 100644--- a/src/main/java/io/github/hectorvent/floci/services/msk/MskController.java+++ b/src/main/java/io/github/hectorvent/floci/services/msk/MskController.java@@ -30,10 +30,12 @@ public class MskController {private static final String PROVISIONED_CLUSTER_TYPE = "PROVISIONED";private final MskService mskService;+ private final MskScramSecretsService mskScramSecretsService;@Inject- public MskController(MskService mskService) {+ public MskController(MskService mskService, MskScramSecretsService mskScramSecretsService) {this.mskService = mskService;+ this.mskScramSecretsService = mskScramSecretsService;}@POST@@ -96,6 +98,43 @@ public class MskController {return Response.ok(Map.of("bootstrapBrokerString", bootstrapBrokers)).build();}+ @POST+ @Path("/v1/clusters/{clusterArn}/scram-secrets")+ public Response batchAssociateScramSecret(@PathParam("clusterArn") String clusterArn,+ Map<String, Object> request) {+ List<String> secretArns = request == null ? null+ : asStringList(request.get("secretArnList"), "secretArnList");+ List<Map<String, String>> unprocessedScramSecrets =+ mskScramSecretsService.batchAssociateScramSecret(clusterArn, secretArns);+ return Response.ok(Map.of("clusterArn", clusterArn, "unprocessedScramSecrets", unprocessedScramSecrets)).build();+ }++ @GET+ @Path("/v1/clusters/{clusterArn}/scram-secrets")+ public Response listScramSecrets(@PathParam("clusterArn") String clusterArn,+ @QueryParam("maxResults") String maxResults,+ @QueryParam("nextToken") String nextToken) {+ PaginatedResult<String> result = mskScramSecretsService.listScramSecrets(clusterArn,+ Pagination.parseMaxResults(maxResults, "BadRequestException"), nextToken);+ Map<String, Object> response = new HashMap<>();+ response.put("secretArnList", result.items());+ if (result.nextToken() != null) {+ response.put("nextToken", result.nextToken());+ }+ return Response.ok(response).build();+ }++ @PATCH+ @Path("/v1/clusters/{clusterArn}/scram-secrets")+ public Response batchDisassociateScramSecret(@PathParam("clusterArn") String clusterArn,+ Map<String, Object> request) {+ List<String> secretArns = request == null ? null+ : asStringList(request.get("secretArnList"), "secretArnList");+ List<Map<String, String>> unprocessedScramSecrets =+ mskScramSecretsService.batchDisassociateScramSecret(clusterArn, secretArns);+ return Response.ok(Map.of("clusterArn", clusterArn, "unprocessedScramSecrets", unprocessedScramSecrets)).build();+ }+// ── Configurations ───────────────────────────────────────────────────────@POSTdiff --git a/src/main/java/io/github/hectorvent/floci/services/msk/MskScramSecretsService.java b/src/main/java/io/github/hectorvent/floci/services/msk/MskScramSecretsService.javanew file mode 100644index 0000000000..cda8220c68--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/msk/MskScramSecretsService.java@@ -0,0 +1,297 @@+package io.github.hectorvent.floci.services.msk;++import com.fasterxml.jackson.core.JsonProcessingException;+import com.fasterxml.jackson.core.type.TypeReference;+import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.AwsArnUtils;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.PaginatedResult;+import io.github.hectorvent.floci.core.common.Pagination;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.kms.KmsService;+import io.github.hectorvent.floci.services.kms.model.KmsKey;+import io.github.hectorvent.floci.services.kms.model.KmsKeySpec;+import io.github.hectorvent.floci.services.kms.model.KmsKeyUsage;+import io.github.hectorvent.floci.services.msk.model.MskCluster;+import io.github.hectorvent.floci.services.msk.model.MskScramAssociations;+import io.github.hectorvent.floci.services.msk.model.ScramSecretAssociation;+import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;+import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService.BatchGetSecretValueError;+import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService.BatchGetSecretValueResult;+import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService.BatchSecretValue;+import io.github.hectorvent.floci.services.secretsmanager.model.Secret;+import jakarta.annotation.PostConstruct;+import jakarta.annotation.PreDestroy;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.inject.Inject;+import org.jboss.logging.Logger;++import java.util.ArrayList;+import java.util.HashSet;+import java.util.List;+import java.util.Map;+import java.util.concurrent.Executors;+import java.util.concurrent.ScheduledExecutorService;+import java.util.concurrent.TimeUnit;++@ApplicationScoped+public class MskScramSecretsService {++ private static final Logger LOG = Logger.getLogger(MskScramSecretsService.class);+ private static final int MAX_PAGE = 100;++ private final AccountAwareStorageBackend<MskScramAssociations> storage;+ private final MskService mskService;+ private final SecretsManagerService secretsManagerService;+ private final KmsService kmsService;+ private final ObjectMapper objectMapper;+ private final ScheduledExecutorService reconciler = Executors.newSingleThreadScheduledExecutor();+ private final Object lock = new Object();++ @Inject+ public MskScramSecretsService(StorageFactory storageFactory, MskService mskService,+ SecretsManagerService secretsManagerService,+ KmsService kmsService, ObjectMapper objectMapper) {+ this.storage = storageFactory.create("msk", "msk-scram-associations.json",+ new TypeReference<Map<String, MskScramAssociations>>() {});+ this.mskService = mskService;+ this.secretsManagerService = secretsManagerService;+ this.kmsService = kmsService;+ this.objectMapper = objectMapper;+ }++ @PostConstruct+ public void init() {+ reconciler.scheduleWithFixedDelay(() -> {+ try {+ reconcileScramSecrets();+ } catch (RuntimeException e) {+ LOG.error("MSK SCRAM reconciliation failed; retrying on the next scheduled run", e);+ }+ }, 5, 5, TimeUnit.SECONDS);+ }++ @PreDestroy+ public void shutdown() {+ reconciler.shutdown();+ }++ void reconcileScramSecrets() {+ for (AccountAwareStorageBackend.AccountEntry<MskScramAssociations> entry+ : storage.scanAllAccountEntries(key -> true)) {+ synchronized (lock) {+ if (mskService.getClusterForAccount(entry.accountId(), entry.key()).isEmpty()) {+ storage.deleteForAccount(entry.accountId(), entry.key());+ }+ }+ }+ }++ public List<Map<String, String>> batchAssociateScramSecret(String clusterArn, List<String> secretArns) {+ synchronized (lock) {+ MskCluster cluster = scramCluster(clusterArn);+ validateSecretArns(secretArns);+ if (new HashSet<>(secretArns).size() != secretArns.size()) {+ throw new AwsException("BadRequestException", "secretArnList must not contain duplicate ARNs.", 400,+ Map.of("invalidParameter", "secretArnList"));+ }+ MskScramAssociations state = associationsFor(clusterArn);+ List<Map<String, String>> unprocessedScramSecrets = new ArrayList<>();+ for (String secretArn : secretArns) {+ boolean newAssociation = !state.getAssociations().containsKey(secretArn);+ try {+ Secret secret = validateSecretArn(clusterArn, secretArn);+ if (!newAssociation) {+ throw new AwsException("InvalidSecretArn", "The provided secret ARN is invalid.", 400);+ }+ validateSecretEncryption(secret);+ // Amazon MSK reads only the AWSCURRENT version of an associated secret+ BatchSecretValue secretValue = readSecret(secretArn);+ ScramSecretAssociation association = state.getAssociations()+ .computeIfAbsent(secretArn, ignored -> new ScramSecretAssociation());+ ScramCredentials credentials = readScramCredentials(secretValue.secretString());+ association.setUsername(credentials.username());+ putAssociations(clusterArn, state);+ } catch (AwsException e) {+ if (newAssociation && state.getAssociations().remove(secretArn) != null) {+ putAssociations(clusterArn, state);+ }+ unprocessedScramSecrets.add(scramFailure(secretArn, e));+ }+ }+ return unprocessedScramSecrets;+ }+ }++ public PaginatedResult<String> listScramSecrets(String clusterArn, Integer maxResults, String nextToken) {+ synchronized (lock) {+ scramCluster(clusterArn);+ return Pagination.paginate(new ArrayList<>(associationsFor(clusterArn).getAssociations().keySet()),+ arn -> arn, maxResults, nextToken, MAX_PAGE, "BadRequestException");+ }+ }++ public List<Map<String, String>> batchDisassociateScramSecret(String clusterArn, List<String> secretArns) {+ synchronized (lock) {+ scramCluster(clusterArn);+ validateSecretArns(secretArns);+ MskScramAssociations state = associationsFor(clusterArn);+ List<Map<String, String>> unprocessedScramSecrets = new ArrayList<>();+ for (String secretArn : secretArns) {+ try {+ if (state.getAssociations().remove(secretArn) == null) {+ throw new AwsException("InvalidSecretArn", "The provided secret ARN is invalid.", 400);+ }+ putAssociations(clusterArn, state);+ } catch (AwsException e) {+ unprocessedScramSecrets.add(scramFailure(secretArn, e));+ }+ }+ return unprocessedScramSecrets;+ }+ }++ private MskCluster scramCluster(String clusterArn) {+ MskCluster cluster = mskService.describeCluster(clusterArn);+ if (mskService.isServerless(cluster)) {+ throw new AwsException("BadRequestException",+ "SCRAM secrets are not supported for serverless clusters.", 400,+ Map.of("invalidParameter", "clusterArn"));+ }+ return cluster;+ }++ private void validateSecretArns(List<String> secretArns) {+ if (secretArns == null || secretArns.isEmpty() || secretArns.size() > 10+ || secretArns.stream().anyMatch(arn -> arn == null || arn.isBlank())) {+ throw new AwsException("BadRequestException",+ "secretArnList must contain between 1 and 10 secret ARNs.", 400,+ Map.of("invalidParameter", "secretArnList"));+ }+ }++ private Secret validateSecretArn(String clusterArn, String secretArn) {+ if (!AwsArnUtils.isArn(secretArn)) {+ throw new AwsException("BadRequestException", "Invalid secret ARN.", 400,+ Map.of("invalidParameter", "secretArnList"));+ }+ AwsArnUtils.Arn arn = AwsArnUtils.parse(secretArn);+ AwsArnUtils.Arn owner = AwsArnUtils.parse(clusterArn);+ if (!"secretsmanager".equals(arn.service()) || !arn.resource().startsWith("secret:")+ || !owner.partition().equals(arn.partition()) || !owner.region().equals(arn.region())+ || !owner.accountId().equals(arn.accountId())) {+ throw new AwsException("BadRequestException",+ "Secret must belong to the cluster's account, region and partition.", 400,+ Map.of("invalidParameter", "secretArnList"));+ }+ String secretName = arn.resource().substring("secret:".length());+ if (!secretName.startsWith("AmazonMSK_")) {+ throw new AwsException("BadRequestException",+ "The name of secrets associated with an Amazon MSK cluster must have the prefix AmazonMSK_.", 400,+ Map.of("invalidParameter", "secretArnList"));+ }+ Secret secret = secretsManagerService.describeSecret(secretArn, arn.region());+ if (!secretArn.equals(secret.getArn())) {+ throw new AwsException("InvalidSecretArn", "Amazon MSK wasn't able to access the provided secret.", 400);+ }+ return secret;+ }++ private void validateSecretEncryption(Secret secret) {+ String region = AwsArnUtils.parse(secret.getArn()).region();+ String keyId = secret.getKmsKeyId();+ boolean hasAliasPrefix = keyId != null && keyId.startsWith("alias/");+ boolean hasAliasArn = AwsArnUtils.isArnFor(keyId, "kms") && AwsArnUtils.parse(keyId).resource().startsWith("alias/");+ if (hasAliasPrefix || hasAliasArn) {+ throw new AwsException("BadRequestException",+ "SCRAM secrets must reference a KMS key ID or key ARN instead of an alias.", 400,+ Map.of("invalidParameter", "secretArnList"));+ }++ if (keyId != null && !keyId.isBlank()) {+ KmsKey key = kmsService.describeKey(keyId, region);+ if ("CUSTOMER".equals(key.getKeyManager())+ && key.getKeySpec() == KmsKeySpec.SYMMETRIC_DEFAULT+ && key.getKeyUsage() == KmsKeyUsage.ENCRYPT_DECRYPT) {+ return;+ }+ }+ throw new AwsException("BadRequestException",+ "SCRAM secrets must be encrypted with a customer-managed symmetric KMS key.", 400,+ Map.of("invalidParameter", "secretArnList"));+ }++ private Map<String, String> scramFailure(String secretArn, AwsException error) {+ return Map.of("secretArn", secretArn, "errorCode", error.getErrorCode(), "errorMessage", error.getMessage());+ }++ private MskScramAssociations associationsFor(String clusterArn) {+ AwsArnUtils.Arn arn = AwsArnUtils.parse(clusterArn);+ return storage.getForAccount(arn.accountId(), clusterArn).orElseGet(MskScramAssociations::new);+ }++ private void putAssociations(String clusterArn, MskScramAssociations associations) {+ AwsArnUtils.Arn arn = AwsArnUtils.parse(clusterArn);+ if (associations.getAssociations().isEmpty()) {+ storage.deleteForAccount(arn.accountId(), clusterArn);+ } else {+ storage.putForAccount(arn.accountId(), clusterArn, associations);+ }+ }++ private BatchSecretValue readSecret(String secretArn) {+ // Exact-ARN batch reads select AWSCURRENT without updating access timestamps.+ BatchGetSecretValueResult result = secretsManagerService.batchGetSecretValue(+ List.of(secretArn), AwsArnUtils.parse(secretArn).region());+ if (result != null && result.values() != null && result.errors() != null) {+ if (result.values().isEmpty() && result.errors().size() == 1) {+ BatchGetSecretValueError error = result.errors().getFirst();+ if (error != null && secretArn.equals(error.secretId())+ && error.errorCode() != null && error.message() != null) {+ int status = "AccessDeniedException".equals(error.errorCode()) ? 403 : 400;+ throw new AwsException(error.errorCode(), error.message(), status);+ }+ } else if (result.values().size() == 1 && result.errors().isEmpty()) {+ BatchSecretValue value = result.values().getFirst();+ if (value != null && secretArn.equals(value.arn()) && value.versionId() != null) {+ return value;+ }+ }+ }+ throw new AwsException("InternalServerError", "Unexpected Secrets Manager batch response.", 500);+ }++ record ScramCredentials(String username, String password) {+ }++ ScramCredentials readScramCredentials(String secretString) {+ if (secretString == null) {+ throw new AwsException("InternalServerError",+ "The request could not be processed because of an internal error. Try again.", 500);+ }+ JsonNode credentials;+ try {+ credentials = objectMapper.readTree(secretString);+ } catch (JsonProcessingException e) {+ throw new AwsException("InvalidSecretArn", "The provided secret has an invalid schema.", 400);+ }+ if (credentials != null && credentials.isNull()) {+ throw new AwsException("InternalServerError",+ "The request could not be processed because of an internal error. Try again.", 500);+ }+ if (credentials == null || !credentials.isObject()) {+ throw new AwsException("InvalidSecretArn", "The provided secret has an invalid schema.", 400);+ }+ JsonNode username = credentials.path("username");+ if ((!username.isTextual() && !username.isNumber()) || username.asText().isBlank()) {+ throw new AwsException("InvalidSecretArn", "The provided secret has an invalid schema.", 400);+ }+ JsonNode password = credentials.path("password");+ if ((!password.isTextual() && !password.isBoolean()) || password.asText().isBlank()) {+ throw new AwsException("InvalidSecretArn", "The provided secret has an invalid schema.", 400);+ }+ return new ScramCredentials(username.asText(), password.asText());+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/msk/MskService.java b/src/main/java/io/github/hectorvent/floci/services/msk/MskService.javaindex 5a64492002..108d85d187 100644--- a/src/main/java/io/github/hectorvent/floci/services/msk/MskService.java+++ b/src/main/java/io/github/hectorvent/floci/services/msk/MskService.java@@ -38,6 +38,7 @@ import java.util.ArrayList;import java.util.HashMap;import java.util.List;import java.util.Map;+import java.util.Optional;import java.util.Set;import java.util.UUID;import java.util.concurrent.Executors;@@ -647,6 +648,13 @@ public class MskService implements ResourceProvider {return storage.scan(k -> true);}+ public Optional<MskCluster> getClusterForAccount(String accountId, String clusterArn) {+ if (storage instanceof AccountAwareStorageBackend<MskCluster> aware) {+ return aware.getForAccount(accountId, clusterArn);+ }+ return storage.get(clusterArn);+ }+private void putCluster(MskCluster cluster) {if (cluster.getAccountId() != null && storage instanceof AccountAwareStorageBackend<MskCluster> aware) {aware.putForAccount(cluster.getAccountId(), cluster.getClusterArn(), cluster);diff --git a/src/main/java/io/github/hectorvent/floci/services/msk/model/MskScramAssociations.java b/src/main/java/io/github/hectorvent/floci/services/msk/model/MskScramAssociations.javanew file mode 100644index 0000000000..1304db64fc--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/msk/model/MskScramAssociations.java@@ -0,0 +1,20 @@+package io.github.hectorvent.floci.services.msk.model;++import io.quarkus.runtime.annotations.RegisterForReflection;++import java.util.LinkedHashMap;+import java.util.Map;++@RegisterForReflection+public class MskScramAssociations {++ private Map<String, ScramSecretAssociation> associations = new LinkedHashMap<>();++ public Map<String, ScramSecretAssociation> getAssociations() {+ return associations;+ }++ public void setAssociations(Map<String, ScramSecretAssociation> associations) {+ this.associations = associations;+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/msk/model/ScramSecretAssociation.java b/src/main/java/io/github/hectorvent/floci/services/msk/model/ScramSecretAssociation.javanew file mode 100644index 0000000000..668656e53f--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/msk/model/ScramSecretAssociation.java@@ -0,0 +1,18 @@+package io.github.hectorvent.floci.services.msk.model;++import io.quarkus.runtime.annotations.RegisterForReflection;++@RegisterForReflection+public class ScramSecretAssociation {++ private String username;++ public String getUsername() {+ return username;+ }++ public void setUsername(String username) {+ this.username = username;+ }++}diff --git a/src/test/java/io/github/hectorvent/floci/services/msk/MskControllerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/msk/MskControllerIntegrationTest.javaindex 19aff81555..39172da866 100644--- a/src/test/java/io/github/hectorvent/floci/services/msk/MskControllerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/msk/MskControllerIntegrationTest.java@@ -1,11 +1,14 @@package io.github.hectorvent.floci.services.msk;+import io.github.hectorvent.floci.testing.RestAssuredJsonUtils;import io.quarkus.test.junit.QuarkusTest;import io.restassured.path.json.JsonPath;import org.junit.jupiter.api.Test;import java.nio.charset.StandardCharsets;import java.util.Base64;+import java.util.List;+import java.util.Map;import java.util.UUID;import static io.restassured.RestAssured.given;@@ -1221,4 +1224,145 @@ class MskControllerIntegrationTest {.statusCode(400).body("message", containsString("Exactly one"));}+++ private String createScramKey() {+ return given()+ .contentType("application/x-amz-json-1.1")+ .header("X-Amz-Target", "TrentService.CreateKey")+ .body(Map.of())+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .extract().path("KeyMetadata.Arn");+ }++ @Test+ void batchAssociateScramSecret() {+ String clusterArn = createScramCluster();+ String secretArn = createScramSecret();++ given()+ .contentType("application/json")+ .body(Map.of("secretArnList", List.of(secretArn)))+ .when()+ .post("/v1/clusters/{clusterArn}/scram-secrets", clusterArn)+ .then()+ .statusCode(200)+ .body("clusterArn", equalTo(clusterArn))+ .body("unprocessedScramSecrets", empty());+ given()+ .when()+ .get("/v1/clusters/{clusterArn}/scram-secrets", clusterArn)+ .then()+ .statusCode(200)+ .body("secretArnList", equalTo(List.of(secretArn)));+ }++ @Test+ void batchAssociateScramSecretReportsUnprocessedSecret() {+ String clusterArn = createScramCluster();+ String secretArn = createScramSecret();+ associateScramSecret(clusterArn, secretArn);++ given().contentType("application/json")+ .body(Map.of("secretArnList", List.of(secretArn)))+ .when()+ .post("/v1/clusters/{clusterArn}/scram-secrets", clusterArn)+ .then()+ .statusCode(200)+ .body("unprocessedScramSecrets", hasSize(1))+ .body("unprocessedScramSecrets[0].secretArn", equalTo(secretArn))+ .body("unprocessedScramSecrets[0].errorCode", equalTo("InvalidSecretArn"));+ }++ @Test+ void listScramSecrets() {+ String clusterArn = createScramCluster();+ String secretArn = createScramSecret();+ associateScramSecret(clusterArn, secretArn);++ given()+ .get("/v1/clusters/{clusterArn}/scram-secrets", clusterArn)+ .then()+ .statusCode(200)+ .body("secretArnList", equalTo(List.of(secretArn)));+ }++ @Test+ void batchDisassociateScramSecret() {+ String clusterArn = createScramCluster();+ String secretArn = createScramSecret();+ associateScramSecret(clusterArn, secretArn);++ given().contentType("application/json")+ .body(Map.of("secretArnList", List.of(secretArn)))+ .when()+ .patch("/v1/clusters/{clusterArn}/scram-secrets", clusterArn)+ .then()+ .statusCode(200)+ .body("clusterArn", equalTo(clusterArn))+ .body("unprocessedScramSecrets", empty());+ given().get("/v1/clusters/{clusterArn}/scram-secrets", clusterArn)+ .then()+ .statusCode(200)+ .body("secretArnList", empty());+ }++ @Test+ void batchDisassociateScramSecretReportsUnassociatedSecret() {+ String clusterArn = createScramCluster();+ String secretArn = createScramSecret();++ given()+ .contentType("application/json")+ .body(Map.of("secretArnList", List.of(secretArn)))+ .when()+ .patch("/v1/clusters/{clusterArn}/scram-secrets", clusterArn)+ .then()+ .statusCode(200)+ .body("unprocessedScramSecrets", hasSize(1))+ .body("unprocessedScramSecrets[0].secretArn", equalTo(secretArn))+ .body("unprocessedScramSecrets[0].errorCode", equalTo("InvalidSecretArn"));+ }++ private String createScramCluster() {+ return given()+ .contentType("application/json")+ .body(Map.of("clusterName", "scram-" + UUID.randomUUID()))+ .when()+ .post("/v1/clusters")+ .then()+ .statusCode(200)+ .extract().path("clusterArn");+ }++ private String createScramSecret() {+ RestAssuredJsonUtils.configureAwsContentTypes();+ return given()+ .contentType("application/x-amz-json-1.1")+ .header("X-Amz-Target", "secretsmanager.CreateSecret")+ .body(Map.of(+ "Name", "AmazonMSK_" + UUID.randomUUID(),+ "KmsKeyId", createScramKey(),+ "SecretString", "{\"username\":\"alice\",\"password\":\"password\"}")+ )+ .when()+ .post("/")+ .then()+ .statusCode(200)+ .extract().path("ARN");+ }++ private void associateScramSecret(String clusterArn, String secretArn) {+ given()+ .contentType("application/json")+ .body(Map.of("secretArnList", List.of(secretArn)))+ .when()+ .post("/v1/clusters/{clusterArn}/scram-secrets", clusterArn)+ .then()+ .statusCode(200)+ .body("unprocessedScramSecrets", empty());+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/msk/MskControllerTest.java b/src/test/java/io/github/hectorvent/floci/services/msk/MskControllerTest.javaindex 14fb79db77..55a78d04ff 100644--- a/src/test/java/io/github/hectorvent/floci/services/msk/MskControllerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/msk/MskControllerTest.java@@ -25,6 +25,7 @@ import static org.mockito.Mockito.when;class MskControllerTest {private MskService mskService;+ private MskScramSecretsService mskScramSecretsService;private MskController controller;// A configuration exactly as the pre-revision-history schema persisted it - the one case@@ -45,7 +46,8 @@ class MskControllerTest {@BeforeEachvoid setUp() {mskService = Mockito.mock(MskService.class);- controller = new MskController(mskService);+ mskScramSecretsService = Mockito.mock(MskScramSecretsService.class);+ controller = new MskController(mskService, mskScramSecretsService);}private static MskConfiguration legacyConfiguration() throws Exception {diff --git a/src/test/java/io/github/hectorvent/floci/services/msk/MskScramServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/msk/MskScramServiceTest.javanew file mode 100644index 0000000000..125a2edfe4--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/msk/MskScramServiceTest.java@@ -0,0 +1,259 @@+package io.github.hectorvent.floci.services.msk;++import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.kms.KmsService;+import io.github.hectorvent.floci.services.kms.model.KmsKey;+import io.github.hectorvent.floci.services.msk.model.MskCluster;+import io.github.hectorvent.floci.services.msk.model.MskScramAssociations;+import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;+import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService.BatchGetSecretValueResult;+import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService.BatchSecretValue;+import io.github.hectorvent.floci.services.secretsmanager.model.Secret;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;++import java.util.List;+import java.util.Map;+import java.util.Optional;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.clearInvocations;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.never;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.verifyNoInteractions;+import static org.mockito.Mockito.when;++class MskScramServiceTest {++ private static final String ACCOUNT = "000000000000";+ private static final String CLUSTER = "arn:aws:kafka:us-east-1:000000000000:cluster/scram/id";+ private static final String SECRET =+ "arn:aws:secretsmanager:us-east-1:000000000000:secret:AmazonMSK_alice";+ private AccountAwareStorageBackend<MskScramAssociations> associations;+ private SecretsManagerService secrets;+ private KmsService kms;+ private MskService clusters;+ private MskScramSecretsService service;++ @BeforeEach+ void setUp() {+ associations = AccountAwareStorageBackend.inMemory(ACCOUNT);+ StorageFactory storageFactory = mock(StorageFactory.class);+ when(storageFactory.<MskScramAssociations>create(+ eq("msk"), eq("msk-scram-associations.json"), any())).thenReturn(associations);+ clusters = mock(MskService.class);+ MskCluster cluster = new MskCluster();+ cluster.setClusterArn(CLUSTER);+ when(clusters.describeCluster(CLUSTER)).thenReturn(cluster);+ when(clusters.getClusterForAccount(ACCOUNT, CLUSTER)).thenReturn(Optional.of(cluster));+ secrets = mock(SecretsManagerService.class);+ when(secrets.describeSecret(anyString(), eq("us-east-1"))).thenAnswer(invocation -> {+ Secret metadata = new Secret();+ metadata.setArn(invocation.getArgument(0));+ metadata.setKmsKeyId("customer-key");+ return metadata;+ });+ kms = mock(KmsService.class);+ when(kms.describeKey("customer-key", "us-east-1")).thenReturn(new KmsKey());+ service = new MskScramSecretsService(storageFactory, clusters, secrets, kms, new ObjectMapper());+ secretValue(SECRET, "{\"username\":\"alice\",\"password\":\"password\"}");+ }++ private void secretValue(String arn, String value) {+ when(secrets.batchGetSecretValue(List.of(arn), "us-east-1")).thenReturn(+ new BatchGetSecretValueResult(List.of(new BatchSecretValue(+ arn, "AmazonMSK_fixture", value, null, "version", List.of("AWSCURRENT"), null)), List.of()));+ }++ @Test+ void reconciliationRemovesOnlyDeletedClusterAssociationsAcrossAccounts() {+ service.batchAssociateScramSecret(CLUSTER, List.of(SECRET));+ String otherAccount = "111111111111";+ String deletedCluster = CLUSTER.replace(ACCOUNT, otherAccount);+ associations.putForAccount(otherAccount, deletedCluster, new MskScramAssociations());+ when(clusters.getClusterForAccount(otherAccount, deletedCluster)).thenReturn(Optional.empty());++ service.reconcileScramSecrets();++ assertTrue(associations.getForAccount(ACCOUNT, CLUSTER).isPresent());+ assertTrue(associations.getForAccount(otherAccount, deletedCluster).isEmpty());+ verify(clusters).getClusterForAccount(otherAccount, deletedCluster);+ }++ @Test+ void reconciliationPreservesAssociationsWhenClusterLookupFails() {+ service.batchAssociateScramSecret(CLUSTER, List.of(SECRET));+ AwsException failure = new AwsException("InternalServerErrorException", "Lookup failed.", 500);+ when(clusters.getClusterForAccount(ACCOUNT, CLUSTER)).thenThrow(failure);++ assertEquals(failure, assertThrows(AwsException.class, () -> service.reconcileScramSecrets()));+ assertTrue(associations.getForAccount(ACCOUNT, CLUSTER).isPresent());+ }++ @Test+ void associatesListsAndDisassociatesMetadata() {+ assertTrue(service.batchAssociateScramSecret(CLUSTER, List.of(SECRET)).isEmpty());+ assertEquals(List.of(SECRET), service.listScramSecrets(CLUSTER, null, null).items());+ assertEquals("alice", associations.getForAccount(ACCOUNT, CLUSTER).orElseThrow()+ .getAssociations().get(SECRET).getUsername());++ assertTrue(service.batchDisassociateScramSecret(CLUSTER, List.of(SECRET)).isEmpty());+ assertTrue(service.listScramSecrets(CLUSTER, null, null).items().isEmpty());+ assertTrue(associations.getForAccount(ACCOUNT, CLUSTER).isEmpty());+ }++ @ParameterizedTest+ @ValueSource(strings = {"alias/msk-scram",+ "arn:aws:kms:us-east-1:000000000000:alias/msk-scram"})+ void rejectsKmsAliasReferencesBeforeKeyLookup(String keyReference) {+ Secret metadata = new Secret();+ metadata.setArn(SECRET);+ metadata.setKmsKeyId(keyReference);+ when(secrets.describeSecret(SECRET, "us-east-1")).thenReturn(metadata);++ List<Map<String, String>> failures = service.batchAssociateScramSecret(CLUSTER, List.of(SECRET));++ assertEquals("BadRequestException", failures.getFirst().get("errorCode"));+ assertTrue(service.listScramSecrets(CLUSTER, null, null).items().isEmpty());+ verifyNoInteractions(kms);+ verify(secrets, never()).batchGetSecretValue(any(), anyString());+ }++ @ParameterizedTest+ @ValueSource(strings = {"12345678-1234-1234-1234-123456789012",+ "arn:aws:kms:us-east-1:000000000000:key/12345678-1234-1234-1234-123456789012"})+ void acceptsKmsKeyIdsAndKeyArns(String keyReference) {+ Secret metadata = new Secret();+ metadata.setArn(SECRET);+ metadata.setKmsKeyId(keyReference);+ when(secrets.describeSecret(SECRET, "us-east-1")).thenReturn(metadata);+ when(kms.describeKey(keyReference, "us-east-1")).thenReturn(new KmsKey());++ assertTrue(service.batchAssociateScramSecret(CLUSTER, List.of(SECRET)).isEmpty());++ verify(kms).describeKey(keyReference, "us-east-1");+ assertEquals(List.of(SECRET), service.listScramSecrets(CLUSTER, null, null).items());+ }++ @Test+ void rejectsPartialSecretArnForAssociation() {+ String fullArn = SECRET + "-Ab12Cd";+ Secret metadata = new Secret();+ metadata.setArn(fullArn);+ metadata.setKmsKeyId("customer-key");+ when(secrets.describeSecret(SECRET, "us-east-1")).thenReturn(metadata);++ List<Map<String, String>> failures = service.batchAssociateScramSecret(CLUSTER, List.of(SECRET));++ assertEquals(List.of(Map.of("secretArn", SECRET, "errorCode", "InvalidSecretArn",+ "errorMessage", "Amazon MSK wasn't able to access the provided secret.")), failures);+ assertTrue(service.listScramSecrets(CLUSTER, null, null).items().isEmpty());+ verifyNoInteractions(kms);+ verify(secrets, never()).batchGetSecretValue(any(), anyString());+ secretValue(fullArn, "{\"username\":\"alice\",\"password\":\"password\"}");+ assertTrue(service.batchAssociateScramSecret(CLUSTER, List.of(fullArn)).isEmpty());+ assertEquals(List.of(fullArn), service.listScramSecrets(CLUSTER, null, null).items());+ }++ @Test+ void rejectsPartialSecretArnForDisassociation() {+ String fullArn = SECRET + "-Ab12Cd";+ secretValue(fullArn, "{\"username\":\"alice\",\"password\":\"password\"}");+ assertTrue(service.batchAssociateScramSecret(CLUSTER, List.of(fullArn)).isEmpty());+ clearInvocations(secrets);++ List<Map<String, String>> failures = service.batchDisassociateScramSecret(CLUSTER, List.of(SECRET));++ assertEquals(List.of(Map.of("secretArn", SECRET, "errorCode", "InvalidSecretArn",+ "errorMessage", "The provided secret ARN is invalid.")), failures);+ assertEquals(List.of(fullArn), service.listScramSecrets(CLUSTER, null, null).items());+ assertTrue(service.batchDisassociateScramSecret(CLUSTER, List.of(fullArn)).isEmpty());+ verifyNoInteractions(secrets);+ }++ @Test+ void disassociatesDeletedSecretWithoutLookingItUp() {+ assertTrue(service.batchAssociateScramSecret(CLUSTER, List.of(SECRET)).isEmpty());+ when(secrets.describeSecret(SECRET, "us-east-1"))+ .thenThrow(new AwsException("ResourceNotFoundException", "Secret not found.", 400));+ clearInvocations(secrets);++ assertTrue(service.batchDisassociateScramSecret(CLUSTER, List.of(SECRET)).isEmpty());++ assertTrue(service.listScramSecrets(CLUSTER, null, null).items().isEmpty());+ assertTrue(associations.getForAccount(ACCOUNT, CLUSTER).isEmpty());+ verifyNoInteractions(secrets);+ }++ @Test+ void rejectsReturnedSecretArnForDifferentSecret() {+ when(secrets.batchGetSecretValue(List.of(SECRET), "us-east-1"))+ .thenReturn(+ new BatchGetSecretValueResult(List.of(new BatchSecretValue(+ SECRET + "Other", "AmazonMSK_fixture", "{\"username\":\"alice\",\"password\":\"password\"}",+ null, "version", List.of("AWSCURRENT"), null)), List.of())+ );++ List<Map<String, String>> failures = service.batchAssociateScramSecret(CLUSTER, List.of(SECRET));++ assertEquals("InternalServerError", failures.getFirst().get("errorCode"));+ assertTrue(service.listScramSecrets(CLUSTER, null, null).items().isEmpty());+ }++ @Test+ void reportsItemFailuresWithoutBlockingValidSecrets() {+ String invalid = SECRET + "-invalid";+ secretValue(invalid, "{}");++ List<Map<String, String>> failures = service.batchAssociateScramSecret(CLUSTER, List.of(invalid, SECRET));++ assertEquals(1, failures.size());+ assertEquals(invalid, failures.getFirst().get("secretArn"));+ assertEquals("InvalidSecretArn", failures.getFirst().get("errorCode"));+ assertEquals(List.of(SECRET), service.listScramSecrets(CLUSTER, null, null).items());+ }++ @Test+ void rejectsRepeatedAssociationAndDisassociation() {+ service.batchAssociateScramSecret(CLUSTER, List.of(SECRET));+ assertEquals("InvalidSecretArn", service.batchAssociateScramSecret(CLUSTER, List.of(SECRET))+ .getFirst().get("errorCode"));+ service.batchDisassociateScramSecret(CLUSTER, List.of(SECRET));+ assertEquals("InvalidSecretArn", service.batchDisassociateScramSecret(CLUSTER, List.of(SECRET))+ .getFirst().get("errorCode"));+ }++ @Test+ void rejectsDuplicateAssociationBatchBeforeStoringMetadata() {+ AwsException error = assertThrows(AwsException.class,+ () -> service.batchAssociateScramSecret(CLUSTER, List.of(SECRET, SECRET)));++ assertEquals("BadRequestException", error.getErrorCode());+ assertTrue(service.listScramSecrets(CLUSTER, null, null).items().isEmpty());+ }++ @Test+ void listsPersistedAssociationsAfterServiceRecreation() {+ service.batchAssociateScramSecret(CLUSTER, List.of(SECRET));+ StorageFactory storageFactory = mock(StorageFactory.class);+ when(storageFactory.<MskScramAssociations>create(+ eq("msk"), eq("msk-scram-associations.json"), any())).thenReturn(associations);+ MskService clusters = mock(MskService.class);+ when(clusters.describeCluster(CLUSTER)).thenReturn(new MskCluster());+ MskScramSecretsService restored = new MskScramSecretsService(+ storageFactory, clusters, secrets, mock(KmsService.class), new ObjectMapper());++ assertEquals(List.of(SECRET), restored.listScramSecrets(CLUSTER, null, null).items());+ }+}