Light, fluffy, and always free - The AWS Local Emulator alternative
feat(msk): apply SCRAM credentials in redpanda cluster (#6353)
Isak Lindbeck6914401edecc
diff --git a/docs/services/msk.md b/docs/services/msk.mdindex f92b9188cc..5ce00f86da 100644--- a/docs/services/msk.md+++ b/docs/services/msk.md@@ -17,9 +17,9 @@ Floci emulates Amazon MSK by orchestrating **Redpanda** containers. This provide| `DescribeClusterV2` | Get cluster metadata and state using V2 API || `DeleteCluster` | Stops and removes the Redpanda container || `GetBootstrapBrokers` | Get the connection strings for the cluster |-| `BatchAssociateScramSecret` | Validate Secrets Manager credentials and store secret associations |+| `BatchAssociateScramSecret` | Associate secret SCRAM credential with cluster || `ListScramSecrets` | List the secret ARNs associated with a cluster |-| `BatchDisassociateScramSecret` | Remove secret associations |+| `BatchDisassociateScramSecret` | Remove secret SCRAM credential associated with cluster || `CreateConfiguration` | Create a broker configuration (`server.properties`) || `ListConfigurations` | List all configurations || `DescribeConfiguration` | Get configuration metadata and latest revision |@@ -117,7 +117,12 @@ Configuration tags are reachable only through the tag endpoints, because AWS's### SCRAM Secrets Association-`BatchAssociateScramSecret` accepts up to 10 Secrets Manager ARNs and stores their associations.+Manages credentials only: the broker does not enable SASL authentication or+validate these users when clients connect. Clients can still connect without+credentials. Authenticated SASL_SSL listeners is not implemented.++`BatchAssociateScramSecret` accepts up to 10 Secrets Manager ARNs and creates kafka+`SCRAM-SHA-512` users.Secrets must belong to the cluster's account, region and partition, use the `AmazonMSK_`name prefix and a customer-managed symmetric KMS key ID, not an `alias/...` reference.The secrets must contain JSON such as `{"username":"alice","password":"pw"}`.diff --git a/src/main/java/io/github/hectorvent/floci/services/msk/MskScramSecretsService.java b/src/main/java/io/github/hectorvent/floci/services/msk/MskScramSecretsService.javaindex cda8220c68..59f1d77335 100644--- a/src/main/java/io/github/hectorvent/floci/services/msk/MskScramSecretsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/msk/MskScramSecretsService.java@@ -4,6 +4,7 @@ import com.fasterxml.jackson.core.JsonProcessingException;import com.fasterxml.jackson.core.type.TypeReference;import com.fasterxml.jackson.databind.JsonNode;import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsArnUtils;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.PaginatedResult;@@ -32,6 +33,7 @@ import java.util.ArrayList;import java.util.HashSet;import java.util.List;import java.util.Map;+import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.Executors;import java.util.concurrent.ScheduledExecutorService;import java.util.concurrent.TimeUnit;@@ -43,20 +45,24 @@ public class MskScramSecretsService {private static final int MAX_PAGE = 100;private final AccountAwareStorageBackend<MskScramAssociations> storage;+ private final EmulatorConfig config;private final MskService mskService;+ private final RedpandaManager redpandaManager;private final SecretsManagerService secretsManagerService;private final KmsService kmsService;private final ObjectMapper objectMapper;private final ScheduledExecutorService reconciler = Executors.newSingleThreadScheduledExecutor();- private final Object lock = new Object();+ private final Map<String, Object> clusterLocks = new ConcurrentHashMap<>();@Inject- public MskScramSecretsService(StorageFactory storageFactory, MskService mskService,- SecretsManagerService secretsManagerService,+ public MskScramSecretsService(StorageFactory storageFactory, EmulatorConfig config, MskService mskService,+ RedpandaManager redpandaManager, SecretsManagerService secretsManagerService,KmsService kmsService, ObjectMapper objectMapper) {this.storage = storageFactory.create("msk", "msk-scram-associations.json",new TypeReference<Map<String, MskScramAssociations>>() {});+ this.config = config;this.mskService = mskService;+ this.redpandaManager = redpandaManager;this.secretsManagerService = secretsManagerService;this.kmsService = kmsService;this.objectMapper = objectMapper;@@ -78,10 +84,14 @@ public class MskScramSecretsService {reconciler.shutdown();}+ private Object clusterLock(String clusterArn) {+ return clusterLocks.computeIfAbsent(String.valueOf(clusterArn), ignored -> new Object());+ }+void reconcileScramSecrets() {for (AccountAwareStorageBackend.AccountEntry<MskScramAssociations> entry: storage.scanAllAccountEntries(key -> true)) {- synchronized (lock) {+ synchronized (clusterLock(entry.key())) {if (mskService.getClusterForAccount(entry.accountId(), entry.key()).isEmpty()) {storage.deleteForAccount(entry.accountId(), entry.key());}@@ -90,7 +100,7 @@ public class MskScramSecretsService {}public List<Map<String, String>> batchAssociateScramSecret(String clusterArn, List<String> secretArns) {- synchronized (lock) {+ synchronized (clusterLock(clusterArn)) {MskCluster cluster = scramCluster(clusterArn);validateSecretArns(secretArns);if (new HashSet<>(secretArns).size() != secretArns.size()) {@@ -109,11 +119,7 @@ public class MskScramSecretsService {validateSecretEncryption(secret);// Amazon MSK reads only the AWSCURRENT version of an associated secretBatchSecretValue secretValue = readSecret(secretArn);- ScramSecretAssociation association = state.getAssociations()- .computeIfAbsent(secretArn, ignored -> new ScramSecretAssociation());- ScramCredentials credentials = readScramCredentials(secretValue.secretString());- association.setUsername(credentials.username());- putAssociations(clusterArn, state);+ applyScramCredentials(cluster, state, secretArn, secretValue);} catch (AwsException e) {if (newAssociation && state.getAssociations().remove(secretArn) != null) {putAssociations(clusterArn, state);@@ -126,7 +132,7 @@ public class MskScramSecretsService {}public PaginatedResult<String> listScramSecrets(String clusterArn, Integer maxResults, String nextToken) {- synchronized (lock) {+ synchronized (clusterLock(clusterArn)) {scramCluster(clusterArn);return Pagination.paginate(new ArrayList<>(associationsFor(clusterArn).getAssociations().keySet()),arn -> arn, maxResults, nextToken, MAX_PAGE, "BadRequestException");@@ -134,17 +140,17 @@ public class MskScramSecretsService {}public List<Map<String, String>> batchDisassociateScramSecret(String clusterArn, List<String> secretArns) {- synchronized (lock) {- scramCluster(clusterArn);+ synchronized (clusterLock(clusterArn)) {+ MskCluster cluster = scramCluster(clusterArn);validateSecretArns(secretArns);MskScramAssociations state = associationsFor(clusterArn);List<Map<String, String>> unprocessedScramSecrets = new ArrayList<>();for (String secretArn : secretArns) {try {- if (state.getAssociations().remove(secretArn) == null) {+ if (!state.getAssociations().containsKey(secretArn)) {throw new AwsException("InvalidSecretArn", "The provided secret ARN is invalid.", 400);}- putAssociations(clusterArn, state);+ removeScramAssociation(cluster, state, secretArn);} catch (AwsException e) {unprocessedScramSecrets.add(scramFailure(secretArn, e));}@@ -241,6 +247,30 @@ public class MskScramSecretsService {}}+ private void applyScramCredentials(MskCluster cluster, MskScramAssociations state,+ String secretArn, BatchSecretValue secretValue) {+ ScramCredentials credentials = readScramCredentials(secretValue.secretString());+ if (!config.services().msk().mock()) {+ redpandaManager.createUser(cluster, credentials.username(), credentials.password());+ }+ ScramSecretAssociation association = new ScramSecretAssociation();+ association.setUsername(credentials.username());+ state.getAssociations().put(secretArn, association);+ putAssociations(cluster.getClusterArn(), state);+ }++ private void removeScramAssociation(MskCluster cluster, MskScramAssociations state, String secretArn) {+ ScramSecretAssociation association = state.getAssociations().get(secretArn);+ if (association == null) {+ return;+ }+ if (association.getUsername() != null && !config.services().msk().mock()) {+ redpandaManager.deleteUser(cluster, association.getUsername());+ }+ state.getAssociations().remove(secretArn);+ putAssociations(cluster.getClusterArn(), state);+ }+private BatchSecretValue readSecret(String secretArn) {// Exact-ARN batch reads select AWSCURRENT without updating access timestamps.BatchGetSecretValueResult result = secretsManagerService.batchGetSecretValue(diff --git a/src/main/java/io/github/hectorvent/floci/services/msk/RedpandaManager.java b/src/main/java/io/github/hectorvent/floci/services/msk/RedpandaManager.javaindex 8984b60b79..958d690716 100644--- a/src/main/java/io/github/hectorvent/floci/services/msk/RedpandaManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/msk/RedpandaManager.java@@ -1,7 +1,11 @@package io.github.hectorvent.floci.services.msk;+import com.fasterxml.jackson.core.JsonProcessingException;+import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsArnUtils;+import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;import io.github.hectorvent.floci.core.common.docker.ContainerDetector;@@ -18,14 +22,21 @@ import jakarta.inject.Inject;import org.jboss.logging.Logger;import java.io.Closeable;+import java.io.IOException;+import java.io.InputStream;+import java.io.OutputStream;import java.net.HttpURLConnection;import java.net.URI;+import java.net.URLEncoder;+import java.nio.charset.StandardCharsets;import java.nio.file.Files;import java.nio.file.Path;import java.util.ArrayList;+import java.util.LinkedHashSet;import java.util.List;import java.util.Map;import java.util.Optional;+import java.util.Set;import java.util.concurrent.ConcurrentHashMap;@ApplicationScoped@@ -40,6 +51,8 @@ public class RedpandaManager {// Redpanda's Admin API exposes readiness at /v1/status/ready. /ready always returns 404private static final String ADMIN_READY_PATH = "/v1/status/ready";+ private static final String ADMIN_SCRAM_USERS_PATH = "/v1/security/users";+ private static final String ADMIN_SCRAM_USER_PATH_TEMPLATE = "/v1/security/users/%s";private record StartedBroker(int hostPort, ContainerInfo info) {}@@ -51,6 +64,7 @@ public class RedpandaManager {private final EmulatorConfig config;private final RegionResolver regionResolver;private final PortAllocator portAllocator;+ private final ObjectMapper objectMapper;private final Map<String, Closeable> logStreams = new ConcurrentHashMap<>();// Container mode only, keyed like logStreams: the host port the host listener is published// on (not recoverable from the bootstrap string, which may be the Docker-network address).@@ -64,7 +78,8 @@ public class RedpandaManager {ContainerDetector containerDetector,EmulatorConfig config,RegionResolver regionResolver,- PortAllocator portAllocator) {+ PortAllocator portAllocator,+ ObjectMapper objectMapper) {this.containerBuilder = containerBuilder;this.lifecycleManager = lifecycleManager;this.logStreamer = logStreamer;@@ -72,6 +87,7 @@ public class RedpandaManager {this.config = config;this.regionResolver = regionResolver;this.portAllocator = portAllocator;+ this.objectMapper = objectMapper;}/**@@ -262,7 +278,7 @@ public class RedpandaManager {}try {- String adminUrl = adminReadyUrl(cluster);+ String adminUrl = adminBaseUrl(cluster) + ADMIN_READY_PATH;HttpURLConnection conn = (HttpURLConnection) URI.create(adminUrl).toURL().openConnection();conn.setRequestMethod("GET");conn.setConnectTimeout(1000);@@ -288,7 +304,7 @@ public class RedpandaManager {}/**- * The broker's admin readiness URL, resolved from the persisted container id the same way+ * The broker's admin base URL, resolved from the persisted container id the same way* container start resolves endpoints: the bound host port in native mode, the container's* reachable address in container mode. Nothing is kept in memory, so it still works for a* cluster left in CREATING by a Floci process that was killed and restarted, and an IP is@@ -298,10 +314,10 @@ public class RedpandaManager {* resolved the Kafka endpoint: in container mode the broker publishes the host listener, so* it is on the default bridge as well, and the bridge address is not one Floci can reach.*/- String adminReadyUrl(MskCluster cluster) {+ String adminBaseUrl(MskCluster cluster) {String network = containerBuilder.resolveDockerNetwork(config.services().dockerNetwork()).orElse(null);EndpointInfo admin = lifecycleManager.resolveEndpoint(cluster.getContainerId(), ADMIN_PORT, network);- return "http://" + admin.host() + ":" + admin.port() + ADMIN_READY_PATH;+ return "http://" + admin.host() + ":" + admin.port();}private void releaseKafkaHostPort(MskCluster cluster) {@@ -377,4 +393,105 @@ public class RedpandaManager {return cluster.getDockerVolumeName();}+ public void createUser(MskCluster cluster, String username, String password) {+ byte[] payload;+ try {+ // Amazon MSK supports only SCRAM-SHA-512 for SCRAM authentication+ payload = objectMapper.writeValueAsBytes(Map.of(+ "username", username, "password", password, "algorithm", "SCRAM-SHA-512"));+ } catch (JsonProcessingException e) {+ throw new AwsException("InternalServerErrorException", "Failed to serialize SCRAM credentials.", 500);+ }+ String encodedUsername = URLEncoder.encode(username, StandardCharsets.UTF_8).replace("+", "%20");+ AdminResponse response = listUsers(cluster).contains(username)+ ? sendAdminRequest(cluster, ADMIN_SCRAM_USER_PATH_TEMPLATE.formatted(encodedUsername), "PUT", payload)+ : sendAdminRequest(cluster, ADMIN_SCRAM_USERS_PATH, "POST", payload);++ int status = response.status();+ if (status < 200 || status >= 300) {+ throw new AwsException("InternalServerErrorException",+ "Kafka user creation failed with HTTP status " + status + ".", 500);+ }+ }++ public void deleteUser(MskCluster cluster, String username) {+ String encodedUsername = URLEncoder.encode(username, StandardCharsets.UTF_8).replace("+", "%20");+ AdminResponse response = sendAdminRequest(cluster, ADMIN_SCRAM_USER_PATH_TEMPLATE.formatted(encodedUsername), "DELETE", null);+ int status = response.status();+ if (status != 404 && (status < 200 || status >= 300)) {+ throw new AwsException("InternalServerErrorException",+ "Kafka user deletion failed with HTTP status " + status + ".", 500);+ }+ }++ public Set<String> listUsers(MskCluster cluster) {+ AdminResponse response = sendAdminRequest(cluster, ADMIN_SCRAM_USERS_PATH, "GET", null);+ if (response.status() != 200) {+ throw new AwsException("InternalServerErrorException", "Unable to list Kafka SCRAM users.", 500);+ }+ JsonNode users;+ try {+ users = objectMapper.readTree(response.body());+ } catch (IOException e) {+ throw new AwsException("InternalServerErrorException", "Invalid Kafka user list response.", 500);+ }+ if (users == null || !users.isArray()) {+ throw new AwsException("InternalServerErrorException", "Invalid Kafka user list response.", 500);+ }+ Set<String> result = new LinkedHashSet<>();+ for (JsonNode user : users) {+ if (user.isTextual()) {+ result.add(user.textValue());+ }+ }+ return result;+ }++ private record AdminResponse(int status, byte[] body) {+ }++ AdminResponse sendAdminRequest(MskCluster cluster, String path, String method, byte[] payload) {+ if (cluster.getContainerId() == null || cluster.getBootstrapBrokers() == null) {+ throw new AwsException("ServiceUnavailableException", "Cluster has no backing Kafka broker.", 503);+ }+ String adminUrl;+ try {+ adminUrl = adminBaseUrl(cluster);+ } catch (RuntimeException e) {+ LOG.warnv("Failed to resolve the admin endpoint of container {0} for MSK cluster {1}: {2}",+ cluster.getContainerId(), cluster.getClusterName(), e.getMessage());+ throw new AwsException("ServiceUnavailableException", "Cluster has no backing Kafka broker.", 503);+ }+ String url = path.startsWith("/") ? adminUrl + path : adminUrl + "/" + path;+ try {+ HttpURLConnection connection = (HttpURLConnection) URI.create(url).toURL().openConnection();+ try {+ connection.setRequestMethod(method);+ connection.setConnectTimeout(5000);+ connection.setReadTimeout(5000);+ if (payload != null) {+ connection.setRequestProperty("Content-Type", "application/json");+ connection.setDoOutput(true);+ connection.setFixedLengthStreamingMode(payload.length);+ try (OutputStream output = connection.getOutputStream()) {+ output.write(payload);+ }+ }+ int status = connection.getResponseCode();+ byte[] body = new byte[0];+ InputStream responseStream = status >= 400 ? connection.getErrorStream() : connection.getInputStream();+ if (responseStream != null) {+ try (InputStream input = responseStream) {+ body = input.readAllBytes();+ }+ }+ return new AdminResponse(status, body);+ } finally {+ connection.disconnect();+ }+ } catch (IOException e) {+ // Do not include the broker response or exception text: either can contain credentials.+ throw new AwsException("ServiceUnavailableException", "Unable to contact the Kafka management API.", 503);+ }+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/msk/MskScramServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/msk/MskScramServiceTest.javaindex 125a2edfe4..6b41f59ff8 100644--- a/src/test/java/io/github/hectorvent/floci/services/msk/MskScramServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/msk/MskScramServiceTest.java@@ -1,35 +1,66 @@package io.github.hectorvent.floci.services.msk;import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.datatype.jsr310.JavaTimeModule;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.AwsArnUtils;import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.PaginatedResult;+import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.kms.KmsService;import io.github.hectorvent.floci.services.kms.model.KmsKey;+import io.github.hectorvent.floci.services.kms.model.KmsKeySpec;+import io.github.hectorvent.floci.services.kms.model.KmsKeyUsage;import io.github.hectorvent.floci.services.msk.model.MskCluster;+import io.github.hectorvent.floci.services.msk.model.MskConfiguration;import io.github.hectorvent.floci.services.msk.model.MskScramAssociations;+import io.github.hectorvent.floci.services.msk.model.ScramSecretAssociation;import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;+import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService.BatchGetSecretValueError;import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService.BatchGetSecretValueResult;import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService.BatchSecretValue;import io.github.hectorvent.floci.services.secretsmanager.model.Secret;+import io.github.hectorvent.floci.services.secretsmanager.model.SecretVersion;import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Named;import org.junit.jupiter.api.Test;import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.Arguments;+import org.junit.jupiter.params.provider.MethodSource;import org.junit.jupiter.params.provider.ValueSource;+import java.util.Collections;import java.util.List;import java.util.Map;-import java.util.Optional;+import java.util.UUID;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;+import java.util.concurrent.TimeUnit;+import java.util.stream.Stream;import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertNull;import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;import static org.mockito.ArgumentMatchers.anyString;import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.RETURNS_DEEP_STUBS;import static org.mockito.Mockito.clearInvocations;+import static org.mockito.Mockito.doAnswer;+import static org.mockito.Mockito.doNothing;+import static org.mockito.Mockito.doReturn;+import static org.mockito.Mockito.doThrow;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.never;+import static org.mockito.Mockito.spy;+import static org.mockito.Mockito.times;import static org.mockito.Mockito.verify;import static org.mockito.Mockito.verifyNoInteractions;import static org.mockito.Mockito.when;@@ -37,80 +68,200 @@ import static org.mockito.Mockito.when;class MskScramServiceTest {private static final String ACCOUNT = "000000000000";- private static final String CLUSTER = "arn:aws:kafka:us-east-1:000000000000:cluster/scram/id";private static final String SECRET =- "arn:aws:secretsmanager:us-east-1:000000000000:secret:AmazonMSK_alice";+ "arn:aws:secretsmanager:us-east-1:000000000000:secret:AmazonMSK_alice";+ private AccountAwareStorageBackend<MskCluster> clusters;private AccountAwareStorageBackend<MskScramAssociations> associations;+ private StorageFactory storageFactory;+ private EmulatorConfig config;+ private RegionResolver regionResolver;+ private RedpandaManager redpanda;private SecretsManagerService secrets;private KmsService kms;- private MskService clusters;+ private ObjectMapper mapper;+ private MskService clusterService;private MskScramSecretsService service;+ private MskCluster cluster;@BeforeEachvoid setUp() {- associations = AccountAwareStorageBackend.inMemory(ACCOUNT);- StorageFactory storageFactory = mock(StorageFactory.class);- when(storageFactory.<MskScramAssociations>create(- eq("msk"), eq("msk-scram-associations.json"), any())).thenReturn(associations);- clusters = mock(MskService.class);- MskCluster cluster = new MskCluster();- cluster.setClusterArn(CLUSTER);- when(clusters.describeCluster(CLUSTER)).thenReturn(cluster);- when(clusters.getClusterForAccount(ACCOUNT, CLUSTER)).thenReturn(Optional.of(cluster));+ clusters = AccountAwareStorageBackend.inMemory(ACCOUNT);+ associations = spy(AccountAwareStorageBackend.<MskScramAssociations>inMemory(ACCOUNT));+ storageFactory = mock(StorageFactory.class);+ when(storageFactory.<MskCluster>create(eq("msk"), eq("msk-clusters.json"), any()))+ .thenReturn(clusters);+ when(storageFactory.<MskScramAssociations>create(eq("msk"), eq("msk-scram-associations.json"), any()))+ .thenReturn(associations);+ when(storageFactory.<MskConfiguration>create(eq("msk"), eq("msk-configurations.json"), any()))+ .thenReturn(AccountAwareStorageBackend.inMemory(ACCOUNT));+ config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);+ when(config.services().msk().mock()).thenReturn(true);+ regionResolver = mock(RegionResolver.class);+ when(regionResolver.getRegion()).thenReturn("us-east-1");+ when(regionResolver.getAccountId()).thenReturn(ACCOUNT);+ redpanda = mock(RedpandaManager.class);secrets = mock(SecretsManagerService.class);- when(secrets.describeSecret(anyString(), eq("us-east-1"))).thenAnswer(invocation -> {+ kms = mock(KmsService.class);+ when(secrets.describeSecret(anyString(), anyString())).thenAnswer(invocation -> {Secret metadata = new Secret();metadata.setArn(invocation.getArgument(0));metadata.setKmsKeyId("customer-key");return metadata;});- kms = mock(KmsService.class);- when(kms.describeKey("customer-key", "us-east-1")).thenReturn(new KmsKey());- service = new MskScramSecretsService(storageFactory, clusters, secrets, kms, new ObjectMapper());- secretValue(SECRET, "{\"username\":\"alice\",\"password\":\"password\"}");+ when(kms.describeKey(eq("customer-key"), anyString()))+ .thenReturn(new KmsKey());+ mapper = new ObjectMapper().registerModule(new JavaTimeModule());+ clusterService = spy(new MskService(storageFactory, config, regionResolver, redpanda));+ service = new MskScramSecretsService(storageFactory, config, clusterService, redpanda, secrets, kms, mapper);+ cluster = clusterService.createCluster("scram");+ cluster.setContainerId("broker-1");+ when(config.services().msk().mock())+ .thenReturn(false);}- private void secretValue(String arn, String value) {- when(secrets.batchGetSecretValue(List.of(arn), "us-east-1")).thenReturn(- new BatchGetSecretValueResult(List.of(new BatchSecretValue(- arn, "AmazonMSK_fixture", value, null, "version", List.of("AWSCURRENT"), null)), List.of()));+ private SecretVersion version(String username, String password) {+ SecretVersion version = new SecretVersion();+ version.setVersionId(UUID.randomUUID().toString());+ version.setSecretString(+ "{\"username\":\"" + username + "\",\"password\":\"" + password + "\"}");+ return version;+ }++ @Test+ void brokerOperationDoesNotBlockAnotherCluster() throws Exception {+ when(config.services().msk().mock())+ .thenReturn(true);+ MskCluster otherCluster = clusterService.createCluster("other-scram");+ when(config.services().msk().mock())+ .thenReturn(false);+ CountDownLatch operationInFirstClusterStarted = new CountDownLatch(1);+ CountDownLatch operationInSecondClusterDone = new CountDownLatch(1);+ doAnswer(invocation -> {+ operationInFirstClusterStarted.countDown();+ assertTrue(operationInSecondClusterDone.await(10, TimeUnit.SECONDS));+ return null;+ }).when(redpanda).createUser(cluster, "alice", "password");++ try (ExecutorService executor = Executors.newFixedThreadPool(2)) {+ Future<?> associationInFirstCluster = executor.submit(+ () -> associate(SECRET, version("alice", "password")));+ try {+ assertTrue(operationInFirstClusterStarted.await(5, TimeUnit.SECONDS));+ Future<List<String>> listInSecondCluster = executor.submit(+ () -> service.listScramSecrets(otherCluster.getClusterArn(), null, null).items());+ assertTrue(listInSecondCluster.get(5, TimeUnit.SECONDS).isEmpty());+ } finally {+ operationInSecondClusterDone.countDown();+ }+ associationInFirstCluster.get(5, TimeUnit.SECONDS);+ }+ }++ private void associate(String arn, SecretVersion version) {+ setSecretValue(arn, version);+ assertTrue(+ service.batchAssociateScramSecret(cluster.getClusterArn(), List.of(arn)).isEmpty());+ }++ private void setSecretValue(String arn, SecretVersion version) {+ BatchGetSecretValueResult batchGetSecretValueResult = new BatchGetSecretValueResult(List.of(new BatchSecretValue(+ arn, "AmazonMSK_fixture", version.getSecretString(), version.getSecretBinary(),+ version.getVersionId(), List.of("AWSCURRENT"), version.getCreatedDate())+ ), List.of());+ doReturn(batchGetSecretValueResult)+ .when(secrets)+ .batchGetSecretValue(List.of(arn), AwsArnUtils.parse(arn).region());+ }++ private void setSecretReadError(String errorCode, String message) {+ doReturn(new BatchGetSecretValueResult(List.of(), List.of(new BatchGetSecretValueError(SECRET, errorCode, message))))+ .when(secrets).batchGetSecretValue(List.of(SECRET), "us-east-1");+ }++ private MskScramAssociations state() {+ return associations+ .getForAccount(ACCOUNT, cluster.getClusterArn())+ .orElseGet(MskScramAssociations::new);+ }++ private ScramSecretAssociation association(String secretArn) {+ return state().getAssociations().get(secretArn);+ }++ private static Stream<Arguments> invalidCredentials() {+ return Stream.of(+ Arguments.of(Named.of("empty content", "")),+ Arguments.of(Named.of("malformed JSON", "{")),+ Arguments.of(Named.of("array instead of object", "[]")),+ Arguments.of(Named.of("string instead of object", "\"alice\"")),+ Arguments.of(Named.of("number instead of object", "42")),+ Arguments.of(Named.of("empty object", "{}")),+ Arguments.of(Named.of("missing username", "{\"password\":\"pw\"}")),+ Arguments.of(Named.of("missing password", "{\"username\":\"alice\"}")),+ Arguments.of(Named.of("null username", "{\"username\":null,\"password\":\"pw\"}")),+ Arguments.of(Named.of("empty username", "{\"username\":\"\",\"password\":\"pw\"}")),+ Arguments.of(Named.of("blank username", "{\"username\":\" \",\"password\":\"pw\"}")),+ Arguments.of(Named.of("empty password", "{\"username\":\"alice\",\"password\":\"\"}")),+ Arguments.of(Named.of("blank password", "{\"username\":\"alice\",\"password\":\" \"}")),+ Arguments.of(Named.of("array username", "{\"username\":[],\"password\":\"pw\"}")),+ Arguments.of(Named.of("object username", "{\"username\":{},\"password\":\"pw\"}")),+ Arguments.of(Named.of("boolean username", "{\"username\":true,\"password\":\"pw\"}")),+ Arguments.of(Named.of("null password", "{\"username\":\"alice\",\"password\":null}")),+ Arguments.of(Named.of("array password", "{\"username\":\"alice\",\"password\":[]}")),+ Arguments.of(Named.of("object password", "{\"username\":\"alice\",\"password\":{}}")),+ Arguments.of(Named.of("numeric password", "{\"username\":\"alice\",\"password\":123}")));+ }++ private static Stream<Arguments> validCredentials() {+ return Stream.of(+ Arguments.of(Named.of("string credentials", "{\"username\":\"alice\",\"password\":\"pw\"}"), "alice", "pw"),+ Arguments.of(Named.of("numeric username converted to string", "{\"username\":123,\"password\":\"pw\"}"), "123", "pw"),+ Arguments.of(Named.of("boolean password converted to string", "{\"username\":\"alice\",\"password\":true}"), "alice", "true"),+ Arguments.of(Named.of("extra fields ignored", "{\"username\":\"alice\",\"password\":\"pw\",\"extra\":42}"), "alice", "pw"));+ }++ @ParameterizedTest(name = "{index}: {0}")+ @MethodSource("validCredentials")+ void readScramCredentialsWithValidValues(+ String content, String username, String password) {+ // Supported scalar values become strings, while unrelated JSON fields are ignored.+ MskScramSecretsService.ScramCredentials credentials = service.readScramCredentials(content);++ assertEquals(username, credentials.username());+ assertEquals(password, credentials.password());}@Testvoid reconciliationRemovesOnlyDeletedClusterAssociationsAcrossAccounts() {- service.batchAssociateScramSecret(CLUSTER, List.of(SECRET));+ associate(SECRET, version("alice", "password"));String otherAccount = "111111111111";- String deletedCluster = CLUSTER.replace(ACCOUNT, otherAccount);+ String deletedCluster = cluster.getClusterArn().replace(ACCOUNT, otherAccount);associations.putForAccount(otherAccount, deletedCluster, new MskScramAssociations());- when(clusters.getClusterForAccount(otherAccount, deletedCluster)).thenReturn(Optional.empty());service.reconcileScramSecrets();- assertTrue(associations.getForAccount(ACCOUNT, CLUSTER).isPresent());+ assertTrue(associations.getForAccount(ACCOUNT, cluster.getClusterArn()).isPresent());assertTrue(associations.getForAccount(otherAccount, deletedCluster).isEmpty());- verify(clusters).getClusterForAccount(otherAccount, deletedCluster);+ verify(clusterService).getClusterForAccount(otherAccount, deletedCluster);}@Testvoid reconciliationPreservesAssociationsWhenClusterLookupFails() {- service.batchAssociateScramSecret(CLUSTER, List.of(SECRET));+ associate(SECRET, version("alice", "password"));AwsException failure = new AwsException("InternalServerErrorException", "Lookup failed.", 500);- when(clusters.getClusterForAccount(ACCOUNT, CLUSTER)).thenThrow(failure);+ doThrow(failure).when(clusterService).getClusterForAccount(ACCOUNT, cluster.getClusterArn());assertEquals(failure, assertThrows(AwsException.class, () -> service.reconcileScramSecrets()));- assertTrue(associations.getForAccount(ACCOUNT, CLUSTER).isPresent());+ assertTrue(associations.getForAccount(ACCOUNT, cluster.getClusterArn()).isPresent());}@Test- void associatesListsAndDisassociatesMetadata() {- assertTrue(service.batchAssociateScramSecret(CLUSTER, List.of(SECRET)).isEmpty());- assertEquals(List.of(SECRET), service.listScramSecrets(CLUSTER, null, null).items());- assertEquals("alice", associations.getForAccount(ACCOUNT, CLUSTER).orElseThrow()- .getAssociations().get(SECRET).getUsername());+ void readScramCredentialsRejectsNullSecretStringWithInternalServerError() {+ // A missing secret string maps to an internal error rather than invalid credentials.+ AwsException error = assertThrows(AwsException.class, () -> service.readScramCredentials(null));- assertTrue(service.batchDisassociateScramSecret(CLUSTER, List.of(SECRET)).isEmpty());- assertTrue(service.listScramSecrets(CLUSTER, null, null).items().isEmpty());- assertTrue(associations.getForAccount(ACCOUNT, CLUSTER).isEmpty());+ assertEquals("InternalServerError", error.getErrorCode());+ assertEquals(500, error.getHttpStatus());}@ParameterizedTest@@ -122,10 +273,10 @@ class MskScramServiceTest {metadata.setKmsKeyId(keyReference);when(secrets.describeSecret(SECRET, "us-east-1")).thenReturn(metadata);- List<Map<String, String>> failures = service.batchAssociateScramSecret(CLUSTER, List.of(SECRET));+ List<Map<String, String>> failures = service.batchAssociateScramSecret(cluster.getClusterArn(), List.of(SECRET));assertEquals("BadRequestException", failures.getFirst().get("errorCode"));- assertTrue(service.listScramSecrets(CLUSTER, null, null).items().isEmpty());+ assertTrue(service.listScramSecrets(cluster.getClusterArn(), null, null).items().isEmpty());verifyNoInteractions(kms);verify(secrets, never()).batchGetSecretValue(any(), anyString());}@@ -139,11 +290,12 @@ class MskScramServiceTest {metadata.setKmsKeyId(keyReference);when(secrets.describeSecret(SECRET, "us-east-1")).thenReturn(metadata);when(kms.describeKey(keyReference, "us-east-1")).thenReturn(new KmsKey());+ setSecretValue(SECRET, version("alice", "password"));- assertTrue(service.batchAssociateScramSecret(CLUSTER, List.of(SECRET)).isEmpty());+ assertTrue(service.batchAssociateScramSecret(cluster.getClusterArn(), List.of(SECRET)).isEmpty());verify(kms).describeKey(keyReference, "us-east-1");- assertEquals(List.of(SECRET), service.listScramSecrets(CLUSTER, null, null).items());+ assertEquals(List.of(SECRET), service.listScramSecrets(cluster.getClusterArn(), null, null).items());}@Test@@ -154,45 +306,43 @@ class MskScramServiceTest {metadata.setKmsKeyId("customer-key");when(secrets.describeSecret(SECRET, "us-east-1")).thenReturn(metadata);- List<Map<String, String>> failures = service.batchAssociateScramSecret(CLUSTER, List.of(SECRET));+ List<Map<String, String>> failures = service.batchAssociateScramSecret(cluster.getClusterArn(), List.of(SECRET));assertEquals(List.of(Map.of("secretArn", SECRET, "errorCode", "InvalidSecretArn","errorMessage", "Amazon MSK wasn't able to access the provided secret.")), failures);- assertTrue(service.listScramSecrets(CLUSTER, null, null).items().isEmpty());+ assertTrue(service.listScramSecrets(cluster.getClusterArn(), null, null).items().isEmpty());verifyNoInteractions(kms);verify(secrets, never()).batchGetSecretValue(any(), anyString());- secretValue(fullArn, "{\"username\":\"alice\",\"password\":\"password\"}");- assertTrue(service.batchAssociateScramSecret(CLUSTER, List.of(fullArn)).isEmpty());- assertEquals(List.of(fullArn), service.listScramSecrets(CLUSTER, null, null).items());+ associate(fullArn, version("alice", "password"));+ assertEquals(List.of(fullArn), service.listScramSecrets(cluster.getClusterArn(), null, null).items());}@Testvoid rejectsPartialSecretArnForDisassociation() {String fullArn = SECRET + "-Ab12Cd";- secretValue(fullArn, "{\"username\":\"alice\",\"password\":\"password\"}");- assertTrue(service.batchAssociateScramSecret(CLUSTER, List.of(fullArn)).isEmpty());+ associate(fullArn, version("alice", "password"));clearInvocations(secrets);- List<Map<String, String>> failures = service.batchDisassociateScramSecret(CLUSTER, List.of(SECRET));+ List<Map<String, String>> failures = service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET));assertEquals(List.of(Map.of("secretArn", SECRET, "errorCode", "InvalidSecretArn","errorMessage", "The provided secret ARN is invalid.")), failures);- assertEquals(List.of(fullArn), service.listScramSecrets(CLUSTER, null, null).items());- assertTrue(service.batchDisassociateScramSecret(CLUSTER, List.of(fullArn)).isEmpty());+ assertEquals(List.of(fullArn), service.listScramSecrets(cluster.getClusterArn(), null, null).items());+ assertTrue(service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(fullArn)).isEmpty());verifyNoInteractions(secrets);}@Testvoid disassociatesDeletedSecretWithoutLookingItUp() {- assertTrue(service.batchAssociateScramSecret(CLUSTER, List.of(SECRET)).isEmpty());+ associate(SECRET, version("alice", "password"));when(secrets.describeSecret(SECRET, "us-east-1")).thenThrow(new AwsException("ResourceNotFoundException", "Secret not found.", 400));clearInvocations(secrets);- assertTrue(service.batchDisassociateScramSecret(CLUSTER, List.of(SECRET)).isEmpty());+ assertTrue(service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET)).isEmpty());- assertTrue(service.listScramSecrets(CLUSTER, null, null).items().isEmpty());- assertTrue(associations.getForAccount(ACCOUNT, CLUSTER).isEmpty());+ assertTrue(service.listScramSecrets(cluster.getClusterArn(), null, null).items().isEmpty());+ assertTrue(associations.getForAccount(ACCOUNT, cluster.getClusterArn()).isEmpty());verifyNoInteractions(secrets);}@@ -205,55 +355,490 @@ class MskScramServiceTest {null, "version", List.of("AWSCURRENT"), null)), List.of()));- List<Map<String, String>> failures = service.batchAssociateScramSecret(CLUSTER, List.of(SECRET));+ List<Map<String, String>> failures = service.batchAssociateScramSecret(cluster.getClusterArn(), List.of(SECRET));assertEquals("InternalServerError", failures.getFirst().get("errorCode"));- assertTrue(service.listScramSecrets(CLUSTER, null, null).items().isEmpty());+ assertTrue(service.listScramSecrets(cluster.getClusterArn(), null, null).items().isEmpty());}@Test- void reportsItemFailuresWithoutBlockingValidSecrets() {- String invalid = SECRET + "-invalid";- secretValue(invalid, "{}");+ void readScramCredentialsRejectsJsonNullWithInternalServerError() {+ // JSON null has the same internal-error contract as a missing secret string.+ AwsException error =+ assertThrows(AwsException.class, () -> service.readScramCredentials("null"));- List<Map<String, String>> failures = service.batchAssociateScramSecret(CLUSTER, List.of(invalid, SECRET));+ assertEquals("InternalServerError", error.getErrorCode());+ assertEquals(500, error.getHttpStatus());+ }++ @ParameterizedTest(name = "{index}: {0}")+ @MethodSource("invalidCredentials")+ void readScramCredentialsWithInvalidValues(String content) {+ // Malformed content and unsupported credential values produce InvalidSecretArn.+ AwsException error =+ assertThrows(AwsException.class, () -> service.readScramCredentials(content));++ assertEquals("InvalidSecretArn", error.getErrorCode());+ }++ @Test+ void associationRejectsDefaultAndIncompatibleKmsKeysWithoutChangingCredentials() {+ // Only compatible customer-managed keys are accepted, without disturbing working credentials.+ SecretVersion original = version("alice", "password");+ associate(SECRET, original);+ clearInvocations(redpanda);+ Secret metadata = new Secret();+ String invalidArn = SECRET + "-invalid-key";+ metadata.setArn(invalidArn);+ when(secrets.describeSecret(invalidArn, "us-east-1")).thenReturn(metadata);+ KmsKey key = new KmsKey();+ when(kms.describeKey("incompatible-key", "us-east-1")).thenReturn(key);++ for (String keyKind : List.of("default", "aws-managed", "asymmetric", "hmac")) {+ metadata.setKmsKeyId("default".equals(keyKind) ? null : "incompatible-key");+ key.setKeyManager("aws-managed".equals(keyKind) ? "AWS" : "CUSTOMER");+ key.setKeySpec(+ "asymmetric".equals(keyKind)+ ? KmsKeySpec.RSA_2048+ : "hmac".equals(keyKind)+ ? KmsKeySpec.HMAC_256+ : KmsKeySpec.SYMMETRIC_DEFAULT);+ key.setKeyUsage(+ "hmac".equals(keyKind)+ ? KmsKeyUsage.GENERATE_VERIFY_MAC+ : KmsKeyUsage.ENCRYPT_DECRYPT);++ List<Map<String, String>> failures =+ service.batchAssociateScramSecret(cluster.getClusterArn(), List.of(invalidArn));++ assertEquals(1, failures.size(), keyKind);+ assertEquals("BadRequestException", failures.getFirst().get("errorCode"), keyKind);+ assertNull(association(invalidArn));+ assertEquals("alice", association(SECRET).getUsername(), keyKind);+ }+ verifyNoInteractions(redpanda);+ assertTrue(service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET)).isEmpty());+ }++ @Test+ void associationRejectsKmsAliasBeforeLookupWithoutReadingCredentials() {+ // KMS aliases fail validation before any key lookup, credential read, or broker change.+ Secret metadata = new Secret();+ metadata.setArn(SECRET);+ metadata.setKmsKeyId("alias/msk-scram");+ when(secrets.describeSecret(SECRET, "us-east-1")).thenReturn(metadata);++ List<Map<String, String>> failures = service.batchAssociateScramSecret(cluster.getClusterArn(), List.of(SECRET));assertEquals(1, failures.size());- assertEquals(invalid, failures.getFirst().get("secretArn"));- assertEquals("InvalidSecretArn", failures.getFirst().get("errorCode"));- assertEquals(List.of(SECRET), service.listScramSecrets(CLUSTER, null, null).items());+ assertEquals(SECRET, failures.getFirst().get("secretArn"));+ assertEquals("BadRequestException", failures.getFirst().get("errorCode"));+ assertNull(association(SECRET));+ verifyNoInteractions(kms, redpanda);+ verify(secrets, never()).batchGetSecretValue(any(), anyString());}@Test- void rejectsRepeatedAssociationAndDisassociation() {- service.batchAssociateScramSecret(CLUSTER, List.of(SECRET));- assertEquals("InvalidSecretArn", service.batchAssociateScramSecret(CLUSTER, List.of(SECRET))- .getFirst().get("errorCode"));- service.batchDisassociateScramSecret(CLUSTER, List.of(SECRET));- assertEquals("InvalidSecretArn", service.batchDisassociateScramSecret(CLUSTER, List.of(SECRET))- .getFirst().get("errorCode"));+ void associationReportsKmsLookupFailureWithoutCreatingAssociation() {+ // Key lookup errors remain item failures and must not create association state.+ doThrow(new AwsException("NotFoundException", "Key not found.", 400))+ .when(kms)+ .describeKey("customer-key", "us-east-1");++ List<Map<String, String>> failures =+ service.batchAssociateScramSecret(cluster.getClusterArn(), List.of(SECRET));++ assertEquals("NotFoundException", failures.getFirst().get("errorCode"));+ assertNull(association(SECRET));+ verifyNoInteractions(redpanda);}@Test- void rejectsDuplicateAssociationBatchBeforeStoringMetadata() {- AwsException error = assertThrows(AwsException.class,- () -> service.batchAssociateScramSecret(CLUSTER, List.of(SECRET, SECRET)));+ void duplicateAssociationBatchIsRejectedBeforeAnySecretIsProcessed() {+ // Duplicate ARNs invalidate the whole batch before any item can cause side effects.+ String fresh = SECRET + "-fresh";++ AwsException error = assertThrows(+ AwsException.class,+ () -> service.batchAssociateScramSecret(cluster.getClusterArn(), List.of(fresh, SECRET, SECRET))+ );assertEquals("BadRequestException", error.getErrorCode());- assertTrue(service.listScramSecrets(CLUSTER, null, null).items().isEmpty());+ assertEquals(400, error.getHttpStatus());+ assertTrue(state().getAssociations().isEmpty());+ verifyNoInteractions(redpanda, secrets, kms);+ verify(associations, never()).putForAccount(anyString(), anyString(), any());+ }++ @Test+ void duplicateAssociationBatchPreservesExistingAssociation() {+ // Rejecting a duplicate batch must leave existing credentials and membership untouched.+ SecretVersion original = version("alice", "original");+ associate(SECRET, original);+ clearInvocations(redpanda, secrets, kms, associations);+ String fresh = SECRET + "-fresh";++ AwsException error = assertThrows(+ AwsException.class,+ () -> service.batchAssociateScramSecret(cluster.getClusterArn(), List.of(fresh, SECRET, SECRET))+ );++ assertEquals("BadRequestException", error.getErrorCode());+ assertEquals(400, error.getHttpStatus());+ assertEquals(+ List.of(SECRET),+ service.listScramSecrets(cluster.getClusterArn(), null, null).items());+ assertEquals("alice", association(SECRET).getUsername());+ verifyNoInteractions(redpanda, secrets, kms);+ verify(associations, never()).putForAccount(anyString(), anyString(), any());+ }++ @Test+ void associationRejectsSecretsWithoutAmazonMskPrefix() {+ String nonPrefixSecret =+ "arn:aws:secretsmanager:us-east-1:000000000000:secret:NotAmazonMSK_user";+ List<Map<String, String>> associateFailures =+ service.batchAssociateScramSecret(+ cluster.getClusterArn(), List.of(nonPrefixSecret));+ assertEquals(1, associateFailures.size());+ assertEquals("BadRequestException", associateFailures.getFirst().get("errorCode"));+ }++ @Test+ void listingAndDisassociationRejectServerlessClusters() {+ // Serverless clusters do not support these SCRAM management operations.+ cluster.setClusterType("SERVERLESS");+ assertThrows(AwsException.class, () -> service.listScramSecrets(cluster.getClusterArn(), null, null));+ assertThrows(AwsException.class, () -> service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET)));+ }++ @Test+ void associationReadsCurrentVersionThroughExactArnBatchGetSecretValue() {+ // Association applies the AWSCURRENT secret value and records its version only after success.+ SecretVersion version = version("alice", "password");+ setSecretValue(SECRET, version);++ assertTrue(service.batchAssociateScramSecret(cluster.getClusterArn(), List.of(SECRET)).isEmpty());++ verify(secrets).batchGetSecretValue(List.of(SECRET), "us-east-1");+ verify(secrets, never()).getSecretValue(anyString(), any(), any(), anyString());+ verify(secrets, never()).batchGetSecretValueByFilters(any(), anyString());+ verify(redpanda).createUser(cluster, "alice", "password");+ }++ @ParameterizedTest+ @ValueSource(strings = {"ResourceNotFoundException", "InvalidRequestException", "AccessDeniedException"})+ void batchReadErrorsAreReportedPerSecretWithoutBlockingOtherAssociations(String errorCode) {+ setSecretReadError(errorCode, "Secret cannot be read.");+ String validArn = SECRET + "-valid";+ setSecretValue(validArn, version("bob", "password"));++ List<Map<String, String>> failures = service.batchAssociateScramSecret(+ cluster.getClusterArn(), List.of(SECRET, validArn));++ assertEquals(List.of(Map.of("secretArn", SECRET, "errorCode", errorCode,+ "errorMessage", "Secret cannot be read.")), failures);+ assertNull(association(SECRET));+ assertNotNull(association(validArn));+ verify(redpanda).createUser(cluster, "bob", "password");+ }++ @Test+ void associationIsStoredAfterBrokerSideEffect() {+ doAnswer(+ invocation -> {+ verify(associations, never()).putForAccount(anyString(), anyString(), any());+ return null;+ })+ .when(redpanda)+ .createUser(cluster, "alice", "password");++ associate(SECRET, version("alice", "password"));+ assertEquals("alice", association(SECRET).getUsername());+ verify(associations).putForAccount(eq(ACCOUNT), eq(cluster.getClusterArn()), any());+ }++ @Test+ void failedAssociationCanBeRetried() {+ SecretVersion version = version("alice", "password");+ setSecretValue(SECRET, version);+ doThrow(new AwsException("ServiceUnavailableException", "Broker unavailable.", 503))+ .when(redpanda)+ .createUser(cluster, "alice", "password");+ assertEquals(+ 1,+ service.batchAssociateScramSecret(cluster.getClusterArn(), List.of(SECRET)).size());+ assertTrue(service.listScramSecrets(cluster.getClusterArn(), null, null).items().isEmpty());++ doNothing().when(redpanda).createUser(cluster, "alice", "password");+ associate(SECRET, version);++ assertEquals("alice", association(SECRET).getUsername());+ assertEquals(List.of(SECRET), service.listScramSecrets(cluster.getClusterArn(), null, null).items());+ }++ @Test+ void successfulAssociationCannotBeRepeated() {+ SecretVersion version = version("alice", "password");+ associate(SECRET, version);+ clearInvocations(redpanda, associations);+ setSecretValue(SECRET, version("bob", "changed"));+ List<Map<String, String>> failures =+ service.batchAssociateScramSecret(cluster.getClusterArn(), List.of(SECRET));+ assertEquals("InvalidSecretArn", failures.getFirst().get("errorCode"));+ assertEquals("alice", association(SECRET).getUsername());+ verifyNoInteractions(redpanda);+ verify(associations, never()).putForAccount(anyString(), anyString(), any());+ assertEquals(List.of(SECRET), service.listScramSecrets(cluster.getClusterArn(), null, null).items());+ }++ @Test+ void repeatedAssociationIsRejectedBeforeReadingUnavailableSecret() {+ // Existing membership determines duplicate rejection even when the secret cannot be read.+ associate(SECRET, version("alice", "password"));+ clearInvocations(redpanda);+ for (String errorCode : List.of("ResourceNotFoundException", "InvalidRequestException")) {+ setSecretReadError(errorCode, "Secret cannot be read.");++ List<Map<String, String>> failures =+ service.batchAssociateScramSecret(cluster.getClusterArn(), List.of(SECRET));++ assertEquals("InvalidSecretArn", failures.getFirst().get("errorCode"));+ assertEquals("alice", association(SECRET).getUsername());+ }+ verifyNoInteractions(redpanda);+ }++ @Test+ void disassociationRejectsNullEmptyAndOversizedBatches() {+ // Disassociation requires a nonempty batch containing at most ten secret ARNs.+ String clusterArn = cluster.getClusterArn();+ assertThrows(AwsException.class, () -> service.batchDisassociateScramSecret(clusterArn, null));+ assertThrows(AwsException.class, () -> service.batchDisassociateScramSecret(clusterArn, List.of()));+ assertThrows(AwsException.class, () -> service.batchDisassociateScramSecret(clusterArn, Collections.nCopies(11, SECRET)));+ }++ @Test+ void disassociationReportsMalformedSecretArnAsAnItemFailure() {+ // A malformed secret ARN is reported per item rather than failing the whole request.+ assertEquals(1, service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of("malformed")).size());+ }++ @Test+ void unassociatedDisassociationReturnsOneFailurePerOccurrenceWithoutBrokerCalls() {+ // Each unassociated ARN occurrence fails independently without storage or broker changes.+ List<Map<String, String>> failures =+ service.batchDisassociateScramSecret(+ cluster.getClusterArn(), List.of(SECRET, SECRET));++ assertEquals(2, failures.size());+ for (Map<String, String> failure : failures) {+ assertEquals(SECRET, failure.get("secretArn"));+ assertEquals("InvalidSecretArn", failure.get("errorCode"));+ }+ assertTrue(state().getAssociations().isEmpty());+ verifyNoInteractions(redpanda);+ verify(associations, never()).putForAccount(anyString(), anyString(), any());}@Test- void listsPersistedAssociationsAfterServiceRecreation() {- service.batchAssociateScramSecret(CLUSTER, List.of(SECRET));- StorageFactory storageFactory = mock(StorageFactory.class);- when(storageFactory.<MskScramAssociations>create(- eq("msk"), eq("msk-scram-associations.json"), any())).thenReturn(associations);- MskService clusters = mock(MskService.class);- when(clusters.describeCluster(CLUSTER)).thenReturn(new MskCluster());- MskScramSecretsService restored = new MskScramSecretsService(- storageFactory, clusters, secrets, mock(KmsService.class), new ObjectMapper());+ void repeatedDisassociationPreservesOtherAssociations() {+ // Removing an already absent secret must not affect other cluster associations.+ String other = SECRET + "-other";+ associate(SECRET, version("alice", "pw"));+ associate(other, version("bob", "pw"));+ assertTrue(service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET)).isEmpty());+ clearInvocations(redpanda);++ List<Map<String, String>> failures = service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET));++ assertEquals("InvalidSecretArn", failures.getFirst().get("errorCode"));+ assertEquals(List.of(other), service.listScramSecrets(cluster.getClusterArn(), null, null).items());+ verifyNoInteractions(redpanda);+ }++ @Test+ void failedDisassociationRetriesOriginalUsernameWithoutApplyingChangedCredentials() {+ // Pending cleanup retains the original username even if the secret rotates before retry.+ associate(SECRET, version("alice", "password"));+ doThrow(new AwsException("ServiceUnavailableException", "Broker unavailable.", 503))+ .when(redpanda)+ .deleteUser(cluster, "alice");++ assertEquals(1, service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET)).size());+ assertEquals("alice", association(SECRET).getUsername());+ clearInvocations(redpanda);+ setSecretValue(SECRET, version("charlie", "changed-during-cleanup"));+ doNothing().when(redpanda).deleteUser(cluster, "alice");+ assertTrue(service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET)).isEmpty());+ assertTrue(state().getAssociations().isEmpty());+ verify(redpanda).deleteUser(cluster, "alice");+ verify(redpanda, never()).createUser(any(), anyString(), anyString());+ }- assertEquals(List.of(SECRET), restored.listScramSecrets(CLUSTER, null, null).items());+ @Test+ void deletionRetryRemovesAssociationWithoutRewritingIt() {+ associate(SECRET, version("alice", "password"));+ doThrow(new AwsException("ServiceUnavailableException", "Broker unavailable.", 503))+ .when(redpanda)+ .deleteUser(cluster, "alice");+ service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET));+ clearInvocations(associations);+ doNothing().when(redpanda).deleteUser(cluster, "alice");++ assertTrue(service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET)).isEmpty());++ verify(associations, never()).putForAccount(anyString(), anyString(), any());+ verify(associations, times(1)).deleteForAccount(ACCOUNT, cluster.getClusterArn());+ assertTrue(state().getAssociations().isEmpty());+ }++ @Test+ void listScramSecretsReturnsEachPageAndOnlyProvidesTokenWhenMoreResultsExist() {+ // Pagination exposes each association once and omits the token on the final page.+ String second = SECRET + "-second";+ associate(SECRET, version("alice", "password"));+ associate(second, version("bob", "password"));+ PaginatedResult<String> page = service.listScramSecrets(cluster.getClusterArn(), 1, null);+ assertEquals(List.of(SECRET), page.items());+ assertNotNull(page.nextToken());+ PaginatedResult<String> last =+ service.listScramSecrets(cluster.getClusterArn(), 1, page.nextToken());+ assertEquals(List.of(second), last.items());+ assertNull(last.nextToken());+ }++ @Test+ void listScramSecretsRejectsPageSizesOutsideOneToOneHundred() {+ // List page sizes must stay within the AWS range of one through one hundred.+ assertThrows(AwsException.class, () -> service.listScramSecrets(cluster.getClusterArn(), 0, null));+ assertThrows(AwsException.class, () -> service.listScramSecrets(cluster.getClusterArn(), 101, null));+ }++ @Test+ void listScramSecretsRejectsMalformedNextToken() {+ // Invalid pagination tokens must fail instead of silently restarting the listing.+ assertThrows(AwsException.class, () -> service.listScramSecrets(cluster.getClusterArn(), 1, "!"));+ }++ @Test+ void listScramSecretsRejectsMissingCluster() {+ // Listing an unknown cluster must fail rather than appear to return an empty membership.+ assertThrows(AwsException.class, () -> service.listScramSecrets("missing", null, null));+ }++ private static Stream<Arguments> sharedUsernameAssociations() {+ return Stream.of(+ Arguments.of(Named.of("same password, sequential", false), false, "password"),+ Arguments.of(Named.of("same password, reversed sequential", false), true, "password"),+ Arguments.of(Named.of("same password, batch", true), false, "password"),+ Arguments.of(Named.of("same password, reversed batch", true), true, "password"),+ Arguments.of(Named.of("different passwords, sequential", false), false, "another-password"),+ Arguments.of(Named.of("different passwords, reversed sequential", false), true, "another-password"),+ Arguments.of(Named.of("different passwords, batch", true), false, "another-password"),+ Arguments.of(Named.of("different passwords, reversed batch", true), true, "another-password")+ );+ }++ @ParameterizedTest(name = "{index}: {0}")+ @MethodSource("sharedUsernameAssociations")+ void associationAcceptsSharedUsernameRegardlessOfPasswordAndOrder(+ boolean batch, boolean reversed, String secondPassword) {+ // Distinct secrets may share a broker username regardless of passwords, ordering, or batching.+ String second = SECRET + "-second";+ SecretVersion firstVersion = version("alice", "password");+ SecretVersion secondVersion = version("alice", secondPassword);+ setSecretValue(SECRET, firstVersion);+ setSecretValue(second, secondVersion);+ List<String> ordered = reversed ? List.of(second, SECRET) : List.of(SECRET, second);++ if (batch) {+ assertTrue(+ service.batchAssociateScramSecret(cluster.getClusterArn(), ordered).isEmpty());+ } else {+ for (String arn : ordered) {+ assertTrue(+ service.batchAssociateScramSecret(cluster.getClusterArn(), List.of(arn))+ .isEmpty());+ }+ }++ assertEquals(+ ordered.stream().sorted().toList(),+ service.listScramSecrets(cluster.getClusterArn(), null, null).items().stream()+ .sorted()+ .toList());+ assertEquals("alice", association(SECRET).getUsername());+ assertEquals("alice", association(second).getUsername());+ verify(redpanda, times("password".equals(secondPassword) ? 2 : 1))+ .createUser(cluster, "alice", "password");+ verify(redpanda, times("password".equals(secondPassword) ? 2 : 1))+ .createUser(cluster, "alice", secondPassword);+ }++ @Test+ void failedUserDeletionDoesNotInvalidateSharedAssociations() {+ // Failed broker deletion leaves both associations available for a caller to retry.+ String second = SECRET + "-second";+ SecretVersion surviving = version("alice", "surviving-password");+ associate(SECRET, version("alice", "password"));+ associate(second, surviving);+ doThrow(new AwsException("ServiceUnavailableException", "Broker unavailable.", 503))+ .when(redpanda)+ .deleteUser(cluster, "alice");++ assertEquals(+ 1,+ service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET))+ .size());++ assertEquals("alice", association(second).getUsername());+ assertEquals("alice", association(SECRET).getUsername());+ }++ @Test+ void associationsPersistSeparatelyWithoutPasswords() throws Exception {+ // Separate association storage survives restart without persisting passwords or reapplying credentials.+ associate(SECRET, version("alice", "sensitive-password"));+ String json = mapper.writeValueAsString(state());+ assertFalse(json.contains("sensitive-password"));+ assertFalse(mapper.readTree(mapper.writeValueAsString(cluster)).has("scramSecrets"));+ MskScramAssociations restored = mapper.readValue(json, MskScramAssociations.class);+ associations.putForAccount(ACCOUNT, cluster.getClusterArn(), restored);+ MskScramSecretsService restarted =+ new MskScramSecretsService(+ storageFactory, config, clusterService, redpanda, secrets, kms, mapper);++ assertEquals(+ List.of(SECRET),+ restarted.listScramSecrets(cluster.getClusterArn(), null, null).items());+ assertEquals("alice", restored.getAssociations().get(SECRET).getUsername());+ clearInvocations(redpanda);+ verifyNoInteractions(redpanda);+ }++ @Test+ void mockModeTracksAssociationsWithoutBrokerCalls() {+ // Mock mode maintains association lifecycle state while suppressing all broker operations.+ when(config.services().msk().mock()).thenReturn(true);+ associate(SECRET, version("alice", "password"));+ assertEquals("alice", association(SECRET).getUsername());+ service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET));+ verifyNoInteractions(redpanda);+ }++ @Test+ void failedClusterDeletionRetainsScramAssociationState() {+ // Failed cluster deletion retains both the cluster and its recoverable SCRAM association state.+ associate(SECRET, version("alice", "password"));+ doThrow(new AwsException("ServiceUnavailableException", "Broker unavailable.", 503))+ .when(redpanda)+ .stopContainer(cluster);++ assertThrows(+ AwsException.class, () -> clusterService.deleteCluster(cluster.getClusterArn()));++ assertNotNull(association(SECRET));+ assertTrue(clusters.get(cluster.getClusterArn()).isPresent());}}diff --git a/src/test/java/io/github/hectorvent/floci/services/msk/RedpandaManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/msk/RedpandaManagerTest.javaindex 124c4b69b4..744ec95003 100644--- a/src/test/java/io/github/hectorvent/floci/services/msk/RedpandaManagerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/msk/RedpandaManagerTest.java@@ -1,9 +1,11 @@package io.github.hectorvent.floci.services.msk;+import com.fasterxml.jackson.databind.ObjectMapper;import com.github.dockerjava.api.DockerClient;import com.github.dockerjava.api.exception.NotFoundException;import com.sun.net.httpserver.HttpServer;import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;@@ -26,11 +28,14 @@ import org.mockito.Mock;import org.mockito.junit.jupiter.MockitoExtension;import java.net.InetSocketAddress;+import java.nio.charset.StandardCharsets;import java.nio.file.Files;import java.nio.file.Path;import java.util.Map;import java.util.Optional;+import java.util.concurrent.atomic.AtomicReference;+import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;import static org.junit.jupiter.api.Assertions.assertThrows;@@ -101,7 +106,7 @@ class RedpandaManagerTest {ContainerBuilder containerBuilder = new ContainerBuilder(config, dockerHostResolver, embeddedDnsServer);manager = new RedpandaManager(containerBuilder, lifecycleManager, logStreamer, containerDetector,- config, regionResolver, portAllocator);+ config, regionResolver, portAllocator, new ObjectMapper());lenient().when(logStreamer.generateLogStreamName(any())).thenReturn("log-stream");lenient().when(regionResolver.getDefaultRegion()).thenReturn("us-east-1");@@ -181,6 +186,183 @@ class RedpandaManagerTest {return adminServer.getAddress().getPort();}+ private MskCluster clusterWithAdminPort(int adminHostPort) {+ when(lifecycleManager.resolveEndpoint("container-id", RedpandaManager.ADMIN_PORT, null))+ .thenReturn(new EndpointInfo("localhost", adminHostPort));+ MskCluster cluster = newCluster();+ cluster.setContainerId("container-id");+ cluster.setBootstrapBrokers("localhost:19092");+ return cluster;+ }++ @Test+ void createScramUser() throws Exception {+ int port = startFakeAdminServer();+ AtomicReference<String> body = new AtomicReference<>();+ AtomicReference<String> method = new AtomicReference<>();+ adminServer.createContext("/v1/security/users", exchange -> {+ if ("GET".equals(exchange.getRequestMethod())) {+ byte[] users = "[]".getBytes(StandardCharsets.UTF_8);+ exchange.sendResponseHeaders(200, users.length);+ exchange.getResponseBody().write(users);+ exchange.close();+ return;+ }+ method.set(exchange.getRequestMethod());+ body.set(new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8));+ exchange.sendResponseHeaders(200, -1);+ exchange.close();+ });++ manager.createUser(clusterWithAdminPort(port), "alice", "password");++ assertEquals("POST", method.get());+ assertEquals(Map.of("username", "alice", "password", "password", "algorithm", "SCRAM-SHA-512"),+ new ObjectMapper().readValue(body.get(), Map.class));+ }++ @Test+ void createScramUserUpdatesExistingUser() throws Exception {+ int port = startFakeAdminServer();+ AtomicReference<String> updatePath = new AtomicReference<>();+ AtomicReference<String> updateBody = new AtomicReference<>();+ AtomicReference<String> updateMethod = new AtomicReference<>();+ adminServer.createContext("/v1/security/users", exchange -> {+ if ("GET".equals(exchange.getRequestMethod())) {+ byte[] users = "[\"alice /+\"]".getBytes(StandardCharsets.UTF_8);+ exchange.sendResponseHeaders(200, users.length);+ exchange.getResponseBody().write(users);+ } else {+ updateMethod.set(exchange.getRequestMethod());+ updatePath.set(exchange.getRequestURI().getRawPath());+ updateBody.set(new String(exchange.getRequestBody().readAllBytes(), StandardCharsets.UTF_8));+ exchange.sendResponseHeaders(200, -1);+ }+ exchange.close();+ });++ manager.createUser(clusterWithAdminPort(port), "alice /+", "new-password");++ assertEquals("/v1/security/users/alice%20%2F%2B", updatePath.get());+ assertEquals("PUT", updateMethod.get());+ assertEquals("new-password", new ObjectMapper().readTree(updateBody.get()).get("password").textValue());+ }++ @Test+ void createScramUserSurfacesBrokerErrorsWithoutLeakingResponse() throws Exception {+ int port = startFakeAdminServer();+ adminServer.createContext("/v1/security/users", exchange -> {+ if ("GET".equals(exchange.getRequestMethod())) {+ byte[] users = "[]".getBytes(StandardCharsets.UTF_8);+ exchange.sendResponseHeaders(200, users.length);+ exchange.getResponseBody().write(users);+ exchange.close();+ return;+ }+ byte[] body = "sensitive-password".getBytes(StandardCharsets.UTF_8);+ exchange.sendResponseHeaders(500, body.length);+ exchange.getResponseBody().write(body);+ exchange.close();+ });++ AwsException error = assertThrows(AwsException.class,+ () -> manager.createUser(clusterWithAdminPort(port), "alice", "sensitive-password"));++ assertEquals("InternalServerErrorException", error.getErrorCode());+ assertFalse(error.getMessage().contains("sensitive-password"));+ }++ @Test+ void deleteScramUser() throws Exception {+ int port = startFakeAdminServer();+ AtomicReference<String> path = new AtomicReference<>();+ AtomicReference<String> method = new AtomicReference<>();+ adminServer.createContext("/v1/security/users", exchange -> {+ path.set(exchange.getRequestURI().getRawPath());+ method.set(exchange.getRequestMethod());+ exchange.sendResponseHeaders(200, -1);+ exchange.close();+ });++ manager.deleteUser(clusterWithAdminPort(port), "alice /+");++ assertEquals("DELETE", method.get());+ assertEquals("/v1/security/users/alice%20%2F%2B", path.get());+ }++ @Test+ void deleteScramUserToleratesAlreadyDeletedUser() throws Exception {+ int port = startFakeAdminServer();+ adminServer.createContext("/v1/security/users", exchange -> {+ exchange.sendResponseHeaders(404, -1);+ exchange.close();+ });++ assertDoesNotThrow(() -> manager.deleteUser(clusterWithAdminPort(port), "alice"));+ }++ @Test+ void checkScramUserExistsReturnsTrueWhenPresent() throws Exception {+ int port = startFakeAdminServer();+ adminServer.createContext("/v1/security/users", exchange -> {+ byte[] users = "[\"alice\", \"bob\"]".getBytes(StandardCharsets.UTF_8);+ exchange.sendResponseHeaders(200, users.length);+ exchange.getResponseBody().write(users);+ exchange.close();+ });++ MskCluster cluster = clusterWithAdminPort(port);+ assertTrue(manager.listUsers(cluster).contains("alice"));+ }++ @Test+ void checkScramUserExistsReturnsFalseWhenUserNotFound() throws Exception {+ int port = startFakeAdminServer();+ adminServer.createContext("/v1/security/users", exchange -> {+ byte[] users = "[\"bob\"]".getBytes(StandardCharsets.UTF_8);+ exchange.sendResponseHeaders(200, users.length);+ exchange.getResponseBody().write(users);+ exchange.close();+ });++ MskCluster cluster = clusterWithAdminPort(port);+ assertFalse(manager.listUsers(cluster).contains("alice"));+ }++ @Test+ void deleteScramUserReportsBrokerFailure() throws Exception {+ int port = startFakeAdminServer();+ adminServer.createContext("/v1/security/users", exchange -> {+ exchange.sendResponseHeaders(503, -1);+ exchange.close();+ });+ MskCluster cluster = clusterWithAdminPort(port);+ AwsException error = assertThrows(AwsException.class, () -> manager.deleteUser(cluster, "alice"));+ assertEquals("InternalServerErrorException", error.getErrorCode());+ }++ @Test+ void sendAdminRequestRejectsClusterWithoutBroker() {+ AwsException error = assertThrows(AwsException.class,+ () -> manager.sendAdminRequest(newCluster(), "/v1/security/users", "GET", null));+ assertEquals("ServiceUnavailableException", error.getErrorCode());+ }++ @Test+ void adminRequestsReportMissingBrokerWhenEndpointResolutionFails() {+ when(lifecycleManager.resolveEndpoint("container-fail", RedpandaManager.ADMIN_PORT, null))+ .thenThrow(new NotFoundException("container not found"));++ MskCluster cluster = newCluster();+ cluster.setContainerId("container-fail");+ cluster.setBootstrapBrokers("localhost:19092");++ assertFalse(manager.isReady(cluster));+ AwsException error = assertThrows(AwsException.class, () -> manager.deleteUser(cluster, "alice"));+ assertEquals("ServiceUnavailableException", error.getErrorCode());+ assertEquals("Cluster has no backing Kafka broker.", error.getMessage());+ }+@Testvoid isReadyPollsTheCorrectAdminReadinessPathInNativeMode() throws Exception {int adminHostPort = startFakeAdminServer();@@ -198,7 +380,7 @@ class RedpandaManagerTest {}@Test- void adminReadyUrlResolvesFromThePersistedContainerIdOnTheBrokersNetworkAfterRestart() {+ void adminBaseUrlResolvesFromThePersistedContainerIdOnTheBrokersNetworkAfterRestart() {when(config.services().dockerNetwork()).thenReturn(Optional.of("floci-net"));when(lifecycleManager.resolveEndpoint("container-460", RedpandaManager.ADMIN_PORT, "floci-net")).thenReturn(new EndpointInfo("172.18.0.9", RedpandaManager.ADMIN_PORT));@@ -209,7 +391,7 @@ class RedpandaManagerTest {cluster.setContainerId("container-460");cluster.setBootstrapBrokers("localhost:9300");- assertEquals("http://172.18.0.9:9644/v1/status/ready", manager.adminReadyUrl(cluster),+ assertEquals("http://172.18.0.9:9644", manager.adminBaseUrl(cluster),"readiness must not depend on in-memory state or on the bootstrap string's format");}