Light, fluffy, and always free - The AWS Local Emulator alternative
feat(msk): scheduled reconcile for SCRAM secret changes (#6384)
Isak Lindbeckbc18cbbdaa39
diff --git a/docs/services/msk.md b/docs/services/msk.mdindex 5ce00f86da..eb9fd28fd6 100644--- a/docs/services/msk.md+++ b/docs/services/msk.md@@ -40,6 +40,7 @@ Floci emulates Amazon MSK by orchestrating **Redpanda** containers. This provide| `FLOCI_SERVICES_MSK_DEFAULT_IMAGE` | `redpandadata/redpanda:latest` | Docker image for Redpanda (Kafka) containers || `FLOCI_SERVICES_MSK_KAFKA_HOST_PORT_BASE` | `9300` | First host port a broker's Kafka listener can be published on || `FLOCI_SERVICES_MSK_KAFKA_HOST_PORT_MAX` | `9399` | Last host port a broker's Kafka listener can be published on |+| `FLOCI_SERVICES_MSK_SCRAM_RECONCILIATION_INTERVAL_MS` | `5000` | SCRAM secret reconciliation interval (ms) || `FLOCI_SERVICES_MSK_BOOTSTRAP_HOSTNAME` | _(unset)_ | Floci in a container only: hostname the host listener advertises, and when set, `GetBootstrapBrokers` returns `<container-name>:9092,<hostname>:<host port>` instead of the Docker-network address |## How it works@@ -132,6 +133,19 @@ The secrets must contain JSON such as `{"username":"alice","password":"pw"}`.`unprocessedScramSecrets`; reassociation and removal of an unassociated ARN return`InvalidSecretArn`.+Floci schedules SCRAM reconciliation jobs every five seconds by default, measured as+a delay after each scheduling sweep. Sweeps do not wait for broker operations to finish,+and clusters with queued or running jobs are skipped. Set+`FLOCI_SERVICES_MSK_SCRAM_RECONCILIATION_INTERVAL_MS` to change this delay in milliseconds.+This is Floci's timing, not an AWS synchronization guarantee.++When an associated secret is scheduled for deletion or permanently deleted, reconciliation+revokes its broker user but retains the association in `ListScramSecrets`. Restoring a secret+whose deletion was scheduled causes reconciliation to recreate its broker user using the+current credentials, without requiring reassociation. Permanently deleted secrets cannot+be restored; creating a secret with the same name produces a different ARN and requires+a new association.+## Examples```bashdiff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex 9405bd460a..4837631ef9 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -1575,6 +1575,9 @@ public interface EmulatorConfig {@WithDefault("9399")int kafkaHostPortMax();+ @WithDefault("5000")+ long scramReconciliationIntervalMs();+/*** Only applies when Floci runs in a container. Each broker then has a second Kafka* listener, published to a host port from the kafka-host-port range so clients on thediff --git a/src/main/java/io/github/hectorvent/floci/services/msk/MskScramSecretsService.java b/src/main/java/io/github/hectorvent/floci/services/msk/MskScramSecretsService.javaindex 59f1d77335..a5b18834a7 100644--- a/src/main/java/io/github/hectorvent/floci/services/msk/MskScramSecretsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/msk/MskScramSecretsService.java@@ -15,17 +15,17 @@ import io.github.hectorvent.floci.services.kms.KmsService;import io.github.hectorvent.floci.services.kms.model.KmsKey;import io.github.hectorvent.floci.services.kms.model.KmsKeySpec;import io.github.hectorvent.floci.services.kms.model.KmsKeyUsage;+import io.github.hectorvent.floci.services.msk.model.ClusterState;import io.github.hectorvent.floci.services.msk.model.MskCluster;import io.github.hectorvent.floci.services.msk.model.MskScramAssociations;import io.github.hectorvent.floci.services.msk.model.ScramSecretAssociation;import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;-import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService.BatchGetSecretValueError;-import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService.BatchGetSecretValueResult;-import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService.BatchSecretValue;import io.github.hectorvent.floci.services.secretsmanager.model.Secret;-import jakarta.annotation.PostConstruct;+import io.github.hectorvent.floci.services.secretsmanager.model.SecretVersion;+import io.quarkus.runtime.StartupEvent;import jakarta.annotation.PreDestroy;import jakarta.enterprise.context.ApplicationScoped;+import jakarta.enterprise.event.Observes;import jakarta.inject.Inject;import org.jboss.logging.Logger;@@ -33,9 +33,15 @@ import java.util.ArrayList;import java.util.HashSet;import java.util.List;import java.util.Map;+import java.util.Objects;+import java.util.Optional;+import java.util.Set;+import java.util.concurrent.ArrayBlockingQueue;import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.Executors;+import java.util.concurrent.RejectedExecutionException;import java.util.concurrent.ScheduledExecutorService;+import java.util.concurrent.ThreadPoolExecutor;import java.util.concurrent.TimeUnit;@ApplicationScoped@@ -51,9 +57,19 @@ public class MskScramSecretsService {private final SecretsManagerService secretsManagerService;private final KmsService kmsService;private final ObjectMapper objectMapper;+private final ScheduledExecutorService reconciler = Executors.newSingleThreadScheduledExecutor();+ private final ThreadPoolExecutor reconciliationWorkers = new ThreadPoolExecutor(+ 4, 4, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(128));+ private final Set<String> pendingReconciliations = ConcurrentHashMap.newKeySet();private final Map<String, Object> clusterLocks = new ConcurrentHashMap<>();+ private final Map<String, ClusterReconciliationFailure> clusterReconciliationFailures = new ConcurrentHashMap<>();+ private record ClusterReconciliationFailure(Class<?> type, String message) {}+ private final Map<ReconciliationKey, ReconciliationFailure> secretReconciliationFailures = new ConcurrentHashMap<>();+ private record ReconciliationKey(String clusterArn, String secretArn) {}+ private record ReconciliationFailure(String errorCode, String message) {}+@Injectpublic MskScramSecretsService(StorageFactory storageFactory, EmulatorConfig config, MskService mskService,RedpandaManager redpandaManager, SecretsManagerService secretsManagerService,@@ -68,37 +84,147 @@ public class MskScramSecretsService {this.objectMapper = objectMapper;}- @PostConstruct- public void init() {+ void onStart(@Observes StartupEvent ignored) {+ if (!config.services().msk().enabled()) {+ return;+ }+ long intervalMs = config.services().msk().scramReconciliationIntervalMs();+ if (intervalMs <= 0) {+ throw new IllegalArgumentException("floci.services.msk.scram-reconciliation-interval-ms must be greater than zero.");+ }reconciler.scheduleWithFixedDelay(() -> {try {- reconcileScramSecrets();+ scheduleScramReconciliation();} catch (RuntimeException e) {LOG.error("MSK SCRAM reconciliation failed; retrying on the next scheduled run", e);}- }, 5, 5, TimeUnit.SECONDS);+ }, intervalMs, intervalMs, TimeUnit.MILLISECONDS);}@PreDestroypublic void shutdown() {reconciler.shutdown();+ reconciliationWorkers.shutdownNow();}private Object clusterLock(String clusterArn) {return clusterLocks.computeIfAbsent(String.valueOf(clusterArn), ignored -> new Object());}- void reconcileScramSecrets() {- for (AccountAwareStorageBackend.AccountEntry<MskScramAssociations> entry- : storage.scanAllAccountEntries(key -> true)) {- synchronized (clusterLock(entry.key())) {- if (mskService.getClusterForAccount(entry.accountId(), entry.key()).isEmpty()) {- storage.deleteForAccount(entry.accountId(), entry.key());+ private void scheduleScramReconciliation() {+ int deferredClusters = 0;+ for (AccountAwareStorageBackend.AccountEntry<MskScramAssociations> entry : storage.scanAllAccountEntries(key -> true)) {+ String clusterArn = entry.key();+ if (!pendingReconciliations.add(clusterArn)) {+ continue;+ }+ try {+ reconciliationWorkers.execute(() -> runClusterReconciliation(entry));+ } catch (RejectedExecutionException e) {+ pendingReconciliations.remove(clusterArn);+ if (!reconciliationWorkers.isShutdown()) {+ deferredClusters++;}}}+ if (deferredClusters > 0) {+ LOG.warnv("MSK SCRAM reconciliation workers are full; retrying {0} clusters on the next scheduled run",+ deferredClusters);+ }}+ private void runClusterReconciliation(AccountAwareStorageBackend.AccountEntry<MskScramAssociations> entry) {+ String clusterArn = entry.key();+ try {+ reconcileCluster(clusterArn, entry.accountId());+ clusterReconciliationFailures.remove(clusterArn);+ } catch (RuntimeException e) {+ ClusterReconciliationFailure failure = new ClusterReconciliationFailure(e.getClass(), e.getMessage());+ if (!failure.equals(clusterReconciliationFailures.put(clusterArn, failure))) {+ LOG.errorv(e, "MSK SCRAM reconciliation failed for cluster {0}; retrying on the next scheduled run",+ clusterArn);+ } else {+ LOG.debugv(e, "MSK SCRAM reconciliation failed for cluster {0}; retrying on the next scheduled run",+ clusterArn);+ }+ } finally {+ pendingReconciliations.remove(clusterArn);+ }+ }++ void reconcileCluster(String clusterArn, String accountId) {+ synchronized (clusterLock(clusterArn)) {+ Optional<MskCluster> clusterGet = mskService.getClusterForAccount(accountId, clusterArn);+ if (clusterGet.isEmpty()) {+ storage.deleteForAccount(accountId, clusterArn);+ secretReconciliationFailures.keySet().removeIf(key -> key.clusterArn().equals(clusterArn));+ } else {+ reconcileScramSecretsForCluster(clusterGet.get(), clusterArn);+ }+ }+ }++ private void reconcileScramSecretsForCluster(MskCluster cluster, String clusterArn) {+ if (mskService.isServerless(cluster) || cluster.getState() != ClusterState.ACTIVE) {+ return;+ }+ MskScramAssociations clusterAssociations = associationsFor(clusterArn);+ for (String secretArn : new ArrayList<>(clusterAssociations.getAssociations().keySet())) {+ ReconciliationKey reconciliationKey = new ReconciliationKey(clusterArn, secretArn);+ try {+ reconcileScramSecret(secretArn, clusterAssociations, cluster);+ secretReconciliationFailures.remove(reconciliationKey);+ } catch (AwsException e) {+ ReconciliationFailure failure = new ReconciliationFailure(e.getErrorCode(), e.getMessage());+ if (!failure.equals(secretReconciliationFailures.put(reconciliationKey, failure))) {+ LOG.warnv("Could not reconcile MSK SCRAM secret {0} for cluster {1}: {2}",+ secretArn, clusterArn, e.getErrorCode());+ } else {+ LOG.debugv("Could not reconcile MSK SCRAM secret {0} for cluster {1}: {2}",+ secretArn, clusterArn, e.getErrorCode());+ }+ }+ }+ }++ private void reconcileScramSecret(String secretArn, MskScramAssociations clusterAssociations, MskCluster cluster) {+ ScramSecretAssociation association = clusterAssociations.getAssociations().get(secretArn);+ if (association == null) {+ // Should never happen+ throw new AwsException("InternalServerErrorException", "SCRAM association is missing during reconciliation.", 500);+ }+ if (association.isDisassociating()) {+ // Recover if disassociation was interrupted+ removeScramAssociation(cluster, clusterAssociations, secretArn);+ return;+ }+ Optional<Secret> secret = describeSecret(secretArn);+ if (secret.isEmpty() || secret.get().getDeletedDate() != null) {+ // Revoke credentials for secret that are deleted or scheduled for deletion, but keep the association+ revokeScramCredentials(cluster, clusterAssociations, association);+ return;+ }++ SecretVersion secretVersion = getCurrentSecretVersion(secret.get());+ boolean versionIdChanged = !Objects.equals(secretVersion.getVersionId(), association.getAppliedVersionId());+ boolean containerIdChanged = !Objects.equals(cluster.getContainerId(), association.getAppliedContainerId());+ // Reapply if the secret or cluster container has changed+ // Reapply if credentials where revoked, which can happen if the secret was deleted and then restored+ if (association.isCredentialsRevoked() || versionIdChanged || containerIdChanged) {+ applyScramCredentials(cluster, clusterAssociations, association, secretVersion);+ }+ }++ private void revokeScramCredentials(MskCluster cluster, MskScramAssociations state, ScramSecretAssociation association) {+ if (association.isCredentialsRevoked()) {+ // Skip repeated delete calls to the cluster+ return;+ }+ deleteScramUser(cluster, state, association);+ association.setCredentialsRevoked(true);+ putAssociations(cluster.getClusterArn(), state);+ }+public List<Map<String, String>> batchAssociateScramSecret(String clusterArn, List<String> secretArns) {synchronized (clusterLock(clusterArn)) {MskCluster cluster = scramCluster(clusterArn);@@ -118,8 +244,10 @@ public class MskScramSecretsService {}validateSecretEncryption(secret);// Amazon MSK reads only the AWSCURRENT version of an associated secret- BatchSecretValue secretValue = readSecret(secretArn);- applyScramCredentials(cluster, state, secretArn, secretValue);+ SecretVersion secretVersion = getCurrentSecretVersion(secret);+ ScramSecretAssociation association = state.getAssociations()+ .computeIfAbsent(secretArn, ignored -> new ScramSecretAssociation());+ applyScramCredentials(cluster, state, association, secretVersion);} catch (AwsException e) {if (newAssociation && state.getAssociations().remove(secretArn) != null) {putAssociations(clusterArn, state);@@ -248,14 +376,21 @@ public class MskScramSecretsService {}private void applyScramCredentials(MskCluster cluster, MskScramAssociations state,- String secretArn, BatchSecretValue secretValue) {- ScramCredentials credentials = readScramCredentials(secretValue.secretString());+ ScramSecretAssociation association, SecretVersion secretValue) {+ ScramCredentials credentials = readScramCredentials(secretValue.getSecretString());+ boolean ownershipChanged = !Objects.equals(association.getUsername(), credentials.username());+ if (ownershipChanged) {+ deleteScramUser(cluster, state, association);+ association.setUsername(credentials.username());+ // Persist ownership changes before updating the broker; mark the version as applied only after success.+ putAssociations(cluster.getClusterArn(), state);+ }if (!config.services().msk().mock()) {redpandaManager.createUser(cluster, credentials.username(), credentials.password());}- ScramSecretAssociation association = new ScramSecretAssociation();- association.setUsername(credentials.username());- state.getAssociations().put(secretArn, association);+ association.setAppliedVersionId(secretValue.getVersionId());+ association.setAppliedContainerId(cluster.getContainerId());+ association.setCredentialsRevoked(false);putAssociations(cluster.getClusterArn(), state);}@@ -264,33 +399,64 @@ public class MskScramSecretsService {if (association == null) {return;}- if (association.getUsername() != null && !config.services().msk().mock()) {- redpandaManager.deleteUser(cluster, association.getUsername());+ if (!association.isDisassociating()) {+ association.setDisassociating(true);+ // Persist the removal intent before deleting broker credentials so a failure does not lose it.+ putAssociations(cluster.getClusterArn(), state);}+ deleteScramUser(cluster, state, association);state.getAssociations().remove(secretArn);putAssociations(cluster.getClusterArn(), state);+ secretReconciliationFailures.remove(new ReconciliationKey(cluster.getClusterArn(), secretArn));}- private BatchSecretValue readSecret(String secretArn) {- // Exact-ARN batch reads select AWSCURRENT without updating access timestamps.- BatchGetSecretValueResult result = secretsManagerService.batchGetSecretValue(- List.of(secretArn), AwsArnUtils.parse(secretArn).region());- if (result != null && result.values() != null && result.errors() != null) {- if (result.values().isEmpty() && result.errors().size() == 1) {- BatchGetSecretValueError error = result.errors().getFirst();- if (error != null && secretArn.equals(error.secretId())- && error.errorCode() != null && error.message() != null) {- int status = "AccessDeniedException".equals(error.errorCode()) ? 403 : 400;- throw new AwsException(error.errorCode(), error.message(), status);- }- } else if (result.values().size() == 1 && result.errors().isEmpty()) {- BatchSecretValue value = result.values().getFirst();- if (value != null && secretArn.equals(value.arn()) && value.versionId() != null) {- return value;+ private void deleteScramUser(MskCluster cluster, MskScramAssociations state, ScramSecretAssociation association) {+ String username = association.getUsername();+ // Revoked credentials have already been deleted+ if (username != null && !association.isCredentialsRevoked()) {+ if (!config.services().msk().mock()) {+ redpandaManager.deleteUser(cluster, username);+ }+ // Allow a user existing in multiple secrets to be reconciled again+ invalidateUserAssociations(state, username);+ }+ }++ private void invalidateUserAssociations(MskScramAssociations state, String username) {+ for (ScramSecretAssociation association : state.getAssociations().values()) {+ if (username.equals(association.getUsername())) {+ association.setAppliedVersionId(null);+ association.setAppliedContainerId(null);+ }+ }+ }++ private Optional<Secret> describeSecret(String secretArn) {+ try {+ Secret secret = secretsManagerService.describeSecret(secretArn, AwsArnUtils.parse(secretArn).region());+ return Optional.of(secret);+ } catch (AwsException e) {+ if ("ResourceNotFoundException".equals(e.getErrorCode())) {+ return Optional.empty();+ }+ throw e;+ }+ }++ private SecretVersion getCurrentSecretVersion(Secret secret) {+ if (secret.getDeletedDate() != null) {+ throw new AwsException("InternalServerError",+ SecretsManagerService.SCHEDULED_DELETION_ERROR_MESSAGE, 500);+ }+ if (secret.getVersions() != null) {+ for (SecretVersion version : secret.getVersions().values()) {+ if (version.getVersionStages() != null && version.getVersionStages().contains("AWSCURRENT")) {+ return version;}}}- throw new AwsException("InternalServerError", "Unexpected Secrets Manager batch response.", 500);+ throw new AwsException("ResourceNotFoundException",+ "Secrets Manager can't find the specified secret value for staging label: AWSCURRENT", 400);}record ScramCredentials(String username, String password) {diff --git a/src/main/java/io/github/hectorvent/floci/services/msk/model/ScramSecretAssociation.java b/src/main/java/io/github/hectorvent/floci/services/msk/model/ScramSecretAssociation.javaindex 668656e53f..f724ff0c43 100644--- a/src/main/java/io/github/hectorvent/floci/services/msk/model/ScramSecretAssociation.java+++ b/src/main/java/io/github/hectorvent/floci/services/msk/model/ScramSecretAssociation.java@@ -6,6 +6,10 @@ import io.quarkus.runtime.annotations.RegisterForReflection;public class ScramSecretAssociation {private String username;+ private String appliedVersionId;+ private String appliedContainerId;+ private boolean credentialsRevoked;+ private boolean disassociating;public String getUsername() {return username;@@ -15,4 +19,35 @@ public class ScramSecretAssociation {this.username = username;}+ public String getAppliedVersionId() {+ return appliedVersionId;+ }++ public void setAppliedVersionId(String appliedVersionId) {+ this.appliedVersionId = appliedVersionId;+ }++ public String getAppliedContainerId() {+ return appliedContainerId;+ }++ public void setAppliedContainerId(String appliedContainerId) {+ this.appliedContainerId = appliedContainerId;+ }++ public boolean isCredentialsRevoked() {+ return credentialsRevoked;+ }++ public void setCredentialsRevoked(boolean credentialsRevoked) {+ this.credentialsRevoked = credentialsRevoked;+ }++ public boolean isDisassociating() {+ return disassociating;+ }++ public void setDisassociating(boolean disassociating) {+ this.disassociating = disassociating;+ }}diff --git a/src/main/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerService.java b/src/main/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerService.javaindex 0f6c79846c..44702eff50 100644--- a/src/main/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerService.java+++ b/src/main/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerService.java@@ -62,6 +62,9 @@ public class SecretsManagerService implements ResourceProvider {private static final String PENDING_DELETION_STATE = "PendingDeletion";private static final Pattern SECRET_NAME_PATTERN = Pattern.compile("[a-zA-Z0-9/_+=.@-]+");+ public static final String SCHEDULED_DELETION_ERROR_MESSAGE =+ "You can't perform this operation on the secret because it was marked for deletion.";+private final StorageBackend<String, Secret> store;private final int defaultRecoveryWindowDays;private final RegionResolver regionResolver;@@ -1566,7 +1569,7 @@ public class SecretsManagerService implements ResourceProvider {Secret secret = resolveSecret(secretId, region);if (secret.getDeletedDate() != null) {errors.add(new BatchGetSecretValueError(secretId, "InvalidRequestException",- "You can't perform this operation on the secret because it was marked for deletion."));+ SCHEDULED_DELETION_ERROR_MESSAGE));continue;}@@ -1792,7 +1795,7 @@ public class SecretsManagerService implements ResourceProvider {private void throwIfPendingDeletion(Secret secret) {if (secret.getDeletedDate() != null) {throw new AwsException("InvalidRequestException",- "You can't perform this operation on the secret because it was marked for deletion.", 400);+ SCHEDULED_DELETION_ERROR_MESSAGE, 400);}}diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex fa389a7869..bc68b88a7c 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -478,6 +478,7 @@ floci:msk:enabled: truemock: false+ scram-reconciliation-interval-ms: 5000default-image: "redpandadata/redpanda:latest"kafka-host-port-base: 9300kafka-host-port-max: 9399diff --git a/src/test/java/io/github/hectorvent/floci/services/msk/MskScramServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/msk/MskScramServiceTest.javaindex 6b41f59ff8..647573508b 100644--- a/src/test/java/io/github/hectorvent/floci/services/msk/MskScramServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/msk/MskScramServiceTest.java@@ -13,16 +13,15 @@ import io.github.hectorvent.floci.services.kms.KmsService;import io.github.hectorvent.floci.services.kms.model.KmsKey;import io.github.hectorvent.floci.services.kms.model.KmsKeySpec;import io.github.hectorvent.floci.services.kms.model.KmsKeyUsage;+import io.github.hectorvent.floci.services.msk.model.ClusterState;import io.github.hectorvent.floci.services.msk.model.MskCluster;import io.github.hectorvent.floci.services.msk.model.MskConfiguration;import io.github.hectorvent.floci.services.msk.model.MskScramAssociations;import io.github.hectorvent.floci.services.msk.model.ScramSecretAssociation;import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;-import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService.BatchGetSecretValueError;-import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService.BatchGetSecretValueResult;-import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService.BatchSecretValue;import io.github.hectorvent.floci.services.secretsmanager.model.Secret;import io.github.hectorvent.floci.services.secretsmanager.model.SecretVersion;+import io.github.hectorvent.floci.testutil.LogCapture;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Named;import org.junit.jupiter.api.Test;@@ -31,6 +30,8 @@ import org.junit.jupiter.params.provider.Arguments;import org.junit.jupiter.params.provider.MethodSource;import org.junit.jupiter.params.provider.ValueSource;+import java.time.Duration;+import java.time.Instant;import java.util.Collections;import java.util.List;import java.util.Map;@@ -40,8 +41,10 @@ import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;import java.util.concurrent.Future;import java.util.concurrent.TimeUnit;+import java.util.logging.LogRecord;import java.util.stream.Stream;+import static org.awaitility.Awaitility.await;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertFalse;import static org.junit.jupiter.api.Assertions.assertNotNull;@@ -95,6 +98,7 @@ class MskScramServiceTest {when(storageFactory.<MskConfiguration>create(eq("msk"), eq("msk-configurations.json"), any())).thenReturn(AccountAwareStorageBackend.inMemory(ACCOUNT));config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);+ when(config.services().msk().enabled()).thenReturn(true);when(config.services().msk().mock()).thenReturn(true);regionResolver = mock(RegionResolver.class);when(regionResolver.getRegion()).thenReturn("us-east-1");@@ -157,6 +161,127 @@ class MskScramServiceTest {}}+ @Test+ void startupSkipsReconciliationWhenMskIsDisabled() {+ when(config.services().msk().enabled()).thenReturn(false);++ try {+ service.onStart(null);++ verify(config.services().msk(), never()).scramReconciliationIntervalMs();+ verifyNoInteractions(redpanda);+ } finally {+ service.shutdown();+ }+ }++ @ParameterizedTest+ @ValueSource(longs = {0, -1})+ void reconciliationRejectsNonPositiveInterval(long intervalMs) {+ when(config.services().msk().scramReconciliationIntervalMs()).thenReturn(intervalMs);++ try {+ IllegalArgumentException error = assertThrows(IllegalArgumentException.class, () -> service.onStart(null));+ assertEquals("floci.services.msk.scram-reconciliation-interval-ms must be greater than zero.",+ error.getMessage());+ } finally {+ service.shutdown();+ }+ }++ @Test+ void scheduledReconciliationKeepsOtherClustersSyncingWhileOneBrokerIsBlocked() throws Exception {+ when(config.services().msk().mock()).thenReturn(true);+ MskCluster healthyCluster = clusterService.createCluster("healthy-scram");+ when(config.services().msk().mock()).thenReturn(false);+ String healthySecret = SECRET + "-healthy";+ associate(SECRET, version("alice", "original"));+ setSecretValue(healthySecret, version("bob", "original"));+ assertTrue(service.batchAssociateScramSecret(healthyCluster.getClusterArn(), List.of(healthySecret)).isEmpty());++ CountDownLatch operationForFirstClusterStarted = new CountDownLatch(1);+ CountDownLatch operationForSecondClusterDone = new CountDownLatch(1);+ CountDownLatch operationForFirstClusterDone = new CountDownLatch(1);+ doAnswer(invocation -> {+ operationForFirstClusterStarted.countDown();+ try {+ assertTrue(operationForSecondClusterDone.await(30, TimeUnit.SECONDS));+ } finally {+ operationForFirstClusterDone.countDown();+ }+ return null;+ }).when(redpanda).createUser(cluster, "alice", "rotated");+ setSecretValue(SECRET, version("alice", "rotated"));+ when(config.services().msk().scramReconciliationIntervalMs()).thenReturn(5L);++ try {+ service.onStart(null);+ assertTrue(operationForFirstClusterStarted.await(5, TimeUnit.SECONDS));++ // More rotations than workers expose duplicate blocked jobs exhausting the pool.+ for (int rotation = 1; rotation <= 6; rotation++) {+ String password = "healthy-rotation-" + rotation;+ setSecretValue(healthySecret, version("bob", password));+ await()+ .atMost(Duration.ofSeconds(5))+ .untilAsserted(() -> verify(redpanda).createUser(healthyCluster, "bob", password));+ }++ verify(redpanda, times(1)).createUser(cluster, "alice", "rotated");+ } finally {+ operationForSecondClusterDone.countDown();+ try {+ assertTrue(operationForFirstClusterDone.await(5, TimeUnit.SECONDS));+ } finally {+ service.shutdown();+ }+ }+ }++ @Test+ void scheduledReconciliationLogsRepeatedFailuresAndRecovers() {+ associate(SECRET, version("alice", "original"));+ setSecretValue(SECRET, version("alice", "rotated"));+ doThrow(new IllegalStateException("Broker unavailable"))+ .doThrow(new IllegalStateException("Broker unavailable"))+ .doThrow(new IllegalArgumentException("Invalid broker"))+ .doNothing()+ .when(redpanda)+ .createUser(cluster, "alice", "rotated");+ when(config.services().msk().scramReconciliationIntervalMs()).thenReturn(5L);++ try {+ List<LogRecord> logs = LogCapture.capture(MskScramSecretsService.class, () -> {+ service.onStart(null);+ await()+ .atMost(Duration.ofSeconds(5))+ .untilAsserted(() -> verify(redpanda, times(4)).createUser(cluster, "alice", "rotated"));+ });+ assertEquals(List.of("ERROR", "DEBUG", "ERROR"), logLevels(logs));+ } finally {+ service.shutdown();+ }+ }++ @Test+ void scheduledReconciliationAppliesRotationsUsingConfiguredInterval() {+ associate(SECRET, version("alice", "original"));+ when(config.services().msk().scramReconciliationIntervalMs()).thenReturn(5L);+ setSecretValue(SECRET, version("alice", "rotated"));++ try {+ service.onStart(null);+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() ->+ verify(redpanda).createUser(cluster, "alice", "rotated"));++ setSecretValue(SECRET, version("alice", "rotated-again"));+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() ->+ verify(redpanda).createUser(cluster, "alice", "rotated-again"));+ } finally {+ service.shutdown();+ }+ }+private void associate(String arn, SecretVersion version) {setSecretValue(arn, version);assertTrue(@@ -164,18 +289,24 @@ class MskScramServiceTest {}private void setSecretValue(String arn, SecretVersion version) {- BatchGetSecretValueResult batchGetSecretValueResult = new BatchGetSecretValueResult(List.of(new BatchSecretValue(- arn, "AmazonMSK_fixture", version.getSecretString(), version.getSecretBinary(),- version.getVersionId(), List.of("AWSCURRENT"), version.getCreatedDate())- ), List.of());- doReturn(batchGetSecretValueResult)- .when(secrets)- .batchGetSecretValue(List.of(arn), AwsArnUtils.parse(arn).region());+ Secret secret = new Secret();+ secret.setArn(arn);+ secret.setKmsKeyId("customer-key");+ version.setVersionStages(List.of("AWSCURRENT"));+ secret.setVersions(Map.of(version.getVersionId(), version));+ doReturn(secret).when(secrets).describeSecret(arn, AwsArnUtils.parse(arn).region());}private void setSecretReadError(String errorCode, String message) {- doReturn(new BatchGetSecretValueResult(List.of(), List.of(new BatchGetSecretValueError(SECRET, errorCode, message))))- .when(secrets).batchGetSecretValue(List.of(SECRET), "us-east-1");+ doThrow(new AwsException(errorCode, message, 400))+ .when(secrets).describeSecret(SECRET, "us-east-1");+ }++ private void scheduleDeletion(String arn) {+ Secret metadata = new Secret();+ metadata.setArn(arn);+ metadata.setDeletedDate(Instant.now().plusSeconds(86400));+ doReturn(metadata).when(secrets).describeSecret(arn, "us-east-1");}private MskScramAssociations state() {@@ -238,7 +369,8 @@ class MskScramServiceTest {String deletedCluster = cluster.getClusterArn().replace(ACCOUNT, otherAccount);associations.putForAccount(otherAccount, deletedCluster, new MskScramAssociations());- service.reconcileScramSecrets();+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);+ service.reconcileCluster(deletedCluster, otherAccount);assertTrue(associations.getForAccount(ACCOUNT, cluster.getClusterArn()).isPresent());assertTrue(associations.getForAccount(otherAccount, deletedCluster).isEmpty());@@ -251,7 +383,7 @@ class MskScramServiceTest {AwsException failure = new AwsException("InternalServerErrorException", "Lookup failed.", 500);doThrow(failure).when(clusterService).getClusterForAccount(ACCOUNT, cluster.getClusterArn());- assertEquals(failure, assertThrows(AwsException.class, () -> service.reconcileScramSecrets()));+ assertEquals(failure, assertThrows(AwsException.class, () -> service.reconcileCluster(cluster.getClusterArn(), ACCOUNT)));assertTrue(associations.getForAccount(ACCOUNT, cluster.getClusterArn()).isPresent());}@@ -291,6 +423,8 @@ class MskScramServiceTest {when(secrets.describeSecret(SECRET, "us-east-1")).thenReturn(metadata);when(kms.describeKey(keyReference, "us-east-1")).thenReturn(new KmsKey());setSecretValue(SECRET, version("alice", "password"));+ Secret secret = secrets.describeSecret(SECRET, "us-east-1");+ secret.setKmsKeyId(keyReference);assertTrue(service.batchAssociateScramSecret(cluster.getClusterArn(), List.of(SECRET)).isEmpty());@@ -348,16 +482,13 @@ class MskScramServiceTest {@Testvoid rejectsReturnedSecretArnForDifferentSecret() {- when(secrets.batchGetSecretValue(List.of(SECRET), "us-east-1"))- .thenReturn(- new BatchGetSecretValueResult(List.of(new BatchSecretValue(- SECRET + "Other", "AmazonMSK_fixture", "{\"username\":\"alice\",\"password\":\"password\"}",- null, "version", List.of("AWSCURRENT"), null)), List.of())- );+ Secret secret = new Secret();+ secret.setArn(SECRET + "Other");+ when(secrets.describeSecret(SECRET, "us-east-1")).thenReturn(secret);List<Map<String, String>> failures = service.batchAssociateScramSecret(cluster.getClusterArn(), List.of(SECRET));- assertEquals("InternalServerError", failures.getFirst().get("errorCode"));+ assertEquals("InvalidSecretArn", failures.getFirst().get("errorCode"));assertTrue(service.listScramSecrets(cluster.getClusterArn(), null, null).items().isEmpty());}@@ -414,7 +545,7 @@ class MskScramServiceTest {assertEquals(1, failures.size(), keyKind);assertEquals("BadRequestException", failures.getFirst().get("errorCode"), keyKind);assertNull(association(invalidArn));- assertEquals("alice", association(SECRET).getUsername(), keyKind);+ assertEquals(original.getVersionId(), association(SECRET).getAppliedVersionId(), keyKind);}verifyNoInteractions(redpanda);assertTrue(service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET)).isEmpty());@@ -489,6 +620,7 @@ class MskScramServiceTest {List.of(SECRET),service.listScramSecrets(cluster.getClusterArn(), null, null).items());assertEquals("alice", association(SECRET).getUsername());+ assertEquals(original.getVersionId(), association(SECRET).getAppliedVersionId());verifyNoInteractions(redpanda, secrets, kms);verify(associations, never()).putForAccount(anyString(), anyString(), any());}@@ -513,22 +645,49 @@ class MskScramServiceTest {}@Test- void associationReadsCurrentVersionThroughExactArnBatchGetSecretValue() {+ void associationReadsCurrentVersionFromSecretMetadata() {// Association applies the AWSCURRENT secret value and records its version only after success.SecretVersion version = version("alice", "password");setSecretValue(SECRET, version);+ Secret secret = secrets.describeSecret(SECRET, "us-east-1");+ SecretVersion previous = version("bob", "old-password");+ previous.setVersionStages(List.of("AWSPREVIOUS"));+ secret.setVersions(Map.of(version.getVersionId(), version, previous.getVersionId(), previous));+ secret.setCurrentVersionId(previous.getVersionId());+ Instant lastAccessed = Instant.parse("2026-01-01T00:00:00Z");+ secret.setLastAccessedDate(lastAccessed);+ clearInvocations(secrets);assertTrue(service.batchAssociateScramSecret(cluster.getClusterArn(), List.of(SECRET)).isEmpty());- verify(secrets).batchGetSecretValue(List.of(SECRET), "us-east-1");+ verify(secrets).describeSecret(SECRET, "us-east-1");+ verify(secrets, never()).batchGetSecretValue(any(), anyString());verify(secrets, never()).getSecretValue(anyString(), any(), any(), anyString());verify(secrets, never()).batchGetSecretValueByFilters(any(), anyString());verify(redpanda).createUser(cluster, "alice", "password");+ assertEquals(version.getVersionId(), association(SECRET).getAppliedVersionId());+ assertEquals(lastAccessed, secret.getLastAccessedDate());+ }++ @Test+ void associationOfSecretScheduledForDeletionReportsInternalServerError() {+ setSecretValue(SECRET, version("alice", "password"));+ Secret secret = secrets.describeSecret(SECRET, "us-east-1");+ secret.setDeletedDate(Instant.now().plusSeconds(86400));++ List<Map<String, String>> failures =+ service.batchAssociateScramSecret(cluster.getClusterArn(), List.of(SECRET));++ assertEquals(List.of(Map.of("secretArn", SECRET, "errorCode", "InternalServerError",+ "errorMessage", "You can't perform this operation on the secret because it was marked for deletion.")),+ failures);+ assertNull(association(SECRET));+ verifyNoInteractions(redpanda);}@ParameterizedTest@ValueSource(strings = {"ResourceNotFoundException", "InvalidRequestException", "AccessDeniedException"})- void batchReadErrorsAreReportedPerSecretWithoutBlockingOtherAssociations(String errorCode) {+ void secretLookupErrorsAreReportedPerSecretWithoutBlockingOtherAssociations(String errorCode) {setSecretReadError(errorCode, "Secret cannot be read.");String validArn = SECRET + "-valid";setSecretValue(validArn, version("bob", "password"));@@ -544,18 +703,21 @@ class MskScramServiceTest {}@Test- void associationIsStoredAfterBrokerSideEffect() {+ void associationIntentIsStoredBeforeBrokerUpdate() {+ // Persisted ownership must precede broker creation so an interrupted operation remains recoverable.doAnswer(invocation -> {- verify(associations, never()).putForAccount(anyString(), anyString(), any());+ ScramSecretAssociation intent = association(SECRET);+ assertNotNull(intent);+ assertEquals("alice", intent.getUsername());+ assertNull(intent.getAppliedVersionId());return null;}).when(redpanda).createUser(cluster, "alice", "password");associate(SECRET, version("alice", "password"));- assertEquals("alice", association(SECRET).getUsername());- verify(associations).putForAccount(eq(ACCOUNT), eq(cluster.getClusterArn()), any());+ assertNotNull(association(SECRET).getAppliedVersionId());}@Test@@ -574,6 +736,7 @@ class MskScramServiceTest {associate(SECRET, version);assertEquals("alice", association(SECRET).getUsername());+ assertEquals(version.getVersionId(), association(SECRET).getAppliedVersionId());assertEquals(List.of(SECRET), service.listScramSecrets(cluster.getClusterArn(), null, null).items());}@@ -587,6 +750,7 @@ class MskScramServiceTest {service.batchAssociateScramSecret(cluster.getClusterArn(), List.of(SECRET));assertEquals("InvalidSecretArn", failures.getFirst().get("errorCode"));assertEquals("alice", association(SECRET).getUsername());+ assertEquals(version.getVersionId(), association(SECRET).getAppliedVersionId());verifyNoInteractions(redpanda);verify(associations, never()).putForAccount(anyString(), anyString(), any());assertEquals(List.of(SECRET), service.listScramSecrets(cluster.getClusterArn(), null, null).items());@@ -597,8 +761,11 @@ class MskScramServiceTest {// Existing membership determines duplicate rejection even when the secret cannot be read.associate(SECRET, version("alice", "password"));clearInvocations(redpanda);- for (String errorCode : List.of("ResourceNotFoundException", "InvalidRequestException")) {- setSecretReadError(errorCode, "Secret cannot be read.");+ for (boolean scheduledForDeletion : List.of(false, true)) {+ Secret secret = new Secret();+ secret.setArn(SECRET);+ secret.setDeletedDate(scheduledForDeletion ? Instant.now().plusSeconds(86400) : null);+ doReturn(secret).when(secrets).describeSecret(SECRET, "us-east-1");List<Map<String, String>> failures =service.batchAssociateScramSecret(cluster.getClusterArn(), List.of(SECRET));@@ -667,32 +834,16 @@ class MskScramServiceTest {assertEquals(1, service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET)).size());assertEquals("alice", association(SECRET).getUsername());+ assertTrue(association(SECRET).isDisassociating());clearInvocations(redpanda);setSecretValue(SECRET, version("charlie", "changed-during-cleanup"));doNothing().when(redpanda).deleteUser(cluster, "alice");- assertTrue(service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET)).isEmpty());+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);assertTrue(state().getAssociations().isEmpty());verify(redpanda).deleteUser(cluster, "alice");verify(redpanda, never()).createUser(any(), anyString(), anyString());}- @Test- void deletionRetryRemovesAssociationWithoutRewritingIt() {- associate(SECRET, version("alice", "password"));- doThrow(new AwsException("ServiceUnavailableException", "Broker unavailable.", 503))- .when(redpanda)- .deleteUser(cluster, "alice");- service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET));- clearInvocations(associations);- doNothing().when(redpanda).deleteUser(cluster, "alice");-- assertTrue(service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET)).isEmpty());-- verify(associations, never()).putForAccount(anyString(), anyString(), any());- verify(associations, times(1)).deleteForAccount(ACCOUNT, cluster.getClusterArn());- assertTrue(state().getAssociations().isEmpty());- }-@Testvoid listScramSecretsReturnsEachPageAndOnlyProvidesTokenWhenMoreResultsExist() {// Pagination exposes each association once and omits the token on the final page.@@ -727,6 +878,411 @@ class MskScramServiceTest {assertThrows(AwsException.class, () -> service.listScramSecrets("missing", null, null));}+ @Test+ void reconcileUpdatesPasswordAndSkipsUnchangedVersion() {+ // Reconciliation applies each new password version once and skips already applied versions.+ associate(SECRET, version("alice", "old-password"));+ clearInvocations(redpanda);+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);+ verifyNoInteractions(redpanda);++ SecretVersion changed = version("alice", "new-password");+ setSecretValue(SECRET, changed);+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);+ verify(redpanda, times(1)).createUser(cluster, "alice", "new-password");+ assertEquals(changed.getVersionId(), association(SECRET).getAppliedVersionId());+ }++ @Test+ void passwordRotationWritesOnlyCompletedState() {+ // Password-only rotation persists completion once without an extra intent write.+ associate(SECRET, version("alice", "password"));+ clearInvocations(associations);+ SecretVersion changed = version("alice", "new-password");+ setSecretValue(SECRET, changed);++ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);++ verify(associations, times(1)).putForAccount(eq(ACCOUNT), eq(cluster.getClusterArn()), any());+ assertEquals(changed.getVersionId(), association(SECRET).getAppliedVersionId());+ }++ @Test+ void usernameRotationReplacesOldUserAndDisassociationDeletesTheNewUser() {+ // Username rotation removes the old broker user and makes later cleanup target the new one.+ associate(SECRET, version("alice", "password"));+ setSecretValue(SECRET, version("bob", "new-password"));+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);++ assertEquals("bob", association(SECRET).getUsername());+ verify(redpanda).deleteUser(cluster, "alice");+ verify(redpanda).createUser(cluster, "bob", "new-password");+ assertTrue(+ service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET))+ .isEmpty());+ verify(redpanda).deleteUser(cluster, "bob");+ assertTrue(state().getAssociations().isEmpty());+ }++ @Test+ void associationIsRejectedWhenRotatedContentIsInvalid() {+ associate(SECRET, version("alice", "original"));+ clearInvocations(redpanda);+ SecretVersion invalid = version("unused", "unused");+ invalid.setSecretString("{}");+ setSecretValue(SECRET, invalid);+ clearInvocations(secrets);++ List<Map<String, String>> failures =+ service.batchAssociateScramSecret(cluster.getClusterArn(), List.of(SECRET));++ assertEquals(1, failures.size());+ assertEquals("InvalidSecretArn", failures.getFirst().get("errorCode"));+ verify(secrets).describeSecret(SECRET, "us-east-1");+ verify(secrets, never()).batchGetSecretValue(any(), anyString());+ verifyNoInteractions(redpanda);+ }++ @Test+ void reconciliationPreservesAppliedCredentialsWhenRotatedContentIsInvalid() {+ SecretVersion original = version("alice", "original");+ associate(SECRET, original);+ clearInvocations(redpanda);+ SecretVersion invalid = version("unused", "unused");+ invalid.setSecretString("{}");+ setSecretValue(SECRET, invalid);++ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);++ assertEquals("alice", association(SECRET).getUsername());+ assertEquals(original.getVersionId(), association(SECRET).getAppliedVersionId());+ verifyNoInteractions(redpanda);+ }++ @Test+ void repeatedReconciliationFailuresLogAtDebugUntilErrorChangesOrRecovery() {+ associate(SECRET, version("alice", "original"));+ setSecretReadError("AccessDeniedException", "Denied.");++ List<LogRecord> logs = LogCapture.capture(MskScramSecretsService.class, () -> {+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);+ setSecretReadError("AccessDeniedException", "Different denial.");+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);+ setSecretReadError("InvalidRequestException", "Invalid request.");+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);+ });+ assertEquals(List.of("WARN", "DEBUG", "WARN", "WARN"), logLevels(logs));++ setSecretValue(SECRET, version("alice", "recovered"));+ logs = LogCapture.capture(MskScramSecretsService.class, () -> service.reconcileCluster(cluster.getClusterArn(), ACCOUNT));+ assertTrue(logs.isEmpty());+ verify(redpanda).createUser(cluster, "alice", "recovered");++ setSecretReadError("InvalidRequestException", "Invalid request.");+ logs = LogCapture.capture(MskScramSecretsService.class, () -> service.reconcileCluster(cluster.getClusterArn(), ACCOUNT));+ assertEquals(List.of("WARN"), logLevels(logs));+ }++ private List<String> logLevels(List<LogRecord> logs) {+ return logs.stream().map(logRecord -> logRecord.getLevel().getName()).toList();+ }++ @Test+ void invalidRequestsMalformedUpdatesAndAccessDenialsPreserveAppliedVersion() {+ // Unreadable or malformed updates are not deletion signals and must preserve the applied version.+ SecretVersion original = version("alice", "password");+ associate(SECRET, original);+ clearInvocations(redpanda);+ setSecretReadError("InvalidRequestException", "The requested operation is not valid for this secret.");+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);+ SecretVersion invalid = new SecretVersion();+ invalid.setVersionId("invalid");+ invalid.setSecretString("{}");+ setSecretValue(SECRET, invalid);+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);+ setSecretReadError("AccessDeniedException", "Denied.");+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);++ assertEquals(original.getVersionId(), association(SECRET).getAppliedVersionId());+ verifyNoInteractions(redpanda);+ }++ @Test+ void metadataAccessDenialPreservesWorkingCredentials() {+ // Metadata access denial must not revoke credentials that are already working.+ SecretVersion original = version("alice", "password");+ associate(SECRET, original);+ doThrow(new AwsException("AccessDeniedException", "Denied.", 403))+ .when(secrets)+ .describeSecret(SECRET, "us-east-1");+ clearInvocations(redpanda);++ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);++ assertFalse(association(SECRET).isCredentialsRevoked());+ assertEquals(original.getVersionId(), association(SECRET).getAppliedVersionId());+ verifyNoInteractions(redpanda);+ }++ @Test+ void missingAwsCurrentStagePreservesAssociationAndAppliedCredentials() {+ associate(SECRET, version("alice", "password"));+ clearInvocations(redpanda);++ Secret secret = new Secret();+ secret.setVersions(Map.of());+ doReturn(secret).when(secrets).describeSecret(SECRET, "us-east-1");++ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);++ assertNotNull(association(SECRET));+ assertFalse(association(SECRET).isCredentialsRevoked());+ verifyNoInteractions(redpanda);+ }++ @Test+ void brokerResourceNotFoundDoesNotTreatSecretAsDeleted() {+ // Broker not-found errors must not be mistaken for a deleted Secrets Manager secret.+ associate(SECRET, version("alice", "old-password"));+ SecretVersion changed = version("alice", "new-password");+ setSecretValue(SECRET, changed);+ doThrow(new AwsException("ResourceNotFoundException", "Broker user not found.", 404))+ .when(redpanda)+ .createUser(cluster, "alice", "new-password");++ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);++ assertNotNull(association(SECRET));+ assertTrue(state().getAssociations().containsKey(SECRET));+ verify(redpanda, never()).deleteUser(any(), anyString());+ }++ @Test+ void brokerFailureDoesNotMarkVersionAppliedAndIsRetried() {+ // Failed broker updates leave the old applied version intact so reconciliation can retry.+ SecretVersion original = version("alice", "old-password");+ associate(SECRET, original);+ SecretVersion changed = version("alice", "new-password");+ setSecretValue(SECRET, changed);+ doThrow(new AwsException("ServiceUnavailableException", "Broker unavailable.", 503))+ .when(redpanda)+ .createUser(cluster, "alice", "new-password");+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);+ assertEquals(original.getVersionId(), association(SECRET).getAppliedVersionId());+ doNothing().when(redpanda).createUser(cluster, "alice", "new-password");+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);+ assertEquals(changed.getVersionId(), association(SECRET).getAppliedVersionId());+ }++ @Test+ void reconcileUsesOwningAccountAndRegionWithoutRequestContext() {+ // Background reconciliation resolves credentials and storage using the resource owner's context.+ String account = "111111111111";+ String region = "eu-west-1";+ String arn = AwsArnUtils.Arn.of("kafka", region, account, "cluster/foreign/id").toString();+ String secretArn =+ AwsArnUtils.Arn.of("secretsmanager", region, account, "secret:foreign").toString();+ MskCluster foreign = new MskCluster(arn, "foreign", "3.6.0");+ foreign.setState(ClusterState.ACTIVE);+ foreign.setContainerId("foreign-broker");+ MskScramAssociations foreignAssociations = new MskScramAssociations();+ foreignAssociations.getAssociations().put(secretArn, new ScramSecretAssociation());+ clusters.putForAccount(account, arn, foreign);+ associations.putForAccount(account, arn, foreignAssociations);+ setSecretValue(secretArn, version("foreign", "password"));++ service.reconcileCluster(arn, account);++ verify(secrets).describeSecret(secretArn, region);+ verify(redpanda).createUser(foreign, "foreign", "password");+ assertTrue(clusters.get(arn).isEmpty());+ assertEquals(+ "foreign",+ associations+ .getForAccount(account, arn)+ .orElseThrow()+ .getAssociations()+ .get(secretArn)+ .getUsername());+ }++ @Test+ void permanentDeletionRevokesCredentialsButRetainsMembershipAndRetriesBrokerFailure() {+ // Deleted secrets retain membership, with revocation recorded only after broker cleanup succeeds.+ SecretVersion original = version("alice", "password");+ associate(SECRET, original);+ clearInvocations(redpanda, secrets, associations);+ doThrow(+ new AwsException(+ "ResourceNotFoundException",+ "Secrets Manager can't find the specified secret.",+ 400))+ .when(secrets)+ .describeSecret(SECRET, "us-east-1");+ doThrow(new AwsException("ServiceUnavailableException", "Broker unavailable.", 503))+ .when(redpanda)+ .deleteUser(cluster, "alice");++ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);++ assertFalse(association(SECRET).isCredentialsRevoked());+ assertEquals(original.getVersionId(), association(SECRET).getAppliedVersionId());+ verify(associations, never()).putForAccount(anyString(), anyString(), any());+ doNothing().when(redpanda).deleteUser(cluster, "alice");+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);++ assertTrue(association(SECRET).isCredentialsRevoked());+ assertNull(association(SECRET).getAppliedVersionId());+ assertNull(association(SECRET).getAppliedContainerId());+ assertEquals(+ List.of(SECRET),+ service.listScramSecrets(cluster.getClusterArn(), null, null).items());+ verify(secrets, never()).batchGetSecretValue(any(), anyString());+ verify(associations, times(1))+ .putForAccount(eq(ACCOUNT), eq(cluster.getClusterArn()), any());+ clearInvocations(redpanda, associations);++ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);++ verifyNoInteractions(redpanda);+ verify(associations, never()).putForAccount(anyString(), anyString(), any());+ assertTrue(service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET)).isEmpty());+ assertTrue(state().getAssociations().isEmpty());+ verifyNoInteractions(redpanda);+ }++ @Test+ void scheduledDeletionRevokesCredentialsAndRestorationReappliesUnchangedVersion() {+ // Scheduled deletion revokes credentials, and restoration reapplies them even without a new version.+ SecretVersion original = version("alice", "password");+ associate(SECRET, original);+ scheduleDeletion(SECRET);+ clearInvocations(redpanda, secrets, associations);++ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);++ assertTrue(association(SECRET).isCredentialsRevoked());+ assertEquals(+ List.of(SECRET),+ service.listScramSecrets(cluster.getClusterArn(), null, null).items());+ verify(redpanda, times(1)).deleteUser(cluster, "alice");+ verify(redpanda, never()).createUser(any(), anyString(), anyString());+ verify(secrets, never()).batchGetSecretValue(any(), anyString());+ verify(associations, times(1))+ .putForAccount(eq(ACCOUNT), eq(cluster.getClusterArn()), any());++ setSecretValue(SECRET, original);+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);++ assertFalse(association(SECRET).isCredentialsRevoked());+ assertEquals(original.getVersionId(), association(SECRET).getAppliedVersionId());+ assertEquals(cluster.getContainerId(), association(SECRET).getAppliedContainerId());+ verify(redpanda, times(1)).createUser(cluster, "alice", "password");+ verify(associations, times(2))+ .putForAccount(eq(ACCOUNT), eq(cluster.getClusterArn()), any());+ }++ @Test+ void revokedAssociationSurvivesRestartWithoutRepeatedDeletionOrStorageWrites()+ throws Exception {+ // Persisted revocation survives restart without repeated cleanup and still permits restoration.+ SecretVersion original = version("alice", "password");+ associate(SECRET, original);+ scheduleDeletion(SECRET);+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);+ MskScramAssociations restored =+ mapper.readValue(mapper.writeValueAsString(state()), MskScramAssociations.class);+ associations.putForAccount(ACCOUNT, cluster.getClusterArn(), restored);+ MskScramSecretsService restarted =+ new MskScramSecretsService(+ storageFactory, config, clusterService, redpanda, secrets, kms, mapper);+ clearInvocations(redpanda, associations);++ restarted.reconcileCluster(cluster.getClusterArn(), ACCOUNT);++ assertTrue(association(SECRET).isCredentialsRevoked());+ assertEquals(+ List.of(SECRET),+ restarted.listScramSecrets(cluster.getClusterArn(), null, null).items());+ verifyNoInteractions(redpanda);+ verify(associations, never()).putForAccount(anyString(), anyString(), any());++ setSecretValue(SECRET, original);+ restarted.reconcileCluster(cluster.getClusterArn(), ACCOUNT);++ assertFalse(association(SECRET).isCredentialsRevoked());+ assertEquals(original.getVersionId(), association(SECRET).getAppliedVersionId());+ verify(redpanda).createUser(cluster, "alice", "password");+ }++ @Test+ void failedRevocationIsRetriedAfterRestart() throws Exception {+ // Restart must preserve an unfinished revocation so broker cleanup is retried once successfully.+ associate(SECRET, version("alice", "password"));+ scheduleDeletion(SECRET);+ doThrow(new AwsException("ServiceUnavailableException", "Broker unavailable.", 503))+ .when(redpanda)+ .deleteUser(cluster, "alice");+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);+ MskScramAssociations restored =+ mapper.readValue(mapper.writeValueAsString(state()), MskScramAssociations.class);+ associations.putForAccount(ACCOUNT, cluster.getClusterArn(), restored);+ MskScramSecretsService restarted =+ new MskScramSecretsService(+ storageFactory, config, clusterService, redpanda, secrets, kms, mapper);+ doNothing().when(redpanda).deleteUser(cluster, "alice");+ clearInvocations(redpanda);++ restarted.reconcileCluster(cluster.getClusterArn(), ACCOUNT);++ assertTrue(association(SECRET).isCredentialsRevoked());+ verify(redpanda, times(1)).deleteUser(cluster, "alice");+ }++ @Test+ void failedRestorationRemainsRetryable() {+ // Failed credential restoration keeps revocation state intact until broker creation succeeds.+ SecretVersion original = version("alice", "password");+ associate(SECRET, original);+ scheduleDeletion(SECRET);+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);+ setSecretValue(SECRET, original);+ doThrow(new AwsException("ServiceUnavailableException", "Broker unavailable.", 503))+ .when(redpanda)+ .createUser(cluster, "alice", "password");++ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);++ assertTrue(association(SECRET).isCredentialsRevoked());+ assertNull(association(SECRET).getAppliedVersionId());+ doNothing().when(redpanda).createUser(cluster, "alice", "password");+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);+ assertFalse(association(SECRET).isCredentialsRevoked());+ assertEquals(original.getVersionId(), association(SECRET).getAppliedVersionId());+ }++ @Test+ void deletionAfterMetadataReadIsHandledOnNextReconciliation() {+ SecretVersion original = version("alice", "password");+ associate(SECRET, original);+ Secret metadata = secrets.describeSecret(SECRET, "us-east-1");+ doReturn(metadata)+ .doThrow(new AwsException("ResourceNotFoundException", "Secret deleted.", 400))+ .when(secrets)+ .describeSecret(SECRET, "us-east-1");++ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);++ assertFalse(association(SECRET).isCredentialsRevoked());+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);++ assertTrue(association(SECRET).isCredentialsRevoked());+ assertEquals(+ List.of(SECRET),+ service.listScramSecrets(cluster.getClusterArn(), null, null).items());+ }+private static Stream<Arguments> sharedUsernameAssociations() {return Stream.of(Arguments.of(Named.of("same password, sequential", false), false, "password"),@@ -770,15 +1326,112 @@ class MskScramServiceTest {.toList());assertEquals("alice", association(SECRET).getUsername());assertEquals("alice", association(second).getUsername());+ assertEquals(firstVersion.getVersionId(), association(SECRET).getAppliedVersionId());+ assertEquals(secondVersion.getVersionId(), association(second).getAppliedVersionId());verify(redpanda, times("password".equals(secondPassword) ? 2 : 1)).createUser(cluster, "alice", "password");verify(redpanda, times("password".equals(secondPassword) ? 2 : 1)).createUser(cluster, "alice", secondPassword);}+ @Test+ void disassociationInvalidatesSharedUserAndReconciliationReappliesUnchangedSecret() {+ // Deleting a shared broker user invalidates surviving associations so unchanged credentials reapply.+ String second = SECRET + "-second";+ SecretVersion surviving = version("alice", "surviving-password");+ associate(SECRET, version("alice", "password"));+ associate(second, surviving);+ clearInvocations(redpanda);++ assertTrue(+ service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET))+ .isEmpty());++ verify(redpanda).deleteUser(cluster, "alice");+ assertNull(association(SECRET));+ assertEquals("alice", association(second).getUsername());+ assertFalse(association(second).isDisassociating());+ assertNull(association(second).getAppliedVersionId());+ assertNull(association(second).getAppliedContainerId());+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);++ verify(redpanda, times(1)).createUser(cluster, "alice", "surviving-password");+ assertEquals(surviving.getVersionId(), association(second).getAppliedVersionId());+ assertEquals(cluster.getContainerId(), association(second).getAppliedContainerId());+ }++ @Test+ void disassociationPersistsRemovalAndSharedUserInvalidationInTwoWrites() {+ // Disassociation persists cleanup intent and then the removal with shared-user invalidation.+ String second = SECRET + "-second";+ associate(SECRET, version("alice", "password"));+ associate(second, version("alice", "surviving-password"));+ clearInvocations(associations);++ assertTrue(+ service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET))+ .isEmpty());++ verify(associations, times(2))+ .putForAccount(eq(ACCOUNT), eq(cluster.getClusterArn()), any());+ assertNull(association(SECRET));+ assertNull(association(second).getAppliedVersionId());+ assertNull(association(second).getAppliedContainerId());+ }++ @Test+ void usernameRotationDoesNotReapplyUnrelatedUsers() {+ String unrelated = SECRET + "-unrelated";+ SecretVersion other = version("charlie", "other-password");+ associate(SECRET, version("alice", "password"));+ associate(unrelated, other);+ clearInvocations(redpanda);++ setSecretValue(SECRET, version("bob", "new-password"));+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);++ assertEquals(other.getVersionId(), association(unrelated).getAppliedVersionId());+ assertEquals(cluster.getContainerId(), association(unrelated).getAppliedContainerId());+ verify(redpanda).createUser(cluster, "bob", "new-password");+ verify(redpanda, never()).createUser(cluster, "charlie", "other-password");+ }++ @Test+ void disassociationAfterRenameOnlyDeletesTheRelevantUsername() {+ // After shared secrets diverge in username, each disassociation deletes only its own broker user.+ String second = SECRET + "-second";+ associate(SECRET, version("alice", "password"));+ associate(second, version("alice", "shared-password"));+ setSecretValue(second, version("bob", "renamed-password"));+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);+ clearInvocations(redpanda);++ assertTrue(+ service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET))+ .isEmpty());+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);++ assertEquals(+ List.of(second),+ service.listScramSecrets(cluster.getClusterArn(), null, null).items());+ assertEquals("bob", association(second).getUsername());+ verify(redpanda).deleteUser(cluster, "alice");+ verify(redpanda, never()).deleteUser(cluster, "bob");+ verify(redpanda, never()).createUser(any(), anyString(), anyString());+ clearInvocations(redpanda);++ assertTrue(+ service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(second))+ .isEmpty());++ assertTrue(service.listScramSecrets(cluster.getClusterArn(), null, null).items().isEmpty());+ verify(redpanda).deleteUser(cluster, "bob");+ verify(redpanda, never()).deleteUser(cluster, "alice");+ }+@Testvoid failedUserDeletionDoesNotInvalidateSharedAssociations() {- // Failed broker deletion leaves both associations available for a caller to retry.+ // Failed broker deletion leaves shared credentials applied while retaining the cleanup intent.String second = SECRET + "-second";SecretVersion surviving = version("alice", "surviving-password");associate(SECRET, version("alice", "password"));@@ -792,8 +1445,84 @@ class MskScramServiceTest {service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET)).size());- assertEquals("alice", association(second).getUsername());- assertEquals("alice", association(SECRET).getUsername());+ assertEquals(surviving.getVersionId(), association(second).getAppliedVersionId());+ assertEquals(cluster.getContainerId(), association(second).getAppliedContainerId());+ assertTrue(association(SECRET).isDisassociating());+ }++ @Test+ void revocationRestoresSharedUserAndDisassociationDoesNotDeleteItAgain() {+ // Revocation restores surviving shared credentials, and later removal must not delete them again.+ String second = SECRET + "-second";+ SecretVersion surviving = version("alice", "surviving-password");+ associate(SECRET, version("alice", "password"));+ associate(second, surviving);+ scheduleDeletion(SECRET);+ clearInvocations(redpanda);++ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);++ assertTrue(association(SECRET).isCredentialsRevoked());+ assertEquals(surviving.getVersionId(), association(second).getAppliedVersionId());+ assertEquals(+ List.of(SECRET, second),+ service.listScramSecrets(cluster.getClusterArn(), null, null).items());+ verify(redpanda, times(1)).deleteUser(cluster, "alice");+ verify(redpanda, times(1)).createUser(cluster, "alice", "surviving-password");+ clearInvocations(redpanda);++ assertTrue(+ service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET))+ .isEmpty());+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);++ assertEquals(surviving.getVersionId(), association(second).getAppliedVersionId());+ assertEquals(+ List.of(second),+ service.listScramSecrets(cluster.getClusterArn(), null, null).items());+ verifyNoInteractions(redpanda);+ }++ @Test+ void failedRevocationDoesNotInvalidateSharedUser() {+ // A failed revocation must not invalidate or rewrite still-working shared-user state.+ String second = SECRET + "-second";+ SecretVersion surviving = version("alice", "surviving-password");+ associate(SECRET, version("alice", "password"));+ associate(second, surviving);+ scheduleDeletion(SECRET);+ doThrow(new AwsException("ServiceUnavailableException", "Broker unavailable.", 503))+ .when(redpanda)+ .deleteUser(cluster, "alice");+ clearInvocations(associations);++ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);++ assertFalse(association(SECRET).isCredentialsRevoked());+ assertEquals(surviving.getVersionId(), association(second).getAppliedVersionId());+ verify(associations, never()).putForAccount(anyString(), anyString(), any());+ }++ @Test+ void restoredSecretWithNewUsernameDoesNotDeleteSurvivingSharedUser() {+ // Restoring a revoked secret under a new username must preserve the surviving shared user.+ String second = SECRET + "-second";+ SecretVersion surviving = version("alice", "surviving-password");+ associate(SECRET, version("alice", "password"));+ associate(second, surviving);+ scheduleDeletion(SECRET);+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);+ clearInvocations(redpanda);+ doReturn(new Secret()).when(secrets).describeSecret(SECRET, "us-east-1");+ setSecretValue(SECRET, version("bob", "restored-password"));++ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);++ assertFalse(association(SECRET).isCredentialsRevoked());+ assertEquals("bob", association(SECRET).getUsername());+ assertEquals(surviving.getVersionId(), association(second).getAppliedVersionId());+ verify(redpanda).createUser(cluster, "bob", "restored-password");+ verify(redpanda, never()).deleteUser(any(), anyString());}@Test@@ -814,19 +1543,77 @@ class MskScramServiceTest {restarted.listScramSecrets(cluster.getClusterArn(), null, null).items());assertEquals("alice", restored.getAssociations().get(SECRET).getUsername());clearInvocations(redpanda);+ restarted.reconcileCluster(cluster.getClusterArn(), ACCOUNT);+ verifyNoInteractions(redpanda);+ }++ @Test+ void recreatedBrokerReceivesCurrentCredentialsOnlyAfterClusterBecomesActive() {+ // A replacement broker receives existing credentials only once the cluster is active.+ associate(SECRET, version("alice", "password"));+ clearInvocations(redpanda);+ cluster.setContainerId("broker-2");+ cluster.setState(ClusterState.CREATING);+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);verifyNoInteractions(redpanda);+ cluster.setState(ClusterState.ACTIVE);+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);+ verify(redpanda).createUser(cluster, "alice", "password");+ assertEquals("broker-2", association(SECRET).getAppliedContainerId());}@Test- void mockModeTracksAssociationsWithoutBrokerCalls() {+ void reconciliationRemovesOrphansFromTheirOwningAccount() {+ // Orphan cleanup uses the owning account and must not remove another account's valid associations.+ String account = "111111111111";+ String arn =+ AwsArnUtils.Arn.of("kafka", "eu-west-1", account, "cluster/deleted/id").toString();+ MskScramAssociations orphan = new MskScramAssociations();+ orphan.getAssociations().put(SECRET, new ScramSecretAssociation());+ associations.putForAccount(account, arn, orphan);+ associate(SECRET, version("alice", "password"));+ clearInvocations(redpanda, secrets);++ service.reconcileCluster(arn, account);++ assertTrue(associations.getForAccount(account, arn).isEmpty());+ assertNotNull(association(SECRET));+ verifyNoInteractions(redpanda);+ }++ @Test+ void mockModeTracksAndReconcilesAssociationsWithoutBrokerCalls() {// Mock mode maintains association lifecycle state while suppressing all broker operations.when(config.services().msk().mock()).thenReturn(true);associate(SECRET, version("alice", "password"));- assertEquals("alice", association(SECRET).getUsername());+ setSecretValue(SECRET, version("bob", "password"));+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);+ assertEquals("bob", association(SECRET).getUsername());service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET));verifyNoInteractions(redpanda);}+ @Test+ void reconcileSkipsServerlessClusterWithStaleAssociation() {+ // Stale SCRAM state on a serverless cluster must not trigger credential reads or broker calls.+ String serverlessArn =+ AwsArnUtils.Arn.of("kafka", "us-east-1", ACCOUNT, "cluster/serverless/id")+ .toString();+ MskCluster serverless = new MskCluster(serverlessArn, "serverless", "3.6.0");+ serverless.setClusterType("SERVERLESS");+ serverless.setState(ClusterState.ACTIVE);+ serverless.setContainerId("unrelated-broker");+ clusters.putForAccount(ACCOUNT, serverlessArn, serverless);+ MskScramAssociations staleAssociations = new MskScramAssociations();+ staleAssociations.getAssociations().put(SECRET, new ScramSecretAssociation());+ associations.putForAccount(ACCOUNT, serverlessArn, staleAssociations);+ clearInvocations(secrets, redpanda);++ service.reconcileCluster(serverlessArn, ACCOUNT);++ verifyNoInteractions(secrets, redpanda);+ }+@Testvoid failedClusterDeletionRetainsScramAssociationState() {// Failed cluster deletion retains both the cluster and its recoverable SCRAM association state.@@ -837,6 +1624,7 @@ class MskScramServiceTest {assertThrows(AwsException.class, () -> clusterService.deleteCluster(cluster.getClusterArn()));+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);assertNotNull(association(SECRET));assertTrue(clusters.get(cluster.getClusterArn()).isPresent());diff --git a/src/test/resources/application.yml b/src/test/resources/application.ymlindex e642926088..3efa98900c 100644--- a/src/test/resources/application.yml+++ b/src/test/resources/application.yml@@ -182,6 +182,7 @@ floci:msk:enabled: truemock: true+ scram-reconciliation-interval-ms: 5000default-image: "redpandadata/redpanda:latest"amazonmq:enabled: true