Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(msk): scheduled reconcile for SCRAM secret changes (#6384)

Isak Lindbeckbc18cbbdaa39
diff --git a/docs/services/msk.md b/docs/services/msk.md
index 5ce00f86da..eb9fd28fd6 100644
--- a/docs/services/msk.md
+++ b/docs/services/msk.md
@@ -40,6 +40,7 @@ Floci emulates Amazon MSK by orchestrating **Redpanda** containers. This provide
| `FLOCI_SERVICES_MSK_DEFAULT_IMAGE` | `redpandadata/redpanda:latest` | Docker image for Redpanda (Kafka) containers |
| `FLOCI_SERVICES_MSK_KAFKA_HOST_PORT_BASE` | `9300` | First host port a broker's Kafka listener can be published on |
| `FLOCI_SERVICES_MSK_KAFKA_HOST_PORT_MAX` | `9399` | Last host port a broker's Kafka listener can be published on |
+| `FLOCI_SERVICES_MSK_SCRAM_RECONCILIATION_INTERVAL_MS` | `5000` | SCRAM secret reconciliation interval (ms) |
| `FLOCI_SERVICES_MSK_BOOTSTRAP_HOSTNAME` | _(unset)_ | Floci in a container only: hostname the host listener advertises, and when set, `GetBootstrapBrokers` returns `<container-name>:9092,<hostname>:<host port>` instead of the Docker-network address |
## How it works
@@ -132,6 +133,19 @@ The secrets must contain JSON such as `{"username":"alice","password":"pw"}`.
`unprocessedScramSecrets`; reassociation and removal of an unassociated ARN return
`InvalidSecretArn`.
+Floci schedules SCRAM reconciliation jobs every five seconds by default, measured as
+a delay after each scheduling sweep. Sweeps do not wait for broker operations to finish,
+and clusters with queued or running jobs are skipped. Set
+`FLOCI_SERVICES_MSK_SCRAM_RECONCILIATION_INTERVAL_MS` to change this delay in milliseconds.
+This is Floci's timing, not an AWS synchronization guarantee.
+
+When an associated secret is scheduled for deletion or permanently deleted, reconciliation
+revokes its broker user but retains the association in `ListScramSecrets`. Restoring a secret
+whose deletion was scheduled causes reconciliation to recreate its broker user using the
+current credentials, without requiring reassociation. Permanently deleted secrets cannot
+be restored; creating a secret with the same name produces a different ARN and requires
+a new association.
+
## Examples
```bash
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
index 9405bd460a..4837631ef9 100644
--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
@@ -1575,6 +1575,9 @@ public interface EmulatorConfig {
@WithDefault("9399")
int kafkaHostPortMax();
+ @WithDefault("5000")
+ long scramReconciliationIntervalMs();
+
/**
* Only applies when Floci runs in a container. Each broker then has a second Kafka
* listener, published to a host port from the kafka-host-port range so clients on the
diff --git a/src/main/java/io/github/hectorvent/floci/services/msk/MskScramSecretsService.java b/src/main/java/io/github/hectorvent/floci/services/msk/MskScramSecretsService.java
index 59f1d77335..a5b18834a7 100644
--- a/src/main/java/io/github/hectorvent/floci/services/msk/MskScramSecretsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/msk/MskScramSecretsService.java
@@ -15,17 +15,17 @@ import io.github.hectorvent.floci.services.kms.KmsService;
import io.github.hectorvent.floci.services.kms.model.KmsKey;
import io.github.hectorvent.floci.services.kms.model.KmsKeySpec;
import io.github.hectorvent.floci.services.kms.model.KmsKeyUsage;
+import io.github.hectorvent.floci.services.msk.model.ClusterState;
import io.github.hectorvent.floci.services.msk.model.MskCluster;
import io.github.hectorvent.floci.services.msk.model.MskScramAssociations;
import io.github.hectorvent.floci.services.msk.model.ScramSecretAssociation;
import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;
-import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService.BatchGetSecretValueError;
-import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService.BatchGetSecretValueResult;
-import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService.BatchSecretValue;
import io.github.hectorvent.floci.services.secretsmanager.model.Secret;
-import jakarta.annotation.PostConstruct;
+import io.github.hectorvent.floci.services.secretsmanager.model.SecretVersion;
+import io.quarkus.runtime.StartupEvent;
import jakarta.annotation.PreDestroy;
import jakarta.enterprise.context.ApplicationScoped;
+import jakarta.enterprise.event.Observes;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
@@ -33,9 +33,15 @@ import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.Executors;
+import java.util.concurrent.RejectedExecutionException;
import java.util.concurrent.ScheduledExecutorService;
+import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
@ApplicationScoped
@@ -51,9 +57,19 @@ public class MskScramSecretsService {
private final SecretsManagerService secretsManagerService;
private final KmsService kmsService;
private final ObjectMapper objectMapper;
+
private final ScheduledExecutorService reconciler = Executors.newSingleThreadScheduledExecutor();
+ private final ThreadPoolExecutor reconciliationWorkers = new ThreadPoolExecutor(
+ 4, 4, 0L, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(128));
+ private final Set<String> pendingReconciliations = ConcurrentHashMap.newKeySet();
private final Map<String, Object> clusterLocks = new ConcurrentHashMap<>();
+ private final Map<String, ClusterReconciliationFailure> clusterReconciliationFailures = new ConcurrentHashMap<>();
+ private record ClusterReconciliationFailure(Class<?> type, String message) {}
+ private final Map<ReconciliationKey, ReconciliationFailure> secretReconciliationFailures = new ConcurrentHashMap<>();
+ private record ReconciliationKey(String clusterArn, String secretArn) {}
+ private record ReconciliationFailure(String errorCode, String message) {}
+
@Inject
public MskScramSecretsService(StorageFactory storageFactory, EmulatorConfig config, MskService mskService,
RedpandaManager redpandaManager, SecretsManagerService secretsManagerService,
@@ -68,37 +84,147 @@ public class MskScramSecretsService {
this.objectMapper = objectMapper;
}
- @PostConstruct
- public void init() {
+ void onStart(@Observes StartupEvent ignored) {
+ if (!config.services().msk().enabled()) {
+ return;
+ }
+ long intervalMs = config.services().msk().scramReconciliationIntervalMs();
+ if (intervalMs <= 0) {
+ throw new IllegalArgumentException("floci.services.msk.scram-reconciliation-interval-ms must be greater than zero.");
+ }
reconciler.scheduleWithFixedDelay(() -> {
try {
- reconcileScramSecrets();
+ scheduleScramReconciliation();
} catch (RuntimeException e) {
LOG.error("MSK SCRAM reconciliation failed; retrying on the next scheduled run", e);
}
- }, 5, 5, TimeUnit.SECONDS);
+ }, intervalMs, intervalMs, TimeUnit.MILLISECONDS);
}
@PreDestroy
public void shutdown() {
reconciler.shutdown();
+ reconciliationWorkers.shutdownNow();
}
private Object clusterLock(String clusterArn) {
return clusterLocks.computeIfAbsent(String.valueOf(clusterArn), ignored -> new Object());
}
- void reconcileScramSecrets() {
- for (AccountAwareStorageBackend.AccountEntry<MskScramAssociations> entry
- : storage.scanAllAccountEntries(key -> true)) {
- synchronized (clusterLock(entry.key())) {
- if (mskService.getClusterForAccount(entry.accountId(), entry.key()).isEmpty()) {
- storage.deleteForAccount(entry.accountId(), entry.key());
+ private void scheduleScramReconciliation() {
+ int deferredClusters = 0;
+ for (AccountAwareStorageBackend.AccountEntry<MskScramAssociations> entry : storage.scanAllAccountEntries(key -> true)) {
+ String clusterArn = entry.key();
+ if (!pendingReconciliations.add(clusterArn)) {
+ continue;
+ }
+ try {
+ reconciliationWorkers.execute(() -> runClusterReconciliation(entry));
+ } catch (RejectedExecutionException e) {
+ pendingReconciliations.remove(clusterArn);
+ if (!reconciliationWorkers.isShutdown()) {
+ deferredClusters++;
}
}
}
+ if (deferredClusters > 0) {
+ LOG.warnv("MSK SCRAM reconciliation workers are full; retrying {0} clusters on the next scheduled run",
+ deferredClusters);
+ }
}
+ private void runClusterReconciliation(AccountAwareStorageBackend.AccountEntry<MskScramAssociations> entry) {
+ String clusterArn = entry.key();
+ try {
+ reconcileCluster(clusterArn, entry.accountId());
+ clusterReconciliationFailures.remove(clusterArn);
+ } catch (RuntimeException e) {
+ ClusterReconciliationFailure failure = new ClusterReconciliationFailure(e.getClass(), e.getMessage());
+ if (!failure.equals(clusterReconciliationFailures.put(clusterArn, failure))) {
+ LOG.errorv(e, "MSK SCRAM reconciliation failed for cluster {0}; retrying on the next scheduled run",
+ clusterArn);
+ } else {
+ LOG.debugv(e, "MSK SCRAM reconciliation failed for cluster {0}; retrying on the next scheduled run",
+ clusterArn);
+ }
+ } finally {
+ pendingReconciliations.remove(clusterArn);
+ }
+ }
+
+ void reconcileCluster(String clusterArn, String accountId) {
+ synchronized (clusterLock(clusterArn)) {
+ Optional<MskCluster> clusterGet = mskService.getClusterForAccount(accountId, clusterArn);
+ if (clusterGet.isEmpty()) {
+ storage.deleteForAccount(accountId, clusterArn);
+ secretReconciliationFailures.keySet().removeIf(key -> key.clusterArn().equals(clusterArn));
+ } else {
+ reconcileScramSecretsForCluster(clusterGet.get(), clusterArn);
+ }
+ }
+ }
+
+ private void reconcileScramSecretsForCluster(MskCluster cluster, String clusterArn) {
+ if (mskService.isServerless(cluster) || cluster.getState() != ClusterState.ACTIVE) {
+ return;
+ }
+ MskScramAssociations clusterAssociations = associationsFor(clusterArn);
+ for (String secretArn : new ArrayList<>(clusterAssociations.getAssociations().keySet())) {
+ ReconciliationKey reconciliationKey = new ReconciliationKey(clusterArn, secretArn);
+ try {
+ reconcileScramSecret(secretArn, clusterAssociations, cluster);
+ secretReconciliationFailures.remove(reconciliationKey);
+ } catch (AwsException e) {
+ ReconciliationFailure failure = new ReconciliationFailure(e.getErrorCode(), e.getMessage());
+ if (!failure.equals(secretReconciliationFailures.put(reconciliationKey, failure))) {
+ LOG.warnv("Could not reconcile MSK SCRAM secret {0} for cluster {1}: {2}",
+ secretArn, clusterArn, e.getErrorCode());
+ } else {
+ LOG.debugv("Could not reconcile MSK SCRAM secret {0} for cluster {1}: {2}",
+ secretArn, clusterArn, e.getErrorCode());
+ }
+ }
+ }
+ }
+
+ private void reconcileScramSecret(String secretArn, MskScramAssociations clusterAssociations, MskCluster cluster) {
+ ScramSecretAssociation association = clusterAssociations.getAssociations().get(secretArn);
+ if (association == null) {
+ // Should never happen
+ throw new AwsException("InternalServerErrorException", "SCRAM association is missing during reconciliation.", 500);
+ }
+ if (association.isDisassociating()) {
+ // Recover if disassociation was interrupted
+ removeScramAssociation(cluster, clusterAssociations, secretArn);
+ return;
+ }
+ Optional<Secret> secret = describeSecret(secretArn);
+ if (secret.isEmpty() || secret.get().getDeletedDate() != null) {
+ // Revoke credentials for secret that are deleted or scheduled for deletion, but keep the association
+ revokeScramCredentials(cluster, clusterAssociations, association);
+ return;
+ }
+
+ SecretVersion secretVersion = getCurrentSecretVersion(secret.get());
+ boolean versionIdChanged = !Objects.equals(secretVersion.getVersionId(), association.getAppliedVersionId());
+ boolean containerIdChanged = !Objects.equals(cluster.getContainerId(), association.getAppliedContainerId());
+ // Reapply if the secret or cluster container has changed
+ // Reapply if credentials where revoked, which can happen if the secret was deleted and then restored
+ if (association.isCredentialsRevoked() || versionIdChanged || containerIdChanged) {
+ applyScramCredentials(cluster, clusterAssociations, association, secretVersion);
+ }
+ }
+
+ private void revokeScramCredentials(MskCluster cluster, MskScramAssociations state, ScramSecretAssociation association) {
+ if (association.isCredentialsRevoked()) {
+ // Skip repeated delete calls to the cluster
+ return;
+ }
+ deleteScramUser(cluster, state, association);
+ association.setCredentialsRevoked(true);
+ putAssociations(cluster.getClusterArn(), state);
+ }
+
public List<Map<String, String>> batchAssociateScramSecret(String clusterArn, List<String> secretArns) {
synchronized (clusterLock(clusterArn)) {
MskCluster cluster = scramCluster(clusterArn);
@@ -118,8 +244,10 @@ public class MskScramSecretsService {
}
validateSecretEncryption(secret);
// Amazon MSK reads only the AWSCURRENT version of an associated secret
- BatchSecretValue secretValue = readSecret(secretArn);
- applyScramCredentials(cluster, state, secretArn, secretValue);
+ SecretVersion secretVersion = getCurrentSecretVersion(secret);
+ ScramSecretAssociation association = state.getAssociations()
+ .computeIfAbsent(secretArn, ignored -> new ScramSecretAssociation());
+ applyScramCredentials(cluster, state, association, secretVersion);
} catch (AwsException e) {
if (newAssociation && state.getAssociations().remove(secretArn) != null) {
putAssociations(clusterArn, state);
@@ -248,14 +376,21 @@ public class MskScramSecretsService {
}
private void applyScramCredentials(MskCluster cluster, MskScramAssociations state,
- String secretArn, BatchSecretValue secretValue) {
- ScramCredentials credentials = readScramCredentials(secretValue.secretString());
+ ScramSecretAssociation association, SecretVersion secretValue) {
+ ScramCredentials credentials = readScramCredentials(secretValue.getSecretString());
+ boolean ownershipChanged = !Objects.equals(association.getUsername(), credentials.username());
+ if (ownershipChanged) {
+ deleteScramUser(cluster, state, association);
+ association.setUsername(credentials.username());
+ // Persist ownership changes before updating the broker; mark the version as applied only after success.
+ putAssociations(cluster.getClusterArn(), state);
+ }
if (!config.services().msk().mock()) {
redpandaManager.createUser(cluster, credentials.username(), credentials.password());
}
- ScramSecretAssociation association = new ScramSecretAssociation();
- association.setUsername(credentials.username());
- state.getAssociations().put(secretArn, association);
+ association.setAppliedVersionId(secretValue.getVersionId());
+ association.setAppliedContainerId(cluster.getContainerId());
+ association.setCredentialsRevoked(false);
putAssociations(cluster.getClusterArn(), state);
}
@@ -264,33 +399,64 @@ public class MskScramSecretsService {
if (association == null) {
return;
}
- if (association.getUsername() != null && !config.services().msk().mock()) {
- redpandaManager.deleteUser(cluster, association.getUsername());
+ if (!association.isDisassociating()) {
+ association.setDisassociating(true);
+ // Persist the removal intent before deleting broker credentials so a failure does not lose it.
+ putAssociations(cluster.getClusterArn(), state);
}
+ deleteScramUser(cluster, state, association);
state.getAssociations().remove(secretArn);
putAssociations(cluster.getClusterArn(), state);
+ secretReconciliationFailures.remove(new ReconciliationKey(cluster.getClusterArn(), secretArn));
}
- private BatchSecretValue readSecret(String secretArn) {
- // Exact-ARN batch reads select AWSCURRENT without updating access timestamps.
- BatchGetSecretValueResult result = secretsManagerService.batchGetSecretValue(
- List.of(secretArn), AwsArnUtils.parse(secretArn).region());
- if (result != null && result.values() != null && result.errors() != null) {
- if (result.values().isEmpty() && result.errors().size() == 1) {
- BatchGetSecretValueError error = result.errors().getFirst();
- if (error != null && secretArn.equals(error.secretId())
- && error.errorCode() != null && error.message() != null) {
- int status = "AccessDeniedException".equals(error.errorCode()) ? 403 : 400;
- throw new AwsException(error.errorCode(), error.message(), status);
- }
- } else if (result.values().size() == 1 && result.errors().isEmpty()) {
- BatchSecretValue value = result.values().getFirst();
- if (value != null && secretArn.equals(value.arn()) && value.versionId() != null) {
- return value;
+ private void deleteScramUser(MskCluster cluster, MskScramAssociations state, ScramSecretAssociation association) {
+ String username = association.getUsername();
+ // Revoked credentials have already been deleted
+ if (username != null && !association.isCredentialsRevoked()) {
+ if (!config.services().msk().mock()) {
+ redpandaManager.deleteUser(cluster, username);
+ }
+ // Allow a user existing in multiple secrets to be reconciled again
+ invalidateUserAssociations(state, username);
+ }
+ }
+
+ private void invalidateUserAssociations(MskScramAssociations state, String username) {
+ for (ScramSecretAssociation association : state.getAssociations().values()) {
+ if (username.equals(association.getUsername())) {
+ association.setAppliedVersionId(null);
+ association.setAppliedContainerId(null);
+ }
+ }
+ }
+
+ private Optional<Secret> describeSecret(String secretArn) {
+ try {
+ Secret secret = secretsManagerService.describeSecret(secretArn, AwsArnUtils.parse(secretArn).region());
+ return Optional.of(secret);
+ } catch (AwsException e) {
+ if ("ResourceNotFoundException".equals(e.getErrorCode())) {
+ return Optional.empty();
+ }
+ throw e;
+ }
+ }
+
+ private SecretVersion getCurrentSecretVersion(Secret secret) {
+ if (secret.getDeletedDate() != null) {
+ throw new AwsException("InternalServerError",
+ SecretsManagerService.SCHEDULED_DELETION_ERROR_MESSAGE, 500);
+ }
+ if (secret.getVersions() != null) {
+ for (SecretVersion version : secret.getVersions().values()) {
+ if (version.getVersionStages() != null && version.getVersionStages().contains("AWSCURRENT")) {
+ return version;
}
}
}
- throw new AwsException("InternalServerError", "Unexpected Secrets Manager batch response.", 500);
+ throw new AwsException("ResourceNotFoundException",
+ "Secrets Manager can't find the specified secret value for staging label: AWSCURRENT", 400);
}
record ScramCredentials(String username, String password) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/msk/model/ScramSecretAssociation.java b/src/main/java/io/github/hectorvent/floci/services/msk/model/ScramSecretAssociation.java
index 668656e53f..f724ff0c43 100644
--- a/src/main/java/io/github/hectorvent/floci/services/msk/model/ScramSecretAssociation.java
+++ b/src/main/java/io/github/hectorvent/floci/services/msk/model/ScramSecretAssociation.java
@@ -6,6 +6,10 @@ import io.quarkus.runtime.annotations.RegisterForReflection;
public class ScramSecretAssociation {
private String username;
+ private String appliedVersionId;
+ private String appliedContainerId;
+ private boolean credentialsRevoked;
+ private boolean disassociating;
public String getUsername() {
return username;
@@ -15,4 +19,35 @@ public class ScramSecretAssociation {
this.username = username;
}
+ public String getAppliedVersionId() {
+ return appliedVersionId;
+ }
+
+ public void setAppliedVersionId(String appliedVersionId) {
+ this.appliedVersionId = appliedVersionId;
+ }
+
+ public String getAppliedContainerId() {
+ return appliedContainerId;
+ }
+
+ public void setAppliedContainerId(String appliedContainerId) {
+ this.appliedContainerId = appliedContainerId;
+ }
+
+ public boolean isCredentialsRevoked() {
+ return credentialsRevoked;
+ }
+
+ public void setCredentialsRevoked(boolean credentialsRevoked) {
+ this.credentialsRevoked = credentialsRevoked;
+ }
+
+ public boolean isDisassociating() {
+ return disassociating;
+ }
+
+ public void setDisassociating(boolean disassociating) {
+ this.disassociating = disassociating;
+ }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerService.java b/src/main/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerService.java
index 0f6c79846c..44702eff50 100644
--- a/src/main/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerService.java
@@ -62,6 +62,9 @@ public class SecretsManagerService implements ResourceProvider {
private static final String PENDING_DELETION_STATE = "PendingDeletion";
private static final Pattern SECRET_NAME_PATTERN = Pattern.compile("[a-zA-Z0-9/_+=.@-]+");
+ public static final String SCHEDULED_DELETION_ERROR_MESSAGE =
+ "You can't perform this operation on the secret because it was marked for deletion.";
+
private final StorageBackend<String, Secret> store;
private final int defaultRecoveryWindowDays;
private final RegionResolver regionResolver;
@@ -1566,7 +1569,7 @@ public class SecretsManagerService implements ResourceProvider {
Secret secret = resolveSecret(secretId, region);
if (secret.getDeletedDate() != null) {
errors.add(new BatchGetSecretValueError(secretId, "InvalidRequestException",
- "You can't perform this operation on the secret because it was marked for deletion."));
+ SCHEDULED_DELETION_ERROR_MESSAGE));
continue;
}
@@ -1792,7 +1795,7 @@ public class SecretsManagerService implements ResourceProvider {
private void throwIfPendingDeletion(Secret secret) {
if (secret.getDeletedDate() != null) {
throw new AwsException("InvalidRequestException",
- "You can't perform this operation on the secret because it was marked for deletion.", 400);
+ SCHEDULED_DELETION_ERROR_MESSAGE, 400);
}
}
diff --git a/src/main/resources/application.yml b/src/main/resources/application.yml
index fa389a7869..bc68b88a7c 100644
--- a/src/main/resources/application.yml
+++ b/src/main/resources/application.yml
@@ -478,6 +478,7 @@ floci:
msk:
enabled: true
mock: false
+ scram-reconciliation-interval-ms: 5000
default-image: "redpandadata/redpanda:latest"
kafka-host-port-base: 9300
kafka-host-port-max: 9399
diff --git a/src/test/java/io/github/hectorvent/floci/services/msk/MskScramServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/msk/MskScramServiceTest.java
index 6b41f59ff8..647573508b 100644
--- a/src/test/java/io/github/hectorvent/floci/services/msk/MskScramServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/msk/MskScramServiceTest.java
@@ -13,16 +13,15 @@ import io.github.hectorvent.floci.services.kms.KmsService;
import io.github.hectorvent.floci.services.kms.model.KmsKey;
import io.github.hectorvent.floci.services.kms.model.KmsKeySpec;
import io.github.hectorvent.floci.services.kms.model.KmsKeyUsage;
+import io.github.hectorvent.floci.services.msk.model.ClusterState;
import io.github.hectorvent.floci.services.msk.model.MskCluster;
import io.github.hectorvent.floci.services.msk.model.MskConfiguration;
import io.github.hectorvent.floci.services.msk.model.MskScramAssociations;
import io.github.hectorvent.floci.services.msk.model.ScramSecretAssociation;
import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;
-import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService.BatchGetSecretValueError;
-import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService.BatchGetSecretValueResult;
-import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService.BatchSecretValue;
import io.github.hectorvent.floci.services.secretsmanager.model.Secret;
import io.github.hectorvent.floci.services.secretsmanager.model.SecretVersion;
+import io.github.hectorvent.floci.testutil.LogCapture;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Named;
import org.junit.jupiter.api.Test;
@@ -31,6 +30,8 @@ import org.junit.jupiter.params.provider.Arguments;
import org.junit.jupiter.params.provider.MethodSource;
import org.junit.jupiter.params.provider.ValueSource;
+import java.time.Duration;
+import java.time.Instant;
import java.util.Collections;
import java.util.List;
import java.util.Map;
@@ -40,8 +41,10 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import java.util.logging.LogRecord;
import java.util.stream.Stream;
+import static org.awaitility.Awaitility.await;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -95,6 +98,7 @@ class MskScramServiceTest {
when(storageFactory.<MskConfiguration>create(eq("msk"), eq("msk-configurations.json"), any()))
.thenReturn(AccountAwareStorageBackend.inMemory(ACCOUNT));
config = mock(EmulatorConfig.class, RETURNS_DEEP_STUBS);
+ when(config.services().msk().enabled()).thenReturn(true);
when(config.services().msk().mock()).thenReturn(true);
regionResolver = mock(RegionResolver.class);
when(regionResolver.getRegion()).thenReturn("us-east-1");
@@ -157,6 +161,127 @@ class MskScramServiceTest {
}
}
+ @Test
+ void startupSkipsReconciliationWhenMskIsDisabled() {
+ when(config.services().msk().enabled()).thenReturn(false);
+
+ try {
+ service.onStart(null);
+
+ verify(config.services().msk(), never()).scramReconciliationIntervalMs();
+ verifyNoInteractions(redpanda);
+ } finally {
+ service.shutdown();
+ }
+ }
+
+ @ParameterizedTest
+ @ValueSource(longs = {0, -1})
+ void reconciliationRejectsNonPositiveInterval(long intervalMs) {
+ when(config.services().msk().scramReconciliationIntervalMs()).thenReturn(intervalMs);
+
+ try {
+ IllegalArgumentException error = assertThrows(IllegalArgumentException.class, () -> service.onStart(null));
+ assertEquals("floci.services.msk.scram-reconciliation-interval-ms must be greater than zero.",
+ error.getMessage());
+ } finally {
+ service.shutdown();
+ }
+ }
+
+ @Test
+ void scheduledReconciliationKeepsOtherClustersSyncingWhileOneBrokerIsBlocked() throws Exception {
+ when(config.services().msk().mock()).thenReturn(true);
+ MskCluster healthyCluster = clusterService.createCluster("healthy-scram");
+ when(config.services().msk().mock()).thenReturn(false);
+ String healthySecret = SECRET + "-healthy";
+ associate(SECRET, version("alice", "original"));
+ setSecretValue(healthySecret, version("bob", "original"));
+ assertTrue(service.batchAssociateScramSecret(healthyCluster.getClusterArn(), List.of(healthySecret)).isEmpty());
+
+ CountDownLatch operationForFirstClusterStarted = new CountDownLatch(1);
+ CountDownLatch operationForSecondClusterDone = new CountDownLatch(1);
+ CountDownLatch operationForFirstClusterDone = new CountDownLatch(1);
+ doAnswer(invocation -> {
+ operationForFirstClusterStarted.countDown();
+ try {
+ assertTrue(operationForSecondClusterDone.await(30, TimeUnit.SECONDS));
+ } finally {
+ operationForFirstClusterDone.countDown();
+ }
+ return null;
+ }).when(redpanda).createUser(cluster, "alice", "rotated");
+ setSecretValue(SECRET, version("alice", "rotated"));
+ when(config.services().msk().scramReconciliationIntervalMs()).thenReturn(5L);
+
+ try {
+ service.onStart(null);
+ assertTrue(operationForFirstClusterStarted.await(5, TimeUnit.SECONDS));
+
+ // More rotations than workers expose duplicate blocked jobs exhausting the pool.
+ for (int rotation = 1; rotation <= 6; rotation++) {
+ String password = "healthy-rotation-" + rotation;
+ setSecretValue(healthySecret, version("bob", password));
+ await()
+ .atMost(Duration.ofSeconds(5))
+ .untilAsserted(() -> verify(redpanda).createUser(healthyCluster, "bob", password));
+ }
+
+ verify(redpanda, times(1)).createUser(cluster, "alice", "rotated");
+ } finally {
+ operationForSecondClusterDone.countDown();
+ try {
+ assertTrue(operationForFirstClusterDone.await(5, TimeUnit.SECONDS));
+ } finally {
+ service.shutdown();
+ }
+ }
+ }
+
+ @Test
+ void scheduledReconciliationLogsRepeatedFailuresAndRecovers() {
+ associate(SECRET, version("alice", "original"));
+ setSecretValue(SECRET, version("alice", "rotated"));
+ doThrow(new IllegalStateException("Broker unavailable"))
+ .doThrow(new IllegalStateException("Broker unavailable"))
+ .doThrow(new IllegalArgumentException("Invalid broker"))
+ .doNothing()
+ .when(redpanda)
+ .createUser(cluster, "alice", "rotated");
+ when(config.services().msk().scramReconciliationIntervalMs()).thenReturn(5L);
+
+ try {
+ List<LogRecord> logs = LogCapture.capture(MskScramSecretsService.class, () -> {
+ service.onStart(null);
+ await()
+ .atMost(Duration.ofSeconds(5))
+ .untilAsserted(() -> verify(redpanda, times(4)).createUser(cluster, "alice", "rotated"));
+ });
+ assertEquals(List.of("ERROR", "DEBUG", "ERROR"), logLevels(logs));
+ } finally {
+ service.shutdown();
+ }
+ }
+
+ @Test
+ void scheduledReconciliationAppliesRotationsUsingConfiguredInterval() {
+ associate(SECRET, version("alice", "original"));
+ when(config.services().msk().scramReconciliationIntervalMs()).thenReturn(5L);
+ setSecretValue(SECRET, version("alice", "rotated"));
+
+ try {
+ service.onStart(null);
+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() ->
+ verify(redpanda).createUser(cluster, "alice", "rotated"));
+
+ setSecretValue(SECRET, version("alice", "rotated-again"));
+ await().atMost(Duration.ofSeconds(5)).untilAsserted(() ->
+ verify(redpanda).createUser(cluster, "alice", "rotated-again"));
+ } finally {
+ service.shutdown();
+ }
+ }
+
private void associate(String arn, SecretVersion version) {
setSecretValue(arn, version);
assertTrue(
@@ -164,18 +289,24 @@ class MskScramServiceTest {
}
private void setSecretValue(String arn, SecretVersion version) {
- BatchGetSecretValueResult batchGetSecretValueResult = new BatchGetSecretValueResult(List.of(new BatchSecretValue(
- arn, "AmazonMSK_fixture", version.getSecretString(), version.getSecretBinary(),
- version.getVersionId(), List.of("AWSCURRENT"), version.getCreatedDate())
- ), List.of());
- doReturn(batchGetSecretValueResult)
- .when(secrets)
- .batchGetSecretValue(List.of(arn), AwsArnUtils.parse(arn).region());
+ Secret secret = new Secret();
+ secret.setArn(arn);
+ secret.setKmsKeyId("customer-key");
+ version.setVersionStages(List.of("AWSCURRENT"));
+ secret.setVersions(Map.of(version.getVersionId(), version));
+ doReturn(secret).when(secrets).describeSecret(arn, AwsArnUtils.parse(arn).region());
}
private void setSecretReadError(String errorCode, String message) {
- doReturn(new BatchGetSecretValueResult(List.of(), List.of(new BatchGetSecretValueError(SECRET, errorCode, message))))
- .when(secrets).batchGetSecretValue(List.of(SECRET), "us-east-1");
+ doThrow(new AwsException(errorCode, message, 400))
+ .when(secrets).describeSecret(SECRET, "us-east-1");
+ }
+
+ private void scheduleDeletion(String arn) {
+ Secret metadata = new Secret();
+ metadata.setArn(arn);
+ metadata.setDeletedDate(Instant.now().plusSeconds(86400));
+ doReturn(metadata).when(secrets).describeSecret(arn, "us-east-1");
}
private MskScramAssociations state() {
@@ -238,7 +369,8 @@ class MskScramServiceTest {
String deletedCluster = cluster.getClusterArn().replace(ACCOUNT, otherAccount);
associations.putForAccount(otherAccount, deletedCluster, new MskScramAssociations());
- service.reconcileScramSecrets();
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+ service.reconcileCluster(deletedCluster, otherAccount);
assertTrue(associations.getForAccount(ACCOUNT, cluster.getClusterArn()).isPresent());
assertTrue(associations.getForAccount(otherAccount, deletedCluster).isEmpty());
@@ -251,7 +383,7 @@ class MskScramServiceTest {
AwsException failure = new AwsException("InternalServerErrorException", "Lookup failed.", 500);
doThrow(failure).when(clusterService).getClusterForAccount(ACCOUNT, cluster.getClusterArn());
- assertEquals(failure, assertThrows(AwsException.class, () -> service.reconcileScramSecrets()));
+ assertEquals(failure, assertThrows(AwsException.class, () -> service.reconcileCluster(cluster.getClusterArn(), ACCOUNT)));
assertTrue(associations.getForAccount(ACCOUNT, cluster.getClusterArn()).isPresent());
}
@@ -291,6 +423,8 @@ class MskScramServiceTest {
when(secrets.describeSecret(SECRET, "us-east-1")).thenReturn(metadata);
when(kms.describeKey(keyReference, "us-east-1")).thenReturn(new KmsKey());
setSecretValue(SECRET, version("alice", "password"));
+ Secret secret = secrets.describeSecret(SECRET, "us-east-1");
+ secret.setKmsKeyId(keyReference);
assertTrue(service.batchAssociateScramSecret(cluster.getClusterArn(), List.of(SECRET)).isEmpty());
@@ -348,16 +482,13 @@ class MskScramServiceTest {
@Test
void rejectsReturnedSecretArnForDifferentSecret() {
- when(secrets.batchGetSecretValue(List.of(SECRET), "us-east-1"))
- .thenReturn(
- new BatchGetSecretValueResult(List.of(new BatchSecretValue(
- SECRET + "Other", "AmazonMSK_fixture", "{\"username\":\"alice\",\"password\":\"password\"}",
- null, "version", List.of("AWSCURRENT"), null)), List.of())
- );
+ Secret secret = new Secret();
+ secret.setArn(SECRET + "Other");
+ when(secrets.describeSecret(SECRET, "us-east-1")).thenReturn(secret);
List<Map<String, String>> failures = service.batchAssociateScramSecret(cluster.getClusterArn(), List.of(SECRET));
- assertEquals("InternalServerError", failures.getFirst().get("errorCode"));
+ assertEquals("InvalidSecretArn", failures.getFirst().get("errorCode"));
assertTrue(service.listScramSecrets(cluster.getClusterArn(), null, null).items().isEmpty());
}
@@ -414,7 +545,7 @@ class MskScramServiceTest {
assertEquals(1, failures.size(), keyKind);
assertEquals("BadRequestException", failures.getFirst().get("errorCode"), keyKind);
assertNull(association(invalidArn));
- assertEquals("alice", association(SECRET).getUsername(), keyKind);
+ assertEquals(original.getVersionId(), association(SECRET).getAppliedVersionId(), keyKind);
}
verifyNoInteractions(redpanda);
assertTrue(service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET)).isEmpty());
@@ -489,6 +620,7 @@ class MskScramServiceTest {
List.of(SECRET),
service.listScramSecrets(cluster.getClusterArn(), null, null).items());
assertEquals("alice", association(SECRET).getUsername());
+ assertEquals(original.getVersionId(), association(SECRET).getAppliedVersionId());
verifyNoInteractions(redpanda, secrets, kms);
verify(associations, never()).putForAccount(anyString(), anyString(), any());
}
@@ -513,22 +645,49 @@ class MskScramServiceTest {
}
@Test
- void associationReadsCurrentVersionThroughExactArnBatchGetSecretValue() {
+ void associationReadsCurrentVersionFromSecretMetadata() {
// Association applies the AWSCURRENT secret value and records its version only after success.
SecretVersion version = version("alice", "password");
setSecretValue(SECRET, version);
+ Secret secret = secrets.describeSecret(SECRET, "us-east-1");
+ SecretVersion previous = version("bob", "old-password");
+ previous.setVersionStages(List.of("AWSPREVIOUS"));
+ secret.setVersions(Map.of(version.getVersionId(), version, previous.getVersionId(), previous));
+ secret.setCurrentVersionId(previous.getVersionId());
+ Instant lastAccessed = Instant.parse("2026-01-01T00:00:00Z");
+ secret.setLastAccessedDate(lastAccessed);
+ clearInvocations(secrets);
assertTrue(service.batchAssociateScramSecret(cluster.getClusterArn(), List.of(SECRET)).isEmpty());
- verify(secrets).batchGetSecretValue(List.of(SECRET), "us-east-1");
+ verify(secrets).describeSecret(SECRET, "us-east-1");
+ verify(secrets, never()).batchGetSecretValue(any(), anyString());
verify(secrets, never()).getSecretValue(anyString(), any(), any(), anyString());
verify(secrets, never()).batchGetSecretValueByFilters(any(), anyString());
verify(redpanda).createUser(cluster, "alice", "password");
+ assertEquals(version.getVersionId(), association(SECRET).getAppliedVersionId());
+ assertEquals(lastAccessed, secret.getLastAccessedDate());
+ }
+
+ @Test
+ void associationOfSecretScheduledForDeletionReportsInternalServerError() {
+ setSecretValue(SECRET, version("alice", "password"));
+ Secret secret = secrets.describeSecret(SECRET, "us-east-1");
+ secret.setDeletedDate(Instant.now().plusSeconds(86400));
+
+ List<Map<String, String>> failures =
+ service.batchAssociateScramSecret(cluster.getClusterArn(), List.of(SECRET));
+
+ assertEquals(List.of(Map.of("secretArn", SECRET, "errorCode", "InternalServerError",
+ "errorMessage", "You can't perform this operation on the secret because it was marked for deletion.")),
+ failures);
+ assertNull(association(SECRET));
+ verifyNoInteractions(redpanda);
}
@ParameterizedTest
@ValueSource(strings = {"ResourceNotFoundException", "InvalidRequestException", "AccessDeniedException"})
- void batchReadErrorsAreReportedPerSecretWithoutBlockingOtherAssociations(String errorCode) {
+ void secretLookupErrorsAreReportedPerSecretWithoutBlockingOtherAssociations(String errorCode) {
setSecretReadError(errorCode, "Secret cannot be read.");
String validArn = SECRET + "-valid";
setSecretValue(validArn, version("bob", "password"));
@@ -544,18 +703,21 @@ class MskScramServiceTest {
}
@Test
- void associationIsStoredAfterBrokerSideEffect() {
+ void associationIntentIsStoredBeforeBrokerUpdate() {
+ // Persisted ownership must precede broker creation so an interrupted operation remains recoverable.
doAnswer(
invocation -> {
- verify(associations, never()).putForAccount(anyString(), anyString(), any());
+ ScramSecretAssociation intent = association(SECRET);
+ assertNotNull(intent);
+ assertEquals("alice", intent.getUsername());
+ assertNull(intent.getAppliedVersionId());
return null;
})
.when(redpanda)
.createUser(cluster, "alice", "password");
associate(SECRET, version("alice", "password"));
- assertEquals("alice", association(SECRET).getUsername());
- verify(associations).putForAccount(eq(ACCOUNT), eq(cluster.getClusterArn()), any());
+ assertNotNull(association(SECRET).getAppliedVersionId());
}
@Test
@@ -574,6 +736,7 @@ class MskScramServiceTest {
associate(SECRET, version);
assertEquals("alice", association(SECRET).getUsername());
+ assertEquals(version.getVersionId(), association(SECRET).getAppliedVersionId());
assertEquals(List.of(SECRET), service.listScramSecrets(cluster.getClusterArn(), null, null).items());
}
@@ -587,6 +750,7 @@ class MskScramServiceTest {
service.batchAssociateScramSecret(cluster.getClusterArn(), List.of(SECRET));
assertEquals("InvalidSecretArn", failures.getFirst().get("errorCode"));
assertEquals("alice", association(SECRET).getUsername());
+ assertEquals(version.getVersionId(), association(SECRET).getAppliedVersionId());
verifyNoInteractions(redpanda);
verify(associations, never()).putForAccount(anyString(), anyString(), any());
assertEquals(List.of(SECRET), service.listScramSecrets(cluster.getClusterArn(), null, null).items());
@@ -597,8 +761,11 @@ class MskScramServiceTest {
// Existing membership determines duplicate rejection even when the secret cannot be read.
associate(SECRET, version("alice", "password"));
clearInvocations(redpanda);
- for (String errorCode : List.of("ResourceNotFoundException", "InvalidRequestException")) {
- setSecretReadError(errorCode, "Secret cannot be read.");
+ for (boolean scheduledForDeletion : List.of(false, true)) {
+ Secret secret = new Secret();
+ secret.setArn(SECRET);
+ secret.setDeletedDate(scheduledForDeletion ? Instant.now().plusSeconds(86400) : null);
+ doReturn(secret).when(secrets).describeSecret(SECRET, "us-east-1");
List<Map<String, String>> failures =
service.batchAssociateScramSecret(cluster.getClusterArn(), List.of(SECRET));
@@ -667,32 +834,16 @@ class MskScramServiceTest {
assertEquals(1, service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET)).size());
assertEquals("alice", association(SECRET).getUsername());
+ assertTrue(association(SECRET).isDisassociating());
clearInvocations(redpanda);
setSecretValue(SECRET, version("charlie", "changed-during-cleanup"));
doNothing().when(redpanda).deleteUser(cluster, "alice");
- assertTrue(service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET)).isEmpty());
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
assertTrue(state().getAssociations().isEmpty());
verify(redpanda).deleteUser(cluster, "alice");
verify(redpanda, never()).createUser(any(), anyString(), anyString());
}
- @Test
- void deletionRetryRemovesAssociationWithoutRewritingIt() {
- associate(SECRET, version("alice", "password"));
- doThrow(new AwsException("ServiceUnavailableException", "Broker unavailable.", 503))
- .when(redpanda)
- .deleteUser(cluster, "alice");
- service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET));
- clearInvocations(associations);
- doNothing().when(redpanda).deleteUser(cluster, "alice");
-
- assertTrue(service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET)).isEmpty());
-
- verify(associations, never()).putForAccount(anyString(), anyString(), any());
- verify(associations, times(1)).deleteForAccount(ACCOUNT, cluster.getClusterArn());
- assertTrue(state().getAssociations().isEmpty());
- }
-
@Test
void listScramSecretsReturnsEachPageAndOnlyProvidesTokenWhenMoreResultsExist() {
// Pagination exposes each association once and omits the token on the final page.
@@ -727,6 +878,411 @@ class MskScramServiceTest {
assertThrows(AwsException.class, () -> service.listScramSecrets("missing", null, null));
}
+ @Test
+ void reconcileUpdatesPasswordAndSkipsUnchangedVersion() {
+ // Reconciliation applies each new password version once and skips already applied versions.
+ associate(SECRET, version("alice", "old-password"));
+ clearInvocations(redpanda);
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+ verifyNoInteractions(redpanda);
+
+ SecretVersion changed = version("alice", "new-password");
+ setSecretValue(SECRET, changed);
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+ verify(redpanda, times(1)).createUser(cluster, "alice", "new-password");
+ assertEquals(changed.getVersionId(), association(SECRET).getAppliedVersionId());
+ }
+
+ @Test
+ void passwordRotationWritesOnlyCompletedState() {
+ // Password-only rotation persists completion once without an extra intent write.
+ associate(SECRET, version("alice", "password"));
+ clearInvocations(associations);
+ SecretVersion changed = version("alice", "new-password");
+ setSecretValue(SECRET, changed);
+
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+
+ verify(associations, times(1)).putForAccount(eq(ACCOUNT), eq(cluster.getClusterArn()), any());
+ assertEquals(changed.getVersionId(), association(SECRET).getAppliedVersionId());
+ }
+
+ @Test
+ void usernameRotationReplacesOldUserAndDisassociationDeletesTheNewUser() {
+ // Username rotation removes the old broker user and makes later cleanup target the new one.
+ associate(SECRET, version("alice", "password"));
+ setSecretValue(SECRET, version("bob", "new-password"));
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+
+ assertEquals("bob", association(SECRET).getUsername());
+ verify(redpanda).deleteUser(cluster, "alice");
+ verify(redpanda).createUser(cluster, "bob", "new-password");
+ assertTrue(
+ service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET))
+ .isEmpty());
+ verify(redpanda).deleteUser(cluster, "bob");
+ assertTrue(state().getAssociations().isEmpty());
+ }
+
+ @Test
+ void associationIsRejectedWhenRotatedContentIsInvalid() {
+ associate(SECRET, version("alice", "original"));
+ clearInvocations(redpanda);
+ SecretVersion invalid = version("unused", "unused");
+ invalid.setSecretString("{}");
+ setSecretValue(SECRET, invalid);
+ clearInvocations(secrets);
+
+ List<Map<String, String>> failures =
+ service.batchAssociateScramSecret(cluster.getClusterArn(), List.of(SECRET));
+
+ assertEquals(1, failures.size());
+ assertEquals("InvalidSecretArn", failures.getFirst().get("errorCode"));
+ verify(secrets).describeSecret(SECRET, "us-east-1");
+ verify(secrets, never()).batchGetSecretValue(any(), anyString());
+ verifyNoInteractions(redpanda);
+ }
+
+ @Test
+ void reconciliationPreservesAppliedCredentialsWhenRotatedContentIsInvalid() {
+ SecretVersion original = version("alice", "original");
+ associate(SECRET, original);
+ clearInvocations(redpanda);
+ SecretVersion invalid = version("unused", "unused");
+ invalid.setSecretString("{}");
+ setSecretValue(SECRET, invalid);
+
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+
+ assertEquals("alice", association(SECRET).getUsername());
+ assertEquals(original.getVersionId(), association(SECRET).getAppliedVersionId());
+ verifyNoInteractions(redpanda);
+ }
+
+ @Test
+ void repeatedReconciliationFailuresLogAtDebugUntilErrorChangesOrRecovery() {
+ associate(SECRET, version("alice", "original"));
+ setSecretReadError("AccessDeniedException", "Denied.");
+
+ List<LogRecord> logs = LogCapture.capture(MskScramSecretsService.class, () -> {
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+ setSecretReadError("AccessDeniedException", "Different denial.");
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+ setSecretReadError("InvalidRequestException", "Invalid request.");
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+ });
+ assertEquals(List.of("WARN", "DEBUG", "WARN", "WARN"), logLevels(logs));
+
+ setSecretValue(SECRET, version("alice", "recovered"));
+ logs = LogCapture.capture(MskScramSecretsService.class, () -> service.reconcileCluster(cluster.getClusterArn(), ACCOUNT));
+ assertTrue(logs.isEmpty());
+ verify(redpanda).createUser(cluster, "alice", "recovered");
+
+ setSecretReadError("InvalidRequestException", "Invalid request.");
+ logs = LogCapture.capture(MskScramSecretsService.class, () -> service.reconcileCluster(cluster.getClusterArn(), ACCOUNT));
+ assertEquals(List.of("WARN"), logLevels(logs));
+ }
+
+ private List<String> logLevels(List<LogRecord> logs) {
+ return logs.stream().map(logRecord -> logRecord.getLevel().getName()).toList();
+ }
+
+ @Test
+ void invalidRequestsMalformedUpdatesAndAccessDenialsPreserveAppliedVersion() {
+ // Unreadable or malformed updates are not deletion signals and must preserve the applied version.
+ SecretVersion original = version("alice", "password");
+ associate(SECRET, original);
+ clearInvocations(redpanda);
+ setSecretReadError("InvalidRequestException", "The requested operation is not valid for this secret.");
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+ SecretVersion invalid = new SecretVersion();
+ invalid.setVersionId("invalid");
+ invalid.setSecretString("{}");
+ setSecretValue(SECRET, invalid);
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+ setSecretReadError("AccessDeniedException", "Denied.");
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+
+ assertEquals(original.getVersionId(), association(SECRET).getAppliedVersionId());
+ verifyNoInteractions(redpanda);
+ }
+
+ @Test
+ void metadataAccessDenialPreservesWorkingCredentials() {
+ // Metadata access denial must not revoke credentials that are already working.
+ SecretVersion original = version("alice", "password");
+ associate(SECRET, original);
+ doThrow(new AwsException("AccessDeniedException", "Denied.", 403))
+ .when(secrets)
+ .describeSecret(SECRET, "us-east-1");
+ clearInvocations(redpanda);
+
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+
+ assertFalse(association(SECRET).isCredentialsRevoked());
+ assertEquals(original.getVersionId(), association(SECRET).getAppliedVersionId());
+ verifyNoInteractions(redpanda);
+ }
+
+ @Test
+ void missingAwsCurrentStagePreservesAssociationAndAppliedCredentials() {
+ associate(SECRET, version("alice", "password"));
+ clearInvocations(redpanda);
+
+ Secret secret = new Secret();
+ secret.setVersions(Map.of());
+ doReturn(secret).when(secrets).describeSecret(SECRET, "us-east-1");
+
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+
+ assertNotNull(association(SECRET));
+ assertFalse(association(SECRET).isCredentialsRevoked());
+ verifyNoInteractions(redpanda);
+ }
+
+ @Test
+ void brokerResourceNotFoundDoesNotTreatSecretAsDeleted() {
+ // Broker not-found errors must not be mistaken for a deleted Secrets Manager secret.
+ associate(SECRET, version("alice", "old-password"));
+ SecretVersion changed = version("alice", "new-password");
+ setSecretValue(SECRET, changed);
+ doThrow(new AwsException("ResourceNotFoundException", "Broker user not found.", 404))
+ .when(redpanda)
+ .createUser(cluster, "alice", "new-password");
+
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+
+ assertNotNull(association(SECRET));
+ assertTrue(state().getAssociations().containsKey(SECRET));
+ verify(redpanda, never()).deleteUser(any(), anyString());
+ }
+
+ @Test
+ void brokerFailureDoesNotMarkVersionAppliedAndIsRetried() {
+ // Failed broker updates leave the old applied version intact so reconciliation can retry.
+ SecretVersion original = version("alice", "old-password");
+ associate(SECRET, original);
+ SecretVersion changed = version("alice", "new-password");
+ setSecretValue(SECRET, changed);
+ doThrow(new AwsException("ServiceUnavailableException", "Broker unavailable.", 503))
+ .when(redpanda)
+ .createUser(cluster, "alice", "new-password");
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+ assertEquals(original.getVersionId(), association(SECRET).getAppliedVersionId());
+ doNothing().when(redpanda).createUser(cluster, "alice", "new-password");
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+ assertEquals(changed.getVersionId(), association(SECRET).getAppliedVersionId());
+ }
+
+ @Test
+ void reconcileUsesOwningAccountAndRegionWithoutRequestContext() {
+ // Background reconciliation resolves credentials and storage using the resource owner's context.
+ String account = "111111111111";
+ String region = "eu-west-1";
+ String arn = AwsArnUtils.Arn.of("kafka", region, account, "cluster/foreign/id").toString();
+ String secretArn =
+ AwsArnUtils.Arn.of("secretsmanager", region, account, "secret:foreign").toString();
+ MskCluster foreign = new MskCluster(arn, "foreign", "3.6.0");
+ foreign.setState(ClusterState.ACTIVE);
+ foreign.setContainerId("foreign-broker");
+ MskScramAssociations foreignAssociations = new MskScramAssociations();
+ foreignAssociations.getAssociations().put(secretArn, new ScramSecretAssociation());
+ clusters.putForAccount(account, arn, foreign);
+ associations.putForAccount(account, arn, foreignAssociations);
+ setSecretValue(secretArn, version("foreign", "password"));
+
+ service.reconcileCluster(arn, account);
+
+ verify(secrets).describeSecret(secretArn, region);
+ verify(redpanda).createUser(foreign, "foreign", "password");
+ assertTrue(clusters.get(arn).isEmpty());
+ assertEquals(
+ "foreign",
+ associations
+ .getForAccount(account, arn)
+ .orElseThrow()
+ .getAssociations()
+ .get(secretArn)
+ .getUsername());
+ }
+
+ @Test
+ void permanentDeletionRevokesCredentialsButRetainsMembershipAndRetriesBrokerFailure() {
+ // Deleted secrets retain membership, with revocation recorded only after broker cleanup succeeds.
+ SecretVersion original = version("alice", "password");
+ associate(SECRET, original);
+ clearInvocations(redpanda, secrets, associations);
+ doThrow(
+ new AwsException(
+ "ResourceNotFoundException",
+ "Secrets Manager can't find the specified secret.",
+ 400))
+ .when(secrets)
+ .describeSecret(SECRET, "us-east-1");
+ doThrow(new AwsException("ServiceUnavailableException", "Broker unavailable.", 503))
+ .when(redpanda)
+ .deleteUser(cluster, "alice");
+
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+
+ assertFalse(association(SECRET).isCredentialsRevoked());
+ assertEquals(original.getVersionId(), association(SECRET).getAppliedVersionId());
+ verify(associations, never()).putForAccount(anyString(), anyString(), any());
+ doNothing().when(redpanda).deleteUser(cluster, "alice");
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+
+ assertTrue(association(SECRET).isCredentialsRevoked());
+ assertNull(association(SECRET).getAppliedVersionId());
+ assertNull(association(SECRET).getAppliedContainerId());
+ assertEquals(
+ List.of(SECRET),
+ service.listScramSecrets(cluster.getClusterArn(), null, null).items());
+ verify(secrets, never()).batchGetSecretValue(any(), anyString());
+ verify(associations, times(1))
+ .putForAccount(eq(ACCOUNT), eq(cluster.getClusterArn()), any());
+ clearInvocations(redpanda, associations);
+
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+
+ verifyNoInteractions(redpanda);
+ verify(associations, never()).putForAccount(anyString(), anyString(), any());
+ assertTrue(service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET)).isEmpty());
+ assertTrue(state().getAssociations().isEmpty());
+ verifyNoInteractions(redpanda);
+ }
+
+ @Test
+ void scheduledDeletionRevokesCredentialsAndRestorationReappliesUnchangedVersion() {
+ // Scheduled deletion revokes credentials, and restoration reapplies them even without a new version.
+ SecretVersion original = version("alice", "password");
+ associate(SECRET, original);
+ scheduleDeletion(SECRET);
+ clearInvocations(redpanda, secrets, associations);
+
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+
+ assertTrue(association(SECRET).isCredentialsRevoked());
+ assertEquals(
+ List.of(SECRET),
+ service.listScramSecrets(cluster.getClusterArn(), null, null).items());
+ verify(redpanda, times(1)).deleteUser(cluster, "alice");
+ verify(redpanda, never()).createUser(any(), anyString(), anyString());
+ verify(secrets, never()).batchGetSecretValue(any(), anyString());
+ verify(associations, times(1))
+ .putForAccount(eq(ACCOUNT), eq(cluster.getClusterArn()), any());
+
+ setSecretValue(SECRET, original);
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+
+ assertFalse(association(SECRET).isCredentialsRevoked());
+ assertEquals(original.getVersionId(), association(SECRET).getAppliedVersionId());
+ assertEquals(cluster.getContainerId(), association(SECRET).getAppliedContainerId());
+ verify(redpanda, times(1)).createUser(cluster, "alice", "password");
+ verify(associations, times(2))
+ .putForAccount(eq(ACCOUNT), eq(cluster.getClusterArn()), any());
+ }
+
+ @Test
+ void revokedAssociationSurvivesRestartWithoutRepeatedDeletionOrStorageWrites()
+ throws Exception {
+ // Persisted revocation survives restart without repeated cleanup and still permits restoration.
+ SecretVersion original = version("alice", "password");
+ associate(SECRET, original);
+ scheduleDeletion(SECRET);
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+ MskScramAssociations restored =
+ mapper.readValue(mapper.writeValueAsString(state()), MskScramAssociations.class);
+ associations.putForAccount(ACCOUNT, cluster.getClusterArn(), restored);
+ MskScramSecretsService restarted =
+ new MskScramSecretsService(
+ storageFactory, config, clusterService, redpanda, secrets, kms, mapper);
+ clearInvocations(redpanda, associations);
+
+ restarted.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+
+ assertTrue(association(SECRET).isCredentialsRevoked());
+ assertEquals(
+ List.of(SECRET),
+ restarted.listScramSecrets(cluster.getClusterArn(), null, null).items());
+ verifyNoInteractions(redpanda);
+ verify(associations, never()).putForAccount(anyString(), anyString(), any());
+
+ setSecretValue(SECRET, original);
+ restarted.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+
+ assertFalse(association(SECRET).isCredentialsRevoked());
+ assertEquals(original.getVersionId(), association(SECRET).getAppliedVersionId());
+ verify(redpanda).createUser(cluster, "alice", "password");
+ }
+
+ @Test
+ void failedRevocationIsRetriedAfterRestart() throws Exception {
+ // Restart must preserve an unfinished revocation so broker cleanup is retried once successfully.
+ associate(SECRET, version("alice", "password"));
+ scheduleDeletion(SECRET);
+ doThrow(new AwsException("ServiceUnavailableException", "Broker unavailable.", 503))
+ .when(redpanda)
+ .deleteUser(cluster, "alice");
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+ MskScramAssociations restored =
+ mapper.readValue(mapper.writeValueAsString(state()), MskScramAssociations.class);
+ associations.putForAccount(ACCOUNT, cluster.getClusterArn(), restored);
+ MskScramSecretsService restarted =
+ new MskScramSecretsService(
+ storageFactory, config, clusterService, redpanda, secrets, kms, mapper);
+ doNothing().when(redpanda).deleteUser(cluster, "alice");
+ clearInvocations(redpanda);
+
+ restarted.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+
+ assertTrue(association(SECRET).isCredentialsRevoked());
+ verify(redpanda, times(1)).deleteUser(cluster, "alice");
+ }
+
+ @Test
+ void failedRestorationRemainsRetryable() {
+ // Failed credential restoration keeps revocation state intact until broker creation succeeds.
+ SecretVersion original = version("alice", "password");
+ associate(SECRET, original);
+ scheduleDeletion(SECRET);
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+ setSecretValue(SECRET, original);
+ doThrow(new AwsException("ServiceUnavailableException", "Broker unavailable.", 503))
+ .when(redpanda)
+ .createUser(cluster, "alice", "password");
+
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+
+ assertTrue(association(SECRET).isCredentialsRevoked());
+ assertNull(association(SECRET).getAppliedVersionId());
+ doNothing().when(redpanda).createUser(cluster, "alice", "password");
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+ assertFalse(association(SECRET).isCredentialsRevoked());
+ assertEquals(original.getVersionId(), association(SECRET).getAppliedVersionId());
+ }
+
+ @Test
+ void deletionAfterMetadataReadIsHandledOnNextReconciliation() {
+ SecretVersion original = version("alice", "password");
+ associate(SECRET, original);
+ Secret metadata = secrets.describeSecret(SECRET, "us-east-1");
+ doReturn(metadata)
+ .doThrow(new AwsException("ResourceNotFoundException", "Secret deleted.", 400))
+ .when(secrets)
+ .describeSecret(SECRET, "us-east-1");
+
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+
+ assertFalse(association(SECRET).isCredentialsRevoked());
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+
+ assertTrue(association(SECRET).isCredentialsRevoked());
+ assertEquals(
+ List.of(SECRET),
+ service.listScramSecrets(cluster.getClusterArn(), null, null).items());
+ }
+
private static Stream<Arguments> sharedUsernameAssociations() {
return Stream.of(
Arguments.of(Named.of("same password, sequential", false), false, "password"),
@@ -770,15 +1326,112 @@ class MskScramServiceTest {
.toList());
assertEquals("alice", association(SECRET).getUsername());
assertEquals("alice", association(second).getUsername());
+ assertEquals(firstVersion.getVersionId(), association(SECRET).getAppliedVersionId());
+ assertEquals(secondVersion.getVersionId(), association(second).getAppliedVersionId());
verify(redpanda, times("password".equals(secondPassword) ? 2 : 1))
.createUser(cluster, "alice", "password");
verify(redpanda, times("password".equals(secondPassword) ? 2 : 1))
.createUser(cluster, "alice", secondPassword);
}
+ @Test
+ void disassociationInvalidatesSharedUserAndReconciliationReappliesUnchangedSecret() {
+ // Deleting a shared broker user invalidates surviving associations so unchanged credentials reapply.
+ String second = SECRET + "-second";
+ SecretVersion surviving = version("alice", "surviving-password");
+ associate(SECRET, version("alice", "password"));
+ associate(second, surviving);
+ clearInvocations(redpanda);
+
+ assertTrue(
+ service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET))
+ .isEmpty());
+
+ verify(redpanda).deleteUser(cluster, "alice");
+ assertNull(association(SECRET));
+ assertEquals("alice", association(second).getUsername());
+ assertFalse(association(second).isDisassociating());
+ assertNull(association(second).getAppliedVersionId());
+ assertNull(association(second).getAppliedContainerId());
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+
+ verify(redpanda, times(1)).createUser(cluster, "alice", "surviving-password");
+ assertEquals(surviving.getVersionId(), association(second).getAppliedVersionId());
+ assertEquals(cluster.getContainerId(), association(second).getAppliedContainerId());
+ }
+
+ @Test
+ void disassociationPersistsRemovalAndSharedUserInvalidationInTwoWrites() {
+ // Disassociation persists cleanup intent and then the removal with shared-user invalidation.
+ String second = SECRET + "-second";
+ associate(SECRET, version("alice", "password"));
+ associate(second, version("alice", "surviving-password"));
+ clearInvocations(associations);
+
+ assertTrue(
+ service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET))
+ .isEmpty());
+
+ verify(associations, times(2))
+ .putForAccount(eq(ACCOUNT), eq(cluster.getClusterArn()), any());
+ assertNull(association(SECRET));
+ assertNull(association(second).getAppliedVersionId());
+ assertNull(association(second).getAppliedContainerId());
+ }
+
+ @Test
+ void usernameRotationDoesNotReapplyUnrelatedUsers() {
+ String unrelated = SECRET + "-unrelated";
+ SecretVersion other = version("charlie", "other-password");
+ associate(SECRET, version("alice", "password"));
+ associate(unrelated, other);
+ clearInvocations(redpanda);
+
+ setSecretValue(SECRET, version("bob", "new-password"));
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+
+ assertEquals(other.getVersionId(), association(unrelated).getAppliedVersionId());
+ assertEquals(cluster.getContainerId(), association(unrelated).getAppliedContainerId());
+ verify(redpanda).createUser(cluster, "bob", "new-password");
+ verify(redpanda, never()).createUser(cluster, "charlie", "other-password");
+ }
+
+ @Test
+ void disassociationAfterRenameOnlyDeletesTheRelevantUsername() {
+ // After shared secrets diverge in username, each disassociation deletes only its own broker user.
+ String second = SECRET + "-second";
+ associate(SECRET, version("alice", "password"));
+ associate(second, version("alice", "shared-password"));
+ setSecretValue(second, version("bob", "renamed-password"));
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+ clearInvocations(redpanda);
+
+ assertTrue(
+ service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET))
+ .isEmpty());
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+
+ assertEquals(
+ List.of(second),
+ service.listScramSecrets(cluster.getClusterArn(), null, null).items());
+ assertEquals("bob", association(second).getUsername());
+ verify(redpanda).deleteUser(cluster, "alice");
+ verify(redpanda, never()).deleteUser(cluster, "bob");
+ verify(redpanda, never()).createUser(any(), anyString(), anyString());
+ clearInvocations(redpanda);
+
+ assertTrue(
+ service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(second))
+ .isEmpty());
+
+ assertTrue(service.listScramSecrets(cluster.getClusterArn(), null, null).items().isEmpty());
+ verify(redpanda).deleteUser(cluster, "bob");
+ verify(redpanda, never()).deleteUser(cluster, "alice");
+ }
+
@Test
void failedUserDeletionDoesNotInvalidateSharedAssociations() {
- // Failed broker deletion leaves both associations available for a caller to retry.
+ // Failed broker deletion leaves shared credentials applied while retaining the cleanup intent.
String second = SECRET + "-second";
SecretVersion surviving = version("alice", "surviving-password");
associate(SECRET, version("alice", "password"));
@@ -792,8 +1445,84 @@ class MskScramServiceTest {
service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET))
.size());
- assertEquals("alice", association(second).getUsername());
- assertEquals("alice", association(SECRET).getUsername());
+ assertEquals(surviving.getVersionId(), association(second).getAppliedVersionId());
+ assertEquals(cluster.getContainerId(), association(second).getAppliedContainerId());
+ assertTrue(association(SECRET).isDisassociating());
+ }
+
+ @Test
+ void revocationRestoresSharedUserAndDisassociationDoesNotDeleteItAgain() {
+ // Revocation restores surviving shared credentials, and later removal must not delete them again.
+ String second = SECRET + "-second";
+ SecretVersion surviving = version("alice", "surviving-password");
+ associate(SECRET, version("alice", "password"));
+ associate(second, surviving);
+ scheduleDeletion(SECRET);
+ clearInvocations(redpanda);
+
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+
+ assertTrue(association(SECRET).isCredentialsRevoked());
+ assertEquals(surviving.getVersionId(), association(second).getAppliedVersionId());
+ assertEquals(
+ List.of(SECRET, second),
+ service.listScramSecrets(cluster.getClusterArn(), null, null).items());
+ verify(redpanda, times(1)).deleteUser(cluster, "alice");
+ verify(redpanda, times(1)).createUser(cluster, "alice", "surviving-password");
+ clearInvocations(redpanda);
+
+ assertTrue(
+ service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET))
+ .isEmpty());
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+
+ assertEquals(surviving.getVersionId(), association(second).getAppliedVersionId());
+ assertEquals(
+ List.of(second),
+ service.listScramSecrets(cluster.getClusterArn(), null, null).items());
+ verifyNoInteractions(redpanda);
+ }
+
+ @Test
+ void failedRevocationDoesNotInvalidateSharedUser() {
+ // A failed revocation must not invalidate or rewrite still-working shared-user state.
+ String second = SECRET + "-second";
+ SecretVersion surviving = version("alice", "surviving-password");
+ associate(SECRET, version("alice", "password"));
+ associate(second, surviving);
+ scheduleDeletion(SECRET);
+ doThrow(new AwsException("ServiceUnavailableException", "Broker unavailable.", 503))
+ .when(redpanda)
+ .deleteUser(cluster, "alice");
+ clearInvocations(associations);
+
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+
+ assertFalse(association(SECRET).isCredentialsRevoked());
+ assertEquals(surviving.getVersionId(), association(second).getAppliedVersionId());
+ verify(associations, never()).putForAccount(anyString(), anyString(), any());
+ }
+
+ @Test
+ void restoredSecretWithNewUsernameDoesNotDeleteSurvivingSharedUser() {
+ // Restoring a revoked secret under a new username must preserve the surviving shared user.
+ String second = SECRET + "-second";
+ SecretVersion surviving = version("alice", "surviving-password");
+ associate(SECRET, version("alice", "password"));
+ associate(second, surviving);
+ scheduleDeletion(SECRET);
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+ clearInvocations(redpanda);
+ doReturn(new Secret()).when(secrets).describeSecret(SECRET, "us-east-1");
+ setSecretValue(SECRET, version("bob", "restored-password"));
+
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+
+ assertFalse(association(SECRET).isCredentialsRevoked());
+ assertEquals("bob", association(SECRET).getUsername());
+ assertEquals(surviving.getVersionId(), association(second).getAppliedVersionId());
+ verify(redpanda).createUser(cluster, "bob", "restored-password");
+ verify(redpanda, never()).deleteUser(any(), anyString());
}
@Test
@@ -814,19 +1543,77 @@ class MskScramServiceTest {
restarted.listScramSecrets(cluster.getClusterArn(), null, null).items());
assertEquals("alice", restored.getAssociations().get(SECRET).getUsername());
clearInvocations(redpanda);
+ restarted.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+ verifyNoInteractions(redpanda);
+ }
+
+ @Test
+ void recreatedBrokerReceivesCurrentCredentialsOnlyAfterClusterBecomesActive() {
+ // A replacement broker receives existing credentials only once the cluster is active.
+ associate(SECRET, version("alice", "password"));
+ clearInvocations(redpanda);
+ cluster.setContainerId("broker-2");
+ cluster.setState(ClusterState.CREATING);
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
verifyNoInteractions(redpanda);
+ cluster.setState(ClusterState.ACTIVE);
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+ verify(redpanda).createUser(cluster, "alice", "password");
+ assertEquals("broker-2", association(SECRET).getAppliedContainerId());
}
@Test
- void mockModeTracksAssociationsWithoutBrokerCalls() {
+ void reconciliationRemovesOrphansFromTheirOwningAccount() {
+ // Orphan cleanup uses the owning account and must not remove another account's valid associations.
+ String account = "111111111111";
+ String arn =
+ AwsArnUtils.Arn.of("kafka", "eu-west-1", account, "cluster/deleted/id").toString();
+ MskScramAssociations orphan = new MskScramAssociations();
+ orphan.getAssociations().put(SECRET, new ScramSecretAssociation());
+ associations.putForAccount(account, arn, orphan);
+ associate(SECRET, version("alice", "password"));
+ clearInvocations(redpanda, secrets);
+
+ service.reconcileCluster(arn, account);
+
+ assertTrue(associations.getForAccount(account, arn).isEmpty());
+ assertNotNull(association(SECRET));
+ verifyNoInteractions(redpanda);
+ }
+
+ @Test
+ void mockModeTracksAndReconcilesAssociationsWithoutBrokerCalls() {
// Mock mode maintains association lifecycle state while suppressing all broker operations.
when(config.services().msk().mock()).thenReturn(true);
associate(SECRET, version("alice", "password"));
- assertEquals("alice", association(SECRET).getUsername());
+ setSecretValue(SECRET, version("bob", "password"));
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
+ assertEquals("bob", association(SECRET).getUsername());
service.batchDisassociateScramSecret(cluster.getClusterArn(), List.of(SECRET));
verifyNoInteractions(redpanda);
}
+ @Test
+ void reconcileSkipsServerlessClusterWithStaleAssociation() {
+ // Stale SCRAM state on a serverless cluster must not trigger credential reads or broker calls.
+ String serverlessArn =
+ AwsArnUtils.Arn.of("kafka", "us-east-1", ACCOUNT, "cluster/serverless/id")
+ .toString();
+ MskCluster serverless = new MskCluster(serverlessArn, "serverless", "3.6.0");
+ serverless.setClusterType("SERVERLESS");
+ serverless.setState(ClusterState.ACTIVE);
+ serverless.setContainerId("unrelated-broker");
+ clusters.putForAccount(ACCOUNT, serverlessArn, serverless);
+ MskScramAssociations staleAssociations = new MskScramAssociations();
+ staleAssociations.getAssociations().put(SECRET, new ScramSecretAssociation());
+ associations.putForAccount(ACCOUNT, serverlessArn, staleAssociations);
+ clearInvocations(secrets, redpanda);
+
+ service.reconcileCluster(serverlessArn, ACCOUNT);
+
+ verifyNoInteractions(secrets, redpanda);
+ }
+
@Test
void failedClusterDeletionRetainsScramAssociationState() {
// Failed cluster deletion retains both the cluster and its recoverable SCRAM association state.
@@ -837,6 +1624,7 @@ class MskScramServiceTest {
assertThrows(
AwsException.class, () -> clusterService.deleteCluster(cluster.getClusterArn()));
+ service.reconcileCluster(cluster.getClusterArn(), ACCOUNT);
assertNotNull(association(SECRET));
assertTrue(clusters.get(cluster.getClusterArn()).isPresent());
diff --git a/src/test/resources/application.yml b/src/test/resources/application.yml
index e642926088..3efa98900c 100644
--- a/src/test/resources/application.yml
+++ b/src/test/resources/application.yml
@@ -182,6 +182,7 @@ floci:
msk:
enabled: true
mock: true
+ scram-reconciliation-interval-ms: 5000
default-image: "redpandadata/redpanda:latest"
amazonmq:
enabled: true