Light, fluffy, and always free - The AWS Local Emulator alternative
feat(redshift): rotate managed master passwords (#6505)
Vũ Thế Huyce0933be9bd0
diff --git a/docs/services/redshift.md b/docs/services/redshift.mdindex 1c91b874f9..0c93f73745 100644--- a/docs/services/redshift.md+++ b/docs/services/redshift.md@@ -273,7 +273,9 @@ For `AWS::Redshift::ClusterParameterGroup`, `Parameters` is applied via `ModifyC- `ManageMasterPassword` creates a Redshift-owned Secrets Manager secret. The secret contains themanaged username, password, endpoint, port, and database name. `MasterPasswordSecretKmsKeyId`selects the KMS key used for the secret metadata and is validated through KMS. Switching managed- password mode or its KMS key in place is not implemented.+ password mode or its KMS key in place is not implemented. `RotateSecret` and scheduled rotation+ update both the cluster's master password and its secret value; rotation requires an available+ cluster and does not use a Lambda function.- Snapshot restoration has the same limitations as `RestoreFromClusterSnapshot`: snapshots arelocal SQL dumps, not cross-account backups or a full Redshift engine image.- `AWS::Redshift::ClusterSecurityGroup` is accepted as metadata: Floci does not emulate the legacy EC2-Classic security group model.@@ -699,5 +701,6 @@ These views expose the documented Redshift column names, types, and ordering map- The auth proxy validates the master user's password and any live `GetClusterCredentials` credential. Other non-master users pass straight through to PostgreSQL, which remains the authority for their credentials.- A cluster using `ManageMasterPassword` keeps its generated password in sync with theRedshift-owned Secrets Manager secret. Updating `MasterUserPassword` through `ModifyCluster`- updates the current secret version as well.+ updates the current secret version as well. Service-managed `RotateSecret` and scheduled rotation+ change the backend password and publish the matching secret version without invoking Lambda.- IAM database authentication over the wire, and `sslmode=verify-full` against the self-signed proxy certificate.diff --git a/docs/services/secrets-manager.md b/docs/services/secrets-manager.mdindex 9382fd6d82..f8759df6a7 100644--- a/docs/services/secrets-manager.md+++ b/docs/services/secrets-manager.md@@ -137,6 +137,9 @@ aws secretsmanager validate-resource-policy \- **Rotation schedules fire.** A background sweep (see the config table) rotates any secret whose`NextRotationDate` has passed. `AutomaticallyAfterDays`, `rate()` and `cron()` are all honoured.+- **Redshift-owned master secrets rotate with their cluster.** `RotateSecret` and scheduled+ rotation change the provisioned Redshift cluster password, then publish the matching secret+ version. The cluster must be available, and a rotation Lambda is not used.- **`RotateSecret` also needs `lambda:InvokeFunction` under IAM enforcement.** As on AWS, the callermust be allowed to invoke the rotation function as well as to call `RotateSecret`. A caller thatis not gets `AccessDeniedException` with HTTP 400, as Secrets Manager reports it, and the secret isdiff --git a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.javaindex 2222dbf03a..f0134bbaa3 100644--- a/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java+++ b/src/main/java/io/github/hectorvent/floci/services/redshift/RedshiftService.java@@ -26,6 +26,7 @@ import io.github.hectorvent.floci.services.redshift.model.Snapshot;import io.github.hectorvent.floci.services.redshift.model.SnapshotCopyGrant;import io.github.hectorvent.floci.services.redshift.model.SnapshotSchedule;import io.github.hectorvent.floci.services.redshift.proxy.RedshiftProxyManager;+import io.github.hectorvent.floci.services.secretsmanager.ManagedSecretRotator;import io.github.hectorvent.floci.services.secretsmanager.RandomPasswordGenerator;import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;import io.github.hectorvent.floci.services.secretsmanager.model.Secret;@@ -59,7 +60,7 @@ import java.util.regex.Matcher;import java.util.regex.Pattern;@ApplicationScoped-public class RedshiftService {+public class RedshiftService implements ManagedSecretRotator {private static final Logger LOG = Logger.getLogger(RedshiftService.class);// The group every cluster starts on until one is named.@@ -95,6 +96,7 @@ public class RedshiftService {// Proxy ports currently handed out, so allocateProxyPort never double-assigns within this JVM.private final Set<Integer> usedPorts = ConcurrentHashMap.newKeySet();private final ConcurrentHashMap<ResourceKey, ResourceLock> resourceLocks = new ConcurrentHashMap<>();+ private final ConcurrentHashMap<String, ResourceKey> managedRotationReservations = new ConcurrentHashMap<>();private record ResourceKey(String accountId, String kind, String identifier) { }@@ -104,7 +106,11 @@ public class RedshiftService {}private <T> T withResourceLock(String kind, String identifier, Supplier<T> work) {- ResourceKey key = new ResourceKey(clusters.accountId(), kind, identifier);+ return withResourceLock(clusters.accountId(), kind, identifier, work);+ }++ private <T> T withResourceLock(String accountId, String kind, String identifier, Supplier<T> work) {+ ResourceKey key = new ResourceKey(accountId, kind, identifier);ResourceLock scoped = resourceLocks.compute(key, (ignored, existing) -> {ResourceLock selected = existing == null ? new ResourceLock() : existing;selected.users++;@@ -121,7 +127,18 @@ public class RedshiftService {// Acquire resource locks before the service monitor; snapshot I/O never takes the monitor.private <T> T mutateCluster(String identifier, Supplier<T> mutation) {- return withResourceLock("cluster", identifier, () -> mutate(mutation));+ return withResourceLock("cluster", identifier, () -> {+ ensureNoManagedRotation(clusters.accountId(), identifier);+ return mutate(mutation);+ });+ }++ private void ensureNoManagedRotation(String accountId, String identifier) {+ ResourceKey key = new ResourceKey(accountId, "cluster", identifier);+ if (managedRotationReservations.containsValue(key)) {+ throw new AwsException("InvalidRequestException",+ "The cluster's managed master password is being rotated. Try again after rotation completes.", 400);+ }}private synchronized <T> T mutate(Supplier<T> mutation) {@@ -699,6 +716,7 @@ public class RedshiftService {public Cluster deleteCluster(String identifier, Boolean skipFinalClusterSnapshot,String finalSnapshotIdentifier, Integer finalSnapshotRetentionPeriod) {return withResourceLock("cluster", identifier, () -> {+ ensureNoManagedRotation(clusters.accountId(), identifier);Supplier<Cluster> deletion = () -> deleteClusterWithFinalSnapshot(identifier, skipFinalClusterSnapshot,finalSnapshotIdentifier, finalSnapshotRetentionPeriod);return Boolean.TRUE.equals(skipFinalClusterSnapshot) ? deletion.get()@@ -911,6 +929,98 @@ public class RedshiftService {}}+ @Override+ public boolean supports(String secretArn, String region) {+ if (!AwsArnUtils.isArn(secretArn) || !region.equals(AwsArnUtils.parse(secretArn).region())) {+ return false;+ }+ return clusters.scanAllAccountEntries(key -> true).stream()+ .anyMatch(entry -> secretArn.equals(entry.value().getMasterPasswordSecretArn()));+ }++ @Override+ public void rotate(String secretArn, String region, ManagedSecretRotator.Rotation rotation) {+ AccountAwareStorageBackend.AccountEntry<Cluster> entry = clusters.scanAllAccountEntries(key -> true).stream()+ .filter(candidate -> secretArn.equals(candidate.value().getMasterPasswordSecretArn()))+ .findFirst()+ .orElseThrow(() -> new AwsException("ResourceNotFoundException",+ "The Redshift cluster for this managed secret no longer exists.", 400));+ withResourceLock(entry.accountId(), "cluster", entry.key(), () -> mutate(() -> {+ Cluster cluster = clusters.getForAccount(entry.accountId(), entry.key())+ .orElseThrow(() -> new AwsException("ResourceNotFoundException",+ "The Redshift cluster for this managed secret no longer exists.", 400));+ if (!secretArn.equals(cluster.getMasterPasswordSecretArn())) {+ throw new AwsException("ResourceNotFoundException",+ "The Redshift cluster for this managed secret no longer exists.", 400);+ }+ if (!"available".equals(cluster.getClusterStatus())) {+ throw new AwsException("InvalidRequestException",+ "The Redshift cluster must be available to rotate its master password.", 400);+ }+ String oldPassword = cluster.getMasterPassword();+ String newPassword = RandomPasswordGenerator.generate(objectMapper.createObjectNode()+ .put("ExcludeCharacters", "'\"/@\\"));+ rotation.commit(() -> {+ containerManager.alterUserPassword(entry.accountId(), cluster.getClusterIdentifier(),+ cluster.getMasterUsername(), newPassword);+ proxyManager.updateMasterPassword(+ relayKey(entry.accountId(), cluster.getClusterIdentifier()), newPassword);+ cluster.setMasterPassword(newPassword);+ clusters.putForAccount(entry.accountId(), cluster.getClusterIdentifier(), cluster);+ clusters.flush();+ return managedMasterSecret(cluster, newPassword);+ }, () -> {+ containerManager.alterUserPassword(entry.accountId(), cluster.getClusterIdentifier(),+ cluster.getMasterUsername(), oldPassword);+ proxyManager.updateMasterPassword(+ relayKey(entry.accountId(), cluster.getClusterIdentifier()), oldPassword);+ cluster.setMasterPassword(oldPassword);+ clusters.putForAccount(entry.accountId(), cluster.getClusterIdentifier(), cluster);+ clusters.flush();+ });+ return null;+ }));+ }++ @Override+ public void reserve(String secretArn, String region) {+ AccountAwareStorageBackend.AccountEntry<Cluster> entry = clusters.scanAllAccountEntries(key -> true).stream()+ .filter(candidate -> secretArn.equals(candidate.value().getMasterPasswordSecretArn()))+ .findFirst()+ .orElseThrow(() -> new AwsException("ResourceNotFoundException",+ "The Redshift cluster for this managed secret no longer exists.", 400));+ ResourceKey key = new ResourceKey(entry.accountId(), "cluster", entry.key());+ withResourceLock(entry.accountId(), "cluster", entry.key(), () -> {+ Cluster cluster = clusters.getForAccount(entry.accountId(), entry.key())+ .orElseThrow(() -> new AwsException("ResourceNotFoundException",+ "The Redshift cluster for this managed secret no longer exists.", 400));+ if (!secretArn.equals(cluster.getMasterPasswordSecretArn())) {+ throw new AwsException("ResourceNotFoundException",+ "The Redshift cluster for this managed secret no longer exists.", 400);+ }+ if (!"available".equals(cluster.getClusterStatus())) {+ throw new AwsException("InvalidRequestException",+ "The Redshift cluster must be available to rotate its master password.", 400);+ }+ if (managedRotationReservations.putIfAbsent(secretArn, key) != null) {+ throw new AwsException("InvalidRequestException",+ "The cluster's managed master password is already being rotated.", 400);+ }+ return null;+ });+ }++ @Override+ public void release(String secretArn, String region) {+ ResourceKey key = managedRotationReservations.get(secretArn);+ if (key != null) {+ withResourceLock(key.accountId(), key.kind(), key.identifier(), () -> {+ managedRotationReservations.remove(secretArn, key);+ return null;+ });+ }+ }+public Cluster describeLoggingStatus(String clusterIdentifier) {return clusters.get(clusterIdentifier).orElseThrow(() -> new AwsException("ClusterNotFound", "Cluster " + clusterIdentifier + " not found", 404));@@ -1236,8 +1346,11 @@ public class RedshiftService {}public Cluster restoreFromClusterSnapshot(String clusterIdentifier, String snapshotIdentifier, String nodeType) {- return withResourceLock("cluster", clusterIdentifier, () -> withResourceLock("snapshot", snapshotIdentifier,- () -> mutate(() -> restoreSnapshot(clusterIdentifier, snapshotIdentifier, nodeType))));+ return withResourceLock("cluster", clusterIdentifier, () -> {+ ensureNoManagedRotation(clusters.accountId(), clusterIdentifier);+ return withResourceLock("snapshot", snapshotIdentifier,+ () -> mutate(() -> restoreSnapshot(clusterIdentifier, snapshotIdentifier, nodeType)));+ });}public Cluster restoreFromClusterSnapshot(String clusterIdentifier, String snapshotIdentifier, String nodeType,@@ -1248,6 +1361,7 @@ public class RedshiftService {}return withResourceLock("cluster", clusterIdentifier, () -> withResourceLock("snapshot", snapshotIdentifier,() -> {+ ensureNoManagedRotation(clusters.accountId(), clusterIdentifier);Cluster cluster = restoreSnapshot(clusterIdentifier, snapshotIdentifier, nodeType);try {cluster.setClusterSubnetGroupName(clusterSubnetGroupName);diff --git a/src/main/java/io/github/hectorvent/floci/services/secretsmanager/ManagedSecretRotator.java b/src/main/java/io/github/hectorvent/floci/services/secretsmanager/ManagedSecretRotator.javanew file mode 100644index 0000000000..ffac1a9897--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/secretsmanager/ManagedSecretRotator.java@@ -0,0 +1,20 @@+package io.github.hectorvent.floci.services.secretsmanager;++import java.util.function.Supplier;++/** Supplies the backend credential change for a service-owned secret rotation. */+public interface ManagedSecretRotator {++ boolean supports(String secretArn, String region);++ default void reserve(String secretArn, String region) { }++ default void release(String secretArn, String region) { }++ void rotate(String secretArn, String region, Rotation rotation);++ /** Commits a backend password change and its corresponding secret value as one rotation. */+ interface Rotation {+ void commit(Supplier<String> changeBackend, Runnable rollbackBackend);+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/secretsmanager/SecretRotationScheduler.java b/src/main/java/io/github/hectorvent/floci/services/secretsmanager/SecretRotationScheduler.javaindex d6857afd0d..750bbf0a6f 100644--- a/src/main/java/io/github/hectorvent/floci/services/secretsmanager/SecretRotationScheduler.java+++ b/src/main/java/io/github/hectorvent/floci/services/secretsmanager/SecretRotationScheduler.java@@ -84,7 +84,9 @@ public class SecretRotationScheduler {void tick(Instant now) {for (SecretsManagerService.RegionalSecret entry : service.listAllSecrets()) {- if (!isDue(entry.secret(), now)) {+ if (!isDue(entry.secret(), now)+ || (entry.secret().getOwningService() != null+ && !service.supportsManagedRotation(entry.secret().getArn(), entry.region()))) {continue;}try {@@ -109,7 +111,7 @@ public class SecretRotationScheduler {}// A replica is read-only, a deleted secret is not rotatable, and a secret another AWS// service owns is rotated by that service rather than through a rotation Lambda.- if (secret.isReplica() || secret.getDeletedDate() != null || secret.getOwningService() != null) {+ if (secret.isReplica() || secret.getDeletedDate() != null) {return false;}return !secret.getNextRotationDate().isAfter(now);diff --git a/src/main/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerService.java b/src/main/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerService.javaindex ab6e110ac9..974d8de370 100644--- a/src/main/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerService.java+++ b/src/main/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerService.java@@ -25,6 +25,7 @@ import io.github.hectorvent.floci.services.secretsmanager.model.Secret;import io.github.hectorvent.floci.services.secretsmanager.model.SecretVersion;import jakarta.annotation.PreDestroy;import jakarta.enterprise.context.ApplicationScoped;+import jakarta.enterprise.inject.Instance;import jakarta.inject.Inject;import org.jboss.logging.Logger;@@ -48,6 +49,7 @@ import java.util.concurrent.ThreadLocalRandom;import java.util.concurrent.atomic.AtomicInteger;import java.util.function.Consumer;import java.util.function.Predicate;+import java.util.function.Supplier;import java.util.regex.Pattern;@ApplicationScoped@@ -71,6 +73,7 @@ public class SecretsManagerService implements ResourceProvider {private final LambdaService lambdaService;private final ObjectMapper objectMapper;private final KmsService kmsService;+ private final List<ManagedSecretRotator> managedSecretRotators;// Fixed threads with an unbounded queue: a burst of rotations is queued, not rejected,// because RotateSecret documents no overload error.@@ -87,6 +90,7 @@ public class SecretsManagerService implements ResourceProvider {private final Object[] nameLockStripes = newLockStripes(NAME_LOCK_STRIPE_COUNT);// Entries exist only while a service owner changes backend credentials outside the stripe.private final Set<String> managedReleases = ConcurrentHashMap.newKeySet();+ private final Map<String, String> managedRotationTokens = new ConcurrentHashMap<>();private static Object[] newLockStripes(int count) {Object[] stripes = new Object[count];@@ -122,13 +126,28 @@ public class SecretsManagerService implements ResourceProvider {}@Inject+ public SecretsManagerService(StorageFactory factory, EmulatorConfig config, RegionResolver regionResolver,+ LambdaService lambdaService, ObjectMapper objectMapper,+ KmsService kmsService, Instance<ManagedSecretRotator> managedSecretRotators) {+ this(factory.create("secretsmanager", "secretsmanager-secrets.json",+ new TypeReference<Map<String, Secret>>() {}),+ config.services().secretsmanager().defaultRecoveryWindowDays(),+ regionResolver, lambdaService, objectMapper, kmsService, managedSecretRotators.stream().toList());+ }+public SecretsManagerService(StorageFactory factory, EmulatorConfig config, RegionResolver regionResolver,LambdaService lambdaService, ObjectMapper objectMapper,KmsService kmsService) {+ this(factory, config, regionResolver, lambdaService, objectMapper, kmsService, List.of());+ }++ private SecretsManagerService(StorageFactory factory, EmulatorConfig config, RegionResolver regionResolver,+ LambdaService lambdaService, ObjectMapper objectMapper,+ KmsService kmsService, List<ManagedSecretRotator> managedSecretRotators) {this(factory.create("secretsmanager", "secretsmanager-secrets.json",new TypeReference<Map<String, Secret>>() {}),config.services().secretsmanager().defaultRecoveryWindowDays(),- regionResolver, lambdaService, objectMapper, kmsService);+ regionResolver, lambdaService, objectMapper, kmsService, managedSecretRotators);}SecretsManagerService(StorageBackend<String, Secret> store, int defaultRecoveryWindowDays) {@@ -144,12 +163,20 @@ public class SecretsManagerService implements ResourceProvider {SecretsManagerService(StorageBackend<String, Secret> store, int defaultRecoveryWindowDays,RegionResolver regionResolver, LambdaService lambdaService,ObjectMapper objectMapper, KmsService kmsService) {+ this(store, defaultRecoveryWindowDays, regionResolver, lambdaService, objectMapper, kmsService, List.of());+ }++ SecretsManagerService(StorageBackend<String, Secret> store, int defaultRecoveryWindowDays,+ RegionResolver regionResolver, LambdaService lambdaService,+ ObjectMapper objectMapper, KmsService kmsService,+ List<ManagedSecretRotator> managedSecretRotators) {this.store = store;this.defaultRecoveryWindowDays = defaultRecoveryWindowDays;this.regionResolver = regionResolver;this.lambdaService = lambdaService;this.objectMapper = objectMapper;this.kmsService = kmsService;+ this.managedSecretRotators = List.copyOf(managedSecretRotators);}@PreDestroy@@ -311,6 +338,7 @@ public class SecretsManagerService implements ResourceProvider {}synchronized (lockFor(secret.getArn())) {+ throwIfManagedRelease(secret.getArn());if (clientRequestToken != null && secret.getVersions() != null && secret.getVersions().containsKey(clientRequestToken)) {SecretVersion existingVersion = secret.getVersions().get(clientRequestToken);boolean isPendingPlaceholder = existingVersion.getSecretString() == null@@ -418,6 +446,7 @@ public class SecretsManagerService implements ResourceProvider {validateKmsKey(kmsKeyId, region);synchronized (lockFor(secret.getArn())) {+ throwIfManagedRelease(secret.getArn());if (description != null) {secret.setDescription(description);}@@ -963,6 +992,10 @@ public class SecretsManagerService implements ResourceProvider {return all;}+ public boolean supportsManagedRotation(String secretArn, String region) {+ return managedSecretRotators.stream().anyMatch(rotator -> rotator.supports(secretArn, region));+ }+/** A replication target: the region to copy into, and optionally the KMS key to record. */public record ReplicaRegion(String region, String kmsKeyId) {}@@ -1275,7 +1308,12 @@ public class SecretsManagerService implements ResourceProvider {// A service-managed secret is rotated by its owning service, so it has no rotation Lambda:// AWS rejects one here, and does not require one to enable rotation.+ String secretArn = secret.getArn();boolean serviceManaged = secret.getOwningService() != null;+ ManagedSecretRotator managedRotator = serviceManaged ? managedSecretRotators.stream()+ .filter(candidate -> candidate.supports(secretArn, region))+ .findFirst()+ .orElse(null) : null;if (serviceManaged && rotationLambdaArn != null) {throw new AwsException("InvalidRequestException","Rotation Lambda ARN is not supported for a service-managed secret.", 400);@@ -1316,38 +1354,97 @@ public class SecretsManagerService implements ResourceProvider {}}- synchronized (lockFor(secret.getArn())) {- SecretVersion pendingVersion = findVersionByStage(secret, "AWSPENDING");- if (pendingVersion != null) {- SecretVersion currentVersion = findVersionByStage(secret, "AWSCURRENT");- if (currentVersion == null || !pendingVersion.getVersionId().equals(currentVersion.getVersionId())) {+ String managedToken = serviceManaged && rotateImmediately+ ? clientRequestToken != null && !clientRequestToken.isEmpty()+ ? clientRequestToken : UUID.randomUUID().toString() : null;++ if (managedToken != null && managedRotator != null) {+ synchronized (lockFor(secret.getArn())) {+ Secret latest = resolveSecret(secret.getArn(), region);+ SecretVersion existing = latest.getVersions() == null ? null : latest.getVersions().get(managedToken);+ if (existing != null) {+ return latest;+ }+ String activeToken = managedRotationTokens.get(secret.getArn());+ if (activeToken != null) {+ if (activeToken.equals(managedToken)) {+ return latest;+ }throw new AwsException("InvalidRequestException","A previous rotation isn't complete. That rotation will be reattempted.", 400);}+ throwIfManagedRelease(secret.getArn());+ managedRotationTokens.put(secret.getArn(), managedToken);}+ }- if (rotationLambdaArn != null) {- secret.setRotationLambdaArn(rotationLambdaArn);- }- if (rotationRules != null) {- secret.setRotationRules(rotationRules);+ boolean managedReservationAcquired = false;+ boolean rotationPrepared = false;+ try {+ if (managedToken != null && managedRotator != null) {+ managedRotator.reserve(secret.getArn(), region);+ managedReservationAcquired = true;}- secret.setRotationEnabled(true);- secret.setLastChangedDate(Instant.now());- // Scheduled from now rather than from the rotation's completion on purpose: the- // rotation itself runs asynchronously and may fail, and a schedule that only advanced- // on success would leave the background sweep retrying it on every tick forever.- secret.setNextRotationDate(computeNextRotationDate(secret.getRotationRules(), Instant.now()));- // A service-managed secret is rotated by its owning service, which floci does not- // emulate: the master password is not re-issued here, so nothing sets LastRotatedDate.- // Reporting a rotation that did not happen would leave GetSecretValue and the database- // holding a credential the secret claims to have replaced.+ synchronized (lockFor(secret.getArn())) {+ if (managedToken != null && managedRotator != null) {+ secret = resolveSecret(secret.getArn(), region);+ if (secret.getVersions() != null && secret.getVersions().containsKey(managedToken)) {+ return secret;+ }+ managedReleases.add(secret.getArn());+ }+ SecretVersion pendingVersion = findVersionByStage(secret, "AWSPENDING");+ if (pendingVersion != null) {+ SecretVersion currentVersion = findVersionByStage(secret, "AWSCURRENT");+ if (currentVersion == null || !pendingVersion.getVersionId().equals(currentVersion.getVersionId())) {+ throw new AwsException("InvalidRequestException",+ "A previous rotation isn't complete. That rotation will be reattempted.", 400);+ }+ }- persist(secret, region);+ if (rotationLambdaArn != null) {+ secret.setRotationLambdaArn(rotationLambdaArn);+ }+ if (rotationRules != null) {+ secret.setRotationRules(rotationRules);+ }+ secret.setRotationEnabled(true);+ secret.setLastChangedDate(Instant.now());+ // Scheduled from now rather than from the rotation's completion on purpose: the+ // rotation itself runs asynchronously and may fail, and a schedule that only advanced+ // on success would leave the background sweep retrying it on every tick forever.+ secret.setNextRotationDate(computeNextRotationDate(secret.getRotationRules(), Instant.now()));+ persist(secret, region);+ rotationPrepared = true;+ }+ } finally {+ if (!rotationPrepared) {+ if (managedToken != null && managedRotator != null) {+ clearManagedRotation(secret.getArn(), managedToken);+ }+ if (managedReservationAcquired) {+ managedRotator.release(secret.getArn(), region);+ }+ }}if (serviceManaged) {- LOG.infov("Enabled service-managed rotation for secret: {0}", secret.getName());+ if (managedRotator == null) {+ return secret;+ }+ if (!rotateImmediately) {+ LOG.infov("Enabled service-managed rotation for secret: {0}", secret.getName());+ return secret;+ }+ String arn = secret.getArn();+ try {+ rotationExecutor.submit(() -> executeManagedRotation(arn, managedToken, region, managedRotator));+ } catch (RuntimeException e) {+ clearManagedRotation(arn, managedToken);+ managedRotator.release(arn, region);+ throw e;+ }+ LOG.infov("Started service-managed rotation for secret: {0}", secret.getName());return secret;}@@ -1367,6 +1464,90 @@ public class SecretsManagerService implements ResourceProvider {return secret;}+ private void executeManagedRotation(String secretArn, String token, String region,+ ManagedSecretRotator rotator) {+ try {+ AtomicInteger committed = new AtomicInteger();+ rotator.rotate(secretArn, region, new ManagedSecretRotator.Rotation() {+ @Override+ public void commit(Supplier<String> changeBackend, Runnable rollbackBackend) {+ try {+ String newValue = changeBackend.get();+ synchronized (lockFor(secretArn)) {+ Secret current = resolveSecret(secretArn, region);+ putManagedRotationVersion(current, token, current.getCurrentVersionId(), newValue);+ current.setLastRotatedDate(Instant.now());+ current.setLastChangedDate(Instant.now());+ persist(current, region);+ }+ committed.set(1);+ } catch (RuntimeException e) {+ rollbackBackend.run();+ throw e;+ }+ }+ });+ if (committed.get() == 0) {+ throw new IllegalStateException("The managed secret rotator did not commit a new value");+ }+ } catch (Exception e) {+ LOG.errorv(e, "Managed rotation failed for secret {0}", secretArn);+ synchronized (lockFor(secretArn)) {+ Secret current = resolveSecret(secretArn, region);+ if (current.getVersions() != null) {+ SecretVersion pending = current.getVersions().remove(token);+ if (pending != null) {+ persist(current, region);+ }+ }+ persist(current, region);+ }+ } finally {+ clearManagedRotation(secretArn, token);+ rotator.release(secretArn, region);+ }+ }++ private void clearManagedRotation(String secretArn, String token) {+ synchronized (lockFor(secretArn)) {+ if (managedRotationTokens.remove(secretArn, token)) {+ managedReleases.remove(secretArn);+ }+ }+ }++ private void putManagedRotationVersion(Secret secret, String token, String previousVersionId,+ String secretString) {+ if (secret.getVersions() == null) {+ secret.setVersions(new HashMap<>());+ }+ for (SecretVersion version : secret.getVersions().values()) {+ if (version.getVersionStages().contains(AWSPREVIOUS)) {+ List<String> stages = new ArrayList<>(version.getVersionStages());+ stages.remove(AWSPREVIOUS);+ version.setVersionStages(stages);+ }+ }+ if (previousVersionId != null) {+ SecretVersion previous = secret.getVersions().get(previousVersionId);+ if (previous != null) {+ List<String> stages = new ArrayList<>(previous.getVersionStages());+ stages.remove(AWSCURRENT);+ if (!stages.contains(AWSPREVIOUS)) {+ stages.add(AWSPREVIOUS);+ }+ previous.setVersionStages(stages);+ }+ }+ SecretVersion rotated = new SecretVersion();+ rotated.setVersionId(token);+ rotated.setSecretString(secretString);+ rotated.setVersionStages(List.of(AWSCURRENT));+ rotated.setCreatedDate(Instant.now());+ secret.getVersions().put(token, rotated);+ secret.setCurrentVersionId(token);+ }+/*** The full ARN of the rotation function {@code reference} names, in the secret's partition and* account unless the reference names its own. A reference Lambda cannot read is refused withdiff --git a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.javaindex d50d7450d3..6d81f89ae8 100644--- a/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/redshift/RedshiftServiceTest.java@@ -777,6 +777,51 @@ class RedshiftServiceTest {eq("arn:aws:kms:us-east-1:111111111111:key/key-1"), anyList(), eq("redshift"), eq("us-east-1"));}+ @Test+ void managedMasterSecretRotationChangesTheClusterAndProxyPassword() throws Exception {+ Cluster cluster = new Cluster();+ cluster.setClusterIdentifier("managed-cluster");+ cluster.setMasterUsername("admin");+ cluster.setMasterPassword("OldPassword123");+ cluster.setMasterPasswordSecretArn("arn:aws:secretsmanager:us-east-1:111111111111:secret:redshift-managed");+ cluster.setClusterStatus("available");+ when(clusterBackend.scanAllAccountEntries(any())).thenReturn(List.of(+ new AccountAwareStorageBackend.AccountEntry<>("111111111111", "managed-cluster", cluster)));+ when(clusterBackend.getForAccount("111111111111", "managed-cluster")).thenReturn(Optional.of(cluster));++ AtomicReference<String> secretValue = new AtomicReference<>();+ assertTrue(service.supports(cluster.getMasterPasswordSecretArn(), "us-east-1"));+ service.rotate(cluster.getMasterPasswordSecretArn(), "us-east-1", (changeBackend, rollbackBackend) ->+ secretValue.set(changeBackend.get()));++ verify(cm).alterUserPassword(eq("111111111111"), eq("managed-cluster"), eq("admin"),+ argThat(password -> !"OldPassword123".equals(password)));+ verify(proxyManager).updateMasterPassword(eq("111111111111:managed-cluster"),+ argThat(password -> !"OldPassword123".equals(password)));+ assertNotNull(secretValue.get());+ String rotatedPassword = new ObjectMapper().readTree(secretValue.get()).path("password").asText();+ assertEquals(cluster.getMasterPassword(), rotatedPassword);+ assertFalse(rotatedPassword.chars().anyMatch(character -> "'\"/@\\".indexOf(character) >= 0));+ }++ @Test+ void managedRotationReservationBlocksClusterDeletionBeforeMutation() {+ Cluster cluster = new Cluster();+ cluster.setClusterIdentifier("managed-cluster");+ cluster.setMasterPasswordSecretArn("arn:aws:secretsmanager:us-east-1:111111111111:secret:redshift-managed");+ cluster.setClusterStatus("available");+ when(clusterBackend.scanAllAccountEntries(any())).thenReturn(List.of(+ new AccountAwareStorageBackend.AccountEntry<>("111111111111", "managed-cluster", cluster)));+ when(clusterBackend.getForAccount("111111111111", "managed-cluster")).thenReturn(Optional.of(cluster));++ service.reserve(cluster.getMasterPasswordSecretArn(), "us-east-1");++ assertThrows(AwsException.class, () -> service.deleteCluster("managed-cluster"));+ verify(cm, never()).stop("111111111111", "managed-cluster");++ service.release(cluster.getMasterPasswordSecretArn(), "us-east-1");+ }+@Testvoid managedMasterPasswordRollsBackClusterWhenSecretCreationFails() {when(cm.start(eq("111111111111"), eq("failed-managed-cluster"), eq("admin"), anyString()))diff --git a/src/test/java/io/github/hectorvent/floci/services/secretsmanager/ManagedSecretRotationServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/secretsmanager/ManagedSecretRotationServiceTest.javanew file mode 100644index 0000000000..c5219e65ba--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/secretsmanager/ManagedSecretRotationServiceTest.java@@ -0,0 +1,419 @@+package io.github.hectorvent.floci.services.secretsmanager;++import com.fasterxml.jackson.databind.ObjectMapper;+import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.InMemoryStorage;+import io.github.hectorvent.floci.services.secretsmanager.model.Secret;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;++import java.time.Instant;+import java.util.List;+import java.util.UUID;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicInteger;+import java.util.function.BooleanSupplier;+import java.util.function.Consumer;++import static org.junit.jupiter.api.Assertions.*;++class ManagedSecretRotationServiceTest {+ private static final String REGION = "us-east-1";++ @Test+ void rotatesOwnerAndPublishesRequestedVersionExactlyOnce() throws Exception {+ AtomicInteger changes = new AtomicInteger();+ CountDownLatch firstRelease = new CountDownLatch(1);+ ManagedSecretRotator owner = owner(rotation -> rotation.commit(() -> {+ changes.incrementAndGet();+ return "new-password";+ }, () -> fail("unexpected rollback")), firstRelease::countDown);+ SecretsManagerService service = service(owner);+ try {+ Secret original = create(service);+ String previous = original.getCurrentVersionId();+ String token = UUID.randomUUID().toString();+ service.rotateSecret(original.getArn(), token, null, rules(), true, REGION);+ await(() -> service.describeSecret(original.getArn(), REGION).getLastRotatedDate() != null);+ assertEquals(token, service.describeSecret(original.getArn(), REGION).getCurrentVersionId());+ assertEquals("new-password", service.getSecretValue(original.getArn(), null, null, REGION).getSecretString());+ assertEquals(previous, service.getSecretValue(original.getArn(), null, "AWSPREVIOUS", REGION).getVersionId());+ service.rotateSecret(original.getArn(), token, null, null, true, REGION);+ assertEquals(1, changes.get());+ assertTrue(firstRelease.await(5, TimeUnit.SECONDS));+ String secondToken = UUID.randomUUID().toString();+ service.rotateSecret(original.getArn(), secondToken, null, null, true, REGION);+ await(() -> secondToken.equals(service.describeSecret(original.getArn(), REGION).getCurrentVersionId()));+ assertEquals(token, service.getSecretValue(original.getArn(), null, "AWSPREVIOUS", REGION).getVersionId());+ service.rotateSecret(original.getArn(), token, null, null, true, REGION);+ assertEquals(secondToken, service.describeSecret(original.getArn(), REGION).getCurrentVersionId());+ assertEquals(2, changes.get());+ } finally {+ service.shutdown();+ }+ }++ @Test+ void sameTokenRetryDuringReservationDoesNotReserveOrRotateTwice() throws Exception {+ CountDownLatch reserving = new CountDownLatch(1);+ CountDownLatch finishReservation = new CountDownLatch(1);+ CountDownLatch released = new CountDownLatch(1);+ AtomicInteger reservations = new AtomicInteger();+ AtomicInteger changes = new AtomicInteger();+ ManagedSecretRotator owner = new ManagedSecretRotator() {+ public boolean supports(String arn, String region) {+ return true;+ }++ public void reserve(String arn, String region) {+ if (reservations.incrementAndGet() != 1) {+ throw new AwsException("InvalidRequestException", "Owner already reserved", 400);+ }+ blockReserve(reserving, finishReservation);+ }++ public void rotate(String arn, String region, Rotation rotation) {+ rotation.commit(() -> {+ changes.incrementAndGet();+ return "new-password";+ }, () -> fail("unexpected rollback"));+ }++ public void release(String arn, String region) {+ released.countDown();+ }+ };+ SecretsManagerService service = service(owner);+ try (ExecutorService executor = Executors.newSingleThreadExecutor()) {+ Secret original = create(service);+ String token = UUID.randomUUID().toString();+ Future<Secret> first = executor.submit(() -> service.rotateSecret(+ original.getArn(), token, null, rules(), true, REGION));+ try {+ assertTrue(reserving.await(5, TimeUnit.SECONDS));+ assertDoesNotThrow(() -> service.rotateSecret(original.getArn(), token, null, rules(), true, REGION));+ assertThrows(AwsException.class, () -> service.rotateSecret(original.getArn(),+ UUID.randomUUID().toString(), null, rules(), true, REGION));+ assertDoesNotThrow(() -> service.putSecretValue(+ original.getArn(), "updated-password", null, null, REGION, null));+ assertEquals("updated-password",+ service.getSecretValue(original.getArn(), null, null, REGION).getSecretString());+ } finally {+ finishReservation.countDown();+ }+ first.get(5, TimeUnit.SECONDS);+ assertTrue(released.await(5, TimeUnit.SECONDS));+ assertDoesNotThrow(() -> service.rotateSecret(original.getArn(), token, null, rules(), true, REGION));+ assertEquals(1, reservations.get());+ assertEquals(1, changes.get());+ assertEquals(token, service.describeSecret(original.getArn(), REGION).getCurrentVersionId());+ } finally {+ finishReservation.countDown();+ service.shutdown();+ }+ }++ @Test+ void ownerCanDeleteSecretWhileRotationWaitsForReservation() throws Exception {+ CountDownLatch reserving = new CountDownLatch(1);+ CountDownLatch finishReservation = new CountDownLatch(1);+ AtomicInteger releases = new AtomicInteger();+ SecretsManagerService service = service(new ManagedSecretRotator() {+ public boolean supports(String arn, String region) {+ return true;+ }++ public void reserve(String arn, String region) {+ blockReserve(reserving, finishReservation);+ throw new AwsException("ResourceNotFoundException", "Owner deleted", 400);+ }++ public void rotate(String arn, String region, Rotation rotation) {+ fail("deleted owner's secret must not rotate");+ }++ public void release(String arn, String region) {+ releases.incrementAndGet();+ }+ });+ try (ExecutorService executor = Executors.newSingleThreadExecutor()) {+ Secret original = create(service);+ Future<AwsException> rotation = executor.submit(() -> assertThrows(AwsException.class,+ () -> service.rotateSecret(original.getArn(), UUID.randomUUID().toString(),+ null, rules(), true, REGION)));+ try {+ assertTrue(reserving.await(5, TimeUnit.SECONDS));+ assertTrue(service.deleteManagedSecretAfter(original.getArn(), REGION, () -> { }));+ AwsException missing = assertThrows(AwsException.class,+ () -> service.describeSecret(original.getArn(), REGION));+ assertEquals("ResourceNotFoundException", missing.getErrorCode());+ } finally {+ finishReservation.countDown();+ }+ assertEquals("ResourceNotFoundException", rotation.get(5, TimeUnit.SECONDS).getErrorCode());+ assertEquals(0, releases.get());+ } finally {+ finishReservation.countDown();+ service.shutdown();+ }+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void versionSavedDuringReservationSurvivesWithoutRunningOwner(boolean ownerFails) throws Exception {+ CountDownLatch reserving = new CountDownLatch(1);+ CountDownLatch finishReservation = new CountDownLatch(1);+ CountDownLatch released = new CountDownLatch(1);+ AtomicInteger changes = new AtomicInteger();+ AtomicInteger reservations = new AtomicInteger();+ SecretsManagerService service = service(new ManagedSecretRotator() {+ public boolean supports(String arn, String region) {+ return true;+ }++ public void reserve(String arn, String region) {+ if (reservations.incrementAndGet() > 1) {+ throw new AwsException("InvalidRequestException", "Owner unavailable", 400);+ }+ blockReserve(reserving, finishReservation);+ }++ public void rotate(String arn, String region, Rotation rotation) {+ changes.incrementAndGet();+ if (ownerFails) {+ throw new IllegalStateException("backend refused password");+ }+ rotation.commit(() -> "rotated-password", () -> fail("unexpected rollback"));+ }++ public void release(String arn, String region) {+ released.countDown();+ }+ });+ try (ExecutorService executor = Executors.newSingleThreadExecutor()) {+ Secret original = create(service);+ String token = UUID.randomUUID().toString();+ Future<Secret> rotation = executor.submit(() -> service.rotateSecret(+ original.getArn(), token, null, rules(), true, REGION));+ try {+ assertTrue(reserving.await(5, TimeUnit.SECONDS));+ service.putSecretValue(original.getArn(), "saved-password", null, token, REGION, null);+ } finally {+ finishReservation.countDown();+ }+ Secret result = rotation.get(5, TimeUnit.SECONDS);+ assertTrue(released.await(5, TimeUnit.SECONDS));+ assertEquals("saved-password",+ service.getSecretValue(original.getArn(), token, null, REGION).getSecretString());+ assertEquals(token, result.getCurrentVersionId());+ assertEquals(0, changes.get());+ assertFalse(result.isRotationEnabled());+ assertNull(result.getLastRotatedDate());+ service.rotateSecret(original.getArn(), token, null, rules(), true, REGION);+ assertEquals(1, reservations.get());+ assertDoesNotThrow(() -> service.putSecretValue(+ original.getArn(), "updated-password", null, null, REGION, null));+ AwsException unavailable = assertThrows(AwsException.class, () -> service.rotateSecret(+ original.getArn(), UUID.randomUUID().toString(), null, rules(), true, REGION));+ assertEquals("Owner unavailable", unavailable.getMessage());+ assertEquals(2, reservations.get());+ } finally {+ finishReservation.countDown();+ service.shutdown();+ }+ }++ @Test+ void failedReservationAllowsCredentialUpdatesAndSameTokenRetry() throws Exception {+ AtomicInteger reservations = new AtomicInteger();+ AtomicInteger releases = new AtomicInteger();+ CountDownLatch released = new CountDownLatch(1);+ SecretsManagerService service = service(new ManagedSecretRotator() {+ public boolean supports(String arn, String region) {+ return true;+ }++ public void reserve(String arn, String region) {+ if (reservations.incrementAndGet() == 1) {+ throw new AwsException("InvalidRequestException", "Owner unavailable", 400);+ }+ }++ public void rotate(String arn, String region, Rotation rotation) {+ rotation.commit(() -> "new-password", () -> fail("unexpected rollback"));+ }++ public void release(String arn, String region) {+ releases.incrementAndGet();+ released.countDown();+ }+ });+ try {+ Secret original = create(service);+ String token = UUID.randomUUID().toString();+ assertThrows(AwsException.class, () -> service.rotateSecret(+ original.getArn(), token, null, rules(), true, REGION));+ assertEquals(0, releases.get());+ assertFalse(service.describeSecret(original.getArn(), REGION).isRotationEnabled());+ assertDoesNotThrow(() -> service.putSecretValue(original.getArn(), "updated-password", null, null,+ REGION, null));+ service.rotateSecret(original.getArn(), token, null, rules(), true, REGION);+ assertTrue(released.await(5, TimeUnit.SECONDS));+ assertEquals(2, reservations.get());+ assertEquals(1, releases.get());+ assertEquals(token, service.describeSecret(original.getArn(), REGION).getCurrentVersionId());+ } finally {+ service.shutdown();+ }+ }++ @Test+ void activeRotationBlocksMutationsAndCancellationSurvivesPublication() throws Exception {+ CountDownLatch changing = new CountDownLatch(1);+ CountDownLatch finish = new CountDownLatch(1);+ SecretsManagerService service = service(owner(rotation -> rotation.commit(() -> {+ changing.countDown();+ try {+ assertTrue(finish.await(5, TimeUnit.SECONDS));+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new IllegalStateException(e);+ }+ return "new-password";+ }, () -> { })));+ try {+ Secret original = create(service);+ String activeToken = UUID.randomUUID().toString();+ service.rotateSecret(original.getArn(), activeToken, null, rules(), true, REGION);+ assertTrue(changing.await(5, TimeUnit.SECONDS));+ assertThrows(AwsException.class, () -> service.rotateSecret(original.getArn(),+ UUID.randomUUID().toString(), null, rules(), true, REGION));+ assertThrows(AwsException.class, () -> service.putSecretValue(original.getArn(), "other", null, null, REGION, null));+ assertThrows(AwsException.class, () -> service.updateSecret(original.getArn(), "other", null, REGION));+ assertThrows(AwsException.class, () -> service.deleteSecret(original.getArn(), null, true, REGION));+ service.cancelRotateSecret(original.getArn(), REGION);+ finish.countDown();+ await(() -> service.describeSecret(original.getArn(), REGION).getLastRotatedDate() != null);+ assertFalse(service.describeSecret(original.getArn(), REGION).isRotationEnabled());+ assertNull(service.describeSecret(original.getArn(), REGION).getNextRotationDate());+ } finally {+ finish.countDown();+ service.shutdown();+ }+ }++ @Test+ void failedOwnerChangeRollsBackAndSameTokenCanRetry() throws Exception {+ AtomicInteger attempts = new AtomicInteger();+ AtomicInteger rollbacks = new AtomicInteger();+ CountDownLatch firstFailure = new CountDownLatch(1);+ CountDownLatch firstRelease = new CountDownLatch(1);+ SecretsManagerService service = service(owner(rotation -> {+ try {+ rotation.commit(() -> {+ if (attempts.incrementAndGet() == 1) {+ throw new IllegalStateException("backend refused password");+ }+ return "new-password";+ }, rollbacks::incrementAndGet);+ } catch (IllegalStateException e) {+ firstFailure.countDown();+ throw e;+ }+ }, firstRelease::countDown));+ try {+ Secret original = create(service);+ String token = UUID.randomUUID().toString();+ service.rotateSecret(original.getArn(), token, null, rules(), true, REGION);+ assertTrue(firstFailure.await(5, TimeUnit.SECONDS));+ assertTrue(firstRelease.await(5, TimeUnit.SECONDS));+ await(() -> rollbacks.get() == 1);+ assertTrue(service.describeSecret(original.getArn(), REGION).isRotationEnabled());+ assertEquals("old-password", service.getSecretValue(original.getArn(), null, null, REGION).getSecretString());+ assertNull(service.describeSecret(original.getArn(), REGION).getLastRotatedDate());+ assertEquals(1, rollbacks.get());+ service.rotateSecret(original.getArn(), token, null, null, true, REGION);+ await(() -> service.describeSecret(original.getArn(), REGION).getLastRotatedDate() != null);+ assertEquals(token, service.describeSecret(original.getArn(), REGION).getCurrentVersionId());+ } finally {+ service.shutdown();+ }+ }++ @Test+ void configureOnlyDoesNotChangePasswordAndSchedulerRotatesSupportedOwners() throws Exception {+ AtomicInteger changes = new AtomicInteger();+ SecretsManagerService service = service(owner(rotation -> rotation.commit(() -> {+ changes.incrementAndGet();+ return "new-password";+ }, () -> { })));+ try {+ Secret original = create(service);+ service.rotateSecret(original.getArn(), UUID.randomUUID().toString(), null, rules(), false, REGION);+ assertEquals(0, changes.get());+ assertNull(original.getLastRotatedDate());+ new SecretRotationScheduler(service).tick(Instant.now().plusSeconds(31L * 86400));+ await(() -> service.describeSecret(original.getArn(), REGION).getLastRotatedDate() != null);+ assertEquals(1, changes.get());+ } finally {+ service.shutdown();+ }+ }++ private static SecretsManagerService service(ManagedSecretRotator owner) {+ return new SecretsManagerService(new InMemoryStorage<>(), 30,+ new RegionResolver(REGION, "000000000000"), null, new ObjectMapper(), null, List.of(owner));+ }++ private static Secret create(SecretsManagerService service) {+ return service.createSecret("managed", "old-password", null, null, null, null, "redshift", REGION);+ }++ private static Secret.RotationRules rules() {+ return new Secret.RotationRules(30, null, null);+ }++ private static ManagedSecretRotator owner(Consumer<ManagedSecretRotator.Rotation> action) {+ return owner(action, () -> { });+ }++ private static ManagedSecretRotator owner(Consumer<ManagedSecretRotator.Rotation> action, Runnable releaseAction) {+ return new ManagedSecretRotator() {+ public boolean supports(String arn, String region) {+ return true;+ }++ public void rotate(String arn, String region, Rotation rotation) {+ action.accept(rotation);+ }++ public void release(String arn, String region) {+ releaseAction.run();+ }+ };+ }++ private static void blockReserve(CountDownLatch reserving, CountDownLatch finishReservation) {+ reserving.countDown();+ try {+ assertTrue(finishReservation.await(5, TimeUnit.SECONDS));+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new IllegalStateException(e);+ }+ }++ private static void await(BooleanSupplier condition) throws Exception {+ long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);+ while (!condition.getAsBoolean() && System.nanoTime() < deadline) {+ Thread.sleep(10);+ }+ assertTrue(condition.getAsBoolean(), "rotation did not finish");+ }+}