Light, fluffy, and always free - The AWS Local Emulator alternative
fix(rds): preserve credentials when managed-secret opt-out is refused (#5553)
David Phillips2927a4520e65
diff --git a/docs/services/rds.md b/docs/services/rds.mdindex 322f3cf81b..986238d70a 100644--- a/docs/services/rds.md+++ b/docs/services/rds.md@@ -581,6 +581,20 @@ Custom endpoints are modelled, not routed: a custom endpoint reports the clusterreachable, but connections through it go where the cluster endpoint sends them rather than beingspread across the chosen members.+## Aurora managed password opt-out++When `ModifyDBCluster` disables `ManageMasterUserPassword` while the managed secret has+cross-Region replicas, the request succeeds but retains the cluster password and its+`MasterUserSecret` metadata, as Aurora does. Remove the secret's replicas and retry to+release password management. Other requested cluster settings still apply.++Floci reserves the secret while changing backend credentials, without holding a shared+Secrets Manager lock during backend I/O. Replication and deletion requests for that secret+fail while release is in progress; unrelated secrets remain available.+A failed backend password rotation retains the managed secret and stored credentials.+If Secrets Manager support is unavailable, opt-out fails before changing cluster state.+This does not provide a transaction across database backends and persistent storage.+## Point in time restore`RestoreDBInstanceToPointInTime` and `RestoreDBClusterToPointInTime` check the request as AWS does:diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java b/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.javaindex 010e1ca5e7..f285ab802d 100644--- a/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java@@ -4483,20 +4483,33 @@ public class RdsService implements Resettable, ResourceProvider {String effectiveSecretKmsKeyId = Boolean.TRUE.equals(manageMasterUserPassword)&& cluster.getMasterUserSecretArn() == null? resolveManagedSecretKmsKeyId(masterUserSecretKmsKeyId, effectiveRegion) : null;- boolean passwordRotated = false;- if (newPassword != null && !newPassword.isBlank()) {- String oldPassword = cluster.getMasterPassword();- boolean backendRunning = !config.services().rds().mock()- && cluster.getContainerId() != null;- passwordRotated = backendRunning && !newPassword.equals(oldPassword);- // Propagate the rotation into the running backend DB before overwriting the stored- // password — the last moment the old credential (which the backend still holds) is- // known — mirroring the ModifyDBInstance path.- if (passwordRotated) {+ boolean hasNewPassword = newPassword != null && !newPassword.isBlank();+ String oldPassword = cluster.getMasterPassword();+ boolean rotateBackend = hasNewPassword && !config.services().rds().mock()+ && cluster.getContainerId() != null && !newPassword.equals(oldPassword);+ Runnable changeCredentials = () -> {+ // Keep the old credential and secret until the running backend accepts the change.+ if (rotateBackend) {containerManager.rotateMasterPassword(cluster.getDockerVolumeName(), cluster.getContainerId(),cluster.getEngine(), cluster.getMasterUsername(), oldPassword, newPassword);}+ };+ boolean releasesManagedSecret = Boolean.FALSE.equals(manageMasterUserPassword)+ && cluster.getMasterUserSecretArn() != null;+ boolean credentialsChanged = true;+ if (releasesManagedSecret) {+ if (secretsManagerService == null) {+ throw new AwsException("InvalidParameterCombination",+ "ManageMasterUserPassword requires Secrets Manager support.", 400);+ }+ credentialsChanged = secretsManagerService.deleteManagedSecretAfter(+ cluster.getMasterUserSecretArn(), effectiveRegion, changeCredentials);+ } else {+ changeCredentials.run();+ }+ boolean passwordRotated = credentialsChanged && rotateBackend;+ if (credentialsChanged && hasNewPassword) {cluster.setMasterPassword(newPassword);}if (iamEnabled != null) {@@ -4507,8 +4520,12 @@ public class RdsService implements Resettable, ResourceProvider {cluster.setMasterPassword(generatedMasterPassword());}attachManagedMasterUserSecret(cluster, effectiveRegion, masterUserSecretKmsKeyId, effectiveSecretKmsKeyId);- } else if (Boolean.FALSE.equals(manageMasterUserPassword) && cluster.getMasterUserSecretArn() != null) {- detachManagedMasterUserSecret(cluster, effectiveRegion);+ } else if (releasesManagedSecret) {+ if (credentialsChanged) {+ cluster.setMasterUserSecretArn(null);+ cluster.setMasterUserSecretStatus(null);+ cluster.setMasterUserSecretKmsKeyId(null);+ }} else if (cluster.getMasterUserSecretArn() != null&& masterUserSecretKmsKeyId != null&& !masterUserSecretKmsKeyId.equals(cluster.getMasterUserSecretKmsKeyId())) {diff --git a/src/main/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerService.java b/src/main/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerService.javaindex fd561528ce..2aa168a04c 100644--- a/src/main/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerService.java+++ b/src/main/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerService.java@@ -39,6 +39,7 @@ import java.util.Map;import java.util.Objects;import java.util.Set;import java.util.UUID;+import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;import java.util.concurrent.ThreadFactory;@@ -77,6 +78,8 @@ public class SecretsManagerService implements ResourceProvider {// stripe only contend; no lock is held while a rotation Lambda runs.private static final int ROTATION_LOCK_STRIPE_COUNT = 32;private final Object[] rotationLockStripes = newLockStripes(ROTATION_LOCK_STRIPE_COUNT);+ // Entries exist only while a service owner changes backend credentials outside the stripe.+ private final Set<String> managedReleases = ConcurrentHashMap.newKeySet();private static Object[] newLockStripes(int count) {Object[] stripes = new Object[count];@@ -658,6 +661,55 @@ public class SecretsManagerService implements ResourceProvider {public record Filter(String key, List<String> values) {}+ /**+ * Reserves release under the secret monitor, then changes backend credentials without holding+ * a shared stripe. Replication and deletion refuse a reserved secret until the action finishes.+ * Returns false without running the action when replicas prevent deletion.+ */+ public boolean deleteManagedSecretAfter(String secretArn, String region, Runnable changeCredentials) {+ boolean secretExists = true;+ synchronized (lockFor(secretArn)) {+ throwIfManagedRelease(secretArn);+ Secret secret = null;+ try {+ secret = resolveSecret(secretArn, region);+ } catch (AwsException e) {+ if (!"ResourceNotFoundException".equals(e.getErrorCode())) {+ throw e;+ }+ // An already removed secret does not prevent its owner from releasing management.+ secretExists = false;+ }+ if (secret != null) {+ throwIfReplica(secret);+ if (secret.getReplicationStatus() != null && !secret.getReplicationStatus().isEmpty()) {+ return false;+ }+ }+ managedReleases.add(secretArn);+ }+ boolean credentialsChanged = false;+ try {+ changeCredentials.run();+ credentialsChanged = true;+ return true;+ } finally {+ synchronized (lockFor(secretArn)) {+ managedReleases.remove(secretArn);+ if (credentialsChanged && secretExists) {+ deleteSecret(secretArn, null, true, region);+ }+ }+ }+ }++ private void throwIfManagedRelease(String secretArn) {+ if (managedReleases.contains(secretArn)) {+ throw new AwsException("InvalidRequestException",+ "The secret's managed credentials are being released. Try again after the release completes.", 400);+ }+ }+public Secret deleteSecret(String secretId, Integer recoveryWindowInDays, boolean forceDelete, String region) {// Parameters are checked before the secret is resolved, the way AWS orders it: an invalid// window is reported as such even when the secret does not exist.@@ -676,6 +728,7 @@ public class SecretsManagerService implements ResourceProvider {Secret resolved = resolveSecret(secretId, region);synchronized (lockFor(resolved.getArn())) {Secret secret = resolveSecret(resolved.getArn(), region);+ throwIfManagedRelease(secret.getArn());throwIfReplica(secret);// AWS: "You can't delete a primary secret that is replicated to other Regions. You must// first delete the replicas." Deleting it here would strand every replica with no primary@@ -868,11 +921,12 @@ public class SecretsManagerService implements ResourceProvider {"AddReplicaRegions must contain at least one Region.", 400);}- Secret primary = resolveSecret(secretId, region);- throwIfPendingDeletion(primary);- throwIfReplica(primary);-- synchronized (lockFor(primary.getArn())) {+ Secret resolved = resolveSecret(secretId, region);+ synchronized (lockFor(resolved.getArn())) {+ Secret primary = resolveSecret(resolved.getArn(), region);+ throwIfManagedRelease(primary.getArn());+ throwIfPendingDeletion(primary);+ throwIfReplica(primary);List<Secret.ReplicaStatus> statuses = primary.getReplicationStatus() != null? new ArrayList<>(primary.getReplicationStatus()) : new ArrayList<>();@@ -967,10 +1021,11 @@ public class SecretsManagerService implements ResourceProvider {"RemoveReplicaRegions must contain at least one Region.", 400);}- Secret primary = resolveSecret(secretId, region);- throwIfReplica(primary);-- synchronized (lockFor(primary.getArn())) {+ Secret resolved = resolveSecret(secretId, region);+ synchronized (lockFor(resolved.getArn())) {+ Secret primary = resolveSecret(resolved.getArn(), region);+ throwIfManagedRelease(primary.getArn());+ throwIfReplica(primary);List<Secret.ReplicaStatus> statuses = primary.getReplicationStatus() != null? new ArrayList<>(primary.getReplicationStatus()) : new ArrayList<>();@@ -1013,6 +1068,18 @@ public class SecretsManagerService implements ResourceProvider {// halves of the promotion run inside one critical section for the same reason: a sync// landing between them would write the replica back over the copy just promoted.synchronized (lockFor(primary != null ? primary.getArn() : replica.getArn())) {+ // A removal or promotion may have completed while this call waited. Never write the+ // pre-lock objects back over a deletion or over a newly independent secret.+ replica = resolveSecret(replica.getArn(), region);+ if (!replica.isReplica() || !primaryRegionName.equals(replica.getPrimaryRegion())) {+ throw new AwsException("InvalidRequestException",+ "The secret is no longer a replica of the original primary.", 400);+ }+ if (primary != null) {+ primary = resolveSecret(primary.getArn(), primaryRegionName);+ throwIfManagedRelease(primary.getArn());+ }+ throwIfManagedRelease(replica.getArn());replica.setPrimaryRegion(null);replica.setLastChangedDate(Instant.now());putSecret(region, replica);diff --git a/src/test/java/io/github/hectorvent/floci/services/rds/RdsReplicatedSecretOptOutIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/rds/RdsReplicatedSecretOptOutIntegrationTest.javanew file mode 100644index 0000000000..192269a25f--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/rds/RdsReplicatedSecretOptOutIntegrationTest.java@@ -0,0 +1,107 @@+package io.github.hectorvent.floci.services.rds;++import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RequestScopes;+import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;+import io.github.hectorvent.floci.testing.RdsMockProfile;+import io.quarkus.test.junit.QuarkusTest;+import io.quarkus.test.junit.TestProfile;+import io.restassured.RestAssured;+import jakarta.inject.Inject;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.CsvSource;+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;+import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;+import software.amazon.awssdk.regions.Region;+import software.amazon.awssdk.services.rds.RdsClient;+import software.amazon.awssdk.services.rds.model.DBCluster;++import java.net.URI;+import java.util.List;+import java.util.UUID;++import static org.junit.jupiter.api.Assertions.assertAll;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertThrows;++@QuarkusTest+@TestProfile(RdsMockProfile.class)+class RdsReplicatedSecretOptOutIntegrationTest {+ @Inject RdsService rdsService;+ @Inject SecretsManagerService secrets;++ @ParameterizedTest+ @CsvSource({"aurora-postgresql,16.4", "aurora-mysql,8.0"})+ void replicatedSecretOptOutReturnsManagedCredentials(String engine, String version) {+ String account = "000000000000";+ String region = "us-east-1";+ String id = "optout-" + UUID.randomUUID().toString().substring(0, 8);+ try (RdsClient rds = RdsClient.builder()+ .endpointOverride(URI.create("http://localhost:" + RestAssured.port))+ .httpClientBuilder(UrlConnectionHttpClient.builder()).region(Region.US_EAST_1)+ .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(account, "test")))+ .build()) {+ DBCluster created = rds.createDBCluster(r -> r.dbClusterIdentifier(id).engine(engine)+ .engineVersion(version).masterUsername("admin").manageMasterUserPassword(true)).dbCluster();+ String arn = created.masterUserSecret().secretArn();+ try {+ String password = RequestScopes.callAs(account, region,+ () -> rdsService.getDbCluster(id, region).getMasterPassword());+ RequestScopes.runAs(account, region, () -> secrets.replicateSecretToRegions(arn,+ List.of(new SecretsManagerService.ReplicaRegion("us-west-2", null)), false, region));+ String secretValue = RequestScopes.callAs(account, region,+ () -> secrets.getSecretValue(arn, null, null, region).getSecretString());++ DBCluster modified = rds.modifyDBCluster(r -> r.dbClusterIdentifier(id)+ .manageMasterUserPassword(false).masterUserPassword("replacement-password")).dbCluster();+ DBCluster described = rds.describeDBClusters(r -> r.dbClusterIdentifier(id)).dbClusters().get(0);+ assertAll(+ () -> assertEquals(created.masterUserSecret(), modified.masterUserSecret()),+ () -> assertEquals(created.masterUserSecret(), described.masterUserSecret()),+ () -> assertEquals(password, RequestScopes.callAs(account, region,+ () -> rdsService.getDbCluster(id, region).getMasterPassword())),+ () -> assertEquals(secretValue, RequestScopes.callAs(account, region,+ () -> secrets.getSecretValue(arn, null, null, region).getSecretString())));+ RequestScopes.runAs(account, region,+ () -> secrets.removeRegionsFromReplication(arn, List.of("us-west-2"), region));+ DBCluster released = rds.modifyDBCluster(r -> r.dbClusterIdentifier(id)+ .manageMasterUserPassword(false).masterUserPassword("replacement-password")).dbCluster();+ assertNull(released.masterUserSecret());+ assertNull(rds.describeDBClusters(r -> r.dbClusterIdentifier(id)).dbClusters().get(0).masterUserSecret());+ assertEquals("replacement-password", RequestScopes.callAs(account, region,+ () -> rdsService.getDbCluster(id, region).getMasterPassword()));+ AwsException missing = assertThrows(AwsException.class, () -> RequestScopes.callAs(account, region,+ () -> secrets.describeSecret(arn, region)));+ assertEquals("ResourceNotFoundException", missing.getErrorCode());+ } finally {+ try {+ RequestScopes.runAs(account, region, () -> {+ try {+ secrets.removeRegionsFromReplication(arn, List.of("us-west-2"), region);+ } catch (AwsException e) {+ if (!"ResourceNotFoundException".equals(e.getErrorCode())) {+ throw e;+ }+ }+ });+ } finally {+ try {+ rds.deleteDBCluster(r -> r.dbClusterIdentifier(id).skipFinalSnapshot(true));+ } finally {+ RequestScopes.runAs(account, region, () -> {+ try {+ secrets.deleteSecret(arn, null, true, region);+ } catch (AwsException e) {+ if (!"ResourceNotFoundException".equals(e.getErrorCode())) {+ throw e;+ }+ }+ });+ }+ }+ }+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.javaindex 411cd87228..82e6203c87 100644--- a/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java@@ -1438,6 +1438,177 @@ class RdsServiceTest {verify(secretsManager).deleteSecret(eq(secretArn), any(), anyBoolean(), eq("us-east-1"));}+ @ParameterizedTest+ @CsvSource({"aurora-postgresql,16.3,true", "aurora-mysql,8.0,true",+ "aurora-postgresql,16.3,false", "aurora-mysql,8.0,false"})+ void replicatedClusterOptOutPreservesCredentials(String engine, String version, boolean mockMode) {+ when(rdsConfig.mock()).thenReturn(mockMode);+ SecretsManagerService secrets = realSecretsForOptOut();+ RdsService service = newService(containerManager, proxyManager,+ new InMemoryStorage<>(), new InMemoryStorage<>(),+ new InMemoryStorage<>(), new InMemoryStorage<>(), secrets);+ DbCluster cluster = service.createDbCluster("optout", engine, version,+ "admin", null, "dbname", false, null, null, null, false, "us-east-1",+ null, null, null, true, null);+ String oldPassword = cluster.getMasterPassword();+ String arn = cluster.getMasterUserSecretArn();+ String key = cluster.getMasterUserSecretKmsKeyId();+ DbInstance member = service.createDbInstance("optout-member", engine, version,+ "admin", oldPassword, "dbname", "db.r5.large", 20, false, null, null, "optout");+ secrets.replicateSecretToRegions(arn,+ List.of(new SecretsManagerService.ReplicaRegion("us-west-2", null)), false, "us-east-1");+ String secretValue = secrets.getSecretValue(arn, null, null, "us-east-1").getSecretString();+ clearInvocations(containerManager, proxyManager);++ DbCluster modified = service.modifyDbCluster("optout", "replacement-password", true,+ null, null, null, false, null, "us-east-1");++ assertAll(+ () -> assertEquals(oldPassword, modified.getMasterPassword()),+ () -> assertTrue(modified.isIamDatabaseAuthenticationEnabled()),+ () -> assertEquals(oldPassword, member.getMasterPassword()),+ () -> assertEquals(arn, modified.getMasterUserSecretArn()),+ () -> assertEquals("active", modified.getMasterUserSecretStatus()),+ () -> assertEquals(key, modified.getMasterUserSecretKmsKeyId()),+ () -> assertEquals(secretValue, secrets.getSecretValue(arn, null, null, "us-east-1").getSecretString()),+ () -> verify(containerManager, never()).rotateMasterPassword(any(), any(), any(), any(), any(), any()),+ () -> verify(proxyManager, never()).updateMasterPassword(any(), any()));+ }++ @ParameterizedTest+ @CsvSource({"aurora-postgresql,16.3", "aurora-mysql,8.0"})+ void clusterOptOutRotationFailureRetainsManagedSecret(String engine, String version) {+ SecretsManagerService secrets = realSecretsForOptOut();+ RdsService service = newService(containerManager, proxyManager,+ new InMemoryStorage<>(), new InMemoryStorage<>(),+ new InMemoryStorage<>(), new InMemoryStorage<>(), secrets);+ DbCluster cluster = service.createDbCluster("optout-fail", engine, version,+ "admin", null, "dbname", false, null, null, null, false, "us-east-1",+ null, null, null, true, null);+ String oldPassword = cluster.getMasterPassword();+ String arn = cluster.getMasterUserSecretArn();+ String key = cluster.getMasterUserSecretKmsKeyId();+ String secretValue = secrets.getSecretValue(arn, null, null, "us-east-1").getSecretString();+ doThrow(new IllegalStateException("backend rotation failed"))+ .when(containerManager).rotateMasterPassword(any(), any(), any(), any(), any(), any());+ clearInvocations(proxyManager);++ assertThrows(IllegalStateException.class, () -> service.modifyDbCluster("optout-fail",+ "replacement-password", null, null, null, null, false, null, "us-east-1"));++ assertAll(+ () -> assertEquals(oldPassword, cluster.getMasterPassword()),+ () -> assertEquals(arn, cluster.getMasterUserSecretArn()),+ () -> assertEquals("active", cluster.getMasterUserSecretStatus()),+ () -> assertEquals(key, cluster.getMasterUserSecretKmsKeyId()),+ () -> assertEquals(secretValue, secrets.getSecretValue(arn, null, null, "us-east-1").getSecretString()),+ () -> verify(proxyManager, never()).updateMasterPassword(any(), any()));+ }++ @ParameterizedTest+ @CsvSource({"aurora-postgresql,16.3,true", "aurora-mysql,8.0,true",+ "aurora-postgresql,16.3,false", "aurora-mysql,8.0,false"})+ void clusterOptOutRetryDeletesSecretAfterReplicationRemoved(String engine, String version, boolean mockMode) {+ when(rdsConfig.mock()).thenReturn(mockMode);+ SecretsManagerService secrets = realSecretsForOptOut();+ RdsService service = newService(containerManager, proxyManager,+ new InMemoryStorage<>(), new InMemoryStorage<>(),+ new InMemoryStorage<>(), new InMemoryStorage<>(), secrets);+ DbCluster cluster = service.createDbCluster("optout-retry", engine, version,+ "admin", null, "dbname", false, null, null, null, false, "us-east-1",+ null, null, null, true, null);+ String arn = cluster.getMasterUserSecretArn();+ String oldPassword = cluster.getMasterPassword();+ DbInstance member = service.createDbInstance("retry-member", engine, version,+ "admin", oldPassword, "dbname", "db.r5.large", 20, false, null, null, "optout-retry");+ secrets.replicateSecretToRegions(arn,+ List.of(new SecretsManagerService.ReplicaRegion("us-west-2", null)), false, "us-east-1");+ service.modifyDbCluster("optout-retry", "replacement-password", null,+ null, null, null, false, null, "us-east-1");+ secrets.removeRegionsFromReplication(arn, List.of("us-west-2"), "us-east-1");++ DbCluster modified = service.modifyDbCluster("optout-retry", "replacement-password", null,+ null, null, null, false, null, "us-east-1");++ assertAll(+ () -> assertNull(modified.getMasterUserSecretArn()),+ () -> assertNull(modified.getMasterUserSecretStatus()),+ () -> assertNull(modified.getMasterUserSecretKmsKeyId()),+ () -> assertEquals("replacement-password", modified.getMasterPassword()),+ () -> assertEquals("ResourceNotFoundException", assertThrows(AwsException.class,+ () -> secrets.getSecretValue(arn, null, null, "us-east-1")).getErrorCode()));+ if (!mockMode) {+ assertEquals("replacement-password", member.getMasterPassword());+ verify(containerManager).rotateMasterPassword(cluster.getDockerVolumeName(), cluster.getContainerId(),+ cluster.getEngine(), cluster.getMasterUsername(), oldPassword, "replacement-password");+ verify(proxyManager).updateMasterPassword("rds-resource:" + cluster.getDbClusterArn(), "replacement-password");+ verify(proxyManager).updateMasterPassword("rds-resource:" + member.getDbInstanceArn(), "replacement-password");+ }+ }++ @ParameterizedTest+ @CsvSource({"aurora-postgresql,16.3,true", "aurora-mysql,8.0,true",+ "aurora-postgresql,16.3,false", "aurora-mysql,8.0,false"})+ void clusterOptOutWithoutSecretsManagerFailsBeforeChangingState(String engine, String version, boolean mockMode) {+ when(rdsConfig.mock()).thenReturn(mockMode);+ DbCluster cluster = rdsService.createDbCluster("missing-secrets", engine, version,+ "admin", "original-password", "dbname", false, null, null, null, false, "us-east-1");+ String arn = "arn:aws:secretsmanager:us-east-1:123456789012:secret:rds!missing-secrets";+ cluster.setMasterUserSecretArn(arn);+ cluster.setMasterUserSecretStatus("active");+ cluster.setMasterUserSecretKmsKeyId("managed-key");+ DbInstance member = rdsService.createDbInstance("missing-secrets-member", engine, version,+ "admin", "original-password", "dbname", "db.r5.large", 20, false, null, null, "missing-secrets");+ clearInvocations(containerManager, proxyManager);++ AwsException error = assertThrows(AwsException.class, () -> rdsService.modifyDbCluster(+ "missing-secrets", "replacement-password", true, 0.5, 8.0, null,+ false, null, "us-east-1"));++ assertAll(+ () -> assertEquals("InvalidParameterCombination", error.getErrorCode()),+ () -> assertEquals(400, error.getHttpStatus()),+ () -> assertEquals("original-password", cluster.getMasterPassword()),+ () -> assertEquals("original-password", member.getMasterPassword()),+ () -> assertEquals(arn, cluster.getMasterUserSecretArn()),+ () -> assertEquals("active", cluster.getMasterUserSecretStatus()),+ () -> assertEquals("managed-key", cluster.getMasterUserSecretKmsKeyId()),+ () -> assertFalse(cluster.isIamDatabaseAuthenticationEnabled()),+ () -> assertNull(cluster.getServerlessV2MinCapacity()),+ () -> assertNull(cluster.getServerlessV2MaxCapacity()),+ () -> assertSame(cluster, rdsService.getDbCluster("missing-secrets", "us-east-1")),+ () -> verifyNoInteractions(containerManager, proxyManager));+ }++ @ParameterizedTest+ @CsvSource({"aurora-postgresql,16.3,true", "aurora-mysql,8.0,true",+ "aurora-postgresql,16.3,false", "aurora-mysql,8.0,false"})+ void unmanagedClusterPasswordChangeDoesNotRequireSecretsManager(String engine, String version, boolean mockMode) {+ when(rdsConfig.mock()).thenReturn(mockMode);+ DbCluster cluster = rdsService.createDbCluster("unmanaged-optout", engine, version,+ "admin", "original-password", "dbname", false, null, null, null, false, "us-east-1");+ clearInvocations(containerManager, proxyManager);++ rdsService.modifyDbCluster("unmanaged-optout", "replacement-password", true,+ null, null, null, false, null, "us-east-1");++ assertEquals("replacement-password", cluster.getMasterPassword());+ assertTrue(cluster.isIamDatabaseAuthenticationEnabled());+ assertNull(cluster.getMasterUserSecretArn());+ verify(containerManager, times(mockMode ? 0 : 1))+ .rotateMasterPassword(any(), any(), any(), any(), eq("original-password"), eq("replacement-password"));+ }++ private SecretsManagerService realSecretsForOptOut() {+ StorageFactory factory = mock(StorageFactory.class);+ when(factory.<Secret>create(eq("secretsmanager"), any(), any()))+ .thenReturn(AccountAwareStorageBackend.inMemory("123456789012"));+ EmulatorConfig.SecretsManagerServiceConfig secretsConfig = mock(EmulatorConfig.SecretsManagerServiceConfig.class);+ when(config.services().secretsmanager()).thenReturn(secretsConfig);+ when(secretsConfig.defaultRecoveryWindowDays()).thenReturn(30);+ return new SecretsManagerService(factory, config, regionResolver, null, new ObjectMapper(), null);+ }+@Testvoid modifyDbClusterRekeysExistingManagedSecret() {when(config.services().rds().mock()).thenReturn(true);diff --git a/src/test/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerWriteLockTest.java b/src/test/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerWriteLockTest.javaindex 022068e45b..34631ca373 100644--- a/src/test/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerWriteLockTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerWriteLockTest.java@@ -10,14 +10,19 @@ import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;import io.github.hectorvent.floci.services.secretsmanager.model.Secret;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;+import java.util.HashMap;import java.util.List;+import java.util.Map;import java.util.concurrent.CountDownLatch;import java.util.concurrent.TimeUnit;import java.util.concurrent.atomic.AtomicReference;import java.util.function.BiPredicate;import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;import static org.junit.jupiter.api.Assertions.assertNull;import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;@@ -47,6 +52,238 @@ class SecretsManagerWriteLockTest {service.createSecret(SECRET_NAME, "v1", null, "original", null, null, REGION);}+ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void backendReleaseDoesNotBlockAnotherSecretOnTheSameStripe(boolean failBackend) throws Exception {+ Map<Integer, String> stripes = new HashMap<>();+ String primary = null;+ String unrelated = null;+ for (int index = 0; index <= 32; index++) {+ String arn = service.createSecret("collision-" + index, "v1", null, "original", null, null, REGION).getArn();+ String previous = stripes.putIfAbsent(Math.floorMod(arn.hashCode(), 32), arn);+ if (previous != null) {+ primary = previous;+ unrelated = arn;+ break;+ }+ }+ assertTrue(primary != null && unrelated != null);+ String releasingArn = primary;+ String otherArn = unrelated;+ CountDownLatch entered = new CountDownLatch(1);+ CountDownLatch release = new CountDownLatch(1);+ CountDownLatch unrelatedUpdated = new CountDownLatch(1);+ Watched deletion = start("managed-release", () -> {+ Runnable credentials = () -> {+ entered.countDown();+ try {+ assertTrue(release.await(10, TimeUnit.SECONDS));+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new AssertionError(e);+ }+ if (failBackend) {+ throw new IllegalStateException("backend failed");+ }+ };+ if (failBackend) {+ assertThrows(IllegalStateException.class,+ () -> service.deleteManagedSecretAfter(releasingArn, REGION, credentials));+ } else {+ assertTrue(service.deleteManagedSecretAfter(releasingArn, REGION, credentials));+ }+ });+ Watched other = null;+ try {+ assertTrue(entered.await(5, TimeUnit.SECONDS));+ other = start("unrelated-update", () -> {+ service.updateSecret(otherArn, "independent", null, REGION);+ unrelatedUpdated.countDown();+ });+ assertTrue(unrelatedUpdated.await(2, TimeUnit.SECONDS), "Backend I/O blocked an unrelated secret");+ } finally {+ release.countDown();+ deletion.awaitSuccess();+ if (other != null) {+ other.awaitSuccess();+ }+ }+ assertEquals("independent", service.describeSecret(otherArn, REGION).getDescription());+ if (failBackend) {+ assertEquals("v1", service.getSecretValue(releasingArn, null, null, REGION).getSecretString());+ service.replicateSecretToRegions(releasingArn,+ List.of(new SecretsManagerService.ReplicaRegion(REPLICA_REGION, null)), false, REGION);+ assertEquals(1, service.describeSecret(releasingArn, REGION).getReplicationStatus().size());+ } else {+ assertThrows(AwsException.class, () -> service.describeSecret(releasingArn, REGION));+ }+ }++ @Test+ void managedDeletionWaitsForReplicationAndPreservesCredentials() throws Exception {+ String arn = service.describeSecret(SECRET_NAME, REGION).getArn();+ storage.parkOn(REPLICA_REGION + "::" + SECRET_NAME);+ Watched replication = start("replication", () -> service.replicateSecretToRegions(arn,+ List.of(new SecretsManagerService.ReplicaRegion(REPLICA_REGION, null)), false, REGION));+ AtomicReference<Boolean> changed = new AtomicReference<>(false);+ AtomicReference<Boolean> deleted = new AtomicReference<>();+ Watched deletion = null;+ try {+ assertTrue(storage.awaitEntered());+ deletion = start("managed-deletion", () -> deleted.set(+ service.deleteManagedSecretAfter(arn, REGION, () -> changed.set(true))));+ assertTrue(deletion.awaitBlocked());+ } finally {+ storage.release();+ replication.awaitSuccess();+ if (deletion != null) {+ deletion.awaitSuccess();+ }+ }+ assertFalse(deleted.get());+ assertFalse(changed.get());+ assertEquals(1, service.describeSecret(arn, REGION).getReplicationStatus().size());+ assertEquals("v1", service.getSecretValue(SECRET_NAME, null, null, REPLICA_REGION).getSecretString());+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void replicationChangesCannotResurrectAReleasedManagedSecret(boolean removeReplica) throws Exception {+ String arn = service.describeSecret(SECRET_NAME, REGION).getArn();+ if (removeReplica) {+ service.replicateSecretToRegions(arn,+ List.of(new SecretsManagerService.ReplicaRegion(REPLICA_REGION, null)), false, REGION);+ service.removeRegionsFromReplication(arn, List.of(REPLICA_REGION), REGION);+ }+ CountDownLatch changing = new CountDownLatch(1);+ CountDownLatch release = new CountDownLatch(1);+ Watched deletion = start("managed-deletion", () -> assertTrue(+ service.deleteManagedSecretAfter(arn, REGION, () -> {+ changing.countDown();+ try {+ assertTrue(release.await(5, TimeUnit.SECONDS));+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new AssertionError(e);+ }+ })));+ Watched replication = null;+ try {+ assertTrue(changing.await(5, TimeUnit.SECONDS));+ replication = start("replication", () -> {+ AwsException missing = assertThrows(AwsException.class, () -> {+ if (removeReplica) {+ service.removeRegionsFromReplication(arn, List.of(REPLICA_REGION), REGION);+ } else {+ service.replicateSecretToRegions(arn,+ List.of(new SecretsManagerService.ReplicaRegion(REPLICA_REGION, null)), false, REGION);+ }+ });+ assertEquals("InvalidRequestException", missing.getErrorCode());+ });+ replication.awaitSuccess();+ } finally {+ release.countDown();+ deletion.awaitSuccess();+ if (replication != null) {+ replication.awaitSuccess();+ }+ }+ assertThrows(AwsException.class, () -> service.describeSecret(arn, REGION));+ assertThrows(AwsException.class, () -> service.describeSecret(SECRET_NAME, REPLICA_REGION));+ }++ @Test+ void queuedPromotionCannotRestoreSecretsAfterManagedRelease() throws Exception {+ String arn = service.describeSecret(SECRET_NAME, REGION).getArn();+ service.replicateSecretToRegions(arn,+ List.of(new SecretsManagerService.ReplicaRegion(REPLICA_REGION, null)), false, REGION);+ storage.parkOn(REGION + "::" + SECRET_NAME);+ storage.afterParkedWrite = () -> {+ service.removeRegionsFromReplication(arn, List.of(REPLICA_REGION), REGION);+ assertTrue(service.deleteManagedSecretAfter(arn, REGION, () -> { }));+ };+ Watched writer = start("primary-writer", () -> service.updateSecret(arn, "updated", null, REGION));+ Watched promotion = null;+ try {+ assertTrue(storage.awaitEntered());+ promotion = start("queued-promotion", () -> {+ AwsException missing = assertThrows(AwsException.class,+ () -> service.stopReplicationToReplica(SECRET_NAME, REPLICA_REGION));+ assertEquals("ResourceNotFoundException", missing.getErrorCode());+ });+ assertTrue(promotion.awaitBlocked());+ } finally {+ storage.release();+ writer.awaitSuccess();+ if (promotion != null) {+ promotion.awaitSuccess();+ }+ }+ assertThrows(AwsException.class, () -> service.describeSecret(arn, REGION));+ assertThrows(AwsException.class, () -> service.describeSecret(SECRET_NAME, REPLICA_REGION));+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void activeReleaseRejectsDeletionAndDuplicateReleaseUntilBackendFailure(boolean forceDelete) throws Exception {+ String arn = service.describeSecret(SECRET_NAME, REGION).getArn();+ CountDownLatch entered = new CountDownLatch(1);+ CountDownLatch release = new CountDownLatch(1);+ Watched owner = start("failing-release", () -> assertThrows(IllegalStateException.class,+ () -> service.deleteManagedSecretAfter(arn, REGION, () -> {+ entered.countDown();+ try {+ assertTrue(release.await(10, TimeUnit.SECONDS));+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new AssertionError(e);+ }+ throw new IllegalStateException("backend failed");+ })));+ Watched concurrent = null;+ try {+ assertTrue(entered.await(5, TimeUnit.SECONDS));+ concurrent = start("concurrent-release", () -> {+ AwsException deletion = assertThrows(AwsException.class,+ () -> service.deleteSecret(arn, null, forceDelete, REGION));+ assertEquals("InvalidRequestException", deletion.getErrorCode());+ AwsException duplicate = assertThrows(AwsException.class,+ () -> service.deleteManagedSecretAfter(arn, REGION,+ () -> { throw new AssertionError("Duplicate backend call"); }));+ assertEquals("InvalidRequestException", duplicate.getErrorCode());+ });+ concurrent.awaitSuccess();+ } finally {+ release.countDown();+ owner.awaitSuccess();+ if (concurrent != null) {+ concurrent.awaitSuccess();+ }+ }+ assertEquals("v1", service.getSecretValue(arn, null, null, REGION).getSecretString());+ assertTrue(service.deleteManagedSecretAfter(arn, REGION, () -> { }));+ assertThrows(AwsException.class, () -> service.describeSecret(arn, REGION));+ }++ @Test+ void failedCredentialChangeRetainsTheSecret() {+ String arn = service.describeSecret(SECRET_NAME, REGION).getArn();+ assertThrows(IllegalStateException.class, () -> service.deleteManagedSecretAfter(arn, REGION, () -> {+ throw new IllegalStateException("backend refused rotation");+ }));+ assertEquals("v1", service.getSecretValue(arn, null, null, REGION).getSecretString());+ }++ @Test+ void alreadyRemovedManagedSecretStillAllowsCredentialChange() {+ String arn = service.describeSecret(SECRET_NAME, REGION).getArn();+ service.deleteSecret(arn, null, true, REGION);+ AtomicReference<Boolean> changed = new AtomicReference<>(false);+ assertTrue(service.deleteManagedSecretAfter(arn, REGION, () -> changed.set(true)));+ assertTrue(changed.get());+ }+@Testvoid updateSecretWaitsForAnInFlightUpdate() throws Exception {storage.parkOn(REGION + "::" + SECRET_NAME);@@ -155,6 +392,7 @@ class SecretsManagerWriteLockTest {private final CountDownLatch entered = new CountDownLatch(1);private final CountDownLatch released = new CountDownLatch(1);private volatile BiPredicate<String, Secret> parkWhen;+ private Runnable afterParkedWrite;void parkOn(String key) {parkOn((written, secret) -> written.equals(key));@@ -185,6 +423,11 @@ class SecretsManagerWriteLockTest {}}super.put(key, value);+ Runnable afterWrite = afterParkedWrite;+ if (afterWrite != null) {+ afterParkedWrite = null;+ afterWrite.run();+ }}}