Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(rds): preserve credentials when managed-secret opt-out is refused (#5553)

David Phillips2927a4520e65
diff --git a/docs/services/rds.md b/docs/services/rds.md
index 322f3cf81b..986238d70a 100644
--- a/docs/services/rds.md
+++ b/docs/services/rds.md
@@ -581,6 +581,20 @@ Custom endpoints are modelled, not routed: a custom endpoint reports the cluster
reachable, but connections through it go where the cluster endpoint sends them rather than being
spread across the chosen members.
+## Aurora managed password opt-out
+
+When `ModifyDBCluster` disables `ManageMasterUserPassword` while the managed secret has
+cross-Region replicas, the request succeeds but retains the cluster password and its
+`MasterUserSecret` metadata, as Aurora does. Remove the secret's replicas and retry to
+release password management. Other requested cluster settings still apply.
+
+Floci reserves the secret while changing backend credentials, without holding a shared
+Secrets Manager lock during backend I/O. Replication and deletion requests for that secret
+fail while release is in progress; unrelated secrets remain available.
+A failed backend password rotation retains the managed secret and stored credentials.
+If Secrets Manager support is unavailable, opt-out fails before changing cluster state.
+This does not provide a transaction across database backends and persistent storage.
+
## Point in time restore
`RestoreDBInstanceToPointInTime` and `RestoreDBClusterToPointInTime` check the request as AWS does:
diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java b/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java
index 010e1ca5e7..f285ab802d 100644
--- a/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java
@@ -4483,20 +4483,33 @@ public class RdsService implements Resettable, ResourceProvider {
String effectiveSecretKmsKeyId = Boolean.TRUE.equals(manageMasterUserPassword)
&& cluster.getMasterUserSecretArn() == null
? resolveManagedSecretKmsKeyId(masterUserSecretKmsKeyId, effectiveRegion) : null;
- boolean passwordRotated = false;
- if (newPassword != null && !newPassword.isBlank()) {
- String oldPassword = cluster.getMasterPassword();
- boolean backendRunning = !config.services().rds().mock()
- && cluster.getContainerId() != null;
- passwordRotated = backendRunning && !newPassword.equals(oldPassword);
- // Propagate the rotation into the running backend DB before overwriting the stored
- // password — the last moment the old credential (which the backend still holds) is
- // known — mirroring the ModifyDBInstance path.
- if (passwordRotated) {
+ boolean hasNewPassword = newPassword != null && !newPassword.isBlank();
+ String oldPassword = cluster.getMasterPassword();
+ boolean rotateBackend = hasNewPassword && !config.services().rds().mock()
+ && cluster.getContainerId() != null && !newPassword.equals(oldPassword);
+ Runnable changeCredentials = () -> {
+ // Keep the old credential and secret until the running backend accepts the change.
+ if (rotateBackend) {
containerManager.rotateMasterPassword(
cluster.getDockerVolumeName(), cluster.getContainerId(),
cluster.getEngine(), cluster.getMasterUsername(), oldPassword, newPassword);
}
+ };
+ boolean releasesManagedSecret = Boolean.FALSE.equals(manageMasterUserPassword)
+ && cluster.getMasterUserSecretArn() != null;
+ boolean credentialsChanged = true;
+ if (releasesManagedSecret) {
+ if (secretsManagerService == null) {
+ throw new AwsException("InvalidParameterCombination",
+ "ManageMasterUserPassword requires Secrets Manager support.", 400);
+ }
+ credentialsChanged = secretsManagerService.deleteManagedSecretAfter(
+ cluster.getMasterUserSecretArn(), effectiveRegion, changeCredentials);
+ } else {
+ changeCredentials.run();
+ }
+ boolean passwordRotated = credentialsChanged && rotateBackend;
+ if (credentialsChanged && hasNewPassword) {
cluster.setMasterPassword(newPassword);
}
if (iamEnabled != null) {
@@ -4507,8 +4520,12 @@ public class RdsService implements Resettable, ResourceProvider {
cluster.setMasterPassword(generatedMasterPassword());
}
attachManagedMasterUserSecret(cluster, effectiveRegion, masterUserSecretKmsKeyId, effectiveSecretKmsKeyId);
- } else if (Boolean.FALSE.equals(manageMasterUserPassword) && cluster.getMasterUserSecretArn() != null) {
- detachManagedMasterUserSecret(cluster, effectiveRegion);
+ } else if (releasesManagedSecret) {
+ if (credentialsChanged) {
+ cluster.setMasterUserSecretArn(null);
+ cluster.setMasterUserSecretStatus(null);
+ cluster.setMasterUserSecretKmsKeyId(null);
+ }
} else if (cluster.getMasterUserSecretArn() != null
&& masterUserSecretKmsKeyId != null
&& !masterUserSecretKmsKeyId.equals(cluster.getMasterUserSecretKmsKeyId())) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerService.java b/src/main/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerService.java
index fd561528ce..2aa168a04c 100644
--- a/src/main/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerService.java
@@ -39,6 +39,7 @@ import java.util.Map;
import java.util.Objects;
import java.util.Set;
import java.util.UUID;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ThreadFactory;
@@ -77,6 +78,8 @@ public class SecretsManagerService implements ResourceProvider {
// stripe only contend; no lock is held while a rotation Lambda runs.
private static final int ROTATION_LOCK_STRIPE_COUNT = 32;
private final Object[] rotationLockStripes = newLockStripes(ROTATION_LOCK_STRIPE_COUNT);
+ // Entries exist only while a service owner changes backend credentials outside the stripe.
+ private final Set<String> managedReleases = ConcurrentHashMap.newKeySet();
private static Object[] newLockStripes(int count) {
Object[] stripes = new Object[count];
@@ -658,6 +661,55 @@ public class SecretsManagerService implements ResourceProvider {
public record Filter(String key, List<String> values) {}
+ /**
+ * Reserves release under the secret monitor, then changes backend credentials without holding
+ * a shared stripe. Replication and deletion refuse a reserved secret until the action finishes.
+ * Returns false without running the action when replicas prevent deletion.
+ */
+ public boolean deleteManagedSecretAfter(String secretArn, String region, Runnable changeCredentials) {
+ boolean secretExists = true;
+ synchronized (lockFor(secretArn)) {
+ throwIfManagedRelease(secretArn);
+ Secret secret = null;
+ try {
+ secret = resolveSecret(secretArn, region);
+ } catch (AwsException e) {
+ if (!"ResourceNotFoundException".equals(e.getErrorCode())) {
+ throw e;
+ }
+ // An already removed secret does not prevent its owner from releasing management.
+ secretExists = false;
+ }
+ if (secret != null) {
+ throwIfReplica(secret);
+ if (secret.getReplicationStatus() != null && !secret.getReplicationStatus().isEmpty()) {
+ return false;
+ }
+ }
+ managedReleases.add(secretArn);
+ }
+ boolean credentialsChanged = false;
+ try {
+ changeCredentials.run();
+ credentialsChanged = true;
+ return true;
+ } finally {
+ synchronized (lockFor(secretArn)) {
+ managedReleases.remove(secretArn);
+ if (credentialsChanged && secretExists) {
+ deleteSecret(secretArn, null, true, region);
+ }
+ }
+ }
+ }
+
+ private void throwIfManagedRelease(String secretArn) {
+ if (managedReleases.contains(secretArn)) {
+ throw new AwsException("InvalidRequestException",
+ "The secret's managed credentials are being released. Try again after the release completes.", 400);
+ }
+ }
+
public Secret deleteSecret(String secretId, Integer recoveryWindowInDays, boolean forceDelete, String region) {
// Parameters are checked before the secret is resolved, the way AWS orders it: an invalid
// window is reported as such even when the secret does not exist.
@@ -676,6 +728,7 @@ public class SecretsManagerService implements ResourceProvider {
Secret resolved = resolveSecret(secretId, region);
synchronized (lockFor(resolved.getArn())) {
Secret secret = resolveSecret(resolved.getArn(), region);
+ throwIfManagedRelease(secret.getArn());
throwIfReplica(secret);
// AWS: "You can't delete a primary secret that is replicated to other Regions. You must
// first delete the replicas." Deleting it here would strand every replica with no primary
@@ -868,11 +921,12 @@ public class SecretsManagerService implements ResourceProvider {
"AddReplicaRegions must contain at least one Region.", 400);
}
- Secret primary = resolveSecret(secretId, region);
- throwIfPendingDeletion(primary);
- throwIfReplica(primary);
-
- synchronized (lockFor(primary.getArn())) {
+ Secret resolved = resolveSecret(secretId, region);
+ synchronized (lockFor(resolved.getArn())) {
+ Secret primary = resolveSecret(resolved.getArn(), region);
+ throwIfManagedRelease(primary.getArn());
+ throwIfPendingDeletion(primary);
+ throwIfReplica(primary);
List<Secret.ReplicaStatus> statuses = primary.getReplicationStatus() != null
? new ArrayList<>(primary.getReplicationStatus()) : new ArrayList<>();
@@ -967,10 +1021,11 @@ public class SecretsManagerService implements ResourceProvider {
"RemoveReplicaRegions must contain at least one Region.", 400);
}
- Secret primary = resolveSecret(secretId, region);
- throwIfReplica(primary);
-
- synchronized (lockFor(primary.getArn())) {
+ Secret resolved = resolveSecret(secretId, region);
+ synchronized (lockFor(resolved.getArn())) {
+ Secret primary = resolveSecret(resolved.getArn(), region);
+ throwIfManagedRelease(primary.getArn());
+ throwIfReplica(primary);
List<Secret.ReplicaStatus> statuses = primary.getReplicationStatus() != null
? new ArrayList<>(primary.getReplicationStatus()) : new ArrayList<>();
@@ -1013,6 +1068,18 @@ public class SecretsManagerService implements ResourceProvider {
// halves of the promotion run inside one critical section for the same reason: a sync
// landing between them would write the replica back over the copy just promoted.
synchronized (lockFor(primary != null ? primary.getArn() : replica.getArn())) {
+ // A removal or promotion may have completed while this call waited. Never write the
+ // pre-lock objects back over a deletion or over a newly independent secret.
+ replica = resolveSecret(replica.getArn(), region);
+ if (!replica.isReplica() || !primaryRegionName.equals(replica.getPrimaryRegion())) {
+ throw new AwsException("InvalidRequestException",
+ "The secret is no longer a replica of the original primary.", 400);
+ }
+ if (primary != null) {
+ primary = resolveSecret(primary.getArn(), primaryRegionName);
+ throwIfManagedRelease(primary.getArn());
+ }
+ throwIfManagedRelease(replica.getArn());
replica.setPrimaryRegion(null);
replica.setLastChangedDate(Instant.now());
putSecret(region, replica);
diff --git a/src/test/java/io/github/hectorvent/floci/services/rds/RdsReplicatedSecretOptOutIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/rds/RdsReplicatedSecretOptOutIntegrationTest.java
new file mode 100644
index 0000000000..192269a25f
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/rds/RdsReplicatedSecretOptOutIntegrationTest.java
@@ -0,0 +1,107 @@
+package io.github.hectorvent.floci.services.rds;
+
+import io.github.hectorvent.floci.core.common.AwsException;
+import io.github.hectorvent.floci.core.common.RequestScopes;
+import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;
+import io.github.hectorvent.floci.testing.RdsMockProfile;
+import io.quarkus.test.junit.QuarkusTest;
+import io.quarkus.test.junit.TestProfile;
+import io.restassured.RestAssured;
+import jakarta.inject.Inject;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.http.urlconnection.UrlConnectionHttpClient;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.rds.RdsClient;
+import software.amazon.awssdk.services.rds.model.DBCluster;
+
+import java.net.URI;
+import java.util.List;
+import java.util.UUID;
+
+import static org.junit.jupiter.api.Assertions.assertAll;
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+
+@QuarkusTest
+@TestProfile(RdsMockProfile.class)
+class RdsReplicatedSecretOptOutIntegrationTest {
+ @Inject RdsService rdsService;
+ @Inject SecretsManagerService secrets;
+
+ @ParameterizedTest
+ @CsvSource({"aurora-postgresql,16.4", "aurora-mysql,8.0"})
+ void replicatedSecretOptOutReturnsManagedCredentials(String engine, String version) {
+ String account = "000000000000";
+ String region = "us-east-1";
+ String id = "optout-" + UUID.randomUUID().toString().substring(0, 8);
+ try (RdsClient rds = RdsClient.builder()
+ .endpointOverride(URI.create("http://localhost:" + RestAssured.port))
+ .httpClientBuilder(UrlConnectionHttpClient.builder()).region(Region.US_EAST_1)
+ .credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(account, "test")))
+ .build()) {
+ DBCluster created = rds.createDBCluster(r -> r.dbClusterIdentifier(id).engine(engine)
+ .engineVersion(version).masterUsername("admin").manageMasterUserPassword(true)).dbCluster();
+ String arn = created.masterUserSecret().secretArn();
+ try {
+ String password = RequestScopes.callAs(account, region,
+ () -> rdsService.getDbCluster(id, region).getMasterPassword());
+ RequestScopes.runAs(account, region, () -> secrets.replicateSecretToRegions(arn,
+ List.of(new SecretsManagerService.ReplicaRegion("us-west-2", null)), false, region));
+ String secretValue = RequestScopes.callAs(account, region,
+ () -> secrets.getSecretValue(arn, null, null, region).getSecretString());
+
+ DBCluster modified = rds.modifyDBCluster(r -> r.dbClusterIdentifier(id)
+ .manageMasterUserPassword(false).masterUserPassword("replacement-password")).dbCluster();
+ DBCluster described = rds.describeDBClusters(r -> r.dbClusterIdentifier(id)).dbClusters().get(0);
+ assertAll(
+ () -> assertEquals(created.masterUserSecret(), modified.masterUserSecret()),
+ () -> assertEquals(created.masterUserSecret(), described.masterUserSecret()),
+ () -> assertEquals(password, RequestScopes.callAs(account, region,
+ () -> rdsService.getDbCluster(id, region).getMasterPassword())),
+ () -> assertEquals(secretValue, RequestScopes.callAs(account, region,
+ () -> secrets.getSecretValue(arn, null, null, region).getSecretString())));
+ RequestScopes.runAs(account, region,
+ () -> secrets.removeRegionsFromReplication(arn, List.of("us-west-2"), region));
+ DBCluster released = rds.modifyDBCluster(r -> r.dbClusterIdentifier(id)
+ .manageMasterUserPassword(false).masterUserPassword("replacement-password")).dbCluster();
+ assertNull(released.masterUserSecret());
+ assertNull(rds.describeDBClusters(r -> r.dbClusterIdentifier(id)).dbClusters().get(0).masterUserSecret());
+ assertEquals("replacement-password", RequestScopes.callAs(account, region,
+ () -> rdsService.getDbCluster(id, region).getMasterPassword()));
+ AwsException missing = assertThrows(AwsException.class, () -> RequestScopes.callAs(account, region,
+ () -> secrets.describeSecret(arn, region)));
+ assertEquals("ResourceNotFoundException", missing.getErrorCode());
+ } finally {
+ try {
+ RequestScopes.runAs(account, region, () -> {
+ try {
+ secrets.removeRegionsFromReplication(arn, List.of("us-west-2"), region);
+ } catch (AwsException e) {
+ if (!"ResourceNotFoundException".equals(e.getErrorCode())) {
+ throw e;
+ }
+ }
+ });
+ } finally {
+ try {
+ rds.deleteDBCluster(r -> r.dbClusterIdentifier(id).skipFinalSnapshot(true));
+ } finally {
+ RequestScopes.runAs(account, region, () -> {
+ try {
+ secrets.deleteSecret(arn, null, true, region);
+ } catch (AwsException e) {
+ if (!"ResourceNotFoundException".equals(e.getErrorCode())) {
+ throw e;
+ }
+ }
+ });
+ }
+ }
+ }
+ }
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java
index 411cd87228..82e6203c87 100644
--- a/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java
@@ -1438,6 +1438,177 @@ class RdsServiceTest {
verify(secretsManager).deleteSecret(eq(secretArn), any(), anyBoolean(), eq("us-east-1"));
}
+ @ParameterizedTest
+ @CsvSource({"aurora-postgresql,16.3,true", "aurora-mysql,8.0,true",
+ "aurora-postgresql,16.3,false", "aurora-mysql,8.0,false"})
+ void replicatedClusterOptOutPreservesCredentials(String engine, String version, boolean mockMode) {
+ when(rdsConfig.mock()).thenReturn(mockMode);
+ SecretsManagerService secrets = realSecretsForOptOut();
+ RdsService service = newService(containerManager, proxyManager,
+ new InMemoryStorage<>(), new InMemoryStorage<>(),
+ new InMemoryStorage<>(), new InMemoryStorage<>(), secrets);
+ DbCluster cluster = service.createDbCluster("optout", engine, version,
+ "admin", null, "dbname", false, null, null, null, false, "us-east-1",
+ null, null, null, true, null);
+ String oldPassword = cluster.getMasterPassword();
+ String arn = cluster.getMasterUserSecretArn();
+ String key = cluster.getMasterUserSecretKmsKeyId();
+ DbInstance member = service.createDbInstance("optout-member", engine, version,
+ "admin", oldPassword, "dbname", "db.r5.large", 20, false, null, null, "optout");
+ secrets.replicateSecretToRegions(arn,
+ List.of(new SecretsManagerService.ReplicaRegion("us-west-2", null)), false, "us-east-1");
+ String secretValue = secrets.getSecretValue(arn, null, null, "us-east-1").getSecretString();
+ clearInvocations(containerManager, proxyManager);
+
+ DbCluster modified = service.modifyDbCluster("optout", "replacement-password", true,
+ null, null, null, false, null, "us-east-1");
+
+ assertAll(
+ () -> assertEquals(oldPassword, modified.getMasterPassword()),
+ () -> assertTrue(modified.isIamDatabaseAuthenticationEnabled()),
+ () -> assertEquals(oldPassword, member.getMasterPassword()),
+ () -> assertEquals(arn, modified.getMasterUserSecretArn()),
+ () -> assertEquals("active", modified.getMasterUserSecretStatus()),
+ () -> assertEquals(key, modified.getMasterUserSecretKmsKeyId()),
+ () -> assertEquals(secretValue, secrets.getSecretValue(arn, null, null, "us-east-1").getSecretString()),
+ () -> verify(containerManager, never()).rotateMasterPassword(any(), any(), any(), any(), any(), any()),
+ () -> verify(proxyManager, never()).updateMasterPassword(any(), any()));
+ }
+
+ @ParameterizedTest
+ @CsvSource({"aurora-postgresql,16.3", "aurora-mysql,8.0"})
+ void clusterOptOutRotationFailureRetainsManagedSecret(String engine, String version) {
+ SecretsManagerService secrets = realSecretsForOptOut();
+ RdsService service = newService(containerManager, proxyManager,
+ new InMemoryStorage<>(), new InMemoryStorage<>(),
+ new InMemoryStorage<>(), new InMemoryStorage<>(), secrets);
+ DbCluster cluster = service.createDbCluster("optout-fail", engine, version,
+ "admin", null, "dbname", false, null, null, null, false, "us-east-1",
+ null, null, null, true, null);
+ String oldPassword = cluster.getMasterPassword();
+ String arn = cluster.getMasterUserSecretArn();
+ String key = cluster.getMasterUserSecretKmsKeyId();
+ String secretValue = secrets.getSecretValue(arn, null, null, "us-east-1").getSecretString();
+ doThrow(new IllegalStateException("backend rotation failed"))
+ .when(containerManager).rotateMasterPassword(any(), any(), any(), any(), any(), any());
+ clearInvocations(proxyManager);
+
+ assertThrows(IllegalStateException.class, () -> service.modifyDbCluster("optout-fail",
+ "replacement-password", null, null, null, null, false, null, "us-east-1"));
+
+ assertAll(
+ () -> assertEquals(oldPassword, cluster.getMasterPassword()),
+ () -> assertEquals(arn, cluster.getMasterUserSecretArn()),
+ () -> assertEquals("active", cluster.getMasterUserSecretStatus()),
+ () -> assertEquals(key, cluster.getMasterUserSecretKmsKeyId()),
+ () -> assertEquals(secretValue, secrets.getSecretValue(arn, null, null, "us-east-1").getSecretString()),
+ () -> verify(proxyManager, never()).updateMasterPassword(any(), any()));
+ }
+
+ @ParameterizedTest
+ @CsvSource({"aurora-postgresql,16.3,true", "aurora-mysql,8.0,true",
+ "aurora-postgresql,16.3,false", "aurora-mysql,8.0,false"})
+ void clusterOptOutRetryDeletesSecretAfterReplicationRemoved(String engine, String version, boolean mockMode) {
+ when(rdsConfig.mock()).thenReturn(mockMode);
+ SecretsManagerService secrets = realSecretsForOptOut();
+ RdsService service = newService(containerManager, proxyManager,
+ new InMemoryStorage<>(), new InMemoryStorage<>(),
+ new InMemoryStorage<>(), new InMemoryStorage<>(), secrets);
+ DbCluster cluster = service.createDbCluster("optout-retry", engine, version,
+ "admin", null, "dbname", false, null, null, null, false, "us-east-1",
+ null, null, null, true, null);
+ String arn = cluster.getMasterUserSecretArn();
+ String oldPassword = cluster.getMasterPassword();
+ DbInstance member = service.createDbInstance("retry-member", engine, version,
+ "admin", oldPassword, "dbname", "db.r5.large", 20, false, null, null, "optout-retry");
+ secrets.replicateSecretToRegions(arn,
+ List.of(new SecretsManagerService.ReplicaRegion("us-west-2", null)), false, "us-east-1");
+ service.modifyDbCluster("optout-retry", "replacement-password", null,
+ null, null, null, false, null, "us-east-1");
+ secrets.removeRegionsFromReplication(arn, List.of("us-west-2"), "us-east-1");
+
+ DbCluster modified = service.modifyDbCluster("optout-retry", "replacement-password", null,
+ null, null, null, false, null, "us-east-1");
+
+ assertAll(
+ () -> assertNull(modified.getMasterUserSecretArn()),
+ () -> assertNull(modified.getMasterUserSecretStatus()),
+ () -> assertNull(modified.getMasterUserSecretKmsKeyId()),
+ () -> assertEquals("replacement-password", modified.getMasterPassword()),
+ () -> assertEquals("ResourceNotFoundException", assertThrows(AwsException.class,
+ () -> secrets.getSecretValue(arn, null, null, "us-east-1")).getErrorCode()));
+ if (!mockMode) {
+ assertEquals("replacement-password", member.getMasterPassword());
+ verify(containerManager).rotateMasterPassword(cluster.getDockerVolumeName(), cluster.getContainerId(),
+ cluster.getEngine(), cluster.getMasterUsername(), oldPassword, "replacement-password");
+ verify(proxyManager).updateMasterPassword("rds-resource:" + cluster.getDbClusterArn(), "replacement-password");
+ verify(proxyManager).updateMasterPassword("rds-resource:" + member.getDbInstanceArn(), "replacement-password");
+ }
+ }
+
+ @ParameterizedTest
+ @CsvSource({"aurora-postgresql,16.3,true", "aurora-mysql,8.0,true",
+ "aurora-postgresql,16.3,false", "aurora-mysql,8.0,false"})
+ void clusterOptOutWithoutSecretsManagerFailsBeforeChangingState(String engine, String version, boolean mockMode) {
+ when(rdsConfig.mock()).thenReturn(mockMode);
+ DbCluster cluster = rdsService.createDbCluster("missing-secrets", engine, version,
+ "admin", "original-password", "dbname", false, null, null, null, false, "us-east-1");
+ String arn = "arn:aws:secretsmanager:us-east-1:123456789012:secret:rds!missing-secrets";
+ cluster.setMasterUserSecretArn(arn);
+ cluster.setMasterUserSecretStatus("active");
+ cluster.setMasterUserSecretKmsKeyId("managed-key");
+ DbInstance member = rdsService.createDbInstance("missing-secrets-member", engine, version,
+ "admin", "original-password", "dbname", "db.r5.large", 20, false, null, null, "missing-secrets");
+ clearInvocations(containerManager, proxyManager);
+
+ AwsException error = assertThrows(AwsException.class, () -> rdsService.modifyDbCluster(
+ "missing-secrets", "replacement-password", true, 0.5, 8.0, null,
+ false, null, "us-east-1"));
+
+ assertAll(
+ () -> assertEquals("InvalidParameterCombination", error.getErrorCode()),
+ () -> assertEquals(400, error.getHttpStatus()),
+ () -> assertEquals("original-password", cluster.getMasterPassword()),
+ () -> assertEquals("original-password", member.getMasterPassword()),
+ () -> assertEquals(arn, cluster.getMasterUserSecretArn()),
+ () -> assertEquals("active", cluster.getMasterUserSecretStatus()),
+ () -> assertEquals("managed-key", cluster.getMasterUserSecretKmsKeyId()),
+ () -> assertFalse(cluster.isIamDatabaseAuthenticationEnabled()),
+ () -> assertNull(cluster.getServerlessV2MinCapacity()),
+ () -> assertNull(cluster.getServerlessV2MaxCapacity()),
+ () -> assertSame(cluster, rdsService.getDbCluster("missing-secrets", "us-east-1")),
+ () -> verifyNoInteractions(containerManager, proxyManager));
+ }
+
+ @ParameterizedTest
+ @CsvSource({"aurora-postgresql,16.3,true", "aurora-mysql,8.0,true",
+ "aurora-postgresql,16.3,false", "aurora-mysql,8.0,false"})
+ void unmanagedClusterPasswordChangeDoesNotRequireSecretsManager(String engine, String version, boolean mockMode) {
+ when(rdsConfig.mock()).thenReturn(mockMode);
+ DbCluster cluster = rdsService.createDbCluster("unmanaged-optout", engine, version,
+ "admin", "original-password", "dbname", false, null, null, null, false, "us-east-1");
+ clearInvocations(containerManager, proxyManager);
+
+ rdsService.modifyDbCluster("unmanaged-optout", "replacement-password", true,
+ null, null, null, false, null, "us-east-1");
+
+ assertEquals("replacement-password", cluster.getMasterPassword());
+ assertTrue(cluster.isIamDatabaseAuthenticationEnabled());
+ assertNull(cluster.getMasterUserSecretArn());
+ verify(containerManager, times(mockMode ? 0 : 1))
+ .rotateMasterPassword(any(), any(), any(), any(), eq("original-password"), eq("replacement-password"));
+ }
+
+ private SecretsManagerService realSecretsForOptOut() {
+ StorageFactory factory = mock(StorageFactory.class);
+ when(factory.<Secret>create(eq("secretsmanager"), any(), any()))
+ .thenReturn(AccountAwareStorageBackend.inMemory("123456789012"));
+ EmulatorConfig.SecretsManagerServiceConfig secretsConfig = mock(EmulatorConfig.SecretsManagerServiceConfig.class);
+ when(config.services().secretsmanager()).thenReturn(secretsConfig);
+ when(secretsConfig.defaultRecoveryWindowDays()).thenReturn(30);
+ return new SecretsManagerService(factory, config, regionResolver, null, new ObjectMapper(), null);
+ }
+
@Test
void modifyDbClusterRekeysExistingManagedSecret() {
when(config.services().rds().mock()).thenReturn(true);
diff --git a/src/test/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerWriteLockTest.java b/src/test/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerWriteLockTest.java
index 022068e45b..34631ca373 100644
--- a/src/test/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerWriteLockTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/secretsmanager/SecretsManagerWriteLockTest.java
@@ -10,14 +10,19 @@ import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;
import io.github.hectorvent.floci.services.secretsmanager.model.Secret;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
import java.util.function.BiPredicate;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -47,6 +52,238 @@ class SecretsManagerWriteLockTest {
service.createSecret(SECRET_NAME, "v1", null, "original", null, null, REGION);
}
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void backendReleaseDoesNotBlockAnotherSecretOnTheSameStripe(boolean failBackend) throws Exception {
+ Map<Integer, String> stripes = new HashMap<>();
+ String primary = null;
+ String unrelated = null;
+ for (int index = 0; index <= 32; index++) {
+ String arn = service.createSecret("collision-" + index, "v1", null, "original", null, null, REGION).getArn();
+ String previous = stripes.putIfAbsent(Math.floorMod(arn.hashCode(), 32), arn);
+ if (previous != null) {
+ primary = previous;
+ unrelated = arn;
+ break;
+ }
+ }
+ assertTrue(primary != null && unrelated != null);
+ String releasingArn = primary;
+ String otherArn = unrelated;
+ CountDownLatch entered = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ CountDownLatch unrelatedUpdated = new CountDownLatch(1);
+ Watched deletion = start("managed-release", () -> {
+ Runnable credentials = () -> {
+ entered.countDown();
+ try {
+ assertTrue(release.await(10, TimeUnit.SECONDS));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError(e);
+ }
+ if (failBackend) {
+ throw new IllegalStateException("backend failed");
+ }
+ };
+ if (failBackend) {
+ assertThrows(IllegalStateException.class,
+ () -> service.deleteManagedSecretAfter(releasingArn, REGION, credentials));
+ } else {
+ assertTrue(service.deleteManagedSecretAfter(releasingArn, REGION, credentials));
+ }
+ });
+ Watched other = null;
+ try {
+ assertTrue(entered.await(5, TimeUnit.SECONDS));
+ other = start("unrelated-update", () -> {
+ service.updateSecret(otherArn, "independent", null, REGION);
+ unrelatedUpdated.countDown();
+ });
+ assertTrue(unrelatedUpdated.await(2, TimeUnit.SECONDS), "Backend I/O blocked an unrelated secret");
+ } finally {
+ release.countDown();
+ deletion.awaitSuccess();
+ if (other != null) {
+ other.awaitSuccess();
+ }
+ }
+ assertEquals("independent", service.describeSecret(otherArn, REGION).getDescription());
+ if (failBackend) {
+ assertEquals("v1", service.getSecretValue(releasingArn, null, null, REGION).getSecretString());
+ service.replicateSecretToRegions(releasingArn,
+ List.of(new SecretsManagerService.ReplicaRegion(REPLICA_REGION, null)), false, REGION);
+ assertEquals(1, service.describeSecret(releasingArn, REGION).getReplicationStatus().size());
+ } else {
+ assertThrows(AwsException.class, () -> service.describeSecret(releasingArn, REGION));
+ }
+ }
+
+ @Test
+ void managedDeletionWaitsForReplicationAndPreservesCredentials() throws Exception {
+ String arn = service.describeSecret(SECRET_NAME, REGION).getArn();
+ storage.parkOn(REPLICA_REGION + "::" + SECRET_NAME);
+ Watched replication = start("replication", () -> service.replicateSecretToRegions(arn,
+ List.of(new SecretsManagerService.ReplicaRegion(REPLICA_REGION, null)), false, REGION));
+ AtomicReference<Boolean> changed = new AtomicReference<>(false);
+ AtomicReference<Boolean> deleted = new AtomicReference<>();
+ Watched deletion = null;
+ try {
+ assertTrue(storage.awaitEntered());
+ deletion = start("managed-deletion", () -> deleted.set(
+ service.deleteManagedSecretAfter(arn, REGION, () -> changed.set(true))));
+ assertTrue(deletion.awaitBlocked());
+ } finally {
+ storage.release();
+ replication.awaitSuccess();
+ if (deletion != null) {
+ deletion.awaitSuccess();
+ }
+ }
+ assertFalse(deleted.get());
+ assertFalse(changed.get());
+ assertEquals(1, service.describeSecret(arn, REGION).getReplicationStatus().size());
+ assertEquals("v1", service.getSecretValue(SECRET_NAME, null, null, REPLICA_REGION).getSecretString());
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void replicationChangesCannotResurrectAReleasedManagedSecret(boolean removeReplica) throws Exception {
+ String arn = service.describeSecret(SECRET_NAME, REGION).getArn();
+ if (removeReplica) {
+ service.replicateSecretToRegions(arn,
+ List.of(new SecretsManagerService.ReplicaRegion(REPLICA_REGION, null)), false, REGION);
+ service.removeRegionsFromReplication(arn, List.of(REPLICA_REGION), REGION);
+ }
+ CountDownLatch changing = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ Watched deletion = start("managed-deletion", () -> assertTrue(
+ service.deleteManagedSecretAfter(arn, REGION, () -> {
+ changing.countDown();
+ try {
+ assertTrue(release.await(5, TimeUnit.SECONDS));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError(e);
+ }
+ })));
+ Watched replication = null;
+ try {
+ assertTrue(changing.await(5, TimeUnit.SECONDS));
+ replication = start("replication", () -> {
+ AwsException missing = assertThrows(AwsException.class, () -> {
+ if (removeReplica) {
+ service.removeRegionsFromReplication(arn, List.of(REPLICA_REGION), REGION);
+ } else {
+ service.replicateSecretToRegions(arn,
+ List.of(new SecretsManagerService.ReplicaRegion(REPLICA_REGION, null)), false, REGION);
+ }
+ });
+ assertEquals("InvalidRequestException", missing.getErrorCode());
+ });
+ replication.awaitSuccess();
+ } finally {
+ release.countDown();
+ deletion.awaitSuccess();
+ if (replication != null) {
+ replication.awaitSuccess();
+ }
+ }
+ assertThrows(AwsException.class, () -> service.describeSecret(arn, REGION));
+ assertThrows(AwsException.class, () -> service.describeSecret(SECRET_NAME, REPLICA_REGION));
+ }
+
+ @Test
+ void queuedPromotionCannotRestoreSecretsAfterManagedRelease() throws Exception {
+ String arn = service.describeSecret(SECRET_NAME, REGION).getArn();
+ service.replicateSecretToRegions(arn,
+ List.of(new SecretsManagerService.ReplicaRegion(REPLICA_REGION, null)), false, REGION);
+ storage.parkOn(REGION + "::" + SECRET_NAME);
+ storage.afterParkedWrite = () -> {
+ service.removeRegionsFromReplication(arn, List.of(REPLICA_REGION), REGION);
+ assertTrue(service.deleteManagedSecretAfter(arn, REGION, () -> { }));
+ };
+ Watched writer = start("primary-writer", () -> service.updateSecret(arn, "updated", null, REGION));
+ Watched promotion = null;
+ try {
+ assertTrue(storage.awaitEntered());
+ promotion = start("queued-promotion", () -> {
+ AwsException missing = assertThrows(AwsException.class,
+ () -> service.stopReplicationToReplica(SECRET_NAME, REPLICA_REGION));
+ assertEquals("ResourceNotFoundException", missing.getErrorCode());
+ });
+ assertTrue(promotion.awaitBlocked());
+ } finally {
+ storage.release();
+ writer.awaitSuccess();
+ if (promotion != null) {
+ promotion.awaitSuccess();
+ }
+ }
+ assertThrows(AwsException.class, () -> service.describeSecret(arn, REGION));
+ assertThrows(AwsException.class, () -> service.describeSecret(SECRET_NAME, REPLICA_REGION));
+ }
+
+ @ParameterizedTest
+ @ValueSource(booleans = {false, true})
+ void activeReleaseRejectsDeletionAndDuplicateReleaseUntilBackendFailure(boolean forceDelete) throws Exception {
+ String arn = service.describeSecret(SECRET_NAME, REGION).getArn();
+ CountDownLatch entered = new CountDownLatch(1);
+ CountDownLatch release = new CountDownLatch(1);
+ Watched owner = start("failing-release", () -> assertThrows(IllegalStateException.class,
+ () -> service.deleteManagedSecretAfter(arn, REGION, () -> {
+ entered.countDown();
+ try {
+ assertTrue(release.await(10, TimeUnit.SECONDS));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new AssertionError(e);
+ }
+ throw new IllegalStateException("backend failed");
+ })));
+ Watched concurrent = null;
+ try {
+ assertTrue(entered.await(5, TimeUnit.SECONDS));
+ concurrent = start("concurrent-release", () -> {
+ AwsException deletion = assertThrows(AwsException.class,
+ () -> service.deleteSecret(arn, null, forceDelete, REGION));
+ assertEquals("InvalidRequestException", deletion.getErrorCode());
+ AwsException duplicate = assertThrows(AwsException.class,
+ () -> service.deleteManagedSecretAfter(arn, REGION,
+ () -> { throw new AssertionError("Duplicate backend call"); }));
+ assertEquals("InvalidRequestException", duplicate.getErrorCode());
+ });
+ concurrent.awaitSuccess();
+ } finally {
+ release.countDown();
+ owner.awaitSuccess();
+ if (concurrent != null) {
+ concurrent.awaitSuccess();
+ }
+ }
+ assertEquals("v1", service.getSecretValue(arn, null, null, REGION).getSecretString());
+ assertTrue(service.deleteManagedSecretAfter(arn, REGION, () -> { }));
+ assertThrows(AwsException.class, () -> service.describeSecret(arn, REGION));
+ }
+
+ @Test
+ void failedCredentialChangeRetainsTheSecret() {
+ String arn = service.describeSecret(SECRET_NAME, REGION).getArn();
+ assertThrows(IllegalStateException.class, () -> service.deleteManagedSecretAfter(arn, REGION, () -> {
+ throw new IllegalStateException("backend refused rotation");
+ }));
+ assertEquals("v1", service.getSecretValue(arn, null, null, REGION).getSecretString());
+ }
+
+ @Test
+ void alreadyRemovedManagedSecretStillAllowsCredentialChange() {
+ String arn = service.describeSecret(SECRET_NAME, REGION).getArn();
+ service.deleteSecret(arn, null, true, REGION);
+ AtomicReference<Boolean> changed = new AtomicReference<>(false);
+ assertTrue(service.deleteManagedSecretAfter(arn, REGION, () -> changed.set(true)));
+ assertTrue(changed.get());
+ }
+
@Test
void updateSecretWaitsForAnInFlightUpdate() throws Exception {
storage.parkOn(REGION + "::" + SECRET_NAME);
@@ -155,6 +392,7 @@ class SecretsManagerWriteLockTest {
private final CountDownLatch entered = new CountDownLatch(1);
private final CountDownLatch released = new CountDownLatch(1);
private volatile BiPredicate<String, Secret> parkWhen;
+ private Runnable afterParkedWrite;
void parkOn(String key) {
parkOn((written, secret) -> written.equals(key));
@@ -185,6 +423,11 @@ class SecretsManagerWriteLockTest {
}
}
super.put(key, value);
+ Runnable afterWrite = afterParkedWrite;
+ if (afterWrite != null) {
+ afterParkedWrite = null;
+ afterWrite.run();
+ }
}
}