Light, fluffy, and always free - The AWS Local Emulator alternative
fix(rds): warn once for failed cluster member relays (#4721)
XuJianbc42cee76779
diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java b/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.javaindex 9e9371f9d9..c6d2aa27b9 100644--- a/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java@@ -189,6 +189,7 @@ public class RdsService implements Resettable, ResourceProvider {// Null when a test constructs the service without CloudWatch; auto-pause then reports no metrics.private final CloudWatchMetricsService metricsService;private final Set<Integer> usedPorts = ConcurrentHashMap.newKeySet();+ private final Set<String> reportedMemberRelayFailures = ConcurrentHashMap.newKeySet();private static final Pattern IMAGE_TAG_VERSION_PATTERN = Pattern.compile("^(\\d+(?:\\.\\d+)*)(.*)$");private static final Pattern SAFE_IMAGE_TAG_PATTERN = Pattern.compile("[A-Za-z0-9._-]+");private static final int SERVERLESS_V2_DEFAULT_AUTO_PAUSE_SECONDS = 300;@@ -415,6 +416,7 @@ public class RdsService implements Resettable, ResourceProvider {public void clear() {usedPorts.clear();+ reportedMemberRelayFailures.clear();}// ── DB Instances ──────────────────────────────────────────────────────────@@ -3205,6 +3207,7 @@ public class RdsService implements Resettable, ResourceProvider {instance.setStatus(DbInstanceStatus.AVAILABLE);}putInstanceForScope(currentAccountId(), effectiveRegion, id, instance);+ reportedMemberRelayFailures.remove(instance.getDbInstanceArn());LOG.infov("Backing database container for DB instance {0} started on retry", id);return instance;}@@ -3330,6 +3333,7 @@ public class RdsService implements Resettable, ResourceProvider {member.setContainerHost(cluster.getContainerHost());member.setContainerPort(cluster.getContainerPort());}+ // A restart without a backend has not restored the relay; retry success clears the warning.if (restart) {member.setStatus(DbInstanceStatus.AVAILABLE);putInstanceForScope(accountId, region, memberId, member);@@ -3341,7 +3345,9 @@ public class RdsService implements Resettable, ResourceProvider {} catch (RuntimeException persistFailure) {e.addSuppressed(persistFailure);}- LOG.debugv(e, "Failed to restore RDS cluster member {0}; its relay can be retried", memberId);+ Logger.Level level = reportedMemberRelayFailures.add(member.getDbInstanceArn())+ ? Logger.Level.WARN : Logger.Level.DEBUG;+ LOG.logv(level, e, "Failed to restore RDS cluster member {0}; its relay can be retried", memberId);if (restart) {if (restartFailure == null) {restartFailure = e;@@ -3437,6 +3443,7 @@ public class RdsService implements Resettable, ResourceProvider {releaseProxyPort(instance.getProxyPort());deleteInstanceForScope(currentAccountId(), effectiveRegion, id);+ reportedMemberRelayFailures.remove(instance.getDbInstanceArn());LOG.infov("DB instance {0} deleted", id);}diff --git a/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.javaindex 7ca487e14d..ed86416a24 100644--- a/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java@@ -44,6 +44,7 @@ import io.github.hectorvent.floci.services.rds.proxy.RdsProxyBinding;import io.github.hectorvent.floci.services.rds.proxy.RdsProxyManager;import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;import io.github.hectorvent.floci.services.secretsmanager.model.Secret;+import io.github.hectorvent.floci.testutil.LogCapture;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;import org.junit.jupiter.params.ParameterizedTest;@@ -57,13 +58,14 @@ import java.util.Collection;import java.util.List;import java.util.Map;import java.util.Optional;+import java.util.OptionalInt;import java.util.concurrent.CountDownLatch;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;import java.util.concurrent.Future;import java.util.concurrent.TimeUnit;import java.util.concurrent.atomic.AtomicBoolean;-import java.util.OptionalInt;+import java.util.logging.LogRecord;import java.util.stream.Collectors;import static org.junit.jupiter.api.Assertions.*;@@ -4243,6 +4245,91 @@ class RdsServiceTest {any(), any(), any(), any(), any(), any());}+ @Test+ void clusterMemberRelayFailureWarnsOncePerMemberUntilRecovery() {+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ .thenReturn(null);+ DbCluster cluster = rdsService.createDbCluster("log-cluster", "aurora-postgresql", "16.3",+ "admin", "password", "dbname", false, null);+ DbInstance first = rdsService.createDbInstance("log-first", "aurora-postgresql", "16.3",+ "admin", "password", "dbname", "db.serverless", 0, false, null, null, "log-cluster");+ DbInstance second = rdsService.createDbInstance("log-second", "aurora-postgresql", "16.3",+ "admin", "password", "dbname", "db.serverless", 0, false, null, null, "log-cluster");+ IllegalStateException firstFailure = new IllegalStateException("first relay failed");+ IllegalStateException secondFailure = new IllegalStateException("second relay failed");+ IllegalStateException afterRecoveryFailure = new IllegalStateException("relay failed again");+ doThrow(firstFailure).doThrow(firstFailure).doNothing().doThrow(afterRecoveryFailure)+ .when(proxyManager).startProxy(eq("rds-resource:" + first.getDbInstanceArn()),+ any(), anyBoolean(), anyInt(), any(), anyInt(), any(), any(), any(), any(), any(), any());+ doThrow(secondFailure).doThrow(secondFailure).doNothing()+ .when(proxyManager).startProxy(eq("rds-resource:" + second.getDbInstanceArn()),+ any(), anyBoolean(), anyInt(), any(), anyInt(), any(), any(), any(), any(), any(), any());+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ .thenReturn(new RdsContainerHandle("log-container", cluster.getDbClusterArn(),+ "log-cluster", "127.0.0.1", 15432));++ List<LogRecord> records = LogCapture.capture(RdsService.class, () -> {+ rdsService.ensureClusterBackend("log-cluster", "us-east-1");+ rdsService.ensureClusterBackend("log-cluster", "us-east-1");+ rdsService.ensureClusterBackend("log-cluster", "us-east-1");+ assertEquals(DbInstanceStatus.AVAILABLE, first.getStatus());+ assertEquals(DbInstanceStatus.AVAILABLE, second.getStatus());+ first.setContainerHost(null);+ first.setContainerPort(0);+ rdsService.ensureClusterBackend("log-cluster", "us-east-1");+ });+ List<LogRecord> failures = records.stream()+ .filter(record -> record.getMessage().contains("Failed to restore RDS cluster member"))+ .toList();+ assertEquals(List.of("WARN", "WARN", "DEBUG", "DEBUG", "WARN"),+ failures.stream().map(record -> record.getLevel().getName()).toList());+ assertEquals(List.of(firstFailure, secondFailure, firstFailure, secondFailure,+ afterRecoveryFailure), failures.stream().map(LogRecord::getThrown).toList());+ }++ @Test+ void clusterMemberRestartWithoutBackendKeepsRelayWarningUntilRecovery() {+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ .thenReturn(null);+ DbCluster cluster = rdsService.createDbCluster("restart-log-cluster", "aurora-postgresql", "16.3",+ "admin", "password", "dbname", false, null);+ DbInstance member = rdsService.createDbInstance("restart-log-member", "aurora-postgresql", "16.3",+ "admin", "password", "dbname", "db.serverless", 0, false, null, null,+ "restart-log-cluster");+ IllegalStateException firstFailure = new IllegalStateException("first relay failed");+ IllegalStateException ongoingFailure = new IllegalStateException("relay still failed after restart");+ IllegalStateException afterRecoveryFailure = new IllegalStateException("relay failed after recovery");+ doThrow(firstFailure).doThrow(ongoingFailure).doNothing().doThrow(afterRecoveryFailure)+ .when(proxyManager).startProxy(eq("rds-resource:" + member.getDbInstanceArn()),+ any(), anyBoolean(), anyInt(), any(), anyInt(), any(), any(), any(), any(), any(), any());+ RdsContainerHandle handle = new RdsContainerHandle("restart-log-container",+ cluster.getDbClusterArn(), "restart-log-cluster", "127.0.0.1", 15432);+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ .thenReturn(handle, null, handle);++ List<LogRecord> records = LogCapture.capture(RdsService.class, () -> {+ rdsService.ensureClusterBackend("restart-log-cluster", "us-east-1");+ assertEquals(DbInstanceStatus.FAILED, member.getStatus());+ rdsService.rebootDbCluster("restart-log-cluster", "us-east-1");+ assertEquals(DbInstanceStatus.AVAILABLE, member.getStatus());+ assertNull(member.getContainerHost());+ rdsService.ensureClusterBackend("restart-log-cluster", "us-east-1");+ assertEquals(DbInstanceStatus.FAILED, member.getStatus());+ rdsService.ensureClusterBackend("restart-log-cluster", "us-east-1");+ assertEquals(DbInstanceStatus.AVAILABLE, member.getStatus());+ member.setContainerHost(null);+ member.setContainerPort(0);+ rdsService.ensureClusterBackend("restart-log-cluster", "us-east-1");+ });+ List<LogRecord> failures = records.stream()+ .filter(record -> record.getMessage().contains("Failed to restore RDS cluster member"))+ .toList();+ assertEquals(List.of("WARN", "DEBUG", "WARN"),+ failures.stream().map(record -> record.getLevel().getName()).toList());+ assertEquals(List.of(firstFailure, ongoingFailure, afterRecoveryFailure),+ failures.stream().map(LogRecord::getThrown).toList());+ }+@ParameterizedTest@CsvSource({"true", "false"})void clusterControlPlaneReportsMemberFailureAfterTryingTheRemainingMembers(boolean reboot) {