Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(rds): warn once for failed cluster member relays (#4721)

XuJianbc42cee76779
diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java b/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java
index 9e9371f9d9..c6d2aa27b9 100644
--- a/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java
@@ -189,6 +189,7 @@ public class RdsService implements Resettable, ResourceProvider {
// Null when a test constructs the service without CloudWatch; auto-pause then reports no metrics.
private final CloudWatchMetricsService metricsService;
private final Set<Integer> usedPorts = ConcurrentHashMap.newKeySet();
+ private final Set<String> reportedMemberRelayFailures = ConcurrentHashMap.newKeySet();
private static final Pattern IMAGE_TAG_VERSION_PATTERN = Pattern.compile("^(\\d+(?:\\.\\d+)*)(.*)$");
private static final Pattern SAFE_IMAGE_TAG_PATTERN = Pattern.compile("[A-Za-z0-9._-]+");
private static final int SERVERLESS_V2_DEFAULT_AUTO_PAUSE_SECONDS = 300;
@@ -415,6 +416,7 @@ public class RdsService implements Resettable, ResourceProvider {
public void clear() {
usedPorts.clear();
+ reportedMemberRelayFailures.clear();
}
// ── DB Instances ──────────────────────────────────────────────────────────
@@ -3205,6 +3207,7 @@ public class RdsService implements Resettable, ResourceProvider {
instance.setStatus(DbInstanceStatus.AVAILABLE);
}
putInstanceForScope(currentAccountId(), effectiveRegion, id, instance);
+ reportedMemberRelayFailures.remove(instance.getDbInstanceArn());
LOG.infov("Backing database container for DB instance {0} started on retry", id);
return instance;
}
@@ -3330,6 +3333,7 @@ public class RdsService implements Resettable, ResourceProvider {
member.setContainerHost(cluster.getContainerHost());
member.setContainerPort(cluster.getContainerPort());
}
+ // A restart without a backend has not restored the relay; retry success clears the warning.
if (restart) {
member.setStatus(DbInstanceStatus.AVAILABLE);
putInstanceForScope(accountId, region, memberId, member);
@@ -3341,7 +3345,9 @@ public class RdsService implements Resettable, ResourceProvider {
} catch (RuntimeException persistFailure) {
e.addSuppressed(persistFailure);
}
- LOG.debugv(e, "Failed to restore RDS cluster member {0}; its relay can be retried", memberId);
+ Logger.Level level = reportedMemberRelayFailures.add(member.getDbInstanceArn())
+ ? Logger.Level.WARN : Logger.Level.DEBUG;
+ LOG.logv(level, e, "Failed to restore RDS cluster member {0}; its relay can be retried", memberId);
if (restart) {
if (restartFailure == null) {
restartFailure = e;
@@ -3437,6 +3443,7 @@ public class RdsService implements Resettable, ResourceProvider {
releaseProxyPort(instance.getProxyPort());
deleteInstanceForScope(currentAccountId(), effectiveRegion, id);
+ reportedMemberRelayFailures.remove(instance.getDbInstanceArn());
LOG.infov("DB instance {0} deleted", id);
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java
index 7ca487e14d..ed86416a24 100644
--- a/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java
@@ -44,6 +44,7 @@ import io.github.hectorvent.floci.services.rds.proxy.RdsProxyBinding;
import io.github.hectorvent.floci.services.rds.proxy.RdsProxyManager;
import io.github.hectorvent.floci.services.secretsmanager.SecretsManagerService;
import io.github.hectorvent.floci.services.secretsmanager.model.Secret;
+import io.github.hectorvent.floci.testutil.LogCapture;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.params.ParameterizedTest;
@@ -57,13 +58,14 @@ import java.util.Collection;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.OptionalInt;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.OptionalInt;
+import java.util.logging.LogRecord;
import java.util.stream.Collectors;
import static org.junit.jupiter.api.Assertions.*;
@@ -4243,6 +4245,91 @@ class RdsServiceTest {
any(), any(), any(), any(), any(), any());
}
+ @Test
+ void clusterMemberRelayFailureWarnsOncePerMemberUntilRecovery() {
+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))
+ .thenReturn(null);
+ DbCluster cluster = rdsService.createDbCluster("log-cluster", "aurora-postgresql", "16.3",
+ "admin", "password", "dbname", false, null);
+ DbInstance first = rdsService.createDbInstance("log-first", "aurora-postgresql", "16.3",
+ "admin", "password", "dbname", "db.serverless", 0, false, null, null, "log-cluster");
+ DbInstance second = rdsService.createDbInstance("log-second", "aurora-postgresql", "16.3",
+ "admin", "password", "dbname", "db.serverless", 0, false, null, null, "log-cluster");
+ IllegalStateException firstFailure = new IllegalStateException("first relay failed");
+ IllegalStateException secondFailure = new IllegalStateException("second relay failed");
+ IllegalStateException afterRecoveryFailure = new IllegalStateException("relay failed again");
+ doThrow(firstFailure).doThrow(firstFailure).doNothing().doThrow(afterRecoveryFailure)
+ .when(proxyManager).startProxy(eq("rds-resource:" + first.getDbInstanceArn()),
+ any(), anyBoolean(), anyInt(), any(), anyInt(), any(), any(), any(), any(), any(), any());
+ doThrow(secondFailure).doThrow(secondFailure).doNothing()
+ .when(proxyManager).startProxy(eq("rds-resource:" + second.getDbInstanceArn()),
+ any(), anyBoolean(), anyInt(), any(), anyInt(), any(), any(), any(), any(), any(), any());
+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))
+ .thenReturn(new RdsContainerHandle("log-container", cluster.getDbClusterArn(),
+ "log-cluster", "127.0.0.1", 15432));
+
+ List<LogRecord> records = LogCapture.capture(RdsService.class, () -> {
+ rdsService.ensureClusterBackend("log-cluster", "us-east-1");
+ rdsService.ensureClusterBackend("log-cluster", "us-east-1");
+ rdsService.ensureClusterBackend("log-cluster", "us-east-1");
+ assertEquals(DbInstanceStatus.AVAILABLE, first.getStatus());
+ assertEquals(DbInstanceStatus.AVAILABLE, second.getStatus());
+ first.setContainerHost(null);
+ first.setContainerPort(0);
+ rdsService.ensureClusterBackend("log-cluster", "us-east-1");
+ });
+ List<LogRecord> failures = records.stream()
+ .filter(record -> record.getMessage().contains("Failed to restore RDS cluster member"))
+ .toList();
+ assertEquals(List.of("WARN", "WARN", "DEBUG", "DEBUG", "WARN"),
+ failures.stream().map(record -> record.getLevel().getName()).toList());
+ assertEquals(List.of(firstFailure, secondFailure, firstFailure, secondFailure,
+ afterRecoveryFailure), failures.stream().map(LogRecord::getThrown).toList());
+ }
+
+ @Test
+ void clusterMemberRestartWithoutBackendKeepsRelayWarningUntilRecovery() {
+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))
+ .thenReturn(null);
+ DbCluster cluster = rdsService.createDbCluster("restart-log-cluster", "aurora-postgresql", "16.3",
+ "admin", "password", "dbname", false, null);
+ DbInstance member = rdsService.createDbInstance("restart-log-member", "aurora-postgresql", "16.3",
+ "admin", "password", "dbname", "db.serverless", 0, false, null, null,
+ "restart-log-cluster");
+ IllegalStateException firstFailure = new IllegalStateException("first relay failed");
+ IllegalStateException ongoingFailure = new IllegalStateException("relay still failed after restart");
+ IllegalStateException afterRecoveryFailure = new IllegalStateException("relay failed after recovery");
+ doThrow(firstFailure).doThrow(ongoingFailure).doNothing().doThrow(afterRecoveryFailure)
+ .when(proxyManager).startProxy(eq("rds-resource:" + member.getDbInstanceArn()),
+ any(), anyBoolean(), anyInt(), any(), anyInt(), any(), any(), any(), any(), any(), any());
+ RdsContainerHandle handle = new RdsContainerHandle("restart-log-container",
+ cluster.getDbClusterArn(), "restart-log-cluster", "127.0.0.1", 15432);
+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))
+ .thenReturn(handle, null, handle);
+
+ List<LogRecord> records = LogCapture.capture(RdsService.class, () -> {
+ rdsService.ensureClusterBackend("restart-log-cluster", "us-east-1");
+ assertEquals(DbInstanceStatus.FAILED, member.getStatus());
+ rdsService.rebootDbCluster("restart-log-cluster", "us-east-1");
+ assertEquals(DbInstanceStatus.AVAILABLE, member.getStatus());
+ assertNull(member.getContainerHost());
+ rdsService.ensureClusterBackend("restart-log-cluster", "us-east-1");
+ assertEquals(DbInstanceStatus.FAILED, member.getStatus());
+ rdsService.ensureClusterBackend("restart-log-cluster", "us-east-1");
+ assertEquals(DbInstanceStatus.AVAILABLE, member.getStatus());
+ member.setContainerHost(null);
+ member.setContainerPort(0);
+ rdsService.ensureClusterBackend("restart-log-cluster", "us-east-1");
+ });
+ List<LogRecord> failures = records.stream()
+ .filter(record -> record.getMessage().contains("Failed to restore RDS cluster member"))
+ .toList();
+ assertEquals(List.of("WARN", "DEBUG", "WARN"),
+ failures.stream().map(record -> record.getLevel().getName()).toList());
+ assertEquals(List.of(firstFailure, ongoingFailure, afterRecoveryFailure),
+ failures.stream().map(LogRecord::getThrown).toList());
+ }
+
@ParameterizedTest
@CsvSource({"true", "false"})
void clusterControlPlaneReportsMemberFailureAfterTryingTheRemainingMembers(boolean reboot) {