Light, fluffy, and always free - The AWS Local Emulator alternative
fix(rds): recover metadata after backend startup failures (#4532)
Aniket Shukla9ae71eceda13
diff --git a/docs/services/rds-data.md b/docs/services/rds-data.mdindex 275deaadcf..01a615ae65 100644--- a/docs/services/rds-data.md+++ b/docs/services/rds-data.md@@ -31,6 +31,7 @@ The deprecated `ExecuteSql` operation is recognized at `POST /ExecuteSql` and re## Compatibility Notes- `resourceArn` and `secretArn` are required on Data API requests. `resourceArn` must identify an existing local RDS cluster or instance.+- Resolving a cluster ARN retries missing active member relays even if the cluster backend is already running. A failed member relay does not block SQL through the healthy cluster backend and is retried on a subsequent request. Healthy relays are not rebound and stopped members remain stopped. See [recoverable RDS backend startup failures](rds.md#recoverable-backend-startup-failures).- `database` is optional when the resolved RDS resource has a database name; otherwise it must be provided. Transactional `ExecuteStatement` requests must use the same database as the active transaction when `database` is present.- Transaction requests validate `resourceArn` against the active transaction resource. Floci resolves accepted ARN aliases to the local resource before comparing transaction identity.- MySQL, MariaDB, and PostgreSQL resources are supported. Aurora PostgreSQL resources resolve to the same PostgreSQL execution path.diff --git a/docs/services/rds.md b/docs/services/rds.mdindex 3a4799e614..6fc369a02c 100644--- a/docs/services/rds.md+++ b/docs/services/rds.md@@ -4,6 +4,36 @@**Management Endpoint:** `POST http://localhost:4566/`**Data Endpoint:** `localhost:<proxy-port>` (TCP)+### Recoverable backend startup failures++If a persisted database's container or authentication relay cannot start when Floci restarts,+Floci retains the database record, data volume, and reserved endpoint port and retries when its+backend is next needed. A failed restore does not release or renumber a reserved endpoint; a retry+or the next boot uses the same port.+This recoverable emulator condition uses `available`, the same metadata-only status as a+restart without a reachable Docker daemon, not AWS's terminal `failed` state. It does not+guarantee that a database connection is ready while Docker is unavailable. Port allocation+failures still report `failed` because no endpoint could be allocated.++A successful lazy retry updates legacy `failed` records to `available` only after the container+and relay are ready. Older records whose endpoint was cleared receive a new relay endpoint on+retry. A failed retry leaves their status unchanged and cleans up the attempted backend and+any newly allocated relay port.++Instance and cluster start operations, and cluster reboot, reconstruct a missing relay endpoint+through the same retry path.+Retrying a cluster directly, or resolving its ARN for a Data API request, also restores its missing+member relays without restarting a healthy cluster backend or rebinding members that already+recovered. Data API cluster-ARN resolution checks for missing members even when the cluster backend+is already running, so a member relay that fails once is retried on the next request. Stopped+members remain stopped until a control-plane start operation.+When the cluster and all active members have backends, Data API resolution skips the synchronized+recovery path, so healthy queries do not wait for unrelated container starts or image pulls.+Lazy cluster recovery tries every active member even if one relay fails; a failed member does+not prevent Data API access through a recovered cluster backend, and repeated member failures are+logged at debug level. Start and reboot report member relay errors after trying all members and+mark those members `failed` until a successful retry.+Floci manages real PostgreSQL, MySQL, MariaDB, and SQL Server Docker containers and proxies TCP connections to them, including IAM authentication support where the protocol supports it. SQL Server uses a transparent TCP relay for its native TDS protocol.RDS Data API (`rds-data`) is documented separately because it uses REST JSON routes instead of the RDS Query protocol. See [RDS Data API](rds-data.md).diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java b/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.javaindex 1808674ff3..9e9371f9d9 100644--- a/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java@@ -2800,6 +2800,16 @@ public class RdsService implements Resettable, ResourceProvider {* it, the way a reboot does after stopping them.*/private void startStandaloneInstanceBackend(DbInstance instance, String id, String effectiveRegion) {+ if (instance.getEndpoint() == null) {+ try {+ ensureInstanceBackend(id, effectiveRegion);+ } catch (RuntimeException | Error e) {+ instance.setStatus(DbInstanceStatus.FAILED);+ putInstanceForScope(currentAccountId(), effectiveRegion, id, instance);+ throw e;+ }+ return;+ }String image = imageForEngine(instance.getEngine(), instance.getEngineVersion());String storageResourceId = resolvedInstanceStorageResourceId(instance);String dockerVolumeName = resolvedInstanceDockerVolumeName(instance);@@ -2905,28 +2915,7 @@ public class RdsService implements Resettable, ResourceProvider {}cluster.setStatus(DbInstanceStatus.AVAILABLE);putClusterForScope(accountId, effectiveRegion, id, cluster);- for (String memberId : cluster.getDbClusterMembers()) {- DbInstance member = findInstanceForScope(accountId, effectiveRegion, memberId);- if (member == null) {- continue;- }- member.setContainerId(cluster.getContainerId());- member.setContainerHost(cluster.getContainerHost());- member.setContainerPort(cluster.getContainerPort());- if (!config.services().rds().mock() && hasBackend(cluster.getContainerHost(), cluster.getContainerPort())) {- final String memberRegion = regionFromArn(member.getDbInstanceArn());- proxyManager.startProxy(rdsResourceRelayKey(member.getDbInstanceArn(), memberId),- member.getEngine(), member.isIamDatabaseAuthenticationEnabled(), member.getProxyPort(),- cluster.getContainerHost(), cluster.getContainerPort(), member.getEndpoint().address(),- member.getMasterUsername() != null ? member.getMasterUsername() : "root",- member.getMasterPassword(), member.getDbName(),- (user, pw) -> validateDbPasswordForScope(accountId, memberRegion, memberId, user, pw),- proxyBinding(member.getEngine(), member.getEndpoint().address(), member.getProxyPort(),- memberRegion, accountId, member.getDbiResourceId()));- }- member.setStatus(DbInstanceStatus.AVAILABLE);- putInstanceForScope(accountId, effectiveRegion, memberId, member);- }+ restoreClusterMemberBackends(cluster, effectiveRegion, true);LOG.infov("DB cluster {0} started", id);return response;}@@ -2949,28 +2938,7 @@ public class RdsService implements Resettable, ResourceProvider {}cluster.setStatus(DbInstanceStatus.AVAILABLE);putClusterForScope(accountId, effectiveRegion, id, cluster);- for (String memberId : cluster.getDbClusterMembers()) {- DbInstance member = findInstanceForScope(accountId, effectiveRegion, memberId);- if (member == null) {- continue;- }- member.setContainerId(cluster.getContainerId());- member.setContainerHost(cluster.getContainerHost());- member.setContainerPort(cluster.getContainerPort());- if (!config.services().rds().mock() && hasBackend(cluster.getContainerHost(), cluster.getContainerPort())) {- final String memberRegion = regionFromArn(member.getDbInstanceArn());- proxyManager.startProxy(rdsResourceRelayKey(member.getDbInstanceArn(), memberId),- member.getEngine(), member.isIamDatabaseAuthenticationEnabled(), member.getProxyPort(),- cluster.getContainerHost(), cluster.getContainerPort(), member.getEndpoint().address(),- member.getMasterUsername() != null ? member.getMasterUsername() : "root",- member.getMasterPassword(), member.getDbName(),- (user, pw) -> validateDbPasswordForScope(accountId, memberRegion, memberId, user, pw),- proxyBinding(member.getEngine(), member.getEndpoint().address(), member.getProxyPort(),- memberRegion, accountId, member.getDbiResourceId()));- }- member.setStatus(DbInstanceStatus.AVAILABLE);- putInstanceForScope(accountId, effectiveRegion, memberId, member);- }+ restoreClusterMemberBackends(cluster, effectiveRegion, true);LOG.infov("DB cluster {0} rebooted", id);return response;}@@ -3002,6 +2970,16 @@ public class RdsService implements Resettable, ResourceProvider {/** Brings a cluster's container up on its existing volume and its proxy with it. */private void startClusterBackend(DbCluster cluster, String id, String effectiveRegion) {+ if (cluster.getEndpoint() == null) {+ try {+ ensureClusterBackend(id, effectiveRegion, false);+ } catch (RuntimeException | Error e) {+ cluster.setStatus(DbInstanceStatus.FAILED);+ putClusterForScope(currentAccountId(), effectiveRegion, id, cluster);+ throw e;+ }+ return;+ }String image = imageForEngine(cluster.getEngine(), cluster.getEngineVersion());String storageResourceId = resolvedClusterStorageResourceId(cluster);String dockerVolumeName = resolvedClusterDockerVolumeName(cluster);@@ -3152,8 +3130,10 @@ public class RdsService implements Resettable, ResourceProvider {public synchronized DbInstance ensureInstanceBackend(String id, String region) {String effectiveRegion = effectiveRegion(region);DbInstance instance = getDbInstance(id, effectiveRegion);- if (config.services().rds().mock()- || hasBackend(instance.getContainerHost(), instance.getContainerPort())) {+ if (config.services().rds().mock()) {+ return instance;+ }+ if (hasBackend(instance.getContainerHost(), instance.getContainerPort())) {return instance;}@@ -3163,7 +3143,7 @@ public class RdsService implements Resettable, ResourceProvider {String backendHost;int backendPort;if (clusterId != null && !clusterId.isBlank()) {- DbCluster cluster = ensureClusterBackend(clusterId, effectiveRegion);+ DbCluster cluster = ensureClusterBackend(clusterId, effectiveRegion, false);if (!hasBackend(cluster.getContainerHost(), cluster.getContainerPort())) {return instance;}@@ -3188,17 +3168,27 @@ public class RdsService implements Resettable, ResourceProvider {? instance.getMasterUsername() : "root";final String accountId = accountIdFromArn(instance.getDbInstanceArn());final String instanceRegion = regionFromArn(instance.getDbInstanceArn());+ int relayPort = instance.getProxyPort();+ DbEndpoint relayEndpoint = instance.getEndpoint();+ boolean allocatedRelayPort = false;try {+ if (relayEndpoint == null) {+ relayPort = allocateProxyPort();+ allocatedRelayPort = true;+ relayEndpoint = proxyEndpoint(relayPort);+ }proxyManager.startProxy(rdsResourceRelayKey(instance.getDbInstanceArn(), id),instance.getEngine(), instance.isIamDatabaseAuthenticationEnabled(),- instance.getProxyPort(), backendHost, backendPort,- instance.getEndpoint().address(),+ relayPort, backendHost, backendPort, relayEndpoint.address(),effectiveMasterUser, instance.getMasterPassword(), instance.getDbName(),(user, pw) -> validateDbPasswordForScope(accountId, instanceRegion, id, user, pw),- proxyBinding(instance.getEngine(), instance.getEndpoint().address(),- instance.getProxyPort(), instanceRegion, accountId, instance.getDbiResourceId()));+ proxyBinding(instance.getEngine(), relayEndpoint.address(),+ relayPort, instanceRegion, accountId, instance.getDbiResourceId()));} catch (RuntimeException e) {+ if (allocatedRelayPort) {+ releaseProxyPort(relayPort);+ }stopStartedBackend(started, e);throw e;}@@ -3209,6 +3199,11 @@ public class RdsService implements Resettable, ResourceProvider {instance.setContainerId(backendContainerId);instance.setContainerHost(backendHost);instance.setContainerPort(backendPort);+ instance.setProxyPort(relayPort);+ instance.setEndpoint(relayEndpoint);+ if (instance.getStatus() == DbInstanceStatus.FAILED) {+ instance.setStatus(DbInstanceStatus.AVAILABLE);+ }putInstanceForScope(currentAccountId(), effectiveRegion, id, instance);LOG.infov("Backing database container for DB instance {0} started on retry", id);return instance;@@ -3220,10 +3215,34 @@ public class RdsService implements Resettable, ResourceProvider {* @return the cluster, with its container fields populated when a backend became available*/public synchronized DbCluster ensureClusterBackend(String id, String region) {+ return ensureClusterBackend(id, region, true);+ }++ /** Checks whether active members need the synchronized backend-recovery path. */+ public boolean hasMissingClusterMemberBackends(DbCluster cluster, String region) {+ String accountId = currentAccountId();+ String effectiveRegion = effectiveRegion(region);+ for (String memberId : new ArrayList<>(cluster.getDbClusterMembers())) {+ DbInstance member = findInstanceForScope(accountId, effectiveRegion, memberId);+ if (member != null && member.getStatus() != DbInstanceStatus.STOPPED+ && member.getStatus() != DbInstanceStatus.DELETING+ && !hasBackend(member.getContainerHost(), member.getContainerPort())) {+ return true;+ }+ }+ return false;+ }++ private DbCluster ensureClusterBackend(String id, String region, boolean restoreMembers) {String effectiveRegion = effectiveRegion(region);DbCluster cluster = getDbCluster(id, effectiveRegion);- if (config.services().rds().mock()- || hasBackend(cluster.getContainerHost(), cluster.getContainerPort())) {+ if (config.services().rds().mock()) {+ return cluster;+ }+ if (hasBackend(cluster.getContainerHost(), cluster.getContainerPort())) {+ if (restoreMembers) {+ restoreClusterMemberBackends(cluster, effectiveRegion, false);+ }return cluster;}@@ -3242,17 +3261,27 @@ public class RdsService implements Resettable, ResourceProvider {? cluster.getMasterUsername() : "root";final String accountId = accountIdFromArn(cluster.getDbClusterArn());final String clusterRegion = regionFromArn(cluster.getDbClusterArn());+ int relayPort = cluster.getProxyPort();+ DbEndpoint relayEndpoint = cluster.getEndpoint();+ boolean allocatedRelayPort = false;try {+ if (relayEndpoint == null) {+ relayPort = allocateProxyPort();+ allocatedRelayPort = true;+ relayEndpoint = proxyEndpoint(relayPort);+ }proxyManager.startProxy(rdsResourceRelayKey(cluster.getDbClusterArn(), id),cluster.getEngine(), cluster.isIamDatabaseAuthenticationEnabled(),- cluster.getProxyPort(), started.getHost(), started.getPort(),- cluster.getEndpoint().address(),+ relayPort, started.getHost(), started.getPort(), relayEndpoint.address(),effectiveMasterUser, cluster.getMasterPassword(), cluster.getDatabaseName(),(user, pw) -> validateDbClusterPasswordForScope(accountId, clusterRegion, id, user, pw),- proxyBinding(cluster.getEngine(), cluster.getEndpoint().address(),- cluster.getProxyPort(), clusterRegion, accountId, cluster.getDbClusterResourceId()));+ proxyBinding(cluster.getEngine(), relayEndpoint.address(),+ relayPort, clusterRegion, accountId, cluster.getDbClusterResourceId()));} catch (RuntimeException e) {+ if (allocatedRelayPort) {+ releaseProxyPort(relayPort);+ }stopStartedBackend(started, e);throw e;}@@ -3261,12 +3290,74 @@ public class RdsService implements Resettable, ResourceProvider {cluster.setContainerId(started.getContainerId());cluster.setContainerHost(started.getHost());cluster.setContainerPort(started.getPort());+ cluster.setProxyPort(relayPort);+ cluster.setEndpoint(relayEndpoint);+ if (cluster.getReaderEndpoint() == null) {+ cluster.setReaderEndpoint(relayEndpoint);+ }+ if (cluster.getStatus() == DbInstanceStatus.FAILED) {+ cluster.setStatus(DbInstanceStatus.AVAILABLE);+ }putClusterForScope(currentAccountId(), effectiveRegion, id, cluster);applyAutoPause(cluster);LOG.infov("Backing database container for DB cluster {0} started on retry", id);+ if (restoreMembers) {+ restoreClusterMemberBackends(cluster, effectiveRegion, false);+ }return cluster;}+ private void restoreClusterMemberBackends(DbCluster cluster, String region, boolean restart) {+ String accountId = currentAccountId();+ RuntimeException restartFailure = null;+ for (String memberId : cluster.getDbClusterMembers()) {+ DbInstance member = findInstanceForScope(accountId, region, memberId);+ if (member == null || member.getStatus() == DbInstanceStatus.DELETING+ || (!restart && member.getStatus() == DbInstanceStatus.STOPPED)) {+ continue;+ }+ try {+ if (!config.services().rds().mock()+ && hasBackend(cluster.getContainerHost(), cluster.getContainerPort())) {+ if (restart) {+ member.setContainerHost(null);+ member.setContainerPort(0);+ }+ // Member retry must not recurse into the cluster's other members.+ ensureInstanceBackend(memberId, region);+ } else if (restart) {+ member.setContainerId(cluster.getContainerId());+ member.setContainerHost(cluster.getContainerHost());+ member.setContainerPort(cluster.getContainerPort());+ }+ if (restart) {+ member.setStatus(DbInstanceStatus.AVAILABLE);+ putInstanceForScope(accountId, region, memberId, member);+ }+ } catch (RuntimeException e) {+ member.setStatus(DbInstanceStatus.FAILED);+ try {+ putInstanceForScope(accountId, region, memberId, member);+ } catch (RuntimeException persistFailure) {+ e.addSuppressed(persistFailure);+ }+ LOG.debugv(e, "Failed to restore RDS cluster member {0}; its relay can be retried", memberId);+ if (restart) {+ if (restartFailure == null) {+ restartFailure = e;+ } else {+ restartFailure.addSuppressed(e);+ }+ }+ }+ }+ // A Data API request only needs the cluster backend, which is already ready. Start+ // and reboot operations still report a member failure, after trying every member.+ if (restartFailure != null) {+ throw restartFailure;+ }+ }+private void stopStartedBackend(RdsContainerHandle started, RuntimeException proxyFailure) {if (started == null) {return;@@ -7183,7 +7274,7 @@ public class RdsService implements Resettable, ResourceProvider {cluster.setEndpoint(null);cluster.setReaderEndpoint(null);}- cluster.setStatus(DbInstanceStatus.FAILED);+ cluster.setStatus(portReserved ? DbInstanceStatus.AVAILABLE : DbInstanceStatus.FAILED);String retainedContainerId = !containerCleaned && cleanupHandle != null? cleanupHandle.getContainerId(): restoredHandle == null ? persistedContainerId : null;@@ -7324,7 +7415,7 @@ public class RdsService implements Resettable, ResourceProvider {instance.setProxyPort(0);instance.setEndpoint(null);}- instance.setStatus(DbInstanceStatus.FAILED);+ instance.setStatus(portReserved ? DbInstanceStatus.AVAILABLE : DbInstanceStatus.FAILED);String retainedContainerId = !containerCleaned && cleanupHandle != null? cleanupHandle.getContainerId(): restoredHandle == null ? persistedContainerId : null;@@ -7675,7 +7766,7 @@ public class RdsService implements Resettable, ResourceProvider {return instance;}- private synchronized DbCluster findClusterForScope(+ private DbCluster findClusterForScope(String accountId, String region, String clusterId) {String effectiveAccountId = accountId != null ? accountId : currentAccountId();String effectiveRegion = effectiveRegion(region);@@ -7684,26 +7775,29 @@ public class RdsService implements Resettable, ResourceProvider {cluster.getDbClusterArn(), effectiveAccountId, effectiveRegion,"cluster", clusterId);if (clusters instanceof AccountAwareStorageBackend<DbCluster> aware) {+ // The store already serializes legacy migration, independently of container starts.return aware.getForAccountMigratingLegacyKeys(effectiveAccountId, key, List.of(clusterId), owner).filter(owner).orElse(null);}- Optional<DbCluster> canonical = clusters.get(key).filter(owner);- if (canonical.isPresent()) {- clusters.get(clusterId).filter(owner).ifPresent(ignored -> clusters.delete(clusterId));- return canonical.get();- }- Optional<DbCluster> legacy = clusters.get(clusterId).filter(owner);- if (legacy.isPresent()) {- clusters.put(key, legacy.get());- clusters.delete(clusterId);+ synchronized (this) {+ Optional<DbCluster> canonical = clusters.get(key).filter(owner);+ if (canonical.isPresent()) {+ clusters.get(clusterId).filter(owner).ifPresent(ignored -> clusters.delete(clusterId));+ return canonical.get();+ }+ Optional<DbCluster> legacy = clusters.get(clusterId).filter(owner);+ if (legacy.isPresent()) {+ clusters.put(key, legacy.get());+ clusters.delete(clusterId);+ }+ return legacy.orElse(null);}- return legacy.orElse(null);}- private synchronized DbInstance findInstanceForScope(+ private DbInstance findInstanceForScope(String accountId, String region, String instanceId) {String effectiveAccountId = accountId != null ? accountId : currentAccountId();String effectiveRegion = effectiveRegion(region);@@ -7712,23 +7806,26 @@ public class RdsService implements Resettable, ResourceProvider {instance.getDbInstanceArn(), effectiveAccountId, effectiveRegion,"db", instanceId);if (instances instanceof AccountAwareStorageBackend<DbInstance> aware) {+ // The store already serializes legacy migration, independently of container starts.return aware.getForAccountMigratingLegacyKeys(effectiveAccountId, key, List.of(instanceId), owner).filter(owner).orElse(null);}- Optional<DbInstance> canonical = instances.get(key).filter(owner);- if (canonical.isPresent()) {- instances.get(instanceId).filter(owner).ifPresent(ignored -> instances.delete(instanceId));- return canonical.get();- }- Optional<DbInstance> legacy = instances.get(instanceId).filter(owner);- if (legacy.isPresent()) {- instances.put(key, legacy.get());- instances.delete(instanceId);+ synchronized (this) {+ Optional<DbInstance> canonical = instances.get(key).filter(owner);+ if (canonical.isPresent()) {+ instances.get(instanceId).filter(owner).ifPresent(ignored -> instances.delete(instanceId));+ return canonical.get();+ }+ Optional<DbInstance> legacy = instances.get(instanceId).filter(owner);+ if (legacy.isPresent()) {+ instances.put(key, legacy.get());+ instances.delete(instanceId);+ }+ return legacy.orElse(null);}- return legacy.orElse(null);}private void putClusterForScope(diff --git a/src/main/java/io/github/hectorvent/floci/services/rdsdata/RdsDataResourceResolver.java b/src/main/java/io/github/hectorvent/floci/services/rdsdata/RdsDataResourceResolver.javaindex 818d70b4e5..1b107072c1 100644--- a/src/main/java/io/github/hectorvent/floci/services/rdsdata/RdsDataResourceResolver.java+++ b/src/main/java/io/github/hectorvent/floci/services/rdsdata/RdsDataResourceResolver.java@@ -73,7 +73,9 @@ class RdsDataResourceResolver {}private DatabaseTarget fromCluster(DbCluster cluster, String region) {+ // Healthy queries must not wait behind unrelated container starts or image pulls.DbCluster resolved = hasRuntime(cluster.getContainerHost(), cluster.getContainerPort())+ && !rdsService.hasMissingClusterMemberBackends(cluster, region)? cluster: rdsService.ensureClusterBackend(cluster.getDbClusterIdentifier(), region);return target(resolved.getDbClusterArn(), resolved.getEngine(), resolved.getContainerHost(),diff --git a/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.javaindex f16c54c3d0..7ca487e14d 100644--- a/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java@@ -2124,6 +2124,7 @@ class RdsServiceTest {20, false, null, null, null);assertNull(rdsService.ensureInstanceBackend("probe-db", "us-east-1").getContainerId());+ created.setStatus(DbInstanceStatus.FAILED);// A Docker daemon appears.when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))@@ -2131,6 +2132,8 @@ class RdsServiceTest {"probe-db", "127.0.0.1", 15432));DbInstance started = rdsService.ensureInstanceBackend("probe-db", "us-east-1");+ assertEquals(DbInstanceStatus.AVAILABLE, started.getStatus());+ assertEquals(DbInstanceStatus.AVAILABLE, rdsService.getDbInstance("probe-db").getStatus());assertEquals("late-container", started.getContainerId());assertEquals("127.0.0.1", started.getContainerHost());assertEquals(15432, started.getContainerPort());@@ -2162,6 +2165,7 @@ class RdsServiceTest {20, false, null, null, null);RdsContainerHandle late = new RdsContainerHandle("late-container", created.getDbInstanceArn(),"probe-db", "127.0.0.1", 15432);+ created.setStatus(DbInstanceStatus.FAILED);when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any())).thenReturn(late);doThrow(new IllegalStateException("port in use")).doNothing()@@ -2175,8 +2179,10 @@ class RdsServiceTest {// call retries both halves instead of returning a container nothing listens on.verify(containerManager).stop(late);assertNull(rdsService.getDbInstance("probe-db").getContainerId());+ assertEquals(DbInstanceStatus.FAILED, rdsService.getDbInstance("probe-db").getStatus());DbInstance started = rdsService.ensureInstanceBackend("probe-db", "us-east-1");+ assertEquals(DbInstanceStatus.AVAILABLE, started.getStatus());assertEquals("late-container", started.getContainerId());verify(containerManager, times(3)).tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());@@ -2192,12 +2198,18 @@ class RdsServiceTest {"admin", "password", "dbname", "db.serverless",0, false, null, null, "probe-cluster");+ cluster.setStatus(DbInstanceStatus.FAILED);+ member.setStatus(DbInstanceStatus.FAILED);+when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any())).thenReturn(new RdsContainerHandle("late-cluster-container", cluster.getDbClusterArn(),"probe-cluster", "127.0.0.1", 15432));DbInstance started = rdsService.ensureInstanceBackend("probe-member", "us-east-1");+ assertEquals(DbInstanceStatus.AVAILABLE, started.getStatus());+ assertEquals(DbInstanceStatus.AVAILABLE, rdsService.getDbCluster("probe-cluster").getStatus());+assertEquals("late-cluster-container", started.getContainerId());assertEquals("late-cluster-container",rdsService.getDbCluster("probe-cluster").getContainerId());@@ -3850,7 +3862,7 @@ class RdsServiceTest {restoredService.restorePersistedRuntime();DbInstance failed = restoredService.getDbInstance("broken");- assertEquals(DbInstanceStatus.FAILED, failed.getStatus());+ assertEquals(DbInstanceStatus.AVAILABLE, failed.getStatus());assertEquals(7000, failed.getEndpoint().port());assertEquals("restored-container", failed.getContainerId());assertNull(failed.getContainerHost());@@ -3894,8 +3906,11 @@ class RdsServiceTest {failedBoot.restorePersistedRuntime();for (String id : List.of("user", "auth", "oms")) {- assertEquals(DbInstanceStatus.FAILED, failedBoot.getDbInstance(id).getStatus());+ assertEquals(DbInstanceStatus.AVAILABLE, failedBoot.getDbInstance(id).getStatus());}+ assertEquals(Map.of("user", 7001, "auth", 7002, "oms", 7003),+ failedBoot.listDbInstances(null).stream().collect(Collectors.toMap(+ DbInstance::getDbInstanceIdentifier, instance -> instance.getEndpoint().port())));RdsContainerManager healthyContainerManager = mock(RdsContainerManager.class);when(healthyContainerManager.tryStart(@@ -3982,7 +3997,7 @@ class RdsServiceTest {restoredService.restorePersistedRuntime();DbInstance failed = restoredService.getDbInstance("broken");- assertEquals(DbInstanceStatus.FAILED, failed.getStatus());+ assertEquals(DbInstanceStatus.AVAILABLE, failed.getStatus());assertEquals(7000, failed.getEndpoint().port());assertEquals(7000, failed.getProxyPort());assertEquals("persisted-container", failed.getContainerId());@@ -4026,6 +4041,241 @@ class RdsServiceTest {assertEquals(0, failed.getProxyPort());}+ @Test+ void failedStartupBackendRetriesOnItsRetainedEndpoint() {+ InMemoryStorage<String, DbInstance> instances = new InMemoryStorage<>();+ instances.put("broken", persistedInstance("broken", "123456789012", "secret", 7000));+ RdsContainerManager manager = mock(RdsContainerManager.class);+ RdsProxyManager proxies = mock(RdsProxyManager.class);+ RdsContainerHandle recovered = new RdsContainerHandle("recovered-container",+ "arn:aws:rds:us-east-1:123456789012:db:broken", "broken", "127.0.0.1", 15432);+ when(manager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ .thenThrow(new IllegalStateException("temporary Docker error"))+ .thenReturn(recovered);+ RdsService service = newService(manager, proxies, instances, new InMemoryStorage<>(),+ new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>());+ service.restorePersistedRuntime();+ assertEquals(DbInstanceStatus.AVAILABLE, service.getDbInstance("broken").getStatus());+ assertEquals(7000, service.getDbInstance("broken").getEndpoint().port());+ assertEquals(7000, service.getDbInstance("broken").getProxyPort());+ DbInstance retried = service.ensureInstanceBackend("broken", "us-east-1");+ assertEquals(DbInstanceStatus.AVAILABLE, retried.getStatus());+ assertEquals("recovered-container", retried.getContainerId());+ assertNotNull(retried.getEndpoint());+ assertEquals(7000, retried.getProxyPort());+ verify(proxies).startProxy(any(), any(), anyBoolean(), eq(retried.getProxyPort()),+ eq("127.0.0.1"), eq(15432), eq(retried.getEndpoint().address()),+ any(), any(), any(), any(), any());+ }++ @Test+ void retryOfLegacyClusterWithoutAnEndpointRollsBackTheNewPortOnRelayFailure() {+ InMemoryStorage<String, DbCluster> clusters = new InMemoryStorage<>();+ DbCluster cluster = persistedCluster("123456789012", "secret", 7000);+ cluster.setStatus(DbInstanceStatus.FAILED);+ cluster.setEndpoint(null);+ cluster.setReaderEndpoint(null);+ cluster.setProxyPort(0);+ cluster.setContainerHost(null);+ cluster.setContainerPort(0);+ clusters.put("cluster1", cluster);+ RdsContainerManager manager = mock(RdsContainerManager.class);+ RdsProxyManager proxies = mock(RdsProxyManager.class);+ RdsContainerHandle recovered = new RdsContainerHandle("recovered-container",+ cluster.getDbClusterArn(), "cluster1", "127.0.0.1", 15432);+ when(manager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ .thenReturn(recovered);+ doThrow(new IllegalStateException("temporary relay error")).doNothing()+ .when(proxies).startProxy(any(), any(), anyBoolean(), anyInt(), any(), anyInt(),+ any(), any(), any(), any(), any(), any());+ RdsService service = newService(manager, proxies, new InMemoryStorage<>(), clusters,+ new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>());+ assertThrows(IllegalStateException.class, () -> service.ensureClusterBackend("cluster1", "us-east-1"));+ assertEquals(DbInstanceStatus.FAILED, service.getDbCluster("cluster1").getStatus());+ assertNull(service.getDbCluster("cluster1").getEndpoint());+ verify(manager).stop(recovered);+ DbCluster retried = service.ensureClusterBackend("cluster1", "us-east-1");+ assertEquals(DbInstanceStatus.AVAILABLE, retried.getStatus());+ assertEquals(7000, retried.getProxyPort());+ assertEquals(retried.getEndpoint(), retried.getReaderEndpoint());+ }++ @ParameterizedTest+ @CsvSource({"true", "false"})+ void stopAndStartRecoverAnInstanceWithRetainedOrLegacyMissingEndpoint(boolean legacy) {+ InMemoryStorage<String, DbInstance> instances = new InMemoryStorage<>();+ DbInstance persisted = persistedInstance("broken", "123456789012", "secret", 7000);+ instances.put("broken", persisted);+ RdsContainerManager manager = mock(RdsContainerManager.class);+ RdsProxyManager proxies = mock(RdsProxyManager.class);+ RdsContainerHandle recovered = new RdsContainerHandle("recovered-container",+ persisted.getDbInstanceArn(), "broken", "127.0.0.1", 15432);+ when(manager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ .thenThrow(new IllegalStateException("temporary Docker error"))+ .thenReturn(recovered);+ if (legacy) {+ persisted.setEndpoint(null);+ persisted.setProxyPort(0);+ doReturn(recovered).when(manager).tryStart(+ any(), any(), any(), any(), any(), any(), any(), any(), any());+ }+ RdsService service = newService(manager, proxies, instances, new InMemoryStorage<>(),+ new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>());+ if (legacy) {+ assertNull(service.getDbInstance("broken").getEndpoint());+ } else {+ service.restorePersistedRuntime();+ assertEquals(7000, service.getDbInstance("broken").getEndpoint().port());+ }+ service.stopDbInstance("broken", null);+ assertEquals(DbInstanceStatus.STARTING, service.startDbInstance("broken").getStatus());+ DbInstance started = service.getDbInstance("broken");+ assertEquals(DbInstanceStatus.AVAILABLE, started.getStatus());+ assertNotNull(started.getEndpoint());+ assertEquals(7000, started.getProxyPort());+ verify(proxies).startProxy(any(), any(), anyBoolean(), eq(7000), eq("127.0.0.1"),+ eq(15432), eq(started.getEndpoint().address()), any(), any(), any(), any(), any());+ }++ @ParameterizedTest+ @CsvSource({"true, true", "true, false", "false, true", "false, false"})+ void clusterControlPlaneRecoveryHandlesRetainedAndLegacyMissingEndpoints(boolean reboot, boolean legacy) {+ InMemoryStorage<String, DbCluster> clusters = new InMemoryStorage<>();+ InMemoryStorage<String, DbInstance> instances = new InMemoryStorage<>();+ DbCluster persisted = persistedCluster("123456789012", "secret", 7000);+ DbInstance member = persistedInstance("member", "123456789012", "secret", 7001);+ member.setDbClusterIdentifier("cluster1");+ persisted.getDbClusterMembers().add("member");+ instances.put("member", member);+ clusters.put("cluster1", persisted);+ RdsContainerManager manager = mock(RdsContainerManager.class);+ RdsProxyManager proxies = mock(RdsProxyManager.class);+ RdsContainerHandle recovered = new RdsContainerHandle("recovered-container",+ persisted.getDbClusterArn(), "cluster1", "127.0.0.1", 15432);+ when(manager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ .thenThrow(new IllegalStateException("temporary Docker error"))+ .thenReturn(recovered);+ if (legacy) {+ persisted.setEndpoint(null);+ persisted.setReaderEndpoint(null);+ persisted.setProxyPort(0);+ member.setEndpoint(null);+ member.setProxyPort(0);+ doReturn(recovered).when(manager).tryStart(+ any(), any(), any(), any(), any(), any(), any(), any(), any());+ }+ RdsService service = newService(manager, proxies, instances, clusters,+ new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>());+ if (legacy) {+ assertNull(service.getDbCluster("cluster1").getEndpoint());+ assertNull(service.getDbInstance("member").getEndpoint());+ } else {+ service.restorePersistedRuntime();+ assertEquals(7000, service.getDbCluster("cluster1").getEndpoint().port());+ assertEquals(7000, service.getDbCluster("cluster1").getReaderEndpoint().port());+ member = service.getDbInstance("member");+ assertEquals(7001, member.getEndpoint().port());+ }+ if (reboot) {+ service.rebootDbCluster("cluster1", "us-east-1");+ } else {+ service.stopDbCluster("cluster1", "us-east-1");+ service.startDbCluster("cluster1", "us-east-1");+ }+ DbCluster started = service.getDbCluster("cluster1");+ assertEquals(DbInstanceStatus.AVAILABLE, started.getStatus());+ assertNotNull(started.getEndpoint());+ assertEquals(7000, started.getEndpoint().port());+ assertEquals(started.getEndpoint(), started.getReaderEndpoint());+ verify(proxies).startProxy(eq("rds-resource:" + started.getDbClusterArn()),+ any(), anyBoolean(), eq(started.getProxyPort()),+ eq("127.0.0.1"), eq(15432), eq(started.getEndpoint().address()),+ any(), any(), any(), any(), any());+ assertEquals(DbInstanceStatus.AVAILABLE, member.getStatus());+ assertNotNull(member.getEndpoint());+ assertEquals(7001, member.getEndpoint().port());+ assertEquals("recovered-container", member.getContainerId());+ verify(proxies).startProxy(eq("rds-resource:" + member.getDbInstanceArn()),+ any(), anyBoolean(), eq(member.getProxyPort()), eq("127.0.0.1"), eq(15432),+ eq(member.getEndpoint().address()), any(), any(), any(), any(), any());+ }++ @Test+ void directClusterRetryRestoresAllMissingMemberRelaysAndRetriesFailures() {+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ .thenReturn(null);+ DbCluster cluster = rdsService.createDbCluster("retry-cluster", "aurora-postgresql", "16.3",+ "admin", "password", "dbname", false, null);+ DbInstance first = rdsService.createDbInstance("first-member", "aurora-postgresql", "16.3",+ "admin", "password", "dbname", "db.serverless", 0, false, null, null, "retry-cluster");+ DbInstance second = rdsService.createDbInstance("second-member", "aurora-postgresql", "16.3",+ "admin", "password", "dbname", "db.serverless", 0, false, null, null, "retry-cluster");+ DbInstance stopped = rdsService.createDbInstance("stopped-member", "aurora-postgresql", "16.3",+ "admin", "password", "dbname", "db.serverless", 0, false, null, null, "retry-cluster");+ stopped.setStatus(DbInstanceStatus.STOPPED);+ first.setEndpoint(null);+ first.setProxyPort(0);+ first.setStatus(DbInstanceStatus.FAILED);+ doThrow(new IllegalStateException("temporary member relay error")).doNothing()+ .when(proxyManager).startProxy(eq("rds-resource:" + first.getDbInstanceArn()),+ any(), anyBoolean(), anyInt(), any(), anyInt(), any(), any(), any(), any(), any(), any());+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ .thenReturn(new RdsContainerHandle("recovered-container", cluster.getDbClusterArn(),+ "retry-cluster", "127.0.0.1", 15432));+ assertDoesNotThrow(() -> rdsService.ensureClusterBackend("retry-cluster", "us-east-1"));+ assertNull(first.getContainerHost());+ assertNull(first.getEndpoint());+ assertEquals(DbInstanceStatus.FAILED, first.getStatus());+ assertEquals("recovered-container", second.getContainerId());+ rdsService.ensureClusterBackend("retry-cluster", "us-east-1");+ assertEquals(DbInstanceStatus.AVAILABLE, first.getStatus());+ assertNotNull(first.getEndpoint());+ assertEquals("recovered-container", first.getContainerId());+ assertEquals("recovered-container", second.getContainerId());+ rdsService.ensureClusterBackend("retry-cluster", "us-east-1");+ assertEquals(DbInstanceStatus.STOPPED, stopped.getStatus());+ verify(proxyManager, never()).startProxy(eq("rds-resource:" + stopped.getDbInstanceArn()),+ any(), anyBoolean(), anyInt(), any(), anyInt(), any(), any(), any(), any(), any(), any());+ verify(proxyManager).startProxy(eq("rds-resource:" + second.getDbInstanceArn()),+ any(), anyBoolean(), eq(second.getProxyPort()), eq("127.0.0.1"), eq(15432),+ any(), any(), any(), any(), any(), any());+ verify(containerManager, times(2)).tryStart(eq(cluster.getDbClusterArn()), any(), any(),+ any(), any(), any(), any(), any(), any());+ }++ @ParameterizedTest+ @CsvSource({"true", "false"})+ void clusterControlPlaneReportsMemberFailureAfterTryingTheRemainingMembers(boolean reboot) {+ DbCluster cluster = rdsService.createDbCluster("control-cluster", "aurora-postgresql", "16.3",+ "admin", "password", "dbname", false, null);+ DbInstance first = rdsService.createDbInstance("control-first", "aurora-postgresql", "16.3",+ "admin", "password", "dbname", "db.serverless", 0, false, null, null, "control-cluster");+ DbInstance second = rdsService.createDbInstance("control-second", "aurora-postgresql", "16.3",+ "admin", "password", "dbname", "db.serverless", 0, false, null, null, "control-cluster");+ doThrow(new IllegalStateException("temporary member relay error")).doNothing()+ .when(proxyManager).startProxy(eq("rds-resource:" + first.getDbInstanceArn()),+ any(), anyBoolean(), anyInt(), any(), anyInt(), any(), any(), any(), any(), any(), any());+ if (reboot) {+ assertThrows(IllegalStateException.class,+ () -> rdsService.rebootDbCluster("control-cluster", "us-east-1"));+ } else {+ rdsService.stopDbCluster("control-cluster", "us-east-1");+ assertThrows(IllegalStateException.class,+ () -> rdsService.startDbCluster("control-cluster", "us-east-1"));+ }+ assertEquals(DbInstanceStatus.FAILED, first.getStatus());+ assertNull(first.getContainerHost());+ assertEquals(DbInstanceStatus.AVAILABLE, second.getStatus());+ assertNotNull(second.getContainerHost());+ assertEquals(DbInstanceStatus.AVAILABLE, cluster.getStatus(), "the cluster relay itself is ready");+ rdsService.ensureClusterBackend("control-cluster", "us-east-1");+ assertEquals(DbInstanceStatus.AVAILABLE, first.getStatus());+ verify(proxyManager, times(2)).startProxy(eq("rds-resource:" + second.getDbInstanceArn()),+ any(), anyBoolean(), anyInt(), any(), anyInt(), any(), any(), any(), any(), any(), any());+ verify(containerManager, times(2)).tryStart(eq(cluster.getDbClusterArn()), any(), any(),+ any(), any(), any(), any(), any(), any());+ }+@Testvoid failedClusterRestoreClearsRuntimeStateAndKeepsItsEndpointPort() {InMemoryStorage<String, DbCluster> clusters = new InMemoryStorage<>();@@ -4059,7 +4309,7 @@ class RdsServiceTest {restoredService.restorePersistedRuntime();DbCluster failed = restoredService.getDbCluster("cluster1");- assertEquals(DbInstanceStatus.FAILED, failed.getStatus());+ assertEquals(DbInstanceStatus.AVAILABLE, failed.getStatus());assertEquals(7000, failed.getEndpoint().port());assertEquals(7000, failed.getReaderEndpoint().port());assertEquals("restored-container", failed.getContainerId());diff --git a/src/test/java/io/github/hectorvent/floci/services/rdsdata/RdsDataResourceResolverTest.java b/src/test/java/io/github/hectorvent/floci/services/rdsdata/RdsDataResourceResolverTest.javaindex 4c20a0b460..1f61f94085 100644--- a/src/test/java/io/github/hectorvent/floci/services/rdsdata/RdsDataResourceResolverTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/rdsdata/RdsDataResourceResolverTest.java@@ -1,22 +1,46 @@package io.github.hectorvent.floci.services.rdsdata;+import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsException;+import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;+import io.github.hectorvent.floci.core.storage.StorageFactory;+import io.github.hectorvent.floci.services.ec2.Ec2Service;import io.github.hectorvent.floci.services.rds.RdsService;+import io.github.hectorvent.floci.services.rds.container.RdsContainerManager;import io.github.hectorvent.floci.services.rds.model.DatabaseEngine;import io.github.hectorvent.floci.services.rds.model.DbCluster;import io.github.hectorvent.floci.services.rds.model.DbEndpoint;import io.github.hectorvent.floci.services.rds.model.DbInstance;import io.github.hectorvent.floci.services.rds.model.DbInstanceStatus;+import io.github.hectorvent.floci.services.rds.proxy.RdsProxyManager;import org.junit.jupiter.api.Test;+import org.junit.jupiter.params.ParameterizedTest;+import org.junit.jupiter.params.provider.ValueSource;import java.time.Instant;import java.util.ArrayList;import java.util.List;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;+import java.util.concurrent.TimeUnit;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertThrows;import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.ArgumentMatchers.anyBoolean;+import static org.mockito.ArgumentMatchers.anyInt;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.ArgumentMatchers.eq;+import static org.mockito.Mockito.doThrow;import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.never;+import static org.mockito.Mockito.spy;+import static org.mockito.Mockito.times;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.verifyNoInteractions;import static org.mockito.Mockito.when;class RdsDataResourceResolverTest {@@ -39,6 +63,7 @@ class RdsDataResourceResolverTest {assertEquals("127.0.0.1", target.host());assertEquals(3306, target.port());assertEquals("app", target.databaseName());+ verify(rdsService, never()).ensureClusterBackend(anyString(), anyString());}@Test@@ -109,6 +134,146 @@ class RdsDataResourceResolverTest {assertEquals(5432, target.port());}+ @Test+ void repeatedClusterArnResolutionRetriesFailedMembersOfAnAlreadyRunningCluster() {+ ClusterFixture fixture = readyClusterFixture();+ DbCluster cluster = fixture.cluster();+ AccountAwareStorageBackend<DbInstance> instances = fixture.instances();+ RdsProxyManager proxies = fixture.proxies();+ RdsService rdsService = fixture.service();+ cluster.getDbClusterMembers().addAll(List.of("retry-member", "other-member", "stopped-member"));+ DbInstance retryMember = clusterMember("retry-member", 7002, DbInstanceStatus.FAILED);+ DbInstance otherMember = clusterMember("other-member", 7003, DbInstanceStatus.AVAILABLE);+ DbInstance stoppedMember = clusterMember("stopped-member", 7004, DbInstanceStatus.STOPPED);+ instances.put("retry-member", retryMember);+ instances.put("other-member", otherMember);+ instances.put("stopped-member", stoppedMember);+ doThrow(new IllegalStateException("temporary member relay error")).doNothing()+ .when(proxies).startProxy(eq("rds-resource:" + retryMember.getDbInstanceArn()),+ any(), anyBoolean(), anyInt(), any(), anyInt(), any(), any(), any(), any(), any(), any());+ RdsDataResourceResolver resolver = new RdsDataResourceResolver(rdsService);++ RdsDataResourceResolver.DatabaseTarget firstTarget = resolver.resolve(cluster.getDbClusterArn(), "us-west-2");++ assertEquals("127.0.0.1", firstTarget.host());+ assertEquals(15432, firstTarget.port());+ assertEquals(DbInstanceStatus.FAILED, retryMember.getStatus());+ assertEquals("cluster-container", otherMember.getContainerId());++ assertEquals(firstTarget, resolver.resolve(cluster.getDbClusterArn(), "us-west-2"));+ assertEquals(DbInstanceStatus.AVAILABLE, retryMember.getStatus());+ assertEquals("cluster-container", retryMember.getContainerId());+ assertEquals("127.0.0.1", retryMember.getContainerHost());+ assertEquals(15432, retryMember.getContainerPort());+ assertEquals(7002, retryMember.getEndpoint().port());+ assertEquals(firstTarget, resolver.resolve(cluster.getDbClusterArn(), "us-west-2"));+ assertEquals(DbInstanceStatus.STOPPED, stoppedMember.getStatus());+ verify(rdsService, times(2)).ensureClusterBackend("cluster1", "us-west-2");+ verify(proxies, times(2)).startProxy(eq("rds-resource:" + retryMember.getDbInstanceArn()),+ any(), anyBoolean(), eq(7002), eq("127.0.0.1"), eq(15432), any(), any(), any(), any(), any(), any());+ verify(proxies).startProxy(eq("rds-resource:" + otherMember.getDbInstanceArn()),+ any(), anyBoolean(), eq(7003), eq("127.0.0.1"), eq(15432), any(), any(), any(), any(), any(), any());+ verify(proxies, never()).startProxy(eq("rds-resource:" + stoppedMember.getDbInstanceArn()),+ any(), anyBoolean(), anyInt(), any(), anyInt(), any(), any(), any(), any(), any(), any());+ verify(proxies, never()).startProxy(eq("rds-resource:" + cluster.getDbClusterArn()),+ any(), anyBoolean(), anyInt(), any(), anyInt(), any(), any(), any(), any(), any(), any());+ verifyNoInteractions(fixture.containers());+ }++ @Test+ void healthyClusterResolutionDoesNotWaitForTheServiceMonitor() throws Exception {+ ClusterFixture fixture = readyClusterFixture();+ DbInstance readyMember = clusterMember("ready-member", 7002, DbInstanceStatus.AVAILABLE);+ readyMember.setContainerHost("127.0.0.1");+ readyMember.setContainerPort(15432);+ fixture.instances().put("ready-member", readyMember);+ fixture.instances().put("stopped-member", clusterMember("stopped-member", 7003, DbInstanceStatus.STOPPED));+ fixture.instances().put("deleting-member", clusterMember("deleting-member", 7004, DbInstanceStatus.DELETING));+ fixture.cluster().getDbClusterMembers().addAll(+ List.of("ready-member", "stopped-member", "deleting-member", "removed-member"));+ RdsDataResourceResolver resolver = new RdsDataResourceResolver(fixture.service());++ try (ExecutorService executor = Executors.newSingleThreadExecutor()) {+ // Container starts on unrelated databases hold this monitor while pulling images.+ synchronized (fixture.service()) {+ Future<RdsDataResourceResolver.DatabaseTarget> resolved = executor.submit(() ->+ resolver.resolve(fixture.cluster().getDbClusterArn(), "us-west-2"));+ RdsDataResourceResolver.DatabaseTarget target = resolved.get(5, TimeUnit.SECONDS);+ assertEquals("127.0.0.1", target.host());+ assertEquals(15432, target.port());+ }+ }++ verify(fixture.service(), never()).ensureClusterBackend(anyString(), anyString());+ verifyNoInteractions(fixture.containers(), fixture.proxies());+ }++ @ParameterizedTest+ @ValueSource(booleans = {false, true})+ void activeMemberWithAnIncompleteBackendIsRetried(boolean blankHost) {+ ClusterFixture fixture = readyClusterFixture();+ DbInstance member = clusterMember("retry-member", 7002, DbInstanceStatus.AVAILABLE);+ member.setContainerHost(blankHost ? " " : "127.0.0.1");+ member.setContainerPort(blankHost ? 15432 : 0);+ fixture.instances().put("retry-member", member);+ fixture.cluster().getDbClusterMembers().add("retry-member");++ RdsDataResourceResolver.DatabaseTarget target = new RdsDataResourceResolver(fixture.service())+ .resolve(fixture.cluster().getDbClusterArn(), "us-west-2");++ assertEquals(15432, target.port());+ assertEquals("127.0.0.1", member.getContainerHost());+ assertEquals(15432, member.getContainerPort());+ verify(fixture.service()).ensureClusterBackend("cluster1", "us-west-2");+ verify(fixture.proxies()).startProxy(eq("rds-resource:" + member.getDbInstanceArn()),+ any(), anyBoolean(), eq(7002), eq("127.0.0.1"), eq(15432), any(), any(), any(), any(), any(), any());+ verifyNoInteractions(fixture.containers());+ }++ private static ClusterFixture readyClusterFixture() {+ EmulatorConfig config = mock(EmulatorConfig.class);+ EmulatorConfig.ServicesConfig servicesConfig = mock(EmulatorConfig.ServicesConfig.class);+ EmulatorConfig.RdsServiceConfig rdsConfig = mock(EmulatorConfig.RdsServiceConfig.class);+ when(config.services()).thenReturn(servicesConfig);+ when(config.defaultAccountId()).thenReturn("000000000000");+ when(servicesConfig.rds()).thenReturn(rdsConfig);+ when(rdsConfig.iamTokenEndpointBinding()).thenReturn(true);+ AccountAwareStorageBackend<DbCluster> clusters = AccountAwareStorageBackend.inMemory("000000000000");+ AccountAwareStorageBackend<DbInstance> instances = AccountAwareStorageBackend.inMemory("000000000000");+ StorageFactory storageFactory = mock(StorageFactory.class);+ when(storageFactory.create(anyString(), anyString(), any()))+ .thenAnswer(invocation -> AccountAwareStorageBackend.inMemory("000000000000"));+ when(storageFactory.<DbCluster>create(eq("rds"), eq("rds-clusters.json"), any())).thenReturn(clusters);+ when(storageFactory.<DbInstance>create(eq("rds"), eq("rds-instances.json"), any())).thenReturn(instances);+ DbCluster cluster = new DbCluster("cluster1", DatabaseEngine.POSTGRES, "16.3", "admin", "secret",+ "app", DbInstanceStatus.AVAILABLE, new DbEndpoint("localhost", 7001),+ new DbEndpoint("localhost", 7001), false, new ArrayList<>(), null, Instant.now(), 7001);+ cluster.setDbClusterArn("arn:aws:rds:us-west-2:000000000000:cluster:cluster1");+ cluster.setContainerId("cluster-container");+ cluster.setContainerHost("127.0.0.1");+ cluster.setContainerPort(15432);+ clusters.put("cluster1", cluster);+ RdsContainerManager containers = mock(RdsContainerManager.class);+ RdsProxyManager proxies = mock(RdsProxyManager.class);+ RdsService rdsService = spy(new RdsService(containers, proxies, mock(Ec2Service.class),+ new RegionResolver("us-east-1", "000000000000"), config, storageFactory,+ null, null, null, null, null, null));+ return new ClusterFixture(rdsService, cluster, instances, containers, proxies);+ }++ private record ClusterFixture(RdsService service, DbCluster cluster,+ AccountAwareStorageBackend<DbInstance> instances,+ RdsContainerManager containers, RdsProxyManager proxies) { }++ private static DbInstance clusterMember(String id, int port, DbInstanceStatus status) {+ DbInstance instance = new DbInstance(id, DatabaseEngine.POSTGRES, "16.3", "admin", "secret",+ "app", "db.serverless", 0, status, new DbEndpoint("localhost", port),+ false, null, null, Instant.now(), port);+ instance.setDbInstanceArn("arn:aws:rds:us-west-2:000000000000:db:" + id);+ instance.setDbClusterIdentifier("cluster1");+ return instance;+ }+private static DbInstance daemonlessInstance() {DbInstance instance = new DbInstance("probe-db", DatabaseEngine.POSTGRES, "16.3", "admin","secret", "app", "db.t3.micro", 20, DbInstanceStatus.AVAILABLE,