Light, fluffy, and always free - The AWS Local Emulator alternative
fix(rds): keep the control plane working with no Docker daemon (#3170)
lex0049dfd1b18abb
diff --git a/docs/services/rds.md b/docs/services/rds.mdindex 8c9bbe61a6..a2448ec8e9 100644--- a/docs/services/rds.md+++ b/docs/services/rds.md@@ -107,6 +107,21 @@ services:FLOCI_SERVICES_RDS_PROXY_BASE_PORT: "7001"```+### Without a reachable Docker daemon++A DB instance or cluster record is metadata. Its identifier, ARN, endpoint address and tags come+from Floci's configuration, not from Docker. When no daemon is reachable, because Floci runs inside+Docker with no socket mounted or the daemon on the host is stopped, `CreateDBInstance` and+`CreateDBCluster` still succeed and the resource reaches `available`. `DescribeDBInstances`,+`ModifyDBInstance`, the tagging APIs and `DeleteDBInstance` all work on that record, and Floci logs+a warning naming the missing daemon.++Nothing listens behind the endpoint in that state. The backing container is retried by every+operation that needs the live database, so it starts as soon as a daemon becomes reachable. Until+then, RDS Data API calls fail with a modelled `InternalServerErrorException` that names the missing+daemon. A daemon that is reachable but cannot start the container still fails `CreateDBInstance`+outright, since that is a real error rather than a degraded mode.+### Mock mode (CI / tests)Set `FLOCI_SERVICES_RDS_MOCK=true` when you only need the management API shape: clusters anddiff --git a/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java b/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.javaindex f2cf7bb226..c8c05bba5e 100644--- a/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java@@ -618,20 +618,28 @@ public class RdsService implements Resettable, ResourceProvider {} else {placement = resolvePlacement(dbSubnetGroupName, availabilityZone, multiAz, effectiveRegion);if (!mock) {- // Standalone instance — start its own container+ // Standalone instance, so it starts its own container. The record itself is metadata:+ // identifier, ARN, endpoint and tags come from configuration, so the instance is+ // created and reaches 'available' even when no Docker daemon is reachable.String image = imageForEngine(engine, engineVersion);instanceVolumeId = String.format("%06x", new SecureRandom().nextInt(0xFFFFFF));instanceDockerVolumeName = newVolumeName(instanceVolumeId, instanceStorageResourceId);- RdsContainerHandle handle = containerManager.start(+ RdsContainerHandle handle = containerManager.tryStart(dbInstanceArn, id, instanceStorageResourceId,instanceDockerVolumeName, engine, image,masterUsername, masterPassword, dbName);- backendHost = handle.getHost();- backendPort = handle.getPort();- containerId = handle.getContainerId();- containerHost = handle.getHost();- containerPort = handle.getPort();+ if (handle != null) {+ backendHost = handle.getHost();+ backendPort = handle.getPort();+ containerId = handle.getContainerId();+ containerHost = handle.getHost();+ containerPort = handle.getPort();+ } else {+ LOG.warnv("DB instance {0} created without a backing database container: no "+ + "Docker daemon is reachable. Metadata operations work; connections to "+ + "the database do not until a daemon appears.", id);+ }}}@@ -664,7 +672,7 @@ public class RdsService implements Resettable, ResourceProvider {attachManagedMasterUserSecret(instance, effectiveRegion, masterUserSecretKmsKeyId);}- if (!mock) {+ if (!mock && hasBackend(backendHost, backendPort)) {final String accountId = accountIdFromArn(instance.getDbInstanceArn());final String instanceRegion = regionFromArn(instance.getDbInstanceArn());proxyManager.startProxy(rdsResourceRelayKey(instance.getDbInstanceArn(), id),@@ -1374,15 +1382,15 @@ public class RdsService implements Resettable, ResourceProvider {String image = imageForEngine(instance.getEngine(), instance.getEngineVersion());String storageResourceId = resolvedInstanceStorageResourceId(instance);String dockerVolumeName = resolvedInstanceDockerVolumeName(instance);- RdsContainerHandle handle = containerManager.start(+ RdsContainerHandle handle = containerManager.tryStart(instance.getDbInstanceArn(), id, storageResourceId,dockerVolumeName, instance.getEngine(), image, instance.getMasterUsername(),instance.getMasterPassword(), instance.getDbName());instance.setContainerStorageResourceId(storageResourceId);instance.setDockerVolumeName(dockerVolumeName);- instance.setContainerId(handle.getContainerId());- instance.setContainerHost(handle.getHost());- instance.setContainerPort(handle.getPort());+ instance.setContainerId(handle != null ? handle.getContainerId() : null);+ instance.setContainerHost(handle != null ? handle.getHost() : null);+ instance.setContainerPort(handle != null ? handle.getPort() : 0);}}@@ -1390,24 +1398,190 @@ public class RdsService implements Resettable, ResourceProvider {putInstanceForScope(currentAccountId(), effectiveRegion, id, instance);if (!mock) {- String effectiveMasterUser = instance.getMasterUsername() != null- ? instance.getMasterUsername() : "root";- final String accountId = accountIdFromArn(instance.getDbInstanceArn());- final String instanceRegion = regionFromArn(instance.getDbInstanceArn());+ if (hasBackend(instance.getContainerHost(), instance.getContainerPort())) {+ String effectiveMasterUser = instance.getMasterUsername() != null+ ? instance.getMasterUsername() : "root";+ final String accountId = accountIdFromArn(instance.getDbInstanceArn());+ final String instanceRegion = regionFromArn(instance.getDbInstanceArn());+ proxyManager.startProxy(rdsResourceRelayKey(instance.getDbInstanceArn(), id),+ instance.getEngine(),+ instance.isIamDatabaseAuthenticationEnabled(),+ instance.getProxyPort(), instance.getContainerHost(), instance.getContainerPort(),+ instance.getEndpoint().address(),+ effectiveMasterUser, instance.getMasterPassword(), instance.getDbName(),+ (user, pw) -> validateDbPasswordForScope(+ accountId, instanceRegion, id, user, pw));+ } else {+ // No backing container: created or last rebooted while no daemon was reachable.+ instance = ensureInstanceBackend(id, effectiveRegion);+ }+ }++ LOG.infov("DB instance {0} rebooted", id);+ return instance;+ }++ private static boolean hasBackend(String host, int port) {+ return host != null && !host.isBlank() && port > 0;+ }++ /**+ * Whether a delete has anything for Docker to clean up. A record with a container, or with a+ * cleanup identity retained from an earlier failure, always has. One with neither was created+ * or restored while no daemon was reachable, and only a reachable daemon could remove whatever+ * might still exist for it.+ */+ private boolean backendCleanupPossible(String containerId, String runtimeArn) {+ return containerId != null+ || containerManager.getActiveHandle(runtimeArn) != null+ || containerManager.isDockerReachable();+ }++ /**+ * Starts the backing database container and auth proxy for a DB instance recorded without+ * one, because no Docker daemon was reachable when it was created, rebooted or restored.+ * Every operation that needs the live database calls this first, so the backend comes up as+ * soon as a daemon appears. Instances that already have a backend, and mock-mode instances,+ * are returned untouched. The record is only updated once both halves are up, so a proxy+ * failure leaves it retrying next time instead of pointing at a container nothing listens on.+ *+ * @return the instance, with its container fields populated when a backend became available+ */+ public synchronized DbInstance ensureInstanceBackend(String id, String region) {+ String effectiveRegion = effectiveRegion(region);+ DbInstance instance = getDbInstance(id, effectiveRegion);+ if (config.services().rds().mock()+ || hasBackend(instance.getContainerHost(), instance.getContainerPort())) {+ return instance;+ }++ String clusterId = instance.getDbClusterIdentifier();+ RdsContainerHandle started = null;+ String backendContainerId;+ String backendHost;+ int backendPort;+ if (clusterId != null && !clusterId.isBlank()) {+ DbCluster cluster = ensureClusterBackend(clusterId, effectiveRegion);+ if (!hasBackend(cluster.getContainerHost(), cluster.getContainerPort())) {+ return instance;+ }+ backendContainerId = cluster.getContainerId();+ backendHost = cluster.getContainerHost();+ backendPort = cluster.getContainerPort();+ } else {+ String image = imageForEngine(instance.getEngine(), instance.getEngineVersion());+ started = containerManager.tryStart(+ instance.getDbInstanceArn(), id, resolvedInstanceStorageResourceId(instance),+ resolvedInstanceDockerVolumeName(instance), instance.getEngine(), image,+ instance.getMasterUsername(), instance.getMasterPassword(), instance.getDbName());+ if (started == null) {+ return instance;+ }+ backendContainerId = started.getContainerId();+ backendHost = started.getHost();+ backendPort = started.getPort();+ }++ String effectiveMasterUser = instance.getMasterUsername() != null+ ? instance.getMasterUsername() : "root";+ final String accountId = accountIdFromArn(instance.getDbInstanceArn());+ final String instanceRegion = regionFromArn(instance.getDbInstanceArn());+ try {proxyManager.startProxy(rdsResourceRelayKey(instance.getDbInstanceArn(), id),- instance.getEngine(),- instance.isIamDatabaseAuthenticationEnabled(),- instance.getProxyPort(), instance.getContainerHost(), instance.getContainerPort(),+ instance.getEngine(), instance.isIamDatabaseAuthenticationEnabled(),+ instance.getProxyPort(), backendHost, backendPort,instance.getEndpoint().address(),effectiveMasterUser, instance.getMasterPassword(), instance.getDbName(),(user, pw) -> validateDbPasswordForScope(accountId, instanceRegion, id, user, pw));+ } catch (RuntimeException e) {+ stopStartedBackend(started, e);+ throw e;}-- LOG.infov("DB instance {0} rebooted", id);+ if (started != null) {+ instance.setContainerStorageResourceId(resolvedInstanceStorageResourceId(instance));+ instance.setDockerVolumeName(resolvedInstanceDockerVolumeName(instance));+ }+ instance.setContainerId(backendContainerId);+ instance.setContainerHost(backendHost);+ instance.setContainerPort(backendPort);+ putInstanceForScope(currentAccountId(), effectiveRegion, id, instance);+ LOG.infov("Backing database container for DB instance {0} started on retry", id);return instance;}+ /**+ * The {@link #ensureInstanceBackend} counterpart for DB clusters.+ *+ * @return the cluster, with its container fields populated when a backend became available+ */+ public synchronized DbCluster ensureClusterBackend(String id, String region) {+ String effectiveRegion = effectiveRegion(region);+ DbCluster cluster = getDbCluster(id, effectiveRegion);+ if (config.services().rds().mock()+ || hasBackend(cluster.getContainerHost(), cluster.getContainerPort())) {+ return cluster;+ }++ String image = imageForEngine(cluster.getEngine(), cluster.getEngineVersion());+ String storageResourceId = resolvedClusterStorageResourceId(cluster);+ String dockerVolumeName = resolvedClusterDockerVolumeName(cluster);+ RdsContainerHandle started = containerManager.tryStart(+ cluster.getDbClusterArn(), id, storageResourceId, dockerVolumeName,+ cluster.getEngine(), image, cluster.getMasterUsername(),+ cluster.getMasterPassword(), cluster.getDatabaseName());+ if (started == null) {+ return cluster;+ }++ String effectiveMasterUser = cluster.getMasterUsername() != null+ ? cluster.getMasterUsername() : "root";+ final String accountId = accountIdFromArn(cluster.getDbClusterArn());+ final String clusterRegion = regionFromArn(cluster.getDbClusterArn());+ try {+ proxyManager.startProxy(rdsResourceRelayKey(cluster.getDbClusterArn(), id),+ cluster.getEngine(), cluster.isIamDatabaseAuthenticationEnabled(),+ cluster.getProxyPort(), started.getHost(), started.getPort(),+ cluster.getEndpoint().address(),+ effectiveMasterUser, cluster.getMasterPassword(), cluster.getDatabaseName(),+ (user, pw) -> validateDbClusterPasswordForScope(+ accountId, clusterRegion, id, user, pw));+ } catch (RuntimeException e) {+ stopStartedBackend(started, e);+ throw e;+ }+ cluster.setContainerStorageResourceId(storageResourceId);+ cluster.setDockerVolumeName(dockerVolumeName);+ cluster.setContainerId(started.getContainerId());+ cluster.setContainerHost(started.getHost());+ cluster.setContainerPort(started.getPort());+ putClusterForScope(currentAccountId(), effectiveRegion, id, cluster);+ LOG.infov("Backing database container for DB cluster {0} started on retry", id);+ return cluster;+ }++ private void stopStartedBackend(RdsContainerHandle started, RuntimeException proxyFailure) {+ if (started == null) {+ return;+ }+ try {+ containerManager.stop(started);+ } catch (RuntimeException | Error stopFailure) {+ proxyFailure.addSuppressed(stopFailure);+ LOG.warnv(stopFailure, "Failed to stop container {0} after its auth proxy did not start",+ started.getContainerId());+ }+ }++ /**+ * Whether Floci can reach a Docker daemon at all. The RDS data plane is a real database+ * connection, which cannot be emulated without one, so callers use this to raise a modelled+ * error naming the missing daemon instead of a generic runtime failure.+ */+ public boolean isBackendRuntimeAvailable() {+ return config.services().rds().mock() || containerManager.isDockerReachable();+ }+public synchronized void deleteDbInstance(String id) {deleteDbInstance(id, regionResolver.getDefaultRegion());}@@ -1435,8 +1609,11 @@ public class RdsService implements Resettable, ResourceProvider {String clusterId = instance.getDbClusterIdentifier();if (clusterId == null || clusterId.isBlank()) {- // Standalone — stop its container and clean up its Docker volume (neither exists in mock mode)- if (!mock) {+ // Standalone, so stop its container and clean up its Docker volume. Neither exists in+ // mock mode, and an instance with no container and no reachable daemon has nothing+ // Docker could clean up, so its delete stays pure metadata.+ if (!mock && backendCleanupPossible(+ instance.getContainerId(), instance.getDbInstanceArn())) {if (instance.getContainerId() != null) {containerManager.stop(buildHandle(instance));} else {@@ -1585,13 +1762,19 @@ public class RdsService implements Resettable, ResourceProvider {String clusterVolumeId = String.format("%06x", new SecureRandom().nextInt(0xFFFFFF));String clusterDockerVolumeName = newVolumeName(clusterVolumeId, clusterResourceId);- RdsContainerHandle handle = containerManager.start(+ RdsContainerHandle handle = containerManager.tryStart(clusterArn, id, clusterResourceId, clusterDockerVolumeName,engine, image,masterUsername, masterPassword, databaseName);- cluster.setContainerId(handle.getContainerId());- cluster.setContainerHost(handle.getHost());- cluster.setContainerPort(handle.getPort());+ if (handle != null) {+ cluster.setContainerId(handle.getContainerId());+ cluster.setContainerHost(handle.getHost());+ cluster.setContainerPort(handle.getPort());+ } else {+ LOG.warnv("DB cluster {0} created without a backing database container: no Docker "+ + "daemon is reachable. Metadata operations work; connections to the "+ + "database do not until a daemon appears.", id);+ }cluster.setVolumeId(clusterVolumeId);cluster.setDockerVolumeName(clusterDockerVolumeName);}@@ -1609,7 +1792,7 @@ public class RdsService implements Resettable, ResourceProvider {}try {- if (!mock) {+ if (!mock && hasBackend(cluster.getContainerHost(), cluster.getContainerPort())) {String effectiveMasterUser = masterUsername != null ? masterUsername : "root";final String accountId = accountIdFromArn(cluster.getDbClusterArn());final String clusterRegion = regionFromArn(cluster.getDbClusterArn());@@ -1919,15 +2102,17 @@ public class RdsService implements Resettable, ResourceProvider {if (!config.services().rds().mock()) {proxyManager.stopProxy(rdsResourceRelayKey(cluster.getDbClusterArn(), id));- if (cluster.getContainerId() != null) {- containerManager.stop(buildClusterHandle(cluster));- } else {- containerManager.stopByRuntimeId(cluster.getDbClusterArn());+ if (backendCleanupPossible(cluster.getContainerId(), cluster.getDbClusterArn())) {+ if (cluster.getContainerId() != null) {+ containerManager.stop(buildClusterHandle(cluster));+ } else {+ containerManager.stopByRuntimeId(cluster.getDbClusterArn());+ }+ containerManager.removeVolume(+ cluster.getDbClusterArn(),+ resolvedClusterStorageResourceId(cluster),+ resolvedClusterDockerVolumeName(cluster));}- containerManager.removeVolume(- cluster.getDbClusterArn(),- resolvedClusterStorageResourceId(cluster),- resolvedClusterDockerVolumeName(cluster));}releaseProxyPort(cluster.getProxyPort());@@ -4246,25 +4431,29 @@ public class RdsService implements Resettable, ResourceProvider {cluster.setEndpoint(endpoint);cluster.setReaderEndpoint(endpoint);String image = imageForEngine(cluster.getEngine(), cluster.getEngineVersion());- restoredHandle = containerManager.start(+ restoredHandle = containerManager.tryStart(cluster.getDbClusterArn(), cluster.getDbClusterIdentifier(),storageResourceId, dockerVolumeName, cluster.getEngine(), image,cluster.getMasterUsername(), cluster.getMasterPassword(), cluster.getDatabaseName());- cluster.setContainerId(restoredHandle.getContainerId());- cluster.setContainerHost(restoredHandle.getHost());- cluster.setContainerPort(restoredHandle.getPort());-- String effectiveMasterUser = cluster.getMasterUsername() != null- ? cluster.getMasterUsername() : "root";- proxyManager.startProxy(rdsResourceRelayKey(- cluster.getDbClusterArn(), cluster.getDbClusterIdentifier()),- cluster.getEngine(),- cluster.isIamDatabaseAuthenticationEnabled(), proxyPort,- restoredHandle.getHost(), restoredHandle.getPort(), cluster.getEndpoint().address(),- effectiveMasterUser, cluster.getMasterPassword(), cluster.getDatabaseName(),- (user, pw) -> validateDbClusterPasswordForScope(- accountId, clusterRegion,- cluster.getDbClusterIdentifier(), user, pw));+ cluster.setContainerId(restoredHandle != null ? restoredHandle.getContainerId() : null);+ cluster.setContainerHost(restoredHandle != null ? restoredHandle.getHost() : null);+ cluster.setContainerPort(restoredHandle != null ? restoredHandle.getPort() : 0);++ // With no reachable daemon the record survives the restart without a container;+ // the container is retried the next time something needs the live database.+ if (restoredHandle != null) {+ String effectiveMasterUser = cluster.getMasterUsername() != null+ ? cluster.getMasterUsername() : "root";+ proxyManager.startProxy(rdsResourceRelayKey(+ cluster.getDbClusterArn(), cluster.getDbClusterIdentifier()),+ cluster.getEngine(),+ cluster.isIamDatabaseAuthenticationEnabled(), proxyPort,+ restoredHandle.getHost(), restoredHandle.getPort(), cluster.getEndpoint().address(),+ effectiveMasterUser, cluster.getMasterPassword(), cluster.getDatabaseName(),+ (user, pw) -> validateDbClusterPasswordForScope(+ accountId, clusterRegion,+ cluster.getDbClusterIdentifier(), user, pw));+ }cluster.setStatus(DbInstanceStatus.AVAILABLE);putClusterForScope(accountId, clusterRegion,cluster.getDbClusterIdentifier(), cluster);@@ -4354,7 +4543,10 @@ public class RdsService implements Resettable, ResourceProvider {}backendHost = cluster.getContainerHost();backendPort = cluster.getContainerPort();- if (backendHost == null || backendPort <= 0) {+ // A cluster restored 'available' without a container had no reachable daemon;+ // its members share that state and are retried together with it.+ if (!hasBackend(backendHost, backendPort)+ && cluster.getStatus() != DbInstanceStatus.AVAILABLE) {throw new AwsException("InvalidDBClusterStateFault","DB cluster " + clusterId + " runtime is not available.", 400);}@@ -4363,29 +4555,31 @@ public class RdsService implements Resettable, ResourceProvider {instance.setContainerPort(cluster.getContainerPort());} else {String image = imageForEngine(instance.getEngine(), instance.getEngineVersion());- restoredHandle = containerManager.start(+ restoredHandle = containerManager.tryStart(instance.getDbInstanceArn(), instance.getDbInstanceIdentifier(),instance.getContainerStorageResourceId(),instance.getDockerVolumeName(), instance.getEngine(), image,instance.getMasterUsername(), instance.getMasterPassword(), instance.getDbName());- backendHost = restoredHandle.getHost();- backendPort = restoredHandle.getPort();- instance.setContainerId(restoredHandle.getContainerId());- instance.setContainerHost(restoredHandle.getHost());- instance.setContainerPort(restoredHandle.getPort());+ backendHost = restoredHandle != null ? restoredHandle.getHost() : null;+ backendPort = restoredHandle != null ? restoredHandle.getPort() : 0;+ instance.setContainerId(restoredHandle != null ? restoredHandle.getContainerId() : null);+ instance.setContainerHost(backendHost);+ instance.setContainerPort(backendPort);}- String effectiveMasterUser = instance.getMasterUsername() != null- ? instance.getMasterUsername() : "root";- proxyManager.startProxy(rdsResourceRelayKey(- instance.getDbInstanceArn(), instance.getDbInstanceIdentifier()),- instance.getEngine(),- instance.isIamDatabaseAuthenticationEnabled(), proxyPort,- backendHost, backendPort, instance.getEndpoint().address(),- effectiveMasterUser, instance.getMasterPassword(), instance.getDbName(),- (user, pw) -> validateDbPasswordForScope(- accountId, instanceRegion,- instance.getDbInstanceIdentifier(), user, pw));+ if (hasBackend(backendHost, backendPort)) {+ String effectiveMasterUser = instance.getMasterUsername() != null+ ? instance.getMasterUsername() : "root";+ proxyManager.startProxy(rdsResourceRelayKey(+ instance.getDbInstanceArn(), instance.getDbInstanceIdentifier()),+ instance.getEngine(),+ instance.isIamDatabaseAuthenticationEnabled(), proxyPort,+ backendHost, backendPort, instance.getEndpoint().address(),+ effectiveMasterUser, instance.getMasterPassword(), instance.getDbName(),+ (user, pw) -> validateDbPasswordForScope(+ accountId, instanceRegion,+ instance.getDbInstanceIdentifier(), user, pw));+ }instance.setStatus(DbInstanceStatus.AVAILABLE);putInstanceForScope(accountId, instanceRegion,instance.getDbInstanceIdentifier(), instance);diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.javaindex e520efe8db..d4ca14fb60 100644--- a/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java@@ -57,6 +57,8 @@ public class RdsContainerManager {private final Map<String, RdsContainerHandle> activeContainers = new ConcurrentHashMap<>();private final Map<String, String> activeStorageOwners = new ConcurrentHashMap<>();private final Set<String> claimedRuntimes = ConcurrentHashMap.newKeySet();+ private final Set<String> cleanupPending = ConcurrentHashMap.newKeySet();+ private volatile boolean dockerUnavailableLogged;@Injectpublic RdsContainerManager(ContainerBuilder containerBuilder,@@ -91,6 +93,86 @@ public class RdsContainerManager {engine, image, masterUsername, masterPassword, dbName);}+ /**+ * Attempts {@link #start} and reports the backend as unavailable instead of propagating the+ * failure when the cause is that no Docker daemon is reachable from Floci: Floci running+ * inside Docker without a mounted socket, or a stopped daemon on the host. A failure raised+ * while the daemon <em>is</em> reachable is a genuine container problem and still propagates,+ * so nothing changes for a Floci that can start database containers.+ * <p>+ * A handle retained by an earlier cleanup failure is cleaned up first, so a container+ * created just before the daemon went away, or one whose stop failed after its proxy did+ * not start, is removed once the daemon is back instead of blocking the runtime's next+ * start. An identity that names only the fixed container name came from a+ * start that failed before Docker created anything, and is dropped when the daemon is+ * unreachable so the runtime can retry.+ *+ * @return the container handle, or {@code null} when no Docker daemon is reachable+ */+ public RdsContainerHandle tryStart(+ String runtimeId, String instanceId, String containerStorageResourceId,+ String dockerVolumeName, DatabaseEngine engine, String image,+ String masterUsername, String masterPassword, String dbName) {+ String effectiveRuntimeId = runtimeId == null || runtimeId.isBlank() ? instanceId : runtimeId;+ try {+ retryRetainedCleanup(effectiveRuntimeId);+ RdsContainerHandle handle = start(runtimeId, instanceId, containerStorageResourceId,+ dockerVolumeName, engine, image, masterUsername, masterPassword, dbName);+ dockerUnavailableLogged = false;+ return handle;+ } catch (RuntimeException e) {+ if (isDockerReachable()) {+ throw e;+ }+ discardNeverCreatedIdentity(effectiveRuntimeId, dockerVolumeName);+ if (!dockerUnavailableLogged) {+ dockerUnavailableLogged = true;+ LOG.warnv("No Docker daemon is reachable from Floci ({0}). RDS metadata operations "+ + "keep working and DB instances still reach 'available', but they have no "+ + "backing database container until a daemon becomes reachable.",+ e.getMessage());+ }+ return null;+ }+ }++ /**+ * Probes the configured Docker endpoint, which is how a missing daemon is told apart from a+ * container that failed for its own reasons.+ */+ public boolean isDockerReachable() {+ try {+ lifecycleManager.getDockerClient().pingCmd().exec();+ return true;+ } catch (Exception e) {+ LOG.debugv("Docker daemon is not reachable: {0}", e.getMessage());+ return false;+ }+ }++ /**+ * A handle retained because its cleanup failed, whether from a failed start or from a stop+ * that could not remove the container, is cleaned up before the runtime starts again. A live+ * handle that is not pending cleanup is left alone, so a concurrent duplicate start is still+ * rejected by {@link #start}.+ */+ private void retryRetainedCleanup(String runtimeId) {+ RdsContainerHandle retained = activeContainers.get(runtimeId);+ if (retained != null && cleanupPending.contains(runtimeId)) {+ stop(retained);+ }+ }++ private void discardNeverCreatedIdentity(String runtimeId, String containerName) {+ RdsContainerHandle retained = activeContainers.get(runtimeId);+ if (retained == null || retained.getHost() != null+ || !retained.getContainerId().equals(containerName)) {+ return;+ }+ activeContainers.remove(runtimeId, retained);+ releaseOwnership(retained.getStorageKey(), retained.getContainerKey(), runtimeId);+ }+public RdsContainerHandle start(String runtimeId, String instanceId, String containerStorageResourceId,String dockerVolumeName, DatabaseEngine engine, String image,@@ -220,6 +302,7 @@ public class RdsContainerManager {cleanupContainerId, effectiveRuntimeId, instanceId,null, 0, storageKey, containerKey);activeContainers.put(effectiveRuntimeId, retained);+ cleanupPending.add(effectiveRuntimeId);LOG.errorv(cleanupFailure,"Failed to clean up RDS container {0}; retaining storage ownership for {1}",cleanupContainerId, effectiveRuntimeId);@@ -272,6 +355,7 @@ public class RdsContainerManager {} catch (RuntimeException | Error e) {activeContainers.putIfAbsent(effectiveHandle.getRuntimeId(), effectiveHandle);claimedRuntimes.add(effectiveHandle.getRuntimeId());+ cleanupPending.add(effectiveHandle.getRuntimeId());throw e;}activeContainers.remove(effectiveHandle.getRuntimeId(), effectiveHandle);@@ -624,6 +708,7 @@ public class RdsContainerManager {activeStorageOwners.remove(containerKey, runtimeId);}claimedRuntimes.remove(runtimeId);+ cleanupPending.remove(runtimeId);}private static String requireSafeStorageComponent(String value, String label) {diff --git a/src/main/java/io/github/hectorvent/floci/services/rdsdata/RdsDataResourceResolver.java b/src/main/java/io/github/hectorvent/floci/services/rdsdata/RdsDataResourceResolver.javaindex 0dd0b505b7..818d70b4e5 100644--- a/src/main/java/io/github/hectorvent/floci/services/rdsdata/RdsDataResourceResolver.java+++ b/src/main/java/io/github/hectorvent/floci/services/rdsdata/RdsDataResourceResolver.java@@ -28,6 +28,9 @@ class RdsDataResourceResolver {throw new AwsException("BadRequestException", "resourceArn is required.", 400);}+ DbCluster cluster = null;+ DbInstance instance = null;+ String region = null;try {AwsArnUtils.Arn arn = AwsArnUtils.parse(resourceArn);if (!"rds".equals(arn.service())) {@@ -43,38 +46,66 @@ class RdsDataResourceResolver {}String type = arn.resource().substring(0, separator);String id = arn.resource().substring(separator + 1);+ region = arn.region();if ("cluster".equals(type)) {- DbCluster cluster = rdsService.getDbCluster(id, arn.region());- if (resourceArn.equals(cluster.getDbClusterArn())) {- return fromCluster(cluster);+ DbCluster found = rdsService.getDbCluster(id, region);+ if (resourceArn.equals(found.getDbClusterArn())) {+ cluster = found;}} else if ("db".equals(type)) {- DbInstance instance = rdsService.getDbInstance(id, arn.region());- if (resourceArn.equals(instance.getDbInstanceArn())) {- return fromInstance(instance);+ DbInstance found = rdsService.getDbInstance(id, region);+ if (resourceArn.equals(found.getDbInstanceArn())) {+ instance = found;}}} catch (AwsException | IllegalArgumentException ignored) {// Normalize lookup and ARN parsing failures to the RDS Data API error shape below.}+ if (cluster != null) {+ return fromCluster(cluster, region);+ }+ if (instance != null) {+ return fromInstance(instance, region);+ }throw new AwsException("BadRequestException","resourceArn does not resolve to a local RDS resource: " + resourceArn, 400);}- private static DatabaseTarget fromCluster(DbCluster cluster) {- return target(cluster.getDbClusterArn(), cluster.getEngine(), cluster.getContainerHost(), cluster.getContainerPort(),- cluster.getMasterUsername(), cluster.getMasterPassword(), cluster.getDatabaseName());+ private DatabaseTarget fromCluster(DbCluster cluster, String region) {+ DbCluster resolved = hasRuntime(cluster.getContainerHost(), cluster.getContainerPort())+ ? cluster+ : rdsService.ensureClusterBackend(cluster.getDbClusterIdentifier(), region);+ return target(resolved.getDbClusterArn(), resolved.getEngine(), resolved.getContainerHost(),+ resolved.getContainerPort(), resolved.getMasterUsername(), resolved.getMasterPassword(),+ resolved.getDatabaseName());}- private static DatabaseTarget fromInstance(DbInstance instance) {- return target(instance.getDbInstanceArn(), instance.getEngine(), instance.getContainerHost(), instance.getContainerPort(),- instance.getMasterUsername(), instance.getMasterPassword(), instance.getDbName());+ private DatabaseTarget fromInstance(DbInstance instance, String region) {+ DbInstance resolved = hasRuntime(instance.getContainerHost(), instance.getContainerPort())+ ? instance+ : rdsService.ensureInstanceBackend(instance.getDbInstanceIdentifier(), region);+ return target(resolved.getDbInstanceArn(), resolved.getEngine(), resolved.getContainerHost(),+ resolved.getContainerPort(), resolved.getMasterUsername(), resolved.getMasterPassword(),+ resolved.getDbName());}- private static DatabaseTarget target(String arn, DatabaseEngine engine, String host, int port,- String username, String password, String databaseName) {- if (host == null || host.isBlank() || port <= 0) {+ private static boolean hasRuntime(String host, int port) {+ return host != null && !host.isBlank() && port > 0;+ }++ private DatabaseTarget target(String arn, DatabaseEngine engine, String host, int port,+ String username, String password, String databaseName) {+ if (!hasRuntime(host, port)) {+ // The Data API is RDS's data plane: it needs a real database, which Floci can only+ // provide through a Docker container. Name the missing daemon rather than reporting a+ // generic runtime failure, in the Data API's modelled server-side error shape.+ if (!rdsService.isBackendRuntimeAvailable()) {+ throw new AwsException("InternalServerErrorException",+ "The RDS backing database is unavailable because no Docker daemon is reachable "+ + "from Floci. DB instance and cluster metadata operations are supported; "+ + "Data API execution requires Docker.", 500);+ }throw new AwsException("BadRequestException","RDS resource runtime is not available for Data API execution.", 400);}diff --git a/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.javaindex cca15858e5..06f6e50816 100644--- a/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java@@ -139,7 +139,7 @@ class RdsServiceTest {new InMemoryStorage<>(), new InMemoryStorage<>(),new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>());- when(containerManager.start(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any())).thenReturn(new RdsContainerHandle("cont-id", "id", "localhost", 5432));when(ec2Service.resolveDefaultVpcId(any())).thenAnswer(invocation -> Ec2Service.defaultVpcId(invocation.getArgument(0)));@@ -220,7 +220,7 @@ class RdsServiceTest {rdsService.createDbCluster("dup-cluster", "postgres", "17.5","admin", "password", "dbname", false, null, null, null, false));assertEquals("DBClusterAlreadyExistsFault", exception.getErrorCode());- verify(containerManager, times(1)).start(any(), any(), any(), any(), any(), any(), any(), any(), any());+ verify(containerManager, times(1)).tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());}/**@@ -234,7 +234,7 @@ class RdsServiceTest {void createDbClusterRejectsADuplicateWhileTheFirstIsStillProvisioning() throws Exception {CountDownLatch insideStart = new CountDownLatch(1);CountDownLatch releaseStart = new CountDownLatch(1);- when(containerManager.start(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any())).thenAnswer(invocation -> {insideStart.countDown();assertTrue(releaseStart.await(5, TimeUnit.SECONDS), "test released the latch");@@ -259,7 +259,7 @@ class RdsServiceTest {} finally {executor.shutdownNow();}- verify(containerManager, times(1)).start(any(), any(), any(), any(), any(), any(), any(), any(), any());+ verify(containerManager, times(1)).tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());assertNotNull(rdsService.getDbCluster("racy-cluster"));}@@ -267,7 +267,7 @@ class RdsServiceTest {void createDbInstanceRejectsADuplicateWhileTheFirstIsStillProvisioning() throws Exception {CountDownLatch insideStart = new CountDownLatch(1);CountDownLatch releaseStart = new CountDownLatch(1);- when(containerManager.start(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any())).thenAnswer(invocation -> {insideStart.countDown();assertTrue(releaseStart.await(5, TimeUnit.SECONDS), "test released the latch");@@ -293,7 +293,7 @@ class RdsServiceTest {} finally {executor.shutdownNow();}- verify(containerManager, times(1)).start(any(), any(), any(), any(), any(), any(), any(), any(), any());+ verify(containerManager, times(1)).tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());assertNotNull(rdsService.getDbInstance("racy-db"));}@@ -307,7 +307,7 @@ class RdsServiceTest {CountDownLatch insideStart = new CountDownLatch(1);CountDownLatch releaseStart = new CountDownLatch(1);AtomicBoolean firstCall = new AtomicBoolean(true);- when(containerManager.start(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any())).thenAnswer(invocation -> {if (firstCall.getAndSet(false)) {insideStart.countDown();@@ -342,7 +342,7 @@ class RdsServiceTest {rdsService.deleteDbCluster("reused-cluster");assertNotNull(rdsService.createDbCluster("reused-cluster", "postgres", "17.5","admin", "password", "dbname", false, null, null, null, false));- verify(containerManager, times(2)).start(any(), any(), any(), any(), any(), any(), any(), any(), any());+ verify(containerManager, times(2)).tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());}/** The instance-side twin: a successful create must release the identifier too. */@@ -355,7 +355,7 @@ class RdsServiceTest {assertNotNull(rdsService.createDbInstance("reused-db", "postgres", "17.5","admin", "password", "dbname", "db.t3.micro",20, false, null, null, null, null, false));- verify(containerManager, times(2)).start(any(), any(), any(), any(), any(), any(), any(), any(), any());+ verify(containerManager, times(2)).tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());}/**@@ -405,7 +405,7 @@ class RdsServiceTest {/** A failed create must release the identifier so a clean retry can proceed. */@Testvoid createDbClusterFailureReleasesTheIdentifierForRetry() {- when(containerManager.start(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any())).thenThrow(new RuntimeException("image pull failed")).thenReturn(new RdsContainerHandle("cont-id", "id", "localhost", 5432));@@ -415,7 +415,7 @@ class RdsServiceTest {assertNotNull(rdsService.createDbCluster("retry-cluster", "postgres", "17.5","admin", "password", "dbname", false, null, null, null, false));- verify(containerManager, times(2)).start(any(), any(), any(), any(), any(), any(), any(), any(), any());+ verify(containerManager, times(2)).tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());}@Test@@ -424,7 +424,7 @@ class RdsServiceTest {"admin", "password", "dbname", "db.t3.micro",20, false, null, null, null);- verify(containerManager).start(+ verify(containerManager).tryStart(eq("arn:aws:rds:us-east-1:123456789012:db:mydb"), eq("mydb"),any(), any(), eq(DatabaseEngine.POSTGRES),eq("postgres:18.1-alpine"), eq("admin"), eq("password"), eq("dbname"));@@ -437,7 +437,7 @@ class RdsServiceTest {rdsService.createDbCluster("cluster1", "postgres", "16.3","admin", "password", "dbname", false, null);- verify(containerManager).start(+ verify(containerManager).tryStart(eq("arn:aws:rds:us-east-1:123456789012:cluster:cluster1"), eq("cluster1"),any(), any(), eq(DatabaseEngine.POSTGRES),eq("postgres:16.14-alpine3.23"), eq("admin"), eq("password"), eq("dbname"));@@ -451,7 +451,7 @@ class RdsServiceTest {rdsService.createDbCluster("cluster1", "postgres", "18.1","admin", "password", "dbname", false, null);- verify(containerManager).start(+ verify(containerManager).tryStart(eq("arn:aws:rds:us-east-1:123456789012:cluster:cluster1"), eq("cluster1"),any(), any(), eq(DatabaseEngine.POSTGRES),eq(EmulatorConfig.RdsServiceConfig.DEFAULT_POSTGRES_IMAGE),@@ -1246,7 +1246,7 @@ class RdsServiceTest {20, false, null, "missing-subnet-group", null));assertEquals("DBSubnetGroupNotFoundFault", exception.getErrorCode());- verify(containerManager, never()).start(any(), any(), any(), any(), any(), any(), any(), any(), any());+ verify(containerManager, never()).tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());verify(proxyManager, never()).startProxy(any(), any(), anyBoolean(), anyInt(),any(), anyInt(), any(), any(), any(), any(), any());}@@ -1311,7 +1311,7 @@ class RdsServiceTest {assertEquals("localhost", cluster.getEndpoint().address());assertTrue(cluster.getEndpoint().port() > 0);assertNull(cluster.getContainerId());- verify(containerManager, never()).start(any(), any(), any(), any(), any(), any(), any(), any(), any());+ verify(containerManager, never()).tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());verify(proxyManager, never()).startProxy(any(), any(), anyBoolean(), anyInt(), any(), anyInt(),any(), any(), any(), any(), any());}@@ -1544,11 +1544,289 @@ class RdsServiceTest {// No Docker volume name may be persisted: the mock cluster has a null volume id, so the// fallback would fabricate a name that a later non-mock restore could try to reference.assertNull(instance.getDockerVolumeName());- verify(containerManager, never()).start(any(), any(), any(), any(), any(), any(), any(), any(), any());+ verify(containerManager, never()).tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());verify(proxyManager, never()).startProxy(any(), any(), anyBoolean(), anyInt(), any(), anyInt(),any(), any(), any(), any(), any());}+ // ------------------------------------------------------------+ // No Docker daemon reachable (Floci inside Docker with no mounted socket)+ // ------------------------------------------------------------++ @Test+ void createDbInstanceSucceedsAsMetadataWhenNoDockerDaemonIsReachable() {+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ .thenReturn(null);++ DbInstance instance = rdsService.createDbInstance("probe-db", "postgres", "16.3",+ "probeadmin", "ProbePassw0rd!", "dbname", "db.t3.micro",+ 20, false, null, null, null, false, null,+ Map.of("tofu-estate", "probe1"));++ assertEquals(DbInstanceStatus.AVAILABLE, instance.getStatus());+ assertEquals("arn:aws:rds:us-east-1:123456789012:db:probe-db", instance.getDbInstanceArn());+ assertNotNull(instance.getEndpoint());+ assertEquals("probe1", instance.getTags().get("tofu-estate"));+ // Nothing was started, so no runtime is claimed; the storage identity is kept for the retry.+ assertNull(instance.getContainerId());+ assertNull(instance.getContainerHost());+ assertEquals(0, instance.getContainerPort());+ assertNotNull(instance.getVolumeId());+ verify(proxyManager, never()).startProxy(any(), any(), anyBoolean(), anyInt(), any(), anyInt(),+ any(), any(), any(), any(), any());+ }++ @Test+ void instanceMetadataCrudWorksWhenNoDockerDaemonIsReachable() {+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ .thenReturn(null);++ DbInstance created = rdsService.createDbInstance("probe-db", "postgres", "16.3",+ "probeadmin", "ProbePassw0rd!", "dbname", "db.t3.micro",+ 20, false, null, null, null, false, null,+ Map.of("tofu-estate", "probe1"));++ assertEquals(1, rdsService.listDbInstances("probe-db").size());+ assertEquals("probe1", rdsService.listTagsForResource(created.getDbInstanceArn()).get("tofu-estate"));++ rdsService.addTagsToResource(created.getDbInstanceArn(), Map.of("Name", "probe-db"), "us-east-1");+ assertEquals("probe-db", rdsService.listTagsForResource(created.getDbInstanceArn()).get("Name"));++ assertEquals(DbInstanceStatus.AVAILABLE,+ rdsService.modifyDbInstance("probe-db", "NewPassw0rd!", null, null).getStatus());+ verify(containerManager, never()).rotateMasterPassword(any(), any(), any(), any(), any(), any());++ rdsService.deleteDbInstance("probe-db");+ assertThrows(AwsException.class, () -> rdsService.getDbInstance("probe-db"));+ // Cleanup must not reach for a container or volume that was never created.+ verify(containerManager, never()).stop(any());+ verify(containerManager, never()).stopByRuntimeId(any());+ verify(containerManager, never()).removeVolume(any(), any(), any());+ }++ @Test+ void deleteStillRemovesTheVolumeWhenADaemonIsReachableAgain() {+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ .thenReturn(null);+ DbInstance created = rdsService.createDbInstance("probe-db", "postgres", "16.3",+ "admin", "password", "dbname", "db.t3.micro",+ 20, false, null, null, null);++ // The daemon is back, so whatever Docker holds for this record is cleaned up as before.+ when(containerManager.isDockerReachable()).thenReturn(true);+ rdsService.deleteDbInstance("probe-db");++ verify(containerManager, never()).stop(any());+ verify(containerManager).stopByRuntimeId(created.getDbInstanceArn());+ verify(containerManager).removeVolume(+ eq(created.getDbInstanceArn()), any(), eq(created.getDockerVolumeName()));+ }++ @Test+ void createDbClusterAndMemberSucceedAsMetadataWhenNoDockerDaemonIsReachable() {+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ .thenReturn(null);++ DbCluster cluster = rdsService.createDbCluster("probe-cluster", "aurora-postgresql", "16.3",+ "admin", "password", "dbname", false, null);+ DbInstance member = rdsService.createDbInstance("probe-member", "aurora-postgresql", "16.3",+ "admin", "password", "dbname", "db.serverless",+ 0, false, null, null, "probe-cluster");++ assertEquals(DbInstanceStatus.AVAILABLE, cluster.getStatus());+ assertNull(cluster.getContainerId());+ assertNotNull(cluster.getVolumeId());+ assertEquals(DbInstanceStatus.AVAILABLE, member.getStatus());+ assertNull(member.getContainerId());+ verify(proxyManager, never()).startProxy(any(), any(), anyBoolean(), anyInt(), any(), anyInt(),+ any(), any(), any(), any(), any());++ rdsService.deleteDbInstance("probe-member");+ rdsService.deleteDbCluster("probe-cluster");+ assertThrows(AwsException.class, () -> rdsService.getDbCluster("probe-cluster"));+ verify(containerManager, never()).stop(any());+ verify(containerManager, never()).removeVolume(any(), any(), any());+ }++ @Test+ void backendIsRetriedOncePerCallSoTheContainerStartsWhenADaemonAppears() {+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ .thenReturn(null);+ DbInstance created = rdsService.createDbInstance("probe-db", "postgres", "16.3",+ "admin", "password", "dbname", "db.t3.micro",+ 20, false, null, null, null);++ assertNull(rdsService.ensureInstanceBackend("probe-db", "us-east-1").getContainerId());++ // A Docker daemon appears.+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ .thenReturn(new RdsContainerHandle("late-container", created.getDbInstanceArn(),+ "probe-db", "127.0.0.1", 15432));++ DbInstance started = rdsService.ensureInstanceBackend("probe-db", "us-east-1");+ assertEquals("late-container", started.getContainerId());+ assertEquals("127.0.0.1", started.getContainerHost());+ assertEquals(15432, started.getContainerPort());+ assertEquals(created.getDockerVolumeName(), started.getDockerVolumeName());+ // The create and both retries use the record's own storage identity.+ verify(containerManager, times(3)).tryStart(+ eq(created.getDbInstanceArn()), eq("probe-db"),+ eq(created.getContainerStorageResourceId()), eq(created.getDockerVolumeName()),+ eq(DatabaseEngine.POSTGRES), eq("postgres:16.3-alpine"),+ eq("admin"), eq("password"), eq("dbname"));+ verify(proxyManager).startProxy(+ eq("rds-resource:" + created.getDbInstanceArn()), eq(DatabaseEngine.POSTGRES),+ eq(false), eq(created.getProxyPort()), eq("127.0.0.1"), eq(15432), any(),+ eq("admin"), eq("password"), eq("dbname"), any());++ // Already backed, so a further call is a no-op rather than a second container. Three+ // calls in total: the create, the retry that found no daemon, and the retry that started it.+ rdsService.ensureInstanceBackend("probe-db", "us-east-1");+ verify(containerManager, times(3))+ .tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());+ }++ @Test+ void retryLeavesTheRecordWithoutABackendWhenTheProxyDoesNotStart() {+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ .thenReturn(null);+ DbInstance created = rdsService.createDbInstance("probe-db", "postgres", "16.3",+ "admin", "password", "dbname", "db.t3.micro",+ 20, false, null, null, null);+ RdsContainerHandle late = new RdsContainerHandle("late-container", created.getDbInstanceArn(),+ "probe-db", "127.0.0.1", 15432);+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ .thenReturn(late);+ doThrow(new IllegalStateException("port in use")).doNothing()+ .when(proxyManager).startProxy(any(), any(), anyBoolean(), anyInt(), any(), anyInt(),+ any(), any(), any(), any(), any());++ assertThrows(IllegalStateException.class,+ () -> rdsService.ensureInstanceBackend("probe-db", "us-east-1"));++ // The container is stopped and the record still says it has no backend, so the next+ // call retries both halves instead of returning a container nothing listens on.+ verify(containerManager).stop(late);+ assertNull(rdsService.getDbInstance("probe-db").getContainerId());++ DbInstance started = rdsService.ensureInstanceBackend("probe-db", "us-east-1");+ assertEquals("late-container", started.getContainerId());+ verify(containerManager, times(3))+ .tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());+ }++ @Test+ void clusterMemberBackendIsRetriedThroughItsCluster() {+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ .thenReturn(null);+ DbCluster cluster = rdsService.createDbCluster("probe-cluster", "aurora-postgresql", "16.3",+ "admin", "password", "dbname", false, null);+ DbInstance member = rdsService.createDbInstance("probe-member", "aurora-postgresql", "16.3",+ "admin", "password", "dbname", "db.serverless",+ 0, false, null, null, "probe-cluster");++ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ .thenReturn(new RdsContainerHandle("late-cluster-container", cluster.getDbClusterArn(),+ "probe-cluster", "127.0.0.1", 15432));++ DbInstance started = rdsService.ensureInstanceBackend("probe-member", "us-east-1");++ assertEquals("late-cluster-container", started.getContainerId());+ assertEquals("late-cluster-container",+ rdsService.getDbCluster("probe-cluster").getContainerId());+ verify(containerManager, times(2)).tryStart(+ eq(cluster.getDbClusterArn()), eq("probe-cluster"), any(), any(),+ eq(DatabaseEngine.POSTGRES), any(), eq("admin"), eq("password"), eq("dbname"));+ verify(proxyManager).startProxy(+ eq("rds-resource:" + cluster.getDbClusterArn()), any(), anyBoolean(),+ eq(cluster.getProxyPort()), eq("127.0.0.1"), eq(15432), any(), any(), any(), any(), any());+ verify(proxyManager).startProxy(+ eq("rds-resource:" + member.getDbInstanceArn()), any(), anyBoolean(),+ eq(member.getProxyPort()), eq("127.0.0.1"), eq(15432), any(), any(), any(), any(), any());+ }++ @Test+ void rebootRetriesTheBackendForAnInstanceCreatedWithoutADaemon() {+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ .thenReturn(null);+ DbInstance created = rdsService.createDbInstance("probe-db", "postgres", "16.3",+ "admin", "password", "dbname", "db.t3.micro",+ 20, false, null, null, null);++ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ .thenReturn(new RdsContainerHandle("late-container", created.getDbInstanceArn(),+ "probe-db", "127.0.0.1", 15432));++ DbInstance rebooted = rdsService.rebootDbInstance("probe-db");++ assertEquals(DbInstanceStatus.AVAILABLE, rebooted.getStatus());+ assertEquals("late-container", rebooted.getContainerId());+ // There was no container to stop, only one to start.+ verify(containerManager, never()).stop(any());+ verify(proxyManager).startProxy(+ eq("rds-resource:" + created.getDbInstanceArn()), any(), anyBoolean(),+ eq(created.getProxyPort()), eq("127.0.0.1"), eq(15432), any(), any(), any(), any(), any());+ }++ @Test+ void restoreKeepsInstanceAvailableWhenNoDockerDaemonIsReachable() {+ StorageBackend<String, DbInstance> instances = new InMemoryStorage<>();+ StorageBackend<String, DbCluster> clusters = new InMemoryStorage<>();++ RdsService initialService = newService(containerManager, proxyManager,+ instances, clusters, new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>());+ DbInstance created = initialService.createDbInstance("mydb", "postgres", "16.3",+ "admin", "secret", "app", "db.t3.micro",+ 20, false, null, null, null, null, false);++ RdsContainerManager daemonlessContainerManager = mock(RdsContainerManager.class);+ RdsProxyManager daemonlessProxyManager = mock(RdsProxyManager.class);+ when(daemonlessContainerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ .thenReturn(null);++ RdsService restoredService = newService(daemonlessContainerManager, daemonlessProxyManager,+ instances, clusters, new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>());+ restoredService.restorePersistedRuntime();++ DbInstance restored = restoredService.getDbInstance("mydb");+ assertEquals(DbInstanceStatus.AVAILABLE, restored.getStatus());+ assertEquals(created.getProxyPort(), restored.getProxyPort());+ assertEquals(created.getProxyPort(), restored.getEndpoint().port());+ assertNull(restored.getContainerId());+ verify(daemonlessProxyManager, never()).startProxy(any(), any(), anyBoolean(), anyInt(),+ any(), anyInt(), any(), any(), any(), any(), any());+ }++ @Test+ void restoreKeepsClusterAndMemberAvailableWhenNoDockerDaemonIsReachable() {+ StorageBackend<String, DbInstance> instances = new InMemoryStorage<>();+ StorageBackend<String, DbCluster> clusters = new InMemoryStorage<>();++ RdsService initialService = newService(containerManager, proxyManager,+ instances, clusters, new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>());+ initialService.createDbCluster("cluster1", "aurora-postgresql", "16.3",+ "admin", "secret", "app", false, null, null, null, false);+ initialService.createDbInstance("member1", "aurora-postgresql", "16.3",+ "admin", "secret", "app", "db.t3.medium",+ 20, false, null, null, "cluster1", null, false);++ RdsContainerManager daemonlessContainerManager = mock(RdsContainerManager.class);+ RdsProxyManager daemonlessProxyManager = mock(RdsProxyManager.class);+ when(daemonlessContainerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ .thenReturn(null);++ RdsService restoredService = newService(daemonlessContainerManager, daemonlessProxyManager,+ instances, clusters, new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>());+ restoredService.restorePersistedRuntime();++ assertEquals(DbInstanceStatus.AVAILABLE, restoredService.getDbCluster("cluster1").getStatus());+ DbInstance restoredMember = restoredService.getDbInstance("member1");+ assertEquals(DbInstanceStatus.AVAILABLE, restoredMember.getStatus());+ assertNull(restoredMember.getContainerId());+ verify(daemonlessProxyManager, never()).startProxy(any(), any(), anyBoolean(), anyInt(),+ any(), anyInt(), any(), any(), any(), any(), any());+ }+@Testvoid mockModeCreatesStandaloneInstanceAvailableWithoutContainer() {when(config.services().rds().mock()).thenReturn(true);@@ -1559,7 +1837,7 @@ class RdsServiceTest {assertEquals(DbInstanceStatus.AVAILABLE, instance.getStatus());assertNull(instance.getContainerId());- verify(containerManager, never()).start(any(), any(), any(), any(), any(), any(), any(), any(), any());+ verify(containerManager, never()).tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());}@Test@@ -1609,7 +1887,7 @@ class RdsServiceTest {DbInstance rebooted = rdsService.rebootDbInstance("standalone");assertEquals(DbInstanceStatus.AVAILABLE, rebooted.getStatus());- verify(containerManager, never()).start(any(), any(), any(), any(), any(), any(), any(), any(), any());+ verify(containerManager, never()).tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());verify(containerManager, never()).stop(any());verify(proxyManager, never()).startProxy(any(), any(), anyBoolean(), anyInt(), any(), anyInt(),any(), any(), any(), any(), any());@@ -1629,7 +1907,7 @@ class RdsServiceTest {DbInstance failed = rdsService.getDbInstance("standalone");assertEquals(DbInstanceStatus.FAILED, failed.getStatus());assertEquals(instance.getContainerId(), failed.getContainerId());- verify(containerManager, times(1)).start(+ verify(containerManager, times(1)).tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());verify(proxyManager).stopProxy("rds-resource:" + instance.getDbInstanceArn());}@@ -2106,7 +2384,7 @@ class RdsServiceTest {20, false, null, "single-az-group", null, null, true));assertEquals("DBSubnetGroupDoesNotCoverEnoughAZs", exception.getErrorCode());- verify(containerManager, never()).start(any(), any(), any(), any(), any(), any(), any(), any(), any());+ verify(containerManager, never()).tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());}@Test@@ -2117,7 +2395,7 @@ class RdsServiceTest {null, null, "us-east-1a", true));assertEquals("InvalidParameterCombination", exception.getErrorCode());- verify(containerManager, never()).start(any(), any(), any(), any(), any(), any(), any(), any(), any());+ verify(containerManager, never()).tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());}@Test@@ -2229,7 +2507,7 @@ class RdsServiceTest {StorageBackend<String, DbParameterGroup> parameterGroups = new InMemoryStorage<>();StorageBackend<String, DbClusterParameterGroup> clusterParameterGroups = new InMemoryStorage<>();- when(containerManager.start(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any())).thenReturn(new RdsContainerHandle("initial-container", "mydb", "localhost", 5432));RdsService initialService = newService(containerManager, proxyManager,@@ -2245,7 +2523,7 @@ class RdsServiceTest {RdsContainerManager restoredContainerManager = mock(RdsContainerManager.class);RdsProxyManager restoredProxyManager = mock(RdsProxyManager.class);- when(restoredContainerManager.start(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ when(restoredContainerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any())).thenReturn(new RdsContainerHandle("restored-container", "mydb", "127.0.0.1", 15432));RdsService restoredService = newService(restoredContainerManager, restoredProxyManager,@@ -2262,7 +2540,7 @@ class RdsServiceTest {assertEquals("127.0.0.1", restored.getContainerHost());assertEquals(15432, restored.getContainerPort());- verify(restoredContainerManager).start(+ verify(restoredContainerManager).tryStart(eq(restored.getDbInstanceArn()), eq("mydb"),eq(restored.getContainerStorageResourceId()),eq(restored.getDockerVolumeName()), eq(DatabaseEngine.POSTGRES),@@ -2284,7 +2562,7 @@ class RdsServiceTest {StorageBackend<String, DbParameterGroup> parameterGroups = new InMemoryStorage<>();StorageBackend<String, DbClusterParameterGroup> clusterParameterGroups = new InMemoryStorage<>();- when(containerManager.start(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any())).thenReturn(new RdsContainerHandle("initial-cluster-container", "cluster1", "localhost", 5432));RdsService initialService = newService(containerManager, proxyManager,@@ -2297,7 +2575,7 @@ class RdsServiceTest {RdsContainerManager restoredContainerManager = mock(RdsContainerManager.class);RdsProxyManager restoredProxyManager = mock(RdsProxyManager.class);- when(restoredContainerManager.start(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ when(restoredContainerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any())).thenReturn(new RdsContainerHandle("restored-cluster-container", "cluster1", "127.0.0.1", 15432));RdsService restoredService = newService(restoredContainerManager, restoredProxyManager,@@ -2315,7 +2593,7 @@ class RdsServiceTest {assertEquals("127.0.0.1", restoredMember.getContainerHost());assertEquals(15432, restoredMember.getContainerPort());- verify(restoredContainerManager).start(+ verify(restoredContainerManager).tryStart(eq(restoredCluster.getDbClusterArn()), eq("cluster1"),eq(restoredCluster.getContainerStorageResourceId()),eq(restoredCluster.getDockerVolumeName()), eq(DatabaseEngine.POSTGRES),@@ -2342,7 +2620,7 @@ class RdsServiceTest {RdsContainerHandle replacementHandle = new RdsContainerHandle("replacement-container", "arn:aws:rds:us-east-1:123456789012:db:replacement","replacement", "127.0.0.1", 15433);- when(restoredContainerManager.start(+ when(restoredContainerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any())).thenReturn(restoredHandle, replacementHandle);org.mockito.Mockito.doThrow(new IllegalStateException("relay failed"))@@ -2402,7 +2680,7 @@ class RdsServiceTest {persisted.setContainerId("persisted-container");instances.put("broken", persisted);RdsContainerManager restoredContainerManager = mock(RdsContainerManager.class);- when(restoredContainerManager.start(+ when(restoredContainerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any())).thenThrow(new IllegalStateException("Docker start failed"));RdsService restoredService = newService(@@ -2468,7 +2746,7 @@ class RdsServiceTest {RdsContainerHandle replacementHandle = new RdsContainerHandle("replacement-container", "arn:aws:rds:us-east-1:123456789012:cluster:replacement","replacement", "127.0.0.1", 15433);- when(restoredContainerManager.start(+ when(restoredContainerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any())).thenReturn(restoredHandle, replacementHandle);org.mockito.Mockito.doThrow(new IllegalStateException("relay failed"))@@ -4510,7 +4788,7 @@ class RdsServiceTest {StorageBackend<String, DbProxy> proxies = new InMemoryStorage<>();StorageBackend<String, DbProxyTargetGroup> proxyTargetGroups = new InMemoryStorage<>();- when(containerManager.start(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any())).thenReturn(new RdsContainerHandle("initial-container", "cluster1", "localhost", 5432));RdsService initialService = new RdsService(containerManager, proxyManager, ec2Service,@@ -4524,7 +4802,7 @@ class RdsServiceTest {RdsContainerManager restoredContainerManager = mock(RdsContainerManager.class);RdsProxyManager restoredProxyManager = mock(RdsProxyManager.class);- when(restoredContainerManager.start(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ when(restoredContainerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any())).thenReturn(new RdsContainerHandle("restored-container", "cluster1", "127.0.0.1", 15432));RdsService restoredService = new RdsService(restoredContainerManager, restoredProxyManager, ec2Service,@@ -4588,7 +4866,7 @@ class RdsServiceTest {RdsContainerManager restoredContainerManager = mock(RdsContainerManager.class);RdsProxyManager restoredProxyManager = mock(RdsProxyManager.class);- when(restoredContainerManager.start(any(), any(), any(), any(), any(), any(), any(), any(), any()))+ when(restoredContainerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any())).thenReturn(new RdsContainerHandle("restored-container", "cluster1","127.0.0.1", 15432));@@ -5378,7 +5656,7 @@ class RdsServiceTest {+ "or you do not have permissions to access it.", missing.getMessage());assertThrows(AwsException.class, () -> rdsService.getDbInstance("mydb"));// refused before any side effect: no container was started for the rejected create- verify(containerManager, never()).start(any(), any(), any(), any(), any(), any(), any(), any(), any());+ verify(containerManager, never()).tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());KmsKey disabled = knownKey("k1");disabled.setEnabled(false);diff --git a/src/test/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManagerTest.javaindex 4d42194d7c..6c048256df 100644--- a/src/test/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManagerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManagerTest.java@@ -529,6 +529,166 @@ class RdsContainerManagerTest {verify(lifecycleManager).removeVolumeStrict(volume);}+ @Test+ void tryStartReportsUnavailableInsteadOfThrowingWhenNoDockerDaemonIsReachable() {+ // Floci running inside Docker without a mounted daemon socket: the RDS control plane must+ // keep working, so the failure is reported rather than propagated.+ ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);+ doThrow(new IllegalStateException("Failed to remove stale container floci-rds-db1"))+ .when(lifecycleManager).removeIfExistsStrict(any());+ doThrow(new IllegalStateException("Failed to remove container floci-rds-db1"))+ .when(lifecycleManager).stopAndRemoveStrict(any(), any());+ when(lifecycleManager.getDockerClient()).thenThrow(+ new RuntimeException("java.net.SocketException: No such file or directory"));+ RdsContainerManager manager = daemonlessCapableManager(lifecycleManager);+ String runtimeId = "arn:aws:rds:us-east-1:000000000000:db:db1";++ for (int attempt = 0; attempt < 3; attempt++) {+ assertNull(manager.tryStart(runtimeId, "db1", "db1", "floci-rds-db1",+ DatabaseEngine.MYSQL, "mysql:8.0", "root", "password", "db"),+ "attempt " + attempt + " should report unavailable");+ }+ assertFalse(manager.isDockerReachable());+ // No container was created, so no cleanup identity is retained for the runtime.+ assertNull(manager.getActiveHandle(runtimeId));+ }++ @Test+ void tryStartPropagatesFailuresRaisedWhileTheDaemonIsReachable() {+ // A reachable daemon that cannot start the container is a genuine failure, not a+ // degraded mode: CreateDBInstance must still surface it.+ ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);+ when(lifecycleManager.create(any()))+ .thenThrow(new IllegalStateException("no such image: mysql:8.0"));+ DockerClient dockerClient = mock(DockerClient.class, org.mockito.Mockito.RETURNS_DEEP_STUBS);+ when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);+ RdsContainerManager manager = daemonlessCapableManager(lifecycleManager);++ IllegalStateException failure = assertThrows(IllegalStateException.class,+ () -> manager.tryStart("arn:aws:rds:us-east-1:000000000000:db:db1", "db1", "db1",+ "floci-rds-db1", DatabaseEngine.MYSQL, "mysql:8.0", "root", "password", "db"));++ assertEquals("no such image: mysql:8.0", failure.getMessage());+ assertTrue(manager.isDockerReachable());+ }++ @Test+ void tryStartSucceedsForTheSameRuntimeOnceADaemonAppears() {+ ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);+ doThrow(new IllegalStateException("Failed to remove stale container floci-rds-db1"))+ .when(lifecycleManager).removeIfExistsStrict(any());+ doThrow(new IllegalStateException("Failed to remove container floci-rds-db1"))+ .when(lifecycleManager).stopAndRemoveStrict(any(), any());+ when(lifecycleManager.getDockerClient()).thenThrow(+ new RuntimeException("java.net.SocketException: No such file or directory"));+ RdsContainerManager manager = daemonlessCapableManager(lifecycleManager);+ String runtimeId = "arn:aws:rds:us-east-1:000000000000:db:db1";++ assertNull(manager.tryStart(runtimeId, "db1", "db1", "floci-rds-db1",+ DatabaseEngine.MYSQL, "mysql:8.0", "root", "password", "db"));++ // A Docker daemon appears.+ org.mockito.Mockito.reset(lifecycleManager);+ stubStarts(lifecycleManager, new ContainerLifecycleManager.ContainerInfo(+ "late-container", Map.of(3306, new ContainerLifecycleManager.EndpointInfo("db1", 3306))));+ DockerClient dockerClient = mock(DockerClient.class, org.mockito.Mockito.RETURNS_DEEP_STUBS);+ when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);++ RdsContainerHandle handle = manager.tryStart(runtimeId, "db1", "db1", "floci-rds-db1",+ DatabaseEngine.MYSQL, "mysql:8.0", "root", "password", "db");++ assertEquals("late-container", handle.getContainerId());+ assertEquals(3306, handle.getPort());+ assertEquals(handle, manager.getActiveHandle(runtimeId));+ }++ @Test+ void tryStartKeepsAndLaterCleansTheIdentityOfAContainerCreatedBeforeTheDaemonWentAway() {+ // Docker answered the create and then went away: the created container must stay known+ // so the retry removes it once the daemon is back, rather than being forgotten.+ ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);+ when(lifecycleManager.create(any())).thenReturn("created-container");+ when(lifecycleManager.startCreated(any(), any()))+ .thenThrow(new IllegalStateException("java.net.SocketException: Connection reset"));+ doThrow(new IllegalStateException("Failed to remove container created-container"))+ .when(lifecycleManager).stopAndRemoveStrict(any(), any());+ when(lifecycleManager.getDockerClient()).thenThrow(+ new RuntimeException("java.net.SocketException: No such file or directory"));+ RdsContainerManager manager = daemonlessCapableManager(lifecycleManager);+ String runtimeId = "arn:aws:rds:us-east-1:000000000000:db:db1";++ assertNull(manager.tryStart(runtimeId, "db1", "db1", "floci-rds-db1",+ DatabaseEngine.MYSQL, "mysql:8.0", "root", "password", "db"));+ assertEquals("created-container", manager.getActiveHandle(runtimeId).getContainerId());++ // Still no daemon: the retry keeps the identity rather than dropping it.+ assertNull(manager.tryStart(runtimeId, "db1", "db1", "floci-rds-db1",+ DatabaseEngine.MYSQL, "mysql:8.0", "root", "password", "db"));+ assertEquals("created-container", manager.getActiveHandle(runtimeId).getContainerId());++ // The daemon is back: the leftover is removed first, then the start goes through.+ org.mockito.Mockito.reset(lifecycleManager);+ stubStarts(lifecycleManager, new ContainerLifecycleManager.ContainerInfo(+ "late-container", Map.of(3306, new ContainerLifecycleManager.EndpointInfo("db1", 3306))));+ DockerClient dockerClient = mock(DockerClient.class, org.mockito.Mockito.RETURNS_DEEP_STUBS);+ when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);++ RdsContainerHandle handle = manager.tryStart(runtimeId, "db1", "db1", "floci-rds-db1",+ DatabaseEngine.MYSQL, "mysql:8.0", "root", "password", "db");++ assertEquals("late-container", handle.getContainerId());+ verify(lifecycleManager).stopAndRemoveStrict(+ org.mockito.ArgumentMatchers.eq("created-container"), any());+ }++ @Test+ void tryStartRetriesACleanupThatFailedAfterAStop() {+ // The service stops a container whose auth proxy did not start. When that stop fails+ // the handle and claim stay retained, and the runtime's next start must clean them up+ // rather than fail on the claim forever.+ ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);+ stubStarts(lifecycleManager,+ new ContainerLifecycleManager.ContainerInfo("first-container",+ Map.of(3306, new ContainerLifecycleManager.EndpointInfo("db1", 3306))),+ new ContainerLifecycleManager.ContainerInfo("second-container",+ Map.of(3306, new ContainerLifecycleManager.EndpointInfo("db1", 3306))));+ DockerClient dockerClient = mock(DockerClient.class, org.mockito.Mockito.RETURNS_DEEP_STUBS);+ when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);+ doThrow(new IllegalStateException("Failed to remove container first-container"))+ .doNothing()+ .when(lifecycleManager).stopAndRemoveStrict(+ org.mockito.ArgumentMatchers.eq("first-container"), any());+ RdsContainerManager manager = daemonlessCapableManager(lifecycleManager);+ String runtimeId = "arn:aws:rds:us-east-1:000000000000:db:db1";++ RdsContainerHandle first = manager.start(runtimeId, "db1", "db1", "floci-rds-db1",+ DatabaseEngine.MYSQL, "mysql:8.0", "root", "password", "db");+ assertThrows(IllegalStateException.class, () -> manager.stop(first));+ assertEquals(first, manager.getActiveHandle(runtimeId));++ RdsContainerHandle retried = manager.tryStart(runtimeId, "db1", "db1", "floci-rds-db1",+ DatabaseEngine.MYSQL, "mysql:8.0", "root", "password", "db");++ assertEquals("second-container", retried.getContainerId());+ assertEquals(retried, manager.getActiveHandle(runtimeId));+ verify(lifecycleManager, times(2)).stopAndRemoveStrict(+ org.mockito.ArgumentMatchers.eq("first-container"), any());+ }++ private RdsContainerManager daemonlessCapableManager(ContainerLifecycleManager lifecycleManager) {+ EmulatorConfig config = config(tempDir.resolve("host-root"));+ ContainerLogStreamer logStreamer = mock(ContainerLogStreamer.class);+ lenient().when(logStreamer.generateLogStreamName(any())).thenReturn("log-stream");+ return new RdsContainerManager(+ new ContainerBuilder(config, mock(DockerHostResolver.class), mock(EmbeddedDnsServer.class)),+ lifecycleManager,+ logStreamer,+ mock(ContainerDetector.class),+ config,+ new RegionResolver("us-east-1", "000000000000"),+ mock(ServiceConfigAccess.class));+ }+@Testvoid failedStartReleasesClaimsForSameRuntimeRetry() {EmulatorConfig config = config(tempDir.resolve("host-root"));diff --git a/src/test/java/io/github/hectorvent/floci/services/rdsdata/RdsDataResourceResolverTest.java b/src/test/java/io/github/hectorvent/floci/services/rdsdata/RdsDataResourceResolverTest.javaindex 891c7e6b55..4c20a0b460 100644--- a/src/test/java/io/github/hectorvent/floci/services/rdsdata/RdsDataResourceResolverTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/rdsdata/RdsDataResourceResolverTest.java@@ -5,6 +5,7 @@ import io.github.hectorvent.floci.services.rds.RdsService;import io.github.hectorvent.floci.services.rds.model.DatabaseEngine;import io.github.hectorvent.floci.services.rds.model.DbCluster;import io.github.hectorvent.floci.services.rds.model.DbEndpoint;+import io.github.hectorvent.floci.services.rds.model.DbInstance;import io.github.hectorvent.floci.services.rds.model.DbInstanceStatus;import org.junit.jupiter.api.Test;@@ -14,6 +15,7 @@ import java.util.List;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.when;@@ -52,6 +54,69 @@ class RdsDataResourceResolverTest {assertEquals(400, error.getHttpStatus());}+ @Test+ void dataApiNamesTheMissingDockerDaemonWhenTheInstanceHasNoBackingContainer() {+ RdsService rdsService = mock(RdsService.class);+ DbInstance instance = daemonlessInstance();+ when(rdsService.getDbInstance("probe-db", "us-east-1")).thenReturn(instance);+ when(rdsService.ensureInstanceBackend("probe-db", "us-east-1")).thenReturn(instance);+ when(rdsService.isBackendRuntimeAvailable()).thenReturn(false);++ AwsException error = assertThrows(AwsException.class, () -> new RdsDataResourceResolver(rdsService)+ .resolve("arn:aws:rds:us-east-1:000000000000:db:probe-db"));++ assertEquals("InternalServerErrorException", error.getErrorCode());+ assertEquals(500, error.getHttpStatus());+ assertTrue(error.getMessage().contains("Docker"), error.getMessage());+ }++ @Test+ void dataApiRetriesTheBackendSoItResolvesOnceADaemonAppears() {+ RdsService rdsService = mock(RdsService.class);+ DbInstance started = daemonlessInstance();+ started.setContainerHost("127.0.0.1");+ started.setContainerPort(5432);+ when(rdsService.getDbInstance("probe-db", "us-east-1")).thenReturn(daemonlessInstance());+ when(rdsService.ensureInstanceBackend("probe-db", "us-east-1")).thenReturn(started);++ RdsDataResourceResolver.DatabaseTarget target = new RdsDataResourceResolver(rdsService)+ .resolve("arn:aws:rds:us-east-1:000000000000:db:probe-db");++ assertEquals("127.0.0.1", target.host());+ assertEquals(5432, target.port());+ }++ @Test+ void dataApiRetriesAClusterBackendThroughTheClusterEntryPoint() {+ RdsService rdsService = mock(RdsService.class);+ DbCluster cluster = new DbCluster("cluster1", DatabaseEngine.POSTGRES, "16.3", "admin", "secret",+ "app", DbInstanceStatus.AVAILABLE, new DbEndpoint("localhost", 7001),+ new DbEndpoint("localhost", 7001), false, new ArrayList<>(), null, Instant.now(), 7001);+ cluster.setDbClusterArn("arn:aws:rds:us-east-1:000000000000:cluster:cluster1");+ DbCluster started = new DbCluster("cluster1", DatabaseEngine.POSTGRES, "16.3", "admin", "secret",+ "app", DbInstanceStatus.AVAILABLE, new DbEndpoint("localhost", 7001),+ new DbEndpoint("localhost", 7001), false, new ArrayList<>(), null, Instant.now(), 7001);+ started.setDbClusterArn(cluster.getDbClusterArn());+ started.setContainerHost("127.0.0.1");+ started.setContainerPort(5432);+ when(rdsService.getDbCluster("cluster1", "us-east-1")).thenReturn(cluster);+ when(rdsService.ensureClusterBackend("cluster1", "us-east-1")).thenReturn(started);++ RdsDataResourceResolver.DatabaseTarget target = new RdsDataResourceResolver(rdsService)+ .resolve("arn:aws:rds:us-east-1:000000000000:cluster:cluster1");++ assertEquals("127.0.0.1", target.host());+ assertEquals(5432, target.port());+ }++ private static DbInstance daemonlessInstance() {+ DbInstance instance = new DbInstance("probe-db", DatabaseEngine.POSTGRES, "16.3", "admin",+ "secret", "app", "db.t3.micro", 20, DbInstanceStatus.AVAILABLE,+ new DbEndpoint("localhost", 7001), false, null, null, Instant.now(), 7001);+ instance.setDbInstanceArn("arn:aws:rds:us-east-1:000000000000:db:probe-db");+ return instance;+ }+@Testvoid rejectsResourceArnOutsideSignedRequestRegionBeforeLookup() {RdsService rdsService = mock(RdsService.class);