Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(rds): keep the control plane working with no Docker daemon (#3170)

lex0049dfd1b18abb
diff --git a/docs/services/rds.md b/docs/services/rds.md
index 8c9bbe61a6..a2448ec8e9 100644
--- a/docs/services/rds.md
+++ b/docs/services/rds.md
@@ -107,6 +107,21 @@ services:
FLOCI_SERVICES_RDS_PROXY_BASE_PORT: "7001"
```
+### Without a reachable Docker daemon
+
+A DB instance or cluster record is metadata. Its identifier, ARN, endpoint address and tags come
+from Floci's configuration, not from Docker. When no daemon is reachable, because Floci runs inside
+Docker with no socket mounted or the daemon on the host is stopped, `CreateDBInstance` and
+`CreateDBCluster` still succeed and the resource reaches `available`. `DescribeDBInstances`,
+`ModifyDBInstance`, the tagging APIs and `DeleteDBInstance` all work on that record, and Floci logs
+a warning naming the missing daemon.
+
+Nothing listens behind the endpoint in that state. The backing container is retried by every
+operation that needs the live database, so it starts as soon as a daemon becomes reachable. Until
+then, RDS Data API calls fail with a modelled `InternalServerErrorException` that names the missing
+daemon. A daemon that is reachable but cannot start the container still fails `CreateDBInstance`
+outright, since that is a real error rather than a degraded mode.
+
### Mock mode (CI / tests)
Set `FLOCI_SERVICES_RDS_MOCK=true` when you only need the management API shape: clusters and
diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java b/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java
index f2cf7bb226..c8c05bba5e 100644
--- a/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java
@@ -618,20 +618,28 @@ public class RdsService implements Resettable, ResourceProvider {
} else {
placement = resolvePlacement(dbSubnetGroupName, availabilityZone, multiAz, effectiveRegion);
if (!mock) {
- // Standalone instance — start its own container
+ // Standalone instance, so it starts its own container. The record itself is metadata:
+ // identifier, ARN, endpoint and tags come from configuration, so the instance is
+ // created and reaches 'available' even when no Docker daemon is reachable.
String image = imageForEngine(engine, engineVersion);
instanceVolumeId = String.format("%06x", new SecureRandom().nextInt(0xFFFFFF));
instanceDockerVolumeName = newVolumeName(
instanceVolumeId, instanceStorageResourceId);
- RdsContainerHandle handle = containerManager.start(
+ RdsContainerHandle handle = containerManager.tryStart(
dbInstanceArn, id, instanceStorageResourceId,
instanceDockerVolumeName, engine, image,
masterUsername, masterPassword, dbName);
- backendHost = handle.getHost();
- backendPort = handle.getPort();
- containerId = handle.getContainerId();
- containerHost = handle.getHost();
- containerPort = handle.getPort();
+ if (handle != null) {
+ backendHost = handle.getHost();
+ backendPort = handle.getPort();
+ containerId = handle.getContainerId();
+ containerHost = handle.getHost();
+ containerPort = handle.getPort();
+ } else {
+ LOG.warnv("DB instance {0} created without a backing database container: no "
+ + "Docker daemon is reachable. Metadata operations work; connections to "
+ + "the database do not until a daemon appears.", id);
+ }
}
}
@@ -664,7 +672,7 @@ public class RdsService implements Resettable, ResourceProvider {
attachManagedMasterUserSecret(instance, effectiveRegion, masterUserSecretKmsKeyId);
}
- if (!mock) {
+ if (!mock && hasBackend(backendHost, backendPort)) {
final String accountId = accountIdFromArn(instance.getDbInstanceArn());
final String instanceRegion = regionFromArn(instance.getDbInstanceArn());
proxyManager.startProxy(rdsResourceRelayKey(instance.getDbInstanceArn(), id),
@@ -1374,15 +1382,15 @@ public class RdsService implements Resettable, ResourceProvider {
String image = imageForEngine(instance.getEngine(), instance.getEngineVersion());
String storageResourceId = resolvedInstanceStorageResourceId(instance);
String dockerVolumeName = resolvedInstanceDockerVolumeName(instance);
- RdsContainerHandle handle = containerManager.start(
+ RdsContainerHandle handle = containerManager.tryStart(
instance.getDbInstanceArn(), id, storageResourceId,
dockerVolumeName, instance.getEngine(), image, instance.getMasterUsername(),
instance.getMasterPassword(), instance.getDbName());
instance.setContainerStorageResourceId(storageResourceId);
instance.setDockerVolumeName(dockerVolumeName);
- instance.setContainerId(handle.getContainerId());
- instance.setContainerHost(handle.getHost());
- instance.setContainerPort(handle.getPort());
+ instance.setContainerId(handle != null ? handle.getContainerId() : null);
+ instance.setContainerHost(handle != null ? handle.getHost() : null);
+ instance.setContainerPort(handle != null ? handle.getPort() : 0);
}
}
@@ -1390,24 +1398,190 @@ public class RdsService implements Resettable, ResourceProvider {
putInstanceForScope(currentAccountId(), effectiveRegion, id, instance);
if (!mock) {
- String effectiveMasterUser = instance.getMasterUsername() != null
- ? instance.getMasterUsername() : "root";
- final String accountId = accountIdFromArn(instance.getDbInstanceArn());
- final String instanceRegion = regionFromArn(instance.getDbInstanceArn());
+ if (hasBackend(instance.getContainerHost(), instance.getContainerPort())) {
+ String effectiveMasterUser = instance.getMasterUsername() != null
+ ? instance.getMasterUsername() : "root";
+ final String accountId = accountIdFromArn(instance.getDbInstanceArn());
+ final String instanceRegion = regionFromArn(instance.getDbInstanceArn());
+ proxyManager.startProxy(rdsResourceRelayKey(instance.getDbInstanceArn(), id),
+ instance.getEngine(),
+ instance.isIamDatabaseAuthenticationEnabled(),
+ instance.getProxyPort(), instance.getContainerHost(), instance.getContainerPort(),
+ instance.getEndpoint().address(),
+ effectiveMasterUser, instance.getMasterPassword(), instance.getDbName(),
+ (user, pw) -> validateDbPasswordForScope(
+ accountId, instanceRegion, id, user, pw));
+ } else {
+ // No backing container: created or last rebooted while no daemon was reachable.
+ instance = ensureInstanceBackend(id, effectiveRegion);
+ }
+ }
+
+ LOG.infov("DB instance {0} rebooted", id);
+ return instance;
+ }
+
+ private static boolean hasBackend(String host, int port) {
+ return host != null && !host.isBlank() && port > 0;
+ }
+
+ /**
+ * Whether a delete has anything for Docker to clean up. A record with a container, or with a
+ * cleanup identity retained from an earlier failure, always has. One with neither was created
+ * or restored while no daemon was reachable, and only a reachable daemon could remove whatever
+ * might still exist for it.
+ */
+ private boolean backendCleanupPossible(String containerId, String runtimeArn) {
+ return containerId != null
+ || containerManager.getActiveHandle(runtimeArn) != null
+ || containerManager.isDockerReachable();
+ }
+
+ /**
+ * Starts the backing database container and auth proxy for a DB instance recorded without
+ * one, because no Docker daemon was reachable when it was created, rebooted or restored.
+ * Every operation that needs the live database calls this first, so the backend comes up as
+ * soon as a daemon appears. Instances that already have a backend, and mock-mode instances,
+ * are returned untouched. The record is only updated once both halves are up, so a proxy
+ * failure leaves it retrying next time instead of pointing at a container nothing listens on.
+ *
+ * @return the instance, with its container fields populated when a backend became available
+ */
+ public synchronized DbInstance ensureInstanceBackend(String id, String region) {
+ String effectiveRegion = effectiveRegion(region);
+ DbInstance instance = getDbInstance(id, effectiveRegion);
+ if (config.services().rds().mock()
+ || hasBackend(instance.getContainerHost(), instance.getContainerPort())) {
+ return instance;
+ }
+
+ String clusterId = instance.getDbClusterIdentifier();
+ RdsContainerHandle started = null;
+ String backendContainerId;
+ String backendHost;
+ int backendPort;
+ if (clusterId != null && !clusterId.isBlank()) {
+ DbCluster cluster = ensureClusterBackend(clusterId, effectiveRegion);
+ if (!hasBackend(cluster.getContainerHost(), cluster.getContainerPort())) {
+ return instance;
+ }
+ backendContainerId = cluster.getContainerId();
+ backendHost = cluster.getContainerHost();
+ backendPort = cluster.getContainerPort();
+ } else {
+ String image = imageForEngine(instance.getEngine(), instance.getEngineVersion());
+ started = containerManager.tryStart(
+ instance.getDbInstanceArn(), id, resolvedInstanceStorageResourceId(instance),
+ resolvedInstanceDockerVolumeName(instance), instance.getEngine(), image,
+ instance.getMasterUsername(), instance.getMasterPassword(), instance.getDbName());
+ if (started == null) {
+ return instance;
+ }
+ backendContainerId = started.getContainerId();
+ backendHost = started.getHost();
+ backendPort = started.getPort();
+ }
+
+ String effectiveMasterUser = instance.getMasterUsername() != null
+ ? instance.getMasterUsername() : "root";
+ final String accountId = accountIdFromArn(instance.getDbInstanceArn());
+ final String instanceRegion = regionFromArn(instance.getDbInstanceArn());
+ try {
proxyManager.startProxy(rdsResourceRelayKey(instance.getDbInstanceArn(), id),
- instance.getEngine(),
- instance.isIamDatabaseAuthenticationEnabled(),
- instance.getProxyPort(), instance.getContainerHost(), instance.getContainerPort(),
+ instance.getEngine(), instance.isIamDatabaseAuthenticationEnabled(),
+ instance.getProxyPort(), backendHost, backendPort,
instance.getEndpoint().address(),
effectiveMasterUser, instance.getMasterPassword(), instance.getDbName(),
(user, pw) -> validateDbPasswordForScope(
accountId, instanceRegion, id, user, pw));
+ } catch (RuntimeException e) {
+ stopStartedBackend(started, e);
+ throw e;
}
-
- LOG.infov("DB instance {0} rebooted", id);
+ if (started != null) {
+ instance.setContainerStorageResourceId(resolvedInstanceStorageResourceId(instance));
+ instance.setDockerVolumeName(resolvedInstanceDockerVolumeName(instance));
+ }
+ instance.setContainerId(backendContainerId);
+ instance.setContainerHost(backendHost);
+ instance.setContainerPort(backendPort);
+ putInstanceForScope(currentAccountId(), effectiveRegion, id, instance);
+ LOG.infov("Backing database container for DB instance {0} started on retry", id);
return instance;
}
+ /**
+ * The {@link #ensureInstanceBackend} counterpart for DB clusters.
+ *
+ * @return the cluster, with its container fields populated when a backend became available
+ */
+ public synchronized DbCluster ensureClusterBackend(String id, String region) {
+ String effectiveRegion = effectiveRegion(region);
+ DbCluster cluster = getDbCluster(id, effectiveRegion);
+ if (config.services().rds().mock()
+ || hasBackend(cluster.getContainerHost(), cluster.getContainerPort())) {
+ return cluster;
+ }
+
+ String image = imageForEngine(cluster.getEngine(), cluster.getEngineVersion());
+ String storageResourceId = resolvedClusterStorageResourceId(cluster);
+ String dockerVolumeName = resolvedClusterDockerVolumeName(cluster);
+ RdsContainerHandle started = containerManager.tryStart(
+ cluster.getDbClusterArn(), id, storageResourceId, dockerVolumeName,
+ cluster.getEngine(), image, cluster.getMasterUsername(),
+ cluster.getMasterPassword(), cluster.getDatabaseName());
+ if (started == null) {
+ return cluster;
+ }
+
+ String effectiveMasterUser = cluster.getMasterUsername() != null
+ ? cluster.getMasterUsername() : "root";
+ final String accountId = accountIdFromArn(cluster.getDbClusterArn());
+ final String clusterRegion = regionFromArn(cluster.getDbClusterArn());
+ try {
+ proxyManager.startProxy(rdsResourceRelayKey(cluster.getDbClusterArn(), id),
+ cluster.getEngine(), cluster.isIamDatabaseAuthenticationEnabled(),
+ cluster.getProxyPort(), started.getHost(), started.getPort(),
+ cluster.getEndpoint().address(),
+ effectiveMasterUser, cluster.getMasterPassword(), cluster.getDatabaseName(),
+ (user, pw) -> validateDbClusterPasswordForScope(
+ accountId, clusterRegion, id, user, pw));
+ } catch (RuntimeException e) {
+ stopStartedBackend(started, e);
+ throw e;
+ }
+ cluster.setContainerStorageResourceId(storageResourceId);
+ cluster.setDockerVolumeName(dockerVolumeName);
+ cluster.setContainerId(started.getContainerId());
+ cluster.setContainerHost(started.getHost());
+ cluster.setContainerPort(started.getPort());
+ putClusterForScope(currentAccountId(), effectiveRegion, id, cluster);
+ LOG.infov("Backing database container for DB cluster {0} started on retry", id);
+ return cluster;
+ }
+
+ private void stopStartedBackend(RdsContainerHandle started, RuntimeException proxyFailure) {
+ if (started == null) {
+ return;
+ }
+ try {
+ containerManager.stop(started);
+ } catch (RuntimeException | Error stopFailure) {
+ proxyFailure.addSuppressed(stopFailure);
+ LOG.warnv(stopFailure, "Failed to stop container {0} after its auth proxy did not start",
+ started.getContainerId());
+ }
+ }
+
+ /**
+ * Whether Floci can reach a Docker daemon at all. The RDS data plane is a real database
+ * connection, which cannot be emulated without one, so callers use this to raise a modelled
+ * error naming the missing daemon instead of a generic runtime failure.
+ */
+ public boolean isBackendRuntimeAvailable() {
+ return config.services().rds().mock() || containerManager.isDockerReachable();
+ }
+
public synchronized void deleteDbInstance(String id) {
deleteDbInstance(id, regionResolver.getDefaultRegion());
}
@@ -1435,8 +1609,11 @@ public class RdsService implements Resettable, ResourceProvider {
String clusterId = instance.getDbClusterIdentifier();
if (clusterId == null || clusterId.isBlank()) {
- // Standalone — stop its container and clean up its Docker volume (neither exists in mock mode)
- if (!mock) {
+ // Standalone, so stop its container and clean up its Docker volume. Neither exists in
+ // mock mode, and an instance with no container and no reachable daemon has nothing
+ // Docker could clean up, so its delete stays pure metadata.
+ if (!mock && backendCleanupPossible(
+ instance.getContainerId(), instance.getDbInstanceArn())) {
if (instance.getContainerId() != null) {
containerManager.stop(buildHandle(instance));
} else {
@@ -1585,13 +1762,19 @@ public class RdsService implements Resettable, ResourceProvider {
String clusterVolumeId = String.format("%06x", new SecureRandom().nextInt(0xFFFFFF));
String clusterDockerVolumeName = newVolumeName(
clusterVolumeId, clusterResourceId);
- RdsContainerHandle handle = containerManager.start(
+ RdsContainerHandle handle = containerManager.tryStart(
clusterArn, id, clusterResourceId, clusterDockerVolumeName,
engine, image,
masterUsername, masterPassword, databaseName);
- cluster.setContainerId(handle.getContainerId());
- cluster.setContainerHost(handle.getHost());
- cluster.setContainerPort(handle.getPort());
+ if (handle != null) {
+ cluster.setContainerId(handle.getContainerId());
+ cluster.setContainerHost(handle.getHost());
+ cluster.setContainerPort(handle.getPort());
+ } else {
+ LOG.warnv("DB cluster {0} created without a backing database container: no Docker "
+ + "daemon is reachable. Metadata operations work; connections to the "
+ + "database do not until a daemon appears.", id);
+ }
cluster.setVolumeId(clusterVolumeId);
cluster.setDockerVolumeName(clusterDockerVolumeName);
}
@@ -1609,7 +1792,7 @@ public class RdsService implements Resettable, ResourceProvider {
}
try {
- if (!mock) {
+ if (!mock && hasBackend(cluster.getContainerHost(), cluster.getContainerPort())) {
String effectiveMasterUser = masterUsername != null ? masterUsername : "root";
final String accountId = accountIdFromArn(cluster.getDbClusterArn());
final String clusterRegion = regionFromArn(cluster.getDbClusterArn());
@@ -1919,15 +2102,17 @@ public class RdsService implements Resettable, ResourceProvider {
if (!config.services().rds().mock()) {
proxyManager.stopProxy(rdsResourceRelayKey(cluster.getDbClusterArn(), id));
- if (cluster.getContainerId() != null) {
- containerManager.stop(buildClusterHandle(cluster));
- } else {
- containerManager.stopByRuntimeId(cluster.getDbClusterArn());
+ if (backendCleanupPossible(cluster.getContainerId(), cluster.getDbClusterArn())) {
+ if (cluster.getContainerId() != null) {
+ containerManager.stop(buildClusterHandle(cluster));
+ } else {
+ containerManager.stopByRuntimeId(cluster.getDbClusterArn());
+ }
+ containerManager.removeVolume(
+ cluster.getDbClusterArn(),
+ resolvedClusterStorageResourceId(cluster),
+ resolvedClusterDockerVolumeName(cluster));
}
- containerManager.removeVolume(
- cluster.getDbClusterArn(),
- resolvedClusterStorageResourceId(cluster),
- resolvedClusterDockerVolumeName(cluster));
}
releaseProxyPort(cluster.getProxyPort());
@@ -4246,25 +4431,29 @@ public class RdsService implements Resettable, ResourceProvider {
cluster.setEndpoint(endpoint);
cluster.setReaderEndpoint(endpoint);
String image = imageForEngine(cluster.getEngine(), cluster.getEngineVersion());
- restoredHandle = containerManager.start(
+ restoredHandle = containerManager.tryStart(
cluster.getDbClusterArn(), cluster.getDbClusterIdentifier(),
storageResourceId, dockerVolumeName, cluster.getEngine(), image,
cluster.getMasterUsername(), cluster.getMasterPassword(), cluster.getDatabaseName());
- cluster.setContainerId(restoredHandle.getContainerId());
- cluster.setContainerHost(restoredHandle.getHost());
- cluster.setContainerPort(restoredHandle.getPort());
-
- String effectiveMasterUser = cluster.getMasterUsername() != null
- ? cluster.getMasterUsername() : "root";
- proxyManager.startProxy(rdsResourceRelayKey(
- cluster.getDbClusterArn(), cluster.getDbClusterIdentifier()),
- cluster.getEngine(),
- cluster.isIamDatabaseAuthenticationEnabled(), proxyPort,
- restoredHandle.getHost(), restoredHandle.getPort(), cluster.getEndpoint().address(),
- effectiveMasterUser, cluster.getMasterPassword(), cluster.getDatabaseName(),
- (user, pw) -> validateDbClusterPasswordForScope(
- accountId, clusterRegion,
- cluster.getDbClusterIdentifier(), user, pw));
+ cluster.setContainerId(restoredHandle != null ? restoredHandle.getContainerId() : null);
+ cluster.setContainerHost(restoredHandle != null ? restoredHandle.getHost() : null);
+ cluster.setContainerPort(restoredHandle != null ? restoredHandle.getPort() : 0);
+
+ // With no reachable daemon the record survives the restart without a container;
+ // the container is retried the next time something needs the live database.
+ if (restoredHandle != null) {
+ String effectiveMasterUser = cluster.getMasterUsername() != null
+ ? cluster.getMasterUsername() : "root";
+ proxyManager.startProxy(rdsResourceRelayKey(
+ cluster.getDbClusterArn(), cluster.getDbClusterIdentifier()),
+ cluster.getEngine(),
+ cluster.isIamDatabaseAuthenticationEnabled(), proxyPort,
+ restoredHandle.getHost(), restoredHandle.getPort(), cluster.getEndpoint().address(),
+ effectiveMasterUser, cluster.getMasterPassword(), cluster.getDatabaseName(),
+ (user, pw) -> validateDbClusterPasswordForScope(
+ accountId, clusterRegion,
+ cluster.getDbClusterIdentifier(), user, pw));
+ }
cluster.setStatus(DbInstanceStatus.AVAILABLE);
putClusterForScope(accountId, clusterRegion,
cluster.getDbClusterIdentifier(), cluster);
@@ -4354,7 +4543,10 @@ public class RdsService implements Resettable, ResourceProvider {
}
backendHost = cluster.getContainerHost();
backendPort = cluster.getContainerPort();
- if (backendHost == null || backendPort <= 0) {
+ // A cluster restored 'available' without a container had no reachable daemon;
+ // its members share that state and are retried together with it.
+ if (!hasBackend(backendHost, backendPort)
+ && cluster.getStatus() != DbInstanceStatus.AVAILABLE) {
throw new AwsException("InvalidDBClusterStateFault",
"DB cluster " + clusterId + " runtime is not available.", 400);
}
@@ -4363,29 +4555,31 @@ public class RdsService implements Resettable, ResourceProvider {
instance.setContainerPort(cluster.getContainerPort());
} else {
String image = imageForEngine(instance.getEngine(), instance.getEngineVersion());
- restoredHandle = containerManager.start(
+ restoredHandle = containerManager.tryStart(
instance.getDbInstanceArn(), instance.getDbInstanceIdentifier(),
instance.getContainerStorageResourceId(),
instance.getDockerVolumeName(), instance.getEngine(), image,
instance.getMasterUsername(), instance.getMasterPassword(), instance.getDbName());
- backendHost = restoredHandle.getHost();
- backendPort = restoredHandle.getPort();
- instance.setContainerId(restoredHandle.getContainerId());
- instance.setContainerHost(restoredHandle.getHost());
- instance.setContainerPort(restoredHandle.getPort());
+ backendHost = restoredHandle != null ? restoredHandle.getHost() : null;
+ backendPort = restoredHandle != null ? restoredHandle.getPort() : 0;
+ instance.setContainerId(restoredHandle != null ? restoredHandle.getContainerId() : null);
+ instance.setContainerHost(backendHost);
+ instance.setContainerPort(backendPort);
}
- String effectiveMasterUser = instance.getMasterUsername() != null
- ? instance.getMasterUsername() : "root";
- proxyManager.startProxy(rdsResourceRelayKey(
- instance.getDbInstanceArn(), instance.getDbInstanceIdentifier()),
- instance.getEngine(),
- instance.isIamDatabaseAuthenticationEnabled(), proxyPort,
- backendHost, backendPort, instance.getEndpoint().address(),
- effectiveMasterUser, instance.getMasterPassword(), instance.getDbName(),
- (user, pw) -> validateDbPasswordForScope(
- accountId, instanceRegion,
- instance.getDbInstanceIdentifier(), user, pw));
+ if (hasBackend(backendHost, backendPort)) {
+ String effectiveMasterUser = instance.getMasterUsername() != null
+ ? instance.getMasterUsername() : "root";
+ proxyManager.startProxy(rdsResourceRelayKey(
+ instance.getDbInstanceArn(), instance.getDbInstanceIdentifier()),
+ instance.getEngine(),
+ instance.isIamDatabaseAuthenticationEnabled(), proxyPort,
+ backendHost, backendPort, instance.getEndpoint().address(),
+ effectiveMasterUser, instance.getMasterPassword(), instance.getDbName(),
+ (user, pw) -> validateDbPasswordForScope(
+ accountId, instanceRegion,
+ instance.getDbInstanceIdentifier(), user, pw));
+ }
instance.setStatus(DbInstanceStatus.AVAILABLE);
putInstanceForScope(accountId, instanceRegion,
instance.getDbInstanceIdentifier(), instance);
diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java
index e520efe8db..d4ca14fb60 100644
--- a/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java
@@ -57,6 +57,8 @@ public class RdsContainerManager {
private final Map<String, RdsContainerHandle> activeContainers = new ConcurrentHashMap<>();
private final Map<String, String> activeStorageOwners = new ConcurrentHashMap<>();
private final Set<String> claimedRuntimes = ConcurrentHashMap.newKeySet();
+ private final Set<String> cleanupPending = ConcurrentHashMap.newKeySet();
+ private volatile boolean dockerUnavailableLogged;
@Inject
public RdsContainerManager(ContainerBuilder containerBuilder,
@@ -91,6 +93,86 @@ public class RdsContainerManager {
engine, image, masterUsername, masterPassword, dbName);
}
+ /**
+ * Attempts {@link #start} and reports the backend as unavailable instead of propagating the
+ * failure when the cause is that no Docker daemon is reachable from Floci: Floci running
+ * inside Docker without a mounted socket, or a stopped daemon on the host. A failure raised
+ * while the daemon <em>is</em> reachable is a genuine container problem and still propagates,
+ * so nothing changes for a Floci that can start database containers.
+ * <p>
+ * A handle retained by an earlier cleanup failure is cleaned up first, so a container
+ * created just before the daemon went away, or one whose stop failed after its proxy did
+ * not start, is removed once the daemon is back instead of blocking the runtime's next
+ * start. An identity that names only the fixed container name came from a
+ * start that failed before Docker created anything, and is dropped when the daemon is
+ * unreachable so the runtime can retry.
+ *
+ * @return the container handle, or {@code null} when no Docker daemon is reachable
+ */
+ public RdsContainerHandle tryStart(
+ String runtimeId, String instanceId, String containerStorageResourceId,
+ String dockerVolumeName, DatabaseEngine engine, String image,
+ String masterUsername, String masterPassword, String dbName) {
+ String effectiveRuntimeId = runtimeId == null || runtimeId.isBlank() ? instanceId : runtimeId;
+ try {
+ retryRetainedCleanup(effectiveRuntimeId);
+ RdsContainerHandle handle = start(runtimeId, instanceId, containerStorageResourceId,
+ dockerVolumeName, engine, image, masterUsername, masterPassword, dbName);
+ dockerUnavailableLogged = false;
+ return handle;
+ } catch (RuntimeException e) {
+ if (isDockerReachable()) {
+ throw e;
+ }
+ discardNeverCreatedIdentity(effectiveRuntimeId, dockerVolumeName);
+ if (!dockerUnavailableLogged) {
+ dockerUnavailableLogged = true;
+ LOG.warnv("No Docker daemon is reachable from Floci ({0}). RDS metadata operations "
+ + "keep working and DB instances still reach 'available', but they have no "
+ + "backing database container until a daemon becomes reachable.",
+ e.getMessage());
+ }
+ return null;
+ }
+ }
+
+ /**
+ * Probes the configured Docker endpoint, which is how a missing daemon is told apart from a
+ * container that failed for its own reasons.
+ */
+ public boolean isDockerReachable() {
+ try {
+ lifecycleManager.getDockerClient().pingCmd().exec();
+ return true;
+ } catch (Exception e) {
+ LOG.debugv("Docker daemon is not reachable: {0}", e.getMessage());
+ return false;
+ }
+ }
+
+ /**
+ * A handle retained because its cleanup failed, whether from a failed start or from a stop
+ * that could not remove the container, is cleaned up before the runtime starts again. A live
+ * handle that is not pending cleanup is left alone, so a concurrent duplicate start is still
+ * rejected by {@link #start}.
+ */
+ private void retryRetainedCleanup(String runtimeId) {
+ RdsContainerHandle retained = activeContainers.get(runtimeId);
+ if (retained != null && cleanupPending.contains(runtimeId)) {
+ stop(retained);
+ }
+ }
+
+ private void discardNeverCreatedIdentity(String runtimeId, String containerName) {
+ RdsContainerHandle retained = activeContainers.get(runtimeId);
+ if (retained == null || retained.getHost() != null
+ || !retained.getContainerId().equals(containerName)) {
+ return;
+ }
+ activeContainers.remove(runtimeId, retained);
+ releaseOwnership(retained.getStorageKey(), retained.getContainerKey(), runtimeId);
+ }
+
public RdsContainerHandle start(
String runtimeId, String instanceId, String containerStorageResourceId,
String dockerVolumeName, DatabaseEngine engine, String image,
@@ -220,6 +302,7 @@ public class RdsContainerManager {
cleanupContainerId, effectiveRuntimeId, instanceId,
null, 0, storageKey, containerKey);
activeContainers.put(effectiveRuntimeId, retained);
+ cleanupPending.add(effectiveRuntimeId);
LOG.errorv(cleanupFailure,
"Failed to clean up RDS container {0}; retaining storage ownership for {1}",
cleanupContainerId, effectiveRuntimeId);
@@ -272,6 +355,7 @@ public class RdsContainerManager {
} catch (RuntimeException | Error e) {
activeContainers.putIfAbsent(effectiveHandle.getRuntimeId(), effectiveHandle);
claimedRuntimes.add(effectiveHandle.getRuntimeId());
+ cleanupPending.add(effectiveHandle.getRuntimeId());
throw e;
}
activeContainers.remove(effectiveHandle.getRuntimeId(), effectiveHandle);
@@ -624,6 +708,7 @@ public class RdsContainerManager {
activeStorageOwners.remove(containerKey, runtimeId);
}
claimedRuntimes.remove(runtimeId);
+ cleanupPending.remove(runtimeId);
}
private static String requireSafeStorageComponent(String value, String label) {
diff --git a/src/main/java/io/github/hectorvent/floci/services/rdsdata/RdsDataResourceResolver.java b/src/main/java/io/github/hectorvent/floci/services/rdsdata/RdsDataResourceResolver.java
index 0dd0b505b7..818d70b4e5 100644
--- a/src/main/java/io/github/hectorvent/floci/services/rdsdata/RdsDataResourceResolver.java
+++ b/src/main/java/io/github/hectorvent/floci/services/rdsdata/RdsDataResourceResolver.java
@@ -28,6 +28,9 @@ class RdsDataResourceResolver {
throw new AwsException("BadRequestException", "resourceArn is required.", 400);
}
+ DbCluster cluster = null;
+ DbInstance instance = null;
+ String region = null;
try {
AwsArnUtils.Arn arn = AwsArnUtils.parse(resourceArn);
if (!"rds".equals(arn.service())) {
@@ -43,38 +46,66 @@ class RdsDataResourceResolver {
}
String type = arn.resource().substring(0, separator);
String id = arn.resource().substring(separator + 1);
+ region = arn.region();
if ("cluster".equals(type)) {
- DbCluster cluster = rdsService.getDbCluster(id, arn.region());
- if (resourceArn.equals(cluster.getDbClusterArn())) {
- return fromCluster(cluster);
+ DbCluster found = rdsService.getDbCluster(id, region);
+ if (resourceArn.equals(found.getDbClusterArn())) {
+ cluster = found;
}
} else if ("db".equals(type)) {
- DbInstance instance = rdsService.getDbInstance(id, arn.region());
- if (resourceArn.equals(instance.getDbInstanceArn())) {
- return fromInstance(instance);
+ DbInstance found = rdsService.getDbInstance(id, region);
+ if (resourceArn.equals(found.getDbInstanceArn())) {
+ instance = found;
}
}
} catch (AwsException | IllegalArgumentException ignored) {
// Normalize lookup and ARN parsing failures to the RDS Data API error shape below.
}
+ if (cluster != null) {
+ return fromCluster(cluster, region);
+ }
+ if (instance != null) {
+ return fromInstance(instance, region);
+ }
throw new AwsException("BadRequestException",
"resourceArn does not resolve to a local RDS resource: " + resourceArn, 400);
}
- private static DatabaseTarget fromCluster(DbCluster cluster) {
- return target(cluster.getDbClusterArn(), cluster.getEngine(), cluster.getContainerHost(), cluster.getContainerPort(),
- cluster.getMasterUsername(), cluster.getMasterPassword(), cluster.getDatabaseName());
+ private DatabaseTarget fromCluster(DbCluster cluster, String region) {
+ DbCluster resolved = hasRuntime(cluster.getContainerHost(), cluster.getContainerPort())
+ ? cluster
+ : rdsService.ensureClusterBackend(cluster.getDbClusterIdentifier(), region);
+ return target(resolved.getDbClusterArn(), resolved.getEngine(), resolved.getContainerHost(),
+ resolved.getContainerPort(), resolved.getMasterUsername(), resolved.getMasterPassword(),
+ resolved.getDatabaseName());
}
- private static DatabaseTarget fromInstance(DbInstance instance) {
- return target(instance.getDbInstanceArn(), instance.getEngine(), instance.getContainerHost(), instance.getContainerPort(),
- instance.getMasterUsername(), instance.getMasterPassword(), instance.getDbName());
+ private DatabaseTarget fromInstance(DbInstance instance, String region) {
+ DbInstance resolved = hasRuntime(instance.getContainerHost(), instance.getContainerPort())
+ ? instance
+ : rdsService.ensureInstanceBackend(instance.getDbInstanceIdentifier(), region);
+ return target(resolved.getDbInstanceArn(), resolved.getEngine(), resolved.getContainerHost(),
+ resolved.getContainerPort(), resolved.getMasterUsername(), resolved.getMasterPassword(),
+ resolved.getDbName());
}
- private static DatabaseTarget target(String arn, DatabaseEngine engine, String host, int port,
- String username, String password, String databaseName) {
- if (host == null || host.isBlank() || port <= 0) {
+ private static boolean hasRuntime(String host, int port) {
+ return host != null && !host.isBlank() && port > 0;
+ }
+
+ private DatabaseTarget target(String arn, DatabaseEngine engine, String host, int port,
+ String username, String password, String databaseName) {
+ if (!hasRuntime(host, port)) {
+ // The Data API is RDS's data plane: it needs a real database, which Floci can only
+ // provide through a Docker container. Name the missing daemon rather than reporting a
+ // generic runtime failure, in the Data API's modelled server-side error shape.
+ if (!rdsService.isBackendRuntimeAvailable()) {
+ throw new AwsException("InternalServerErrorException",
+ "The RDS backing database is unavailable because no Docker daemon is reachable "
+ + "from Floci. DB instance and cluster metadata operations are supported; "
+ + "Data API execution requires Docker.", 500);
+ }
throw new AwsException("BadRequestException",
"RDS resource runtime is not available for Data API execution.", 400);
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java
index cca15858e5..06f6e50816 100644
--- a/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java
@@ -139,7 +139,7 @@ class RdsServiceTest {
new InMemoryStorage<>(), new InMemoryStorage<>(),
new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>());
- when(containerManager.start(any(), any(), any(), any(), any(), any(), any(), any(), any()))
+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))
.thenReturn(new RdsContainerHandle("cont-id", "id", "localhost", 5432));
when(ec2Service.resolveDefaultVpcId(any()))
.thenAnswer(invocation -> Ec2Service.defaultVpcId(invocation.getArgument(0)));
@@ -220,7 +220,7 @@ class RdsServiceTest {
rdsService.createDbCluster("dup-cluster", "postgres", "17.5",
"admin", "password", "dbname", false, null, null, null, false));
assertEquals("DBClusterAlreadyExistsFault", exception.getErrorCode());
- verify(containerManager, times(1)).start(any(), any(), any(), any(), any(), any(), any(), any(), any());
+ verify(containerManager, times(1)).tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());
}
/**
@@ -234,7 +234,7 @@ class RdsServiceTest {
void createDbClusterRejectsADuplicateWhileTheFirstIsStillProvisioning() throws Exception {
CountDownLatch insideStart = new CountDownLatch(1);
CountDownLatch releaseStart = new CountDownLatch(1);
- when(containerManager.start(any(), any(), any(), any(), any(), any(), any(), any(), any()))
+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))
.thenAnswer(invocation -> {
insideStart.countDown();
assertTrue(releaseStart.await(5, TimeUnit.SECONDS), "test released the latch");
@@ -259,7 +259,7 @@ class RdsServiceTest {
} finally {
executor.shutdownNow();
}
- verify(containerManager, times(1)).start(any(), any(), any(), any(), any(), any(), any(), any(), any());
+ verify(containerManager, times(1)).tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());
assertNotNull(rdsService.getDbCluster("racy-cluster"));
}
@@ -267,7 +267,7 @@ class RdsServiceTest {
void createDbInstanceRejectsADuplicateWhileTheFirstIsStillProvisioning() throws Exception {
CountDownLatch insideStart = new CountDownLatch(1);
CountDownLatch releaseStart = new CountDownLatch(1);
- when(containerManager.start(any(), any(), any(), any(), any(), any(), any(), any(), any()))
+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))
.thenAnswer(invocation -> {
insideStart.countDown();
assertTrue(releaseStart.await(5, TimeUnit.SECONDS), "test released the latch");
@@ -293,7 +293,7 @@ class RdsServiceTest {
} finally {
executor.shutdownNow();
}
- verify(containerManager, times(1)).start(any(), any(), any(), any(), any(), any(), any(), any(), any());
+ verify(containerManager, times(1)).tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());
assertNotNull(rdsService.getDbInstance("racy-db"));
}
@@ -307,7 +307,7 @@ class RdsServiceTest {
CountDownLatch insideStart = new CountDownLatch(1);
CountDownLatch releaseStart = new CountDownLatch(1);
AtomicBoolean firstCall = new AtomicBoolean(true);
- when(containerManager.start(any(), any(), any(), any(), any(), any(), any(), any(), any()))
+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))
.thenAnswer(invocation -> {
if (firstCall.getAndSet(false)) {
insideStart.countDown();
@@ -342,7 +342,7 @@ class RdsServiceTest {
rdsService.deleteDbCluster("reused-cluster");
assertNotNull(rdsService.createDbCluster("reused-cluster", "postgres", "17.5",
"admin", "password", "dbname", false, null, null, null, false));
- verify(containerManager, times(2)).start(any(), any(), any(), any(), any(), any(), any(), any(), any());
+ verify(containerManager, times(2)).tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());
}
/** The instance-side twin: a successful create must release the identifier too. */
@@ -355,7 +355,7 @@ class RdsServiceTest {
assertNotNull(rdsService.createDbInstance("reused-db", "postgres", "17.5",
"admin", "password", "dbname", "db.t3.micro",
20, false, null, null, null, null, false));
- verify(containerManager, times(2)).start(any(), any(), any(), any(), any(), any(), any(), any(), any());
+ verify(containerManager, times(2)).tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());
}
/**
@@ -405,7 +405,7 @@ class RdsServiceTest {
/** A failed create must release the identifier so a clean retry can proceed. */
@Test
void createDbClusterFailureReleasesTheIdentifierForRetry() {
- when(containerManager.start(any(), any(), any(), any(), any(), any(), any(), any(), any()))
+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))
.thenThrow(new RuntimeException("image pull failed"))
.thenReturn(new RdsContainerHandle("cont-id", "id", "localhost", 5432));
@@ -415,7 +415,7 @@ class RdsServiceTest {
assertNotNull(rdsService.createDbCluster("retry-cluster", "postgres", "17.5",
"admin", "password", "dbname", false, null, null, null, false));
- verify(containerManager, times(2)).start(any(), any(), any(), any(), any(), any(), any(), any(), any());
+ verify(containerManager, times(2)).tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());
}
@Test
@@ -424,7 +424,7 @@ class RdsServiceTest {
"admin", "password", "dbname", "db.t3.micro",
20, false, null, null, null);
- verify(containerManager).start(
+ verify(containerManager).tryStart(
eq("arn:aws:rds:us-east-1:123456789012:db:mydb"), eq("mydb"),
any(), any(), eq(DatabaseEngine.POSTGRES),
eq("postgres:18.1-alpine"), eq("admin"), eq("password"), eq("dbname"));
@@ -437,7 +437,7 @@ class RdsServiceTest {
rdsService.createDbCluster("cluster1", "postgres", "16.3",
"admin", "password", "dbname", false, null);
- verify(containerManager).start(
+ verify(containerManager).tryStart(
eq("arn:aws:rds:us-east-1:123456789012:cluster:cluster1"), eq("cluster1"),
any(), any(), eq(DatabaseEngine.POSTGRES),
eq("postgres:16.14-alpine3.23"), eq("admin"), eq("password"), eq("dbname"));
@@ -451,7 +451,7 @@ class RdsServiceTest {
rdsService.createDbCluster("cluster1", "postgres", "18.1",
"admin", "password", "dbname", false, null);
- verify(containerManager).start(
+ verify(containerManager).tryStart(
eq("arn:aws:rds:us-east-1:123456789012:cluster:cluster1"), eq("cluster1"),
any(), any(), eq(DatabaseEngine.POSTGRES),
eq(EmulatorConfig.RdsServiceConfig.DEFAULT_POSTGRES_IMAGE),
@@ -1246,7 +1246,7 @@ class RdsServiceTest {
20, false, null, "missing-subnet-group", null));
assertEquals("DBSubnetGroupNotFoundFault", exception.getErrorCode());
- verify(containerManager, never()).start(any(), any(), any(), any(), any(), any(), any(), any(), any());
+ verify(containerManager, never()).tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());
verify(proxyManager, never()).startProxy(any(), any(), anyBoolean(), anyInt(),
any(), anyInt(), any(), any(), any(), any(), any());
}
@@ -1311,7 +1311,7 @@ class RdsServiceTest {
assertEquals("localhost", cluster.getEndpoint().address());
assertTrue(cluster.getEndpoint().port() > 0);
assertNull(cluster.getContainerId());
- verify(containerManager, never()).start(any(), any(), any(), any(), any(), any(), any(), any(), any());
+ verify(containerManager, never()).tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());
verify(proxyManager, never()).startProxy(any(), any(), anyBoolean(), anyInt(), any(), anyInt(),
any(), any(), any(), any(), any());
}
@@ -1544,11 +1544,289 @@ class RdsServiceTest {
// No Docker volume name may be persisted: the mock cluster has a null volume id, so the
// fallback would fabricate a name that a later non-mock restore could try to reference.
assertNull(instance.getDockerVolumeName());
- verify(containerManager, never()).start(any(), any(), any(), any(), any(), any(), any(), any(), any());
+ verify(containerManager, never()).tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());
verify(proxyManager, never()).startProxy(any(), any(), anyBoolean(), anyInt(), any(), anyInt(),
any(), any(), any(), any(), any());
}
+ // ------------------------------------------------------------
+ // No Docker daemon reachable (Floci inside Docker with no mounted socket)
+ // ------------------------------------------------------------
+
+ @Test
+ void createDbInstanceSucceedsAsMetadataWhenNoDockerDaemonIsReachable() {
+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))
+ .thenReturn(null);
+
+ DbInstance instance = rdsService.createDbInstance("probe-db", "postgres", "16.3",
+ "probeadmin", "ProbePassw0rd!", "dbname", "db.t3.micro",
+ 20, false, null, null, null, false, null,
+ Map.of("tofu-estate", "probe1"));
+
+ assertEquals(DbInstanceStatus.AVAILABLE, instance.getStatus());
+ assertEquals("arn:aws:rds:us-east-1:123456789012:db:probe-db", instance.getDbInstanceArn());
+ assertNotNull(instance.getEndpoint());
+ assertEquals("probe1", instance.getTags().get("tofu-estate"));
+ // Nothing was started, so no runtime is claimed; the storage identity is kept for the retry.
+ assertNull(instance.getContainerId());
+ assertNull(instance.getContainerHost());
+ assertEquals(0, instance.getContainerPort());
+ assertNotNull(instance.getVolumeId());
+ verify(proxyManager, never()).startProxy(any(), any(), anyBoolean(), anyInt(), any(), anyInt(),
+ any(), any(), any(), any(), any());
+ }
+
+ @Test
+ void instanceMetadataCrudWorksWhenNoDockerDaemonIsReachable() {
+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))
+ .thenReturn(null);
+
+ DbInstance created = rdsService.createDbInstance("probe-db", "postgres", "16.3",
+ "probeadmin", "ProbePassw0rd!", "dbname", "db.t3.micro",
+ 20, false, null, null, null, false, null,
+ Map.of("tofu-estate", "probe1"));
+
+ assertEquals(1, rdsService.listDbInstances("probe-db").size());
+ assertEquals("probe1", rdsService.listTagsForResource(created.getDbInstanceArn()).get("tofu-estate"));
+
+ rdsService.addTagsToResource(created.getDbInstanceArn(), Map.of("Name", "probe-db"), "us-east-1");
+ assertEquals("probe-db", rdsService.listTagsForResource(created.getDbInstanceArn()).get("Name"));
+
+ assertEquals(DbInstanceStatus.AVAILABLE,
+ rdsService.modifyDbInstance("probe-db", "NewPassw0rd!", null, null).getStatus());
+ verify(containerManager, never()).rotateMasterPassword(any(), any(), any(), any(), any(), any());
+
+ rdsService.deleteDbInstance("probe-db");
+ assertThrows(AwsException.class, () -> rdsService.getDbInstance("probe-db"));
+ // Cleanup must not reach for a container or volume that was never created.
+ verify(containerManager, never()).stop(any());
+ verify(containerManager, never()).stopByRuntimeId(any());
+ verify(containerManager, never()).removeVolume(any(), any(), any());
+ }
+
+ @Test
+ void deleteStillRemovesTheVolumeWhenADaemonIsReachableAgain() {
+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))
+ .thenReturn(null);
+ DbInstance created = rdsService.createDbInstance("probe-db", "postgres", "16.3",
+ "admin", "password", "dbname", "db.t3.micro",
+ 20, false, null, null, null);
+
+ // The daemon is back, so whatever Docker holds for this record is cleaned up as before.
+ when(containerManager.isDockerReachable()).thenReturn(true);
+ rdsService.deleteDbInstance("probe-db");
+
+ verify(containerManager, never()).stop(any());
+ verify(containerManager).stopByRuntimeId(created.getDbInstanceArn());
+ verify(containerManager).removeVolume(
+ eq(created.getDbInstanceArn()), any(), eq(created.getDockerVolumeName()));
+ }
+
+ @Test
+ void createDbClusterAndMemberSucceedAsMetadataWhenNoDockerDaemonIsReachable() {
+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))
+ .thenReturn(null);
+
+ DbCluster cluster = rdsService.createDbCluster("probe-cluster", "aurora-postgresql", "16.3",
+ "admin", "password", "dbname", false, null);
+ DbInstance member = rdsService.createDbInstance("probe-member", "aurora-postgresql", "16.3",
+ "admin", "password", "dbname", "db.serverless",
+ 0, false, null, null, "probe-cluster");
+
+ assertEquals(DbInstanceStatus.AVAILABLE, cluster.getStatus());
+ assertNull(cluster.getContainerId());
+ assertNotNull(cluster.getVolumeId());
+ assertEquals(DbInstanceStatus.AVAILABLE, member.getStatus());
+ assertNull(member.getContainerId());
+ verify(proxyManager, never()).startProxy(any(), any(), anyBoolean(), anyInt(), any(), anyInt(),
+ any(), any(), any(), any(), any());
+
+ rdsService.deleteDbInstance("probe-member");
+ rdsService.deleteDbCluster("probe-cluster");
+ assertThrows(AwsException.class, () -> rdsService.getDbCluster("probe-cluster"));
+ verify(containerManager, never()).stop(any());
+ verify(containerManager, never()).removeVolume(any(), any(), any());
+ }
+
+ @Test
+ void backendIsRetriedOncePerCallSoTheContainerStartsWhenADaemonAppears() {
+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))
+ .thenReturn(null);
+ DbInstance created = rdsService.createDbInstance("probe-db", "postgres", "16.3",
+ "admin", "password", "dbname", "db.t3.micro",
+ 20, false, null, null, null);
+
+ assertNull(rdsService.ensureInstanceBackend("probe-db", "us-east-1").getContainerId());
+
+ // A Docker daemon appears.
+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))
+ .thenReturn(new RdsContainerHandle("late-container", created.getDbInstanceArn(),
+ "probe-db", "127.0.0.1", 15432));
+
+ DbInstance started = rdsService.ensureInstanceBackend("probe-db", "us-east-1");
+ assertEquals("late-container", started.getContainerId());
+ assertEquals("127.0.0.1", started.getContainerHost());
+ assertEquals(15432, started.getContainerPort());
+ assertEquals(created.getDockerVolumeName(), started.getDockerVolumeName());
+ // The create and both retries use the record's own storage identity.
+ verify(containerManager, times(3)).tryStart(
+ eq(created.getDbInstanceArn()), eq("probe-db"),
+ eq(created.getContainerStorageResourceId()), eq(created.getDockerVolumeName()),
+ eq(DatabaseEngine.POSTGRES), eq("postgres:16.3-alpine"),
+ eq("admin"), eq("password"), eq("dbname"));
+ verify(proxyManager).startProxy(
+ eq("rds-resource:" + created.getDbInstanceArn()), eq(DatabaseEngine.POSTGRES),
+ eq(false), eq(created.getProxyPort()), eq("127.0.0.1"), eq(15432), any(),
+ eq("admin"), eq("password"), eq("dbname"), any());
+
+ // Already backed, so a further call is a no-op rather than a second container. Three
+ // calls in total: the create, the retry that found no daemon, and the retry that started it.
+ rdsService.ensureInstanceBackend("probe-db", "us-east-1");
+ verify(containerManager, times(3))
+ .tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());
+ }
+
+ @Test
+ void retryLeavesTheRecordWithoutABackendWhenTheProxyDoesNotStart() {
+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))
+ .thenReturn(null);
+ DbInstance created = rdsService.createDbInstance("probe-db", "postgres", "16.3",
+ "admin", "password", "dbname", "db.t3.micro",
+ 20, false, null, null, null);
+ RdsContainerHandle late = new RdsContainerHandle("late-container", created.getDbInstanceArn(),
+ "probe-db", "127.0.0.1", 15432);
+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))
+ .thenReturn(late);
+ doThrow(new IllegalStateException("port in use")).doNothing()
+ .when(proxyManager).startProxy(any(), any(), anyBoolean(), anyInt(), any(), anyInt(),
+ any(), any(), any(), any(), any());
+
+ assertThrows(IllegalStateException.class,
+ () -> rdsService.ensureInstanceBackend("probe-db", "us-east-1"));
+
+ // The container is stopped and the record still says it has no backend, so the next
+ // call retries both halves instead of returning a container nothing listens on.
+ verify(containerManager).stop(late);
+ assertNull(rdsService.getDbInstance("probe-db").getContainerId());
+
+ DbInstance started = rdsService.ensureInstanceBackend("probe-db", "us-east-1");
+ assertEquals("late-container", started.getContainerId());
+ verify(containerManager, times(3))
+ .tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());
+ }
+
+ @Test
+ void clusterMemberBackendIsRetriedThroughItsCluster() {
+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))
+ .thenReturn(null);
+ DbCluster cluster = rdsService.createDbCluster("probe-cluster", "aurora-postgresql", "16.3",
+ "admin", "password", "dbname", false, null);
+ DbInstance member = rdsService.createDbInstance("probe-member", "aurora-postgresql", "16.3",
+ "admin", "password", "dbname", "db.serverless",
+ 0, false, null, null, "probe-cluster");
+
+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))
+ .thenReturn(new RdsContainerHandle("late-cluster-container", cluster.getDbClusterArn(),
+ "probe-cluster", "127.0.0.1", 15432));
+
+ DbInstance started = rdsService.ensureInstanceBackend("probe-member", "us-east-1");
+
+ assertEquals("late-cluster-container", started.getContainerId());
+ assertEquals("late-cluster-container",
+ rdsService.getDbCluster("probe-cluster").getContainerId());
+ verify(containerManager, times(2)).tryStart(
+ eq(cluster.getDbClusterArn()), eq("probe-cluster"), any(), any(),
+ eq(DatabaseEngine.POSTGRES), any(), eq("admin"), eq("password"), eq("dbname"));
+ verify(proxyManager).startProxy(
+ eq("rds-resource:" + cluster.getDbClusterArn()), any(), anyBoolean(),
+ eq(cluster.getProxyPort()), eq("127.0.0.1"), eq(15432), any(), any(), any(), any(), any());
+ verify(proxyManager).startProxy(
+ eq("rds-resource:" + member.getDbInstanceArn()), any(), anyBoolean(),
+ eq(member.getProxyPort()), eq("127.0.0.1"), eq(15432), any(), any(), any(), any(), any());
+ }
+
+ @Test
+ void rebootRetriesTheBackendForAnInstanceCreatedWithoutADaemon() {
+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))
+ .thenReturn(null);
+ DbInstance created = rdsService.createDbInstance("probe-db", "postgres", "16.3",
+ "admin", "password", "dbname", "db.t3.micro",
+ 20, false, null, null, null);
+
+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))
+ .thenReturn(new RdsContainerHandle("late-container", created.getDbInstanceArn(),
+ "probe-db", "127.0.0.1", 15432));
+
+ DbInstance rebooted = rdsService.rebootDbInstance("probe-db");
+
+ assertEquals(DbInstanceStatus.AVAILABLE, rebooted.getStatus());
+ assertEquals("late-container", rebooted.getContainerId());
+ // There was no container to stop, only one to start.
+ verify(containerManager, never()).stop(any());
+ verify(proxyManager).startProxy(
+ eq("rds-resource:" + created.getDbInstanceArn()), any(), anyBoolean(),
+ eq(created.getProxyPort()), eq("127.0.0.1"), eq(15432), any(), any(), any(), any(), any());
+ }
+
+ @Test
+ void restoreKeepsInstanceAvailableWhenNoDockerDaemonIsReachable() {
+ StorageBackend<String, DbInstance> instances = new InMemoryStorage<>();
+ StorageBackend<String, DbCluster> clusters = new InMemoryStorage<>();
+
+ RdsService initialService = newService(containerManager, proxyManager,
+ instances, clusters, new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>());
+ DbInstance created = initialService.createDbInstance("mydb", "postgres", "16.3",
+ "admin", "secret", "app", "db.t3.micro",
+ 20, false, null, null, null, null, false);
+
+ RdsContainerManager daemonlessContainerManager = mock(RdsContainerManager.class);
+ RdsProxyManager daemonlessProxyManager = mock(RdsProxyManager.class);
+ when(daemonlessContainerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))
+ .thenReturn(null);
+
+ RdsService restoredService = newService(daemonlessContainerManager, daemonlessProxyManager,
+ instances, clusters, new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>());
+ restoredService.restorePersistedRuntime();
+
+ DbInstance restored = restoredService.getDbInstance("mydb");
+ assertEquals(DbInstanceStatus.AVAILABLE, restored.getStatus());
+ assertEquals(created.getProxyPort(), restored.getProxyPort());
+ assertEquals(created.getProxyPort(), restored.getEndpoint().port());
+ assertNull(restored.getContainerId());
+ verify(daemonlessProxyManager, never()).startProxy(any(), any(), anyBoolean(), anyInt(),
+ any(), anyInt(), any(), any(), any(), any(), any());
+ }
+
+ @Test
+ void restoreKeepsClusterAndMemberAvailableWhenNoDockerDaemonIsReachable() {
+ StorageBackend<String, DbInstance> instances = new InMemoryStorage<>();
+ StorageBackend<String, DbCluster> clusters = new InMemoryStorage<>();
+
+ RdsService initialService = newService(containerManager, proxyManager,
+ instances, clusters, new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>());
+ initialService.createDbCluster("cluster1", "aurora-postgresql", "16.3",
+ "admin", "secret", "app", false, null, null, null, false);
+ initialService.createDbInstance("member1", "aurora-postgresql", "16.3",
+ "admin", "secret", "app", "db.t3.medium",
+ 20, false, null, null, "cluster1", null, false);
+
+ RdsContainerManager daemonlessContainerManager = mock(RdsContainerManager.class);
+ RdsProxyManager daemonlessProxyManager = mock(RdsProxyManager.class);
+ when(daemonlessContainerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))
+ .thenReturn(null);
+
+ RdsService restoredService = newService(daemonlessContainerManager, daemonlessProxyManager,
+ instances, clusters, new InMemoryStorage<>(), new InMemoryStorage<>(), new InMemoryStorage<>());
+ restoredService.restorePersistedRuntime();
+
+ assertEquals(DbInstanceStatus.AVAILABLE, restoredService.getDbCluster("cluster1").getStatus());
+ DbInstance restoredMember = restoredService.getDbInstance("member1");
+ assertEquals(DbInstanceStatus.AVAILABLE, restoredMember.getStatus());
+ assertNull(restoredMember.getContainerId());
+ verify(daemonlessProxyManager, never()).startProxy(any(), any(), anyBoolean(), anyInt(),
+ any(), anyInt(), any(), any(), any(), any(), any());
+ }
+
@Test
void mockModeCreatesStandaloneInstanceAvailableWithoutContainer() {
when(config.services().rds().mock()).thenReturn(true);
@@ -1559,7 +1837,7 @@ class RdsServiceTest {
assertEquals(DbInstanceStatus.AVAILABLE, instance.getStatus());
assertNull(instance.getContainerId());
- verify(containerManager, never()).start(any(), any(), any(), any(), any(), any(), any(), any(), any());
+ verify(containerManager, never()).tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());
}
@Test
@@ -1609,7 +1887,7 @@ class RdsServiceTest {
DbInstance rebooted = rdsService.rebootDbInstance("standalone");
assertEquals(DbInstanceStatus.AVAILABLE, rebooted.getStatus());
- verify(containerManager, never()).start(any(), any(), any(), any(), any(), any(), any(), any(), any());
+ verify(containerManager, never()).tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());
verify(containerManager, never()).stop(any());
verify(proxyManager, never()).startProxy(any(), any(), anyBoolean(), anyInt(), any(), anyInt(),
any(), any(), any(), any(), any());
@@ -1629,7 +1907,7 @@ class RdsServiceTest {
DbInstance failed = rdsService.getDbInstance("standalone");
assertEquals(DbInstanceStatus.FAILED, failed.getStatus());
assertEquals(instance.getContainerId(), failed.getContainerId());
- verify(containerManager, times(1)).start(
+ verify(containerManager, times(1)).tryStart(
any(), any(), any(), any(), any(), any(), any(), any(), any());
verify(proxyManager).stopProxy("rds-resource:" + instance.getDbInstanceArn());
}
@@ -2106,7 +2384,7 @@ class RdsServiceTest {
20, false, null, "single-az-group", null, null, true));
assertEquals("DBSubnetGroupDoesNotCoverEnoughAZs", exception.getErrorCode());
- verify(containerManager, never()).start(any(), any(), any(), any(), any(), any(), any(), any(), any());
+ verify(containerManager, never()).tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());
}
@Test
@@ -2117,7 +2395,7 @@ class RdsServiceTest {
null, null, "us-east-1a", true));
assertEquals("InvalidParameterCombination", exception.getErrorCode());
- verify(containerManager, never()).start(any(), any(), any(), any(), any(), any(), any(), any(), any());
+ verify(containerManager, never()).tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());
}
@Test
@@ -2229,7 +2507,7 @@ class RdsServiceTest {
StorageBackend<String, DbParameterGroup> parameterGroups = new InMemoryStorage<>();
StorageBackend<String, DbClusterParameterGroup> clusterParameterGroups = new InMemoryStorage<>();
- when(containerManager.start(any(), any(), any(), any(), any(), any(), any(), any(), any()))
+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))
.thenReturn(new RdsContainerHandle("initial-container", "mydb", "localhost", 5432));
RdsService initialService = newService(containerManager, proxyManager,
@@ -2245,7 +2523,7 @@ class RdsServiceTest {
RdsContainerManager restoredContainerManager = mock(RdsContainerManager.class);
RdsProxyManager restoredProxyManager = mock(RdsProxyManager.class);
- when(restoredContainerManager.start(any(), any(), any(), any(), any(), any(), any(), any(), any()))
+ when(restoredContainerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))
.thenReturn(new RdsContainerHandle("restored-container", "mydb", "127.0.0.1", 15432));
RdsService restoredService = newService(restoredContainerManager, restoredProxyManager,
@@ -2262,7 +2540,7 @@ class RdsServiceTest {
assertEquals("127.0.0.1", restored.getContainerHost());
assertEquals(15432, restored.getContainerPort());
- verify(restoredContainerManager).start(
+ verify(restoredContainerManager).tryStart(
eq(restored.getDbInstanceArn()), eq("mydb"),
eq(restored.getContainerStorageResourceId()),
eq(restored.getDockerVolumeName()), eq(DatabaseEngine.POSTGRES),
@@ -2284,7 +2562,7 @@ class RdsServiceTest {
StorageBackend<String, DbParameterGroup> parameterGroups = new InMemoryStorage<>();
StorageBackend<String, DbClusterParameterGroup> clusterParameterGroups = new InMemoryStorage<>();
- when(containerManager.start(any(), any(), any(), any(), any(), any(), any(), any(), any()))
+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))
.thenReturn(new RdsContainerHandle("initial-cluster-container", "cluster1", "localhost", 5432));
RdsService initialService = newService(containerManager, proxyManager,
@@ -2297,7 +2575,7 @@ class RdsServiceTest {
RdsContainerManager restoredContainerManager = mock(RdsContainerManager.class);
RdsProxyManager restoredProxyManager = mock(RdsProxyManager.class);
- when(restoredContainerManager.start(any(), any(), any(), any(), any(), any(), any(), any(), any()))
+ when(restoredContainerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))
.thenReturn(new RdsContainerHandle("restored-cluster-container", "cluster1", "127.0.0.1", 15432));
RdsService restoredService = newService(restoredContainerManager, restoredProxyManager,
@@ -2315,7 +2593,7 @@ class RdsServiceTest {
assertEquals("127.0.0.1", restoredMember.getContainerHost());
assertEquals(15432, restoredMember.getContainerPort());
- verify(restoredContainerManager).start(
+ verify(restoredContainerManager).tryStart(
eq(restoredCluster.getDbClusterArn()), eq("cluster1"),
eq(restoredCluster.getContainerStorageResourceId()),
eq(restoredCluster.getDockerVolumeName()), eq(DatabaseEngine.POSTGRES),
@@ -2342,7 +2620,7 @@ class RdsServiceTest {
RdsContainerHandle replacementHandle = new RdsContainerHandle(
"replacement-container", "arn:aws:rds:us-east-1:123456789012:db:replacement",
"replacement", "127.0.0.1", 15433);
- when(restoredContainerManager.start(
+ when(restoredContainerManager.tryStart(
any(), any(), any(), any(), any(), any(), any(), any(), any()))
.thenReturn(restoredHandle, replacementHandle);
org.mockito.Mockito.doThrow(new IllegalStateException("relay failed"))
@@ -2402,7 +2680,7 @@ class RdsServiceTest {
persisted.setContainerId("persisted-container");
instances.put("broken", persisted);
RdsContainerManager restoredContainerManager = mock(RdsContainerManager.class);
- when(restoredContainerManager.start(
+ when(restoredContainerManager.tryStart(
any(), any(), any(), any(), any(), any(), any(), any(), any()))
.thenThrow(new IllegalStateException("Docker start failed"));
RdsService restoredService = newService(
@@ -2468,7 +2746,7 @@ class RdsServiceTest {
RdsContainerHandle replacementHandle = new RdsContainerHandle(
"replacement-container", "arn:aws:rds:us-east-1:123456789012:cluster:replacement",
"replacement", "127.0.0.1", 15433);
- when(restoredContainerManager.start(
+ when(restoredContainerManager.tryStart(
any(), any(), any(), any(), any(), any(), any(), any(), any()))
.thenReturn(restoredHandle, replacementHandle);
org.mockito.Mockito.doThrow(new IllegalStateException("relay failed"))
@@ -4510,7 +4788,7 @@ class RdsServiceTest {
StorageBackend<String, DbProxy> proxies = new InMemoryStorage<>();
StorageBackend<String, DbProxyTargetGroup> proxyTargetGroups = new InMemoryStorage<>();
- when(containerManager.start(any(), any(), any(), any(), any(), any(), any(), any(), any()))
+ when(containerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))
.thenReturn(new RdsContainerHandle("initial-container", "cluster1", "localhost", 5432));
RdsService initialService = new RdsService(containerManager, proxyManager, ec2Service,
@@ -4524,7 +4802,7 @@ class RdsServiceTest {
RdsContainerManager restoredContainerManager = mock(RdsContainerManager.class);
RdsProxyManager restoredProxyManager = mock(RdsProxyManager.class);
- when(restoredContainerManager.start(any(), any(), any(), any(), any(), any(), any(), any(), any()))
+ when(restoredContainerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))
.thenReturn(new RdsContainerHandle("restored-container", "cluster1", "127.0.0.1", 15432));
RdsService restoredService = new RdsService(restoredContainerManager, restoredProxyManager, ec2Service,
@@ -4588,7 +4866,7 @@ class RdsServiceTest {
RdsContainerManager restoredContainerManager = mock(RdsContainerManager.class);
RdsProxyManager restoredProxyManager = mock(RdsProxyManager.class);
- when(restoredContainerManager.start(any(), any(), any(), any(), any(), any(), any(), any(), any()))
+ when(restoredContainerManager.tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any()))
.thenReturn(new RdsContainerHandle("restored-container", "cluster1",
"127.0.0.1", 15432));
@@ -5378,7 +5656,7 @@ class RdsServiceTest {
+ "or you do not have permissions to access it.", missing.getMessage());
assertThrows(AwsException.class, () -> rdsService.getDbInstance("mydb"));
// refused before any side effect: no container was started for the rejected create
- verify(containerManager, never()).start(any(), any(), any(), any(), any(), any(), any(), any(), any());
+ verify(containerManager, never()).tryStart(any(), any(), any(), any(), any(), any(), any(), any(), any());
KmsKey disabled = knownKey("k1");
disabled.setEnabled(false);
diff --git a/src/test/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManagerTest.java
index 4d42194d7c..6c048256df 100644
--- a/src/test/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManagerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManagerTest.java
@@ -529,6 +529,166 @@ class RdsContainerManagerTest {
verify(lifecycleManager).removeVolumeStrict(volume);
}
+ @Test
+ void tryStartReportsUnavailableInsteadOfThrowingWhenNoDockerDaemonIsReachable() {
+ // Floci running inside Docker without a mounted daemon socket: the RDS control plane must
+ // keep working, so the failure is reported rather than propagated.
+ ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);
+ doThrow(new IllegalStateException("Failed to remove stale container floci-rds-db1"))
+ .when(lifecycleManager).removeIfExistsStrict(any());
+ doThrow(new IllegalStateException("Failed to remove container floci-rds-db1"))
+ .when(lifecycleManager).stopAndRemoveStrict(any(), any());
+ when(lifecycleManager.getDockerClient()).thenThrow(
+ new RuntimeException("java.net.SocketException: No such file or directory"));
+ RdsContainerManager manager = daemonlessCapableManager(lifecycleManager);
+ String runtimeId = "arn:aws:rds:us-east-1:000000000000:db:db1";
+
+ for (int attempt = 0; attempt < 3; attempt++) {
+ assertNull(manager.tryStart(runtimeId, "db1", "db1", "floci-rds-db1",
+ DatabaseEngine.MYSQL, "mysql:8.0", "root", "password", "db"),
+ "attempt " + attempt + " should report unavailable");
+ }
+ assertFalse(manager.isDockerReachable());
+ // No container was created, so no cleanup identity is retained for the runtime.
+ assertNull(manager.getActiveHandle(runtimeId));
+ }
+
+ @Test
+ void tryStartPropagatesFailuresRaisedWhileTheDaemonIsReachable() {
+ // A reachable daemon that cannot start the container is a genuine failure, not a
+ // degraded mode: CreateDBInstance must still surface it.
+ ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);
+ when(lifecycleManager.create(any()))
+ .thenThrow(new IllegalStateException("no such image: mysql:8.0"));
+ DockerClient dockerClient = mock(DockerClient.class, org.mockito.Mockito.RETURNS_DEEP_STUBS);
+ when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);
+ RdsContainerManager manager = daemonlessCapableManager(lifecycleManager);
+
+ IllegalStateException failure = assertThrows(IllegalStateException.class,
+ () -> manager.tryStart("arn:aws:rds:us-east-1:000000000000:db:db1", "db1", "db1",
+ "floci-rds-db1", DatabaseEngine.MYSQL, "mysql:8.0", "root", "password", "db"));
+
+ assertEquals("no such image: mysql:8.0", failure.getMessage());
+ assertTrue(manager.isDockerReachable());
+ }
+
+ @Test
+ void tryStartSucceedsForTheSameRuntimeOnceADaemonAppears() {
+ ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);
+ doThrow(new IllegalStateException("Failed to remove stale container floci-rds-db1"))
+ .when(lifecycleManager).removeIfExistsStrict(any());
+ doThrow(new IllegalStateException("Failed to remove container floci-rds-db1"))
+ .when(lifecycleManager).stopAndRemoveStrict(any(), any());
+ when(lifecycleManager.getDockerClient()).thenThrow(
+ new RuntimeException("java.net.SocketException: No such file or directory"));
+ RdsContainerManager manager = daemonlessCapableManager(lifecycleManager);
+ String runtimeId = "arn:aws:rds:us-east-1:000000000000:db:db1";
+
+ assertNull(manager.tryStart(runtimeId, "db1", "db1", "floci-rds-db1",
+ DatabaseEngine.MYSQL, "mysql:8.0", "root", "password", "db"));
+
+ // A Docker daemon appears.
+ org.mockito.Mockito.reset(lifecycleManager);
+ stubStarts(lifecycleManager, new ContainerLifecycleManager.ContainerInfo(
+ "late-container", Map.of(3306, new ContainerLifecycleManager.EndpointInfo("db1", 3306))));
+ DockerClient dockerClient = mock(DockerClient.class, org.mockito.Mockito.RETURNS_DEEP_STUBS);
+ when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);
+
+ RdsContainerHandle handle = manager.tryStart(runtimeId, "db1", "db1", "floci-rds-db1",
+ DatabaseEngine.MYSQL, "mysql:8.0", "root", "password", "db");
+
+ assertEquals("late-container", handle.getContainerId());
+ assertEquals(3306, handle.getPort());
+ assertEquals(handle, manager.getActiveHandle(runtimeId));
+ }
+
+ @Test
+ void tryStartKeepsAndLaterCleansTheIdentityOfAContainerCreatedBeforeTheDaemonWentAway() {
+ // Docker answered the create and then went away: the created container must stay known
+ // so the retry removes it once the daemon is back, rather than being forgotten.
+ ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);
+ when(lifecycleManager.create(any())).thenReturn("created-container");
+ when(lifecycleManager.startCreated(any(), any()))
+ .thenThrow(new IllegalStateException("java.net.SocketException: Connection reset"));
+ doThrow(new IllegalStateException("Failed to remove container created-container"))
+ .when(lifecycleManager).stopAndRemoveStrict(any(), any());
+ when(lifecycleManager.getDockerClient()).thenThrow(
+ new RuntimeException("java.net.SocketException: No such file or directory"));
+ RdsContainerManager manager = daemonlessCapableManager(lifecycleManager);
+ String runtimeId = "arn:aws:rds:us-east-1:000000000000:db:db1";
+
+ assertNull(manager.tryStart(runtimeId, "db1", "db1", "floci-rds-db1",
+ DatabaseEngine.MYSQL, "mysql:8.0", "root", "password", "db"));
+ assertEquals("created-container", manager.getActiveHandle(runtimeId).getContainerId());
+
+ // Still no daemon: the retry keeps the identity rather than dropping it.
+ assertNull(manager.tryStart(runtimeId, "db1", "db1", "floci-rds-db1",
+ DatabaseEngine.MYSQL, "mysql:8.0", "root", "password", "db"));
+ assertEquals("created-container", manager.getActiveHandle(runtimeId).getContainerId());
+
+ // The daemon is back: the leftover is removed first, then the start goes through.
+ org.mockito.Mockito.reset(lifecycleManager);
+ stubStarts(lifecycleManager, new ContainerLifecycleManager.ContainerInfo(
+ "late-container", Map.of(3306, new ContainerLifecycleManager.EndpointInfo("db1", 3306))));
+ DockerClient dockerClient = mock(DockerClient.class, org.mockito.Mockito.RETURNS_DEEP_STUBS);
+ when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);
+
+ RdsContainerHandle handle = manager.tryStart(runtimeId, "db1", "db1", "floci-rds-db1",
+ DatabaseEngine.MYSQL, "mysql:8.0", "root", "password", "db");
+
+ assertEquals("late-container", handle.getContainerId());
+ verify(lifecycleManager).stopAndRemoveStrict(
+ org.mockito.ArgumentMatchers.eq("created-container"), any());
+ }
+
+ @Test
+ void tryStartRetriesACleanupThatFailedAfterAStop() {
+ // The service stops a container whose auth proxy did not start. When that stop fails
+ // the handle and claim stay retained, and the runtime's next start must clean them up
+ // rather than fail on the claim forever.
+ ContainerLifecycleManager lifecycleManager = mock(ContainerLifecycleManager.class);
+ stubStarts(lifecycleManager,
+ new ContainerLifecycleManager.ContainerInfo("first-container",
+ Map.of(3306, new ContainerLifecycleManager.EndpointInfo("db1", 3306))),
+ new ContainerLifecycleManager.ContainerInfo("second-container",
+ Map.of(3306, new ContainerLifecycleManager.EndpointInfo("db1", 3306))));
+ DockerClient dockerClient = mock(DockerClient.class, org.mockito.Mockito.RETURNS_DEEP_STUBS);
+ when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);
+ doThrow(new IllegalStateException("Failed to remove container first-container"))
+ .doNothing()
+ .when(lifecycleManager).stopAndRemoveStrict(
+ org.mockito.ArgumentMatchers.eq("first-container"), any());
+ RdsContainerManager manager = daemonlessCapableManager(lifecycleManager);
+ String runtimeId = "arn:aws:rds:us-east-1:000000000000:db:db1";
+
+ RdsContainerHandle first = manager.start(runtimeId, "db1", "db1", "floci-rds-db1",
+ DatabaseEngine.MYSQL, "mysql:8.0", "root", "password", "db");
+ assertThrows(IllegalStateException.class, () -> manager.stop(first));
+ assertEquals(first, manager.getActiveHandle(runtimeId));
+
+ RdsContainerHandle retried = manager.tryStart(runtimeId, "db1", "db1", "floci-rds-db1",
+ DatabaseEngine.MYSQL, "mysql:8.0", "root", "password", "db");
+
+ assertEquals("second-container", retried.getContainerId());
+ assertEquals(retried, manager.getActiveHandle(runtimeId));
+ verify(lifecycleManager, times(2)).stopAndRemoveStrict(
+ org.mockito.ArgumentMatchers.eq("first-container"), any());
+ }
+
+ private RdsContainerManager daemonlessCapableManager(ContainerLifecycleManager lifecycleManager) {
+ EmulatorConfig config = config(tempDir.resolve("host-root"));
+ ContainerLogStreamer logStreamer = mock(ContainerLogStreamer.class);
+ lenient().when(logStreamer.generateLogStreamName(any())).thenReturn("log-stream");
+ return new RdsContainerManager(
+ new ContainerBuilder(config, mock(DockerHostResolver.class), mock(EmbeddedDnsServer.class)),
+ lifecycleManager,
+ logStreamer,
+ mock(ContainerDetector.class),
+ config,
+ new RegionResolver("us-east-1", "000000000000"),
+ mock(ServiceConfigAccess.class));
+ }
+
@Test
void failedStartReleasesClaimsForSameRuntimeRetry() {
EmulatorConfig config = config(tempDir.resolve("host-root"));
diff --git a/src/test/java/io/github/hectorvent/floci/services/rdsdata/RdsDataResourceResolverTest.java b/src/test/java/io/github/hectorvent/floci/services/rdsdata/RdsDataResourceResolverTest.java
index 891c7e6b55..4c20a0b460 100644
--- a/src/test/java/io/github/hectorvent/floci/services/rdsdata/RdsDataResourceResolverTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/rdsdata/RdsDataResourceResolverTest.java
@@ -5,6 +5,7 @@ import io.github.hectorvent.floci.services.rds.RdsService;
import io.github.hectorvent.floci.services.rds.model.DatabaseEngine;
import io.github.hectorvent.floci.services.rds.model.DbCluster;
import io.github.hectorvent.floci.services.rds.model.DbEndpoint;
+import io.github.hectorvent.floci.services.rds.model.DbInstance;
import io.github.hectorvent.floci.services.rds.model.DbInstanceStatus;
import org.junit.jupiter.api.Test;
@@ -14,6 +15,7 @@ import java.util.List;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
@@ -52,6 +54,69 @@ class RdsDataResourceResolverTest {
assertEquals(400, error.getHttpStatus());
}
+ @Test
+ void dataApiNamesTheMissingDockerDaemonWhenTheInstanceHasNoBackingContainer() {
+ RdsService rdsService = mock(RdsService.class);
+ DbInstance instance = daemonlessInstance();
+ when(rdsService.getDbInstance("probe-db", "us-east-1")).thenReturn(instance);
+ when(rdsService.ensureInstanceBackend("probe-db", "us-east-1")).thenReturn(instance);
+ when(rdsService.isBackendRuntimeAvailable()).thenReturn(false);
+
+ AwsException error = assertThrows(AwsException.class, () -> new RdsDataResourceResolver(rdsService)
+ .resolve("arn:aws:rds:us-east-1:000000000000:db:probe-db"));
+
+ assertEquals("InternalServerErrorException", error.getErrorCode());
+ assertEquals(500, error.getHttpStatus());
+ assertTrue(error.getMessage().contains("Docker"), error.getMessage());
+ }
+
+ @Test
+ void dataApiRetriesTheBackendSoItResolvesOnceADaemonAppears() {
+ RdsService rdsService = mock(RdsService.class);
+ DbInstance started = daemonlessInstance();
+ started.setContainerHost("127.0.0.1");
+ started.setContainerPort(5432);
+ when(rdsService.getDbInstance("probe-db", "us-east-1")).thenReturn(daemonlessInstance());
+ when(rdsService.ensureInstanceBackend("probe-db", "us-east-1")).thenReturn(started);
+
+ RdsDataResourceResolver.DatabaseTarget target = new RdsDataResourceResolver(rdsService)
+ .resolve("arn:aws:rds:us-east-1:000000000000:db:probe-db");
+
+ assertEquals("127.0.0.1", target.host());
+ assertEquals(5432, target.port());
+ }
+
+ @Test
+ void dataApiRetriesAClusterBackendThroughTheClusterEntryPoint() {
+ RdsService rdsService = mock(RdsService.class);
+ DbCluster cluster = new DbCluster("cluster1", DatabaseEngine.POSTGRES, "16.3", "admin", "secret",
+ "app", DbInstanceStatus.AVAILABLE, new DbEndpoint("localhost", 7001),
+ new DbEndpoint("localhost", 7001), false, new ArrayList<>(), null, Instant.now(), 7001);
+ cluster.setDbClusterArn("arn:aws:rds:us-east-1:000000000000:cluster:cluster1");
+ DbCluster started = new DbCluster("cluster1", DatabaseEngine.POSTGRES, "16.3", "admin", "secret",
+ "app", DbInstanceStatus.AVAILABLE, new DbEndpoint("localhost", 7001),
+ new DbEndpoint("localhost", 7001), false, new ArrayList<>(), null, Instant.now(), 7001);
+ started.setDbClusterArn(cluster.getDbClusterArn());
+ started.setContainerHost("127.0.0.1");
+ started.setContainerPort(5432);
+ when(rdsService.getDbCluster("cluster1", "us-east-1")).thenReturn(cluster);
+ when(rdsService.ensureClusterBackend("cluster1", "us-east-1")).thenReturn(started);
+
+ RdsDataResourceResolver.DatabaseTarget target = new RdsDataResourceResolver(rdsService)
+ .resolve("arn:aws:rds:us-east-1:000000000000:cluster:cluster1");
+
+ assertEquals("127.0.0.1", target.host());
+ assertEquals(5432, target.port());
+ }
+
+ private static DbInstance daemonlessInstance() {
+ DbInstance instance = new DbInstance("probe-db", DatabaseEngine.POSTGRES, "16.3", "admin",
+ "secret", "app", "db.t3.micro", 20, DbInstanceStatus.AVAILABLE,
+ new DbEndpoint("localhost", 7001), false, null, null, Instant.now(), 7001);
+ instance.setDbInstanceArn("arn:aws:rds:us-east-1:000000000000:db:probe-db");
+ return instance;
+ }
+
@Test
void rejectsResourceArnOutsideSignedRequestRegionBeforeLookup() {
RdsService rdsService = mock(RdsService.class);