Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(rds): keep instance available when the liveness probe fails (#5070)

Matheus Oliveira97906c09b92d
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
index de43846912..48c28eac85 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java
@@ -950,27 +950,36 @@ public class ContainerLifecycleManager {
/**
* Returns whether the container is currently running. A missing container is treated as
* not-running; any other Docker error (e.g. an inspect timeout under daemon overload) is also
- * treated as not-running, so a hung/dead container is not reused from the warm pool — a false
+ * treated as not-running, so a hung/dead container is not reused from the warm pool. A false
* negative merely triggers a clean cold-start, which is far cheaper than blocking until the
- * function timeout.
+ * function timeout. Callers that must not act on a failed probe use {@link #probeContainer}.
*
* @param containerId the container ID to inspect
* @return true only if the container exists and is reported as running; false on any error
*/
public boolean isContainerRunning(String containerId) {
+ return probeContainer(containerId) == ContainerLiveness.RUNNING;
+ }
+
+ /**
+ * Probes the container and tells a definite answer from a failed probe.
+ *
+ * @param containerId the container ID to inspect
+ * @return RUNNING or NOT_RUNNING when Docker answered (a missing container is NOT_RUNNING),
+ * UNKNOWN when the inspect call itself failed
+ */
+ public ContainerLiveness probeContainer(String containerId) {
try {
InspectContainerResponse inspect = dockerClient.inspectContainerCmd(containerId).exec();
- return Boolean.TRUE.equals(inspect.getState().getRunning());
+ return Boolean.TRUE.equals(inspect.getState().getRunning())
+ ? ContainerLiveness.RUNNING
+ : ContainerLiveness.NOT_RUNNING;
} catch (NotFoundException e) {
- return false;
+ return ContainerLiveness.NOT_RUNNING;
} catch (Exception e) {
- // Treat an inspect failure/timeout as NOT running. Under Docker-daemon overload,
- // returning true here caused the warm pool to "reuse" dead/hung containers, so the
- // invocation blocked until the function timeout (~20-30s) every time. A false
- // negative merely triggers a clean cold-start, which is far cheaper than a hang.
- LOG.warnv("Liveness check failed for container {0}; treating as not running: {1}",
+ LOG.warnv("Liveness check failed for container {0}; state unknown: {1}",
containerId, e.getMessage());
- return false;
+ return ContainerLiveness.UNKNOWN;
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLiveness.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLiveness.java
new file mode 100644
index 0000000000..913b7ef26d
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLiveness.java
@@ -0,0 +1,11 @@
+package io.github.hectorvent.floci.core.common.docker;
+
+/**
+ * Outcome of probing a container's liveness through the Docker API.
+ * {@link #UNKNOWN} means the probe itself failed, so nothing is known about the container.
+ */
+public enum ContainerLiveness {
+ RUNNING,
+ NOT_RUNNING,
+ UNKNOWN
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java b/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java
index cb1905140a..d663c53d3e 100644
--- a/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java
@@ -11,6 +11,7 @@ import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.AwsRegions;
import io.github.hectorvent.floci.core.common.BackupWindows;
import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.common.docker.ContainerLiveness;
import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;
import io.github.hectorvent.floci.core.common.docker.CurrentContainerNetworkResolver;
import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;
@@ -2685,7 +2686,8 @@ public class RdsService implements Resettable, ResourceProvider {
|| instance.getContainerId().isBlank()) {
return instance;
}
- if (containerManager.isContainerRunning(instance.getContainerId())) {
+ ContainerLiveness liveness = containerManager.probeContainer(instance.getContainerId());
+ if (liveness != ContainerLiveness.NOT_RUNNING) {
return instance;
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java
index 009b7fbb41..7beb403f25 100644
--- a/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java
@@ -10,6 +10,7 @@ import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;
import io.github.hectorvent.floci.core.common.docker.ContainerDetector;
import io.github.hectorvent.floci.core.common.docker.ContainerExec;
import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;
+import io.github.hectorvent.floci.core.common.docker.ContainerLiveness;
import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;
import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.ContainerInfo;
import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.EndpointInfo;
@@ -433,11 +434,15 @@ public class RdsContainerManager implements RdsBackendGate, Resettable {
}
}
- /** Returns whether a backing RDS container still exists and is running. */
- public boolean isContainerRunning(String containerId) {
- return containerId != null
- && !containerId.isBlank()
- && lifecycleManager.isContainerRunning(containerId);
+ /**
+ * Probes a backing RDS container. UNKNOWN means the Docker probe failed and the container's
+ * state is not known, which is not the same as the container being gone.
+ */
+ public ContainerLiveness probeContainer(String containerId) {
+ if (containerId == null || containerId.isBlank()) {
+ return ContainerLiveness.NOT_RUNNING;
+ }
+ return lifecycleManager.probeContainer(containerId);
}
/** Returns the retained runtime handle used to persist cleanup identity after a failed start. */
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerLivenessTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerLivenessTest.java
new file mode 100644
index 0000000000..3c4982accd
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerLivenessTest.java
@@ -0,0 +1,100 @@
+package io.github.hectorvent.floci.core.common.docker;
+
+import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.command.InspectContainerCmd;
+import com.github.dockerjava.api.command.InspectContainerResponse;
+import com.github.dockerjava.api.exception.NotFoundException;
+import io.github.hectorvent.floci.config.EmulatorConfig;
+import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.extension.ExtendWith;
+import org.mockito.Mock;
+import org.mockito.junit.jupiter.MockitoExtension;
+
+import java.io.IOException;
+import java.io.UncheckedIOException;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@ExtendWith(MockitoExtension.class)
+class ContainerLifecycleManagerLivenessTest {
+
+ @Mock
+ private DockerClient dockerClient;
+
+ @Mock
+ private ImageCacheService imageCacheService;
+
+ @Mock
+ private ContainerDetector containerDetector;
+
+ @Mock
+ private PortAllocator portAllocator;
+
+ @Mock
+ private EmulatorConfig config;
+
+ @Mock
+ private InspectContainerCmd inspectCmd;
+
+ private ContainerLifecycleManager manager;
+
+ @BeforeEach
+ void setUp() {
+ manager = new ContainerLifecycleManager(
+ dockerClient, imageCacheService, containerDetector, portAllocator, config);
+ when(dockerClient.inspectContainerCmd("c1")).thenReturn(inspectCmd);
+ }
+
+ private InspectContainerResponse inspectResponse(boolean running) {
+ InspectContainerResponse response = mock(InspectContainerResponse.class);
+ InspectContainerResponse.ContainerState state = mock(InspectContainerResponse.ContainerState.class);
+ when(state.getRunning()).thenReturn(running);
+ when(response.getState()).thenReturn(state);
+ return response;
+ }
+
+ @Test
+ void probeContainerReportsRunning() {
+ InspectContainerResponse response = inspectResponse(true);
+ when(inspectCmd.exec()).thenReturn(response);
+
+ assertEquals(ContainerLiveness.RUNNING, manager.probeContainer("c1"));
+ assertTrue(manager.isContainerRunning("c1"));
+ }
+
+ @Test
+ void probeContainerReportsStoppedContainerAsNotRunning() {
+ InspectContainerResponse response = inspectResponse(false);
+ when(inspectCmd.exec()).thenReturn(response);
+
+ assertEquals(ContainerLiveness.NOT_RUNNING, manager.probeContainer("c1"));
+ assertFalse(manager.isContainerRunning("c1"));
+ }
+
+ @Test
+ void probeContainerReportsMissingContainerAsNotRunning() {
+ when(inspectCmd.exec()).thenThrow(new NotFoundException("No such container"));
+
+ assertEquals(ContainerLiveness.NOT_RUNNING, manager.probeContainer("c1"));
+ }
+
+ @Test
+ void probeContainerReportsIoFailureAsUnknownNotStopped() {
+ when(inspectCmd.exec()).thenThrow(new UncheckedIOException(new IOException("Broken pipe")));
+
+ assertEquals(ContainerLiveness.UNKNOWN, manager.probeContainer("c1"));
+ }
+
+ @Test
+ void isContainerRunningStillTreatsProbeFailureAsNotRunningForWarmPoolCallers() {
+ when(inspectCmd.exec()).thenThrow(new UncheckedIOException(new IOException("Broken pipe")));
+
+ assertFalse(manager.isContainerRunning("c1"));
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java
index e7f7ec3579..e9829b164b 100644
--- a/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java
@@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.rds;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.AwsException;
import io.github.hectorvent.floci.core.common.RegionResolver;
+import io.github.hectorvent.floci.core.common.docker.ContainerLiveness;
import io.github.hectorvent.floci.core.common.docker.CurrentContainerNetworkResolver;
import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;
import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;
@@ -7187,7 +7188,7 @@ class RdsServiceTest {
@Test
void refreshRuntimeHealthMarksDeadContainerFailedAndStopsProxy() {
when(rdsConfig.mock()).thenReturn(false);
- when(containerManager.isContainerRunning("cont-id")).thenReturn(false);
+ when(containerManager.probeContainer("cont-id")).thenReturn(ContainerLiveness.NOT_RUNNING);
DbInstance instance = rdsService.createDbInstance(
"dead-db", "postgres", "16", "admin", "password", "dbname",
"db.t3.micro", 20, false, null, null, null, null, false, false,
@@ -7207,6 +7208,44 @@ class RdsServiceTest {
assertEquals(1, rdsService.describeEvents("dead-db", "db-instance", null, null, 60).size());
}
+ @Test
+ void refreshRuntimeHealthKeepsAvailableStatusWhenLivenessProbeFails() {
+ when(rdsConfig.mock()).thenReturn(false);
+ when(containerManager.probeContainer("cont-id")).thenReturn(ContainerLiveness.UNKNOWN);
+ DbInstance instance = rdsService.createDbInstance(
+ "blip-db", "postgres", "16", "admin", "password", "dbname",
+ "db.t3.micro", 20, false, null, null, null, null, false, false,
+ null, Map.of(), List.of(), null, null, true);
+
+ DbInstance refreshed = rdsService.refreshDbInstanceRuntimeHealth(instance);
+
+ assertEquals(DbInstanceStatus.AVAILABLE, refreshed.getStatus());
+ verify(proxyManager, never()).stopProxy("rds-resource:" + refreshed.getDbInstanceArn());
+ assertTrue(rdsService.describeEvents("blip-db", "db-instance", null, null, 60).isEmpty());
+ }
+
+ @Test
+ void refreshRuntimeHealthReprobesAfterAFailedProbeAndActsOnTheNextDefiniteAnswer() {
+ when(rdsConfig.mock()).thenReturn(false);
+ when(containerManager.probeContainer("cont-id"))
+ .thenReturn(ContainerLiveness.UNKNOWN, ContainerLiveness.NOT_RUNNING);
+ DbInstance instance = rdsService.createDbInstance(
+ "recover-db", "postgres", "16", "admin", "password", "dbname",
+ "db.t3.micro", 20, false, null, null, null, null, false, false,
+ null, Map.of(), List.of(), null, null, true);
+ String proxyKey = "rds-resource:" + instance.getDbInstanceArn();
+
+ DbInstance afterBlip = rdsService.refreshDbInstanceRuntimeHealth(instance);
+ assertEquals(DbInstanceStatus.AVAILABLE, afterBlip.getStatus());
+ verify(proxyManager, never()).stopProxy(proxyKey);
+
+ DbInstance afterDefiniteAnswer = rdsService.refreshDbInstanceRuntimeHealth(afterBlip);
+ assertEquals(DbInstanceStatus.FAILED, afterDefiniteAnswer.getStatus());
+ verify(proxyManager).stopProxy(proxyKey);
+ verify(containerManager, times(2)).probeContainer("cont-id");
+ assertEquals(1, rdsService.describeEvents("recover-db", "db-instance", null, null, 60).size());
+ }
+
@Test
void describeEventsPrunesEventsOlderThanFourteenDays() throws Exception {
InMemoryStorage<String, RdsEvent> eventStore = new InMemoryStorage<>();