Light, fluffy, and always free - The AWS Local Emulator alternative
fix(rds): keep instance available when the liveness probe fails (#5070)
Matheus Oliveira97906c09b92d
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.javaindex de43846912..48c28eac85 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java@@ -950,27 +950,36 @@ public class ContainerLifecycleManager {/*** Returns whether the container is currently running. A missing container is treated as* not-running; any other Docker error (e.g. an inspect timeout under daemon overload) is also- * treated as not-running, so a hung/dead container is not reused from the warm pool — a false+ * treated as not-running, so a hung/dead container is not reused from the warm pool. A false* negative merely triggers a clean cold-start, which is far cheaper than blocking until the- * function timeout.+ * function timeout. Callers that must not act on a failed probe use {@link #probeContainer}.** @param containerId the container ID to inspect* @return true only if the container exists and is reported as running; false on any error*/public boolean isContainerRunning(String containerId) {+ return probeContainer(containerId) == ContainerLiveness.RUNNING;+ }++ /**+ * Probes the container and tells a definite answer from a failed probe.+ *+ * @param containerId the container ID to inspect+ * @return RUNNING or NOT_RUNNING when Docker answered (a missing container is NOT_RUNNING),+ * UNKNOWN when the inspect call itself failed+ */+ public ContainerLiveness probeContainer(String containerId) {try {InspectContainerResponse inspect = dockerClient.inspectContainerCmd(containerId).exec();- return Boolean.TRUE.equals(inspect.getState().getRunning());+ return Boolean.TRUE.equals(inspect.getState().getRunning())+ ? ContainerLiveness.RUNNING+ : ContainerLiveness.NOT_RUNNING;} catch (NotFoundException e) {- return false;+ return ContainerLiveness.NOT_RUNNING;} catch (Exception e) {- // Treat an inspect failure/timeout as NOT running. Under Docker-daemon overload,- // returning true here caused the warm pool to "reuse" dead/hung containers, so the- // invocation blocked until the function timeout (~20-30s) every time. A false- // negative merely triggers a clean cold-start, which is far cheaper than a hang.- LOG.warnv("Liveness check failed for container {0}; treating as not running: {1}",+ LOG.warnv("Liveness check failed for container {0}; state unknown: {1}",containerId, e.getMessage());- return false;+ return ContainerLiveness.UNKNOWN;}}diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLiveness.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLiveness.javanew file mode 100644index 0000000000..913b7ef26d--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLiveness.java@@ -0,0 +1,11 @@+package io.github.hectorvent.floci.core.common.docker;++/**+ * Outcome of probing a container's liveness through the Docker API.+ * {@link #UNKNOWN} means the probe itself failed, so nothing is known about the container.+ */+public enum ContainerLiveness {+ RUNNING,+ NOT_RUNNING,+ UNKNOWN+}diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java b/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.javaindex cb1905140a..d663c53d3e 100644--- a/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java+++ b/src/main/java/io/github/hectorvent/floci/services/rds/RdsService.java@@ -11,6 +11,7 @@ import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.AwsRegions;import io.github.hectorvent.floci.core.common.BackupWindows;import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.docker.ContainerLiveness;import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;import io.github.hectorvent.floci.core.common.docker.CurrentContainerNetworkResolver;import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;@@ -2685,7 +2686,8 @@ public class RdsService implements Resettable, ResourceProvider {|| instance.getContainerId().isBlank()) {return instance;}- if (containerManager.isContainerRunning(instance.getContainerId())) {+ ContainerLiveness liveness = containerManager.probeContainer(instance.getContainerId());+ if (liveness != ContainerLiveness.NOT_RUNNING) {return instance;}diff --git a/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.javaindex 009b7fbb41..7beb403f25 100644--- a/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/rds/container/RdsContainerManager.java@@ -10,6 +10,7 @@ import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;import io.github.hectorvent.floci.core.common.docker.ContainerDetector;import io.github.hectorvent.floci.core.common.docker.ContainerExec;import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;+import io.github.hectorvent.floci.core.common.docker.ContainerLiveness;import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.ContainerInfo;import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.EndpointInfo;@@ -433,11 +434,15 @@ public class RdsContainerManager implements RdsBackendGate, Resettable {}}- /** Returns whether a backing RDS container still exists and is running. */- public boolean isContainerRunning(String containerId) {- return containerId != null- && !containerId.isBlank()- && lifecycleManager.isContainerRunning(containerId);+ /**+ * Probes a backing RDS container. UNKNOWN means the Docker probe failed and the container's+ * state is not known, which is not the same as the container being gone.+ */+ public ContainerLiveness probeContainer(String containerId) {+ if (containerId == null || containerId.isBlank()) {+ return ContainerLiveness.NOT_RUNNING;+ }+ return lifecycleManager.probeContainer(containerId);}/** Returns the retained runtime handle used to persist cleanup identity after a failed start. */diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerLivenessTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerLivenessTest.javanew file mode 100644index 0000000000..3c4982accd--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerLivenessTest.java@@ -0,0 +1,100 @@+package io.github.hectorvent.floci.core.common.docker;++import com.github.dockerjava.api.DockerClient;+import com.github.dockerjava.api.command.InspectContainerCmd;+import com.github.dockerjava.api.command.InspectContainerResponse;+import com.github.dockerjava.api.exception.NotFoundException;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.services.lambda.launcher.ImageCacheService;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.extension.ExtendWith;+import org.mockito.Mock;+import org.mockito.junit.jupiter.MockitoExtension;++import java.io.IOException;+import java.io.UncheckedIOException;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.when;++@ExtendWith(MockitoExtension.class)+class ContainerLifecycleManagerLivenessTest {++ @Mock+ private DockerClient dockerClient;++ @Mock+ private ImageCacheService imageCacheService;++ @Mock+ private ContainerDetector containerDetector;++ @Mock+ private PortAllocator portAllocator;++ @Mock+ private EmulatorConfig config;++ @Mock+ private InspectContainerCmd inspectCmd;++ private ContainerLifecycleManager manager;++ @BeforeEach+ void setUp() {+ manager = new ContainerLifecycleManager(+ dockerClient, imageCacheService, containerDetector, portAllocator, config);+ when(dockerClient.inspectContainerCmd("c1")).thenReturn(inspectCmd);+ }++ private InspectContainerResponse inspectResponse(boolean running) {+ InspectContainerResponse response = mock(InspectContainerResponse.class);+ InspectContainerResponse.ContainerState state = mock(InspectContainerResponse.ContainerState.class);+ when(state.getRunning()).thenReturn(running);+ when(response.getState()).thenReturn(state);+ return response;+ }++ @Test+ void probeContainerReportsRunning() {+ InspectContainerResponse response = inspectResponse(true);+ when(inspectCmd.exec()).thenReturn(response);++ assertEquals(ContainerLiveness.RUNNING, manager.probeContainer("c1"));+ assertTrue(manager.isContainerRunning("c1"));+ }++ @Test+ void probeContainerReportsStoppedContainerAsNotRunning() {+ InspectContainerResponse response = inspectResponse(false);+ when(inspectCmd.exec()).thenReturn(response);++ assertEquals(ContainerLiveness.NOT_RUNNING, manager.probeContainer("c1"));+ assertFalse(manager.isContainerRunning("c1"));+ }++ @Test+ void probeContainerReportsMissingContainerAsNotRunning() {+ when(inspectCmd.exec()).thenThrow(new NotFoundException("No such container"));++ assertEquals(ContainerLiveness.NOT_RUNNING, manager.probeContainer("c1"));+ }++ @Test+ void probeContainerReportsIoFailureAsUnknownNotStopped() {+ when(inspectCmd.exec()).thenThrow(new UncheckedIOException(new IOException("Broken pipe")));++ assertEquals(ContainerLiveness.UNKNOWN, manager.probeContainer("c1"));+ }++ @Test+ void isContainerRunningStillTreatsProbeFailureAsNotRunningForWarmPoolCallers() {+ when(inspectCmd.exec()).thenThrow(new UncheckedIOException(new IOException("Broken pipe")));++ assertFalse(manager.isContainerRunning("c1"));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.javaindex e7f7ec3579..e9829b164b 100644--- a/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/rds/RdsServiceTest.java@@ -3,6 +3,7 @@ package io.github.hectorvent.floci.services.rds;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.AwsException;import io.github.hectorvent.floci.core.common.RegionResolver;+import io.github.hectorvent.floci.core.common.docker.ContainerLiveness;import io.github.hectorvent.floci.core.common.docker.CurrentContainerNetworkResolver;import io.github.hectorvent.floci.core.common.docker.DockerHostResolver;import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;@@ -7187,7 +7188,7 @@ class RdsServiceTest {@Testvoid refreshRuntimeHealthMarksDeadContainerFailedAndStopsProxy() {when(rdsConfig.mock()).thenReturn(false);- when(containerManager.isContainerRunning("cont-id")).thenReturn(false);+ when(containerManager.probeContainer("cont-id")).thenReturn(ContainerLiveness.NOT_RUNNING);DbInstance instance = rdsService.createDbInstance("dead-db", "postgres", "16", "admin", "password", "dbname","db.t3.micro", 20, false, null, null, null, null, false, false,@@ -7207,6 +7208,44 @@ class RdsServiceTest {assertEquals(1, rdsService.describeEvents("dead-db", "db-instance", null, null, 60).size());}+ @Test+ void refreshRuntimeHealthKeepsAvailableStatusWhenLivenessProbeFails() {+ when(rdsConfig.mock()).thenReturn(false);+ when(containerManager.probeContainer("cont-id")).thenReturn(ContainerLiveness.UNKNOWN);+ DbInstance instance = rdsService.createDbInstance(+ "blip-db", "postgres", "16", "admin", "password", "dbname",+ "db.t3.micro", 20, false, null, null, null, null, false, false,+ null, Map.of(), List.of(), null, null, true);++ DbInstance refreshed = rdsService.refreshDbInstanceRuntimeHealth(instance);++ assertEquals(DbInstanceStatus.AVAILABLE, refreshed.getStatus());+ verify(proxyManager, never()).stopProxy("rds-resource:" + refreshed.getDbInstanceArn());+ assertTrue(rdsService.describeEvents("blip-db", "db-instance", null, null, 60).isEmpty());+ }++ @Test+ void refreshRuntimeHealthReprobesAfterAFailedProbeAndActsOnTheNextDefiniteAnswer() {+ when(rdsConfig.mock()).thenReturn(false);+ when(containerManager.probeContainer("cont-id"))+ .thenReturn(ContainerLiveness.UNKNOWN, ContainerLiveness.NOT_RUNNING);+ DbInstance instance = rdsService.createDbInstance(+ "recover-db", "postgres", "16", "admin", "password", "dbname",+ "db.t3.micro", 20, false, null, null, null, null, false, false,+ null, Map.of(), List.of(), null, null, true);+ String proxyKey = "rds-resource:" + instance.getDbInstanceArn();++ DbInstance afterBlip = rdsService.refreshDbInstanceRuntimeHealth(instance);+ assertEquals(DbInstanceStatus.AVAILABLE, afterBlip.getStatus());+ verify(proxyManager, never()).stopProxy(proxyKey);++ DbInstance afterDefiniteAnswer = rdsService.refreshDbInstanceRuntimeHealth(afterBlip);+ assertEquals(DbInstanceStatus.FAILED, afterDefiniteAnswer.getStatus());+ verify(proxyManager).stopProxy(proxyKey);+ verify(containerManager, times(2)).probeContainer("cont-id");+ assertEquals(1, rdsService.describeEvents("recover-db", "db-instance", null, null, 60).size());+ }+@Testvoid describeEventsPrunesEventsOlderThanFourteenDays() throws Exception {InMemoryStorage<String, RdsEvent> eventStore = new InMemoryStorage<>();