Light, fluffy, and always free - The AWS Local Emulator alternative
fix(mwaa): also detect a crashed sibling Postgres container, not just Airflow (#3454)
Priamo Germosenfe076212c2e2
diff --git a/docs/services/mwaa.md b/docs/services/mwaa.mdindex 0998663d80..8deecd4b6c 100644--- a/docs/services/mwaa.md+++ b/docs/services/mwaa.md@@ -36,7 +36,7 @@ Floci starts two containers per environment:Legacy environments retain the region recorded in their ARN. A request in another region does notadopt that environment; recreate it in the requested region instead.-Once Airflow's unauthenticated `/health` endpoint reports both `metadatabase` and `scheduler` as `"healthy"`, the environment transitions to `AVAILABLE`. If the Airflow container instead exits before that (a startup script or `airflow db migrate` failing partway through), the same poller detects the stopped container and transitions the environment to `CREATE_FAILED` rather than polling a dead container forever.+Once Airflow's unauthenticated `/health` endpoint reports both `metadatabase` and `scheduler` as `"healthy"`, the environment transitions to `AVAILABLE`. If the Postgres or Airflow container instead exits before that (a startup script or `airflow db migrate` failing partway through, or Postgres itself dying), the same poller detects the stopped container and transitions the environment to `CREATE_FAILED` rather than polling dead containers forever.### Web/CLI proxydiff --git a/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.java b/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.javaindex c693de606b..719293e74c 100644--- a/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.java@@ -264,15 +264,22 @@ public class MwaaEnvironmentManager {}/**- * True when the Airflow container was created but is no longer running, for example because a- * startup script or {@code airflow db migrate} failed after {@code docker start} returned.- * {@code docker start} only launches the entrypoint and returns immediately, so nothing else- * observes a failure like that; {@link MwaaService}'s readiness poller uses this to stop- * waiting on a container that will never answer {@code /health}, instead of leaving the- * environment at CREATING forever.+ * True when either container backing this environment, Postgres or Airflow, was created but is+ * no longer running: for example a startup script or {@code airflow db migrate} failing+ * partway through kills the Airflow container, or the Postgres container itself dies (out of+ * memory, its volume filling up), which never surfaces as an Airflow-side failure since the+ * Airflow process itself keeps running, just never reporting {@code metadatabase} healthy.+ * {@code docker start} only launches a container's entrypoint and returns immediately, so+ * nothing else observes either of those; {@link MwaaService}'s readiness poller uses this to+ * stop waiting on containers that will never let the environment answer {@code /health},+ * instead of leaving the environment at CREATING forever.*/- public boolean hasAirflowContainerExited(Environment environment) {- String containerId = environment.getAirflowContainerId();+ public boolean hasAnyContainerExited(Environment environment) {+ return hasContainerExited(environment.getDbContainerId(), "Postgres", environment)+ || hasContainerExited(environment.getAirflowContainerId(), "Airflow", environment);+ }++ private boolean hasContainerExited(String containerId, String label, Environment environment) {if (containerId == null) {return false;}@@ -287,8 +294,8 @@ public class MwaaEnvironmentManager {// bad inspect doesn't wrongly fail this environment or, since the readiness poller// shares one loop across every CREATING environment, escape and starve every other// environment's check for this poll tick.- LOG.warnv("Could not inspect Airflow container {0} for environment {1}: {2}",- containerId, environment.getName(), e.getMessage());+ LOG.warnv("Could not inspect {0} container {1} for environment {2}: {3}",+ label, containerId, environment.getName(), e.getMessage());return false;}}diff --git a/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaService.java b/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaService.javaindex e74a107c95..2c8e800c3c 100644--- a/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaService.java+++ b/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaService.java@@ -458,9 +458,9 @@ public class MwaaService implements TagHandler {return;}boolean ready = environmentManager.isReady(environment);- boolean exited = !ready && environmentManager.hasAirflowContainerExited(environment);+ boolean exited = !ready && environmentManager.hasAnyContainerExited(environment);- // isReady()/hasAirflowContainerExited() are blocking Docker/HTTP calls, long enough for a+ // isReady()/hasAnyContainerExited() are blocking Docker/HTTP calls, long enough for a// concurrent DeleteEnvironment (which sets DELETING on this same Environment instance// before tearing down its containers) to have moved this environment past CREATING in the// meantime. Re-checking right before writing avoids resurrecting a just-deleted environment@@ -473,12 +473,12 @@ public class MwaaService implements TagHandler {environment.setStatus(EnvironmentStatus.AVAILABLE);putEnvironment(environment);} else if (exited) {- // docker start returns as soon as the entrypoint launches, so a script or migration- // failing partway through never surfaces here on its own; without this check the- // environment would poll a dead container and report CREATING forever instead of the- // CREATE_FAILED a real failure should be.- LOG.errorv("MWAA environment {0}''s Airflow container exited before becoming ready; "- + "marking CREATE_FAILED", environment.getName());+ // docker start returns as soon as a container's entrypoint launches, so its Postgres or+ // Airflow container dying partway through never surfaces here on its own; without this+ // check the environment would poll dead containers and report CREATING forever instead+ // of the CREATE_FAILED a real failure should be.+ LOG.errorv("MWAA environment {0}''s Postgres or Airflow container exited before "+ + "becoming ready; marking CREATE_FAILED", environment.getName());environment.setStatus(EnvironmentStatus.CREATE_FAILED);putEnvironment(environment);}diff --git a/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.javaindex fcd2dccfc3..0e4f009a5b 100644--- a/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.java@@ -365,8 +365,8 @@ class MwaaEnvironmentManagerTest {}}- /** Crash detection the readiness poller relies on to stop waiting on a dead container,- * verified against a mocked DockerClient (mirrors BatchDockerRunnerTest's deep-stub style). */+ /** Crash detection the readiness poller relies on to stop waiting on dead containers, verified+ * against a mocked DockerClient (mirrors BatchDockerRunnerTest's deep-stub style). */@Nestedclass ContainerExitDetection {@@ -378,40 +378,60 @@ class MwaaEnvironmentManagerTest {when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);}- private Environment environmentWithContainerId(String containerId) {+ private Environment environmentWithContainers(String dbContainerId, String airflowContainerId) {Environment environment = new Environment();environment.setName("my-env");- environment.setAirflowContainerId(containerId);+ environment.setDbContainerId(dbContainerId);+ environment.setAirflowContainerId(airflowContainerId);return environment;}@Test- void reportsNotExitedWhileTheContainerIsStillRunning() {+ void reportsNotExitedWhileBothContainersAreStillRunning() {+ when(dockerClient.inspectContainerCmd("db-container-id").exec().getState().getRunning())+ .thenReturn(true);when(dockerClient.inspectContainerCmd("airflow-container-id").exec().getState().getRunning()).thenReturn(true);- assertFalse(manager.hasAirflowContainerExited(environmentWithContainerId("airflow-container-id")));+ assertFalse(manager.hasAnyContainerExited(environmentWithContainers("db-container-id", "airflow-container-id")));}@Test- void reportsExitedOnceTheContainerHasStopped() {+ void reportsExitedOnceTheAirflowContainerHasStopped() {+ when(dockerClient.inspectContainerCmd("db-container-id").exec().getState().getRunning())+ .thenReturn(true);when(dockerClient.inspectContainerCmd("airflow-container-id").exec().getState().getRunning()).thenReturn(false);- assertTrue(manager.hasAirflowContainerExited(environmentWithContainerId("airflow-container-id")));+ assertTrue(manager.hasAnyContainerExited(environmentWithContainers("db-container-id", "airflow-container-id")));}@Test- void reportsExitedWhenTheContainerHasBeenRemoved() {+ void reportsExitedOnceTheSiblingPostgresContainerHasStopped() {+ // Airflow itself keeps running when Postgres dies underneath it, it just never reports+ // metadatabase healthy, so this must be detected independently of the Airflow+ // container's own running state.+ when(dockerClient.inspectContainerCmd("db-container-id").exec().getState().getRunning())+ .thenReturn(false);+ when(dockerClient.inspectContainerCmd("airflow-container-id").exec().getState().getRunning())+ .thenReturn(true);++ assertTrue(manager.hasAnyContainerExited(environmentWithContainers("db-container-id", "airflow-container-id")));+ }++ @Test+ void reportsExitedWhenEitherContainerHasBeenRemoved() {+ when(dockerClient.inspectContainerCmd("db-container-id").exec().getState().getRunning())+ .thenReturn(true);when(dockerClient.inspectContainerCmd("airflow-container-id").exec()).thenThrow(new NotFoundException("no such container"));- assertTrue(manager.hasAirflowContainerExited(environmentWithContainerId("airflow-container-id")));+ assertTrue(manager.hasAnyContainerExited(environmentWithContainers("db-container-id", "airflow-container-id")));}@Test- void reportsNotExitedWhenNoContainerHasBeenCreatedYet() {- assertFalse(manager.hasAirflowContainerExited(new Environment()));+ void reportsNotExitedWhenNeitherContainerHasBeenCreatedYet() {+ assertFalse(manager.hasAnyContainerExited(new Environment()));}@Test@@ -420,10 +440,12 @@ class MwaaEnvironmentManagerTest {// must not propagate, since the readiness poller shares one loop across every CREATING// environment and a thrown exception here would starve every other environment's check// for this poll tick, not just this one's.+ when(dockerClient.inspectContainerCmd("db-container-id").exec().getState().getRunning())+ .thenReturn(true);when(dockerClient.inspectContainerCmd("airflow-container-id").exec()).thenThrow(new RuntimeException("docker daemon connection reset"));- assertFalse(manager.hasAirflowContainerExited(environmentWithContainerId("airflow-container-id")));+ assertFalse(manager.hasAnyContainerExited(environmentWithContainers("db-container-id", "airflow-container-id")));}}}diff --git a/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaServiceTest.javaindex 6a9aa2dac9..fb4e202253 100644--- a/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaServiceTest.java@@ -590,7 +590,7 @@ class MwaaServiceTest {Environment environment = realModeService.createEnvironment("starting-env",createRequest("arn:aws:s3:::my-bucket", "dags"));when(environmentManager.isReady(environment)).thenReturn(false);- when(environmentManager.hasAirflowContainerExited(environment)).thenReturn(false);+ when(environmentManager.hasAnyContainerExited(environment)).thenReturn(false);realModeService.checkReadiness(environment);@@ -598,14 +598,14 @@ class MwaaServiceTest {}@Test- void checkReadinessMarksCreateFailedWhenTheAirflowContainerHasExited() {- // e.g. a startup script or `airflow db migrate` failed after docker start returned;- // without this, the environment would poll a dead container forever and never leave- // CREATING.+ void checkReadinessMarksCreateFailedWhenAContainerHasExited() {+ // e.g. a startup script or `airflow db migrate` failed after docker start returned, or+ // the sibling Postgres container died; without this, the environment would poll dead+ // containers forever and never leave CREATING.Environment environment = realModeService.createEnvironment("crashed-env",createRequest("arn:aws:s3:::my-bucket", "dags"));when(environmentManager.isReady(environment)).thenReturn(false);- when(environmentManager.hasAirflowContainerExited(environment)).thenReturn(true);+ when(environmentManager.hasAnyContainerExited(environment)).thenReturn(true);realModeService.checkReadiness(environment);@@ -622,7 +622,7 @@ class MwaaServiceTest {assertEquals(EnvironmentStatus.AVAILABLE, environment.getStatus());verify(environmentManager, Mockito.never()).isReady(any());- verify(environmentManager, Mockito.never()).hasAirflowContainerExited(any());+ verify(environmentManager, Mockito.never()).hasAnyContainerExited(any());}@Test@@ -643,12 +643,12 @@ class MwaaServiceTest {@Testvoid checkReadinessDoesNotOverwriteAStatusChangedConcurrentlyWhileWaitingOnContainerExitCheck() {- // Same race, on the hasAirflowContainerExited() branch: without the re-check, this would+ // Same race, on the hasAnyContainerExited() branch: without the re-check, this would// resurrect a just-deleted environment into storage as CREATE_FAILED.Environment environment = realModeService.createEnvironment("deleted-mid-crash-check-env",createRequest("arn:aws:s3:::my-bucket", "dags"));when(environmentManager.isReady(environment)).thenReturn(false);- when(environmentManager.hasAirflowContainerExited(environment)).thenAnswer(invocation -> {+ when(environmentManager.hasAnyContainerExited(environment)).thenAnswer(invocation -> {environment.setStatus(EnvironmentStatus.DELETING);return true;});