Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(mwaa): also detect a crashed sibling Postgres container, not just Airflow (#3454)

Priamo Germosenfe076212c2e2
diff --git a/docs/services/mwaa.md b/docs/services/mwaa.md
index 0998663d80..8deecd4b6c 100644
--- a/docs/services/mwaa.md
+++ b/docs/services/mwaa.md
@@ -36,7 +36,7 @@ Floci starts two containers per environment:
Legacy environments retain the region recorded in their ARN. A request in another region does not
adopt that environment; recreate it in the requested region instead.
-Once Airflow's unauthenticated `/health` endpoint reports both `metadatabase` and `scheduler` as `"healthy"`, the environment transitions to `AVAILABLE`. If the Airflow container instead exits before that (a startup script or `airflow db migrate` failing partway through), the same poller detects the stopped container and transitions the environment to `CREATE_FAILED` rather than polling a dead container forever.
+Once Airflow's unauthenticated `/health` endpoint reports both `metadatabase` and `scheduler` as `"healthy"`, the environment transitions to `AVAILABLE`. If the Postgres or Airflow container instead exits before that (a startup script or `airflow db migrate` failing partway through, or Postgres itself dying), the same poller detects the stopped container and transitions the environment to `CREATE_FAILED` rather than polling dead containers forever.
### Web/CLI proxy
diff --git a/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.java b/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.java
index c693de606b..719293e74c 100644
--- a/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.java
@@ -264,15 +264,22 @@ public class MwaaEnvironmentManager {
}
/**
- * True when the Airflow container was created but is no longer running, for example because a
- * startup script or {@code airflow db migrate} failed after {@code docker start} returned.
- * {@code docker start} only launches the entrypoint and returns immediately, so nothing else
- * observes a failure like that; {@link MwaaService}'s readiness poller uses this to stop
- * waiting on a container that will never answer {@code /health}, instead of leaving the
- * environment at CREATING forever.
+ * True when either container backing this environment, Postgres or Airflow, was created but is
+ * no longer running: for example a startup script or {@code airflow db migrate} failing
+ * partway through kills the Airflow container, or the Postgres container itself dies (out of
+ * memory, its volume filling up), which never surfaces as an Airflow-side failure since the
+ * Airflow process itself keeps running, just never reporting {@code metadatabase} healthy.
+ * {@code docker start} only launches a container's entrypoint and returns immediately, so
+ * nothing else observes either of those; {@link MwaaService}'s readiness poller uses this to
+ * stop waiting on containers that will never let the environment answer {@code /health},
+ * instead of leaving the environment at CREATING forever.
*/
- public boolean hasAirflowContainerExited(Environment environment) {
- String containerId = environment.getAirflowContainerId();
+ public boolean hasAnyContainerExited(Environment environment) {
+ return hasContainerExited(environment.getDbContainerId(), "Postgres", environment)
+ || hasContainerExited(environment.getAirflowContainerId(), "Airflow", environment);
+ }
+
+ private boolean hasContainerExited(String containerId, String label, Environment environment) {
if (containerId == null) {
return false;
}
@@ -287,8 +294,8 @@ public class MwaaEnvironmentManager {
// bad inspect doesn't wrongly fail this environment or, since the readiness poller
// shares one loop across every CREATING environment, escape and starve every other
// environment's check for this poll tick.
- LOG.warnv("Could not inspect Airflow container {0} for environment {1}: {2}",
- containerId, environment.getName(), e.getMessage());
+ LOG.warnv("Could not inspect {0} container {1} for environment {2}: {3}",
+ label, containerId, environment.getName(), e.getMessage());
return false;
}
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaService.java b/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaService.java
index e74a107c95..2c8e800c3c 100644
--- a/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaService.java
@@ -458,9 +458,9 @@ public class MwaaService implements TagHandler {
return;
}
boolean ready = environmentManager.isReady(environment);
- boolean exited = !ready && environmentManager.hasAirflowContainerExited(environment);
+ boolean exited = !ready && environmentManager.hasAnyContainerExited(environment);
- // isReady()/hasAirflowContainerExited() are blocking Docker/HTTP calls, long enough for a
+ // isReady()/hasAnyContainerExited() are blocking Docker/HTTP calls, long enough for a
// concurrent DeleteEnvironment (which sets DELETING on this same Environment instance
// before tearing down its containers) to have moved this environment past CREATING in the
// meantime. Re-checking right before writing avoids resurrecting a just-deleted environment
@@ -473,12 +473,12 @@ public class MwaaService implements TagHandler {
environment.setStatus(EnvironmentStatus.AVAILABLE);
putEnvironment(environment);
} else if (exited) {
- // docker start returns as soon as the entrypoint launches, so a script or migration
- // failing partway through never surfaces here on its own; without this check the
- // environment would poll a dead container and report CREATING forever instead of the
- // CREATE_FAILED a real failure should be.
- LOG.errorv("MWAA environment {0}''s Airflow container exited before becoming ready; "
- + "marking CREATE_FAILED", environment.getName());
+ // docker start returns as soon as a container's entrypoint launches, so its Postgres or
+ // Airflow container dying partway through never surfaces here on its own; without this
+ // check the environment would poll dead containers and report CREATING forever instead
+ // of the CREATE_FAILED a real failure should be.
+ LOG.errorv("MWAA environment {0}''s Postgres or Airflow container exited before "
+ + "becoming ready; marking CREATE_FAILED", environment.getName());
environment.setStatus(EnvironmentStatus.CREATE_FAILED);
putEnvironment(environment);
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.java
index fcd2dccfc3..0e4f009a5b 100644
--- a/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.java
@@ -365,8 +365,8 @@ class MwaaEnvironmentManagerTest {
}
}
- /** Crash detection the readiness poller relies on to stop waiting on a dead container,
- * verified against a mocked DockerClient (mirrors BatchDockerRunnerTest's deep-stub style). */
+ /** Crash detection the readiness poller relies on to stop waiting on dead containers, verified
+ * against a mocked DockerClient (mirrors BatchDockerRunnerTest's deep-stub style). */
@Nested
class ContainerExitDetection {
@@ -378,40 +378,60 @@ class MwaaEnvironmentManagerTest {
when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);
}
- private Environment environmentWithContainerId(String containerId) {
+ private Environment environmentWithContainers(String dbContainerId, String airflowContainerId) {
Environment environment = new Environment();
environment.setName("my-env");
- environment.setAirflowContainerId(containerId);
+ environment.setDbContainerId(dbContainerId);
+ environment.setAirflowContainerId(airflowContainerId);
return environment;
}
@Test
- void reportsNotExitedWhileTheContainerIsStillRunning() {
+ void reportsNotExitedWhileBothContainersAreStillRunning() {
+ when(dockerClient.inspectContainerCmd("db-container-id").exec().getState().getRunning())
+ .thenReturn(true);
when(dockerClient.inspectContainerCmd("airflow-container-id").exec().getState().getRunning())
.thenReturn(true);
- assertFalse(manager.hasAirflowContainerExited(environmentWithContainerId("airflow-container-id")));
+ assertFalse(manager.hasAnyContainerExited(environmentWithContainers("db-container-id", "airflow-container-id")));
}
@Test
- void reportsExitedOnceTheContainerHasStopped() {
+ void reportsExitedOnceTheAirflowContainerHasStopped() {
+ when(dockerClient.inspectContainerCmd("db-container-id").exec().getState().getRunning())
+ .thenReturn(true);
when(dockerClient.inspectContainerCmd("airflow-container-id").exec().getState().getRunning())
.thenReturn(false);
- assertTrue(manager.hasAirflowContainerExited(environmentWithContainerId("airflow-container-id")));
+ assertTrue(manager.hasAnyContainerExited(environmentWithContainers("db-container-id", "airflow-container-id")));
}
@Test
- void reportsExitedWhenTheContainerHasBeenRemoved() {
+ void reportsExitedOnceTheSiblingPostgresContainerHasStopped() {
+ // Airflow itself keeps running when Postgres dies underneath it, it just never reports
+ // metadatabase healthy, so this must be detected independently of the Airflow
+ // container's own running state.
+ when(dockerClient.inspectContainerCmd("db-container-id").exec().getState().getRunning())
+ .thenReturn(false);
+ when(dockerClient.inspectContainerCmd("airflow-container-id").exec().getState().getRunning())
+ .thenReturn(true);
+
+ assertTrue(manager.hasAnyContainerExited(environmentWithContainers("db-container-id", "airflow-container-id")));
+ }
+
+ @Test
+ void reportsExitedWhenEitherContainerHasBeenRemoved() {
+ when(dockerClient.inspectContainerCmd("db-container-id").exec().getState().getRunning())
+ .thenReturn(true);
when(dockerClient.inspectContainerCmd("airflow-container-id").exec())
.thenThrow(new NotFoundException("no such container"));
- assertTrue(manager.hasAirflowContainerExited(environmentWithContainerId("airflow-container-id")));
+ assertTrue(manager.hasAnyContainerExited(environmentWithContainers("db-container-id", "airflow-container-id")));
}
@Test
- void reportsNotExitedWhenNoContainerHasBeenCreatedYet() {
- assertFalse(manager.hasAirflowContainerExited(new Environment()));
+ void reportsNotExitedWhenNeitherContainerHasBeenCreatedYet() {
+ assertFalse(manager.hasAnyContainerExited(new Environment()));
}
@Test
@@ -420,10 +440,12 @@ class MwaaEnvironmentManagerTest {
// must not propagate, since the readiness poller shares one loop across every CREATING
// environment and a thrown exception here would starve every other environment's check
// for this poll tick, not just this one's.
+ when(dockerClient.inspectContainerCmd("db-container-id").exec().getState().getRunning())
+ .thenReturn(true);
when(dockerClient.inspectContainerCmd("airflow-container-id").exec())
.thenThrow(new RuntimeException("docker daemon connection reset"));
- assertFalse(manager.hasAirflowContainerExited(environmentWithContainerId("airflow-container-id")));
+ assertFalse(manager.hasAnyContainerExited(environmentWithContainers("db-container-id", "airflow-container-id")));
}
}
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaServiceTest.java
index 6a9aa2dac9..fb4e202253 100644
--- a/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaServiceTest.java
@@ -590,7 +590,7 @@ class MwaaServiceTest {
Environment environment = realModeService.createEnvironment("starting-env",
createRequest("arn:aws:s3:::my-bucket", "dags"));
when(environmentManager.isReady(environment)).thenReturn(false);
- when(environmentManager.hasAirflowContainerExited(environment)).thenReturn(false);
+ when(environmentManager.hasAnyContainerExited(environment)).thenReturn(false);
realModeService.checkReadiness(environment);
@@ -598,14 +598,14 @@ class MwaaServiceTest {
}
@Test
- void checkReadinessMarksCreateFailedWhenTheAirflowContainerHasExited() {
- // e.g. a startup script or `airflow db migrate` failed after docker start returned;
- // without this, the environment would poll a dead container forever and never leave
- // CREATING.
+ void checkReadinessMarksCreateFailedWhenAContainerHasExited() {
+ // e.g. a startup script or `airflow db migrate` failed after docker start returned, or
+ // the sibling Postgres container died; without this, the environment would poll dead
+ // containers forever and never leave CREATING.
Environment environment = realModeService.createEnvironment("crashed-env",
createRequest("arn:aws:s3:::my-bucket", "dags"));
when(environmentManager.isReady(environment)).thenReturn(false);
- when(environmentManager.hasAirflowContainerExited(environment)).thenReturn(true);
+ when(environmentManager.hasAnyContainerExited(environment)).thenReturn(true);
realModeService.checkReadiness(environment);
@@ -622,7 +622,7 @@ class MwaaServiceTest {
assertEquals(EnvironmentStatus.AVAILABLE, environment.getStatus());
verify(environmentManager, Mockito.never()).isReady(any());
- verify(environmentManager, Mockito.never()).hasAirflowContainerExited(any());
+ verify(environmentManager, Mockito.never()).hasAnyContainerExited(any());
}
@Test
@@ -643,12 +643,12 @@ class MwaaServiceTest {
@Test
void checkReadinessDoesNotOverwriteAStatusChangedConcurrentlyWhileWaitingOnContainerExitCheck() {
- // Same race, on the hasAirflowContainerExited() branch: without the re-check, this would
+ // Same race, on the hasAnyContainerExited() branch: without the re-check, this would
// resurrect a just-deleted environment into storage as CREATE_FAILED.
Environment environment = realModeService.createEnvironment("deleted-mid-crash-check-env",
createRequest("arn:aws:s3:::my-bucket", "dags"));
when(environmentManager.isReady(environment)).thenReturn(false);
- when(environmentManager.hasAirflowContainerExited(environment)).thenAnswer(invocation -> {
+ when(environmentManager.hasAnyContainerExited(environment)).thenAnswer(invocation -> {
environment.setStatus(EnvironmentStatus.DELETING);
return true;
});