Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(mwaa): mark CREATE_FAILED when the Airflow container exits before becoming ready (#3446)

Priamo Germosen2a682a68756f
diff --git a/docs/services/mwaa.md b/docs/services/mwaa.md
index 8855a461b8..0998663d80 100644
--- a/docs/services/mwaa.md
+++ b/docs/services/mwaa.md
@@ -36,7 +36,7 @@ Floci starts two containers per environment:
Legacy environments retain the region recorded in their ARN. A request in another region does not
adopt that environment; recreate it in the requested region instead.
-Once Airflow's unauthenticated `/health` endpoint reports both `metadatabase` and `scheduler` as `"healthy"`, the environment transitions to `AVAILABLE`.
+Once Airflow's unauthenticated `/health` endpoint reports both `metadatabase` and `scheduler` as `"healthy"`, the environment transitions to `AVAILABLE`. If the Airflow container instead exits before that (a startup script or `airflow db migrate` failing partway through), the same poller detects the stopped container and transitions the environment to `CREATE_FAILED` rather than polling a dead container forever.
### Web/CLI proxy
diff --git a/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.java b/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.java
index bef24b32e9..c693de606b 100644
--- a/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.java
@@ -12,6 +12,7 @@ import io.github.hectorvent.floci.core.common.docker.LaunchedContainerAwsEnv;
import io.github.hectorvent.floci.services.mwaa.model.Environment;
import com.github.dockerjava.api.async.ResultCallback;
import com.github.dockerjava.api.command.InspectContainerResponse;
+import com.github.dockerjava.api.exception.NotFoundException;
import com.github.dockerjava.api.model.ContainerNetwork;
import com.github.dockerjava.api.model.Frame;
import com.github.dockerjava.api.model.StreamType;
@@ -262,6 +263,36 @@ public class MwaaEnvironmentManager {
}
}
+ /**
+ * True when the Airflow container was created but is no longer running, for example because a
+ * startup script or {@code airflow db migrate} failed after {@code docker start} returned.
+ * {@code docker start} only launches the entrypoint and returns immediately, so nothing else
+ * observes a failure like that; {@link MwaaService}'s readiness poller uses this to stop
+ * waiting on a container that will never answer {@code /health}, instead of leaving the
+ * environment at CREATING forever.
+ */
+ public boolean hasAirflowContainerExited(Environment environment) {
+ String containerId = environment.getAirflowContainerId();
+ if (containerId == null) {
+ return false;
+ }
+ try {
+ InspectContainerResponse inspect = lifecycleManager.getDockerClient().inspectContainerCmd(containerId).exec();
+ return !Boolean.TRUE.equals(inspect.getState().getRunning());
+ } catch (NotFoundException e) {
+ return true;
+ } catch (Exception e) {
+ // Inconclusive (a transient Docker daemon issue, not "the container is gone"): treat as
+ // still running, like isReady()'s own catch-all does for a failed /health call, so one
+ // bad inspect doesn't wrongly fail this environment or, since the readiness poller
+ // shares one loop across every CREATING environment, escape and starve every other
+ // environment's check for this poll tick.
+ LOG.warnv("Could not inspect Airflow container {0} for environment {1}: {2}",
+ containerId, environment.getName(), e.getMessage());
+ return false;
+ }
+ }
+
/** Crude but dependency-free check for {@code "<section>":{"status":"healthy"...}} in the /health JSON. */
static boolean healthySection(String body, String section) {
int idx = body.indexOf("\"" + section + "\"");
diff --git a/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaService.java b/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaService.java
index 6636cc6fca..7c593ba4f6 100644
--- a/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaService.java
@@ -432,12 +432,7 @@ public class MwaaService implements TagHandler {
readinessPoller.scheduleAtFixedRate(() -> {
try {
for (Environment environment : allEnvironments()) {
- if (environment.getStatus() == EnvironmentStatus.CREATING
- && environmentManager.isReady(environment)) {
- LOG.infov("MWAA environment {0} is now AVAILABLE", environment.getName());
- environment.setStatus(EnvironmentStatus.AVAILABLE);
- putEnvironment(environment);
- }
+ checkReadiness(environment);
}
} catch (Exception e) {
LOG.error("Error in MWAA readiness poller", e);
@@ -445,6 +440,39 @@ public class MwaaService implements TagHandler {
}, 2, 3, TimeUnit.SECONDS);
}
+ // Package-private (not private) so MwaaServiceTest can exercise a single poll pass directly,
+ // without waiting on the scheduled executor.
+ void checkReadiness(Environment environment) {
+ if (environment.getStatus() != EnvironmentStatus.CREATING) {
+ return;
+ }
+ boolean ready = environmentManager.isReady(environment);
+ boolean exited = !ready && environmentManager.hasAirflowContainerExited(environment);
+
+ // isReady()/hasAirflowContainerExited() are blocking Docker/HTTP calls, long enough for a
+ // concurrent DeleteEnvironment (which sets DELETING on this same Environment instance
+ // before tearing down its containers) to have moved this environment past CREATING in the
+ // meantime. Re-checking right before writing avoids resurrecting a just-deleted environment
+ // into storage with a status decided from stale, pre-delete information.
+ if (environment.getStatus() != EnvironmentStatus.CREATING) {
+ return;
+ }
+ if (ready) {
+ LOG.infov("MWAA environment {0} is now AVAILABLE", environment.getName());
+ environment.setStatus(EnvironmentStatus.AVAILABLE);
+ putEnvironment(environment);
+ } else if (exited) {
+ // docker start returns as soon as the entrypoint launches, so a script or migration
+ // failing partway through never surfaces here on its own; without this check the
+ // environment would poll a dead container and report CREATING forever instead of the
+ // CREATE_FAILED a real failure should be.
+ LOG.errorv("MWAA environment {0}''s Airflow container exited before becoming ready; "
+ + "marking CREATE_FAILED", environment.getName());
+ environment.setStatus(EnvironmentStatus.CREATE_FAILED);
+ putEnvironment(environment);
+ }
+ }
+
private void startDagSyncPoller() {
int intervalSeconds = Math.max(1, config.services().mwaa().dagSyncIntervalSeconds());
dagSyncPoller.scheduleAtFixedRate(() -> {
diff --git a/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.java
index d067ef64ad..fcd2dccfc3 100644
--- a/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.java
@@ -13,6 +13,7 @@ import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;
import io.github.hectorvent.floci.services.mwaa.model.Environment;
import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.command.CopyArchiveToContainerCmd;
+import com.github.dockerjava.api.exception.NotFoundException;
import com.github.dockerjava.api.model.Mount;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Nested;
@@ -363,4 +364,66 @@ class MwaaEnvironmentManagerTest {
verify(lifecycleManager, never()).startCreated(anyString(), any());
}
}
+
+ /** Crash detection the readiness poller relies on to stop waiting on a dead container,
+ * verified against a mocked DockerClient (mirrors BatchDockerRunnerTest's deep-stub style). */
+ @Nested
+ class ContainerExitDetection {
+
+ private DockerClient dockerClient;
+
+ @BeforeEach
+ void setUpDockerClient() {
+ dockerClient = Mockito.mock(DockerClient.class, Mockito.RETURNS_DEEP_STUBS);
+ when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);
+ }
+
+ private Environment environmentWithContainerId(String containerId) {
+ Environment environment = new Environment();
+ environment.setName("my-env");
+ environment.setAirflowContainerId(containerId);
+ return environment;
+ }
+
+ @Test
+ void reportsNotExitedWhileTheContainerIsStillRunning() {
+ when(dockerClient.inspectContainerCmd("airflow-container-id").exec().getState().getRunning())
+ .thenReturn(true);
+
+ assertFalse(manager.hasAirflowContainerExited(environmentWithContainerId("airflow-container-id")));
+ }
+
+ @Test
+ void reportsExitedOnceTheContainerHasStopped() {
+ when(dockerClient.inspectContainerCmd("airflow-container-id").exec().getState().getRunning())
+ .thenReturn(false);
+
+ assertTrue(manager.hasAirflowContainerExited(environmentWithContainerId("airflow-container-id")));
+ }
+
+ @Test
+ void reportsExitedWhenTheContainerHasBeenRemoved() {
+ when(dockerClient.inspectContainerCmd("airflow-container-id").exec())
+ .thenThrow(new NotFoundException("no such container"));
+
+ assertTrue(manager.hasAirflowContainerExited(environmentWithContainerId("airflow-container-id")));
+ }
+
+ @Test
+ void reportsNotExitedWhenNoContainerHasBeenCreatedYet() {
+ assertFalse(manager.hasAirflowContainerExited(new Environment()));
+ }
+
+ @Test
+ void reportsNotExitedWhenTheDockerDaemonInspectFailsForAnUnrelatedReason() {
+ // Anything other than NotFoundException is inconclusive, not "the container is gone" -
+ // must not propagate, since the readiness poller shares one loop across every CREATING
+ // environment and a thrown exception here would starve every other environment's check
+ // for this poll tick, not just this one's.
+ when(dockerClient.inspectContainerCmd("airflow-container-id").exec())
+ .thenThrow(new RuntimeException("docker daemon connection reset"));
+
+ assertFalse(manager.hasAirflowContainerExited(environmentWithContainerId("airflow-container-id")));
+ }
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaServiceTest.java
index 795f6fb183..932a785d69 100644
--- a/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaServiceTest.java
@@ -540,5 +540,89 @@ class MwaaServiceTest {
verify(environmentManager).stopEnvironment(environment);
verify(proxyManager).stopProxy(MwaaService.environmentIdentity(environment));
}
+
+ @Test
+ void checkReadinessMarksEnvironmentAvailableOnceReady() {
+ Environment environment = realModeService.createEnvironment("ready-env",
+ createRequest("arn:aws:s3:::my-bucket", "dags"));
+ when(environmentManager.isReady(environment)).thenReturn(true);
+
+ realModeService.checkReadiness(environment);
+
+ assertEquals(EnvironmentStatus.AVAILABLE, environment.getStatus());
+ }
+
+ @Test
+ void checkReadinessLeavesEnvironmentCreatingWhileContainerIsStillStartingUp() {
+ Environment environment = realModeService.createEnvironment("starting-env",
+ createRequest("arn:aws:s3:::my-bucket", "dags"));
+ when(environmentManager.isReady(environment)).thenReturn(false);
+ when(environmentManager.hasAirflowContainerExited(environment)).thenReturn(false);
+
+ realModeService.checkReadiness(environment);
+
+ assertEquals(EnvironmentStatus.CREATING, environment.getStatus());
+ }
+
+ @Test
+ void checkReadinessMarksCreateFailedWhenTheAirflowContainerHasExited() {
+ // e.g. a startup script or `airflow db migrate` failed after docker start returned;
+ // without this, the environment would poll a dead container forever and never leave
+ // CREATING.
+ Environment environment = realModeService.createEnvironment("crashed-env",
+ createRequest("arn:aws:s3:::my-bucket", "dags"));
+ when(environmentManager.isReady(environment)).thenReturn(false);
+ when(environmentManager.hasAirflowContainerExited(environment)).thenReturn(true);
+
+ realModeService.checkReadiness(environment);
+
+ assertEquals(EnvironmentStatus.CREATE_FAILED, environment.getStatus());
+ }
+
+ @Test
+ void checkReadinessIgnoresEnvironmentsNotInCreatingStatus() {
+ Environment environment = realModeService.createEnvironment("available-env",
+ createRequest("arn:aws:s3:::my-bucket", "dags"));
+ environment.setStatus(EnvironmentStatus.AVAILABLE);
+
+ realModeService.checkReadiness(environment);
+
+ assertEquals(EnvironmentStatus.AVAILABLE, environment.getStatus());
+ verify(environmentManager, Mockito.never()).isReady(any());
+ verify(environmentManager, Mockito.never()).hasAirflowContainerExited(any());
+ }
+
+ @Test
+ void checkReadinessDoesNotOverwriteAStatusChangedConcurrentlyWhileWaitingOnIsReady() {
+ // isReady() is a blocking HTTP call; simulate a concurrent DeleteEnvironment moving the
+ // same Environment instance to DELETING while that call is in flight.
+ Environment environment = realModeService.createEnvironment("deleted-mid-check-env",
+ createRequest("arn:aws:s3:::my-bucket", "dags"));
+ when(environmentManager.isReady(environment)).thenAnswer(invocation -> {
+ environment.setStatus(EnvironmentStatus.DELETING);
+ return true;
+ });
+
+ realModeService.checkReadiness(environment);
+
+ assertEquals(EnvironmentStatus.DELETING, environment.getStatus());
+ }
+
+ @Test
+ void checkReadinessDoesNotOverwriteAStatusChangedConcurrentlyWhileWaitingOnContainerExitCheck() {
+ // Same race, on the hasAirflowContainerExited() branch: without the re-check, this would
+ // resurrect a just-deleted environment into storage as CREATE_FAILED.
+ Environment environment = realModeService.createEnvironment("deleted-mid-crash-check-env",
+ createRequest("arn:aws:s3:::my-bucket", "dags"));
+ when(environmentManager.isReady(environment)).thenReturn(false);
+ when(environmentManager.hasAirflowContainerExited(environment)).thenAnswer(invocation -> {
+ environment.setStatus(EnvironmentStatus.DELETING);
+ return true;
+ });
+
+ realModeService.checkReadiness(environment);
+
+ assertEquals(EnvironmentStatus.DELETING, environment.getStatus());
+ }
}
}