Light, fluffy, and always free - The AWS Local Emulator alternative
fix(mwaa): mark CREATE_FAILED when the Airflow container exits before becoming ready (#3446)
Priamo Germosen2a682a68756f
diff --git a/docs/services/mwaa.md b/docs/services/mwaa.mdindex 8855a461b8..0998663d80 100644--- a/docs/services/mwaa.md+++ b/docs/services/mwaa.md@@ -36,7 +36,7 @@ Floci starts two containers per environment:Legacy environments retain the region recorded in their ARN. A request in another region does notadopt that environment; recreate it in the requested region instead.-Once Airflow's unauthenticated `/health` endpoint reports both `metadatabase` and `scheduler` as `"healthy"`, the environment transitions to `AVAILABLE`.+Once Airflow's unauthenticated `/health` endpoint reports both `metadatabase` and `scheduler` as `"healthy"`, the environment transitions to `AVAILABLE`. If the Airflow container instead exits before that (a startup script or `airflow db migrate` failing partway through), the same poller detects the stopped container and transitions the environment to `CREATE_FAILED` rather than polling a dead container forever.### Web/CLI proxydiff --git a/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.java b/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.javaindex bef24b32e9..c693de606b 100644--- a/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.java@@ -12,6 +12,7 @@ import io.github.hectorvent.floci.core.common.docker.LaunchedContainerAwsEnv;import io.github.hectorvent.floci.services.mwaa.model.Environment;import com.github.dockerjava.api.async.ResultCallback;import com.github.dockerjava.api.command.InspectContainerResponse;+import com.github.dockerjava.api.exception.NotFoundException;import com.github.dockerjava.api.model.ContainerNetwork;import com.github.dockerjava.api.model.Frame;import com.github.dockerjava.api.model.StreamType;@@ -262,6 +263,36 @@ public class MwaaEnvironmentManager {}}+ /**+ * True when the Airflow container was created but is no longer running, for example because a+ * startup script or {@code airflow db migrate} failed after {@code docker start} returned.+ * {@code docker start} only launches the entrypoint and returns immediately, so nothing else+ * observes a failure like that; {@link MwaaService}'s readiness poller uses this to stop+ * waiting on a container that will never answer {@code /health}, instead of leaving the+ * environment at CREATING forever.+ */+ public boolean hasAirflowContainerExited(Environment environment) {+ String containerId = environment.getAirflowContainerId();+ if (containerId == null) {+ return false;+ }+ try {+ InspectContainerResponse inspect = lifecycleManager.getDockerClient().inspectContainerCmd(containerId).exec();+ return !Boolean.TRUE.equals(inspect.getState().getRunning());+ } catch (NotFoundException e) {+ return true;+ } catch (Exception e) {+ // Inconclusive (a transient Docker daemon issue, not "the container is gone"): treat as+ // still running, like isReady()'s own catch-all does for a failed /health call, so one+ // bad inspect doesn't wrongly fail this environment or, since the readiness poller+ // shares one loop across every CREATING environment, escape and starve every other+ // environment's check for this poll tick.+ LOG.warnv("Could not inspect Airflow container {0} for environment {1}: {2}",+ containerId, environment.getName(), e.getMessage());+ return false;+ }+ }+/** Crude but dependency-free check for {@code "<section>":{"status":"healthy"...}} in the /health JSON. */static boolean healthySection(String body, String section) {int idx = body.indexOf("\"" + section + "\"");diff --git a/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaService.java b/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaService.javaindex 6636cc6fca..7c593ba4f6 100644--- a/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaService.java+++ b/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaService.java@@ -432,12 +432,7 @@ public class MwaaService implements TagHandler {readinessPoller.scheduleAtFixedRate(() -> {try {for (Environment environment : allEnvironments()) {- if (environment.getStatus() == EnvironmentStatus.CREATING- && environmentManager.isReady(environment)) {- LOG.infov("MWAA environment {0} is now AVAILABLE", environment.getName());- environment.setStatus(EnvironmentStatus.AVAILABLE);- putEnvironment(environment);- }+ checkReadiness(environment);}} catch (Exception e) {LOG.error("Error in MWAA readiness poller", e);@@ -445,6 +440,39 @@ public class MwaaService implements TagHandler {}, 2, 3, TimeUnit.SECONDS);}+ // Package-private (not private) so MwaaServiceTest can exercise a single poll pass directly,+ // without waiting on the scheduled executor.+ void checkReadiness(Environment environment) {+ if (environment.getStatus() != EnvironmentStatus.CREATING) {+ return;+ }+ boolean ready = environmentManager.isReady(environment);+ boolean exited = !ready && environmentManager.hasAirflowContainerExited(environment);++ // isReady()/hasAirflowContainerExited() are blocking Docker/HTTP calls, long enough for a+ // concurrent DeleteEnvironment (which sets DELETING on this same Environment instance+ // before tearing down its containers) to have moved this environment past CREATING in the+ // meantime. Re-checking right before writing avoids resurrecting a just-deleted environment+ // into storage with a status decided from stale, pre-delete information.+ if (environment.getStatus() != EnvironmentStatus.CREATING) {+ return;+ }+ if (ready) {+ LOG.infov("MWAA environment {0} is now AVAILABLE", environment.getName());+ environment.setStatus(EnvironmentStatus.AVAILABLE);+ putEnvironment(environment);+ } else if (exited) {+ // docker start returns as soon as the entrypoint launches, so a script or migration+ // failing partway through never surfaces here on its own; without this check the+ // environment would poll a dead container and report CREATING forever instead of the+ // CREATE_FAILED a real failure should be.+ LOG.errorv("MWAA environment {0}''s Airflow container exited before becoming ready; "+ + "marking CREATE_FAILED", environment.getName());+ environment.setStatus(EnvironmentStatus.CREATE_FAILED);+ putEnvironment(environment);+ }+ }+private void startDagSyncPoller() {int intervalSeconds = Math.max(1, config.services().mwaa().dagSyncIntervalSeconds());dagSyncPoller.scheduleAtFixedRate(() -> {diff --git a/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.javaindex d067ef64ad..fcd2dccfc3 100644--- a/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.java@@ -13,6 +13,7 @@ import io.github.hectorvent.floci.core.common.dns.EmbeddedDnsServer;import io.github.hectorvent.floci.services.mwaa.model.Environment;import com.github.dockerjava.api.DockerClient;import com.github.dockerjava.api.command.CopyArchiveToContainerCmd;+import com.github.dockerjava.api.exception.NotFoundException;import com.github.dockerjava.api.model.Mount;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Nested;@@ -363,4 +364,66 @@ class MwaaEnvironmentManagerTest {verify(lifecycleManager, never()).startCreated(anyString(), any());}}++ /** Crash detection the readiness poller relies on to stop waiting on a dead container,+ * verified against a mocked DockerClient (mirrors BatchDockerRunnerTest's deep-stub style). */+ @Nested+ class ContainerExitDetection {++ private DockerClient dockerClient;++ @BeforeEach+ void setUpDockerClient() {+ dockerClient = Mockito.mock(DockerClient.class, Mockito.RETURNS_DEEP_STUBS);+ when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);+ }++ private Environment environmentWithContainerId(String containerId) {+ Environment environment = new Environment();+ environment.setName("my-env");+ environment.setAirflowContainerId(containerId);+ return environment;+ }++ @Test+ void reportsNotExitedWhileTheContainerIsStillRunning() {+ when(dockerClient.inspectContainerCmd("airflow-container-id").exec().getState().getRunning())+ .thenReturn(true);++ assertFalse(manager.hasAirflowContainerExited(environmentWithContainerId("airflow-container-id")));+ }++ @Test+ void reportsExitedOnceTheContainerHasStopped() {+ when(dockerClient.inspectContainerCmd("airflow-container-id").exec().getState().getRunning())+ .thenReturn(false);++ assertTrue(manager.hasAirflowContainerExited(environmentWithContainerId("airflow-container-id")));+ }++ @Test+ void reportsExitedWhenTheContainerHasBeenRemoved() {+ when(dockerClient.inspectContainerCmd("airflow-container-id").exec())+ .thenThrow(new NotFoundException("no such container"));++ assertTrue(manager.hasAirflowContainerExited(environmentWithContainerId("airflow-container-id")));+ }++ @Test+ void reportsNotExitedWhenNoContainerHasBeenCreatedYet() {+ assertFalse(manager.hasAirflowContainerExited(new Environment()));+ }++ @Test+ void reportsNotExitedWhenTheDockerDaemonInspectFailsForAnUnrelatedReason() {+ // Anything other than NotFoundException is inconclusive, not "the container is gone" -+ // must not propagate, since the readiness poller shares one loop across every CREATING+ // environment and a thrown exception here would starve every other environment's check+ // for this poll tick, not just this one's.+ when(dockerClient.inspectContainerCmd("airflow-container-id").exec())+ .thenThrow(new RuntimeException("docker daemon connection reset"));++ assertFalse(manager.hasAirflowContainerExited(environmentWithContainerId("airflow-container-id")));+ }+ }}diff --git a/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaServiceTest.javaindex 795f6fb183..932a785d69 100644--- a/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaServiceTest.java@@ -540,5 +540,89 @@ class MwaaServiceTest {verify(environmentManager).stopEnvironment(environment);verify(proxyManager).stopProxy(MwaaService.environmentIdentity(environment));}++ @Test+ void checkReadinessMarksEnvironmentAvailableOnceReady() {+ Environment environment = realModeService.createEnvironment("ready-env",+ createRequest("arn:aws:s3:::my-bucket", "dags"));+ when(environmentManager.isReady(environment)).thenReturn(true);++ realModeService.checkReadiness(environment);++ assertEquals(EnvironmentStatus.AVAILABLE, environment.getStatus());+ }++ @Test+ void checkReadinessLeavesEnvironmentCreatingWhileContainerIsStillStartingUp() {+ Environment environment = realModeService.createEnvironment("starting-env",+ createRequest("arn:aws:s3:::my-bucket", "dags"));+ when(environmentManager.isReady(environment)).thenReturn(false);+ when(environmentManager.hasAirflowContainerExited(environment)).thenReturn(false);++ realModeService.checkReadiness(environment);++ assertEquals(EnvironmentStatus.CREATING, environment.getStatus());+ }++ @Test+ void checkReadinessMarksCreateFailedWhenTheAirflowContainerHasExited() {+ // e.g. a startup script or `airflow db migrate` failed after docker start returned;+ // without this, the environment would poll a dead container forever and never leave+ // CREATING.+ Environment environment = realModeService.createEnvironment("crashed-env",+ createRequest("arn:aws:s3:::my-bucket", "dags"));+ when(environmentManager.isReady(environment)).thenReturn(false);+ when(environmentManager.hasAirflowContainerExited(environment)).thenReturn(true);++ realModeService.checkReadiness(environment);++ assertEquals(EnvironmentStatus.CREATE_FAILED, environment.getStatus());+ }++ @Test+ void checkReadinessIgnoresEnvironmentsNotInCreatingStatus() {+ Environment environment = realModeService.createEnvironment("available-env",+ createRequest("arn:aws:s3:::my-bucket", "dags"));+ environment.setStatus(EnvironmentStatus.AVAILABLE);++ realModeService.checkReadiness(environment);++ assertEquals(EnvironmentStatus.AVAILABLE, environment.getStatus());+ verify(environmentManager, Mockito.never()).isReady(any());+ verify(environmentManager, Mockito.never()).hasAirflowContainerExited(any());+ }++ @Test+ void checkReadinessDoesNotOverwriteAStatusChangedConcurrentlyWhileWaitingOnIsReady() {+ // isReady() is a blocking HTTP call; simulate a concurrent DeleteEnvironment moving the+ // same Environment instance to DELETING while that call is in flight.+ Environment environment = realModeService.createEnvironment("deleted-mid-check-env",+ createRequest("arn:aws:s3:::my-bucket", "dags"));+ when(environmentManager.isReady(environment)).thenAnswer(invocation -> {+ environment.setStatus(EnvironmentStatus.DELETING);+ return true;+ });++ realModeService.checkReadiness(environment);++ assertEquals(EnvironmentStatus.DELETING, environment.getStatus());+ }++ @Test+ void checkReadinessDoesNotOverwriteAStatusChangedConcurrentlyWhileWaitingOnContainerExitCheck() {+ // Same race, on the hasAirflowContainerExited() branch: without the re-check, this would+ // resurrect a just-deleted environment into storage as CREATE_FAILED.+ Environment environment = realModeService.createEnvironment("deleted-mid-crash-check-env",+ createRequest("arn:aws:s3:::my-bucket", "dags"));+ when(environmentManager.isReady(environment)).thenReturn(false);+ when(environmentManager.hasAirflowContainerExited(environment)).thenAnswer(invocation -> {+ environment.setStatus(EnvironmentStatus.DELETING);+ return true;+ });++ realModeService.checkReadiness(environment);++ assertEquals(EnvironmentStatus.DELETING, environment.getStatus());+ }}}