Light, fluffy, and always free - The AWS Local Emulator alternative
fix(mwaa): match real MWAA's Airflow/Python pairing for the emulated image tag (#3445)
Priamo Germosen473a26aad92a
diff --git a/docs/services/mwaa.md b/docs/services/mwaa.mdindex 5f31b5ea4e..8855a461b8 100644--- a/docs/services/mwaa.md+++ b/docs/services/mwaa.md@@ -31,7 +31,7 @@ Environment metadata is stored in-process. No Docker containers are started. TheFloci starts two containers per environment:- `floci-mwaa-<account>.<region>.<name>-db` — a private `postgres` metadata database, never given a published host port; it's only ever reached by the sibling Airflow container over the Docker network.-- `floci-mwaa-<account>.<region>.<name>-airflow` — a real `apache/airflow` container running **LocalExecutor** (webserver + scheduler in one process tree). `AirflowVersion` genuinely selects the image tag (`apache/airflow:<version>-python3.12`), validated against `supported-versions` — unlike some Floci services where a requested version is echoed back but not actually applied, MWAA always runs the exact Airflow version requested.+- `floci-mwaa-<account>.<region>.<name>-airflow` — a real `apache/airflow` container running **LocalExecutor** (webserver + scheduler in one process tree). `AirflowVersion` genuinely selects the image tag (`apache/airflow:<version>-python3.11` for versions through 2.10.x, `apache/airflow:<version>-python3.12` from 2.11.0 onward, matching the Airflow/Python pairing real Amazon MWAA runs for each version), validated against `supported-versions` — unlike some Floci services where a requested version is echoed back but not actually applied, MWAA always runs the exact Airflow version requested.Legacy environments retain the region recorded in their ARN. A request in another region does notadopt that environment; recreate it in the requested region instead.diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.javaindex 16e105057d..d169987b9d 100644--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java@@ -2520,7 +2520,9 @@ public interface EmulatorConfig {String defaultPostgresImage();/** Airflow versions environments may request. Combined with the image tag- * {@code apache/airflow:<version>-python3.12}. */+ * {@code apache/airflow:<version>-<pythonTag>}, where the Python tag matches the+ * version real Amazon MWAA runs for that Airflow version: see+ * {@code MwaaEnvironmentManager.pythonTagFor}. */@WithDefault("2.10.5,2.9.3,2.8.4")List<String> supportedVersions();diff --git a/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.java b/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.javaindex 69536ae08f..bef24b32e9 100644--- a/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.java@@ -140,7 +140,7 @@ public class MwaaEnvironmentManager {lifecycleManager.ensureVolume(dagsVolume);lifecycleManager.ensureVolume(logsVolume);- String image = "apache/airflow:%s-python3.12".formatted(airflowVersion);+ String image = "apache/airflow:%s-%s".formatted(airflowVersion, pythonTagFor(airflowVersion));String adminUser = "admin";String adminPassword = generateSecret(24);String sqlAlchemyConn = "postgresql+psycopg2://airflow:" + dbPassword + "@" + dbIp + ":" + POSTGRES_PORT + "/airflow";@@ -365,6 +365,20 @@ public class MwaaEnvironmentManager {return AwsArnUtils.regionOrDefault(environment.getArn(), "us-east-1");}+ /**+ * The Python minor version tag real Amazon MWAA runs for a given {@code AirflowVersion}, so+ * the emulated image matches the same Airflow/Python pairing a requirements.txt built the way+ * AWS documents (a constraint file pinned to that pairing) expects. Per AWS's own Airflow+ * versions table, every version through 2.10.x runs Python 3.11, and 2.11.0 onward (including+ * every 3.x release) runs Python 3.12.+ */+ static String pythonTagFor(String airflowVersion) {+ String[] parts = airflowVersion.split("\\.", 3);+ int major = Integer.parseInt(parts[0]);+ int minor = parts.length > 1 ? Integer.parseInt(parts[1]) : 0;+ return (major > 2 || (major == 2 && minor >= 11)) ? "python3.12" : "python3.11";+ }+/*** Environment variables a startup script must not be able to permanently override — mirrors* real MWAA's documented "reserved environment variables" (which AWS silently restores after adiff --git a/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.javaindex 9d5efe1b49..d067ef64ad 100644--- a/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.java@@ -131,7 +131,21 @@ class MwaaEnvironmentManagerTest {@Testvoid airflowImageTagSubstitutesTheRequestedVersion() {ContainerSpec spec = startAirflowAndCaptureSpec(false);- assertEquals("apache/airflow:2.10.5-python3.12", spec.image());+ // Real Amazon MWAA runs 2.10.x on Python 3.11, not 3.12.+ assertEquals("apache/airflow:2.10.5-python3.11", spec.image());+ }++ @Test+ void pythonTagForMatchesRealMwaasAirflowPythonPairing() {+ // Every version through 2.10.x runs Python 3.11 on real Amazon MWAA.+ assertEquals("python3.11", MwaaEnvironmentManager.pythonTagFor("2.7.2"));+ assertEquals("python3.11", MwaaEnvironmentManager.pythonTagFor("2.8.4"));+ assertEquals("python3.11", MwaaEnvironmentManager.pythonTagFor("2.9.3"));+ assertEquals("python3.11", MwaaEnvironmentManager.pythonTagFor("2.10.5"));+ // 2.11.0 onward, and every 3.x release, runs Python 3.12.+ assertEquals("python3.12", MwaaEnvironmentManager.pythonTagFor("2.11.0"));+ assertEquals("python3.12", MwaaEnvironmentManager.pythonTagFor("2.11.2"));+ assertEquals("python3.12", MwaaEnvironmentManager.pythonTagFor("3.0.6"));}@Test