Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(mwaa): match real MWAA's Airflow/Python pairing for the emulated image tag (#3445)

Priamo Germosen473a26aad92a
diff --git a/docs/services/mwaa.md b/docs/services/mwaa.md
index 5f31b5ea4e..8855a461b8 100644
--- a/docs/services/mwaa.md
+++ b/docs/services/mwaa.md
@@ -31,7 +31,7 @@ Environment metadata is stored in-process. No Docker containers are started. The
Floci starts two containers per environment:
- `floci-mwaa-<account>.<region>.<name>-db` — a private `postgres` metadata database, never given a published host port; it's only ever reached by the sibling Airflow container over the Docker network.
-- `floci-mwaa-<account>.<region>.<name>-airflow` — a real `apache/airflow` container running **LocalExecutor** (webserver + scheduler in one process tree). `AirflowVersion` genuinely selects the image tag (`apache/airflow:<version>-python3.12`), validated against `supported-versions` — unlike some Floci services where a requested version is echoed back but not actually applied, MWAA always runs the exact Airflow version requested.
+- `floci-mwaa-<account>.<region>.<name>-airflow` — a real `apache/airflow` container running **LocalExecutor** (webserver + scheduler in one process tree). `AirflowVersion` genuinely selects the image tag (`apache/airflow:<version>-python3.11` for versions through 2.10.x, `apache/airflow:<version>-python3.12` from 2.11.0 onward, matching the Airflow/Python pairing real Amazon MWAA runs for each version), validated against `supported-versions` — unlike some Floci services where a requested version is echoed back but not actually applied, MWAA always runs the exact Airflow version requested.
Legacy environments retain the region recorded in their ARN. A request in another region does not
adopt that environment; recreate it in the requested region instead.
diff --git a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
index 16e105057d..d169987b9d 100644
--- a/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
+++ b/src/main/java/io/github/hectorvent/floci/config/EmulatorConfig.java
@@ -2520,7 +2520,9 @@ public interface EmulatorConfig {
String defaultPostgresImage();
/** Airflow versions environments may request. Combined with the image tag
- * {@code apache/airflow:<version>-python3.12}. */
+ * {@code apache/airflow:<version>-<pythonTag>}, where the Python tag matches the
+ * version real Amazon MWAA runs for that Airflow version: see
+ * {@code MwaaEnvironmentManager.pythonTagFor}. */
@WithDefault("2.10.5,2.9.3,2.8.4")
List<String> supportedVersions();
diff --git a/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.java b/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.java
index 69536ae08f..bef24b32e9 100644
--- a/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.java
@@ -140,7 +140,7 @@ public class MwaaEnvironmentManager {
lifecycleManager.ensureVolume(dagsVolume);
lifecycleManager.ensureVolume(logsVolume);
- String image = "apache/airflow:%s-python3.12".formatted(airflowVersion);
+ String image = "apache/airflow:%s-%s".formatted(airflowVersion, pythonTagFor(airflowVersion));
String adminUser = "admin";
String adminPassword = generateSecret(24);
String sqlAlchemyConn = "postgresql+psycopg2://airflow:" + dbPassword + "@" + dbIp + ":" + POSTGRES_PORT + "/airflow";
@@ -365,6 +365,20 @@ public class MwaaEnvironmentManager {
return AwsArnUtils.regionOrDefault(environment.getArn(), "us-east-1");
}
+ /**
+ * The Python minor version tag real Amazon MWAA runs for a given {@code AirflowVersion}, so
+ * the emulated image matches the same Airflow/Python pairing a requirements.txt built the way
+ * AWS documents (a constraint file pinned to that pairing) expects. Per AWS's own Airflow
+ * versions table, every version through 2.10.x runs Python 3.11, and 2.11.0 onward (including
+ * every 3.x release) runs Python 3.12.
+ */
+ static String pythonTagFor(String airflowVersion) {
+ String[] parts = airflowVersion.split("\\.", 3);
+ int major = Integer.parseInt(parts[0]);
+ int minor = parts.length > 1 ? Integer.parseInt(parts[1]) : 0;
+ return (major > 2 || (major == 2 && minor >= 11)) ? "python3.12" : "python3.11";
+ }
+
/**
* Environment variables a startup script must not be able to permanently override — mirrors
* real MWAA's documented "reserved environment variables" (which AWS silently restores after a
diff --git a/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.java
index 9d5efe1b49..d067ef64ad 100644
--- a/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.java
@@ -131,7 +131,21 @@ class MwaaEnvironmentManagerTest {
@Test
void airflowImageTagSubstitutesTheRequestedVersion() {
ContainerSpec spec = startAirflowAndCaptureSpec(false);
- assertEquals("apache/airflow:2.10.5-python3.12", spec.image());
+ // Real Amazon MWAA runs 2.10.x on Python 3.11, not 3.12.
+ assertEquals("apache/airflow:2.10.5-python3.11", spec.image());
+ }
+
+ @Test
+ void pythonTagForMatchesRealMwaasAirflowPythonPairing() {
+ // Every version through 2.10.x runs Python 3.11 on real Amazon MWAA.
+ assertEquals("python3.11", MwaaEnvironmentManager.pythonTagFor("2.7.2"));
+ assertEquals("python3.11", MwaaEnvironmentManager.pythonTagFor("2.8.4"));
+ assertEquals("python3.11", MwaaEnvironmentManager.pythonTagFor("2.9.3"));
+ assertEquals("python3.11", MwaaEnvironmentManager.pythonTagFor("2.10.5"));
+ // 2.11.0 onward, and every 3.x release, runs Python 3.12.
+ assertEquals("python3.12", MwaaEnvironmentManager.pythonTagFor("2.11.0"));
+ assertEquals("python3.12", MwaaEnvironmentManager.pythonTagFor("2.11.2"));
+ assertEquals("python3.12", MwaaEnvironmentManager.pythonTagFor("3.0.6"));
}
@Test