Light, fluffy, and always free - The AWS Local Emulator alternative
fix(mwaa): apply AirflowConfigurationOptions to the Airflow container (#3457)
Priamo Germosend429d107ff84
diff --git a/docs/services/mwaa.md b/docs/services/mwaa.mdindex 8deecd4b6c..4acb72f6c0 100644--- a/docs/services/mwaa.md+++ b/docs/services/mwaa.md@@ -33,6 +33,8 @@ Floci starts two containers per environment:- `floci-mwaa-<account>.<region>.<name>-db` — a private `postgres` metadata database, never given a published host port; it's only ever reached by the sibling Airflow container over the Docker network.- `floci-mwaa-<account>.<region>.<name>-airflow` — a real `apache/airflow` container running **LocalExecutor** (webserver + scheduler in one process tree). `AirflowVersion` genuinely selects the image tag (`apache/airflow:<version>-python3.11` for versions through 2.10.x, `apache/airflow:<version>-python3.12` from 2.11.0 onward, matching the Airflow/Python pairing real Amazon MWAA runs for each version), validated against `supported-versions` — unlike some Floci services where a requested version is echoed back but not actually applied, MWAA always runs the exact Airflow version requested.+`AirflowConfigurationOptions` is applied too, not just echoed back: each `section.key` entry becomes an `AIRFLOW__SECTION__KEY` environment variable on the Airflow container, the same mechanism real Amazon MWAA uses, so a DAG's `conf.get("section", "key")` sees the configured value. A malformed key (missing the `section.key` dot) or one that would collide with a variable Floci itself manages (the executor, database connection, or security keys) is skipped and logged, rather than silently breaking the environment's required wiring. `CreateEnvironment` also enforces the same key/value shape botocore does for this field (a 1-64 character key matching `[a-z]+([a-z0-9._]*[a-z0-9_]+)?`, a 1-65536 character printable-ASCII value), since Floci is reachable via raw HTTP or CLI `--cli-input-json`, which bypasses the SDKs' own client-side validation. This only applies at creation time: `UpdateEnvironment` still rejects `AirflowConfigurationOptions` (see [Not Implemented](#not-implemented) below), so changing an option on an existing environment requires deleting and recreating it.+Legacy environments retain the region recorded in their ARN. A request in another region does notadopt that environment; recreate it in the requested region instead.diff --git a/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.java b/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.javaindex 719293e74c..ce6b8a5cce 100644--- a/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.java@@ -32,6 +32,7 @@ import java.security.SecureRandom;import java.util.ArrayList;import java.util.Base64;import java.util.List;+import java.util.Locale;import java.util.Map;import java.util.Optional;import java.util.concurrent.TimeUnit;@@ -159,6 +160,7 @@ public class MwaaEnvironmentManager {"AIRFLOW__CORE__LOAD_EXAMPLES=false","_AIRFLOW_WWW_USER_USERNAME=" + adminUser,"_AIRFLOW_WWW_USER_PASSWORD=" + adminPassword));+ env.addAll(airflowConfigurationOptionsEnv(environment.getAirflowConfigurationOptions()));ContainerBuilder.Builder specBuilder = containerBuilder.newContainer(image).withName(airflowContainerName)@@ -417,6 +419,41 @@ public class MwaaEnvironmentManager {return (major > 2 || (major == 2 && minor >= 11)) ? "python3.12" : "python3.11";}+ /**+ * Translates {@code AirflowConfigurationOptions} ({@code "section.key"} to value, e.g.+ * {@code core.dags_are_paused_at_creation}) into the {@code AIRFLOW__SECTION__KEY} environment+ * variables real Amazon MWAA sets for the same option, matching Airflow's own {@code conf.get}+ * section/key model. An entry without a non-empty section and key on both sides of the dot is+ * malformed per AWS's own format and is skipped rather than guessed at. An entry that would+ * collide with one of {@link #PROTECTED_ENV_VARS} is skipped too and logged: those are Floci's+ * own required LocalExecutor/DB/security wiring, not something a configuration option request+ * gets to override.+ */+ static List<String> airflowConfigurationOptionsEnv(Map<String, String> options) {+ if (options == null || options.isEmpty()) {+ return List.of();+ }+ List<String> env = new ArrayList<>();+ for (Map.Entry<String, String> entry : options.entrySet()) {+ String dotted = entry.getKey();+ int dot = dotted.indexOf('.');+ if (dot <= 0 || dot == dotted.length() - 1) {+ LOG.warnv("Ignoring malformed AirflowConfigurationOptions key ''{0}'', expected "+ + "the AWS \"section.key\" format", dotted);+ continue;+ }+ String key = "AIRFLOW__" + dotted.substring(0, dot).toUpperCase(Locale.ROOT)+ + "__" + dotted.substring(dot + 1).toUpperCase(Locale.ROOT);+ if (PROTECTED_ENV_VARS.contains(key)) {+ LOG.warnv("Ignoring AirflowConfigurationOptions entry ''{0}'': it maps to {1}, "+ + "which Floci itself manages for this environment", dotted, key);+ continue;+ }+ env.add(key + "=" + entry.getValue());+ }+ return env;+ }+/*** Environment variables a startup script must not be able to permanently override — mirrors* real MWAA's documented "reserved environment variables" (which AWS silently restores after adiff --git a/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaService.java b/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaService.javaindex 2c8e800c3c..0a110a6269 100644--- a/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaService.java+++ b/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaService.java@@ -124,6 +124,7 @@ public class MwaaService implements TagHandler {}String version = resolveAirflowVersion(request.getAirflowVersion());+ validateAirflowConfigurationOptions(request.getAirflowConfigurationOptions());String arn = AwsArnUtils.Arn.of("airflow", region, accountId, "environment/" + name).toString();@@ -357,6 +358,47 @@ public class MwaaService implements TagHandler {return version;}+ // CreateEnvironment's botocore model caps AirflowConfigurationOptions to these key/value+ // shapes, matched below.+ private static final Pattern AIRFLOW_CONFIGURATION_OPTION_KEY_PATTERN =+ Pattern.compile("[a-z]+([a-z0-9._]*[a-z0-9_]+)?");+ private static final Pattern AIRFLOW_CONFIGURATION_OPTION_VALUE_PATTERN = Pattern.compile("[ -~]+");+ private static final int MAX_AIRFLOW_CONFIGURATION_OPTION_KEY_LENGTH = 64;+ private static final int MAX_AIRFLOW_CONFIGURATION_OPTION_VALUE_LENGTH = 65536;++ /**+ * The AWS SDKs enforce {@code AirflowConfigurationOptions}' key/value shape client-side, but+ * Floci is also reachable via raw HTTP or CLI {@code --cli-input-json}, which bypasses that,+ * and this map now genuinely flows into the Airflow container's environment+ * ({@link MwaaEnvironmentManager#airflowConfigurationOptionsEnv}) rather than being inert, so+ * an oversized or malformed entry is worth rejecting here rather than passing through to+ * Docker container creation.+ */+ private void validateAirflowConfigurationOptions(Map<String, String> options) {+ if (options == null) {+ return;+ }+ for (Map.Entry<String, String> entry : options.entrySet()) {+ String key = entry.getKey();+ if (key == null || key.length() > MAX_AIRFLOW_CONFIGURATION_OPTION_KEY_LENGTH+ || !AIRFLOW_CONFIGURATION_OPTION_KEY_PATTERN.matcher(key).matches()) {+ throw new AwsException("ValidationException",+ "Invalid AirflowConfigurationOptions key '" + key + "': must be 1-"+ + MAX_AIRFLOW_CONFIGURATION_OPTION_KEY_LENGTH+ + " characters matching " + AIRFLOW_CONFIGURATION_OPTION_KEY_PATTERN.pattern(),+ 400);+ }+ String value = entry.getValue();+ if (value == null || value.length() > MAX_AIRFLOW_CONFIGURATION_OPTION_VALUE_LENGTH+ || !AIRFLOW_CONFIGURATION_OPTION_VALUE_PATTERN.matcher(value).matches()) {+ throw new AwsException("ValidationException",+ "Invalid AirflowConfigurationOptions value for key '" + key + "': must be 1-"+ + MAX_AIRFLOW_CONFIGURATION_OPTION_VALUE_LENGTH + " printable ASCII characters",+ 400);+ }+ }+ }+private String buildWebserverUrl(int proxyPort) {String scheme = config.tls().enabled() ? "https" : "http";String host = config.hostname().orElse("localhost");diff --git a/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.javaindex 0e4f009a5b..8f5cbed096 100644--- a/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.java@@ -149,6 +149,62 @@ class MwaaEnvironmentManagerTest {assertEquals("python3.12", MwaaEnvironmentManager.pythonTagFor("3.0.6"));}+ @Test+ void airflowConfigurationOptionsEnvTranslatesSectionDotKeyToAirflowDoubleUnderscoreFormat() {+ assertEquals(List.of("AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION=false"),+ MwaaEnvironmentManager.airflowConfigurationOptionsEnv(+ Map.of("core.dags_are_paused_at_creation", "false")));+ }++ @Test+ void airflowConfigurationOptionsEnvSkipsAKeyMissingTheSectionDotKeySeparator() {+ assertEquals(List.of(), MwaaEnvironmentManager.airflowConfigurationOptionsEnv(+ Map.of("not_dotted", "value")));+ }++ @Test+ void airflowConfigurationOptionsEnvSkipsAnEmptySectionOrKey() {+ assertEquals(List.of(), MwaaEnvironmentManager.airflowConfigurationOptionsEnv(+ Map.of(".key", "value", "section.", "value")));+ }++ @Test+ void airflowConfigurationOptionsEnvSkipsAnEntryThatWouldOverrideFlociOwnedWiring() {+ // core.executor -> AIRFLOW__CORE__EXECUTOR, one of Floci's own mandatory (LocalExecutor)+ // vars. A configuration option request doesn't get to override that.+ assertEquals(List.of(), MwaaEnvironmentManager.airflowConfigurationOptionsEnv(+ Map.of("core.executor", "CeleryExecutor")));+ }++ @Test+ void airflowConfigurationOptionsEnvHandlesANullOrEmptyMap() {+ assertEquals(List.of(), MwaaEnvironmentManager.airflowConfigurationOptionsEnv(null));+ assertEquals(List.of(), MwaaEnvironmentManager.airflowConfigurationOptionsEnv(Map.of()));+ }++ @Test+ void startAirflowContainerAppliesConfiguredAirflowConfigurationOptions() {+ when(containerDetector.isRunningInContainer()).thenReturn(false);+ when(lifecycleManager.create(any())).thenReturn("airflow-container-id");+ when(lifecycleManager.startCreated(eq("airflow-container-id"), any())).thenReturn(+ new ContainerInfo("airflow-container-id", Map.of(8080, new EndpointInfo("172.18.0.5", 8080))));++ Environment environment = new Environment();+ environment.setName("my-env");+ environment.setAirflowConfigurationOptions(Map.of(+ "core.dags_are_paused_at_creation", "false",+ "core.executor", "CeleryExecutor"));++ manager.startAirflowContainer(environment, "2.10.5", "172.18.0.9", "db-secret-pw", null);++ ArgumentCaptor<ContainerSpec> captor = ArgumentCaptor.forClass(ContainerSpec.class);+ verify(lifecycleManager).create(captor.capture());+ assertTrue(captor.getValue().env().contains("AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION=false"));+ // The colliding entry must not have replaced Floci's own required executor setting.+ assertTrue(captor.getValue().env().contains("AIRFLOW__CORE__EXECUTOR=LocalExecutor"));+ assertFalse(captor.getValue().env().contains("AIRFLOW__CORE__EXECUTOR=CeleryExecutor"));+ }+@Testvoid airflowEnvVarsWireTheResolvedPostgresDsn() {ContainerSpec spec = startAirflowAndCaptureSpec(false);diff --git a/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaServiceTest.javaindex fb4e202253..8b31e6a93a 100644--- a/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaServiceTest.java@@ -336,6 +336,60 @@ class MwaaServiceTest {assertEquals("2.9.3", environment.getAirflowVersion());}+ @Test+ void createEnvironmentWithValidAirflowConfigurationOptionsRoundTrips() {+ CreateEnvironmentRequest request = createRequest("arn:aws:s3:::my-bucket", "dags");+ request.setAirflowConfigurationOptions(Map.of("core.dags_are_paused_at_creation", "false"));++ Environment environment = mwaaService.createEnvironment("config-options-env", request);+ assertEquals(Map.of("core.dags_are_paused_at_creation", "false"),+ environment.getAirflowConfigurationOptions());+ }++ @Test+ void createEnvironmentRejectsAnOverlongAirflowConfigurationOptionKey() {+ CreateEnvironmentRequest request = createRequest("arn:aws:s3:::my-bucket", "dags");+ request.setAirflowConfigurationOptions(Map.of("core." + "x".repeat(64), "value"));++ AwsException ex = assertThrows(AwsException.class,+ () -> mwaaService.createEnvironment("bad-key-length-env", request));+ assertEquals(400, ex.getHttpStatus());+ assertEquals("ValidationException", ex.getErrorCode());+ }++ @Test+ void createEnvironmentRejectsAnAirflowConfigurationOptionKeyWithInvalidCharacters() {+ CreateEnvironmentRequest request = createRequest("arn:aws:s3:::my-bucket", "dags");+ request.setAirflowConfigurationOptions(Map.of("Core.DagsArePaused", "false"));++ AwsException ex = assertThrows(AwsException.class,+ () -> mwaaService.createEnvironment("bad-key-chars-env", request));+ assertEquals(400, ex.getHttpStatus());+ assertEquals("ValidationException", ex.getErrorCode());+ }++ @Test+ void createEnvironmentRejectsAnOverlongAirflowConfigurationOptionValue() {+ CreateEnvironmentRequest request = createRequest("arn:aws:s3:::my-bucket", "dags");+ request.setAirflowConfigurationOptions(Map.of("core.some_key", "x".repeat(65537)));++ AwsException ex = assertThrows(AwsException.class,+ () -> mwaaService.createEnvironment("bad-value-length-env", request));+ assertEquals(400, ex.getHttpStatus());+ assertEquals("ValidationException", ex.getErrorCode());+ }++ @Test+ void createEnvironmentRejectsANonPrintableAirflowConfigurationOptionValue() {+ CreateEnvironmentRequest request = createRequest("arn:aws:s3:::my-bucket", "dags");+ request.setAirflowConfigurationOptions(Map.of("core.some_key", "line one\nline two"));++ AwsException ex = assertThrows(AwsException.class,+ () -> mwaaService.createEnvironment("bad-value-chars-env", request));+ assertEquals(400, ex.getHttpStatus());+ assertEquals("ValidationException", ex.getErrorCode());+ }+@Testvoid getEnvironment() {mwaaService.createEnvironment("my-env", createRequest("arn:aws:s3:::my-bucket", "dags"));