Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(mwaa): apply AirflowConfigurationOptions to the Airflow container (#3457)

Priamo Germosend429d107ff84
diff --git a/docs/services/mwaa.md b/docs/services/mwaa.md
index 8deecd4b6c..4acb72f6c0 100644
--- a/docs/services/mwaa.md
+++ b/docs/services/mwaa.md
@@ -33,6 +33,8 @@ Floci starts two containers per environment:
- `floci-mwaa-<account>.<region>.<name>-db` — a private `postgres` metadata database, never given a published host port; it's only ever reached by the sibling Airflow container over the Docker network.
- `floci-mwaa-<account>.<region>.<name>-airflow` — a real `apache/airflow` container running **LocalExecutor** (webserver + scheduler in one process tree). `AirflowVersion` genuinely selects the image tag (`apache/airflow:<version>-python3.11` for versions through 2.10.x, `apache/airflow:<version>-python3.12` from 2.11.0 onward, matching the Airflow/Python pairing real Amazon MWAA runs for each version), validated against `supported-versions` — unlike some Floci services where a requested version is echoed back but not actually applied, MWAA always runs the exact Airflow version requested.
+`AirflowConfigurationOptions` is applied too, not just echoed back: each `section.key` entry becomes an `AIRFLOW__SECTION__KEY` environment variable on the Airflow container, the same mechanism real Amazon MWAA uses, so a DAG's `conf.get("section", "key")` sees the configured value. A malformed key (missing the `section.key` dot) or one that would collide with a variable Floci itself manages (the executor, database connection, or security keys) is skipped and logged, rather than silently breaking the environment's required wiring. `CreateEnvironment` also enforces the same key/value shape botocore does for this field (a 1-64 character key matching `[a-z]+([a-z0-9._]*[a-z0-9_]+)?`, a 1-65536 character printable-ASCII value), since Floci is reachable via raw HTTP or CLI `--cli-input-json`, which bypasses the SDKs' own client-side validation. This only applies at creation time: `UpdateEnvironment` still rejects `AirflowConfigurationOptions` (see [Not Implemented](#not-implemented) below), so changing an option on an existing environment requires deleting and recreating it.
+
Legacy environments retain the region recorded in their ARN. A request in another region does not
adopt that environment; recreate it in the requested region instead.
diff --git a/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.java b/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.java
index 719293e74c..ce6b8a5cce 100644
--- a/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.java
@@ -32,6 +32,7 @@ import java.security.SecureRandom;
import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.TimeUnit;
@@ -159,6 +160,7 @@ public class MwaaEnvironmentManager {
"AIRFLOW__CORE__LOAD_EXAMPLES=false",
"_AIRFLOW_WWW_USER_USERNAME=" + adminUser,
"_AIRFLOW_WWW_USER_PASSWORD=" + adminPassword));
+ env.addAll(airflowConfigurationOptionsEnv(environment.getAirflowConfigurationOptions()));
ContainerBuilder.Builder specBuilder = containerBuilder.newContainer(image)
.withName(airflowContainerName)
@@ -417,6 +419,41 @@ public class MwaaEnvironmentManager {
return (major > 2 || (major == 2 && minor >= 11)) ? "python3.12" : "python3.11";
}
+ /**
+ * Translates {@code AirflowConfigurationOptions} ({@code "section.key"} to value, e.g.
+ * {@code core.dags_are_paused_at_creation}) into the {@code AIRFLOW__SECTION__KEY} environment
+ * variables real Amazon MWAA sets for the same option, matching Airflow's own {@code conf.get}
+ * section/key model. An entry without a non-empty section and key on both sides of the dot is
+ * malformed per AWS's own format and is skipped rather than guessed at. An entry that would
+ * collide with one of {@link #PROTECTED_ENV_VARS} is skipped too and logged: those are Floci's
+ * own required LocalExecutor/DB/security wiring, not something a configuration option request
+ * gets to override.
+ */
+ static List<String> airflowConfigurationOptionsEnv(Map<String, String> options) {
+ if (options == null || options.isEmpty()) {
+ return List.of();
+ }
+ List<String> env = new ArrayList<>();
+ for (Map.Entry<String, String> entry : options.entrySet()) {
+ String dotted = entry.getKey();
+ int dot = dotted.indexOf('.');
+ if (dot <= 0 || dot == dotted.length() - 1) {
+ LOG.warnv("Ignoring malformed AirflowConfigurationOptions key ''{0}'', expected "
+ + "the AWS \"section.key\" format", dotted);
+ continue;
+ }
+ String key = "AIRFLOW__" + dotted.substring(0, dot).toUpperCase(Locale.ROOT)
+ + "__" + dotted.substring(dot + 1).toUpperCase(Locale.ROOT);
+ if (PROTECTED_ENV_VARS.contains(key)) {
+ LOG.warnv("Ignoring AirflowConfigurationOptions entry ''{0}'': it maps to {1}, "
+ + "which Floci itself manages for this environment", dotted, key);
+ continue;
+ }
+ env.add(key + "=" + entry.getValue());
+ }
+ return env;
+ }
+
/**
* Environment variables a startup script must not be able to permanently override — mirrors
* real MWAA's documented "reserved environment variables" (which AWS silently restores after a
diff --git a/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaService.java b/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaService.java
index 2c8e800c3c..0a110a6269 100644
--- a/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaService.java
+++ b/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaService.java
@@ -124,6 +124,7 @@ public class MwaaService implements TagHandler {
}
String version = resolveAirflowVersion(request.getAirflowVersion());
+ validateAirflowConfigurationOptions(request.getAirflowConfigurationOptions());
String arn = AwsArnUtils.Arn.of("airflow", region, accountId, "environment/" + name).toString();
@@ -357,6 +358,47 @@ public class MwaaService implements TagHandler {
return version;
}
+ // CreateEnvironment's botocore model caps AirflowConfigurationOptions to these key/value
+ // shapes, matched below.
+ private static final Pattern AIRFLOW_CONFIGURATION_OPTION_KEY_PATTERN =
+ Pattern.compile("[a-z]+([a-z0-9._]*[a-z0-9_]+)?");
+ private static final Pattern AIRFLOW_CONFIGURATION_OPTION_VALUE_PATTERN = Pattern.compile("[ -~]+");
+ private static final int MAX_AIRFLOW_CONFIGURATION_OPTION_KEY_LENGTH = 64;
+ private static final int MAX_AIRFLOW_CONFIGURATION_OPTION_VALUE_LENGTH = 65536;
+
+ /**
+ * The AWS SDKs enforce {@code AirflowConfigurationOptions}' key/value shape client-side, but
+ * Floci is also reachable via raw HTTP or CLI {@code --cli-input-json}, which bypasses that,
+ * and this map now genuinely flows into the Airflow container's environment
+ * ({@link MwaaEnvironmentManager#airflowConfigurationOptionsEnv}) rather than being inert, so
+ * an oversized or malformed entry is worth rejecting here rather than passing through to
+ * Docker container creation.
+ */
+ private void validateAirflowConfigurationOptions(Map<String, String> options) {
+ if (options == null) {
+ return;
+ }
+ for (Map.Entry<String, String> entry : options.entrySet()) {
+ String key = entry.getKey();
+ if (key == null || key.length() > MAX_AIRFLOW_CONFIGURATION_OPTION_KEY_LENGTH
+ || !AIRFLOW_CONFIGURATION_OPTION_KEY_PATTERN.matcher(key).matches()) {
+ throw new AwsException("ValidationException",
+ "Invalid AirflowConfigurationOptions key '" + key + "': must be 1-"
+ + MAX_AIRFLOW_CONFIGURATION_OPTION_KEY_LENGTH
+ + " characters matching " + AIRFLOW_CONFIGURATION_OPTION_KEY_PATTERN.pattern(),
+ 400);
+ }
+ String value = entry.getValue();
+ if (value == null || value.length() > MAX_AIRFLOW_CONFIGURATION_OPTION_VALUE_LENGTH
+ || !AIRFLOW_CONFIGURATION_OPTION_VALUE_PATTERN.matcher(value).matches()) {
+ throw new AwsException("ValidationException",
+ "Invalid AirflowConfigurationOptions value for key '" + key + "': must be 1-"
+ + MAX_AIRFLOW_CONFIGURATION_OPTION_VALUE_LENGTH + " printable ASCII characters",
+ 400);
+ }
+ }
+ }
+
private String buildWebserverUrl(int proxyPort) {
String scheme = config.tls().enabled() ? "https" : "http";
String host = config.hostname().orElse("localhost");
diff --git a/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.java
index 0e4f009a5b..8f5cbed096 100644
--- a/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.java
@@ -149,6 +149,62 @@ class MwaaEnvironmentManagerTest {
assertEquals("python3.12", MwaaEnvironmentManager.pythonTagFor("3.0.6"));
}
+ @Test
+ void airflowConfigurationOptionsEnvTranslatesSectionDotKeyToAirflowDoubleUnderscoreFormat() {
+ assertEquals(List.of("AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION=false"),
+ MwaaEnvironmentManager.airflowConfigurationOptionsEnv(
+ Map.of("core.dags_are_paused_at_creation", "false")));
+ }
+
+ @Test
+ void airflowConfigurationOptionsEnvSkipsAKeyMissingTheSectionDotKeySeparator() {
+ assertEquals(List.of(), MwaaEnvironmentManager.airflowConfigurationOptionsEnv(
+ Map.of("not_dotted", "value")));
+ }
+
+ @Test
+ void airflowConfigurationOptionsEnvSkipsAnEmptySectionOrKey() {
+ assertEquals(List.of(), MwaaEnvironmentManager.airflowConfigurationOptionsEnv(
+ Map.of(".key", "value", "section.", "value")));
+ }
+
+ @Test
+ void airflowConfigurationOptionsEnvSkipsAnEntryThatWouldOverrideFlociOwnedWiring() {
+ // core.executor -> AIRFLOW__CORE__EXECUTOR, one of Floci's own mandatory (LocalExecutor)
+ // vars. A configuration option request doesn't get to override that.
+ assertEquals(List.of(), MwaaEnvironmentManager.airflowConfigurationOptionsEnv(
+ Map.of("core.executor", "CeleryExecutor")));
+ }
+
+ @Test
+ void airflowConfigurationOptionsEnvHandlesANullOrEmptyMap() {
+ assertEquals(List.of(), MwaaEnvironmentManager.airflowConfigurationOptionsEnv(null));
+ assertEquals(List.of(), MwaaEnvironmentManager.airflowConfigurationOptionsEnv(Map.of()));
+ }
+
+ @Test
+ void startAirflowContainerAppliesConfiguredAirflowConfigurationOptions() {
+ when(containerDetector.isRunningInContainer()).thenReturn(false);
+ when(lifecycleManager.create(any())).thenReturn("airflow-container-id");
+ when(lifecycleManager.startCreated(eq("airflow-container-id"), any())).thenReturn(
+ new ContainerInfo("airflow-container-id", Map.of(8080, new EndpointInfo("172.18.0.5", 8080))));
+
+ Environment environment = new Environment();
+ environment.setName("my-env");
+ environment.setAirflowConfigurationOptions(Map.of(
+ "core.dags_are_paused_at_creation", "false",
+ "core.executor", "CeleryExecutor"));
+
+ manager.startAirflowContainer(environment, "2.10.5", "172.18.0.9", "db-secret-pw", null);
+
+ ArgumentCaptor<ContainerSpec> captor = ArgumentCaptor.forClass(ContainerSpec.class);
+ verify(lifecycleManager).create(captor.capture());
+ assertTrue(captor.getValue().env().contains("AIRFLOW__CORE__DAGS_ARE_PAUSED_AT_CREATION=false"));
+ // The colliding entry must not have replaced Floci's own required executor setting.
+ assertTrue(captor.getValue().env().contains("AIRFLOW__CORE__EXECUTOR=LocalExecutor"));
+ assertFalse(captor.getValue().env().contains("AIRFLOW__CORE__EXECUTOR=CeleryExecutor"));
+ }
+
@Test
void airflowEnvVarsWireTheResolvedPostgresDsn() {
ContainerSpec spec = startAirflowAndCaptureSpec(false);
diff --git a/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaServiceTest.java
index fb4e202253..8b31e6a93a 100644
--- a/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaServiceTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaServiceTest.java
@@ -336,6 +336,60 @@ class MwaaServiceTest {
assertEquals("2.9.3", environment.getAirflowVersion());
}
+ @Test
+ void createEnvironmentWithValidAirflowConfigurationOptionsRoundTrips() {
+ CreateEnvironmentRequest request = createRequest("arn:aws:s3:::my-bucket", "dags");
+ request.setAirflowConfigurationOptions(Map.of("core.dags_are_paused_at_creation", "false"));
+
+ Environment environment = mwaaService.createEnvironment("config-options-env", request);
+ assertEquals(Map.of("core.dags_are_paused_at_creation", "false"),
+ environment.getAirflowConfigurationOptions());
+ }
+
+ @Test
+ void createEnvironmentRejectsAnOverlongAirflowConfigurationOptionKey() {
+ CreateEnvironmentRequest request = createRequest("arn:aws:s3:::my-bucket", "dags");
+ request.setAirflowConfigurationOptions(Map.of("core." + "x".repeat(64), "value"));
+
+ AwsException ex = assertThrows(AwsException.class,
+ () -> mwaaService.createEnvironment("bad-key-length-env", request));
+ assertEquals(400, ex.getHttpStatus());
+ assertEquals("ValidationException", ex.getErrorCode());
+ }
+
+ @Test
+ void createEnvironmentRejectsAnAirflowConfigurationOptionKeyWithInvalidCharacters() {
+ CreateEnvironmentRequest request = createRequest("arn:aws:s3:::my-bucket", "dags");
+ request.setAirflowConfigurationOptions(Map.of("Core.DagsArePaused", "false"));
+
+ AwsException ex = assertThrows(AwsException.class,
+ () -> mwaaService.createEnvironment("bad-key-chars-env", request));
+ assertEquals(400, ex.getHttpStatus());
+ assertEquals("ValidationException", ex.getErrorCode());
+ }
+
+ @Test
+ void createEnvironmentRejectsAnOverlongAirflowConfigurationOptionValue() {
+ CreateEnvironmentRequest request = createRequest("arn:aws:s3:::my-bucket", "dags");
+ request.setAirflowConfigurationOptions(Map.of("core.some_key", "x".repeat(65537)));
+
+ AwsException ex = assertThrows(AwsException.class,
+ () -> mwaaService.createEnvironment("bad-value-length-env", request));
+ assertEquals(400, ex.getHttpStatus());
+ assertEquals("ValidationException", ex.getErrorCode());
+ }
+
+ @Test
+ void createEnvironmentRejectsANonPrintableAirflowConfigurationOptionValue() {
+ CreateEnvironmentRequest request = createRequest("arn:aws:s3:::my-bucket", "dags");
+ request.setAirflowConfigurationOptions(Map.of("core.some_key", "line one\nline two"));
+
+ AwsException ex = assertThrows(AwsException.class,
+ () -> mwaaService.createEnvironment("bad-value-chars-env", request));
+ assertEquals(400, ex.getHttpStatus());
+ assertEquals("ValidationException", ex.getErrorCode());
+ }
+
@Test
void getEnvironment() {
mwaaService.createEnvironment("my-env", createRequest("arn:aws:s3:::my-bucket", "dags"));