Light, fluffy, and always free - The AWS Local Emulator alternative
fix: run MWAA CLI commands without a shell (#4750)
Niels Kootstraefbd990ef765
diff --git a/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.java b/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.javaindex f9979cf731..87acdef85a 100644--- a/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManager.java@@ -367,9 +367,89 @@ public class MwaaEnvironmentManager {}}- /** Runs {@code airflow <cliCommand>} inside the Airflow container via {@code sh -c}. */+ /**+ * Runs {@code airflow <cliCommand>} inside the Airflow container. The command is split into+ * arguments with POSIX quoting rules and executed without a shell, so shell syntax such as+ * {@code ;}, {@code |} or {@code $(...)} reaches Airflow as plain arguments.+ */public ContainerExec.Result runAirflowCli(String airflowContainerId, String cliCommand) throws Exception {- return execInContainer(airflowContainerId, new String[]{"sh", "-c", "airflow " + cliCommand});+ List<String> argv = new ArrayList<>();+ argv.add("airflow");+ argv.addAll(splitCliArguments(cliCommand));+ return execInContainer(airflowContainerId, argv.toArray(new String[0]));+ }++ /**+ * Splits a CLI command line into arguments using POSIX shell quoting without any expansion:+ * spaces, tabs and newlines separate arguments, single quotes are literal, double quotes honour+ * backslash escapes of {@code $}, backtick, {@code "}, backslash and newline, an unquoted+ * backslash escapes the next character, and a backslash-newline pair is removed.+ */+ private static List<String> splitCliArguments(String command) {+ List<String> args = new ArrayList<>();+ StringBuilder current = new StringBuilder();+ boolean inWord = false;+ int i = 0;+ while (i < command.length()) {+ char c = command.charAt(i);+ if (c == ' ' || c == '\t' || c == '\n') {+ if (inWord) {+ args.add(current.toString());+ current.setLength(0);+ inWord = false;+ }+ i++;+ } else if (c == '\'') {+ int end = command.indexOf('\'', i + 1);+ if (end < 0) {+ throw new IllegalArgumentException("Unterminated single quote in CLI command");+ }+ current.append(command, i + 1, end);+ inWord = true;+ i = end + 1;+ } else if (c == '"') {+ i++;+ boolean closed = false;+ while (i < command.length()) {+ char d = command.charAt(i);+ if (d == '"') {+ closed = true;+ i++;+ break;+ }+ if (d == '\\' && i + 1 < command.length() && "$`\"\\\n".indexOf(command.charAt(i + 1)) >= 0) {+ if (command.charAt(i + 1) != '\n') {+ current.append(command.charAt(i + 1));+ }+ i += 2;+ } else {+ current.append(d);+ i++;+ }+ }+ if (!closed) {+ throw new IllegalArgumentException("Unterminated double quote in CLI command");+ }+ inWord = true;+ } else if (c == '\\') {+ if (i + 1 >= command.length()) {+ throw new IllegalArgumentException("Trailing backslash in CLI command");+ }+ if (command.charAt(i + 1) != '\n') {+ current.append(command.charAt(i + 1));+ inWord = true;+ }+ i += 2;+ } else {+ current.append(c);+ inWord = true;+ i++;+ }+ }+ if (inWord) {+ args.add(current.toString());+ }+ return args;}/** Routed through {@link ContainerStorageHelper} so multiple Floci instances sharing one Dockerdiff --git a/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.javaindex e135cb78d6..60fd402463 100644--- a/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/mwaa/MwaaEnvironmentManagerTest.java@@ -528,7 +528,7 @@ class MwaaEnvironmentManagerTest {}@Test- void runAirflowCliRunsTheCommandThroughAShellInTheAirflowContainer() throws Exception {+ void runAirflowCliRunsTheAirflowCommandInTheAirflowContainer() throws Exception {DockerClient dockerClient = Mockito.mock(DockerClient.class);when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);List<List<String>> commands = ContainerExecStubs.completeEveryExec(dockerClient, "airflow-container-id", 2,@@ -536,9 +536,73 @@ class MwaaEnvironmentManagerTest {ContainerExec.Result result = manager.runAirflowCli("airflow-container-id", "dags list");- assertEquals(List.of(List.of("sh", "-c", "airflow dags list")), commands);+ assertEquals(List.of(List.of("airflow", "dags", "list")), commands);assertEquals(2, result.exitCode());assertEquals("dag1\n", result.stdout());assertEquals("warning\n", result.stderr());}++ @Nested+ class AirflowCli {++ private static final String AIRFLOW_ID = "airflow-container-id";+ private DockerClient dockerClient;++ @BeforeEach+ void stubDocker() {+ dockerClient = Mockito.mock(DockerClient.class);+ when(lifecycleManager.getDockerClient()).thenReturn(dockerClient);+ }++ @Test+ void shellSyntaxInACliCommandReachesAirflowAsPlainArguments() throws Exception {+ List<List<String>> commands = ContainerExecStubs.completeEveryExec(dockerClient, AIRFLOW_ID, 0, "", "");++ manager.runAirflowCli(AIRFLOW_ID, "dags list; touch /tmp/pwned $(id) `id` | cat > /tmp/out");++ assertEquals(List.of(List.of("airflow", "dags", "list;", "touch", "/tmp/pwned", "$(id)", "`id`",+ "|", "cat", ">", "/tmp/out")), commands);+ }++ @Test+ void quotedCliArgumentsReachAirflowWhole() throws Exception {+ List<List<String>> commands = ContainerExecStubs.completeEveryExec(dockerClient, AIRFLOW_ID, 0, "", "");++ manager.runAirflowCli(AIRFLOW_ID, "dags trigger -c '{\"key\": \"a b\"}' my_dag");+ manager.runAirflowCli(AIRFLOW_ID, "variables set greeting \"say \\\"hi\\\" to $USER\" ''");+ manager.runAirflowCli(AIRFLOW_ID, "variables get two\\ words");++ assertEquals(List.of(+ List.of("airflow", "dags", "trigger", "-c", "{\"key\": \"a b\"}", "my_dag"),+ List.of("airflow", "variables", "set", "greeting", "say \"hi\" to $USER", ""),+ List.of("airflow", "variables", "get", "two words")), commands);+ }++ @Test+ void backslashNewlineJoinsTheTextOnEitherSide() throws Exception {+ List<List<String>> commands = ContainerExecStubs.completeEveryExec(dockerClient, AIRFLOW_ID, 0, "", "");++ manager.runAirflowCli(AIRFLOW_ID, "variables set greeting hel\\\nlo \"wor\\\nld\"");++ assertEquals(List.of(List.of("airflow", "variables", "set", "greeting", "hello", "world")), commands);+ }++ @Test+ void onlySpacesTabsAndNewlinesSeparateArguments() throws Exception {+ List<List<String>> commands = ContainerExecStubs.completeEveryExec(dockerClient, AIRFLOW_ID, 0, "", "");++ manager.runAirflowCli(AIRFLOW_ID, "variables\tset\ngreeting hello\u2003world");++ assertEquals(List.of(List.of("airflow", "variables", "set", "greeting", "hello\u2003world")), commands);+ }++ @Test+ void unterminatedQuoteIsRejectedWithoutRunningAnything() {+ List<List<String>> commands = ContainerExecStubs.completeEveryExec(dockerClient, AIRFLOW_ID, 0, "", "");++ assertThrows(IllegalArgumentException.class,+ () -> manager.runAirflowCli(AIRFLOW_ID, "dags trigger -c '{\"key\": 1}"));+ assertEquals(List.of(), commands);+ }+ }}