Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(mwaa): run CLI commands off the event loop (#4343)

Niels Kootstrac9c01729f28a
diff --git a/src/main/java/io/github/hectorvent/floci/services/mwaa/proxy/MwaaWebProxy.java b/src/main/java/io/github/hectorvent/floci/services/mwaa/proxy/MwaaWebProxy.java
index 0138ea4682..8173a0d3b3 100644
--- a/src/main/java/io/github/hectorvent/floci/services/mwaa/proxy/MwaaWebProxy.java
+++ b/src/main/java/io/github/hectorvent/floci/services/mwaa/proxy/MwaaWebProxy.java
@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.mwaa.proxy;
import io.github.hectorvent.floci.services.mwaa.MwaaEnvironmentManager;
import io.vertx.core.Vertx;
+import io.vertx.core.WorkerExecutor;
import io.vertx.core.http.HttpClient;
import io.vertx.core.http.HttpClientOptions;
import io.vertx.core.http.HttpMethod;
@@ -34,6 +35,10 @@ public class MwaaWebProxy {
private static final Logger LOG = Logger.getLogger(MwaaWebProxy.class);
private static final String CLI_PATH = "/aws_mwaa/cli";
+ // CLI execs get their own small pool, shared by name across environments, so a burst of
+ // slow Docker execs cannot occupy the Vert.x worker pool the rest of the emulator uses.
+ private static final String CLI_WORKER_POOL_NAME = "mwaa-cli";
+ private static final int CLI_WORKER_POOL_SIZE = 4;
private static final List<String> HOP_BY_HOP_HEADERS = List.of(
"connection", "keep-alive", "transfer-encoding", "upgrade", "te", "trailers",
@@ -45,6 +50,7 @@ public class MwaaWebProxy {
private final int backendPort;
private final CliTokenValidator tokenValidator;
private final CliExecutor cliExecutor;
+ private final WorkerExecutor cliWorkers;
private HttpServer server;
private HttpClient client;
@@ -57,6 +63,7 @@ public class MwaaWebProxy {
this.backendPort = backendPort;
this.tokenValidator = tokenValidator;
this.cliExecutor = cliExecutor;
+ this.cliWorkers = vertx.createSharedWorkerExecutor(CLI_WORKER_POOL_NAME, CLI_WORKER_POOL_SIZE);
}
/** Starts listening on {@code proxyPort}, blocking until bound (or throwing on failure). */
@@ -71,9 +78,12 @@ public class MwaaWebProxy {
try {
server.listen().toCompletionStage().toCompletableFuture().get(10, TimeUnit.SECONDS);
} catch (ExecutionException e) {
+ // The manager only tracks proxies that started, so release what this one holds here.
+ stop();
throw new IllegalStateException("Could not bind MWAA web proxy for environment "
+ environmentName + " on port " + proxyPort, e.getCause() != null ? e.getCause() : e);
} catch (TimeoutException e) {
+ stop();
throw new IllegalStateException("Timed out binding MWAA web proxy for environment "
+ environmentName + " on port " + proxyPort, e);
}
@@ -88,6 +98,7 @@ public class MwaaWebProxy {
if (client != null) {
client.close();
}
+ cliWorkers.close();
}
private void handleRequest(HttpServerRequest req) {
@@ -108,19 +119,22 @@ public class MwaaWebProxy {
}
req.bodyHandler(body -> {
String command = body.toString(StandardCharsets.UTF_8).trim();
- try {
- MwaaEnvironmentManager.ExecResult result = cliExecutor.execute(command);
- String stdoutB64 = Base64.getEncoder().encodeToString(result.stdout().getBytes(StandardCharsets.UTF_8));
- String stderrB64 = Base64.getEncoder().encodeToString(result.stderr().getBytes(StandardCharsets.UTF_8));
- req.response()
- .putHeader("Content-Type", "application/json")
- .end("{\"stdout\":\"" + stdoutB64 + "\",\"stderr\":\"" + stderrB64 + "\"}");
- } catch (Exception e) {
- LOG.warnv("MWAA CLI exec failed for environment {0}: {1}", environmentName, e.getMessage());
- req.response().setStatusCode(500)
- .putHeader("Content-Type", "application/json")
- .end("{\"message\":\"" + escapeJson(e.getMessage()) + "\"}");
- }
+ // The Docker exec can block for up to 30 seconds, so keep it off the event loop.
+ // Unordered: independent CLI calls should not queue behind each other.
+ cliWorkers.<MwaaEnvironmentManager.ExecResult>executeBlocking(() -> cliExecutor.execute(command), false)
+ .onSuccess(result -> {
+ String stdoutB64 = Base64.getEncoder().encodeToString(result.stdout().getBytes(StandardCharsets.UTF_8));
+ String stderrB64 = Base64.getEncoder().encodeToString(result.stderr().getBytes(StandardCharsets.UTF_8));
+ req.response()
+ .putHeader("Content-Type", "application/json")
+ .end("{\"stdout\":\"" + stdoutB64 + "\",\"stderr\":\"" + stderrB64 + "\"}");
+ })
+ .onFailure(e -> {
+ LOG.warnv("MWAA CLI exec failed for environment {0}: {1}", environmentName, e.getMessage());
+ req.response().setStatusCode(500)
+ .putHeader("Content-Type", "application/json")
+ .end("{\"message\":\"" + escapeJson(e.getMessage()) + "\"}");
+ });
});
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/mwaa/proxy/MwaaWebProxyCliOffloadTest.java b/src/test/java/io/github/hectorvent/floci/services/mwaa/proxy/MwaaWebProxyCliOffloadTest.java
new file mode 100644
index 0000000000..e92851b3f8
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/mwaa/proxy/MwaaWebProxyCliOffloadTest.java
@@ -0,0 +1,226 @@
+package io.github.hectorvent.floci.services.mwaa.proxy;
+
+import io.github.hectorvent.floci.services.mwaa.MwaaEnvironmentManager;
+import io.vertx.core.Vertx;
+import io.vertx.core.VertxOptions;
+import io.vertx.core.buffer.Buffer;
+import io.vertx.core.http.HttpClient;
+import io.vertx.core.http.HttpMethod;
+import io.vertx.core.http.HttpServer;
+import io.vertx.core.http.RequestOptions;
+import io.vertx.core.json.JsonObject;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.Test;
+
+import java.net.ServerSocket;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Base64;
+import java.util.List;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+/**
+ * Confirms {@code POST /aws_mwaa/cli} runs its Docker exec off the Vert.x event loop.
+ * {@code MwaaEnvironmentManager#execInContainer} can block for up to 30 seconds
+ * ({@code awaitCompletion(30, TimeUnit.SECONDS)}), so running it synchronously on the event
+ * loop that also serves the rest of this environment's proxied Airflow traffic would stall
+ * every other request for the duration of one CLI call.
+ *
+ * <p>Each test pins the proxy, its forwarded backend, and the CLI/forwarded requests onto a
+ * single-event-loop-thread {@link Vertx}, so a slow CLI call can only avoid starving unrelated
+ * traffic if it no longer runs on that thread.
+ */
+class MwaaWebProxyCliOffloadTest {
+
+ private Vertx vertx;
+
+ @AfterEach
+ void tearDown() throws Exception {
+ if (vertx != null) {
+ vertx.close().toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+ }
+ }
+
+ @Test
+ void slowCliExecutionDoesNotStallConcurrentProxyTraffic() throws Exception {
+ vertx = Vertx.vertx(new VertxOptions().setEventLoopPoolSize(1));
+
+ HttpServer backend = vertx.createHttpServer()
+ .requestHandler(req -> req.response().setStatusCode(200).end("backend-ok"))
+ .listen(0, "127.0.0.1")
+ .toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+
+ CountDownLatch cliStarted = new CountDownLatch(1);
+ CountDownLatch releaseCli = new CountDownLatch(1);
+ MwaaWebProxy.CliExecutor slowExecutor = command -> {
+ cliStarted.countDown();
+ if (!releaseCli.await(5, TimeUnit.SECONDS)) {
+ throw new IllegalStateException("test did not release the CLI exec in time");
+ }
+ return new MwaaEnvironmentManager.ExecResult(0, "dag1\ndag2\n", "");
+ };
+
+ MwaaWebProxy proxy = new MwaaWebProxy("env1", vertx, "127.0.0.1", backend.actualPort(),
+ (environmentName, token) -> true, slowExecutor);
+ int proxyPort;
+ try (ServerSocket freePort = new ServerSocket(0)) {
+ proxyPort = freePort.getLocalPort();
+ }
+ proxy.start(proxyPort);
+
+ try {
+ HttpClient client = vertx.createHttpClient();
+
+ CompletableFuture<ProxyResponse> cliResponseFuture = client.request(new RequestOptions()
+ .setHost("127.0.0.1")
+ .setPort(proxyPort)
+ .setMethod(HttpMethod.POST)
+ .setURI("/aws_mwaa/cli"))
+ .compose(request -> {
+ request.putHeader("Authorization", "Bearer test-token");
+ request.putHeader("Content-Type", "text/plain");
+ return request.send(Buffer.buffer("dags list"));
+ })
+ .compose(response -> response.body()
+ .map(body -> new ProxyResponse(response.statusCode(), body.toString())))
+ .toCompletionStage().toCompletableFuture();
+
+ assertTrue(cliStarted.await(2, TimeUnit.SECONDS), "CLI executor was never invoked");
+
+ // Fired while the CLI request above is still parked inside the fake Docker exec. On a
+ // single-event-loop-thread Vertx, this can only complete promptly if that thread is
+ // free to accept the connection, route it, and relay the backend's response -- i.e.
+ // only once the CLI call is no longer running on that same thread.
+ ProxyResponse forwarded = client.request(new RequestOptions()
+ .setHost("127.0.0.1")
+ .setPort(proxyPort)
+ .setMethod(HttpMethod.GET)
+ .setURI("/"))
+ .compose(request -> request.send())
+ .compose(response -> response.body()
+ .map(body -> new ProxyResponse(response.statusCode(), body.toString())))
+ .toCompletionStage().toCompletableFuture()
+ .get(3, TimeUnit.SECONDS);
+
+ assertEquals(200, forwarded.statusCode());
+ assertEquals("backend-ok", forwarded.body());
+
+ releaseCli.countDown();
+
+ ProxyResponse cliResponse = cliResponseFuture.get(3, TimeUnit.SECONDS);
+ assertEquals(200, cliResponse.statusCode());
+ JsonObject json = new JsonObject(cliResponse.body());
+ assertEquals("dag1\ndag2\n",
+ new String(Base64.getDecoder().decode(json.getString("stdout")), StandardCharsets.UTF_8));
+ assertEquals("",
+ new String(Base64.getDecoder().decode(json.getString("stderr")), StandardCharsets.UTF_8));
+ } finally {
+ releaseCli.countDown();
+ proxy.stop();
+ backend.close().toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+ }
+ }
+
+ @Test
+ void burstOfSlowCliCallsLeavesTheSharedWorkerPoolFree() throws Exception {
+ int sharedWorkers = 2;
+ vertx = Vertx.vertx(new VertxOptions().setEventLoopPoolSize(1).setWorkerPoolSize(sharedWorkers));
+
+ CountDownLatch cliStarted = new CountDownLatch(sharedWorkers);
+ CountDownLatch releaseCli = new CountDownLatch(1);
+ MwaaWebProxy.CliExecutor slowExecutor = command -> {
+ cliStarted.countDown();
+ if (!releaseCli.await(5, TimeUnit.SECONDS)) {
+ throw new IllegalStateException("test did not release the CLI exec in time");
+ }
+ return new MwaaEnvironmentManager.ExecResult(0, "", "");
+ };
+ MwaaWebProxy proxy = new MwaaWebProxy("env1", vertx, "127.0.0.1", 1,
+ (environmentName, token) -> true, slowExecutor);
+ int proxyPort;
+ try (ServerSocket freePort = new ServerSocket(0)) {
+ proxyPort = freePort.getLocalPort();
+ }
+ proxy.start(proxyPort);
+
+ try {
+ HttpClient client = vertx.createHttpClient();
+ List<CompletableFuture<Integer>> cliCalls = new ArrayList<>();
+ for (int i = 0; i < sharedWorkers * 2; i++) {
+ cliCalls.add(client.request(new RequestOptions()
+ .setHost("127.0.0.1")
+ .setPort(proxyPort)
+ .setMethod(HttpMethod.POST)
+ .setURI("/aws_mwaa/cli"))
+ .compose(request -> {
+ request.putHeader("Authorization", "Bearer test-token");
+ request.putHeader("Content-Type", "text/plain");
+ return request.send(Buffer.buffer("dags list"));
+ })
+ .map(response -> response.statusCode())
+ .toCompletionStage().toCompletableFuture());
+ }
+ assertTrue(cliStarted.await(2, TimeUnit.SECONDS), "CLI executor was never invoked");
+
+ // Unrelated blocking work, such as a Lambda invocation, must not queue behind CLI execs.
+ String unrelated = vertx.<String>executeBlocking(() -> "ran", false)
+ .toCompletionStage().toCompletableFuture()
+ .get(2, TimeUnit.SECONDS);
+ assertEquals("ran", unrelated);
+
+ releaseCli.countDown();
+ for (CompletableFuture<Integer> cliCall : cliCalls) {
+ assertEquals(200, cliCall.get(3, TimeUnit.SECONDS));
+ }
+ } finally {
+ releaseCli.countDown();
+ proxy.stop();
+ }
+ }
+
+ @Test
+ void cliExecutorFailureStillReturnsFiveHundredWithTheErrorMessage() throws Exception {
+ vertx = Vertx.vertx(new VertxOptions().setEventLoopPoolSize(1));
+
+ MwaaWebProxy.CliExecutor failingExecutor = command -> {
+ throw new IllegalStateException("exec timed out in container abc123");
+ };
+ MwaaWebProxy proxy = new MwaaWebProxy("env1", vertx, "127.0.0.1", 1,
+ (environmentName, token) -> true, failingExecutor);
+ int proxyPort;
+ try (ServerSocket freePort = new ServerSocket(0)) {
+ proxyPort = freePort.getLocalPort();
+ }
+ proxy.start(proxyPort);
+
+ try {
+ HttpClient client = vertx.createHttpClient();
+ ProxyResponse response = client.request(new RequestOptions()
+ .setHost("127.0.0.1")
+ .setPort(proxyPort)
+ .setMethod(HttpMethod.POST)
+ .setURI("/aws_mwaa/cli"))
+ .compose(request -> {
+ request.putHeader("Authorization", "Bearer test-token");
+ request.putHeader("Content-Type", "text/plain");
+ return request.send(Buffer.buffer("dags list"));
+ })
+ .compose(resp -> resp.body().map(body -> new ProxyResponse(resp.statusCode(), body.toString())))
+ .toCompletionStage().toCompletableFuture().get(2, TimeUnit.SECONDS);
+
+ assertEquals(500, response.statusCode());
+ JsonObject json = new JsonObject(response.body());
+ assertEquals("exec timed out in container abc123", json.getString("message"));
+ } finally {
+ proxy.stop();
+ }
+ }
+
+ private record ProxyResponse(int statusCode, String body) {
+ }
+}