Light, fluffy, and always free - The AWS Local Emulator alternative
fix(ec2): bound launches and close userdata streams (#3135)
Niels Kootstradd33308e7eca
diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.javaindex 273ba447f1..3bd93296e7 100644--- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.java@@ -30,9 +30,11 @@ import org.jboss.logging.Logger;import java.io.ByteArrayInputStream;import java.io.ByteArrayOutputStream;+import java.io.Closeable;import java.io.IOException;import java.io.OutputStream;import java.nio.charset.StandardCharsets;+import java.time.Duration;import java.util.ArrayList;import java.util.Base64;import java.util.List;@@ -41,12 +43,17 @@ import java.util.Map;import java.util.Objects;import java.util.Optional;import java.util.Set;+import java.util.concurrent.ArrayBlockingQueue;import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ConcurrentHashMap;import java.util.concurrent.ExecutorService;-import java.util.concurrent.Executors;+import java.util.concurrent.RejectedExecutionException;+import java.util.concurrent.RejectedExecutionHandler;import java.util.concurrent.Semaphore;+import java.util.concurrent.ThreadPoolExecutor;import java.util.concurrent.TimeUnit;import java.util.function.BiPredicate;+import java.util.concurrent.atomic.AtomicBoolean;import java.util.regex.Matcher;import java.util.regex.Pattern;import java.util.zip.GZIPInputStream;@@ -76,7 +83,7 @@ public class Ec2ContainerManager {private static final int MAX_DECOMPRESSED_USER_DATA_BYTES = 10 * 1024 * 1024;private static final int MAX_EXEC_OUTPUT_BYTES = 2048;/** Caps concurrent UserData gzip decompressions across ALL instance launches, not just one.- * Launches run independently on {@link #executor}, an unbounded cached thread pool, so the+ * Launches run independently on {@link #executor}, so the* per-payload cap above only bounds a single launch's allocation: without this, N concurrent* RunInstances/CreateLaunchConfiguration calls, each smuggling a near-cap gzip payload, could* together decompress N * 10 MB at once in the shared emulator JVM with no aggregate ceiling.@@ -89,6 +96,26 @@ public class Ec2ContainerManager {/** Gates entry to {@link #gunzip}; see {@link #MAX_CONCURRENT_USER_DATA_DECOMPRESSIONS}. */private static final Semaphore USER_DATA_DECOMPRESSION_BUDGET =new Semaphore(MAX_CONCURRENT_USER_DATA_DECOMPRESSIONS);+ private static final int LAUNCH_CORE_THREADS = 4;+ private static final int LAUNCH_MAX_THREADS = 8;+ private static final int LAUNCH_QUEUE_CAPACITY = 64;+ private static final long USER_DATA_EXECUTION_TIMEOUT_MINUTES = 30;+ // Wait for capacity so accepted launches are not dropped, but never run launch work on callers.+ static final RejectedExecutionHandler BLOCKING_BACKPRESSURE = (runnable, executor) -> {+ if (executor.isShutdown()) {+ throw new RejectedExecutionException("EC2 container manager is stopped");+ }+ try {+ executor.getQueue().put(runnable);+ if (executor.isShutdown() && executor.getQueue().remove(runnable)) {+ throw new RejectedExecutionException("EC2 container manager is stopped");+ }+ }+ catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new RejectedExecutionException("Interrupted while waiting for EC2 launch capacity", e);+ }+ };/** Test seam: when non-null, invoked by {@link #gunzip} right after it acquires a* decompression-budget permit and before it starts decompressing, so tests can observe and* serialize concurrent decompressions deterministically. Always null in production. */@@ -138,15 +165,12 @@ public class Ec2ContainerManager {private final Ec2PortForwardManager portForwardManager;private final RegionResolver regionResolver;private final ContainerNetworkReachability containerNetworkReachability;+ private final ExecutorService executor;+ private final Duration userDataExecutionTimeout;+ private final Set<ResultCallback<Frame>> activeUserDataCallbacks = ConcurrentHashMap.newKeySet();private volatile boolean dockerUnavailableLogged;- private final ExecutorService executor = Executors.newCachedThreadPool(r -> {- Thread t = new Thread(r, "ec2-container-launcher");- t.setDaemon(true);- return t;- });-@Injectpublic Ec2ContainerManager(ContainerBuilder containerBuilder,ContainerLifecycleManager lifecycleManager,@@ -160,6 +184,26 @@ public class Ec2ContainerManager {Ec2PortForwardManager portForwardManager,RegionResolver regionResolver,ContainerNetworkReachability containerNetworkReachability) {+ this(containerBuilder, lifecycleManager, logStreamer, containerDetector, dockerHostResolver, dockerClient,+ portAllocator, config, metadataServer, portForwardManager, regionResolver,+ containerNetworkReachability, createLaunchExecutor(),+ Duration.ofMinutes(USER_DATA_EXECUTION_TIMEOUT_MINUTES));+ }++ Ec2ContainerManager(ContainerBuilder containerBuilder,+ ContainerLifecycleManager lifecycleManager,+ ContainerLogStreamer logStreamer,+ ContainerDetector containerDetector,+ DockerHostResolver dockerHostResolver,+ DockerClient dockerClient,+ PortAllocator portAllocator,+ EmulatorConfig config,+ Ec2MetadataServer metadataServer,+ Ec2PortForwardManager portForwardManager,+ RegionResolver regionResolver,+ ContainerNetworkReachability containerNetworkReachability,+ ExecutorService executor,+ Duration userDataExecutionTimeout) {this.containerBuilder = containerBuilder;this.lifecycleManager = lifecycleManager;this.logStreamer = logStreamer;@@ -172,11 +216,30 @@ public class Ec2ContainerManager {this.metadataServer = metadataServer;this.portForwardManager = portForwardManager;this.containerNetworkReachability = containerNetworkReachability;+ this.executor = executor;+ this.userDataExecutionTimeout = userDataExecutionTimeout;+ }++ private static ExecutorService createLaunchExecutor() {+ return new ThreadPoolExecutor(+ LAUNCH_CORE_THREADS,+ LAUNCH_MAX_THREADS,+ 60L,+ TimeUnit.SECONDS,+ new ArrayBlockingQueue<>(LAUNCH_QUEUE_CAPACITY),+ runnable -> {+ Thread thread = new Thread(runnable, "ec2-container-launcher");+ thread.setDaemon(true);+ return thread;+ },+ BLOCKING_BACKPRESSURE);}@PreDestroyvoid stop() {+ closeActiveUserDataCallbacks();executor.shutdownNow();+ closeActiveUserDataCallbacks();}/**@@ -215,9 +278,10 @@ public class Ec2ContainerManager {return;}- executor.submit(() -> {- try {- String instanceId = instance.getInstanceId();+ try {+ executor.execute(() -> {+ try {+ String instanceId = instance.getInstanceId();// IMDS endpoint that this container should useString flociHost = dockerHostResolver.resolve();int imdsPort = config.services().ec2().imdsPort();@@ -315,21 +379,26 @@ public class Ec2ContainerManager {executeUserData(containerId, instanceId, userData, region);}- } catch (InterruptedException e) {- Thread.currentThread().interrupt();- failLaunch(instance);- } catch (Exception e) {- LOG.warnv("Failed to launch EC2 instance {0}: {1}", instance.getInstanceId(), e.getMessage());- // The daemon can disappear between the probe above and any of the calls in- // this block. Losing Docker is not the instance's fault, so degrade to a- // metadata-only instance; a genuine container failure still fails the launch.- if (isDockerAvailable()) {+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();failLaunch(instance);- } else {- markContainerlessRunning(instance);+ } catch (Exception e) {+ LOG.warnv("Failed to launch EC2 instance {0}: {1}", instance.getInstanceId(), e.getMessage());+ // The daemon can disappear between the probe above and any of the calls in+ // this block. Losing Docker is not the instance's fault, so degrade to a+ // metadata-only instance; a genuine container failure still fails the launch.+ if (isDockerAvailable()) {+ failLaunch(instance);+ } else {+ markContainerlessRunning(instance);+ }}- }- });+ });+ } catch (RejectedExecutionException e) {+ LOG.warnv("Could not schedule EC2 instance {0} launch because the launch executor is saturated or stopping",+ instance.getInstanceId());+ failLaunch(instance);+ }}private StartedContainer createAndStartContainer(Instance instance, ResolvedAmiImage image, String region,@@ -976,6 +1045,9 @@ public class Ec2ContainerManager {logGroup, logStream, region);}+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ LOG.warnv("UserData execution interrupted for EC2 instance {0}", instanceId);} catch (Exception e) {LOG.warnv("UserData execution failed for EC2 instance {0}: {1}", instanceId, e.getMessage());}@@ -1003,11 +1075,29 @@ public class Ec2ContainerManager {BoundedOutput output = new BoundedOutput(MAX_EXEC_OUTPUT_BYTES);CountDownLatch latch = new CountDownLatch(1);- dockerClient.execStartCmd(execId).exec(new ResultCallback.Adapter<Frame>() {+ AtomicBoolean cancelled = new AtomicBoolean();+ ResultCallback.Adapter<Frame> callback = new ResultCallback.Adapter<>() {+ @Override+ public void onStart(Closeable stream) {+ if (cancelled.get()) {+ closeUserDataStream(stream, instanceId);+ return;+ }+ super.onStart(stream);+ if (cancelled.get()) {+ closeUserDataStream(stream, instanceId);+ }+ }+@Overridepublic void onNext(Frame frame) {+ if (cancelled.get()) {+ return;+ }byte[] payload = frame.getPayload();- if (payload == null) return;+ if (payload == null) {+ return;+ }try { output.write(payload); } catch (IOException ignored) {}String line = new String(payload, StandardCharsets.UTF_8).stripTrailing();if (!line.isEmpty()) {@@ -1018,23 +1108,58 @@ public class Ec2ContainerManager {public void onComplete() { latch.countDown(); }@Overridepublic void onError(Throwable t) { latch.countDown(); }- });+ @Override+ public void close() throws IOException {+ cancelled.set(true);+ super.close();+ }+ };+ activeUserDataCallbacks.add(callback);- boolean completed = latch.await(30, TimeUnit.MINUTES);- if (!completed) {- LOG.warnv("UserData shellscript part {0}/{1} timed out for EC2 instance {2}", partNumber, partCount, instanceId);- return;+ try {+ dockerClient.execStartCmd(execId).exec(callback);++ boolean completed = latch.await(userDataExecutionTimeout.toMillis(), TimeUnit.MILLISECONDS);+ if (!completed) {+ LOG.warnv("UserData shellscript part {0}/{1} timed out for EC2 instance {2}",+ partNumber, partCount, instanceId);+ return;+ }++ Long exitCode = dockerClient.inspectExecCmd(execId).exec().getExitCodeLong();+ if (exitCode != null && exitCode != 0) {+ LOG.warnv("UserData shellscript part {0}/{1} failed for EC2 instance {2} with exit code {3}: {4}",+ partNumber, partCount, instanceId, exitCode, summarizeUserDataOutput(output));+ return;+ }++ LOG.infov("UserData shellscript part {0}/{1} completed for EC2 instance {2}: {3}",+ partNumber, partCount, instanceId, summarizeUserDataOutput(output));+ } finally {+ activeUserDataCallbacks.remove(callback);+ closeUserDataCallback(callback, instanceId);}+ }- Long exitCode = dockerClient.inspectExecCmd(execId).exec().getExitCodeLong();- if (exitCode != null && exitCode != 0) {- LOG.warnv("UserData shellscript part {0}/{1} failed for EC2 instance {2} with exit code {3}: {4}",- partNumber, partCount, instanceId, exitCode, summarizeUserDataOutput(output));- return;+ private void closeActiveUserDataCallbacks() {+ activeUserDataCallbacks.forEach(callback -> closeUserDataCallback(callback, "active UserData"));+ }++ private void closeUserDataCallback(ResultCallback<Frame> callback, String context) {+ try {+ callback.close();+ } catch (IOException e) {+ LOG.warnv("Could not close Docker UserData callback for {0}: {1}", context, e.getMessage());}+ }- LOG.infov("UserData shellscript part {0}/{1} completed for EC2 instance {2}: {3}",- partNumber, partCount, instanceId, summarizeUserDataOutput(output));+ private void closeUserDataStream(Closeable stream, String instanceId) {+ try {+ stream.close();+ } catch (IOException e) {+ LOG.warnv("Could not close Docker UserData stream for EC2 instance {0}: {1}",+ instanceId, e.getMessage());+ }}static List<String> userDataShellScripts(String userData) {diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.javaindex dc87b7713c..cde168843d 100644--- a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.java@@ -32,17 +32,20 @@ import io.github.hectorvent.floci.services.ec2.model.InstanceNetworkInterface;import org.junit.jupiter.api.Test;import java.io.InputStream;+import java.io.Closeable;import java.time.Duration;import java.util.ArrayList;import java.util.Arrays;import java.util.List;import java.util.Map;import java.util.Optional;+import java.util.concurrent.ArrayBlockingQueue;import java.util.concurrent.CopyOnWriteArrayList;import java.util.concurrent.CountDownLatch;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;import java.util.concurrent.Future;+import java.util.concurrent.ThreadPoolExecutor;import java.util.concurrent.TimeUnit;import java.util.concurrent.atomic.AtomicInteger;import java.util.concurrent.atomic.AtomicReference;@@ -809,6 +812,89 @@ class Ec2ContainerManagerTest {awaitUntil(() -> "running".equals(instance.getState().getName()), Duration.ofSeconds(2));}+ @Test+ void launchAppliesBackpressureWhenDockerLaunchesAreSaturated() throws Exception {+ ThreadPoolExecutor launchExecutor = new ThreadPoolExecutor(+ 1, 1, 0, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(1),+ Ec2ContainerManager.BLOCKING_BACKPRESSURE);+ LaunchHarness harness = launchHarness(launchExecutor, Duration.ofSeconds(1));+ ExecutorService callerExecutor = Executors.newSingleThreadExecutor();+ CountDownLatch createEntered = new CountDownLatch(1);+ CountDownLatch releaseCreate = new CountDownLatch(1);+ when(harness.lifecycleManager().create(any(ContainerSpec.class))).thenAnswer(invocation -> {+ createEntered.countDown();+ releaseCreate.await(2, TimeUnit.SECONDS);+ throw new RuntimeException("test launch failure");+ });++ try {+ harness.manager().launch(instance("i-backpressure-1"), "ubuntu:24.04", null, "us-west-2");+ harness.manager().launch(instance("i-backpressure-2"), "ubuntu:24.04", null, "us-west-2");+ Future<?> waitingLaunch = callerExecutor.submit(() ->+ harness.manager().launch(instance("i-backpressure-3"), "ubuntu:24.04", null, "us-west-2"));++ assertTrue(createEntered.await(2, TimeUnit.SECONDS), "worker launch should enter Docker launch");+ assertFalse(waitingLaunch.isDone(), "saturated launch should wait for queue capacity");++ releaseCreate.countDown();+ waitingLaunch.get(2, TimeUnit.SECONDS);+ } finally {+ releaseCreate.countDown();+ harness.manager().stop();+ callerExecutor.shutdownNow();+ launchExecutor.shutdownNow();+ }+ }++ @Test+ void userDataTimeoutClosesDockerExecStream() throws Exception {+ LaunchHarness harness = launchHarness(new ThreadPoolExecutor(+ 1, 1, 0, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(1),+ new ThreadPoolExecutor.CallerRunsPolicy()), Duration.ofMillis(50));+ Ec2ContainerManager.containerBridgeIpAttempts = 1;+ Ec2ContainerManager.containerBridgeIpPollMillis = 1;+ InspectContainerCmd inspect = mock(InspectContainerCmd.class);+ when(harness.dockerClient().inspectContainerCmd(TEST_CONTAINER_ID)).thenReturn(inspect);+ InspectContainerResponse withIp = inspectResponse("172.18.0.30");+ when(inspect.exec()).thenReturn(withIp);+ Closeable stream = mock(Closeable.class);+ CountDownLatch userDataStarted = new CountDownLatch(1);+ stubUserDataCallback(harness, stream, userDataStarted);+ Instance instance = instance("i-userdata-timeout");+ instance.setUserData("#!/bin/sh\necho ready\n");++ try {+ harness.manager().launch(instance, "ubuntu:24.04", null, "us-west-2");+ assertTrue(userDataStarted.await(2, TimeUnit.SECONDS), "user data should start");+ verify(stream, timeout(2_000)).close();+ } finally {+ harness.manager().stop();+ }+ }++ @Test+ void shutdownClosesActiveUserDataExecStream() throws Exception {+ LaunchHarness harness = launchHarness();+ Ec2ContainerManager.containerBridgeIpAttempts = 1;+ Ec2ContainerManager.containerBridgeIpPollMillis = 1;+ InspectContainerCmd inspect = mock(InspectContainerCmd.class);+ when(harness.dockerClient().inspectContainerCmd(TEST_CONTAINER_ID)).thenReturn(inspect);+ InspectContainerResponse withIp = inspectResponse("172.18.0.31");+ when(inspect.exec()).thenReturn(withIp);+ Closeable stream = mock(Closeable.class);+ CountDownLatch userDataStarted = new CountDownLatch(1);+ stubUserDataCallback(harness, stream, userDataStarted);+ Instance instance = instance("i-userdata-shutdown");+ instance.setUserData("#!/bin/sh\necho ready\n");++ harness.manager().launch(instance, "ubuntu:24.04", null, "us-west-2");+ assertTrue(userDataStarted.await(2, TimeUnit.SECONDS), "user data should start");++ harness.manager().stop();++ verify(stream, timeout(2_000)).close();+ }+@Testvoid launchCreatesSshdPrivilegeSeparationDirectoryBeforeStartingSshd() throws Exception {Ec2ContainerManager.containerBridgeIpAttempts = 1;@@ -960,6 +1046,10 @@ class Ec2ContainerManagerTest {}private static LaunchHarness launchHarness() {+ return launchHarness(null, Duration.ofMinutes(30));+ }++ private static LaunchHarness launchHarness(ExecutorService executor, Duration userDataTimeout) {ContainerBuilder containerBuilder = mock(ContainerBuilder.class);ContainerBuilder.Builder builder = mock(ContainerBuilder.Builder.class, withSettings().defaultAnswer(RETURNS_SELF));when(containerBuilder.newContainer(anyString())).thenReturn(builder);@@ -990,19 +1080,35 @@ class Ec2ContainerManagerTest {Ec2PortForwardManager portForwardManager = mock(Ec2PortForwardManager.class);RegionResolver regionResolver = mock(RegionResolver.class);when(regionResolver.getAccountId()).thenReturn("000000000000");- Ec2ContainerManager manager = new Ec2ContainerManager(- containerBuilder,- lifecycleManager,- logStreamer,- mock(ContainerDetector.class),- dockerHostResolver,- dockerClient,- portAllocator,- config,- metadataServer,- portForwardManager,- regionResolver,- mock(ContainerNetworkReachability.class));+ Ec2ContainerManager manager = executor == null+ ? new Ec2ContainerManager(+ containerBuilder,+ lifecycleManager,+ logStreamer,+ mock(ContainerDetector.class),+ dockerHostResolver,+ dockerClient,+ portAllocator,+ config,+ metadataServer,+ portForwardManager,+ regionResolver,+ mock(ContainerNetworkReachability.class))+ : new Ec2ContainerManager(+ containerBuilder,+ lifecycleManager,+ logStreamer,+ mock(ContainerDetector.class),+ dockerHostResolver,+ dockerClient,+ portAllocator,+ config,+ metadataServer,+ portForwardManager,+ regionResolver,+ mock(ContainerNetworkReachability.class),+ executor,+ userDataTimeout);return new LaunchHarness(manager, lifecycleManager, dockerClient, metadataServer, logStreamer, builder,portAllocator, portForwardManager, config, new CopyOnWriteArrayList<>());}@@ -1137,6 +1243,56 @@ class Ec2ContainerManagerTest {return harness;}+ private static void stubUserDataCallback(LaunchHarness harness, Closeable stream,+ CountDownLatch userDataStarted) throws Exception {+ AtomicReference<String[]> currentCommand = new AtomicReference<>();+ ExecCreateCmd execCreate = mock(ExecCreateCmd.class, withSettings().defaultAnswer(RETURNS_SELF));+ ExecCreateCmdResponse metadataExec = mock(ExecCreateCmdResponse.class);+ ExecCreateCmdResponse userDataExec = mock(ExecCreateCmdResponse.class);+ when(metadataExec.getId()).thenReturn("metadata-exec");+ when(userDataExec.getId()).thenReturn("userdata-exec");+ when(harness.dockerClient().execCreateCmd(TEST_CONTAINER_ID)).thenReturn(execCreate);+ when(execCreate.withCmd(any(String[].class))).thenAnswer(invocation -> {+ Object[] args = invocation.getArguments();+ currentCommand.set(args.length == 1 && args[0] instanceof String[] command+ ? command : Arrays.copyOf(args, args.length, String[].class));+ return execCreate;+ });+ when(execCreate.exec()).thenAnswer(invocation -> {+ String[] command = currentCommand.get();+ return command != null && command.length == 1 && "/tmp/user-data.sh".equals(command[0])+ ? userDataExec : metadataExec;+ });++ when(harness.dockerClient().execStartCmd(anyString())).thenAnswer(invocation -> {+ String execId = invocation.getArgument(0);+ ExecStartCmd execStart = mock(ExecStartCmd.class);+ when(execStart.exec(any())).thenAnswer(startInvocation -> {+ @SuppressWarnings("unchecked")+ ResultCallback<Frame> callback = startInvocation.getArgument(0);+ if ("userdata-exec".equals(execId)) {+ callback.onStart(stream);+ userDataStarted.countDown();+ } else {+ callback.onComplete();+ }+ return callback;+ });+ return execStart;+ });++ InspectExecCmd inspectExec = mock(InspectExecCmd.class);+ InspectExecResponse inspectExecResponse = mock(InspectExecResponse.class);+ when(inspectExecResponse.getExitCodeLong()).thenReturn(0L);+ when(inspectExec.exec()).thenReturn(inspectExecResponse);+ when(harness.dockerClient().inspectExecCmd(anyString())).thenReturn(inspectExec);++ CopyArchiveToContainerCmd copy = mock(CopyArchiveToContainerCmd.class, withSettings().defaultAnswer(RETURNS_SELF));+ when(harness.dockerClient().copyArchiveToContainerCmd(TEST_CONTAINER_ID)).thenReturn(copy);+ when(copy.withTarInputStream(any(InputStream.class))).thenReturn(copy);+ when(harness.logStreamer().generateLogStreamName(anyString())).thenReturn(TEST_LOG_STREAM_NAME);+ }+/*** Index of the first exec matching {@code expected} exactly, or -1 when it was never run.*/