Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(ec2): bound launches and close userdata streams (#3135)

Niels Kootstradd33308e7eca
diff --git a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.java b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.java
index 273ba447f1..3bd93296e7 100644
--- a/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManager.java
@@ -30,9 +30,11 @@ import org.jboss.logging.Logger;
import java.io.ByteArrayInputStream;
import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
import java.io.IOException;
import java.io.OutputStream;
import java.nio.charset.StandardCharsets;
+import java.time.Duration;
import java.util.ArrayList;
import java.util.Base64;
import java.util.List;
@@ -41,12 +43,17 @@ import java.util.Map;
import java.util.Objects;
import java.util.Optional;
import java.util.Set;
+import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
+import java.util.concurrent.RejectedExecutionException;
+import java.util.concurrent.RejectedExecutionHandler;
import java.util.concurrent.Semaphore;
+import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.function.BiPredicate;
+import java.util.concurrent.atomic.AtomicBoolean;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
import java.util.zip.GZIPInputStream;
@@ -76,7 +83,7 @@ public class Ec2ContainerManager {
private static final int MAX_DECOMPRESSED_USER_DATA_BYTES = 10 * 1024 * 1024;
private static final int MAX_EXEC_OUTPUT_BYTES = 2048;
/** Caps concurrent UserData gzip decompressions across ALL instance launches, not just one.
- * Launches run independently on {@link #executor}, an unbounded cached thread pool, so the
+ * Launches run independently on {@link #executor}, so the
* per-payload cap above only bounds a single launch's allocation: without this, N concurrent
* RunInstances/CreateLaunchConfiguration calls, each smuggling a near-cap gzip payload, could
* together decompress N * 10 MB at once in the shared emulator JVM with no aggregate ceiling.
@@ -89,6 +96,26 @@ public class Ec2ContainerManager {
/** Gates entry to {@link #gunzip}; see {@link #MAX_CONCURRENT_USER_DATA_DECOMPRESSIONS}. */
private static final Semaphore USER_DATA_DECOMPRESSION_BUDGET =
new Semaphore(MAX_CONCURRENT_USER_DATA_DECOMPRESSIONS);
+ private static final int LAUNCH_CORE_THREADS = 4;
+ private static final int LAUNCH_MAX_THREADS = 8;
+ private static final int LAUNCH_QUEUE_CAPACITY = 64;
+ private static final long USER_DATA_EXECUTION_TIMEOUT_MINUTES = 30;
+ // Wait for capacity so accepted launches are not dropped, but never run launch work on callers.
+ static final RejectedExecutionHandler BLOCKING_BACKPRESSURE = (runnable, executor) -> {
+ if (executor.isShutdown()) {
+ throw new RejectedExecutionException("EC2 container manager is stopped");
+ }
+ try {
+ executor.getQueue().put(runnable);
+ if (executor.isShutdown() && executor.getQueue().remove(runnable)) {
+ throw new RejectedExecutionException("EC2 container manager is stopped");
+ }
+ }
+ catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new RejectedExecutionException("Interrupted while waiting for EC2 launch capacity", e);
+ }
+ };
/** Test seam: when non-null, invoked by {@link #gunzip} right after it acquires a
* decompression-budget permit and before it starts decompressing, so tests can observe and
* serialize concurrent decompressions deterministically. Always null in production. */
@@ -138,15 +165,12 @@ public class Ec2ContainerManager {
private final Ec2PortForwardManager portForwardManager;
private final RegionResolver regionResolver;
private final ContainerNetworkReachability containerNetworkReachability;
+ private final ExecutorService executor;
+ private final Duration userDataExecutionTimeout;
+ private final Set<ResultCallback<Frame>> activeUserDataCallbacks = ConcurrentHashMap.newKeySet();
private volatile boolean dockerUnavailableLogged;
- private final ExecutorService executor = Executors.newCachedThreadPool(r -> {
- Thread t = new Thread(r, "ec2-container-launcher");
- t.setDaemon(true);
- return t;
- });
-
@Inject
public Ec2ContainerManager(ContainerBuilder containerBuilder,
ContainerLifecycleManager lifecycleManager,
@@ -160,6 +184,26 @@ public class Ec2ContainerManager {
Ec2PortForwardManager portForwardManager,
RegionResolver regionResolver,
ContainerNetworkReachability containerNetworkReachability) {
+ this(containerBuilder, lifecycleManager, logStreamer, containerDetector, dockerHostResolver, dockerClient,
+ portAllocator, config, metadataServer, portForwardManager, regionResolver,
+ containerNetworkReachability, createLaunchExecutor(),
+ Duration.ofMinutes(USER_DATA_EXECUTION_TIMEOUT_MINUTES));
+ }
+
+ Ec2ContainerManager(ContainerBuilder containerBuilder,
+ ContainerLifecycleManager lifecycleManager,
+ ContainerLogStreamer logStreamer,
+ ContainerDetector containerDetector,
+ DockerHostResolver dockerHostResolver,
+ DockerClient dockerClient,
+ PortAllocator portAllocator,
+ EmulatorConfig config,
+ Ec2MetadataServer metadataServer,
+ Ec2PortForwardManager portForwardManager,
+ RegionResolver regionResolver,
+ ContainerNetworkReachability containerNetworkReachability,
+ ExecutorService executor,
+ Duration userDataExecutionTimeout) {
this.containerBuilder = containerBuilder;
this.lifecycleManager = lifecycleManager;
this.logStreamer = logStreamer;
@@ -172,11 +216,30 @@ public class Ec2ContainerManager {
this.metadataServer = metadataServer;
this.portForwardManager = portForwardManager;
this.containerNetworkReachability = containerNetworkReachability;
+ this.executor = executor;
+ this.userDataExecutionTimeout = userDataExecutionTimeout;
+ }
+
+ private static ExecutorService createLaunchExecutor() {
+ return new ThreadPoolExecutor(
+ LAUNCH_CORE_THREADS,
+ LAUNCH_MAX_THREADS,
+ 60L,
+ TimeUnit.SECONDS,
+ new ArrayBlockingQueue<>(LAUNCH_QUEUE_CAPACITY),
+ runnable -> {
+ Thread thread = new Thread(runnable, "ec2-container-launcher");
+ thread.setDaemon(true);
+ return thread;
+ },
+ BLOCKING_BACKPRESSURE);
}
@PreDestroy
void stop() {
+ closeActiveUserDataCallbacks();
executor.shutdownNow();
+ closeActiveUserDataCallbacks();
}
/**
@@ -215,9 +278,10 @@ public class Ec2ContainerManager {
return;
}
- executor.submit(() -> {
- try {
- String instanceId = instance.getInstanceId();
+ try {
+ executor.execute(() -> {
+ try {
+ String instanceId = instance.getInstanceId();
// IMDS endpoint that this container should use
String flociHost = dockerHostResolver.resolve();
int imdsPort = config.services().ec2().imdsPort();
@@ -315,21 +379,26 @@ public class Ec2ContainerManager {
executeUserData(containerId, instanceId, userData, region);
}
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- failLaunch(instance);
- } catch (Exception e) {
- LOG.warnv("Failed to launch EC2 instance {0}: {1}", instance.getInstanceId(), e.getMessage());
- // The daemon can disappear between the probe above and any of the calls in
- // this block. Losing Docker is not the instance's fault, so degrade to a
- // metadata-only instance; a genuine container failure still fails the launch.
- if (isDockerAvailable()) {
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
failLaunch(instance);
- } else {
- markContainerlessRunning(instance);
+ } catch (Exception e) {
+ LOG.warnv("Failed to launch EC2 instance {0}: {1}", instance.getInstanceId(), e.getMessage());
+ // The daemon can disappear between the probe above and any of the calls in
+ // this block. Losing Docker is not the instance's fault, so degrade to a
+ // metadata-only instance; a genuine container failure still fails the launch.
+ if (isDockerAvailable()) {
+ failLaunch(instance);
+ } else {
+ markContainerlessRunning(instance);
+ }
}
- }
- });
+ });
+ } catch (RejectedExecutionException e) {
+ LOG.warnv("Could not schedule EC2 instance {0} launch because the launch executor is saturated or stopping",
+ instance.getInstanceId());
+ failLaunch(instance);
+ }
}
private StartedContainer createAndStartContainer(Instance instance, ResolvedAmiImage image, String region,
@@ -976,6 +1045,9 @@ public class Ec2ContainerManager {
logGroup, logStream, region
);
}
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ LOG.warnv("UserData execution interrupted for EC2 instance {0}", instanceId);
} catch (Exception e) {
LOG.warnv("UserData execution failed for EC2 instance {0}: {1}", instanceId, e.getMessage());
}
@@ -1003,11 +1075,29 @@ public class Ec2ContainerManager {
BoundedOutput output = new BoundedOutput(MAX_EXEC_OUTPUT_BYTES);
CountDownLatch latch = new CountDownLatch(1);
- dockerClient.execStartCmd(execId).exec(new ResultCallback.Adapter<Frame>() {
+ AtomicBoolean cancelled = new AtomicBoolean();
+ ResultCallback.Adapter<Frame> callback = new ResultCallback.Adapter<>() {
+ @Override
+ public void onStart(Closeable stream) {
+ if (cancelled.get()) {
+ closeUserDataStream(stream, instanceId);
+ return;
+ }
+ super.onStart(stream);
+ if (cancelled.get()) {
+ closeUserDataStream(stream, instanceId);
+ }
+ }
+
@Override
public void onNext(Frame frame) {
+ if (cancelled.get()) {
+ return;
+ }
byte[] payload = frame.getPayload();
- if (payload == null) return;
+ if (payload == null) {
+ return;
+ }
try { output.write(payload); } catch (IOException ignored) {}
String line = new String(payload, StandardCharsets.UTF_8).stripTrailing();
if (!line.isEmpty()) {
@@ -1018,23 +1108,58 @@ public class Ec2ContainerManager {
public void onComplete() { latch.countDown(); }
@Override
public void onError(Throwable t) { latch.countDown(); }
- });
+ @Override
+ public void close() throws IOException {
+ cancelled.set(true);
+ super.close();
+ }
+ };
+ activeUserDataCallbacks.add(callback);
- boolean completed = latch.await(30, TimeUnit.MINUTES);
- if (!completed) {
- LOG.warnv("UserData shellscript part {0}/{1} timed out for EC2 instance {2}", partNumber, partCount, instanceId);
- return;
+ try {
+ dockerClient.execStartCmd(execId).exec(callback);
+
+ boolean completed = latch.await(userDataExecutionTimeout.toMillis(), TimeUnit.MILLISECONDS);
+ if (!completed) {
+ LOG.warnv("UserData shellscript part {0}/{1} timed out for EC2 instance {2}",
+ partNumber, partCount, instanceId);
+ return;
+ }
+
+ Long exitCode = dockerClient.inspectExecCmd(execId).exec().getExitCodeLong();
+ if (exitCode != null && exitCode != 0) {
+ LOG.warnv("UserData shellscript part {0}/{1} failed for EC2 instance {2} with exit code {3}: {4}",
+ partNumber, partCount, instanceId, exitCode, summarizeUserDataOutput(output));
+ return;
+ }
+
+ LOG.infov("UserData shellscript part {0}/{1} completed for EC2 instance {2}: {3}",
+ partNumber, partCount, instanceId, summarizeUserDataOutput(output));
+ } finally {
+ activeUserDataCallbacks.remove(callback);
+ closeUserDataCallback(callback, instanceId);
}
+ }
- Long exitCode = dockerClient.inspectExecCmd(execId).exec().getExitCodeLong();
- if (exitCode != null && exitCode != 0) {
- LOG.warnv("UserData shellscript part {0}/{1} failed for EC2 instance {2} with exit code {3}: {4}",
- partNumber, partCount, instanceId, exitCode, summarizeUserDataOutput(output));
- return;
+ private void closeActiveUserDataCallbacks() {
+ activeUserDataCallbacks.forEach(callback -> closeUserDataCallback(callback, "active UserData"));
+ }
+
+ private void closeUserDataCallback(ResultCallback<Frame> callback, String context) {
+ try {
+ callback.close();
+ } catch (IOException e) {
+ LOG.warnv("Could not close Docker UserData callback for {0}: {1}", context, e.getMessage());
}
+ }
- LOG.infov("UserData shellscript part {0}/{1} completed for EC2 instance {2}: {3}",
- partNumber, partCount, instanceId, summarizeUserDataOutput(output));
+ private void closeUserDataStream(Closeable stream, String instanceId) {
+ try {
+ stream.close();
+ } catch (IOException e) {
+ LOG.warnv("Could not close Docker UserData stream for EC2 instance {0}: {1}",
+ instanceId, e.getMessage());
+ }
}
static List<String> userDataShellScripts(String userData) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.java
index dc87b7713c..cde168843d 100644
--- a/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/ec2/Ec2ContainerManagerTest.java
@@ -32,17 +32,20 @@ import io.github.hectorvent.floci.services.ec2.model.InstanceNetworkInterface;
import org.junit.jupiter.api.Test;
import java.io.InputStream;
+import java.io.Closeable;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.concurrent.ArrayBlockingQueue;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
+import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.concurrent.atomic.AtomicReference;
@@ -809,6 +812,89 @@ class Ec2ContainerManagerTest {
awaitUntil(() -> "running".equals(instance.getState().getName()), Duration.ofSeconds(2));
}
+ @Test
+ void launchAppliesBackpressureWhenDockerLaunchesAreSaturated() throws Exception {
+ ThreadPoolExecutor launchExecutor = new ThreadPoolExecutor(
+ 1, 1, 0, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(1),
+ Ec2ContainerManager.BLOCKING_BACKPRESSURE);
+ LaunchHarness harness = launchHarness(launchExecutor, Duration.ofSeconds(1));
+ ExecutorService callerExecutor = Executors.newSingleThreadExecutor();
+ CountDownLatch createEntered = new CountDownLatch(1);
+ CountDownLatch releaseCreate = new CountDownLatch(1);
+ when(harness.lifecycleManager().create(any(ContainerSpec.class))).thenAnswer(invocation -> {
+ createEntered.countDown();
+ releaseCreate.await(2, TimeUnit.SECONDS);
+ throw new RuntimeException("test launch failure");
+ });
+
+ try {
+ harness.manager().launch(instance("i-backpressure-1"), "ubuntu:24.04", null, "us-west-2");
+ harness.manager().launch(instance("i-backpressure-2"), "ubuntu:24.04", null, "us-west-2");
+ Future<?> waitingLaunch = callerExecutor.submit(() ->
+ harness.manager().launch(instance("i-backpressure-3"), "ubuntu:24.04", null, "us-west-2"));
+
+ assertTrue(createEntered.await(2, TimeUnit.SECONDS), "worker launch should enter Docker launch");
+ assertFalse(waitingLaunch.isDone(), "saturated launch should wait for queue capacity");
+
+ releaseCreate.countDown();
+ waitingLaunch.get(2, TimeUnit.SECONDS);
+ } finally {
+ releaseCreate.countDown();
+ harness.manager().stop();
+ callerExecutor.shutdownNow();
+ launchExecutor.shutdownNow();
+ }
+ }
+
+ @Test
+ void userDataTimeoutClosesDockerExecStream() throws Exception {
+ LaunchHarness harness = launchHarness(new ThreadPoolExecutor(
+ 1, 1, 0, TimeUnit.MILLISECONDS, new ArrayBlockingQueue<>(1),
+ new ThreadPoolExecutor.CallerRunsPolicy()), Duration.ofMillis(50));
+ Ec2ContainerManager.containerBridgeIpAttempts = 1;
+ Ec2ContainerManager.containerBridgeIpPollMillis = 1;
+ InspectContainerCmd inspect = mock(InspectContainerCmd.class);
+ when(harness.dockerClient().inspectContainerCmd(TEST_CONTAINER_ID)).thenReturn(inspect);
+ InspectContainerResponse withIp = inspectResponse("172.18.0.30");
+ when(inspect.exec()).thenReturn(withIp);
+ Closeable stream = mock(Closeable.class);
+ CountDownLatch userDataStarted = new CountDownLatch(1);
+ stubUserDataCallback(harness, stream, userDataStarted);
+ Instance instance = instance("i-userdata-timeout");
+ instance.setUserData("#!/bin/sh\necho ready\n");
+
+ try {
+ harness.manager().launch(instance, "ubuntu:24.04", null, "us-west-2");
+ assertTrue(userDataStarted.await(2, TimeUnit.SECONDS), "user data should start");
+ verify(stream, timeout(2_000)).close();
+ } finally {
+ harness.manager().stop();
+ }
+ }
+
+ @Test
+ void shutdownClosesActiveUserDataExecStream() throws Exception {
+ LaunchHarness harness = launchHarness();
+ Ec2ContainerManager.containerBridgeIpAttempts = 1;
+ Ec2ContainerManager.containerBridgeIpPollMillis = 1;
+ InspectContainerCmd inspect = mock(InspectContainerCmd.class);
+ when(harness.dockerClient().inspectContainerCmd(TEST_CONTAINER_ID)).thenReturn(inspect);
+ InspectContainerResponse withIp = inspectResponse("172.18.0.31");
+ when(inspect.exec()).thenReturn(withIp);
+ Closeable stream = mock(Closeable.class);
+ CountDownLatch userDataStarted = new CountDownLatch(1);
+ stubUserDataCallback(harness, stream, userDataStarted);
+ Instance instance = instance("i-userdata-shutdown");
+ instance.setUserData("#!/bin/sh\necho ready\n");
+
+ harness.manager().launch(instance, "ubuntu:24.04", null, "us-west-2");
+ assertTrue(userDataStarted.await(2, TimeUnit.SECONDS), "user data should start");
+
+ harness.manager().stop();
+
+ verify(stream, timeout(2_000)).close();
+ }
+
@Test
void launchCreatesSshdPrivilegeSeparationDirectoryBeforeStartingSshd() throws Exception {
Ec2ContainerManager.containerBridgeIpAttempts = 1;
@@ -960,6 +1046,10 @@ class Ec2ContainerManagerTest {
}
private static LaunchHarness launchHarness() {
+ return launchHarness(null, Duration.ofMinutes(30));
+ }
+
+ private static LaunchHarness launchHarness(ExecutorService executor, Duration userDataTimeout) {
ContainerBuilder containerBuilder = mock(ContainerBuilder.class);
ContainerBuilder.Builder builder = mock(ContainerBuilder.Builder.class, withSettings().defaultAnswer(RETURNS_SELF));
when(containerBuilder.newContainer(anyString())).thenReturn(builder);
@@ -990,19 +1080,35 @@ class Ec2ContainerManagerTest {
Ec2PortForwardManager portForwardManager = mock(Ec2PortForwardManager.class);
RegionResolver regionResolver = mock(RegionResolver.class);
when(regionResolver.getAccountId()).thenReturn("000000000000");
- Ec2ContainerManager manager = new Ec2ContainerManager(
- containerBuilder,
- lifecycleManager,
- logStreamer,
- mock(ContainerDetector.class),
- dockerHostResolver,
- dockerClient,
- portAllocator,
- config,
- metadataServer,
- portForwardManager,
- regionResolver,
- mock(ContainerNetworkReachability.class));
+ Ec2ContainerManager manager = executor == null
+ ? new Ec2ContainerManager(
+ containerBuilder,
+ lifecycleManager,
+ logStreamer,
+ mock(ContainerDetector.class),
+ dockerHostResolver,
+ dockerClient,
+ portAllocator,
+ config,
+ metadataServer,
+ portForwardManager,
+ regionResolver,
+ mock(ContainerNetworkReachability.class))
+ : new Ec2ContainerManager(
+ containerBuilder,
+ lifecycleManager,
+ logStreamer,
+ mock(ContainerDetector.class),
+ dockerHostResolver,
+ dockerClient,
+ portAllocator,
+ config,
+ metadataServer,
+ portForwardManager,
+ regionResolver,
+ mock(ContainerNetworkReachability.class),
+ executor,
+ userDataTimeout);
return new LaunchHarness(manager, lifecycleManager, dockerClient, metadataServer, logStreamer, builder,
portAllocator, portForwardManager, config, new CopyOnWriteArrayList<>());
}
@@ -1137,6 +1243,56 @@ class Ec2ContainerManagerTest {
return harness;
}
+ private static void stubUserDataCallback(LaunchHarness harness, Closeable stream,
+ CountDownLatch userDataStarted) throws Exception {
+ AtomicReference<String[]> currentCommand = new AtomicReference<>();
+ ExecCreateCmd execCreate = mock(ExecCreateCmd.class, withSettings().defaultAnswer(RETURNS_SELF));
+ ExecCreateCmdResponse metadataExec = mock(ExecCreateCmdResponse.class);
+ ExecCreateCmdResponse userDataExec = mock(ExecCreateCmdResponse.class);
+ when(metadataExec.getId()).thenReturn("metadata-exec");
+ when(userDataExec.getId()).thenReturn("userdata-exec");
+ when(harness.dockerClient().execCreateCmd(TEST_CONTAINER_ID)).thenReturn(execCreate);
+ when(execCreate.withCmd(any(String[].class))).thenAnswer(invocation -> {
+ Object[] args = invocation.getArguments();
+ currentCommand.set(args.length == 1 && args[0] instanceof String[] command
+ ? command : Arrays.copyOf(args, args.length, String[].class));
+ return execCreate;
+ });
+ when(execCreate.exec()).thenAnswer(invocation -> {
+ String[] command = currentCommand.get();
+ return command != null && command.length == 1 && "/tmp/user-data.sh".equals(command[0])
+ ? userDataExec : metadataExec;
+ });
+
+ when(harness.dockerClient().execStartCmd(anyString())).thenAnswer(invocation -> {
+ String execId = invocation.getArgument(0);
+ ExecStartCmd execStart = mock(ExecStartCmd.class);
+ when(execStart.exec(any())).thenAnswer(startInvocation -> {
+ @SuppressWarnings("unchecked")
+ ResultCallback<Frame> callback = startInvocation.getArgument(0);
+ if ("userdata-exec".equals(execId)) {
+ callback.onStart(stream);
+ userDataStarted.countDown();
+ } else {
+ callback.onComplete();
+ }
+ return callback;
+ });
+ return execStart;
+ });
+
+ InspectExecCmd inspectExec = mock(InspectExecCmd.class);
+ InspectExecResponse inspectExecResponse = mock(InspectExecResponse.class);
+ when(inspectExecResponse.getExitCodeLong()).thenReturn(0L);
+ when(inspectExec.exec()).thenReturn(inspectExecResponse);
+ when(harness.dockerClient().inspectExecCmd(anyString())).thenReturn(inspectExec);
+
+ CopyArchiveToContainerCmd copy = mock(CopyArchiveToContainerCmd.class, withSettings().defaultAnswer(RETURNS_SELF));
+ when(harness.dockerClient().copyArchiveToContainerCmd(TEST_CONTAINER_ID)).thenReturn(copy);
+ when(copy.withTarInputStream(any(InputStream.class))).thenReturn(copy);
+ when(harness.logStreamer().generateLogStreamName(anyString())).thenReturn(TEST_LOG_STREAM_NAME);
+ }
+
/**
* Index of the first exec matching {@code expected} exactly, or -1 when it was never run.
*/