Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(docker): share one deadline across container teardown (#6077)

Andre Kurait4fa11104b4d6
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/ContainerTeardowns.java b/src/main/java/io/github/hectorvent/floci/core/common/ContainerTeardowns.java
index 22a4ed9626..af3a82c7c3 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/ContainerTeardowns.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/ContainerTeardowns.java
@@ -3,6 +3,16 @@ package io.github.hectorvent.floci.core.common;
import jakarta.enterprise.inject.Instance;
import org.jboss.logging.Logger;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+
/**
* Runs every {@link ContainerTeardown} implementation, tolerating a failure in one so it
* doesn't block the others or whatever cleanup the caller runs next. {@code EmulatorLifecycle}
@@ -24,4 +34,50 @@ public final class ContainerTeardowns {
}
}
}
+
+ public static <T> void stopInParallel(List<T> resources, Consumer<T> stop, Logger log) {
+ stopInParallel(resources, stop, log, System.nanoTime() + TimeUnit.SECONDS.toNanos(15));
+ }
+
+ public static <T> void stopInParallel(List<T> resources, Consumer<T> stop, Logger log,
+ long deadlineNanos) {
+ if (resources.isEmpty()) {
+ return;
+ }
+ long remaining = Math.max(0, deadlineNanos - System.nanoTime());
+ if (remaining == 0) {
+ log.warnv("Container drain deadline expired with {0} resource(s) pending", resources.size());
+ return;
+ }
+ // Every owned container must receive its stop within the same deadline. A fixed worker
+ // queue lets slow Docker calls consume that budget before later containers can even stop.
+ ExecutorService executor = Executors.newThreadPerTaskExecutor(
+ Thread.ofVirtual().name("container-teardown-", 0).factory());
+ try {
+ List<Callable<Void>> tasks = new ArrayList<>(resources.size());
+ for (T resource : resources) {
+ tasks.add(() -> {
+ stop.accept(resource);
+ return null;
+ });
+ }
+ for (Future<Void> task : executor.invokeAll(tasks,
+ Math.max(0, deadlineNanos - System.nanoTime()), TimeUnit.NANOSECONDS)) {
+ if (task.isCancelled()) {
+ log.warn("Container drain did not finish before the shared deadline");
+ } else {
+ try {
+ task.get();
+ } catch (ExecutionException failure) {
+ log.warnv(failure.getCause(), "Container drain failed");
+ }
+ }
+ }
+ } catch (InterruptedException interrupted) {
+ Thread.currentThread().interrupt();
+ log.warn("Container drain interrupted");
+ } finally {
+ executor.shutdownNow();
+ }
+ }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java b/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java
index c2a320875e..e96452d5a7 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java
@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.lambda;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.ContainerTeardown;
+import io.github.hectorvent.floci.core.common.ContainerTeardowns;
import io.github.hectorvent.floci.services.lambda.launcher.ContainerHandle;
import io.github.hectorvent.floci.services.lambda.launcher.LambdaRuntimeLauncher;
import io.github.hectorvent.floci.services.lambda.model.ContainerState;
@@ -21,9 +22,7 @@ import java.util.List;
import java.util.Map;
import java.util.Optional;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
import java.util.concurrent.ScheduledExecutorService;
import java.util.concurrent.TimeUnit;
@@ -384,26 +383,7 @@ public class WarmPool implements ContainerTeardown {
}
private void stopInParallel(List<ContainerHandle> handles) {
- if (handles.isEmpty()) {
- return;
- }
- ExecutorService pool = Executors.newThreadPerTaskExecutor(
- Thread.ofVirtual().name("warm-pool-drainer-", 0).factory());
- try {
- List<Future<?>> futures = new ArrayList<>(handles.size());
- for (ContainerHandle handle : handles) {
- futures.add(pool.submit(() -> stopQuietly(handle)));
- }
- for (Future<?> f : futures) {
- try {
- f.get(15, TimeUnit.SECONDS);
- } catch (Exception e) {
- LOG.warnv("Drain task did not finish cleanly: {0}", e.getMessage());
- }
- }
- } finally {
- pool.shutdownNow();
- }
+ ContainerTeardowns.stopInParallel(handles, this::stopQuietly, LOG);
}
private void drainAll() {
diff --git a/src/test/java/io/github/hectorvent/floci/core/common/ContainerTeardownsTest.java b/src/test/java/io/github/hectorvent/floci/core/common/ContainerTeardownsTest.java
new file mode 100644
index 0000000000..9546887f3e
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/core/common/ContainerTeardownsTest.java
@@ -0,0 +1,75 @@
+package io.github.hectorvent.floci.core.common;
+
+import org.jboss.logging.Logger;
+import org.junit.jupiter.api.Test;
+
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.stream.IntStream;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class ContainerTeardownsTest {
+
+ private static final Logger LOG = Logger.getLogger(ContainerTeardownsTest.class);
+
+ @Test
+ void admitsEveryOwnedStopBeforeWaitingForOtherContainers() throws Exception {
+ List<Integer> resources = IntStream.range(0, 83).boxed().toList();
+ CountDownLatch started = new CountDownLatch(resources.size());
+ CountDownLatch release = new CountDownLatch(1);
+ Set<Integer> stopped = ConcurrentHashMap.newKeySet();
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ Future<?> drain = caller.submit(() -> ContainerTeardowns.stopInParallel(resources, resource -> {
+ started.countDown();
+ try {
+ release.await();
+ } catch (InterruptedException interrupted) {
+ Thread.currentThread().interrupt();
+ throw new IllegalStateException(interrupted);
+ }
+ stopped.add(resource);
+ }, LOG));
+ try {
+ assertTrue(started.await(3, TimeUnit.SECONDS),
+ "Queued containers must receive their stop before slower containers consume the shared deadline");
+ } finally {
+ release.countDown();
+ try {
+ drain.get(5, TimeUnit.SECONDS);
+ } finally {
+ caller.shutdownNow();
+ }
+ }
+ assertEquals(Set.copyOf(resources), stopped);
+ }
+
+ @Test
+ void failedStopDoesNotPreventTheOtherOwnedStops() {
+ List<Integer> resources = IntStream.range(0, 83).boxed().toList();
+ Set<Integer> attempted = ConcurrentHashMap.newKeySet();
+ ContainerTeardowns.stopInParallel(resources, resource -> {
+ attempted.add(resource);
+ if (resource == 0) {
+ throw new IllegalStateException("Docker removal failed");
+ }
+ }, LOG);
+ assertEquals(Set.copyOf(resources), attempted);
+ }
+
+ @Test
+ void anExpiredDeadlineDoesNotStartAnotherDrainBudget() {
+ AtomicInteger attempts = new AtomicInteger();
+ ContainerTeardowns.stopInParallel(List.of(1, 2),
+ ignored -> attempts.incrementAndGet(), LOG, System.nanoTime() - 1);
+ assertEquals(0, attempts.get());
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java
index 5459eea0f7..702b0eb965 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java
@@ -25,6 +25,7 @@ import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotSame;
@@ -178,6 +179,60 @@ class WarmPoolTest {
pool.shutdown();
}
+ @Test
+ void interruptedDrainCancelsEveryStopAndPreservesTheCallerInterrupt() throws Exception {
+ WarmPool pool = buildPool();
+ pool.init();
+ LambdaFunction fn = function("interrupted-drain");
+ AtomicInteger ids = new AtomicInteger();
+ when(containerLauncher.launch(any())).thenAnswer(invocation ->
+ new ContainerHandle("cid-interrupt-" + ids.incrementAndGet(),
+ fn.getFunctionName(), null, ContainerState.WARM));
+ List<ContainerHandle> handles = new ArrayList<>();
+ for (int i = 0; i < 3; i++) {
+ handles.add(pool.acquire(fn));
+ }
+ handles.forEach(pool::release);
+
+ CountDownLatch allStopping = new CountDownLatch(handles.size());
+ CountDownLatch finishStops = new CountDownLatch(1);
+ CountDownLatch interruptedStops = new CountDownLatch(handles.size());
+ doAnswer(invocation -> {
+ allStopping.countDown();
+ try {
+ finishStops.await();
+ } catch (InterruptedException interrupted) {
+ interruptedStops.countDown();
+ throw interrupted;
+ }
+ return null;
+ }).when(containerLauncher).stop(any());
+ AtomicReference<Thread> drainThread = new AtomicReference<>();
+ ExecutorService caller = Executors.newSingleThreadExecutor();
+ Future<Boolean> drain = caller.submit(() -> {
+ drainThread.set(Thread.currentThread());
+ pool.stopManagedContainers();
+ return Thread.currentThread().isInterrupted();
+ });
+ try {
+ assertTrue(allStopping.await(3, TimeUnit.SECONDS));
+ drainThread.get().interrupt();
+ assertTrue(drain.get(2, TimeUnit.SECONDS));
+ assertTrue(interruptedStops.await(3, TimeUnit.SECONDS));
+ } finally {
+ finishStops.countDown();
+ try {
+ drain.get(5, TimeUnit.SECONDS);
+ } finally {
+ caller.shutdownNow();
+ pool.shutdown();
+ }
+ }
+ for (ContainerHandle handle : handles) {
+ verify(containerLauncher, times(1)).stop(handle);
+ }
+ }
+
@Test
void destroyHandleStopsContainerAndDoesNotReturnToPool() {
WarmPool pool = buildPool();