Light, fluffy, and always free - The AWS Local Emulator alternative
fix(docker): share one deadline across container teardown (#6077)
Andre Kurait4fa11104b4d6
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/ContainerTeardowns.java b/src/main/java/io/github/hectorvent/floci/core/common/ContainerTeardowns.javaindex 22a4ed9626..af3a82c7c3 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/ContainerTeardowns.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/ContainerTeardowns.java@@ -3,6 +3,16 @@ package io.github.hectorvent.floci.core.common;import jakarta.enterprise.inject.Instance;import org.jboss.logging.Logger;+import java.util.ArrayList;+import java.util.List;+import java.util.concurrent.Callable;+import java.util.concurrent.ExecutionException;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;+import java.util.concurrent.TimeUnit;+import java.util.function.Consumer;+/*** Runs every {@link ContainerTeardown} implementation, tolerating a failure in one so it* doesn't block the others or whatever cleanup the caller runs next. {@code EmulatorLifecycle}@@ -24,4 +34,50 @@ public final class ContainerTeardowns {}}}++ public static <T> void stopInParallel(List<T> resources, Consumer<T> stop, Logger log) {+ stopInParallel(resources, stop, log, System.nanoTime() + TimeUnit.SECONDS.toNanos(15));+ }++ public static <T> void stopInParallel(List<T> resources, Consumer<T> stop, Logger log,+ long deadlineNanos) {+ if (resources.isEmpty()) {+ return;+ }+ long remaining = Math.max(0, deadlineNanos - System.nanoTime());+ if (remaining == 0) {+ log.warnv("Container drain deadline expired with {0} resource(s) pending", resources.size());+ return;+ }+ // Every owned container must receive its stop within the same deadline. A fixed worker+ // queue lets slow Docker calls consume that budget before later containers can even stop.+ ExecutorService executor = Executors.newThreadPerTaskExecutor(+ Thread.ofVirtual().name("container-teardown-", 0).factory());+ try {+ List<Callable<Void>> tasks = new ArrayList<>(resources.size());+ for (T resource : resources) {+ tasks.add(() -> {+ stop.accept(resource);+ return null;+ });+ }+ for (Future<Void> task : executor.invokeAll(tasks,+ Math.max(0, deadlineNanos - System.nanoTime()), TimeUnit.NANOSECONDS)) {+ if (task.isCancelled()) {+ log.warn("Container drain did not finish before the shared deadline");+ } else {+ try {+ task.get();+ } catch (ExecutionException failure) {+ log.warnv(failure.getCause(), "Container drain failed");+ }+ }+ }+ } catch (InterruptedException interrupted) {+ Thread.currentThread().interrupt();+ log.warn("Container drain interrupted");+ } finally {+ executor.shutdownNow();+ }+ }}diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java b/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.javaindex c2a320875e..e96452d5a7 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java@@ -2,6 +2,7 @@ package io.github.hectorvent.floci.services.lambda;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.ContainerTeardown;+import io.github.hectorvent.floci.core.common.ContainerTeardowns;import io.github.hectorvent.floci.services.lambda.launcher.ContainerHandle;import io.github.hectorvent.floci.services.lambda.launcher.LambdaRuntimeLauncher;import io.github.hectorvent.floci.services.lambda.model.ContainerState;@@ -21,9 +22,7 @@ import java.util.List;import java.util.Map;import java.util.Optional;import java.util.concurrent.ConcurrentHashMap;-import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;-import java.util.concurrent.Future;import java.util.concurrent.ScheduledExecutorService;import java.util.concurrent.TimeUnit;@@ -384,26 +383,7 @@ public class WarmPool implements ContainerTeardown {}private void stopInParallel(List<ContainerHandle> handles) {- if (handles.isEmpty()) {- return;- }- ExecutorService pool = Executors.newThreadPerTaskExecutor(- Thread.ofVirtual().name("warm-pool-drainer-", 0).factory());- try {- List<Future<?>> futures = new ArrayList<>(handles.size());- for (ContainerHandle handle : handles) {- futures.add(pool.submit(() -> stopQuietly(handle)));- }- for (Future<?> f : futures) {- try {- f.get(15, TimeUnit.SECONDS);- } catch (Exception e) {- LOG.warnv("Drain task did not finish cleanly: {0}", e.getMessage());- }- }- } finally {- pool.shutdownNow();- }+ ContainerTeardowns.stopInParallel(handles, this::stopQuietly, LOG);}private void drainAll() {diff --git a/src/test/java/io/github/hectorvent/floci/core/common/ContainerTeardownsTest.java b/src/test/java/io/github/hectorvent/floci/core/common/ContainerTeardownsTest.javanew file mode 100644index 0000000000..9546887f3e--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/core/common/ContainerTeardownsTest.java@@ -0,0 +1,75 @@+package io.github.hectorvent.floci.core.common;++import org.jboss.logging.Logger;+import org.junit.jupiter.api.Test;++import java.util.List;+import java.util.Set;+import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.Future;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicInteger;+import java.util.stream.IntStream;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertTrue;++class ContainerTeardownsTest {++ private static final Logger LOG = Logger.getLogger(ContainerTeardownsTest.class);++ @Test+ void admitsEveryOwnedStopBeforeWaitingForOtherContainers() throws Exception {+ List<Integer> resources = IntStream.range(0, 83).boxed().toList();+ CountDownLatch started = new CountDownLatch(resources.size());+ CountDownLatch release = new CountDownLatch(1);+ Set<Integer> stopped = ConcurrentHashMap.newKeySet();+ ExecutorService caller = Executors.newSingleThreadExecutor();+ Future<?> drain = caller.submit(() -> ContainerTeardowns.stopInParallel(resources, resource -> {+ started.countDown();+ try {+ release.await();+ } catch (InterruptedException interrupted) {+ Thread.currentThread().interrupt();+ throw new IllegalStateException(interrupted);+ }+ stopped.add(resource);+ }, LOG));+ try {+ assertTrue(started.await(3, TimeUnit.SECONDS),+ "Queued containers must receive their stop before slower containers consume the shared deadline");+ } finally {+ release.countDown();+ try {+ drain.get(5, TimeUnit.SECONDS);+ } finally {+ caller.shutdownNow();+ }+ }+ assertEquals(Set.copyOf(resources), stopped);+ }++ @Test+ void failedStopDoesNotPreventTheOtherOwnedStops() {+ List<Integer> resources = IntStream.range(0, 83).boxed().toList();+ Set<Integer> attempted = ConcurrentHashMap.newKeySet();+ ContainerTeardowns.stopInParallel(resources, resource -> {+ attempted.add(resource);+ if (resource == 0) {+ throw new IllegalStateException("Docker removal failed");+ }+ }, LOG);+ assertEquals(Set.copyOf(resources), attempted);+ }++ @Test+ void anExpiredDeadlineDoesNotStartAnotherDrainBudget() {+ AtomicInteger attempts = new AtomicInteger();+ ContainerTeardowns.stopInParallel(List.of(1, 2),+ ignored -> attempts.incrementAndGet(), LOG, System.nanoTime() - 1);+ assertEquals(0, attempts.get());+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.javaindex 5459eea0f7..702b0eb965 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java@@ -25,6 +25,7 @@ import java.util.concurrent.Executors;import java.util.concurrent.Future;import java.util.concurrent.TimeUnit;import java.util.concurrent.atomic.AtomicInteger;+import java.util.concurrent.atomic.AtomicReference;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertNotSame;@@ -178,6 +179,60 @@ class WarmPoolTest {pool.shutdown();}+ @Test+ void interruptedDrainCancelsEveryStopAndPreservesTheCallerInterrupt() throws Exception {+ WarmPool pool = buildPool();+ pool.init();+ LambdaFunction fn = function("interrupted-drain");+ AtomicInteger ids = new AtomicInteger();+ when(containerLauncher.launch(any())).thenAnswer(invocation ->+ new ContainerHandle("cid-interrupt-" + ids.incrementAndGet(),+ fn.getFunctionName(), null, ContainerState.WARM));+ List<ContainerHandle> handles = new ArrayList<>();+ for (int i = 0; i < 3; i++) {+ handles.add(pool.acquire(fn));+ }+ handles.forEach(pool::release);++ CountDownLatch allStopping = new CountDownLatch(handles.size());+ CountDownLatch finishStops = new CountDownLatch(1);+ CountDownLatch interruptedStops = new CountDownLatch(handles.size());+ doAnswer(invocation -> {+ allStopping.countDown();+ try {+ finishStops.await();+ } catch (InterruptedException interrupted) {+ interruptedStops.countDown();+ throw interrupted;+ }+ return null;+ }).when(containerLauncher).stop(any());+ AtomicReference<Thread> drainThread = new AtomicReference<>();+ ExecutorService caller = Executors.newSingleThreadExecutor();+ Future<Boolean> drain = caller.submit(() -> {+ drainThread.set(Thread.currentThread());+ pool.stopManagedContainers();+ return Thread.currentThread().isInterrupted();+ });+ try {+ assertTrue(allStopping.await(3, TimeUnit.SECONDS));+ drainThread.get().interrupt();+ assertTrue(drain.get(2, TimeUnit.SECONDS));+ assertTrue(interruptedStops.await(3, TimeUnit.SECONDS));+ } finally {+ finishStops.countDown();+ try {+ drain.get(5, TimeUnit.SECONDS);+ } finally {+ caller.shutdownNow();+ pool.shutdown();+ }+ }+ for (ContainerHandle handle : handles) {+ verify(containerLauncher, times(1)).stop(handle);+ }+ }+@Testvoid destroyHandleStopsContainerAndDoesNotReturnToPool() {WarmPool pool = buildPool();