Light, fluffy, and always free - The AWS Local Emulator alternative
fix(lambda): drain warm containers across functions concurrently (#5899)
Andre Kurait9ceedde585af
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java b/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.javaindex e5e789ed2a..c2a320875e 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java@@ -367,6 +367,10 @@ public class WarmPool implements ContainerTeardown {* Called on function deletion and emulator shutdown.*/public void drainFunction(String functionName) {+ stopInParallel(detachFunction(functionName));+ }++ private List<ContainerHandle> detachFunction(String functionName) {PoolState poolState = poolStates.computeIfAbsent(functionName, ignored -> new PoolState());List<ContainerHandle> toStop;synchronized (poolState) {@@ -376,16 +380,15 @@ public class WarmPool implements ContainerTeardown {poolState.idleByEnvironment.clear();}LOG.infov("Draining {0} container(s) for function: {1}", toStop.size(), functionName);- stopInParallel(toStop);+ return toStop;}private void stopInParallel(List<ContainerHandle> handles) {if (handles.isEmpty()) {return;}- int parallelism = Math.min(handles.size(), 16);- ExecutorService pool = Executors.newFixedThreadPool(parallelism,- r -> { Thread t = new Thread(r, "warm-pool-drainer"); t.setDaemon(true); return t; });+ ExecutorService pool = Executors.newThreadPerTaskExecutor(+ Thread.ofVirtual().name("warm-pool-drainer-", 0).factory());try {List<Future<?>> futures = new ArrayList<>(handles.size());for (ContainerHandle handle : handles) {@@ -404,9 +407,11 @@ public class WarmPool implements ContainerTeardown {}private void drainAll() {+ List<ContainerHandle> toStop = new ArrayList<>();for (String functionName : new ArrayList<>(poolStates.keySet())) {- drainFunction(functionName);+ toStop.addAll(detachFunction(functionName));}+ stopInParallel(toStop);}private void evictIdleContainers() {diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.javaindex da650b1a8a..5459eea0f7 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java@@ -16,18 +16,22 @@ import java.time.Duration;import java.time.Instant;import java.time.ZoneId;import java.time.ZoneOffset;+import java.util.ArrayList;+import java.util.List;import java.util.Optional;import java.util.concurrent.CountDownLatch;import java.util.concurrent.ExecutorService;import java.util.concurrent.Executors;import java.util.concurrent.Future;import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicInteger;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertNotSame;import static org.junit.jupiter.api.Assertions.assertSame;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.mockito.ArgumentMatchers.any;+import static org.mockito.Mockito.doAnswer;import static org.mockito.Mockito.lenient;import static org.mockito.Mockito.mock;import static org.mockito.Mockito.never;@@ -119,6 +123,51 @@ class WarmPoolTest {pool.shutdown();}+ @Test+ void stopManagedContainersDoesNotLetOneFunctionOccupyEveryWorker() throws Exception {+ WarmPool pool = buildPool(Optional.of(17), 0);+ pool.init();+ AtomicInteger ids = new AtomicInteger();+ when(containerLauncher.launch(any())).thenAnswer(invocation -> {+ LambdaFunction fn = invocation.getArgument(0);+ return new ContainerHandle("cid-" + ids.incrementAndGet(),+ fn.getFunctionName(), null, ContainerState.WARM);+ });+ List<ContainerHandle> handles = new ArrayList<>();+ for (String name : List.of("first", "second")) {+ LambdaFunction fn = function(name);+ for (int i = 0; i < 17; i++) {+ handles.add(pool.acquire(fn));+ }+ }+ handles.forEach(pool::release);++ CountDownLatch allStopping = new CountDownLatch(handles.size());+ CountDownLatch finishStops = new CountDownLatch(1);+ doAnswer(invocation -> {+ allStopping.countDown();+ assertTrue(finishStops.await(10, TimeUnit.SECONDS));+ return null;+ }).when(containerLauncher).stop(any());+ ExecutorService executor = Executors.newSingleThreadExecutor();+ Future<?> drain = executor.submit(pool::stopManagedContainers);+ try {+ assertTrue(allStopping.await(5, TimeUnit.SECONDS),+ "Every container must begin stopping before blocked stops finish");+ } finally {+ finishStops.countDown();+ try {+ drain.get(10, TimeUnit.SECONDS);+ } finally {+ executor.shutdownNow();+ pool.shutdown();+ }+ }+ for (ContainerHandle handle : handles) {+ verify(containerLauncher, times(1)).stop(handle);+ }+ }+@Testvoid stopManagedContainersOnEmptyPoolIsNoOp() {WarmPool pool = buildPool();