Light, fluffy, and always free - The AWS Local Emulator alternative
fix(docker): bound concurrent container teardown (#5860)
Andre Kuraitae285fc3a801
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.javaindex 6397598452..aeed6b382b 100644--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManager.java@@ -48,6 +48,7 @@ import java.util.OptionalLong;import java.util.Set;import java.util.UUID;import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.Semaphore;import java.util.concurrent.TimeUnit;import java.util.concurrent.atomic.AtomicReference;import java.util.regex.Pattern;@@ -84,6 +85,8 @@ public class ContainerLifecycleManager {/** Host interface a port marked loopback-only publishes on. */private static final String LOOPBACK_HOST_IP = "127.0.0.1";+ private final Semaphore teardownSlots = new Semaphore(8, true);+private final DockerClient dockerClient;private final ImageCacheService imageCacheService;private final ContainerDetector containerDetector;@@ -484,6 +487,10 @@ public class ContainerLifecycleManager {* SIGTERM before Docker sends SIGKILL. Must be a non-negative integer.*/public void stopAndRemove(String containerId, Closeable logStream, int stopTimeoutSeconds) {+ withTeardownSlot(() -> stopAndRemoveAdmitted(containerId, logStream, stopTimeoutSeconds));+ }++ private void stopAndRemoveAdmitted(String containerId, Closeable logStream, int stopTimeoutSeconds) {LOG.infov("Stopping container {0}", containerId);boolean stoppedOrMissing = false;@@ -537,6 +544,10 @@ public class ContainerLifecycleManager {* only once the container is gone; when removal fails it is left open for the retry.*/public void stopAndRemoveStrict(String containerId, Closeable logStream) {+ withTeardownSlot(() -> stopAndRemoveStrictAdmitted(containerId, logStream));+ }++ private void stopAndRemoveStrictAdmitted(String containerId, Closeable logStream) {LOG.infov("Stopping container {0}", containerId);Exception stopFailure = null;@@ -571,6 +582,15 @@ public class ContainerLifecycleManager {closeLogStreamIfPresent(logStream);}+ private void withTeardownSlot(Runnable stopAndRemove) {+ teardownSlots.acquireUninterruptibly();+ try {+ stopAndRemove.run();+ } finally {+ teardownSlots.release();+ }+ }+private void closeLogStreamIfPresent(Closeable logStream) {if (logStream != null) {closeLogStreamAfterContainerStop(logStream);diff --git a/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerTeardownTest.java b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerTeardownTest.javanew file mode 100644index 0000000000..a57b24b4ba--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/core/common/docker/ContainerLifecycleManagerTeardownTest.java@@ -0,0 +1,145 @@+package io.github.hectorvent.floci.core.common.docker;++import com.github.dockerjava.api.DockerClient;+import com.github.dockerjava.api.command.RemoveContainerCmd;+import com.github.dockerjava.api.command.StopContainerCmd;+import org.junit.jupiter.api.Test;++import java.util.ArrayList;+import java.util.List;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.Executors;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicBoolean;+import java.util.concurrent.atomic.AtomicReference;++import static io.github.hectorvent.floci.testing.ThreadStates.awaitBlockedOrDone;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNull;+import static org.junit.jupiter.api.Assertions.assertThrows;+import static org.junit.jupiter.api.Assertions.assertTrue;+import static org.mockito.ArgumentMatchers.anyInt;+import static org.mockito.ArgumentMatchers.anyString;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.never;+import static org.mockito.Mockito.verify;+import static org.mockito.Mockito.when;++class ContainerLifecycleManagerTeardownTest {++ @Test+ void interruptedWaitersStillCleanUpWithoutReleasingAnotherOwnersSlot() throws Exception {+ DockerClient docker = mock(DockerClient.class);+ ContainerLifecycleManager manager = new ContainerLifecycleManager(docker, null, null, null, null);+ StopContainerCmd stop = mock(StopContainerCmd.class);+ when(docker.stopContainerCmd(anyString())).thenReturn(stop);+ when(stop.withTimeout(anyInt())).thenReturn(stop);+ RemoveContainerCmd remove = mock(RemoveContainerCmd.class);+ when(docker.removeContainerCmd(anyString())).thenAnswer(invocation -> {+ if (invocation.<String>getArgument(0).startsWith("owned-")) {+ return remove;+ }+ RemoveContainerCmd waitingRemove = mock(RemoveContainerCmd.class);+ when(waitingRemove.withForce(true)).thenReturn(waitingRemove);+ return waitingRemove;+ });+ when(remove.withForce(true)).thenReturn(remove);+ CountDownLatch removing = new CountDownLatch(8);+ CountDownLatch release = new CountDownLatch(1);+ when(remove.exec()).thenAnswer(invocation -> {+ removing.countDown();+ assertTrue(release.await(30, TimeUnit.SECONDS));+ return null;+ });+ List<Thread> owners = new ArrayList<>();+ AtomicReference<Throwable> failure = new AtomicReference<>();+ for (int i = 0; i < 8; i++) {+ String id = "owned-" + i;+ boolean strict = i % 2 == 0;+ owners.add(Thread.ofVirtual().start(() -> {+ try {+ if (strict) {+ manager.stopAndRemoveStrict(id, null);+ } else {+ manager.stopAndRemove(id, null, 1);+ }+ } catch (Throwable problem) {+ failure.compareAndSet(null, problem);+ }+ }));+ }+ List<Thread> waiters = new ArrayList<>();+ List<AtomicBoolean> interrupted = new ArrayList<>();+ try {+ assertTrue(removing.await(10, TimeUnit.SECONDS));+ for (boolean strict : List.of(false, true)) {+ String id = "waiting-" + strict;+ AtomicBoolean preservedInterrupt = new AtomicBoolean();+ interrupted.add(preservedInterrupt);+ Thread waiter = Thread.ofVirtual().start(() -> {+ try {+ if (strict) {+ manager.stopAndRemoveStrict(id, null);+ } else {+ manager.stopAndRemove(id, null);+ }+ preservedInterrupt.set(Thread.currentThread().isInterrupted());+ } catch (Throwable problem) {+ failure.compareAndSet(null, problem);+ }+ });+ waiters.add(waiter);+ awaitBlockedOrDone(waiter);+ waiter.interrupt();+ assertTrue(waiter.isAlive());+ verify(docker, never()).stopContainerCmd(id);+ verify(docker, never()).removeContainerCmd(id);+ }+ } finally {+ release.countDown();+ for (Thread thread : owners) {+ thread.join(10000);+ assertFalse(thread.isAlive());+ }+ for (Thread thread : waiters) {+ thread.join(10000);+ assertFalse(thread.isAlive());+ }+ }+ assertNull(failure.get());+ for (AtomicBoolean preservedInterrupt : interrupted) {+ assertTrue(preservedInterrupt.get());+ }+ for (boolean strict : List.of(false, true)) {+ verify(docker).stopContainerCmd("waiting-" + strict);+ verify(docker).removeContainerCmd("waiting-" + strict);+ }+ }++ @Test+ void removalFailuresReleaseAdmissionForLaterOwners() throws Exception {+ DockerClient docker = mock(DockerClient.class);+ ContainerLifecycleManager manager = new ContainerLifecycleManager(docker, null, null, null, null);+ StopContainerCmd stop = mock(StopContainerCmd.class);+ when(docker.stopContainerCmd(anyString())).thenReturn(stop);+ when(stop.withTimeout(anyInt())).thenReturn(stop);+ RemoveContainerCmd remove = mock(RemoveContainerCmd.class);+ when(docker.removeContainerCmd(anyString())).thenReturn(remove);+ when(remove.withForce(true)).thenReturn(remove);+ when(remove.exec()).thenThrow(new IllegalStateException("daemon refused removal"));+ ExecutorService executor = Executors.newVirtualThreadPerTaskExecutor();+ try {+ executor.submit(() -> {+ for (int i = 0; i < 16; i++) {+ String id = "failed-" + i;+ assertThrows(IllegalStateException.class,+ () -> manager.stopAndRemoveStrict(id, null));+ }+ }).get(10, TimeUnit.SECONDS);+ } finally {+ executor.shutdownNow();+ }+ }++}