Light, fluffy, and always free - The AWS Local Emulator alternative
fix(init-hooks): terminate hook process trees on timeout (#3372)
Niels Kootstrad99f162eb6e2
diff --git a/src/main/java/io/github/hectorvent/floci/lifecycle/inithook/HookScriptExecutor.java b/src/main/java/io/github/hectorvent/floci/lifecycle/inithook/HookScriptExecutor.javaindex ee7fb12d19..b30bf46004 100644--- a/src/main/java/io/github/hectorvent/floci/lifecycle/inithook/HookScriptExecutor.java+++ b/src/main/java/io/github/hectorvent/floci/lifecycle/inithook/HookScriptExecutor.java@@ -7,7 +7,11 @@ import org.jboss.logging.Logger;import java.io.File;import java.io.IOException;+import java.util.ArrayList;+import java.util.List;+import java.util.concurrent.ExecutionException;import java.util.concurrent.TimeUnit;+import java.util.concurrent.TimeoutException;@ApplicationScopedpublic class HookScriptExecutor {@@ -57,23 +61,91 @@ public class HookScriptExecutor {} finally {if (process.isAlive()) {LOG.debugv("Hook script process still alive during cleanup, forcing termination: {0}", scriptFileName);- process.destroyForcibly();+ forceTerminateProcessTree(process);}}}private void terminateProcess(final Process process, final String scriptFileName) throws InterruptedException {// Try a graceful shutdown first, then force termination if the process does not exit in time.+ List<ProcessHandle> descendants = processDescendants(process);+ descendants.forEach(ProcessHandle::destroy);process.destroy();- if (process.isAlive()) {- final long shutdownGracePeriodSeconds = initHooksConfig.shutdownGracePeriodSeconds();- final boolean terminatedGracefully = process.waitFor(shutdownGracePeriodSeconds, TimeUnit.SECONDS);- if (!terminatedGracefully) {- LOG.debugv("Hook script process did not terminate gracefully, forcing termination: {0}", scriptFileName);- process.destroyForcibly();- process.waitFor(shutdownGracePeriodSeconds, TimeUnit.SECONDS);+ final long shutdownGracePeriodSeconds = initHooksConfig.shutdownGracePeriodSeconds();+ final long gracefulDeadline = deadlineNanos(shutdownGracePeriodSeconds);+ final boolean terminatedGracefully = waitForProcess(process, gracefulDeadline);+ final boolean descendantsTerminated = waitForDescendants(descendants, gracefulDeadline);+ if (!terminatedGracefully || !descendantsTerminated) {+ LOG.debugv("Hook script process tree did not terminate gracefully, forcing termination: {0}",+ scriptFileName);+ process.destroyForcibly();+ List<ProcessHandle> remainingDescendants = new ArrayList<>(descendants);+ remainingDescendants.addAll(processDescendants(process));+ remainingDescendants.forEach(ProcessHandle::destroyForcibly);+ final long forceDeadline = deadlineNanos(shutdownGracePeriodSeconds);+ waitForProcess(process, forceDeadline);+ waitForDescendants(remainingDescendants, forceDeadline);+ }+ }++ private List<ProcessHandle> processDescendants(final Process process) {+ ProcessHandle handle = process.toHandle();+ return handle == null ? List.of() : handle.descendants().toList();+ }++ private boolean waitForDescendants(List<ProcessHandle> descendants, long deadlineNanos)+ throws InterruptedException {+ for (ProcessHandle descendant : descendants) {+ if (!waitForExit(descendant, deadlineNanos)) {+ return false;}}+ return true;+ }++ private boolean waitForExit(ProcessHandle process, long deadlineNanos) throws InterruptedException {+ if (!process.isAlive()) {+ return true;+ }+ long remainingNanos = deadlineNanos - System.nanoTime();+ if (remainingNanos <= 0) {+ return false;+ }+ try {+ process.onExit().get(remainingNanos, TimeUnit.NANOSECONDS);+ return true;+ } catch (TimeoutException e) {+ return false;+ } catch (ExecutionException e) {+ return !process.isAlive();+ }+ }++ private boolean waitForProcess(Process process, long deadlineNanos) throws InterruptedException {+ if (!process.isAlive()) {+ return true;+ }+ long remainingNanos = deadlineNanos - System.nanoTime();+ return remainingNanos > 0 && process.waitFor(remainingNanos, TimeUnit.NANOSECONDS);+ }++ private long deadlineNanos(long timeoutSeconds) {+ return System.nanoTime() + TimeUnit.SECONDS.toNanos(timeoutSeconds);+ }++ private void forceTerminateProcessTree(Process process) {+ final long shutdownGracePeriodSeconds = initHooksConfig.shutdownGracePeriodSeconds();+ final long deadline = deadlineNanos(shutdownGracePeriodSeconds);+ List<ProcessHandle> descendants = new ArrayList<>(processDescendants(process));+ process.destroyForcibly();+ descendants.addAll(processDescendants(process));+ descendants.forEach(ProcessHandle::destroyForcibly);+ try {+ waitForProcess(process, deadline);+ waitForDescendants(descendants, deadline);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ }}}diff --git a/src/test/java/io/github/hectorvent/floci/lifecycle/inithook/HookScriptExecutorTest.java b/src/test/java/io/github/hectorvent/floci/lifecycle/inithook/HookScriptExecutorTest.javaindex 45ebb7b6aa..8e0af798f0 100644--- a/src/test/java/io/github/hectorvent/floci/lifecycle/inithook/HookScriptExecutorTest.java+++ b/src/test/java/io/github/hectorvent/floci/lifecycle/inithook/HookScriptExecutorTest.java@@ -13,13 +13,17 @@ import org.mockito.junit.jupiter.MockitoExtension;import java.io.File;import java.io.IOException;+import java.nio.file.Files;+import java.nio.file.Path;import java.util.concurrent.TimeUnit;-import static org.mockito.Mockito.times;@ExtendWith(MockitoExtension.class)class HookScriptExecutorTest {+ @org.junit.jupiter.api.io.TempDir+ Path tempDirectory;+@Mock(answer = Answers.RETURNS_DEEP_STUBS)private EmulatorConfig emulatorConfigMock;@@ -74,12 +78,12 @@ class HookScriptExecutorTest {Mockito.when(emulatorConfigMock.initHooks().shutdownGracePeriodSeconds()).thenReturn(2L);Mockito.when(process.waitFor(30L, TimeUnit.SECONDS)).thenReturn(false);Mockito.when(process.isAlive()).thenReturn(true, false);- Mockito.when(process.waitFor(2L, TimeUnit.SECONDS)).thenReturn(false);+ Mockito.when(process.waitFor(Mockito.anyLong(), Mockito.eq(TimeUnit.NANOSECONDS))).thenReturn(false);IllegalStateException exception = Assertions.assertThrows(IllegalStateException.class, () -> hookScriptExecutor.run(process, "script.sh"));Mockito.verify(process).destroy();- Mockito.verify(process, times(2)).waitFor(2L, TimeUnit.SECONDS);+ Mockito.verify(process).waitFor(Mockito.anyLong(), Mockito.eq(TimeUnit.NANOSECONDS));Mockito.verify(process).destroyForcibly();Assertions.assertAll(@@ -97,7 +101,7 @@ class HookScriptExecutorTest {Mockito.when(emulatorConfigMock.initHooks().shutdownGracePeriodSeconds()).thenReturn(2L);Mockito.when(process.waitFor(30L, TimeUnit.SECONDS)).thenReturn(false);Mockito.when(process.isAlive()).thenReturn(true, false);- Mockito.when(process.waitFor(2L, TimeUnit.SECONDS)).thenReturn(true);+ Mockito.when(process.waitFor(Mockito.anyLong(), Mockito.eq(TimeUnit.NANOSECONDS))).thenReturn(true);IllegalStateException exception = Assertions.assertThrows(IllegalStateException.class, () -> hookScriptExecutor.run(process, "script.sh"));@@ -153,7 +157,7 @@ class HookScriptExecutorTest {IllegalStateException exception = Assertions.assertThrows(IllegalStateException.class, () -> hookScriptExecutor.run(process, "script.sh"));Mockito.verify(process).destroy();- Mockito.verify(process, Mockito.never()).waitFor(2L, TimeUnit.SECONDS);+ Mockito.verify(process, Mockito.never()).waitFor(Mockito.anyLong(), Mockito.eq(TimeUnit.NANOSECONDS));Mockito.verify(process, Mockito.never()).destroyForcibly();Assertions.assertAll(@@ -162,6 +166,33 @@ class HookScriptExecutorTest {);}+ @Test+ @DisplayName("Should terminate hook descendants when the timeout expires")+ void shouldTerminateHookDescendantsWhenTheTimeoutExpires() throws Exception {+ Mockito.when(emulatorConfigMock.initHooks().timeoutSeconds()).thenReturn(1L);+ Mockito.when(emulatorConfigMock.initHooks().shutdownGracePeriodSeconds()).thenReturn(1L);+ Mockito.when(emulatorConfigMock.initHooks().shellExecutable()).thenReturn("/bin/sh");++ Path childPidFile = tempDirectory.resolve("child.pid");+ Path script = tempDirectory.resolve("timeout.sh");+ Files.writeString(script, "(sleep 30) &\n"+ + "echo $! > child.pid\n"+ + "wait\n");++ Assertions.assertThrows(IllegalStateException.class, () -> hookScriptExecutor.run(script.toFile()));++ long childPid = Long.parseLong(Files.readString(childPidFile).trim());+ try {+ Assertions.assertFalse(ProcessHandle.of(childPid).map(ProcessHandle::isAlive).orElse(false));+ } finally {+ ProcessHandle.of(childPid).ifPresent(handle -> {+ if (handle.isAlive()) {+ handle.destroyForcibly();+ }+ });+ }+ }+@Test@DisplayName("Should throw IOException when shell executable does not exist")void shouldThrowIOExceptionWhenShellExecutableDoesNotExist() {