Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(lambda): keep watching for a runtime exit after the wait stream fails (#5284)

Hector Venturaa3027c07cf9d
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java
index 14818aaa2c..6881d76be3 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java
@@ -21,8 +21,6 @@ import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServer;
import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServerFactory;
import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.exception.NotFoundException;
-import com.github.dockerjava.api.model.WaitResponse;
-import com.github.dockerjava.core.command.WaitContainerResultCallback;
import jakarta.annotation.PostConstruct;
import jakarta.annotation.PreDestroy;
import jakarta.enterprise.context.ApplicationScoped;
@@ -1389,26 +1387,14 @@ public class ContainerLauncher implements LambdaRuntimeLauncher {
* invocation waiting out the full function timeout to be told {@code Function.TimedOut}
* instead of the {@code Runtime.ExitError} AWS reports immediately.
*
- * <p>Fires exactly once per container, on whatever exit eventually happens - including an
- * intentional {@code docker stop} during normal teardown. {@link RuntimeApiServer
- * #handleRuntimeProcessExited} itself distinguishes a genuine crash from that case (its own
- * {@code stopped}/{@code faulted} guard), so this method only needs to forward the event;
- * it does not need to be un-armed on the teardown path.
+ * <p>Fires once per container exit, including an intentional {@code docker stop} during
+ * normal teardown. {@link RuntimeApiServer#handleRuntimeProcessExited} itself distinguishes
+ * a genuine crash from that case (its own {@code stopped}/{@code faulted} guard). The watch
+ * survives its stream timing out on a quiet container; see {@link LambdaExitWatcher}.
*/
private void watchForUnexpectedExit(DockerClient dockerClient, String containerId,
RuntimeApiServer runtimeApiServer) {
- try {
- dockerClient.waitContainerCmd(containerId).exec(new WaitContainerResultCallback() {
- @Override
- public void onNext(WaitResponse response) {
- super.onNext(response);
- Integer statusCode = response.getStatusCode();
- runtimeApiServer.handleRuntimeProcessExited(statusCode != null ? statusCode : -1);
- }
- });
- } catch (RuntimeException e) {
- LOG.debugv(e, "Could not arm exit watcher for container {0}", containerId);
- }
+ LambdaExitWatcher.watch(dockerClient, containerId, runtimeApiServer::handleRuntimeProcessExited);
}
private void launchExtensions(DockerClient dockerClient, String containerId, String functionName,
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/LambdaExitWatcher.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/LambdaExitWatcher.java
new file mode 100644
index 0000000000..b40ecaf020
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/LambdaExitWatcher.java
@@ -0,0 +1,132 @@
+package io.github.hectorvent.floci.services.lambda.launcher;
+
+import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.command.InspectContainerResponse;
+import com.github.dockerjava.api.exception.NotFoundException;
+import com.github.dockerjava.api.model.WaitResponse;
+import com.github.dockerjava.core.command.WaitContainerResultCallback;
+import org.jboss.logging.Logger;
+
+import java.time.Duration;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+import java.util.function.BiConsumer;
+import java.util.function.IntConsumer;
+import java.util.function.LongSupplier;
+
+/**
+ * Reports when a Lambda container's main process exits, through Docker's wait-for-exit stream.
+ *
+ * <p>The stream sends nothing until the container exits, so on a quiet container it ends with a read
+ * timeout (the Docker client's response timeout) while the container still runs. A watch that simply
+ * stopped there would miss every later crash, turning a {@code Runtime.ExitError} into a
+ * {@code Function.TimedOut}. When the stream ends with an error, the container is inspected instead:
+ * still running re-arms the watch, already exited reports the inspected exit code, and gone (removed
+ * by teardown) reports nothing.
+ *
+ * <p>A stream that fails soon after it was armed, or an inspect that fails, usually means the daemon
+ * itself is unreachable (a dropped connection, a restart). Those re-arm after a delay that doubles each
+ * time, so the watch outlasts a short outage instead of spending its attempts within milliseconds; after
+ * {@link #MAX_QUICK_FAILURES} in a row it stops. A stream that lived longer than {@link #QUICK_FAILURE}
+ * resets the count and re-arms at once.
+ */
+final class LambdaExitWatcher {
+
+ private static final Logger LOG = Logger.getLogger(LambdaExitWatcher.class);
+
+ /** A stream that fails sooner than this after it was armed counts as a quick failure. */
+ static final Duration QUICK_FAILURE = Duration.ofSeconds(5);
+ /** Consecutive quick failures tolerated before the watch gives up. */
+ static final int MAX_QUICK_FAILURES = 5;
+ /** Delay before re-arming after the first quick failure; it doubles with each further one. */
+ static final Duration FIRST_RETRY_DELAY = Duration.ofSeconds(1);
+
+ private final DockerClient dockerClient;
+ private final String containerId;
+ private final IntConsumer onExit;
+ private final BiConsumer<Duration, Runnable> scheduler;
+ private final LongSupplier nanoClock;
+ private int quickFailures;
+
+ LambdaExitWatcher(DockerClient dockerClient, String containerId, IntConsumer onExit,
+ BiConsumer<Duration, Runnable> scheduler, LongSupplier nanoClock) {
+ this.dockerClient = dockerClient;
+ this.containerId = containerId;
+ this.onExit = onExit;
+ this.scheduler = scheduler;
+ this.nanoClock = nanoClock;
+ }
+
+ /** Arms a watch that calls {@code onExit} with the exit status, at most once per exit. */
+ static void watch(DockerClient dockerClient, String containerId, IntConsumer onExit) {
+ new LambdaExitWatcher(dockerClient, containerId, onExit, LambdaExitWatcher::schedule, System::nanoTime)
+ .arm();
+ }
+
+ private static void schedule(Duration delay, Runnable task) {
+ CompletableFuture.delayedExecutor(delay.toMillis(), TimeUnit.MILLISECONDS).execute(task);
+ }
+
+ void arm() {
+ long armedAt = nanoClock.getAsLong();
+ try {
+ dockerClient.waitContainerCmd(containerId).exec(new WaitContainerResultCallback() {
+ @Override
+ public void onNext(WaitResponse response) {
+ super.onNext(response);
+ Integer statusCode = response.getStatusCode();
+ onExit.accept(statusCode != null ? statusCode : -1);
+ }
+
+ @Override
+ public void onError(Throwable throwable) {
+ super.onError(throwable);
+ streamFailed(throwable, armedAt);
+ }
+ });
+ } catch (RuntimeException e) {
+ LOG.debugv(e, "Could not arm exit watcher for container {0}", containerId);
+ reArm(e, armedAt);
+ }
+ }
+
+ private void streamFailed(Throwable cause, long armedAt) {
+ InspectContainerResponse.ContainerState state;
+ try {
+ state = dockerClient.inspectContainerCmd(containerId).exec().getState();
+ } catch (NotFoundException e) {
+ LOG.debugv("Exit watcher for container {0} stopped: the container is gone", containerId);
+ return;
+ } catch (RuntimeException e) {
+ LOG.debugv(e, "Exit watcher for container {0} could not inspect the container", containerId);
+ reArm(e, armedAt);
+ return;
+ }
+ if (state == null || !Boolean.TRUE.equals(state.getRunning())) {
+ Long exitCode = state == null ? null : state.getExitCodeLong();
+ onExit.accept(exitCode != null ? exitCode.intValue() : -1);
+ return;
+ }
+ reArm(cause, armedAt);
+ }
+
+ private void reArm(Throwable cause, long armedAt) {
+ boolean quick = nanoClock.getAsLong() - armedAt < QUICK_FAILURE.toNanos();
+ quickFailures = quick ? quickFailures + 1 : 0;
+ if (quickFailures > MAX_QUICK_FAILURES) {
+ LOG.warnv(cause, "Exit watcher for container {0} gave up after {1} quick failures; a later exit is unreported",
+ containerId, quickFailures);
+ return;
+ }
+ if (quickFailures == 0) {
+ LOG.debugv("Exit watcher stream for running container {0} ended ({1}); re-arming",
+ containerId, cause.toString());
+ arm();
+ return;
+ }
+ Duration delay = FIRST_RETRY_DELAY.multipliedBy(1L << (quickFailures - 1));
+ LOG.debugv("Exit watcher for container {0} failed quickly ({1}); re-arming in {2} ms",
+ containerId, cause.toString(), delay.toMillis());
+ scheduler.accept(delay, this::arm);
+ }
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/LambdaExitWatcherTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/LambdaExitWatcherTest.java
new file mode 100644
index 0000000000..37159c48ab
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/LambdaExitWatcherTest.java
@@ -0,0 +1,181 @@
+package io.github.hectorvent.floci.services.lambda.launcher;
+
+import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.command.InspectContainerCmd;
+import com.github.dockerjava.api.command.InspectContainerResponse;
+import com.github.dockerjava.api.command.WaitContainerCmd;
+import com.github.dockerjava.api.exception.NotFoundException;
+import com.github.dockerjava.api.model.WaitResponse;
+import com.github.dockerjava.core.command.WaitContainerResultCallback;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+import java.net.SocketTimeoutException;
+import java.time.Duration;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+class LambdaExitWatcherTest {
+
+ private static final String CONTAINER = "lambda-container";
+ private static final long LONG_LIVED_NANOS = LambdaExitWatcher.QUICK_FAILURE.toNanos() + 1;
+
+ private final List<WaitContainerResultCallback> armed = new ArrayList<>();
+ private final List<Integer> exits = new ArrayList<>();
+ private final List<Duration> delays = new ArrayList<>();
+ private long now;
+ private DockerClient docker;
+ private InspectContainerCmd inspect;
+
+ @BeforeEach
+ void setUp() {
+ docker = mock(DockerClient.class);
+ WaitContainerCmd wait = mock(WaitContainerCmd.class);
+ when(docker.waitContainerCmd(CONTAINER)).thenReturn(wait);
+ when(wait.exec(any())).thenAnswer(invocation -> {
+ WaitContainerResultCallback callback = invocation.getArgument(0);
+ armed.add(callback);
+ return callback;
+ });
+ inspect = mock(InspectContainerCmd.class);
+ when(docker.inspectContainerCmd(CONTAINER)).thenReturn(inspect);
+ }
+
+ /** A watch whose clock is {@link #now} and whose scheduler records each delay and runs the task at once. */
+ private void watch() {
+ new LambdaExitWatcher(docker, CONTAINER, exits::add, (delay, task) -> {
+ delays.add(delay);
+ task.run();
+ }, () -> now).arm();
+ }
+
+ private InspectContainerResponse state(Boolean running, Long exitCode) {
+ InspectContainerResponse response = mock(InspectContainerResponse.class);
+ InspectContainerResponse.ContainerState state = mock(InspectContainerResponse.ContainerState.class);
+ when(state.getRunning()).thenReturn(running);
+ when(state.getExitCodeLong()).thenReturn(exitCode);
+ when(response.getState()).thenReturn(state);
+ return response;
+ }
+
+ private void containerState(Boolean running, Long exitCode) {
+ InspectContainerResponse response = state(running, exitCode);
+ when(inspect.exec()).thenReturn(response);
+ }
+
+ private static WaitResponse exited(int status) {
+ WaitResponse response = mock(WaitResponse.class);
+ when(response.getStatusCode()).thenReturn(status);
+ return response;
+ }
+
+ private void failLatestStream(String message) {
+ armed.get(armed.size() - 1).onError(new SocketTimeoutException(message));
+ }
+
+ private static List<Duration> seconds(long... values) {
+ List<Duration> result = new ArrayList<>();
+ for (long value : values) {
+ result.add(Duration.ofSeconds(value));
+ }
+ return result;
+ }
+
+ // Catches: a watch that stops when its stream times out on a quiet container, so a crash after
+ // the Docker client's response timeout is reported as Function.TimedOut.
+ @Test
+ void aStreamThatTimesOutOnARunningContainerIsReArmedAndStillReportsTheExit() {
+ containerState(true, null);
+ watch();
+
+ now += LONG_LIVED_NANOS;
+ failLatestStream("Read timed out");
+ assertEquals(2, armed.size());
+ assertEquals(List.of(), delays);
+ assertEquals(List.of(), exits);
+
+ armed.get(1).onNext(exited(137));
+ assertEquals(List.of(137), exits);
+ }
+
+ @Test
+ void aStreamThatFailsAfterTheContainerExitedReportsTheInspectedExitCode() {
+ containerState(false, 2L);
+ watch();
+
+ failLatestStream("Read timed out");
+
+ assertEquals(1, armed.size());
+ assertEquals(List.of(2), exits);
+ }
+
+ @Test
+ void aStreamThatFailsAfterTheContainerWasRemovedReportsNothing() {
+ when(inspect.exec()).thenThrow(new NotFoundException("No such container"));
+ watch();
+
+ failLatestStream("Read timed out");
+
+ assertEquals(1, armed.size());
+ assertEquals(List.of(), exits);
+ }
+
+ // Catches: quick re-arms spent back to back, which exhaust the budget within milliseconds of the
+ // daemon going away instead of riding out a short outage.
+ @Test
+ void quickFailuresReArmAfterADoublingDelayThenStop() {
+ containerState(true, null);
+ watch();
+
+ for (int i = 0; i <= LambdaExitWatcher.MAX_QUICK_FAILURES; i++) {
+ failLatestStream("Connection reset");
+ }
+
+ assertEquals(seconds(1, 2, 4, 8, 16), delays);
+ assertEquals(LambdaExitWatcher.MAX_QUICK_FAILURES + 1, armed.size());
+ assertEquals(List.of(), exits);
+ }
+
+ // Catches: a failed inspect ending the watch for good, so a daemon restart (which stops the
+ // container) loses the Runtime.ExitError the next inspect would report.
+ @Test
+ void anInspectThatFailsIsRetriedAndThenReportsTheExit() {
+ InspectContainerResponse stopped = state(false, 137L);
+ when(inspect.exec())
+ .thenThrow(new RuntimeException("Cannot connect to the Docker daemon"))
+ .thenReturn(stopped);
+ watch();
+
+ failLatestStream("Connection reset");
+ assertEquals(seconds(1), delays);
+ assertEquals(2, armed.size());
+
+ failLatestStream("Connection reset");
+ assertEquals(List.of(137), exits);
+ }
+
+ // Catches: the quick-failure count surviving a stream that lived past the threshold, so a few
+ // unrelated blips spread over a long-running container would end the watch.
+ @Test
+ void aLongLivedStreamResetsTheQuickFailureCount() {
+ containerState(true, null);
+ watch();
+
+ failLatestStream("Connection reset");
+ failLatestStream("Connection reset");
+ assertEquals(seconds(1, 2), delays);
+
+ now += LONG_LIVED_NANOS;
+ failLatestStream("Read timed out");
+ assertEquals(seconds(1, 2), delays);
+
+ failLatestStream("Connection reset");
+ assertEquals(seconds(1, 2, 1), delays);
+ assertEquals(5, armed.size());
+ }
+}