Light, fluffy, and always free - The AWS Local Emulator alternative
fix(lambda): keep watching for a runtime exit after the wait stream fails (#5284)
Hector Venturaa3027c07cf9d
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.javaindex 14818aaa2c..6881d76be3 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java@@ -21,8 +21,6 @@ import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServer;import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServerFactory;import com.github.dockerjava.api.DockerClient;import com.github.dockerjava.api.exception.NotFoundException;-import com.github.dockerjava.api.model.WaitResponse;-import com.github.dockerjava.core.command.WaitContainerResultCallback;import jakarta.annotation.PostConstruct;import jakarta.annotation.PreDestroy;import jakarta.enterprise.context.ApplicationScoped;@@ -1389,26 +1387,14 @@ public class ContainerLauncher implements LambdaRuntimeLauncher {* invocation waiting out the full function timeout to be told {@code Function.TimedOut}* instead of the {@code Runtime.ExitError} AWS reports immediately.*- * <p>Fires exactly once per container, on whatever exit eventually happens - including an- * intentional {@code docker stop} during normal teardown. {@link RuntimeApiServer- * #handleRuntimeProcessExited} itself distinguishes a genuine crash from that case (its own- * {@code stopped}/{@code faulted} guard), so this method only needs to forward the event;- * it does not need to be un-armed on the teardown path.+ * <p>Fires once per container exit, including an intentional {@code docker stop} during+ * normal teardown. {@link RuntimeApiServer#handleRuntimeProcessExited} itself distinguishes+ * a genuine crash from that case (its own {@code stopped}/{@code faulted} guard). The watch+ * survives its stream timing out on a quiet container; see {@link LambdaExitWatcher}.*/private void watchForUnexpectedExit(DockerClient dockerClient, String containerId,RuntimeApiServer runtimeApiServer) {- try {- dockerClient.waitContainerCmd(containerId).exec(new WaitContainerResultCallback() {- @Override- public void onNext(WaitResponse response) {- super.onNext(response);- Integer statusCode = response.getStatusCode();- runtimeApiServer.handleRuntimeProcessExited(statusCode != null ? statusCode : -1);- }- });- } catch (RuntimeException e) {- LOG.debugv(e, "Could not arm exit watcher for container {0}", containerId);- }+ LambdaExitWatcher.watch(dockerClient, containerId, runtimeApiServer::handleRuntimeProcessExited);}private void launchExtensions(DockerClient dockerClient, String containerId, String functionName,diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/LambdaExitWatcher.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/LambdaExitWatcher.javanew file mode 100644index 0000000000..b40ecaf020--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/LambdaExitWatcher.java@@ -0,0 +1,132 @@+package io.github.hectorvent.floci.services.lambda.launcher;++import com.github.dockerjava.api.DockerClient;+import com.github.dockerjava.api.command.InspectContainerResponse;+import com.github.dockerjava.api.exception.NotFoundException;+import com.github.dockerjava.api.model.WaitResponse;+import com.github.dockerjava.core.command.WaitContainerResultCallback;+import org.jboss.logging.Logger;++import java.time.Duration;+import java.util.concurrent.CompletableFuture;+import java.util.concurrent.TimeUnit;+import java.util.function.BiConsumer;+import java.util.function.IntConsumer;+import java.util.function.LongSupplier;++/**+ * Reports when a Lambda container's main process exits, through Docker's wait-for-exit stream.+ *+ * <p>The stream sends nothing until the container exits, so on a quiet container it ends with a read+ * timeout (the Docker client's response timeout) while the container still runs. A watch that simply+ * stopped there would miss every later crash, turning a {@code Runtime.ExitError} into a+ * {@code Function.TimedOut}. When the stream ends with an error, the container is inspected instead:+ * still running re-arms the watch, already exited reports the inspected exit code, and gone (removed+ * by teardown) reports nothing.+ *+ * <p>A stream that fails soon after it was armed, or an inspect that fails, usually means the daemon+ * itself is unreachable (a dropped connection, a restart). Those re-arm after a delay that doubles each+ * time, so the watch outlasts a short outage instead of spending its attempts within milliseconds; after+ * {@link #MAX_QUICK_FAILURES} in a row it stops. A stream that lived longer than {@link #QUICK_FAILURE}+ * resets the count and re-arms at once.+ */+final class LambdaExitWatcher {++ private static final Logger LOG = Logger.getLogger(LambdaExitWatcher.class);++ /** A stream that fails sooner than this after it was armed counts as a quick failure. */+ static final Duration QUICK_FAILURE = Duration.ofSeconds(5);+ /** Consecutive quick failures tolerated before the watch gives up. */+ static final int MAX_QUICK_FAILURES = 5;+ /** Delay before re-arming after the first quick failure; it doubles with each further one. */+ static final Duration FIRST_RETRY_DELAY = Duration.ofSeconds(1);++ private final DockerClient dockerClient;+ private final String containerId;+ private final IntConsumer onExit;+ private final BiConsumer<Duration, Runnable> scheduler;+ private final LongSupplier nanoClock;+ private int quickFailures;++ LambdaExitWatcher(DockerClient dockerClient, String containerId, IntConsumer onExit,+ BiConsumer<Duration, Runnable> scheduler, LongSupplier nanoClock) {+ this.dockerClient = dockerClient;+ this.containerId = containerId;+ this.onExit = onExit;+ this.scheduler = scheduler;+ this.nanoClock = nanoClock;+ }++ /** Arms a watch that calls {@code onExit} with the exit status, at most once per exit. */+ static void watch(DockerClient dockerClient, String containerId, IntConsumer onExit) {+ new LambdaExitWatcher(dockerClient, containerId, onExit, LambdaExitWatcher::schedule, System::nanoTime)+ .arm();+ }++ private static void schedule(Duration delay, Runnable task) {+ CompletableFuture.delayedExecutor(delay.toMillis(), TimeUnit.MILLISECONDS).execute(task);+ }++ void arm() {+ long armedAt = nanoClock.getAsLong();+ try {+ dockerClient.waitContainerCmd(containerId).exec(new WaitContainerResultCallback() {+ @Override+ public void onNext(WaitResponse response) {+ super.onNext(response);+ Integer statusCode = response.getStatusCode();+ onExit.accept(statusCode != null ? statusCode : -1);+ }++ @Override+ public void onError(Throwable throwable) {+ super.onError(throwable);+ streamFailed(throwable, armedAt);+ }+ });+ } catch (RuntimeException e) {+ LOG.debugv(e, "Could not arm exit watcher for container {0}", containerId);+ reArm(e, armedAt);+ }+ }++ private void streamFailed(Throwable cause, long armedAt) {+ InspectContainerResponse.ContainerState state;+ try {+ state = dockerClient.inspectContainerCmd(containerId).exec().getState();+ } catch (NotFoundException e) {+ LOG.debugv("Exit watcher for container {0} stopped: the container is gone", containerId);+ return;+ } catch (RuntimeException e) {+ LOG.debugv(e, "Exit watcher for container {0} could not inspect the container", containerId);+ reArm(e, armedAt);+ return;+ }+ if (state == null || !Boolean.TRUE.equals(state.getRunning())) {+ Long exitCode = state == null ? null : state.getExitCodeLong();+ onExit.accept(exitCode != null ? exitCode.intValue() : -1);+ return;+ }+ reArm(cause, armedAt);+ }++ private void reArm(Throwable cause, long armedAt) {+ boolean quick = nanoClock.getAsLong() - armedAt < QUICK_FAILURE.toNanos();+ quickFailures = quick ? quickFailures + 1 : 0;+ if (quickFailures > MAX_QUICK_FAILURES) {+ LOG.warnv(cause, "Exit watcher for container {0} gave up after {1} quick failures; a later exit is unreported",+ containerId, quickFailures);+ return;+ }+ if (quickFailures == 0) {+ LOG.debugv("Exit watcher stream for running container {0} ended ({1}); re-arming",+ containerId, cause.toString());+ arm();+ return;+ }+ Duration delay = FIRST_RETRY_DELAY.multipliedBy(1L << (quickFailures - 1));+ LOG.debugv("Exit watcher for container {0} failed quickly ({1}); re-arming in {2} ms",+ containerId, cause.toString(), delay.toMillis());+ scheduler.accept(delay, this::arm);+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/LambdaExitWatcherTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/LambdaExitWatcherTest.javanew file mode 100644index 0000000000..37159c48ab--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/LambdaExitWatcherTest.java@@ -0,0 +1,181 @@+package io.github.hectorvent.floci.services.lambda.launcher;++import com.github.dockerjava.api.DockerClient;+import com.github.dockerjava.api.command.InspectContainerCmd;+import com.github.dockerjava.api.command.InspectContainerResponse;+import com.github.dockerjava.api.command.WaitContainerCmd;+import com.github.dockerjava.api.exception.NotFoundException;+import com.github.dockerjava.api.model.WaitResponse;+import com.github.dockerjava.core.command.WaitContainerResultCallback;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;++import java.net.SocketTimeoutException;+import java.time.Duration;+import java.util.ArrayList;+import java.util.List;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.mockito.ArgumentMatchers.any;+import static org.mockito.Mockito.mock;+import static org.mockito.Mockito.when;++class LambdaExitWatcherTest {++ private static final String CONTAINER = "lambda-container";+ private static final long LONG_LIVED_NANOS = LambdaExitWatcher.QUICK_FAILURE.toNanos() + 1;++ private final List<WaitContainerResultCallback> armed = new ArrayList<>();+ private final List<Integer> exits = new ArrayList<>();+ private final List<Duration> delays = new ArrayList<>();+ private long now;+ private DockerClient docker;+ private InspectContainerCmd inspect;++ @BeforeEach+ void setUp() {+ docker = mock(DockerClient.class);+ WaitContainerCmd wait = mock(WaitContainerCmd.class);+ when(docker.waitContainerCmd(CONTAINER)).thenReturn(wait);+ when(wait.exec(any())).thenAnswer(invocation -> {+ WaitContainerResultCallback callback = invocation.getArgument(0);+ armed.add(callback);+ return callback;+ });+ inspect = mock(InspectContainerCmd.class);+ when(docker.inspectContainerCmd(CONTAINER)).thenReturn(inspect);+ }++ /** A watch whose clock is {@link #now} and whose scheduler records each delay and runs the task at once. */+ private void watch() {+ new LambdaExitWatcher(docker, CONTAINER, exits::add, (delay, task) -> {+ delays.add(delay);+ task.run();+ }, () -> now).arm();+ }++ private InspectContainerResponse state(Boolean running, Long exitCode) {+ InspectContainerResponse response = mock(InspectContainerResponse.class);+ InspectContainerResponse.ContainerState state = mock(InspectContainerResponse.ContainerState.class);+ when(state.getRunning()).thenReturn(running);+ when(state.getExitCodeLong()).thenReturn(exitCode);+ when(response.getState()).thenReturn(state);+ return response;+ }++ private void containerState(Boolean running, Long exitCode) {+ InspectContainerResponse response = state(running, exitCode);+ when(inspect.exec()).thenReturn(response);+ }++ private static WaitResponse exited(int status) {+ WaitResponse response = mock(WaitResponse.class);+ when(response.getStatusCode()).thenReturn(status);+ return response;+ }++ private void failLatestStream(String message) {+ armed.get(armed.size() - 1).onError(new SocketTimeoutException(message));+ }++ private static List<Duration> seconds(long... values) {+ List<Duration> result = new ArrayList<>();+ for (long value : values) {+ result.add(Duration.ofSeconds(value));+ }+ return result;+ }++ // Catches: a watch that stops when its stream times out on a quiet container, so a crash after+ // the Docker client's response timeout is reported as Function.TimedOut.+ @Test+ void aStreamThatTimesOutOnARunningContainerIsReArmedAndStillReportsTheExit() {+ containerState(true, null);+ watch();++ now += LONG_LIVED_NANOS;+ failLatestStream("Read timed out");+ assertEquals(2, armed.size());+ assertEquals(List.of(), delays);+ assertEquals(List.of(), exits);++ armed.get(1).onNext(exited(137));+ assertEquals(List.of(137), exits);+ }++ @Test+ void aStreamThatFailsAfterTheContainerExitedReportsTheInspectedExitCode() {+ containerState(false, 2L);+ watch();++ failLatestStream("Read timed out");++ assertEquals(1, armed.size());+ assertEquals(List.of(2), exits);+ }++ @Test+ void aStreamThatFailsAfterTheContainerWasRemovedReportsNothing() {+ when(inspect.exec()).thenThrow(new NotFoundException("No such container"));+ watch();++ failLatestStream("Read timed out");++ assertEquals(1, armed.size());+ assertEquals(List.of(), exits);+ }++ // Catches: quick re-arms spent back to back, which exhaust the budget within milliseconds of the+ // daemon going away instead of riding out a short outage.+ @Test+ void quickFailuresReArmAfterADoublingDelayThenStop() {+ containerState(true, null);+ watch();++ for (int i = 0; i <= LambdaExitWatcher.MAX_QUICK_FAILURES; i++) {+ failLatestStream("Connection reset");+ }++ assertEquals(seconds(1, 2, 4, 8, 16), delays);+ assertEquals(LambdaExitWatcher.MAX_QUICK_FAILURES + 1, armed.size());+ assertEquals(List.of(), exits);+ }++ // Catches: a failed inspect ending the watch for good, so a daemon restart (which stops the+ // container) loses the Runtime.ExitError the next inspect would report.+ @Test+ void anInspectThatFailsIsRetriedAndThenReportsTheExit() {+ InspectContainerResponse stopped = state(false, 137L);+ when(inspect.exec())+ .thenThrow(new RuntimeException("Cannot connect to the Docker daemon"))+ .thenReturn(stopped);+ watch();++ failLatestStream("Connection reset");+ assertEquals(seconds(1), delays);+ assertEquals(2, armed.size());++ failLatestStream("Connection reset");+ assertEquals(List.of(137), exits);+ }++ // Catches: the quick-failure count surviving a stream that lived past the threshold, so a few+ // unrelated blips spread over a long-running container would end the watch.+ @Test+ void aLongLivedStreamResetsTheQuickFailureCount() {+ containerState(true, null);+ watch();++ failLatestStream("Connection reset");+ failLatestStream("Connection reset");+ assertEquals(seconds(1, 2), delays);++ now += LONG_LIVED_NANOS;+ failLatestStream("Read timed out");+ assertEquals(seconds(1, 2), delays);++ failLatestStream("Connection reset");+ assertEquals(seconds(1, 2, 1), delays);+ assertEquals(5, armed.size());+ }+}