Light, fluffy, and always free - The AWS Local Emulator alternative
fix(lambdaweb): never run a request twice when the app crashes (#5842)
TATSUNO “Taz” Yasuhiroc8de16ffc7ca
diff --git a/docs/services/lambda-web-functions.md b/docs/services/lambda-web-functions.mdindex 7ab7f4114d..3704ecb80c 100644--- a/docs/services/lambda-web-functions.md+++ b/docs/services/lambda-web-functions.md@@ -84,7 +84,7 @@ The platform answers these cases itself, with the AWS HTML pages and `lambda-web| 429 | `Platform.EndpointRequestThrottle` | The requests exceed the endpoint's `rateLimit` || 502 | `Application.TimeoutListenError`, `Application.InitCodeError`, `Runtime.Exit` | The server did not listen within 10 seconds, threw at import, or exited at start || 503 | `Platform.EndpointDisabled` | The endpoint's `rateLimit` is 0 |-| 503 | `Runtime.OriginRequestError` | The process died while handling the request |+| 503 | `Runtime.OriginRequestError` | The process died while handling the request, or tens of milliseconds before it || 504 | `Runtime.RequestTimeout` | No response within the revision's `timeoutSeconds` || 505 | `Platform.UnsupportedHttpVersion` | HTTP/1.0 |diff --git a/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlane.java b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlane.javaindex 77b2b13f68..e580ec1d1c 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlane.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlane.java@@ -29,6 +29,7 @@ import jakarta.enterprise.event.Observes;import jakarta.inject.Inject;import org.jboss.logging.Logger;+import java.time.Duration;import java.time.Instant;import java.time.ZoneOffset;import java.time.format.DateTimeFormatter;@@ -64,6 +65,8 @@ public class LambdaWebDataPlane {private static final String BASE32 = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";private static final int REQUEST_ID_LENGTH = 26;private static final DateTimeFormatter HTTP_DATE = DateTimeFormatter.RFC_1123_DATE_TIME.withZone(ZoneOffset.UTC);+ /** Floci sees an exit some tens of milliseconds after it. A process that keeps running is not waited for longer. */+ private static final Duration EXIT_WAIT = Duration.ofSeconds(1);private final LambdaWebService service;private final LambdaWebEnvironmentManager environments;@@ -172,7 +175,7 @@ public class LambdaWebDataPlane {*/private void forward(HttpServerRequest request, String host, WebFunctionTarget target,WebFunctionRevision revision, Environment environment, String requestId, boolean firstTry) {- Instant started = Instant.now();+ long started = System.nanoTime();Exchange exchange = new Exchange(request, environment);boolean hasBody = hasRequestBody(request);AtomicBoolean bodyIn = new AtomicBoolean(!hasBody);@@ -235,18 +238,19 @@ public class LambdaWebDataPlane {/*** A process that exited before the request arrived is invisible to the client, which gets a* fresh environment, as on AWS. One that died under the request, or that cannot be retried- * because the body is spent, fails the request.+ * because the body is spent, fails the request. The answer waits for Floci to see the exit, so+ * the next request does not meet the dead environment.*/private void failed(HttpServerRequest request, String host, WebFunctionTarget target,- WebFunctionRevision revision, Exchange exchange, String requestId, Instant started,+ WebFunctionRevision revision, Exchange exchange, String requestId, long started,boolean retry) {if (!exchange.finish()) {return;}Environment environment = exchange.environment;- vertx.<Boolean>executeBlocking(() -> environments.exitedBefore(environment, started), false)- .onComplete(exited -> {- if (retry && exited.succeeded() && exited.result()) {+ Future.fromCompletionStage(environment.awaitExit(EXIT_WAIT), vertx.getOrCreateContext())+ .onComplete(ignored -> {+ if (retry && environment.exitedBefore(started)) {acquire(request, host, target, revision, requestId, environment);return;}diff --git a/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebEnvironmentManager.java b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebEnvironmentManager.javaindex 9c082423af..73514be00a 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebEnvironmentManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebEnvironmentManager.java@@ -1,11 +1,11 @@package io.github.hectorvent.floci.services.lambdaweb;+import com.github.dockerjava.api.DockerClient;import com.github.dockerjava.api.async.ResultCallback;import com.github.dockerjava.api.command.InspectContainerResponse;import com.github.dockerjava.api.command.WaitContainerResultCallback;import com.github.dockerjava.api.exception.NotFoundException;import com.github.dockerjava.api.model.Frame;-import com.github.dockerjava.api.model.WaitResponse;import io.github.hectorvent.floci.config.EmulatorConfig;import io.github.hectorvent.floci.core.common.ContainerTeardown;import io.github.hectorvent.floci.core.common.RequestScopes;@@ -18,6 +18,7 @@ import io.github.hectorvent.floci.core.common.docker.ContainerSpec;import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;import io.github.hectorvent.floci.core.common.docker.LaunchedContainerAwsEnv;import io.github.hectorvent.floci.core.common.docker.RetryingTarCopier;+import io.github.hectorvent.floci.core.common.docker.StreamingDocker;import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager;import io.github.hectorvent.floci.services.iam.model.SessionCreds;import io.github.hectorvent.floci.services.lambda.launcher.ImageResolver;@@ -86,6 +87,7 @@ public class LambdaWebEnvironmentManager implements ContainerTeardown {private static final int LOG_TAIL_LINES = 20;private static final Duration LOG_TAIL_TIMEOUT = Duration.ofSeconds(2);private static final Duration EXIT_WATCH_REARM = Duration.ofSeconds(1);+ private static final Duration EXIT_STATUS_TIMEOUT = Duration.ofSeconds(5);/*** One container of an endpoint's pool. It joins the pool before it starts, so requests that@@ -106,7 +108,8 @@ public class LambdaWebEnvironmentManager implements ContainerTeardown {private final AtomicInteger inFlight = new AtomicInteger();private final AtomicLong lastUsed = new AtomicLong(System.nanoTime());private final AtomicBoolean alive = new AtomicBoolean(true);- private final AtomicReference<Instant> exitedAt = new AtomicReference<>();+ /** The {@link System#nanoTime()} at which Floci saw the process end. */+ private final CompletableFuture<Long> exitSeen = new CompletableFuture<>();Environment(String poolKey, String revisionId, String accountId) {this.poolKey = poolKey;@@ -147,6 +150,25 @@ public class LambdaWebEnvironmentManager implements ContainerTeardown {inFlight.incrementAndGet();lastUsed.set(System.nanoTime());}++ /**+ * Whether Floci saw the process end before {@code nanoTime}. A request sent after that never+ * reached the app, so it gets a fresh environment, as on AWS. Floci sees an exit tens of+ * milliseconds after it happens. A request sent in between fails, so it never runs twice.+ */+ public boolean exitedBefore(long nanoTime) {+ Long seen = exitSeen.getNow(null);+ return seen != null && seen - nanoTime < 0;+ }++ /** Completes once Floci has seen the process end, or after {@code limit} while it still runs. */+ public CompletableFuture<Void> awaitExit(Duration limit) {+ return exitSeen.copy().orTimeout(limit.toNanos(), TimeUnit.NANOSECONDS).handle((seen, timedOut) -> null);+ }++ void sawExit(long nanoTime) {+ exitSeen.complete(nanoTime);+ }}/** The environment did not come up. {@code page} is what the endpoint answers. */@@ -173,6 +195,7 @@ public class LambdaWebEnvironmentManager implements ContainerTeardown {private final EmulatorConfig config;private final ContainerBuilder containerBuilder;private final ContainerLifecycleManager lifecycleManager;+ private final DockerClient streamingDocker;private final ImageResolver imageResolver;private final EcrRegistryManager ecrRegistryManager;private final S3Service s3Service;@@ -189,13 +212,15 @@ public class LambdaWebEnvironmentManager implements ContainerTeardown {@Injectpublic LambdaWebEnvironmentManager(EmulatorConfig config, ContainerBuilder containerBuilder,- ContainerLifecycleManager lifecycleManager, ImageResolver imageResolver,+ ContainerLifecycleManager lifecycleManager,+ @StreamingDocker DockerClient streamingDocker, ImageResolver imageResolver,EcrRegistryManager ecrRegistryManager, S3Service s3Service,ZipExtractor zipExtractor, LaunchedContainerAwsEnv awsEnv,LambdaExecutionRoleCredentials executionRoleCredentials) {this.config = config;this.containerBuilder = containerBuilder;this.lifecycleManager = lifecycleManager;+ this.streamingDocker = streamingDocker;this.imageResolver = imageResolver;this.ecrRegistryManager = ecrRegistryManager;this.s3Service = s3Service;@@ -342,27 +367,19 @@ public class LambdaWebEnvironmentManager implements ContainerTeardown {}}- /**- * Whether the process had already exited when a request that found it dead was sent. Such a- * request gets a fresh environment, as on AWS, while a process that dies under a request fails it.- * A container that is already gone answers no: nothing proves the exit came first.- */- public boolean exitedBefore(Environment environment, Instant moment) {- Instant exited = environment.exitedAt.get();- if (exited == null) {- exited = finishedAt(environment.containerId).orElse(null);- if (exited != null) {- environment.exitedAt.compareAndSet(null, exited);- }- }- return exited != null && exited.isBefore(moment);- }-@Overridepublic void stopManagedContainers() {allEnvironments().forEach(this::discard);}+ /** The live environment that writes {@code logStream}, the one a response names. */+ Environment environmentOf(String logStream) {+ return allEnvironments().stream()+ .filter(each -> logStream.equals(each.logStream))+ .findFirst()+ .orElseThrow();+ }+private List<Environment> allEnvironments() {List<Environment> all = new ArrayList<>();for (String key : pools.keySet()) {@@ -612,31 +629,32 @@ public class LambdaWebEnvironmentManager implements ContainerTeardown {});}- private Optional<Instant> finishedAt(String containerId) {+ private Integer exitStatus(String containerId) {try {- return stoppedState(containerId).map(state -> {- String finishedAt = state.getFinishedAt();- return finishedAt == null || finishedAt.startsWith("0001") ? Instant.now() : Instant.parse(finishedAt);- });+ return lifecycleManager.getDockerClient().waitContainerCmd(containerId)+ .exec(new WaitContainerResultCallback())+ .awaitStatusCode(EXIT_STATUS_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);} catch (RuntimeException e) {- LOG.debugv(e, "Could not inspect the Lambda Web environment {0}", containerId);- return Optional.empty();+ LOG.debugv(e, "Could not read the exit status of the Lambda Web environment {0}", containerId);+ return null;}}/*** A process that exits between requests is invisible to clients: the next request starts anew.- * The wait stream can drop on a quiet container, so a dropped watch looks at the container and- * arms itself again while the environment lives.+ * The attach stream ends when the process exits, and Floci times that on its own clock. The exit+ * time Docker records comes from the daemon's clock, which can be seconds off. The output is+ * dropped here. A reader that falls behind would block the app's writes.*/private void watchExit(Environment environment) {try {- lifecycleManager.getDockerClient().waitContainerCmd(environment.containerId)- .exec(new WaitContainerResultCallback() {+ streamingDocker.attachContainerCmd(environment.containerId)+ .withStdOut(true).withStdErr(true).withFollowStream(true)+ .exec(new ResultCallback.Adapter<Frame>() {@Override- public void onNext(WaitResponse response) {- super.onNext(response);- exited(environment, response.getStatusCode());+ public void onComplete() {+ super.onComplete();+ exited(environment, System.nanoTime());}@Override@@ -668,24 +686,29 @@ public class LambdaWebEnvironmentManager implements ContainerTeardown {stopped = Optional.empty();}if (stopped.isPresent()) {- exited(environment, stopped.get().getExitCodeLong() == null- ? null : stopped.get().getExitCodeLong().intValue());+ exited(environment, System.nanoTime());return;}watchExit(environment);}, EXIT_WATCH_REARM.toMillis(), TimeUnit.MILLISECONDS);}- private void exited(Environment environment, Integer statusCode) {- if (!environment.alive.compareAndSet(true, false)) {+ /** The environment leaves the pool before waiters hear of the exit, so the request after a failed one starts anew. */+ private void exited(Environment environment, long seenAt) {+ boolean running = environment.alive.compareAndSet(true, false);+ if (running) {+ removeFromPool(environment);+ }+ environment.sawExit(seenAt);+ if (!running) {return;}- environment.exitedAt.compareAndSet(null, finishedAt(environment.containerId).orElse(Instant.now()));- removeFromPool(environment);- reaper.execute(() -> {+ // Docker can take seconds to report the status under load. The reaper stays free to rearm dropped watches.+ Thread.ofVirtual().name("lambda-web-exit").start(() -> {try {LOG.infov("The Lambda Web environment {0} exited with status {1}: {2}",- environment.containerId, statusCode, lastLogLines(environment.containerId));+ environment.containerId, exitStatus(environment.containerId),+ lastLogLines(environment.containerId));lifecycleManager.stopAndRemove(environment.containerId, null, 0);} finally {forgetCredentials(environment);diff --git a/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneDockerIntegrationTest.javaindex 6c700263a0..3ffd2b1aea 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneDockerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneDockerIntegrationTest.java@@ -1,6 +1,7 @@package io.github.hectorvent.floci.services.lambdaweb;import com.github.dockerjava.api.DockerClient;+import io.github.hectorvent.floci.services.lambdaweb.LambdaWebEnvironmentManager.Environment;import io.github.hectorvent.floci.services.s3.S3Service;import io.quarkus.test.junit.QuarkusTest;import io.restassured.RestAssured;@@ -18,6 +19,7 @@ import java.io.InputStream;import java.io.OutputStream;import java.net.Socket;import java.nio.charset.StandardCharsets;+import java.time.Duration;import java.util.ArrayList;import java.util.HashSet;import java.util.List;@@ -74,6 +76,7 @@ class LambdaWebDataPlaneDockerIntegrationTest {res.writeHead(Number(req.url.split('/')[2]), {'x-app': 'yes'}); res.end('status body'); return;}if (req.url.startsWith('/crash')) { setTimeout(() => process.exit(1), 50); return; }+ if (req.url.startsWith('/exit')) { res.end('bye', () => process.exit(0)); return; }let body = '';req.on('data', (chunk) => { body += chunk; });req.on('end', () => {@@ -192,6 +195,22 @@ class LambdaWebDataPlaneDockerIntegrationTest {assertNotEquals(logStream, after.header("lambda-web-logstream"), "a new environment serves the next request");}+ @Test+ @Timeout(180)+ void anEnvironmentThatExitsBetweenRequestsIsReplacedUnseen() throws Exception {+ String domain = createFunction(30);+ String logStream = given().header("Host", domain).when().get("/status/200")+ .then().statusCode(200).extract().header("lambda-web-logstream");+ Environment environment = environments.environmentOf(logStream);++ given().header("Host", domain).when().get("/exit").then().statusCode(200).body(equalTo("bye"));+ environment.awaitExit(Duration.ofSeconds(30)).get();+ assertTrue(environment.exitedBefore(System.nanoTime()), "Floci did not see the environment exit");++ given().header("Host", domain).when().get("/status/200")+ .then().statusCode(200).header("lambda-web-logstream", not(equalTo(logStream)));+ }+@Test@Timeout(240)void environmentsScaleUpToMaxEnvironmentsAndTheRateLimitThrottles() throws Exception {diff --git a/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebEnvironmentManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebEnvironmentManagerTest.javanew file mode 100644index 0000000000..897ac54ca9--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebEnvironmentManagerTest.java@@ -0,0 +1,49 @@+package io.github.hectorvent.floci.services.lambdaweb;++import io.github.hectorvent.floci.services.lambdaweb.LambdaWebEnvironmentManager.Environment;+import org.junit.jupiter.api.Test;++import java.time.Duration;+import java.util.concurrent.CompletableFuture;+import java.util.concurrent.TimeUnit;++import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertTrue;++class LambdaWebEnvironmentManagerTest {++ @Test+ void onlyAnExitSeenBeforeTheRequestStartedAllowsARetry() {+ Environment environment = new Environment("pool", "rev-1", "000000000000");+ long started = System.nanoTime();+ assertFalse(environment.exitedBefore(started));++ environment.sawExit(started);++ assertFalse(environment.exitedBefore(started));+ assertFalse(environment.exitedBefore(started - 1));+ assertTrue(environment.exitedBefore(started + 1));+ }++ @Test+ void awaitExitEndsWhenTheExitIsSeen() throws Exception {+ Environment environment = new Environment("pool", "rev-1", "000000000000");+ CompletableFuture<Void> waited = environment.awaitExit(Duration.ofMinutes(1));+ assertFalse(waited.isDone());++ environment.sawExit(System.nanoTime());++ waited.get(5, TimeUnit.SECONDS);+ }++ @Test+ void awaitExitGivesUpOnAProcessThatKeepsRunning() throws Exception {+ Environment environment = new Environment("pool", "rev-1", "000000000000");++ environment.awaitExit(Duration.ofMillis(10)).get(5, TimeUnit.SECONDS);++ assertFalse(environment.exitedBefore(System.nanoTime()));+ environment.sawExit(System.nanoTime());+ assertTrue(environment.exitedBefore(System.nanoTime()));+ }+}