Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(lambdaweb): never run a request twice when the app crashes (#5842)

TATSUNO “Taz” Yasuhiroc8de16ffc7ca
diff --git a/docs/services/lambda-web-functions.md b/docs/services/lambda-web-functions.md
index 7ab7f4114d..3704ecb80c 100644
--- a/docs/services/lambda-web-functions.md
+++ b/docs/services/lambda-web-functions.md
@@ -84,7 +84,7 @@ The platform answers these cases itself, with the AWS HTML pages and `lambda-web
| 429 | `Platform.EndpointRequestThrottle` | The requests exceed the endpoint's `rateLimit` |
| 502 | `Application.TimeoutListenError`, `Application.InitCodeError`, `Runtime.Exit` | The server did not listen within 10 seconds, threw at import, or exited at start |
| 503 | `Platform.EndpointDisabled` | The endpoint's `rateLimit` is 0 |
-| 503 | `Runtime.OriginRequestError` | The process died while handling the request |
+| 503 | `Runtime.OriginRequestError` | The process died while handling the request, or tens of milliseconds before it |
| 504 | `Runtime.RequestTimeout` | No response within the revision's `timeoutSeconds` |
| 505 | `Platform.UnsupportedHttpVersion` | HTTP/1.0 |
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlane.java b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlane.java
index 77b2b13f68..e580ec1d1c 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlane.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlane.java
@@ -29,6 +29,7 @@ import jakarta.enterprise.event.Observes;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
+import java.time.Duration;
import java.time.Instant;
import java.time.ZoneOffset;
import java.time.format.DateTimeFormatter;
@@ -64,6 +65,8 @@ public class LambdaWebDataPlane {
private static final String BASE32 = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";
private static final int REQUEST_ID_LENGTH = 26;
private static final DateTimeFormatter HTTP_DATE = DateTimeFormatter.RFC_1123_DATE_TIME.withZone(ZoneOffset.UTC);
+ /** Floci sees an exit some tens of milliseconds after it. A process that keeps running is not waited for longer. */
+ private static final Duration EXIT_WAIT = Duration.ofSeconds(1);
private final LambdaWebService service;
private final LambdaWebEnvironmentManager environments;
@@ -172,7 +175,7 @@ public class LambdaWebDataPlane {
*/
private void forward(HttpServerRequest request, String host, WebFunctionTarget target,
WebFunctionRevision revision, Environment environment, String requestId, boolean firstTry) {
- Instant started = Instant.now();
+ long started = System.nanoTime();
Exchange exchange = new Exchange(request, environment);
boolean hasBody = hasRequestBody(request);
AtomicBoolean bodyIn = new AtomicBoolean(!hasBody);
@@ -235,18 +238,19 @@ public class LambdaWebDataPlane {
/**
* A process that exited before the request arrived is invisible to the client, which gets a
* fresh environment, as on AWS. One that died under the request, or that cannot be retried
- * because the body is spent, fails the request.
+ * because the body is spent, fails the request. The answer waits for Floci to see the exit, so
+ * the next request does not meet the dead environment.
*/
private void failed(HttpServerRequest request, String host, WebFunctionTarget target,
- WebFunctionRevision revision, Exchange exchange, String requestId, Instant started,
+ WebFunctionRevision revision, Exchange exchange, String requestId, long started,
boolean retry) {
if (!exchange.finish()) {
return;
}
Environment environment = exchange.environment;
- vertx.<Boolean>executeBlocking(() -> environments.exitedBefore(environment, started), false)
- .onComplete(exited -> {
- if (retry && exited.succeeded() && exited.result()) {
+ Future.fromCompletionStage(environment.awaitExit(EXIT_WAIT), vertx.getOrCreateContext())
+ .onComplete(ignored -> {
+ if (retry && environment.exitedBefore(started)) {
acquire(request, host, target, revision, requestId, environment);
return;
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebEnvironmentManager.java b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebEnvironmentManager.java
index 9c082423af..73514be00a 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebEnvironmentManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebEnvironmentManager.java
@@ -1,11 +1,11 @@
package io.github.hectorvent.floci.services.lambdaweb;
+import com.github.dockerjava.api.DockerClient;
import com.github.dockerjava.api.async.ResultCallback;
import com.github.dockerjava.api.command.InspectContainerResponse;
import com.github.dockerjava.api.command.WaitContainerResultCallback;
import com.github.dockerjava.api.exception.NotFoundException;
import com.github.dockerjava.api.model.Frame;
-import com.github.dockerjava.api.model.WaitResponse;
import io.github.hectorvent.floci.config.EmulatorConfig;
import io.github.hectorvent.floci.core.common.ContainerTeardown;
import io.github.hectorvent.floci.core.common.RequestScopes;
@@ -18,6 +18,7 @@ import io.github.hectorvent.floci.core.common.docker.ContainerSpec;
import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;
import io.github.hectorvent.floci.core.common.docker.LaunchedContainerAwsEnv;
import io.github.hectorvent.floci.core.common.docker.RetryingTarCopier;
+import io.github.hectorvent.floci.core.common.docker.StreamingDocker;
import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager;
import io.github.hectorvent.floci.services.iam.model.SessionCreds;
import io.github.hectorvent.floci.services.lambda.launcher.ImageResolver;
@@ -86,6 +87,7 @@ public class LambdaWebEnvironmentManager implements ContainerTeardown {
private static final int LOG_TAIL_LINES = 20;
private static final Duration LOG_TAIL_TIMEOUT = Duration.ofSeconds(2);
private static final Duration EXIT_WATCH_REARM = Duration.ofSeconds(1);
+ private static final Duration EXIT_STATUS_TIMEOUT = Duration.ofSeconds(5);
/**
* One container of an endpoint's pool. It joins the pool before it starts, so requests that
@@ -106,7 +108,8 @@ public class LambdaWebEnvironmentManager implements ContainerTeardown {
private final AtomicInteger inFlight = new AtomicInteger();
private final AtomicLong lastUsed = new AtomicLong(System.nanoTime());
private final AtomicBoolean alive = new AtomicBoolean(true);
- private final AtomicReference<Instant> exitedAt = new AtomicReference<>();
+ /** The {@link System#nanoTime()} at which Floci saw the process end. */
+ private final CompletableFuture<Long> exitSeen = new CompletableFuture<>();
Environment(String poolKey, String revisionId, String accountId) {
this.poolKey = poolKey;
@@ -147,6 +150,25 @@ public class LambdaWebEnvironmentManager implements ContainerTeardown {
inFlight.incrementAndGet();
lastUsed.set(System.nanoTime());
}
+
+ /**
+ * Whether Floci saw the process end before {@code nanoTime}. A request sent after that never
+ * reached the app, so it gets a fresh environment, as on AWS. Floci sees an exit tens of
+ * milliseconds after it happens. A request sent in between fails, so it never runs twice.
+ */
+ public boolean exitedBefore(long nanoTime) {
+ Long seen = exitSeen.getNow(null);
+ return seen != null && seen - nanoTime < 0;
+ }
+
+ /** Completes once Floci has seen the process end, or after {@code limit} while it still runs. */
+ public CompletableFuture<Void> awaitExit(Duration limit) {
+ return exitSeen.copy().orTimeout(limit.toNanos(), TimeUnit.NANOSECONDS).handle((seen, timedOut) -> null);
+ }
+
+ void sawExit(long nanoTime) {
+ exitSeen.complete(nanoTime);
+ }
}
/** The environment did not come up. {@code page} is what the endpoint answers. */
@@ -173,6 +195,7 @@ public class LambdaWebEnvironmentManager implements ContainerTeardown {
private final EmulatorConfig config;
private final ContainerBuilder containerBuilder;
private final ContainerLifecycleManager lifecycleManager;
+ private final DockerClient streamingDocker;
private final ImageResolver imageResolver;
private final EcrRegistryManager ecrRegistryManager;
private final S3Service s3Service;
@@ -189,13 +212,15 @@ public class LambdaWebEnvironmentManager implements ContainerTeardown {
@Inject
public LambdaWebEnvironmentManager(EmulatorConfig config, ContainerBuilder containerBuilder,
- ContainerLifecycleManager lifecycleManager, ImageResolver imageResolver,
+ ContainerLifecycleManager lifecycleManager,
+ @StreamingDocker DockerClient streamingDocker, ImageResolver imageResolver,
EcrRegistryManager ecrRegistryManager, S3Service s3Service,
ZipExtractor zipExtractor, LaunchedContainerAwsEnv awsEnv,
LambdaExecutionRoleCredentials executionRoleCredentials) {
this.config = config;
this.containerBuilder = containerBuilder;
this.lifecycleManager = lifecycleManager;
+ this.streamingDocker = streamingDocker;
this.imageResolver = imageResolver;
this.ecrRegistryManager = ecrRegistryManager;
this.s3Service = s3Service;
@@ -342,27 +367,19 @@ public class LambdaWebEnvironmentManager implements ContainerTeardown {
}
}
- /**
- * Whether the process had already exited when a request that found it dead was sent. Such a
- * request gets a fresh environment, as on AWS, while a process that dies under a request fails it.
- * A container that is already gone answers no: nothing proves the exit came first.
- */
- public boolean exitedBefore(Environment environment, Instant moment) {
- Instant exited = environment.exitedAt.get();
- if (exited == null) {
- exited = finishedAt(environment.containerId).orElse(null);
- if (exited != null) {
- environment.exitedAt.compareAndSet(null, exited);
- }
- }
- return exited != null && exited.isBefore(moment);
- }
-
@Override
public void stopManagedContainers() {
allEnvironments().forEach(this::discard);
}
+ /** The live environment that writes {@code logStream}, the one a response names. */
+ Environment environmentOf(String logStream) {
+ return allEnvironments().stream()
+ .filter(each -> logStream.equals(each.logStream))
+ .findFirst()
+ .orElseThrow();
+ }
+
private List<Environment> allEnvironments() {
List<Environment> all = new ArrayList<>();
for (String key : pools.keySet()) {
@@ -612,31 +629,32 @@ public class LambdaWebEnvironmentManager implements ContainerTeardown {
});
}
- private Optional<Instant> finishedAt(String containerId) {
+ private Integer exitStatus(String containerId) {
try {
- return stoppedState(containerId).map(state -> {
- String finishedAt = state.getFinishedAt();
- return finishedAt == null || finishedAt.startsWith("0001") ? Instant.now() : Instant.parse(finishedAt);
- });
+ return lifecycleManager.getDockerClient().waitContainerCmd(containerId)
+ .exec(new WaitContainerResultCallback())
+ .awaitStatusCode(EXIT_STATUS_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);
} catch (RuntimeException e) {
- LOG.debugv(e, "Could not inspect the Lambda Web environment {0}", containerId);
- return Optional.empty();
+ LOG.debugv(e, "Could not read the exit status of the Lambda Web environment {0}", containerId);
+ return null;
}
}
/**
* A process that exits between requests is invisible to clients: the next request starts anew.
- * The wait stream can drop on a quiet container, so a dropped watch looks at the container and
- * arms itself again while the environment lives.
+ * The attach stream ends when the process exits, and Floci times that on its own clock. The exit
+ * time Docker records comes from the daemon's clock, which can be seconds off. The output is
+ * dropped here. A reader that falls behind would block the app's writes.
*/
private void watchExit(Environment environment) {
try {
- lifecycleManager.getDockerClient().waitContainerCmd(environment.containerId)
- .exec(new WaitContainerResultCallback() {
+ streamingDocker.attachContainerCmd(environment.containerId)
+ .withStdOut(true).withStdErr(true).withFollowStream(true)
+ .exec(new ResultCallback.Adapter<Frame>() {
@Override
- public void onNext(WaitResponse response) {
- super.onNext(response);
- exited(environment, response.getStatusCode());
+ public void onComplete() {
+ super.onComplete();
+ exited(environment, System.nanoTime());
}
@Override
@@ -668,24 +686,29 @@ public class LambdaWebEnvironmentManager implements ContainerTeardown {
stopped = Optional.empty();
}
if (stopped.isPresent()) {
- exited(environment, stopped.get().getExitCodeLong() == null
- ? null : stopped.get().getExitCodeLong().intValue());
+ exited(environment, System.nanoTime());
return;
}
watchExit(environment);
}, EXIT_WATCH_REARM.toMillis(), TimeUnit.MILLISECONDS);
}
- private void exited(Environment environment, Integer statusCode) {
- if (!environment.alive.compareAndSet(true, false)) {
+ /** The environment leaves the pool before waiters hear of the exit, so the request after a failed one starts anew. */
+ private void exited(Environment environment, long seenAt) {
+ boolean running = environment.alive.compareAndSet(true, false);
+ if (running) {
+ removeFromPool(environment);
+ }
+ environment.sawExit(seenAt);
+ if (!running) {
return;
}
- environment.exitedAt.compareAndSet(null, finishedAt(environment.containerId).orElse(Instant.now()));
- removeFromPool(environment);
- reaper.execute(() -> {
+ // Docker can take seconds to report the status under load. The reaper stays free to rearm dropped watches.
+ Thread.ofVirtual().name("lambda-web-exit").start(() -> {
try {
LOG.infov("The Lambda Web environment {0} exited with status {1}: {2}",
- environment.containerId, statusCode, lastLogLines(environment.containerId));
+ environment.containerId, exitStatus(environment.containerId),
+ lastLogLines(environment.containerId));
lifecycleManager.stopAndRemove(environment.containerId, null, 0);
} finally {
forgetCredentials(environment);
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneDockerIntegrationTest.java
index 6c700263a0..3ffd2b1aea 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneDockerIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneDockerIntegrationTest.java
@@ -1,6 +1,7 @@
package io.github.hectorvent.floci.services.lambdaweb;
import com.github.dockerjava.api.DockerClient;
+import io.github.hectorvent.floci.services.lambdaweb.LambdaWebEnvironmentManager.Environment;
import io.github.hectorvent.floci.services.s3.S3Service;
import io.quarkus.test.junit.QuarkusTest;
import io.restassured.RestAssured;
@@ -18,6 +19,7 @@ import java.io.InputStream;
import java.io.OutputStream;
import java.net.Socket;
import java.nio.charset.StandardCharsets;
+import java.time.Duration;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@@ -74,6 +76,7 @@ class LambdaWebDataPlaneDockerIntegrationTest {
res.writeHead(Number(req.url.split('/')[2]), {'x-app': 'yes'}); res.end('status body'); return;
}
if (req.url.startsWith('/crash')) { setTimeout(() => process.exit(1), 50); return; }
+ if (req.url.startsWith('/exit')) { res.end('bye', () => process.exit(0)); return; }
let body = '';
req.on('data', (chunk) => { body += chunk; });
req.on('end', () => {
@@ -192,6 +195,22 @@ class LambdaWebDataPlaneDockerIntegrationTest {
assertNotEquals(logStream, after.header("lambda-web-logstream"), "a new environment serves the next request");
}
+ @Test
+ @Timeout(180)
+ void anEnvironmentThatExitsBetweenRequestsIsReplacedUnseen() throws Exception {
+ String domain = createFunction(30);
+ String logStream = given().header("Host", domain).when().get("/status/200")
+ .then().statusCode(200).extract().header("lambda-web-logstream");
+ Environment environment = environments.environmentOf(logStream);
+
+ given().header("Host", domain).when().get("/exit").then().statusCode(200).body(equalTo("bye"));
+ environment.awaitExit(Duration.ofSeconds(30)).get();
+ assertTrue(environment.exitedBefore(System.nanoTime()), "Floci did not see the environment exit");
+
+ given().header("Host", domain).when().get("/status/200")
+ .then().statusCode(200).header("lambda-web-logstream", not(equalTo(logStream)));
+ }
+
@Test
@Timeout(240)
void environmentsScaleUpToMaxEnvironmentsAndTheRateLimitThrottles() throws Exception {
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebEnvironmentManagerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebEnvironmentManagerTest.java
new file mode 100644
index 0000000000..897ac54ca9
--- /dev/null
+++ b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebEnvironmentManagerTest.java
@@ -0,0 +1,49 @@
+package io.github.hectorvent.floci.services.lambdaweb;
+
+import io.github.hectorvent.floci.services.lambdaweb.LambdaWebEnvironmentManager.Environment;
+import org.junit.jupiter.api.Test;
+
+import java.time.Duration;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+class LambdaWebEnvironmentManagerTest {
+
+ @Test
+ void onlyAnExitSeenBeforeTheRequestStartedAllowsARetry() {
+ Environment environment = new Environment("pool", "rev-1", "000000000000");
+ long started = System.nanoTime();
+ assertFalse(environment.exitedBefore(started));
+
+ environment.sawExit(started);
+
+ assertFalse(environment.exitedBefore(started));
+ assertFalse(environment.exitedBefore(started - 1));
+ assertTrue(environment.exitedBefore(started + 1));
+ }
+
+ @Test
+ void awaitExitEndsWhenTheExitIsSeen() throws Exception {
+ Environment environment = new Environment("pool", "rev-1", "000000000000");
+ CompletableFuture<Void> waited = environment.awaitExit(Duration.ofMinutes(1));
+ assertFalse(waited.isDone());
+
+ environment.sawExit(System.nanoTime());
+
+ waited.get(5, TimeUnit.SECONDS);
+ }
+
+ @Test
+ void awaitExitGivesUpOnAProcessThatKeepsRunning() throws Exception {
+ Environment environment = new Environment("pool", "rev-1", "000000000000");
+
+ environment.awaitExit(Duration.ofMillis(10)).get(5, TimeUnit.SECONDS);
+
+ assertFalse(environment.exitedBefore(System.nanoTime()));
+ environment.sawExit(System.nanoTime());
+ assertTrue(environment.exitedBefore(System.nanoTime()));
+ }
+}