Light, fluffy, and always free - The AWS Local Emulator alternative
feat(lambdaweb): serve endpoint domains from a Node.js environment per revision (#5760)
TATSUNO “Taz” Yasuhiro6e01216f88de
diff --git a/.github/ci/prefetch-images.sh b/.github/ci/prefetch-images.shindex ac63b6b10c..53c85adb91 100755--- a/.github/ci/prefetch-images.sh+++ b/.github/ci/prefetch-images.sh@@ -87,6 +87,7 @@ starts EcsContainerManagerFirelensDockerIntegrationTest && [ -n "$FLUENT_BIT_IMA# shard reaches it; the two Python runtimes each have one Docker-backed class; nodejs:18 was# pulled by every shard and started by none. A class missed here pulls inline, as before.grep -qE '/apigateway/|/apigatewayv2/|/lambda/' "$SHARD_FILE" && ecr public.ecr.aws/lambda/nodejs:20+grep -qE '/lambdaweb/' "$SHARD_FILE" && ecr public.ecr.aws/lambda/nodejs:24starts SwfLambdaIntegrationTest && ecr public.ecr.aws/lambda/python:3.12starts ElbV2LambdaTargetDataPlaneIntegrationTest && ecr public.ecr.aws/lambda/python:3.14diff --git a/compatibility-tests/sdk-test-awscli/test/lambda-web.bats b/compatibility-tests/sdk-test-awscli/test/lambda-web.batsindex 1d8d960aeb..06063ecfa6 100644--- a/compatibility-tests/sdk-test-awscli/test/lambda-web.bats+++ b/compatibility-tests/sdk-test-awscli/test/lambda-web.bats@@ -17,6 +17,8 @@ setup() {teardown() {if [ -n "$FUNCTION_NAME" ]; thenaws_cmd lambda-web delete-web-function --function-name "$FUNCTION_NAME" >/dev/null 2>&1 || true+ aws_cmd iam delete-role-policy --role-name "awscli-lambdaweb-$FUNCTION_NAME" --policy-name awscli-lambdaweb-logs >/dev/null 2>&1 || true+ aws_cmd iam delete-role --role-name "awscli-lambdaweb-$FUNCTION_NAME" >/dev/null 2>&1 || truefiaws_cmd s3 rb "s3://$BUCKET" --force >/dev/null 2>&1 || true}@@ -131,6 +133,27 @@ create_function() {assert_output --partial "Invalid function name or ARN '-bad'"}+@test "Lambda Web: deploy the hello world app and invoke its endpoint" {+ FUNCTION_NAME=$(unique_name "cli-web")+ # deploy passes --endpoint-url only to its Lambda Web client, so the environment variable routes the rest.+ run bash -c "cd '$BATS_TEST_TMPDIR' && AWS_ENDPOINT_URL='$FLOCI_ENDPOINT' aws lambda-web deploy --name '$FUNCTION_NAME' --hello-world --create"+ assert_success+ domain=$(aws_cmd lambda-web get-web-function-endpoint --function-name "$FUNCTION_NAME" --endpoint-name dev | jq -r '.domainName')++ run curl -sS -i -H "Host: $domain" "$FLOCI_ENDPOINT/hello?from=bats"+ assert_success+ assert_output --partial "HTTP/1.1 200"+ assert_output --partial "lambda-web-request-id: "+ assert_output --partial "lambda-web-logstream: "+ assert_output --partial "Hello from Lambda Web!"++ # A domain Floci never minted, in the shape it mints.+ run curl -sS -i -H "Host: nopenopenopenope.001.lambda-web.us-east-1.localhost:4566" "$FLOCI_ENDPOINT/"+ assert_success+ assert_output --partial "HTTP/1.1 404"+ assert_output --partial "lambda-web-error-type: Platform.InvalidEndpoint"+}+@test "Lambda Web: delete removes the function and the waiter settles" {FUNCTION_NAME=$(unique_name "cli-web")create_function "$FUNCTION_NAME" >/dev/nulldiff --git a/docs/services/lambda-web-functions.md b/docs/services/lambda-web-functions.mdindex 38ebb94aa0..f4e3b19aff 100644--- a/docs/services/lambda-web-functions.md+++ b/docs/services/lambda-web-functions.md@@ -41,7 +41,7 @@ The `Lambda Web` service model (apiVersion 2025-03-07, `aws lambda-web` in the A- **ARNs** given as identifiers must name the request's region and the caller's account, as on AWS. A revision or endpoint ARN resolves by its id or name only.- **Page tokens** are base64 cursors that carry the request and the position of the page, not the UUIDs AWS issues. A token reused with other parameters gets the AWS "does not match the request" error, and any other token the "invalid or has expired" error.- **Domain names** take the form `<id>.001.lambda-web.<region>.<floci host>` and `<id>.lambda-web.global.<floci host>`, with the host and port of `floci.base-url`. A `PerRegion` endpoint gets one domain per region in `regionalEndpoints`.-- **Not implemented yet:** invoking a function through its endpoint domain, execution environments, `IamAuth` on requests, and the automatic deletion of revisions past 50. The endpoint domain does not answer requests yet.+- **Not implemented yet:** `IamAuth` on requests, the automatic deletion of revisions past 50, the scaling limits (`maxConcurrencyPerEnvironment`, `maxEnvironments`, a `rateLimit` other than 0), the metadata API, the logs and metrics, and serving a `MultiRegion` or `PerRegion` endpoint in its other regions.### `aws lambda-web deploy`@@ -51,7 +51,40 @@ The CLI's `deploy` command works against Floci with `AWS_ENDPOINT_URL` set:AWS_ENDPOINT_URL=http://localhost:4566 aws lambda-web deploy --name my-app --hello-world --create```-The command passes `--endpoint-url` only to the Lambda Web client. Its STS, S3 and IAM calls use the default endpoints, so the environment variable is what routes them to Floci. The `endpointUrl` it prints is the Floci domain name behind `https://`, which Floci does not serve yet.+The command passes `--endpoint-url` only to the Lambda Web client. Its STS, S3 and IAM calls use the default endpoints, so the environment variable is what routes them to Floci. The `endpointUrl` it prints is the Floci domain name behind `https://`. Floci serves it over `http://`, as below.++## Invoking an endpoint++A request whose `Host` is an endpoint's `domainName` goes to the function. The domain carries the host and port of `floci.base-url`, so either address works:++```bash+curl http://<domainName>/+curl -H "Host: <domainName>" http://localhost:4566/+```++The `Host` must match the domain exactly, as on AWS: another case or another port is `404` with `lambda-web-error-type: Platform.InvalidEndpoint`.++Floci runs each revision in one execution environment, a `public.ecr.aws/lambda/nodejs:24` container started by the first request. The code is the revision's S3 object unpacked into `/var/task`, the entry file is `index.js` or the file `AWS_LAMBDA_NODEJS_ENTRYPOINT` names, and node runs with the flags AWS uses. The app sees `AWS_LAMBDA_HTTP_ENDPOINT=0.0.0.0:3000`, `AWS_LAMBDA_INITIALIZATION_TYPE=native-http`, `AWS_EXECUTION_ENV`, `AWS_LAMBDA_FUNCTION_MEMORY_SIZE=2048`, `AWS_LAMBDA_LOG_LEVEL`, `AWS_ACCOUNT_ID`, `AWS_REGION`, credentials and `AWS_ENDPOINT_URL` for Floci, `NODE_PATH`, and its own variables. The server has 10 seconds to listen. An environment that sees no request for `floci.services.lambda.container-idle-timeout-seconds` is stopped with `SIGTERM` and one second, as on scale to zero. A process that exits between requests is replaced by a fresh environment on the next request.++Requests are forwarded over HTTP/1.1 with the headers AWS adds and drops: `lambda-web-request-id`, `lambda-web-endpoint-name`, `x-forwarded-proto`, `x-forwarded-port` and `Connection: close` are set, the client address is appended as an `X-Forwarded-For` line, and `Connection`, `Keep-Alive`, `Upgrade`, `Expect`, `Trailer`, the `Proxy-*` headers, `X-Real-IP`, the method override headers and any `lambda-web-*` header the client sent are dropped. A body without a `Content-Length` reaches the app chunked. Responses carry `lambda-web-request-id` and `lambda-web-logstream`, lose their connection headers and trailers, and stream as the app writes them. Requests with two revisions in `revisionWeights` are split by weight.++The revision's `timeoutSeconds` bounds the whole exchange, as on AWS: a response that has not started by then is the 504 page, and one that is still streaming is cut. The request body is taken in full before the answer, so a slow upload gets its 504 once the body is in.++The container gets the execution role's session credentials when the role exists in IAM, and the account's test credentials otherwise, so SDK calls from the app land in the function's account.++The platform answers these cases itself, with the AWS HTML pages and `lambda-web-error-type`:++| Status | Error type | When |+| --- | --- | --- |+| 400 | `Platform.MalformedRequest` | A method other than `GET`, `HEAD`, `POST`, `PUT`, `PATCH`, `OPTIONS`, `DELETE` |+| 404 | `Platform.InvalidEndpoint` | The `Host` is a `lambda-web` domain Floci did not mint |+| 502 | `Application.TimeoutListenError`, `Application.InitCodeError`, `Runtime.Exit` | The server did not listen within 10 seconds, threw at import, or exited at start |+| 503 | `Platform.EndpointDisabled` | The endpoint's `rateLimit` is 0 |+| 503 | `Runtime.OriginRequestError` | The process died while handling the request |+| 504 | `Runtime.RequestTimeout` | No response within the revision's `timeoutSeconds` |+| 505 | `Platform.UnsupportedHttpVersion` | HTTP/1.0 |++A request head over 128 KiB is refused with `431`, without the AWS page. A request without a `Host` header cannot be told apart from one for another Floci service, so it does not get `Platform.MissingHostHeader`.## CloudFormationdiff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/LambdaExecutionRoleCredentials.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/LambdaExecutionRoleCredentials.javaindex 639798caa4..a022398d77 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/LambdaExecutionRoleCredentials.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/LambdaExecutionRoleCredentials.java@@ -53,8 +53,11 @@ public class LambdaExecutionRoleCredentials {* the existing container credential fallback rather than becoming an implicit deny-all caller.*/public Optional<SessionCreds> forFunction(LambdaFunction function) {- String roleArn = function.getRole();- String functionAccountId = sessionAccountId(function);+ return forRole(function.getRole(), sessionAccountId(function), function.getFunctionName());+ }++ /** A session for {@code roleArn} assumed on behalf of {@code functionName} in {@code functionAccountId}. */+ public Optional<SessionCreds> forRole(String roleArn, String functionAccountId, String functionName) {if (roleArn == null || roleArn.isBlank()|| functionAccountId == null || functionAccountId.isBlank()) {return Optional.empty();@@ -84,7 +87,6 @@ public class LambdaExecutionRoleCredentials {"ASIA" + random(UPPER_ALPHANUMERIC, 16),random(SECRET_CHARACTERS, 40),random(SECRET_CHARACTERS, 200));- String functionName = function.getFunctionName();String assumedRoleId = role.get().getRoleId() != null ? role.get().getRoleId() + ":" + functionName : null;iamService.registerLambdaExecutionRoleSession(functionAccountId, credentials.accessKeyId(), credentials.secretAccessKey(),diff --git a/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlane.java b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlane.javanew file mode 100644index 0000000000..ffbfdc754a--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlane.java@@ -0,0 +1,455 @@+package io.github.hectorvent.floci.services.lambdaweb;++import io.github.hectorvent.floci.core.common.RequestHost;+import io.github.hectorvent.floci.core.common.ServiceConfigAccess;+import io.github.hectorvent.floci.services.lambdaweb.LambdaWebEnvironmentManager.Environment;+import io.github.hectorvent.floci.services.lambdaweb.LambdaWebEnvironmentManager.StartFailure;+import io.github.hectorvent.floci.services.lambdaweb.LambdaWebService.WebFunctionTarget;+import io.github.hectorvent.floci.services.lambdaweb.model.WebFunctionRevision;+import io.vertx.core.Future;+import io.vertx.core.Handler;+import io.vertx.core.MultiMap;+import io.vertx.core.Vertx;+import io.vertx.core.buffer.Buffer;+import io.vertx.core.http.HttpClient;+import io.vertx.core.http.HttpClientOptions;+import io.vertx.core.http.HttpClientRequest;+import io.vertx.core.http.HttpClientResponse;+import io.vertx.core.http.HttpMethod;+import io.vertx.core.http.HttpServerRequest;+import io.vertx.core.http.HttpServerResponse;+import io.vertx.core.http.HttpVersion;+import io.vertx.core.http.RequestOptions;+import io.vertx.core.streams.ReadStream;+import io.vertx.ext.web.Router;+import io.vertx.ext.web.RoutingContext;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.enterprise.event.Observes;+import jakarta.inject.Inject;+import org.jboss.logging.Logger;++import java.time.Instant;+import java.time.ZoneOffset;+import java.time.format.DateTimeFormatter;+import java.util.HashSet;+import java.util.Locale;+import java.util.Map;+import java.util.Optional;+import java.util.Set;+import java.util.concurrent.ThreadLocalRandom;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicBoolean;+import java.util.regex.Pattern;++/**+ * Serves the endpoint domains of Web Functions. A request whose Host is a {@code lambda-web} domain+ * goes over HTTP/1.1 to the revision's environment with the headers AWS adds and drops; any other+ * request falls through to the next route.+ */+@ApplicationScoped+public class LambdaWebDataPlane {++ static final Set<String> SUPPORTED_METHODS = Set.of("GET", "HEAD", "POST", "PUT", "PATCH", "OPTIONS", "DELETE");+ /** Dropped before forwarding. {@code te} passes through, as recorded. */+ static final Set<String> DROPPED_REQUEST_HEADERS = Set.of("host", "connection", "keep-alive", "upgrade", "expect",+ "trailer", "transfer-encoding", "proxy-authenticate", "proxy-authorization", "proxy-connection",+ "x-real-ip", "x-http-method-override", "x-http-method", "x-method-override");+ static final Set<String> DROPPED_RESPONSE_HEADERS = Set.of("connection", "keep-alive", "transfer-encoding",+ "upgrade", "te", "trailer", "trailers", "proxy-authenticate", "proxy-authorization");+ static final String REQUEST_ID_HEADER = "lambda-web-request-id";+ private static final Pattern ENDPOINT_DOMAIN = Pattern.compile(+ "[a-z0-9]{16}\\.(?:[0-9]{3}\\.lambda-web\\.[a-z0-9-]+|lambda-web\\.global)\\..+", Pattern.CASE_INSENSITIVE);+ private static final Logger LOG = Logger.getLogger(LambdaWebDataPlane.class);+ private static final String BASE32 = "ABCDEFGHIJKLMNOPQRSTUVWXYZ234567";+ private static final int REQUEST_ID_LENGTH = 26;+ private static final DateTimeFormatter HTTP_DATE = DateTimeFormatter.RFC_1123_DATE_TIME.withZone(ZoneOffset.UTC);++ private final LambdaWebService service;+ private final LambdaWebEnvironmentManager environments;+ private final ServiceConfigAccess serviceConfigAccess;+ private final Vertx vertx;+ private final HttpClient client;++ @Inject+ public LambdaWebDataPlane(LambdaWebService service, LambdaWebEnvironmentManager environments,+ ServiceConfigAccess serviceConfigAccess, Vertx vertx) {+ this.service = service;+ this.environments = environments;+ this.serviceConfigAccess = serviceConfigAccess;+ this.vertx = vertx;+ this.client = vertx.createHttpClient(new HttpClientOptions()+ .setMaxPoolSize(256)+ .setConnectTimeout(5_000)+ .setKeepAlive(false));+ }++ void register(@Observes Router router) {+ router.route().handler(this::handle);+ }++ void handle(RoutingContext context) {+ HttpServerRequest request = context.request();+ String host = RequestHost.of(request);+ if (host == null || !isLambdaWebHost(host) || !serviceConfigAccess.isEnabled("lambda")) {+ context.next();+ return;+ }+ String requestId = newRequestId();+ if (request.version() == HttpVersion.HTTP_1_0) {+ page(request, LambdaWebErrorPages.UNSUPPORTED_HTTP_VERSION, requestId, true);+ return;+ }+ if (!SUPPORTED_METHODS.contains(request.method().name())) {+ malformed(request);+ return;+ }+ Optional<WebFunctionTarget> located = service.locate(host);+ if (located.isEmpty()) {+ page(request, LambdaWebErrorPages.INVALID_ENDPOINT, requestId, false);+ return;+ }+ WebFunctionTarget target = located.get();+ if (target.endpoint().throttleConfig != null+ && Integer.valueOf(0).equals(target.endpoint().throttleConfig.rateLimit)) {+ page(request, LambdaWebErrorPages.ENDPOINT_DISABLED, requestId, false);+ return;+ }+ WebFunctionRevision revision = LambdaWebService.pickRevision(target);+ if (revision == null) {+ page(request, LambdaWebErrorPages.ORIGIN_REQUEST_ERROR, requestId, false);+ return;+ }+ if ("100-continue".equalsIgnoreCase(request.getHeader("Expect"))) {+ request.response().writeContinue();+ }+ request.pause();+ acquire(request, host, target, revision, requestId, null);+ }++ /** Starts or reuses the revision's environment off the event loop, then forwards. */+ private void acquire(HttpServerRequest request, String host, WebFunctionTarget target,+ WebFunctionRevision revision, String requestId, Environment dead) {+ vertx.<Environment>executeBlocking(() -> {+ if (dead != null) {+ environments.discard(dead);+ }+ return environments.acquire(target.accountId(), target.region(), target.function(), revision);+ }, false)+ .onSuccess(environment -> forward(request, host, target, revision, environment, requestId, dead == null))+ .onFailure(error -> {+ request.resume();+ if (error instanceof StartFailure failure) {+ page(request, failure.page(), requestId, false);+ return;+ }+ LOG.errorv(error, "Could not start an environment for {0}", host);+ internalError(request, requestId);+ });+ }++ /**+ * One forwarded request. {@code timeoutSeconds} bounds the whole exchange: past it an answer+ * that has not started is the 504 page and one that has is cut. The edge takes the whole+ * request body before it answers, so a slow upload gets its 504 once the body is in.+ */+ private void forward(HttpServerRequest request, String host, WebFunctionTarget target,+ WebFunctionRevision revision, Environment environment, String requestId, boolean firstTry) {+ Instant started = Instant.now();+ Exchange exchange = new Exchange(request, environment);+ boolean hasBody = hasRequestBody(request);+ AtomicBoolean bodyIn = new AtomicBoolean(!hasBody);+ AtomicBoolean expired = new AtomicBoolean();+ RequestOptions options = new RequestOptions()+ .setHost(environment.host())+ .setPort(environment.port())+ .setURI(request.uri())+ .setMethod(request.method());+ client.request(options).onSuccess(upstream -> {+ copyRequestHeaders(request, upstream, host, target.endpoint().endpointName, requestId);+ if (hasBody && request.getHeader("Content-Length") == null) {+ upstream.setChunked(true);+ }+ Runnable timeOut = () -> {+ if (exchange.finish()) {+ upstream.reset();+ if (request.response().headWritten()) {+ request.response().close();+ } else {+ page(request, LambdaWebErrorPages.REQUEST_TIMEOUT, requestId, false);+ }+ }+ };+ exchange.deadline = vertx.setTimer(TimeUnit.SECONDS.toMillis(revision.serviceConfig.timeoutSeconds), id -> {+ expired.set(true);+ if (bodyIn.get()) {+ timeOut.run();+ }+ });+ Future<HttpClientResponse> sent = hasBody+ ? upstream.send(new RequestBody(request, () -> {+ bodyIn.set(true);+ if (expired.get()) {+ timeOut.run();+ }+ }))+ : upstream.send();+ sent.onSuccess(response -> relay(request, response, exchange, requestId))+ .onFailure(error -> vertx.runOnContext(ignored -> {+ LOG.debugv("The environment {0} did not answer request {1}: {2}",+ environment.containerId(), requestId, error.toString());+ if (request.response().headWritten()) {+ if (exchange.finish()) {+ request.response().close();+ }+ return;+ }+ failed(request, host, target, revision, exchange, requestId, started, firstTry && !hasBody);+ }));+ if (hasBody) {+ request.resume();+ }+ }).onFailure(error -> {+ LOG.debugv("The environment {0} refused the connection: {1}", environment.containerId(), error.toString());+ failed(request, host, target, revision, exchange, requestId, started, firstTry);+ });+ }++ /**+ * A process that exited before the request arrived is invisible to the client, which gets a+ * fresh environment, as on AWS. One that died under the request, or that cannot be retried+ * because the body is spent, fails the request.+ */+ private void failed(HttpServerRequest request, String host, WebFunctionTarget target,+ WebFunctionRevision revision, Exchange exchange, String requestId, Instant started,+ boolean retry) {+ if (!exchange.finish()) {+ return;+ }+ Environment environment = exchange.environment;+ vertx.<Boolean>executeBlocking(() -> environments.exitedBefore(environment, started), false)+ .onComplete(exited -> {+ if (retry && exited.succeeded() && exited.result()) {+ acquire(request, host, target, revision, requestId, environment);+ return;+ }+ request.resume();+ page(request, LambdaWebErrorPages.ORIGIN_REQUEST_ERROR, requestId, false);+ });+ }++ private void relay(HttpServerRequest request, HttpClientResponse response, Exchange exchange, String requestId) {+ if (exchange.isFinished()) {+ return;+ }+ HttpServerResponse out = request.response();+ out.setStatusCode(response.statusCode());+ copyResponseHeaders(response.headers(), out);+ out.headers().set("lambda-web-logstream", exchange.environment.logStream());+ out.headers().set(REQUEST_ID_HEADER, requestId);+ if (response.getHeader("Content-Length") == null && hasBody(request.method(), response.statusCode())) {+ out.setChunked(true);+ }+ response.pipeTo(out).onComplete(piped -> {+ if (!exchange.finish()) {+ return;+ }+ if (piped.succeeded()) {+ return;+ }+ if (out.headWritten()) {+ out.close();+ } else {+ page(request, LambdaWebErrorPages.ORIGIN_REQUEST_ERROR, requestId, false);+ }+ });+ }++ /** One request against one environment. The first finisher releases the reservation and the deadline. */+ private final class Exchange {+ private final HttpServerRequest request;+ private final Environment environment;+ private final AtomicBoolean finished = new AtomicBoolean();+ private long deadline = -1;++ Exchange(HttpServerRequest request, Environment environment) {+ this.request = request;+ this.environment = environment;+ }++ boolean finish() {+ if (!finished.compareAndSet(false, true)) {+ return false;+ }+ if (deadline >= 0) {+ vertx.cancelTimer(deadline);+ }+ environment.release();+ return true;+ }++ boolean isFinished() {+ return finished.get();+ }+ }++ /** The request body as the upstream reads it, telling {@code onEnd} when the last byte went by. */+ private static final class RequestBody implements ReadStream<Buffer> {+ private final HttpServerRequest request;+ private final Runnable onEnd;++ RequestBody(HttpServerRequest request, Runnable onEnd) {+ this.request = request;+ this.onEnd = onEnd;+ }++ @Override+ public ReadStream<Buffer> exceptionHandler(Handler<Throwable> handler) {+ request.exceptionHandler(handler);+ return this;+ }++ @Override+ public ReadStream<Buffer> handler(Handler<Buffer> handler) {+ request.handler(handler);+ return this;+ }++ @Override+ public ReadStream<Buffer> pause() {+ request.pause();+ return this;+ }++ @Override+ public ReadStream<Buffer> resume() {+ request.resume();+ return this;+ }++ @Override+ public ReadStream<Buffer> fetch(long amount) {+ request.fetch(amount);+ return this;+ }++ @Override+ public ReadStream<Buffer> endHandler(Handler<Void> handler) {+ request.endHandler(end -> {+ onEnd.run();+ if (handler != null) {+ handler.handle(end);+ }+ });+ return this;+ }+ }++ static boolean hasRequestBody(HttpServerRequest request) {+ String length = request.getHeader("Content-Length");+ return (length != null && !"0".equals(length.trim())) || request.headers().contains("Transfer-Encoding");+ }++ /** The shapes Floci mints, in any case and with any port, so a near miss gets the AWS 404 and not another service. */+ static boolean isLambdaWebHost(String host) {+ return ENDPOINT_DOMAIN.matcher(host).matches();+ }++ static boolean hasBody(HttpMethod method, int status) {+ return method != HttpMethod.HEAD && status >= 200 && status != 204 && status != 304;+ }++ static void copyRequestHeaders(HttpServerRequest request, HttpClientRequest upstream, String host,+ String endpointName, String requestId) {+ Set<String> dropped = new HashSet<>(DROPPED_REQUEST_HEADERS);+ for (String connection : request.headers().getAll("Connection")) {+ for (String token : connection.split(",")) {+ dropped.add(token.trim().toLowerCase(Locale.ROOT));+ }+ }+ MultiMap headers = upstream.headers();+ for (Map.Entry<String, String> header : request.headers()) {+ String name = header.getKey().toLowerCase(Locale.ROOT);+ if (!dropped.contains(name) && !name.startsWith("lambda-web-")) {+ headers.add(header.getKey(), header.getValue());+ }+ }+ headers.set("Host", host);+ headers.add("X-Forwarded-For", request.remoteAddress() == null ? "" : request.remoteAddress().host());+ headers.set("x-forwarded-proto", request.scheme());+ headers.set("x-forwarded-port", String.valueOf(port(host, request.scheme())));+ headers.set(REQUEST_ID_HEADER, requestId);+ headers.set("lambda-web-endpoint-name", endpointName);+ headers.set("Connection", "close");+ }++ static void copyResponseHeaders(MultiMap headers, HttpServerResponse out) {+ Set<String> dropped = new HashSet<>(DROPPED_RESPONSE_HEADERS);+ for (String connection : headers.getAll("Connection")) {+ for (String token : connection.split(",")) {+ dropped.add(token.trim().toLowerCase(Locale.ROOT));+ }+ }+ for (Map.Entry<String, String> header : headers) {+ if (!dropped.contains(header.getKey().toLowerCase(Locale.ROOT))) {+ out.headers().add(header.getKey(), header.getValue());+ }+ }+ }++ static int port(String host, String scheme) {+ int colon = host.lastIndexOf(':');+ if (colon > 0 && host.indexOf(']') < colon) {+ try {+ return Integer.parseInt(host.substring(colon + 1));+ } catch (NumberFormatException e) {+ return "https".equals(scheme) ? 443 : 80;+ }+ }+ return "https".equals(scheme) ? 443 : 80;+ }++ static String newRequestId() {+ StringBuilder id = new StringBuilder(REQUEST_ID_LENGTH);+ ThreadLocalRandom random = ThreadLocalRandom.current();+ for (int i = 0; i < REQUEST_ID_LENGTH; i++) {+ id.append(BASE32.charAt(random.nextInt(BASE32.length())));+ }+ return id.toString();+ }++ static void page(HttpServerRequest request, LambdaWebErrorPages.Page page, String requestId, boolean close) {+ HttpServerResponse response = request.response();+ response.setStatusCode(page.status()).setStatusMessage(page.reason());+ response.headers().set("content-type", "text/html");+ response.headers().set("connection", close ? "close" : "keep-alive");+ response.headers().set(REQUEST_ID_HEADER, requestId);+ response.headers().set("lambda-web-error-type", page.errorType());+ response.headers().set("date", HTTP_DATE.format(Instant.now()));+ response.end(LambdaWebErrorPages.html(page, requestId)).onComplete(ignored -> {+ if (close) {+ response.close();+ }+ });+ }++ /** The edge refuses a method before the platform sees the request, with its own header casing. */+ static void malformed(HttpServerRequest request) {+ HttpServerResponse response = request.response();+ response.setStatusCode(400).setStatusMessage("Bad Request");+ response.headers().set("Date", LambdaWebErrorPages.edgeDate(Instant.now()));+ response.headers().set("Connection", "close");+ response.headers().set("x-amz-request-id", Long.toHexString(ThreadLocalRandom.current().nextLong())+ .toUpperCase(Locale.ROOT));+ response.headers().set("Content-Type", "text/html; charset=utf-8");+ response.headers().set("Lambda-Web-Error-Type", LambdaWebErrorPages.MALFORMED_REQUEST_TYPE);+ response.end(LambdaWebErrorPages.MALFORMED_REQUEST_HTML).onComplete(ignored -> response.close());+ }++ /** AWS answers its own failures with 502 and no error type, so nothing is revealed. */+ private static void internalError(HttpServerRequest request, String requestId) {+ HttpServerResponse response = request.response();+ response.setStatusCode(502);+ response.headers().set(REQUEST_ID_HEADER, requestId);+ response.end();+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebEnvironmentManager.java b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebEnvironmentManager.javanew file mode 100644index 0000000000..2e2d639fd4--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebEnvironmentManager.java@@ -0,0 +1,573 @@+package io.github.hectorvent.floci.services.lambdaweb;++import com.github.dockerjava.api.async.ResultCallback;+import com.github.dockerjava.api.command.InspectContainerResponse;+import com.github.dockerjava.api.command.WaitContainerResultCallback;+import com.github.dockerjava.api.exception.NotFoundException;+import com.github.dockerjava.api.model.Frame;+import com.github.dockerjava.api.model.WaitResponse;+import io.github.hectorvent.floci.config.EmulatorConfig;+import io.github.hectorvent.floci.core.common.ContainerTeardown;+import io.github.hectorvent.floci.core.common.RequestScopes;+import io.github.hectorvent.floci.core.common.docker.ContainerBuilder;+import io.github.hectorvent.floci.core.common.docker.ContainerExec;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.ContainerInfo;+import io.github.hectorvent.floci.core.common.docker.ContainerLifecycleManager.EndpointInfo;+import io.github.hectorvent.floci.core.common.docker.ContainerSpec;+import io.github.hectorvent.floci.core.common.docker.ContainerStorageHelper;+import io.github.hectorvent.floci.core.common.docker.LaunchedContainerAwsEnv;+import io.github.hectorvent.floci.core.common.docker.RetryingTarCopier;+import io.github.hectorvent.floci.services.ecr.registry.EcrRegistryManager;+import io.github.hectorvent.floci.services.iam.model.SessionCreds;+import io.github.hectorvent.floci.services.lambda.launcher.ImageResolver;+import io.github.hectorvent.floci.services.lambda.launcher.LambdaExecutionRoleCredentials;+import io.github.hectorvent.floci.services.lambda.zip.ZipExtractor;+import io.github.hectorvent.floci.services.lambdaweb.model.WebFunction;+import io.github.hectorvent.floci.services.lambdaweb.model.WebFunctionRevision;+import io.github.hectorvent.floci.services.s3.S3Service;+import jakarta.annotation.PreDestroy;+import jakarta.enterprise.context.ApplicationScoped;+import jakarta.inject.Inject;+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;+import org.jboss.logging.Logger;++import java.io.IOException;+import java.io.OutputStream;+import java.io.UncheckedIOException;+import java.nio.charset.StandardCharsets;+import java.nio.file.Files;+import java.nio.file.Path;+import java.time.Duration;+import java.time.Instant;+import java.time.ZoneOffset;+import java.time.format.DateTimeFormatter;+import java.util.ArrayList;+import java.util.Comparator;+import java.util.List;+import java.util.Map;+import java.util.Optional;+import java.util.UUID;+import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.Executors;+import java.util.concurrent.ScheduledExecutorService;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicBoolean;+import java.util.concurrent.atomic.AtomicInteger;+import java.util.concurrent.atomic.AtomicLong;+import java.util.concurrent.atomic.AtomicReference;+import java.util.stream.Stream;++/**+ * The execution environments of Web Function revisions: one Node.js server container per revision,+ * started by the first request, reused while requests keep coming and removed after the Lambda idle+ * timeout. The runtime loads {@code index.js}, or the file {@code AWS_LAMBDA_NODEJS_ENTRYPOINT} names,+ * from {@code /var/task} with the node flags AWS uses.+ */+@ApplicationScoped+public class LambdaWebEnvironmentManager implements ContainerTeardown {++ static final int APP_PORT = 3000;+ static final String TASK_DIR = "/var/task";+ static final int MEMORY_MB = 2048;+ static final Duration LISTEN_TIMEOUT = Duration.ofSeconds(10);+ static final List<String> NODE_FLAGS = List.of("--expose-gc", "--max-http-header-size", "135168",+ "--no-experimental-require-module", "--max-semi-space-size=102", "--max-old-space-size=1024");+ private static final Logger LOG = Logger.getLogger(LambdaWebEnvironmentManager.class);+ private static final DateTimeFormatter LOG_STREAM_DATE =+ DateTimeFormatter.ofPattern("yyyy/MM/dd").withZone(ZoneOffset.UTC);+ private static final Duration READY_POLL = Duration.ofMillis(200);+ private static final int LISTEN_PROBE_TIMEOUT_SECONDS = 2;+ private static final String LISTEN_PROBE = "require('node:net').connect(" + APP_PORT + ", '127.0.0.1')"+ + ".once('connect', () => process.exit(0)).once('error', () => process.exit(1))";+ private static final int LOG_TAIL_LINES = 20;+ private static final Duration LOG_TAIL_TIMEOUT = Duration.ofSeconds(2);+ private static final Duration EXIT_WATCH_REARM = Duration.ofSeconds(1);++ /**+ * A started container. {@code acquire} reserves it for one request and {@code release} gives+ * the reservation back; the idle reaper only stops an environment with no reservation.+ */+ public static final class Environment {+ private final String key;+ private final String accountId;+ private final String containerId;+ private final String host;+ private final int port;+ private final String logStream;+ private final Optional<SessionCreds> roleCredentials;+ private final AtomicInteger inFlight = new AtomicInteger();+ private final AtomicLong lastUsed = new AtomicLong(System.nanoTime());+ private final AtomicBoolean alive = new AtomicBoolean(true);+ private final AtomicReference<Instant> exitedAt = new AtomicReference<>();++ Environment(String key, String accountId, String containerId, String host, int port, String logStream,+ Optional<SessionCreds> roleCredentials) {+ this.key = key;+ this.accountId = accountId;+ this.containerId = containerId;+ this.host = host;+ this.port = port;+ this.logStream = logStream;+ this.roleCredentials = roleCredentials;+ }++ public String containerId() {+ return containerId;+ }++ public String host() {+ return host;+ }++ public int port() {+ return port;+ }++ public String logStream() {+ return logStream;+ }++ public boolean isAlive() {+ return alive.get();+ }++ public void release() {+ inFlight.decrementAndGet();+ lastUsed.set(System.nanoTime());+ }++ private void reserve() {+ inFlight.incrementAndGet();+ lastUsed.set(System.nanoTime());+ }+ }++ /** The environment did not come up. {@code page} is what the endpoint answers. */+ public static final class StartFailure extends RuntimeException {+ private final transient LambdaWebErrorPages.Page page;++ StartFailure(LambdaWebErrorPages.Page page) {+ super(page.errorType());+ this.page = page;+ }++ LambdaWebErrorPages.Page page() {+ return page;+ }+ }++ private final EmulatorConfig config;+ private final ContainerBuilder containerBuilder;+ private final ContainerLifecycleManager lifecycleManager;+ private final ImageResolver imageResolver;+ private final EcrRegistryManager ecrRegistryManager;+ private final S3Service s3Service;+ private final ZipExtractor zipExtractor;+ private final LaunchedContainerAwsEnv awsEnv;+ private final LambdaExecutionRoleCredentials executionRoleCredentials;+ private final ConcurrentHashMap<String, Environment> environments = new ConcurrentHashMap<>();+ private final ConcurrentHashMap<String, Object> locks = new ConcurrentHashMap<>();+ private final ScheduledExecutorService reaper = Executors.newSingleThreadScheduledExecutor(task -> {+ Thread thread = new Thread(task, "lambda-web-environments");+ thread.setDaemon(true);+ return thread;+ });++ @Inject+ public LambdaWebEnvironmentManager(EmulatorConfig config, ContainerBuilder containerBuilder,+ ContainerLifecycleManager lifecycleManager, ImageResolver imageResolver,+ EcrRegistryManager ecrRegistryManager, S3Service s3Service,+ ZipExtractor zipExtractor, LaunchedContainerAwsEnv awsEnv,+ LambdaExecutionRoleCredentials executionRoleCredentials) {+ this.config = config;+ this.containerBuilder = containerBuilder;+ this.lifecycleManager = lifecycleManager;+ this.imageResolver = imageResolver;+ this.ecrRegistryManager = ecrRegistryManager;+ this.s3Service = s3Service;+ this.zipExtractor = zipExtractor;+ this.awsEnv = awsEnv;+ this.executionRoleCredentials = executionRoleCredentials;+ long period = Math.max(1, Math.min(30, config.services().lambda().containerIdleTimeoutSeconds() / 2 + 1));+ reaper.scheduleWithFixedDelay(this::reapIdle, period, period, TimeUnit.SECONDS);+ }++ /**+ * The running environment of {@code revision}, started if there is none, reserved for one+ * request. The caller releases it when the request is over. Blocks on Docker and S3.+ */+ public Environment acquire(String accountId, String region, WebFunction fn, WebFunctionRevision revision) {+ String key = String.join("|", accountId, region, fn.functionName, revision.revisionId);+ synchronized (lockFor(key)) {+ Environment existing = environments.get(key);+ if (existing != null && existing.isAlive()) {+ existing.reserve();+ return existing;+ }+ Environment started = start(key, accountId, region, fn, revision);+ started.reserve();+ environments.put(key, started);+ return started;+ }+ }++ /** Stops an environment, with the one second AWS gives the process after SIGTERM. */+ public void discard(Environment environment) {+ if (!environment.alive.compareAndSet(true, false)) {+ environments.remove(environment.key, environment);+ return;+ }+ environments.remove(environment.key, environment);+ try {+ exitCodeIfStopped(environment.containerId).ifPresent(exitCode ->+ LOG.infov("The Lambda Web environment {0} had exited with status {1}: {2}",+ environment.containerId, exitCode, lastLogLines(environment.containerId)));+ } catch (RuntimeException e) {+ LOG.debugv(e, "Could not inspect the Lambda Web environment {0}", environment.containerId);+ }+ try {+ lifecycleManager.stopAndRemove(environment.containerId, null, 1);+ } finally {+ forgetCredentials(environment);+ }+ }++ /**+ * Whether the process had already exited when a request that found it dead was sent. Such a+ * request gets a fresh environment, as on AWS, while a process that dies under a request fails it.+ * A container that is already gone answers no: nothing proves the exit came first.+ */+ public boolean exitedBefore(Environment environment, Instant moment) {+ Instant exited = environment.exitedAt.get();+ if (exited == null) {+ exited = finishedAt(environment.containerId).orElse(null);+ if (exited != null) {+ environment.exitedAt.compareAndSet(null, exited);+ }+ }+ return exited != null && exited.isBefore(moment);+ }++ @Override+ public void stopManagedContainers() {+ new ArrayList<>(environments.values()).forEach(this::discard);+ }++ @PreDestroy+ void shutdown() {+ reaper.shutdownNow();+ }++ private Object lockFor(String key) {+ return locks.computeIfAbsent(key, ignored -> new Object());+ }++ private void reapIdle() {+ long idleNanos = TimeUnit.SECONDS.toNanos(config.services().lambda().containerIdleTimeoutSeconds());+ for (Environment environment : new ArrayList<>(environments.values())) {+ try {+ synchronized (lockFor(environment.key)) {+ if (environment.inFlight.get() != 0+ || System.nanoTime() - environment.lastUsed.get() <= idleNanos) {+ continue;+ }+ LOG.infov("Removing the idle Lambda Web environment {0}", environment.containerId);+ discard(environment);+ }+ } catch (RuntimeException e) {+ LOG.warnv(e, "Could not remove the idle Lambda Web environment {0}", environment.containerId);+ }+ }+ }++ private Environment start(String key, String accountId, String region, WebFunction fn,+ WebFunctionRevision revision) {+ String image = ecrRegistryManager.rewriteImageUri(imageResolver.resolve(revision.buildConfig.runtime));+ String shortId = UUID.randomUUID().toString().replace("-", "").substring(0, 8);+ String name = ContainerStorageHelper.prefixedDockerName(config, "floci-lambda-web",+ fn.functionName + "-" + revision.revisionId + "-" + shortId);+ Map<String, String> userEnv = revision.serviceConfig.environmentVariables == null+ ? Map.of() : revision.serviceConfig.environmentVariables;+ List<String> cmd = new ArrayList<>(NODE_FLAGS);+ cmd.add(userEnv.getOrDefault("AWS_LAMBDA_NODEJS_ENTRYPOINT", "index.js"));+ Optional<SessionCreds> roleCredentials = executionRoleCredentials.forRole(+ revision.serviceConfig.executionRoleArn, accountId, fn.functionName);+ ContainerSpec spec = containerBuilder.newContainer(image)+ .withName(name)+ .withEnv(environment(accountId, region, revision, userEnv, roleCredentials))+ .withEntrypoint(List.of("/var/lang/bin/node"))+ .withCmd(cmd)+ .withWorkingDir(TASK_DIR)+ .withMemoryMb(MEMORY_MB)+ .withLoopbackPortBinding(APP_PORT, 0)+ .withDockerNetwork(config.services().lambda().dockerNetwork())+ .withHostDockerInternalOnLinux()+ .withEmbeddedDns()+ .withLogRotation()+ .withLabels(ContainerStorageHelper.resourceIdentityLabels("lambda-web", fn.functionName, accountId, region))+ .build();+ String containerId;+ try {+ containerId = lifecycleManager.create(spec);+ } catch (RuntimeException e) {+ roleCredentials.ifPresent(creds -> executionRoleCredentials.unregister(accountId, creds.accessKeyId()));+ throw e;+ }+ try {+ copyCode(containerId, accountId, region, fn, revision);+ ContainerInfo info = lifecycleManager.startCreated(containerId, spec);+ EndpointInfo endpoint = info.getEndpoint(APP_PORT);+ awaitListening(containerId);+ String logStream = LOG_STREAM_DATE.format(Instant.now()) + "/" + fn.functionName + "[" + revision.revisionId+ + "]" + UUID.randomUUID().toString().replace("-", "");+ Environment environment = new Environment(key, accountId, containerId, endpoint.host(), endpoint.port(),+ logStream, roleCredentials);+ watchExit(environment);+ LOG.infov("Started the Lambda Web environment {0} of {1} revision {2} at {3}:{4}",+ containerId, fn.functionName, revision.revisionId, endpoint.host(), endpoint.port());+ return environment;+ } catch (RuntimeException e) {+ lifecycleManager.stopAndRemove(containerId, null, 1);+ roleCredentials.ifPresent(creds -> executionRoleCredentials.unregister(accountId, creds.accessKeyId()));+ throw e;+ }+ }++ private List<String> environment(String accountId, String region, WebFunctionRevision revision,+ Map<String, String> userEnv, Optional<SessionCreds> roleCredentials) {+ List<String> env = new ArrayList<>();+ env.add("AWS_LAMBDA_HTTP_ENDPOINT=0.0.0.0:" + APP_PORT);+ env.add("AWS_LAMBDA_INITIALIZATION_TYPE=native-http");+ env.add("AWS_EXECUTION_ENV=AWS_Lambda_" + revision.buildConfig.runtime + "-lite");+ env.add("AWS_LAMBDA_FUNCTION_MEMORY_SIZE=" + MEMORY_MB);+ env.add("AWS_LAMBDA_LOG_LEVEL=" + revision.serviceConfig.applicationLogLevel);+ env.add("AWS_ACCOUNT_ID=" + accountId);+ env.add("NODE_PATH=" + TASK_DIR);+ env.addAll(awsEnv.sdkBaselineEnv(region, Optional.empty(), roleCredentials, accountId));+ userEnv.forEach((name, value) -> env.add(name + "=" + value));+ return env;+ }++ private void forgetCredentials(Environment environment) {+ environment.roleCredentials.ifPresent(creds ->+ executionRoleCredentials.unregister(environment.accountId, creds.accessKeyId()));+ }++ /** The code goes in by {@code docker cp}, so it works when Floci itself runs in a container. */+ private void copyCode(String containerId, String accountId, String region, WebFunction fn,+ WebFunctionRevision revision) {+ WebFunctionRevision.BuildConfig build = revision.buildConfig;+ byte[] zip = RequestScopes.callAs(accountId, region,+ () -> s3Service.getObject(build.bucket, build.key, build.versionId)).getData();+ Path dir;+ try {+ dir = Files.createTempDirectory("floci-lambda-web-");+ } catch (IOException e) {+ throw new UncheckedIOException("Could not make room for the code of " + fn.functionName, e);+ }+ try {+ zipExtractor.extractTo(zip, dir, config.services().lambda().zipMaxEntries());+ RetryingTarCopier.copyStreamed(lifecycleManager.getDockerClient(), containerId, TASK_DIR,+ "web-" + fn.functionName, out -> tar(dir, out));+ } catch (IOException e) {+ throw new UncheckedIOException("Could not unpack the code of " + fn.functionName + " revision "+ + revision.revisionId, e);+ } finally {+ deleteRecursively(dir);+ }+ }++ private static void tar(Path dir, OutputStream out) throws IOException {+ try (TarArchiveOutputStream tar = new TarArchiveOutputStream(out); Stream<Path> walk = Files.walk(dir)) {+ tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);+ tar.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX);+ List<Path> paths = walk.filter(path -> !path.equals(dir)).sorted().toList();+ for (Path path : paths) {+ boolean directory = Files.isDirectory(path);+ String name = dir.relativize(path).toString().replace('\\', '/');+ TarArchiveEntry entry = new TarArchiveEntry(path.toFile(), directory ? name + "/" : name);+ entry.setMode(directory || Files.isExecutable(path) ? 0755 : 0644);+ tar.putArchiveEntry(entry);+ if (!directory) {+ Files.copy(path, tar);+ }+ tar.closeArchiveEntry();+ }+ }+ }++ private static void deleteRecursively(Path dir) {+ try (Stream<Path> walk = Files.walk(dir)) {+ List<Path> paths = walk.sorted(Comparator.reverseOrder()).toList();+ for (Path path : paths) {+ Files.deleteIfExists(path);+ }+ } catch (IOException e) {+ LOG.debugv(e, "Could not remove the unpacked code at {0}", dir);+ }+ }++ private void awaitListening(String containerId) {+ long deadline = System.nanoTime() + LISTEN_TIMEOUT.toNanos();+ while (true) {+ Optional<Long> exitCode = exitCodeIfStopped(containerId);+ if (exitCode.isPresent()) {+ LOG.infov("The Lambda Web environment {0} exited with status {1} before it listened: {2}",+ containerId, exitCode.get(), lastLogLines(containerId));+ throw new StartFailure(exitCode.get() == 0+ ? LambdaWebErrorPages.RUNTIME_EXIT : LambdaWebErrorPages.INIT_CODE_ERROR);+ }+ if (isListening(containerId)) {+ return;+ }+ if (System.nanoTime() > deadline) {+ throw new StartFailure(LambdaWebErrorPages.TIMEOUT_LISTEN_ERROR);+ }+ try {+ Thread.sleep(READY_POLL.toMillis());+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ throw new StartFailure(LambdaWebErrorPages.TIMEOUT_LISTEN_ERROR);+ }+ }+ }++ /**+ * Probes from inside the container. A published port on Docker Desktop accepts a connection+ * before anything listens behind it, so a connect from the host proves nothing.+ */+ private boolean isListening(String containerId) {+ try {+ ContainerExec.Result result = ContainerExec.runMerged(lifecycleManager.getDockerClient(), containerId,+ new String[] {"/var/lang/bin/node", "-e", LISTEN_PROBE}, LISTEN_PROBE_TIMEOUT_SECONDS);+ return !result.timedOut() && result.exitCode() == 0;+ } catch (RuntimeException e) {+ LOG.debugv(e, "Could not probe the Lambda Web environment {0}", containerId);+ return false;+ }+ }++ private String lastLogLines(String containerId) {+ StringBuilder lines = new StringBuilder();+ try {+ lifecycleManager.getDockerClient().logContainerCmd(containerId)+ .withStdOut(true).withStdErr(true).withTail(LOG_TAIL_LINES)+ .exec(new ResultCallback.Adapter<Frame>() {+ @Override+ public void onNext(Frame frame) {+ lines.append(new String(frame.getPayload(), StandardCharsets.UTF_8));+ }+ }).awaitCompletion(LOG_TAIL_TIMEOUT.toMillis(), TimeUnit.MILLISECONDS);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ } catch (RuntimeException e) {+ LOG.debugv(e, "Could not read the log of the Lambda Web environment {0}", containerId);+ }+ return lines.toString().strip();+ }++ private Optional<InspectContainerResponse.ContainerState> stoppedState(String containerId) {+ try {+ InspectContainerResponse.ContainerState state =+ lifecycleManager.getDockerClient().inspectContainerCmd(containerId).exec().getState();+ return state == null || Boolean.TRUE.equals(state.getRunning()) ? Optional.empty() : Optional.of(state);+ } catch (NotFoundException e) {+ return Optional.empty();+ }+ }++ private Optional<Long> exitCodeIfStopped(String containerId) {+ try {+ lifecycleManager.getDockerClient().inspectContainerCmd(containerId).exec();+ } catch (NotFoundException e) {+ return Optional.of(-1L);+ }+ return stoppedState(containerId).map(state -> {+ Long exitCode = state.getExitCodeLong();+ return exitCode == null ? -1L : exitCode;+ });+ }++ private Optional<Instant> finishedAt(String containerId) {+ try {+ return stoppedState(containerId).map(state -> {+ String finishedAt = state.getFinishedAt();+ return finishedAt == null || finishedAt.startsWith("0001") ? Instant.now() : Instant.parse(finishedAt);+ });+ } catch (RuntimeException e) {+ LOG.debugv(e, "Could not inspect the Lambda Web environment {0}", containerId);+ return Optional.empty();+ }+ }++ /**+ * A process that exits between requests is invisible to clients: the next request starts anew.+ * The wait stream can drop on a quiet container, so a dropped watch looks at the container and+ * arms itself again while the environment lives.+ */+ private void watchExit(Environment environment) {+ try {+ lifecycleManager.getDockerClient().waitContainerCmd(environment.containerId)+ .exec(new WaitContainerResultCallback() {+ @Override+ public void onNext(WaitResponse response) {+ super.onNext(response);+ exited(environment, response.getStatusCode());+ }++ @Override+ public void onError(Throwable throwable) {+ super.onError(throwable);+ LOG.debugv(throwable, "The exit watch of the Lambda Web environment {0} dropped",+ environment.containerId);+ rearmExitWatch(environment);+ }+ });+ } catch (RuntimeException e) {+ LOG.debugv(e, "Could not watch the exit of the Lambda Web environment {0}", environment.containerId);+ rearmExitWatch(environment);+ }+ }++ private void rearmExitWatch(Environment environment) {+ if (!environment.isAlive() || reaper.isShutdown()) {+ return;+ }+ reaper.schedule(() -> {+ if (!environment.isAlive()) {+ return;+ }+ Optional<InspectContainerResponse.ContainerState> stopped;+ try {+ stopped = stoppedState(environment.containerId);+ } catch (RuntimeException e) {+ stopped = Optional.empty();+ }+ if (stopped.isPresent()) {+ exited(environment, stopped.get().getExitCodeLong() == null+ ? null : stopped.get().getExitCodeLong().intValue());+ return;+ }+ watchExit(environment);+ }, EXIT_WATCH_REARM.toMillis(), TimeUnit.MILLISECONDS);+ }++ private void exited(Environment environment, Integer statusCode) {+ if (!environment.alive.compareAndSet(true, false)) {+ return;+ }+ environment.exitedAt.compareAndSet(null, finishedAt(environment.containerId).orElse(Instant.now()));+ environments.remove(environment.key, environment);+ reaper.execute(() -> {+ try {+ LOG.infov("The Lambda Web environment {0} exited with status {1}: {2}",+ environment.containerId, statusCode, lastLogLines(environment.containerId));+ lifecycleManager.stopAndRemove(environment.containerId, null, 0);+ } finally {+ forgetCredentials(environment);+ }+ });+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebErrorPages.java b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebErrorPages.javanew file mode 100644index 0000000000..d62add6c65--- /dev/null+++ b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebErrorPages.java@@ -0,0 +1,53 @@+package io.github.hectorvent.floci.services.lambdaweb;++import java.time.Instant;+import java.time.ZoneOffset;+import java.time.format.DateTimeFormatter;+import java.util.Locale;++/** The HTML pages a Web Function endpoint answers with, as recorded against AWS. */+final class LambdaWebErrorPages {++ record Page(int status, String reason, String errorType, String message) {+ }++ private static final String APPLICATION_ERROR = "Request failed due to an application error.";++ static final Page UNSUPPORTED_HTTP_VERSION = new Page(505, "HTTP Version Not Supported",+ "Platform.UnsupportedHttpVersion", "The provided HTTP version is not supported. Please use HTTP/1.1.");+ static final Page INVALID_ENDPOINT = new Page(404, "Not Found", "Platform.InvalidEndpoint",+ "The requested endpoint could not be found.");+ static final Page ENDPOINT_DISABLED = new Page(503, "Service Unavailable", "Platform.EndpointDisabled",+ "The service is temporarily unable to handle your request. Please try again later.");+ static final Page REQUEST_TIMEOUT = new Page(504, "Gateway Timeout", "Runtime.RequestTimeout",+ "The server failed to respond within the required time limit.");+ static final Page ORIGIN_REQUEST_ERROR = new Page(503, "Service Unavailable", "Runtime.OriginRequestError",+ APPLICATION_ERROR);+ static final Page TIMEOUT_LISTEN_ERROR = new Page(502, "Bad Gateway", "Application.TimeoutListenError",+ APPLICATION_ERROR);+ static final Page INIT_CODE_ERROR = new Page(502, "Bad Gateway", "Application.InitCodeError", APPLICATION_ERROR);+ static final Page RUNTIME_EXIT = new Page(502, "Bad Gateway", "Runtime.Exit", APPLICATION_ERROR);++ /** The edge in front of the platform refuses a method itself, with its own page and header casing. */+ static final String MALFORMED_REQUEST_TYPE = "Platform.MalformedRequest";+ static final String MALFORMED_REQUEST_HTML = "<!DOCTYPE html>\n<html>\n<head>\n<title>HTTP Error 400</title>\n"+ + "</head>\n<body>\n<h1>400 Bad Request</h1>\n"+ + "<p>The request could not be parsed as a valid HTTP request.</p>\n</body>\n</html>\n";+ /** The edge writes a two-digit year in its Date header. */+ private static final DateTimeFormatter EDGE_DATE =+ DateTimeFormatter.ofPattern("EEE, dd MMM yy HH:mm:ss 'GMT'", Locale.ROOT).withZone(ZoneOffset.UTC);++ private LambdaWebErrorPages() {+ }++ static String html(Page page, String requestId) {+ return "<!DOCTYPE html>\n<html>\n<head>\n <title>HTTP Error " + page.status() + "</title>\n</head>\n<body>\n"+ + " <h1>" + page.status() + " " + page.reason() + "</h1>\n"+ + " <p>" + page.message() + "</p>\n"+ + " <p>Request ID: " + requestId + "</p>\n</body>\n</html>";+ }++ static String edgeDate(Instant now) {+ return EDGE_DATE.format(now);+ }+}diff --git a/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebService.java b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebService.javaindex 2131bcec81..a06586b703 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebService.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebService.java@@ -10,7 +10,9 @@ import io.github.hectorvent.floci.core.common.PaginatedResult;import io.github.hectorvent.floci.core.common.Pagination;import io.github.hectorvent.floci.core.common.RegionResolver;import io.github.hectorvent.floci.core.common.RequestContext;+import io.github.hectorvent.floci.core.common.RequestScopes;import io.github.hectorvent.floci.core.common.Resettable;+import io.github.hectorvent.floci.core.storage.AccountAwareStorageBackend;import io.github.hectorvent.floci.core.storage.StorageBackend;import io.github.hectorvent.floci.core.storage.StorageFactory;import io.github.hectorvent.floci.services.lambdaweb.model.WebFunction;@@ -41,6 +43,7 @@ import java.util.Optional;import java.util.Set;import java.util.UUID;import java.util.concurrent.ConcurrentHashMap;+import java.util.concurrent.ThreadLocalRandom;import java.util.function.Consumer;import java.util.function.Function;import java.util.regex.Matcher;@@ -137,6 +140,8 @@ public class LambdaWebService implements Resettable {/** Keyed {@code region/functionName}. The backend adds the caller's account to every key. */private StorageBackend<String, WebFunction> functions;private final ConcurrentHashMap<String, Object> locks = new ConcurrentHashMap<>();+ /** Every endpoint domain, filled from storage at start and kept by the create and delete paths. */+ private final ConcurrentHashMap<String, EndpointLocator> domains = new ConcurrentHashMap<>();@Injectpublic LambdaWebService(RegionResolver regionResolver, StorageFactory storageFactory, EmulatorConfig config,@@ -157,11 +162,74 @@ public class LambdaWebService implements Resettable {void initializeStorage() {this.functions = storageFactory.create("lambda", "lambda-web-functions.json",new TypeReference<Map<String, WebFunction>>() {});+ domains.clear();+ List<WebFunction> all = functions instanceof AccountAwareStorageBackend<WebFunction> aware+ ? aware.scanAllAccounts()+ : functions.scan(key -> true);+ for (WebFunction fn : all) {+ fn.endpoints.values().forEach(endpoint -> indexDomains(fn, endpoint));+ }}@Overridepublic void clear() {functions.clear();+ domains.clear();+ }++ /** Where a request to an endpoint domain goes. Domains are unique across accounts and regions. */+ public record WebFunctionTarget(String accountId, String region, WebFunction function, WebFunctionEndpoint endpoint) {+ }++ private record EndpointLocator(String accountId, String region, String functionName, String endpointName) {+ }++ public Optional<WebFunctionTarget> locate(String host) {+ EndpointLocator locator = domains.get(host);+ if (locator == null) {+ return Optional.empty();+ }+ return RequestScopes.callAs(locator.accountId(), locator.region(), () -> {+ synchronized (lockFor(locator.region(), locator.functionName())) {+ Optional<WebFunction> stored = functions.get(key(locator.region(), locator.functionName()));+ if (stored.isEmpty() || !stored.get().endpoints.containsKey(locator.endpointName())) {+ domains.remove(host, locator);+ return Optional.empty();+ }+ WebFunction fn = copy(stored.get(), WebFunction.class);+ return Optional.of(new WebFunctionTarget(locator.accountId(), locator.region(), fn,+ fn.endpoints.get(locator.endpointName())));+ }+ });+ }++ private void indexDomains(WebFunction fn, WebFunctionEndpoint endpoint) {+ AwsArnUtils.Arn arn = AwsArnUtils.parse(fn.functionArn);+ EndpointLocator locator = new EndpointLocator(arn.accountId(), arn.region(), fn.functionName, endpoint.endpointName);+ domains.put(endpoint.domainName, locator);+ endpoint.regionalDomainNames.values().forEach(domain -> domains.put(domain, locator));+ }++ private void unindexDomains(WebFunctionEndpoint endpoint) {+ domains.remove(endpoint.domainName);+ endpoint.regionalDomainNames.values().forEach(domains::remove);+ }++ /** The revision a request goes to, drawn by the endpoint's weights. */+ public static WebFunctionRevision pickRevision(WebFunctionTarget target) {+ List<WebFunctionEndpoint.RevisionWeight> weights = target.endpoint().revisionWeights;+ if (weights.isEmpty()) {+ return null;+ }+ int draw = ThreadLocalRandom.current().nextInt(100);+ int cumulative = 0;+ for (WebFunctionEndpoint.RevisionWeight weight : weights) {+ cumulative += weight.weight;+ if (draw < cumulative) {+ return target.function().revisions.get(weight.revisionId);+ }+ }+ return target.function().revisions.get(weights.get(weights.size() - 1).revisionId);}/** An endpoint before and after an update. The update response shows both. */@@ -206,6 +274,7 @@ public class LambdaWebService implements Resettable {attachEndpoint(region, fn, endpoint, revision);fn.revisions.put(revision.revisionId, revision);fn.endpoints.put(endpoint.endpointName, endpoint);+ indexDomains(fn, endpoint);LOG.infov("Created web function {0} with revision {1} and endpoint {2}",functionName, revision.revisionId, endpoint.endpointName);});@@ -276,7 +345,8 @@ public class LambdaWebService implements Resettable {public void deleteFunction(String region, String functionNameOrArn) {String functionName = resolveFunctionName(region, functionNameOrArn, "DeleteWebFunction");synchronized (lockFor(region, functionName)) {- function(region, functionName);+ WebFunction fn = function(region, functionName);+ fn.endpoints.values().forEach(this::unindexDomains);functions.delete(key(region, functionName));}LOG.infov("Deleted web function {0}", functionName);@@ -569,6 +639,7 @@ public class LambdaWebService implements Resettable {WebFunction fn = function(region, functionName);attachEndpoint(region, fn, endpoint, null);fn.endpoints.put(endpoint.endpointName, endpoint);+ indexDomains(fn, endpoint);fn.updatedAt = now();persist(region, fn);}@@ -665,6 +736,7 @@ public class LambdaWebService implements Resettable {WebFunction fn = function(region, functionName);WebFunctionEndpoint endpoint = endpoint(fn, endpointName);fn.endpoints.remove(endpoint.endpointName);+ unindexDomains(endpoint);fn.updatedAt = now();persist(region, fn);}diff --git a/src/main/java/io/github/hectorvent/floci/services/s3/S3VirtualHostFilter.java b/src/main/java/io/github/hectorvent/floci/services/s3/S3VirtualHostFilter.javaindex ca1e0293ef..f6f246bb5a 100644--- a/src/main/java/io/github/hectorvent/floci/services/s3/S3VirtualHostFilter.java+++ b/src/main/java/io/github/hectorvent/floci/services/s3/S3VirtualHostFilter.java@@ -59,6 +59,7 @@ public class S3VirtualHostFilter implements ContainerRequestFilter {private static final Set<String> NON_S3_SERVICE_LABELS = Set.of("execute-api", // ApiGatewayExecuteApiHostFilter: <api-id>.execute-api[.<region>].<host>"lambda-url", // LambdaUrlRoutingFilter: <url-id>.lambda-url.<region>.<host>+ "lambda-web", // LambdaWebDataPlane: <id>.<shard>.lambda-web.<region>.<host>, <id>.lambda-web.global.<host>"emr-serverless", // EmrServerlessRouteFilter: emr-serverless.<host>"cloudfront"); // CloudFrontDistributionFilter: <dist-id>.cloudfront.netdiff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex 56849e3fc4..35d17cc67e 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -31,6 +31,8 @@ quarkus:require-localhost: falselimits:max-body-size: ${floci.protocols.max-request-size}M+ # Lambda Web Functions accept a request head of up to 128 KiB and answer 431 past it.+ max-header-size: 128Kshutdown:# Enables the pre-shutdown phase so stop hooks run before the HTTP server stops.# No 'delay' value is set - observers run synchronously, no artificial sleep is introduced.diff --git a/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneDockerIntegrationTest.javanew file mode 100644index 0000000000..beedfb9251--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneDockerIntegrationTest.java@@ -0,0 +1,234 @@+package io.github.hectorvent.floci.services.lambdaweb;++import com.github.dockerjava.api.DockerClient;+import io.github.hectorvent.floci.services.s3.S3Service;+import io.quarkus.test.junit.QuarkusTest;+import io.restassured.RestAssured;+import io.restassured.response.Response;+import jakarta.inject.Inject;+import org.junit.jupiter.api.AfterEach;+import org.junit.jupiter.api.Assumptions;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.Timeout;++import java.io.ByteArrayOutputStream;+import java.io.IOException;+import java.io.InputStream;+import java.io.OutputStream;+import java.net.Socket;+import java.nio.charset.StandardCharsets;+import java.util.Map;+import java.util.zip.ZipEntry;+import java.util.zip.ZipOutputStream;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.matchesPattern;+import static org.hamcrest.Matchers.not;+import static org.hamcrest.Matchers.nullValue;+import static org.hamcrest.Matchers.startsWith;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertNotEquals;+import static org.junit.jupiter.api.Assertions.assertTrue;++/**+ * Requests to an endpoint domain reach a Node.js server that Floci starts from the revision's S3+ * code, with the headers AWS adds and drops, the recorded error pages and a fresh environment after+ * a crash.+ */+@QuarkusTest+class LambdaWebDataPlaneDockerIntegrationTest {++ private static final String BASE = "/2025-03-07";+ private static final String BUCKET = "web-code-docker";+ private static final String ROLE = "arn:aws:iam::000000000000:role/web";+ private static final String REQUEST_ID = "[A-Z2-7]{26}";+ private static final String APP = """+ const http = require('node:http');+ const [host, port] = process.env.AWS_LAMBDA_HTTP_ENDPOINT.split(':');+ http.createServer((req, res) => {+ if (req.url.startsWith('/slow')) { setTimeout(() => res.end('late'), 8000); return; }+ if (req.url.startsWith('/stream')) {+ res.writeHead(200, {'content-type': 'text/plain'});+ let i = 0;+ const timer = setInterval(() => {+ res.write('chunk ' + (i++) + '\\n');+ if (i >= 10) { clearInterval(timer); res.end('done\\n'); }+ }, 500);+ return;+ }+ if (req.url.startsWith('/status/')) {+ res.writeHead(Number(req.url.split('/')[2]), {'x-app': 'yes'}); res.end('status body'); return;+ }+ if (req.url.startsWith('/crash')) { setTimeout(() => process.exit(1), 50); return; }+ let body = '';+ req.on('data', (chunk) => { body += chunk; });+ req.on('end', () => {+ res.writeHead(200, {'content-type': 'application/json', 'connection': 'keep-alive', 'trailer': 'x-t'});+ res.end(JSON.stringify({method: req.method, url: req.url, headers: req.headers, bodyLength: body.length,+ env: {endpoint: process.env.AWS_LAMBDA_HTTP_ENDPOINT, init: process.env.AWS_LAMBDA_INITIALIZATION_TYPE,+ exec: process.env.AWS_EXECUTION_ENV, account: process.env.AWS_ACCOUNT_ID,+ region: process.env.AWS_REGION, custom: process.env.CUSTOM}}));+ });+ }).listen(Number(port), host);+ """;++ @Inject+ DockerClient dockerClient;++ @Inject+ S3Service s3Service;++ @Inject+ LambdaWebEnvironmentManager environments;++ private String functionName;++ @BeforeEach+ void requireDocker() {+ Assumptions.assumeTrue(isDockerAvailable(), "Docker must be available for the Lambda Web data plane");+ }++ @AfterEach+ void stopEnvironments() {+ environments.stopManagedContainers();+ if (functionName != null) {+ given().when().delete(BASE + "/web-functions/" + functionName);+ }+ }++ @Test+ @Timeout(180)+ void requestsReachTheServerWithTheHeadersAwsAddsAndDrops() throws Exception {+ String domain = createFunction(30);++ Response echo = given().header("Host", domain)+ .header("x-real-ip", "1.2.3.4")+ .header("x-http-method-override", "PUT")+ .header("lambda-web-request-id", "fake")+ .header("x-forwarded-for", "9.9.9.9")+ .header("x-custom", "1")+ .when().get("/a/b?x=1");+ echo.then().statusCode(200)+ .header("content-type", "application/json")+ .header("lambda-web-request-id", matchesPattern(REQUEST_ID))+ .header("lambda-web-logstream", matchesPattern("\\d{4}/\\d{2}/\\d{2}/" + functionName + "\\[rev-[a-z0-9]{5}\\][0-9a-f]{32}"))+ .header("trailer", nullValue())+ .body("method", equalTo("GET"))+ .body("url", equalTo("/a/b?x=1"))+ .body("headers.host", equalTo(domain))+ .body("headers.x-custom", equalTo("1"))+ .body("headers.x-real-ip", nullValue())+ .body("headers.x-http-method-override", nullValue())+ .body("headers.lambda-web-endpoint-name", equalTo("dev"))+ .body("headers.x-forwarded-proto", equalTo("http"))+ .body("headers.x-forwarded-port", equalTo("4566"))+ .body("headers.x-forwarded-for", startsWith("9.9.9.9, "))+ .body("headers.connection", equalTo("close"))+ .body("env.endpoint", equalTo("0.0.0.0:3000"))+ .body("env.init", equalTo("native-http"))+ .body("env.exec", equalTo("AWS_Lambda_nodejs24.x-lite"))+ .body("env.account", equalTo("000000000000"))+ .body("env.region", equalTo("us-east-1"))+ .body("env.custom", equalTo("yes"));+ assertEquals(echo.header("lambda-web-request-id"), echo.path("headers.lambda-web-request-id"),+ "the request id the app saw is the one in the response");++ given().header("Host", domain).body("hello body").when().post("/")+ .then().statusCode(200).body("bodyLength", equalTo(10));+ given().header("Host", domain).when().get("/status/418")+ .then().statusCode(418).header("x-app", "yes").body(equalTo("status body"));++ // A body without a Content-Length reaches the app chunked, as on AWS.+ String chunked = raw("POST /upload HTTP/1.1\r\nHost: " + domain + "\r\nTransfer-Encoding: chunked\r\n"+ + "Connection: close\r\n\r\n11\r\nchunked body here\r\n0\r\n\r\n");+ assertTrue(chunked.startsWith("HTTP/1.1 200 "), chunked);+ assertTrue(chunked.contains("\"bodyLength\":17"), chunked);+ assertTrue(chunked.contains("\"transfer-encoding\":\"chunked\""), chunked);+ }++ @Test+ @Timeout(180)+ void aTimeoutAndACrashAnswerWithThePlatformPagesAndTheNextRequestGetsAFreshEnvironment() throws Exception {+ String domain = createFunction(3);++ Response first = given().header("Host", domain).when().get("/status/200");+ first.then().statusCode(200);+ String logStream = first.header("lambda-web-logstream");++ Response timeout = given().header("Host", domain).when().get("/slow");+ timeout.then().statusCode(504)+ .header("lambda-web-error-type", "Runtime.RequestTimeout")+ .body(equalTo(LambdaWebErrorPages.html(LambdaWebErrorPages.REQUEST_TIMEOUT,+ timeout.header("lambda-web-request-id"))));++ // A response that is still streaming at the deadline is cut, as on AWS.+ String streamed = raw("GET /stream HTTP/1.1\r\nHost: " + domain + "\r\nConnection: close\r\n\r\n");+ assertTrue(streamed.startsWith("HTTP/1.1 200 "), streamed);+ assertTrue(streamed.contains("chunk 0"), streamed);+ assertFalse(streamed.contains("done"), streamed);++ Response crash = given().header("Host", domain).when().get("/crash");+ crash.then().statusCode(503)+ .header("lambda-web-error-type", "Runtime.OriginRequestError")+ .body(equalTo(LambdaWebErrorPages.html(LambdaWebErrorPages.ORIGIN_REQUEST_ERROR,+ crash.header("lambda-web-request-id"))));++ Response after = given().header("Host", domain).when().get("/status/200");+ after.then().statusCode(200).header("lambda-web-logstream", not(equalTo(logStream)));+ assertNotEquals(logStream, after.header("lambda-web-logstream"), "a new environment serves the next request");+ }++ private String createFunction(int timeoutSeconds) throws IOException {+ functionName = "dp-docker-" + Long.toString(System.nanoTime(), 36);+ s3Service.createBucket(BUCKET, "us-east-1");+ s3Service.putObject(BUCKET, functionName + ".zip", zip(APP), "application/zip", Map.of());+ return given().contentType("application/json")+ .body("""+ {"functionName": "%s",+ "revisionConfig": {+ "buildConfig": {"codeConfig": {"s3Object": {"bucket": "%s", "key": "%s.zip"}},+ "runtimeConfig": {"runtime": "nodejs24.x"}},+ "serviceConfig": {"executionRoleArn": "%s", "timeoutSeconds": %d,+ "environmentVariables": {"CUSTOM": "yes"}}},+ "endpointConfig": {"endpointName": "dev", "endpointType": "HomeRegion",+ "authType": "ApplicationManaged", "autoDeploymentMode": "LatestRevision"}}+ """.formatted(functionName, BUCKET, functionName, ROLE, timeoutSeconds))+ .when().put(BASE + "/web-functions")+ .then().statusCode(202)+ .extract().path("endpoint.domainName");+ }++ /** Sends one raw HTTP/1.1 request with {@code Connection: close} and reads until the server closes. */+ private static String raw(String request) throws IOException {+ try (Socket socket = new Socket("localhost", RestAssured.port)) {+ socket.setSoTimeout(30_000);+ OutputStream out = socket.getOutputStream();+ out.write(request.getBytes(StandardCharsets.US_ASCII));+ out.flush();+ InputStream in = socket.getInputStream();+ return new String(in.readAllBytes(), StandardCharsets.UTF_8);+ }+ }++ private static byte[] zip(String indexJs) throws IOException {+ ByteArrayOutputStream bytes = new ByteArrayOutputStream();+ try (ZipOutputStream zip = new ZipOutputStream(bytes)) {+ zip.putNextEntry(new ZipEntry("index.js"));+ zip.write(indexJs.getBytes(StandardCharsets.UTF_8));+ zip.closeEntry();+ }+ return bytes.toByteArray();+ }++ private boolean isDockerAvailable() {+ try {+ dockerClient.pingCmd().exec();+ return true;+ } catch (Exception e) {+ return false;+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneIntegrationTest.javanew file mode 100644index 0000000000..dc1682927a--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneIntegrationTest.java@@ -0,0 +1,123 @@+package io.github.hectorvent.floci.services.lambdaweb;++import io.github.hectorvent.floci.services.s3.S3Service;+import io.quarkus.test.junit.QuarkusTest;+import io.restassured.RestAssured;+import io.restassured.response.Response;+import jakarta.inject.Inject;+import org.junit.jupiter.api.Test;++import java.io.IOException;+import java.io.InputStream;+import java.io.OutputStream;+import java.net.Socket;+import java.nio.charset.StandardCharsets;+import java.util.Map;++import static io.restassured.RestAssured.given;+import static org.hamcrest.Matchers.equalTo;+import static org.hamcrest.Matchers.matchesPattern;+import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertTrue;++/** The platform answers that need no execution environment, so no Docker. */+@QuarkusTest+class LambdaWebDataPlaneIntegrationTest {++ private static final String BASE = "/2025-03-07";+ private static final String BUCKET = "web-code-data-plane";+ private static final String ROLE = "arn:aws:iam::000000000000:role/web";+ private static final String UNKNOWN_HOST = "abcdefghijklmnop.001.lambda-web.us-east-1.localhost:4566";+ private static final String REQUEST_ID = "[A-Z2-7]{26}";++ @Inject+ S3Service s3Service;++ @Test+ void anUnknownEndpointDomainIsNotFound() {+ Response response = given().header("Host", UNKNOWN_HOST).when().get("/some/path?q=1");+ response.then().statusCode(404)+ .header("content-type", "text/html")+ .header("connection", "keep-alive")+ .header("lambda-web-error-type", "Platform.InvalidEndpoint")+ .header("lambda-web-request-id", matchesPattern(REQUEST_ID))+ .header("content-length", "225");+ String requestId = response.header("lambda-web-request-id");+ assertEquals(LambdaWebErrorPages.html(LambdaWebErrorPages.INVALID_ENDPOINT, requestId), response.asString());++ // A domain the control plane minted but with another case or port is another domain.+ given().header("Host", UNKNOWN_HOST.toUpperCase()).when().get("/")+ .then().statusCode(404).header("lambda-web-error-type", "Platform.InvalidEndpoint");+ }++ @Test+ void aDisabledEndpointAnswersBeforeAnyEnvironmentStarts() {+ String functionName = "dp-disabled-" + Long.toString(System.nanoTime(), 36);+ String domain = createFunction(functionName);+ given().contentType("application/json").body("{\"throttleConfig\": {\"rateLimit\": 0}}")+ .when().patch(BASE + "/web-functions/" + functionName + "/endpoints/dev")+ .then().statusCode(202);++ Response response = given().header("Host", domain).when().get("/");+ response.then().statusCode(503)+ .header("lambda-web-error-type", "Platform.EndpointDisabled")+ .header("content-length", "274")+ .body(equalTo(LambdaWebErrorPages.html(LambdaWebErrorPages.ENDPOINT_DISABLED,+ response.header("lambda-web-request-id"))));++ given().when().delete(BASE + "/web-functions/" + functionName).then().statusCode(204);+ }++ @Test+ void httpOneDotZeroIsRefused() throws IOException {+ String response = raw("GET / HTTP/1.0\r\nHost: " + UNKNOWN_HOST + "\r\n\r\n");+ assertTrue(response.startsWith("HTTP/1.0 505 HTTP Version Not Supported\r\n"), response);+ assertTrue(response.contains("\r\nlambda-web-error-type: Platform.UnsupportedHttpVersion\r\n"), response);+ assertTrue(response.contains("\r\nconnection: close\r\n"), response);+ assertTrue(response.contains("<p>The provided HTTP version is not supported. Please use HTTP/1.1.</p>"), response);+ }++ @Test+ void anUnsupportedMethodIsRefusedByTheEdge() throws IOException {+ String response = raw("TRACE / HTTP/1.1\r\nHost: " + UNKNOWN_HOST + "\r\n\r\n");+ assertTrue(response.startsWith("HTTP/1.1 400 Bad Request\r\n"), response);+ assertTrue(response.contains("\r\nLambda-Web-Error-Type: Platform.MalformedRequest\r\n"), response);+ assertTrue(response.contains("\r\nContent-Type: text/html; charset=utf-8\r\n"), response);+ assertTrue(response.contains("\r\nConnection: close\r\n"), response);+ assertTrue(response.endsWith(LambdaWebErrorPages.MALFORMED_REQUEST_HTML), response);+ }++ @Test+ void otherHostsStillReachTheControlPlane() {+ given().when().get(BASE + "/web-account-settings").then().statusCode(200);+ }++ private String createFunction(String functionName) {+ s3Service.createBucket(BUCKET, "us-east-1");+ s3Service.putObject(BUCKET, "app.zip", "zip".getBytes(StandardCharsets.UTF_8), "application/zip", Map.of());+ return given().contentType("application/json")+ .body("""+ {"functionName": "%s",+ "revisionConfig": {+ "buildConfig": {"codeConfig": {"s3Object": {"bucket": "%s", "key": "app.zip"}},+ "runtimeConfig": {"runtime": "nodejs24.x"}},+ "serviceConfig": {"executionRoleArn": "%s"}},+ "endpointConfig": {"endpointName": "dev", "endpointType": "HomeRegion",+ "authType": "ApplicationManaged", "autoDeploymentMode": "LatestRevision"}}+ """.formatted(functionName, BUCKET, ROLE))+ .when().put(BASE + "/web-functions")+ .then().statusCode(202)+ .extract().path("endpoint.domainName");+ }++ private static String raw(String request) throws IOException {+ try (Socket socket = new Socket("localhost", RestAssured.port)) {+ socket.setSoTimeout(10_000);+ OutputStream out = socket.getOutputStream();+ out.write(request.getBytes(StandardCharsets.US_ASCII));+ out.flush();+ InputStream in = socket.getInputStream();+ return new String(in.readAllBytes(), StandardCharsets.UTF_8);+ }+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneTest.java b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneTest.javanew file mode 100644index 0000000000..5eca67e88f--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneTest.java@@ -0,0 +1,36 @@+package io.github.hectorvent.floci.services.lambdaweb;++import org.junit.jupiter.api.Test;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;+import static org.junit.jupiter.api.Assertions.assertTrue;++class LambdaWebDataPlaneTest {++ @Test+ void onlyTheMintedDomainShapesAreClaimed() {+ assertTrue(LambdaWebDataPlane.isLambdaWebHost("abcdefghijklmnop.001.lambda-web.us-east-1.localhost:4566"));+ assertTrue(LambdaWebDataPlane.isLambdaWebHost("abcdefghijklmnop.001.lambda-web.us-east-1.localhost"));+ assertTrue(LambdaWebDataPlane.isLambdaWebHost("abcdefghijklmnop.lambda-web.global.localhost:4566"));+ // A near miss is the AWS 404, so the case and the port are claimed too.+ assertTrue(LambdaWebDataPlane.isLambdaWebHost("ABCDEFGHIJKLMNOP.001.LAMBDA-WEB.US-EAST-1.LOCALHOST:4566"));+ // A dotted S3 bucket or another service's host is not.+ assertFalse(LambdaWebDataPlane.isLambdaWebHost("my.lambda-web.archive.localhost:4566"));+ assertFalse(LambdaWebDataPlane.isLambdaWebHost("abcdefghijklmnop.lambda-url.us-east-1.localhost:4566"));+ assertFalse(LambdaWebDataPlane.isLambdaWebHost("localhost:4566"));+ }++ @Test+ void requestIdsAre26CharactersOfBase32() {+ String id = LambdaWebDataPlane.newRequestId();+ assertTrue(id.matches("[A-Z2-7]{26}"), id);+ }++ @Test+ void thePortComesFromTheHostOrTheScheme() {+ assertEquals(4566, LambdaWebDataPlane.port("x.001.lambda-web.us-east-1.localhost:4566", "http"));+ assertEquals(80, LambdaWebDataPlane.port("x.001.lambda-web.us-east-1.localhost", "http"));+ assertEquals(443, LambdaWebDataPlane.port("x.001.lambda-web.us-east-1.floci.example", "https"));+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebErrorPagesTest.java b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebErrorPagesTest.javanew file mode 100644index 0000000000..5599ac5351--- /dev/null+++ b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebErrorPagesTest.java@@ -0,0 +1,45 @@+package io.github.hectorvent.floci.services.lambdaweb;++import org.junit.jupiter.api.Test;++import java.nio.charset.StandardCharsets;+import java.time.Instant;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertTrue;++/** The page bodies must be byte for byte what AWS sends, so their recorded content lengths pin them. */+class LambdaWebErrorPagesTest {++ private static final String REQUEST_ID = "WLYFST5BARDELKBHCQDDIBYUPM";++ @Test+ void pagesHaveTheRecordedContentLengths() {+ assertEquals(225, length(LambdaWebErrorPages.INVALID_ENDPOINT));+ assertEquals(264, length(LambdaWebErrorPages.UNSUPPORTED_HTTP_VERSION));+ assertEquals(274, length(LambdaWebErrorPages.ENDPOINT_DISABLED));+ assertEquals(249, length(LambdaWebErrorPages.REQUEST_TIMEOUT));+ assertEquals(236, length(LambdaWebErrorPages.ORIGIN_REQUEST_ERROR));+ assertEquals(228, length(LambdaWebErrorPages.TIMEOUT_LISTEN_ERROR));+ assertEquals(228, length(LambdaWebErrorPages.INIT_CODE_ERROR));+ assertEquals(228, length(LambdaWebErrorPages.RUNTIME_EXIT));+ assertEquals(180, LambdaWebErrorPages.MALFORMED_REQUEST_HTML.getBytes(StandardCharsets.UTF_8).length);+ }++ @Test+ void pageNamesTheStatusAndTheRequestId() {+ String html = LambdaWebErrorPages.html(LambdaWebErrorPages.INVALID_ENDPOINT, REQUEST_ID);+ assertTrue(html.startsWith("<!DOCTYPE html>\n<html>\n<head>\n <title>HTTP Error 404</title>\n"), html);+ assertTrue(html.contains(" <h1>404 Not Found</h1>\n <p>The requested endpoint could not be found.</p>\n"), html);+ assertTrue(html.endsWith(" <p>Request ID: " + REQUEST_ID + "</p>\n</body>\n</html>"), html);+ }++ @Test+ void edgeDateHasATwoDigitYear() {+ assertEquals("Thu, 15 Oct 26 23:32:32 GMT", LambdaWebErrorPages.edgeDate(Instant.parse("2026-10-15T23:32:32Z")));+ }++ private static int length(LambdaWebErrorPages.Page page) {+ return LambdaWebErrorPages.html(page, REQUEST_ID).getBytes(StandardCharsets.UTF_8).length;+ }+}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebServiceTest.java b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebServiceTest.javaindex 3d95891840..b55618c81e 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebServiceTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebServiceTest.java@@ -529,9 +529,18 @@ class LambdaWebServiceTest {ObjectNode update = mapper.createObjectNode().put("description", "updated");first.updateEndpoint(REGION, "durable", "dev", update);+ String domain = first.getEndpoint(REGION, "durable", "dev").domainName;+ assertEquals("dev", first.locate(domain).orElseThrow().endpoint().endpointName);+LambdaWebService second = service(new FileStorageFactory(storageDir));WebFunction restored = second.getFunction(REGION, "durable");assertEquals(created.functionArn, restored.functionArn);+ LambdaWebService.WebFunctionTarget target = second.locate(domain).orElseThrow();+ assertEquals(ACCOUNT_ID, target.accountId());+ assertEquals(REGION, target.region());+ assertEquals("durable", target.function().functionName);+ assertEquals("dev", target.endpoint().endpointName);+ assertTrue(second.locate("nope.001.lambda-web.us-east-1.localhost:4566").isEmpty());assertEquals(2, restored.revisions.size());WebFunctionEndpoint endpoint = second.getEndpoint(REGION, "durable", "dev");assertEquals("updated", endpoint.description);@@ -541,6 +550,8 @@ class LambdaWebServiceTest {assertEquals(endpoint.revisionWeights.get(0).revisionId,second.listRevisions(REGION, "durable", mapper.createObjectNode()).items().get(0).revisionId,"the LatestRevision endpoint moved to the newer revision and that move was written back");+ second.deleteEndpoint(REGION, "durable", "dev");+ assertTrue(second.locate(domain).isEmpty(), "a deleted endpoint's domain is gone");}private static final class MemoryStorageFactory extends StorageFactory {