Light, fluffy, and always free - The AWS Local Emulator alternative
feat(lambdaweb): refuse a long request head as AWS does (#6083)
TATSUNO “Taz” Yasuhiro5aaa37284c44
diff --git a/docs/services/lambda-web-functions.md b/docs/services/lambda-web-functions.mdindex ffa07bf8ac..4a459b25c8 100644--- a/docs/services/lambda-web-functions.md+++ b/docs/services/lambda-web-functions.md@@ -87,13 +87,14 @@ The platform answers these cases itself, with the AWS HTML pages and `lambda-web| 404 | `Platform.InvalidEndpoint` | The `Host` is a `lambda-web` domain Floci did not mint || 429 | `Platform.EndpointMaxEnvThrottle` | Every environment is full and the endpoint runs `maxEnvironments` || 429 | `Platform.EndpointRequestThrottle` | The requests exceed the endpoint's `rateLimit` |+| 431 | `Platform.HeaderTooLarge` | The request line, the headers and the blank line pass 128 KiB, counted as `Name: value` lines. This page answers before every other check || 502 | `Application.TimeoutListenError`, `Application.InitCodeError`, `Runtime.ImportCodeError`, `Runtime.Exit`, `Runtime.ExitError` | The server did not listen within 9 seconds, threw at import, could not find a module at import, or exited at start with status 0 or another status || 503 | `Platform.EndpointDisabled` | The endpoint's `rateLimit` is 0 || 503 | `Runtime.OriginRequestError` | The process died while handling the request, or tens of milliseconds before it || 504 | `Runtime.RequestTimeout` | No response within the revision's `timeoutSeconds` || 505 | `Platform.UnsupportedHttpVersion` | HTTP/1.0 |-A request head over 128 KiB is refused with `431`, without the AWS page. A request without a `Host` header cannot be told apart from one for another Floci service, so it does not get `Platform.MissingHostHeader`.+A URI of 65,535 bytes or more gets an empty `414`, as on AWS. Headers or a request line past 256 KiB get the HTTP server's empty `431` or `414` instead of the AWS answer. A request without a `Host` header cannot be told apart from one for another Floci service, so it does not get `Platform.MissingHostHeader`.## Logsdiff --git a/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlane.java b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlane.javaindex a93581b81a..b327003328 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlane.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlane.java@@ -7,6 +7,7 @@ import io.github.hectorvent.floci.services.lambdaweb.LambdaWebEnvironmentManagerimport io.github.hectorvent.floci.services.lambdaweb.LambdaWebEnvironmentManager.Throttled;import io.github.hectorvent.floci.services.lambdaweb.LambdaWebService.WebFunctionTarget;import io.github.hectorvent.floci.services.lambdaweb.model.WebFunctionRevision;+import io.quarkus.vertx.http.runtime.RouteConstants;import io.vertx.core.Future;import io.vertx.core.Handler;import io.vertx.core.MultiMap;@@ -69,6 +70,13 @@ public class LambdaWebDataPlane {private static final DateTimeFormatter HTTP_DATE = DateTimeFormatter.RFC_1123_DATE_TIME.withZone(ZoneOffset.UTC);/** Floci sees an exit some tens of milliseconds after it. A process that keeps running is not waited for longer. */private static final Duration EXIT_WAIT = Duration.ofSeconds(1);+ /** The edge counts the request line, the header lines and the blank line, as the client sent them. */+ private static final int MAX_HEAD_LENGTH = 128 * 1024;+ /** The layer behind the edge refuses a URI this long. */+ private static final int URI_TOO_LONG = 65_535;+ /** The Quarkus defaults, which every other host keeps. */+ private static final int MAX_INITIAL_LINE_LENGTH = 4096;+ private static final int MAX_HEADER_SIZE = 20 * 1024;private final LambdaWebService service;private final LambdaWebEnvironmentManager environments;@@ -96,9 +104,51 @@ public class LambdaWebDataPlane {}void register(@Observes Router router) {+ // A head is refused before its body is, as on AWS.+ router.route().order(RouteConstants.ROUTE_ORDER_UPLOAD_LIMIT - 1).handler(this::limitHead);router.route().handler(this::handle);}+ /**+ * The server reads heads past every limit, so the limits are kept here. Lambda Web hosts get the+ * AWS ones and every other host the Quarkus defaults. A head within the defaults is within both,+ * so only a longer one has its host looked up.+ */+ void limitHead(RoutingContext context) {+ HttpServerRequest request = context.request();+ if (request.version() == HttpVersion.HTTP_2) {+ context.next();+ return;+ }+ int initialLine = request.method().name().length() + 1 + request.uri().length() + 1 + "HTTP/1.1".length();+ int headerBytes = 0;+ int headerLines = 0;+ for (Map.Entry<String, String> header : request.headers()) {+ headerBytes += header.getKey().length() + ": ".length() + header.getValue().length();+ headerLines++;+ }+ if (initialLine <= MAX_INITIAL_LINE_LENGTH && headerBytes <= MAX_HEADER_SIZE) {+ context.next();+ return;+ }+ String host = RequestHost.of(request);+ if (host != null && isLambdaWebHost(host) && serviceConfigAccess.isEnabled("lambda")) {+ int head = initialLine + headerBytes + 2 * (1 + headerLines + 1);+ if (head > MAX_HEAD_LENGTH) {+ edge(request, 431, "Request Header Fields Too Large", LambdaWebErrorPages.HEADER_TOO_LARGE_TYPE,+ LambdaWebErrorPages.HEADER_TOO_LARGE_HTML);+ } else if (request.uri().length() >= URI_TOO_LONG) {+ uriTooLong(request);+ } else {+ context.next();+ }+ return;+ }+ HttpServerResponse response = request.response();+ response.setStatusCode(initialLine > MAX_INITIAL_LINE_LENGTH ? 414 : 431).end();+ response.close();+ }+void handle(RoutingContext context) {HttpServerRequest request = context.request();String host = RequestHost.of(request);@@ -589,15 +639,30 @@ public class LambdaWebDataPlane {/** The edge refuses a method before the platform sees the request, with its own header casing. */static void malformed(HttpServerRequest request) {+ edge(request, 400, "Bad Request", LambdaWebErrorPages.MALFORMED_REQUEST_TYPE,+ LambdaWebErrorPages.MALFORMED_REQUEST_HTML);+ }++ private static void edge(HttpServerRequest request, int status, String reason, String errorType, String html) {HttpServerResponse response = request.response();- response.setStatusCode(400).setStatusMessage("Bad Request");+ response.setStatusCode(status).setStatusMessage(reason);response.headers().set("Date", LambdaWebErrorPages.edgeDate(Instant.now()));response.headers().set("Connection", "close");response.headers().set("x-amz-request-id", Long.toHexString(ThreadLocalRandom.current().nextLong()).toUpperCase(Locale.ROOT));response.headers().set("Content-Type", "text/html; charset=utf-8");- response.headers().set("Lambda-Web-Error-Type", LambdaWebErrorPages.MALFORMED_REQUEST_TYPE);- response.end(LambdaWebErrorPages.MALFORMED_REQUEST_HTML).onComplete(ignored -> response.close());+ response.headers().set("Lambda-Web-Error-Type", errorType);+ response.headers().set("Content-Length", String.valueOf(html.getBytes(StandardCharsets.UTF_8).length));+ response.end(html).onComplete(ignored -> response.close());+ }++ private static void uriTooLong(HttpServerRequest request) {+ HttpServerResponse response = request.response();+ response.setStatusCode(414).setStatusMessage("URI Too Long");+ response.headers().set("connection", "close");+ response.headers().set("content-length", "0");+ response.headers().set("date", HTTP_DATE.format(Instant.now()));+ response.end().onComplete(ignored -> response.close());}/** AWS answers its own failures with 502 and no error type, so nothing is revealed. */diff --git a/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebErrorPages.java b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebErrorPages.javaindex 9954dcc3cf..0d15c21943 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebErrorPages.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebErrorPages.java@@ -37,11 +37,18 @@ final class LambdaWebErrorPages {static final Page ENDPOINT_REQUEST_THROTTLE = new Page(429, "Too Many Requests","Platform.EndpointRequestThrottle", RATE_EXCEEDED);- /** The edge in front of the platform refuses a method itself, with its own page and header casing. */+ /**+ * The edge in front of the platform refuses a method or a large head itself, with its own pages+ * and header casing.+ */static final String MALFORMED_REQUEST_TYPE = "Platform.MalformedRequest";static final String MALFORMED_REQUEST_HTML = "<!DOCTYPE html>\n<html>\n<head>\n<title>HTTP Error 400</title>\n"+ "</head>\n<body>\n<h1>400 Bad Request</h1>\n"+ "<p>The request could not be parsed as a valid HTTP request.</p>\n</body>\n</html>\n";+ static final String HEADER_TOO_LARGE_TYPE = "Platform.HeaderTooLarge";+ static final String HEADER_TOO_LARGE_HTML = "<!DOCTYPE html>\n<html>\n<head>\n<title>HTTP Error 431</title>\n"+ + "</head>\n<body>\n<h1>431 Request Header Fields Too Large</h1>\n"+ + "<p>The request headers exceed the maximum allowed size.</p>\n</body>\n</html>\n";/** The edge writes a two-digit year in its Date header. */private static final DateTimeFormatter EDGE_DATE =DateTimeFormatter.ofPattern("EEE, dd MMM yy HH:mm:ss 'GMT'", Locale.ROOT).withZone(ZoneOffset.UTC);diff --git a/src/main/resources/application.yml b/src/main/resources/application.ymlindex 5626b6cd4f..7abe15142b 100644--- a/src/main/resources/application.yml+++ b/src/main/resources/application.yml@@ -35,8 +35,10 @@ quarkus:require-localhost: falselimits:max-body-size: ${floci.protocols.max-request-size}M- # Lambda Web Functions accept a request head of up to 128 KiB and answer 431 past it.- max-header-size: 128K+ # The server reads heads past the Lambda Web limits so their own 431 and 414 can answer them.+ # LambdaWebDataPlane keeps these two at the Quarkus defaults for every other host.+ max-header-size: 256K+ max-initial-line-length: 262144shutdown:# Enables the pre-shutdown phase so stop hooks run before the HTTP server stops.# No 'delay' value is set - observers run synchronously, no artificial sleep is introduced.diff --git a/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneIntegrationTest.javaindex 079cf36023..52d649c863 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneIntegrationTest.java@@ -13,6 +13,7 @@ import java.io.OutputStream;import java.net.Socket;import java.nio.charset.StandardCharsets;import java.time.Duration;+import java.util.List;import java.util.Map;import static io.restassured.RestAssured.given;@@ -20,6 +21,7 @@ import static org.hamcrest.Matchers.equalTo;import static org.hamcrest.Matchers.matchesPattern;import static org.hamcrest.Matchers.nullValue;import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertFalse;import static org.junit.jupiter.api.Assertions.assertTrue;/** The platform answers that need no execution environment, so no Docker. */@@ -181,9 +183,59 @@ class LambdaWebDataPlaneIntegrationTest {assertTrue(response.contains("\r\nLambda-Web-Error-Type: Platform.MalformedRequest\r\n"), response);assertTrue(response.contains("\r\nContent-Type: text/html; charset=utf-8\r\n"), response);assertTrue(response.contains("\r\nConnection: close\r\n"), response);+ assertTrue(response.contains("\r\nContent-Length: 180\r\n"), response);assertTrue(response.endsWith(LambdaWebErrorPages.MALFORMED_REQUEST_HTML), response);}+ @Test+ void aHeadPastTheLimitIsRefusedByTheEdgeBeforeAnyOtherCheck() throws IOException {+ String host = "Host: " + UNKNOWN_HOST + "\r\n";+ String closing = host + "Connection: close\r\n";+ assertTrue(raw(head("GET / HTTP/1.1", closing, 1, 131_072)).startsWith("HTTP/1.1 404 Not Found\r\n"));+ assertTrue(raw(head("GET / HTTP/1.1", closing, 120, 131_072)).startsWith("HTTP/1.1 404 Not Found\r\n"));++ for (String line : List.of("GET / HTTP/1.1", "TRACE / HTTP/1.1", "GET / HTTP/1.0")) {+ for (int headers : List.of(1, 120)) {+ String response = raw(head(line, host, headers, 131_073));+ String message = line + " " + headers + "\n" + response;+ assertTrue(response.contains(" 431 Request Header Fields Too Large\r\n"), message);+ assertTrue(response.contains("\r\nLambda-Web-Error-Type: Platform.HeaderTooLarge\r\n"), message);+ assertTrue(response.contains("\r\nContent-Type: text/html; charset=utf-8\r\n"), message);+ assertTrue(response.contains("\r\nConnection: close\r\n"), message);+ assertTrue(response.contains("\r\nContent-Length: 196\r\n"), message);+ assertTrue(response.endsWith(LambdaWebErrorPages.HEADER_TOO_LARGE_HTML), message);+ }+ }+ }++ @Test+ void aUriOf64KiBIsTooLong() throws IOException {+ String fits = raw("GET /" + "a".repeat(65_533) + " HTTP/1.1\r\nHost: " + UNKNOWN_HOST+ + "\r\nConnection: close\r\n\r\n");+ assertTrue(fits.startsWith("HTTP/1.1 404 Not Found\r\n"), fits);++ String response = raw("GET /" + "a".repeat(65_534) + " HTTP/1.1\r\nHost: " + UNKNOWN_HOST+ + "\r\nConnection: close\r\n\r\n");+ assertTrue(response.startsWith("HTTP/1.1 414 URI Too Long\r\n"), response);+ assertTrue(response.contains("\r\nconnection: close\r\n"), response);+ assertTrue(response.contains("\r\ncontent-length: 0\r\n"), response);+ assertTrue(response.endsWith("\r\n\r\n"), response);+ }++ @Test+ void otherHostsKeepTheQuarkusLimits() throws IOException {+ String path = "/" + "a".repeat(4096 - "GET / HTTP/1.1".length());+ assertFalse(raw("GET " + path + " HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n")+ .startsWith("HTTP/1.1 414 "));+ assertTrue(raw("GET " + path + "a HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n")+ .startsWith("HTTP/1.1 414 "));++ String headers = "Host: localhost\r\nConnection: close\r\nX-Pad: ";+ String pad = "a".repeat(20 * 1024 - "Host: localhost".length() - "Connection: close".length() - "X-Pad: ".length());+ assertFalse(raw("GET / HTTP/1.1\r\n" + headers + pad + "\r\n\r\n").startsWith("HTTP/1.1 431 "));+ assertTrue(raw("GET / HTTP/1.1\r\n" + headers + pad + "a\r\n\r\n").startsWith("HTTP/1.1 431 "));+ }+@Testvoid otherHostsStillReachTheControlPlane() {given().when().get(BASE + "/web-account-settings").then().statusCode(200);@@ -211,6 +263,21 @@ class LambdaWebDataPlaneIntegrationTest {.extract().path("endpoint.domainName");}+ /** A head of exactly {@code length} bytes, padded over {@code headers} headers after the fixed ones. */+ private static String head(String requestLine, String fixedHeaders, int headers, int length) {+ StringBuilder head = new StringBuilder(requestLine + "\r\n" + fixedHeaders);+ int padding = length - head.length() - 2;+ int share = padding / headers;+ for (int i = 0; i < headers; i++) {+ String name = "X-Pad-%03d: ".formatted(i);+ int line = i < headers - 1 ? share : padding - (headers - 1) * share;+ head.append(name).append("a".repeat(line - name.length() - 2)).append("\r\n");+ }+ String result = head.append("\r\n").toString();+ assertEquals(length, result.length());+ return result;+ }+private static String raw(String request) throws IOException {try (Socket socket = new Socket("localhost", RestAssured.port)) {socket.setSoTimeout(10_000);diff --git a/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebErrorPagesTest.java b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebErrorPagesTest.javaindex c5cfdddf1e..50c801fa10 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebErrorPagesTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebErrorPagesTest.java@@ -29,6 +29,7 @@ class LambdaWebErrorPagesTest {assertEquals(237, length(LambdaWebErrorPages.ENDPOINT_MAX_ENV_THROTTLE));assertEquals(237, length(LambdaWebErrorPages.ENDPOINT_REQUEST_THROTTLE));assertEquals(180, LambdaWebErrorPages.MALFORMED_REQUEST_HTML.getBytes(StandardCharsets.UTF_8).length);+ assertEquals(196, LambdaWebErrorPages.HEADER_TOO_LARGE_HTML.getBytes(StandardCharsets.UTF_8).length);}@Testdiff --git a/src/test/resources/application.yml b/src/test/resources/application.ymlindex 245f424632..d283e045fc 100644--- a/src/test/resources/application.yml+++ b/src/test/resources/application.yml@@ -7,6 +7,8 @@ quarkus:host: 0.0.0.0limits:max-body-size: ${floci.protocols.max-request-size}M+ max-header-size: 256K+ max-initial-line-length: 262144# Groovy (a REST Assured dependency) starts DFA-cache-cleaner threads that never stop.# Loaded parent-first they live in the shared system classloader, so they cannot pin a# per-profile application classloader across the many app restarts of a test run.