Light, fluffy, and always free - The AWS Local Emulator alternative
fix(lambdaweb): report init failures as AWS does (#5960)
TATSUNO “Taz” Yasuhirob6bc3032f1b8
diff --git a/docs/services/lambda-web-functions.md b/docs/services/lambda-web-functions.mdindex 6f0f721f65..f4d133ab8b 100644--- a/docs/services/lambda-web-functions.md+++ b/docs/services/lambda-web-functions.md@@ -85,7 +85,7 @@ The platform answers these cases itself, with the AWS HTML pages and `lambda-web| 404 | `Platform.InvalidEndpoint` | The `Host` is a `lambda-web` domain Floci did not mint || 429 | `Platform.EndpointMaxEnvThrottle` | Every environment is full and the endpoint runs `maxEnvironments` || 429 | `Platform.EndpointRequestThrottle` | The requests exceed the endpoint's `rateLimit` |-| 502 | `Application.TimeoutListenError`, `Application.InitCodeError`, `Runtime.Exit` | The server did not listen within 9 seconds, threw at import, or exited at start |+| 502 | `Application.TimeoutListenError`, `Application.InitCodeError`, `Runtime.ImportCodeError`, `Runtime.Exit`, `Runtime.ExitError` | The server did not listen within 9 seconds, threw at import, could not find a module at import, or exited at start with status 0 or another status || 503 | `Platform.EndpointDisabled` | The endpoint's `rateLimit` is 0 || 503 | `Runtime.OriginRequestError` | The process died while handling the request, or tens of milliseconds before it || 504 | `Runtime.RequestTimeout` | No response within the revision's `timeoutSeconds` |@@ -106,7 +106,7 @@ The platform writes the `platform.*` events of the [system log schema](https://d| Event | When || --- | --- || `platform.initStart` | The environment starts |-| `platform.initReport` | The server listens, or fails to, with the error type of the `502` page. A server that does not listen also gets an `ERROR` event that says so, unless `applicationLogLevel` is `FATAL` |+| `platform.initReport` | The server listens, or fails to, with the error type of the `502` page. A server that does not listen or an app that fails at import also gets an `ERROR` event that says so, unless `applicationLogLevel` is `FATAL` || `platform.invokeStart` | Right after a successful `initReport` || `platform.requestComplete` | At `DEBUG`, for each request the environment took, with the status, the body bytes and the error type of a `503` or `504` page || `platform.invokeSummary` | Every 60 seconds from `invokeStart`, and once more when the environment ends |diff --git a/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebEnvironmentManager.java b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebEnvironmentManager.javaindex 7277d49310..22ef1ff2de 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebEnvironmentManager.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebEnvironmentManager.java@@ -32,6 +32,7 @@ import jakarta.annotation.PreDestroy;import jakarta.enterprise.context.ApplicationScoped;import jakarta.inject.Inject;import org.apache.commons.compress.archivers.tar.TarArchiveEntry;+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;import org.jboss.logging.Logger;@@ -101,6 +102,9 @@ public class LambdaWebEnvironmentManager implements ContainerTeardown {private static final String[] READ_USAGE = {"/bin/cat", "/sys/fs/cgroup/cpu.stat", "/sys/fs/cgroup/memory.peak"};private static final long SIGKILL_STATUS = 137;private static final long SIGTERM_STATUS = 143;+ /** Where index.mjs writes the error type of an app that fails to load. */+ private static final String INIT_ERROR_FILE = "/tmp/floci-init-error";+ private static final int INIT_ERROR_MAX_BYTES = 64;/** Longer than the data plane waits to see an exit, so a request the exit failed is counted. */private static final Duration EXCHANGES_WAIT = Duration.ofSeconds(2);@@ -722,7 +726,7 @@ public class LambdaWebEnvironmentManager implements ContainerTeardown {LOG.infov("The Lambda Web environment {0} exited with status {1} before it listened: {2}",containerId, exitCode.get(), lastLogLines(containerId));throw new StartFailure(exitCode.get() == 0- ? LambdaWebErrorPages.RUNTIME_EXIT : LambdaWebErrorPages.INIT_CODE_ERROR);+ ? LambdaWebErrorPages.RUNTIME_EXIT : initErrorPage(containerId));}if (isListening(containerId)) {return;@@ -743,6 +747,27 @@ public class LambdaWebEnvironmentManager implements ContainerTeardown {}}+ /** The page for an app that ended with an error before it listened. The container has stopped but still exists. */+ private LambdaWebErrorPages.Page initErrorPage(String containerId) {+ String errorType = "";+ try (InputStream archive = lifecycleManager.getDockerClient()+ .copyArchiveFromContainerCmd(containerId, INIT_ERROR_FILE).exec();+ TarArchiveInputStream tar = new TarArchiveInputStream(archive)) {+ if (tar.getNextEntry() != null) {+ errorType = new String(tar.readNBytes(INIT_ERROR_MAX_BYTES), StandardCharsets.UTF_8);+ }+ } catch (NotFoundException expected) {+ // The app exited on its own, so index.mjs wrote no error type.+ } catch (IOException | RuntimeException e) {+ LOG.warnv(e, "Could not read the init error of the Lambda Web environment {0}", containerId);+ }+ return switch (errorType) {+ case "Application.InitCodeError" -> LambdaWebErrorPages.INIT_CODE_ERROR;+ case "Runtime.ImportCodeError" -> LambdaWebErrorPages.IMPORT_CODE_ERROR;+ default -> LambdaWebErrorPages.RUNTIME_EXIT_ERROR;+ };+ }+/*** Probes from inside the container. A published port on Docker Desktop accepts a connection* before anything listens behind it, so a connect from the host proves nothing.diff --git a/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebErrorPages.java b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebErrorPages.javaindex 5ab10025a9..9954dcc3cf 100644--- a/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebErrorPages.java+++ b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebErrorPages.java@@ -28,7 +28,9 @@ final class LambdaWebErrorPages {static final Page TIMEOUT_LISTEN_ERROR = new Page(502, "Bad Gateway", "Application.TimeoutListenError",APPLICATION_ERROR);static final Page INIT_CODE_ERROR = new Page(502, "Bad Gateway", "Application.InitCodeError", APPLICATION_ERROR);+ static final Page IMPORT_CODE_ERROR = new Page(502, "Bad Gateway", "Runtime.ImportCodeError", APPLICATION_ERROR);static final Page RUNTIME_EXIT = new Page(502, "Bad Gateway", "Runtime.Exit", APPLICATION_ERROR);+ static final Page RUNTIME_EXIT_ERROR = new Page(502, "Bad Gateway", "Runtime.ExitError", APPLICATION_ERROR);private static final String RATE_EXCEEDED = "Request rate exceeded. Please try again later.";static final Page ENDPOINT_MAX_ENV_THROTTLE = new Page(429, "Too Many Requests","Platform.EndpointMaxEnvThrottle", RATE_EXCEEDED);diff --git a/src/main/resources/lambdaweb/index.mjs b/src/main/resources/lambdaweb/index.mjsindex 654ed40b22..9927e3de06 100644--- a/src/main/resources/lambdaweb/index.mjs+++ b/src/main/resources/lambdaweb/index.mjs@@ -2,6 +2,7 @@// console output into the JSON log events AWS writes, with the request each one belongs to,// filters the output by applicationLogLevel, then imports the app.import { AsyncLocalStorage } from 'node:async_hooks';+import fs from 'node:fs';import http from 'node:http';import path from 'node:path';import { pathToFileURL } from 'node:url';@@ -92,5 +93,34 @@ if (LEVELS.INFO < minimum) {process.stderr.write = drop;}+// AWS writes one ERROR event in place of Node's stack when the app fails to load. Floci reads the+// error type from INIT_ERROR_FILE once the process is gone, so any exit status of the app keeps its meaning.+// /tmp is writable by any user, also in an image that does not run as root.+const INIT_ERROR_FILE = '/tmp/floci-init-error';const entry = process.env.AWS_LAMBDA_NODEJS_ENTRYPOINT || 'index.js';-await import(pathToFileURL(path.resolve('/var/task', entry)).href);+// A runtime image could carry the file from a container that failed before.+try {+ fs.rmSync(INIT_ERROR_FILE, { force: true });+} catch (ignored) {+ // The app still runs. Only a failed start would get the old error type.+}+try {+ await import(pathToFileURL(path.resolve('/var/task', entry)).href);+} catch (error) {+ // AWS goes by the code alone, also for an error the app throws itself.+ const missing = error?.code === 'MODULE_NOT_FOUND' || error?.code === 'ERR_MODULE_NOT_FOUND';+ const errorType = missing ? 'Runtime.ImportCodeError' : 'Application.InitCodeError';+ const reason = error instanceof Error ? error.message : String(error);+ if (LEVELS.ERROR >= minimum) {+ writeErr(line('ERROR', `${errorType}: could not load customer code: ${reason}` + (missing+ ? ". The entrypoint must be 'index.js' at the root of the deployment package, or the"+ + ' AWS_LAMBDA_NODEJS_ENTRYPOINT environment variable must point to your entry file'+ : '')));+ }+ try {+ fs.writeFileSync(INIT_ERROR_FILE, errorType);+ } catch (ignored) {+ // A full disk leaves Floci with Runtime.ExitError, without a stack trace AWS would not write.+ }+ process.exit(1);+}diff --git a/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneDockerIntegrationTest.javaindex 4680d3a881..5546cb22ca 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneDockerIntegrationTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneDockerIntegrationTest.java@@ -66,6 +66,8 @@ class LambdaWebDataPlaneDockerIntegrationTest {const [host, port] = process.env.AWS_LAMBDA_HTTP_ENDPOINT.split(':');console.log('starting');if (process.env.MODE === 'throw') { throw new Error('init boom'); }+ if (process.env.MODE === 'exit') { process.exit(1); }+ if (process.env.MODE === 'missing') { require('no-such-module'); }http.createServer((req, res) => {if (req.url.startsWith('/log')) {console.info({route: 'log', main: require.main === module, argv: process.argv.slice(1)});@@ -504,8 +506,40 @@ class LambdaWebDataPlaneDockerIntegrationTest {assertEquals("WARN", initReport.getString("level"));assertEquals("failure", initReport.getString("record.status"));assertEquals("Application.InitCodeError", initReport.getString("record.errorType"));- assertTrue(events.stream().anyMatch(event -> event.contains("\"message\":\"starting\"")), events.toString());- assertTrue(events.stream().anyMatch(event -> event.contains("init boom")), events.toString());+ List<JsonPath> output = events.stream().filter(event -> !isPlatformEvent(event)).map(JsonPath::from).toList();+ assertEquals(List.of("ERROR Application.InitCodeError: could not load customer code: init boom", "INFO starting"),+ output.stream().map(event -> event.getString("level") + " " + event.getString("message")).sorted()+ .toList(), events.toString());+ }++ @Test+ @Timeout(180)+ void anAppThatExitsWithAnErrorAtStartGetsTheExitErrorPage() throws Exception {+ String domain = createFunction(30, 64, "WARN", "exit");+ String group = "/aws/lambda/web/" + functionName + ".local";+ given().header("Host", domain).when().get("/").then().statusCode(502)+ .header("lambda-web-error-type", "Runtime.ExitError");++ List<String> events = awaitLogEvents(group, awaitLogStream(group),+ each -> types(each).contains("platform.initReport"));+ assertEquals("Runtime.ExitError", platformEvent(events, "platform.initReport").getString("record.errorType"));+ }++ @Test+ @Timeout(180)+ void anAppThatCannotFindAModuleGetsTheImportCodeErrorPage() throws Exception {+ String domain = createFunction(30, 64, "WARN", "missing");+ String group = "/aws/lambda/web/" + functionName + ".local";+ given().header("Host", domain).when().get("/").then().statusCode(502)+ .header("lambda-web-error-type", "Runtime.ImportCodeError");++ List<String> events = awaitLogEvents(group, awaitLogStream(group), each -> types(each).contains(+ "platform.initReport") && each.stream().anyMatch(event -> event.contains("ImportCodeError:")));+ assertEquals("Runtime.ImportCodeError",+ platformEvent(events, "platform.initReport").getString("record.errorType"));+ assertTrue(events.stream().anyMatch(event -> event.contains("\"message\":\"Runtime.ImportCodeError: could not"+ + " load customer code: Cannot find module 'no-such-module'\\nRequire stack:\\n- /var/task/index.js."+ + " The entrypoint must be")), events.toString());}private static boolean isPlatformEvent(String event) {diff --git a/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebErrorPagesTest.java b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebErrorPagesTest.javaindex 39f562afb6..c5cfdddf1e 100644--- a/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebErrorPagesTest.java+++ b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebErrorPagesTest.java@@ -23,7 +23,9 @@ class LambdaWebErrorPagesTest {assertEquals(236, length(LambdaWebErrorPages.ORIGIN_REQUEST_ERROR));assertEquals(228, length(LambdaWebErrorPages.TIMEOUT_LISTEN_ERROR));assertEquals(228, length(LambdaWebErrorPages.INIT_CODE_ERROR));+ assertEquals(228, length(LambdaWebErrorPages.IMPORT_CODE_ERROR));assertEquals(228, length(LambdaWebErrorPages.RUNTIME_EXIT));+ assertEquals(228, length(LambdaWebErrorPages.RUNTIME_EXIT_ERROR));assertEquals(237, length(LambdaWebErrorPages.ENDPOINT_MAX_ENV_THROTTLE));assertEquals(237, length(LambdaWebErrorPages.ENDPOINT_REQUEST_THROTTLE));assertEquals(180, LambdaWebErrorPages.MALFORMED_REQUEST_HTML.getBytes(StandardCharsets.UTF_8).length);