Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

fix(lambdaweb): report init failures as AWS does (#5960)

TATSUNO “Taz” Yasuhirob6bc3032f1b8
diff --git a/docs/services/lambda-web-functions.md b/docs/services/lambda-web-functions.md
index 6f0f721f65..f4d133ab8b 100644
--- a/docs/services/lambda-web-functions.md
+++ b/docs/services/lambda-web-functions.md
@@ -85,7 +85,7 @@ The platform answers these cases itself, with the AWS HTML pages and `lambda-web
| 404 | `Platform.InvalidEndpoint` | The `Host` is a `lambda-web` domain Floci did not mint |
| 429 | `Platform.EndpointMaxEnvThrottle` | Every environment is full and the endpoint runs `maxEnvironments` |
| 429 | `Platform.EndpointRequestThrottle` | The requests exceed the endpoint's `rateLimit` |
-| 502 | `Application.TimeoutListenError`, `Application.InitCodeError`, `Runtime.Exit` | The server did not listen within 9 seconds, threw at import, or exited at start |
+| 502 | `Application.TimeoutListenError`, `Application.InitCodeError`, `Runtime.ImportCodeError`, `Runtime.Exit`, `Runtime.ExitError` | The server did not listen within 9 seconds, threw at import, could not find a module at import, or exited at start with status 0 or another status |
| 503 | `Platform.EndpointDisabled` | The endpoint's `rateLimit` is 0 |
| 503 | `Runtime.OriginRequestError` | The process died while handling the request, or tens of milliseconds before it |
| 504 | `Runtime.RequestTimeout` | No response within the revision's `timeoutSeconds` |
@@ -106,7 +106,7 @@ The platform writes the `platform.*` events of the [system log schema](https://d
| Event | When |
| --- | --- |
| `platform.initStart` | The environment starts |
-| `platform.initReport` | The server listens, or fails to, with the error type of the `502` page. A server that does not listen also gets an `ERROR` event that says so, unless `applicationLogLevel` is `FATAL` |
+| `platform.initReport` | The server listens, or fails to, with the error type of the `502` page. A server that does not listen or an app that fails at import also gets an `ERROR` event that says so, unless `applicationLogLevel` is `FATAL` |
| `platform.invokeStart` | Right after a successful `initReport` |
| `platform.requestComplete` | At `DEBUG`, for each request the environment took, with the status, the body bytes and the error type of a `503` or `504` page |
| `platform.invokeSummary` | Every 60 seconds from `invokeStart`, and once more when the environment ends |
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebEnvironmentManager.java b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebEnvironmentManager.java
index 7277d49310..22ef1ff2de 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebEnvironmentManager.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebEnvironmentManager.java
@@ -32,6 +32,7 @@ import jakarta.annotation.PreDestroy;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.jboss.logging.Logger;
@@ -101,6 +102,9 @@ public class LambdaWebEnvironmentManager implements ContainerTeardown {
private static final String[] READ_USAGE = {"/bin/cat", "/sys/fs/cgroup/cpu.stat", "/sys/fs/cgroup/memory.peak"};
private static final long SIGKILL_STATUS = 137;
private static final long SIGTERM_STATUS = 143;
+ /** Where index.mjs writes the error type of an app that fails to load. */
+ private static final String INIT_ERROR_FILE = "/tmp/floci-init-error";
+ private static final int INIT_ERROR_MAX_BYTES = 64;
/** Longer than the data plane waits to see an exit, so a request the exit failed is counted. */
private static final Duration EXCHANGES_WAIT = Duration.ofSeconds(2);
@@ -722,7 +726,7 @@ public class LambdaWebEnvironmentManager implements ContainerTeardown {
LOG.infov("The Lambda Web environment {0} exited with status {1} before it listened: {2}",
containerId, exitCode.get(), lastLogLines(containerId));
throw new StartFailure(exitCode.get() == 0
- ? LambdaWebErrorPages.RUNTIME_EXIT : LambdaWebErrorPages.INIT_CODE_ERROR);
+ ? LambdaWebErrorPages.RUNTIME_EXIT : initErrorPage(containerId));
}
if (isListening(containerId)) {
return;
@@ -743,6 +747,27 @@ public class LambdaWebEnvironmentManager implements ContainerTeardown {
}
}
+ /** The page for an app that ended with an error before it listened. The container has stopped but still exists. */
+ private LambdaWebErrorPages.Page initErrorPage(String containerId) {
+ String errorType = "";
+ try (InputStream archive = lifecycleManager.getDockerClient()
+ .copyArchiveFromContainerCmd(containerId, INIT_ERROR_FILE).exec();
+ TarArchiveInputStream tar = new TarArchiveInputStream(archive)) {
+ if (tar.getNextEntry() != null) {
+ errorType = new String(tar.readNBytes(INIT_ERROR_MAX_BYTES), StandardCharsets.UTF_8);
+ }
+ } catch (NotFoundException expected) {
+ // The app exited on its own, so index.mjs wrote no error type.
+ } catch (IOException | RuntimeException e) {
+ LOG.warnv(e, "Could not read the init error of the Lambda Web environment {0}", containerId);
+ }
+ return switch (errorType) {
+ case "Application.InitCodeError" -> LambdaWebErrorPages.INIT_CODE_ERROR;
+ case "Runtime.ImportCodeError" -> LambdaWebErrorPages.IMPORT_CODE_ERROR;
+ default -> LambdaWebErrorPages.RUNTIME_EXIT_ERROR;
+ };
+ }
+
/**
* Probes from inside the container. A published port on Docker Desktop accepts a connection
* before anything listens behind it, so a connect from the host proves nothing.
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebErrorPages.java b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebErrorPages.java
index 5ab10025a9..9954dcc3cf 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebErrorPages.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebErrorPages.java
@@ -28,7 +28,9 @@ final class LambdaWebErrorPages {
static final Page TIMEOUT_LISTEN_ERROR = new Page(502, "Bad Gateway", "Application.TimeoutListenError",
APPLICATION_ERROR);
static final Page INIT_CODE_ERROR = new Page(502, "Bad Gateway", "Application.InitCodeError", APPLICATION_ERROR);
+ static final Page IMPORT_CODE_ERROR = new Page(502, "Bad Gateway", "Runtime.ImportCodeError", APPLICATION_ERROR);
static final Page RUNTIME_EXIT = new Page(502, "Bad Gateway", "Runtime.Exit", APPLICATION_ERROR);
+ static final Page RUNTIME_EXIT_ERROR = new Page(502, "Bad Gateway", "Runtime.ExitError", APPLICATION_ERROR);
private static final String RATE_EXCEEDED = "Request rate exceeded. Please try again later.";
static final Page ENDPOINT_MAX_ENV_THROTTLE = new Page(429, "Too Many Requests",
"Platform.EndpointMaxEnvThrottle", RATE_EXCEEDED);
diff --git a/src/main/resources/lambdaweb/index.mjs b/src/main/resources/lambdaweb/index.mjs
index 654ed40b22..9927e3de06 100644
--- a/src/main/resources/lambdaweb/index.mjs
+++ b/src/main/resources/lambdaweb/index.mjs
@@ -2,6 +2,7 @@
// console output into the JSON log events AWS writes, with the request each one belongs to,
// filters the output by applicationLogLevel, then imports the app.
import { AsyncLocalStorage } from 'node:async_hooks';
+import fs from 'node:fs';
import http from 'node:http';
import path from 'node:path';
import { pathToFileURL } from 'node:url';
@@ -92,5 +93,34 @@ if (LEVELS.INFO < minimum) {
process.stderr.write = drop;
}
+// AWS writes one ERROR event in place of Node's stack when the app fails to load. Floci reads the
+// error type from INIT_ERROR_FILE once the process is gone, so any exit status of the app keeps its meaning.
+// /tmp is writable by any user, also in an image that does not run as root.
+const INIT_ERROR_FILE = '/tmp/floci-init-error';
const entry = process.env.AWS_LAMBDA_NODEJS_ENTRYPOINT || 'index.js';
-await import(pathToFileURL(path.resolve('/var/task', entry)).href);
+// A runtime image could carry the file from a container that failed before.
+try {
+ fs.rmSync(INIT_ERROR_FILE, { force: true });
+} catch (ignored) {
+ // The app still runs. Only a failed start would get the old error type.
+}
+try {
+ await import(pathToFileURL(path.resolve('/var/task', entry)).href);
+} catch (error) {
+ // AWS goes by the code alone, also for an error the app throws itself.
+ const missing = error?.code === 'MODULE_NOT_FOUND' || error?.code === 'ERR_MODULE_NOT_FOUND';
+ const errorType = missing ? 'Runtime.ImportCodeError' : 'Application.InitCodeError';
+ const reason = error instanceof Error ? error.message : String(error);
+ if (LEVELS.ERROR >= minimum) {
+ writeErr(line('ERROR', `${errorType}: could not load customer code: ${reason}` + (missing
+ ? ". The entrypoint must be 'index.js' at the root of the deployment package, or the"
+ + ' AWS_LAMBDA_NODEJS_ENTRYPOINT environment variable must point to your entry file'
+ : '')));
+ }
+ try {
+ fs.writeFileSync(INIT_ERROR_FILE, errorType);
+ } catch (ignored) {
+ // A full disk leaves Floci with Runtime.ExitError, without a stack trace AWS would not write.
+ }
+ process.exit(1);
+}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneDockerIntegrationTest.java b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneDockerIntegrationTest.java
index 4680d3a881..5546cb22ca 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneDockerIntegrationTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebDataPlaneDockerIntegrationTest.java
@@ -66,6 +66,8 @@ class LambdaWebDataPlaneDockerIntegrationTest {
const [host, port] = process.env.AWS_LAMBDA_HTTP_ENDPOINT.split(':');
console.log('starting');
if (process.env.MODE === 'throw') { throw new Error('init boom'); }
+ if (process.env.MODE === 'exit') { process.exit(1); }
+ if (process.env.MODE === 'missing') { require('no-such-module'); }
http.createServer((req, res) => {
if (req.url.startsWith('/log')) {
console.info({route: 'log', main: require.main === module, argv: process.argv.slice(1)});
@@ -504,8 +506,40 @@ class LambdaWebDataPlaneDockerIntegrationTest {
assertEquals("WARN", initReport.getString("level"));
assertEquals("failure", initReport.getString("record.status"));
assertEquals("Application.InitCodeError", initReport.getString("record.errorType"));
- assertTrue(events.stream().anyMatch(event -> event.contains("\"message\":\"starting\"")), events.toString());
- assertTrue(events.stream().anyMatch(event -> event.contains("init boom")), events.toString());
+ List<JsonPath> output = events.stream().filter(event -> !isPlatformEvent(event)).map(JsonPath::from).toList();
+ assertEquals(List.of("ERROR Application.InitCodeError: could not load customer code: init boom", "INFO starting"),
+ output.stream().map(event -> event.getString("level") + " " + event.getString("message")).sorted()
+ .toList(), events.toString());
+ }
+
+ @Test
+ @Timeout(180)
+ void anAppThatExitsWithAnErrorAtStartGetsTheExitErrorPage() throws Exception {
+ String domain = createFunction(30, 64, "WARN", "exit");
+ String group = "/aws/lambda/web/" + functionName + ".local";
+ given().header("Host", domain).when().get("/").then().statusCode(502)
+ .header("lambda-web-error-type", "Runtime.ExitError");
+
+ List<String> events = awaitLogEvents(group, awaitLogStream(group),
+ each -> types(each).contains("platform.initReport"));
+ assertEquals("Runtime.ExitError", platformEvent(events, "platform.initReport").getString("record.errorType"));
+ }
+
+ @Test
+ @Timeout(180)
+ void anAppThatCannotFindAModuleGetsTheImportCodeErrorPage() throws Exception {
+ String domain = createFunction(30, 64, "WARN", "missing");
+ String group = "/aws/lambda/web/" + functionName + ".local";
+ given().header("Host", domain).when().get("/").then().statusCode(502)
+ .header("lambda-web-error-type", "Runtime.ImportCodeError");
+
+ List<String> events = awaitLogEvents(group, awaitLogStream(group), each -> types(each).contains(
+ "platform.initReport") && each.stream().anyMatch(event -> event.contains("ImportCodeError:")));
+ assertEquals("Runtime.ImportCodeError",
+ platformEvent(events, "platform.initReport").getString("record.errorType"));
+ assertTrue(events.stream().anyMatch(event -> event.contains("\"message\":\"Runtime.ImportCodeError: could not"
+ + " load customer code: Cannot find module 'no-such-module'\\nRequire stack:\\n- /var/task/index.js."
+ + " The entrypoint must be")), events.toString());
}
private static boolean isPlatformEvent(String event) {
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebErrorPagesTest.java b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebErrorPagesTest.java
index 39f562afb6..c5cfdddf1e 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebErrorPagesTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambdaweb/LambdaWebErrorPagesTest.java
@@ -23,7 +23,9 @@ class LambdaWebErrorPagesTest {
assertEquals(236, length(LambdaWebErrorPages.ORIGIN_REQUEST_ERROR));
assertEquals(228, length(LambdaWebErrorPages.TIMEOUT_LISTEN_ERROR));
assertEquals(228, length(LambdaWebErrorPages.INIT_CODE_ERROR));
+ assertEquals(228, length(LambdaWebErrorPages.IMPORT_CODE_ERROR));
assertEquals(228, length(LambdaWebErrorPages.RUNTIME_EXIT));
+ assertEquals(228, length(LambdaWebErrorPages.RUNTIME_EXIT_ERROR));
assertEquals(237, length(LambdaWebErrorPages.ENDPOINT_MAX_ENV_THROTTLE));
assertEquals(237, length(LambdaWebErrorPages.ENDPOINT_REQUEST_THROTTLE));
assertEquals(180, LambdaWebErrorPages.MALFORMED_REQUEST_HTML.getBytes(StandardCharsets.UTF_8).length);