Skip to content

armgit / floci

Public Imported from GitHub
0

Light, fluffy, and always free - The AWS Local Emulator alternative

feat(lambda): implement the Lambda Extensions API (#1773)

Jay9f0dec9317d2
diff --git a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.java b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.java
index 89bda916ff..0996536ff3 100644
--- a/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.java
+++ b/src/main/java/io/github/hectorvent/floci/core/common/docker/ContainerLogStreamer.java
@@ -57,22 +57,53 @@ public class ContainerLogStreamer {
.withStdErr(true)
.withFollowStream(true)
.withTimestamps(false)
- .exec(new ResultCallback.Adapter<>() {
- @Override
- public void onNext(Frame frame) {
- String line = new String(frame.getPayload(), StandardCharsets.UTF_8).stripTrailing();
- if (!line.isEmpty()) {
- LOG.infov("[{0}] {1}", logPrefix, line);
- forwardToCloudWatchLogs(logGroup, logStream, region, line);
- }
- }
- });
+ .exec(frameCallback(logGroup, logStream, region, logPrefix));
} catch (Exception e) {
LOG.warnv("Could not attach log stream for container {0}: {1}", containerId, e.getMessage());
return null;
}
}
+ /**
+ * Streams the output of a {@code docker exec} to the same destinations as {@link #attach}.
+ *
+ * <p>Needed because {@code docker logs} (and therefore {@link #attach}, which uses
+ * {@code logContainerCmd}) only covers the container's PID 1 stdout/stderr. An exec runs on a
+ * separate Docker stream that never reaches that log, so a Lambda extension started via exec
+ * would otherwise produce no CloudWatch output at all — the opposite of what an observability
+ * extension is for.
+ *
+ * <p>The returned callback must be handed to {@code execStartCmd(...).exec(...)} by the caller,
+ * which owns the exec's lifecycle. Draining it is required regardless of logging: an undrained
+ * exec output pipe fills up and stalls the process on the other end.
+ *
+ * @param logPrefix console-only prefix (e.g. {@code "lambda:myFn:my-extension"}); the message
+ * forwarded to CloudWatch is left unprefixed so the log stream matches real
+ * AWS, which interleaves extension output with the function's own.
+ */
+ public ResultCallback.Adapter<Frame> execLogCallback(String logGroup, String logStream,
+ String region, String logPrefix) {
+ return frameCallback(logGroup, logStream, region, logPrefix);
+ }
+
+ /** Shared frame handling for both the container log stream and exec streams. */
+ private ResultCallback.Adapter<Frame> frameCallback(String logGroup, String logStream,
+ String region, String logPrefix) {
+ return new ResultCallback.Adapter<>() {
+ @Override
+ public void onNext(Frame frame) {
+ if (frame == null || frame.getPayload() == null) {
+ return;
+ }
+ String line = new String(frame.getPayload(), StandardCharsets.UTF_8).stripTrailing();
+ if (!line.isEmpty()) {
+ LOG.infov("[{0}] {1}", logPrefix, line);
+ forwardToCloudWatchLogs(logGroup, logStream, region, line);
+ }
+ }
+ };
+ }
+
/**
* Creates a CloudWatch log group and stream if they don't already exist.
*/
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java b/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java
index e6b4361d9a..b81829a83e 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/WarmPool.java
@@ -117,6 +117,15 @@ public class WarmPool implements ContainerTeardown {
if (candidate == null) {
break;
}
+ // A container whose extension reported a fatal error is still *running*, so the
+ // liveness probe alone would hand it back out. Skip it for the same reason a dead
+ // one is skipped: it can no longer serve invocations correctly.
+ if (candidate.isFaulted()) {
+ LOG.infov("Discarding pooled container {0} for function {1}: an extension reported a fatal error",
+ candidate.getContainerId(), fn.getFunctionName());
+ stopQuietly(candidate);
+ continue;
+ }
if (containerLauncher.isAlive(candidate)) {
handle = candidate;
break;
@@ -145,6 +154,16 @@ public class WarmPool implements ContainerTeardown {
*/
public void release(ContainerHandle handle) {
boolean ephemeral = config != null && config.services().lambda().ephemeral();
+ // An extension reporting an init/exit error is fatal to the execution environment in real
+ // AWS. RuntimeApiServer already refuses new work at that point; the container is torn down
+ // here rather than at fault time so the invocation that was in flight when the extension
+ // failed still completes normally through the runtime.
+ if (handle.isFaulted()) {
+ LOG.infov("Retiring container {0} for function {1}: an extension reported a fatal error",
+ handle.getContainerId(), handle.getFunctionName());
+ stopQuietly(handle);
+ return;
+ }
if (ephemeral || handle.isHotReload()) {
LOG.debugv("{0}: stopping container {1} after invocation",
handle.isHotReload() ? "Hot-reload" : "Ephemeral", handle.getContainerId());
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerHandle.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerHandle.java
index abd458d7f7..7e78bd1487 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerHandle.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerHandle.java
@@ -44,6 +44,18 @@ public class ContainerHandle {
public void touchLastUsed() { this.lastUsedMs = System.currentTimeMillis(); }
public ContainerState getState() { return state; }
public void setState(ContainerState state) { this.state = state; }
+
public Closeable getLogStream() { return logStream; }
public void setLogStream(Closeable logStream) { this.logStream = logStream; }
+
+ /**
+ * True once an extension in this container reported an init or exit error. Real AWS treats
+ * both as fatal to the execution environment, so {@code WarmPool} must retire the container
+ * instead of pooling or reusing it. Read from the container's {@code RuntimeApiServer}, which
+ * is where extensions report the error; a handle with no runtime server (test fixtures) is
+ * never faulted.
+ */
+ public boolean isFaulted() {
+ return runtimeApiServer != null && runtimeApiServer.isFaulted();
+ }
}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java
index 1f4e1cc99d..0925c9d04f 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncher.java
@@ -17,11 +17,13 @@ import io.github.hectorvent.floci.services.lambda.model.LambdaLayerVersion;
import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServer;
import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServerFactory;
import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.exception.NotFoundException;
import jakarta.enterprise.context.ApplicationScoped;
import jakarta.inject.Inject;
import org.jboss.logging.Logger;
import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveInputStream;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import java.io.Closeable;
@@ -147,6 +149,8 @@ public class ContainerLauncher {
// Start Runtime API server first so container can connect on boot
RuntimeApiServer runtimeApiServer = runtimeApiServerFactory.create();
+ runtimeApiServer.setFunctionMetadata(fn.getFunctionName(), fn.getVersion(), fn.getHandler(),
+ fn.getAccountId());
// Everything after the runtime-api server is allocated runs inside one try/catch: a failure
// ANYWHERE below — image/host resolve, the code-volume populate (ensureCodeVolume), the spec
@@ -328,6 +332,29 @@ public class ContainerLauncher {
// Now start the container with code in place
lifecycleManager.startCreated(containerId, spec);
+ // Extensions can log as soon as they start, which is before the container's own log stream
+ // is attached below. Create the group/stream up front so those early lines are not dropped
+ // by CloudWatch; the call is idempotent, so attach() repeating it is harmless.
+ LogDestination logDestination = new LogDestination(cwLogGroup, cwLogStream, lambdaRegion);
+ logStreamer.ensureLogGroupAndStream(cwLogGroup, cwLogStream, lambdaRegion);
+
+ // Real AWS's runtime interface client discovers and launches every binary under
+ // /opt/extensions/ as a sibling process to the main entrypoint before the runtime is
+ // considered ready; Floci only runs the image's own ENTRYPOINT/CMD, so extensions
+ // (e.g. aws-lambda-web-adapter) never start without this. Best-effort: an extension
+ // launch failure shouldn't fail the whole container launch, since a function with no
+ // extensions is the common case and this must be a no-op for it.
+ launchExtensions(dockerClient, containerId, fn.getFunctionName(), runtimeApiServer, logDestination);
+
+ // Init-readiness barrier: the execs above are detached, so without waiting here the caller
+ // could enqueue the first invocation before an extension is ready to receive it — the
+ // adapter would silently miss that invoke. Real AWS likewise holds the environment out of
+ // service until every extension is init-ready, which it defines as the extension's first
+ // /extension/event/next rather than its register call. A no-extensions function (the
+ // common case) does not wait at all; a timeout is non-fatal, since a container that serves
+ // invocations without a slow extension is strictly better than failing the launch.
+ awaitExtensionReadiness(runtimeApiServer, fn.getFunctionName());
+
ContainerHandle handle = new ContainerHandle(containerId, fn.getFunctionName(), runtimeApiServer, ContainerState.WARM, fn.isHotReload());
// Attach log streaming
@@ -600,6 +627,136 @@ public class ContainerLauncher {
return runtime != null && runtime.startsWith("provided");
}
+ /** Directory Lambda's real init system scans for extension binaries. */
+ private static final String EXTENSIONS_DIR = "/opt/extensions";
+
+ /**
+ * How long a launch waits for extensions to call {@code /extension/register} before giving up
+ * and serving invocations without them. Real AWS allows extensions the function's full init
+ * budget (10s); this is deliberately shorter, since exceeding it here is non-fatal and the
+ * cost of the wait is paid on every cold start of a function that has extensions.
+ */
+ private static final long EXTENSION_REGISTRATION_TIMEOUT_MS = 5000;
+
+ /**
+ * Discovers binaries under {@link #EXTENSIONS_DIR} inside the (already started) container and
+ * launches each as a detached process — the piece of real AWS's init system (which starts every
+ * registered extension alongside the runtime) that Floci otherwise has no equivalent for. Uses
+ * {@code docker exec} rather than baking a wrapper entrypoint into the image, since the image is
+ * user-supplied and unmodified.
+ *
+ * <p>Extensions inherit the container's env (already set on the container itself, including
+ * {@code AWS_LAMBDA_RUNTIME_API}) automatically — {@code docker exec} does not need it re-supplied.
+ * Best-effort throughout: a function with no {@code /opt/extensions} directory (the common case)
+ * must be a silent no-op, and a failure launching one extension must not prevent the container
+ * (or other extensions) from running.
+ */
+ /** Where a container's output is sent: the CloudWatch log group/stream and its region. */
+ private record LogDestination(String logGroup, String logStream, String region) { }
+
+ private void launchExtensions(DockerClient dockerClient, String containerId, String functionName,
+ RuntimeApiServer runtimeApiServer, LogDestination logDestination) {
+ List<String> extensionNames = listExtensionBinaries(dockerClient, containerId, functionName);
+ // Arm the readiness barrier before starting any extension process, so the latch already
+ // exists when the first one starts polling /extension/event/next — otherwise a
+ // fast-starting extension could become ready before there is anything to count it down.
+ runtimeApiServer.expectExtensions(extensionNames.size());
+ for (String name : extensionNames) {
+ try {
+ String path = EXTENSIONS_DIR + "/" + name;
+ var create = dockerClient.execCreateCmd(containerId)
+ .withCmd(path)
+ .withAttachStdout(true)
+ .withAttachStderr(true);
+ String execId = create.exec().getId();
+ // Detached: an extension runs for the container's whole lifetime, so this must not
+ // block the launch waiting for it to exit.
+ //
+ // The callback forwards the exec's output to the function's CloudWatch log group,
+ // the same destination the container's PID 1 output goes to. That has to be done
+ // explicitly: `docker logs` only covers PID 1, so an exec's stream never reaches
+ // the container log and an observability extension's output would vanish entirely.
+ // Draining is also required in its own right — an unread exec pipe fills up and
+ // stalls the extension process.
+ dockerClient.execStartCmd(execId).exec(logStreamer.execLogCallback(
+ logDestination.logGroup(), logDestination.logStream(), logDestination.region(),
+ "lambda:" + functionName + ":" + name));
+ LOG.infov("Launched extension {0} for function {1} (container {2})",
+ name, functionName, containerId);
+ } catch (Exception e) {
+ LOG.warnv(e, "Failed to launch extension {0} for function {1}; continuing without it",
+ name, functionName);
+ }
+ }
+ }
+
+ /**
+ * Waits for every launched extension to become init-ready — its first
+ * {@code /extension/event/next}, the point at which AWS considers an extension initialised —
+ * before the container is handed to the caller for its first invocation.
+ *
+ * <p>Best-effort by design: a timeout logs and proceeds rather than failing the launch. An
+ * extension that is slow or crashes on startup should degrade to "invocations run without it"
+ * — the same outcome as before this barrier existed — not take the whole function down. A
+ * genuinely broken extension reports {@code init/error} instead, which condemns the
+ * environment through {@code RuntimeApiServer}'s fault path.
+ */
+ private void awaitExtensionReadiness(RuntimeApiServer runtimeApiServer, String functionName) {
+ try {
+ if (!runtimeApiServer.awaitExtensionsReady(EXTENSION_REGISTRATION_TIMEOUT_MS)) {
+ LOG.warnv("Not all extensions for function {0} became ready within {1}ms; "
+ + "continuing without them",
+ functionName, String.valueOf(EXTENSION_REGISTRATION_TIMEOUT_MS));
+ }
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ LOG.debugv("Interrupted waiting for extensions of function {0} to become ready", functionName);
+ }
+ }
+
+ /**
+ * Lists file names directly under {@link #EXTENSIONS_DIR} inside the container, or an empty
+ * list if the directory doesn't exist or the probe fails (most functions have no extensions).
+ *
+ * <p>Uses Docker's archive API ({@code copyArchiveFromContainerCmd}) rather than {@code exec}ing
+ * a shell/{@code find}/{@code basename} pipeline inside the container: a minimal or distroless
+ * function image can have a valid extension binary under {@code /opt/extensions} without any of
+ * those utilities present, which would silently look like "no extensions" to a shell-based probe.
+ * The archive API only talks to the Docker daemon, so it works regardless of the image contents.
+ */
+ private List<String> listExtensionBinaries(DockerClient dockerClient, String containerId, String functionName) {
+ try (InputStream tarStream = dockerClient.copyArchiveFromContainerCmd(containerId, EXTENSIONS_DIR).exec();
+ TarArchiveInputStream tar = new TarArchiveInputStream(tarStream)) {
+
+ List<String> names = new ArrayList<>();
+ TarArchiveEntry entry;
+ while ((entry = tar.getNextEntry()) != null) {
+ if (!tar.canReadEntryData(entry) || entry.isDirectory()) {
+ continue;
+ }
+ // Docker wraps the requested directory itself as a leading path component (e.g.
+ // "extensions/lambda-adapter"); only direct children count as extension binaries,
+ // matching the real init system's non-recursive scan of the directory.
+ String name = entry.getName();
+ int slash = name.indexOf('/');
+ if (slash < 0 || name.indexOf('/', slash + 1) >= 0) {
+ continue;
+ }
+ if ((entry.getMode() & 0111) == 0) {
+ continue;
+ }
+ names.add(name.substring(slash + 1));
+ }
+ return names;
+ } catch (NotFoundException e) {
+ return List.of();
+ } catch (Exception e) {
+ LOG.debugv("Could not list {0} for function {1} ({2}); assuming no extensions",
+ EXTENSIONS_DIR, functionName, e.getMessage());
+ return List.of();
+ }
+ }
+
/**
* Resolves the host path of Floci's CA certificate to inject into Lambda containers, or
* empty when TLS is disabled or no readable certificate exists. Mirrors {@code TlsConfigSource}:
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/model/ExtensionEvent.java b/src/main/java/io/github/hectorvent/floci/services/lambda/model/ExtensionEvent.java
new file mode 100644
index 0000000000..ed851f6c78
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/model/ExtensionEvent.java
@@ -0,0 +1,39 @@
+package io.github.hectorvent.floci.services.lambda.model;
+
+/**
+ * A lifecycle event delivered to a registered extension via
+ * {@code GET /2020-01-01/extension/event/next} — either {@code INVOKE} (mirrors a runtime
+ * invocation) or {@code SHUTDOWN} (container is about to stop).
+ */
+public class ExtensionEvent {
+
+ public enum Type { INVOKE, SHUTDOWN }
+
+ private final Type type;
+ private final String requestId;
+ private final long deadlineMs;
+ private final String functionArn;
+ private final String shutdownReason;
+
+ public static ExtensionEvent invoke(String requestId, long deadlineMs, String functionArn) {
+ return new ExtensionEvent(Type.INVOKE, requestId, deadlineMs, functionArn, null);
+ }
+
+ public static ExtensionEvent shutdown(long deadlineMs, String shutdownReason) {
+ return new ExtensionEvent(Type.SHUTDOWN, null, deadlineMs, null, shutdownReason);
+ }
+
+ private ExtensionEvent(Type type, String requestId, long deadlineMs, String functionArn, String shutdownReason) {
+ this.type = type;
+ this.requestId = requestId;
+ this.deadlineMs = deadlineMs;
+ this.functionArn = functionArn;
+ this.shutdownReason = shutdownReason;
+ }
+
+ public Type getType() { return type; }
+ public String getRequestId() { return requestId; }
+ public long getDeadlineMs() { return deadlineMs; }
+ public String getFunctionArn() { return functionArn; }
+ public String getShutdownReason() { return shutdownReason; }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/model/RegisteredExtension.java b/src/main/java/io/github/hectorvent/floci/services/lambda/model/RegisteredExtension.java
new file mode 100644
index 0000000000..6eb5feda09
--- /dev/null
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/model/RegisteredExtension.java
@@ -0,0 +1,67 @@
+package io.github.hectorvent.floci.services.lambda.model;
+
+import java.util.ArrayDeque;
+import java.util.List;
+import io.vertx.ext.web.RoutingContext;
+
+/**
+ * Tracks one extension's Extensions API registration state within a single container's
+ * {@code RuntimeApiServer} — its subscribed event types and the queue/parked-poller pair
+ * used to deliver {@link ExtensionEvent}s to its {@code /extension/event/next} polling loop.
+ *
+ * Not thread-safe on its own — all access to {@code pendingEvents}/{@code waitingContext}
+ * must happen while holding the owning {@code RuntimeApiServer}'s lock. This class
+ * intentionally has no internal synchronization; see {@code RuntimeApiServer}'s class doc
+ * for the locking discipline.
+ */
+public class RegisteredExtension {
+
+ private final String identifier;
+ private final String name;
+ private final List<String> subscribedEvents;
+ private final ArrayDeque<ExtensionEvent> pendingEvents = new ArrayDeque<>();
+ private RoutingContext waitingContext;
+ private boolean firstNextReceived;
+
+ public RegisteredExtension(String identifier, String name, List<String> subscribedEvents) {
+ this.identifier = identifier;
+ this.name = name;
+ this.subscribedEvents = subscribedEvents;
+ }
+
+ public String getIdentifier() { return identifier; }
+ public String getName() { return name; }
+ public List<String> getSubscribedEvents() { return subscribedEvents; }
+
+ public boolean isSubscribedTo(ExtensionEvent.Type type) {
+ return subscribedEvents.contains(type.name());
+ }
+
+ public ArrayDeque<ExtensionEvent> getPendingEvents() { return pendingEvents; }
+
+ public RoutingContext takeWaitingContext() {
+ RoutingContext ctx = waitingContext;
+ waitingContext = null;
+ return ctx;
+ }
+
+ public void setWaitingContext(RoutingContext ctx) {
+ waitingContext = ctx;
+ }
+
+ /**
+ * Records that this extension has issued its first {@code /extension/event/next}, which is
+ * what AWS treats as the extension being init-ready (registering alone only obtains an
+ * identifier). Called under the owning server's lock, like the rest of this class.
+ *
+ * @return true only on the first call, so the caller counts the readiness barrier down once
+ * per extension no matter how many times it polls.
+ */
+ public boolean markFirstNextReceived() {
+ if (firstNextReceived) {
+ return false;
+ }
+ firstNextReceived = true;
+ return true;
+ }
+}
diff --git a/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java b/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java
index 9e5d2095ea..8dd861a27d 100644
--- a/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java
+++ b/src/main/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServer.java
@@ -1,27 +1,67 @@
package io.github.hectorvent.floci.services.lambda.runtime;
+import io.github.hectorvent.floci.services.lambda.model.ExtensionEvent;
import io.github.hectorvent.floci.services.lambda.model.InvokeResult;
import io.github.hectorvent.floci.services.lambda.model.PendingInvocation;
+import io.github.hectorvent.floci.services.lambda.model.RegisteredExtension;
import io.vertx.core.Vertx;
import io.vertx.core.http.HttpServer;
import io.vertx.core.http.HttpServerOptions;
+import io.vertx.core.json.JsonObject;
import io.vertx.ext.web.Router;
import io.vertx.ext.web.RoutingContext;
import io.vertx.ext.web.handler.BodyHandler;
import org.jboss.logging.Logger;
+import java.util.ArrayDeque;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.UUID;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.ConcurrentLinkedQueue;
+import java.util.concurrent.CountDownLatch;
/**
- * Per-container HTTP server implementing the AWS Lambda Runtime API.
+ * Per-container HTTP server implementing the AWS Lambda Runtime API and, for
+ * Lambda Extensions (e.g. aws-lambda-web-adapter), the Extensions API.
* NOT a CDI bean — instances are created by RuntimeApiServerFactory.
*
* The container's language runtime connects to this server to:
* - Poll for the next invocation (GET /runtime/invocation/next)
* - Report success (POST /runtime/invocation/{requestId}/response)
* - Report failure (POST /runtime/invocation/{requestId}/error)
+ *
+ * Extension processes (binaries under /opt/extensions/) connect to this server to:
+ * - Register for lifecycle events (POST /extension/register)
+ * - Poll for the next event (GET /extension/event/next)
+ * - Report an init/exit error (POST /extension/init/error, /extension/exit/error)
+ *
+ * Extension event delivery is intentionally decoupled from invoke completion: unlike real
+ * AWS (where the Invoke phase only ends once the runtime AND every registered extension have
+ * each signaled done via /next), completing a PendingInvocation's resultFuture here depends
+ * only on the runtime's own /response or /error call. Extensions are still notified of every
+ * INVOKE/SHUTDOWN via their own /event/next polling loop. This is a deliberate simplification:
+ * it's enough for extensions like aws-lambda-web-adapter (whose registration alone is what
+ * gates it starting its proxy loop) without taking on exact post-invoke completion timing,
+ * which mainly matters for extensions doing work after the response is already returned to
+ * the client (e.g. flushing telemetry) — not a fidelity floci's local dev/test use case needs.
+ *
+ * Locking discipline: every read/write of {@code stopped}, {@code pendingQueue},
+ * {@code waitingContexts}, {@code extensions}, and each {@code RegisteredExtension}'s
+ * {@code pendingEvents}/waiting context happens inside a single {@code synchronized(lock)}
+ * block per operation. That block only *decides* what to do (dispatch to a specific
+ * RoutingContext, complete a future, or nothing) and returns/collects that decision — the
+ * actual {@code ctx.response()...end()}/{@code vertx.runOnContext(...)} calls always happen
+ * after releasing the lock, so the lock is never held across a call into Vert.x and hold
+ * times stay to sub-microsecond in-memory queue operations. One lock per server instance
+ * (up to ~100 instances live at once, one per container) means no cross-container contention.
+ * This intentionally stays non-blocking/event-loop-based throughout — NEXT_PATH and
+ * EXTENSION_NEXT_PATH never park a real thread in {@code wait()} — because the container's
+ * language-runtime bootstrap process holds one of these long-polls open for its entire idle
+ * lifetime; blocking a bounded worker-thread pool (Quarkus's default is 20 threads) for that
+ * long would exhaust it with only a handful of warm containers.
*/
public class RuntimeApiServer {
@@ -33,6 +73,26 @@ public class RuntimeApiServer {
private static final String ERROR_PATH = "/" + RUNTIME_API_VERSION + "/runtime/invocation/:requestId/error";
private static final String INIT_ERROR_PATH = "/" + RUNTIME_API_VERSION + "/runtime/init/error";
+ private static final String EXTENSIONS_API_VERSION = "2020-01-01";
+ private static final String EXTENSION_REGISTER_PATH = "/" + EXTENSIONS_API_VERSION + "/extension/register";
+ private static final String EXTENSION_NEXT_PATH = "/" + EXTENSIONS_API_VERSION + "/extension/event/next";
+ private static final String EXTENSION_INIT_ERROR_PATH = "/" + EXTENSIONS_API_VERSION + "/extension/init/error";
+ private static final String EXTENSION_EXIT_ERROR_PATH = "/" + EXTENSIONS_API_VERSION + "/extension/exit/error";
+
+ private static final String EXTENSION_NAME_HEADER = "Lambda-Extension-Name";
+ private static final String EXTENSION_ID_HEADER = "Lambda-Extension-Identifier";
+ private static final String EXTENSION_EVENT_ID_HEADER = "Lambda-Extension-Event-Identifier";
+ private static final String EXTENSION_ACCEPT_FEATURE_HEADER = "Lambda-Extension-Accept-Feature";
+
+ /** Required by AWS on both /extension/init/error and /extension/exit/error. */
+ private static final String EXTENSION_ERROR_TYPE_HEADER = "Lambda-Extension-Function-Error-Type";
+
+ /** Opt-in feature that adds {@code accountId} to the register response. */
+ private static final String ACCOUNT_ID_FEATURE = "accountId";
+
+ /** Matches the account used elsewhere when a function carries none (see LambdaService). */
+ private static final String DEFAULT_ACCOUNT_ID = "000000000000";
+
private static final byte[] CONTAINER_STOPPED_PAYLOAD =
"{\"errorMessage\":\"Container stopped\",\"errorType\":\"ContainerStopped\"}".getBytes();
@@ -44,27 +104,120 @@ public class RuntimeApiServer {
private final Vertx vertx;
private final int port;
- // Invocations queued before a /next poller arrived.
- private final ConcurrentLinkedQueue<PendingInvocation> pendingQueue = new ConcurrentLinkedQueue<>();
+ // Guards stopped, pendingQueue, waitingContexts, extensions, closeFuture, and every
+ // RegisteredExtension's pendingEvents/waitingContext. See class doc for the discipline.
+ private final Object lock = new Object();
- // /next callers parked while the pending queue is empty.
- private final ConcurrentLinkedQueue<RoutingContext> waitingContexts = new ConcurrentLinkedQueue<>();
+ // Invocations queued before a /next poller arrived. Guarded by lock.
+ private final ArrayDeque<PendingInvocation> pendingQueue = new ArrayDeque<>();
+
+ // /next callers parked while the pending queue is empty. Guarded by lock.
+ private final ArrayDeque<RoutingContext> waitingContexts = new ArrayDeque<>();
private final ConcurrentHashMap<String, PendingInvocation> inFlight = new ConcurrentHashMap<>();
+ // Extensions registered via /extension/register, keyed by their generated identifier.
+ // Guarded by lock.
+ private final Map<String, RegisteredExtension> extensions = new HashMap<>();
+
private volatile HttpServer httpServer;
- private volatile boolean stopped;
- private volatile CompletableFuture<Void> closeFuture;
+ private boolean stopped;
+ private CompletableFuture<Void> closeFuture;
+
+ // Set once an extension reports an init/exit error. Real AWS treats both as fatal to the
+ // execution environment, so the container must neither accept new work nor be reused.
+ //
+ // Read under `lock` alongside `stopped` in enqueue()/NEXT_PATH — a condemned environment
+ // refuses new invocations rather than queueing them to a runtime that will never receive
+ // them — and read without the lock by WarmPool (hence `volatile`) to decide whether to
+ // retire the container instead of pooling it.
+ //
+ // Distinct from `stopped`: this does *not* tear the server down. The invocation already
+ // in flight when the extension failed still completes normally through the runtime's own
+ // /response call, matching AWS's treatment of the environment as condemned for future work
+ // rather than aborted mid-invoke. WarmPool performs the actual teardown afterwards.
+ private volatile boolean faulted;
+
+ // Init-readiness barrier. ContainerLauncher launches extension binaries as detached `docker
+ // exec`s and returns immediately, so without this the first invocation can reach the runtime
+ // before an extension is ready — the adapter never sees that invoke and silently misses it.
+ // ContainerLauncher declares how many binaries it launched via expectExtensions();
+ // awaitExtensionsReady() blocks the launch until they are all ready (or the deadline passes).
+ //
+ // Readiness is the extension's *first /extension/event/next*, not its register call, matching
+ // AWS's lifecycle: registering only obtains an identifier, whereas an extension that is
+ // polling for events has finished its own initialisation and can actually receive an INVOKE.
+ // An extension that registers and then spends time initialising is correctly still counted as
+ // not-ready during that gap.
+ //
+ // A CountDownLatch is level-triggered, so an await() that arrives after the last extension
+ // became ready returns immediately rather than missing the signal — the latch is created
+ // before any extension can register, which is what makes that safe.
+ private volatile CountDownLatch extensionsReady;
+
+ // Set by ContainerLauncher once it knows which function this server instance is for (the
+ // factory creates the server generically, before that's known) — used only to populate the
+ // Extensions API register response.
+ private volatile String functionName = "function";
+ private volatile String functionVersion = "$LATEST";
+ private volatile String handler = "";
+ private volatile String accountId = DEFAULT_ACCOUNT_ID;
RuntimeApiServer(Vertx vertx, int port) {
this.vertx = vertx;
this.port = port;
}
+ public void setFunctionMetadata(String functionName, String functionVersion, String handler,
+ String accountId) {
+ this.functionName = functionName != null ? functionName : "function";
+ this.functionVersion = functionVersion != null ? functionVersion : "$LATEST";
+ this.handler = handler != null ? handler : "";
+ this.accountId = accountId != null && !accountId.isBlank() ? accountId : DEFAULT_ACCOUNT_ID;
+ }
+
public int getPort() {
return port;
}
+ /**
+ * True once a registered extension reported an init or exit error. The execution environment
+ * is condemned: {@code WarmPool} must not return this container to the pool or hand it out
+ * again, matching real AWS's treatment of both errors as fatal to the environment.
+ */
+ public boolean isFaulted() {
+ return faulted;
+ }
+
+ /**
+ * Declares how many extension binaries were launched for this container, arming the
+ * init-readiness barrier that {@link #awaitExtensionsReady(long)} waits on.
+ *
+ * <p>Must be called <em>before</em> the extension processes are started, so the latch exists
+ * before any of them can call {@code /extension/register}. A count of zero (the common case:
+ * no {@code /opt/extensions} directory) leaves the barrier permanently open.
+ */
+ public void expectExtensions(int count) {
+ extensionsReady = new CountDownLatch(Math.max(0, count));
+ }
+
+ /**
+ * Blocks until every extension declared via {@link #expectExtensions(int)} is init-ready —
+ * that is, until each has issued its first {@code /extension/event/next} — or the timeout
+ * elapses.
+ *
+ * @return true if all expected extensions became ready; false if the wait timed out with some
+ * still missing, in which case the container is still usable — it simply starts
+ * serving invocations without the extensions that never checked in.
+ */
+ public boolean awaitExtensionsReady(long timeoutMs) throws InterruptedException {
+ CountDownLatch latch = extensionsReady;
+ if (latch == null) {
+ return true;
+ }
+ return latch.await(timeoutMs, java.util.concurrent.TimeUnit.MILLISECONDS);
+ }
+
public CompletableFuture<Void> start() {
CompletableFuture<Void> started = new CompletableFuture<>();
@@ -76,34 +229,26 @@ public class RuntimeApiServer {
// Uses a reactive pattern (no thread held while waiting) to avoid Vert.x worker pool
// exhaustion when many warm containers poll concurrently.
router.get(NEXT_PATH).handler(ctx -> {
- if (stopped) {
- ctx.response().setStatusCode(204).end();
- return;
- }
- PendingInvocation invocation = pendingQueue.poll();
- if (invocation != null) {
- if (stopped) {
- invocation.getResultFuture().complete(
- new InvokeResult(200, "Unhandled", CONTAINER_STOPPED_PAYLOAD, null, invocation.getRequestId()));
- ctx.response().setStatusCode(204).end();
- return;
+ PendingInvocation toDispatch = null;
+ boolean send204 = false;
+ synchronized (lock) {
+ // `faulted` alongside `stopped`: a condemned environment must not be handed work,
+ // including an invocation that was queued just before the fault was reported.
+ if (stopped || faulted) {
+ send204 = true;
+ } else {
+ toDispatch = pendingQueue.poll();
+ if (toDispatch == null) {
+ waitingContexts.add(ctx);
+ }
}
- sendInvocation(ctx, invocation);
- return;
}
- // No invocation queued yet — park this context until enqueue() wakes it.
- waitingContexts.add(ctx);
- // Re-check stop race: stop() may have drained waitingContexts before our add().
- if (stopped && waitingContexts.remove(ctx)) {
+ if (send204) {
ctx.response().setStatusCode(204).end();
- return;
+ } else if (toDispatch != null) {
+ sendInvocation(ctx, toDispatch);
}
- // Re-check enqueue race: an invocation may have arrived between our poll() and add().
- PendingInvocation raced = pendingQueue.poll();
- if (raced != null && waitingContexts.remove(ctx)) {
- sendInvocation(ctx, raced);
- }
- // else: still parked — enqueue() will dispatch via vertx.runOnContext().
+ // else: parked — enqueue() or stop() will dispatch this ctx later.
});
// POST /runtime/invocation/{requestId}/response — success
@@ -138,6 +283,143 @@ public class RuntimeApiServer {
sendStatusOk(ctx);
});
+ // POST /extension/register — an extension process (e.g. aws-lambda-web-adapter)
+ // registers to receive lifecycle events. Real AWS requires the Lambda-Extension-Name
+ // header to equal the extension's own file name; floci does not validate that here
+ // (the identifier it hands back is sufficient for the extension to poll /event/next).
+ router.post(EXTENSION_REGISTER_PATH).handler(ctx -> {
+ String name = ctx.request().getHeader(EXTENSION_NAME_HEADER);
+ if (name == null || name.isBlank()) {
+ ctx.response().setStatusCode(400)
+ .putHeader("Content-Type", "application/json")
+ .end("{\"errorMessage\":\"Missing Lambda-Extension-Name header\"}");
+ return;
+ }
+ List<String> events = List.of("INVOKE", "SHUTDOWN");
+ var body = ctx.body().asJsonObject();
+ if (body != null && body.getJsonArray("events") != null) {
+ events = body.getJsonArray("events").stream().map(String::valueOf).toList();
+ }
+ String identifier = UUID.randomUUID().toString();
+ RegisteredExtension extension = new RegisteredExtension(identifier, name, events);
+ boolean environmentFaulted;
+ synchronized (lock) {
+ // AWS makes an init/exit error terminal for the whole Extensions API, not just for
+ // runtime work: once the environment is condemned no further extension call
+ // succeeds. Registering into a container about to be retired would hand back an
+ // identifier that can never receive an event.
+ //
+ // Checked *inside* the lock, like every other guard on this flag:
+ // handleExtensionFatalError sets `faulted` under the lock, so a volatile read
+ // before acquiring it can be overtaken between the check and the put() below,
+ // registering into an environment condemned a moment later.
+ environmentFaulted = faulted;
+ if (!environmentFaulted) {
+ extensions.put(identifier, extension);
+ if (stopped && extension.isSubscribedTo(ExtensionEvent.Type.SHUTDOWN)) {
+ // The server is already stopping — this extension will never see stop()'s
+ // own SHUTDOWN fan-out (that already ran), so queue one now to preserve
+ // "every registered extension eventually sees SHUTDOWN."
+ extension.getPendingEvents().offer(
+ ExtensionEvent.shutdown(System.currentTimeMillis() + 2000, "SPINDOWN"));
+ }
+ }
+ }
+ if (environmentFaulted) {
+ sendExtensionFaulted(ctx);
+ return;
+ }
+ LOG.infov("Extension registered: {0} ({1}), events={2}", name, identifier, events);
+
+ JsonObject responseBody = new JsonObject()
+ .put("functionName", functionName)
+ .put("functionVersion", functionVersion)
+ .put("handler", handler);
+ // AWS only includes accountId when the extension opts in via Lambda-Extension-Accept-
+ // Feature; adding it unconditionally would diverge from the real response shape for
+ // extensions that never asked. The header is a comma-separated feature list.
+ if (acceptsFeature(ctx.request().getHeader(EXTENSION_ACCEPT_FEATURE_HEADER), ACCOUNT_ID_FEATURE)) {
+ responseBody.put("accountId", accountId);
+ }
+ ctx.response()
+ .setStatusCode(200)
+ .putHeader(EXTENSION_ID_HEADER, identifier)
+ .putHeader("Content-Type", "application/json")
+ .end(responseBody.encode());
+ });
+
+ // GET /extension/event/next — blocks until the next INVOKE or SHUTDOWN event for this
+ // extension. Mirrors NEXT_PATH's park/dispatch pattern, scoped per-extension since each
+ // extension has its own independent event queue.
+ router.get(EXTENSION_NEXT_PATH).handler(ctx -> {
+ String identifier = ctx.request().getHeader(EXTENSION_ID_HEADER);
+ ExtensionEvent toDispatch = null;
+ boolean send204 = false;
+ boolean unknownExtension = false;
+ boolean environmentFaulted = false;
+ CountDownLatch readyLatch = null;
+ synchronized (lock) {
+ RegisteredExtension extension = identifier != null ? extensions.get(identifier) : null;
+ if (extension == null) {
+ unknownExtension = true;
+ } else if (faulted) {
+ // Read under the lock alongside `stopped`: handleExtensionFatalError() sets
+ // `faulted` and drains the parked pollers as one atomic change, so checking it
+ // here is what stops a poller parking again straight after that drain.
+ environmentFaulted = true;
+ } else {
+ // AWS treats an extension as init-ready at its first /event/next, not at
+ // register: the extension has finished its own initialisation only once it
+ // starts polling. Count the readiness barrier down here, once per extension.
+ if (extension.markFirstNextReceived()) {
+ readyLatch = extensionsReady;
+ }
+ // Poll before checking `stopped`: stop() offers a SHUTDOWN to pendingEvents for
+ // any extension it doesn't find already parked, so a request arriving just
+ // after `stopped` was set — but after that offer landed — must still see it.
+ // Checking `stopped` first would answer 204 and orphan the queued SHUTDOWN.
+ toDispatch = extension.getPendingEvents().poll();
+ if (toDispatch == null) {
+ // Deliberate deviation: AWS documents only 200/403/500 here and never ends
+ // the long-poll with an empty response. Floci answers 204 once the server
+ // is stopping and there is genuinely nothing left to deliver, because the
+ // alternative is holding the connection open while the listener is torn
+ // down — the extension would see a dropped socket instead of a clean
+ // close. Only reachable for an extension not subscribed to SHUTDOWN (a
+ // subscribed one gets the real SHUTDOWN event from stop()'s fan-out, which
+ // the poll above returns), so no extension that asked to be told about
+ // shutdown learns about it this way.
+ if (stopped) {
+ send204 = true;
+ } else {
+ extension.setWaitingContext(ctx);
+ }
+ }
+ }
+ }
+ if (readyLatch != null) {
+ readyLatch.countDown();
+ }
+ if (unknownExtension) {
+ ctx.response().setStatusCode(403)
+ .putHeader("Content-Type", "application/json")
+ .end("{\"errorMessage\":\"Unknown or missing Lambda-Extension-Identifier\"}");
+ } else if (environmentFaulted) {
+ sendExtensionFaulted(ctx);
+ } else if (send204) {
+ ctx.response().setStatusCode(204).end();
+ } else if (toDispatch != null) {
+ sendExtensionEvent(ctx, toDispatch);
+ }
+ // else: parked — notifyExtensionsOfInvoke()/stop() will dispatch this ctx later.
+ });
+
+ // POST /extension/init/error, /extension/exit/error — an extension reports it can't
+ // continue. Floci doesn't restart the execution environment on this (unlike real AWS);
+ // it's enough to unregister the extension so future event fan-out skips it.
+ router.post(EXTENSION_INIT_ERROR_PATH).handler(ctx -> handleExtensionFatalError(ctx, "init"));
+ router.post(EXTENSION_EXIT_ERROR_PATH).handler(ctx -> handleExtensionFatalError(ctx, "exit"));
+
long deadline = System.currentTimeMillis() + 5000;
tryListen(started, router, deadline);
@@ -164,13 +446,44 @@ public class RuntimeApiServer {
});
}
- public synchronized CompletableFuture<Void> stop() {
- if (closeFuture != null) {
- return closeFuture;
+ public CompletableFuture<Void> stop() {
+ CompletableFuture<Void> closed;
+ List<RoutingContext> contextsToClose204;
+ List<Runnable> dispatches = new ArrayList<>();
+ List<PendingInvocation> invocationsToFailContainerStopped;
+
+ synchronized (lock) {
+ if (closeFuture != null) {
+ return closeFuture;
+ }
+ stopped = true;
+ closed = new CompletableFuture<>();
+ closeFuture = closed;
+
+ contextsToClose204 = new ArrayList<>(waitingContexts);
+ waitingContexts.clear();
+
+ ExtensionEvent shutdownEvent = ExtensionEvent.shutdown(System.currentTimeMillis() + 2000, "SPINDOWN");
+ for (RegisteredExtension ext : extensions.values()) {
+ if (!ext.isSubscribedTo(ExtensionEvent.Type.SHUTDOWN)) {
+ continue;
+ }
+ RoutingContext waitingCtx = ext.takeWaitingContext();
+ if (waitingCtx != null) {
+ dispatches.add(() -> {
+ if (!waitingCtx.response().ended()) {
+ sendExtensionEvent(waitingCtx, shutdownEvent);
+ }
+ });
+ } else {
+ ext.getPendingEvents().offer(shutdownEvent);
+ }
+ }
+
+ invocationsToFailContainerStopped = new ArrayList<>(pendingQueue);
+ pendingQueue.clear();
}
- stopped = true;
- CompletableFuture<Void> closed = new CompletableFuture<>();
- closeFuture = closed;
+
if (httpServer != null) {
httpServer.close(ar -> {
if (ar.succeeded()) {
@@ -186,9 +499,7 @@ public class RuntimeApiServer {
}
// Wake any parked /next pollers with 204 (container shutting down — runtime will exit).
- RoutingContext waiting;
- while ((waiting = waitingContexts.poll()) != null) {
- final RoutingContext ctx = waiting;
+ for (RoutingContext ctx : contextsToClose204) {
vertx.runOnContext(v -> {
if (!ctx.response().ended()) {
ctx.response().setStatusCode(204).end();
@@ -196,9 +507,13 @@ public class RuntimeApiServer {
});
}
- // Drain queued invocations that were never consumed by /next.
- PendingInvocation pending;
- while ((pending = pendingQueue.poll()) != null) {
+ // Dispatch SHUTDOWN to every extension that was already parked on /event/next.
+ for (Runnable dispatch : dispatches) {
+ vertx.runOnContext(v -> dispatch.run());
+ }
+
+ // Fail queued invocations that were never consumed by /next.
+ for (PendingInvocation pending : invocationsToFailContainerStopped) {
pending.getResultFuture().complete(
new InvokeResult(200, "Unhandled", CONTAINER_STOPPED_PAYLOAD, null, pending.getRequestId()));
}
@@ -213,35 +528,93 @@ public class RuntimeApiServer {
}
public CompletableFuture<InvokeResult> enqueue(PendingInvocation invocation) {
- if (stopped) {
+ boolean rejected;
+ List<Runnable> dispatches = new ArrayList<>();
+ RoutingContext waitingCtxForInvocation = null;
+
+ synchronized (lock) {
+ // A condemned environment (an extension reported init/exit error) refuses new work the
+ // same way a stopping one does. Without this the invocation is queued to a runtime that
+ // will never be given it, and the caller hangs until the function timeout rather than
+ // failing fast. Read inside the lock alongside `stopped` so the accept/reject decision
+ // stays a single atomic step.
+ rejected = stopped || faulted;
+ if (!rejected) {
+ if (!extensions.isEmpty()) {
+ ExtensionEvent event = ExtensionEvent.invoke(
+ invocation.getRequestId(), invocation.getDeadlineMs(), invocation.getFunctionArn());
+ for (RegisteredExtension ext : extensions.values()) {
+ if (!ext.isSubscribedTo(ExtensionEvent.Type.INVOKE)) {
+ continue;
+ }
+ RoutingContext waitingCtx = ext.takeWaitingContext();
+ if (waitingCtx != null) {
+ dispatches.add(() -> {
+ if (!waitingCtx.response().ended()) {
+ sendExtensionEvent(waitingCtx, event);
+ } else {
+ synchronized (lock) {
+ ext.getPendingEvents().offer(event);
+ }
+ }
+ });
+ } else {
+ ext.getPendingEvents().offer(event);
+ }
+ }
+ }
+
+ waitingCtxForInvocation = waitingContexts.poll();
+ if (waitingCtxForInvocation == null) {
+ pendingQueue.offer(invocation);
+ }
+ }
+ }
+
+ if (rejected) {
invocation.getResultFuture().complete(
new InvokeResult(200, "Unhandled", CONTAINER_STOPPED_PAYLOAD, null, invocation.getRequestId()));
return invocation.getResultFuture();
}
- // If a /next poller is already parked, dispatch immediately on the event loop.
- RoutingContext ctx = waitingContexts.poll();
- if (ctx != null) {
- final RoutingContext waitingCtx = ctx;
+
+ for (Runnable dispatch : dispatches) {
+ vertx.runOnContext(v -> dispatch.run());
+ }
+
+ if (waitingCtxForInvocation != null) {
+ final RoutingContext waitingCtx = waitingCtxForInvocation;
vertx.runOnContext(v -> {
if (!waitingCtx.response().ended()) {
sendInvocation(waitingCtx, invocation);
} else {
// Connection closed between park and dispatch — re-queue.
- pendingQueue.offer(invocation);
+ synchronized (lock) {
+ pendingQueue.offer(invocation);
+ }
}
});
- return invocation.getResultFuture();
- }
- pendingQueue.offer(invocation);
- // Close the check-then-offer race: if stop() ran between the guard and offer(),
- // the drain is done and our invocation would sit forever. Remove and complete.
- if (stopped && pendingQueue.remove(invocation)) {
- invocation.getResultFuture().complete(
- new InvokeResult(200, "Unhandled", CONTAINER_STOPPED_PAYLOAD, null, invocation.getRequestId()));
}
return invocation.getResultFuture();
}
+ /**
+ * True if {@code headerValue} — the comma-separated {@code Lambda-Extension-Accept-Feature}
+ * list an extension sends at registration — opts into {@code feature}. Matching is
+ * case-insensitive and tolerant of surrounding whitespace; a null/blank header opts into
+ * nothing.
+ */
+ private static boolean acceptsFeature(String headerValue, String feature) {
+ if (headerValue == null || headerValue.isBlank()) {
+ return false;
+ }
+ for (String candidate : headerValue.split(",")) {
+ if (candidate.trim().equalsIgnoreCase(feature)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
private void sendStatusOk(RoutingContext ctx) {
ctx.response()
.setStatusCode(202)
@@ -251,7 +624,7 @@ public class RuntimeApiServer {
private void sendInvocation(RoutingContext ctx, PendingInvocation invocation) {
inFlight.put(invocation.getRequestId(), invocation);
-
+
byte[] payload = invocation.getPayload();
String body = (payload != null && payload.length > 0)
? new String(payload)
@@ -264,4 +637,140 @@ public class RuntimeApiServer {
.putHeader("Lambda-Runtime-Deadline-Ms", String.valueOf(invocation.getDeadlineMs()))
.end(body);
}
+
+ private void sendExtensionEvent(RoutingContext ctx, ExtensionEvent event) {
+ JsonObject body = new JsonObject().put("eventType", event.getType().name());
+ if (event.getType() == ExtensionEvent.Type.INVOKE) {
+ body.put("requestId", event.getRequestId())
+ .put("invokedFunctionArn", event.getFunctionArn())
+ .put("deadlineMs", event.getDeadlineMs());
+ } else {
+ body.put("shutdownReason", event.getShutdownReason())
+ .put("deadlineMs", event.getDeadlineMs());
+ }
+ ctx.response()
+ .setStatusCode(200)
+ .putHeader("Content-Type", "application/json")
+ .putHeader(EXTENSION_EVENT_ID_HEADER, UUID.randomUUID().toString())
+ .end(body.encode());
+ }
+
+ /**
+ * Response for any Extensions API call made after an extension reported an init/exit error.
+ * AWS makes the fault terminal for the whole environment — no further extension call succeeds
+ * — so this is a hard error rather than the 204 used for an orderly shutdown, which an
+ * extension is entitled to read as "nothing right now, poll again".
+ */
+ private void sendExtensionFaulted(RoutingContext ctx) {
+ ctx.response()
+ .setStatusCode(500)
+ .putHeader("Content-Type", "application/json")
+ .end("{\"errorType\":\"Extension.SandboxFaulted\","
+ + "\"errorMessage\":\"Execution environment condemned by an extension "
+ + "init/exit error\"}");
+ }
+
+ private void handleExtensionFatalError(RoutingContext ctx, String phase) {
+ String identifier = ctx.request().getHeader(EXTENSION_ID_HEADER);
+ String errorType = ctx.request().getHeader(EXTENSION_ERROR_TYPE_HEADER);
+
+ // Validate before mutating anything. Retiring the execution environment is destructive and
+ // unrecoverable, so a malformed report must not trigger it — otherwise a caller that got
+ // the contract wrong silently kills a healthy container and the 202 tells them it worked.
+ // AWS requires Lambda-Extension-Function-Error-Type on both error endpoints, and rejects
+ // an unknown identifier rather than treating it as a fatal report.
+ if (identifier == null || identifier.isBlank()) {
+ ctx.response().setStatusCode(403)
+ .putHeader("Content-Type", "application/json")
+ .end("{\"errorMessage\":\"Unknown or missing Lambda-Extension-Identifier\"}");
+ return;
+ }
+ if (errorType == null || errorType.isBlank()) {
+ ctx.response().setStatusCode(400)
+ .putHeader("Content-Type", "application/json")
+ .end("{\"errorMessage\":\"Missing Lambda-Extension-Function-Error-Type header\"}");
+ return;
+ }
+
+ RegisteredExtension removed;
+ List<PendingInvocation> strandedInvocations;
+ List<RoutingContext> strandedPollers;
+ List<RoutingContext> strandedExtensionPollers;
+ synchronized (lock) {
+ removed = extensions.remove(identifier);
+ if (removed == null) {
+ // Unknown identifier: reject without condemning. Answered outside the lock below.
+ strandedInvocations = List.of();
+ strandedPollers = List.of();
+ strandedExtensionPollers = List.of();
+ } else {
+ // Set under the lock alongside the unregistration so both land as one atomic
+ // change.
+ faulted = true;
+
+ // Anything already queued or parked would otherwise wait forever, since the guards
+ // in enqueue()/NEXT_PATH now refuse to move work through a condemned environment.
+ // Drain both here so callers fail fast instead of hanging until the function
+ // timeout.
+ strandedInvocations = new ArrayList<>(pendingQueue);
+ pendingQueue.clear();
+ strandedPollers = new ArrayList<>(waitingContexts);
+ waitingContexts.clear();
+
+ // Sibling extensions parked on /event/next need the same treatment, and for the
+ // same reason: the faulted guard in that handler means nothing will ever wake
+ // them, so without this their long-poll stays open until stop() eventually runs
+ // — up to a full function timeout away. Mirrors the extensions.values() sweep in
+ // stop(); harmless with a single extension, but a second one hangs without it.
+ strandedExtensionPollers = new ArrayList<>();
+ for (RegisteredExtension ext : extensions.values()) {
+ RoutingContext waitingCtx = ext.takeWaitingContext();
+ if (waitingCtx != null) {
+ strandedExtensionPollers.add(waitingCtx);
+ }
+ }
+ }
+ }
+
+ if (removed == null) {
+ LOG.warnv("Unknown extension identifier {0} reported {1} error on port {2}; ignoring",
+ identifier, phase, String.valueOf(port));
+ ctx.response().setStatusCode(403)
+ .putHeader("Content-Type", "application/json")
+ .end("{\"errorMessage\":\"Unknown or missing Lambda-Extension-Identifier\"}");
+ return;
+ }
+
+ LOG.warnv("Extension {0} ({1}) reported {2} error ({3}) on port {4}; "
+ + "retiring execution environment",
+ removed.getName(), identifier, phase, errorType, String.valueOf(port));
+
+ for (PendingInvocation stranded : strandedInvocations) {
+ stranded.getResultFuture().complete(
+ new InvokeResult(200, "Unhandled", CONTAINER_STOPPED_PAYLOAD, null, stranded.getRequestId()));
+ }
+ for (RoutingContext poller : strandedPollers) {
+ vertx.runOnContext(v -> {
+ if (!poller.response().ended()) {
+ poller.response().setStatusCode(204).end();
+ }
+ });
+ }
+ // Reuses the same 500 that /event/next returns to any extension polling after a fault,
+ // rather than inventing a separate response for the already-parked case: AWS specifies
+ // that the environment is terminal but not what an open long-poll receives, so matching
+ // the behaviour we already ship is the conservative choice.
+ for (RoutingContext poller : strandedExtensionPollers) {
+ vertx.runOnContext(v -> {
+ if (!poller.response().ended()) {
+ sendExtensionFaulted(poller);
+ }
+ });
+ }
+
+ ctx.response()
+ .setStatusCode(202)
+ .putHeader("Content-Type", "application/json")
+ .end(STATUS_OK_BODY);
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java
index e1d10754f0..1315055ece 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/WarmPoolTest.java
@@ -5,6 +5,7 @@ import io.github.hectorvent.floci.services.lambda.launcher.ContainerHandle;
import io.github.hectorvent.floci.services.lambda.launcher.ContainerLauncher;
import io.github.hectorvent.floci.services.lambda.model.ContainerState;
import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;
+import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServer;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
import org.mockito.Mock;
@@ -14,6 +15,7 @@ import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertNotSame;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.Mockito.lenient;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.times;
@@ -157,6 +159,80 @@ class WarmPoolTest {
pool.shutdown();
}
+ /**
+ * An extension reporting init/exit error is fatal to the execution environment in real AWS,
+ * so the container must be retired after the invocation rather than returned to the pool.
+ */
+ @Test
+ void releaseAfterExtensionFatalError_retiresContainerInsteadOfPooling() {
+ WarmPool pool = buildPool();
+ pool.init();
+
+ LambdaFunction fn = mock(LambdaFunction.class);
+ when(fn.getFunctionName()).thenReturn("faulted-fn");
+
+ RuntimeApiServer faultedServer = mock(RuntimeApiServer.class);
+ when(faultedServer.isFaulted()).thenReturn(true);
+ ContainerHandle faulted = new ContainerHandle("cid-faulted", "faulted-fn", faultedServer, ContainerState.WARM);
+ ContainerHandle fresh = new ContainerHandle("cid-fresh", "faulted-fn", null, ContainerState.WARM);
+
+ // Both acquires below cold-start (the pool is empty, then the faulted handle is discarded
+ // before any liveness check), so no isAlive stubbing is needed.
+ when(containerLauncher.launch(any())).thenReturn(faulted, fresh);
+
+ ContainerHandle first = pool.acquire(fn);
+ assertSame(faulted, first);
+
+ // The extension died during this invocation; releasing must tear the container down.
+ pool.release(first);
+ verify(containerLauncher, times(1)).stop(faulted);
+
+ // The next acquire cold-starts rather than handing the condemned container back out.
+ ContainerHandle second = pool.acquire(fn);
+ assertSame(fresh, second);
+ assertNotSame(faulted, second);
+ verify(containerLauncher, times(2)).launch(any());
+
+ pool.shutdown();
+ }
+
+ /**
+ * A container whose extension faults while it sits WARM in the pool is still *running*, so the
+ * liveness probe alone would hand it back out. It must be skipped and discarded on acquire.
+ */
+ @Test
+ void acquire_discardsPooledHandleWhoseExtensionFaulted() {
+ WarmPool pool = buildPool();
+ pool.init();
+
+ LambdaFunction fn = mock(LambdaFunction.class);
+ when(fn.getFunctionName()).thenReturn("pooled-fault-fn");
+
+ RuntimeApiServer server = mock(RuntimeApiServer.class);
+ ContainerHandle pooled = new ContainerHandle("cid-pooled", "pooled-fault-fn", server, ContainerState.WARM);
+ ContainerHandle fresh = new ContainerHandle("cid-fresh", "pooled-fault-fn", null, ContainerState.WARM);
+
+ when(containerLauncher.launch(any())).thenReturn(pooled, fresh);
+ // Healthy at release time, so it goes into the pool as normal.
+ when(server.isFaulted()).thenReturn(false);
+ ContainerHandle seeded = pool.acquire(fn);
+ assertSame(pooled, seeded);
+ pool.release(seeded);
+
+ // The extension now faults while the container sits idle in the pool. isAlive() is stubbed
+ // true so the container is unambiguously still running: if the faulted check were removed,
+ // this handle would be considered reusable and handed straight back out.
+ when(server.isFaulted()).thenReturn(true);
+ lenient().when(containerLauncher.isAlive(pooled)).thenReturn(true);
+
+ ContainerHandle acquired = pool.acquire(fn);
+ assertSame(fresh, acquired);
+ assertNotSame(pooled, acquired);
+ verify(containerLauncher, times(1)).stop(pooled);
+
+ pool.shutdown();
+ }
+
@Test
void acquire_discardsDeadPooledHandleAndColdStarts() {
WarmPool pool = buildPool();
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java
index ac14850c91..e767f6f109 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/launcher/ContainerLauncherTest.java
@@ -14,9 +14,19 @@ import io.github.hectorvent.floci.services.lambda.model.LambdaFunction;
import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServer;
import io.github.hectorvent.floci.services.lambda.runtime.RuntimeApiServerFactory;
import com.github.dockerjava.api.DockerClient;
+import com.github.dockerjava.api.async.ResultCallback;
import com.github.dockerjava.api.command.CopyArchiveToContainerCmd;
+import com.github.dockerjava.api.command.ExecCreateCmd;
+import com.github.dockerjava.api.command.ExecCreateCmdResponse;
+import com.github.dockerjava.api.command.ExecStartCmd;
+import com.github.dockerjava.api.command.CopyArchiveFromContainerCmd;
+import com.github.dockerjava.api.exception.NotFoundException;
+import com.github.dockerjava.api.model.Frame;
+import com.github.dockerjava.api.model.StreamType;
import com.github.dockerjava.api.model.Mount;
import com.github.dockerjava.api.model.MountType;
+import org.apache.commons.compress.archivers.tar.TarArchiveEntry;
+import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.extension.ExtendWith;
@@ -26,11 +36,16 @@ import org.mockito.InOrder;
import org.mockito.Mock;
import org.mockito.junit.jupiter.MockitoExtension;
+import io.github.hectorvent.floci.services.cloudwatch.logs.CloudWatchLogsService;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
import java.nio.file.Files;
import java.nio.file.Path;
import java.util.List;
import java.util.Map;
import java.util.Optional;
+import java.util.concurrent.atomic.AtomicBoolean;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertNotNull;
@@ -38,6 +53,8 @@ import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyLong;
+import static org.mockito.ArgumentMatchers.anyString;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.*;
@@ -565,4 +582,334 @@ class ContainerLauncherTest {
verify(lifecycleManager, never()).create(any());
verify(lifecycleManager, never()).startCreated(any(), any());
}
+
+ /** Builds a tar archive matching what {@code docker cp}/{@code copyArchiveFromContainerCmd}
+ * returns for a directory: the directory itself as a leading entry, then each name as a direct
+ * child, executable. */
+ private static byte[] tarOf(String... binaryNames) throws IOException {
+ var entries = new java.util.LinkedHashMap<String, Boolean>();
+ for (String name : binaryNames) {
+ entries.put(name, true);
+ }
+ return tarOfRaw(entries);
+ }
+
+ /** Like {@link #tarOf}, but lets each entry's executable bit be controlled individually
+ * (name -> executable), so filtering logic (non-executable files, nested paths) can be
+ * exercised. Names containing "/" are written as-is (not prefixed), to model entries nested
+ * more than one level below the extensions directory. */
+ private static byte[] tarOfRaw(java.util.Map<String, Boolean> nameToExecutable) throws IOException {
+ var bos = new java.io.ByteArrayOutputStream();
+ try (TarArchiveOutputStream tar = new TarArchiveOutputStream(bos)) {
+ TarArchiveEntry dir = new TarArchiveEntry("extensions/");
+ tar.putArchiveEntry(dir);
+ tar.closeArchiveEntry();
+ for (var e : nameToExecutable.entrySet()) {
+ TarArchiveEntry entry = new TarArchiveEntry("extensions/" + e.getKey());
+ entry.setMode(e.getValue() ? 0100755 : 0100644); // executable vs. non-executable regular file
+ entry.setSize(0);
+ tar.putArchiveEntry(entry);
+ tar.closeArchiveEntry();
+ }
+ }
+ return bos.toByteArray();
+ }
+
+ /** Stubs container-123's /opt/extensions listing (via the Docker archive API, not exec) to
+ * return the given binary names, and stubs exec of each resulting /opt/extensions/<name> path
+ * to succeed immediately. */
+ private List<ExecCreateCmd> stubExtensionDiscovery(String... binaryNames) throws IOException {
+ return stubExtensionDiscoveryFromTar(tarOf(binaryNames));
+ }
+
+ /** Same as {@link #stubExtensionDiscovery}, but takes a caller-built tar so tests can exercise
+ * {@code listExtensionBinaries}' entry filtering (non-executable files, nested paths) rather
+ * than only the convenience all-executable-direct-children shape. Returns the {@code ExecCreateCmd}
+ * mock for each launched binary, in launch order — pass it to {@link #capturedLaunchPaths} after
+ * calling {@code launch()} to assert on exactly which discovered names were (and weren't) launched. */
+ private List<ExecCreateCmd> stubExtensionDiscoveryFromTar(byte[] tar) {
+ CopyArchiveFromContainerCmd copyCmd = mock(CopyArchiveFromContainerCmd.class);
+ when(copyCmd.exec()).thenReturn(new java.io.ByteArrayInputStream(tar));
+ when(dockerClient.copyArchiveFromContainerCmd("container-123", "/opt/extensions"))
+ .thenReturn(copyCmd);
+
+ List<ExecCreateCmd> launchCmds = new java.util.ArrayList<>();
+ lenient().when(dockerClient.execCreateCmd("container-123"))
+ .thenAnswer(invocation -> {
+ ExecCreateCmd launchCmd = mock(ExecCreateCmd.class, withSettings().defaultAnswer(RETURNS_SELF));
+ ExecCreateCmdResponse launchResponse = mock(ExecCreateCmdResponse.class);
+ when(launchResponse.getId()).thenReturn("exec-launch-" + java.util.UUID.randomUUID());
+ when(launchCmd.exec()).thenReturn(launchResponse);
+ launchCmds.add(launchCmd);
+ return launchCmd;
+ });
+ lenient().when(dockerClient.execStartCmd(argThat(id -> id != null && id.startsWith("exec-launch-"))))
+ .thenAnswer(invocation -> {
+ ExecStartCmd start = mock(ExecStartCmd.class);
+ when(start.exec(any())).thenAnswer(startInvocation -> {
+ @SuppressWarnings("unchecked")
+ ResultCallback<Frame> cb = startInvocation.getArgument(0);
+ cb.onComplete();
+ return cb;
+ });
+ return start;
+ });
+ // launcher.launch(fn) populates this list after this method returns.
+ return launchCmds;
+ }
+
+ /** Overrides the default exec-start stub so the callback the launcher supplies is driven with a
+ * real STDOUT {@link Frame} carrying {@code output}, exercising the frame-to-CloudWatch path
+ * rather than only completing the callback. */
+ private void stubExecStartEmittingFrame(String output) {
+ lenient().when(dockerClient.execStartCmd(argThat(id -> id != null && id.startsWith("exec-launch-"))))
+ .thenAnswer(invocation -> {
+ ExecStartCmd start = mock(ExecStartCmd.class);
+ when(start.exec(any())).thenAnswer(startInvocation -> {
+ @SuppressWarnings("unchecked")
+ ResultCallback<Frame> cb = startInvocation.getArgument(0);
+ cb.onNext(new Frame(StreamType.STDOUT, output.getBytes(StandardCharsets.UTF_8)));
+ cb.onComplete();
+ return cb;
+ });
+ return start;
+ });
+ }
+
+ /** Extracts the single command-line argument each mock in {@code launchCmds} was called with
+ * via {@code withCmd(String...)}, in call order — the {@code /opt/extensions/<name>} path
+ * each discovered extension binary was launched with. */
+ private static List<String> capturedLaunchPaths(List<ExecCreateCmd> launchCmds) {
+ List<String> paths = new java.util.ArrayList<>();
+ for (ExecCreateCmd cmd : launchCmds) {
+ ArgumentCaptor<String[]> captor = ArgumentCaptor.forClass(String[].class);
+ verify(cmd).withCmd(captor.capture());
+ paths.add(captor.getValue()[0]);
+ }
+ return paths;
+ }
+
+ @Test
+ void launchFunction_discoversAndLaunchesExtensionBinaries() throws Exception {
+ stubExtensionDiscovery("lambda-adapter");
+
+ LambdaFunction fn = new LambdaFunction();
+ fn.setFunctionName("with-extension-fn");
+ fn.setPackageType("Image");
+ fn.setImageUri("123456789012.dkr.ecr.us-east-1.amazonaws.com/repo:latest");
+
+ launcher.launch(fn);
+
+ // The extension binary was discovered via the archive API and execed by its full
+ // /opt/extensions path, after the container was started (real AWS starts extensions once
+ // the container is up, alongside the runtime).
+ verify(dockerClient).copyArchiveFromContainerCmd("container-123", "/opt/extensions");
+ verify(dockerClient, atLeastOnce()).execCreateCmd(eq("container-123"));
+
+ InOrder inOrder = inOrder(lifecycleManager, dockerClient);
+ inOrder.verify(lifecycleManager).startCreated(eq("container-123"), any());
+ inOrder.verify(dockerClient).copyArchiveFromContainerCmd("container-123", "/opt/extensions");
+ inOrder.verify(dockerClient, atLeastOnce()).execCreateCmd("container-123");
+ }
+
+ /**
+ * Extension stdout/stderr must reach the function's CloudWatch log group. `docker logs` — and
+ * therefore ContainerLogStreamer.attach(), which uses logContainerCmd — only covers the
+ * container's PID 1 output, so an exec's stream never reaches the container log. Without
+ * explicit forwarding an observability extension's output is dropped entirely.
+ *
+ * <p>Uses a real ContainerLogStreamer over a mocked CloudWatchLogsService so the assertion
+ * covers the actual frame-to-log-event path rather than just that a mock was called.
+ */
+ @Test
+ void launchFunction_forwardsExtensionOutputToCloudWatchLogs() throws Exception {
+ CloudWatchLogsService cloudWatchLogs = mock(CloudWatchLogsService.class);
+ ContainerReachableEndpoint reachableEndpoint =
+ new ContainerReachableEndpoint(config, dockerHostResolver, embeddedDnsServer);
+ ContainerLauncher launcherWithRealStreamer = new ContainerLauncher(
+ new ContainerBuilder(config, dockerHostResolver, embeddedDnsServer),
+ lifecycleManager,
+ new ContainerLogStreamer(dockerClient, cloudWatchLogs),
+ imageResolver, runtimeApiServerFactory, dockerHostResolver, config,
+ ecrRegistryManager,
+ mock(io.github.hectorvent.floci.services.lambda.LambdaLayerService.class),
+ new LaunchedContainerAwsEnv(reachableEndpoint));
+
+ stubExtensionDiscovery("otel-collector");
+ // Feed a real stdout frame through whatever callback the launcher hands to execStartCmd.
+ stubExecStartEmittingFrame("extension started on :8080\n");
+
+ LambdaFunction fn = new LambdaFunction();
+ fn.setFunctionName("observability-fn");
+ fn.setPackageType("Image");
+ fn.setImageUri("123456789012.dkr.ecr.us-east-1.amazonaws.com/repo:latest");
+
+ launcherWithRealStreamer.launch(fn);
+
+ // The frame became a CloudWatch log event in the function's own log group.
+ ArgumentCaptor<List<Map<String, Object>>> events = ArgumentCaptor.forClass(List.class);
+ verify(cloudWatchLogs, atLeastOnce()).putLogEvents(
+ eq("/aws/lambda/observability-fn"), anyString(), events.capture(), anyString());
+ assertTrue(events.getAllValues().stream()
+ .flatMap(List::stream)
+ .anyMatch(e -> "extension started on :8080".equals(e.get("message"))),
+ "extension stdout must be forwarded to the function's CloudWatch log group");
+ }
+
+ /**
+ * The log group and stream must exist before extensions are launched: they can log immediately,
+ * and putLogEvents against a missing stream is swallowed at debug level — silently losing
+ * exactly the early startup output this forwarding exists to capture.
+ */
+ @Test
+ void launchFunction_createsLogGroupBeforeLaunchingExtensions() throws Exception {
+ stubExtensionDiscovery("lambda-adapter");
+
+ LambdaFunction fn = new LambdaFunction();
+ fn.setFunctionName("ordering-fn");
+ fn.setPackageType("Image");
+ fn.setImageUri("123456789012.dkr.ecr.us-east-1.amazonaws.com/repo:latest");
+
+ launcher.launch(fn);
+
+ InOrder inOrder = inOrder(logStreamer, dockerClient);
+ inOrder.verify(logStreamer).ensureLogGroupAndStream(
+ eq("/aws/lambda/ordering-fn"), anyString(), anyString());
+ inOrder.verify(dockerClient, atLeastOnce()).execCreateCmd("container-123");
+ }
+
+ /**
+ * The init-readiness barrier abanna asked for on PR #1773: extension processes are started as
+ * detached execs, so without waiting the caller could enqueue the first invocation before an
+ * extension was ready for it and the adapter would silently miss that invoke. The launch must
+ * arm the barrier with the discovered binary count *before* starting any exec (so a
+ * fast-starting extension can't become ready before there is a latch to count it down), and
+ * must not return until the extensions are init-ready.
+ */
+ @Test
+ void launchFunction_waitsForExtensionsToBecomeReadyBeforeReturning() throws Exception {
+ stubExtensionDiscovery("lambda-adapter", "otel-collector");
+
+ LambdaFunction fn = new LambdaFunction();
+ fn.setFunctionName("barrier-fn");
+ fn.setPackageType("Image");
+ fn.setImageUri("123456789012.dkr.ecr.us-east-1.amazonaws.com/repo:latest");
+
+ // Readiness is delayed: awaitExtensionsReady only reports success after a pause, standing
+ // in for extension processes that take a moment to start up and poll for their first event.
+ AtomicBoolean readinessComplete = new AtomicBoolean(false);
+ when(runtimeApiServer.awaitExtensionsReady(anyLong())).thenAnswer(inv -> {
+ Thread.sleep(150);
+ readinessComplete.set(true);
+ return true;
+ });
+
+ launcher.launch(fn);
+
+ assertTrue(readinessComplete.get(),
+ "launch() must not return before the extensions are init-ready");
+
+ // The barrier is armed with the number of binaries actually discovered, and armed before
+ // any extension process is started.
+ verify(runtimeApiServer).expectExtensions(2);
+ InOrder inOrder = inOrder(runtimeApiServer, dockerClient);
+ inOrder.verify(runtimeApiServer).expectExtensions(2);
+ inOrder.verify(dockerClient, atLeastOnce()).execCreateCmd("container-123");
+ inOrder.verify(runtimeApiServer).awaitExtensionsReady(anyLong());
+ }
+
+ /**
+ * A slow or crashed extension must degrade to "invocations run without it", not fail the whole
+ * function launch — the same outcome as before the barrier existed.
+ */
+ @Test
+ void launchFunction_extensionReadinessTimeout_doesNotFailLaunch() throws Exception {
+ stubExtensionDiscovery("never-registers");
+
+ LambdaFunction fn = new LambdaFunction();
+ fn.setFunctionName("timeout-fn");
+ fn.setPackageType("Image");
+ fn.setImageUri("123456789012.dkr.ecr.us-east-1.amazonaws.com/repo:latest");
+
+ when(runtimeApiServer.awaitExtensionsReady(anyLong())).thenReturn(false);
+
+ ContainerHandle handle = launcher.launch(fn);
+
+ assertNotNull(handle, "a readiness timeout must not fail the launch");
+ assertEquals("container-123", handle.getContainerId());
+ }
+
+ /**
+ * The common case — no /opt/extensions directory — must not wait at all: the barrier is armed
+ * with zero and the launch proceeds immediately.
+ */
+ @Test
+ void launchFunction_noExtensions_armsBarrierWithZeroAndDoesNotBlock() throws Exception {
+ CopyArchiveFromContainerCmd copyCmd = mock(CopyArchiveFromContainerCmd.class);
+ when(copyCmd.exec()).thenThrow(new NotFoundException("no such directory"));
+ when(dockerClient.copyArchiveFromContainerCmd("container-123", "/opt/extensions"))
+ .thenReturn(copyCmd);
+
+ LambdaFunction fn = new LambdaFunction();
+ fn.setFunctionName("no-extension-fn");
+ fn.setPackageType("Image");
+ fn.setImageUri("123456789012.dkr.ecr.us-east-1.amazonaws.com/repo:latest");
+
+ launcher.launch(fn);
+
+ verify(runtimeApiServer).expectExtensions(0);
+ }
+
+ @Test
+ void launchFunction_noExtensionsDirectory_doesNotFailLaunch() throws Exception {
+ CopyArchiveFromContainerCmd copyCmd = mock(CopyArchiveFromContainerCmd.class);
+ when(copyCmd.exec()).thenThrow(new NotFoundException("no such directory"));
+ when(dockerClient.copyArchiveFromContainerCmd("container-123", "/opt/extensions"))
+ .thenReturn(copyCmd);
+
+ LambdaFunction fn = new LambdaFunction();
+ fn.setFunctionName("no-extension-fn");
+ fn.setPackageType("Image");
+ fn.setImageUri("123456789012.dkr.ecr.us-east-1.amazonaws.com/repo:latest");
+
+ launcher.launch(fn);
+
+ // The discovery probe still runs (best-effort), but nothing beyond it — no extension
+ // binary path is ever execed since the directory doesn't exist.
+ verify(dockerClient, never()).execCreateCmd("container-123");
+ }
+
+ @Test
+ void launchFunction_extensionDiscovery_filtersNonExecutableAndNestedEntries() throws Exception {
+ var entries = new java.util.LinkedHashMap<String, Boolean>();
+ entries.put("lambda-adapter", true); // direct child, executable: launched
+ entries.put("README.md", false); // direct child, not executable: skipped
+ entries.put("nested/inner-binary", true); // nested (not a direct child): skipped
+ List<ExecCreateCmd> launchCmds = stubExtensionDiscoveryFromTar(tarOfRaw(entries));
+
+ LambdaFunction fn = new LambdaFunction();
+ fn.setFunctionName("mixed-extensions-fn");
+ fn.setPackageType("Image");
+ fn.setImageUri("123456789012.dkr.ecr.us-east-1.amazonaws.com/repo:latest");
+
+ launcher.launch(fn);
+
+ assertEquals(List.of("/opt/extensions/lambda-adapter"), capturedLaunchPaths(launchCmds),
+ "only the direct, executable entry should be launched");
+ }
+
+ @Test
+ void launchFunction_multipleExtensionBinaries_allDiscoveredAndLaunched() throws Exception {
+ List<ExecCreateCmd> launchCmds = stubExtensionDiscoveryFromTar(tarOf("lambda-adapter", "otel-collector"));
+
+ LambdaFunction fn = new LambdaFunction();
+ fn.setFunctionName("multi-extension-fn");
+ fn.setPackageType("Image");
+ fn.setImageUri("123456789012.dkr.ecr.us-east-1.amazonaws.com/repo:latest");
+
+ launcher.launch(fn);
+
+ assertEquals(List.of("/opt/extensions/lambda-adapter", "/opt/extensions/otel-collector"),
+ capturedLaunchPaths(launchCmds));
+ }
}
diff --git a/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java b/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java
index fed89321ac..eaaf27d637 100644
--- a/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java
+++ b/src/test/java/io/github/hectorvent/floci/services/lambda/runtime/RuntimeApiServerTest.java
@@ -17,6 +17,8 @@ import java.net.URI;
import java.net.http.HttpClient;
import java.net.http.HttpRequest;
import java.net.http.HttpResponse;
+import java.util.ArrayList;
+import java.util.List;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
@@ -28,6 +30,7 @@ import org.junit.jupiter.api.condition.OS;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -55,7 +58,13 @@ class RuntimeApiServerTest {
void tearDown() throws Exception {
server.stop().get(5, TimeUnit.SECONDS);
scheduler.shutdownNow();
- vertx.close();
+ httpClient.close();
+ // Await the close rather than firing it and moving on: Vertx.close() is asynchronous, so
+ // an unawaited call lets the next test's setUp() create a new Vertx and bind a port while
+ // this one's event loops and server sockets are still tearing down. Across a class with
+ // stress tests that stand up hundreds of servers, that backlog surfaces as BindException
+ // or a start() timeout in an unrelated test's setUp().
+ vertx.close().toCompletionStage().toCompletableFuture().get(10, TimeUnit.SECONDS);
}
@Test
@@ -343,9 +352,1024 @@ class RuntimeApiServerTest {
assertEquals(404, resp.statusCode());
}
+ @Test
+ @Timeout(10)
+ void extensionRegister_returnsIdentifierHeaderAndFunctionBody() throws Exception {
+ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/register"))
+ .header("Lambda-Extension-Name", "lambda-adapter")
+ .POST(HttpRequest.BodyPublishers.ofString("{\"events\":[\"INVOKE\",\"SHUTDOWN\"]}"))
+ .build(),
+ HttpResponse.BodyHandlers.ofString());
+
+ assertEquals(200, response.statusCode());
+ assertTrue(response.headers().firstValue("Lambda-Extension-Identifier").isPresent(),
+ "register must return a Lambda-Extension-Identifier header");
+ JsonObject body = new JsonObject(response.body());
+ assertNotNull(body.getString("functionName"));
+ assertNotNull(body.getString("functionVersion"));
+ }
+
+ /**
+ * Regression: the register response previously hardcoded functionName/functionVersion/handler
+ * to placeholder values regardless of which function the server was actually serving — extensions
+ * that key telemetry off this response (e.g. per-function metrics tagging) would mislabel every
+ * function identically. ContainerLauncher calls setFunctionMetadata once it knows which
+ * LambdaFunction a given RuntimeApiServer instance belongs to.
+ */
+ @Test
+ @Timeout(10)
+ void extensionRegister_returnsRealFunctionMetadataOnceSet() throws Exception {
+ server.setFunctionMetadata("my-real-function", "3", "index.handler", "123456789012");
+
+ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/register"))
+ .header("Lambda-Extension-Name", "lambda-adapter")
+ .POST(HttpRequest.BodyPublishers.ofString("{}"))
+ .build(),
+ HttpResponse.BodyHandlers.ofString());
+
+ assertEquals(200, response.statusCode());
+ JsonObject body = new JsonObject(response.body());
+ assertEquals("my-real-function", body.getString("functionName"));
+ assertEquals("3", body.getString("functionVersion"));
+ assertEquals("index.handler", body.getString("handler"));
+ }
+
+ /**
+ * AWS includes {@code accountId} in the register response when the extension opts in via
+ * {@code Lambda-Extension-Accept-Feature: accountId}. Without it a telemetry extension
+ * registers successfully but silently loses account attribution.
+ */
+ @Test
+ @Timeout(10)
+ void extensionRegister_withAccountIdFeature_includesAccountId() throws Exception {
+ server.setFunctionMetadata("my-fn", "$LATEST", "index.handler", "210987654321");
+
+ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/register"))
+ .header("Lambda-Extension-Name", "telemetry-extension")
+ .header("Lambda-Extension-Accept-Feature", "accountId")
+ .POST(HttpRequest.BodyPublishers.ofString("{}"))
+ .build(),
+ HttpResponse.BodyHandlers.ofString());
+
+ assertEquals(200, response.statusCode());
+ assertEquals("210987654321", new JsonObject(response.body()).getString("accountId"));
+ }
+
+ /**
+ * The feature is opt-in: an extension that does not request it must get the unchanged response
+ * shape, since real AWS omits the field entirely rather than always sending it.
+ */
+ @Test
+ @Timeout(10)
+ void extensionRegister_withoutAccountIdFeature_omitsAccountId() throws Exception {
+ server.setFunctionMetadata("my-fn", "$LATEST", "index.handler", "210987654321");
+
+ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/register"))
+ .header("Lambda-Extension-Name", "plain-extension")
+ .POST(HttpRequest.BodyPublishers.ofString("{}"))
+ .build(),
+ HttpResponse.BodyHandlers.ofString());
+
+ assertEquals(200, response.statusCode());
+ assertNull(new JsonObject(response.body()).getString("accountId"),
+ "accountId must only appear when the extension opts in");
+ }
+
+ /**
+ * The header carries a comma-separated feature list, so accountId must still be honoured
+ * alongside other requested features and regardless of casing/spacing.
+ */
+ @Test
+ @Timeout(10)
+ void extensionRegister_accountIdFeatureAmongOthers_isHonoured() throws Exception {
+ server.setFunctionMetadata("my-fn", "$LATEST", "index.handler", "210987654321");
+
+ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/register"))
+ .header("Lambda-Extension-Name", "telemetry-extension")
+ .header("Lambda-Extension-Accept-Feature", "someOtherFeature, AccountID")
+ .POST(HttpRequest.BodyPublishers.ofString("{}"))
+ .build(),
+ HttpResponse.BodyHandlers.ofString());
+
+ assertEquals(200, response.statusCode());
+ assertEquals("210987654321", new JsonObject(response.body()).getString("accountId"));
+ }
+
+ /** A function with no account falls back to the same placeholder used elsewhere in Floci. */
+ @Test
+ @Timeout(10)
+ void extensionRegister_withNoFunctionAccountId_fallsBackToPlaceholder() throws Exception {
+ server.setFunctionMetadata("my-fn", "$LATEST", "index.handler", null);
+
+ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/register"))
+ .header("Lambda-Extension-Name", "telemetry-extension")
+ .header("Lambda-Extension-Accept-Feature", "accountId")
+ .POST(HttpRequest.BodyPublishers.ofString("{}"))
+ .build(),
+ HttpResponse.BodyHandlers.ofString());
+
+ assertEquals(200, response.statusCode());
+ assertEquals("000000000000", new JsonObject(response.body()).getString("accountId"));
+ }
+
+ @Test
+ @Timeout(10)
+ void extensionRegister_missingNameHeader_returns400() throws Exception {
+ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/register"))
+ .POST(HttpRequest.BodyPublishers.ofString("{}"))
+ .build(),
+ HttpResponse.BodyHandlers.ofString());
+
+ assertEquals(400, response.statusCode());
+ }
+
+ @Test
+ @Timeout(10)
+ void extensionEventNext_unknownIdentifier_returns403() throws Exception {
+ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/event/next"))
+ .header("Lambda-Extension-Identifier", "not-a-real-id")
+ .GET().build(),
+ HttpResponse.BodyHandlers.ofString());
+
+ assertEquals(403, response.statusCode());
+ }
+
+ @Test
+ @Timeout(15)
+ void extensionEventNext_receivesInvokeEventWhenRuntimeInvocationEnqueued() throws Exception {
+ String extensionId = registerExtension("lambda-adapter", "INVOKE", "SHUTDOWN");
+
+ CompletableFuture<HttpResponse<String>> asyncNext = httpClient.sendAsync(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/event/next"))
+ .header("Lambda-Extension-Identifier", extensionId)
+ .GET().build(),
+ HttpResponse.BodyHandlers.ofString());
+
+ Thread.sleep(300);
+ assertFalse(asyncNext.isDone(), "extension /event/next should be parked with no pending event");
+
+ PendingInvocation invocation = new PendingInvocation(
+ "req-ext-invoke", "{}".getBytes(), System.currentTimeMillis() + 60_000,
+ "arn:aws:lambda:us-east-1:000000000000:function:test",
+ new CompletableFuture<>());
+ server.enqueue(invocation);
+
+ HttpResponse<String> response = asyncNext.get(2, TimeUnit.SECONDS);
+ assertEquals(200, response.statusCode());
+ assertTrue(response.headers().firstValue("Lambda-Extension-Event-Identifier").isPresent());
+ JsonObject body = new JsonObject(response.body());
+ assertEquals("INVOKE", body.getString("eventType"));
+ assertEquals("req-ext-invoke", body.getString("requestId"));
+ }
+
+ @Test
+ @Timeout(15)
+ void extensionEventNext_notSubscribedToInvoke_isNotNotified() throws Exception {
+ // Registers for SHUTDOWN only — real AWS never delivers INVOKE to an extension that
+ // didn't ask for it.
+ String extensionId = registerExtension("shutdown-only-extension", "SHUTDOWN");
+
+ CompletableFuture<HttpResponse<String>> asyncNext = httpClient.sendAsync(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/event/next"))
+ .header("Lambda-Extension-Identifier", extensionId)
+ .GET().build(),
+ HttpResponse.BodyHandlers.ofString());
+
+ server.enqueue(new PendingInvocation(
+ "req-not-subscribed", "{}".getBytes(), System.currentTimeMillis() + 60_000,
+ "arn:aws:lambda:us-east-1:000000000000:function:test",
+ new CompletableFuture<>()));
+
+ Thread.sleep(500);
+ assertFalse(asyncNext.isDone(),
+ "extension not subscribed to INVOKE must not be woken by an invocation");
+ }
+
+ @Test
+ @Timeout(15)
+ void extensionEventNext_receivesShutdownEventWhenServerStops() throws Exception {
+ String extensionId = registerExtension("lambda-adapter", "INVOKE", "SHUTDOWN");
+
+ CompletableFuture<HttpResponse<String>> asyncNext = httpClient.sendAsync(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/event/next"))
+ .header("Lambda-Extension-Identifier", extensionId)
+ .GET().build(),
+ HttpResponse.BodyHandlers.ofString());
+
+ Thread.sleep(300);
+ assertFalse(asyncNext.isDone());
+
+ server.stop();
+
+ HttpResponse<String> response = asyncNext.get(2, TimeUnit.SECONDS);
+ assertEquals(200, response.statusCode());
+ JsonObject body = new JsonObject(response.body());
+ assertEquals("SHUTDOWN", body.getString("eventType"));
+ }
+
+ @Test
+ @Timeout(10)
+ void extensionInitError_returns202AndUnregistersExtension() throws Exception {
+ String extensionId = registerExtension("failing-extension", "INVOKE");
+
+ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/init/error"))
+ .header("Lambda-Extension-Identifier", extensionId)
+ .header("Lambda-Extension-Function-Error-Type", "Extension.ConfigInvalid")
+ .POST(HttpRequest.BodyPublishers.ofString(
+ "{\"errorMessage\":\"bad config\",\"errorType\":\"Extension.ConfigInvalid\"}"))
+ .build(),
+ HttpResponse.BodyHandlers.ofString());
+
+ assertEquals(202, response.statusCode());
+ assertEquals("OK", new JsonObject(response.body()).getString("status"));
+
+ // The unregistered extension is no longer a valid target for /event/next.
+ HttpResponse<String> nextResponse = httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/event/next"))
+ .header("Lambda-Extension-Identifier", extensionId)
+ .GET().build(),
+ HttpResponse.BodyHandlers.ofString());
+ assertEquals(403, nextResponse.statusCode());
+
+ // Both init/error and exit/error are fatal to the execution environment in real AWS, so
+ // the server is marked faulted for WarmPool to retire the container rather than reuse it.
+ assertTrue(server.isFaulted(), "init/error must condemn the execution environment");
+ }
+
+ @Test
+ @Timeout(30)
+ void extensionExitError_condemnsExecutionEnvironment() throws Exception {
+ String extensionId = registerExtension("failing-extension", "INVOKE");
+ assertFalse(server.isFaulted(), "environment must not be faulted before any error is reported");
+
+ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/exit/error"))
+ .header("Lambda-Extension-Identifier", extensionId)
+ .header("Lambda-Extension-Function-Error-Type", "Extension.TestError")
+ .POST(HttpRequest.BodyPublishers.ofString(
+ "{\"errorMessage\":\"crashed\",\"errorType\":\"Extension.Crash\"}"))
+ .build(),
+ HttpResponse.BodyHandlers.ofString());
+
+ assertEquals(202, response.statusCode());
+ assertTrue(server.isFaulted(), "exit/error must condemn the execution environment");
+ }
+
+ /**
+ * The init-readiness barrier: awaitExtensionsReady() must block until every expected extension
+ * is init-ready, which AWS defines as its first /extension/event/next.
+ */
+ @Test
+ @Timeout(30)
+ void awaitExtensionsReady_blocksUntilAllExpectedExtensionsPollForEvents() throws Exception {
+ server.expectExtensions(2);
+
+ CompletableFuture<Boolean> awaited = CompletableFuture.supplyAsync(() -> {
+ try {
+ return server.awaitExtensionsReady(10_000);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ return false;
+ }
+ });
+
+ String first = registerExtension("first-extension", "INVOKE");
+ String second = registerExtension("second-extension", "INVOKE");
+ Thread.sleep(200);
+ assertFalse(awaited.isDone(), "registering alone must not open the barrier");
+
+ pollExtensionEventNext(first);
+ Thread.sleep(200);
+ assertFalse(awaited.isDone(), "must still be waiting while one extension has not polled");
+
+ pollExtensionEventNext(second);
+ assertTrue(awaited.get(5, TimeUnit.SECONDS), "must unblock once all extensions are ready");
+ }
+
+ /**
+ * The distinction abanna raised on PR #1773: an extension that registers immediately but then
+ * spends time initialising before its first Next is *not* ready, and the barrier must hold the
+ * container out of service for that whole gap. Gating on register would release it early and
+ * let the first invoke race the extension's own startup — the bug the barrier exists to stop.
+ */
+ @Test
+ @Timeout(30)
+ void awaitExtensionsReady_waitsForDelayedFirstNextAfterFastRegistration() throws Exception {
+ server.expectExtensions(1);
+ String extensionId = registerExtension("slow-starting-extension", "INVOKE");
+
+ CompletableFuture<Boolean> awaited = CompletableFuture.supplyAsync(() -> {
+ try {
+ return server.awaitExtensionsReady(10_000);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ return false;
+ }
+ });
+
+ Thread.sleep(700);
+ assertFalse(awaited.isDone(),
+ "a registered-but-not-yet-polling extension must keep the barrier closed");
+
+ pollExtensionEventNext(extensionId);
+ assertTrue(awaited.get(5, TimeUnit.SECONDS),
+ "the delayed first /event/next must open the barrier");
+ }
+
+ /**
+ * The missed-signal case: readiness completing *before* anyone waits must not strand the
+ * launch. A CountDownLatch is level-triggered, so a late await returns immediately — this
+ * pins that behaviour, since an edge-triggered signal here would hang the cold start.
+ */
+ @Test
+ @Timeout(30)
+ void awaitExtensionsReady_returnsImmediatelyWhenExtensionAlreadyReady() throws Exception {
+ server.expectExtensions(1);
+ pollExtensionEventNext(registerExtension("fast-extension", "INVOKE"));
+
+ long start = System.currentTimeMillis();
+ assertTrue(server.awaitExtensionsReady(10_000));
+ assertTrue(System.currentTimeMillis() - start < 1000,
+ "an await arriving after readiness must not wait");
+ }
+
+ /**
+ * Repeated polling must not count the barrier down more than once per extension, which would
+ * let one chatty extension open a barrier that others have not reached yet.
+ */
+ @Test
+ @Timeout(30)
+ void awaitExtensionsReady_countsEachExtensionOnceAcrossRepeatedPolls() throws Exception {
+ server.expectExtensions(2);
+ String chatty = registerExtension("chatty-extension", "INVOKE");
+ registerExtension("quiet-extension", "INVOKE");
+
+ for (int i = 0; i < 3; i++) {
+ pollExtensionEventNext(chatty);
+ }
+
+ assertFalse(server.awaitExtensionsReady(300),
+ "one extension polling repeatedly must not satisfy the other's slot");
+ }
+
+ /** A function with no extensions must not wait at all. */
+ @Test
+ @Timeout(30)
+ void awaitExtensionsReady_withNoExpectedExtensions_doesNotBlock() throws Exception {
+ server.expectExtensions(0);
+
+ long start = System.currentTimeMillis();
+ assertTrue(server.awaitExtensionsReady(10_000));
+ assertTrue(System.currentTimeMillis() - start < 1000, "zero extensions must not wait");
+ }
+
+ /** A never-ready extension must time out rather than block forever. */
+ @Test
+ @Timeout(30)
+ void awaitExtensionsReady_timesOutWhenExtensionNeverPolls() throws Exception {
+ server.expectExtensions(1);
+
+ assertFalse(server.awaitExtensionsReady(300),
+ "a missing first /event/next must report timeout, not hang");
+ }
+
+ /**
+ * The behaviour abanna reported on PR #1773: "a later invoke can still be served after
+ * init/error". A *new* invocation enqueued after the environment was condemned must not be
+ * dispatched to the runtime — serving it would hide a failed adapter or security extension.
+ * This is distinct from the in-flight invocation covered below, which must still complete.
+ */
+ @Test
+ @Timeout(30)
+ void invokeEnqueuedAfterExtensionFatalError_isNotServed() throws Exception {
+ String extensionId = registerExtension("failing-extension", "INVOKE");
+
+ httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/init/error"))
+ .header("Lambda-Extension-Identifier", extensionId)
+ .header("Lambda-Extension-Function-Error-Type", "Extension.TestError")
+ .POST(HttpRequest.BodyPublishers.ofString("{\"errorMessage\":\"bad config\"}"))
+ .build(),
+ HttpResponse.BodyHandlers.ofString());
+ assertTrue(server.isFaulted());
+
+ // A brand-new invocation arrives at the condemned environment.
+ PendingInvocation later = new PendingInvocation(
+ "req-after-fault", "{}".getBytes(), System.currentTimeMillis() + 60_000,
+ "arn:aws:lambda:us-east-1:000000000000:function:test",
+ new CompletableFuture<>());
+ CompletableFuture<InvokeResult> laterFuture = server.enqueue(later);
+
+ // It must be rejected outright rather than handed to the runtime's /next poller.
+ InvokeResult result = laterFuture.get(5, TimeUnit.SECONDS);
+ assertEquals("Unhandled", result.getFunctionError(),
+ "an invoke enqueued after a fatal extension error must not be served");
+
+ HttpResponse<String> next = httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))
+ .timeout(java.time.Duration.ofSeconds(5))
+ .GET().build(),
+ HttpResponse.BodyHandlers.ofString());
+ assertEquals(204, next.statusCode(),
+ "the runtime must not receive work from a condemned environment");
+ }
+
+ /**
+ * A sibling extension already parked on /event/next when another reports a fatal error must be
+ * released, not left hanging. The faulted guard in that handler means nothing will ever wake
+ * it, so without an explicit drain the long-poll stays open until stop() eventually runs — up
+ * to a full function timeout later. Harmless with one extension (the aws-lambda-web-adapter
+ * case), but a second one hangs.
+ *
+ * <p>The response is the same 500 that a *new* poll receives after a fault; AWS specifies the
+ * terminal state but not what an already-open long-poll gets, so this matches shipped
+ * behaviour rather than inventing a second convention.
+ */
+ @Test
+ @Timeout(30)
+ void extensionFatalError_releasesSiblingExtensionParkedOnEventNext() throws Exception {
+ String survivorId = registerExtension("surviving-extension", "INVOKE", "SHUTDOWN");
+ String failingId = registerExtension("failing-extension", "INVOKE");
+
+ // The survivor is parked on /event/next before the fault fires.
+ CompletableFuture<HttpResponse<String>> parked = pollExtensionEventNext(survivorId);
+ assertFalse(parked.isDone(), "sibling extension should be parked with no pending event");
+
+ reportExtensionInitError(failingId);
+
+ HttpResponse<String> response = parked.get(5, TimeUnit.SECONDS);
+ assertEquals(500, response.statusCode(),
+ "a parked sibling must be released when the environment is condemned");
+ assertTrue(response.body().contains("Extension.SandboxFaulted"));
+ }
+
+ /**
+ * AWS requires {@code Lambda-Extension-Function-Error-Type} on both error endpoints. Retiring
+ * the environment is destructive and unrecoverable, so a report missing the required header
+ * must be rejected *before* anything is condemned — otherwise a caller that got the contract
+ * wrong silently kills a healthy container and the 202 tells them it worked.
+ */
+ @Test
+ @Timeout(30)
+ void extensionInitError_withoutErrorTypeHeader_isRejectedAndDoesNotFault() throws Exception {
+ String extensionId = registerExtension("failing-extension", "INVOKE");
+
+ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/init/error"))
+ .header("Lambda-Extension-Identifier", extensionId)
+ .POST(HttpRequest.BodyPublishers.ofString("{\"errorMessage\":\"bad\"}"))
+ .build(),
+ HttpResponse.BodyHandlers.ofString());
+
+ assertEquals(400, response.statusCode(),
+ "a report missing the required error-type header must be rejected");
+ assertFalse(server.isFaulted(),
+ "a malformed report must not condemn the execution environment");
+ }
+
+ /** The same requirement on exit/error — both endpoints carry it in AWS. */
+ @Test
+ @Timeout(30)
+ void extensionExitError_withoutErrorTypeHeader_isRejectedAndDoesNotFault() throws Exception {
+ String extensionId = registerExtension("failing-extension", "INVOKE");
+
+ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/exit/error"))
+ .header("Lambda-Extension-Identifier", extensionId)
+ .POST(HttpRequest.BodyPublishers.ofString("{\"errorMessage\":\"bad\"}"))
+ .build(),
+ HttpResponse.BodyHandlers.ofString());
+
+ assertEquals(400, response.statusCode());
+ assertFalse(server.isFaulted(),
+ "a malformed report must not condemn the execution environment");
+ }
+
+ /**
+ * An unknown identifier must be rejected rather than treated as a fatal report. This reverses
+ * an earlier deliberate choice here (condemn regardless, on the theory that *some* extension
+ * had failed): AWS validates the identifier first, and honouring an unrecognised one lets any
+ * unauthenticated caller retire a healthy container.
+ */
+ @Test
+ @Timeout(30)
+ void extensionInitError_withUnknownIdentifier_isRejectedAndDoesNotFault() throws Exception {
+ registerExtension("healthy-extension", "INVOKE");
+
+ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/init/error"))
+ .header("Lambda-Extension-Identifier", "not-a-real-id")
+ .header("Lambda-Extension-Function-Error-Type", "Extension.TestError")
+ .POST(HttpRequest.BodyPublishers.ofString("{\"errorMessage\":\"bad\"}"))
+ .build(),
+ HttpResponse.BodyHandlers.ofString());
+
+ assertEquals(403, response.statusCode(),
+ "an unknown identifier must be rejected, not honoured as a fatal report");
+ assertFalse(server.isFaulted(),
+ "an unknown extension must not condemn the execution environment");
+ }
+
+ /** A missing identifier is rejected the same way, and likewise must not condemn. */
+ @Test
+ @Timeout(30)
+ void extensionInitError_withMissingIdentifier_isRejectedAndDoesNotFault() throws Exception {
+ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/init/error"))
+ .header("Lambda-Extension-Function-Error-Type", "Extension.TestError")
+ .POST(HttpRequest.BodyPublishers.ofString("{\"errorMessage\":\"bad\"}"))
+ .build(),
+ HttpResponse.BodyHandlers.ofString());
+
+ assertEquals(403, response.statusCode());
+ assertFalse(server.isFaulted());
+ }
+
+ /**
+ * The positive path still works with both required headers present: the environment is
+ * condemned exactly as before. Guards against the validation above over-rejecting.
+ */
+ @Test
+ @Timeout(30)
+ void extensionInitError_withRequiredHeaders_faultsAsBefore() throws Exception {
+ String extensionId = registerExtension("failing-extension", "INVOKE");
+
+ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/init/error"))
+ .header("Lambda-Extension-Identifier", extensionId)
+ .header("Lambda-Extension-Function-Error-Type", "Extension.BadConfig")
+ .POST(HttpRequest.BodyPublishers.ofString("{\"errorMessage\":\"bad\"}"))
+ .build(),
+ HttpResponse.BodyHandlers.ofString());
+
+ assertEquals(202, response.statusCode());
+ assertTrue(server.isFaulted(),
+ "a well-formed report must still condemn the execution environment");
+ }
+
+ /**
+ * AWS makes an init/exit error terminal for the Extensions API too, not only for runtime work:
+ * once the environment is condemned no further extension call succeeds. A registration that
+ * still returned 200 would hand a second extension an identifier that can never receive an
+ * event, leaving it polling a container that is on its way to being retired.
+ */
+ @Test
+ @Timeout(30)
+ void extensionRegisterAfterFatalError_isRejected() throws Exception {
+ String extensionId = registerExtension("failing-extension", "INVOKE");
+ reportExtensionInitError(extensionId);
+
+ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/register"))
+ .header("Lambda-Extension-Name", "late-extension")
+ .POST(HttpRequest.BodyPublishers.ofString("{\"events\":[\"INVOKE\"]}"))
+ .build(),
+ HttpResponse.BodyHandlers.ofString());
+
+ assertEquals(500, response.statusCode(),
+ "registration into a condemned environment must fail, not hand back an identifier");
+ assertTrue(response.body().contains("Extension.SandboxFaulted"));
+ }
+
+ /**
+ * The same rule for /event/next, and the reason it is a 500 rather than the 204 used for an
+ * orderly shutdown: an extension reads 204 as "nothing right now" and polls again, so a 204
+ * here would spin it against a dead environment instead of telling it to stop.
+ */
+ @Test
+ @Timeout(30)
+ void extensionEventNextAfterFatalError_isRejected() throws Exception {
+ String survivorId = registerExtension("surviving-extension", "INVOKE");
+ String failingId = registerExtension("failing-extension", "INVOKE");
+ reportExtensionInitError(failingId);
+
+ // The still-registered extension polls after a *different* extension condemned the
+ // environment — the fault is environment-wide, not scoped to the reporter.
+ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/event/next"))
+ .header("Lambda-Extension-Identifier", survivorId)
+ .timeout(java.time.Duration.ofSeconds(5))
+ .GET().build(),
+ HttpResponse.BodyHandlers.ofString());
+
+ assertEquals(500, response.statusCode(),
+ "a condemned environment must not park or serve further extension polls");
+ assertTrue(response.body().contains("Extension.SandboxFaulted"));
+ }
+
+ /**
+ * The in-flight invocation must still complete normally: real AWS condemns the environment for
+ * *future* work, and tearing the runtime down mid-invoke would lose a result that did compute.
+ */
+ @Test
+ @Timeout(30)
+ void extensionFatalError_doesNotBreakInFlightInvocation() throws Exception {
+ String extensionId = registerExtension("failing-extension", "INVOKE");
+
+ PendingInvocation invocation = new PendingInvocation(
+ "req-fatal", "{\"hello\":\"world\"}".getBytes(), System.currentTimeMillis() + 60_000,
+ "arn:aws:lambda:us-east-1:000000000000:function:test",
+ new CompletableFuture<>());
+ CompletableFuture<InvokeResult> resultFuture = server.enqueue(invocation);
+
+ // Runtime picks the invocation up, then the extension dies mid-invoke.
+ HttpResponse<String> next = httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2018-06-01/runtime/invocation/next"))
+ .GET().build(),
+ HttpResponse.BodyHandlers.ofString());
+ assertEquals(200, next.statusCode());
+
+ httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/exit/error"))
+ .header("Lambda-Extension-Identifier", extensionId)
+ .header("Lambda-Extension-Function-Error-Type", "Extension.TestError")
+ .POST(HttpRequest.BodyPublishers.ofString("{\"errorMessage\":\"crashed\"}"))
+ .build(),
+ HttpResponse.BodyHandlers.ofString());
+ assertTrue(server.isFaulted());
+
+ // The runtime's own response still lands and completes the invocation successfully.
+ httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port
+ + "/2018-06-01/runtime/invocation/req-fatal/response"))
+ .POST(HttpRequest.BodyPublishers.ofString("{\"ok\":true}"))
+ .build(),
+ HttpResponse.BodyHandlers.ofString());
+
+ InvokeResult result = resultFuture.get(10, TimeUnit.SECONDS);
+ assertEquals(200, result.getStatusCode());
+ assertNull(result.getFunctionError(), "in-flight invocation must not be failed by the extension fault");
+ assertEquals("{\"ok\":true}", new String(result.getPayload()));
+ }
+
+ /**
+ * Regression for the historical orphaned-SHUTDOWN race (floci-io/floci#1882): stop()
+ * could flip stopped=true and offer a SHUTDOWN event in the exact window a concurrent
+ * /event/next request had already polled-empty and was about to check stopped, so the
+ * request got a bare 204 with the SHUTDOWN never delivered. Stress it with many
+ * iterations of a jittered race between stop() and /event/next, asserting SHUTDOWN is
+ * always delivered exactly once — never orphaned (zero deliveries) and never duplicated.
+ */
+ @Test
+ @Timeout(60)
+ void extensionShutdown_racedAgainstStop_isNeverOrphaned() throws Exception {
+ int iterations = 500;
+ java.util.concurrent.atomic.AtomicInteger totalDelivered = new java.util.concurrent.atomic.AtomicInteger();
+
+ for (int i = 0; i < iterations; i++) {
+ // A fresh client per iteration — the port may be reused from an earlier iteration, and
+ // a shared client's connection pool could otherwise hand back a stale pooled
+ // connection to that iteration's now-closed server.
+ HttpClient client = HttpClient.newBuilder().connectTimeout(java.time.Duration.ofSeconds(5)).build();
+ int freshPort = findFreePort();
+ RuntimeApiServer freshServer = new RuntimeApiServer(vertx, freshPort);
+ freshServer.start().get(5, TimeUnit.SECONDS);
+ try {
+ String extensionId = registerExtensionOn(client, freshPort, "lambda-adapter", "SHUTDOWN");
+
+ CompletableFuture<HttpResponse<String>> asyncNext = client.sendAsync(
+ HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + freshPort + "/2020-01-01/extension/event/next"))
+ .header("Lambda-Extension-Identifier", extensionId)
+ .GET().build(),
+ HttpResponse.BodyHandlers.ofString());
+
+ // Give the request just enough time to actually connect and park server-side
+ // (a few ms is plenty on localhost) before racing stop() against it — without
+ // this, stop() can close the listening socket before the connection is even
+ // established, which isn't the race we're testing.
+ Thread.sleep(5);
+ freshServer.stop();
+
+ HttpResponse<String> response = asyncNext.get(5, TimeUnit.SECONDS);
+ boolean deliveredHere = response.statusCode() == 200
+ && "SHUTDOWN".equals(new JsonObject(response.body()).getString("eventType"));
+ if (deliveredHere) {
+ totalDelivered.incrementAndGet();
+ } else {
+ // Didn't land in this specific request — a subsequent poll with the same
+ // identifier must still find the SHUTDOWN queued (never orphaned).
+ HttpResponse<String> retry = client.send(HttpRequest.newBuilder()
+ .uri(URI.create(
+ "http://localhost:" + freshPort + "/2020-01-01/extension/event/next"))
+ .header("Lambda-Extension-Identifier", extensionId)
+ .GET().build(),
+ HttpResponse.BodyHandlers.ofString());
+ if (retry.statusCode() == 200
+ && "SHUTDOWN".equals(new JsonObject(retry.body()).getString("eventType"))) {
+ totalDelivered.incrementAndGet();
+ }
+ }
+ } finally {
+ freshServer.stop().get(5, TimeUnit.SECONDS);
+ // Each HttpClient owns a selector thread and connection pool; leaking one per
+ // iteration across hundreds of iterations starves the JVM and makes even an
+ // unrelated server's bind time out.
+ client.close();
+ }
+ }
+
+ assertEquals(iterations, totalDelivered.get(),
+ "every iteration's SHUTDOWN must be delivered exactly once (never orphaned)");
+ }
+
+ /**
+ * Equivalent race for the runtime invocation queue: races enqueue() against stop() with
+ * jitter across many iterations, asserting the invocation's resultFuture always completes
+ * (real dispatch or ContainerStopped) — a hang here trips the test timeout, an unambiguous
+ * regression signal for an orphaned invocation.
+ */
+ @Test
+ @Timeout(60)
+ void enqueueRacedAgainstStop_alwaysCompletesResultFuture() throws Exception {
+ int iterations = 500;
+ java.util.concurrent.ThreadLocalRandom random = java.util.concurrent.ThreadLocalRandom.current();
+
+ for (int i = 0; i < iterations; i++) {
+ int freshPort = findFreePort();
+ RuntimeApiServer freshServer = new RuntimeApiServer(vertx, freshPort);
+ freshServer.start().get(5, TimeUnit.SECONDS);
+ try {
+ PendingInvocation invocation = new PendingInvocation(
+ "req-race-" + i, "{}".getBytes(), System.currentTimeMillis() + 60_000,
+ "arn:aws:lambda:us-east-1:000000000000:function:test",
+ new CompletableFuture<>());
+
+ Thread stopper = new Thread(() -> {
+ try {
+ Thread.sleep(random.nextInt(0, 5));
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ }
+ freshServer.stop();
+ });
+ stopper.start();
+ freshServer.enqueue(invocation);
+ stopper.join(5000);
+
+ InvokeResult result = invocation.getResultFuture().get(5, TimeUnit.SECONDS);
+ assertNotNull(result, "resultFuture must always complete, never hang");
+ } finally {
+ freshServer.stop().get(5, TimeUnit.SECONDS);
+ }
+ }
+ }
+
+ /**
+ * /extension/register racing stop() (floci-io/floci#1882: register previously didn't
+ * check stopped at all). Registration itself always completes before stop() can begin
+ * (the race is purely over which one observes the other's effect first — stop() draining
+ * an empty extensions map vs. register() finding stopped already true), and whatever
+ * identifier comes back, a subsequent /event/next with it must eventually return SHUTDOWN
+ * rather than hanging or 403ing inconsistently.
+ */
+ @Test
+ @Timeout(60)
+ void extensionRegister_racedAgainstStop_eventuallyDeliversShutdown() throws Exception {
+ int iterations = 300;
+
+ for (int i = 0; i < iterations; i++) {
+ HttpClient client = HttpClient.newBuilder().connectTimeout(java.time.Duration.ofSeconds(5)).build();
+ int freshPort = findFreePort();
+ RuntimeApiServer freshServer = new RuntimeApiServer(vertx, freshPort);
+ freshServer.start().get(5, TimeUnit.SECONDS);
+ try {
+ String extensionId = registerExtensionOn(client, freshPort, "lambda-adapter", "SHUTDOWN");
+ // Race stop() against the extension's very first /event/next poll, which may
+ // land before or after stop() — either way SHUTDOWN must still be delivered.
+ CompletableFuture<HttpResponse<String>> asyncNext = client.sendAsync(
+ HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + freshPort + "/2020-01-01/extension/event/next"))
+ .header("Lambda-Extension-Identifier", extensionId)
+ .GET().build(),
+ HttpResponse.BodyHandlers.ofString());
+ // Give the request time to actually connect and park before stop() closes
+ // the listening socket — see extensionShutdown_racedAgainstStop_isNeverOrphaned.
+ Thread.sleep(5);
+ freshServer.stop();
+
+ HttpResponse<String> response = asyncNext.get(5, TimeUnit.SECONDS);
+ assertEquals(200, response.statusCode(),
+ "a registered extension must eventually see SHUTDOWN, never hang/403");
+ assertEquals("SHUTDOWN", new JsonObject(response.body()).getString("eventType"));
+ } finally {
+ freshServer.stop().get(5, TimeUnit.SECONDS);
+ client.close();
+ }
+ }
+ }
+
+ /**
+ * handleExtensionFatalError racing stop()'s SHUTDOWN fan-out: no exception, no
+ * double-processing. The fatal-error POST completes before stop() begins tearing down
+ * the connection (dispatched synchronously, unlike the long-polling /event/next above),
+ * so the only race is stop()'s SHUTDOWN fan-out landing concurrently with the extension
+ * having just been removed from the map.
+ */
+ @Test
+ @Timeout(30)
+ void extensionFatalError_racedAgainstStop_noExceptionNoDoubleProcessing() throws Exception {
+ int iterations = 300;
+
+ for (int i = 0; i < iterations; i++) {
+ HttpClient client = HttpClient.newBuilder().connectTimeout(java.time.Duration.ofSeconds(5)).build();
+ int freshPort = findFreePort();
+ RuntimeApiServer freshServer = new RuntimeApiServer(vertx, freshPort);
+ freshServer.start().get(5, TimeUnit.SECONDS);
+ try {
+ String extensionId = registerExtensionOn(client, freshPort, "failing-extension", "INVOKE", "SHUTDOWN");
+
+ HttpResponse<String> response = client.send(HttpRequest.newBuilder()
+ .uri(URI.create(
+ "http://localhost:" + freshPort + "/2020-01-01/extension/init/error"))
+ .header("Lambda-Extension-Identifier", extensionId)
+ .header("Lambda-Extension-Function-Error-Type", "Extension.TestError")
+ .POST(HttpRequest.BodyPublishers.ofString("{}"))
+ .build(),
+ HttpResponse.BodyHandlers.ofString());
+ assertEquals(202, response.statusCode(), "fatal-error endpoint must not throw under the race");
+
+ // stop() runs immediately after — races its SHUTDOWN fan-out against the
+ // extension having just been removed by the fatal-error handler above.
+ freshServer.stop().get(5, TimeUnit.SECONDS);
+ } finally {
+ freshServer.stop().get(5, TimeUnit.SECONDS);
+ client.close();
+ }
+ }
+ }
+
+ /**
+ * Regression guard against reintroducing worker-thread pinning: NEXT_PATH must stay
+ * non-blocking/event-loop-based, since the container's language-runtime bootstrap holds
+ * this long-poll open for its entire idle lifetime. Parking more concurrent /next polls
+ * than the default Quarkus worker pool (20 threads) must still succeed — this would fail
+ * or hang if a future change moved NEXT_PATH to a blockingHandler/wait() design.
+ */
+ @Test
+ @Timeout(30)
+ void manyConcurrentNextPollers_exceedingWorkerPoolSize_allParkAndComplete() throws Exception {
+ int serverCount = 30;
+ List<RuntimeApiServer> servers = new ArrayList<>();
+ List<Integer> ports = new ArrayList<>();
+ try {
+ for (int i = 0; i < serverCount; i++) {
+ int freshPort = findFreePort();
+ RuntimeApiServer freshServer = new RuntimeApiServer(vertx, freshPort);
+ freshServer.start().get(5, TimeUnit.SECONDS);
+ servers.add(freshServer);
+ ports.add(freshPort);
+ }
+
+ List<CompletableFuture<HttpResponse<String>>> pending = new ArrayList<>();
+ for (int freshPort : ports) {
+ pending.add(httpClient.sendAsync(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + freshPort + "/2018-06-01/runtime/invocation/next"))
+ .GET().build(),
+ HttpResponse.BodyHandlers.ofString()));
+ }
+
+ Thread.sleep(500);
+ for (CompletableFuture<HttpResponse<String>> f : pending) {
+ assertFalse(f.isDone(), "all pollers should be parked, none held on a blocking thread");
+ }
+
+ for (int i = 0; i < serverCount; i++) {
+ servers.get(i).enqueue(new PendingInvocation(
+ "req-many-" + i, "{}".getBytes(), System.currentTimeMillis() + 60_000,
+ "arn:aws:lambda:us-east-1:000000000000:function:test",
+ new CompletableFuture<>()));
+ }
+
+ for (CompletableFuture<HttpResponse<String>> f : pending) {
+ assertEquals(200, f.get(5, TimeUnit.SECONDS).statusCode());
+ }
+ } finally {
+ for (RuntimeApiServer s : servers) {
+ s.stop().get(5, TimeUnit.SECONDS);
+ }
+ }
+ }
+
+ private String registerExtensionOn(HttpClient client, int targetPort, String name, String... events)
+ throws Exception {
+ String eventsJson = String.join(",", java.util.Arrays.stream(events).map(e -> "\"" + e + "\"").toList());
+ HttpResponse<String> response = client.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + targetPort + "/2020-01-01/extension/register"))
+ .header("Lambda-Extension-Name", name)
+ .POST(HttpRequest.BodyPublishers.ofString("{\"events\":[" + eventsJson + "]}"))
+ .build(),
+ HttpResponse.BodyHandlers.ofString());
+ assertEquals(200, response.statusCode());
+ return response.headers().firstValue("Lambda-Extension-Identifier").orElseThrow();
+ }
+
+ /** Has the given extension report an init error, condemning the execution environment. */
+ private void reportExtensionInitError(String extensionId) throws Exception {
+ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/init/error"))
+ .header("Lambda-Extension-Identifier", extensionId)
+ .header("Lambda-Extension-Function-Error-Type", "Extension.TestError")
+ .POST(HttpRequest.BodyPublishers.ofString("{\"errorMessage\":\"bad config\"}"))
+ .build(),
+ HttpResponse.BodyHandlers.ofString());
+ assertEquals(202, response.statusCode());
+ assertTrue(server.isFaulted());
+ }
+
+ /**
+ * Issues one /extension/event/next for the given extension, which is what marks it init-ready.
+ *
+ * <p>Deliberately async and un-awaited: with no event pending the handler parks the request
+ * until an INVOKE or SHUTDOWN arrives, so a blocking send here would hang the test. The
+ * readiness countdown happens as the handler runs, before it parks — callers observe it
+ * through the barrier rather than through this response.
+ *
+ * <p>Sleeps briefly before returning so the request has actually reached the handler. Without
+ * that, a caller asserting on the barrier immediately afterwards could observe the state from
+ * before this poll and pass (or fail) for the wrong reason.
+ */
+ private CompletableFuture<HttpResponse<String>> pollExtensionEventNext(String extensionId)
+ throws Exception {
+ CompletableFuture<HttpResponse<String>> pending = httpClient.sendAsync(
+ HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/event/next"))
+ .header("Lambda-Extension-Identifier", extensionId)
+ .GET().build(),
+ HttpResponse.BodyHandlers.ofString());
+ Thread.sleep(150);
+ return pending;
+ }
+
+ private String registerExtension(String name, String... events) throws Exception {
+ String eventsJson = String.join(",", java.util.Arrays.stream(events).map(e -> "\"" + e + "\"").toList());
+ HttpResponse<String> response = httpClient.send(HttpRequest.newBuilder()
+ .uri(URI.create("http://localhost:" + port + "/2020-01-01/extension/register"))
+ .header("Lambda-Extension-Name", name)
+ .POST(HttpRequest.BodyPublishers.ofString("{\"events\":[" + eventsJson + "]}"))
+ .build(),
+ HttpResponse.BodyHandlers.ofString());
+ assertEquals(200, response.statusCode());
+ return response.headers().firstValue("Lambda-Extension-Identifier").orElseThrow();
+ }
+
+ /**
+ * Lowest port used by {@link #findFreePort()}. Deliberately below the OS ephemeral range
+ * (49152-65535 on macOS, 32768-60999 on typical Linux) — see {@link #findFreePort()}.
+ */
+ private static final int TEST_PORT_BASE = 20000;
+ /**
+ * Wide enough that the rotation does not wrap back onto a port still in TIME_WAIT: one run of
+ * this class performs ~1600 port draws, and a closed listener's port stays unusable for tens of
+ * seconds afterwards. Running several test classes in the same JVM multiplies that count, which
+ * is when a narrower window started producing sporadic connect failures.
+ */
+ private static final int TEST_PORT_RANGE = 25000;
+
+ /** Rotates across the range so consecutive calls don't retry a port still in TIME_WAIT. */
+ private static final java.util.concurrent.atomic.AtomicInteger NEXT_PORT_OFFSET =
+ new java.util.concurrent.atomic.AtomicInteger(
+ java.util.concurrent.ThreadLocalRandom.current().nextInt(TEST_PORT_RANGE));
+
+ /**
+ * Returns a port that is free <em>and</em> outside the OS ephemeral range.
+ *
+ * <p>Binding port 0 (the obvious implementation) draws from the ephemeral range, which is
+ * exactly where the OS and background applications allocate their own listeners. Long-running
+ * desktop agents routinely hold stable ports in that range — on the machine this was diagnosed
+ * on, {@code LogiPlugin} (49200-49204), {@code rapportd} (56698, 63091-63092) and
+ * {@code CommCenter} (65000-65001) all sit inside it. The stress tests below perform well over
+ * a thousand port draws per run, so a collision is near-certain: the probe socket closes, the
+ * other process (or the OS) claims the port before Vert.x binds it, and the test's request is
+ * answered by a stranger — observed as a spurious {@code 501} from an unrelated HTTP daemon, or
+ * as a bind failure/start() timeout. Neither has anything to do with the concurrency under test.
+ *
+ * <p>Ports 20000-29999 sit below every common ephemeral range, so nothing else on the host is
+ * handing them out. The bind check still guards against a genuinely occupied port.
+ */
private static int findFreePort() throws IOException {
- try (ServerSocket socket = new ServerSocket(0)) {
- return socket.getLocalPort();
+ for (int attempt = 0; attempt < TEST_PORT_RANGE; attempt++) {
+ int candidate = TEST_PORT_BASE
+ + Math.floorMod(NEXT_PORT_OFFSET.getAndIncrement(), TEST_PORT_RANGE);
+ try (ServerSocket socket = new ServerSocket()) {
+ // Without SO_REUSEADDR the probe leaves the port in TIME_WAIT, which would block
+ // the server that is about to bind it for real.
+ socket.setReuseAddress(true);
+ socket.bind(new InetSocketAddress("0.0.0.0", candidate));
+ return candidate;
+ } catch (IOException inUse) {
+ // Occupied — try the next port in the rotation.
+ }
}
+ throw new IOException("no free port in " + TEST_PORT_BASE + "-" + (TEST_PORT_BASE + TEST_PORT_RANGE));
}
}